Incremental Updates in AI Data Processing: Save 70% on Compute Costs

import pixeltable as pxt
from pixeltable.functions import openai

# Create table with expensive AI processing
documents = pxt.create_table('knowledge_base', {
    'document': pxt.Document,
    'title': pxt.String
})

# Expensive embedding generation
documents.add_computed_column(
    text_embedding=openai.embeddings(
        pxt.functions.extract_text(documents.document),
        model='text-embedding-3-large'
    )
)

# Add embedding index
documents.add_embedding_index(
    'text',
    embedding=documents.text_embedding
)

# Initial insert: processes all documents
documents.insert([...100_documents])

# Add 1 new document: only processes the new one!
documents.insert({'document': 'new_doc.pdf', 'title': 'Latest'})
# ✅ Saves 99% of compute vs reprocessing everything