Pixeltable vs LanceDB
Complete Infrastructure vs Multimodal Database

import lancedb
import pandas as pd
from sentence_transformers import SentenceTransformer

# Manual setup and orchestration required
db = lancedb.connect("~/lancedb")
model = SentenceTransformer('all-MiniLM-L6-v2')

# Step 1: Create table with source data
docs = pd.DataFrame([
    {"id": 1, "content": "Document content..."},
    {"id": 2, "content": "More content..."}
])
table = db.create_table("documents", docs)

# Step 2: Define UDF for embedding generation
@lancedb.batch_udf
def embed_func(batch):
    texts = batch["content"].to_pylist()
    embeddings = model.encode(texts)
    return pa.RecordBatch.from_arrays(
        [pa.FixedSizeListArray.from_arrays(embeddings, 384)], 
        ["embedding"]
    )

# Step 3: Apply UDF (processes entire column)
table.add_columns(embed_func)

# Step 4: Manual query processing
def query_rag(question: str):
    query_embedding = model.encode([question])
    results = table.search(query_embedding[0]).limit(5)
    
    context = "\n".join(results["content"].tolist())
    response = llm_client.complete(
        f"Context: {context}\nQuestion: {question}"
    )
    return response

# Updates require manual reprocessing

import pixeltable as pxt
from pixeltable.functions import openai, document

# Step 1: Create documents table
docs = pxt.create_table('documents', {
    'document': pxt.Document,
    'metadata': pxt.Json
})

# Step 2: Add computed columns (declarative)
docs.add_computed_column(
    chunks=document.split_text(
        docs.document, 
        separators='sentence', 
        limit=500
    )
)

# Step 3: Auto-embedding index (incremental)
docs.add_embedding_index(
    'chunks', 
    string_embed=openai.using(model='text-embedding-ada-002')
)

# Step 4: Insert documents (auto-processing)
docs.insert([
    {'document': '/path/to/doc1.pdf', 'metadata': {...}},
    {'document': '/path/to/doc2.docx', 'metadata': {...}}
])

# Step 5: Query with automatic RAG
@pxt.udf
def rag_query(question: str) -> str:
    context_chunks = docs.chunks.similarity(question).limit(5)
    context = "\n".join([c.chunks for c in context_chunks])
    
    return openai.chat_completions(
        model='gpt-4',
        messages=[{
            'role': 'user', 
            'content': f'Context: {context}\nQ: {question}'
        }]
    ).choices[0].message.content

# Auto-updates when files change