Pixeltable vs LanceDB

import pixeltable as pxt
from pixeltable.functions import openai, document

docs = pxt.create_table('documents', {
    'document': pxt.Document,
    'metadata': pxt.Json
})

docs.add_computed_column(
    chunks=document.split_text(
        docs.document,
        separators='sentence',
        limit=500
    )
)

docs.add_embedding_index(
    'chunks',
    string_embed=openai.using(model='text-embedding-ada-002')
)

docs.insert([
    {'document': '/path/to/doc1.pdf', 'metadata': {...}},
    {'document': '/path/to/doc2.docx', 'metadata': {...}}
])

@pxt.udf
def rag_query(question: str) -> str:
    context_chunks = docs.chunks.similarity(string=question).limit(5)
    context = "\n".join([c.chunks for c in context_chunks])
    return openai.chat_completions(
        model='gpt-4',
        messages=[{'role': 'user', 'content': f'Context: {context}\nQ: {question}'}]
    ).choices[0].message.content

# Auto-updates when files change

import lancedb
import pandas as pd
from sentence_transformers import SentenceTransformer

db = lancedb.connect("~/lancedb")
model = SentenceTransformer('all-MiniLM-L6-v2')

docs = pd.DataFrame([
    {"id": 1, "content": "Document content..."},
    {"id": 2, "content": "More content..."}
])
table = db.create_table("documents", docs)

@lancedb.batch_udf
def embed_func(batch):
    texts = batch["content"].to_pylist()
    embeddings = model.encode(texts)
    return pa.RecordBatch.from_arrays(
        [pa.FixedSizeListArray.from_arrays(embeddings, 384)],
        ["embedding"]
    )

table.add_columns(embed_func)

def query_rag(question: str):
    query_embedding = model.encode([question])
    results = table.search(query_embedding[0]).limit(5)
    context = "\n".join(results["content"].tolist())
    return llm_client.complete(f"Context: {context}\nQuestion: {question}")

# Updates require manual reprocessing