Pixeltable vs Neon

import pixeltable as pxt
from pixeltable.functions.document import document_splitter
from pixeltable.functions.huggingface import sentence_transformer

TableModel = pxt.model_base()
embed = sentence_transformer.using(
    model_id='sentence-transformers/all-MiniLM-L6-v2'
)

class Docs(TableModel, name='docs'):
    document: pxt.Document
    title: pxt.String

class Chunks(
    TableModel,
    name='chunks',
    base=Docs,
    iterator=document_splitter(Docs.document, separators='sentence', limit=512),
):
    __indexes__ = [pxt.EmbeddingIndex(text, embedding=embed)]

# Apply schema & insert: transforms and embeddings run automatically
# pxt schema update app.py rag
docs = pxt.get_table('rag.docs')
docs.insert([{'document': 'annual_report.pdf', 'title': '2025 Annual Report'}])

# Query vector similarity in-engine
chunks = pxt.get_table('rag.chunks')
sim = chunks.text.similarity(string='operating margin growth')
results = chunks.order_by(sim, asc=False).limit(5).select(chunks.text, chunks.title)

# 1. Database schema in Neon
# CREATE EXTENSION IF NOT EXISTS vector;
# CREATE TABLE docs (id UUID PRIMARY KEY, title TEXT, s3_url TEXT);
# CREATE TABLE doc_chunks (id UUID PRIMARY KEY, doc_id UUID, text TEXT, embedding vector(384));

import fitz  # PyMuPDF
from sentence_transformers import SentenceTransformer
import psycopg2
from psycopg2.extras import execute_batch

model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
conn = psycopg2.connect("postgresql://neondb_owner:[email protected]/neondb")

def ingest_document(doc_id, title, pdf_path):
    doc = fitz.open(pdf_path)
    chunks = [page.get_text() for page in doc]
    embeddings = model.encode(chunks)
    
    with conn.cursor() as cur:
        cur.execute("INSERT INTO docs VALUES (%s, %s, %s)", (doc_id, title, pdf_path))
        rows = [(str(uuid.uuid4()), doc_id, c, e.tolist()) for c, e in zip(chunks, embeddings)]
        execute_batch(cur, "INSERT INTO doc_chunks VALUES (%s, %s, %s, %s)", rows)
    conn.commit()
# Plus: Celery task runner, S3 download glue, retry on network failure...

# Update embedder in app.py:
new_embed = sentence_transformer.using(model_id='BAAI/bge-small-en-v1.5')

class Chunks(
    TableModel,
    name='chunks',
    base=Docs,
    iterator=document_splitter(Docs.document, separators='sentence', limit=512),
):
    __indexes__ = [pxt.EmbeddingIndex(text, embedding=new_embed)]

# Run: pxt schema update app.py rag
# Pixeltable calculates the delta, computes new embeddings,
# and updates the index without duplicating rows or manual scripts.

# 1. Neon DDL migration
# ALTER TABLE doc_chunks ADD COLUMN new_embedding vector(384);

# 2. Write and run custom backfill script
new_model = SentenceTransformer('BAAI/bge-small-en-v1.5')
batch_size = 100
offset = 0

while True:
    with conn.cursor() as cur:
        cur.execute("SELECT id, text FROM doc_chunks WHERE new_embedding IS NULL LIMIT %s", (batch_size,))
        rows = cur.fetchall()
        if not rows:
            break
        ids, texts = zip(*rows)
        vectors = new_model.encode(list(texts))
        update_data = [(v.tolist(), i) for v, i in zip(vectors, ids)]
        execute_batch(cur, "UPDATE doc_chunks SET new_embedding = %s WHERE id = %s", update_data)
        conn.commit()
# Risk: transaction timeouts, connection drops, and drift between old and new columns.