feat: add incremental embeddings refresh workflow · j4ck.xyz/atmosphere-vods@0542693

+75

.github/workflows/refresh-embeddings.yml

··· 1 + name: Refresh Embeddings Index 2 + 3 + on: 4 + schedule: 5 + - cron: '15 */12 * * *' 6 + workflow_dispatch: 7 + 8 + permissions: 9 + contents: write 10 + 11 + jobs: 12 + refresh-embeddings: 13 + runs-on: ubuntu-latest 14 + 15 + steps: 16 + - name: Checkout main 17 + uses: actions/checkout@v4 18 + with: 19 + fetch-depth: 0 20 + 21 + - name: Setup Node 22 + uses: actions/setup-node@v4 23 + with: 24 + node-version: '20' 25 + cache: npm 26 + 27 + - name: Install dependencies 28 + run: npm ci 29 + 30 + - name: Fetch latest embeddings-data branch 31 + run: | 32 + git fetch origin embeddings-data || true 33 + 34 + - name: Prepare previous embeddings snapshot 35 + run: | 36 + mkdir -p .cache 37 + if git show-ref --verify --quiet refs/remotes/origin/embeddings-data; then 38 + git show origin/embeddings-data:video-embeddings.json > .cache/previous-video-embeddings.json || true 39 + fi 40 + 41 + - name: Generate incremental embeddings index 42 + env: 43 + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} 44 + OPENROUTER_EMBEDDING_MODEL: qwen/qwen3-embedding-8b 45 + EXISTING_EMBEDDINGS_PATH: .cache/previous-video-embeddings.json 46 + EMBEDDINGS_OUTPUT_PATH: video-embeddings.json 47 + run: npm run embeddings:generate 48 + 49 + - name: Detect index changes 50 + id: diff 51 + run: | 52 + if [ -f .cache/previous-video-embeddings.json ] && cmp -s .cache/previous-video-embeddings.json video-embeddings.json; then 53 + echo "changed=false" >> "$GITHUB_OUTPUT" 54 + else 55 + echo "changed=true" >> "$GITHUB_OUTPUT" 56 + fi 57 + 58 + - name: Publish embeddings-data branch 59 + if: steps.diff.outputs.changed == 'true' 60 + env: 61 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 62 + run: | 63 + mkdir -p .out 64 + cp video-embeddings.json .out/video-embeddings.json 65 + git checkout --orphan embeddings-data 66 + git rm -rf . >/dev/null 2>&1 || true 67 + cp .out/video-embeddings.json ./video-embeddings.json 68 + rm -rf .out 69 + git add video-embeddings.json 70 + git -c user.name="jack" -c user.email="jack@j4ck.xyz" commit -m "chore: refresh embeddings index" 71 + git push --force-with-lease origin embeddings-data 72 + 73 + - name: No changes 74 + if: steps.diff.outputs.changed != 'true' 75 + run: echo "No embedding updates required."

+17 -2

README.md

··· 58 58 ### Required env vars for embeddings generation 59 59 60 60 - `OPENROUTER_API_KEY` 61 - - optional: `OPENROUTER_EMBEDDING_MODEL` (defaults to `openai/text-embedding-3-small`) 61 + - optional: `OPENROUTER_EMBEDDING_MODEL` (defaults to `qwen/qwen3-embedding-8b`) 62 + - optional: `EXISTING_EMBEDDINGS_PATH` for incremental reuse (defaults to current output file) 63 + - optional: `EMBEDDINGS_OUTPUT_PATH` (defaults to `public/video-embeddings.json`) 62 64 63 65 ## Cloudflare Pages function search backend 64 66 ··· 72 74 Set this env var in Cloudflare Pages project settings for semantic mode: 73 75 74 76 - `OPENROUTER_API_KEY` 75 - - optional: `OPENROUTER_EMBEDDING_MODEL` 77 + - optional: `OPENROUTER_EMBEDDING_MODEL` (defaults to `qwen/qwen3-embedding-8b`) 78 + - optional: `EMBEDDINGS_INDEX_URL` (raw URL for externally refreshed index) 76 79 77 80 If the key is not set, search falls back gracefully to lexical title ranking. 78 81 ··· 82 85 from relay + PDS APIs. 83 86 - Those fresh videos are lexical-ranked until you refresh `public/video-embeddings.json`. 84 87 - The UI shows index snapshot time and how many videos are currently embedded. 88 + 89 + ## GitHub Actions (twice daily incremental refresh) 90 + 91 + Workflow file: `.github/workflows/refresh-embeddings.yml` 92 + 93 + - Runs every 12 hours and on manual dispatch. 94 + - Reuses existing vectors from `embeddings-data` branch and embeds only new VOD URIs. 95 + - Publishes updated `video-embeddings.json` to `embeddings-data` only when changed. 96 + 97 + Recommended Cloudflare env for zero-redeploy index updates: 98 + 99 + - `EMBEDDINGS_INDEX_URL=https://raw.githubusercontent.com/j4ckxyz/atmosphere-vods/embeddings-data/video-embeddings.json` 85 100 86 101 ## Deploy to Vercel 87 102

+39 -15

functions/api/search.ts

··· 25 25 env: { 26 26 OPENROUTER_API_KEY?: string 27 27 OPENROUTER_EMBEDDING_MODEL?: string 28 + EMBEDDINGS_INDEX_URL?: string 28 29 ASSETS?: { 29 30 fetch: (request: Request) => Promise<Response> 30 31 } ··· 253 254 return { index: cachedIndex.index, norms: cachedIndex.norms } 254 255 } 255 256 256 - const assets = context.env.ASSETS 257 - if (!assets) { 258 - throw new Error('Cloudflare ASSETS binding unavailable') 257 + let index: EmbeddingIndex | null = null 258 + 259 + const remoteIndexUrl = context.env.EMBEDDINGS_INDEX_URL 260 + if (remoteIndexUrl) { 261 + try { 262 + const remoteResponse = await fetch(remoteIndexUrl, { 263 + headers: { 264 + Accept: 'application/json', 265 + 'Cache-Control': 'no-cache', 266 + }, 267 + }) 268 + 269 + if (remoteResponse.ok) { 270 + index = (await remoteResponse.json()) as EmbeddingIndex 271 + } 272 + } catch { 273 + index = null 274 + } 259 275 } 260 276 261 - const origin = new URL(context.request.url).origin 262 - const assetsRequest = new Request(`${origin}/video-embeddings.json`, { 263 - method: 'GET', 264 - headers: { 265 - Accept: 'application/json', 266 - }, 267 - }) 268 - const response = await assets.fetch(assetsRequest) 277 + if (!index) { 278 + const assets = context.env.ASSETS 279 + if (!assets) { 280 + throw new Error('Cloudflare ASSETS binding unavailable') 281 + } 282 + 283 + const origin = new URL(context.request.url).origin 284 + const assetsRequest = new Request(`${origin}/video-embeddings.json`, { 285 + method: 'GET', 286 + headers: { 287 + Accept: 'application/json', 288 + }, 289 + }) 290 + const response = await assets.fetch(assetsRequest) 269 291 270 - if (!response.ok) { 271 - throw new Error(`Embeddings asset unavailable (${response.status})`) 292 + if (!response.ok) { 293 + throw new Error(`Embeddings asset unavailable (${response.status})`) 294 + } 295 + 296 + index = (await response.json()) as EmbeddingIndex 272 297 } 273 298 274 - const index = (await response.json()) as EmbeddingIndex 275 299 const norms = (index.entries ?? []).map((entry) => normalizeVector(entry.embedding ?? [])) 276 300 277 301 cachedIndex = { ··· 289 313 return null 290 314 } 291 315 292 - const model = context.env.OPENROUTER_EMBEDDING_MODEL || 'openai/text-embedding-3-small' 316 + const model = context.env.OPENROUTER_EMBEDDING_MODEL || 'qwen/qwen3-embedding-8b' 293 317 const response = await fetch('https://openrouter.ai/api/v1/embeddings', { 294 318 method: 'POST', 295 319 headers: {

+124 -15

scripts/generate-video-embeddings.mjs

··· 6 6 const BSKY_RELAY_SYNC_API = 'https://bsky.network' 7 7 const STREAMPLACE_VIDEO_COLLECTION = 'place.stream.video' 8 8 const OPENROUTER_API_URL = 'https://openrouter.ai/api/v1' 9 - const EMBEDDING_MODEL = process.env.OPENROUTER_EMBEDDING_MODEL ?? 'openai/text-embedding-3-small' 10 - const OUTPUT_PATH = path.resolve(process.cwd(), 'public/video-embeddings.json') 9 + const EMBEDDING_MODEL = process.env.OPENROUTER_EMBEDDING_MODEL ?? 'qwen/qwen3-embedding-8b' 10 + const OUTPUT_PATH = path.resolve( 11 + process.cwd(), 12 + process.env.EMBEDDINGS_OUTPUT_PATH ?? 'public/video-embeddings.json', 13 + ) 14 + const EXISTING_PATH = path.resolve( 15 + process.cwd(), 16 + process.env.EXISTING_EMBEDDINGS_PATH ?? process.env.EMBEDDINGS_OUTPUT_PATH ?? 'public/video-embeddings.json', 17 + ) 11 18 const TAXONOMY_PATH = path.resolve(process.cwd(), 'src/lib/video-taxonomy.json') 19 + const EMBEDDING_BATCH_SIZE = Number.parseInt(process.env.EMBEDDING_BATCH_SIZE ?? '64', 10) 12 20 13 21 function parseEnvFile(content) { 14 22 const vars = {} ··· 64 72 throw new Error(`Request failed (${response.status}) for ${url}`) 65 73 } 66 74 return response.json() 75 + } 76 + 77 + function isFiniteNumber(value) { 78 + return typeof value === 'number' && Number.isFinite(value) 79 + } 80 + 81 + async function loadExistingIndex() { 82 + if (!existsSync(EXISTING_PATH)) { 83 + return null 84 + } 85 + 86 + const raw = await readFile(EXISTING_PATH, 'utf8') 87 + try { 88 + const parsed = JSON.parse(raw) 89 + if (!Array.isArray(parsed.entries)) { 90 + return null 91 + } 92 + return parsed 93 + } catch { 94 + return null 95 + } 67 96 } 68 97 69 98 async function callOpenRouter(pathname, body) { ··· 180 209 .join('\n') 181 210 } 182 211 212 + async function embedBatch(inputs) { 213 + const embeddingResponse = await callOpenRouter('/embeddings', { 214 + model: EMBEDDING_MODEL, 215 + input: inputs, 216 + input_type: 'search_document', 217 + }) 218 + 219 + return embeddingResponse.data ?? [] 220 + } 221 + 222 + function catalogSignature(entries) { 223 + return entries 224 + .map((entry) => `${entry.uri}|${entry.sourceRepoDid}|${entry.createdAt ?? ''}|${entry.title}`) 225 + .join('\n') 226 + } 227 + 183 228 async function main() { 184 229 await loadEnv() 185 230 ··· 215 260 throw new Error('No video records available for embedding generation') 216 261 } 217 262 218 - const inputs = allRecords.map((record) => toEmbeddingInput(record)) 219 - const embeddingResponse = await callOpenRouter('/embeddings', { 220 - model: EMBEDDING_MODEL, 221 - input: inputs, 222 - input_type: 'search_document', 263 + const existing = await loadExistingIndex() 264 + const canReuseExisting = existing?.model === EMBEDDING_MODEL 265 + const existingByUri = new Map( 266 + canReuseExisting 267 + ? (existing.entries ?? []) 268 + .filter((entry) => Array.isArray(entry.embedding) && entry.embedding.length > 0) 269 + .map((entry) => [entry.uri, entry]) 270 + : [], 271 + ) 272 + 273 + if (existing && !canReuseExisting) { 274 + console.log( 275 + `Existing index model (${existing.model}) differs from ${EMBEDDING_MODEL}; rebuilding all embeddings`, 276 + ) 277 + } 278 + 279 + const recordsByUri = new Map(allRecords.map((record) => [record.uri, record])) 280 + const orderedUris = [...recordsByUri.keys()].sort((left, right) => left.localeCompare(right)) 281 + const orderedRecords = orderedUris.map((uri) => recordsByUri.get(uri)) 282 + 283 + const reuseCandidates = [] 284 + const missingRecords = [] 285 + 286 + for (const record of orderedRecords) { 287 + const existingEntry = existingByUri.get(record.uri) 288 + if (existingEntry) { 289 + reuseCandidates.push({ record, existingEntry }) 290 + } else { 291 + missingRecords.push(record) 292 + } 293 + } 294 + 295 + const newEmbeddingsByUri = new Map() 296 + 297 + for (let start = 0; start < missingRecords.length; start += EMBEDDING_BATCH_SIZE) { 298 + const batchRecords = missingRecords.slice(start, start + EMBEDDING_BATCH_SIZE) 299 + const inputs = batchRecords.map((record) => toEmbeddingInput(record)) 300 + const data = await embedBatch(inputs) 301 + 302 + for (let i = 0; i < batchRecords.length; i += 1) { 303 + const embedding = data[i]?.embedding 304 + if (!Array.isArray(embedding) || embedding.length === 0) { 305 + throw new Error(`Embedding missing for URI ${batchRecords[i].uri}`) 306 + } 307 + if (!embedding.every((value) => isFiniteNumber(value))) { 308 + throw new Error(`Invalid embedding values for URI ${batchRecords[i].uri}`) 309 + } 310 + newEmbeddingsByUri.set(batchRecords[i].uri, embedding) 311 + } 312 + 313 + console.log( 314 + `Embedded ${Math.min(start + EMBEDDING_BATCH_SIZE, missingRecords.length)}/${missingRecords.length} new records`, 315 + ) 316 + } 317 + 318 + const nextEntries = orderedRecords.map((record) => { 319 + const reused = existingByUri.get(record.uri) 320 + const embedding = reused?.embedding ?? newEmbeddingsByUri.get(record.uri) 321 + return { 322 + uri: record.uri, 323 + sourceRepoDid: record.sourceRepoDid, 324 + createdAt: record.value?.createdAt, 325 + title: record.value?.title ?? 'Untitled', 326 + embedding, 327 + } 223 328 }) 224 329 330 + const previousSignature = existing ? catalogSignature(existing.entries ?? []) : '' 331 + const nextSignature = catalogSignature(nextEntries) 332 + const hasCatalogChanges = previousSignature !== nextSignature 333 + const hasNewEmbeddings = missingRecords.length > 0 334 + 225 335 const output = { 226 336 version: 1, 227 - generatedAt: new Date().toISOString(), 337 + generatedAt: 338 + hasNewEmbeddings || hasCatalogChanges || !existing?.generatedAt 339 + ? new Date().toISOString() 340 + : existing.generatedAt, 228 341 model: EMBEDDING_MODEL, 229 - entries: allRecords.map((record, index) => ({ 230 - uri: record.uri, 231 - sourceRepoDid: record.sourceRepoDid, 232 - createdAt: record.value?.createdAt, 233 - title: record.value?.title ?? 'Untitled', 234 - embedding: embeddingResponse.data[index]?.embedding ?? [], 235 - })), 342 + entries: nextEntries, 236 343 } 237 344 238 345 await writeFile(OUTPUT_PATH, `${JSON.stringify(output, null, 2)}\n`, 'utf8') 346 + console.log(`Reused embeddings: ${reuseCandidates.length}`) 347 + console.log(`New embeddings: ${missingRecords.length}`) 239 348 console.log(`Wrote embeddings to ${OUTPUT_PATH}`) 240 349 } 241 350

Configure Feed

Configure Feed