search for standard sites
pub-search.waow.tech
search
zig
blog
atproto
feat: hybrid BM25 + recency ranking, since filter, CI deploy
- add recency decay to FTS5 ranking: ORDER BY rank + (days_old / 30)
- add ?since=<ISO-date> parameter to filter by creation date
- add GitHub Actions workflow to deploy backend on push to main
- update docs with ranking explanation
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
+83
-17
+27
.github/workflows/deploy-backend.yml
+27
.github/workflows/deploy-backend.yml
···
1
+
name: Deploy Backend
2
+
3
+
on:
4
+
push:
5
+
branches: [main]
6
+
paths:
7
+
- 'backend/**'
8
+
9
+
concurrency:
10
+
group: backend-deploy
11
+
cancel-in-progress: true
12
+
13
+
env:
14
+
FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
15
+
16
+
jobs:
17
+
deploy:
18
+
runs-on: ubuntu-latest
19
+
steps:
20
+
- uses: actions/checkout@v4
21
+
22
+
- name: Setup flyctl
23
+
uses: superfly/flyctl-actions/setup-flyctl@master
24
+
25
+
- name: Deploy to Fly.io
26
+
working-directory: backend
27
+
run: flyctl deploy --remote-only
+8
-6
README.md
+8
-6
README.md
···
27
27
## api
28
28
29
29
```
30
-
GET /search?q=<query>&tag=<tag>&platform=<platform> # full-text search
31
-
GET /similar?uri=<at-uri> # find similar documents
32
-
GET /tags # list all tags with counts
33
-
GET /popular # popular search queries
34
-
GET /stats # document/publication counts
35
-
GET /health # health check
30
+
GET /search?q=<query>&tag=<tag>&platform=<platform>&since=<date> # full-text search
31
+
GET /similar?uri=<at-uri> # find similar documents
32
+
GET /tags # list all tags with counts
33
+
GET /popular # popular search queries
34
+
GET /stats # document/publication counts
35
+
GET /health # health check
36
36
```
37
37
38
38
search returns three entity types: `article` (document in a publication), `looseleaf` (standalone document), `publication` (newsletter itself). each result includes a `platform` field (leaflet, pckt, etc). tag and platform filtering apply to documents only.
39
+
40
+
**ranking**: results use hybrid BM25 + recency scoring. text relevance is primary, but recent documents get a boost (~1 point per 30 days). the `since` parameter filters to documents created after the given ISO date (e.g., `since=2025-01-01`).
39
41
40
42
`/similar` uses [Voyage AI](https://voyageai.com) embeddings with brute-force cosine similarity (~0.15s for 3500 docs).
41
43
+43
-8
backend/src/search.zig
+43
-8
backend/src/search.zig
···
88
88
\\LEFT JOIN publications p ON d.publication_uri = p.uri
89
89
\\JOIN document_tags dt ON d.uri = dt.document_uri
90
90
\\WHERE documents_fts MATCH :query AND dt.tag = :tag
91
-
\\ORDER BY rank LIMIT 40
91
+
\\ORDER BY rank + (julianday('now') - julianday(d.created_at)) / 30.0 LIMIT 40
92
92
);
93
93
94
94
const DocsByFts = zql.Query(
···
102
102
\\JOIN documents d ON f.uri = d.uri
103
103
\\LEFT JOIN publications p ON d.publication_uri = p.uri
104
104
\\WHERE documents_fts MATCH :query
105
-
\\ORDER BY rank LIMIT 40
105
+
\\ORDER BY rank + (julianday('now') - julianday(d.created_at)) / 30.0 LIMIT 40
106
+
);
107
+
108
+
const DocsByFtsAndSince = zql.Query(
109
+
\\SELECT f.uri, d.did, d.title,
110
+
\\ snippet(documents_fts, 2, '', '', '...', 32) as snippet,
111
+
\\ d.created_at, d.rkey,
112
+
\\ COALESCE(p.base_path, (SELECT base_path FROM publications WHERE did = d.did LIMIT 1), '') as base_path,
113
+
\\ CASE WHEN d.publication_uri != '' THEN 1 ELSE 0 END as has_publication,
114
+
\\ d.platform, COALESCE(d.path, '') as path
115
+
\\FROM documents_fts f
116
+
\\JOIN documents d ON f.uri = d.uri
117
+
\\LEFT JOIN publications p ON d.publication_uri = p.uri
118
+
\\WHERE documents_fts MATCH :query AND d.created_at >= :since
119
+
\\ORDER BY rank + (julianday('now') - julianday(d.created_at)) / 30.0 LIMIT 40
106
120
);
107
121
108
122
const DocsByFtsAndPlatform = zql.Query(
···
116
130
\\JOIN documents d ON f.uri = d.uri
117
131
\\LEFT JOIN publications p ON d.publication_uri = p.uri
118
132
\\WHERE documents_fts MATCH :query AND d.platform = :platform
119
-
\\ORDER BY rank LIMIT 40
133
+
\\ORDER BY rank + (julianday('now') - julianday(d.created_at)) / 30.0 LIMIT 40
134
+
);
135
+
136
+
const DocsByFtsAndPlatformAndSince = zql.Query(
137
+
\\SELECT f.uri, d.did, d.title,
138
+
\\ snippet(documents_fts, 2, '', '', '...', 32) as snippet,
139
+
\\ d.created_at, d.rkey,
140
+
\\ COALESCE(p.base_path, (SELECT base_path FROM publications WHERE did = d.did LIMIT 1), '') as base_path,
141
+
\\ CASE WHEN d.publication_uri != '' THEN 1 ELSE 0 END as has_publication,
142
+
\\ d.platform, COALESCE(d.path, '') as path
143
+
\\FROM documents_fts f
144
+
\\JOIN documents d ON f.uri = d.uri
145
+
\\LEFT JOIN publications p ON d.publication_uri = p.uri
146
+
\\WHERE documents_fts MATCH :query AND d.platform = :platform AND d.created_at >= :since
147
+
\\ORDER BY rank + (julianday('now') - julianday(d.created_at)) / 30.0 LIMIT 40
120
148
);
121
149
122
150
const DocsByTagAndPlatform = zql.Query(
···
144
172
\\LEFT JOIN publications p ON d.publication_uri = p.uri
145
173
\\JOIN document_tags dt ON d.uri = dt.document_uri
146
174
\\WHERE documents_fts MATCH :query AND dt.tag = :tag AND d.platform = :platform
147
-
\\ORDER BY rank LIMIT 40
175
+
\\ORDER BY rank + (julianday('now') - julianday(d.created_at)) / 30.0 LIMIT 40
148
176
);
149
177
150
178
const DocsByPlatform = zql.Query(
···
161
189
162
190
// Find documents by their publication's base_path (subdomain search)
163
191
// e.g., searching "gyst" finds all docs on gyst.leaflet.pub
192
+
// Uses recency decay: recent docs rank higher than old ones with same match
164
193
const DocsByPubBasePath = zql.Query(
165
194
\\SELECT d.uri, d.did, d.title, '' as snippet,
166
195
\\ d.created_at, d.rkey,
···
171
200
\\JOIN publications p ON d.publication_uri = p.uri
172
201
\\JOIN publications_fts pf ON p.uri = pf.uri
173
202
\\WHERE publications_fts MATCH :query
174
-
\\ORDER BY d.created_at DESC LIMIT 40
203
+
\\ORDER BY rank + (julianday('now') - julianday(d.created_at)) / 30.0 LIMIT 40
175
204
);
176
205
177
206
const DocsByPubBasePathAndPlatform = zql.Query(
···
184
213
\\JOIN publications p ON d.publication_uri = p.uri
185
214
\\JOIN publications_fts pf ON p.uri = pf.uri
186
215
\\WHERE publications_fts MATCH :query AND d.platform = :platform
187
-
\\ORDER BY d.created_at DESC LIMIT 40
216
+
\\ORDER BY rank + (julianday('now') - julianday(d.created_at)) / 30.0 LIMIT 40
188
217
);
189
218
190
219
/// Publication search result (internal)
···
230
259
\\FROM publications_fts f
231
260
\\JOIN publications p ON f.uri = p.uri
232
261
\\WHERE publications_fts MATCH :query
233
-
\\ORDER BY rank LIMIT 10
262
+
\\ORDER BY rank + (julianday('now') - julianday(p.created_at)) / 30.0 LIMIT 10
234
263
);
235
264
236
-
pub fn search(alloc: Allocator, query: []const u8, tag_filter: ?[]const u8, platform_filter: ?[]const u8) ![]const u8 {
265
+
pub fn search(alloc: Allocator, query: []const u8, tag_filter: ?[]const u8, platform_filter: ?[]const u8, since_filter: ?[]const u8) ![]const u8 {
237
266
const c = db.getClient() orelse return error.NotInitialized;
238
267
239
268
var output: std.Io.Writer.Allocating = .init(alloc);
···
246
275
const has_query = query.len > 0;
247
276
const has_tag = tag_filter != null;
248
277
const has_platform = platform_filter != null;
278
+
const has_since = since_filter != null;
249
279
250
280
// track seen URIs for deduplication (content match + base_path match)
251
281
var seen_uris = std.StringHashMap(void).init(alloc);
252
282
defer seen_uris.deinit();
253
283
254
284
// search documents by content (title, content) - handle all filter combinations
285
+
// note: since filter only supported with query (not tag-only searches)
255
286
var doc_result = if (has_query and has_tag and has_platform)
256
287
c.query(DocsByFtsAndTagAndPlatform.positional, DocsByFtsAndTagAndPlatform.bind(.{
257
288
.query = fts_query,
···
260
291
})) catch null
261
292
else if (has_query and has_tag)
262
293
c.query(DocsByFtsAndTag.positional, DocsByFtsAndTag.bind(.{ .query = fts_query, .tag = tag_filter.? })) catch null
294
+
else if (has_query and has_platform and has_since)
295
+
c.query(DocsByFtsAndPlatformAndSince.positional, DocsByFtsAndPlatformAndSince.bind(.{ .query = fts_query, .platform = platform_filter.?, .since = since_filter.? })) catch null
263
296
else if (has_query and has_platform)
264
297
c.query(DocsByFtsAndPlatform.positional, DocsByFtsAndPlatform.bind(.{ .query = fts_query, .platform = platform_filter.? })) catch null
298
+
else if (has_query and has_since)
299
+
c.query(DocsByFtsAndSince.positional, DocsByFtsAndSince.bind(.{ .query = fts_query, .since = since_filter.? })) catch null
265
300
else if (has_query)
266
301
c.query(DocsByFts.positional, DocsByFts.bind(.{ .query = fts_query })) catch null
267
302
else if (has_tag and has_platform)
+3
-2
backend/src/server.zig
+3
-2
backend/src/server.zig
···
76
76
defer arena.deinit();
77
77
const alloc = arena.allocator();
78
78
79
-
// parse query params: /search?q=something&tag=foo&platform=leaflet
79
+
// parse query params: /search?q=something&tag=foo&platform=leaflet&since=2025-01-01
80
80
const query = parseQueryParam(alloc, target, "q") catch "";
81
81
const tag_filter = parseQueryParam(alloc, target, "tag") catch null;
82
82
const platform_filter = parseQueryParam(alloc, target, "platform") catch null;
83
+
const since_filter = parseQueryParam(alloc, target, "since") catch null;
83
84
84
85
if (query.len == 0 and tag_filter == null) {
85
86
try sendJson(request, "{\"error\":\"enter a search term\"}");
···
87
88
}
88
89
89
90
// perform FTS search - arena handles cleanup
90
-
const results = search.search(alloc, query, tag_filter, platform_filter) catch |err| {
91
+
const results = search.search(alloc, query, tag_filter, platform_filter, since_filter) catch |err| {
91
92
stats.recordError();
92
93
return err;
93
94
};
+2
-1
docs/search-architecture.md
+2
-1
docs/search-architecture.md
···
21
21
↓
22
22
buildFtsQuery(): "crypto OR casino*"
23
23
↓
24
-
FTS5 MATCH query with BM25 ranking
24
+
FTS5 MATCH query with BM25 + recency decay
25
25
↓
26
26
results with snippet()
27
27
```
···
30
30
- **OR between terms** for better recall (deliberate, see commit 35ad4b5)
31
31
- **prefix match on last word** for type-ahead feel
32
32
- **unicode61 tokenizer** splits on non-alphanumeric (we match this in buildFtsQuery)
33
+
- **recency decay** boosts recent docs: `ORDER BY rank + (days_old / 30)`
33
34
34
35
### what's coupled to FTS5
35
36