search for standard sites
pub-search.waow.tech
search
zig
blog
atproto
feat: dedupe documents by (did, rkey) across collections
When platforms dual-publish to both their own schema and site.standard
(like pckt does), the same content gets indexed under different URIs:
- at://did/blog.pckt.document/abc123
- at://did/site.standard.document/abc123
This adds:
- UNIQUE index on (did, rkey) for documents and publications
- Dedupe logic in insertDocument/insertPublication that cleans up
old records when same (did, rkey) appears with a different URI
The last-seen URI wins, so site.standard takes precedence if it
arrives after the platform-specific record.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
+33
+5
backend/src/db/schema.zig
+5
backend/src/db/schema.zig
···
134
134
client.exec("UPDATE publications SET source_collection = 'pub.leaflet.publication' WHERE source_collection IS NULL", &.{}) catch {};
135
135
136
136
// vector embeddings column already added by backfill script
137
+
138
+
// dedupe index: same (did, rkey) across collections = same document
139
+
// e.g., pub.leaflet.document/abc and site.standard.document/abc are the same content
140
+
client.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_documents_did_rkey ON documents(did, rkey)", &.{}) catch {};
141
+
client.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_publications_did_rkey ON publications(did, rkey)", &.{}) catch {};
137
142
}
+28
backend/src/indexer.zig
+28
backend/src/indexer.zig
···
15
15
) !void {
16
16
const c = db.getClient() orelse return error.NotInitialized;
17
17
18
+
// dedupe: if (did, rkey) exists with different uri, clean up old record first
19
+
// this handles cross-collection duplicates (e.g., pub.leaflet.document + site.standard.document)
20
+
if (c.query("SELECT uri FROM documents WHERE did = ? AND rkey = ?", &.{ did, rkey })) |result_val| {
21
+
var result = result_val;
22
+
defer result.deinit();
23
+
if (result.first()) |row| {
24
+
const old_uri = row.text(0);
25
+
if (!std.mem.eql(u8, old_uri, uri)) {
26
+
c.exec("DELETE FROM documents_fts WHERE uri = ?", &.{old_uri}) catch {};
27
+
c.exec("DELETE FROM document_tags WHERE document_uri = ?", &.{old_uri}) catch {};
28
+
c.exec("DELETE FROM documents WHERE uri = ?", &.{old_uri}) catch {};
29
+
}
30
+
}
31
+
} else |_| {}
32
+
18
33
try c.exec(
19
34
"INSERT OR REPLACE INTO documents (uri, did, rkey, title, content, created_at, publication_uri, platform, source_collection) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
20
35
&.{ uri, did, rkey, title, content, created_at orelse "", publication_uri orelse "", platform, source_collection },
···
46
61
base_path: ?[]const u8,
47
62
) !void {
48
63
const c = db.getClient() orelse return error.NotInitialized;
64
+
65
+
// dedupe: if (did, rkey) exists with different uri, clean up old record first
66
+
if (c.query("SELECT uri FROM publications WHERE did = ? AND rkey = ?", &.{ did, rkey })) |result_val| {
67
+
var result = result_val;
68
+
defer result.deinit();
69
+
if (result.first()) |row| {
70
+
const old_uri = row.text(0);
71
+
if (!std.mem.eql(u8, old_uri, uri)) {
72
+
c.exec("DELETE FROM publications_fts WHERE uri = ?", &.{old_uri}) catch {};
73
+
c.exec("DELETE FROM publications WHERE uri = ?", &.{old_uri}) catch {};
74
+
}
75
+
}
76
+
} else |_| {}
49
77
50
78
try c.exec(
51
79
"INSERT OR REPLACE INTO publications (uri, did, rkey, name, description, base_path) VALUES (?, ?, ?, ?, ?, ?)",