feat: cross-platform dedup + date filter + load more count · zzstoatzz.io/leaflet-search@e10929f

+132 -7

2 changed files

expand all

backend

src

search.zig

site

index.html

+44 -3

backend/src/search.zig

··· 283 283 return searchKeyword(alloc, query, tag_filter, platform_filter, since_filter); 284 284 } 285 285 286 + /// Check if we've already seen a result from the same author with the same title. 287 + /// Used to collapse cross-platform duplicates (same content published to multiple ATProto apps). 288 + fn isDuplicateAuthorTitle(seen: *std.StringHashMap(void), alloc: Allocator, did: []const u8, title: []const u8) !bool { 289 + if (did.len == 0 or title.len == 0) return false; 290 + const key = try std.fmt.allocPrint(alloc, "{s}\x00{s}", .{ did, title }); 291 + const result = try seen.getOrPut(key); 292 + if (result.found_existing) { 293 + alloc.free(key); 294 + return true; 295 + } 296 + return false; 297 + } 298 + 286 299 /// Keyword search: FTS5 via local SQLite or Turso fallback. 287 300 fn searchKeyword(alloc: Allocator, query: []const u8, tag_filter: ?[]const u8, platform_filter: ?[]const u8, since_filter: ?[]const u8) ![]const u8 { 288 301 // try local SQLite first (faster for FTS queries) ··· 316 329 // track seen URIs for deduplication (content match + base_path match) 317 330 var seen_uris = std.StringHashMap(void).init(alloc); 318 331 defer seen_uris.deinit(); 332 + 333 + // track seen (did, title) pairs for cross-platform dedup 334 + var seen_authors = std.StringHashMap(void).init(alloc); 335 + defer seen_authors.deinit(); 319 336 320 337 // build batch of queries to execute in single HTTP request 321 338 var statements: [3]db.Client.Statement = undefined; ··· 364 381 if (doc_sql != null) { 365 382 for (batch.get(query_idx)) |row| { 366 383 const doc = Doc.fromRow(row); 384 + if (try isDuplicateAuthorTitle(&seen_authors, alloc, doc.did, doc.title)) continue; 367 385 const uri_dupe = try alloc.dupe(u8, doc.uri); 368 386 try seen_uris.put(uri_dupe, {}); 369 387 try jw.write(doc.toJson()); ··· 375 393 if (run_basepath) { 376 394 for (batch.get(query_idx)) |row| { 377 395 const doc = Doc.fromRow(row); 378 - if (!seen_uris.contains(doc.uri)) { 396 + if (!seen_uris.contains(doc.uri) and !try isDuplicateAuthorTitle(&seen_authors, alloc, doc.did, doc.title)) { 379 397 try jw.write(doc.toJson()); 380 398 } 381 399 } ··· 413 431 // track seen URIs for deduplication 414 432 var seen_uris = std.StringHashMap(void).init(alloc); 415 433 defer seen_uris.deinit(); 434 + 435 + // track seen (did, title) pairs for cross-platform dedup 436 + var seen_authors = std.StringHashMap(void).init(alloc); 437 + defer seen_authors.deinit(); 416 438 417 439 // document content search 418 440 if (platform_filter) |platform| { ··· 430 452 431 453 while (rows.next()) |row| { 432 454 const doc = Doc.fromLocalRow(row); 455 + if (try isDuplicateAuthorTitle(&seen_authors, alloc, doc.did, doc.title)) continue; 433 456 const uri_dupe = try alloc.dupe(u8, doc.uri); 434 457 try seen_uris.put(uri_dupe, {}); 435 458 try jw.write(doc.toJson()); ··· 450 473 451 474 while (bp_rows.next()) |row| { 452 475 const doc = Doc.fromLocalRow(row); 453 - if (!seen_uris.contains(doc.uri)) { 476 + if (!seen_uris.contains(doc.uri) and !try isDuplicateAuthorTitle(&seen_authors, alloc, doc.did, doc.title)) { 454 477 try jw.write(doc.toJson()); 455 478 } 456 479 } ··· 474 497 var doc_count: u32 = 0; 475 498 while (rows.next()) |row| { 476 499 const doc = Doc.fromLocalRow(row); 500 + if (try isDuplicateAuthorTitle(&seen_authors, alloc, doc.did, doc.title)) continue; 477 501 const uri_dupe = try alloc.dupe(u8, doc.uri); 478 502 try seen_uris.put(uri_dupe, {}); 479 503 try jw.write(doc.toJson()); ··· 501 525 var bp_count: u32 = 0; 502 526 while (bp_rows.next()) |row| { 503 527 const doc = Doc.fromLocalRow(row); 504 - if (!seen_uris.contains(doc.uri)) { 528 + if (!seen_uris.contains(doc.uri) and !try isDuplicateAuthorTitle(&seen_authors, alloc, doc.did, doc.title)) { 505 529 try jw.write(doc.toJson()); 506 530 } 507 531 bp_count += 1; ··· 658 682 var output: std.Io.Writer.Allocating = .init(alloc); 659 683 errdefer output.deinit(); 660 684 685 + // track seen (did, title) pairs for cross-platform dedup 686 + var seen_authors = std.StringHashMap(void).init(alloc); 687 + defer seen_authors.deinit(); 688 + 661 689 var jw: json.Stringify = .{ .writer = &output.writer }; 662 690 try jw.beginArray(); 663 691 var count: usize = 0; 664 692 for (results) |r| { 665 693 if (std.mem.eql(u8, r.uri, uri)) continue; 666 694 if (count >= limit) break; 695 + if (try isDuplicateAuthorTitle(&seen_authors, alloc, r.did, r.title)) continue; 667 696 try jw.write(SearchResultJson{ 668 697 .type = if (r.has_publication) "article" else "looseleaf", 669 698 .uri = r.uri, ··· 841 870 var jw: json.Stringify = .{ .writer = &output.writer }; 842 871 try jw.beginArray(); 843 872 873 + // track seen (did, title) pairs for cross-platform dedup 874 + var seen_authors = std.StringHashMap(void).init(alloc); 875 + defer seen_authors.deinit(); 876 + 844 877 for (scored.items[0..limit]) |entry| { 845 878 const bits = source_bits.get(entry.uri) orelse 0; 846 879 // prefer keyword version (has FTS snippet) 847 880 const obj = kw_objects.get(entry.uri) orelse sem_objects.get(entry.uri) orelse continue; 881 + 882 + // cross-platform dedup: skip if same author+title already emitted 883 + if (try isDuplicateAuthorTitle(&seen_authors, alloc, jsonStr(obj, "did"), jsonStr(obj, "title"))) continue; 848 884 849 885 const source_label: []const u8 = switch (bits) { 850 886 0b01 => "keyword", ··· 932 968 var seen: [20][]const u8 = undefined; 933 969 var seen_count: usize = 0; 934 970 971 + // track seen (did, title) pairs for cross-platform dedup 972 + var seen_authors = std.StringHashMap(void).init(alloc); 973 + defer seen_authors.deinit(); 974 + 935 975 for (results, 0..) |r, idx| { 936 976 if (filtered_count >= 20) break; 937 977 if (r.title.len == 0) continue; ··· 946 986 } 947 987 } 948 988 if (is_dup) continue; 989 + if (try isDuplicateAuthorTitle(&seen_authors, alloc, r.did, r.title)) continue; 949 990 if (seen_count < 20) { 950 991 seen[seen_count] = r.uri; 951 992 seen_count += 1;

+88 -4

site/index.html

··· 408 408 color: #d4956a; 409 409 } 410 410 411 + .date-filter { 412 + margin-bottom: 1rem; 413 + } 414 + 415 + .date-filter-label { 416 + font-size: 11px; 417 + color: #444; 418 + margin-bottom: 0.5rem; 419 + } 420 + 421 + .date-filter-list { 422 + display: flex; 423 + gap: 0.5rem; 424 + } 425 + 426 + .date-option { 427 + font-size: 11px; 428 + padding: 3px 8px; 429 + background: #151515; 430 + border: 1px solid #252525; 431 + border-radius: 3px; 432 + cursor: pointer; 433 + color: #777; 434 + } 435 + 436 + .date-option:hover { 437 + background: #1a1a1a; 438 + border-color: #333; 439 + color: #aaa; 440 + } 441 + 442 + .date-option.active { 443 + background: rgba(59, 130, 246, 0.2); 444 + border-color: #60a5fa; 445 + color: #60a5fa; 446 + } 447 + 411 448 .active-filter { 412 449 display: flex; 413 450 align-items: center; ··· 502 539 } 503 540 504 541 /* ensure minimum 44px touch targets */ 505 - .tag, .platform-option, .mode-option, .suggestion, input.tag-input { 542 + .tag, .platform-option, .mode-option, .date-option, .suggestion, input.tag-input { 506 543 min-height: 44px; 507 544 display: inline-flex; 508 545 align-items: center; ··· 561 598 } 562 599 563 600 /* tags wrap better on mobile */ 564 - .tags-list, .platform-filter-list { 601 + .tags-list, .platform-filter-list, .date-filter-list { 565 602 gap: 0.5rem; 566 603 } 567 604 ··· 580 617 581 618 /* ensure touch targets on tablets too */ 582 619 @media (hover: none) and (pointer: coarse) { 583 - .tag, .platform-option, .mode-option, .suggestion, .related-item, input.tag-input { 620 + .tag, .platform-option, .mode-option, .date-option, .suggestion, .related-item, input.tag-input { 584 621 min-height: 44px; 585 622 display: inline-flex; 586 623 align-items: center; ··· 607 644 608 645 <div id="platform-filter" class="platform-filter"></div> 609 646 647 + <div id="date-filter" class="date-filter"></div> 648 + 610 649 <div id="results" class="results"> 611 650 <div class="empty-state"> 612 651 <p>search atproto publishing platforms</p> ··· 630 669 const activeFilterDiv = document.getElementById('active-filter'); 631 670 const suggestionsDiv = document.getElementById('suggestions'); 632 671 const platformFilterDiv = document.getElementById('platform-filter'); 672 + const dateFilterDiv = document.getElementById('date-filter'); 633 673 const modeToggleDiv = document.getElementById('mode-toggle'); 634 674 let currentTag = null; 635 675 let currentPlatform = null; 676 + let currentSince = null; 636 677 let currentMode = 'keyword'; 637 678 let allTags = []; 638 679 let popularSearches = []; ··· 665 706 if (tag) searchUrl += `&tag=${encodeURIComponent(tag)}`; 666 707 if (platform) searchUrl += `&platform=${encodeURIComponent(platform)}`; 667 708 if (currentMode !== 'keyword') searchUrl += `&mode=${currentMode}`; 709 + const sinceDate = getSinceDate(); 710 + if (sinceDate) searchUrl += `&since=${encodeURIComponent(sinceDate)}`; 668 711 669 712 try { 670 713 const res = await fetch(searchUrl); ··· 759 802 760 803 // add "load more" button if there are more results 761 804 if (currentHasMore) { 762 - const loadMoreHtml = `<div class="load-more" style="text-align:center;padding:1rem"><button onclick="loadMore()" style="font-size:12px">load more</button></div>`; 805 + const remaining = extracted.total - totalShown; 806 + const remainingText = remaining > 0 ? ` (${remaining} remaining)` : ''; 807 + const loadMoreHtml = `<div class="load-more" style="text-align:center;padding:1rem"><button onclick="loadMore()" style="font-size:12px">load more${remainingText}</button></div>`; 763 808 resultsDiv.insertAdjacentHTML('beforeend', loadMoreHtml); 764 809 } 765 810 ··· 957 1002 if (q) params.set('q', q); 958 1003 if (currentTag) params.set('tag', currentTag); 959 1004 if (currentPlatform) params.set('platform', currentPlatform); 1005 + if (currentSince) params.set('since', currentSince); 960 1006 if (currentMode !== 'keyword') params.set('mode', currentMode); 961 1007 const url = params.toString() ? `?${params}` : '/'; 962 1008 history.pushState(null, '', url); ··· 1027 1073 platformFilterDiv.innerHTML = `<div class="platform-filter-label">filter by platform:</div><div class="platform-filter-list">${html}</div>`; 1028 1074 } 1029 1075 1076 + function renderDateFilter() { 1077 + const options = [ 1078 + { id: null, label: 'any' }, 1079 + { id: 'week', label: 'week' }, 1080 + { id: 'month', label: 'month' }, 1081 + { id: 'year', label: 'year' }, 1082 + ]; 1083 + const html = options.map(o => { 1084 + const active = currentSince === o.id; 1085 + return `<span class="date-option${active ? ' active' : ''}" onclick="setDateFilter(${o.id ? `'${o.id}'` : 'null'})">${o.label}</span>`; 1086 + }).join(''); 1087 + dateFilterDiv.innerHTML = `<div class="date-filter-label">date:</div><div class="date-filter-list">${html}</div>`; 1088 + } 1089 + 1090 + function setDateFilter(period) { 1091 + currentSince = currentSince === period ? null : period; 1092 + renderDateFilter(); 1093 + if (queryInput.value.trim() || currentTag || currentPlatform) { 1094 + doSearch(); 1095 + } 1096 + } 1097 + 1098 + function getSinceDate() { 1099 + if (!currentSince) return null; 1100 + const now = new Date(); 1101 + switch (currentSince) { 1102 + case 'week': now.setDate(now.getDate() - 7); break; 1103 + case 'month': now.setMonth(now.getMonth() - 1); break; 1104 + case 'year': now.setFullYear(now.getFullYear() - 1); break; 1105 + } 1106 + return now.toISOString(); 1107 + } 1108 + 1030 1109 function renderModeToggle() { 1031 1110 const modes = [ 1032 1111 { id: 'keyword', label: 'keyword' }, ··· 1164 1243 queryInput.value = params.get('q') || ''; 1165 1244 currentTag = params.get('tag') || null; 1166 1245 currentPlatform = params.get('platform') || null; 1246 + currentSince = params.get('since') || null; 1167 1247 currentMode = params.get('mode') || 'keyword'; 1168 1248 renderActiveFilter(); 1169 1249 renderTags(); 1170 1250 renderPlatformFilter(); 1251 + renderDateFilter(); 1171 1252 renderModeToggle(); 1172 1253 tagsDiv.style.display = currentMode === 'keyword' ? '' : 'none'; 1173 1254 if (queryInput.value || currentTag || currentPlatform) search(queryInput.value, currentTag, currentPlatform); ··· 1178 1259 const initialQuery = initialParams.get('q'); 1179 1260 const initialTag = initialParams.get('tag'); 1180 1261 const initialPlatform = initialParams.get('platform'); 1262 + const initialSince = initialParams.get('since'); 1181 1263 const initialMode = initialParams.get('mode'); 1182 1264 if (initialQuery) queryInput.value = initialQuery; 1183 1265 if (initialTag) currentTag = initialTag; 1184 1266 if (initialPlatform) currentPlatform = initialPlatform; 1267 + if (initialSince) currentSince = initialSince; 1185 1268 if (initialMode) currentMode = initialMode; 1186 1269 renderActiveFilter(); 1187 1270 renderPlatformFilter(); 1271 + renderDateFilter(); 1188 1272 renderModeToggle(); 1189 1273 tagsDiv.style.display = currentMode === 'keyword' ? '' : 'none'; 1190 1274

Configure Feed

Configure Feed