feat: API quality improvements — snippets, scores, v2 wrapper, pagination

+95 -18

backend/src/search.zig

··· 642 642 alloc.free(results); 643 643 } 644 644 645 + // collect filtered URIs for snippet lookup 646 + var uri_buf: [21][]const u8 = undefined; // limit+1 647 + var uri_count: usize = 0; 648 + for (results) |r| { 649 + if (std.mem.eql(u8, r.uri, uri)) continue; 650 + if (uri_count >= limit) break; 651 + uri_buf[uri_count] = r.uri; 652 + uri_count += 1; 653 + } 654 + 655 + const snippets = fetchSnippets(alloc, uri_buf[0..uri_count]); 656 + 645 657 // serialize, filtering out the source URI 646 658 var output: std.Io.Writer.Allocating = .init(alloc); 647 659 errdefer output.deinit(); ··· 657 669 .uri = r.uri, 658 670 .did = r.did, 659 671 .title = r.title, 660 - .snippet = "", 672 + .snippet = snippets.get(r.uri) orelse "", 661 673 .createdAt = r.created_at, 662 674 .rkey = r.rkey, 663 675 .basePath = r.base_path, ··· 805 817 } 806 818 }.lessThan); 807 819 808 - // 6. serialize top 20 with source annotation 820 + // 6. fetch content previews for semantic-only results (they have no FTS snippet) 821 + const limit = @min(scored.items.len, 20); 822 + var sem_uri_buf: [20][]const u8 = undefined; 823 + var sem_uri_count: usize = 0; 824 + for (scored.items[0..limit]) |entry| { 825 + const bits = source_bits.get(entry.uri) orelse 0; 826 + if (bits == 0b10) { // semantic-only 827 + const obj = sem_objects.get(entry.uri) orelse continue; 828 + const existing_snippet = jsonStr(obj, "snippet"); 829 + if (existing_snippet.len == 0 and sem_uri_count < 20) { 830 + sem_uri_buf[sem_uri_count] = entry.uri; 831 + sem_uri_count += 1; 832 + } 833 + } 834 + } 835 + const hybrid_snippets = fetchSnippets(alloc, sem_uri_buf[0..sem_uri_count]); 836 + 837 + // 7. serialize top 20 with source annotation 809 838 var output: std.Io.Writer.Allocating = .init(alloc); 810 839 errdefer output.deinit(); 811 840 812 841 var jw: json.Stringify = .{ .writer = &output.writer }; 813 842 try jw.beginArray(); 814 843 815 - const limit = @min(scored.items.len, 20); 816 844 for (scored.items[0..limit]) |entry| { 817 845 const bits = source_bits.get(entry.uri) orelse 0; 818 846 // prefer keyword version (has FTS snippet) ··· 825 853 else => "", 826 854 }; 827 855 856 + // for semantic-only results with empty snippet, use fetched preview 857 + const snippet = blk: { 858 + const existing = jsonStr(obj, "snippet"); 859 + if (existing.len > 0) break :blk existing; 860 + if (bits == 0b10) { 861 + break :blk hybrid_snippets.get(entry.uri) orelse ""; 862 + } 863 + break :blk existing; 864 + }; 865 + 828 866 try jw.beginObject(); 829 867 // write all standard fields from the source object 830 - inline for (.{ "type", "uri", "did", "title", "snippet", "rkey", "basePath", "platform", "path" }) |field| { 868 + inline for (.{ "type", "uri", "did", "title" }) |field| { 869 + try jw.objectField(field); 870 + try jw.write(jsonStr(obj, field)); 871 + } 872 + try jw.objectField("snippet"); 873 + try jw.write(snippet); 874 + inline for (.{ "rkey", "basePath", "platform", "path" }) |field| { 831 875 try jw.objectField(field); 832 876 try jw.write(jsonStr(obj, field)); 833 877 } ··· 835 879 try jw.write(jsonStr(obj, "createdAt")); 836 880 try jw.objectField("source"); 837 881 try jw.write(source_label); 882 + try jw.objectField("score"); 883 + try jw.write(entry.score); 838 884 try jw.endObject(); 839 885 } 840 886 ··· 880 926 alloc.free(results); 881 927 } 882 928 883 - // serialize results, filtering by distance + platform + dedup, capped at 20 884 - var output: std.Io.Writer.Allocating = .init(alloc); 885 - errdefer output.deinit(); 886 - 887 - // track seen URIs to deduplicate 929 + // first pass: filter and collect URIs for snippet lookup 930 + var filtered_indices: [20]usize = undefined; 931 + var filtered_count: usize = 0; 888 932 var seen: [20][]const u8 = undefined; 889 933 var seen_count: usize = 0; 890 934 891 - var jw: json.Stringify = .{ .writer = &output.writer }; 892 - try jw.beginArray(); 893 - var count: usize = 0; 894 - for (results) |r| { 895 - if (count >= 20) break; 896 - // skip documents with empty/test titles 935 + for (results, 0..) |r, idx| { 936 + if (filtered_count >= 20) break; 897 937 if (r.title.len == 0) continue; 898 938 if (platform_filter) |pf| { 899 939 if (!std.mem.eql(u8, r.platform, pf)) continue; 900 940 } 901 - // deduplicate by URI 902 941 var is_dup = false; 903 942 for (seen[0..seen_count]) |s| { 904 943 if (std.mem.eql(u8, s, r.uri)) { ··· 911 950 seen[seen_count] = r.uri; 912 951 seen_count += 1; 913 952 } 914 - count += 1; 953 + filtered_indices[filtered_count] = idx; 954 + filtered_count += 1; 955 + } 956 + 957 + // fetch content previews from local SQLite (arena-allocated, no explicit cleanup) 958 + const snippets = fetchSnippets(alloc, seen[0..seen_count]); 959 + 960 + // serialize results with snippets 961 + var output: std.Io.Writer.Allocating = .init(alloc); 962 + errdefer output.deinit(); 963 + 964 + var jw: json.Stringify = .{ .writer = &output.writer }; 965 + try jw.beginArray(); 966 + for (filtered_indices[0..filtered_count]) |idx| { 967 + const r = results[idx]; 915 968 try jw.write(SearchResultJson{ 916 969 .type = if (r.has_publication) "article" else "looseleaf", 917 970 .uri = r.uri, 918 971 .did = r.did, 919 972 .title = r.title, 920 - .snippet = "", 973 + .snippet = snippets.get(r.uri) orelse "", 921 974 .createdAt = r.created_at, 922 975 .rkey = r.rkey, 923 976 .basePath = r.base_path, ··· 928 981 try jw.endArray(); 929 982 930 983 return try output.toOwnedSlice(); 984 + } 985 + 986 + // --- snippet helpers (for semantic/similar results) --- 987 + 988 + /// Fetch content previews from local SQLite for a list of URIs. 989 + /// Returns a map of URI -> preview text (first 200 chars of content). 990 + /// Gracefully returns empty map if local db is unavailable. 991 + fn fetchSnippets(alloc: Allocator, uris: []const []const u8) std.StringHashMap([]const u8) { 992 + var map = std.StringHashMap([]const u8).init(alloc); 993 + const local = db.getLocalDb() orelse return map; 994 + for (uris) |uri| { 995 + var rows = local.query( 996 + "SELECT substr(content, 1, 200) FROM documents WHERE uri = ?", 997 + .{uri}, 998 + ) catch continue; 999 + defer rows.deinit(); 1000 + if (rows.next()) |row| { 1001 + const preview = row.text(0); 1002 + if (preview.len > 0) { 1003 + map.put(uri, preview) catch continue; 1004 + } 1005 + } 1006 + } 1007 + return map; 931 1008 } 932 1009 933 1010 // --- JSON helpers (for hybrid search parsing) ---

+142 -18

backend/src/server.zig

··· 68 68 return; 69 69 } 70 70 71 - if (mem.startsWith(u8, target, "/search")) { 71 + // extract path (before query string) for routing 72 + const path = if (mem.indexOf(u8, target, "?")) |qi| target[0..qi] else target; 73 + 74 + if (mem.startsWith(u8, path, "/search")) { 72 75 try handleSearch(request, target); 73 - } else if (mem.eql(u8, target, "/tags")) { 74 - try handleTags(request); 75 - } else if (mem.eql(u8, target, "/stats")) { 76 + } else if (mem.eql(u8, path, "/tags")) { 77 + try handleTags(request, target); 78 + } else if (mem.eql(u8, path, "/stats")) { 76 79 try handleStats(request); 77 - } else if (mem.eql(u8, target, "/health")) { 80 + } else if (mem.eql(u8, path, "/health")) { 78 81 try sendJson(request, "{\"status\":\"ok\"}"); 79 - } else if (mem.eql(u8, target, "/popular")) { 80 - try handlePopular(request); 81 - } else if (mem.eql(u8, target, "/dashboard")) { 82 + } else if (mem.eql(u8, path, "/popular")) { 83 + try handlePopular(request, target); 84 + } else if (mem.eql(u8, path, "/dashboard")) { 82 85 try handleDashboard(request); 83 - } else if (mem.eql(u8, target, "/api/dashboard")) { 86 + } else if (mem.eql(u8, path, "/api/dashboard")) { 84 87 try handleDashboardApi(request); 85 - } else if (mem.startsWith(u8, target, "/similar")) { 88 + } else if (mem.startsWith(u8, path, "/similar")) { 86 89 try handleSimilar(request, target); 87 - } else if (mem.eql(u8, target, "/activity")) { 90 + } else if (mem.eql(u8, path, "/activity")) { 88 91 try handleActivity(request); 89 92 } else { 90 93 try sendNotFound(request); ··· 104 107 const since_filter = parseQueryParam(alloc, target, "since") catch null; 105 108 const mode_str = parseQueryParam(alloc, target, "mode") catch null; 106 109 const mode = search.SearchMode.fromString(mode_str); 110 + const format = parseQueryParam(alloc, target, "format") catch "v1"; 111 + const limit_str = parseQueryParam(alloc, target, "limit") catch null; 112 + const offset_str = parseQueryParam(alloc, target, "offset") catch null; 113 + const limit = if (limit_str) |s| std.fmt.parseInt(usize, s, 10) catch 20 else 20; 114 + const offset = if (offset_str) |s| std.fmt.parseInt(usize, s, 10) catch 0 else 0; 107 115 108 116 // record per-mode latency 109 117 const timing_endpoint: metrics.timing.Endpoint = switch (mode) { ··· 135 143 }; 136 144 metrics.stats.recordSearch(query); 137 145 logfire.counter("search.requests", 1); 138 - try sendJson(request, results); 146 + 147 + if (mem.eql(u8, format, "v2")) { 148 + const wrapped = try wrapResponse(alloc, results, query, @tagName(mode), limit, offset); 149 + try sendJson(request, wrapped); 150 + } else { 151 + // v1: apply pagination by slicing the JSON array 152 + if (offset > 0 or limit < 40) { 153 + const paginated = try paginateJsonArray(alloc, results, limit, offset); 154 + try sendJson(request, paginated); 155 + } else { 156 + try sendJson(request, results); 157 + } 158 + } 139 159 } 140 160 141 - fn handleTags(request: *http.Server.Request) !void { 161 + fn handleTags(request: *http.Server.Request, target: []const u8) !void { 142 162 const start_time = std.time.microTimestamp(); 143 163 defer metrics.timing.record(.tags, start_time); 144 164 ··· 149 169 defer arena.deinit(); 150 170 const alloc = arena.allocator(); 151 171 172 + const format = parseQueryParam(alloc, target, "format") catch "v1"; 152 173 const tags = try getTags(alloc); 153 - try sendJson(request, tags); 174 + 175 + if (mem.eql(u8, format, "v2")) { 176 + const wrapped = try wrapResponse(alloc, tags, "", "tags", 100, 0); 177 + try sendJson(request, wrapped); 178 + } else { 179 + try sendJson(request, tags); 180 + } 154 181 } 155 182 156 - fn handlePopular(request: *http.Server.Request) !void { 183 + fn handlePopular(request: *http.Server.Request, target: []const u8) !void { 157 184 const start_time = std.time.microTimestamp(); 158 185 defer metrics.timing.record(.popular, start_time); 159 186 ··· 164 191 defer arena.deinit(); 165 192 const alloc = arena.allocator(); 166 193 194 + const format = parseQueryParam(alloc, target, "format") catch "v1"; 167 195 const popular = try getPopular(alloc, 5); 168 - try sendJson(request, popular); 196 + 197 + if (mem.eql(u8, format, "v2")) { 198 + const wrapped = try wrapResponse(alloc, popular, "", "popular", 100, 0); 199 + try sendJson(request, wrapped); 200 + } else { 201 + try sendJson(request, popular); 202 + } 169 203 } 170 204 171 205 // --- tags/popular query logic --- ··· 428 462 return; 429 463 }; 430 464 465 + const format = parseQueryParam(alloc, target, "format") catch "v1"; 466 + 431 467 // span attributes are copied internally, safe to use arena strings 432 468 const span = logfire.span("http.similar", .{ .uri = uri }); 433 469 defer span.end(); 434 470 435 471 const results = search.findSimilar(alloc, uri, 5) catch { 436 - try sendJson(request, "[]"); 472 + if (mem.eql(u8, format, "v2")) { 473 + try sendJson(request, "{\"results\":[],\"total\":0,\"hasMore\":false}"); 474 + } else { 475 + try sendJson(request, "[]"); 476 + } 437 477 return; 438 478 }; 439 479 440 - try sendJson(request, results); 480 + if (mem.eql(u8, format, "v2")) { 481 + const wrapped = try wrapResponse(alloc, results, "", "similar", 20, 0); 482 + try sendJson(request, wrapped); 483 + } else { 484 + try sendJson(request, results); 485 + } 486 + } 487 + 488 + /// Wrap a JSON array response in v2 envelope: {"results": [...], "total": N, "hasMore": bool} 489 + fn wrapResponse(alloc: Allocator, array_json: []const u8, query: []const u8, mode: []const u8, limit: usize, offset: usize) ![]const u8 { 490 + // parse the array to count items and apply pagination 491 + const parsed = json.parseFromSlice(json.Value, alloc, array_json, .{}) catch { 492 + return array_json; // fallback to raw if parse fails 493 + }; 494 + defer parsed.deinit(); 495 + 496 + const items = switch (parsed.value) { 497 + .array => |arr| arr.items, 498 + else => return array_json, 499 + }; 500 + 501 + const total = items.len; 502 + const start = @min(offset, total); 503 + const end = @min(start + limit, total); 504 + const has_more = end < total; 505 + 506 + var output: std.Io.Writer.Allocating = .init(alloc); 507 + errdefer output.deinit(); 508 + 509 + var jw: json.Stringify = .{ .writer = &output.writer }; 510 + try jw.beginObject(); 511 + 512 + try jw.objectField("results"); 513 + try jw.beginArray(); 514 + for (items[start..end]) |item| { 515 + try jw.write(item); 516 + } 517 + try jw.endArray(); 518 + 519 + try jw.objectField("total"); 520 + try jw.write(total); 521 + 522 + try jw.objectField("hasMore"); 523 + try jw.write(has_more); 524 + 525 + if (query.len > 0) { 526 + try jw.objectField("query"); 527 + try jw.write(query); 528 + } 529 + 530 + if (mode.len > 0) { 531 + try jw.objectField("mode"); 532 + try jw.write(mode); 533 + } 534 + 535 + try jw.endObject(); 536 + return try output.toOwnedSlice(); 537 + } 538 + 539 + /// Apply pagination to a JSON array (for v1 format with limit/offset) 540 + fn paginateJsonArray(alloc: Allocator, array_json: []const u8, limit: usize, offset: usize) ![]const u8 { 541 + const parsed = json.parseFromSlice(json.Value, alloc, array_json, .{}) catch { 542 + return array_json; 543 + }; 544 + defer parsed.deinit(); 545 + 546 + const items = switch (parsed.value) { 547 + .array => |arr| arr.items, 548 + else => return array_json, 549 + }; 550 + 551 + const total = items.len; 552 + const start = @min(offset, total); 553 + const end = @min(start + limit, total); 554 + 555 + var output: std.Io.Writer.Allocating = .init(alloc); 556 + errdefer output.deinit(); 557 + 558 + var jw: json.Stringify = .{ .writer = &output.writer }; 559 + try jw.beginArray(); 560 + for (items[start..end]) |item| { 561 + try jw.write(item); 562 + } 563 + try jw.endArray(); 564 + return try output.toOwnedSlice(); 441 565 } 442 566 443 567 fn handleActivity(request: *http.Server.Request) !void {

+13 -1

mcp/src/pub_search/_types.py

··· 17 17 rkey: str 18 18 basePath: str = "" 19 19 platform: Literal["leaflet", "pckt", "offprint", "greengale", "other"] = "leaflet" 20 + path: str = "" 21 + source: str = "" 22 + score: float = 0.0 20 23 21 24 @computed_field 22 25 @property 23 26 def url(self) -> str: 24 27 """web URL for this document.""" 25 - if self.basePath: 28 + if self.type == "publication" and self.basePath: 29 + return f"https://{self.basePath}" 30 + if self.platform == "leaflet" and self.basePath and self.rkey: 26 31 return f"https://{self.basePath}/{self.rkey}" 32 + if self.basePath and self.path: 33 + sep = "" if self.path.startswith("/") else "/" 34 + return f"https://{self.basePath}{sep}{self.path}" 35 + if self.platform == "leaflet" and self.did and self.rkey: 36 + return f"https://leaflet.pub/p/{self.did}/{self.rkey}" 37 + if self.uri: 38 + return f"https://pdsls.dev/{self.uri}" 27 39 return "" 28 40 29 41

+110 -14

mcp/src/pub_search/server.py

··· 27 27 28 28 ## tools 29 29 30 - - `search(query, tag, platform, since)` - full-text search with filters 30 + - `search(query, tag, platform, since)` - keyword search with filters 31 + - `search_semantic(query)` - meaning-based search (natural language queries) 32 + - `search_hybrid(query)` - combined keyword + semantic with source annotations 31 33 - `get_document(uri)` - fetch full content by AT-URI 32 - - `find_similar(uri)` - semantic similarity search 34 + - `find_similar(uri)` - find related documents 33 35 - `get_tags()` - available tags 34 36 - `get_stats()` - index statistics 35 37 - `get_popular()` - popular queries 36 38 37 39 ## workflow 38 40 39 - 1. `search("topic")` or `search("topic", platform="leaflet")` 41 + 1. `search("topic")` for keyword search, `search_hybrid("topic")` for best results 40 42 2. `get_document(uri)` for full text 41 43 3. `find_similar(uri)` for related content 42 44 45 + ## search modes 46 + 47 + - **keyword**: fast exact match (~100ms), supports tag/since filters 48 + - **semantic**: meaning-based (~500ms), good for natural language queries 49 + - **hybrid**: both combined with rank fusion, `source` field shows how each result was found 50 + 43 51 ## result types 44 52 45 53 - **article**: document in a publication ··· 59 67 - prefix matching on last word: "cat dog" matches "cat dogs" 60 68 - combine filters: `search("python", tag="tutorial", platform="leaflet")` 61 69 - use `since="2025-01-01"` for recent content 62 - - `find_similar(uri)` for semantic similarity (voyage-3-lite embeddings) 70 + - `search_semantic("natural language query")` for meaning-based search 71 + - `search_hybrid("query")` for best of both — results show `source` field 72 + - `find_similar(uri)` to discover related documents 63 73 - `get_tags()` to discover available tags 64 74 """ 65 75 ··· 72 82 Platform = Literal["leaflet", "pckt", "offprint", "greengale", "other"] 73 83 74 84 85 + def _extract_results(data: Any) -> list[dict[str, Any]]: 86 + """extract results array from API response (handles both v1 and v2 formats).""" 87 + if isinstance(data, dict) and "results" in data: 88 + return data["results"] 89 + if isinstance(data, list): 90 + return data 91 + return [] 92 + 93 + 75 94 @mcp.tool 76 95 async def search( 77 96 query: str = "", ··· 95 114 if not query and not tag: 96 115 return [] 97 116 98 - params: dict[str, Any] = {} 117 + params: dict[str, Any] = {"format": "v2", "limit": str(limit)} 99 118 if query: 100 119 params["q"] = query 101 120 if tag: ··· 108 127 async with get_http_client() as client: 109 128 response = await client.get("/search", params=params) 110 129 response.raise_for_status() 111 - results = response.json() 130 + data = response.json() 112 131 132 + results = _extract_results(data) 133 + return [SearchResult(**r) for r in results[:limit]] 134 + 135 + 136 + @mcp.tool 137 + async def search_semantic( 138 + query: str, 139 + platform: Platform | None = None, 140 + limit: int = 5, 141 + ) -> list[SearchResult]: 142 + """semantic search using vector embeddings. 143 + 144 + finds documents by meaning rather than exact keyword match. 145 + good for natural language queries like "essays about loneliness" 146 + or oblique descriptions like "guy from south africa with lots of kids". 147 + 148 + args: 149 + query: natural language query 150 + platform: filter by platform (leaflet, pckt, offprint, greengale, other) 151 + limit: max results (default 5, max 40) 152 + 153 + returns: 154 + list of results ranked by semantic similarity 155 + """ 156 + params: dict[str, Any] = {"q": query, "mode": "semantic", "format": "v2", "limit": str(limit)} 157 + if platform: 158 + params["platform"] = platform 159 + 160 + async with get_http_client() as client: 161 + response = await client.get("/search", params=params) 162 + response.raise_for_status() 163 + data = response.json() 164 + 165 + if isinstance(data, dict) and "error" in data: 166 + return [] 167 + 168 + results = _extract_results(data) 169 + return [SearchResult(**r) for r in results[:limit]] 170 + 171 + 172 + @mcp.tool 173 + async def search_hybrid( 174 + query: str, 175 + platform: Platform | None = None, 176 + limit: int = 5, 177 + ) -> list[SearchResult]: 178 + """hybrid search combining keyword and semantic results. 179 + 180 + runs both keyword (exact match) and semantic (meaning-based) search, 181 + then merges results using Reciprocal Rank Fusion. documents found by 182 + both methods rank highest. results include a `source` field indicating 183 + how each result was found: "keyword", "semantic", or "keyword+semantic". 184 + 185 + args: 186 + query: search query 187 + platform: filter by platform (leaflet, pckt, offprint, greengale, other) 188 + limit: max results (default 5, max 40) 189 + 190 + returns: 191 + list of results with source annotations, ranked by combined relevance 192 + """ 193 + params: dict[str, Any] = {"q": query, "mode": "hybrid", "format": "v2", "limit": str(limit)} 194 + if platform: 195 + params["platform"] = platform 196 + 197 + async with get_http_client() as client: 198 + response = await client.get("/search", params=params) 199 + response.raise_for_status() 200 + data = response.json() 201 + 202 + if isinstance(data, dict) and "error" in data: 203 + return [] 204 + 205 + results = _extract_results(data) 113 206 return [SearchResult(**r) for r in results[:limit]] 114 207 115 208 ··· 189 282 async def find_similar(uri: str, limit: int = 5) -> list[SearchResult]: 190 283 """find documents similar to a given document. 191 284 192 - uses vector similarity (voyage-3-lite embeddings) to find semantically 193 - related documents. great for discovering related content after finding 285 + uses vector similarity to find semantically related documents. 286 + great for discovering related content after finding 194 287 an interesting document. 195 288 196 289 args: ··· 201 294 list of similar documents with uri, title, and metadata 202 295 """ 203 296 async with get_http_client() as client: 204 - response = await client.get("/similar", params={"uri": uri}) 297 + response = await client.get("/similar", params={"uri": uri, "format": "v2"}) 205 298 response.raise_for_status() 206 - results = response.json() 299 + data = response.json() 207 300 301 + results = _extract_results(data) 208 302 return [SearchResult(**r) for r in results[:limit]] 209 303 210 304 ··· 219 313 list of tags with their document counts 220 314 """ 221 315 async with get_http_client() as client: 222 - response = await client.get("/tags") 316 + response = await client.get("/tags", params={"format": "v2"}) 223 317 response.raise_for_status() 224 - results = response.json() 318 + data = response.json() 225 319 320 + results = _extract_results(data) 226 321 return [Tag(**t) for t in results] 227 322 228 323 ··· 253 348 list of popular queries with search counts 254 349 """ 255 350 async with get_http_client() as client: 256 - response = await client.get("/popular") 351 + response = await client.get("/popular", params={"format": "v2"}) 257 352 response.raise_for_status() 258 - results = response.json() 353 + data = response.json() 259 354 355 + results = _extract_results(data) 260 356 return [PopularSearch(**p) for p in results[:limit]] 261 357 262 358

+1 -1

mcp/tests/test_mcp.py

··· 123 123 tools = await client.list_tools() 124 124 125 125 tool_names = {t.name for t in tools} 126 - expected = {"search", "get_document", "find_similar", "get_tags", "get_stats", "get_popular"} 126 + expected = {"search", "search_semantic", "search_hybrid", "get_document", "find_similar", "get_tags", "get_stats", "get_popular"} 127 127 assert expected == tool_names 128 128 129 129 async def test_list_prompts(self, client):

+71 -18

site/index.html

··· 637 637 let allTags = []; 638 638 let popularSearches = []; 639 639 const authorCache = new Map(); 640 - async function search(query, tag = null, platform = null) { 640 + let currentOffset = 0; 641 + let currentHasMore = false; 642 + const PAGE_SIZE = 20; 643 + 644 + // extract results array from API response (handles both v1 and v2 formats) 645 + function extractResults(parsed) { 646 + if (parsed.results && Array.isArray(parsed.results)) { 647 + return { results: parsed.results, total: parsed.total || parsed.results.length, hasMore: parsed.hasMore || false }; 648 + } 649 + if (Array.isArray(parsed)) { 650 + return { results: parsed, total: parsed.length, hasMore: false }; 651 + } 652 + return null; 653 + } 654 + 655 + async function search(query, tag = null, platform = null, append = false) { 641 656 if (!query.trim() && !tag && !platform) return; 642 657 643 - searchBtn.disabled = true; 644 - resultsDiv.innerHTML = `<div class="status">searching...</div>`; 658 + if (!append) { 659 + searchBtn.disabled = true; 660 + resultsDiv.innerHTML = `<div class="status">searching...</div>`; 661 + currentOffset = 0; 662 + } 645 663 646 - let searchUrl = `${API_URL}/search?q=${encodeURIComponent(query || '')}`; 664 + let searchUrl = `${API_URL}/search?q=${encodeURIComponent(query || '')}&format=v2&limit=${PAGE_SIZE}&offset=${currentOffset}`; 647 665 if (tag) searchUrl += `&tag=${encodeURIComponent(tag)}`; 648 666 if (platform) searchUrl += `&platform=${encodeURIComponent(platform)}`; 649 667 if (currentMode !== 'keyword') searchUrl += `&mode=${currentMode}`; ··· 651 669 try { 652 670 const res = await fetch(searchUrl); 653 671 const rawText = await res.text(); 654 - let results; 672 + let parsed; 655 673 656 674 try { 657 - results = JSON.parse(rawText); 675 + parsed = JSON.parse(rawText); 658 676 } catch (parseErr) { 659 677 resultsDiv.innerHTML = `<div class="status error">JSON parse error<pre style="text-align:left;font-size:0.7rem;overflow:auto;max-height:200px">${escapeHtml(rawText)}</pre></div>`; 660 678 return; 661 679 } 662 680 663 - if (results.error) { 664 - resultsDiv.innerHTML = `<div class="status error">${results.error}</div>`; 681 + if (parsed.error) { 682 + resultsDiv.innerHTML = `<div class="status error">${parsed.error}</div>`; 683 + return; 684 + } 685 + 686 + const extracted = extractResults(parsed); 687 + if (!extracted) { 688 + resultsDiv.innerHTML = `<div class="status error">unexpected response format</div>`; 665 689 return; 666 690 } 667 691 668 - if (results.length === 0) { 692 + const results = extracted.results; 693 + currentHasMore = extracted.hasMore; 694 + 695 + if (results.length === 0 && !append) { 669 696 resultsDiv.innerHTML = ` 670 697 <div class="empty-state"> 671 698 <p>no results${query ? ` for ${formatQueryForDisplay(query)}` : ''}${tag ? ` in #${escapeHtml(tag)}` : ''}${platform ? ` on ${escapeHtml(platform)}` : ''}</p> ··· 717 744 `; 718 745 } 719 746 720 - resultsDiv.innerHTML = html; 721 - statsDiv.textContent = `${results.length} result${results.length === 1 ? '' : 's'}`; 747 + if (append) { 748 + // remove existing "load more" button and related section before appending 749 + const existingLoadMore = resultsDiv.querySelector('.load-more'); 750 + if (existingLoadMore) existingLoadMore.remove(); 751 + resultsDiv.insertAdjacentHTML('beforeend', html); 752 + } else { 753 + resultsDiv.innerHTML = html; 754 + } 755 + 756 + const totalShown = resultsDiv.querySelectorAll('.result').length; 757 + statsDiv.textContent = `${totalShown} result${totalShown === 1 ? '' : 's'}${extracted.total > totalShown ? ` of ${extracted.total}` : ''}`; 722 758 resolveAuthors(results); 723 759 724 - if (results.length > 0 && results[0].uri) { 760 + // add "load more" button if there are more results 761 + if (currentHasMore) { 762 + const loadMoreHtml = `<div class="load-more" style="text-align:center;padding:1rem"><button onclick="loadMore()" style="font-size:12px">load more</button></div>`; 763 + resultsDiv.insertAdjacentHTML('beforeend', loadMoreHtml); 764 + } 765 + 766 + if (!append && results.length > 0 && results[0].uri) { 725 767 loadRelated(results[0]); 726 768 } 727 769 ··· 730 772 } finally { 731 773 searchBtn.disabled = false; 732 774 } 775 + } 776 + 777 + function loadMore() { 778 + currentOffset += PAGE_SIZE; 779 + search(queryInput.value, currentTag, currentPlatform, true); 733 780 } 734 781 735 782 async function resolveAuthors(results) { ··· 1049 1096 1050 1097 async function loadTags() { 1051 1098 try { 1052 - const res = await fetch(`${API_URL}/tags`); 1099 + const res = await fetch(`${API_URL}/tags?format=v2`); 1053 1100 const data = await res.json(); 1054 1101 if (data.error) { 1055 1102 console.error('failed to load tags:', data.error); 1056 1103 return; 1057 1104 } 1058 - allTags = data; 1105 + const extracted = extractResults(data); 1106 + allTags = extracted ? extracted.results : (Array.isArray(data) ? data : []); 1059 1107 renderTags(); 1060 1108 } catch (e) { 1061 1109 console.error('failed to load tags', e); ··· 1064 1112 1065 1113 async function loadPopular() { 1066 1114 try { 1067 - const res = await fetch(`${API_URL}/popular`); 1115 + const res = await fetch(`${API_URL}/popular?format=v2`); 1068 1116 const data = await res.json(); 1069 1117 if (!data.error) { 1070 - popularSearches = data.slice(0, 5); 1118 + const extracted = extractResults(data); 1119 + const items = extracted ? extracted.results : (Array.isArray(data) ? data : []); 1120 + popularSearches = items.slice(0, 5); 1071 1121 renderSuggestions(); 1072 1122 } 1073 1123 } catch (e) { ··· 1150 1200 resultsDiv.appendChild(loadingEl); 1151 1201 1152 1202 try { 1153 - const res = await fetch(`${API_URL}/similar?uri=${encodeURIComponent(topResult.uri)}`); 1154 - const related = await res.json(); 1203 + const res = await fetch(`${API_URL}/similar?uri=${encodeURIComponent(topResult.uri)}&format=v2`); 1204 + const data = await res.json(); 1155 1205 1156 1206 loadingEl.remove(); 1207 + 1208 + const extracted = extractResults(data); 1209 + const related = extracted ? extracted.results : (Array.isArray(data) ? data : []); 1157 1210 1158 1211 if (!related || related.length === 0) return; 1159 1212

Configure Feed

Configure Feed