thisismissem.social / slices
Highly ambitious ATProtocol AppView service and sdks
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

index actors based on slice, add getActors method to generated client

Chad Miller 5d086fda 080d8fe1

+317 -39
+18
api/migrations/006_slice_actors.sql
··· 1 + -- Add slice_uri column to actor table to associate actors with specific slices 2 + ALTER TABLE "actor" ADD COLUMN "slice_uri" TEXT; 3 + 4 + -- Create a new primary key that includes both did and slice_uri 5 + -- First drop the existing primary key 6 + ALTER TABLE "actor" DROP CONSTRAINT "actor_pkey"; 7 + 8 + -- Add the new composite primary key 9 + ALTER TABLE "actor" ADD PRIMARY KEY ("did", "slice_uri"); 10 + 11 + -- Create new indexes for slice-based queries 12 + CREATE INDEX IF NOT EXISTS idx_actor_slice_uri ON "actor"("slice_uri"); 13 + CREATE INDEX IF NOT EXISTS idx_actor_slice_handle ON "actor"("slice_uri", "handle"); 14 + CREATE INDEX IF NOT EXISTS idx_actor_slice_indexed_at ON "actor"("slice_uri", "indexedAt"); 15 + 16 + -- Update existing records to have a default slice_uri (if any exist) 17 + -- This will need to be handled manually or with a data migration 18 + -- For now, we'll leave existing records as-is since they'll have NULL slice_uri
+56
api/scripts/generate-typescript.ts
··· 154 154 ], 155 155 }); 156 156 157 + // GetActorsResponse interface 158 + sourceFile.addInterface({ 159 + name: "GetActorsResponse", 160 + isExported: true, 161 + properties: [ 162 + { name: "actors", type: "Actor[]" }, 163 + { name: "cursor", type: "string", hasQuestionToken: true }, 164 + ], 165 + }); 166 + 157 167 // ListRecordsParams interface (generic with sort fields) 158 168 sourceFile.addInterface({ 159 169 name: "ListRecordsParams", ··· 175 185 properties: [{ name: "uri", type: "string" }], 176 186 }); 177 187 188 + // GetActorsParams interface 189 + sourceFile.addInterface({ 190 + name: "GetActorsParams", 191 + isExported: true, 192 + properties: [ 193 + { name: "search", type: "string", hasQuestionToken: true }, 194 + { name: "dids", type: "string[]", hasQuestionToken: true }, 195 + { name: "limit", type: "number", hasQuestionToken: true }, 196 + { name: "cursor", type: "string", hasQuestionToken: true }, 197 + ], 198 + }); 199 + 178 200 // SearchRecordsParams interface (generic with sort fields) 179 201 sourceFile.addInterface({ 180 202 name: "SearchRecordsParams", ··· 199 221 { name: "did", type: "string" }, 200 222 { name: "collection", type: "string" }, 201 223 { name: "value", type: "Record<string, unknown>" }, 224 + { name: "indexedAt", type: "string" }, 225 + ], 226 + }); 227 + 228 + // Actor interface (used in getActors response) 229 + sourceFile.addInterface({ 230 + name: "Actor", 231 + isExported: true, 232 + properties: [ 233 + { name: "did", type: "string" }, 234 + { name: "handle", type: "string", hasQuestionToken: true }, 235 + { name: "sliceUri", type: "string" }, 202 236 { name: "indexedAt", type: "string" }, 203 237 ], 204 238 }); ··· 940 974 `return await this.makeRequest<SliceRecordsOutput>('social.slices.slice.records', 'POST', params);`, 941 975 ], 942 976 }); 977 + 978 + classDeclaration.addMethod({ 979 + name: "getActors", 980 + parameters: [{ name: "params", type: "GetActorsParams", hasQuestionToken: true }], 981 + returnType: "Promise<GetActorsResponse>", 982 + isAsync: true, 983 + statements: [ 984 + `const requestParams = { ...params, slice: this.sliceUri };`, 985 + `return await this.makeRequest<GetActorsResponse>('social.slices.slice.getActors', 'GET', requestParams);`, 986 + ], 987 + }); 943 988 } 944 989 945 990 // Add sync methods to the social.slices.slice class ··· 978 1023 979 1024 // Add blob upload method to the main AtProtoClient 980 1025 if (className === "Client") { 1026 + classDeclaration.addMethod({ 1027 + name: "getActors", 1028 + parameters: [{ name: "params", type: "GetActorsParams" }], 1029 + returnType: "Promise<GetActorsResponse>", 1030 + isAsync: true, 1031 + statements: [ 1032 + `const requestParams = { ...params, slice: this.sliceUri };`, 1033 + `return await this.makeRequest<GetActorsResponse>('social.slices.slice.getActors', 'GET', requestParams);`, 1034 + ], 1035 + }); 1036 + 981 1037 classDeclaration.addMethod({ 982 1038 name: "uploadBlob", 983 1039 parameters: [{ name: "request", type: "UploadBlobRequest" }],
+120 -15
api/src/database.rs
··· 448 448 449 449 for actor in actors { 450 450 sqlx::query!( 451 - r#"INSERT INTO "actor" ("did", "handle", "indexed_at") 452 - VALUES ($1, $2, $3) 453 - ON CONFLICT ("did") 451 + r#"INSERT INTO "actor" ("did", "handle", "slice_uri", "indexed_at") 452 + VALUES ($1, $2, $3, $4) 453 + ON CONFLICT ("did", "slice_uri") 454 454 DO UPDATE SET 455 455 "handle" = EXCLUDED."handle", 456 456 "indexed_at" = EXCLUDED."indexed_at""#, 457 457 actor.did, 458 458 actor.handle, 459 + actor.slice_uri, 459 460 actor.indexed_at 460 461 ) 461 462 .execute(&mut *tx) ··· 547 548 pub async fn get_slice_total_actors(&self, slice_uri: &str) -> Result<i64, DatabaseError> { 548 549 let count = sqlx::query!( 549 550 r#" 550 - WITH slice_collections AS ( 551 - SELECT DISTINCT 552 - json->>'nsid' as collection_nsid 553 - FROM record 554 - WHERE collection = 'social.slices.lexicon' 555 - AND json->>'slice' = $1 556 - AND json->>'nsid' IS NOT NULL 557 - AND (json->>'definitions')::jsonb->'main'->>'type' = 'record' 558 - ) 559 - SELECT COUNT(DISTINCT r.did) as count 560 - FROM record r 561 - INNER JOIN slice_collections sc ON r.collection = sc.collection_nsid 551 + SELECT COUNT(*) as count 552 + FROM actor 553 + WHERE slice_uri = $1 562 554 "#, 563 555 slice_uri 564 556 ) ··· 566 558 .await?; 567 559 568 560 Ok(count.count.unwrap_or(0)) 561 + } 562 + 563 + pub async fn get_slice_actors( 564 + &self, 565 + slice_uri: &str, 566 + search: Option<&str>, 567 + dids: Option<&Vec<String>>, 568 + limit: Option<i32>, 569 + cursor: Option<&str>, 570 + ) -> Result<(Vec<Actor>, Option<String>), DatabaseError> { 571 + let limit = limit.unwrap_or(50).min(100); // Cap at 100 572 + 573 + // If specific DIDs are requested, prioritize that over search and pagination 574 + let records = if let Some(did_list) = dids { 575 + sqlx::query_as!( 576 + Actor, 577 + r#" 578 + SELECT did, handle, slice_uri, indexed_at 579 + FROM actor 580 + WHERE slice_uri = $1 AND did = ANY($2) 581 + ORDER BY did ASC 582 + "#, 583 + slice_uri, 584 + did_list 585 + ) 586 + .fetch_all(&self.pool) 587 + .await? 588 + } else { 589 + // Handle search and pagination when not filtering by specific DIDs 590 + match (search, cursor) { 591 + (Some(search_term), Some(cursor_did)) => { 592 + sqlx::query_as!( 593 + Actor, 594 + r#" 595 + SELECT did, handle, slice_uri, indexed_at 596 + FROM actor 597 + WHERE slice_uri = $1 598 + AND (handle ILIKE '%' || $2 || '%' OR did ILIKE '%' || $2 || '%') 599 + AND did > $3 600 + ORDER BY did ASC 601 + LIMIT $4 602 + "#, 603 + slice_uri, 604 + search_term, 605 + cursor_did, 606 + limit as i64 607 + ) 608 + .fetch_all(&self.pool) 609 + .await? 610 + }, 611 + (Some(search_term), None) => { 612 + sqlx::query_as!( 613 + Actor, 614 + r#" 615 + SELECT did, handle, slice_uri, indexed_at 616 + FROM actor 617 + WHERE slice_uri = $1 618 + AND (handle ILIKE '%' || $2 || '%' OR did ILIKE '%' || $2 || '%') 619 + ORDER BY did ASC 620 + LIMIT $3 621 + "#, 622 + slice_uri, 623 + search_term, 624 + limit as i64 625 + ) 626 + .fetch_all(&self.pool) 627 + .await? 628 + }, 629 + (None, Some(cursor_did)) => { 630 + sqlx::query_as!( 631 + Actor, 632 + r#" 633 + SELECT did, handle, slice_uri, indexed_at 634 + FROM actor 635 + WHERE slice_uri = $1 AND did > $2 636 + ORDER BY did ASC 637 + LIMIT $3 638 + "#, 639 + slice_uri, 640 + cursor_did, 641 + limit as i64 642 + ) 643 + .fetch_all(&self.pool) 644 + .await? 645 + }, 646 + (None, None) => { 647 + sqlx::query_as!( 648 + Actor, 649 + r#" 650 + SELECT did, handle, slice_uri, indexed_at 651 + FROM actor 652 + WHERE slice_uri = $1 653 + ORDER BY did ASC 654 + LIMIT $2 655 + "#, 656 + slice_uri, 657 + limit as i64 658 + ) 659 + .fetch_all(&self.pool) 660 + .await? 661 + }, 662 + } 663 + }; 664 + 665 + // Generate cursor from the last record if there are any records 666 + // But not when filtering by specific DIDs (no pagination needed) 667 + let cursor = if records.is_empty() || dids.is_some() { 668 + None 669 + } else { 670 + records.last().map(|actor| actor.did.clone()) 671 + }; 672 + 673 + Ok((records, cursor)) 569 674 } 570 675 571 676 pub async fn get_slice_lexicon_count(&self, slice_uri: &str) -> Result<i64, DatabaseError> {
+44
api/src/handler_xrpc_dynamic.rs
··· 47 47 pub sort: Option<String>, 48 48 } 49 49 50 + #[derive(Deserialize)] 51 + pub struct GetActorsParams { 52 + pub slice: String, 53 + pub search: Option<String>, 54 + pub dids: Option<String>, // Will be comma-separated string from query params 55 + pub limit: Option<i32>, 56 + pub cursor: Option<String>, 57 + } 58 + 50 59 51 60 52 61 // Dynamic XRPC handler that routes based on method name (for GET requests) ··· 65 74 } else if method.ends_with(".searchRecords") { 66 75 let collection = method.trim_end_matches(".searchRecords").to_string(); 67 76 dynamic_search_records_impl(collection, state, params).await 77 + } else if method == "social.slices.slice.getActors" { 78 + dynamic_get_actors_impl(state, params).await 68 79 } else { 69 80 Err(StatusCode::NOT_FOUND) 70 81 } ··· 415 426 416 427 Ok(Json(serde_json::json!({}))) 417 428 } 429 + 430 + // Implementation for getActors 431 + async fn dynamic_get_actors_impl( 432 + state: AppState, 433 + params: serde_json::Value, 434 + ) -> Result<Json<serde_json::Value>, StatusCode> { 435 + let get_actors_params: GetActorsParams = serde_json::from_value(params) 436 + .map_err(|_| StatusCode::BAD_REQUEST)?; 437 + 438 + // Handle dids parameter - split comma-separated string if present 439 + let dids_param = if let Some(dids_str) = get_actors_params.dids.as_ref() { 440 + Some(dids_str.split(',').map(|s| s.trim().to_string()).collect()) 441 + } else { 442 + None 443 + }; 444 + 445 + match state.database.get_slice_actors( 446 + &get_actors_params.slice, 447 + get_actors_params.search.as_deref(), 448 + dids_param.as_ref(), 449 + get_actors_params.limit, 450 + get_actors_params.cursor.as_deref(), 451 + ).await { 452 + Ok((actors, cursor)) => { 453 + let response = serde_json::json!({ 454 + "actors": actors, 455 + "cursor": cursor 456 + }); 457 + Ok(Json(response)) 458 + }, 459 + Err(_) => Err(StatusCode::INTERNAL_SERVER_ERROR), 460 + } 461 + }
+1
api/src/jobs.rs
··· 56 56 // Perform the sync 57 57 match sync_service 58 58 .backfill_collections( 59 + &payload.slice_uri, 59 60 payload.params.collections.as_deref(), 60 61 payload.params.external_collections.as_deref(), 61 62 payload.params.repos.as_deref(),
+1
api/src/models.rs
··· 55 55 pub struct Actor { 56 56 pub did: String, 57 57 pub handle: Option<String>, 58 + pub slice_uri: String, 58 59 pub indexed_at: String, 59 60 } 60 61
+4 -3
api/src/sync.rs
··· 66 66 67 67 68 68 69 - pub async fn backfill_collections(&self, collections: Option<&[String]>, external_collections: Option<&[String]>, repos: Option<&[String]>) -> Result<(i64, i64), SyncError> { 69 + pub async fn backfill_collections(&self, slice_uri: &str, collections: Option<&[String]>, external_collections: Option<&[String]>, repos: Option<&[String]>) -> Result<(i64, i64), SyncError> { 70 70 info!("🔄 Starting backfill operation"); 71 71 72 72 let primary_collections = collections.map(|c| c.to_vec()).unwrap_or_default(); ··· 203 203 204 204 // Index actors first (like the TypeScript version) 205 205 info!("📝 Indexing actors..."); 206 - self.index_actors(&valid_repos, &atp_map).await?; 206 + self.index_actors(slice_uri, &valid_repos, &atp_map).await?; 207 207 info!("✓ Indexed {} actors", valid_repos.len()); 208 208 209 209 // Single batch insert for new/changed records only ··· 421 421 } 422 422 } 423 423 424 - async fn index_actors(&self, repos: &[String], atp_map: &std::collections::HashMap<String, AtpData>) -> Result<(), SyncError> { 424 + async fn index_actors(&self, slice_uri: &str, repos: &[String], atp_map: &std::collections::HashMap<String, AtpData>) -> Result<(), SyncError> { 425 425 let mut actors = Vec::new(); 426 426 let now = chrono::Utc::now().to_rfc3339(); 427 427 ··· 430 430 actors.push(Actor { 431 431 did: atp_data.did.clone(), 432 432 handle: atp_data.handle.clone(), 433 + slice_uri: slice_uri.to_string(), 433 434 indexed_at: now.clone(), 434 435 }); 435 436 }
+2 -2
frontend/deno.json
··· 1 1 { 2 2 "tasks": { 3 - "start": "deno run -A --env-file=.env --unstable-kv src/main.ts", 4 - "dev": "deno run -A --env-file=.env --unstable-kv --watch src/main.ts" 3 + "start": "deno run -A --env-file=.env src/main.ts", 4 + "dev": "deno run -A --env-file=.env --watch src/main.ts" 5 5 }, 6 6 "fmt": { 7 7 "useTabs": false,
+67 -9
frontend/src/client.ts
··· 1 1 // Generated TypeScript client for AT Protocol records 2 - // Generated at: 2025-08-27 05:10:34 UTC 2 + // Generated at: 2025-08-28 00:21:00 UTC 3 3 // Lexicons: 3 4 4 5 5 /** ··· 41 41 cursor?: string; 42 42 } 43 43 44 - export interface ListRecordsParams { 44 + export interface GetActorsResponse { 45 + actors: Actor[]; 46 + cursor?: string; 47 + } 48 + 49 + export interface ListRecordsParams<TSortField extends string = string> { 45 50 author?: string; 51 + authors?: string[]; 46 52 limit?: number; 47 53 cursor?: string; 54 + sort?: 55 + | `${TSortField}:${"asc" | "desc"}` 56 + | `${TSortField}:${"asc" | "desc"},${TSortField}:${"asc" | "desc"}`; 48 57 } 49 58 50 59 export interface GetRecordParams { 51 60 uri: string; 52 61 } 53 62 54 - export interface SearchRecordsParams { 63 + export interface GetActorsParams { 64 + search?: string; 65 + dids?: string[]; 66 + limit?: number; 67 + cursor?: string; 68 + } 69 + 70 + export interface SearchRecordsParams<TSortField extends string = string> { 55 71 query: string; 56 72 field?: string; 57 73 limit?: number; 58 74 cursor?: string; 75 + sort?: 76 + | `${TSortField}:${"asc" | "desc"}` 77 + | `${TSortField}:${"asc" | "desc"},${TSortField}:${"asc" | "desc"}`; 59 78 } 60 79 61 80 export interface IndexedRecord { ··· 64 83 did: string; 65 84 collection: string; 66 85 value: Record<string, unknown>; 86 + indexedAt: string; 87 + } 88 + 89 + export interface Actor { 90 + did: string; 91 + handle?: string; 92 + sliceUri: string; 67 93 indexedAt: string; 68 94 } 69 95 ··· 194 220 createdAt: string; 195 221 } 196 222 223 + export type SocialSlicesSliceRecordSortFields = "name" | "createdAt"; 224 + 197 225 export interface SocialSlicesLexiconRecord { 198 226 /** Namespaced identifier for the lexicon */ 199 227 nsid: string; ··· 207 235 slice: string; 208 236 } 209 237 238 + export type SocialSlicesLexiconRecordSortFields = 239 + | "nsid" 240 + | "definitions" 241 + | "createdAt" 242 + | "updatedAt" 243 + | "slice"; 244 + 210 245 export interface SocialSlicesActorProfileRecord { 211 246 displayName?: string; 212 247 /** Free-form profile description text. */ ··· 215 250 avatar?: BlobRef; 216 251 createdAt?: string; 217 252 } 253 + 254 + export type SocialSlicesActorProfileRecordSortFields = 255 + | "displayName" 256 + | "description" 257 + | "createdAt"; 218 258 219 259 class BaseClient { 220 260 protected readonly baseUrl: string; ··· 339 379 } 340 380 341 381 async listRecords( 342 - params?: ListRecordsParams 382 + params?: ListRecordsParams<SocialSlicesSliceRecordSortFields> 343 383 ): Promise<ListRecordsResponse<SocialSlicesSliceRecord>> { 344 384 const requestParams = { ...params, slice: this.sliceUri }; 345 385 return await this.makeRequest<ListRecordsResponse<SocialSlicesSliceRecord>>( ··· 361 401 } 362 402 363 403 async searchRecords( 364 - params: SearchRecordsParams 404 + params: SearchRecordsParams<SocialSlicesSliceRecordSortFields> 365 405 ): Promise<ListRecordsResponse<SocialSlicesSliceRecord>> { 366 406 const requestParams = { ...params, slice: this.sliceUri }; 367 407 return await this.makeRequest<ListRecordsResponse<SocialSlicesSliceRecord>>( ··· 428 468 ); 429 469 } 430 470 471 + async getActors(params?: GetActorsParams): Promise<GetActorsResponse> { 472 + const requestParams = { ...params, slice: this.sliceUri }; 473 + return await this.makeRequest<GetActorsResponse>( 474 + "social.slices.slice.getActors", 475 + "GET", 476 + requestParams 477 + ); 478 + } 479 + 431 480 async startSync(params: BulkSyncParams): Promise<SyncJobResponse> { 432 481 const requestParams = { ...params, slice: this.sliceUri }; 433 482 return await this.makeRequest<SyncJobResponse>( ··· 465 514 } 466 515 467 516 async listRecords( 468 - params?: ListRecordsParams 517 + params?: ListRecordsParams<SocialSlicesLexiconRecordSortFields> 469 518 ): Promise<ListRecordsResponse<SocialSlicesLexiconRecord>> { 470 519 const requestParams = { ...params, slice: this.sliceUri }; 471 520 return await this.makeRequest< ··· 485 534 } 486 535 487 536 async searchRecords( 488 - params: SearchRecordsParams 537 + params: SearchRecordsParams<SocialSlicesLexiconRecordSortFields> 489 538 ): Promise<ListRecordsResponse<SocialSlicesLexiconRecord>> { 490 539 const requestParams = { ...params, slice: this.sliceUri }; 491 540 return await this.makeRequest< ··· 538 587 } 539 588 540 589 async listRecords( 541 - params?: ListRecordsParams 590 + params?: ListRecordsParams<SocialSlicesActorProfileRecordSortFields> 542 591 ): Promise<ListRecordsResponse<SocialSlicesActorProfileRecord>> { 543 592 const requestParams = { ...params, slice: this.sliceUri }; 544 593 return await this.makeRequest< ··· 556 605 } 557 606 558 607 async searchRecords( 559 - params: SearchRecordsParams 608 + params: SearchRecordsParams<SocialSlicesActorProfileRecordSortFields> 560 609 ): Promise<ListRecordsResponse<SocialSlicesActorProfileRecord>> { 561 610 const requestParams = { ...params, slice: this.sliceUri }; 562 611 return await this.makeRequest< ··· 655 704 this.sliceUri = sliceUri; 656 705 this.social = new SocialClient(baseUrl, sliceUri, oauthClient); 657 706 this.oauth = this.oauthClient; 707 + } 708 + 709 + async getActors(params: GetActorsParams): Promise<GetActorsResponse> { 710 + const requestParams = { ...params, slice: this.sliceUri }; 711 + return await this.makeRequest<GetActorsResponse>( 712 + "social.slices.slice.getActors", 713 + "GET", 714 + requestParams 715 + ); 658 716 } 659 717 660 718 uploadBlob(request: UploadBlobRequest): Promise<UploadBlobResponse> {
+4 -10
frontend/src/pages/SliceSyncPage.tsx
··· 73 73 name="external_collections" 74 74 rows={4} 75 75 className="block w-full border border-gray-300 rounded-md px-3 py-2" 76 - placeholder="Add external collections not in your slice lexicons: 77 - 78 - app.bsky.feed.post 79 - app.bsky.actor.profile 80 - com.atproto.repo.strongRef" 76 + placeholder="Add external collections e.g. app.bsky.actor.profile to sync collections not in your domain" 81 77 /> 82 - <p className="text-xs text-gray-500 mt-1"> 83 - These collections will be synced even if they're not defined in your slice lexicons 84 - </p> 85 78 </div> 86 79 87 80 <div> ··· 122 115 </div> 123 116 124 117 {/* Job History */} 125 - <div 118 + <div 126 119 hx-get={`/api/slices/${sliceId}/job-history`} 127 120 hx-trigger="load, every 10s" 128 121 hx-swap="innerHTML" ··· 137 130 </h3> 138 131 <ul className="text-blue-700 space-y-1 text-sm"> 139 132 <li> 140 - • Collections from your slice lexicons are automatically loaded above 133 + • Collections from your slice lexicons are automatically loaded 134 + above 141 135 </li> 142 136 <li> 143 137 • Use External Collections to sync popular collections like{" "}