Speed up avatar fetching · zzstoatzz.io/at-me@c71032e

+1

src/main.rs

··· 50 50 .service(routes::get_mst) 51 51 .service(routes::init) 52 52 .service(routes::get_avatar) 53 + .service(routes::get_avatar_batch) 53 54 .service(routes::validate_url) 54 55 .service(routes::get_record) 55 56 .service(routes::auth_status)

+93 -45

src/routes.rs

··· 6 6 use atrium_oauth::{AuthorizeOptions, CallbackParams, KnownScope, OAuthSession, Scope}; 7 7 use dashmap::DashMap; 8 8 use futures_util::future; 9 + use futures_util::stream::{FuturesUnordered, StreamExt}; 9 10 use once_cell::sync::Lazy; 10 11 use serde::Deserialize; 11 12 use serde::Serialize; 12 - use std::collections::HashMap; 13 + use std::collections::{HashMap, HashSet}; 13 14 use std::sync::{Arc, Mutex}; 14 - use std::time::Instant; 15 + use std::time::{Duration, Instant}; 15 16 16 17 use crate::constants; 17 18 use crate::firehose::FirehoseManager; ··· 36 37 37 38 static DID_CACHE: Lazy<DashMap<String, CachedDid>> = Lazy::new(DashMap::new); 38 39 40 + static HTTP_CLIENT: Lazy<reqwest::Client> = Lazy::new(|| { 41 + reqwest::Client::builder() 42 + .user_agent("at-me/1.0 (+https://at-me.zzstoatzz.io)") 43 + .pool_idle_timeout(Some(Duration::from_secs(30))) 44 + .connect_timeout(Duration::from_secs(4)) 45 + .timeout(Duration::from_secs(6)) 46 + .build() 47 + .expect("failed to build shared http client") 48 + }); 49 + 50 + async fn http_get(url: &str) -> Result<reqwest::Response, reqwest::Error> { 51 + HTTP_CLIENT.get(url).send().await 52 + } 53 + 39 54 // Guestbook signature struct 40 55 #[derive(Serialize, Clone)] 41 56 #[serde(rename_all = "camelCase")] ··· 102 117 // Handle provided - resolve to DID 103 118 let resolve_url = format!("{}?handle={}", constants::BSKY_API_RESOLVE_HANDLE, handle); 104 119 105 - match reqwest::get(&resolve_url).await { 120 + match http_get(&resolve_url).await { 106 121 Ok(response) => match response.json::<serde_json::Value>().await { 107 122 Ok(data) => match data["did"].as_str() { 108 123 Some(did) => did.to_string(), ··· 339 354 async fn try_resolve_handle_to_did(handle: &str) -> Option<String> { 340 355 let resolve_url = format!("{}?handle={}", constants::BSKY_API_RESOLVE_HANDLE, handle); 341 356 342 - match reqwest::get(&resolve_url).await { 357 + match http_get(&resolve_url).await { 343 358 Ok(response) => match response.json::<serde_json::Value>().await { 344 359 Ok(data) => data["did"].as_str().map(String::from), 345 360 Err(_) => None, ··· 354 369 355 370 // Fetch DID document 356 371 let did_doc_url = format!("{}/{}", constants::PLC_DIRECTORY, did); 357 - let did_doc_response = match reqwest::get(&did_doc_url).await { 372 + let did_doc_response = match http_get(&did_doc_url).await { 358 373 Ok(r) => r, 359 374 Err(e) => { 360 375 return HttpResponse::InternalServerError().json(serde_json::json!({ ··· 396 411 397 412 // Fetch collections from PDS 398 413 let repo_url = format!("{}/xrpc/com.atproto.repo.describeRepo?repo={}", pds, did); 399 - let repo_response = match reqwest::get(&repo_url).await { 414 + let repo_response = match http_get(&repo_url).await { 400 415 Ok(r) => r, 401 416 Err(e) => { 402 417 return HttpResponse::InternalServerError().json(serde_json::json!({ ··· 584 599 585 600 async fn fetch_user_avatar(did: &str) -> Option<String> { 586 601 let profile_url = format!("{}?actor={}", constants::BSKY_API_GET_PROFILE, did); 587 - if let Ok(response) = reqwest::get(&profile_url).await { 602 + if let Ok(response) = http_get(&profile_url).await { 588 603 if let Ok(profile) = response.json::<serde_json::Value>().await { 589 604 return profile["avatar"].as_str().map(String::from); 590 605 } ··· 595 610 #[derive(Deserialize)] 596 611 pub struct AvatarQuery { 597 612 namespace: String, 613 + } 614 + 615 + #[derive(Deserialize)] 616 + pub struct AvatarBatchRequest { 617 + namespaces: Vec<String>, 598 618 } 599 619 600 620 #[get("/api/avatar")] ··· 637 657 })) 638 658 } 639 659 640 - async fn fetch_avatar_for_namespace(namespace: &str) -> Option<String> { 641 - // Reverse namespace to get domain (e.g., io.zzstoatzz -> zzstoatzz.io) 642 - let reversed: String = namespace.split('.').rev().collect::<Vec<&str>>().join("."); 643 - let handles = [reversed.clone(), format!("{}.bsky.social", reversed)]; 660 + #[post("/api/avatar/batch")] 661 + pub async fn get_avatar_batch(payload: web::Json<AvatarBatchRequest>) -> HttpResponse { 662 + let mut requested: Vec<String> = Vec::new(); 663 + let mut seen = HashSet::new(); 644 664 645 - // Try all handles concurrently 646 - let futures: Vec<_> = handles 647 - .iter() 648 - .map(|handle| try_fetch_avatar_for_handle(handle)) 649 - .collect(); 665 + for raw in &payload.namespaces { 666 + let trimmed = raw.trim(); 667 + if trimmed.is_empty() { 668 + continue; 669 + } 670 + if seen.insert(trimmed.to_string()) { 671 + requested.push(trimmed.to_string()); 672 + } 673 + } 650 674 651 - // Wait for all futures and return first successful result 652 - let results = future::join_all(futures).await; 675 + let mut avatars: HashMap<String, Option<String>> = HashMap::new(); 676 + let mut to_fetch: Vec<String> = Vec::new(); 653 677 654 - results.into_iter().flatten().next() 655 - } 678 + { 679 + let cache = AVATAR_CACHE.lock().unwrap(); 680 + for namespace in &requested { 681 + if let Some(entry) = cache.get(namespace) { 682 + if entry.timestamp.elapsed() < constants::CACHE_TTL { 683 + avatars.insert(namespace.clone(), entry.url.clone()); 684 + continue; 685 + } 686 + } 687 + to_fetch.push(namespace.clone()); 688 + } 689 + } 656 690 657 - async fn try_fetch_avatar_for_handle(handle: &str) -> Option<String> { 658 - // Try to resolve handle to DID 659 - let resolve_url = format!("{}?handle={}", constants::BSKY_API_RESOLVE_HANDLE, handle); 691 + if !to_fetch.is_empty() { 692 + let mut futures: FuturesUnordered<_> = to_fetch 693 + .into_iter() 694 + .map(|namespace| async move { 695 + let avatar_url = fetch_avatar_for_namespace(&namespace).await; 696 + (namespace, avatar_url) 697 + }) 698 + .collect(); 660 699 661 - let did = match reqwest::get(&resolve_url).await { 662 - Ok(response) => match response.json::<serde_json::Value>().await { 663 - Ok(data) => data["did"].as_str()?.to_string(), 664 - Err(_) => return None, 665 - }, 666 - Err(_) => return None, 667 - }; 700 + while let Some((namespace, avatar_url)) = futures.next().await { 701 + { 702 + let mut cache = AVATAR_CACHE.lock().unwrap(); 703 + cache.insert( 704 + namespace.clone(), 705 + CachedAvatar { 706 + url: avatar_url.clone(), 707 + timestamp: Instant::now(), 708 + }, 709 + ); 710 + } 711 + avatars.insert(namespace, avatar_url); 712 + } 713 + } 668 714 669 - // Try to get profile 670 - let profile_url = format!("{}?actor={}", constants::BSKY_API_GET_PROFILE, did); 715 + HttpResponse::Ok() 716 + .insert_header(("Cache-Control", constants::HTTP_CACHE_CONTROL)) 717 + .json(serde_json::json!({ 718 + "avatars": avatars 719 + })) 720 + } 671 721 672 - match reqwest::get(&profile_url).await { 673 - Ok(response) => match response.json::<serde_json::Value>().await { 674 - Ok(profile) => profile["avatar"].as_str().map(String::from), 675 - Err(_) => None, 676 - }, 677 - Err(_) => None, 722 + async fn fetch_avatar_for_namespace(namespace: &str) -> Option<String> { 723 + if let Some(did) = resolve_namespace_to_did(namespace).await { 724 + return fetch_user_avatar(&did).await; 678 725 } 726 + None 679 727 } 680 728 681 729 #[derive(Deserialize)] ··· 737 785 query.pds, query.did, query.collection, query.rkey 738 786 ); 739 787 740 - match reqwest::get(&record_url).await { 788 + match http_get(&record_url).await { 741 789 Ok(response) => { 742 790 if !response.status().is_success() { 743 791 return HttpResponse::Ok().json(serde_json::json!({ ··· 1123 1171 async fn fetch_profile_info(did: &str) -> (Option<String>, Option<String>) { 1124 1172 // Fetch DID document for handle 1125 1173 let did_doc_url = format!("{}/{}", constants::PLC_DIRECTORY, did); 1126 - let handle = if let Ok(response) = reqwest::get(&did_doc_url).await { 1174 + let handle = if let Ok(response) = http_get(&did_doc_url).await { 1127 1175 if let Ok(doc) = response.json::<serde_json::Value>().await { 1128 1176 doc["alsoKnownAs"] 1129 1177 .as_array() ··· 1152 1200 1153 1201 log::info!("Fetching guestbook signatures from UFOs API"); 1154 1202 1155 - let response = reqwest::get(&ufos_url) 1203 + let response = http_get(&ufos_url) 1156 1204 .await 1157 1205 .map_err(|e| format!("failed to fetch from UFOs API: {}", e))?; 1158 1206 ··· 1221 1269 1222 1270 // Fetch DID document to get PDS URL 1223 1271 let did_doc_url = format!("{}/{}", constants::PLC_DIRECTORY, did); 1224 - let pds = match reqwest::get(&did_doc_url).await { 1272 + let pds = match http_get(&did_doc_url).await { 1225 1273 Ok(response) => match response.json::<serde_json::Value>().await { 1226 1274 Ok(doc) => doc["service"] 1227 1275 .as_array() ··· 1256 1304 constants::GUESTBOOK_COLLECTION 1257 1305 ); 1258 1306 1259 - match reqwest::get(&records_url).await { 1307 + match http_get(&records_url).await { 1260 1308 Ok(response) => { 1261 1309 if !response.status().is_success() { 1262 1310 // No records found or collection doesn't exist ··· 1305 1353 1306 1354 // Fetch DID document to get PDS 1307 1355 let did_doc_url = format!("{}/{}", constants::PLC_DIRECTORY, did); 1308 - let did_doc = match reqwest::get(&did_doc_url).await { 1356 + let did_doc = match http_get(&did_doc_url).await { 1309 1357 Ok(r) => match r.json::<serde_json::Value>().await { 1310 1358 Ok(d) => d, 1311 1359 Err(e) => { ··· 1336 1384 1337 1385 // Fetch collections from PDS 1338 1386 let repo_url = format!("{}/xrpc/com.atproto.repo.describeRepo?repo={}", pds, did); 1339 - let mut collections = match reqwest::get(&repo_url).await { 1387 + let mut collections = match http_get(&repo_url).await { 1340 1388 Ok(r) => match r.json::<serde_json::Value>().await { 1341 1389 Ok(repo_data) => repo_data["collections"] 1342 1390 .as_array()

+26 -21

src/templates/landing.html

··· 427 427 } 428 428 } 429 429 430 - async function fetchAppAvatar(namespace) { 431 - const reversed = namespace.split('.').reverse().join('.'); 432 - const handles = [reversed, `${reversed}.bsky.social`]; 430 + async function fetchAppAvatars(namespaces) { 431 + if (!Array.isArray(namespaces) || !namespaces.length) return {}; 432 + const deduped = [...new Set(namespaces.filter(Boolean))]; 433 + if (!deduped.length) return {}; 433 434 434 - for (const handle of handles) { 435 - try { 436 - const didRes = await fetch(`https://public.api.bsky.app/xrpc/com.atproto.identity.resolveHandle?handle=${handle}`); 437 - if (!didRes.ok) continue; 438 - 439 - const { did } = await didRes.json(); 440 - const profileRes = await fetch(`https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=${did}`); 441 - if (!profileRes.ok) continue; 442 - 443 - const profile = await profileRes.json(); 444 - if (profile.avatar) return profile.avatar; 445 - } catch (e) { 446 - continue; 447 - } 435 + try { 436 + const response = await fetch('/api/avatar/batch', { 437 + method: 'POST', 438 + headers: { 'Content-Type': 'application/json' }, 439 + body: JSON.stringify({ namespaces: deduped }) 440 + }); 441 + if (!response.ok) return {}; 442 + const data = await response.json(); 443 + return data.avatars || {}; 444 + } catch (e) { 445 + return {}; 448 446 } 449 - return null; 450 447 } 451 448 452 449 async function renderAtmosphere() { ··· 455 452 456 453 const atmosphere = document.getElementById('atmosphere'); 457 454 const maxSize = Math.max(...data.map(d => d.dids_total)); 455 + 456 + const namespaces = data.map(app => app.namespace); 457 + const avatarPromise = fetchAppAvatars(namespaces); 458 + const orbRegistry = []; 458 459 459 460 data.forEach((app, i) => { 460 461 const orb = document.createElement('div'); ··· 494 495 495 496 atmosphere.appendChild(orb); 496 497 497 - // Fetch and apply avatar 498 - fetchAppAvatar(app.namespace).then(avatarUrl => { 498 + orbRegistry.push({ orb, tooltip, namespace: app.namespace }); 499 + }); 500 + 501 + avatarPromise.then(avatarMap => { 502 + orbRegistry.forEach(({ orb, tooltip, namespace }) => { 503 + const avatarUrl = avatarMap[namespace]; 499 504 if (avatarUrl) { 500 - orb.innerHTML = `<img src="${avatarUrl}" alt="${app.namespace}" />`; 505 + orb.innerHTML = `<img src="${avatarUrl}" alt="${namespace}" />`; 501 506 orb.appendChild(tooltip); 502 507 } 503 508 });

+28 -9

static/app.js

··· 51 51 } 52 52 } 53 53 54 + async function fetchAppAvatars(namespaces) { 55 + if (!Array.isArray(namespaces) || namespaces.length === 0) return {}; 56 + const uniqueNamespaces = [...new Set(namespaces.filter(Boolean))]; 57 + if (!uniqueNamespaces.length) return {}; 58 + 59 + try { 60 + const response = await fetch('/api/avatar/batch', { 61 + method: 'POST', 62 + headers: { 'Content-Type': 'application/json' }, 63 + body: JSON.stringify({ namespaces: uniqueNamespaces }) 64 + }); 65 + if (!response.ok) return {}; 66 + const data = await response.json(); 67 + return data.avatars || {}; 68 + } catch (e) { 69 + return {}; 70 + } 71 + } 72 + 54 73 // Info modal handlers 55 74 document.getElementById('infoBtn').addEventListener('click', () => { 56 75 document.getElementById('infoModal').classList.add('visible'); ··· 274 293 return { div, namespace }; 275 294 }); 276 295 277 - // Fetch all avatars concurrently 278 - Promise.all(appNames.map(namespace => fetchAppAvatar(namespace))) 279 - .then(avatarUrls => { 280 - avatarUrls.forEach((avatarUrl, i) => { 281 - if (avatarUrl) { 282 - const circle = appDivs[i].div.querySelector('.app-circle'); 283 - circle.innerHTML = `<img src="${avatarUrl}" class="app-logo" alt="${appDivs[i].namespace}" />`; 284 - } 285 - }); 296 + // Fetch all avatars concurrently via batch endpoint 297 + fetchAppAvatars(appNames).then(avatarMap => { 298 + appDivs.forEach(({ div, namespace }) => { 299 + const avatarUrl = avatarMap[namespace]; 300 + if (avatarUrl) { 301 + const circle = div.querySelector('.app-circle'); 302 + circle.innerHTML = `<img src="${avatarUrl}" class="app-logo" alt="${namespace}" />`; 303 + } 286 304 }); 305 + }); 287 306 288 307 appDivs.forEach(({ div, namespace }, i) => { 289 308 // Reverse namespace for display and create URL

+26 -23

static/login.js

·········

Configure Feed

Configure Feed