feat: add proper backfill stats tracking · gamesgamesgamesgames.games/happyview@a791aab

+2

src/admin/mod.rs

··· 5 5 mod network_lexicons; 6 6 mod records; 7 7 mod stats; 8 + mod tap_stats; 8 9 mod types; 9 10 10 11 use axum::Router; ··· 31 32 ) 32 33 .route("/admins/{id}", delete(admins::delete_admin)) 33 34 .route("/records", get(records::list_records)) 35 + .route("/tap/stats", get(tap_stats::tap_stats)) 34 36 .route( 35 37 "/network-lexicons", 36 38 post(network_lexicons::add).get(network_lexicons::list),

+24

src/admin/tap_stats.rs

··· 1 + use axum::Json; 2 + use axum::extract::State; 3 + 4 + use crate::AppState; 5 + use crate::error::AppError; 6 + use crate::tap; 7 + 8 + use super::auth::AdminAuth; 9 + 10 + /// GET /admin/tap/stats — aggregate stats from Tap. 11 + pub(super) async fn tap_stats( 12 + State(state): State<AppState>, 13 + _admin: AdminAuth, 14 + ) -> Result<Json<tap::TapStats>, AppError> { 15 + let stats = tap::get_stats( 16 + &state.http, 17 + &state.config.tap_url, 18 + state.config.tap_admin_password.as_deref(), 19 + ) 20 + .await 21 + .map_err(AppError::BadGateway)?; 22 + 23 + Ok(Json(stats)) 24 + }

+4

src/error.rs

··· 7 7 Auth(String), 8 8 /// Auth failure with a DPoP nonce that the client should retry with. 9 9 AuthDpopNonce(String), 10 + BadGateway(String), 10 11 BadRequest(String), 11 12 Forbidden(String), 12 13 Internal(String), ··· 19 20 match self { 20 21 AppError::Auth(msg) => write!(f, "auth error: {msg}"), 21 22 AppError::AuthDpopNonce(nonce) => write!(f, "auth error: use_dpop_nonce ({nonce})"), 23 + AppError::BadGateway(msg) => write!(f, "bad gateway: {msg}"), 22 24 AppError::BadRequest(msg) => write!(f, "bad request: {msg}"), 23 25 AppError::Forbidden(msg) => write!(f, "forbidden: {msg}"), 24 26 AppError::Internal(msg) => write!(f, "internal error: {msg}"), ··· 48 50 other => { 49 51 let (status, message) = match &other { 50 52 AppError::Auth(msg) => (StatusCode::UNAUTHORIZED, msg.clone()), 53 + AppError::BadGateway(msg) => (StatusCode::BAD_GATEWAY, msg.clone()), 51 54 AppError::BadRequest(msg) => (StatusCode::BAD_REQUEST, msg.clone()), 55 + 52 56 AppError::Forbidden(msg) => (StatusCode::FORBIDDEN, msg.clone()), 53 57 AppError::Internal(msg) => { 54 58 tracing::error!("{msg}");

+71 -1

src/tap.rs

··· 1 1 use futures_util::{SinkExt, StreamExt}; 2 - use serde::Deserialize; 2 + use serde::{Deserialize, Serialize}; 3 3 use serde_json::Value; 4 4 use sqlx::PgPool; 5 5 use tokio::sync::watch; ··· 93 93 return Err(format!("tap returned {status}: {body}")); 94 94 } 95 95 Ok(()) 96 + } 97 + 98 + async fn tap_get<T: serde::de::DeserializeOwned>( 99 + http: &reqwest::Client, 100 + tap_url: &str, 101 + path: &str, 102 + password: Option<&str>, 103 + ) -> Result<T, String> { 104 + let url = format!("{}{}", tap_url.trim_end_matches('/'), path); 105 + let mut req = http.get(&url); 106 + if let Some(pw) = password { 107 + req = req.basic_auth("admin", Some(pw)); 108 + } 109 + let resp = req 110 + .send() 111 + .await 112 + .map_err(|e| format!("tap HTTP request failed: {e}"))?; 113 + if !resp.status().is_success() { 114 + let status = resp.status(); 115 + let body = resp.text().await.unwrap_or_default(); 116 + return Err(format!("tap returned {status}: {body}")); 117 + } 118 + resp.json::<T>() 119 + .await 120 + .map_err(|e| format!("failed to parse tap response: {e}")) 121 + } 122 + 123 + // --------------------------------------------------------------------------- 124 + // Tap stats 125 + // --------------------------------------------------------------------------- 126 + 127 + #[derive(Serialize)] 128 + pub struct TapStats { 129 + pub repo_count: u64, 130 + pub record_count: u64, 131 + pub outbox_buffer: u64, 132 + } 133 + 134 + #[derive(Deserialize)] 135 + struct RepoCountResponse { 136 + repo_count: u64, 137 + } 138 + 139 + #[derive(Deserialize)] 140 + struct RecordCountResponse { 141 + record_count: u64, 142 + } 143 + 144 + #[derive(Deserialize)] 145 + struct OutboxBufferResponse { 146 + outbox_buffer: u64, 147 + } 148 + 149 + /// Fetch aggregate stats from Tap's monitoring endpoints in parallel. 150 + pub async fn get_stats( 151 + http: &reqwest::Client, 152 + tap_url: &str, 153 + tap_admin_password: Option<&str>, 154 + ) -> Result<TapStats, String> { 155 + let (repo, record, outbox) = tokio::try_join!( 156 + tap_get::<RepoCountResponse>(http, tap_url, "/stats/repo-count", tap_admin_password), 157 + tap_get::<RecordCountResponse>(http, tap_url, "/stats/record-count", tap_admin_password), 158 + tap_get::<OutboxBufferResponse>(http, tap_url, "/stats/outbox-buffer", tap_admin_password), 159 + )?; 160 + 161 + Ok(TapStats { 162 + repo_count: repo.repo_count, 163 + record_count: record.record_count, 164 + outbox_buffer: outbox.outbox_buffer, 165 + }) 96 166 } 97 167 98 168 /// Sync Tap's collection filters and signal collections with HappyView's

+78 -61

web/src/app/(dashboard)/backfill/page.tsx

··· 1 - "use client" 1 + "use client"; 2 2 3 - import { useCallback, useEffect, useState } from "react" 3 + import { useCallback, useEffect, useState } from "react"; 4 4 5 - import { useAuth } from "@/lib/auth-context" 5 + import { useAuth } from "@/lib/auth-context"; 6 6 import { 7 7 createBackfillJob, 8 8 getBackfillJobs, 9 + getTapStats, 9 10 type BackfillJob, 10 - } from "@/lib/api" 11 - import { SiteHeader } from "@/components/site-header" 12 - import { Badge } from "@/components/ui/badge" 13 - import { Button } from "@/components/ui/button" 11 + type TapStatsResponse, 12 + } from "@/lib/api"; 13 + import { SiteHeader } from "@/components/site-header"; 14 + import { Button } from "@/components/ui/button"; 15 + import { 16 + Card, 17 + CardDescription, 18 + CardHeader, 19 + CardTitle, 20 + } from "@/components/ui/card"; 14 21 import { 15 22 Dialog, 16 23 DialogClose, ··· 20 27 DialogHeader, 21 28 DialogTitle, 22 29 DialogTrigger, 23 - } from "@/components/ui/dialog" 24 - import { Input } from "@/components/ui/input" 25 - import { Label } from "@/components/ui/label" 30 + } from "@/components/ui/dialog"; 31 + import { Input } from "@/components/ui/input"; 32 + import { Label } from "@/components/ui/label"; 26 33 import { 27 34 Table, 28 35 TableBody, ··· 30 37 TableHead, 31 38 TableHeader, 32 39 TableRow, 33 - } from "@/components/ui/table" 34 - 35 - function statusVariant(status: string) { 36 - switch (status) { 37 - case "completed": 38 - return "default" as const 39 - case "running": 40 - return "secondary" as const 41 - case "failed": 42 - return "destructive" as const 43 - default: 44 - return "outline" as const 45 - } 46 - } 40 + } from "@/components/ui/table"; 47 41 48 42 export default function BackfillPage() { 49 - const { getToken } = useAuth() 50 - const [jobs, setJobs] = useState<BackfillJob[]>([]) 51 - const [error, setError] = useState<string | null>(null) 43 + const { getToken } = useAuth(); 44 + const [jobs, setJobs] = useState<BackfillJob[]>([]); 45 + const [tapStats, setTapStats] = useState<TapStatsResponse | null>(null); 46 + const [error, setError] = useState<string | null>(null); 52 47 53 48 const load = useCallback(() => { 54 - getBackfillJobs(getToken).then(setJobs).catch((e) => setError(e.message)) 55 - }, [getToken]) 49 + getBackfillJobs(getToken) 50 + .then(setJobs) 51 + .catch((e) => setError(e.message)); 52 + getTapStats(getToken) 53 + .then(setTapStats) 54 + .catch(() => setTapStats(null)); 55 + }, [getToken]); 56 56 57 57 useEffect(() => { 58 - load() 59 - }, [load]) 58 + load(); 59 + }, [load]); 60 60 61 - // Auto-refresh every 5 seconds when there are active jobs 61 + // Auto-refresh every 5 seconds 62 62 useEffect(() => { 63 - const hasActive = jobs.some( 64 - (j) => j.status === "pending" || j.status === "running" 65 - ) 66 - if (!hasActive) return 67 - const interval = setInterval(load, 5000) 68 - return () => clearInterval(interval) 69 - }, [jobs, load]) 63 + const interval = setInterval(load, 5000); 64 + return () => clearInterval(interval); 65 + }, [load]); 70 66 71 67 return ( 72 68 <> ··· 74 70 <div className="flex flex-1 flex-col gap-4 p-4 md:p-6"> 75 71 {error && <p className="text-destructive text-sm">{error}</p>} 76 72 73 + <div className="grid grid-cols-1 gap-4 sm:grid-cols-3"> 74 + <Card> 75 + <CardHeader className={"text-center sm:text-left"}> 76 + <CardDescription>Tap Repos</CardDescription> 77 + <CardTitle className="text-xl sm:text-2xl font-semibold tabular-nums"> 78 + {tapStats ? tapStats.repo_count.toLocaleString() : "--"} 79 + </CardTitle> 80 + </CardHeader> 81 + </Card> 82 + <Card> 83 + <CardHeader className={"text-center sm:text-left"}> 84 + <CardDescription>Tap Records</CardDescription> 85 + <CardTitle className="text-xl sm:text-2xl font-semibold tabular-nums"> 86 + {tapStats ? tapStats.record_count.toLocaleString() : "--"} 87 + </CardTitle> 88 + </CardHeader> 89 + </Card> 90 + <Card> 91 + <CardHeader className={"text-center sm:text-left"}> 92 + <CardDescription>Outbox Buffer</CardDescription> 93 + <CardTitle className="text-xl sm:text-2xl font-semibold tabular-nums"> 94 + {tapStats ? tapStats.outbox_buffer.toLocaleString() : "--"} 95 + </CardTitle> 96 + </CardHeader> 97 + </Card> 98 + </div> 99 + 77 100 <div className="flex items-center justify-between"> 78 101 <h2 className="text-lg font-semibold">Backfill Jobs</h2> 79 102 <CreateDialog getToken={getToken} onSuccess={load} /> ··· 86 109 <TableHead>ID</TableHead> 87 110 <TableHead>Collection</TableHead> 88 111 <TableHead>DID</TableHead> 89 - <TableHead>Status</TableHead> 90 112 <TableHead>Progress</TableHead> 91 113 <TableHead>Records</TableHead> 92 114 <TableHead>Started</TableHead> ··· 96 118 {jobs.length === 0 && ( 97 119 <TableRow> 98 120 <TableCell 99 - colSpan={7} 121 + colSpan={6} 100 122 className="text-muted-foreground text-center" 101 123 > 102 124 No backfill jobs yet. ··· 114 136 <TableCell className="font-mono text-sm"> 115 137 {job.did ?? "All"} 116 138 </TableCell> 117 - <TableCell> 118 - <Badge variant={statusVariant(job.status)}> 119 - {job.status} 120 - </Badge> 121 - </TableCell> 122 139 <TableCell className="tabular-nums"> 123 140 {job.processed_repos != null && job.total_repos != null 124 141 ? `${job.processed_repos} / ${job.total_repos}` ··· 139 156 </div> 140 157 </div> 141 158 </> 142 - ) 159 + ); 143 160 } 144 161 145 162 function CreateDialog({ 146 163 getToken, 147 164 onSuccess, 148 165 }: { 149 - getToken: () => Promise<string | null> 150 - onSuccess: () => void 166 + getToken: () => Promise<string | null>; 167 + onSuccess: () => void; 151 168 }) { 152 - const [collection, setCollection] = useState("") 153 - const [did, setDid] = useState("") 154 - const [error, setError] = useState<string | null>(null) 155 - const [open, setOpen] = useState(false) 169 + const [collection, setCollection] = useState(""); 170 + const [did, setDid] = useState(""); 171 + const [error, setError] = useState<string | null>(null); 172 + const [open, setOpen] = useState(false); 156 173 157 174 async function handleCreate() { 158 - setError(null) 175 + setError(null); 159 176 try { 160 177 await createBackfillJob(getToken, { 161 178 collection: collection || undefined, 162 179 did: did || undefined, 163 - }) 164 - setCollection("") 165 - setDid("") 166 - setOpen(false) 167 - onSuccess() 180 + }); 181 + setCollection(""); 182 + setDid(""); 183 + setOpen(false); 184 + onSuccess(); 168 185 } catch (e: unknown) { 169 - setError(e instanceof Error ? e.message : String(e)) 186 + setError(e instanceof Error ? e.message : String(e)); 170 187 } 171 188 } 172 189 ··· 212 229 </DialogFooter> 213 230 </DialogContent> 214 231 </Dialog> 215 - ) 232 + ); 216 233 }

+21 -21

web/src/app/(dashboard)/page.tsx

··· 1 - "use client" 1 + "use client"; 2 2 3 - import { useEffect, useState } from "react" 3 + import { useEffect, useState } from "react"; 4 4 5 - import { useAuth } from "@/lib/auth-context" 6 - import { getStats, type StatsResponse } from "@/lib/api" 7 - import { SiteHeader } from "@/components/site-header" 5 + import { useAuth } from "@/lib/auth-context"; 6 + import { getStats, type StatsResponse } from "@/lib/api"; 7 + import { SiteHeader } from "@/components/site-header"; 8 8 import { 9 9 Card, 10 10 CardDescription, 11 11 CardHeader, 12 12 CardTitle, 13 - } from "@/components/ui/card" 13 + } from "@/components/ui/card"; 14 14 import { 15 15 Table, 16 16 TableBody, ··· 18 18 TableHead, 19 19 TableHeader, 20 20 TableRow, 21 - } from "@/components/ui/table" 21 + } from "@/components/ui/table"; 22 22 23 23 export default function DashboardPage() { 24 - const { getToken } = useAuth() 25 - const [stats, setStats] = useState<StatsResponse | null>(null) 26 - const [error, setError] = useState<string | null>(null) 24 + const { getToken } = useAuth(); 25 + const [stats, setStats] = useState<StatsResponse | null>(null); 26 + const [error, setError] = useState<string | null>(null); 27 27 28 28 useEffect(() => { 29 - getStats(getToken).then(setStats).catch((e) => setError(e.message)) 30 - }, [getToken]) 29 + getStats(getToken) 30 + .then(setStats) 31 + .catch((e) => setError(e.message)); 32 + }, [getToken]); 31 33 32 34 return ( 33 35 <> 34 36 <SiteHeader title="Dashboard" /> 35 37 <div className="flex flex-1 flex-col gap-4 p-4 md:gap-6 md:p-6"> 36 - {error && ( 37 - <p className="text-destructive text-sm">{error}</p> 38 - )} 39 - <div className="grid grid-cols-1 gap-4 @xl:grid-cols-2 @3xl:grid-cols-3"> 38 + {error && <p className="text-destructive text-sm">{error}</p>} 39 + <div className="grid grid-cols-1 gap-4 sm:grid-cols-2"> 40 40 <Card> 41 - <CardHeader> 41 + <CardHeader className={"text-center sm:text-left"}> 42 42 <CardDescription>Total Records</CardDescription> 43 - <CardTitle className="text-2xl font-semibold tabular-nums"> 43 + <CardTitle className="text-xl sm:text-2xl font-semibold tabular-nums"> 44 44 {stats ? stats.total_records.toLocaleString() : "--"} 45 45 </CardTitle> 46 46 </CardHeader> 47 47 </Card> 48 48 <Card> 49 - <CardHeader> 49 + <CardHeader className={"text-center sm:text-left"}> 50 50 <CardDescription>Collections</CardDescription> 51 - <CardTitle className="text-2xl font-semibold tabular-nums"> 51 + <CardTitle className="text-xl sm:text-2xl font-semibold tabular-nums"> 52 52 {stats ? stats.collections.length : "--"} 53 53 </CardTitle> 54 54 </CardHeader> ··· 81 81 )} 82 82 </div> 83 83 </> 84 - ) 84 + ); 85 85 }

+2 -1

web/src/app/globals.css

··· 7 7 @theme inline { 8 8 --color-background: var(--background); 9 9 --color-foreground: var(--foreground); 10 + --font-display: var(--font-zen-tokyo-zoo); 10 11 --font-sans: var(--font-geist-sans); 11 12 --font-mono: var(--font-geist-mono); 12 13 --color-sidebar-ring: var(--sidebar-ring); ··· 143 144 font-style: var(--shiki-dark-font-style) !important; 144 145 font-weight: var(--shiki-dark-font-weight) !important; 145 146 text-decoration: var(--shiki-dark-text-decoration) !important; 146 - } 147 + }

+18 -13

web/src/app/layout.tsx

··· 1 - import type { Metadata } from "next" 2 - import { Geist, Geist_Mono } from "next/font/google" 3 - import { ThemeProvider } from "next-themes" 4 - import "./globals.css" 5 - import { ConfigProvider } from "@/lib/config-context" 6 - import { AuthProvider } from "@/lib/auth-context" 7 - import { TooltipProvider } from "@/components/ui/tooltip" 1 + import type { Metadata } from "next"; 2 + import { Geist, Geist_Mono, Zen_Tokyo_Zoo } from "next/font/google"; 3 + import { ThemeProvider } from "next-themes"; 4 + import "./globals.css"; 5 + import { ConfigProvider } from "@/lib/config-context"; 6 + import { AuthProvider } from "@/lib/auth-context"; 7 + import { TooltipProvider } from "@/components/ui/tooltip"; 8 + 9 + const zenTokyoZoo = Zen_Tokyo_Zoo({ 10 + subsets: ["latin"], 11 + weight: "400", 12 + }); 8 13 9 14 const geistSans = Geist({ 10 15 variable: "--font-geist-sans", 11 16 subsets: ["latin"], 12 - }) 17 + }); 13 18 14 19 const geistMono = Geist_Mono({ 15 20 variable: "--font-geist-mono", 16 21 subsets: ["latin"], 17 - }) 22 + }); 18 23 19 24 export const metadata: Metadata = { 20 25 title: "HappyView Admin", 21 26 description: "Admin dashboard for HappyView AppView", 22 - } 27 + }; 23 28 24 29 export default function RootLayout({ 25 30 children, 26 31 }: Readonly<{ 27 - children: React.ReactNode 32 + children: React.ReactNode; 28 33 }>) { 29 34 return ( 30 35 <html lang="en" suppressHydrationWarning> 31 36 <body 32 - className={`${geistSans.variable} ${geistMono.variable} antialiased`} 37 + className={`${geistSans.variable} ${geistMono.variable} ${zenTokyoZoo} antialiased`} 33 38 > 34 39 <ThemeProvider attribute="class" defaultTheme="system" enableSystem> 35 40 <ConfigProvider> ··· 40 45 </ThemeProvider> 41 46 </body> 42 47 </html> 43 - ) 48 + ); 44 49 }

+11

web/src/lib/api.ts

··· 171 171 ) 172 172 } 173 173 174 + // Tap Stats 175 + export interface TapStatsResponse { 176 + repo_count: number 177 + record_count: number 178 + outbox_buffer: number 179 + } 180 + 181 + export function getTapStats(getToken: () => Promise<string | null>) { 182 + return apiFetch<TapStatsResponse>("/admin/tap/stats", getToken) 183 + } 184 + 174 185 // Backfill 175 186 export interface BackfillJob { 176 187 id: string

Configure Feed

Configure Feed