+24

migrations/0003_add_incidents.sql

reviewed

··· 1 1 + CREATE TABLE incidents ( 2 2 + id INTEGER PRIMARY KEY AUTOINCREMENT, 3 3 + service_id TEXT NOT NULL, 4 4 + title TEXT NOT NULL, 5 5 + status TEXT NOT NULL CHECK (status IN ('investigating', 'identified', 'monitoring', 'resolved')), 6 6 + severity TEXT NOT NULL CHECK (severity IN ('critical', 'major', 'minor')), 7 7 + triage_report TEXT, 8 8 + started_at INTEGER NOT NULL, 9 9 + resolved_at INTEGER, 10 10 + created_at INTEGER NOT NULL, 11 11 + updated_at INTEGER NOT NULL 12 12 + ); 13 13 + 14 14 + CREATE TABLE incident_updates ( 15 15 + id INTEGER PRIMARY KEY AUTOINCREMENT, 16 16 + incident_id INTEGER NOT NULL REFERENCES incidents(id), 17 17 + status TEXT NOT NULL, 18 18 + message TEXT NOT NULL, 19 19 + created_at INTEGER NOT NULL 20 20 + ); 21 21 + 22 22 + CREATE INDEX idx_incidents_service ON incidents (service_id, status); 23 23 + CREATE INDEX idx_incidents_active ON incidents (status) WHERE status != 'resolved'; 24 24 + CREATE INDEX idx_incident_updates ON incident_updates (incident_id, created_at DESC);

+2

migrations/0004_add_github_fields.sql

reviewed

··· 1 1 + ALTER TABLE incidents ADD COLUMN github_repo TEXT; 2 2 + ALTER TABLE incidents ADD COLUMN github_issue_number INTEGER;

+148

src/db.ts

reviewed

··· 156 156 .bind(cutoff) 157 157 .run(); 158 158 } 159 159 + 160 160 + import type { Incident, IncidentUpdate, IncidentWithUpdates } from "./types"; 161 161 + 162 162 + export async function createIncident( 163 163 + db: D1Database, 164 164 + data: { service_id: string; title: string; severity: "critical" | "major" | "minor"; github_repo?: string; github_issue_number?: number }, 165 165 + ): Promise<number> { 166 166 + const now = Math.floor(Date.now() / 1000); 167 167 + const result = await db 168 168 + .prepare( 169 169 + "INSERT INTO incidents (service_id, title, status, severity, github_repo, github_issue_number, started_at, created_at, updated_at) VALUES (?, ?, 'investigating', ?, ?, ?, ?, ?, ?)", 170 170 + ) 171 171 + .bind(data.service_id, data.title, data.severity, data.github_repo ?? null, data.github_issue_number ?? null, now, now, now) 172 172 + .run(); 173 173 + const id = result.meta.last_row_id; 174 174 + await db 175 175 + .prepare( 176 176 + "INSERT INTO incident_updates (incident_id, status, message, created_at) VALUES (?, 'investigating', 'Incident detected automatically', ?)", 177 177 + ) 178 178 + .bind(id, now) 179 179 + .run(); 180 180 + return id as number; 181 181 + } 182 182 + 183 183 + export async function updateIncident( 184 184 + db: D1Database, 185 185 + id: number, 186 186 + data: { status?: string; triage_report?: string; resolved_at?: number }, 187 187 + ): Promise<void> { 188 188 + const sets: string[] = []; 189 189 + const values: unknown[] = []; 190 190 + if (data.status) { sets.push("status = ?"); values.push(data.status); } 191 191 + if (data.triage_report !== undefined) { sets.push("triage_report = ?"); values.push(data.triage_report); } 192 192 + if (data.resolved_at) { sets.push("resolved_at = ?"); values.push(data.resolved_at); } 193 193 + sets.push("updated_at = ?"); 194 194 + values.push(Math.floor(Date.now() / 1000)); 195 195 + values.push(id); 196 196 + await db 197 197 + .prepare(`UPDATE incidents SET ${sets.join(", ")} WHERE id = ?`) 198 198 + .bind(...values) 199 199 + .run(); 200 200 + } 201 201 + 202 202 + export async function addIncidentUpdate( 203 203 + db: D1Database, 204 204 + incident_id: number, 205 205 + status: string, 206 206 + message: string, 207 207 + ): Promise<void> { 208 208 + const now = Math.floor(Date.now() / 1000); 209 209 + await db 210 210 + .prepare("INSERT INTO incident_updates (incident_id, status, message, created_at) VALUES (?, ?, ?, ?)") 211 211 + .bind(incident_id, status, message, now) 212 212 + .run(); 213 213 + } 214 214 + 215 215 + export async function getActiveIncidents(db: D1Database): Promise<Incident[]> { 216 216 + const rows = await db 217 217 + .prepare("SELECT * FROM incidents WHERE status != 'resolved' ORDER BY created_at DESC") 218 218 + .all(); 219 219 + return rows.results as unknown as Incident[]; 220 220 + } 221 221 + 222 222 + export async function getActiveIncidentsWithUpdates(db: D1Database): Promise<IncidentWithUpdates[]> { 223 223 + const incidents = await getActiveIncidents(db); 224 224 + return Promise.all( 225 225 + incidents.map(async (incident) => { 226 226 + const updates = await db 227 227 + .prepare("SELECT * FROM incident_updates WHERE incident_id = ? ORDER BY created_at ASC") 228 228 + .bind(incident.id) 229 229 + .all(); 230 230 + return { ...incident, updates: updates.results as unknown as IncidentUpdate[] }; 231 231 + }), 232 232 + ); 233 233 + } 234 234 + 235 235 + export async function getActiveIncidentForService( 236 236 + db: D1Database, 237 237 + service_id: string, 238 238 + ): Promise<Incident | null> { 239 239 + const row = await db 240 240 + .prepare("SELECT * FROM incidents WHERE service_id = ? AND status != 'resolved' ORDER BY created_at DESC LIMIT 1") 241 241 + .bind(service_id) 242 242 + .first(); 243 243 + return (row as unknown as Incident) ?? null; 244 244 + } 245 245 + 246 246 + export async function getRecentIncidents(db: D1Database, days: number): Promise<Incident[]> { 247 247 + const since = Math.floor(Date.now() / 1000) - days * 24 * 60 * 60; 248 248 + const rows = await db 249 249 + .prepare("SELECT * FROM incidents WHERE resolved_at >= ? OR status != 'resolved' ORDER BY created_at DESC") 250 250 + .bind(since) 251 251 + .all(); 252 252 + return rows.results as unknown as Incident[]; 253 253 + } 254 254 + 255 255 + export async function getIncident(db: D1Database, id: number): Promise<IncidentWithUpdates | null> { 256 256 + const incident = await db 257 257 + .prepare("SELECT * FROM incidents WHERE id = ?") 258 258 + .bind(id) 259 259 + .first(); 260 260 + if (!incident) return null; 261 261 + const updates = await db 262 262 + .prepare("SELECT * FROM incident_updates WHERE incident_id = ? ORDER BY created_at ASC") 263 263 + .bind(id) 264 264 + .all(); 265 265 + return { 266 266 + ...(incident as unknown as Incident), 267 267 + updates: updates.results as unknown as IncidentUpdate[], 268 268 + }; 269 269 + } 270 270 + 271 271 + export async function getIncidentByGitHubIssue( 272 272 + db: D1Database, 273 273 + repo: string, 274 274 + issueNumber: number, 275 275 + ): Promise<Incident | null> { 276 276 + const row = await db 277 277 + .prepare("SELECT * FROM incidents WHERE github_repo = ? AND github_issue_number = ? LIMIT 1") 278 278 + .bind(repo, issueNumber) 279 279 + .first(); 280 280 + return (row as unknown as Incident) ?? null; 281 281 + } 282 282 + 283 283 + export async function setIncidentGitHub( 284 284 + db: D1Database, 285 285 + id: number, 286 286 + repo: string, 287 287 + issueNumber: number, 288 288 + ): Promise<void> { 289 289 + await db 290 290 + .prepare("UPDATE incidents SET github_repo = ?, github_issue_number = ?, updated_at = ? WHERE id = ?") 291 291 + .bind(repo, issueNumber, Math.floor(Date.now() / 1000), id) 292 292 + .run(); 293 293 + } 294 294 + 295 295 + export async function getRecentlyResolvedIncident( 296 296 + db: D1Database, 297 297 + service_id: string, 298 298 + withinSeconds: number, 299 299 + ): Promise<Incident | null> { 300 300 + const since = Math.floor(Date.now() / 1000) - withinSeconds; 301 301 + const row = await db 302 302 + .prepare("SELECT * FROM incidents WHERE service_id = ? AND status = 'resolved' AND resolved_at >= ? ORDER BY resolved_at DESC LIMIT 1") 303 303 + .bind(service_id, since) 304 304 + .first(); 305 305 + return (row as unknown as Incident) ?? null; 306 306 + }

+193

src/github.ts

reviewed

··· 1 1 + // GitHub Issues integration for incidents 2 2 + 3 3 + import type { Incident } from "./types"; 4 4 + import { updateIncident, addIncidentUpdate } from "./db"; 5 5 + 6 6 + export function parseRepo(repoUrl: string): { owner: string; repo: string } | null { 7 7 + const match = repoUrl.match(/github\.com\/([^/]+)\/([^/]+)/); 8 8 + if (!match) return null; 9 9 + return { owner: match[1], repo: match[2].replace(/\.git$/, "") }; 10 10 + } 11 11 + 12 12 + export async function createIssue( 13 13 + token: string, 14 14 + owner: string, 15 15 + repo: string, 16 16 + opts: { title: string; body: string; assignees?: string[]; labels?: string[] }, 17 17 + ): Promise<number> { 18 18 + const res = await fetch(`https://api.github.com/repos/${owner}/${repo}/issues`, { 19 19 + method: "POST", 20 20 + headers: { 21 21 + Authorization: `Bearer ${token}`, 22 22 + Accept: "application/vnd.github+json", 23 23 + "User-Agent": "infra-status-worker", 24 24 + }, 25 25 + body: JSON.stringify(opts), 26 26 + }); 27 27 + if (!res.ok) { 28 28 + const text = await res.text(); 29 29 + throw new Error(`GitHub create issue failed: ${res.status} ${text}`); 30 30 + } 31 31 + const data = await res.json<{ number: number }>(); 32 32 + return data.number; 33 33 + } 34 34 + 35 35 + export async function commentOnIssue( 36 36 + token: string, 37 37 + owner: string, 38 38 + repo: string, 39 39 + issueNumber: number, 40 40 + body: string, 41 41 + ): Promise<void> { 42 42 + const res = await fetch( 43 43 + `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}/comments`, 44 44 + { 45 45 + method: "POST", 46 46 + headers: { 47 47 + Authorization: `Bearer ${token}`, 48 48 + Accept: "application/vnd.github+json", 49 49 + "User-Agent": "infra-status-worker", 50 50 + }, 51 51 + body: JSON.stringify({ body }), 52 52 + }, 53 53 + ); 54 54 + if (!res.ok) { 55 55 + const text = await res.text(); 56 56 + throw new Error(`GitHub comment failed: ${res.status} ${text}`); 57 57 + } 58 58 + } 59 59 + 60 60 + export async function closeIssue( 61 61 + token: string, 62 62 + owner: string, 63 63 + repo: string, 64 64 + issueNumber: number, 65 65 + ): Promise<void> { 66 66 + const res = await fetch( 67 67 + `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}`, 68 68 + { 69 69 + method: "PATCH", 70 70 + headers: { 71 71 + Authorization: `Bearer ${token}`, 72 72 + Accept: "application/vnd.github+json", 73 73 + "User-Agent": "infra-status-worker", 74 74 + }, 75 75 + body: JSON.stringify({ state: "closed" }), 76 76 + }, 77 77 + ); 78 78 + if (!res.ok) { 79 79 + const text = await res.text(); 80 80 + throw new Error(`GitHub close issue failed: ${res.status} ${text}`); 81 81 + } 82 82 + } 83 83 + 84 84 + export async function editIssueBody( 85 85 + token: string, 86 86 + owner: string, 87 87 + repo: string, 88 88 + issueNumber: number, 89 89 + body: string, 90 90 + ): Promise<void> { 91 91 + const res = await fetch( 92 92 + `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}`, 93 93 + { 94 94 + method: "PATCH", 95 95 + headers: { 96 96 + Authorization: `Bearer ${token}`, 97 97 + Accept: "application/vnd.github+json", 98 98 + "User-Agent": "infra-status-worker", 99 99 + }, 100 100 + body: JSON.stringify({ body }), 101 101 + }, 102 102 + ); 103 103 + if (!res.ok) { 104 104 + const text = await res.text(); 105 105 + throw new Error(`GitHub edit issue failed: ${res.status} ${text}`); 106 106 + } 107 107 + } 108 108 + 109 109 + interface GitHubIssue { 110 110 + state: string; 111 111 + body: string | null; 112 112 + } 113 113 + 114 114 + interface GitHubComment { 115 115 + id: number; 116 116 + body: string; 117 117 + created_at: string; 118 118 + user: { login: string; type: string }; 119 119 + } 120 120 + 121 121 + async function fetchIssue(token: string, owner: string, repo: string, issueNumber: number): Promise<GitHubIssue> { 122 122 + const res = await fetch(`https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}`, { 123 123 + headers: { 124 124 + Authorization: `Bearer ${token}`, 125 125 + Accept: "application/vnd.github+json", 126 126 + "User-Agent": "infra-status-worker", 127 127 + }, 128 128 + }); 129 129 + if (!res.ok) throw new Error(`GitHub fetch issue failed: ${res.status}`); 130 130 + return res.json(); 131 131 + } 132 132 + 133 133 + async function fetchComments(token: string, owner: string, repo: string, issueNumber: number, since?: string): Promise<GitHubComment[]> { 134 134 + let url = `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}/comments?per_page=50`; 135 135 + if (since) url += `&since=${since}`; 136 136 + const res = await fetch(url, { 137 137 + headers: { 138 138 + Authorization: `Bearer ${token}`, 139 139 + Accept: "application/vnd.github+json", 140 140 + "User-Agent": "infra-status-worker", 141 141 + }, 142 142 + }); 143 143 + if (!res.ok) throw new Error(`GitHub fetch comments failed: ${res.status}`); 144 144 + return res.json(); 145 145 + } 146 146 + 147 147 + export async function syncGitHubIncidents( 148 148 + db: D1Database, 149 149 + kv: KVNamespace, 150 150 + token: string, 151 151 + incidents: Incident[], 152 152 + ): Promise<void> { 153 153 + for (const incident of incidents) { 154 154 + if (!incident.github_repo || !incident.github_issue_number) continue; 155 155 + 156 156 + const parsed = parseRepo(`https://github.com/${incident.github_repo}`); 157 157 + if (!parsed) continue; 158 158 + 159 159 + try { 160 160 + // Check issue state (closed = resolved) and sync body edits 161 161 + const issue = await fetchIssue(token, parsed.owner, parsed.repo, incident.github_issue_number); 162 162 + if (issue.state === "closed" && incident.status !== "resolved") { 163 163 + const now = Math.floor(Date.now() / 1000); 164 164 + await updateIncident(db, incident.id, { status: "resolved", resolved_at: now }); 165 165 + await addIncidentUpdate(db, incident.id, "resolved", "Issue closed on GitHub"); 166 166 + continue; 167 167 + } 168 168 + 169 169 + // Sync issue body edits back to triage_report 170 170 + if (issue.body && issue.body !== incident.triage_report) { 171 171 + await updateIncident(db, incident.id, { triage_report: issue.body }); 172 172 + } 173 173 + 174 174 + // Sync new comments since last check 175 175 + const kvKey = `gh_sync:${incident.id}:last`; 176 176 + const lastSeen = await kv.get(kvKey); 177 177 + const comments = await fetchComments(token, parsed.owner, parsed.repo, incident.github_issue_number, lastSeen ?? undefined); 178 178 + 179 179 + // Filter to human comments only (skip bots and our own posts) 180 180 + const human = comments.filter((c) => c.user.type !== "Bot" && !c.body.startsWith("Automated incident detected") && !c.body.startsWith("## Triage Report") && !c.body.startsWith("Service recovered automatically")); 181 181 + 182 182 + for (const comment of human) { 183 183 + await addIncidentUpdate(db, incident.id, incident.status, comment.body); 184 184 + } 185 185 + 186 186 + // Track last comment time so we don't re-import 187 187 + if (comments.length > 0) { 188 188 + const latest = comments[comments.length - 1].created_at; 189 189 + await kv.put(kvKey, latest, { expirationTtl: 86400 * 7 }); 190 190 + } 191 191 + } catch (_) {} // best effort, don't block other syncs 192 192 + } 193 193 + }

+105 -7

src/index.ts

reviewed

··· 1 1 import type { Env } from "./types"; 2 2 import { getManifest } from "./manifest"; 3 3 import { checkHealth } from "./health"; 4 4 - import { insertPing, pruneOldPings } from "./db"; 4 4 + import { insertPing, getLatestPing, pruneOldPings, createIncident, updateIncident, addIncidentUpdate, getActiveIncidentForService, getActiveIncidents, getRecentlyResolvedIncident, setIncidentGitHub } from "./db"; 5 5 import { refreshDevices } from "./tailscale"; 6 6 import { handleStatusRoute } from "./routes/status"; 7 7 import { handleFavicon } from "./routes/favicon"; 8 8 import { handleUptime } from "./routes/uptime"; 9 9 import { handleBadgeRoute } from "./routes/badge"; 10 10 import { handleIndex } from "./routes/index"; 11 11 + import { handleIncidentRoute } from "./routes/incidents"; 12 12 + import { createIssue, commentOnIssue, closeIssue, parseRepo, syncGitHubIncidents } from "./github"; 11 13 import { schemas } from "./schemas"; 12 14 13 15 async function handleRequest(request: Request, env: Env): Promise<Response> { ··· 54 56 if (badge) return badge; 55 57 } 56 58 59 59 + if (path.startsWith("/api/incidents")) { 60 60 + const res = await handleIncidentRoute(request, env, path); 61 61 + if (res) return res; 62 62 + } 63 63 + 57 64 return new Response("Not Found", { status: 404 }); 58 65 } 59 66 ··· 63 70 return new Response(null, { 64 71 headers: { 65 72 "Access-Control-Allow-Origin": "*", 66 66 - "Access-Control-Allow-Methods": "GET, OPTIONS", 67 67 - "Access-Control-Allow-Headers": "Content-Type", 73 73 + "Access-Control-Allow-Methods": "GET, POST, PATCH, OPTIONS", 74 74 + "Access-Control-Allow-Headers": "Content-Type, Authorization", 68 75 }, 69 76 }); 70 77 } ··· 81 88 refreshDevices(env), 82 89 ]); 83 90 84 84 - const checks = Object.values(manifest).flatMap((machine) => 85 85 - machine.services 91 91 + const checks = Object.values(manifest).flatMap((machine) => { 92 92 + const triageUrl = machine.triage_url; 93 93 + return machine.services 86 94 .filter((svc) => svc.health_url) 87 95 .map(async (svc) => { 96 96 + const previous = await getLatestPing(env.DB, svc.name); 88 97 const result = await checkHealth(svc); 89 98 await insertPing(env.DB, svc.name, result.status, result.latency_ms); 90 90 - }), 91 91 - ); 99 99 + 100 100 + const isDown = result.status === "down" || result.status === "timeout"; 101 101 + const wasUp = !previous || previous.status === "up" || previous.status === "degraded"; 102 102 + 103 103 + if (isDown) { 104 104 + // Track consecutive failures in KV for flap prevention 105 105 + const failKey = `triage:${svc.name}:failures`; 106 106 + const current = parseInt((await env.KV.get(failKey)) ?? "0"); 107 107 + const failures = current + 1; 108 108 + await env.KV.put(failKey, String(failures), { expirationTtl: 1800 }); 109 109 + 110 110 + // Only trigger after 2 consecutive failures (10 min of downtime) 111 111 + if (failures >= 2) { 112 112 + const existing = await getActiveIncidentForService(env.DB, svc.name); 113 113 + if (!existing) { 114 114 + // Check cooldown: no incident resolved in last 15 min 115 115 + const recent = await getRecentlyResolvedIncident(env.DB, svc.name, 900); 116 116 + if (!recent) { 117 117 + const id = await createIncident(env.DB, { 118 118 + service_id: svc.name, 119 119 + title: `${svc.name} is ${result.status}`, 120 120 + severity: "major", 121 121 + }); 122 122 + 123 123 + // Create GitHub issue on the service's repo 124 124 + if (env.GITHUB_TOKEN && svc.repository) { 125 125 + const parsed = parseRepo(svc.repository); 126 126 + if (parsed) { 127 127 + try { 128 128 + const issueNumber = await createIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, { 129 129 + title: `${svc.name} is ${result.status}`, 130 130 + body: `Automated incident detected by [infra.dunkirk.sh](https://infra.dunkirk.sh)\n\n**Service:** ${svc.name}\n**Health URL:** ${svc.health_url}\n**Status:** ${result.status}\n**Detected at:** ${new Date().toISOString()}\n\n---\n*Comments on this issue will appear on the status page. Close the issue to resolve the incident.*`, 131 131 + assignees: env.GITHUB_ASSIGNEE ? [env.GITHUB_ASSIGNEE] : [], 132 132 + labels: ["incident"], 133 133 + }); 134 134 + await setIncidentGitHub(env.DB, id, `${parsed.owner}/${parsed.repo}`, issueNumber); 135 135 + } catch (_) {} // best effort 136 136 + } 137 137 + } 138 138 + 139 139 + // Fire webhook to triage agent (non-blocking) 140 140 + if (triageUrl && env.TRIAGE_AUTH_TOKEN) { 141 141 + fetch(triageUrl, { 142 142 + method: "POST", 143 143 + headers: { 144 144 + "Content-Type": "application/json", 145 145 + Authorization: `Bearer ${env.TRIAGE_AUTH_TOKEN}`, 146 146 + }, 147 147 + body: JSON.stringify({ 148 148 + incident_id: id, 149 149 + service_id: svc.name, 150 150 + service_name: svc.name, 151 151 + health_url: svc.health_url, 152 152 + callback_url: `https://infra.dunkirk.sh/api/incidents/${id}`, 153 153 + }), 154 154 + }).catch(() => {}); // fire and forget 155 155 + } 156 156 + } 157 157 + } 158 158 + } 159 159 + } else { 160 160 + // Service is up — clear failure counter 161 161 + await env.KV.delete(`triage:${svc.name}:failures`); 162 162 + 163 163 + // Auto-resolve active incidents 164 164 + const active = await getActiveIncidentForService(env.DB, svc.name); 165 165 + if (active) { 166 166 + await updateIncident(env.DB, active.id, { 167 167 + status: "resolved", 168 168 + resolved_at: Math.floor(Date.now() / 1000), 169 169 + }); 170 170 + await addIncidentUpdate(env.DB, active.id, "resolved", "Service recovered automatically"); 171 171 + 172 172 + // Close the GitHub issue 173 173 + if (env.GITHUB_TOKEN && active.github_repo && active.github_issue_number) { 174 174 + const parsed = parseRepo(`https://github.com/${active.github_repo}`); 175 175 + if (parsed) { 176 176 + commentOnIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, active.github_issue_number, "Service recovered automatically. Closing issue.").catch(() => {}); 177 177 + closeIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, active.github_issue_number).catch(() => {}); 178 178 + } 179 179 + } 180 180 + } 181 181 + } 182 182 + }); 183 183 + }); 92 184 93 185 await Promise.all(checks); 94 186 await pruneOldPings(env.DB, 365); 187 187 + 188 188 + // Sync GitHub issue comments/state back to incidents 189 189 + if (env.GITHUB_TOKEN) { 190 190 + const active = await getActiveIncidents(env.DB); 191 191 + await syncGitHubIncidents(env.DB, env.KV, env.GITHUB_TOKEN, active); 192 192 + } 95 193 }, 96 194 } satisfies ExportedHandler<Env>;

+7 -3

src/overall.ts

reviewed

··· 1 1 import type { Env } from "./types"; 2 2 import { getManifest } from "./manifest"; 3 3 - import { getLatestPing } from "./db"; 3 3 + import { getLatestPing, getActiveIncidents } from "./db"; 4 4 import { getDeviceStatus } from "./tailscale"; 5 5 6 6 export type OverallGrade = "up" | "degraded" | "down"; ··· 42 42 s === "misconfigured", 43 43 ); 44 44 45 45 - if (onFire) return { grade: "down", label: "On fire" }; 46 46 - if (hasDegraded) return { grade: "degraded", label: "Some systems degraded" }; 45 45 + const activeIncidents = await getActiveIncidents(env.DB); 46 46 + const hasCritical = activeIncidents.some((i) => i.severity === "critical"); 47 47 + const hasMajor = activeIncidents.some((i) => i.severity === "major"); 48 48 + 49 49 + if (hasCritical || onFire) return { grade: "down", label: "On fire" }; 50 50 + if (hasMajor || hasDegraded) return { grade: "degraded", label: "Some systems degraded" }; 47 51 return { grade: "up", label: "All systems operational" }; 48 52 }

+101

src/routes/incidents.ts

reviewed

··· 1 1 + import type { Env } from "../types"; 2 2 + import { 3 3 + getActiveIncidents, 4 4 + getRecentIncidents, 5 5 + getIncident, 6 6 + createIncident, 7 7 + updateIncident, 8 8 + addIncidentUpdate, 9 9 + } from "../db"; 10 10 + import { commentOnIssue, editIssueBody, parseRepo } from "../github"; 11 11 + 12 12 + function authCheck(request: Request, env: Env): boolean { 13 13 + const auth = request.headers.get("Authorization"); 14 14 + if (!auth || !env.TRIAGE_AUTH_TOKEN) return false; 15 15 + return auth === `Bearer ${env.TRIAGE_AUTH_TOKEN}`; 16 16 + } 17 17 + 18 18 + export async function handleIncidentRoute( 19 19 + request: Request, 20 20 + env: Env, 21 21 + path: string, 22 22 + ): Promise<Response | null> { 23 23 + // GET /api/incidents 24 24 + if (path === "/api/incidents" && request.method === "GET") { 25 25 + const active = await getActiveIncidents(env.DB); 26 26 + const recent = await getRecentIncidents(env.DB, 7); 27 27 + const resolved = recent.filter((i) => i.status === "resolved"); 28 28 + return Response.json({ active, recent_resolved: resolved }); 29 29 + } 30 30 + 31 31 + // POST /api/incidents 32 32 + if (path === "/api/incidents" && request.method === "POST") { 33 33 + if (!authCheck(request, env)) { 34 34 + return Response.json({ error: "unauthorized" }, { status: 401 }); 35 35 + } 36 36 + const body = await request.json<{ service_id: string; title: string; severity: "critical" | "major" | "minor" }>(); 37 37 + if (!body.service_id || !body.title || !body.severity) { 38 38 + return Response.json({ error: "missing fields" }, { status: 400 }); 39 39 + } 40 40 + const id = await createIncident(env.DB, body); 41 41 + return Response.json({ id }, { status: 201 }); 42 42 + } 43 43 + 44 44 + // GET /api/incidents/:id 45 45 + const singleMatch = path.match(/^\/api\/incidents\/(\d+)$/); 46 46 + if (singleMatch && request.method === "GET") { 47 47 + const incident = await getIncident(env.DB, parseInt(singleMatch[1])); 48 48 + if (!incident) return Response.json({ error: "not found" }, { status: 404 }); 49 49 + return Response.json(incident); 50 50 + } 51 51 + 52 52 + // PATCH /api/incidents/:id 53 53 + if (singleMatch && request.method === "PATCH") { 54 54 + if (!authCheck(request, env)) { 55 55 + return Response.json({ error: "unauthorized" }, { status: 401 }); 56 56 + } 57 57 + const id = parseInt(singleMatch[1]); 58 58 + const body = await request.json<{ status?: string; triage_report?: string }>(); 59 59 + const updateData: { status?: string; triage_report?: string; resolved_at?: number } = {}; 60 60 + if (body.status) updateData.status = body.status; 61 61 + if (body.triage_report !== undefined) updateData.triage_report = body.triage_report; 62 62 + if (body.status === "resolved") updateData.resolved_at = Math.floor(Date.now() / 1000); 63 63 + await updateIncident(env.DB, id, updateData); 64 64 + if (body.status) { 65 65 + await addIncidentUpdate(env.DB, id, body.status, body.triage_report ?? `Status changed to ${body.status}`); 66 66 + } 67 67 + 68 68 + // Sync triage report to GitHub issue 69 69 + const incident = await getIncident(env.DB, id); 70 70 + if (env.GITHUB_TOKEN && incident?.github_repo && incident.github_issue_number) { 71 71 + const parsed = parseRepo(`https://github.com/${incident.github_repo}`); 72 72 + if (parsed) { 73 73 + if (body.triage_report) { 74 74 + // Post triage report as comment and update issue body 75 75 + commentOnIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, incident.github_issue_number, `## Triage Report\n\n${body.triage_report}`).catch(() => {}); 76 76 + editIssueBody(env.GITHUB_TOKEN, parsed.owner, parsed.repo, incident.github_issue_number, body.triage_report).catch(() => {}); 77 77 + } else if (body.status) { 78 78 + commentOnIssue(env.GITHUB_TOKEN, parsed.owner, parsed.repo, incident.github_issue_number, `Status changed to **${body.status}**`).catch(() => {}); 79 79 + } 80 80 + } 81 81 + } 82 82 + 83 83 + return Response.json({ ok: true }); 84 84 + } 85 85 + 86 86 + // POST /api/incidents/:id/updates 87 87 + const updatesMatch = path.match(/^\/api\/incidents\/(\d+)\/updates$/); 88 88 + if (updatesMatch && request.method === "POST") { 89 89 + if (!authCheck(request, env)) { 90 90 + return Response.json({ error: "unauthorized" }, { status: 401 }); 91 91 + } 92 92 + const body = await request.json<{ status: string; message: string }>(); 93 93 + if (!body.status || !body.message) { 94 94 + return Response.json({ error: "missing fields" }, { status: 400 }); 95 95 + } 96 96 + await addIncidentUpdate(env.DB, parseInt(updatesMatch[1]), body.status, body.message); 97 97 + return Response.json({ ok: true }, { status: 201 }); 98 98 + } 99 99 + 100 100 + return null; 101 101 + }

+57 -1

src/routes/index.ts

reviewed

··· 1 1 import type { Env } from "../types"; 2 2 import { getManifest } from "../manifest"; 3 3 - import { getLatestPing, getUptime7d, getOverallUptimeDays, getLastCheckTime } from "../db"; 3 3 + import { getLatestPing, getUptime7d, getOverallUptimeDays, getLastCheckTime, getActiveIncidentsWithUpdates, getRecentIncidents } from "../db"; 4 4 import { getDeviceStatus } from "../tailscale"; 5 5 import { getOverallStatus } from "../overall"; 6 6 import { COMMIT_SHA } from "../version"; ··· 40 40 41 41 const { grade: overallClass, label: overallText } = await getOverallStatus(env); 42 42 const uptimeDays = await getOverallUptimeDays(env.DB, 90); 43 43 + const activeIncidents = await getActiveIncidentsWithUpdates(env.DB); 44 44 + const recentIncidents = await getRecentIncidents(env.DB, 7); 45 45 + const resolvedIncidents = recentIncidents.filter((i) => i.status === "resolved"); 43 46 44 47 const html = `<!DOCTYPE html> 45 48 <html lang="en"> ··· 97 100 .uptime-bar .day.degraded { background: #f39c12; } 98 101 .uptime-bar .day.down { background: #e74c3c; } 99 102 .uptime-bar .day.none { background: #21262d; } 103 103 + .incidents { margin-bottom: 1.5rem; } 104 104 + .incident-banner { background: #2d1b1b; border: 1px solid #e74c3c; border-radius: 6px; padding: 0.75rem; margin-bottom: 0.5rem; } 105 105 + .incident-banner.major { border-color: #f39c12; background: #2d2517; } 106 106 + .incident-banner.minor { border-color: #8b949e; background: #21262d; } 107 107 + .incident-title { font-size: 0.85rem; font-weight: 500; margin-bottom: 0.25rem; } 108 108 + .incident-meta { font-size: 0.7rem; color: #8b949e; display: flex; gap: 0.75rem; } 109 109 + .incident-status { text-transform: uppercase; letter-spacing: 0.05em; } 110 110 + .incident-status.investigating { color: #e74c3c; } 111 111 + .incident-status.identified { color: #f39c12; } 112 112 + .incident-status.monitoring { color: #3498db; } 113 113 + .incident-triage { margin-top: 0.5rem; } 114 114 + .incident-triage summary { font-size: 0.75rem; color: #8b949e; cursor: pointer; } 115 115 + .incident-triage pre { font-size: 0.7rem; color: #c9d1d9; background: #161b22; padding: 0.5rem; border-radius: 4px; margin-top: 0.25rem; white-space: pre-wrap; word-break: break-word; max-height: 300px; overflow-y: auto; } 116 116 + .incident-timeline { margin-top: 0.5rem; padding-left: 0.75rem; border-left: 2px solid #21262d; } 117 117 + .timeline-entry { padding: 0.25rem 0 0.25rem 0.5rem; font-size: 0.7rem; position: relative; } 118 118 + .timeline-entry::before { content: ''; position: absolute; left: -0.75rem; top: 0.55rem; width: 6px; height: 6px; border-radius: 50%; background: #30363d; transform: translateX(-2px); } 119 119 + .timeline-entry.investigating::before { background: #e74c3c; } 120 120 + .timeline-entry.identified::before { background: #f39c12; } 121 121 + .timeline-entry.monitoring::before { background: #3498db; } 122 122 + .timeline-entry.resolved::before { background: #2ecc71; } 123 123 + .timeline-status { text-transform: uppercase; letter-spacing: 0.04em; font-size: 0.6rem; font-weight: 600; } 124 124 + .timeline-status.investigating { color: #e74c3c; } 125 125 + .timeline-status.identified { color: #f39c12; } 126 126 + .timeline-status.monitoring { color: #3498db; } 127 127 + .timeline-status.resolved { color: #2ecc71; } 128 128 + .timeline-time { color: #484f58; margin-right: 0.5rem; } 129 129 + .timeline-msg { color: #8b949e; } 130 130 + .resolved-incidents { margin-top: 0.75rem; } 131 131 + .resolved-header { font-size: 0.7rem; text-transform: uppercase; letter-spacing: 0.08em; color: #8b949e; margin-bottom: 0.25rem; } 132 132 + .resolved-item { font-size: 0.75rem; color: #8b949e; padding: 0.25rem 0; border-bottom: 1px solid #21262d; } 133 133 + .resolved-item:last-child { border-bottom: none; } 100 134 footer { margin-top: auto; padding-top: 1rem; border-top: 1px solid #21262d; font-size: 0.7rem; color: #8b949e; } 101 135 .footer-meta { display: flex; justify-content: space-between; } 102 136 footer a { color: #8b949e; text-decoration: none; } ··· 107 141 <div class="uptime-bar">${uptimeDays.map((d) => `<div class="day ${d.status}" title="${d.date}: ${d.status}"></div>`).join("")}</div> 108 142 <h1>infra.dunkirk.sh</h1> 109 143 <p class="overall"><span class="dot ${overallClass}" id="overall-dot" title="${overallClass}"></span><span id="overall-text">${overallText}</span></p> 144 144 + ${activeIncidents.length > 0 ? `<div class="incidents"> 145 145 + ${activeIncidents.map((i) => `<div class="incident-banner ${i.severity}"> 146 146 + <div class="incident-title">${esc(i.title)}</div> 147 147 + <div class="incident-meta"> 148 148 + <span class="incident-status ${i.status}">${i.status}</span> 149 149 + <span>${esc(i.service_id)}</span> 150 150 + <span>started <relative-time datetime="${new Date(i.started_at * 1000).toISOString()}">loading</relative-time></span> 151 151 + </div> 152 152 + ${i.triage_report ? `<details class="incident-triage"><summary>triage report</summary><pre>${esc(i.triage_report)}</pre></details>` : ""} 153 153 + ${i.updates.length > 0 ? `<div class="incident-timeline"> 154 154 + ${i.updates.map((u) => `<div class="timeline-entry ${u.status}"> 155 155 + <span class="timeline-time">${new Date(u.created_at * 1000).toLocaleTimeString("en-US", { hour: "2-digit", minute: "2-digit", hour12: false, timeZone: "America/New_York" })}</span> 156 156 + <span class="timeline-status ${u.status}">${esc(u.status)}</span> 157 157 + <span class="timeline-msg">${esc(u.message)}</span> 158 158 + </div>`).join("\n")} 159 159 + </div>` : ""} 160 160 + </div>`).join("\n")} 161 161 + </div>` : ""} 110 162 ${servers 111 163 .map( 112 164 (m) => `<div class="machine"> ··· 132 184 <div class="clients-list"> 133 185 ${clients.map((m) => `<span class="client"><span class="dot ${m.online ? "online" : "unknown"}" data-machine="${esc(m.name)}" title="${m.online ? "online" : "offline"}"></span>${esc(m.name)}</span>`).join("\n")} 134 186 </div> 187 187 + </div>` : ""} 188 188 + ${resolvedIncidents.length > 0 ? `<div class="resolved-incidents"> 189 189 + <div class="resolved-header">recent incidents</div> 190 190 + ${resolvedIncidents.map((i) => `<div class="resolved-item">${esc(i.title)} — resolved <relative-time datetime="${new Date((i.resolved_at ?? i.updated_at) * 1000).toISOString()}">loading</relative-time></div>`).join("\n")} 135 191 </div>` : ""} 136 192 <footer> 137 193 <div class="footer-meta"><span>${lastCheckISO ? `updated <relative-time datetime="${lastCheckISO}" prefix="">loading</relative-time>` : "no checks yet"}</span><a href="https://github.com/taciturnaxolotl/status/commit/${COMMIT_SHA}">${COMMIT_SHA}</a></div>

+31

src/types.ts

reviewed

··· 17 17 hostname: string; 18 18 tailscale_host: string; 19 19 type: "server" | "client"; 20 20 + triage_url: string | null; 20 21 services: Service[]; 21 22 } 22 23 ··· 31 32 }[]; 32 33 } 33 34 35 35 + export interface Incident { 36 36 + id: number; 37 37 + service_id: string; 38 38 + title: string; 39 39 + status: "investigating" | "identified" | "monitoring" | "resolved"; 40 40 + severity: "critical" | "major" | "minor"; 41 41 + triage_report: string | null; 42 42 + github_repo: string | null; 43 43 + github_issue_number: number | null; 44 44 + started_at: number; 45 45 + resolved_at: number | null; 46 46 + created_at: number; 47 47 + updated_at: number; 48 48 + } 49 49 + 50 50 + export interface IncidentUpdate { 51 51 + id: number; 52 52 + incident_id: number; 53 53 + status: string; 54 54 + message: string; 55 55 + created_at: number; 56 56 + } 57 57 + 58 58 + export interface IncidentWithUpdates extends Incident { 59 59 + updates: IncidentUpdate[]; 60 60 + } 61 61 + 34 62 export interface Env { 35 63 DB: D1Database; 36 64 KV: KVNamespace; 37 65 TAILSCALE_API_KEY?: string; 66 66 + TRIAGE_AUTH_TOKEN?: string; 67 67 + GITHUB_TOKEN?: string; 68 68 + GITHUB_ASSIGNEE?: string; 38 69 }

+3

wrangler.toml

reviewed

··· 14 14 [[kv_namespaces]] 15 15 binding = "KV" 16 16 id = "c4daf2e998ea45e1aad9efb6636b66df" 17 17 + 18 18 + [vars] 19 19 + GITHUB_ASSIGNEE = "taciturnaxolotl"