my harness for niri
metrics
+235
-3
+2
src/index.ts
+2
src/index.ts
···
1
1
import { openBash, closeBash } from "./container/index.js"
2
2
import { createServer } from "./server.js"
3
3
import { initDb } from "./db.js"
4
+
import { initMetricsDb } from "./metrics.js"
4
5
import { shutdown } from "./runner/index.js"
5
6
import { startDiscordGateway } from "./discord/gateway.js"
6
7
import { ensureSoulFilePlacement } from "./bootstrap.js"
···
12
13
13
14
await ensureSoulFilePlacement()
14
15
initDb()
16
+
initMetricsDb()
15
17
await openBash()
16
18
17
19
let discordGateway: Awaited<ReturnType<typeof startDiscordGateway>> = null
+14
-1
src/memory.ts
+14
-1
src/memory.ts
···
4
4
import { fileURLToPath } from "url"
5
5
import type { Message } from "./types.js"
6
6
import { getDb } from "./db.js"
7
+
import { recordMetric } from "./metrics.js"
7
8
8
9
const HOME_DIR = path.resolve(fileURLToPath(import.meta.url), "../../home")
9
10
const MEMORIES_DIR = path.join(HOME_DIR, "memories")
···
655
656
`[memory] recalled query=${JSON.stringify(trimForPrompt(normalizeText(latestUser), 120))} personQuery=${profile.personQuery} eventQuery=${profile.eventQuery}\n${recallContent}`,
656
657
)
657
658
659
+
recordMetric({
660
+
type: "memory",
661
+
query: latestUser,
662
+
results: hits.map(toMemorySearchResult),
663
+
})
664
+
658
665
const recallMessage: Message = {
659
666
role: "user",
660
667
content: recallContent,
···
687
694
const profile = buildSearchProfile(rawQuery)
688
695
if (profile.tokens.length === 0) return []
689
696
690
-
return searchMemory(profile, {}, Number.POSITIVE_INFINITY, Math.max(1, Math.min(limit, 10))).map(toMemorySearchResult)
697
+
const results = searchMemory(profile, {}, Number.POSITIVE_INFINITY, Math.max(1, Math.min(limit, 10))).map(toMemorySearchResult)
698
+
recordMetric({
699
+
type: "memory",
700
+
query: rawQuery,
701
+
results,
702
+
})
703
+
return results
691
704
}
+151
src/metrics.ts
+151
src/metrics.ts
···
1
+
import Database from "better-sqlite3"
2
+
import path from "path"
3
+
import { fileURLToPath } from "url"
4
+
import type OpenAI from "openai"
5
+
import type { Message } from "./types.js"
6
+
import type { MemorySearchResult } from "./memory.js"
7
+
8
+
export interface BaseMetricEvent {
9
+
timestamp: string
10
+
}
11
+
12
+
export interface PromptMetric extends BaseMetricEvent {
13
+
type: "prompt"
14
+
messages: Message[]
15
+
}
16
+
17
+
export interface MemoryMetric extends BaseMetricEvent {
18
+
type: "memory"
19
+
query: string
20
+
results: MemorySearchResult[]
21
+
}
22
+
23
+
export interface CompactionMetric extends BaseMetricEvent {
24
+
type: "compaction"
25
+
before: number
26
+
after: number
27
+
method: string
28
+
summary?: string
29
+
}
30
+
31
+
export interface UsageMetric extends BaseMetricEvent {
32
+
type: "usage"
33
+
usage: OpenAI.Completions.CompletionUsage
34
+
}
35
+
36
+
export type MetricEvent = PromptMetric | MemoryMetric | CompactionMetric | UsageMetric
37
+
38
+
export interface MetricEventSummary extends BaseMetricEvent {
39
+
id: number
40
+
type: MetricEvent["type"]
41
+
// Metadata for the feed
42
+
messageCount?: number
43
+
resultCount?: number
44
+
method?: string
45
+
before?: number
46
+
after?: number
47
+
usage?: OpenAI.Completions.CompletionUsage
48
+
}
49
+
50
+
export type MetricEventInput =
51
+
| Omit<PromptMetric, "timestamp">
52
+
| Omit<MemoryMetric, "timestamp">
53
+
| Omit<CompactionMetric, "timestamp">
54
+
| Omit<UsageMetric, "timestamp">
55
+
56
+
interface MetricRow {
57
+
id: number
58
+
type: string
59
+
payload: string
60
+
createdAt: string
61
+
}
62
+
63
+
const HOME_DIR = path.resolve(fileURLToPath(import.meta.url), "../../home")
64
+
const DB_PATH = path.join(HOME_DIR, "metrics.db")
65
+
66
+
let db: Database.Database
67
+
const events: (MetricEvent & { id: number })[] = []
68
+
const MAX_IN_MEMORY = 100
69
+
70
+
export function initMetricsDb(): void {
71
+
db = new Database(DB_PATH)
72
+
db.pragma("journal_mode = WAL")
73
+
db.exec(`
74
+
create table if not exists metrics (
75
+
id integer primary key autoincrement,
76
+
type text not null,
77
+
payload text not null,
78
+
createdAt text not null
79
+
);
80
+
create index if not exists idx_metrics_type on metrics(type);
81
+
create index if not exists idx_metrics_created on metrics(createdAt desc);
82
+
`)
83
+
console.log("[metrics] ready")
84
+
}
85
+
86
+
export function recordMetric(event: MetricEventInput): void {
87
+
const timestamp = new Date().toISOString()
88
+
89
+
if (db) {
90
+
try {
91
+
const stmt = db.prepare("insert into metrics (type, payload, createdAt) values (?, ?, ?)")
92
+
const result = stmt.run(event.type, JSON.stringify(event), timestamp)
93
+
94
+
const fullEvent = { ...event, timestamp, id: Number(result.lastInsertRowid) } as MetricEvent & { id: number }
95
+
events.push(fullEvent)
96
+
if (events.length > MAX_IN_MEMORY) {
97
+
events.shift()
98
+
}
99
+
} catch (err) {
100
+
console.error("[metrics] failed to record to db:", err)
101
+
}
102
+
}
103
+
}
104
+
105
+
export function getMetrics(limit = 100): MetricEventSummary[] {
106
+
if (db) {
107
+
try {
108
+
const rows = db.prepare("select id, type, payload, createdAt from metrics order by createdAt desc limit ?").all(limit) as MetricRow[]
109
+
return rows.map((row) => {
110
+
const payload = JSON.parse(row.payload)
111
+
const summary: MetricEventSummary = {
112
+
id: row.id,
113
+
type: row.type as any,
114
+
timestamp: row.createdAt,
115
+
}
116
+
117
+
if (row.type === "prompt") {
118
+
summary.messageCount = payload.messages?.length
119
+
} else if (row.type === "memory") {
120
+
summary.resultCount = payload.results?.length
121
+
} else if (row.type === "compaction") {
122
+
summary.method = payload.method
123
+
summary.before = payload.before
124
+
summary.after = payload.after
125
+
} else if (row.type === "usage") {
126
+
summary.usage = payload.usage
127
+
}
128
+
129
+
return summary
130
+
})
131
+
} catch (err) {
132
+
console.error("[metrics] failed to fetch from db:", err)
133
+
}
134
+
}
135
+
return []
136
+
}
137
+
138
+
export function getMetricDetail(id: number): (MetricEvent & { id: number }) | null {
139
+
if (db) {
140
+
try {
141
+
const row = db.prepare("select id, type, payload, createdAt from metrics where id = ?").get(id) as MetricRow | undefined
142
+
if (row) {
143
+
const payload = JSON.parse(row.payload)
144
+
return { ...payload, id: row.id, timestamp: row.createdAt }
145
+
}
146
+
} catch (err) {
147
+
console.error("[metrics] failed to fetch detail from db:", err)
148
+
}
149
+
}
150
+
return null
151
+
}
+54
-1
src/runner/loop.ts
+54
-1
src/runner/loop.ts
···
28
28
estimatePromptTokens,
29
29
fallbackClient,
30
30
fallbackContextWindow,
31
+
findSummaryMessageIndex,
31
32
forceCompactConversation,
32
33
isPromptTooLargeError,
33
34
maybeCompactConversation,
···
38
39
summarizeConversationViaLLM,
39
40
} from "./util.js"
40
41
import { buildCompletionMessages, rememberRecalledMemoryChunks, searchMemories } from "../memory.js"
42
+
import { recordMetric } from "../metrics.js"
41
43
42
44
const LLM_POST_TURN_RECENT_MESSAGES = 40
43
45
···
272
274
stream: true,
273
275
stream_options: { include_usage: true },
274
276
} as const
277
+
278
+
recordMetric({ type: "prompt", messages: request.messages })
275
279
276
280
try {
277
281
const stream = await apiClient.chat.completions.create(streamedRequest)
···
335
339
state.tokenCount += usage.total_tokens
336
340
if (usage.prompt_tokens) state.contextSize = usage.prompt_tokens
337
341
console.log(`[tokens] +${usage.total_tokens} total=${state.tokenCount}`)
342
+
recordMetric({ type: "usage", usage })
338
343
}
339
344
340
345
/**
···
514
519
if (result.compacted) {
515
520
state.conversation = result.messages
516
521
state.contextSize = result.estimateAfter
522
+
523
+
const summaryIdx = findSummaryMessageIndex(state.conversation)
524
+
const summary = summaryIdx >= 0 ? (state.conversation[summaryIdx]?.content as string) : undefined
525
+
517
526
console.warn(
518
527
`[context] recovery: force-compacted ${result.messagesRemoved} msgs across ${result.chunks} chunks (${result.estimateBefore} -> ${result.estimateAfter}, target=${targetTokens})`,
519
528
)
529
+
530
+
recordMetric({
531
+
type: "compaction",
532
+
before: result.estimateBefore,
533
+
after: result.estimateAfter,
534
+
method: "force-heuristic",
535
+
summary,
536
+
})
520
537
return true
521
538
}
522
539
console.warn(`[context] recovery: force-compaction produced no changes (messages=${beforeCount}, est=${beforeEstimate})`)
···
545
562
546
563
state.conversation = summarized
547
564
state.contextSize = afterEstimate
565
+
566
+
const summaryIdx = findSummaryMessageIndex(state.conversation)
567
+
const summary = summaryIdx >= 0 ? (state.conversation[summaryIdx]?.content as string) : undefined
568
+
548
569
console.warn(
549
570
`[context] recovery: llm-summarized conversation (${beforeCount} -> ${summarized.length} msgs, ${beforeEstimate} -> ${afterEstimate} tokens)`,
550
571
)
572
+
573
+
recordMetric({
574
+
type: "compaction",
575
+
before: beforeEstimate,
576
+
after: afterEstimate,
577
+
method: "force-llm",
578
+
summary,
579
+
})
551
580
return true
552
581
}
553
582
···
1134
1163
state.conversation = result.messages
1135
1164
state.contextSize = result.estimateAfter
1136
1165
1166
+
const summaryIdx = findSummaryMessageIndex(state.conversation)
1167
+
const summary = summaryIdx >= 0 ? (state.conversation[summaryIdx]?.content as string) : undefined
1168
+
1137
1169
console.log(
1138
1170
`[context] ${phase} compacted ${result.messagesRemoved} messages across ${result.chunks} chunks (${result.estimateBefore} -> ${result.estimateAfter})`,
1139
1171
)
1172
+
1173
+
recordMetric({
1174
+
type: "compaction",
1175
+
before: result.estimateBefore,
1176
+
after: result.estimateAfter,
1177
+
method: "rolling",
1178
+
summary,
1179
+
})
1140
1180
return true
1141
1181
}
1142
1182
···
1164
1204
if (afterEstimate < beforeEstimate) {
1165
1205
state.conversation = summarized
1166
1206
state.contextSize = afterEstimate
1207
+
1208
+
const summaryIdx = findSummaryMessageIndex(state.conversation)
1209
+
const summary = summaryIdx >= 0 ? (state.conversation[summaryIdx]?.content as string) : undefined
1210
+
1167
1211
console.log(
1168
-
`[context] post-turn llm-summarized (${beforeCount} -> ${summarized.length} msgs, ${beforeEstimate} -> ${afterEstimate})`,
1212
+
`[context] post-turn llm-summarized conversation (${beforeCount} -> ${summarized.length} msgs, ${beforeEstimate} -> ${afterEstimate} tokens)`,
1169
1213
)
1214
+
1215
+
recordMetric({
1216
+
type: "compaction",
1217
+
before: beforeEstimate,
1218
+
after: afterEstimate,
1219
+
method: "post-turn-llm",
1220
+
summary,
1221
+
})
1170
1222
return true
1171
1223
}
1224
+
1172
1225
console.warn(
1173
1226
`[context] post-turn llm summary not smaller (${beforeEstimate} -> ${afterEstimate}); falling back to heuristic`,
1174
1227
)
+1
-1
src/runner/util.ts
+1
-1
src/runner/util.ts
···
926
926
return count
927
927
}
928
928
929
-
function findSummaryMessageIndex(messages: Message[]): number {
929
+
export function findSummaryMessageIndex(messages: Message[]): number {
930
930
return messages.findIndex((message) => {
931
931
const content = messageStringContent(message)
932
932
return content.startsWith(CONTEXT_SUMMARY_HEADER)
+13
src/server.ts
+13
src/server.ts
···
11
11
import { fromCron } from "./triggers/cron.js"
12
12
import { fromChat } from "./triggers/chat.js"
13
13
import { subscribe } from "./stream.js"
14
+
import { getMetrics, getMetricDetail } from "./metrics.js"
14
15
import type { UserMessage } from "./types.js"
15
16
16
17
const SRC_DIR = dirname(fileURLToPath(import.meta.url))
···
190
191
app.get("/status", async () => ({
191
192
running: isRunning(),
192
193
}))
194
+
195
+
app.get("/metrics", async (req) => {
196
+
const { limit } = req.query as { limit?: string }
197
+
return getMetrics(limit ? parseInt(limit, 10) : 100)
198
+
})
199
+
200
+
app.get("/metrics/:id", async (req, reply) => {
201
+
const { id } = req.params as { id: string }
202
+
const metric = getMetricDetail(parseInt(id, 10))
203
+
if (!metric) return reply.code(404).send({ error: "metric not found" })
204
+
return metric
205
+
})
193
206
194
207
return app
195
208
}