cxs is a local-first CLI for searching Codex session logs. It is designed for progressive retrieval: find the right session first, then read
feat(eval): 加 perf-bench 脚本,真实大库 advisory 基准
eval/perf-bench.ts 复用本机 ~/.codex/sessions 真实数据,跑 sync 全量 +
7 个代表性 query (单 token / 多 token / CJK / 中英混合) 各 5 次取后 4
样本 p50/p95,产出 data/cxs-perf/<ts>/report.{json,md}。stdout 打 JSON
summary 对齐 eval/run-manual-eval 风格。
实跑参考: 2867 session 全量 sync ~75s,db 240MB,单 token query p95
~86-90ms,双 token "edge tts" p95 190ms (FTS AND-match 成本)。
仅 advisory,不做 CI gate;P2 阶段如果上 reranker 可用作回归对照。
对应 review 报告 P1-3。
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Entire-Checkpoint: 5506719bb0ff
+236
-1
+234
eval/perf-bench.ts
+234
eval/perf-bench.ts
···
1
+
#!/usr/bin/env bun
2
+
3
+
import { mkdirSync, writeFileSync } from "node:fs";
4
+
import { homedir, tmpdir } from "node:os";
5
+
import { join, resolve } from "node:path";
6
+
import { existsSync } from "node:fs";
7
+
8
+
interface PerQueryRecord {
9
+
query: string;
10
+
runs: number;
11
+
samplesMs: number[];
12
+
p50Ms: number;
13
+
p95Ms: number;
14
+
}
15
+
16
+
interface Report {
17
+
generatedAt: string;
18
+
dbPath: string;
19
+
rootDir: string;
20
+
sessionCount: number;
21
+
syncMs: number;
22
+
dbSizeBytes: number;
23
+
perQuery: PerQueryRecord[];
24
+
}
25
+
26
+
// Bench query 选取原则:
27
+
// - 单 token 高频(hammerspoon/envchain): 检验最常见广义 fts 命中
28
+
// - 短 token(sb): 检验 trigram fallback 路径
29
+
// - 多 token 英文(fly deploy / edge tts): 检验多 term AND 路径
30
+
// - CJK 短语(豆包输入法): 检验 CJK trigram 路径
31
+
// - 中英混合(部署 health check): 检验 mixed match
32
+
const BENCH_QUERIES: string[] = [
33
+
"hammerspoon",
34
+
"envchain",
35
+
"sb",
36
+
"fly deploy",
37
+
"edge tts",
38
+
"豆包输入法",
39
+
"部署 health check",
40
+
];
41
+
42
+
const RUNS_PER_QUERY = 5; // 第 1 次作为 warmup,统计后 4 次
43
+
const ROOT = resolve(import.meta.dir, "..");
44
+
const OUT_BASE = resolve(ROOT, "data", "cxs-perf");
45
+
46
+
interface CliArgs {
47
+
root: string;
48
+
db: string;
49
+
jsonOnly: boolean;
50
+
}
51
+
52
+
function parseArgs(argv: string[]): CliArgs {
53
+
let root = join(homedir(), ".codex", "sessions");
54
+
let db = join(tmpdir(), `cxs-perf-${Date.now()}.db`);
55
+
let jsonOnly = false;
56
+
for (let i = 0; i < argv.length; i++) {
57
+
const a = argv[i];
58
+
if (a === "--root") {
59
+
root = resolve(argv[++i] ?? root);
60
+
} else if (a === "--db") {
61
+
db = resolve(argv[++i] ?? db);
62
+
} else if (a === "--json-only") {
63
+
jsonOnly = true;
64
+
} else if (a === "--help" || a === "-h") {
65
+
console.log("Usage: bun run eval:perf [--root <dir>] [--db <path>] [--json-only]");
66
+
process.exit(0);
67
+
}
68
+
}
69
+
return { root, db, jsonOnly };
70
+
}
71
+
72
+
const args = parseArgs(process.argv.slice(2));
73
+
74
+
if (!existsSync(args.root)) {
75
+
console.error(`error: --root not found: ${args.root}`);
76
+
process.exit(1);
77
+
}
78
+
79
+
interface RunResult {
80
+
stdout: string;
81
+
stderr: string;
82
+
exitCode: number;
83
+
ms: number;
84
+
}
85
+
86
+
async function run(cmd: string[]): Promise<RunResult> {
87
+
const t0 = performance.now();
88
+
const proc = Bun.spawn(cmd, { cwd: ROOT, stdout: "pipe", stderr: "pipe" });
89
+
const [stdout, stderr, exitCode] = await Promise.all([
90
+
new Response(proc.stdout).text(),
91
+
new Response(proc.stderr).text(),
92
+
proc.exited,
93
+
]);
94
+
const ms = performance.now() - t0;
95
+
return { stdout, stderr, exitCode, ms };
96
+
}
97
+
98
+
async function runOrThrow(cmd: string[]): Promise<RunResult> {
99
+
const r = await run(cmd);
100
+
if (r.exitCode !== 0) {
101
+
throw new Error(`command failed (exit ${r.exitCode}): ${cmd.join(" ")}\n${r.stderr}`);
102
+
}
103
+
return r;
104
+
}
105
+
106
+
function median(sorted: number[]): number {
107
+
const n = sorted.length;
108
+
if (n === 0) return 0;
109
+
const mid = Math.floor(n / 2);
110
+
if (n % 2 === 1) return sorted[mid]!;
111
+
return (sorted[mid - 1]! + sorted[mid]!) / 2;
112
+
}
113
+
114
+
function percentile(samplesMs: number[], p: number): number {
115
+
// 4 个样本下 p95 数学意义薄弱: 直接取 max 作为 worst-case 近似
116
+
if (samplesMs.length === 0) return 0;
117
+
if (p >= 0.99) return Math.max(...samplesMs);
118
+
const sorted = [...samplesMs].sort((a, b) => a - b);
119
+
const idx = Math.min(sorted.length - 1, Math.floor(p * sorted.length));
120
+
return sorted[idx]!;
121
+
}
122
+
123
+
function fmtMs(n: number): string {
124
+
return n.toFixed(1).padStart(8);
125
+
}
126
+
127
+
function fmtBytes(n: number): string {
128
+
if (n < 1024) return `${n} B`;
129
+
if (n < 1024 * 1024) return `${(n / 1024).toFixed(1)} KB`;
130
+
if (n < 1024 * 1024 * 1024) return `${(n / 1024 / 1024).toFixed(1)} MB`;
131
+
return `${(n / 1024 / 1024 / 1024).toFixed(2)} GB`;
132
+
}
133
+
134
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
135
+
const outDir = args.jsonOnly ? "" : join(OUT_BASE, stamp);
136
+
if (!args.jsonOnly) {
137
+
mkdirSync(outDir, { recursive: true });
138
+
}
139
+
140
+
// 1. sync
141
+
const syncRun = await runOrThrow(["./bin/cxs", "sync", "--db", args.db, "--root", args.root, "--json"]);
142
+
const syncMs = syncRun.ms;
143
+
let sessionCount = 0;
144
+
try {
145
+
const parsed = JSON.parse(syncRun.stdout) as { scanned?: number };
146
+
sessionCount = typeof parsed.scanned === "number" ? parsed.scanned : 0;
147
+
} catch {
148
+
// 解析失败保持 0
149
+
}
150
+
151
+
// 2. find x N runs per query
152
+
const perQuery: PerQueryRecord[] = [];
153
+
for (const q of BENCH_QUERIES) {
154
+
const samplesAll: number[] = [];
155
+
for (let i = 0; i < RUNS_PER_QUERY; i++) {
156
+
const r = await runOrThrow(["./bin/cxs", "find", q, "--db", args.db, "--limit", "10", "--json"]);
157
+
samplesAll.push(r.ms);
158
+
}
159
+
// 丢弃首次 warmup
160
+
const samples = samplesAll.slice(1);
161
+
const sorted = [...samples].sort((a, b) => a - b);
162
+
perQuery.push({
163
+
query: q,
164
+
runs: samples.length,
165
+
samplesMs: samplesAll.map((x) => Number(x.toFixed(2))),
166
+
p50Ms: Number(median(sorted).toFixed(2)),
167
+
p95Ms: Number(percentile(samples, 0.95).toFixed(2)),
168
+
});
169
+
}
170
+
171
+
// 3. stats -> dbSizeBytes
172
+
const statsRun = await runOrThrow(["./bin/cxs", "stats", "--db", args.db, "--json"]);
173
+
let dbSizeBytes = 0;
174
+
try {
175
+
const parsed = JSON.parse(statsRun.stdout) as { dbSizeBytes?: number; sessionCount?: number };
176
+
if (typeof parsed.dbSizeBytes === "number") dbSizeBytes = parsed.dbSizeBytes;
177
+
if (typeof parsed.sessionCount === "number" && parsed.sessionCount > 0) {
178
+
sessionCount = parsed.sessionCount;
179
+
}
180
+
} catch {
181
+
// 忽略
182
+
}
183
+
184
+
const report: Report = {
185
+
generatedAt: new Date().toISOString(),
186
+
dbPath: args.db,
187
+
rootDir: args.root,
188
+
sessionCount,
189
+
syncMs: Number(syncMs.toFixed(2)),
190
+
dbSizeBytes,
191
+
perQuery,
192
+
};
193
+
194
+
if (!args.jsonOnly) {
195
+
writeFileSync(join(outDir, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
196
+
writeFileSync(join(outDir, "report.md"), buildMarkdown(report));
197
+
}
198
+
199
+
const slowest = [...perQuery].sort((a, b) => b.p95Ms - a.p95Ms)[0];
200
+
console.log(JSON.stringify({
201
+
outDir: outDir || null,
202
+
sessionCount,
203
+
syncMs: Number(syncMs.toFixed(2)),
204
+
dbSizeBytes,
205
+
queryCount: perQuery.length,
206
+
slowestQuery: slowest ? { query: slowest.query, p95Ms: slowest.p95Ms } : null,
207
+
}, null, 2));
208
+
209
+
function buildMarkdown(r: Report): string {
210
+
const lines: string[] = [];
211
+
lines.push("# cxs 性能基准报告");
212
+
lines.push("");
213
+
lines.push(`- generated_at: ${r.generatedAt}`);
214
+
lines.push(`- root: \`${r.rootDir}\``);
215
+
lines.push(`- db: \`${r.dbPath}\``);
216
+
lines.push(`- session_count: ${r.sessionCount}`);
217
+
lines.push(`- sync_ms: ${r.syncMs.toFixed(1)}`);
218
+
lines.push(`- db_size: ${fmtBytes(r.dbSizeBytes)} (${r.dbSizeBytes} bytes)`);
219
+
lines.push("");
220
+
lines.push("## per-query find latency");
221
+
lines.push("");
222
+
lines.push(`运行配置: 每个 query ${RUNS_PER_QUERY} 次,丢弃首次 warmup,统计后 ${RUNS_PER_QUERY - 1} 次。`);
223
+
lines.push("");
224
+
lines.push("| query | runs | p50 ms | p95 ms | samples (incl. warmup) ms |");
225
+
lines.push("|-------|-----:|-------:|-------:|---------------------------|");
226
+
for (const row of r.perQuery) {
227
+
const samples = row.samplesMs.map((x) => x.toFixed(1)).join(", ");
228
+
lines.push(`| \`${row.query}\` | ${row.runs} |${fmtMs(row.p50Ms)} |${fmtMs(row.p95Ms)} | ${samples} |`);
229
+
}
230
+
lines.push("");
231
+
lines.push("> 注: 4 个有效样本下 p95 取最大值作为 worst-case 近似,统计意义有限。");
232
+
lines.push("");
233
+
return lines.join("\n");
234
+
}
+2
-1
package.json
+2
-1
package.json
···
44
44
"test": "bun test",
45
45
"check": "tsc --noEmit && bun test",
46
46
"cxs": "bun run ./cli.ts",
47
-
"eval:manual": "bun run ./eval/run-manual-eval.ts"
47
+
"eval:manual": "bun run ./eval/run-manual-eval.ts",
48
+
"eval:perf": "bun run ./eval/perf-bench.ts"
48
49
},
49
50
"dependencies": {
50
51
"chalk": "^5.6.2",