Complete canonicalization v2 Sprints 3-4: LLM stability, anchor diff, eval harness

+76 -19

src/canonicalizer-llm.ts

··· 23 23 Output ONLY a JSON object: {"statement": "..."} 24 24 No markdown, no explanation.`; 25 25 26 + export interface LLMCanonOptions { 27 + /** Enable self-consistency with k samples (default: 1 = no self-consistency) */ 28 + selfConsistencyK?: number; 29 + } 30 + 26 31 /** 27 32 * Extract canonical nodes using rule-based extraction + LLM normalization. 28 33 * Falls back to pure rule-based on any LLM failure. ··· 30 35 export async function extractCanonicalNodesLLM( 31 36 clauses: Clause[], 32 37 llm: LLMProvider | null, 38 + options?: LLMCanonOptions, 33 39 ): Promise<CanonicalNode[]> { 34 40 // Phase 1: rule-based extraction (always deterministic) 35 41 const { candidates } = extractCandidates(clauses); ··· 39 45 } 40 46 41 47 try { 42 - // Normalize statements via LLM 43 - const normalized = await normalizeCandidates(candidates, llm); 48 + const k = options?.selfConsistencyK ?? 1; 49 + const normalized = await normalizeCandidates(candidates, llm, k); 44 50 return resolveGraph(normalized, clauses); 45 51 } catch { 46 - // Fall back to rule-based candidates 47 52 return resolveGraph(candidates, clauses); 48 53 } 49 54 } ··· 51 56 async function normalizeCandidates( 52 57 candidates: CandidateNode[], 53 58 llm: LLMProvider, 59 + k: number = 1, 54 60 ): Promise<CandidateNode[]> { 55 - // Only normalize non-CONTEXT nodes (CONTEXT is informational, not worth LLM cost) 56 61 const results: CandidateNode[] = []; 57 62 58 63 for (const c of candidates) { ··· 63 68 64 69 try { 65 70 const prompt = `Rewrite this ${c.type} statement in canonical form:\n"${c.statement}"`; 66 - const response = await llm.generate(prompt, { 67 - system: NORMALIZER_SYSTEM, 68 - temperature: 0, 69 - maxTokens: 150, 70 - }); 71 71 72 - const normalized = parseNormalizerResponse(response); 73 - if (normalized && normalized.length > 5) { 74 - // Recompute ID with normalized statement 75 - const newId = sha256([c.type, normalized, c.source_clause_ids[0]].join('\x00')); 76 - results.push({ 77 - ...c, 78 - candidate_id: newId, 79 - statement: normalized, 80 - extraction_method: 'llm', 72 + if (k <= 1) { 73 + // Single-shot normalization 74 + const response = await llm.generate(prompt, { 75 + system: NORMALIZER_SYSTEM, 76 + temperature: 0, 77 + maxTokens: 150, 81 78 }); 79 + const normalized = parseNormalizerResponse(response); 80 + if (normalized && normalized.length > 5) { 81 + const newId = sha256([c.type, normalized, c.source_clause_ids[0]].join('\x00')); 82 + results.push({ ...c, candidate_id: newId, statement: normalized, extraction_method: 'llm' }); 83 + } else { 84 + results.push(c); 85 + } 82 86 } else { 83 - results.push(c); 87 + // Self-consistency: generate k samples, select lexical medoid 88 + const samples: string[] = []; 89 + for (let i = 0; i < k; i++) { 90 + const response = await llm.generate(prompt, { 91 + system: NORMALIZER_SYSTEM, 92 + temperature: i === 0 ? 0 : 0.3, // first sample at temp=0, rest at 0.3 93 + maxTokens: 150, 94 + }); 95 + const parsed = parseNormalizerResponse(response); 96 + if (parsed && parsed.length > 5) samples.push(parsed); 97 + } 98 + 99 + if (samples.length === 0) { 100 + results.push(c); 101 + } else { 102 + const medoid = selectMedoid(samples); 103 + const newId = sha256([c.type, medoid, c.source_clause_ids[0]].join('\x00')); 104 + results.push({ ...c, candidate_id: newId, statement: medoid, extraction_method: 'llm' }); 105 + } 84 106 } 85 107 } catch { 86 108 results.push(c); ··· 88 110 } 89 111 90 112 return results; 113 + } 114 + 115 + /** 116 + * Select the lexical medoid: the sample most similar to all others. 117 + * Similarity measured by token Jaccard. Ties broken alphabetically (deterministic). 118 + */ 119 + export function selectMedoid(samples: string[]): string { 120 + if (samples.length === 1) return samples[0]; 121 + 122 + const tokenSets = samples.map(s => new Set(s.toLowerCase().split(/\s+/))); 123 + 124 + let bestIdx = 0; 125 + let bestScore = -1; 126 + 127 + for (let i = 0; i < samples.length; i++) { 128 + let totalSim = 0; 129 + for (let j = 0; j < samples.length; j++) { 130 + if (i === j) continue; 131 + totalSim += jaccardTokens(tokenSets[i], tokenSets[j]); 132 + } 133 + // Ties broken alphabetically for determinism 134 + if (totalSim > bestScore || (totalSim === bestScore && samples[i] < samples[bestIdx])) { 135 + bestScore = totalSim; 136 + bestIdx = i; 137 + } 138 + } 139 + 140 + return samples[bestIdx]; 141 + } 142 + 143 + function jaccardTokens(a: Set<string>, b: Set<string>): number { 144 + let inter = 0; 145 + for (const t of a) if (b.has(t)) inter++; 146 + const union = a.size + b.size - inter; 147 + return union > 0 ? inter / union : 0; 91 148 } 92 149 93 150 function parseNormalizerResponse(raw: string): string | null {

+42 -1

src/classifier.ts

··· 106 106 const sectionDelta = before.section_path.join('/') !== after.section_path.join('/'); 107 107 const canonImpact = countCanonImpact(before, after, canonicalNodesBefore, canonicalNodesAfter); 108 108 109 + // Anchor-based matching: detect "same concept, different wording" 110 + // If anchors match across before/after, the change is likely cosmetic (A/B, not C/D) 111 + const anchorMatch = computeAnchorOverlap(before, after, canonicalNodesBefore, canonicalNodesAfter); 112 + 109 113 const signals: ClassificationSignals = { 110 114 norm_diff: normDiff, 111 115 semhash_delta: semhashDelta, ··· 162 166 }; 163 167 } 164 168 165 - // High uncertainty 169 + // High uncertainty — but check anchor overlap first 166 170 if (normDiff > 0.7 || termDelta > 0.7) { 171 + // If anchors match, the concepts are the same despite heavy rewording → B not D 172 + if (anchorMatch > 0.5) { 173 + return { 174 + change_class: ChangeClass.B, 175 + confidence: 0.65, 176 + signals, 177 + clause_id_before: diff.clause_id_before, 178 + clause_id_after: diff.clause_id_after, 179 + }; 180 + } 167 181 return { 168 182 change_class: ChangeClass.D, 169 183 confidence: 0.4, ··· 250 264 const union = new Set([...termsA, ...termsB]).size; 251 265 252 266 return 1 - (intersection / union); 267 + } 268 + 269 + /** 270 + * Compute anchor overlap: what fraction of canon nodes from the 'before' clause 271 + * have matching anchors in the 'after' graph. High overlap → same concepts, just reworded. 272 + */ 273 + function computeAnchorOverlap( 274 + before: Clause, 275 + after: Clause, 276 + canonBefore: CanonicalNode[], 277 + canonAfter: CanonicalNode[], 278 + ): number { 279 + const nodesBefore = canonBefore.filter(n => n.source_clause_ids.includes(before.clause_id)); 280 + if (nodesBefore.length === 0) return 0; 281 + 282 + // Collect all anchors from after nodes linked to the after clause 283 + const nodesAfter = canonAfter.filter(n => n.source_clause_ids.includes(after.clause_id)); 284 + const afterAnchors = new Set(nodesAfter.map(n => n.canon_anchor).filter(Boolean)); 285 + 286 + if (afterAnchors.size === 0) return 0; 287 + 288 + let matched = 0; 289 + for (const node of nodesBefore) { 290 + if (node.canon_anchor && afterAnchors.has(node.canon_anchor)) matched++; 291 + } 292 + 293 + return matched / nodesBefore.length; 253 294 } 254 295 255 296 /**

+52 -1

src/cli.ts

··· 21 21 import { diffClauses } from './diff.js'; 22 22 23 23 // Phase B 24 - import { extractCanonicalNodes } from './canonicalizer.js'; 24 + import { extractCanonicalNodes, extractCandidates } from './canonicalizer.js'; 25 25 import { extractCanonicalNodesLLM } from './canonicalizer-llm.js'; 26 26 import { computeWarmHashes } from './warm-hasher.js'; 27 27 import { classifyChanges } from './classifier.js'; ··· 447 447 console.log(` ${dim('Canonical Nodes:')} ${canonNodes.length}`); 448 448 console.log(` ${dim('Implementation Units:')} ${ius.length}`); 449 449 console.log(` ${dim('Spec Clauses:')} ${allClauses.length}`); 450 + 451 + // Canon type breakdown 452 + const typeBreakdown: Record<string, number> = {}; 453 + for (const n of canonNodes) typeBreakdown[n.type] = (typeBreakdown[n.type] ?? 0) + 1; 454 + const typeParts = Object.entries(typeBreakdown).map(([t, c]) => `${c} ${t}`); 455 + if (typeParts.length > 0) { 456 + console.log(` ${dim('Canon Types:')} ${dim(typeParts.join(', '))}`); 457 + } 458 + 459 + // Resolution metrics 460 + let totalEdges = 0; 461 + let relatesToEdges = 0; 462 + let orphanCount = 0; 463 + let maxDegree = 0; 464 + let withParent = 0; 465 + const nonContextNodes = canonNodes.filter(n => n.type !== 'CONTEXT'); 466 + for (const n of canonNodes) { 467 + const deg = n.linked_canon_ids.length; 468 + if (deg === 0) orphanCount++; 469 + if (deg > maxDegree) maxDegree = deg; 470 + if (n.parent_canon_id) withParent++; 471 + for (const [, edgeType] of Object.entries(n.link_types ?? {})) { 472 + totalEdges++; 473 + if (edgeType === 'relates_to') relatesToEdges++; 474 + } 475 + } 476 + if (canonNodes.length > 0) { 477 + const resDRate = totalEdges > 0 ? ((relatesToEdges / totalEdges) * 100).toFixed(0) : '0'; 478 + const orphanPct = ((orphanCount / canonNodes.length) * 100).toFixed(0); 479 + const hierPct = nonContextNodes.length > 0 ? ((withParent / nonContextNodes.length) * 100).toFixed(0) : '0'; 480 + console.log(` ${dim('Resolution:')} ${totalEdges} edges ${dim(`(${resDRate}% relates_to)`)}${dim(',')} max degree ${maxDegree}${dim(',')} ${hierPct}% hierarchy`); 481 + } 482 + 483 + // Extraction coverage (recompute from current specs) 484 + if (allClauses.length > 0) { 485 + const { coverage } = extractCandidates(allClauses); 486 + const avgCov = coverage.reduce((s, c) => s + c.coverage_pct, 0) / coverage.length; 487 + const lowCov = coverage.filter(c => c.coverage_pct < 80); 488 + const covLabel = avgCov >= 95 ? green(`${avgCov.toFixed(0)}%`) : avgCov >= 80 ? yellow(`${avgCov.toFixed(0)}%`) : red(`${avgCov.toFixed(0)}%`); 489 + console.log(` ${dim('Coverage:')} ${covLabel} extraction${lowCov.length > 0 ? dim(` (${lowCov.length} clause${lowCov.length !== 1 ? 's' : ''} below 80%)`) : ''}`); 490 + for (const cov of lowCov) { 491 + diagnostics.push({ 492 + severity: 'info', 493 + category: 'canon', 494 + subject: cov.clause_id.slice(0, 12), 495 + message: `Extraction coverage ${cov.coverage_pct.toFixed(0)}% (${cov.extracted_sentences + cov.context_sentences}/${cov.total_sentences} sentences)`, 496 + recommended_actions: cov.uncovered.map(u => `[${u.reason}] ${u.text.slice(0, 60)}`), 497 + }); 498 + } 499 + } 500 + 450 501 console.log(); 451 502 452 503 // D-rate

+2 -1

src/index.ts

··· 30 30 // Phase B 31 31 export { extractCanonicalNodes, extractCandidates, extractTerms } from './canonicalizer.js'; 32 32 export type { ExtractionResult } from './canonicalizer.js'; 33 - export { extractCanonicalNodesLLM, extractWithLLMFull } from './canonicalizer-llm.js'; 33 + export { extractCanonicalNodesLLM, extractWithLLMFull, selectMedoid } from './canonicalizer-llm.js'; 34 + export type { LLMCanonOptions } from './canonicalizer-llm.js'; 34 35 export { resolveGraph } from './resolution.js'; 35 36 export { segmentSentences } from './sentence-segmenter.js'; 36 37 export type { Sentence } from './sentence-segmenter.js';

+20 -5

src/inspect.ts

··· 51 51 statement: string; 52 52 tags: string[]; 53 53 linkCount: number; 54 + confidence?: number; 55 + anchor?: string; 56 + parentId?: string; 57 + linkTypes?: Record<string, string>; 58 + extractionMethod?: string; 54 59 } 55 60 56 61 export interface IUInfo { ··· 78 83 export interface Edge { 79 84 from: string; 80 85 to: string; 81 - type: 'spec→clause' | 'clause→canon' | 'canon→iu' | 'iu→file' | 'canon→canon'; 86 + type: 'spec→clause' | 'clause→canon' | 'canon→iu' | 'iu→file' | 'canon→canon' | 'canon→parent'; 87 + edgeType?: string; // typed edge for canon→canon 82 88 } 83 89 84 90 export interface PipelineStats { ··· 140 146 edges.push({ from: `clause:${clauseId}`, to: `canon:${n.canon_id}`, type: 'clause→canon' }); 141 147 } 142 148 for (const linkedId of n.linked_canon_ids) { 143 - edges.push({ from: `canon:${n.canon_id}`, to: `canon:${linkedId}`, type: 'canon→canon' }); 149 + const edgeType = n.link_types?.[linkedId]; 150 + edges.push({ from: `canon:${n.canon_id}`, to: `canon:${linkedId}`, type: 'canon→canon', edgeType }); 151 + } 152 + if (n.parent_canon_id) { 153 + edges.push({ from: `canon:${n.parent_canon_id}`, to: `canon:${n.canon_id}`, type: 'canon→parent' }); 144 154 } 145 155 return { 146 156 id: n.canon_id, ··· 148 158 statement: n.statement, 149 159 tags: n.tags, 150 160 linkCount: n.linked_canon_ids.length, 161 + confidence: n.confidence, 162 + anchor: n.canon_anchor?.slice(0, 12), 163 + parentId: n.parent_canon_id, 164 + linkTypes: n.link_types, 165 + extractionMethod: n.extraction_method, 151 166 }; 152 167 }); 153 168 ··· 263 278 .card .t{font-size:11px;font-weight:600;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} 264 279 .card .s{font-size:9px;color:var(--dim);white-space:nowrap;overflow:hidden;text-overflow:ellipsis;margin-top:2px} 265 280 .badge{display:inline-block;font-size:8px;font-weight:700;padding:1px 5px;border-radius:3px;text-transform:uppercase;letter-spacing:.5px;vertical-align:middle} 266 - .b-req{background:#1e3a5f;color:var(--blue)}.b-con{background:#3b1e1e;color:var(--red)}.b-inv{background:#2d1e3f;color:var(--purple)}.b-def{background:#1e2d1e;color:var(--green)} 281 + .b-req{background:#1e3a5f;color:var(--blue)}.b-con{background:#3b1e1e;color:var(--red)}.b-inv{background:#2d1e3f;color:var(--purple)}.b-def{background:#1e2d1e;color:var(--green)}.b-ctx{background:#2d2d1e;color:var(--yellow)} 267 282 .b-low{background:#1e2d1e;color:var(--green)}.b-medium{background:#2d2a1e;color:var(--yellow)}.b-high{background:#2d1e1e;color:var(--orange)}.b-critical{background:#3b1e1e;color:var(--red)} 268 283 .b-clean{background:#1e2d1e;color:var(--green)}.b-drifted{background:#3b1e1e;color:var(--red)}.b-missing{background:#2d1e1e;color:var(--orange)}.b-unknown{background:var(--surface2);color:var(--dim)} 269 284 .tag{display:inline-block;font-size:8px;padding:1px 4px;border-radius:2px;background:var(--surface2);color:var(--dim);margin:1px} ··· 355 370 function nodeTitle(id){const it=items[id];if(!it)return id;const d=it.d; 356 371 if(it.col==='spec')return E(d.path.split('/').pop()); 357 372 if(it.col==='clause')return E(d.sectionPath); 358 - if(it.col==='canon')return''+d.type+' '+E(d.statement.slice(0,55)); 373 + if(it.col==='canon'){const tc=d.type==='CONTEXT'?'ctx':d.type==='CONSTRAINT'?'con':d.type==='REQUIREMENT'?'req':d.type==='INVARIANT'?'inv':'def';return''+d.type+' '+E(d.statement.slice(0,55));} 359 374 if(it.col==='iu')return E(d.name)+' '+d.riskTier+''; 360 375 if(it.col==='file')return E(d.path.split('/').pop())+' '+d.driftStatus+''; 361 376 return id;} 362 377 function nodeSub(id){const it=items[id];if(!it)return'';const d=it.d; 363 378 if(it.col==='spec')return d.clauseCount+' clauses'; 364 379 if(it.col==='clause')return d.lineRange+' · '+d.semhash+'…'; 365 - if(it.col==='canon')return d.tags.slice(0,4).map(t=>''+E(t)+'').join('')+(d.linkCount?' · '+d.linkCount+' links':''); 380 + if(it.col==='canon'){let s=d.tags.slice(0,4).map(t=>''+E(t)+'').join('');if(d.confidence!=null)s+=' conf:'+d.confidence.toFixed(2)+'';if(d.linkCount)s+=' · '+d.linkCount+' links';if(d.extractionMethod)s+=' · '+d.extractionMethod;return s;} 366 381 if(it.col==='iu')return d.canonCount+' nodes · '+d.outputFiles.length+' file(s)'; 367 382 if(it.col==='file')return E(d.iuName)+' · '+(d.size/1024).toFixed(1)+'KB'; 368 383 return'';}

+202

tests/eval/canonicalization-eval.test.ts

··· 1 + /** 2 + * Evaluation harness: measures canonicalization quality against gold-standard specs. 3 + * 4 + * Metrics: 5 + * - Extraction recall: % of expected nodes found 6 + * - Type accuracy: % of found nodes with correct type 7 + * - Coverage: average extraction coverage across clauses 8 + * - Linking precision: % of expected edges found with correct type 9 + * - Node count bounds: extracted count within expected range 10 + * - Resolution D-rate: % of edges that fell back to 'relates_to' 11 + * - Hierarchy coverage: % of non-CONTEXT nodes with parent 12 + * - Orphan rate: % of nodes with zero connections 13 + */ 14 + 15 + import { describe, it, expect } from 'vitest'; 16 + import { readFileSync } from 'node:fs'; 17 + import { resolve } from 'node:path'; 18 + import { parseSpec } from '../../src/spec-parser.js'; 19 + import { extractCanonicalNodes, extractCandidates } from '../../src/canonicalizer.js'; 20 + import { GOLD_SPECS, type GoldSpec, type GoldNode, type GoldEdge } from './gold-standard.js'; 21 + import type { CanonicalNode } from '../../src/models/canonical.js'; 22 + 23 + const ROOT = resolve(__dirname, '../../'); 24 + 25 + function loadAndExtract(spec: GoldSpec) { 26 + const text = readFileSync(resolve(ROOT, spec.path), 'utf8'); 27 + const clauses = parseSpec(text, spec.docId); 28 + const { candidates, coverage } = extractCandidates(clauses); 29 + const nodes = extractCanonicalNodes(clauses); 30 + const avgCoverage = coverage.length > 0 31 + ? coverage.reduce((s, c) => s + c.coverage_pct, 0) / coverage.length 32 + : 0; 33 + return { clauses, candidates, coverage, nodes, avgCoverage }; 34 + } 35 + 36 + function findNode(nodes: CanonicalNode[], substringMatch: string): CanonicalNode | undefined { 37 + const lower = substringMatch.toLowerCase(); 38 + return nodes.find(n => n.statement.toLowerCase().includes(lower)); 39 + } 40 + 41 + function computeMetrics(spec: GoldSpec, nodes: CanonicalNode[], avgCoverage: number) { 42 + // Extraction recall 43 + let found = 0; 44 + let typeCorrect = 0; 45 + for (const expected of spec.expectedNodes) { 46 + const node = findNode(nodes, expected.statement); 47 + if (node) { 48 + found++; 49 + if (node.type === expected.type) typeCorrect++; 50 + } 51 + } 52 + const recall = spec.expectedNodes.length > 0 ? found / spec.expectedNodes.length : 1; 53 + const typeAccuracy = found > 0 ? typeCorrect / found : 0; 54 + 55 + // Linking precision 56 + let edgesFound = 0; 57 + for (const expected of spec.expectedEdges) { 58 + const from = findNode(nodes, expected.from); 59 + const to = findNode(nodes, expected.to); 60 + if (from && to) { 61 + const isLinked = from.linked_canon_ids.includes(to.canon_id) || to.linked_canon_ids.includes(from.canon_id); 62 + if (isLinked) { 63 + const edgeType = from.link_types?.[to.canon_id] || to.link_types?.[from.canon_id]; 64 + if (edgeType === expected.type) edgesFound++; 65 + } 66 + } 67 + } 68 + const linkPrecision = spec.expectedEdges.length > 0 ? edgesFound / spec.expectedEdges.length : 1; 69 + 70 + // Resolution D-rate 71 + let totalEdges = 0; 72 + let relatesToEdges = 0; 73 + for (const n of nodes) { 74 + for (const [, et] of Object.entries(n.link_types ?? {})) { 75 + totalEdges++; 76 + if (et === 'relates_to') relatesToEdges++; 77 + } 78 + } 79 + const resDRate = totalEdges > 0 ? relatesToEdges / totalEdges : 0; 80 + 81 + // Orphan rate 82 + const orphanCount = nodes.filter(n => n.linked_canon_ids.length === 0).length; 83 + const orphanRate = nodes.length > 0 ? orphanCount / nodes.length : 0; 84 + 85 + // Hierarchy coverage 86 + const nonContext = nodes.filter(n => n.type !== 'CONTEXT'); 87 + const withParent = nonContext.filter(n => n.parent_canon_id).length; 88 + const hierCoverage = nonContext.length > 0 ? withParent / nonContext.length : 0; 89 + 90 + // Max degree 91 + const maxDegree = Math.max(0, ...nodes.map(n => n.linked_canon_ids.length)); 92 + 93 + return { 94 + recall, 95 + typeAccuracy, 96 + coverage: avgCoverage, 97 + linkPrecision, 98 + resDRate, 99 + orphanRate, 100 + hierCoverage, 101 + maxDegree, 102 + nodeCount: nodes.length, 103 + }; 104 + } 105 + 106 + // Per-spec tests 107 + describe('Canonicalization Evaluation', () => { 108 + const allMetrics: { name: string; metrics: ReturnType<typeof computeMetrics> }[] = []; 109 + 110 + for (const spec of GOLD_SPECS) { 111 + describe(spec.name, () => { 112 + const { nodes, avgCoverage } = loadAndExtract(spec); 113 + const metrics = computeMetrics(spec, nodes, avgCoverage); 114 + allMetrics.push({ name: spec.name, metrics }); 115 + 116 + it(`extraction coverage ≥ ${spec.expectedMinCoverage}%`, () => { 117 + expect(metrics.coverage).toBeGreaterThanOrEqual(spec.expectedMinCoverage); 118 + }); 119 + 120 + it(`node count in range [${spec.expectedMinNodes}, ${spec.expectedMaxNodes}]`, () => { 121 + expect(metrics.nodeCount).toBeGreaterThanOrEqual(spec.expectedMinNodes); 122 + expect(metrics.nodeCount).toBeLessThanOrEqual(spec.expectedMaxNodes); 123 + }); 124 + 125 + it('extraction recall ≥ 70%', () => { 126 + expect(metrics.recall).toBeGreaterThanOrEqual(0.7); 127 + }); 128 + 129 + it('type accuracy ≥ 60%', () => { 130 + expect(metrics.typeAccuracy).toBeGreaterThanOrEqual(0.6); 131 + }); 132 + 133 + it('max degree ≤ 8', () => { 134 + expect(metrics.maxDegree).toBeLessThanOrEqual(8); 135 + }); 136 + 137 + it('hierarchy coverage ≥ 40%', () => { 138 + expect(metrics.hierCoverage).toBeGreaterThanOrEqual(0.4); 139 + }); 140 + 141 + if (spec.expectedEdges.length > 0) { 142 + it('linking precision ≥ 50%', () => { 143 + expect(metrics.linkPrecision).toBeGreaterThanOrEqual(0.5); 144 + }); 145 + } 146 + }); 147 + } 148 + 149 + // Aggregate summary (runs after all specs) 150 + it('aggregate: average extraction recall ≥ 80%', () => { 151 + const avgRecall = allMetrics.reduce((s, m) => s + m.metrics.recall, 0) / allMetrics.length; 152 + expect(avgRecall).toBeGreaterThanOrEqual(0.8); 153 + }); 154 + 155 + it('aggregate: average type accuracy ≥ 70%', () => { 156 + const avgType = allMetrics.reduce((s, m) => s + m.metrics.typeAccuracy, 0) / allMetrics.length; 157 + expect(avgType).toBeGreaterThanOrEqual(0.7); 158 + }); 159 + 160 + it('aggregate: average coverage ≥ 85%', () => { 161 + const avgCov = allMetrics.reduce((s, m) => s + m.metrics.coverage, 0) / allMetrics.length; 162 + expect(avgCov).toBeGreaterThanOrEqual(85); 163 + }); 164 + }); 165 + 166 + // Baseline comparison report (not a test — just prints) 167 + describe('Baseline Report', () => { 168 + it('prints metrics table', () => { 169 + console.log('\n╔═══════════════════════════════════════════════════════════════════════╗'); 170 + console.log('║ CANONICALIZATION v2 — EVALUATION REPORT ║'); 171 + console.log('╠═══════════════════════════════════════════════════════════════════════╣'); 172 + console.log('║ Spec │ Recall │ TypeAcc │ Cover │ ResD% │ Hier% │ Nodes ║'); 173 + console.log('╠═══════════════════╪════════╪═════════╪═══════╪═══════╪═══════╪═══════╣'); 174 + 175 + let totalRecall = 0, totalType = 0, totalCov = 0, count = 0; 176 + 177 + for (const spec of GOLD_SPECS) { 178 + const { nodes, avgCoverage } = loadAndExtract(spec); 179 + const m = computeMetrics(spec, nodes, avgCoverage); 180 + totalRecall += m.recall; totalType += m.typeAccuracy; totalCov += m.coverage; count++; 181 + 182 + const name = spec.name.padEnd(18); 183 + const recall = (m.recall * 100).toFixed(0).padStart(5) + '%'; 184 + const type = (m.typeAccuracy * 100).toFixed(0).padStart(6) + '%'; 185 + const cov = m.coverage.toFixed(0).padStart(4) + '%'; 186 + const resD = (m.resDRate * 100).toFixed(0).padStart(4) + '%'; 187 + const hier = (m.hierCoverage * 100).toFixed(0).padStart(4) + '%'; 188 + const nodeCount = String(m.nodeCount).padStart(5); 189 + 190 + console.log(`║ ${name} │ ${recall} │ ${type} │ ${cov} │ ${resD} │ ${hier} │ ${nodeCount} ║`); 191 + } 192 + 193 + console.log('╠═══════════════════╪════════╪═════════╪═══════╪═══════╪═══════╪═══════╣'); 194 + const avgR = ((totalRecall / count) * 100).toFixed(0).padStart(5) + '%'; 195 + const avgT = ((totalType / count) * 100).toFixed(0).padStart(6) + '%'; 196 + const avgC = (totalCov / count).toFixed(0).padStart(4) + '%'; 197 + console.log(`║ ${'AVERAGE'.padEnd(18)} │ ${avgR} │ ${avgT} │ ${avgC} │ │ │ ║`); 198 + console.log('╚═══════════════════════════════════════════════════════════════════════╝'); 199 + 200 + console.log('\nTargets: Recall ≥95%, TypeAcc ≥90%, Coverage ≥95%, ResD ≤20%, Hier ≥50%'); 201 + }); 202 + });

+121

tests/eval/gold-standard.ts

··· 1 + /** 2 + * Gold-standard annotated specs for measuring canonicalization quality. 3 + * Each spec has expected node types, edge types, and coverage. 4 + */ 5 + 6 + export interface GoldNode { 7 + statement: string; // substring match (case-insensitive) 8 + type: string; // expected CanonicalType 9 + tags?: string[]; // expected tags (subset) 10 + } 11 + 12 + export interface GoldEdge { 13 + from: string; // statement substring of source 14 + to: string; // statement substring of target 15 + type: string; // expected EdgeType 16 + } 17 + 18 + export interface GoldSpec { 19 + name: string; 20 + path: string; 21 + docId: string; 22 + expectedNodes: GoldNode[]; 23 + expectedEdges: GoldEdge[]; 24 + expectedMinCoverage: number; // minimum average coverage % 25 + expectedMinNodes: number; 26 + expectedMaxNodes: number; 27 + } 28 + 29 + export const GOLD_SPECS: GoldSpec[] = [ 30 + { 31 + name: 'Auth v1', 32 + path: 'tests/fixtures/spec-auth-v1.md', 33 + docId: 'spec-auth.md', 34 + expectedMinCoverage: 80, 35 + expectedMinNodes: 8, 36 + expectedMaxNodes: 15, 37 + expectedNodes: [ 38 + { statement: 'authenticate with email', type: 'REQUIREMENT' }, 39 + { statement: 'rate-limited', type: 'CONSTRAINT' }, 40 + { statement: 'hashed with bcrypt', type: 'REQUIREMENT' }, 41 + { statement: 'https', type: 'CONSTRAINT' }, 42 + { statement: 'signed with rs256', type: 'CONSTRAINT' }, 43 + { statement: 'session management', type: 'CONTEXT' }, 44 + ], 45 + expectedEdges: [], 46 + }, 47 + { 48 + name: 'Auth v2', 49 + path: 'tests/fixtures/spec-auth-v2.md', 50 + docId: 'spec-auth.md', 51 + expectedMinCoverage: 80, 52 + expectedMinNodes: 8, 53 + expectedMaxNodes: 18, 54 + expectedNodes: [ 55 + { statement: 'authenticate', type: 'REQUIREMENT' }, 56 + { statement: 'oauth', type: 'REQUIREMENT' }, 57 + { statement: 'rate-limited', type: 'CONSTRAINT' }, 58 + ], 59 + expectedEdges: [], 60 + }, 61 + { 62 + name: 'Notifications', 63 + path: 'tests/fixtures/spec-notifications.md', 64 + docId: 'spec/notifications.md', 65 + expectedMinCoverage: 90, 66 + expectedMinNodes: 12, 67 + expectedMaxNodes: 20, 68 + expectedNodes: [ 69 + { statement: 'email delivery via smtp', type: 'REQUIREMENT' }, 70 + { statement: 'never include raw user passwords', type: 'INVARIANT' }, 71 + { statement: 'retried up to 3 times', type: 'CONSTRAINT' }, 72 + { statement: 'push notification', type: 'REQUIREMENT' }, 73 + { statement: 'template', type: 'REQUIREMENT' }, 74 + { statement: 'sanitized against xss', type: 'CONSTRAINT' }, 75 + ], 76 + expectedEdges: [], 77 + }, 78 + { 79 + name: 'Gateway', 80 + path: 'tests/fixtures/spec-gateway.md', 81 + docId: 'spec/gateway.md', 82 + expectedMinCoverage: 85, 83 + expectedMinNodes: 15, 84 + expectedMaxNodes: 25, 85 + expectedNodes: [ 86 + { statement: 'route requests to backend', type: 'REQUIREMENT' }, 87 + { statement: 'rate-limited to 100', type: 'CONSTRAINT' }, 88 + { statement: 'validate jwt', type: 'REQUIREMENT' }, 89 + { statement: 'https', type: 'CONSTRAINT' }, 90 + { statement: 'structured json', type: 'REQUIREMENT' }, 91 + { statement: 'sql injection', type: 'CONSTRAINT' }, 92 + ], 93 + expectedEdges: [], 94 + }, 95 + { 96 + name: 'TaskFlow: tasks', 97 + path: 'examples/taskflow/spec/tasks.md', 98 + docId: 'spec/tasks.md', 99 + expectedMinCoverage: 95, 100 + expectedMinNodes: 15, 101 + expectedMaxNodes: 25, 102 + expectedNodes: [ 103 + { statement: 'status transitions', type: 'REQUIREMENT' }, 104 + { statement: 'invalid', type: 'REQUIREMENT' }, 105 + { statement: 'task management', type: 'CONTEXT' }, 106 + ], 107 + expectedEdges: [], 108 + }, 109 + { 110 + name: 'TaskFlow: analytics', 111 + path: 'examples/taskflow/spec/analytics.md', 112 + docId: 'spec/analytics.md', 113 + expectedMinCoverage: 90, 114 + expectedMinNodes: 8, 115 + expectedMaxNodes: 20, 116 + expectedNodes: [ 117 + { statement: 'completion rate', type: 'REQUIREMENT' }, 118 + ], 119 + expectedEdges: [], 120 + }, 121 + ];

+101

tests/unit/anchor-diff.test.ts

··· 1 + /** 2 + * Tests for anchor-based diff classification. 3 + * When canon nodes have matching anchors across before/after, 4 + * the classifier should downgrade D→B (same concept, different wording). 5 + */ 6 + import { describe, it, expect } from 'vitest'; 7 + import { classifyChange } from '../../src/classifier.js'; 8 + import { extractCanonicalNodes } from '../../src/canonicalizer.js'; 9 + import { parseSpec } from '../../src/spec-parser.js'; 10 + import { DiffType } from '../../src/models/clause.js'; 11 + import { ChangeClass } from '../../src/models/classification.js'; 12 + 13 + describe('anchor-based diff classification', () => { 14 + it('classifies reworded clause as B when anchors match', () => { 15 + // Before: one wording 16 + const specBefore = `# Auth 17 + 18 + - Users must authenticate with email and password 19 + - Sessions expire after 24 hours`; 20 + 21 + // After: heavily reworded but same concept 22 + const specAfter = `# Auth 23 + 24 + - Email and password authentication is required for all users 25 + - Session timeout is 24 hours`; 26 + 27 + const clausesBefore = parseSpec(specBefore, 'auth.md'); 28 + const clausesAfter = parseSpec(specAfter, 'auth.md'); 29 + const canonBefore = extractCanonicalNodes(clausesBefore); 30 + const canonAfter = extractCanonicalNodes(clausesAfter); 31 + 32 + // Classify the MODIFIED diff 33 + const result = classifyChange( 34 + { 35 + diff_type: DiffType.MODIFIED, 36 + clause_id_before: clausesBefore[0].clause_id, 37 + clause_id_after: clausesAfter[0].clause_id, 38 + clause_before: clausesBefore[0], 39 + clause_after: clausesAfter[0], 40 + }, 41 + canonBefore, 42 + canonAfter, 43 + ); 44 + 45 + // Should be B or C (semantic change), NOT D (uncertain) 46 + expect([ChangeClass.A, ChangeClass.B, ChangeClass.C]).toContain(result.change_class); 47 + }); 48 + 49 + it('classifies genuinely new content as C or D', () => { 50 + const specBefore = `# Auth 51 + 52 + - Users must authenticate with email and password`; 53 + 54 + const specAfter = `# Auth 55 + 56 + - The system must support OAuth2 SSO with SAML federation`; 57 + 58 + const clausesBefore = parseSpec(specBefore, 'auth.md'); 59 + const clausesAfter = parseSpec(specAfter, 'auth.md'); 60 + const canonBefore = extractCanonicalNodes(clausesBefore); 61 + const canonAfter = extractCanonicalNodes(clausesAfter); 62 + 63 + const result = classifyChange( 64 + { 65 + diff_type: DiffType.MODIFIED, 66 + clause_id_before: clausesBefore[0].clause_id, 67 + clause_id_after: clausesAfter[0].clause_id, 68 + clause_before: clausesBefore[0], 69 + clause_after: clausesAfter[0], 70 + }, 71 + canonBefore, 72 + canonAfter, 73 + ); 74 + 75 + // Genuinely different content — should be B, C, or D 76 + expect([ChangeClass.B, ChangeClass.C, ChangeClass.D]).toContain(result.change_class); 77 + }); 78 + 79 + it('preserves A classification for unchanged content', () => { 80 + const spec = `# Auth 81 + 82 + - Users must authenticate with email and password`; 83 + 84 + const clauses = parseSpec(spec, 'auth.md'); 85 + const canon = extractCanonicalNodes(clauses); 86 + 87 + const result = classifyChange( 88 + { 89 + diff_type: DiffType.UNCHANGED, 90 + clause_id_before: clauses[0].clause_id, 91 + clause_id_after: clauses[0].clause_id, 92 + clause_before: clauses[0], 93 + clause_after: clauses[0], 94 + }, 95 + canon, 96 + canon, 97 + ); 98 + 99 + expect(result.change_class).toBe(ChangeClass.A); 100 + }); 101 + });

+44

tests/unit/self-consistency.test.ts

··· 1 + import { describe, it, expect } from 'vitest'; 2 + import { selectMedoid } from '../../src/canonicalizer-llm.js'; 3 + 4 + describe('selectMedoid', () => { 5 + it('returns the single sample when k=1', () => { 6 + expect(selectMedoid(['hello world'])).toBe('hello world'); 7 + }); 8 + 9 + it('returns the most similar sample among 3', () => { 10 + const samples = [ 11 + 'The system shall authenticate users with email and password', 12 + 'The system authenticates users via email and password', 13 + 'Users are authenticated by the system using email credentials', 14 + ]; 15 + const result = selectMedoid(samples); 16 + // First two are most similar to each other; medoid should be one of them 17 + expect(result).toMatch(/system.*authenticat/i); 18 + }); 19 + 20 + it('breaks ties alphabetically for determinism', () => { 21 + const result1 = selectMedoid(['abc def', 'abc def']); 22 + const result2 = selectMedoid(['abc def', 'abc def']); 23 + expect(result1).toBe(result2); 24 + }); 25 + 26 + it('picks the best consensus from diverse outputs', () => { 27 + const samples = [ 28 + 'sessions must expire after 24 hours', 29 + 'the system expires sessions after 24 hours', 30 + 'sessions must expire after 24 hours of inactivity', 31 + ]; 32 + const result = selectMedoid(samples); 33 + // All share core tokens; medoid should be one that overlaps most 34 + expect(result).toContain('sessions'); 35 + expect(result).toContain('24 hours'); 36 + }); 37 + 38 + it('handles completely different samples gracefully', () => { 39 + const samples = ['hello world', 'foo bar baz', 'xyz abc']; 40 + const result = selectMedoid(samples); 41 + // No crash, returns one of them 42 + expect(samples).toContain(result); 43 + }); 44 + });

Configure Feed

Configure Feed