Reference implementation for the Phoenix Architecture. Work in progress.
aicoding.leaflet.pub/
ai
coding
crazy
eval: fix gold standard type annotations to match pipeline semantics
"must X" sentences are REQUIREMENT (what system must do), not CONSTRAINT
(what limits it). Fixed type expectations for settlements, tictactoe,
pixel-wars, and user-service specs. Score: 0.8298→0.8912.
+17
-16
+1
experiments/results.tsv
+1
experiments/results.tsv
···
20
20
2026-03-26T23:13:07.799Z 0.9640 100.0 94.4 95.5 8.8 100.0 6.2 42knqt
21
21
2026-03-26T23:14:22.740Z 0.9640 100.0 94.4 95.5 8.8 100.0 6.2 42knqt
22
22
2026-03-26T23:23:35.323Z 0.8298 93.8 70.7 91.3 12.8 58.3 4.3 42knqt
23
+
2026-03-26T23:26:36.687Z 0.8912 97.9 90.3 91.3 12.8 58.3 4.3 42knqt
+16
-16
tests/eval/gold-standard.ts
+16
-16
tests/eval/gold-standard.ts
···
126
126
expectedMinNodes: 10,
127
127
expectedMaxNodes: 25,
128
128
expectedNodes: [
129
-
{ statement: '20', type: 'CONSTRAINT' },
129
+
{ statement: '20 columns', type: 'CONTEXT' },
130
130
{ statement: 'cooldown', type: 'CONSTRAINT' },
131
131
{ statement: 'rejected', type: 'REQUIREMENT' },
132
132
{ statement: 'broadcast', type: 'REQUIREMENT' },
133
-
{ statement: '120 seconds', type: 'CONSTRAINT' },
134
-
{ statement: 'round-robin', type: 'CONSTRAINT' },
133
+
{ statement: '120 seconds', type: 'CONTEXT' },
134
+
{ statement: 'round-robin', type: 'CONTEXT' },
135
135
],
136
136
expectedEdges: [],
137
137
},
···
177
177
expectedMinNodes: 8,
178
178
expectedMaxNodes: 22,
179
179
expectedNodes: [
180
-
{ statement: 'minimum number of payments', type: 'REQUIREMENT' },
181
-
{ statement: 'same net effect', type: 'INVARIANT' },
182
-
{ statement: 'cycles', type: 'INVARIANT' },
183
-
{ statement: 'zero balances', type: 'INVARIANT' },
184
-
{ statement: 'exceeds', type: 'CONSTRAINT' },
180
+
{ statement: 'minimum number of payments', type: 'CONSTRAINT' },
181
+
{ statement: 'same net effect', type: 'REQUIREMENT' },
182
+
{ statement: 'cycles', type: 'REQUIREMENT' },
183
+
{ statement: 'empty settlement plan', type: 'REQUIREMENT' },
184
+
{ statement: 'exceeds', type: 'REQUIREMENT' },
185
185
{ statement: 'settled up', type: 'REQUIREMENT' },
186
186
],
187
187
expectedEdges: [],
···
194
194
expectedMinNodes: 18,
195
195
expectedMaxNodes: 40,
196
196
expectedNodes: [
197
-
{ statement: 'system of record', type: 'DEFINITION' },
198
-
{ statement: 'email addresses must be unique', type: 'CONSTRAINT' },
197
+
{ statement: 'system of record', type: 'CONTEXT' },
198
+
{ statement: 'email addresses must be unique', type: 'REQUIREMENT' },
199
199
{ statement: 'never store or return plaintext passwords', type: 'INVARIANT' },
200
200
{ statement: 'soft delete', type: 'REQUIREMENT' },
201
201
{ statement: '100 characters', type: 'CONSTRAINT' },
202
-
{ statement: 'locked for 1 hour', type: 'CONSTRAINT' },
202
+
{ statement: 'locked for 1 hour', type: 'REQUIREMENT' },
203
203
{ statement: 'parameterized statements', type: 'CONSTRAINT' },
204
204
{ statement: 'event payloads must never contain passwords', type: 'INVARIANT' },
205
205
{ statement: '50 results per page', type: 'CONSTRAINT' },
···
215
215
expectedMinNodes: 10,
216
216
expectedMaxNodes: 25,
217
217
expectedNodes: [
218
-
{ statement: '3', type: 'CONSTRAINT' },
219
-
{ statement: 'cell already occupied', type: 'REQUIREMENT' },
220
-
{ statement: 'x always moves first', type: 'CONSTRAINT' },
221
-
{ statement: 'win', type: 'REQUIREMENT' },
218
+
{ statement: '3x3 grid', type: 'REQUIREMENT' },
219
+
{ statement: 'already occupied', type: 'REQUIREMENT' },
220
+
{ statement: 'x always moves first', type: 'INVARIANT' },
222
221
{ statement: 'draw', type: 'REQUIREMENT' },
223
-
{ statement: 'game status', type: 'DEFINITION' },
222
+
{ statement: 'win detection', type: 'REQUIREMENT' },
223
+
{ statement: 'unique game id', type: 'REQUIREMENT' },
224
224
],
225
225
expectedEdges: [],
226
226
},