personal memory agent
Replace unprocessed_files stat with pending_segments
Refactor journal stats to use observe.sense.scan_day() for detecting
unprocessed media files, replacing the old defunct *_raw.flac pattern
matching. The new pending_segments stat counts unique segments with
pending files rather than individual file counts.
- Add pending_segments return value to sense.scan_day()
- Update journal_stats.py to use sense_scan_day() (DRY)
- Update dashboard.js repair alerts for new stat name
- Fix conftest.py to import real observe.sense module
- Remove unused fixtures/journal/stats.json
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
+39
-1218
+3
-3
apps/stats/dashboard.js
+3
-3
apps/stats/dashboard.js
···
486
486
const progressSection = document.getElementById('progressSection');
487
487
progressSection.innerHTML = ''; // Clear existing content
488
488
progressSection.appendChild(
489
-
progressCard('Audio Processing', totals.audio_sessions || 0, totals.unprocessed_files || 0)
489
+
progressCard('Audio Processing', totals.audio_sessions || 0, totals.pending_segments || 0)
490
490
);
491
491
progressSection.appendChild(
492
492
progressCard('Insight Summaries', totals.insights_processed || 0, totals.insights_pending || 0)
···
562
562
);
563
563
564
564
// Render repairs if needed
565
-
const repairs = ['unprocessed_files', 'insights_pending'];
565
+
const repairs = ['pending_segments', 'insights_pending'];
566
566
const hasRepairs = repairs.some(key => (totals[key] || 0) > 0);
567
567
568
568
if (hasRepairs) {
···
574
574
575
575
const repairGrid = alert.querySelector('#repairGrid');
576
576
const repairLabels = {
577
-
unprocessed_files: 'Unprocessed Media',
577
+
pending_segments: 'Pending Segments',
578
578
insights_pending: 'Insight Summaries'
579
579
};
580
580
-1130
fixtures/journal/stats.json
-1130
fixtures/journal/stats.json
···
1
-
{
2
-
"days": {
3
-
"20240101": {
4
-
"audio_sessions": 1,
5
-
"audio_segments": 5,
6
-
"unprocessed_files": 0,
7
-
"insights_processed": 2,
8
-
"insights_pending": 13,
9
-
"audio_duration": 44.0,
10
-
"screen_duration": 0.0
11
-
},
12
-
"20240102": {
13
-
"audio_sessions": 1,
14
-
"audio_segments": 3,
15
-
"screen_sessions": 1,
16
-
"screen_frames": 3,
17
-
"unprocessed_files": 0,
18
-
"insights_processed": 1,
19
-
"insights_pending": 14,
20
-
"audio_duration": 29.0,
21
-
"screen_duration": 23.1
22
-
},
23
-
"20250124": {
24
-
"unprocessed_files": 0,
25
-
"insights_processed": 0,
26
-
"insights_pending": 15,
27
-
"audio_duration": 0.0,
28
-
"screen_duration": 0.0
29
-
},
30
-
"20251101": {
31
-
"unprocessed_files": 0,
32
-
"insights_processed": 0,
33
-
"insights_pending": 15,
34
-
"audio_duration": 0.0,
35
-
"screen_duration": 0.0
36
-
},
37
-
"20251102": {
38
-
"unprocessed_files": 0,
39
-
"insights_processed": 0,
40
-
"insights_pending": 15,
41
-
"audio_duration": 0.0,
42
-
"screen_duration": 0.0
43
-
},
44
-
"20251103": {
45
-
"unprocessed_files": 0,
46
-
"insights_processed": 0,
47
-
"insights_pending": 15,
48
-
"audio_duration": 0.0,
49
-
"screen_duration": 0.0
50
-
},
51
-
"20251104": {
52
-
"unprocessed_files": 0,
53
-
"insights_processed": 0,
54
-
"insights_pending": 15,
55
-
"audio_duration": 0.0,
56
-
"screen_duration": 0.0
57
-
},
58
-
"20251107": {
59
-
"unprocessed_files": 0,
60
-
"insights_processed": 0,
61
-
"insights_pending": 15,
62
-
"audio_duration": 0.0,
63
-
"screen_duration": 0.0
64
-
},
65
-
"20251108": {
66
-
"unprocessed_files": 0,
67
-
"insights_processed": 0,
68
-
"insights_pending": 15,
69
-
"audio_duration": 0.0,
70
-
"screen_duration": 0.0
71
-
},
72
-
"20251109": {
73
-
"unprocessed_files": 0,
74
-
"insights_processed": 0,
75
-
"insights_pending": 15,
76
-
"audio_duration": 0.0,
77
-
"screen_duration": 0.0
78
-
},
79
-
"20251110": {
80
-
"unprocessed_files": 0,
81
-
"insights_processed": 0,
82
-
"insights_pending": 15,
83
-
"audio_duration": 0.0,
84
-
"screen_duration": 0.0
85
-
},
86
-
"20251111": {
87
-
"unprocessed_files": 0,
88
-
"insights_processed": 0,
89
-
"insights_pending": 15,
90
-
"audio_duration": 0.0,
91
-
"screen_duration": 0.0
92
-
},
93
-
"20251112": {
94
-
"unprocessed_files": 0,
95
-
"insights_processed": 0,
96
-
"insights_pending": 15,
97
-
"audio_duration": 0.0,
98
-
"screen_duration": 0.0
99
-
},
100
-
"20251117": {
101
-
"unprocessed_files": 0,
102
-
"insights_processed": 0,
103
-
"insights_pending": 15,
104
-
"audio_duration": 0.0,
105
-
"screen_duration": 0.0
106
-
},
107
-
"20251118": {
108
-
"unprocessed_files": 0,
109
-
"insights_processed": 0,
110
-
"insights_pending": 15,
111
-
"audio_duration": 0.0,
112
-
"screen_duration": 0.0
113
-
},
114
-
"20251119": {
115
-
"unprocessed_files": 0,
116
-
"insights_processed": 0,
117
-
"insights_pending": 15,
118
-
"audio_duration": 0.0,
119
-
"screen_duration": 0.0
120
-
},
121
-
"20251120": {
122
-
"unprocessed_files": 0,
123
-
"insights_processed": 0,
124
-
"insights_pending": 15,
125
-
"audio_duration": 0.0,
126
-
"screen_duration": 0.0
127
-
},
128
-
"20251121": {
129
-
"unprocessed_files": 0,
130
-
"insights_processed": 0,
131
-
"insights_pending": 15,
132
-
"audio_duration": 0.0,
133
-
"screen_duration": 0.0
134
-
},
135
-
"20251122": {
136
-
"unprocessed_files": 0,
137
-
"insights_processed": 0,
138
-
"insights_pending": 15,
139
-
"audio_duration": 0.0,
140
-
"screen_duration": 0.0
141
-
},
142
-
"20251124": {
143
-
"unprocessed_files": 0,
144
-
"insights_processed": 0,
145
-
"insights_pending": 15,
146
-
"audio_duration": 0.0,
147
-
"screen_duration": 0.0
148
-
},
149
-
"20251125": {
150
-
"unprocessed_files": 0,
151
-
"insights_processed": 0,
152
-
"insights_pending": 15,
153
-
"audio_duration": 0.0,
154
-
"screen_duration": 0.0
155
-
}
156
-
},
157
-
"totals": {
158
-
"audio_sessions": 2,
159
-
"audio_segments": 8,
160
-
"unprocessed_files": 0,
161
-
"insights_processed": 3,
162
-
"insights_pending": 312,
163
-
"screen_sessions": 1,
164
-
"screen_frames": 3
165
-
},
166
-
"total_audio_duration": 73.0,
167
-
"total_screen_duration": 23.1,
168
-
"topic_counts": {
169
-
"meetings": 4
170
-
},
171
-
"topic_minutes": {
172
-
"meetings": 180.0
173
-
},
174
-
"topic_counts_by_day": {
175
-
"20240101": {
176
-
"meetings": 4
177
-
}
178
-
},
179
-
"facet_counts": {
180
-
"work": 3,
181
-
"personal": 1
182
-
},
183
-
"facet_minutes": {
184
-
"work": 150.0,
185
-
"personal": 30.0
186
-
},
187
-
"facet_counts_by_day": {
188
-
"20240101": {
189
-
"work": 3,
190
-
"personal": 1
191
-
}
192
-
},
193
-
"heatmap": [
194
-
[
195
-
0.0,
196
-
0.0,
197
-
0.0,
198
-
0.0,
199
-
0.0,
200
-
0.0,
201
-
0.0,
202
-
0.0,
203
-
0.0,
204
-
30.0,
205
-
30.0,
206
-
30.0,
207
-
0.0,
208
-
0.0,
209
-
60.0,
210
-
0.0,
211
-
30.0,
212
-
0.0,
213
-
0.0,
214
-
0.0,
215
-
0.0,
216
-
0.0,
217
-
0.0,
218
-
0.0
219
-
],
220
-
[
221
-
0.0,
222
-
0.0,
223
-
0.0,
224
-
0.0,
225
-
0.0,
226
-
0.0,
227
-
0.0,
228
-
0.0,
229
-
0.0,
230
-
0.0,
231
-
0.0,
232
-
0.0,
233
-
0.0,
234
-
0.0,
235
-
0.0,
236
-
0.0,
237
-
0.0,
238
-
0.0,
239
-
0.0,
240
-
0.0,
241
-
0.0,
242
-
0.0,
243
-
0.0,
244
-
0.0
245
-
],
246
-
[
247
-
0.0,
248
-
0.0,
249
-
0.0,
250
-
0.0,
251
-
0.0,
252
-
0.0,
253
-
0.0,
254
-
0.0,
255
-
0.0,
256
-
0.0,
257
-
0.0,
258
-
0.0,
259
-
0.0,
260
-
0.0,
261
-
0.0,
262
-
0.0,
263
-
0.0,
264
-
0.0,
265
-
0.0,
266
-
0.0,
267
-
0.0,
268
-
0.0,
269
-
0.0,
270
-
0.0
271
-
],
272
-
[
273
-
0.0,
274
-
0.0,
275
-
0.0,
276
-
0.0,
277
-
0.0,
278
-
0.0,
279
-
0.0,
280
-
0.0,
281
-
0.0,
282
-
0.0,
283
-
0.0,
284
-
0.0,
285
-
0.0,
286
-
0.0,
287
-
0.0,
288
-
0.0,
289
-
0.0,
290
-
0.0,
291
-
0.0,
292
-
0.0,
293
-
0.0,
294
-
0.0,
295
-
0.0,
296
-
0.0
297
-
],
298
-
[
299
-
0.0,
300
-
0.0,
301
-
0.0,
302
-
0.0,
303
-
0.0,
304
-
0.0,
305
-
0.0,
306
-
0.0,
307
-
0.0,
308
-
0.0,
309
-
0.0,
310
-
0.0,
311
-
0.0,
312
-
0.0,
313
-
0.0,
314
-
0.0,
315
-
0.0,
316
-
0.0,
317
-
0.0,
318
-
0.0,
319
-
0.0,
320
-
0.0,
321
-
0.0,
322
-
0.0
323
-
],
324
-
[
325
-
0.0,
326
-
0.0,
327
-
0.0,
328
-
0.0,
329
-
0.0,
330
-
0.0,
331
-
0.0,
332
-
0.0,
333
-
0.0,
334
-
0.0,
335
-
0.0,
336
-
0.0,
337
-
0.0,
338
-
0.0,
339
-
0.0,
340
-
0.0,
341
-
0.0,
342
-
0.0,
343
-
0.0,
344
-
0.0,
345
-
0.0,
346
-
0.0,
347
-
0.0,
348
-
0.0
349
-
],
350
-
[
351
-
0.0,
352
-
0.0,
353
-
0.0,
354
-
0.0,
355
-
0.0,
356
-
0.0,
357
-
0.0,
358
-
0.0,
359
-
0.0,
360
-
0.0,
361
-
0.0,
362
-
0.0,
363
-
0.0,
364
-
0.0,
365
-
0.0,
366
-
0.0,
367
-
0.0,
368
-
0.0,
369
-
0.0,
370
-
0.0,
371
-
0.0,
372
-
0.0,
373
-
0.0,
374
-
0.0
375
-
]
376
-
],
377
-
"token_usage_by_day": {
378
-
"20250823": {
379
-
"gemini-2.5-flash": {
380
-
"input_tokens": 1350,
381
-
"output_tokens": 606,
382
-
"cached_tokens": 0,
383
-
"reasoning_tokens": 239,
384
-
"total_tokens": 2195
385
-
},
386
-
"gemini-2.5-flash-lite": {
387
-
"input_tokens": 12,
388
-
"output_tokens": 1,
389
-
"cached_tokens": 0,
390
-
"reasoning_tokens": 0,
391
-
"total_tokens": 13
392
-
},
393
-
"models/gemini-2.5-flash": {
394
-
"input_tokens": 1143,
395
-
"output_tokens": 373,
396
-
"reasoning_tokens": 3267,
397
-
"total_tokens": 4783,
398
-
"cached_tokens": 0
399
-
},
400
-
"models/gemini-2.5-flash-lite": {
401
-
"input_tokens": 60,
402
-
"output_tokens": 5,
403
-
"total_tokens": 65,
404
-
"cached_tokens": 0,
405
-
"reasoning_tokens": 0
406
-
}
407
-
},
408
-
"20250824": {
409
-
"gemini-2.5-flash": {
410
-
"input_tokens": 1454,
411
-
"output_tokens": 679,
412
-
"cached_tokens": 0,
413
-
"reasoning_tokens": 528,
414
-
"total_tokens": 2661
415
-
},
416
-
"gemini-2.5-flash-lite": {
417
-
"input_tokens": 12,
418
-
"output_tokens": 1,
419
-
"cached_tokens": 0,
420
-
"reasoning_tokens": 0,
421
-
"total_tokens": 13
422
-
}
423
-
},
424
-
"20250825": {
425
-
"gemini-2.5-flash": {
426
-
"input_tokens": 200,
427
-
"output_tokens": 100,
428
-
"cached_tokens": 0,
429
-
"reasoning_tokens": 0,
430
-
"total_tokens": 300
431
-
},
432
-
"gemini-2.5-flash-lite": {
433
-
"input_tokens": 0,
434
-
"output_tokens": 0,
435
-
"cached_tokens": 0,
436
-
"reasoning_tokens": 0,
437
-
"total_tokens": 0
438
-
}
439
-
},
440
-
"20250826": {
441
-
"gemini-2.5-flash": {
442
-
"input_tokens": 500,
443
-
"output_tokens": 250,
444
-
"cached_tokens": 0,
445
-
"reasoning_tokens": 0,
446
-
"total_tokens": 750
447
-
},
448
-
"gemini-2.5-flash-lite": {
449
-
"input_tokens": 0,
450
-
"output_tokens": 0,
451
-
"cached_tokens": 0,
452
-
"reasoning_tokens": 0,
453
-
"total_tokens": 0
454
-
}
455
-
},
456
-
"20250827": {
457
-
"gemini-2.5-flash": {
458
-
"input_tokens": 1130,
459
-
"output_tokens": 415,
460
-
"cached_tokens": 0,
461
-
"reasoning_tokens": 3246,
462
-
"total_tokens": 4791
463
-
},
464
-
"gemini-2.5-flash-lite": {
465
-
"input_tokens": 60,
466
-
"output_tokens": 5,
467
-
"cached_tokens": 0,
468
-
"reasoning_tokens": 0,
469
-
"total_tokens": 65
470
-
}
471
-
},
472
-
"20250829": {
473
-
"gemini-2.5-flash": {
474
-
"input_tokens": 200,
475
-
"output_tokens": 100,
476
-
"cached_tokens": 0,
477
-
"reasoning_tokens": 0,
478
-
"total_tokens": 300
479
-
},
480
-
"gemini-2.5-flash-lite": {
481
-
"input_tokens": 0,
482
-
"output_tokens": 0,
483
-
"cached_tokens": 0,
484
-
"reasoning_tokens": 0,
485
-
"total_tokens": 0
486
-
}
487
-
},
488
-
"20250905": {
489
-
"gemini-2.5-flash": {
490
-
"input_tokens": 1270,
491
-
"output_tokens": 591,
492
-
"cached_tokens": 0,
493
-
"reasoning_tokens": 3355,
494
-
"total_tokens": 5216
495
-
},
496
-
"gemini-2.5-flash-lite": {
497
-
"input_tokens": 60,
498
-
"output_tokens": 5,
499
-
"cached_tokens": 0,
500
-
"reasoning_tokens": 0,
501
-
"total_tokens": 65
502
-
}
503
-
},
504
-
"20250906": {
505
-
"gemini-2.5-flash": {
506
-
"input_tokens": 674,
507
-
"output_tokens": 328,
508
-
"cached_tokens": 0,
509
-
"reasoning_tokens": 709,
510
-
"total_tokens": 1711
511
-
},
512
-
"gemini-2.5-flash-lite": {
513
-
"input_tokens": 12,
514
-
"output_tokens": 1,
515
-
"cached_tokens": 0,
516
-
"reasoning_tokens": 0,
517
-
"total_tokens": 13
518
-
}
519
-
},
520
-
"20250909": {
521
-
"gemini-2.5-flash": {
522
-
"input_tokens": 1518,
523
-
"output_tokens": 642,
524
-
"cached_tokens": 0,
525
-
"reasoning_tokens": 5004,
526
-
"total_tokens": 7164
527
-
},
528
-
"gemini-2.5-flash-lite": {
529
-
"input_tokens": 84,
530
-
"output_tokens": 7,
531
-
"cached_tokens": 0,
532
-
"reasoning_tokens": 0,
533
-
"total_tokens": 91
534
-
}
535
-
},
536
-
"20250910": {
537
-
"gemini-2.5-flash": {
538
-
"input_tokens": 300,
539
-
"output_tokens": 150,
540
-
"cached_tokens": 0,
541
-
"reasoning_tokens": 0,
542
-
"total_tokens": 450
543
-
},
544
-
"gemini-2.5-flash-lite": {
545
-
"input_tokens": 0,
546
-
"output_tokens": 0,
547
-
"cached_tokens": 0,
548
-
"reasoning_tokens": 0,
549
-
"total_tokens": 0
550
-
}
551
-
},
552
-
"20250914": {
553
-
"gemini-2.5-flash": {
554
-
"input_tokens": 1348,
555
-
"output_tokens": 654,
556
-
"cached_tokens": 0,
557
-
"reasoning_tokens": 1365,
558
-
"total_tokens": 3367
559
-
},
560
-
"gemini-2.5-flash-lite": {
561
-
"input_tokens": 24,
562
-
"output_tokens": 2,
563
-
"cached_tokens": 0,
564
-
"reasoning_tokens": 0,
565
-
"total_tokens": 26
566
-
}
567
-
},
568
-
"20250915": {
569
-
"gemini-2.5-flash": {
570
-
"input_tokens": 474,
571
-
"output_tokens": 218,
572
-
"cached_tokens": 0,
573
-
"reasoning_tokens": 662,
574
-
"total_tokens": 1354
575
-
},
576
-
"gemini-2.5-flash-lite": {
577
-
"input_tokens": 12,
578
-
"output_tokens": 1,
579
-
"cached_tokens": 0,
580
-
"reasoning_tokens": 0,
581
-
"total_tokens": 13
582
-
}
583
-
},
584
-
"20250916": {
585
-
"gemini-2.5-flash": {
586
-
"input_tokens": 348,
587
-
"output_tokens": 153,
588
-
"reasoning_tokens": 1307,
589
-
"total_tokens": 1808
590
-
},
591
-
"gemini-2.5-flash-lite": {
592
-
"input_tokens": 24,
593
-
"output_tokens": 2,
594
-
"total_tokens": 26
595
-
}
596
-
},
597
-
"20250917": {
598
-
"gemini-2.5-flash": {
599
-
"input_tokens": 174,
600
-
"output_tokens": 80,
601
-
"reasoning_tokens": 657,
602
-
"total_tokens": 911
603
-
},
604
-
"gemini-2.5-flash-lite": {
605
-
"input_tokens": 12,
606
-
"output_tokens": 1,
607
-
"total_tokens": 13
608
-
}
609
-
},
610
-
"20250919": {
611
-
"gemini-2.5-flash": {
612
-
"input_tokens": 200,
613
-
"output_tokens": 100,
614
-
"cached_tokens": 0,
615
-
"reasoning_tokens": 0,
616
-
"total_tokens": 300
617
-
},
618
-
"gemini-2.5-flash-lite": {
619
-
"input_tokens": 0,
620
-
"output_tokens": 0,
621
-
"cached_tokens": 0,
622
-
"reasoning_tokens": 0,
623
-
"total_tokens": 0
624
-
}
625
-
},
626
-
"20250920": {
627
-
"gemini-2.5-flash": {
628
-
"input_tokens": 100,
629
-
"output_tokens": 50,
630
-
"cached_tokens": 0,
631
-
"reasoning_tokens": 0,
632
-
"total_tokens": 150
633
-
},
634
-
"gemini-2.5-flash-lite": {
635
-
"input_tokens": 0,
636
-
"output_tokens": 0,
637
-
"cached_tokens": 0,
638
-
"reasoning_tokens": 0,
639
-
"total_tokens": 0
640
-
}
641
-
},
642
-
"20250921": {
643
-
"gemini-2.5-flash": {
644
-
"input_tokens": 800,
645
-
"output_tokens": 400,
646
-
"cached_tokens": 0,
647
-
"reasoning_tokens": 0,
648
-
"total_tokens": 1200
649
-
},
650
-
"gemini-2.5-flash-lite": {
651
-
"input_tokens": 0,
652
-
"output_tokens": 0,
653
-
"cached_tokens": 0,
654
-
"reasoning_tokens": 0,
655
-
"total_tokens": 0
656
-
}
657
-
},
658
-
"20250926": {
659
-
"gemini-2.5-flash": {
660
-
"input_tokens": 174,
661
-
"output_tokens": 79,
662
-
"reasoning_tokens": 648,
663
-
"total_tokens": 901
664
-
},
665
-
"gemini-2.5-flash-lite": {
666
-
"input_tokens": 12,
667
-
"output_tokens": 1,
668
-
"total_tokens": 13
669
-
}
670
-
},
671
-
"20250928": {
672
-
"gemini-2.5-flash": {
673
-
"input_tokens": 200,
674
-
"output_tokens": 100,
675
-
"cached_tokens": 0,
676
-
"reasoning_tokens": 0,
677
-
"total_tokens": 300
678
-
},
679
-
"gemini-2.5-flash-lite": {
680
-
"input_tokens": 0,
681
-
"output_tokens": 0,
682
-
"cached_tokens": 0,
683
-
"reasoning_tokens": 0,
684
-
"total_tokens": 0
685
-
}
686
-
},
687
-
"20251004": {
688
-
"gemini-2.5-flash": {
689
-
"input_tokens": 1000,
690
-
"output_tokens": 500,
691
-
"cached_tokens": 0,
692
-
"reasoning_tokens": 0,
693
-
"total_tokens": 1500
694
-
},
695
-
"gemini-2.5-flash-lite": {
696
-
"input_tokens": 0,
697
-
"output_tokens": 0,
698
-
"cached_tokens": 0,
699
-
"reasoning_tokens": 0,
700
-
"total_tokens": 0
701
-
}
702
-
},
703
-
"20251005": {
704
-
"gemini-2.5-flash": {
705
-
"input_tokens": 1274,
706
-
"output_tokens": 636,
707
-
"cached_tokens": 0,
708
-
"reasoning_tokens": 559,
709
-
"total_tokens": 2469
710
-
},
711
-
"gemini-2.5-flash-lite": {
712
-
"input_tokens": 12,
713
-
"output_tokens": 1,
714
-
"cached_tokens": 0,
715
-
"reasoning_tokens": 0,
716
-
"total_tokens": 13
717
-
}
718
-
},
719
-
"20251007": {
720
-
"gemini-2.5-flash": {
721
-
"input_tokens": 174,
722
-
"output_tokens": 79,
723
-
"reasoning_tokens": 636,
724
-
"total_tokens": 889
725
-
},
726
-
"gemini-2.5-flash-lite": {
727
-
"input_tokens": 12,
728
-
"output_tokens": 1,
729
-
"total_tokens": 13
730
-
}
731
-
},
732
-
"20251011": {
733
-
"gemini-2.5-flash": {
734
-
"input_tokens": 2685,
735
-
"output_tokens": 1137,
736
-
"cached_tokens": 0,
737
-
"reasoning_tokens": 4666,
738
-
"total_tokens": 8488
739
-
},
740
-
"gemini-2.5-flash-lite": {
741
-
"input_tokens": 70,
742
-
"output_tokens": 7,
743
-
"cached_tokens": 0,
744
-
"reasoning_tokens": 0,
745
-
"total_tokens": 77
746
-
}
747
-
},
748
-
"20251012": {
749
-
"gemini-2.5-flash": {
750
-
"input_tokens": 2144,
751
-
"output_tokens": 824,
752
-
"total_tokens": 3371,
753
-
"cached_tokens": 300,
754
-
"reasoning_tokens": 553
755
-
},
756
-
"gpt-5": {
757
-
"input_tokens": 1000,
758
-
"output_tokens": 500,
759
-
"total_tokens": 1500,
760
-
"cached_tokens": 200,
761
-
"reasoning_tokens": 100,
762
-
"requests": 1
763
-
},
764
-
"gemini-2.5-flash-lite": {
765
-
"input_tokens": 5,
766
-
"output_tokens": 1,
767
-
"total_tokens": 6
768
-
}
769
-
},
770
-
"20251013": {
771
-
"gemini-2.5-flash": {
772
-
"input_tokens": 296,
773
-
"output_tokens": 101,
774
-
"total_tokens": 1345,
775
-
"reasoning_tokens": 948
776
-
},
777
-
"gemini-2.5-flash-lite": {
778
-
"input_tokens": 5,
779
-
"output_tokens": 1,
780
-
"total_tokens": 6
781
-
}
782
-
},
783
-
"20251015": {
784
-
"gemini-2.5-flash": {
785
-
"input_tokens": 830,
786
-
"output_tokens": 260,
787
-
"total_tokens": 3781,
788
-
"reasoning_tokens": 2691
789
-
},
790
-
"gemini-2.5-flash-lite": {
791
-
"input_tokens": 34,
792
-
"output_tokens": 4,
793
-
"total_tokens": 38
794
-
}
795
-
},
796
-
"20251016": {
797
-
"clean-format-test": {
798
-
"input_tokens": 100,
799
-
"output_tokens": 50,
800
-
"total_tokens": 150
801
-
}
802
-
},
803
-
"20251025": {
804
-
"gemini-2.5-flash": {
805
-
"input_tokens": 4118,
806
-
"output_tokens": 1816,
807
-
"total_tokens": 9342,
808
-
"reasoning_tokens": 3408
809
-
},
810
-
"gemini-2.5-flash-lite": {
811
-
"input_tokens": 44,
812
-
"output_tokens": 6,
813
-
"total_tokens": 50
814
-
}
815
-
},
816
-
"20251028": {
817
-
"gemini-2.5-flash": {
818
-
"input_tokens": 1215,
819
-
"output_tokens": 533,
820
-
"total_tokens": 3066,
821
-
"reasoning_tokens": 1318
822
-
},
823
-
"gemini-2.5-flash-lite": {
824
-
"input_tokens": 17,
825
-
"output_tokens": 2,
826
-
"total_tokens": 19
827
-
}
828
-
},
829
-
"20251029": {
830
-
"gemini-2.5-flash": {
831
-
"input_tokens": 200,
832
-
"output_tokens": 100,
833
-
"total_tokens": 300
834
-
},
835
-
"gemini-2.5-flash-lite": {
836
-
"input_tokens": 0,
837
-
"output_tokens": 0,
838
-
"total_tokens": 0
839
-
}
840
-
},
841
-
"20251030": {
842
-
"gemini-2.5-flash": {
843
-
"input_tokens": 800,
844
-
"output_tokens": 400,
845
-
"total_tokens": 1200
846
-
},
847
-
"gemini-2.5-flash-lite": {
848
-
"input_tokens": 0,
849
-
"output_tokens": 0,
850
-
"total_tokens": 0
851
-
}
852
-
},
853
-
"20251101": {
854
-
"gemini-2.5-flash": {
855
-
"input_tokens": 3944,
856
-
"output_tokens": 1924,
857
-
"total_tokens": 6298,
858
-
"reasoning_tokens": 430
859
-
},
860
-
"gemini-2.5-flash-lite": {
861
-
"input_tokens": 5,
862
-
"output_tokens": 1,
863
-
"total_tokens": 6
864
-
}
865
-
},
866
-
"20251102": {
867
-
"gemini-2.5-flash": {
868
-
"input_tokens": 4430,
869
-
"output_tokens": 2043,
870
-
"total_tokens": 8839,
871
-
"reasoning_tokens": 2366
872
-
},
873
-
"gemini-2.5-flash-lite": {
874
-
"input_tokens": 34,
875
-
"output_tokens": 4,
876
-
"total_tokens": 38
877
-
}
878
-
},
879
-
"20251104": {
880
-
"gemini-2.5-flash": {
881
-
"input_tokens": 1000,
882
-
"output_tokens": 500,
883
-
"total_tokens": 1500
884
-
},
885
-
"gemini-2.5-flash-lite": {
886
-
"input_tokens": 0,
887
-
"output_tokens": 0,
888
-
"total_tokens": 0
889
-
}
890
-
},
891
-
"20251107": {
892
-
"gemini-2.5-flash": {
893
-
"input_tokens": 200,
894
-
"output_tokens": 100,
895
-
"total_tokens": 300
896
-
},
897
-
"gemini-2.5-flash-lite": {
898
-
"input_tokens": 0,
899
-
"output_tokens": 0,
900
-
"total_tokens": 0
901
-
}
902
-
},
903
-
"20251108": {
904
-
"gemini-2.5-flash": {
905
-
"input_tokens": 1645,
906
-
"output_tokens": 517,
907
-
"total_tokens": 6804,
908
-
"reasoning_tokens": 4642
909
-
},
910
-
"gemini-2.5-flash-lite": {
911
-
"input_tokens": 51,
912
-
"output_tokens": 6,
913
-
"total_tokens": 57
914
-
}
915
-
},
916
-
"20251109": {
917
-
"gemini-2.5-flash": {
918
-
"input_tokens": 1615,
919
-
"output_tokens": 717,
920
-
"total_tokens": 3708,
921
-
"reasoning_tokens": 1376
922
-
},
923
-
"gemini-2.5-flash-lite": {
924
-
"input_tokens": 17,
925
-
"output_tokens": 2,
926
-
"total_tokens": 19
927
-
}
928
-
},
929
-
"20251110": {
930
-
"gemini-2.5-flash": {
931
-
"input_tokens": 1800,
932
-
"output_tokens": 900,
933
-
"total_tokens": 2700
934
-
},
935
-
"gemini-2.5-flash-lite": {
936
-
"input_tokens": 0,
937
-
"output_tokens": 0,
938
-
"total_tokens": 0
939
-
}
940
-
},
941
-
"20251111": {
942
-
"gemini-2.5-flash": {
943
-
"input_tokens": 600,
944
-
"output_tokens": 300,
945
-
"total_tokens": 900
946
-
},
947
-
"gemini-2.5-flash-lite": {
948
-
"input_tokens": 0,
949
-
"output_tokens": 0,
950
-
"total_tokens": 0
951
-
}
952
-
},
953
-
"20251112": {
954
-
"gemini-2.5-flash": {
955
-
"input_tokens": 800,
956
-
"output_tokens": 400,
957
-
"total_tokens": 1200
958
-
},
959
-
"gemini-2.5-flash-lite": {
960
-
"input_tokens": 0,
961
-
"output_tokens": 0,
962
-
"total_tokens": 0
963
-
}
964
-
},
965
-
"20251113": {
966
-
"gemini-2.5-flash": {
967
-
"input_tokens": 1015,
968
-
"output_tokens": 423,
969
-
"total_tokens": 2759,
970
-
"reasoning_tokens": 1321
971
-
},
972
-
"gemini-2.5-flash-lite": {
973
-
"input_tokens": 17,
974
-
"output_tokens": 2,
975
-
"total_tokens": 19
976
-
}
977
-
},
978
-
"20251115": {
979
-
"gemini-2.5-flash-lite": {
980
-
"input_tokens": 3562,
981
-
"output_tokens": 127,
982
-
"total_tokens": 9228,
983
-
"reasoning_tokens": 5539
984
-
}
985
-
},
986
-
"20251118": {
987
-
"gemini-2.5-flash": {
988
-
"input_tokens": 400,
989
-
"output_tokens": 200,
990
-
"total_tokens": 600
991
-
},
992
-
"gemini-2.5-flash-lite": {
993
-
"input_tokens": 0,
994
-
"output_tokens": 0,
995
-
"total_tokens": 0
996
-
}
997
-
},
998
-
"20251119": {
999
-
"gemini-2.5-flash": {
1000
-
"input_tokens": 2144,
1001
-
"output_tokens": 1025,
1002
-
"total_tokens": 3670,
1003
-
"reasoning_tokens": 501
1004
-
},
1005
-
"gemini-2.5-flash-lite": {
1006
-
"input_tokens": 5,
1007
-
"output_tokens": 1,
1008
-
"total_tokens": 6
1009
-
}
1010
-
},
1011
-
"20251121": {
1012
-
"gemini-2.5-flash": {
1013
-
"input_tokens": 544,
1014
-
"output_tokens": 224,
1015
-
"total_tokens": 1225,
1016
-
"reasoning_tokens": 457
1017
-
},
1018
-
"gemini-2.5-flash-lite": {
1019
-
"input_tokens": 5,
1020
-
"output_tokens": 1,
1021
-
"total_tokens": 6
1022
-
}
1023
-
},
1024
-
"20251122": {
1025
-
"gemini-2.5-flash": {
1026
-
"input_tokens": 759,
1027
-
"output_tokens": 256,
1028
-
"total_tokens": 3020,
1029
-
"reasoning_tokens": 2005
1030
-
},
1031
-
"gemini-2.5-flash-lite": {
1032
-
"input_tokens": 22,
1033
-
"output_tokens": 3,
1034
-
"total_tokens": 25
1035
-
}
1036
-
},
1037
-
"20251123": {
1038
-
"gemini-2.5-flash": {
1039
-
"input_tokens": 600,
1040
-
"output_tokens": 300,
1041
-
"total_tokens": 900
1042
-
},
1043
-
"gemini-2.5-flash-lite": {
1044
-
"input_tokens": 87,
1045
-
"output_tokens": 26,
1046
-
"total_tokens": 113
1047
-
}
1048
-
},
1049
-
"20251124": {
1050
-
"gemini-2.5-flash": {
1051
-
"input_tokens": 1615,
1052
-
"output_tokens": 738,
1053
-
"total_tokens": 3921,
1054
-
"reasoning_tokens": 1568
1055
-
},
1056
-
"gemini-2.5-flash-lite": {
1057
-
"input_tokens": 17,
1058
-
"output_tokens": 2,
1059
-
"total_tokens": 19
1060
-
}
1061
-
},
1062
-
"20251125": {
1063
-
"gemini-2.5-flash": {
1064
-
"input_tokens": 200,
1065
-
"output_tokens": 100,
1066
-
"total_tokens": 300
1067
-
},
1068
-
"gemini-2.5-flash-lite": {
1069
-
"input_tokens": 0,
1070
-
"output_tokens": 0,
1071
-
"total_tokens": 0
1072
-
}
1073
-
},
1074
-
"20251128": {
1075
-
"gemini-2.5-flash": {
1076
-
"input_tokens": 200,
1077
-
"output_tokens": 100,
1078
-
"total_tokens": 300
1079
-
},
1080
-
"gemini-2.5-flash-lite": {
1081
-
"input_tokens": 0,
1082
-
"output_tokens": 0,
1083
-
"total_tokens": 0
1084
-
}
1085
-
}
1086
-
},
1087
-
"token_totals_by_model": {
1088
-
"gemini-2.5-flash": {
1089
-
"input_tokens": 50661,
1090
-
"output_tokens": 22848,
1091
-
"cached_tokens": 300,
1092
-
"reasoning_tokens": 47165,
1093
-
"total_tokens": 120524
1094
-
},
1095
-
"gemini-2.5-flash-lite": {
1096
-
"input_tokens": 4345,
1097
-
"output_tokens": 225,
1098
-
"cached_tokens": 0,
1099
-
"reasoning_tokens": 5539,
1100
-
"total_tokens": 10109
1101
-
},
1102
-
"models/gemini-2.5-flash": {
1103
-
"input_tokens": 1143,
1104
-
"output_tokens": 373,
1105
-
"reasoning_tokens": 3267,
1106
-
"total_tokens": 4783,
1107
-
"cached_tokens": 0
1108
-
},
1109
-
"models/gemini-2.5-flash-lite": {
1110
-
"input_tokens": 60,
1111
-
"output_tokens": 5,
1112
-
"total_tokens": 65,
1113
-
"cached_tokens": 0,
1114
-
"reasoning_tokens": 0
1115
-
},
1116
-
"gpt-5": {
1117
-
"input_tokens": 1000,
1118
-
"output_tokens": 500,
1119
-
"total_tokens": 1500,
1120
-
"cached_tokens": 200,
1121
-
"reasoning_tokens": 100,
1122
-
"requests": 1
1123
-
},
1124
-
"clean-format-test": {
1125
-
"input_tokens": 100,
1126
-
"output_tokens": 50,
1127
-
"total_tokens": 150
1128
-
}
1129
-
}
1130
-
}
+17
-2
observe/sense.py
+17
-2
observe/sense.py
···
550
550
logger.info("Batch processing complete")
551
551
552
552
553
-
def scan_day(day_dir: Path) -> dict[str, list[str]]:
553
+
def scan_day(day_dir: Path) -> dict:
554
554
"""Scan a day directory for processed and unprocessed files.
555
555
556
556
Args:
···
560
560
Dictionary with:
561
561
- "processed": List of JSONL output files in segments (HHMMSS_LEN/audio.jsonl, etc)
562
562
- "unprocessed": List of unprocessed source media files in day root
563
+
- "pending_segments": Count of unique segments with pending files
563
564
"""
564
565
# Find processed output files in segments (HHMMSS_LEN/)
565
566
from think.utils import segment_key
566
567
567
568
processed = []
569
+
if not day_dir.exists():
570
+
return {"processed": [], "unprocessed": [], "pending_segments": 0}
571
+
568
572
for segment in day_dir.iterdir():
569
573
if segment.is_dir() and segment_key(segment.name):
570
574
# Check for audio JSONL files (audio.jsonl, mic_audio.jsonl, etc.)
···
584
588
for ext in VIDEO_EXTENSIONS:
585
589
unprocessed.extend(sorted(p.name for p in day_dir.glob(f"*{ext}")))
586
590
587
-
return {"processed": processed, "unprocessed": unprocessed}
591
+
# Count unique segments with pending files
592
+
pending_segment_keys = set()
593
+
for filename in unprocessed:
594
+
key = segment_key(filename)
595
+
if key:
596
+
pending_segment_keys.add(key)
597
+
598
+
return {
599
+
"processed": processed,
600
+
"unprocessed": unprocessed,
601
+
"pending_segments": len(pending_segment_keys),
602
+
}
588
603
589
604
590
605
def main():
+2
-62
tests/conftest.py
+2
-62
tests/conftest.py
···
124
124
observe_pkg = sys.modules.get("observe")
125
125
setattr(observe_pkg, "hear", hear_mod)
126
126
if "observe.sense" not in sys.modules:
127
-
sense_mod = types.ModuleType("observe.sense")
128
-
129
-
def scan_day(day_dir):
130
-
# Stub matching real scan_day behavior:
131
-
# - "raw": processed files in segments
132
-
# - "processed": output JSON files in segments
133
-
# - "repairable": source media files in day root without matching segment
134
-
from pathlib import Path
135
-
136
-
day_path = Path(day_dir)
137
-
raw_files = []
138
-
processed_files = []
139
-
repairable_files = []
140
-
141
-
if day_path.is_dir():
142
-
from think.utils import segment_key
143
-
144
-
# Find raw (processed) files in segments (HHMMSS_LEN/)
145
-
for item in day_path.iterdir():
146
-
if item.is_dir() and segment_key(item.name):
147
-
# Found segment
148
-
for p in item.glob("*.flac"):
149
-
raw_files.append(f"{item.name}/{p.name}")
150
-
for p in item.glob("*.m4a"):
151
-
raw_files.append(f"{item.name}/{p.name}")
152
-
for p in item.glob("*.webm"):
153
-
raw_files.append(f"{item.name}/{p.name}")
154
-
for p in item.glob("*.mp4"):
155
-
raw_files.append(f"{item.name}/{p.name}")
156
-
157
-
# Find processed output files in segments
158
-
for item in day_path.iterdir():
159
-
if item.is_dir() and segment_key(item.name):
160
-
for p in item.glob("*audio.jsonl"):
161
-
processed_files.append(f"{item.name}/{p.name}")
162
-
for p in item.glob("*screen.jsonl"):
163
-
processed_files.append(f"{item.name}/{p.name}")
164
-
165
-
# Find repairable files (source media in root without matching segment)
166
-
for audio_ext in ["*.flac", "*.m4a"]:
167
-
for p in day_path.glob(audio_ext):
168
-
seg = segment_key(p.stem)
169
-
if seg:
170
-
segment_dir = day_path / seg
171
-
if not segment_dir.exists():
172
-
repairable_files.append(p.name)
173
-
174
-
for video_ext in ["*.webm", "*.mp4"]:
175
-
for p in day_path.glob(video_ext):
176
-
seg = segment_key(p.stem)
177
-
if seg:
178
-
segment_dir = day_path / seg
179
-
if not segment_dir.exists():
180
-
repairable_files.append(p.name)
181
-
182
-
return {
183
-
"raw": raw_files,
184
-
"processed": processed_files,
185
-
"repairable": repairable_files,
186
-
}
187
-
188
-
sense_mod.scan_day = scan_day
127
+
# Import the real module - it has minimal dependencies
128
+
sense_mod = importlib.import_module("observe.sense")
189
129
sys.modules["observe.sense"] = sense_mod
190
130
observe_pkg = sys.modules.get("observe")
191
131
setattr(observe_pkg, "sense", sense_mod)
+6
-3
tests/test_journal_stats.py
+6
-3
tests/test_journal_stats.py
···
17
17
'{"start": "10:01:00", "text": "world"}\n'
18
18
)
19
19
20
-
# Create an unprocessed file (remains in day root)
21
-
(day / "123456_300_raw.flac").write_bytes(b"RIFF")
20
+
# Create unprocessed media files (remain in day root, will be moved to segment on processing)
21
+
(day / "123456_300_audio.flac").write_bytes(b"RIFF")
22
+
(day / "123456_300_center_DP-1_screen.webm").write_bytes(b"WEBM")
22
23
23
24
(day / "entities.md").write_text("")
24
25
(day / "insights").mkdir()
···
49
50
js._apply_day_stats("20240101", day_data)
50
51
assert js.days["20240101"]["audio_sessions"] == 1
51
52
assert js.days["20240101"]["audio_segments"] == 2
52
-
assert js.days["20240101"]["unprocessed_files"] == 1
53
+
assert (
54
+
js.days["20240101"]["pending_segments"] == 1
55
+
) # Both files belong to same segment
53
56
assert js.topic_counts["meetings"] == 1
54
57
assert js.facet_counts["work"] == 1
55
58
assert js.facet_minutes["work"] == 5.0
+11
-18
think/journal_stats.py
+11
-18
think/journal_stats.py
···
7
7
from pathlib import Path
8
8
from typing import Dict
9
9
10
+
from observe.sense import scan_day as sense_scan_day
10
11
from observe.utils import VIDEO_EXTENSIONS, load_analysis_frames
11
12
from think.insight import scan_day as insight_scan_day
12
13
from think.utils import day_dirs, setup_cli
···
37
38
def _get_day_mtime(self, day_dir: Path) -> float:
38
39
"""Get latest modification time of files we scan."""
39
40
files = []
40
-
# Check timestamp subdirectories for processed files
41
-
files.extend(day_dir.glob("*/audio.jsonl"))
42
-
files.extend(day_dir.glob("*/*_audio.jsonl")) # Split audio files
43
-
files.extend(day_dir.glob("*/screen.jsonl"))
44
-
files.extend(day_dir.glob("*/*_screen.jsonl")) # Split screen files
45
-
files.extend(day_dir.glob("*/raw.flac"))
41
+
# Check segment subdirectories for processed files
42
+
files.extend(day_dir.glob("*/*audio.jsonl"))
43
+
files.extend(day_dir.glob("*/*screen.jsonl"))
44
+
# Check day root for unprocessed media files
45
+
files.extend(day_dir.glob("*.flac"))
46
+
files.extend(day_dir.glob("*.m4a"))
46
47
for ext in VIDEO_EXTENSIONS:
47
-
files.extend(day_dir.glob(f"*/screen{ext}"))
48
-
# Check day root for unprocessed files
49
-
files.extend(day_dir.glob("*_raw.flac"))
50
-
files.extend(day_dir.glob("*_raw.m4a"))
51
-
for ext in VIDEO_EXTENSIONS:
52
-
files.extend(day_dir.glob(f"*_screen{ext}"))
48
+
files.extend(day_dir.glob(f"*{ext}"))
53
49
54
50
insights = day_dir / "insights"
55
51
if insights.is_dir():
···
243
239
except Exception as e:
244
240
logger.warning(f"Unexpected error processing {jsonl_file}: {e}")
245
241
246
-
# --- Unprocessed files ---
247
-
unprocessed = list(day_dir.glob("*_raw.flac"))
248
-
unprocessed.extend(day_dir.glob("*_raw.m4a"))
249
-
for ext in VIDEO_EXTENSIONS:
250
-
unprocessed.extend(day_dir.glob(f"*_screen{ext}"))
251
-
stats["unprocessed_files"] = len(unprocessed)
242
+
# --- Pending segments (unprocessed media files) ---
243
+
sense_info = sense_scan_day(day_dir)
244
+
stats["pending_segments"] = sense_info["pending_segments"]
252
245
253
246
# --- Insight summaries ---
254
247
insight_info = insight_scan_day(day)