personal memory agent
observe/grab: progressive screencast frame extraction CLI
Adds `sol grab` for walking day → stream → segment → screen → frame_id and
optionally writing a frame image (single or batch). Linear-walk decode only
via observe.see.decode_frames (no av.seek, no ffmpeg -ss) — the 1 Hz capture
pipeline produced corruption with random-access decode. Saved frames are raw
pixels: no annotation, no redaction. Format inferred from --out suffix
(.png/.jpg/.jpeg/.webp). Batch mode decodes all frames first, then writes —
zero files on decode failure. observe.utils.load_analysis_frames gains an
opt-in keep_errors=True so level 4 can surface error-marked frames; default
behavior is unchanged for every existing caller.
+1452
-9
+1
docs/OBSERVE.md
+1
docs/OBSERVE.md
···
39
39
| `sol observe-linux` | Screen and audio capture on Linux (direct) |
40
40
| `sol transcribe` | Audio transcription with faster-whisper |
41
41
| `sol describe` | Visual analysis of screen recordings |
42
+
| `sol grab` | Walk available screen frames and optionally write frame images |
42
43
| `sol sense` | Unified observation coordination |
43
44
44
45
## Architecture
+616
observe/grab.py
+616
observe/grab.py
···
1
+
# SPDX-License-Identifier: AGPL-3.0-only
2
+
# Copyright (c) 2026 sol pbc
3
+
4
+
"""Walk observed screen frames in the journal and optionally write frame images."""
5
+
6
+
from __future__ import annotations
7
+
8
+
import argparse
9
+
import json
10
+
from dataclasses import dataclass
11
+
from datetime import datetime, timedelta
12
+
from pathlib import Path
13
+
from typing import Any
14
+
15
+
from observe.see import decode_frames
16
+
from observe.utils import VIDEO_EXTENSIONS, load_analysis_frames, parse_screen_filename
17
+
from think.utils import (
18
+
get_journal,
19
+
journal_relative_path,
20
+
require_solstone,
21
+
segment_parse,
22
+
segment_path,
23
+
setup_cli,
24
+
)
25
+
26
+
SUPPORTED_OUTPUT_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp"}
27
+
TABLE_LEVELS = {
28
+
"0": ("days", ["day", "streams", "segments", "screens", "frames_analyzed"]),
29
+
"1": ("streams", ["stream", "segments", "screens", "frames_analyzed"]),
30
+
"2": ("segments", ["segment", "start", "end", "screens", "frames_analyzed"]),
31
+
"3": ("screens", ["screen", "position", "connector", "frames_analyzed", "status"]),
32
+
}
33
+
34
+
35
+
@dataclass
36
+
class ScreenBundle:
37
+
video_path: Path | None
38
+
jsonl_rel: str | None
39
+
video_rel: str | None
40
+
frame_records: list[dict[str, Any]]
41
+
frame_index: dict[int, dict[str, Any]]
42
+
legacy_schema: bool
43
+
status: str
44
+
segment_start: datetime
45
+
46
+
47
+
def _is_screen_token(stem: str) -> bool:
48
+
return stem == "screen" or stem.endswith("_screen")
49
+
50
+
51
+
def _normalize_screen_token(stem: str) -> str:
52
+
return stem.removesuffix("_screen") if stem != "screen" else stem
53
+
54
+
55
+
def _screen_stem(screen_token: str) -> str:
56
+
return screen_token if _is_screen_token(screen_token) else f"{screen_token}_screen"
57
+
58
+
59
+
def _frame_notes(frame: dict[str, Any]) -> str:
60
+
lines = str(frame.get("error") or "").splitlines()
61
+
error = lines[0].strip() if lines else ""
62
+
if not error:
63
+
return ""
64
+
text = error if len(error) <= 60 else f"{error[:57]}..."
65
+
return f"error: {text}"
66
+
67
+
68
+
def _frame_primary(frame: dict[str, Any]) -> str:
69
+
analysis = frame.get("analysis")
70
+
value = analysis.get("primary") if isinstance(analysis, dict) else None
71
+
return value if isinstance(value, str) else ""
72
+
73
+
74
+
def _frame_abs_time(segment_start: datetime, timestamp: Any) -> str:
75
+
return (segment_start + timedelta(seconds=float(timestamp or 0.0))).isoformat()
76
+
77
+
78
+
def _segment_bounds(day: str, segment: str) -> tuple[datetime, datetime]:
79
+
start_time, end_time = segment_parse(segment)
80
+
if start_time is None or end_time is None:
81
+
raise ValueError(f"segment {segment} is not a valid HHMMSS_LEN key")
82
+
base = datetime.strptime(day, "%Y%m%d").date()
83
+
return (
84
+
datetime.combine(base, start_time),
85
+
datetime.combine(base, end_time),
86
+
)
87
+
88
+
89
+
def _frame_view(segment_start: datetime, frame: dict[str, Any]) -> dict[str, Any]:
90
+
return {
91
+
"frame_id": int(frame["frame_id"]),
92
+
"timestamp": frame.get("timestamp", 0.0),
93
+
"abs_time": _frame_abs_time(segment_start, frame.get("timestamp", 0.0)),
94
+
"primary": _frame_primary(frame),
95
+
"notes": _frame_notes(frame),
96
+
}
97
+
98
+
99
+
def _scope(day: str, stream: str, segment: str, screen_token: str) -> dict[str, str]:
100
+
return {"day": day, "stream": stream, "segment": segment, "screen": screen_token}
101
+
102
+
103
+
def _source(bundle: ScreenBundle) -> dict[str, str | None]:
104
+
return {"jsonl": bundle.jsonl_rel, "video": bundle.video_rel}
105
+
106
+
107
+
def _print_table(columns: list[str], rows: list[dict[str, Any]]) -> None:
108
+
if not rows:
109
+
return
110
+
widths = {
111
+
col: max(len(col), *(len(str(row.get(col, ""))) for row in rows))
112
+
for col in columns
113
+
}
114
+
print(" ".join(col.ljust(widths[col]) for col in columns))
115
+
print(" ".join("-" * widths[col] for col in columns))
116
+
for row in rows:
117
+
print(" ".join(str(row.get(col, "")).ljust(widths[col]) for col in columns))
118
+
119
+
120
+
def _require_day(day: str) -> Path:
121
+
day_dir = Path(get_journal()) / "chronicle" / day
122
+
if not day_dir.is_dir():
123
+
raise FileNotFoundError(f"day {day} not found")
124
+
return day_dir
125
+
126
+
127
+
def _require_stream(day: str, stream: str) -> Path:
128
+
_require_day(day)
129
+
stream_dir = Path(get_journal()) / "chronicle" / day / stream
130
+
if not stream_dir.is_dir():
131
+
raise FileNotFoundError(f"stream {stream} not found in {day}")
132
+
return stream_dir
133
+
134
+
135
+
def _require_segment(day: str, stream: str, segment: str) -> Path:
136
+
_require_stream(day, stream)
137
+
seg_dir = segment_path(day, segment, stream, create=False)
138
+
if not seg_dir.is_dir():
139
+
raise FileNotFoundError(f"segment {segment} not found in {day}/{stream}")
140
+
return seg_dir
141
+
142
+
143
+
def _load_analyzed_bundle(
144
+
day: str, stream: str, segment: str, screen_token: str
145
+
) -> ScreenBundle:
146
+
bundle = load_screen_bundle(day, stream, segment, screen_token, keep_errors=True)
147
+
if bundle.status == "captured but not analyzed":
148
+
raise ValueError(
149
+
f"screen {screen_token} in {segment} is captured but not analyzed"
150
+
)
151
+
if bundle.legacy_schema:
152
+
raise ValueError(
153
+
"screen file uses pre-frame_id schema; frame selection is unavailable"
154
+
)
155
+
return bundle
156
+
157
+
158
+
def load_screen_bundle(
159
+
day: str, stream: str, segment: str, screen_token: str, *, keep_errors: bool
160
+
) -> ScreenBundle:
161
+
segment_dir = _require_segment(day, stream, segment)
162
+
screen_stem = _screen_stem(screen_token)
163
+
jsonl_path = segment_dir / f"{screen_stem}.jsonl"
164
+
header: dict[str, Any] | None = None
165
+
records: list[dict[str, Any]] = []
166
+
167
+
if jsonl_path.is_file():
168
+
records = load_analysis_frames(jsonl_path, keep_errors=keep_errors)
169
+
if records and "raw" in records[0] and "frame_id" not in records[0]:
170
+
header = records[0]
171
+
else:
172
+
jsonl_path = None
173
+
174
+
video_path: Path | None = None
175
+
if header:
176
+
raw_path = header.get("raw")
177
+
if isinstance(raw_path, str) and raw_path.endswith(VIDEO_EXTENSIONS):
178
+
candidate = segment_dir / raw_path
179
+
if candidate.is_file():
180
+
video_path = candidate
181
+
182
+
if video_path is None:
183
+
for ext in VIDEO_EXTENSIONS:
184
+
candidate = segment_dir / f"{screen_stem}{ext}"
185
+
if candidate.is_file():
186
+
video_path = candidate
187
+
break
188
+
189
+
if jsonl_path is None and video_path is not None:
190
+
status = "captured but not analyzed"
191
+
elif jsonl_path is not None:
192
+
status = "analyzed"
193
+
else:
194
+
raise FileNotFoundError(
195
+
f"screen {screen_token} not found in {day}/{stream}/{segment}"
196
+
)
197
+
198
+
frame_records = [record for record in records if "frame_id" in record]
199
+
frame_index = {int(record["frame_id"]): record for record in frame_records}
200
+
legacy_schema = jsonl_path is not None and not frame_records
201
+
202
+
segment_start, _segment_end = _segment_bounds(day, segment)
203
+
204
+
journal = Path(get_journal())
205
+
return ScreenBundle(
206
+
video_path=video_path,
207
+
jsonl_rel=journal_relative_path(journal, jsonl_path) if jsonl_path else None,
208
+
video_rel=journal_relative_path(journal, video_path) if video_path else None,
209
+
frame_records=sorted(frame_records, key=lambda record: int(record["frame_id"])),
210
+
frame_index=frame_index,
211
+
legacy_schema=legacy_schema,
212
+
status=status,
213
+
segment_start=segment_start,
214
+
)
215
+
216
+
217
+
def list_segment_screens(day: str, stream: str, segment: str) -> dict[str, Any]:
218
+
segment_dir = _require_segment(day, stream, segment)
219
+
tokens: set[str] = set()
220
+
for entry in segment_dir.iterdir():
221
+
if not entry.is_file():
222
+
continue
223
+
suffix = entry.suffix.lower()
224
+
if suffix == ".jsonl" and _is_screen_token(entry.stem):
225
+
tokens.add(entry.stem)
226
+
if suffix in VIDEO_EXTENSIONS and _is_screen_token(entry.stem):
227
+
tokens.add(entry.stem)
228
+
229
+
screens = []
230
+
for token in sorted(tokens):
231
+
bundle = load_screen_bundle(day, stream, segment, token, keep_errors=True)
232
+
position, connector = parse_screen_filename(token)
233
+
screens.append(
234
+
{
235
+
"screen": _normalize_screen_token(token),
236
+
"position": position,
237
+
"connector": connector,
238
+
"frames_analyzed": len(bundle.frame_records),
239
+
"jsonl": bundle.jsonl_rel,
240
+
"video": bundle.video_rel,
241
+
"status": bundle.status,
242
+
}
243
+
)
244
+
245
+
return {
246
+
"level": "3",
247
+
"scope": {"day": day, "stream": stream, "segment": segment},
248
+
"data": {"screens": screens},
249
+
}
250
+
251
+
252
+
def list_stream_segments(day: str, stream: str) -> dict[str, Any]:
253
+
stream_dir = _require_stream(day, stream)
254
+
rows = []
255
+
for segment_dir in sorted(p for p in stream_dir.iterdir() if p.is_dir()):
256
+
try:
257
+
start_dt, end_dt = _segment_bounds(day, segment_dir.name)
258
+
except ValueError:
259
+
continue
260
+
screen_payload = list_segment_screens(day, stream, segment_dir.name)
261
+
screens = screen_payload["data"]["screens"]
262
+
if not screens:
263
+
continue
264
+
rows.append(
265
+
{
266
+
"segment": segment_dir.name,
267
+
"start": start_dt.time().isoformat(),
268
+
"end": end_dt.time().isoformat(),
269
+
"screens": len(screens),
270
+
"frames_analyzed": sum(
271
+
int(screen["frames_analyzed"]) for screen in screens
272
+
),
273
+
}
274
+
)
275
+
276
+
return {
277
+
"level": "2",
278
+
"scope": {"day": day, "stream": stream},
279
+
"data": {"segments": rows},
280
+
}
281
+
282
+
283
+
def list_day_streams(day: str) -> dict[str, Any]:
284
+
day_dir = _require_day(day)
285
+
streams = [
286
+
{
287
+
"stream": stream_dir.name,
288
+
"segments": len(segments),
289
+
"screens": sum(int(segment["screens"]) for segment in segments),
290
+
"frames_analyzed": sum(
291
+
int(segment["frames_analyzed"]) for segment in segments
292
+
),
293
+
}
294
+
for stream_dir in sorted(p for p in day_dir.iterdir() if p.is_dir())
295
+
if stream_dir.name != "health"
296
+
if (segments := list_stream_segments(day, stream_dir.name)["data"]["segments"])
297
+
]
298
+
return {
299
+
"level": "1",
300
+
"scope": {"day": day},
301
+
"data": {"streams": sorted(streams, key=lambda stream: str(stream["stream"]))},
302
+
}
303
+
304
+
305
+
def list_available_days() -> dict[str, Any]:
306
+
chronicle_dir = Path(get_journal()) / "chronicle"
307
+
days = (
308
+
[
309
+
{
310
+
"day": day_dir.name,
311
+
"streams": len(streams),
312
+
"segments": sum(int(stream["segments"]) for stream in streams),
313
+
"screens": sum(int(stream["screens"]) for stream in streams),
314
+
"frames_analyzed": sum(
315
+
int(stream["frames_analyzed"]) for stream in streams
316
+
),
317
+
}
318
+
for day_dir in sorted(p for p in chronicle_dir.iterdir() if p.is_dir())
319
+
if len(day_dir.name) == 8 and day_dir.name.isdigit()
320
+
if (streams := list_day_streams(day_dir.name)["data"]["streams"])
321
+
]
322
+
if chronicle_dir.is_dir()
323
+
else []
324
+
)
325
+
return {"level": "0", "scope": {}, "data": {"days": days}}
326
+
327
+
328
+
def list_screen_frames(
329
+
day: str, stream: str, segment: str, screen_token: str
330
+
) -> dict[str, Any]:
331
+
bundle = load_screen_bundle(day, stream, segment, screen_token, keep_errors=True)
332
+
if bundle.status == "captured but not analyzed":
333
+
raise ValueError(
334
+
f"screen {screen_token} in {segment} is captured but not analyzed"
335
+
)
336
+
frames = (
337
+
[]
338
+
if bundle.legacy_schema
339
+
else [
340
+
_frame_view(bundle.segment_start, frame) for frame in bundle.frame_records
341
+
]
342
+
)
343
+
error_frames = (
344
+
0
345
+
if bundle.legacy_schema
346
+
else sum(1 for frame in bundle.frame_records if "error" in frame)
347
+
)
348
+
349
+
return {
350
+
"level": "4",
351
+
"scope": _scope(day, stream, segment, screen_token),
352
+
"data": {
353
+
"summary": {
354
+
"frames_analyzed": len(bundle.frame_records),
355
+
"error_frames": error_frames,
356
+
"legacy_schema": bundle.legacy_schema,
357
+
},
358
+
"frames": frames,
359
+
},
360
+
}
361
+
362
+
363
+
def show_frame_metadata(
364
+
day: str, stream: str, segment: str, screen_token: str, frame_id: int
365
+
) -> dict[str, Any]:
366
+
bundle = _load_analyzed_bundle(day, stream, segment, screen_token)
367
+
if frame_id not in bundle.frame_index:
368
+
raise FileNotFoundError(
369
+
f"frame id {frame_id} not found in {screen_token} for {segment}"
370
+
)
371
+
372
+
frame = bundle.frame_index[frame_id]
373
+
return {
374
+
"level": "5a",
375
+
"scope": _scope(day, stream, segment, screen_token) | {"frame_id": frame_id},
376
+
"data": {
377
+
"source": _source(bundle),
378
+
"frame": frame,
379
+
"computed": {
380
+
"abs_time": _frame_abs_time(
381
+
bundle.segment_start, frame.get("timestamp", 0.0)
382
+
),
383
+
"notes": _frame_notes(frame),
384
+
},
385
+
},
386
+
}
387
+
388
+
389
+
def parse_frame_id_token(token: str) -> list[int]:
390
+
parts = [part.strip() for part in token.split(",")]
391
+
if any(not part for part in parts):
392
+
raise ValueError(f"frame ids must be positive integers: got '{token}'")
393
+
394
+
frame_ids: list[int] = []
395
+
seen: set[int] = set()
396
+
for part in parts:
397
+
try:
398
+
frame_id = int(part)
399
+
except ValueError as exc:
400
+
raise ValueError(
401
+
f"frame ids must be positive integers: got '{token}'"
402
+
) from exc
403
+
if frame_id < 1:
404
+
raise ValueError(f"frame ids must be positive integers: got '{token}'")
405
+
if frame_id in seen:
406
+
raise ValueError(f"frame ids must be unique: {frame_id}")
407
+
seen.add(frame_id)
408
+
frame_ids.append(frame_id)
409
+
return sorted(frame_ids)
410
+
411
+
412
+
def resolve_output_paths(out_path: str, frame_ids: list[int]) -> list[Path]:
413
+
target = Path(out_path)
414
+
if target.suffix.lower() not in SUPPORTED_OUTPUT_SUFFIXES:
415
+
raise ValueError("--out must end in .png, .jpg, .jpeg, or .webp")
416
+
if len(frame_ids) == 1:
417
+
return [target]
418
+
return [
419
+
target.with_name(f"{target.stem}_{frame_id}{target.suffix}")
420
+
for frame_id in frame_ids
421
+
]
422
+
423
+
424
+
def save_frame_images(
425
+
day: str,
426
+
stream: str,
427
+
segment: str,
428
+
screen_token: str,
429
+
frame_ids: list[int],
430
+
out_path: str,
431
+
force: bool,
432
+
) -> dict[str, Any]:
433
+
bundle = _load_analyzed_bundle(day, stream, segment, screen_token)
434
+
if bundle.video_path is None:
435
+
raise FileNotFoundError(
436
+
f"raw video not found for screen {screen_token} in {segment}"
437
+
)
438
+
439
+
selected = []
440
+
for frame_id in frame_ids:
441
+
if frame_id not in bundle.frame_index:
442
+
raise FileNotFoundError(
443
+
f"frame id {frame_id} not found in {screen_token} for {segment}"
444
+
)
445
+
selected.append(bundle.frame_index[frame_id])
446
+
447
+
output_paths = resolve_output_paths(out_path, frame_ids)
448
+
conflicts = [str(path) for path in output_paths if path.exists() and not force]
449
+
if conflicts:
450
+
joined = ", ".join(conflicts)
451
+
raise FileExistsError(f"output path exists (use --force): {joined}")
452
+
453
+
images = decode_frames(bundle.video_path, selected, annotate_boxes=False)
454
+
missing = [
455
+
frame_id
456
+
for frame_id, image in zip(frame_ids, images, strict=True)
457
+
if image is None
458
+
]
459
+
if missing:
460
+
raise RuntimeError(
461
+
f"failed to decode frame ids: {', '.join(str(i) for i in missing)}"
462
+
)
463
+
464
+
saved = []
465
+
try:
466
+
for frame, image, target in zip(selected, images, output_paths, strict=True):
467
+
suffix = target.suffix.lower()
468
+
save_kwargs = (
469
+
{"quality": 95}
470
+
if suffix in {".jpg", ".jpeg"}
471
+
else {"quality": 90}
472
+
if suffix == ".webp"
473
+
else {}
474
+
)
475
+
assert image is not None
476
+
image.save(target, **save_kwargs)
477
+
saved.append(
478
+
{"path": str(target), **_frame_view(bundle.segment_start, frame)}
479
+
)
480
+
finally:
481
+
for image in images:
482
+
if image is not None:
483
+
image.close()
484
+
485
+
return {
486
+
"level": "5b" if len(frame_ids) == 1 else "5c",
487
+
"scope": _scope(day, stream, segment, screen_token) | {"frame_ids": frame_ids},
488
+
"data": {
489
+
"source": _source(bundle),
490
+
"saved": saved,
491
+
},
492
+
}
493
+
494
+
495
+
def emit_output(payload: dict[str, Any], *, as_json: bool) -> None:
496
+
if as_json:
497
+
print(json.dumps(payload, indent=2))
498
+
return
499
+
500
+
level = payload["level"]
501
+
data = payload["data"]
502
+
if level in TABLE_LEVELS:
503
+
key, columns = TABLE_LEVELS[level]
504
+
_print_table(columns, data[key])
505
+
return
506
+
if level == "4":
507
+
if data["summary"]["legacy_schema"]:
508
+
print("0 frames analyzed: file uses pre-frame_id schema")
509
+
else:
510
+
_print_table(
511
+
["frame_id", "timestamp", "abs_time", "primary", "notes"],
512
+
data["frames"],
513
+
)
514
+
return
515
+
if level == "5a":
516
+
scope = payload["scope"]
517
+
computed = data["computed"]
518
+
for label, value in (
519
+
("Screen", scope["screen"]),
520
+
("JSONL", data["source"]["jsonl"]),
521
+
("Video", data["source"]["video"]),
522
+
("Frame", scope["frame_id"]),
523
+
("Time", computed["abs_time"]),
524
+
):
525
+
print(f"{label}: {value}")
526
+
if computed["notes"]:
527
+
print(f"Notes: {computed['notes']}")
528
+
print()
529
+
print(json.dumps(data["frame"], indent=2))
530
+
return
531
+
if level in {"5b", "5c"}:
532
+
for item in data["saved"]:
533
+
print(f"saved {item['path']}")
534
+
return
535
+
536
+
raise ValueError(f"unsupported output level: {level}")
537
+
538
+
539
+
def main() -> None:
540
+
parser = argparse.ArgumentParser(
541
+
description="Walk observed screen frames and optionally write frame images."
542
+
)
543
+
parser.add_argument(
544
+
"args",
545
+
nargs="*",
546
+
help="Path tokens: [day] [stream] [segment] [screen] [frame-id[,frame-id...]]",
547
+
)
548
+
parser.add_argument(
549
+
"--out",
550
+
type=str,
551
+
help="Write the selected frame image here (.png, .jpg, .jpeg, or .webp).",
552
+
)
553
+
parser.add_argument(
554
+
"--force",
555
+
action="store_true",
556
+
help="Replace an existing output path.",
557
+
)
558
+
parser.add_argument(
559
+
"--json",
560
+
action="store_true",
561
+
help="Emit JSON instead of table or plain output.",
562
+
)
563
+
args = setup_cli(parser)
564
+
require_solstone()
565
+
566
+
tokens = list(args.args)
567
+
if len(tokens) > 5:
568
+
parser.error(
569
+
"grab accepts at most 5 positional tokens: day stream segment screen frame-id"
570
+
)
571
+
if args.force and not args.out:
572
+
parser.error("--force requires --out")
573
+
if args.out and len(tokens) != 5:
574
+
parser.error("--out requires day stream segment screen and frame-id")
575
+
if args.out:
576
+
try:
577
+
resolve_output_paths(args.out, [1])
578
+
except ValueError as exc:
579
+
parser.error(str(exc))
580
+
581
+
list_handlers = {
582
+
0: lambda: list_available_days(),
583
+
1: lambda: list_day_streams(tokens[0]),
584
+
2: lambda: list_stream_segments(tokens[0], tokens[1]),
585
+
3: lambda: list_segment_screens(tokens[0], tokens[1], tokens[2]),
586
+
4: lambda: list_screen_frames(tokens[0], tokens[1], tokens[2], tokens[3]),
587
+
}
588
+
try:
589
+
if len(tokens) < 5:
590
+
payload = list_handlers[len(tokens)]()
591
+
else:
592
+
frame_ids = parse_frame_id_token(tokens[4])
593
+
if len(frame_ids) > 1 and not args.out:
594
+
parser.error("multiple frame ids require --out")
595
+
if args.out:
596
+
payload = save_frame_images(
597
+
tokens[0],
598
+
tokens[1],
599
+
tokens[2],
600
+
tokens[3],
601
+
frame_ids,
602
+
args.out,
603
+
args.force,
604
+
)
605
+
else:
606
+
payload = show_frame_metadata(
607
+
tokens[0], tokens[1], tokens[2], tokens[3], frame_ids[0]
608
+
)
609
+
except (FileNotFoundError, FileExistsError, RuntimeError, ValueError) as exc:
610
+
raise SystemExit(str(exc)) from exc
611
+
612
+
emit_output(payload, as_json=bool(args.json))
613
+
614
+
615
+
if __name__ == "__main__":
616
+
main()
+16
-9
observe/utils.py
+16
-9
observe/utils.py
···
360
360
return monitors
361
361
362
362
363
-
def load_analysis_frames(jsonl_path: Path) -> list[dict]:
363
+
def load_analysis_frames(jsonl_path: Path, *, keep_errors: bool = False) -> list[dict]:
364
364
"""
365
-
Load and parse analysis JSONL, filtering out error frames.
365
+
Load and parse analysis JSONL, with optional error-frame retention.
366
366
367
367
The first line is a header with metadata (e.g., {"raw": "path"}).
368
368
Subsequent frames are sorted by frame_id before being returned.
···
371
371
----------
372
372
jsonl_path : Path
373
373
Path to analysis JSONL file
374
+
keep_errors : bool, optional
375
+
When True, include error records that have a ``frame_id`` in the
376
+
returned frame list. Defaults to False.
374
377
375
378
Returns
376
379
-------
377
380
list[dict]
378
-
List of valid frame analysis results, with header first and frames sorted by frame_id
381
+
List of frame analysis results, with header first and frames sorted by
382
+
frame_id. Error records are excluded unless ``keep_errors`` is True.
379
383
"""
380
384
header = None
381
385
frames = []
···
387
391
continue
388
392
try:
389
393
frame = json.loads(line)
390
-
# Skip frames with errors
391
-
if "error" not in frame:
392
-
# First line without frame_id is the header
393
-
if "frame_id" not in frame and header is None:
394
-
header = frame
395
-
else:
394
+
if "error" in frame:
395
+
if keep_errors and "frame_id" in frame:
396
396
frames.append(frame)
397
+
continue
398
+
399
+
# First line without frame_id is the header
400
+
if "frame_id" not in frame and header is None:
401
+
header = frame
402
+
else:
403
+
frames.append(frame)
397
404
except json.JSONDecodeError as e:
398
405
logger.warning(
399
406
f"Invalid JSON at line {line_num} in {jsonl_path}: {e}"
+6
talent/journal/references/captures.md
+6
talent/journal/references/captures.md
···
119
119
120
120
Captures are the original binary media files recorded by observation tools.
121
121
122
+
`sol grab` walks observed screens from day to stream to segment to screen to frame.
123
+
Without `--out` it lists what is available or shows one frame's details.
124
+
With `--out` it writes one or more frame images using the suffix you choose.
125
+
Use bare `screen` for single-screen segments.
126
+
Use stems like `center_DP-3_screen` for per-monitor segments.
127
+
122
128
### Audio captures
123
129
124
130
Audio files are initially written to the day root with the segment key prefix (Linux) or directly to segment folders (macOS):
+3
tests/fixtures/sol_grab/journal/chronicle/20240101/default/123456_300/screen.jsonl
+3
tests/fixtures/sol_grab/journal/chronicle/20240101/default/123456_300/screen.jsonl
tests/fixtures/sol_grab/journal/chronicle/20240101/default/123456_300/screen.webm
tests/fixtures/sol_grab/journal/chronicle/20240101/default/123456_300/screen.webm
This is a binary file and will not be displayed.
+1
tests/fixtures/sol_grab/journal/chronicle/20240101/default/123456_300/stream.json
+1
tests/fixtures/sol_grab/journal/chronicle/20240101/default/123456_300/stream.json
···
1
+
{"stream": "default", "prev_day": null, "prev_segment": null, "seq": 1}
+10
tests/fixtures/sol_grab/journal/chronicle/20240102/default/233000_300/screen.jsonl
+10
tests/fixtures/sol_grab/journal/chronicle/20240102/default/233000_300/screen.jsonl
···
1
+
{"raw": "screen.webm"}
2
+
{"frame_id": 1, "timestamp": 0.0, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.4}], "analysis": {"visual_description": "Editor with a service module open.", "primary": "code", "secondary": "none", "overlap": true}}
3
+
{"frame_id": 2, "timestamp": 0.166667, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.3}], "analysis": {"visual_description": "Terminal showing a green test run.", "primary": "terminal", "secondary": "none", "overlap": true}}
4
+
{"frame_id": 3, "timestamp": 0.333333, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.4}], "analysis": {"visual_description": "Reference page open in a browser.", "primary": "reading", "secondary": "none", "overlap": true}}
5
+
{"frame_id": 5, "timestamp": 0.666667, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.5}], "analysis": {"visual_description": "Team chat window with a release checklist.", "primary": "messaging", "secondary": "none", "overlap": true}}
6
+
{"frame_id": 7, "timestamp": 1.0, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.4}], "analysis": {"visual_description": "Planning board with sprint tasks.", "primary": "productivity", "secondary": "none", "overlap": true}}
7
+
{"frame_id": 12, "timestamp": 1.833333, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.4}], "analysis": {"visual_description": "Meeting window with shared notes.", "primary": "meeting", "secondary": "none", "overlap": true}}
8
+
{"frame_id": 18, "timestamp": 2.833333, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.7}], "error": "Vision request timed out while describing frame 18.\nRetry budget exhausted after 5 attempts."}
9
+
{"frame_id": 23, "timestamp": 3.666667, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.4}], "analysis": {"visual_description": "Docs page with API examples.", "primary": "browsing", "secondary": "none", "overlap": true}}
10
+
{"frame_id": 24, "timestamp": 3.833333, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.5}], "analysis": {"visual_description": "Music player next to a browser tab.", "primary": "media", "secondary": "browsing", "overlap": false}}
tests/fixtures/sol_grab/journal/chronicle/20240102/default/233000_300/screen.webm
tests/fixtures/sol_grab/journal/chronicle/20240102/default/233000_300/screen.webm
This is a binary file and will not be displayed.
+1
tests/fixtures/sol_grab/journal/chronicle/20240102/default/233000_300/stream.json
+1
tests/fixtures/sol_grab/journal/chronicle/20240102/default/233000_300/stream.json
···
1
+
{"stream": "default", "prev_day": "20240101", "prev_segment": "123456_300", "seq": 2}
+4
tests/fixtures/sol_grab/journal/chronicle/20240103/default/100000_300/left_DP-1_screen.jsonl
+4
tests/fixtures/sol_grab/journal/chronicle/20240103/default/100000_300/left_DP-1_screen.jsonl
···
1
+
{"raw": "left_DP-1_screen.webm"}
2
+
{"frame_id": 1, "timestamp": 0.0, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.3}], "analysis": {"visual_description": "Left monitor with editor tabs.", "primary": "code", "secondary": "none", "overlap": true}}
3
+
{"frame_id": 4, "timestamp": 0.5, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.3}], "analysis": {"visual_description": "Left monitor terminal output.", "primary": "terminal", "secondary": "none", "overlap": true}}
4
+
{"frame_id": 9, "timestamp": 1.333333, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.4}], "analysis": {"visual_description": "Left monitor notes window.", "primary": "reading", "secondary": "none", "overlap": true}}
tests/fixtures/sol_grab/journal/chronicle/20240103/default/100000_300/left_DP-1_screen.webm
tests/fixtures/sol_grab/journal/chronicle/20240103/default/100000_300/left_DP-1_screen.webm
This is a binary file and will not be displayed.
+4
tests/fixtures/sol_grab/journal/chronicle/20240103/default/100000_300/right_HDMI-1_screen.jsonl
+4
tests/fixtures/sol_grab/journal/chronicle/20240103/default/100000_300/right_HDMI-1_screen.jsonl
···
1
+
{"raw": "right_HDMI-1_screen.webm"}
2
+
{"frame_id": 2, "timestamp": 0.166667, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.3}], "analysis": {"visual_description": "Right monitor calendar view.", "primary": "productivity", "secondary": "none", "overlap": true}}
3
+
{"frame_id": 6, "timestamp": 0.833333, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.4}], "analysis": {"visual_description": "Right monitor browser dashboard.", "primary": "browsing", "secondary": "none", "overlap": true}}
4
+
{"frame_id": 10, "timestamp": 1.5, "requests": [{"type": "describe", "model": "gemini-2.5-flash-lite", "duration": 0.4}], "analysis": {"visual_description": "Right monitor chat thread.", "primary": "messaging", "secondary": "none", "overlap": true}}
tests/fixtures/sol_grab/journal/chronicle/20240103/default/100000_300/right_HDMI-1_screen.webm
tests/fixtures/sol_grab/journal/chronicle/20240103/default/100000_300/right_HDMI-1_screen.webm
This is a binary file and will not be displayed.
+1
tests/fixtures/sol_grab/journal/chronicle/20240103/default/100000_300/stream.json
+1
tests/fixtures/sol_grab/journal/chronicle/20240103/default/100000_300/stream.json
···
1
+
{"stream": "default", "prev_day": "20240102", "prev_segment": "233000_300", "seq": 3}
tests/fixtures/sol_grab/journal/chronicle/20240103/default/110000_300/screen.webm
tests/fixtures/sol_grab/journal/chronicle/20240103/default/110000_300/screen.webm
This is a binary file and will not be displayed.
+1
tests/fixtures/sol_grab/journal/chronicle/20240103/default/110000_300/stream.json
+1
tests/fixtures/sol_grab/journal/chronicle/20240103/default/110000_300/stream.json
···
1
+
{"stream": "default", "prev_day": "20240103", "prev_segment": "100000_300", "seq": 4}
+29
tests/fixtures/sol_grab/level_0.json
+29
tests/fixtures/sol_grab/level_0.json
···
1
+
{
2
+
"level": "0",
3
+
"scope": {},
4
+
"data": {
5
+
"days": [
6
+
{
7
+
"day": "20240101",
8
+
"streams": 1,
9
+
"segments": 1,
10
+
"screens": 1,
11
+
"frames_analyzed": 0
12
+
},
13
+
{
14
+
"day": "20240102",
15
+
"streams": 1,
16
+
"segments": 1,
17
+
"screens": 1,
18
+
"frames_analyzed": 9
19
+
},
20
+
{
21
+
"day": "20240103",
22
+
"streams": 1,
23
+
"segments": 2,
24
+
"screens": 3,
25
+
"frames_analyzed": 6
26
+
}
27
+
]
28
+
}
29
+
}
+16
tests/fixtures/sol_grab/level_1.json
+16
tests/fixtures/sol_grab/level_1.json
+25
tests/fixtures/sol_grab/level_2.json
+25
tests/fixtures/sol_grab/level_2.json
···
1
+
{
2
+
"level": "2",
3
+
"scope": {
4
+
"day": "20240103",
5
+
"stream": "default"
6
+
},
7
+
"data": {
8
+
"segments": [
9
+
{
10
+
"segment": "100000_300",
11
+
"start": "10:00:00",
12
+
"end": "10:05:00",
13
+
"screens": 2,
14
+
"frames_analyzed": 6
15
+
},
16
+
{
17
+
"segment": "110000_300",
18
+
"start": "11:00:00",
19
+
"end": "11:05:00",
20
+
"screens": 1,
21
+
"frames_analyzed": 0
22
+
}
23
+
]
24
+
}
25
+
}
+21
tests/fixtures/sol_grab/level_3.json
+21
tests/fixtures/sol_grab/level_3.json
···
1
+
{
2
+
"level": "3",
3
+
"scope": {
4
+
"day": "20240103",
5
+
"stream": "default",
6
+
"segment": "110000_300"
7
+
},
8
+
"data": {
9
+
"screens": [
10
+
{
11
+
"screen": "screen",
12
+
"position": "unknown",
13
+
"connector": "unknown",
14
+
"frames_analyzed": 0,
15
+
"jsonl": null,
16
+
"video": "20240103/default/110000_300/screen.webm",
17
+
"status": "captured but not analyzed"
18
+
}
19
+
]
20
+
}
21
+
}
+81
tests/fixtures/sol_grab/level_4.json
+81
tests/fixtures/sol_grab/level_4.json
···
1
+
{
2
+
"level": "4",
3
+
"scope": {
4
+
"day": "20240102",
5
+
"stream": "default",
6
+
"segment": "233000_300",
7
+
"screen": "screen"
8
+
},
9
+
"data": {
10
+
"summary": {
11
+
"frames_analyzed": 9,
12
+
"error_frames": 1,
13
+
"legacy_schema": false
14
+
},
15
+
"frames": [
16
+
{
17
+
"frame_id": 1,
18
+
"timestamp": 0.0,
19
+
"abs_time": "2024-01-02T23:30:00",
20
+
"primary": "code",
21
+
"notes": ""
22
+
},
23
+
{
24
+
"frame_id": 2,
25
+
"timestamp": 0.166667,
26
+
"abs_time": "2024-01-02T23:30:00.166667",
27
+
"primary": "terminal",
28
+
"notes": ""
29
+
},
30
+
{
31
+
"frame_id": 3,
32
+
"timestamp": 0.333333,
33
+
"abs_time": "2024-01-02T23:30:00.333333",
34
+
"primary": "reading",
35
+
"notes": ""
36
+
},
37
+
{
38
+
"frame_id": 5,
39
+
"timestamp": 0.666667,
40
+
"abs_time": "2024-01-02T23:30:00.666667",
41
+
"primary": "messaging",
42
+
"notes": ""
43
+
},
44
+
{
45
+
"frame_id": 7,
46
+
"timestamp": 1.0,
47
+
"abs_time": "2024-01-02T23:30:01",
48
+
"primary": "productivity",
49
+
"notes": ""
50
+
},
51
+
{
52
+
"frame_id": 12,
53
+
"timestamp": 1.833333,
54
+
"abs_time": "2024-01-02T23:30:01.833333",
55
+
"primary": "meeting",
56
+
"notes": ""
57
+
},
58
+
{
59
+
"frame_id": 18,
60
+
"timestamp": 2.833333,
61
+
"abs_time": "2024-01-02T23:30:02.833333",
62
+
"primary": "",
63
+
"notes": "error: Vision request timed out while describing frame 18."
64
+
},
65
+
{
66
+
"frame_id": 23,
67
+
"timestamp": 3.666667,
68
+
"abs_time": "2024-01-02T23:30:03.666667",
69
+
"primary": "browsing",
70
+
"notes": ""
71
+
},
72
+
{
73
+
"frame_id": 24,
74
+
"timestamp": 3.833333,
75
+
"abs_time": "2024-01-02T23:30:03.833333",
76
+
"primary": "media",
77
+
"notes": ""
78
+
}
79
+
]
80
+
}
81
+
}
+37
tests/fixtures/sol_grab/level_5a.json
+37
tests/fixtures/sol_grab/level_5a.json
···
1
+
{
2
+
"level": "5a",
3
+
"scope": {
4
+
"day": "20240102",
5
+
"stream": "default",
6
+
"segment": "233000_300",
7
+
"screen": "screen",
8
+
"frame_id": 7
9
+
},
10
+
"data": {
11
+
"source": {
12
+
"jsonl": "20240102/default/233000_300/screen.jsonl",
13
+
"video": "20240102/default/233000_300/screen.webm"
14
+
},
15
+
"frame": {
16
+
"frame_id": 7,
17
+
"timestamp": 1.0,
18
+
"requests": [
19
+
{
20
+
"type": "describe",
21
+
"model": "gemini-2.5-flash-lite",
22
+
"duration": 0.4
23
+
}
24
+
],
25
+
"analysis": {
26
+
"visual_description": "Planning board with sprint tasks.",
27
+
"primary": "productivity",
28
+
"secondary": "none",
29
+
"overlap": true
30
+
}
31
+
},
32
+
"computed": {
33
+
"abs_time": "2024-01-02T23:30:01",
34
+
"notes": ""
35
+
}
36
+
}
37
+
}
+28
tests/fixtures/sol_grab/level_5b.json
+28
tests/fixtures/sol_grab/level_5b.json
···
1
+
{
2
+
"level": "5b",
3
+
"scope": {
4
+
"day": "20240102",
5
+
"stream": "default",
6
+
"segment": "233000_300",
7
+
"screen": "screen",
8
+
"frame_ids": [
9
+
7
10
+
]
11
+
},
12
+
"data": {
13
+
"source": {
14
+
"jsonl": "20240102/default/233000_300/screen.jsonl",
15
+
"video": "20240102/default/233000_300/screen.webm"
16
+
},
17
+
"saved": [
18
+
{
19
+
"path": "/home/jer/.local/share/hopper/lodes/hyxf4ukp/worktree/tests/fixtures/sol_grab/_out/frame.png",
20
+
"frame_id": 7,
21
+
"timestamp": 1.0,
22
+
"abs_time": "2024-01-02T23:30:01",
23
+
"primary": "productivity",
24
+
"notes": ""
25
+
}
26
+
]
27
+
}
28
+
}
+46
tests/fixtures/sol_grab/level_5c.json
+46
tests/fixtures/sol_grab/level_5c.json
···
1
+
{
2
+
"level": "5c",
3
+
"scope": {
4
+
"day": "20240102",
5
+
"stream": "default",
6
+
"segment": "233000_300",
7
+
"screen": "screen",
8
+
"frame_ids": [
9
+
7,
10
+
12,
11
+
23
12
+
]
13
+
},
14
+
"data": {
15
+
"source": {
16
+
"jsonl": "20240102/default/233000_300/screen.jsonl",
17
+
"video": "20240102/default/233000_300/screen.webm"
18
+
},
19
+
"saved": [
20
+
{
21
+
"path": "/home/jer/.local/share/hopper/lodes/hyxf4ukp/worktree/tests/fixtures/sol_grab/_out/frame_7.png",
22
+
"frame_id": 7,
23
+
"timestamp": 1.0,
24
+
"abs_time": "2024-01-02T23:30:01",
25
+
"primary": "productivity",
26
+
"notes": ""
27
+
},
28
+
{
29
+
"path": "/home/jer/.local/share/hopper/lodes/hyxf4ukp/worktree/tests/fixtures/sol_grab/_out/frame_12.png",
30
+
"frame_id": 12,
31
+
"timestamp": 1.833333,
32
+
"abs_time": "2024-01-02T23:30:01.833333",
33
+
"primary": "meeting",
34
+
"notes": ""
35
+
},
36
+
{
37
+
"path": "/home/jer/.local/share/hopper/lodes/hyxf4ukp/worktree/tests/fixtures/sol_grab/_out/frame_23.png",
38
+
"frame_id": 23,
39
+
"timestamp": 3.666667,
40
+
"abs_time": "2024-01-02T23:30:03.666667",
41
+
"primary": "browsing",
42
+
"notes": ""
43
+
}
44
+
]
45
+
}
46
+
}
+459
tests/test_grab.py
+459
tests/test_grab.py
···
1
+
# SPDX-License-Identifier: AGPL-3.0-only
2
+
# Copyright (c) 2026 sol pbc
3
+
4
+
"""Tests for the sol grab CLI."""
5
+
6
+
from __future__ import annotations
7
+
8
+
import json
9
+
from pathlib import Path
10
+
from unittest.mock import Mock
11
+
12
+
import pytest
13
+
from PIL import Image
14
+
15
+
FIXTURE_ROOT = Path(__file__).parent / "fixtures" / "sol_grab"
16
+
FIXTURE_JOURNAL = FIXTURE_ROOT / "journal"
17
+
18
+
19
+
def _expected(name: str) -> dict:
20
+
return json.loads((FIXTURE_ROOT / name).read_text(encoding="utf-8"))
21
+
22
+
23
+
def _invoke_grab(monkeypatch, capsys, *argv: str):
24
+
monkeypatch.setenv("SOLSTONE_JOURNAL", str(FIXTURE_JOURNAL))
25
+
monkeypatch.setenv("SOL_SKIP_SUPERVISOR_CHECK", "1")
26
+
monkeypatch.setattr("sys.argv", ["sol grab", *argv])
27
+
28
+
from observe.grab import main
29
+
30
+
exit_code = 0
31
+
exit_message = ""
32
+
try:
33
+
main()
34
+
except SystemExit as exc:
35
+
if isinstance(exc.code, int):
36
+
exit_code = exc.code
37
+
elif exc.code is None:
38
+
exit_code = 0
39
+
else:
40
+
exit_code = 1
41
+
exit_message = str(exc.code)
42
+
captured = capsys.readouterr()
43
+
return exit_code, exit_message, captured.out, captured.err
44
+
45
+
46
+
def _normalize_saved_paths(actual: dict, expected: dict) -> dict:
47
+
normalized = json.loads(json.dumps(actual))
48
+
for actual_item, expected_item in zip(
49
+
normalized["data"]["saved"], expected["data"]["saved"], strict=True
50
+
):
51
+
actual_item["path"] = expected_item["path"]
52
+
return normalized
53
+
54
+
55
+
def test_grab_level_0_json_matches_fixture(monkeypatch, capsys):
56
+
code, message, out, err = _invoke_grab(monkeypatch, capsys, "--json")
57
+
assert code == 0
58
+
assert message == ""
59
+
assert err == ""
60
+
assert json.loads(out) == _expected("level_0.json")
61
+
62
+
63
+
def test_grab_level_0_human_lists_days_with_counts(monkeypatch, capsys):
64
+
code, message, out, err = _invoke_grab(monkeypatch, capsys)
65
+
assert code == 0
66
+
assert message == ""
67
+
assert err == ""
68
+
assert "day" in out
69
+
assert "20240102" in out
70
+
assert "20240103" in out
71
+
72
+
73
+
def test_grab_level_1_json_matches_fixture(monkeypatch, capsys):
74
+
code, message, out, err = _invoke_grab(monkeypatch, capsys, "--json", "20240102")
75
+
assert code == 0
76
+
assert message == ""
77
+
assert err == ""
78
+
assert json.loads(out) == _expected("level_1.json")
79
+
80
+
81
+
def test_grab_missing_day_errors(monkeypatch, capsys):
82
+
code, message, out, err = _invoke_grab(monkeypatch, capsys, "20990101")
83
+
assert code == 1
84
+
assert out == ""
85
+
assert err == ""
86
+
assert message == "day 20990101 not found"
87
+
88
+
89
+
def test_grab_level_2_json_matches_fixture(monkeypatch, capsys):
90
+
code, message, out, err = _invoke_grab(
91
+
monkeypatch, capsys, "--json", "20240103", "default"
92
+
)
93
+
assert code == 0
94
+
assert message == ""
95
+
assert err == ""
96
+
assert json.loads(out) == _expected("level_2.json")
97
+
98
+
99
+
def test_grab_missing_stream_errors(monkeypatch, capsys):
100
+
code, message, out, err = _invoke_grab(monkeypatch, capsys, "20240102", "missing")
101
+
assert code == 1
102
+
assert out == ""
103
+
assert err == ""
104
+
assert message == "stream missing not found in 20240102"
105
+
106
+
107
+
def test_grab_level_3_json_matches_fixture(monkeypatch, capsys):
108
+
code, message, out, err = _invoke_grab(
109
+
monkeypatch, capsys, "--json", "20240103", "default", "110000_300"
110
+
)
111
+
assert code == 0
112
+
assert message == ""
113
+
assert err == ""
114
+
assert json.loads(out) == _expected("level_3.json")
115
+
116
+
117
+
def test_grab_level_3_lists_named_monitors(monkeypatch, capsys):
118
+
code, message, out, err = _invoke_grab(
119
+
monkeypatch, capsys, "--json", "20240103", "default", "100000_300"
120
+
)
121
+
payload = json.loads(out)
122
+
assert code == 0
123
+
assert message == ""
124
+
assert err == ""
125
+
assert [screen["screen"] for screen in payload["data"]["screens"]] == [
126
+
"left_DP-1",
127
+
"right_HDMI-1",
128
+
]
129
+
assert payload["data"]["screens"][0]["position"] == "left"
130
+
assert payload["data"]["screens"][1]["connector"] == "HDMI-1"
131
+
132
+
133
+
def test_grab_missing_segment_errors(monkeypatch, capsys):
134
+
code, message, out, err = _invoke_grab(
135
+
monkeypatch, capsys, "20240103", "default", "999999_300"
136
+
)
137
+
assert code == 1
138
+
assert out == ""
139
+
assert err == ""
140
+
assert message == "segment 999999_300 not found in 20240103/default"
141
+
142
+
143
+
def test_grab_level_4_json_matches_fixture(monkeypatch, capsys):
144
+
code, message, out, err = _invoke_grab(
145
+
monkeypatch, capsys, "--json", "20240102", "default", "233000_300", "screen"
146
+
)
147
+
assert code == 0
148
+
assert message == ""
149
+
assert err == ""
150
+
assert json.loads(out) == _expected("level_4.json")
151
+
152
+
153
+
def test_grab_level_4_human_includes_error_notes(monkeypatch, capsys):
154
+
code, message, out, err = _invoke_grab(
155
+
monkeypatch, capsys, "20240102", "default", "233000_300", "screen"
156
+
)
157
+
assert code == 0
158
+
assert message == ""
159
+
assert err == ""
160
+
assert "frame_id" in out
161
+
assert "error: Vision request timed out while describing frame 18." in out
162
+
163
+
164
+
def test_grab_level_4_legacy_schema_reports_zero_frames(monkeypatch, capsys):
165
+
code, message, out, err = _invoke_grab(
166
+
monkeypatch, capsys, "20240101", "default", "123456_300", "screen"
167
+
)
168
+
assert code == 0
169
+
assert message == ""
170
+
assert err == ""
171
+
assert out.strip() == "0 frames analyzed: file uses pre-frame_id schema"
172
+
173
+
174
+
def test_grab_level_4_captured_but_not_analyzed_errors(monkeypatch, capsys):
175
+
code, message, out, err = _invoke_grab(
176
+
monkeypatch, capsys, "20240103", "default", "110000_300", "screen"
177
+
)
178
+
assert code == 1
179
+
assert out == ""
180
+
assert err == ""
181
+
assert message == "screen screen in 110000_300 is captured but not analyzed"
182
+
183
+
184
+
def test_grab_missing_screen_errors(monkeypatch, capsys):
185
+
code, message, out, err = _invoke_grab(
186
+
monkeypatch, capsys, "20240102", "default", "233000_300", "missing_screen"
187
+
)
188
+
assert code == 1
189
+
assert out == ""
190
+
assert err == ""
191
+
assert message == "screen missing_screen not found in 20240102/default/233000_300"
192
+
193
+
194
+
def test_grab_level_5a_json_matches_fixture(monkeypatch, capsys):
195
+
code, message, out, err = _invoke_grab(
196
+
monkeypatch,
197
+
capsys,
198
+
"--json",
199
+
"20240102",
200
+
"default",
201
+
"233000_300",
202
+
"screen",
203
+
"7",
204
+
)
205
+
assert code == 0
206
+
assert message == ""
207
+
assert err == ""
208
+
assert json.loads(out) == _expected("level_5a.json")
209
+
210
+
211
+
def test_grab_level_5a_human_shows_frame_metadata(monkeypatch, capsys):
212
+
code, message, out, err = _invoke_grab(
213
+
monkeypatch, capsys, "20240102", "default", "233000_300", "screen", "7"
214
+
)
215
+
assert code == 0
216
+
assert message == ""
217
+
assert err == ""
218
+
assert "Screen: screen" in out
219
+
assert '"frame_id": 7' in out
220
+
221
+
222
+
def test_grab_level_5a_legacy_schema_errors(monkeypatch, capsys):
223
+
code, message, out, err = _invoke_grab(
224
+
monkeypatch, capsys, "20240101", "default", "123456_300", "screen", "1"
225
+
)
226
+
assert code == 1
227
+
assert out == ""
228
+
assert err == ""
229
+
assert (
230
+
message
231
+
== "screen file uses pre-frame_id schema; frame selection is unavailable"
232
+
)
233
+
234
+
235
+
def test_grab_missing_frame_id_errors(monkeypatch, capsys):
236
+
code, message, out, err = _invoke_grab(
237
+
monkeypatch, capsys, "20240102", "default", "233000_300", "screen", "999"
238
+
)
239
+
assert code == 1
240
+
assert out == ""
241
+
assert err == ""
242
+
assert message == "frame id 999 not found in screen for 233000_300"
243
+
244
+
245
+
def test_grab_level_5b_json_matches_fixture_and_writes_png(
246
+
monkeypatch, capsys, tmp_path
247
+
):
248
+
out_path = tmp_path / "frame.png"
249
+
code, message, out, err = _invoke_grab(
250
+
monkeypatch,
251
+
capsys,
252
+
"--json",
253
+
"--out",
254
+
str(out_path),
255
+
"20240102",
256
+
"default",
257
+
"233000_300",
258
+
"screen",
259
+
"7",
260
+
)
261
+
actual = json.loads(out)
262
+
expected = _expected("level_5b.json")
263
+
assert code == 0
264
+
assert message == ""
265
+
assert err == ""
266
+
assert _normalize_saved_paths(actual, expected) == expected
267
+
assert out_path.is_file()
268
+
with Image.open(out_path) as image:
269
+
assert image.size == (64, 48)
270
+
271
+
272
+
def test_grab_level_5b_refuses_overwrite_without_force(monkeypatch, capsys, tmp_path):
273
+
out_path = tmp_path / "frame.png"
274
+
out_path.write_bytes(b"existing")
275
+
code, message, out, err = _invoke_grab(
276
+
monkeypatch,
277
+
capsys,
278
+
"--out",
279
+
str(out_path),
280
+
"20240102",
281
+
"default",
282
+
"233000_300",
283
+
"screen",
284
+
"7",
285
+
)
286
+
assert code == 1
287
+
assert out == ""
288
+
assert err == ""
289
+
assert "output path exists" in message
290
+
291
+
292
+
def test_grab_level_5b_force_replaces_existing_file(monkeypatch, capsys, tmp_path):
293
+
out_path = tmp_path / "frame.png"
294
+
out_path.write_bytes(b"existing")
295
+
code, message, out, err = _invoke_grab(
296
+
monkeypatch,
297
+
capsys,
298
+
"--force",
299
+
"--out",
300
+
str(out_path),
301
+
"20240102",
302
+
"default",
303
+
"233000_300",
304
+
"screen",
305
+
"7",
306
+
)
307
+
assert code == 0
308
+
assert message == ""
309
+
assert err == ""
310
+
assert out_path.is_file()
311
+
with Image.open(out_path) as image:
312
+
assert image.size == (64, 48)
313
+
314
+
315
+
def test_grab_level_5b_unknown_suffix_is_argparse_error(monkeypatch, capsys, tmp_path):
316
+
code, message, out, err = _invoke_grab(
317
+
monkeypatch,
318
+
capsys,
319
+
"--out",
320
+
str(tmp_path / "frame.gif"),
321
+
"20240102",
322
+
"default",
323
+
"233000_300",
324
+
"screen",
325
+
"7",
326
+
)
327
+
assert code == 2
328
+
assert message == ""
329
+
assert out == ""
330
+
assert "--out must end in .png, .jpg, .jpeg, or .webp" in err
331
+
332
+
333
+
def test_grab_level_5c_json_matches_fixture_and_writes_numbered_files(
334
+
monkeypatch, capsys, tmp_path
335
+
):
336
+
out_path = tmp_path / "frame.png"
337
+
code, message, out, err = _invoke_grab(
338
+
monkeypatch,
339
+
capsys,
340
+
"--json",
341
+
"--out",
342
+
str(out_path),
343
+
"20240102",
344
+
"default",
345
+
"233000_300",
346
+
"screen",
347
+
"7,12,23",
348
+
)
349
+
actual = json.loads(out)
350
+
expected = _expected("level_5c.json")
351
+
assert code == 0
352
+
assert message == ""
353
+
assert err == ""
354
+
assert _normalize_saved_paths(actual, expected) == expected
355
+
for frame_id in (7, 12, 23):
356
+
saved = tmp_path / f"frame_{frame_id}.png"
357
+
assert saved.is_file()
358
+
with Image.open(saved) as image:
359
+
assert image.size == (64, 48)
360
+
361
+
362
+
def test_grab_level_5c_conflict_scan_happens_before_decode(
363
+
monkeypatch, capsys, tmp_path
364
+
):
365
+
out_path = tmp_path / "frame.png"
366
+
(tmp_path / "frame_12.png").write_bytes(b"existing")
367
+
decode_mock = Mock(side_effect=AssertionError("decode should not run"))
368
+
monkeypatch.setattr("observe.grab.decode_frames", decode_mock)
369
+
code, message, out, err = _invoke_grab(
370
+
monkeypatch,
371
+
capsys,
372
+
"--out",
373
+
str(out_path),
374
+
"20240102",
375
+
"default",
376
+
"233000_300",
377
+
"screen",
378
+
"7,12,23",
379
+
)
380
+
assert code == 1
381
+
assert out == ""
382
+
assert err == ""
383
+
assert "output path exists" in message
384
+
decode_mock.assert_not_called()
385
+
386
+
387
+
def test_grab_level_5c_decode_failure_writes_no_files(monkeypatch, capsys, tmp_path):
388
+
out_path = tmp_path / "frame.png"
389
+
monkeypatch.setattr(
390
+
"observe.grab.decode_frames", Mock(side_effect=RuntimeError("decode blew up"))
391
+
)
392
+
code, message, out, err = _invoke_grab(
393
+
monkeypatch,
394
+
capsys,
395
+
"--out",
396
+
str(out_path),
397
+
"20240102",
398
+
"default",
399
+
"233000_300",
400
+
"screen",
401
+
"7,12,23",
402
+
)
403
+
assert code == 1
404
+
assert out == ""
405
+
assert err == ""
406
+
assert message == "decode blew up"
407
+
assert list(tmp_path.iterdir()) == []
408
+
409
+
410
+
def test_grab_level_5c_requires_out_for_multiple_frame_ids(monkeypatch, capsys):
411
+
code, message, out, err = _invoke_grab(
412
+
monkeypatch, capsys, "20240102", "default", "233000_300", "screen", "7,12,23"
413
+
)
414
+
assert code == 2
415
+
assert message == ""
416
+
assert out == ""
417
+
assert "multiple frame ids require --out" in err
418
+
419
+
420
+
def test_grab_rejects_more_than_five_positionals(monkeypatch, capsys):
421
+
code, message, out, err = _invoke_grab(
422
+
monkeypatch, capsys, "a", "b", "c", "d", "e", "f"
423
+
)
424
+
assert code == 2
425
+
assert message == ""
426
+
assert out == ""
427
+
assert "at most 5 positional tokens" in err
428
+
429
+
430
+
def test_grab_force_requires_out(monkeypatch, capsys):
431
+
code, message, out, err = _invoke_grab(monkeypatch, capsys, "--force")
432
+
assert code == 2
433
+
assert message == ""
434
+
assert out == ""
435
+
assert "--force requires --out" in err
436
+
437
+
438
+
def test_grab_out_requires_level_5(monkeypatch, capsys, tmp_path):
439
+
code, message, out, err = _invoke_grab(
440
+
monkeypatch, capsys, "--out", str(tmp_path / "frame.png"), "20240102"
441
+
)
442
+
assert code == 2
443
+
assert message == ""
444
+
assert out == ""
445
+
assert "--out requires day stream segment screen and frame-id" in err
446
+
447
+
448
+
@pytest.mark.parametrize("token", ["0", "-1", "abc", "7,7", "1,,2"])
449
+
def test_grab_frame_id_token_rejects_invalid_values(token):
450
+
from observe.grab import parse_frame_id_token
451
+
452
+
with pytest.raises(ValueError):
453
+
parse_frame_id_token(token)
454
+
455
+
456
+
def test_grab_frame_id_token_sorts_batch_ids():
457
+
from observe.grab import parse_frame_id_token
458
+
459
+
assert parse_frame_id_token("23,7,12") == [7, 12, 23]
+44
tests/test_see.py
+44
tests/test_see.py
···
91
91
92
92
assert images[0].getpixel((0, 0)) == (10, 10, 10)
93
93
assert images[1].getpixel((0, 0)) == (20, 20, 20)
94
+
95
+
96
+
def test_decode_frames_stops_after_highest_requested_frame(monkeypatch):
97
+
"""Test decode_frames exits once it has filled the highest requested frame."""
98
+
import numpy as np
99
+
100
+
seen = {"count": 0}
101
+
102
+
class FakeFrame:
103
+
def __init__(self, color: int):
104
+
self.pts = 1
105
+
self._color = color
106
+
107
+
def to_ndarray(self, format: str):
108
+
assert format == "rgb24"
109
+
return np.full((2, 2, 3), self._color, dtype=np.uint8)
110
+
111
+
class FakeContainer:
112
+
def __init__(self):
113
+
self.streams = type("Streams", (), {"video": [object()]})
114
+
115
+
def decode(self, stream):
116
+
for index in range(30):
117
+
seen["count"] += 1
118
+
yield FakeFrame(index)
119
+
120
+
def __enter__(self):
121
+
return self
122
+
123
+
def __exit__(self, exc_type, exc, tb):
124
+
return False
125
+
126
+
class FakeAv:
127
+
@staticmethod
128
+
def open(path):
129
+
return FakeContainer()
130
+
131
+
monkeypatch.setitem(__import__("sys").modules, "av", FakeAv)
132
+
133
+
frames = [{"frame_id": 7}, {"frame_id": 12}, {"frame_id": 23}]
134
+
images = decode_frames("dummy.mp4", frames, annotate_boxes=False)
135
+
136
+
assert seen["count"] == 23
137
+
assert all(image is not None for image in images)
+2
think/sol_cli.py
+2
think/sol_cli.py
···
57
57
"sense": "observe.sense",
58
58
"transfer": "observe.transfer",
59
59
"export": "observe.export",
60
+
"grab": "observe.grab",
60
61
"observer": "observe.observer_cli",
61
62
# AI providers and talent execution
62
63
"providers": "think.providers_cli",
···
111
112
"sense",
112
113
"transfer",
113
114
"export",
115
+
"grab",
114
116
"observer",
115
117
],
116
118
"Talent": [