Add content wrapper for screen entry category data

+6 -7

apps/transcripts/routes.py

··· 271 271 # Calculate wall-clock time from segment start + offset 272 272 time_str = _format_time_from_offset(segment_key, offset) 273 273 274 - # Basic frames have <= 1 vision request (just DESCRIBE_JSON) 275 - # Enhanced frames have > 1 (added text extraction or meeting analysis) 276 - requests = source.get("requests", []) 277 - is_basic = len(requests) <= 1 274 + # Basic frames have no enriched content 275 + frame_content = source.get("content", {}) 276 + is_basic = not frame_content 278 277 279 278 # Extract participant boxes for meeting frames 280 279 participants = [] 281 - meeting_analysis = source.get("meeting_analysis") 282 - if meeting_analysis: 283 - for p in meeting_analysis.get("participants", []): 280 + meeting_data = frame_content.get("meeting") 281 + if meeting_data: 282 + for p in meeting_data.get("participants", []): 284 283 box = p.get("box_2d") 285 284 # Only include participants with video and valid box_2d 286 285 if p.get("video") and box and len(box) == 4:

+2 -2

apps/transcripts/workspace.html

··· 1601 1601 const frameId = entry.source_ref?.frame_id; 1602 1602 const boxCoords = entry.source_ref?.box_2d; 1603 1603 const analysis = entry.source_ref?.analysis || {}; 1604 - const category = analysis.visible || 'unknown'; 1604 + const category = analysis.primary || 'unknown'; 1605 1605 const description = analysis.visual_description || category; 1606 1606 const frameIdx = findFrameIndex(entry); 1607 1607 ··· 1675 1675 const monitor = f.source_ref?.monitor || ''; 1676 1676 const monitorPos = getMonitorPosition(monitor); 1677 1677 const analysis = f.source_ref?.analysis || {}; 1678 - const category = analysis.visible || ''; 1678 + const category = analysis.primary || ''; 1679 1679 const description = analysis.visual_description || ''; 1680 1680 const participants = f.source_ref?.participants || []; 1681 1681 const hasPrev = currentFrameIndex > 0;

+4 -2

observe/describe.py

··· 607 607 608 608 result = frame_results[req.frame_id] 609 609 610 - # Merge this follow-up's category result 610 + # Merge this follow-up's category result into content dict 611 + if "content" not in result: 612 + result["content"] = {} 611 613 for category, cat_result in req.category_results.items(): 612 - result[category] = cat_result 614 + result["content"][category] = cat_result 613 615 614 616 # Update requests list (avoid duplicates by using shared list) 615 617 result["requests"] = req.requests

+6 -26

observe/screen.py

··· 208 208 # Add analysis if present 209 209 analysis = frame.get("analysis", {}) 210 210 if analysis: 211 - # New format: primary is a string category 212 - # Fall back to legacy "visible" field for old data 213 - category = analysis.get("primary", analysis.get("visible", "unknown")) 211 + category = analysis.get("primary", "unknown") 214 212 description = analysis.get("visual_description", "") 215 213 216 214 lines.append(f"**Category:** {category}") ··· 227 225 "timestamp_str": timestamp_str, 228 226 } 229 227 230 - # Add category-specific content using formatter dispatch 231 - # New format uses category name as key (e.g., "meeting", "messaging") 232 - # Old format used "extracted_text" and "meeting_analysis" 233 - has_category_content = False 234 - for cat in CATEGORIES: 235 - content = frame.get(cat) 236 - # Also check legacy "meeting_analysis" key for meeting 237 - if cat == "meeting" and not content: 238 - content = frame.get("meeting_analysis") 239 - if content: 240 - formatted = _format_category_content(cat, content, format_context) 228 + # Add category-specific content from content dict 229 + frame_content = frame.get("content", {}) 230 + for cat, cat_data in frame_content.items(): 231 + if cat_data: 232 + formatted = _format_category_content(cat, cat_data, format_context) 241 233 if formatted: 242 234 lines.append(formatted) 243 - has_category_content = True 244 - 245 - # Fall back to legacy extracted_text field if no category content 246 - if not has_category_content: 247 - extracted_text = frame.get("extracted_text") 248 - if extracted_text: 249 - lines.append("**Extracted Text:**") 250 - lines.append("") 251 - lines.append("```") 252 - lines.append(extracted_text.strip()) 253 - lines.append("```") 254 - lines.append("") 255 235 256 236 # Calculate absolute unix timestamp in milliseconds 257 237 frame_timestamp_ms = base_timestamp_ms + int(frame_offset * 1000)

+31 -24

observe/tmux/capture.py

··· 326 326 "secondary": "none", 327 327 "overlap": False, 328 328 }, 329 - "tmux": { 330 - "session": result.session, 331 - "window": { 332 - "id": result.window.id, 333 - "index": result.window.index, 334 - "name": result.window.name, 329 + "content": { 330 + "tmux": { 331 + "session": result.session, 332 + "window": { 333 + "id": result.window.id, 334 + "index": result.window.index, 335 + "name": result.window.name, 336 + }, 337 + "windows": [ 338 + { 339 + "id": w.id, 340 + "index": w.index, 341 + "name": w.name, 342 + "active": w.active, 343 + } 344 + for w in result.windows 345 + ], 346 + "panes": [ 347 + { 348 + "id": p.id, 349 + "index": p.index, 350 + "left": p.left, 351 + "top": p.top, 352 + "width": p.width, 353 + "height": p.height, 354 + "active": p.active, 355 + "content": p.content, 356 + } 357 + for p in result.panes 358 + ], 335 359 }, 336 - "windows": [ 337 - {"id": w.id, "index": w.index, "name": w.name, "active": w.active} 338 - for w in result.windows 339 - ], 340 - "panes": [ 341 - { 342 - "id": p.id, 343 - "index": p.index, 344 - "left": p.left, 345 - "top": p.top, 346 - "width": p.width, 347 - "height": p.height, 348 - "active": p.active, 349 - "content": p.content, 350 - } 351 - for p in result.panes 352 - ], 353 360 }, 354 361 } 355 362 ··· 376 383 # Group captures by session 377 384 by_session: dict[str, list[dict]] = {} 378 385 for capture in captures: 379 - session = capture.get("tmux", {}).get("session", "unknown") 386 + session = capture.get("content", {}).get("tmux", {}).get("session", "unknown") 380 387 if session not in by_session: 381 388 by_session[session] = [] 382 389 by_session[session].append(capture)

+6 -2

tests/test_formatters.py

··· 188 188 "primary": "reading", 189 189 "visual_description": "Documentation page", 190 190 }, 191 - "reading": "# API Reference\n\ndef hello():\n pass", 191 + "content": { 192 + "reading": "# API Reference\n\ndef hello():\n pass", 193 + }, 192 194 } 193 195 ] 194 196 ··· 251 253 { 252 254 "timestamp": 0, 253 255 "analysis": {"primary": "meeting"}, 254 - "meeting": {"participants": ["Alice", "Bob"]}, 256 + "content": { 257 + "meeting": {"participants": ["Alice", "Bob"]}, 258 + }, 255 259 } 256 260 ] 257 261

+23 -13

tests/test_screen_formatter.py

··· 207 207 "primary": "productivity", 208 208 "visual_description": "Spreadsheet view", 209 209 }, 210 - "productivity": "| Name | Value |\n|------|-------|\n| Test | 123 |", 210 + "content": { 211 + "productivity": "| Name | Value |\n|------|-------|\n| Test | 123 |", 212 + }, 211 213 }, 212 214 ] 213 215 ··· 333 335 "primary": "meeting", 334 336 "visual_description": "Video call", 335 337 }, 336 - "meeting": { 337 - "platform": "meet", 338 - "participants": [ 339 - {"name": "Test User", "status": "active", "video": True}, 340 - ], 341 - "screen_share": None, 338 + "content": { 339 + "meeting": { 340 + "platform": "meet", 341 + "participants": [ 342 + {"name": "Test User", "status": "active", "video": True}, 343 + ], 344 + "screen_share": None, 345 + }, 342 346 }, 343 347 }, 344 348 ] ··· 367 371 "primary": "messaging", 368 372 "visual_description": "Chat app", 369 373 }, 370 - "messaging": "**Alice**: Hello!\n**Bob**: Hi there!", 374 + "content": { 375 + "messaging": "**Alice**: Hello!\n**Bob**: Hi there!", 376 + }, 371 377 }, 372 378 ] 373 379 ··· 392 398 "overlap": False, 393 399 "visual_description": "Meeting with shared doc", 394 400 }, 395 - "meeting": { 396 - "platform": "teams", 397 - "participants": [{"name": "User", "status": "active", "video": True}], 398 - "screen_share": None, 401 + "content": { 402 + "meeting": { 403 + "platform": "teams", 404 + "participants": [ 405 + {"name": "User", "status": "active", "video": True} 406 + ], 407 + "screen_share": None, 408 + }, 409 + "productivity": "| Task | Status |\n|------|--------|\n| Review | Done |", 399 410 }, 400 - "productivity": "| Task | Status |\n|------|--------|\n| Review | Done |", 401 411 }, 402 412 ] 403 413

Configure Feed

Configure Feed