personal memory agent
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat(home): add yesterday's processing card to pulse landing

Add the yesterday-processing pulse card, wire its server-side context, and cover the rendering and copy rules with focused home tests and fixtures.

Co-Authored-By: OpenAI Codex <codex@openai.com>

+1588
+585
apps/home/routes.py
··· 45 45 url_prefix="/app/home", 46 46 ) 47 47 48 + _FIRST_WEEK_FRAMING = "Most of what I learn becomes useful in the third or fourth week, when I've seen enough patterns to surface them. For now, here's what's already happening:" 49 + 50 + _ENTITY_TYPE_LABELS = { 51 + "company": ("company", "companies"), 52 + "decision": ("decision", "decisions"), 53 + "person": ("person", "people"), 54 + "place": ("place", "places"), 55 + "project": ("project", "projects"), 56 + "topic": ("topic", "topics"), 57 + "tool": ("tool", "tools"), 58 + "unknown": ("thing", "things"), 59 + } 60 + 48 61 49 62 def _today() -> str: 50 63 return datetime.now().strftime("%Y%m%d") 51 64 52 65 66 + def _yesterday() -> str: 67 + return (datetime.now().astimezone() - timedelta(days=1)).strftime("%Y%m%d") 68 + 69 + 70 + def _count_journal_age_days(today: str) -> int: 71 + chronicle_dir = Path(get_journal()) / "chronicle" 72 + if not chronicle_dir.is_dir(): 73 + return 0 74 + 75 + earliest: datetime | None = None 76 + for child in chronicle_dir.iterdir(): 77 + if not child.is_dir() or not child.name.isdigit() or len(child.name) != 8: 78 + continue 79 + try: 80 + day = datetime.strptime(child.name, "%Y%m%d") 81 + except ValueError: 82 + continue 83 + if earliest is None or day < earliest: 84 + earliest = day 85 + 86 + if earliest is None: 87 + return 0 88 + 89 + try: 90 + today_dt = datetime.strptime(today, "%Y%m%d") 91 + except ValueError: 92 + return 0 93 + return max(0, (today_dt - earliest).days) 94 + 95 + 53 96 def _load_flow_md(today: str) -> tuple[str | None, float | None]: 54 97 """Load today's flow.md content and mtime. Returns (content, mtime) or (None, None).""" 55 98 try: ··· 221 264 return {} 222 265 223 266 267 + def _load_yesterday_stats(yesterday: str) -> dict[str, Any] | None: 268 + try: 269 + stats_path = Path(get_journal()) / "chronicle" / yesterday / "stats.json" 270 + if not stats_path.exists(): 271 + return None 272 + return json.loads(stats_path.read_text(encoding="utf-8")) 273 + except Exception: 274 + logger.warning("home: failed to load yesterday stats", exc_info=True) 275 + return None 276 + 277 + 278 + def _load_yesterday_pipeline_summary(yesterday: str) -> dict[str, Any]: 279 + return summarize_pipeline_day(yesterday) 280 + 281 + 224 282 def _collect_todos(today: str) -> list[dict[str, Any]]: 225 283 """Collect pending todos across all facets.""" 226 284 from apps.todos.todo import get_todos ··· 340 398 return [] 341 399 342 400 401 + def _collect_entities_yesterday(yesterday: str) -> list[dict[str, Any]]: 402 + """Get yesterday's entities from entity_signals table.""" 403 + try: 404 + conn, _ = get_journal_index() 405 + try: 406 + rows = conn.execute( 407 + """SELECT entity_name, COUNT(*) as signal_count, 408 + GROUP_CONCAT(DISTINCT signal_type) as types 409 + FROM entity_signals 410 + WHERE day = ? 411 + GROUP BY entity_name 412 + ORDER BY signal_count DESC""", 413 + (yesterday,), 414 + ).fetchall() 415 + 416 + entity_meta = {} 417 + meta_rows = conn.execute( 418 + "SELECT entity_id, name, type FROM entities WHERE source='identity'" 419 + ).fetchall() 420 + for row in meta_rows: 421 + entity_meta[row[0]] = {"name": row[1], "type": row[2] or "unknown"} 422 + entity_meta[row[1].lower()] = { 423 + "name": row[1], 424 + "type": row[2] or "unknown", 425 + } 426 + 427 + entities = [] 428 + for row in rows: 429 + name = row[0] 430 + meta = entity_meta.get(name, entity_meta.get(name.lower(), {})) 431 + entities.append( 432 + { 433 + "name": meta.get("name", name), 434 + "signal_count": row[1], 435 + "types": row[2] or "", 436 + "entity_type": meta.get("type", "unknown"), 437 + } 438 + ) 439 + return entities 440 + finally: 441 + conn.close() 442 + except Exception: 443 + logger.warning("home: failed to collect yesterday entities", exc_info=True) 444 + return [] 445 + 446 + 447 + def _normalize_activity_title(record: dict[str, Any]) -> str: 448 + title = str(record.get("description") or "").strip() 449 + if title: 450 + return title 451 + activity = str(record.get("activity") or "").strip().replace("_", " ") 452 + if activity: 453 + return activity.title() 454 + return "Untitled activity" 455 + 456 + 457 + def _collect_top_activities_yesterday(yesterday: str) -> list[dict[str, Any]]: 458 + from think.activities import estimate_duration_minutes, load_activity_records 459 + 460 + activities = [] 461 + try: 462 + facets = get_enabled_facets() 463 + except Exception: 464 + logger.warning( 465 + "home: failed to get enabled facets for yesterday activities", 466 + exc_info=True, 467 + ) 468 + return [] 469 + 470 + for facet_name in facets: 471 + for record in load_activity_records(facet_name, yesterday): 472 + segments = record.get("segments", []) 473 + activities.append( 474 + { 475 + **record, 476 + "facet": facet_name, 477 + "title": _normalize_activity_title(record), 478 + "duration_minutes": estimate_duration_minutes(segments), 479 + } 480 + ) 481 + 482 + activities.sort( 483 + key=lambda record: ( 484 + -int(record.get("duration_minutes", 0)), 485 + record.get("title", "").lower(), 486 + record.get("facet", ""), 487 + ) 488 + ) 489 + return activities 490 + 491 + 492 + def _top_heatmap_hours(stats_data: dict[str, Any]) -> list[int]: 493 + hours = stats_data.get("heatmap_data", {}).get("hours", {}) 494 + ranked = [] 495 + for hour, minutes in hours.items(): 496 + try: 497 + hour_int = int(hour) 498 + minutes_value = float(minutes) 499 + except (TypeError, ValueError): 500 + continue 501 + if minutes_value <= 0: 502 + continue 503 + ranked.append((hour_int, minutes_value)) 504 + 505 + ranked.sort(key=lambda item: (-item[1], item[0])) 506 + return [hour for hour, _minutes in ranked[:3]] 507 + 508 + 509 + def _knowledge_graph_freshness(yesterday: str) -> dict[str, Any]: 510 + path = ( 511 + Path(get_journal()) / "chronicle" / yesterday / "agents" / "knowledge_graph.md" 512 + ) 513 + if not path.exists(): 514 + return {"exists": False, "fresh": False, "updated_label": None} 515 + 516 + try: 517 + start_of_yesterday_local = datetime.strptime(yesterday, "%Y%m%d").astimezone() 518 + updated_at = datetime.fromtimestamp(path.stat().st_mtime).astimezone() 519 + except Exception: 520 + logger.warning( 521 + "home: failed to inspect knowledge graph freshness", exc_info=True 522 + ) 523 + return {"exists": True, "fresh": False, "updated_label": None} 524 + 525 + return { 526 + "exists": True, 527 + "fresh": updated_at >= start_of_yesterday_local, 528 + "updated_label": updated_at.strftime("%-I:%M%p").lower(), 529 + } 530 + 531 + 532 + def _briefing_freshness(today: str) -> dict[str, Any]: 533 + briefing_path = Path(get_journal()) / "sol" / "briefing.md" 534 + if not briefing_path.exists(): 535 + return {"exists": False, "valid": False, "generated_label": None} 536 + 537 + try: 538 + post = frontmatter.load(str(briefing_path)) 539 + except Exception: 540 + logger.warning("home: failed to load briefing freshness", exc_info=True) 541 + return {"exists": True, "valid": False, "generated_label": None} 542 + 543 + if post.metadata.get("type") != "morning_briefing": 544 + return {"exists": True, "valid": False, "generated_label": None} 545 + 546 + generated = post.metadata.get("generated") 547 + if generated is None: 548 + return {"exists": True, "valid": False, "generated_label": None} 549 + 550 + try: 551 + if isinstance(generated, str): 552 + generated_dt = datetime.fromisoformat(generated) 553 + else: 554 + generated_dt = generated 555 + if generated_dt.tzinfo is None: 556 + generated_dt = generated_dt.astimezone() 557 + else: 558 + generated_dt = generated_dt.astimezone() 559 + except Exception: 560 + return {"exists": True, "valid": False, "generated_label": None} 561 + 562 + return { 563 + "exists": True, 564 + "valid": generated_dt.strftime("%Y%m%d") == today, 565 + "generated_label": generated_dt.strftime("%-I:%M%p").lower(), 566 + } 567 + 568 + 569 + def _newsletter_attempts_from_dream_logs(yesterday: str) -> tuple[int, int]: 570 + journal = Path(get_journal()) 571 + successful = len(list(journal.glob(f"facets/*/news/{yesterday}.md"))) 572 + 573 + failed = 0 574 + health_dir = journal / "chronicle" / yesterday / "health" 575 + if health_dir.is_dir(): 576 + for path in sorted(health_dir.glob("*_daily_dream.jsonl")): 577 + try: 578 + with path.open(encoding="utf-8") as handle: 579 + for raw_line in handle: 580 + line = raw_line.strip() 581 + if not line: 582 + continue 583 + try: 584 + record = json.loads(line) 585 + except json.JSONDecodeError: 586 + continue 587 + if ( 588 + record.get("event") == "agent.fail" 589 + and record.get("facet") 590 + and record.get("name") == "facet_newsletter" 591 + ): 592 + failed += 1 593 + except OSError: 594 + logger.warning( 595 + "home: failed to read newsletter dream log %s", 596 + path, 597 + exc_info=True, 598 + ) 599 + 600 + return successful, successful + failed 601 + 602 + 603 + def _format_duration(total_minutes: float) -> str: 604 + rounded_minutes = int(round(total_minutes)) 605 + if rounded_minutes < 60: 606 + return f"{rounded_minutes} min" 607 + 608 + rounded_hours = round(total_minutes / 60, 1) 609 + if float(rounded_hours).is_integer(): 610 + hours_int = int(rounded_hours) 611 + return f"{hours_int} hour{'s' if hours_int != 1 else ''}" 612 + return f"{rounded_hours:.1f} hours" 613 + 614 + 615 + def _format_hour_label(start_hour: int, end_hour: int) -> str: 616 + def render(hour: int, *, include_meridiem: bool) -> str: 617 + normalized = hour % 24 618 + display_hour = normalized % 12 or 12 619 + meridiem = "am" if normalized < 12 else "pm" 620 + return f"{display_hour}{meridiem}" if include_meridiem else str(display_hour) 621 + 622 + start_meridiem = "am" if start_hour % 24 < 12 else "pm" 623 + end_meridiem = "am" if end_hour % 24 < 12 else "pm" 624 + return ( 625 + f"{render(start_hour, include_meridiem=start_meridiem != end_meridiem)}-" 626 + f"{render(end_hour, include_meridiem=True)}" 627 + ) 628 + 629 + 630 + def _join_phrases(parts: list[str]) -> str: 631 + if not parts: 632 + return "" 633 + if len(parts) == 1: 634 + return parts[0] 635 + if len(parts) == 2: 636 + return f"{parts[0]} and {parts[1]}" 637 + return ", ".join(parts[:-1]) + f", and {parts[-1]}" 638 + 639 + 640 + def _format_entity_count(type_key: str, count: int) -> str: 641 + singular, plural = _ENTITY_TYPE_LABELS.get(type_key, (type_key, f"{type_key}s")) 642 + label = singular if count == 1 else plural 643 + return f"{count} {label}" 644 + 645 + 646 + def _format_entity_summary(entities: list[dict[str, Any]]) -> str | None: 647 + counts: dict[str, int] = {} 648 + for entity in entities: 649 + type_key = str(entity.get("entity_type") or "unknown").strip().lower() 650 + counts[type_key] = counts.get(type_key, 0) + 1 651 + 652 + if not counts: 653 + return None 654 + 655 + ordered_keys = [] 656 + if counts.get("person"): 657 + ordered_keys.append("person") 658 + ordered_keys.extend( 659 + key 660 + for key, _count in sorted( 661 + ( 662 + (key, count) 663 + for key, count in counts.items() 664 + if key != "person" and count > 0 665 + ), 666 + key=lambda item: (-item[1], item[0]), 667 + ) 668 + ) 669 + 670 + labels = [ 671 + _format_entity_count(key, counts[key]) for key in ordered_keys if counts[key] 672 + ] 673 + if not labels: 674 + return None 675 + return f"I recognized {_join_phrases(labels)}." 676 + 677 + 678 + def _format_activity_label(activity: dict[str, Any]) -> str: 679 + return ( 680 + f"I took notes on {activity.get('title', 'Untitled activity')} " 681 + f"for {_format_duration(activity.get('duration_minutes', 0))} in " 682 + f"{activity.get('facet', 'unknown')}." 683 + ) 684 + 685 + 686 + def _format_newsletter_summary(successful: int, attempted: int) -> str: 687 + if attempted == 0: 688 + return "I didn't produce any facet newsletters." 689 + if attempted > successful: 690 + return ( 691 + f"I wrote {successful} of {attempted} newsletter" 692 + f"{'s' if attempted != 1 else ''}." 693 + ) 694 + return f"I wrote {successful} newsletter{'s' if successful != 1 else ''}." 695 + 696 + 697 + def _format_processing_summary( 698 + mode: str, 699 + successful_newsletters: int, 700 + attempted_newsletters: int, 701 + knowledge_graph: dict[str, Any], 702 + briefing: dict[str, Any], 703 + ) -> str: 704 + if mode == "degraded": 705 + if attempted_newsletters == 0 and successful_newsletters == 0: 706 + return ( 707 + "I didn't produce any facet newsletters, and some overnight " 708 + "processing didn't finish." 709 + ) 710 + if attempted_newsletters > successful_newsletters: 711 + return ( 712 + f"I wrote {successful_newsletters} of {attempted_newsletters} " 713 + "newsletters, but some overnight processing didn't finish." 714 + ) 715 + return ( 716 + f"I wrote {successful_newsletters} newsletter" 717 + f"{'s' if successful_newsletters != 1 else ''}, but some overnight " 718 + "processing didn't finish." 719 + ) 720 + 721 + actions = [] 722 + if successful_newsletters > 0: 723 + actions.append( 724 + f"wrote {successful_newsletters} newsletter" 725 + f"{'s' if successful_newsletters != 1 else ''}" 726 + ) 727 + if knowledge_graph.get("fresh"): 728 + actions.append("refreshed your knowledge graph") 729 + if briefing.get("valid"): 730 + actions.append("prepared your morning briefing") 731 + if not actions: 732 + return _format_newsletter_summary(successful_newsletters, attempted_newsletters) 733 + return f"I {_join_phrases(actions)}." 734 + 735 + 736 + def _format_heatmap_summary(stats_data: dict[str, Any]) -> str | None: 737 + hours = sorted(_top_heatmap_hours(stats_data)) 738 + if not hours: 739 + return None 740 + 741 + ranges = [] 742 + range_start = hours[0] 743 + range_end = hours[0] + 1 744 + for hour in hours[1:]: 745 + if hour == range_end: 746 + range_end += 1 747 + continue 748 + ranges.append(_format_hour_label(range_start, range_end)) 749 + range_start = hour 750 + range_end = hour + 1 751 + ranges.append(_format_hour_label(range_start, range_end)) 752 + return "I watched most closely during " + " · ".join(ranges) + "." 753 + 754 + 755 + def _format_gap_bullets( 756 + pipeline_summary: dict[str, Any], 757 + knowledge_graph: dict[str, Any], 758 + briefing: dict[str, Any], 759 + ) -> list[str]: 760 + bullets = [] 761 + anomalies = pipeline_summary.get("anomalies", []) 762 + has_daily = any( 763 + anomaly.get("kind") == "daily_agents_missing" for anomaly in anomalies 764 + ) 765 + has_activity = any( 766 + anomaly.get("kind") == "activity_agents_missing" for anomaly in anomalies 767 + ) 768 + has_failure = any(anomaly.get("kind") == "agent_failure" for anomaly in anomalies) 769 + 770 + if has_daily: 771 + bullets.append("I didn't finish the full overnight review.") 772 + if has_activity: 773 + bullets.append("I didn't finish writing all of yesterday's notes.") 774 + if has_failure and not has_daily and not has_activity: 775 + bullets.append("Some of my overnight work didn't finish.") 776 + 777 + if not knowledge_graph.get("fresh"): 778 + bullets.append("I didn't refresh your knowledge graph overnight.") 779 + if not briefing.get("valid"): 780 + bullets.append("I didn't prepare your morning briefing overnight.") 781 + return bullets 782 + 783 + 784 + def _summarize_yesterday_processing( 785 + yesterday: str, journal_age_days: int 786 + ) -> dict[str, Any] | None: 787 + stats_data = _load_yesterday_stats(yesterday) 788 + if stats_data is None or journal_age_days == 0: 789 + return None 790 + 791 + stats = stats_data.get("stats", {}) 792 + transcript_seconds = float(stats.get("transcript_duration", 0) or 0) 793 + transcript_segments = int(stats.get("transcript_segments", 0) or 0) 794 + facet_data = stats_data.get("facet_data", {}) 795 + has_facet_activity = any( 796 + float(facet.get("minutes", 0) or 0) > 0 or int(facet.get("count", 0) or 0) > 0 797 + for facet in facet_data.values() 798 + ) 799 + 800 + entities = _collect_entities_yesterday(yesterday) 801 + activities = _collect_top_activities_yesterday(yesterday) 802 + if ( 803 + transcript_seconds <= 0 804 + and transcript_segments <= 0 805 + and not has_facet_activity 806 + and not activities 807 + and not entities 808 + ): 809 + return None 810 + 811 + pipeline_summary = _load_yesterday_pipeline_summary(yesterday) 812 + knowledge_graph = _knowledge_graph_freshness(yesterday) 813 + briefing = _briefing_freshness(_today()) 814 + successful_newsletters, attempted_newsletters = ( 815 + _newsletter_attempts_from_dream_logs(yesterday) 816 + ) 817 + 818 + is_sparse = ( 819 + (transcript_seconds > 0 or transcript_segments > 0) 820 + and not has_facet_activity 821 + and not activities 822 + ) 823 + 824 + status_reasons = [] 825 + if attempted_newsletters > successful_newsletters: 826 + status_reasons.append("newsletter_partial") 827 + if pipeline_summary.get("status") != "healthy": 828 + status_reasons.append("pipeline_warning") 829 + if not knowledge_graph.get("fresh"): 830 + status_reasons.append("knowledge_graph_stale") 831 + if not briefing.get("valid"): 832 + status_reasons.append("briefing_missing") 833 + 834 + if is_sparse: 835 + mode = "sparse" 836 + elif status_reasons: 837 + mode = "degraded" 838 + else: 839 + mode = "healthy" 840 + 841 + first_week_framing = ( 842 + _FIRST_WEEK_FRAMING if journal_age_days <= 7 and mode != "sparse" else None 843 + ) 844 + 845 + if mode == "sparse": 846 + summary_line = ( 847 + f"I watched {_format_duration(transcript_seconds / 60)} yesterday." 848 + ) 849 + return { 850 + "title": "Yesterday's processing", 851 + "mode": mode, 852 + "default_collapsed": False, 853 + "first_week_framing": None, 854 + "summary_line": summary_line, 855 + "details": None, 856 + "sparse_lines": [ 857 + "I didn't produce any facet newsletters.", 858 + "There wasn't much else to process.", 859 + ], 860 + "status_reasons": status_reasons, 861 + } 862 + 863 + details = [] 864 + if mode == "degraded": 865 + details.extend(_format_gap_bullets(pipeline_summary, knowledge_graph, briefing)) 866 + 867 + details.append( 868 + _format_newsletter_summary(successful_newsletters, attempted_newsletters) 869 + ) 870 + if knowledge_graph.get("fresh"): 871 + details.append( 872 + "I refreshed your knowledge graph" 873 + + ( 874 + f" at {knowledge_graph['updated_label']}." 875 + if knowledge_graph.get("updated_label") 876 + else "." 877 + ) 878 + ) 879 + if briefing.get("valid"): 880 + details.append( 881 + "I prepared your morning briefing" 882 + + ( 883 + f" at {briefing['generated_label']}." 884 + if briefing.get("generated_label") 885 + else "." 886 + ) 887 + ) 888 + 889 + heatmap_summary = _format_heatmap_summary(stats_data) 890 + if heatmap_summary: 891 + details.append(heatmap_summary) 892 + 893 + for activity in activities[:2]: 894 + details.append(_format_activity_label(activity)) 895 + 896 + entity_summary = _format_entity_summary(entities) 897 + if entity_summary: 898 + details.append(entity_summary) 899 + 900 + default_collapsed = mode == "healthy" and journal_age_days >= 8 901 + return { 902 + "title": ( 903 + "⚠ Yesterday's processing" 904 + if mode == "degraded" 905 + else "Yesterday's processing" 906 + ), 907 + "mode": mode, 908 + "default_collapsed": default_collapsed, 909 + "first_week_framing": first_week_framing, 910 + "summary_line": _format_processing_summary( 911 + mode, 912 + successful_newsletters, 913 + attempted_newsletters, 914 + knowledge_graph, 915 + briefing, 916 + ), 917 + "details": details, 918 + "sparse_lines": None, 919 + "status_reasons": status_reasons, 920 + } 921 + 922 + 343 923 def _freshness_hours(cadence) -> int: 344 924 """Return freshness window in hours based on routine cadence type.""" 345 925 if isinstance(cadence, dict): ··· 606 1186 def _build_pulse_context() -> dict[str, Any]: 607 1187 """Build the full Pulse page context.""" 608 1188 today = _today() 1189 + yesterday = _yesterday() 609 1190 now = datetime.now() 1191 + journal_age_days = _count_journal_age_days(today) 610 1192 611 1193 capture_status = get_capture_health()["status"] 612 1194 cached = get_cached_state() ··· 775 1357 logger.warning("pipeline_status unavailable", exc_info=True) 776 1358 pipeline_status = None 777 1359 1360 + yesterday_processing = _summarize_yesterday_processing(yesterday, journal_age_days) 1361 + 778 1362 return { 779 1363 "today": today, 780 1364 "now": now, ··· 808 1392 "briefing_needs_deduped": briefing_needs_deduped, 809 1393 "briefing_needs_shared_count": briefing_needs_shared_count, 810 1394 "briefing_needs_badge": briefing_needs_badge, 1395 + "yesterday_processing": yesterday_processing, 811 1396 "show_welcome": show_welcome, 812 1397 "narrative_summary": narrative_summary, 813 1398 "routines_summary": routines_summary,
+109
apps/home/workspace.html
··· 415 415 outline: 2px solid #6366f1; 416 416 outline-offset: 2px; 417 417 } 418 + .pulse-yesterday-header:focus-visible { 419 + outline: 2px solid #6366f1; 420 + outline-offset: 2px; 421 + } 418 422 .pulse-briefing-header:hover { 419 423 background: #f8fafc; 420 424 border-radius: 6px; 421 425 } 426 + .pulse-yesterday-header:hover { 427 + background: #f8fafc; 428 + border-radius: 6px; 429 + } 422 430 .pulse-briefing-header:active { 431 + background: #f1f5f9; 432 + } 433 + .pulse-yesterday-header:active { 423 434 background: #f1f5f9; 424 435 } 425 436 .pulse-tell-more { ··· 663 674 flex-direction: column; 664 675 gap: 1rem; 665 676 } 677 + .pulse-yesterday { 678 + padding: 1.25rem; 679 + background: #fff; 680 + border-radius: 10px; 681 + border: 1px solid #e2e8f0; 682 + } 683 + .pulse-yesterday-header { 684 + display: flex; 685 + align-items: center; 686 + gap: 0.75rem; 687 + cursor: pointer; 688 + transition: background 0.15s ease; 689 + } 690 + .pulse-yesterday-summary { 691 + font-size: 0.85rem; 692 + color: #64748b; 693 + margin-top: 0.5rem; 694 + } 695 + .pulse-yesterday[data-collapsed="false"] .pulse-yesterday-header { 696 + border-bottom: 1px solid #e2e8f0; 697 + padding-bottom: 0.5rem; 698 + margin-bottom: 0.5rem; 699 + } 700 + .pulse-yesterday[data-collapsed="false"] .pulse-yesterday-summary { display: none; } 701 + .pulse-yesterday[data-collapsed="true"] .pulse-yesterday-body { display: none; } 702 + .pulse-yesterday-body { 703 + margin-top: 1rem; 704 + color: #334155; 705 + } 706 + .pulse-yesterday-body p { 707 + margin: 0 0 0.75rem; 708 + line-height: 1.6; 709 + } 710 + .pulse-yesterday-body p:last-child { 711 + margin-bottom: 0; 712 + } 713 + .pulse-yesterday-framing { 714 + font-size: 0.9rem; 715 + color: #475569; 716 + } 717 + .pulse-yesterday-details { 718 + margin: 0; 719 + padding-left: 1.25rem; 720 + color: #334155; 721 + } 722 + .pulse-yesterday-details li { 723 + margin: 0.35rem 0; 724 + line-height: 1.55; 725 + } 666 726 .pulse-briefing-section-toggle { 667 727 font-size: 0.75rem; 668 728 font-weight: 600; ··· 799 859 .pulse-routines, 800 860 .pulse-skills, 801 861 .pulse-network, 862 + .pulse-yesterday, 802 863 .pulse-briefing-card, 803 864 .pulse-empty-state { 804 865 padding-left: 2rem; ··· 877 938 .pulse-today, 878 939 .pulse-needs, 879 940 .pulse-network, 941 + .pulse-yesterday, 880 942 .pulse-briefing-card, 881 943 .pulse-empty-state { 882 944 padding: 0.75rem; ··· 894 956 .pulse-routines, 895 957 .pulse-skills, 896 958 .pulse-network, 959 + .pulse-yesterday, 897 960 .pulse-briefing-card, 898 961 .pulse-empty-state { 899 962 padding-left: 0.75rem; ··· 951 1014 {% endif %} 952 1015 <a href="/app/health">health →</a> 953 1016 </div> 1017 + 1018 + {% if yesterday_processing %} 1019 + <section class="pulse-yesterday" id="pulse-yesterday" data-collapsed="{{ 'true' if yesterday_processing.default_collapsed else 'false' }}"> 1020 + <div class="pulse-yesterday-header" role="button" tabindex="0" aria-expanded="{{ 'false' if yesterday_processing.default_collapsed else 'true' }}"> 1021 + <h2 class="pulse-section-header">{{ yesterday_processing.title }}</h2> 1022 + </div> 1023 + <div class="pulse-yesterday-summary">{{ yesterday_processing.summary_line }}</div> 1024 + <div class="pulse-yesterday-body"> 1025 + {% if yesterday_processing.first_week_framing %} 1026 + <p class="pulse-yesterday-framing">{{ yesterday_processing.first_week_framing }}</p> 1027 + {% endif %} 1028 + {% if yesterday_processing.mode == 'sparse' %} 1029 + {% for line in yesterday_processing.sparse_lines %} 1030 + <p>{{ line }}</p> 1031 + {% endfor %} 1032 + {% elif yesterday_processing.details %} 1033 + <ul class="pulse-yesterday-details"> 1034 + {% for line in yesterday_processing.details %} 1035 + <li>{{ line }}</li> 1036 + {% endfor %} 1037 + </ul> 1038 + {% endif %} 1039 + </div> 1040 + </section> 1041 + {% endif %} 954 1042 955 1043 <!-- Welcome --> 956 1044 {% if show_welcome %} ··· 1353 1441 }); 1354 1442 } 1355 1443 1444 + var yesterdayHeader = document.querySelector('.pulse-yesterday-header'); 1445 + if (yesterdayHeader) { 1446 + yesterdayHeader.addEventListener('click', function() { 1447 + toggleYesterdayCard(); 1448 + }); 1449 + yesterdayHeader.addEventListener('keydown', function(e) { 1450 + if (e.key === 'Enter' || e.key === ' ') { 1451 + e.preventDefault(); 1452 + toggleYesterdayCard(); 1453 + } 1454 + }); 1455 + } 1456 + 1356 1457 // WebSocket live updates 1357 1458 if (window.appEvents) { 1358 1459 // Vital signs: supervisor.status, observe.observed, observe.status ··· 1381 1482 if (!card || card.dataset.phase === 'pending') return; 1382 1483 card.dataset.collapsed = card.dataset.collapsed === 'true' ? 'false' : 'true'; 1383 1484 var header = card.querySelector('.pulse-briefing-header'); 1485 + if (header) header.setAttribute('aria-expanded', card.dataset.collapsed === 'false' ? 'true' : 'false'); 1486 + }; 1487 + 1488 + window.toggleYesterdayCard = function() { 1489 + var card = document.getElementById('pulse-yesterday'); 1490 + if (!card) return; 1491 + card.dataset.collapsed = card.dataset.collapsed === 'true' ? 'false' : 'true'; 1492 + var header = card.querySelector('.pulse-yesterday-header'); 1384 1493 if (header) header.setAttribute('aria-expanded', card.dataset.collapsed === 'false' ? 'true' : 'false'); 1385 1494 }; 1386 1495
+396
docs/design/yesterdays-processing-card.md
··· 1 + # Yesterday's Processing Card 2 + 3 + ## 1. Helper layout in `apps/home/routes.py` 4 + 5 + Use the proposed top-level decomposition. It is the smallest layout that keeps template logic dumb and testable. 6 + 7 + - `_yesterday() -> str` 8 + Returns yesterday in local time as `YYYYMMDD`. 9 + 10 + - `_count_journal_age_days(today: str) -> int` 11 + Scans `Path(get_journal()) / "chronicle"` for `YYYYMMDD` children and returns `(today - earliest).days`. Returns `0` when `chronicle/` is missing or has no day dirs. This matches `think/utils.py:105-124`, `think/utils.py:154-208`. 12 + 13 + - `_summarize_yesterday_processing(yesterday: str, journal_age_days: int) -> dict | None` 14 + Returns the full rendered card contract or `None` to hide the card. 15 + 16 + Internal helpers called only by `_summarize_yesterday_processing`: 17 + 18 + - `_load_yesterday_stats(yesterday: str) -> dict | None` 19 + Reads `chronicle/{yesterday}/stats.json`. Returns `None` on missing/invalid file. 20 + 21 + - `_load_yesterday_pipeline_summary(yesterday: str) -> dict` 22 + Thin wrapper over `summarize_pipeline_day(yesterday)` from `think/pipeline_health.py:23-110`. 23 + 24 + - `_collect_entities_yesterday(yesterday: str) -> list[dict[str, Any]]` 25 + Same DB access pattern as `_collect_entities_today` at `apps/home/routes.py:296+`: `try`/`finally`, `conn.close()`, graceful empty on error. 26 + Query `entity_signals` grouped by `entity_name`, join/fallback through `entities` identity rows the same way the existing helper does. 27 + 28 + - `_collect_top_activities_yesterday(yesterday: str) -> list[dict[str, Any]]` 29 + Iterate enabled facets, call `load_activity_records(facet, yesterday)`, derive `duration_minutes` via `estimate_duration_minutes(record["segments"])`, annotate `facet`, normalize display title, and sort descending by duration. 30 + 31 + - `_top_heatmap_hours(stats_data: dict) -> list[int]` 32 + Reads `stats_data["heatmap_data"]["hours"]`, keeps the top 3 non-zero hours, sorts by minutes desc then hour asc. 33 + 34 + - `_knowledge_graph_freshness(yesterday: str) -> dict` 35 + Reads `chronicle/{yesterday}/agents/knowledge_graph.md`, checks existence and `st_mtime` freshness using the relaxed rule in section 4. 36 + 37 + - `_briefing_freshness(today: str) -> dict` 38 + Reads `journal/sol/briefing.md` with local `frontmatter.load`. Valid only when frontmatter has `type: morning_briefing` and a parseable `generated` timestamp whose local date is `today`. 39 + 40 + - `_newsletter_attempts_from_dream_logs(yesterday: str) -> tuple[int, int]` 41 + Option A helper from section 3. Counts successful facet newsletters from files plus failed facet newsletter attempts from dream logs. 42 + 43 + - Formatting helpers 44 + `_format_duration`, `_format_hour_label`, `_format_entity_summary`, `_format_activity_label`, `_format_newsletter_summary`, `_format_processing_summary`. 45 + All user-facing copy, pluralization, and warning phrasing stays here, not in Jinja. 46 + 47 + Hide conditions: 48 + 49 + - `stats.json` missing. 50 + - `journal_age_days == 0`. 51 + - Stats are effectively empty: no transcript duration, no transcript segments, no facet activity, no activity records, no entity signals. 52 + 53 + Mode resolution: 54 + 55 + - `sparse` 56 + Yesterday has some transcript duration or segments, but no facet-level activity (`facet_data` empty) and no derived top activities. 57 + 58 + - `degraded` 59 + Any overnight processing gap is present: 60 + newsletter partial count, pipeline summary not healthy, missing/invalid briefing, or stale/missing knowledge graph. 61 + 62 + - `healthy` 63 + Non-sparse day with no degradation signals. 64 + 65 + Collapse defaults: 66 + 67 + - `sparse`: expanded. 68 + - `degraded`: expanded. 69 + - `healthy`: expanded for journal age days 1-7, collapsed for day 8+. 70 + 71 + ## 2. Return contract 72 + 73 + Use one dict under the pulse context key `yesterday_processing`. 74 + 75 + Fields: 76 + 77 + - `title` 78 + `"Yesterday's processing"` or `"⚠ Yesterday's processing"`. 79 + 80 + - `mode` 81 + One of `sparse`, `healthy`, `degraded`. 82 + 83 + - `default_collapsed` 84 + Server-computed boolean; template only mirrors it into `data-collapsed` and `aria-expanded`. 85 + 86 + - `first_week_framing` 87 + Optional single paragraph shown only when `journal_age_days <= 7` and `mode != "sparse"`. 88 + 89 + - `summary_line` 90 + Always-visible collapsed summary. 91 + 92 + - `details` 93 + Preformatted bullet strings for healthy/degraded modes. 94 + 95 + - `sparse_lines` 96 + Exactly two preformatted strings when `mode == "sparse"`, else `None`. 97 + 98 + - `status_reasons` 99 + Internal-only list of machine-readable reason tags such as `newsletter_partial`, `pipeline_warning`, `knowledge_graph_stale`, `briefing_missing`. Useful for tests and future copy changes; template does not inspect it. 100 + 101 + Why this shape: 102 + 103 + - It keeps the template mechanical. 104 + - It keeps copy decisions fully in Python. 105 + - It supports stable tests without string-parsing Jinja output. 106 + 107 + Recommended rendered content by mode: 108 + 109 + - `sparse` 110 + Summary line: light-day summary using total duration. 111 + Body: two lines only, no bullet list, no first-week framing. 112 + 113 + - `healthy` 114 + Summary line: “I wrote N newsletters, refreshed your knowledge graph, and prepared your morning briefing.” 115 + Details: 116 + newsletter result, knowledge graph refresh, briefing generation time, top heatmap hours, top activities, top entities. 117 + 118 + - `degraded` 119 + Summary line: “I wrote N of M newsletters, but some overnight processing didn’t finish.” if Option A is selected. 120 + If no denominator is available, summary shifts to “I wrote N newsletters, but some overnight processing didn’t finish.” 121 + Details: 122 + failure/gap bullets first, then normal detail bullets. 123 + 124 + ## 3. Q2 — degraded-day denominator semantics 125 + 126 + ### Ground truth 127 + 128 + - `agent.fail` records include `name`, `agent_id`, `state`, and optional `facet`, but `summarize_pipeline_day()` counts every failure and drops `facet` from `failed_list`. See `think/pipeline_health.py:81-99`. 129 + - `stats.json.facet_data` is not a newsletter ledger. It is built from `events.jsonl` durations in `think/journal_stats.py:296-319` and surfaced in `apps/home/routes.py:616-621`. 130 + - The facet newsletter writer is `sol call journal news`, implemented by `think/tools/facets.py:61-106`. 131 + - The newsletter prompt key is stable: `facet_newsletter`. 132 + Reason: 133 + system talent config keys come from `talent/*.md` filename stems in `think/talent.py:228-235`, and the file is `talent/facet_newsletter.md:1-15`. 134 + Dream logs emit `name=prompt_name` unchanged for dispatch and fail/complete events in `think/dream.py:1277-1292` and `think/dream.py:365-389`. 135 + 136 + ### Option A — re-parse dream JSONL for newsletter-specific facet fails 137 + 138 + Read `chronicle/{yesterday}/health/*_daily_dream.jsonl` and count `agent.fail` records where: 139 + 140 + - `event == "agent.fail"` 141 + - `facet` is present 142 + - `name == "facet_newsletter"` 143 + 144 + Count successes from actual files: 145 + 146 + - `len(list(Path(get_journal()).glob(f"facets/*/news/{yesterday}.md")))` 147 + 148 + Formula: 149 + 150 + - `N = successful_newsletter_files` 151 + - `M = successful_newsletter_files + failed_facet_newsletter_attempts` 152 + 153 + Pros: 154 + 155 + - Faithful to the product language “wrote N of M newsletters”. 156 + - Uses stable prompt key. 157 + - Reads only local files already in the journal. 158 + 159 + Cons: 160 + 161 + - If the runtime is not currently dispatching `facet_newsletter` into daily dream logs, `failed_facet_newsletter_attempts` will often be `0`. 162 + - Non-newsletter pipeline failures still need separate degraded copy. 163 + 164 + ### Option B — re-parse any facet-scoped fail 165 + 166 + Count every `agent.fail` with a `facet` field, regardless of `name`. 167 + 168 + Pros: 169 + 170 + - Captures “something facet-scoped didn’t finish”. 171 + 172 + Cons: 173 + 174 + - Overstates the denominator for “newsletters”. 175 + - Would require copy shift to “facet summaries” or similar. 176 + 177 + ### Option C — drop denominator 178 + 179 + Use only `N` from newsletter files and separate degraded copy for generic pipeline issues. 180 + 181 + Pros: 182 + 183 + - Never lies about `M`. 184 + 185 + Cons: 186 + 187 + - Gives up the specific `N of M` story. 188 + 189 + ### Option D — extend `summarize_pipeline_day` 190 + 191 + Preserve `facet` and maybe richer classification in `failed_list`. 192 + 193 + Pros: 194 + 195 + - Best long-term shape. 196 + 197 + Cons: 198 + 199 + - Wider-than-needed change. Out of scope for this card. 200 + 201 + ### Recommendation 202 + 203 + Pick **Option A**. 204 + 205 + Implementation details: 206 + 207 + - Success path reads `facets/*/news/{yesterday}.md`. 208 + - Failure path reads `chronicle/{yesterday}/health/*_daily_dream.jsonl`. 209 + - Exact agent-name match: `facet_newsletter`. 210 + 211 + Fallback behavior inside Option A: 212 + 213 + - If `M == 0`, do not render `0 of 0`. 214 + Use the non-denominator sentence: “I didn’t produce any facet newsletters.” 215 + - Generic non-newsletter pipeline failures still flip the card to `degraded`, but they appear as separate warning bullets rather than inflating `M`. 216 + 217 + ## 4. Q3 — `knowledge_graph.md` mtime false-negative edge case 218 + 219 + Use the relaxed rule: 220 + 221 + - Fresh when the file exists and `mtime >= start_of_yesterday_local`. 222 + 223 + Do not require `mtime` to fall strictly within yesterday’s wall-clock day. 224 + 225 + Rationale: 226 + 227 + - Prep already found a real case where `chronicle/20260415/agents/knowledge_graph.md` had `mtime` on `2026-04-16 07:23:43`. 228 + - The intent of the card is “did the overnight processing refresh yesterday’s graph?”, not “did the write finish before midnight”. 229 + - This rule admits same-day and overnight-after-midnight completions without introducing an arbitrary 36-hour window. 230 + 231 + Notes: 232 + 233 + - Compare using local time boundaries. 234 + - Use `st_mtime`, not birth time or `ctime`. 235 + 236 + ## 5. Template placement + CSS 237 + 238 + Placement: 239 + 240 + - Insert immediately after the closing `pulse-vitals` block in `apps/home/workspace.html`, before the welcome/briefing blocks. In current file layout that is directly after the `pulse-vitals` div shown around `apps/home/workspace.html:924-953`. 241 + 242 + Markup: 243 + 244 + - Outer element: `<section class="pulse-yesterday" id="pulse-yesterday" data-collapsed="...">` 245 + - Header: 246 + `.pulse-yesterday-header` 247 + `role="button"` 248 + `tabindex="0"` 249 + `aria-expanded="..."` 250 + - Summary row: 251 + `.pulse-yesterday-summary` 252 + - Body: 253 + `.pulse-yesterday-body` 254 + - Optional framing paragraph: 255 + `.pulse-yesterday-framing` 256 + - Bullet list: 257 + `.pulse-yesterday-details` 258 + 259 + Interaction: 260 + 261 + - Mirror the briefing card pattern in `apps/home/workspace.html:966-1005` and `apps/home/workspace.html:1379-1392`. 262 + - Add `toggleYesterdayCard` as a sibling to `toggleBriefingCard`, or generalize both onto a shared helper if that is a pure extraction with no behavior change. 263 + - Do not add `pulse-yesterday` to `SECTION_IDS`; collapse state is server-driven each render. 264 + 265 + CSS: 266 + 267 + - Inline in `apps/home/workspace.html` with the other `.pulse-*` blocks. 268 + - Reuse briefing-card visual structure: 269 + rounded card, bordered shell, hover/focus treatment on header, summary hidden when expanded, body hidden when collapsed. 270 + - New classes only: 271 + `.pulse-yesterday` 272 + `.pulse-yesterday-header` 273 + `.pulse-yesterday-body` 274 + `.pulse-yesterday-summary` 275 + `.pulse-yesterday-details` 276 + `.pulse-yesterday-framing` 277 + - No new global CSS files. 278 + 279 + ## 6. First-week framing 280 + 281 + Behavior: 282 + 283 + - Render one framing paragraph at the top of the expanded body when `journal_age_days <= 7` and `mode != "sparse"`. 284 + 285 + Open issue: 286 + 287 + - I could not recover the exact first-week copy from the checked-in repo or local journal/task artifacts. 288 + - The implementation should use the scope’s verbatim wording once Jer confirms it. 289 + 290 + Interim design assumption: 291 + 292 + - The framing remains a single sentence or short paragraph, not a bullet. 293 + - It is omitted entirely in sparse mode. 294 + 295 + ## 7. Refresh behavior (v1) 296 + 297 + - No new endpoint. 298 + - No new telemetry. 299 + - No new app-event listener dedicated to this card. 300 + - The card data is included in the existing `_build_pulse_context()` return and therefore in `/app/home/api/pulse`. 301 + - Initial page render is the primary path. 302 + - Full-page reload fallback is acceptable in v1. 303 + 304 + This keeps the frontend change small. If later we want live card refresh, it can read from the already-extended `/app/home/api/pulse` payload without another backend change. 305 + 306 + ## 8. Tests plan 307 + 308 + Create a new test module: `tests/test_home_yesterdays_processing.py`. 309 + 310 + Unit tests: 311 + 312 + - `test_yesterdays_card_hidden_when_stats_missing` 313 + - `test_yesterdays_card_hidden_when_all_zero` 314 + - `test_yesterdays_card_sparse_mode_copy` 315 + - `test_yesterdays_card_healthy_collapsed_on_day_8_plus` 316 + - `test_yesterdays_card_healthy_expanded_with_framing_on_days_1_to_7` 317 + - `test_yesterdays_card_degraded_shows_warning_and_partial_count` 318 + - `test_format_duration_boundaries` 319 + - `test_entity_grouping_people_first_zero_dropped_plurals` 320 + - `test_heatmap_peaks_top_3` 321 + - `test_activity_bullet_title_duration_facet` 322 + - `test_knowledge_graph_refresh_detection_yesterday_and_overnight` 323 + - `test_briefing_frontmatter_missing_counts_as_gap` 324 + - `test_newsletter_attempts_option_a_matches_facet_newsletter_failures_only` 325 + 326 + Fixture plan: 327 + 328 + - `tests/fixtures/journal/chronicle/20260415/` 329 + Dense day fixture with: 330 + `stats.json`, 331 + one or two `health/*_daily_dream.jsonl` files, 332 + one activity file under `facets/*/activities/20260415.jsonl`, 333 + `agents/knowledge_graph.md`. 334 + 335 + - `tests/fixtures/journal/chronicle/20260414/` 336 + Sparse day fixture with: 337 + `stats.json` showing transcript duration/segments but empty `facet_data`, 338 + no activities, 339 + optional empty `health/`. 340 + 341 + - `tests/fixtures/journal/chronicle/20260416/` 342 + Existing empty day fixture reused for all-zero/missing cases. 343 + 344 + Supporting non-chronicle fixture: 345 + 346 + - `tests/fixtures/journal/sol/briefing.md` 347 + Valid morning-briefing frontmatter fixture for healthy cases. 348 + Tests that need missing/invalid frontmatter can overwrite or delete it in `tmp_path`. 349 + 350 + Fixture minimization rule: 351 + 352 + - Seed only the fields each test asserts on. 353 + - Keep dream logs to the minimum lines needed: `run.start`, `agent.dispatch`, `agent.complete` or `agent.fail`, `run.complete`. 354 + 355 + ## 9. Non-goals 356 + 357 + - No “today’s processing” surface. 358 + - No historical browser or date picker. 359 + - No weekly rollup. 360 + - No email digest. 361 + - No sharing or screenshots. 362 + - No new endpoints. 363 + - No new telemetry or new event types. 364 + - No changes to briefing preamble or other pulse sections. 365 + - No addition to `SECTION_IDS`. 366 + - No new HTTP dependencies. 367 + 368 + ## Implementation sequence 369 + 370 + 1. Add route helpers and card contract assembly in `apps/home/routes.py`. 371 + 2. Wire the new dict into `_build_pulse_context()` and `/app/home/api/pulse`. 372 + 3. Add template markup and scoped CSS in `apps/home/workspace.html`. 373 + 4. Add the card toggle helper, but no new refresh wiring. 374 + 5. Add focused tests and minimal fixtures. 375 + 376 + ## Review gate — decisions for jer 377 + 378 + - **Q2 denominator choice:** Recommend **Option A**. Match failed newsletter attempts by exact dream-log agent name `facet_newsletter`; count successes from `facets/*/news/{yesterday}.md`. 379 + - **Q3 knowledge-graph freshness rule:** Recommend **fresh when `mtime >= start_of_yesterday_local`**. This intentionally counts overnight-after-midnight completions as fresh. 380 + - **First-week framing copy:** Exact scope text was not recoverable from checked-in artifacts I could search. Need Jer to confirm the verbatim copy before implementation. 381 + 382 + ## Gate answers (VPE, 2026-04-17) 383 + 384 + All three gate items resolved. Proceed to `implement` stage. 385 + 386 + - **Q2 denominator:** Go with **Option A** as recommended. Successes from `facets/*/news/{yesterday}.md`. Failures from dream-log `agent.fail` where `name == "facet_newsletter"` and `facet` is present. When current pipeline emits no `facet_newsletter` fails (which is the common case today), `M == N` and the `N of M` sentence degenerates into a simple `N` — that's fine, honest, and forward-compatible for when we start logging newsletter failures under that exact key. Use the sparse fallback "I didn't produce any facet newsletters." when both are zero. 387 + - **Q3 knowledge-graph freshness:** Use the **relaxed rule**: fresh when `knowledge_graph.md` exists and `st_mtime >= start_of_yesterday_local`. Overnight-after-midnight completions count. Use local time boundaries. Don't use birth/ctime. 388 + - **First-week framing copy (verbatim):** The exact copy IS in the scope (top-level note) and in the approved CPO spec. Use this text, unchanged, when `journal_age_days <= 7` and `mode != "sparse"`: 389 + 390 + > Most of what I learn becomes useful in the third or fourth week, when I've seen enough patterns to surface them. For now, here's what's already happening: 391 + 392 + Render as one `<p class="pulse-yesterday-framing">` at the top of the expanded body. Omit entirely in sparse mode. 393 + 394 + Also confirming by reference (no change to spec): the mockup copy for the other states lives in the scope and in `cpo/specs/in-flight/yesterdays-processing-card.md`. Match those strings where they appear verbatim; don't paraphrase owner-facing language. 395 + 396 + Proceed with the implementation sequence already in the design doc. Run `make test` before `hop processed`.
+3
tests/fixtures/journal/chronicle/20260415/agents/knowledge_graph.md
··· 1 + # Knowledge graph refresh 2 + 3 + Updated overnight.
+1
tests/fixtures/journal/chronicle/20260416/.gitkeep
··· 1 +
+1
tests/fixtures/journal/facets/personal/activities/20260415.jsonl
··· 1 + {"id":"writing_140000_300","activity":"writing","description":"Drafted weekend notes","segments":["140000_300","140500_300"],"active_entities":[]}
+3
tests/fixtures/journal/facets/personal/news/20260415.md
··· 1 + # Personal newsletter 2 + 3 + Summary for personal.
+2
tests/fixtures/journal/facets/work/activities/20260415.jsonl
··· 1 + {"id":"coding_090000_300","activity":"coding","description":"Extended coding session with Git and VS Code","segments":["090000_300","090500_300","091000_300"],"active_entities":["jane_doe"]} 2 + {"id":"meeting_110000_300","activity":"meeting","description":"Project sync with design","segments":["110000_300","110500_300"],"active_entities":["acme_corp"]}
+3
tests/fixtures/journal/facets/work/news/20260415.md
··· 1 + # Work newsletter 2 + 3 + Summary for work.
+485
tests/test_home_yesterdays_processing.py
··· 1 + # SPDX-License-Identifier: AGPL-3.0-only 2 + # Copyright (c) 2026 sol pbc 3 + 4 + """Tests for the Yesterday's processing home card.""" 5 + 6 + from __future__ import annotations 7 + 8 + import json 9 + import os 10 + import shutil 11 + from datetime import datetime 12 + from pathlib import Path 13 + 14 + import pytest 15 + 16 + from apps.home.routes import ( 17 + _briefing_freshness, 18 + _build_pulse_context, 19 + _format_activity_label, 20 + _format_duration, 21 + _format_entity_summary, 22 + _format_gap_bullets, 23 + _format_heatmap_summary, 24 + _knowledge_graph_freshness, 25 + _newsletter_attempts_from_dream_logs, 26 + _summarize_yesterday_processing, 27 + ) 28 + from think.indexer.journal import get_journal_index 29 + 30 + FIXTURES = Path(__file__).parent / "fixtures" / "journal" 31 + 32 + 33 + def _copy_fixture_file(journal: Path, rel_path: str) -> None: 34 + src = FIXTURES / rel_path 35 + dst = journal / rel_path 36 + dst.parent.mkdir(parents=True, exist_ok=True) 37 + shutil.copy2(src, dst) 38 + 39 + 40 + def _write_facet_meta(journal: Path, facet: str, title: str) -> None: 41 + path = journal / "facets" / facet / "facet.json" 42 + path.parent.mkdir(parents=True, exist_ok=True) 43 + path.write_text( 44 + json.dumps( 45 + { 46 + "title": title, 47 + "description": "", 48 + "color": "#0f172a", 49 + "emoji": "📁", 50 + } 51 + ), 52 + encoding="utf-8", 53 + ) 54 + 55 + 56 + def _seed_journal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: 57 + journal = tmp_path / "journal" 58 + journal.mkdir() 59 + monkeypatch.setenv("_SOLSTONE_JOURNAL_OVERRIDE", str(journal)) 60 + 61 + for rel_path in [ 62 + "chronicle/20260415/stats.json", 63 + "chronicle/20260415/agents/knowledge_graph.md", 64 + "chronicle/20260415/health/100_daily_dream.jsonl", 65 + "chronicle/20260414/stats.json", 66 + ]: 67 + _copy_fixture_file(journal, rel_path) 68 + 69 + for rel_path in [ 70 + "facets/work/activities/20260415.jsonl", 71 + "facets/personal/activities/20260415.jsonl", 72 + "facets/work/news/20260415.md", 73 + "facets/personal/news/20260415.md", 74 + ]: 75 + _copy_fixture_file(journal, rel_path) 76 + 77 + (journal / "chronicle" / "20260416").mkdir(parents=True, exist_ok=True) 78 + _write_facet_meta(journal, "work", "Work") 79 + _write_facet_meta(journal, "personal", "Personal") 80 + return journal 81 + 82 + 83 + def _write_briefing( 84 + journal: Path, generated: str, *, metadata_type: str = "morning_briefing" 85 + ) -> None: 86 + path = journal / "sol" / "briefing.md" 87 + path.parent.mkdir(parents=True, exist_ok=True) 88 + path.write_text( 89 + ( 90 + f"---\n" 91 + f"type: {metadata_type}\n" 92 + f'generated: "{generated}"\n' 93 + f"---\n\n" 94 + "## Your Day\n\n" 95 + "- One thing.\n" 96 + ), 97 + encoding="utf-8", 98 + ) 99 + 100 + 101 + def _seed_entities(journal: Path, day: str = "20260415") -> None: 102 + conn, _ = get_journal_index(str(journal)) 103 + try: 104 + conn.execute("DELETE FROM entity_signals") 105 + conn.execute("DELETE FROM entities") 106 + conn.executemany( 107 + """ 108 + INSERT INTO entities(entity_id, source, path, name, type) 109 + VALUES (?, 'identity', ?, ?, ?) 110 + """, 111 + [ 112 + ("jane_doe", "entities/jane_doe/entity.json", "Jane Doe", "person"), 113 + ( 114 + "alice_johnson", 115 + "entities/alice_johnson/entity.json", 116 + "Alice Johnson", 117 + "person", 118 + ), 119 + ( 120 + "product_roadmap", 121 + "entities/product_roadmap/entity.json", 122 + "Product Roadmap", 123 + "topic", 124 + ), 125 + ( 126 + "launch_decision", 127 + "entities/launch_decision/entity.json", 128 + "Launch decision", 129 + "decision", 130 + ), 131 + ], 132 + ) 133 + conn.executemany( 134 + """ 135 + INSERT INTO entity_signals( 136 + signal_type, entity_name, entity_type, target_name, 137 + relationship_type, day, facet, event_title, event_type, path 138 + ) 139 + VALUES (?, ?, NULL, NULL, NULL, ?, ?, NULL, NULL, ?) 140 + """, 141 + [ 142 + ("mention", "jane_doe", day, "work", f"{day}/agents/flow.md"), 143 + ("mention", "alice_johnson", day, "work", f"{day}/agents/flow.md"), 144 + ( 145 + "mention", 146 + "product_roadmap", 147 + day, 148 + "work", 149 + f"{day}/agents/knowledge_graph.md", 150 + ), 151 + ( 152 + "mention", 153 + "launch_decision", 154 + day, 155 + "work", 156 + f"{day}/agents/knowledge_graph.md", 157 + ), 158 + ], 159 + ) 160 + conn.commit() 161 + finally: 162 + conn.close() 163 + 164 + 165 + def _append_dream_log( 166 + journal: Path, day: str, name: str, *, facet: str | None = None 167 + ) -> None: 168 + path = journal / "chronicle" / day / "health" / "101_daily_dream.jsonl" 169 + path.parent.mkdir(parents=True, exist_ok=True) 170 + with path.open("a", encoding="utf-8") as handle: 171 + record = { 172 + "event": "agent.fail", 173 + "mode": "daily", 174 + "name": name, 175 + "state": "error", 176 + } 177 + if facet is not None: 178 + record["facet"] = facet 179 + handle.write(json.dumps(record) + "\n") 180 + 181 + 182 + def _set_mtime(path: Path, dt: datetime) -> None: 183 + ts = dt.timestamp() 184 + os.utime(path, (ts, ts)) 185 + 186 + 187 + def test_yesterdays_card_hidden_when_stats_missing(tmp_path, monkeypatch): 188 + journal = _seed_journal(tmp_path, monkeypatch) 189 + _write_briefing(journal, "2026-04-17T06:45:00") 190 + 191 + monkeypatch.setattr("apps.home.routes._today", lambda: "20260417") 192 + 193 + assert _summarize_yesterday_processing("20260416", 9) is None 194 + 195 + 196 + def test_yesterdays_card_hidden_when_all_zero(tmp_path, monkeypatch): 197 + journal = _seed_journal(tmp_path, monkeypatch) 198 + _write_briefing(journal, "2026-04-17T06:45:00") 199 + (journal / "chronicle" / "20260416" / "stats.json").write_text( 200 + json.dumps( 201 + { 202 + "stats": { 203 + "transcript_segments": 0, 204 + "transcript_duration": 0, 205 + }, 206 + "facet_data": {}, 207 + "heatmap_data": {"weekday": 3, "hours": {}}, 208 + } 209 + ), 210 + encoding="utf-8", 211 + ) 212 + 213 + monkeypatch.setattr("apps.home.routes._today", lambda: "20260417") 214 + 215 + assert _summarize_yesterday_processing("20260416", 9) is None 216 + 217 + 218 + def test_yesterdays_card_sparse_mode_copy(tmp_path, monkeypatch): 219 + journal = _seed_journal(tmp_path, monkeypatch) 220 + _write_briefing(journal, "2026-04-15T06:45:00") 221 + 222 + monkeypatch.setattr("apps.home.routes._today", lambda: "20260415") 223 + 224 + summary = _summarize_yesterday_processing("20260414", 2) 225 + 226 + assert summary["mode"] == "sparse" 227 + assert summary["default_collapsed"] is False 228 + assert summary["first_week_framing"] is None 229 + assert summary["summary_line"] == "I watched 45 min yesterday." 230 + assert summary["sparse_lines"] == [ 231 + "I didn't produce any facet newsletters.", 232 + "There wasn't much else to process.", 233 + ] 234 + 235 + 236 + def test_yesterdays_card_healthy_collapsed_on_day_8_plus(tmp_path, monkeypatch): 237 + journal = _seed_journal(tmp_path, monkeypatch) 238 + _write_briefing(journal, "2026-04-16T06:45:00") 239 + _seed_entities(journal) 240 + 241 + monkeypatch.setattr("apps.home.routes._today", lambda: "20260416") 242 + 243 + summary = _summarize_yesterday_processing("20260415", 8) 244 + 245 + assert summary["mode"] == "healthy" 246 + assert summary["title"] == "Yesterday's processing" 247 + assert summary["default_collapsed"] is True 248 + assert summary["first_week_framing"] is None 249 + assert ( 250 + summary["summary_line"] 251 + == "I wrote 2 newsletters, refreshed your knowledge graph, and prepared your morning briefing." 252 + ) 253 + 254 + 255 + def test_yesterdays_card_healthy_expanded_with_framing_on_days_1_to_7( 256 + tmp_path, monkeypatch 257 + ): 258 + journal = _seed_journal(tmp_path, monkeypatch) 259 + _write_briefing(journal, "2026-04-16T06:45:00") 260 + _seed_entities(journal) 261 + 262 + monkeypatch.setattr("apps.home.routes._today", lambda: "20260416") 263 + 264 + summary = _summarize_yesterday_processing("20260415", 5) 265 + 266 + assert summary["mode"] == "healthy" 267 + assert summary["default_collapsed"] is False 268 + assert ( 269 + summary["first_week_framing"] 270 + == "Most of what I learn becomes useful in the third or fourth week, when I've seen enough patterns to surface them. For now, here's what's already happening:" 271 + ) 272 + assert "I recognized 2 people, 1 decision, and 1 topic." in summary["details"] 273 + 274 + 275 + def test_yesterdays_card_degraded_shows_warning_and_partial_count( 276 + tmp_path, monkeypatch 277 + ): 278 + journal = _seed_journal(tmp_path, monkeypatch) 279 + _write_briefing(journal, "2026-04-16T06:45:00") 280 + _seed_entities(journal) 281 + _append_dream_log(journal, "20260415", "facet_newsletter", facet="personal") 282 + 283 + monkeypatch.setattr("apps.home.routes._today", lambda: "20260416") 284 + 285 + summary = _summarize_yesterday_processing("20260415", 8) 286 + 287 + assert summary["mode"] == "degraded" 288 + assert summary["title"] == "⚠ Yesterday's processing" 289 + assert summary["summary_line"] == ( 290 + "I wrote 2 of 3 newsletters, but some overnight processing didn't finish." 291 + ) 292 + assert summary["status_reasons"] == ["newsletter_partial", "pipeline_warning"] 293 + assert summary["details"][0] == "Some of my overnight work didn't finish." 294 + assert "I wrote 2 of 3 newsletters." in summary["details"] 295 + 296 + 297 + def test_yesterdays_card_degraded_zero_newsletters_keeps_failure_caveat( 298 + tmp_path, monkeypatch 299 + ): 300 + journal = _seed_journal(tmp_path, monkeypatch) 301 + _seed_entities(journal) 302 + for path in journal.glob("facets/*/news/20260415.md"): 303 + path.unlink() 304 + 305 + monkeypatch.setattr("apps.home.routes._today", lambda: "20260416") 306 + 307 + summary = _summarize_yesterday_processing("20260415", 8) 308 + 309 + assert summary["mode"] == "degraded" 310 + assert ( 311 + summary["summary_line"] 312 + == "I didn't produce any facet newsletters, and some overnight processing didn't finish." 313 + ) 314 + 315 + 316 + def test_format_duration_boundaries(): 317 + assert _format_duration(59) == "59 min" 318 + assert _format_duration(60) == "1 hour" 319 + 320 + 321 + def test_entity_grouping_people_first_zero_dropped_plurals(): 322 + assert ( 323 + _format_entity_summary( 324 + [ 325 + {"entity_type": "topic"}, 326 + {"entity_type": "person"}, 327 + {"entity_type": "person"}, 328 + ] 329 + ) 330 + == "I recognized 2 people and 1 topic." 331 + ) 332 + 333 + 334 + def test_heatmap_peaks_top_3(): 335 + assert ( 336 + _format_heatmap_summary( 337 + { 338 + "heatmap_data": { 339 + "hours": { 340 + "9": 45.0, 341 + "11": 40.0, 342 + "14": 30.0, 343 + "10": 20.0, 344 + } 345 + } 346 + } 347 + ) 348 + == "I watched most closely during 9-10am · 11am-12pm · 2-3pm." 349 + ) 350 + 351 + 352 + def test_activity_bullet_title_duration_facet(tmp_path, monkeypatch): 353 + _seed_journal(tmp_path, monkeypatch) 354 + monkeypatch.setattr("apps.home.routes._today", lambda: "20260416") 355 + activity = _summarize_yesterday_processing("20260415", 8) 356 + 357 + first_activity = next( 358 + detail for detail in activity["details"] if detail.startswith("I took notes on") 359 + ) 360 + 361 + assert ( 362 + first_activity 363 + == "I took notes on Extended coding session with Git and VS Code for 15 min in work." 364 + ) 365 + assert ( 366 + _format_activity_label( 367 + { 368 + "title": "Drafted weekend notes", 369 + "duration_minutes": 10, 370 + "facet": "personal", 371 + } 372 + ) 373 + == "I took notes on Drafted weekend notes for 10 min in personal." 374 + ) 375 + 376 + 377 + def test_knowledge_graph_refresh_detection_yesterday_and_overnight( 378 + tmp_path, monkeypatch 379 + ): 380 + journal = _seed_journal(tmp_path, monkeypatch) 381 + path = journal / "chronicle" / "20260415" / "agents" / "knowledge_graph.md" 382 + 383 + _set_mtime(path, datetime(2026, 4, 15, 12, 0, 0)) 384 + assert _knowledge_graph_freshness("20260415")["fresh"] is True 385 + 386 + _set_mtime(path, datetime(2026, 4, 16, 2, 0, 0)) 387 + assert _knowledge_graph_freshness("20260415")["fresh"] is True 388 + 389 + 390 + def test_briefing_frontmatter_missing_counts_as_gap(tmp_path, monkeypatch): 391 + journal = _seed_journal(tmp_path, monkeypatch) 392 + _seed_entities(journal) 393 + 394 + monkeypatch.setattr("apps.home.routes._today", lambda: "20260416") 395 + 396 + summary = _summarize_yesterday_processing("20260415", 8) 397 + 398 + assert summary["mode"] == "degraded" 399 + assert "briefing_missing" in summary["status_reasons"] 400 + assert "I didn't prepare your morning briefing overnight." in summary["details"] 401 + assert _briefing_freshness("20260416") == { 402 + "exists": False, 403 + "valid": False, 404 + "generated_label": None, 405 + } 406 + 407 + 408 + def test_gap_bullets_show_specific_daily_and_activity_without_generic(): 409 + bullets = _format_gap_bullets( 410 + { 411 + "anomalies": [ 412 + {"kind": "daily_agents_missing"}, 413 + {"kind": "activity_agents_missing"}, 414 + {"kind": "agent_failure"}, 415 + ] 416 + }, 417 + {"fresh": True}, 418 + {"valid": True}, 419 + ) 420 + 421 + assert bullets == [ 422 + "I didn't finish the full overnight review.", 423 + "I didn't finish writing all of yesterday's notes.", 424 + ] 425 + 426 + 427 + def test_newsletter_attempts_option_a_matches_facet_newsletter_failures_only( 428 + tmp_path, monkeypatch 429 + ): 430 + journal = _seed_journal(tmp_path, monkeypatch) 431 + _append_dream_log(journal, "20260415", "facet_newsletter", facet="work") 432 + _append_dream_log(journal, "20260415", "knowledge_graph", facet="work") 433 + _append_dream_log(journal, "20260415", "facet_newsletter") 434 + 435 + assert _newsletter_attempts_from_dream_logs("20260415") == (2, 3) 436 + 437 + 438 + def test_build_pulse_context_includes_yesterday_processing(monkeypatch): 439 + monkeypatch.setattr( 440 + "apps.home.routes.get_capture_health", 441 + lambda: {"status": "active", "observers": []}, 442 + ) 443 + monkeypatch.setattr("apps.home.routes.get_cached_state", lambda: {}) 444 + monkeypatch.setattr("apps.home.routes.get_current", lambda: None) 445 + monkeypatch.setattr("apps.home.routes._resolve_attention", lambda awareness: None) 446 + monkeypatch.setattr("apps.home.routes._today", lambda: "20260416") 447 + monkeypatch.setattr("apps.home.routes._yesterday", lambda: "20260415") 448 + monkeypatch.setattr("apps.home.routes._count_journal_age_days", lambda today: 8) 449 + monkeypatch.setattr("apps.home.routes._load_stats", lambda today: {}) 450 + monkeypatch.setattr("apps.home.routes._load_flow_md", lambda today: (None, None)) 451 + monkeypatch.setattr("apps.home.routes._load_pulse_md", lambda: (None, None, [])) 452 + monkeypatch.setattr( 453 + "apps.home.routes._load_briefing_md", lambda today: ({}, None, []) 454 + ) 455 + monkeypatch.setattr("apps.home.routes._collect_events", lambda today: []) 456 + monkeypatch.setattr("apps.home.routes._collect_activities", lambda today: []) 457 + monkeypatch.setattr("apps.home.routes._collect_todos", lambda today: []) 458 + monkeypatch.setattr("apps.home.routes._collect_entities_today", lambda today: []) 459 + monkeypatch.setattr("apps.home.routes._collect_routines", lambda: []) 460 + monkeypatch.setattr("apps.home.routes._collect_skills", lambda: []) 461 + monkeypatch.setattr( 462 + "apps.home.routes.summarize_pipeline_day", 463 + lambda day: {"status": "healthy", "anomalies": []}, 464 + ) 465 + monkeypatch.setattr( 466 + "apps.home.routes.pipeline_status_message", 467 + lambda summary: None, 468 + ) 469 + monkeypatch.setattr( 470 + "apps.home.routes._summarize_yesterday_processing", 471 + lambda yesterday, journal_age_days: { 472 + "title": "Yesterday's processing", 473 + "mode": "healthy", 474 + "default_collapsed": True, 475 + "summary_line": "I wrote 2 newsletters.", 476 + "details": [], 477 + "sparse_lines": None, 478 + "first_week_framing": None, 479 + "status_reasons": [], 480 + }, 481 + ) 482 + 483 + ctx = _build_pulse_context() 484 + 485 + assert ctx["yesterday_processing"]["summary_line"] == "I wrote 2 newsletters."