personal memory agent
convey/chat: refresh watchdog on cortex progress events
Hook the singleton watchdog into _proxy_progress so any cortex
progress event for the active chat-generate or a spawned talent
resets the inactivity timer. A long-running healthy run is no
longer killed at the 3-minute mark; a genuinely stalled run still
trips the timeout after the inactivity threshold.
Co-Authored-By: OpenAI Codex <codex@openai.com>
+164
+8
convey/chat.py
+8
convey/chat.py
···
234
234
raw_chat_use_id = str(_current_chat_state.get("raw_use_id") or "")
235
235
if use_id == raw_chat_use_id:
236
236
logical_use_id = _current_chat_use_id
237
+
_refresh_watchdog_locked(use_id, "chat", str(_current_chat_use_id))
237
238
elif use_id in _active_talents:
238
239
logical_use_id = str(_active_talents[use_id]["chat_use_id"])
240
+
_refresh_watchdog_locked(use_id, "talent", logical_use_id)
239
241
240
242
if logical_use_id is None:
241
243
return
···
739
741
timer = _watchdog_timers.pop(str(use_id), None)
740
742
if timer is not None:
741
743
timer.cancel()
744
+
745
+
746
+
def _refresh_watchdog_locked(use_id: str, kind: str, logical_use_id: str) -> None:
747
+
if not use_id or use_id not in _watchdog_timers:
748
+
return
749
+
_arm_watchdog_locked(use_id, kind, logical_use_id)
742
750
743
751
744
752
def _set_current_raw_use_locked(logical_use_id: str, raw_use_id: str | None) -> None:
+156
tests/test_chat_runtime.py
+156
tests/test_chat_runtime.py
···
532
532
assert spawned_events[-1]["use_id"] in chat._active_talents
533
533
534
534
535
+
def test_watchdog_refreshed_by_progress_event(tmp_path, monkeypatch):
536
+
import convey.chat as chat
537
+
538
+
_setup_journal(tmp_path, monkeypatch)
539
+
_reset_chat_state(chat)
540
+
timers = _install_fake_timers(monkeypatch)
541
+
542
+
monkeypatch.setattr("convey.chat._emit_cortex_event", lambda *args, **kwargs: None)
543
+
544
+
with chat._state_lock:
545
+
start_info = chat._activate_current_locked(
546
+
"1713627800000",
547
+
{"type": "owner_message", "message": "help"},
548
+
{"app": "sol", "path": "/app/sol", "facet": "work"},
549
+
)
550
+
551
+
raw_use_id = start_info["raw_use_id"]
552
+
assert len(timers) == 1
553
+
554
+
chat._proxy_progress(
555
+
{
556
+
"tract": "cortex",
557
+
"event": "thinking",
558
+
"use_id": raw_use_id,
559
+
}
560
+
)
561
+
562
+
assert len(timers) == 2
563
+
assert timers[0].cancelled is True
564
+
assert timers[-1].interval == chat._CHAT_WATCHDOG_SECONDS
565
+
566
+
567
+
def test_watchdog_refresh_is_no_op_when_no_timer_registered(tmp_path, monkeypatch):
568
+
import convey.chat as chat
569
+
570
+
_setup_journal(tmp_path, monkeypatch)
571
+
_reset_chat_state(chat)
572
+
timers = _install_fake_timers(monkeypatch)
573
+
574
+
monkeypatch.setattr("convey.chat._emit_cortex_event", lambda *args, **kwargs: None)
575
+
576
+
with chat._state_lock:
577
+
chat._current_chat_use_id = "1713627850000"
578
+
chat._current_chat_state = {
579
+
"raw_use_id": "1713627850001",
580
+
"trigger": {"type": "owner_message", "message": "help"},
581
+
"location": {"app": "sol", "path": "/app/sol", "facet": "work"},
582
+
"retry_count": 0,
583
+
}
584
+
585
+
chat._proxy_progress(
586
+
{
587
+
"tract": "cortex",
588
+
"event": "thinking",
589
+
"use_id": "1713627850002",
590
+
}
591
+
)
592
+
593
+
assert len(timers) == 0
594
+
595
+
596
+
def test_watchdog_refreshed_by_talent_progress(tmp_path, monkeypatch):
597
+
import convey.chat as chat
598
+
599
+
_setup_journal(tmp_path, monkeypatch)
600
+
_reset_chat_state(chat)
601
+
timers = _install_fake_timers(monkeypatch)
602
+
603
+
monkeypatch.setattr("convey.chat._emit_cortex_event", lambda *args, **kwargs: None)
604
+
605
+
with chat._state_lock:
606
+
chat._current_chat_use_id = "1713627900000"
607
+
chat._current_chat_state = {
608
+
"raw_use_id": None,
609
+
"trigger": {"type": "owner_message", "message": "help"},
610
+
"location": {"app": "sol", "path": "/app/sol", "facet": "work"},
611
+
"retry_count": 0,
612
+
}
613
+
chat._active_talents["1713627900001"] = {
614
+
"chat_use_id": "1713627900000",
615
+
"target": "exec",
616
+
"task": "summarize",
617
+
"location": {"app": "sol", "path": "/app/sol", "facet": "work"},
618
+
}
619
+
chat._arm_watchdog_locked("1713627900001", "talent", "1713627900000")
620
+
621
+
assert len(timers) == 1
622
+
623
+
chat._proxy_progress(
624
+
{
625
+
"tract": "cortex",
626
+
"event": "thinking",
627
+
"use_id": "1713627900001",
628
+
}
629
+
)
630
+
631
+
assert len(timers) == 2
632
+
assert timers[0].cancelled is True
633
+
assert timers[-1].interval == chat._CHAT_WATCHDOG_SECONDS
634
+
635
+
636
+
def test_stalled_run_still_times_out_after_inactivity(tmp_path, monkeypatch):
637
+
import convey.chat as chat
638
+
639
+
_setup_journal(tmp_path, monkeypatch)
640
+
_reset_chat_state(chat)
641
+
timers = _install_fake_timers(monkeypatch)
642
+
643
+
emitted_errors: list[tuple[str, str]] = []
644
+
run_actions: list[dict | None] = []
645
+
monkeypatch.setattr("convey.chat._emit_cortex_event", lambda *args, **kwargs: None)
646
+
monkeypatch.setattr(
647
+
"convey.chat._emit_error",
648
+
lambda use_id, reason: emitted_errors.append((use_id, reason)),
649
+
)
650
+
monkeypatch.setattr(
651
+
"convey.chat._run_next_action", lambda action: run_actions.append(action)
652
+
)
653
+
654
+
with chat._state_lock:
655
+
start_info = chat._activate_current_locked(
656
+
"1713627950000",
657
+
{"type": "owner_message", "message": "help"},
658
+
{"app": "sol", "path": "/app/sol", "facet": "work"},
659
+
)
660
+
661
+
raw_use_id = start_info["raw_use_id"]
662
+
assert len(timers) == 1
663
+
664
+
for _ in range(3):
665
+
chat._proxy_progress(
666
+
{
667
+
"tract": "cortex",
668
+
"event": "thinking",
669
+
"use_id": raw_use_id,
670
+
}
671
+
)
672
+
673
+
assert len(timers) == 4
674
+
timers[-1].fire()
675
+
676
+
errors = [
677
+
event
678
+
for event in read_chat_events(chat._today_day())
679
+
if event["kind"] == "chat_error"
680
+
]
681
+
assert emitted_errors == [("1713627950000", "chat took too long — try again")]
682
+
assert run_actions == [None]
683
+
assert errors[-1]["use_id"] == "1713627950000"
684
+
assert errors[-1]["reason"] == "chat took too long — try again"
685
+
with chat._state_lock:
686
+
assert chat._current_chat_use_id is None
687
+
assert chat._current_chat_state is None
688
+
assert raw_use_id not in chat._watchdog_timers
689
+
690
+
535
691
def test_chat_watchdog_times_out_current_chat_generate(tmp_path, monkeypatch):
536
692
import convey.chat as chat
537
693