personal memory agent
apps/entities: warm caches and memoize obs counts on all-facets path
The all-facets entities listing was re-scanning observations.jsonl line-by-line
and bypassing the in-process entity / relationship caches on every request
(600-950 ms cold, 650-700 ms warm on the founder's journal). Warm both caches
at the API entry and memoize observation counts by (path, mtime_ns).
- think/entities/relationships.py: add load_all_facet_relationships, mirror of
load_all_journal_entities.
- think/entities/observations.py: add count_observations + mtime-keyed
_OBSERVATION_COUNT_CACHE + clear_observation_count_cache.
- apps/entities/routes.py: get_journal_entities_data uses load_all_journal_entities
and warms each facet's relationship cache once before the entity loop;
_get_entity_metadata routes to count_observations.
- think/entities/journal.py: delete_journal_entity now clears the relationship
caches alongside the journal cache (closes a latent invalidation gap surfaced
by warming the cache on read).
- think/entities/__init__.py: export the two new public helpers.
- apps/entities/tests/conftest.py: clear the new cache in all 5 fixture sites.
- apps/entities/tests/test_all_facets_cache.py: prove relationship-cache warm
hits and observation-count mtime invalidation.
No on-the-wire changes; journal-entities / facet-entities / entity-detail API
baselines are unchanged.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
+184
-31
+12
-31
apps/entities/routes.py
+12
-31
apps/entities/routes.py
···
23
23
from convey import state
24
24
from think.entities import (
25
25
block_journal_entity,
26
+
count_observations,
26
27
entity_last_active_ts,
27
28
entity_memory_path,
28
29
entity_slug,
29
30
is_valid_entity_type,
31
+
load_all_facet_relationships,
32
+
load_all_journal_entities,
30
33
load_detected_entities_recent,
31
34
load_entities,
32
35
load_facet_relationship,
···
34
37
rename_entity_memory,
35
38
save_entities,
36
39
save_journal_entity,
37
-
scan_journal_entities,
38
40
unblock_journal_entity,
39
41
validate_aka_uniqueness,
40
42
)
···
51
53
52
54
53
55
def _get_entity_metadata(facet_name: str, entity_name: str) -> dict:
54
-
"""Get observation count and voiceprint status for an entity.
55
-
56
-
Args:
57
-
facet_name: The facet name
58
-
entity_name: The entity name
59
-
60
-
Returns:
61
-
dict with observation_count and has_voiceprint keys
62
-
"""
56
+
"""Get observation count and voiceprint status for an entity."""
63
57
try:
64
58
folder = entity_memory_path(facet_name, entity_name)
65
59
except ValueError:
66
60
return {"observation_count": 0, "has_voiceprint": False}
67
-
68
-
# Count observations
69
-
obs_file = folder / "observations.jsonl"
70
-
obs_count = 0
71
-
if obs_file.exists():
72
-
try:
73
-
with open(obs_file, "r", encoding="utf-8") as f:
74
-
obs_count = sum(1 for line in f if line.strip())
75
-
except OSError:
76
-
pass # File read error, default to 0
77
-
78
-
# Check for voiceprint
79
-
has_voiceprint = (folder / "voiceprints.npz").exists()
80
-
81
-
return {"observation_count": obs_count, "has_voiceprint": has_voiceprint}
61
+
return {
62
+
"observation_count": count_observations(facet_name, entity_name),
63
+
"has_voiceprint": (folder / "voiceprints.npz").exists(),
64
+
}
82
65
83
66
84
67
def get_facet_entities_data(facet_name: str) -> dict:
···
743
726
- entities: list of journal entities enriched with facet info
744
727
"""
745
728
facets_config = get_facets()
746
-
entity_ids = scan_journal_entities()
729
+
journal_entities = load_all_journal_entities()
730
+
for facet_name in facets_config:
731
+
load_all_facet_relationships(facet_name)
747
732
748
733
entities = []
749
-
for entity_id in entity_ids:
750
-
journal_entity = load_journal_entity(entity_id)
751
-
if not journal_entity:
752
-
continue
753
-
734
+
for entity_id, journal_entity in journal_entities.items():
754
735
entity_name = journal_entity.get("name", "")
755
736
756
737
# Build facet relationships
+6
apps/entities/tests/conftest.py
+6
apps/entities/tests/conftest.py
···
30
30
from think.entities.observations import (
31
31
add_observation,
32
32
clear_observation_cache,
33
+
clear_observation_count_cache,
33
34
save_observations,
34
35
)
35
36
from think.entities.relationships import clear_relationship_caches
···
51
52
clear_entity_loading_cache()
52
53
clear_relationship_caches()
53
54
clear_observation_cache()
55
+
clear_observation_count_cache()
54
56
import think.utils
55
57
56
58
think.utils._journal_path_cache = None
···
85
87
clear_entity_loading_cache()
86
88
clear_relationship_caches()
87
89
clear_observation_cache()
90
+
clear_observation_count_cache()
88
91
import think.utils
89
92
90
93
think.utils._journal_path_cache = None
···
111
114
clear_entity_loading_cache()
112
115
clear_relationship_caches()
113
116
clear_observation_cache()
117
+
clear_observation_count_cache()
114
118
import think.utils
115
119
116
120
think.utils._journal_path_cache = None
···
124
128
clear_entity_loading_cache()
125
129
clear_relationship_caches()
126
130
clear_observation_cache()
131
+
clear_observation_count_cache()
127
132
import think.utils
128
133
129
134
think.utils._journal_path_cache = None
···
169
174
clear_entity_loading_cache()
170
175
clear_relationship_caches()
171
176
clear_observation_cache()
177
+
clear_observation_count_cache()
172
178
import think.utils
173
179
174
180
think.utils._journal_path_cache = None
+92
apps/entities/tests/test_all_facets_cache.py
+92
apps/entities/tests/test_all_facets_cache.py
···
1
+
# SPDX-License-Identifier: AGPL-3.0-only
2
+
# Copyright (c) 2026 sol pbc
3
+
4
+
from __future__ import annotations
5
+
6
+
import builtins
7
+
import json
8
+
from pathlib import Path
9
+
from typing import Any
10
+
11
+
from apps.entities.routes import get_journal_entities_data
12
+
from think.entities.observations import add_observation, count_observations
13
+
14
+
15
+
def _entity(name: str) -> dict[str, Any]:
16
+
return {
17
+
"type": "Person",
18
+
"name": name,
19
+
"description": "Test entity",
20
+
"attached_at": 1000,
21
+
"updated_at": 1000,
22
+
}
23
+
24
+
25
+
def test_observation_count_memo(entity_env, monkeypatch):
26
+
facet = "personal"
27
+
entity_name = "Alice Johnson"
28
+
entity_env(
29
+
attached=[_entity(entity_name)],
30
+
observations=["Prefers async updates"],
31
+
observation_entity=entity_name,
32
+
facet=facet,
33
+
)
34
+
35
+
real_open = builtins.open
36
+
observation_opens = 0
37
+
38
+
def counting_open(file, *args, **kwargs):
39
+
nonlocal observation_opens
40
+
path = Path(file)
41
+
if path.name == "observations.jsonl":
42
+
observation_opens += 1
43
+
return real_open(file, *args, **kwargs)
44
+
45
+
monkeypatch.setattr(builtins, "open", counting_open)
46
+
47
+
assert count_observations(facet, entity_name) == 1
48
+
assert count_observations(facet, entity_name) == 1
49
+
assert observation_opens == 1
50
+
51
+
add_observation(facet, entity_name, "Prefers morning meetings", "20260427")
52
+
opens_after_write = observation_opens
53
+
54
+
assert count_observations(facet, entity_name) == 2
55
+
assert observation_opens == opens_after_write + 1
56
+
57
+
58
+
def test_relationship_cache_warm(entity_env, monkeypatch):
59
+
facet = "personal"
60
+
entity_name = "Alice Johnson"
61
+
journal = entity_env(attached=[_entity(entity_name)], facet=facet)
62
+
facet_dir = journal / "facets" / facet
63
+
facet_dir.mkdir(parents=True, exist_ok=True)
64
+
(facet_dir / "facet.json").write_text(
65
+
json.dumps({"title": "Personal", "description": "Personal facet"}),
66
+
encoding="utf-8",
67
+
)
68
+
69
+
real_open = builtins.open
70
+
relationship_opens = 0
71
+
72
+
def counting_open(file, *args, **kwargs):
73
+
nonlocal relationship_opens
74
+
path = Path(file)
75
+
if (
76
+
path.name == "entity.json"
77
+
and "facets" in path.parts
78
+
and path.parent.parent.name == "entities"
79
+
):
80
+
relationship_opens += 1
81
+
return real_open(file, *args, **kwargs)
82
+
83
+
monkeypatch.setattr(builtins, "open", counting_open)
84
+
85
+
first = get_journal_entities_data()
86
+
assert len(first["entities"]) == 1
87
+
assert relationship_opens > 0
88
+
89
+
relationship_opens = 0
90
+
second = get_journal_entities_data()
91
+
assert len(second["entities"]) == 1
92
+
assert relationship_opens == 0
+4
think/entities/__init__.py
+4
think/entities/__init__.py
···
89
89
# Observations
90
90
from think.entities.observations import (
91
91
add_observation,
92
+
count_observations,
92
93
load_observations,
93
94
observations_file_path,
94
95
save_observations,
···
99
100
ensure_entity_memory,
100
101
entity_memory_path,
101
102
facet_relationship_path,
103
+
load_all_facet_relationships,
102
104
load_facet_relationship,
103
105
rename_entity_memory,
104
106
save_facet_relationship,
···
149
151
"ensure_entity_memory",
150
152
"entity_memory_path",
151
153
"facet_relationship_path",
154
+
"load_all_facet_relationships",
152
155
"load_facet_relationship",
153
156
"rename_entity_memory",
154
157
"save_facet_relationship",
···
184
187
"touch_entity",
185
188
# Observations
186
189
"add_observation",
190
+
"count_observations",
187
191
"load_observations",
188
192
"observations_file_path",
189
193
"save_observations",
+3
think/entities/journal.py
+3
think/entities/journal.py
···
350
350
if journal_entity.get("is_principal"):
351
351
raise ValueError("Cannot delete the principal (self) entity")
352
352
353
+
from think.entities.relationships import clear_relationship_caches
354
+
353
355
# Clear cache on modification
354
356
clear_journal_entity_cache()
357
+
clear_relationship_caches()
355
358
356
359
facets_deleted = []
357
360
+39
think/entities/observations.py
+39
think/entities/observations.py
···
23
23
24
24
# Global cache for entity observations: {(facet, entity_slug): list[dict]}
25
25
_OBSERVATION_CACHE: dict[tuple[str, str], list[dict[str, Any]]] | None = None
26
+
# Global cache for observation counts: {path: (mtime_ns, count)}
27
+
_OBSERVATION_COUNT_CACHE: dict[Path, tuple[int, int]] | None = None
26
28
27
29
28
30
def clear_observation_cache() -> None:
29
31
"""Clear the entity observation cache."""
30
32
global _OBSERVATION_CACHE
31
33
_OBSERVATION_CACHE = None
34
+
35
+
36
+
def clear_observation_count_cache() -> None:
37
+
"""Clear the entity observation count cache."""
38
+
global _OBSERVATION_COUNT_CACHE
39
+
_OBSERVATION_COUNT_CACHE = None
32
40
33
41
34
42
def observations_file_path(facet: str, name: str) -> Path:
···
97
105
_OBSERVATION_CACHE[(facet, slug)] = observations
98
106
99
107
return observations
108
+
109
+
110
+
def count_observations(facet: str, name: str) -> int:
111
+
"""Count observations for an entity."""
112
+
global _OBSERVATION_COUNT_CACHE
113
+
try:
114
+
folder = entity_memory_path(facet, name)
115
+
except ValueError:
116
+
return 0
117
+
118
+
obs_file = folder / "observations.jsonl"
119
+
try:
120
+
st = obs_file.stat()
121
+
except OSError:
122
+
return 0
123
+
124
+
if _OBSERVATION_COUNT_CACHE is None:
125
+
_OBSERVATION_COUNT_CACHE = {}
126
+
127
+
cached = _OBSERVATION_COUNT_CACHE.get(obs_file)
128
+
if cached is not None and cached[0] == st.st_mtime_ns:
129
+
return cached[1]
130
+
131
+
try:
132
+
with open(obs_file, "r", encoding="utf-8") as f:
133
+
count = sum(1 for line in f if line.strip())
134
+
except OSError:
135
+
return 0
136
+
137
+
_OBSERVATION_COUNT_CACHE[obs_file] = (st.st_mtime_ns, count)
138
+
return count
100
139
101
140
102
141
def save_observations(
+28
think/entities/relationships.py
+28
think/entities/relationships.py
···
141
141
return entity_ids
142
142
143
143
144
+
def load_all_facet_relationships(facet: str) -> dict[str, EntityDict]:
145
+
"""Load all facet relationships for a facet.
146
+
147
+
Returns:
148
+
Dict mapping entity_id to relationship dict
149
+
"""
150
+
global _RELATIONSHIP_CACHE
151
+
entity_ids = scan_facet_relationships(facet)
152
+
if _RELATIONSHIP_CACHE is not None and all(
153
+
(facet, entity_id) in _RELATIONSHIP_CACHE for entity_id in entity_ids
154
+
):
155
+
return {
156
+
entity_id: _RELATIONSHIP_CACHE[(facet, entity_id)]
157
+
for entity_id in entity_ids
158
+
}
159
+
160
+
if _RELATIONSHIP_CACHE is None:
161
+
_RELATIONSHIP_CACHE = {}
162
+
163
+
relationships = {}
164
+
for entity_id in entity_ids:
165
+
relationship = load_facet_relationship(facet, entity_id)
166
+
if relationship:
167
+
relationships[entity_id] = relationship
168
+
169
+
return relationships
170
+
171
+
144
172
def enrich_relationship_with_journal(
145
173
relationship: EntityDict,
146
174
journal_entity: EntityDict | None,