personal memory agent
Merge branch 'hopper-klgl7taa-export-idempotency'
+38
-40
+2
apps/import/facet_ingest.py
+2
apps/import/facet_ingest.py
···
598
598
"staged_path": str(staged_path),
599
599
},
600
600
)
601
+
received[item_id] = content_hash
601
602
continue
602
603
603
604
parsed_data, path_info = _remap_entity_ids(
···
674
675
result["wrote_files"] = True
675
676
action = "facet_file_created" if new_facet else "facet_file_merged"
676
677
elif status == "staged":
678
+
received[item_id] = content_hash
677
679
result["staged"] += 1
678
680
action = "facet_file_staged"
679
681
else:
+16
-23
apps/import/ingest.py
+16
-23
apps/import/ingest.py
···
46
46
_FACET_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
47
47
_IMPORT_ID_RE = re.compile(r"^\d{8}_\d{6}$")
48
48
49
-
_NEVER_TRANSFER_PATHS = frozenset(
50
-
{
51
-
"convey.password_hash",
52
-
"convey.secret",
53
-
"setup.completed_at",
54
-
"providers.auth",
55
-
"providers.key_validation",
56
-
"transcribe.whisper.device",
57
-
}
58
-
)
59
-
_NEVER_TRANSFER_PREFIXES = ("env.",)
49
+
_NEVER_TRANSFER_PATHS = frozenset({"convey.password_hash"})
60
50
_IDENTITY_PATHS = frozenset(
61
51
{
62
52
"identity.name",
···
101
91
102
92
103
93
def _is_never_transfer(path: str) -> bool:
104
-
if path in _NEVER_TRANSFER_PATHS:
105
-
return True
106
-
return any(path.startswith(prefix) for prefix in _NEVER_TRANSFER_PREFIXES)
94
+
return path in _NEVER_TRANSFER_PATHS
107
95
108
96
109
97
def _categorize_field(path: str) -> str:
···
250
238
for name in expected_names:
251
239
(segment_dir / name).write_bytes(file_infos[name]["content"])
252
240
253
-
file_records = [
254
-
{
255
-
"name": name,
256
-
"sha256": str(file_infos[name]["sha256"]),
257
-
"size": int(file_infos[name]["size"]),
258
-
}
259
-
for name in expected_names
260
-
]
261
-
new_state.setdefault(day, {})[arc_key] = {"files": file_records}
241
+
file_records = [
242
+
{
243
+
"name": name,
244
+
"sha256": str(file_infos[name]["sha256"]),
245
+
"size": int(file_infos[name]["size"]),
246
+
}
247
+
for name in expected_names
248
+
]
249
+
new_state.setdefault(day, {})[arc_key] = {"files": file_records}
250
+
if action == "deconflicted":
251
+
original_arc_key = f"{stream}/{original_segment_key}"
252
+
new_state.setdefault(day, {})[original_arc_key] = {
253
+
"files": file_records
254
+
}
262
255
263
256
entry = {
264
257
"ts": datetime.now(timezone.utc).isoformat(),
+1
-11
observe/export.py
+1
-11
observe/export.py
···
40
40
_DAY_JSONL_RE = re.compile(r"^\d{8}\.jsonl$")
41
41
_DAY_MD_RE = re.compile(r"^\d{8}\.md$")
42
42
_IMPORT_ID_RE = re.compile(r"^\d{8}_\d{6}$")
43
-
_NEVER_TRANSFER_PATHS = frozenset(
44
-
{
45
-
"convey.password_hash",
46
-
"convey.secret",
47
-
"setup.completed_at",
48
-
"providers.auth",
49
-
"providers.key_validation",
50
-
"transcribe.whisper.device",
51
-
}
52
-
)
43
+
_NEVER_TRANSFER_PATHS = frozenset({"convey.password_hash"})
53
44
54
45
55
46
@dataclass
···
234
225
else:
235
226
if isinstance(obj, dict):
236
227
obj.pop(parts[-1], None)
237
-
result.pop("env", None)
238
228
return result
239
229
240
230
+10
-3
tests/test_config_ingest.py
+10
-3
tests/test_config_ingest.py
···
96
96
"trust_localhost": True,
97
97
},
98
98
"setup": {"completed_at": 12345},
99
+
"providers": {"auth": "provider_auth_val", "key_validation": "val123"},
100
+
"transcribe": {"whisper": {"device": "gpu"}},
99
101
"env": {"API_KEY": "xyz"},
100
102
"retention": {"days": 30},
101
103
}
···
179
181
source = load_journal_source(env["key"])
180
182
181
183
assert response.status_code == 200
182
-
assert body == {"staged": True, "skipped": False, "diff_fields": 5}
184
+
assert body == {"staged": True, "skipped": False, "diff_fields": 11}
183
185
assert (state_dir / "source_config.json").exists()
184
186
assert (state_dir / "diff.json").exists()
185
187
assert "last_hash" in _read_json(state_dir / "state.json")
···
211
213
212
214
assert response.status_code == 200
213
215
assert "convey.password_hash" not in diff
214
-
assert "convey.secret" not in diff
215
-
assert not any(key.startswith("env.") for key in diff)
216
+
assert "convey.secret" in diff
217
+
assert diff["convey.secret"]["category"] == "preference"
218
+
assert "setup.completed_at" in diff
219
+
assert "providers.auth" in diff
220
+
assert "providers.key_validation" in diff
221
+
assert "transcribe.whisper.device" in diff
222
+
assert any(key.startswith("env.") for key in diff)
216
223
217
224
218
225
def test_idempotent(ingest_env):
+2
-1
tests/test_export_integration.py
+2
-1
tests/test_export_integration.py
+3
-2
tests/test_facet_ingest.py
+3
-2
tests/test_facet_ingest.py
···
893
893
assert second.status_code == 200
894
894
body = second.get_json()
895
895
assert body["staged"] == 0
896
-
assert body["skipped"] == 0
897
-
assert body["created"] == 1 or body["merged"] == 1
896
+
assert body["skipped"] == 1
897
+
assert body["created"] == 0
898
+
assert body["merged"] == 0
898
899
899
900
900
901
def test_facet_json_conflict_staging(ingest_env):
+4
tests/test_segment_ingest.py
+4
tests/test_segment_ingest.py
···
256
256
257
257
state_data = _read_state(env["key_prefix"])
258
258
assert "laptop/143023_300" in state_data["20260413"]
259
+
assert "laptop/143022_300" in state_data["20260413"]
259
260
260
261
log_entries = _read_log(env["key_prefix"])
261
262
assert log_entries[0]["action"] == "deconflicted"
···
458
459
"errors": [],
459
460
}
460
461
assert (segment_dir / "extra.txt").read_bytes() == b"keep me"
462
+
state_data = _read_state(env["key_prefix"])
463
+
assert "laptop/143022_300" in state_data["20260413"]
464
+
assert state_data["20260413"]["laptop/143022_300"]["files"][0]["name"] == "audio.flac"
461
465
462
466
463
467
def test_ingest_stats_update(ingest_env):