personal memory agent
Add PDF document importer with text extraction, vision OCR, and entity seeding
Implements DocumentImporter (FileImporter protocol) for importing PDF files
as first-class journal segments under import.document stream. Text-based PDFs
use pypdf extraction; scanned PDFs fall back to Gemini vision then tesseract
OCR. Each document produces one segment with original.pdf + document_transcript.md.
+748
-3
+3
-3
apps/import/routes.py
+3
-3
apps/import/routes.py
···
127
127
"display_name": "Document",
128
128
"emoji": "📄",
129
129
"icon": "file",
130
-
"description": "Import a document, PDF, or text file",
130
+
"description": "Import a PDF document",
131
131
"input_type": "file",
132
-
"upload_prompt": "Upload a document (.pdf, .txt, .md)",
132
+
"upload_prompt": "Upload a PDF file",
133
133
"has_guide": False,
134
-
"accept": ".pdf,.txt,.md",
134
+
"accept": ".pdf",
135
135
},
136
136
{
137
137
"name": "quick",
+2
pyproject.toml
+2
pyproject.toml
+337
tests/test_importer_documents.py
+337
tests/test_importer_documents.py
···
1
+
# SPDX-License-Identifier: AGPL-3.0-only
2
+
# Copyright (c) 2026 sol pbc
3
+
4
+
import datetime as dt
5
+
import importlib
6
+
7
+
from think.importers.file_importer import FILE_IMPORTER_REGISTRY
8
+
9
+
10
+
class MockPage:
11
+
def __init__(self, text: str):
12
+
self._text = text
13
+
14
+
def extract_text(self):
15
+
return self._text
16
+
17
+
18
+
class MockPdfReader:
19
+
def __init__(self, path):
20
+
self.path = str(path)
21
+
self.pages = [
22
+
MockPage(
23
+
"Page text content here with enough characters to pass threshold."
24
+
),
25
+
MockPage(
26
+
"Second page text content here with enough characters to pass threshold."
27
+
),
28
+
]
29
+
self.metadata = {"/CreationDate": "D:20260115120000"}
30
+
31
+
32
+
def test_detect_pdf_file(tmp_path):
33
+
mod = importlib.import_module("think.importers.documents")
34
+
pdf = tmp_path / "file.pdf"
35
+
pdf.write_bytes(b"%PDF-1.4")
36
+
assert mod.importer.detect(pdf) is True
37
+
38
+
39
+
def test_detect_non_pdf(tmp_path):
40
+
mod = importlib.import_module("think.importers.documents")
41
+
txt = tmp_path / "file.txt"
42
+
txt.write_text("hello", encoding="utf-8")
43
+
assert mod.importer.detect(txt) is False
44
+
45
+
46
+
def test_detect_directory_with_pdfs(tmp_path):
47
+
mod = importlib.import_module("think.importers.documents")
48
+
(tmp_path / "a.pdf").write_bytes(b"%PDF-1.4")
49
+
(tmp_path / "b.pdf").write_bytes(b"%PDF-1.4")
50
+
assert mod.importer.detect(tmp_path) is True
51
+
52
+
53
+
def test_detect_empty_directory(tmp_path):
54
+
mod = importlib.import_module("think.importers.documents")
55
+
assert mod.importer.detect(tmp_path) is False
56
+
57
+
58
+
def test_preview_single_pdf(tmp_path, monkeypatch):
59
+
mod = importlib.import_module("think.importers.documents")
60
+
pdf = tmp_path / "file.pdf"
61
+
pdf.write_bytes(b"%PDF-1.4")
62
+
monkeypatch.setattr(mod, "PdfReader", MockPdfReader)
63
+
64
+
preview = mod.importer.preview(pdf)
65
+
66
+
assert preview.date_range == ("20260115", "20260115")
67
+
assert preview.item_count == 1
68
+
assert preview.entity_count == 0
69
+
assert preview.summary == "1 PDF documents, 2 total pages"
70
+
71
+
72
+
def test_process_text_pdf(tmp_path, monkeypatch):
73
+
mod = importlib.import_module("think.importers.documents")
74
+
pdf = tmp_path / "contract.pdf"
75
+
pdf.write_bytes(b"%PDF-1.4")
76
+
monkeypatch.setenv("JOURNAL_PATH", str(tmp_path))
77
+
monkeypatch.setattr(mod, "PdfReader", MockPdfReader)
78
+
monkeypatch.setattr(mod, "day_path", lambda day: tmp_path / day)
79
+
monkeypatch.setattr(
80
+
mod,
81
+
"write_content_manifest",
82
+
lambda import_id, entries: tmp_path / "manifest.jsonl",
83
+
)
84
+
monkeypatch.setattr(mod, "seed_entities", lambda facet, day, entities: entities)
85
+
86
+
result = mod.importer.process(
87
+
pdf, tmp_path, facet="work", import_id="20260115_120000"
88
+
)
89
+
90
+
seg_dir = tmp_path / "20260115" / "import.document" / "120000_0"
91
+
md_path = seg_dir / "document_transcript.md"
92
+
93
+
assert result.entries_written == 1
94
+
assert result.entities_seeded >= 0
95
+
assert result.segments == [("20260115", "120000_0")]
96
+
assert result.files_created == [str(md_path)]
97
+
assert md_path.exists()
98
+
content = md_path.read_text(encoding="utf-8")
99
+
assert content.startswith("# contract")
100
+
assert "**Pages:** 2" in content
101
+
assert "Page text content here" in content
102
+
103
+
104
+
def test_process_creates_original_pdf(tmp_path, monkeypatch):
105
+
mod = importlib.import_module("think.importers.documents")
106
+
pdf = tmp_path / "original.pdf"
107
+
pdf.write_bytes(b"%PDF-1.4 fake")
108
+
monkeypatch.setenv("JOURNAL_PATH", str(tmp_path))
109
+
monkeypatch.setattr(mod, "PdfReader", MockPdfReader)
110
+
monkeypatch.setattr(mod, "day_path", lambda day: tmp_path / day)
111
+
monkeypatch.setattr(
112
+
mod,
113
+
"write_content_manifest",
114
+
lambda import_id, entries: tmp_path / "manifest.jsonl",
115
+
)
116
+
monkeypatch.setattr(mod, "seed_entities", lambda facet, day, entities: entities)
117
+
118
+
result = mod.importer.process(pdf, tmp_path, import_id="20260115_120000")
119
+
120
+
copied = tmp_path / "20260115" / "import.document" / "120000_0" / "original.pdf"
121
+
assert copied.exists()
122
+
assert copied.read_bytes() == b"%PDF-1.4 fake"
123
+
assert str(copied) not in result.files_created
124
+
125
+
126
+
def test_process_scanned_detection(tmp_path, monkeypatch):
127
+
mod = importlib.import_module("think.importers.documents")
128
+
129
+
class ScannedReader(MockPdfReader):
130
+
def __init__(self, path):
131
+
self.path = str(path)
132
+
self.pages = [MockPage("x"), MockPage("y")]
133
+
self.metadata = {"/CreationDate": "D:20260115120000"}
134
+
135
+
pdf = tmp_path / "scan.pdf"
136
+
pdf.write_bytes(b"%PDF-1.4")
137
+
monkeypatch.setenv("JOURNAL_PATH", str(tmp_path))
138
+
monkeypatch.setattr(mod, "PdfReader", ScannedReader)
139
+
monkeypatch.setattr(mod, "day_path", lambda day: tmp_path / day)
140
+
monkeypatch.setattr(
141
+
mod,
142
+
"write_content_manifest",
143
+
lambda import_id, entries: tmp_path / "manifest.jsonl",
144
+
)
145
+
monkeypatch.setattr(mod, "seed_entities", lambda facet, day, entities: entities)
146
+
calls = []
147
+
148
+
def fake_vision(pdf_path, page_count):
149
+
calls.append((pdf_path.name, page_count))
150
+
return "Vision extracted text"
151
+
152
+
monkeypatch.setattr(mod, "_extract_text_vision", fake_vision)
153
+
154
+
result = mod.importer.process(pdf, tmp_path, import_id="20260115_120000")
155
+
156
+
assert calls == [("scan.pdf", 2)]
157
+
assert result.errors == []
158
+
md_path = (
159
+
tmp_path
160
+
/ "20260115"
161
+
/ "import.document"
162
+
/ "120000_0"
163
+
/ "document_transcript.md"
164
+
)
165
+
assert "Vision extracted text" in md_path.read_text(encoding="utf-8")
166
+
167
+
168
+
def test_process_scanned_ocr_fallback(tmp_path, monkeypatch):
169
+
mod = importlib.import_module("think.importers.documents")
170
+
171
+
class ScannedReader(MockPdfReader):
172
+
def __init__(self, path):
173
+
self.path = str(path)
174
+
self.pages = [MockPage("x")]
175
+
self.metadata = {"/CreationDate": "D:20260115120000"}
176
+
177
+
pdf = tmp_path / "scan.pdf"
178
+
pdf.write_bytes(b"%PDF-1.4")
179
+
monkeypatch.setenv("JOURNAL_PATH", str(tmp_path))
180
+
monkeypatch.setattr(mod, "PdfReader", ScannedReader)
181
+
monkeypatch.setattr(mod, "day_path", lambda day: tmp_path / day)
182
+
monkeypatch.setattr(
183
+
mod,
184
+
"write_content_manifest",
185
+
lambda import_id, entries: tmp_path / "manifest.jsonl",
186
+
)
187
+
monkeypatch.setattr(mod, "seed_entities", lambda facet, day, entities: entities)
188
+
monkeypatch.setattr(
189
+
mod,
190
+
"_extract_text_vision",
191
+
lambda pdf_path, page_count: (_ for _ in ()).throw(
192
+
RuntimeError("vision failed")
193
+
),
194
+
)
195
+
monkeypatch.setattr(mod, "_extract_text_ocr", lambda pdf_path: "OCR extracted text")
196
+
197
+
mod.importer.process(pdf, tmp_path, import_id="20260115_120000")
198
+
199
+
md_path = (
200
+
tmp_path
201
+
/ "20260115"
202
+
/ "import.document"
203
+
/ "120000_0"
204
+
/ "document_transcript.md"
205
+
)
206
+
assert "OCR extracted text" in md_path.read_text(encoding="utf-8")
207
+
208
+
209
+
def test_process_scanned_all_fallback(tmp_path, monkeypatch):
210
+
mod = importlib.import_module("think.importers.documents")
211
+
212
+
class ScannedReader(MockPdfReader):
213
+
def __init__(self, path):
214
+
self.path = str(path)
215
+
self.pages = [MockPage("x")]
216
+
self.metadata = {"/CreationDate": "D:20260115120000"}
217
+
218
+
pdf = tmp_path / "scan.pdf"
219
+
pdf.write_bytes(b"%PDF-1.4")
220
+
monkeypatch.setenv("JOURNAL_PATH", str(tmp_path))
221
+
monkeypatch.setattr(mod, "PdfReader", ScannedReader)
222
+
monkeypatch.setattr(mod, "day_path", lambda day: tmp_path / day)
223
+
monkeypatch.setattr(
224
+
mod,
225
+
"write_content_manifest",
226
+
lambda import_id, entries: tmp_path / "manifest.jsonl",
227
+
)
228
+
monkeypatch.setattr(mod, "seed_entities", lambda facet, day, entities: entities)
229
+
monkeypatch.setattr(
230
+
mod,
231
+
"_extract_text_vision",
232
+
lambda pdf_path, page_count: (_ for _ in ()).throw(
233
+
RuntimeError("vision failed")
234
+
),
235
+
)
236
+
monkeypatch.setattr(
237
+
mod,
238
+
"_extract_text_ocr",
239
+
lambda pdf_path: (_ for _ in ()).throw(RuntimeError("ocr failed")),
240
+
)
241
+
242
+
result = mod.importer.process(pdf, tmp_path, import_id="20260115_120000")
243
+
244
+
assert result.errors == [
245
+
"scan.pdf: scanned PDF — vision failed (vision failed), OCR failed (ocr failed); using sparse pypdf text"
246
+
]
247
+
md_path = (
248
+
tmp_path
249
+
/ "20260115"
250
+
/ "import.document"
251
+
/ "120000_0"
252
+
/ "document_transcript.md"
253
+
)
254
+
assert "\nx\n" in md_path.read_text(encoding="utf-8")
255
+
256
+
257
+
def test_process_multi_file(tmp_path, monkeypatch):
258
+
mod = importlib.import_module("think.importers.documents")
259
+
pdf_a = tmp_path / "a.pdf"
260
+
pdf_b = tmp_path / "b.pdf"
261
+
pdf_a.write_bytes(b"%PDF-1.4")
262
+
pdf_b.write_bytes(b"%PDF-1.4")
263
+
monkeypatch.setenv("JOURNAL_PATH", str(tmp_path))
264
+
monkeypatch.setattr(mod, "PdfReader", MockPdfReader)
265
+
monkeypatch.setattr(mod, "day_path", lambda day: tmp_path / day)
266
+
monkeypatch.setattr(
267
+
mod,
268
+
"write_content_manifest",
269
+
lambda import_id, entries: tmp_path / "manifest.jsonl",
270
+
)
271
+
monkeypatch.setattr(mod, "seed_entities", lambda facet, day, entities: entities)
272
+
273
+
result = mod.importer.process(tmp_path, tmp_path, import_id="20260115_120000")
274
+
275
+
assert result.entries_written == 2
276
+
assert result.segments == [("20260115", "120000_0"), ("20260115", "120001_0")]
277
+
278
+
279
+
def test_process_entity_seeding(tmp_path, monkeypatch):
280
+
mod = importlib.import_module("think.importers.documents")
281
+
282
+
class EntityReader(MockPdfReader):
283
+
def __init__(self, path):
284
+
self.path = str(path)
285
+
self.pages = [
286
+
MockPage(
287
+
"Signed by Jane Doe on behalf of Example Corp Inc for the purchase agreement and related closing documents."
288
+
)
289
+
]
290
+
self.metadata = {"/CreationDate": "D:20260115120000"}
291
+
292
+
pdf = tmp_path / "parties.pdf"
293
+
pdf.write_bytes(b"%PDF-1.4")
294
+
monkeypatch.setenv("JOURNAL_PATH", str(tmp_path))
295
+
monkeypatch.setattr(mod, "PdfReader", EntityReader)
296
+
monkeypatch.setattr(mod, "day_path", lambda day: tmp_path / day)
297
+
monkeypatch.setattr(
298
+
mod,
299
+
"write_content_manifest",
300
+
lambda import_id, entries: tmp_path / "manifest.jsonl",
301
+
)
302
+
calls = []
303
+
304
+
def fake_seed_entities(facet, day, entities):
305
+
calls.append((facet, day, entities))
306
+
return entities
307
+
308
+
monkeypatch.setattr(mod, "seed_entities", fake_seed_entities)
309
+
310
+
result = mod.importer.process(
311
+
pdf, tmp_path, facet="legal", import_id="20260115_120000"
312
+
)
313
+
314
+
assert result.entities_seeded == len(calls[0][2])
315
+
assert calls[0][0] == "legal"
316
+
assert calls[0][1] == "20260115"
317
+
assert any(entity["type"] == "Person" for entity in calls[0][2])
318
+
assert any(entity["type"] == "Organization" for entity in calls[0][2])
319
+
assert all(entity["observations"] == ["Named in parties"] for entity in calls[0][2])
320
+
321
+
322
+
def test_timestamp_from_metadata(tmp_path):
323
+
mod = importlib.import_module("think.importers.documents")
324
+
pdf = tmp_path / "file.pdf"
325
+
pdf.write_bytes(b"%PDF-1.4")
326
+
reader = MockPdfReader(pdf)
327
+
328
+
timestamp = mod._get_pdf_timestamp(reader, pdf)
329
+
330
+
assert (
331
+
dt.datetime.fromtimestamp(timestamp).strftime("%Y%m%d%H%M%S")
332
+
== "20260115120000"
333
+
)
334
+
335
+
336
+
def test_registry_entry():
337
+
assert FILE_IMPORTER_REGISTRY["document"] == "think.importers.documents"
+6
think/agents.py
+6
think/agents.py
···
116
116
"import.gemini": "an imported Gemini conversation",
117
117
"import.ics": "an imported calendar event",
118
118
"import.obsidian": "an imported note from Obsidian",
119
+
"import.document": "an imported document (PDF)",
119
120
"import.kindle": "imported Kindle reading highlights",
120
121
}
121
122
···
175
176
"import.obsidian": (
176
177
"This is a note. Summarize the key ideas, references, and connections. "
177
178
"What was the author thinking about and working through?"
179
+
),
180
+
"import.document": (
181
+
"This is an imported PDF document. The content has been extracted from the original file. "
182
+
"Focus on the document's purpose, key information, named parties, dates, and any "
183
+
"actionable items. Preserve the document's structure in your analysis."
178
184
),
179
185
"import.kindle": (
180
186
"These are reading highlights. Describe what was being read and what "
+363
think/importers/documents.py
+363
think/importers/documents.py
···
1
+
# SPDX-License-Identifier: AGPL-3.0-only
2
+
# Copyright (c) 2026 sol pbc
3
+
4
+
"""PDF document importer."""
5
+
6
+
from __future__ import annotations
7
+
8
+
import datetime as dt
9
+
import logging
10
+
import re
11
+
import shutil
12
+
from pathlib import Path
13
+
from typing import Callable
14
+
15
+
from pypdf import PdfReader
16
+
17
+
try:
18
+
from pdf2image import convert_from_path
19
+
except ImportError: # pragma: no cover - optional dependency
20
+
convert_from_path = None
21
+
22
+
try:
23
+
import pytesseract
24
+
except ImportError: # pragma: no cover - optional dependency
25
+
pytesseract = None
26
+
27
+
from think.importers.file_importer import ImportPreview, ImportResult
28
+
from think.importers.shared import seed_entities, write_content_manifest
29
+
from think.models import generate
30
+
from think.utils import day_path
31
+
32
+
logger = logging.getLogger(__name__)
33
+
34
+
35
+
def _find_pdfs(path: Path) -> list[Path]:
36
+
"""Return matching PDF files for a file or directory path."""
37
+
if path.is_file() and path.suffix.lower() == ".pdf":
38
+
return [path]
39
+
if path.is_dir():
40
+
return sorted(
41
+
child
42
+
for child in path.iterdir()
43
+
if child.is_file() and child.suffix.lower() == ".pdf"
44
+
)
45
+
return []
46
+
47
+
48
+
def _get_pdf_timestamp(reader: PdfReader, pdf_path: Path) -> float:
49
+
"""Return a best-effort timestamp for a PDF."""
50
+
metadata = reader.metadata or {}
51
+
for key in ("/ModDate", "/CreationDate"):
52
+
value = metadata.get(key)
53
+
if not value:
54
+
continue
55
+
match = re.search(r"(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})", str(value))
56
+
if not match:
57
+
continue
58
+
try:
59
+
parsed = dt.datetime(
60
+
int(match.group(1)),
61
+
int(match.group(2)),
62
+
int(match.group(3)),
63
+
int(match.group(4)),
64
+
int(match.group(5)),
65
+
int(match.group(6)),
66
+
)
67
+
return parsed.timestamp()
68
+
except ValueError:
69
+
continue
70
+
try:
71
+
return pdf_path.stat().st_mtime
72
+
except OSError:
73
+
return dt.datetime.now().timestamp()
74
+
75
+
76
+
def _extract_text_pypdf(reader: PdfReader) -> tuple[str, int, bool]:
77
+
"""Extract text and classify whether the PDF appears scanned."""
78
+
parts: list[str] = []
79
+
total_chars = 0
80
+
for page in reader.pages:
81
+
text = page.extract_text() or ""
82
+
parts.append(text)
83
+
total_chars += len(text)
84
+
page_count = len(reader.pages)
85
+
is_scanned = (total_chars / max(page_count, 1)) < 50
86
+
return "\n\n".join(parts).strip(), page_count, is_scanned
87
+
88
+
89
+
def _extract_text_vision(pdf_path: Path, page_count: int) -> str:
90
+
"""Extract text from scanned PDFs using vision models."""
91
+
if convert_from_path is None:
92
+
raise RuntimeError(
93
+
"pdf2image is required for scanned PDF extraction. Install with: pip install pdf2image"
94
+
)
95
+
96
+
prompt = (
97
+
"Extract all text content from this document. Preserve the document "
98
+
"structure including headings, paragraphs, lists, and tables. Return "
99
+
"the content as clean markdown."
100
+
)
101
+
images = convert_from_path(str(pdf_path), dpi=200)
102
+
if page_count <= 10:
103
+
return generate(
104
+
contents=[prompt, *images], context="import.document.vision"
105
+
).strip()
106
+
107
+
pages: list[str] = []
108
+
for image in images:
109
+
page_text = generate(
110
+
contents=[prompt, image], context="import.document.vision"
111
+
).strip()
112
+
if page_text:
113
+
pages.append(page_text)
114
+
return "\n\n".join(pages).strip()
115
+
116
+
117
+
def _extract_text_ocr(pdf_path: Path) -> str:
118
+
"""Extract text from scanned PDFs using local OCR."""
119
+
if convert_from_path is None:
120
+
raise RuntimeError(
121
+
"pdf2image is required for scanned PDF extraction. Install with: pip install pdf2image"
122
+
)
123
+
if pytesseract is None:
124
+
raise RuntimeError(
125
+
"pytesseract is required for OCR fallback. Install with: pip install pytesseract"
126
+
)
127
+
128
+
images = convert_from_path(str(pdf_path), dpi=200)
129
+
return "\n\n".join(
130
+
pytesseract.image_to_string(image).strip() for image in images
131
+
).strip()
132
+
133
+
134
+
def _render_document_markdown(title: str, text: str, metadata: dict) -> str:
135
+
"""Render extracted document text as markdown."""
136
+
lines = [f"# {title}", "", "**Type:** Document"]
137
+
if metadata.get("page_count") is not None:
138
+
lines.append(f"**Pages:** {metadata['page_count']}")
139
+
if metadata.get("date"):
140
+
lines.append(f"**Date:** {metadata['date']}")
141
+
lines.extend(["", "---", "", text.strip()])
142
+
return "\n".join(lines).rstrip() + "\n"
143
+
144
+
145
+
def _extract_entities(text: str, title: str) -> list[dict]:
146
+
"""Extract simple named people and organizations from document text."""
147
+
names = set(
148
+
m.group(1).strip(" ,.")
149
+
for m in re.finditer(
150
+
r"(?i)\b(?:by|from|to|between|signed by)\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3})",
151
+
text,
152
+
)
153
+
)
154
+
orgs = set(
155
+
m.group(1).strip(" ,.")
156
+
for m in re.finditer(
157
+
r"\b([A-Z][A-Za-z0-9&.,' -]{1,80}\s+(?:LLC|Inc|Corp|Corporation|Trust|Ltd|Company)(?:\s+(?:LLC|Inc|Corp|Corporation|Trust|Ltd|Company))*)\b",
158
+
text,
159
+
)
160
+
)
161
+
observation = f"Named in {title}"
162
+
entities = [
163
+
{"name": name, "type": "Person", "observations": [observation]}
164
+
for name in sorted(names)
165
+
]
166
+
entities.extend(
167
+
{"name": name, "type": "Organization", "observations": [observation]}
168
+
for name in sorted(orgs)
169
+
)
170
+
return entities
171
+
172
+
173
+
class DocumentImporter:
174
+
name = "document"
175
+
display_name = "Documents"
176
+
file_patterns = ["*.pdf"]
177
+
description = "Import PDF documents with text extraction and optional OCR"
178
+
179
+
def detect(self, path: Path) -> bool:
180
+
return bool(_find_pdfs(path))
181
+
182
+
def preview(self, path: Path) -> ImportPreview:
183
+
pdfs = _find_pdfs(path)
184
+
if not pdfs:
185
+
return ImportPreview(
186
+
date_range=("", ""),
187
+
item_count=0,
188
+
entity_count=0,
189
+
summary="No PDF documents found",
190
+
)
191
+
192
+
timestamps: list[float] = []
193
+
total_pages = 0
194
+
for pdf_path in pdfs:
195
+
reader = PdfReader(str(pdf_path))
196
+
timestamps.append(_get_pdf_timestamp(reader, pdf_path))
197
+
total_pages += len(reader.pages)
198
+
199
+
dates = sorted(
200
+
dt.datetime.fromtimestamp(ts).strftime("%Y%m%d") for ts in timestamps
201
+
)
202
+
return ImportPreview(
203
+
date_range=(dates[0], dates[-1]),
204
+
item_count=len(pdfs),
205
+
entity_count=0,
206
+
summary=f"{len(pdfs)} PDF documents, {total_pages} total pages",
207
+
)
208
+
209
+
def process(
210
+
self,
211
+
path: Path,
212
+
journal_root: Path,
213
+
*,
214
+
facet: str | None = None,
215
+
import_id: str | None = None,
216
+
progress_callback: Callable | None = None,
217
+
) -> ImportResult:
218
+
pdfs = _find_pdfs(path)
219
+
import_id = import_id or dt.datetime.now().strftime("%Y%m%d_%H%M%S")
220
+
if not pdfs:
221
+
return ImportResult(
222
+
entries_written=0,
223
+
entities_seeded=0,
224
+
files_created=[],
225
+
errors=[],
226
+
summary="No PDF documents found to import",
227
+
)
228
+
229
+
created_files: list[str] = []
230
+
errors: list[str] = []
231
+
segments: list[tuple[str, str]] = []
232
+
manifest_entries: list[dict] = []
233
+
entities_seeded = 0
234
+
timestamps: list[float] = []
235
+
used_keys: dict[str, set[str]] = {}
236
+
237
+
for index, pdf_path in enumerate(pdfs):
238
+
try:
239
+
reader = PdfReader(str(pdf_path))
240
+
ts = _get_pdf_timestamp(reader, pdf_path)
241
+
text, page_count, is_scanned = _extract_text_pypdf(reader)
242
+
extraction_method = "pypdf"
243
+
244
+
if is_scanned:
245
+
try:
246
+
text = _extract_text_vision(pdf_path, page_count)
247
+
extraction_method = "vision"
248
+
except Exception as vision_exc:
249
+
logger.warning(
250
+
"Vision extraction failed for %s: %s", pdf_path, vision_exc
251
+
)
252
+
try:
253
+
text = _extract_text_ocr(pdf_path)
254
+
extraction_method = "ocr"
255
+
except Exception as ocr_exc:
256
+
logger.warning(
257
+
"OCR extraction failed for %s: %s", pdf_path, ocr_exc
258
+
)
259
+
errors.append(
260
+
f"{pdf_path.name}: scanned PDF — vision failed "
261
+
f"({vision_exc}), OCR failed ({ocr_exc}); using sparse pypdf text"
262
+
)
263
+
264
+
seg_dt = dt.datetime.fromtimestamp(ts)
265
+
day = seg_dt.strftime("%Y%m%d")
266
+
seg_key = f"{seg_dt.strftime('%H%M%S')}_0"
267
+
day_used = used_keys.setdefault(day, set())
268
+
while seg_key in day_used:
269
+
ts += 1
270
+
seg_dt = dt.datetime.fromtimestamp(ts)
271
+
day = seg_dt.strftime("%Y%m%d")
272
+
seg_key = f"{seg_dt.strftime('%H%M%S')}_0"
273
+
day_used = used_keys.setdefault(day, set())
274
+
day_used.add(seg_key)
275
+
timestamps.append(ts)
276
+
277
+
title = pdf_path.stem
278
+
date_str = seg_dt.strftime("%Y-%m-%d")
279
+
segment_dir = day_path(day) / "import.document" / seg_key
280
+
segment_dir.mkdir(parents=True, exist_ok=True)
281
+
282
+
original_path = segment_dir / "original.pdf"
283
+
shutil.copy2(pdf_path, original_path)
284
+
md_path = segment_dir / "document_transcript.md"
285
+
md_text = _render_document_markdown(
286
+
title,
287
+
text,
288
+
{"page_count": page_count, "date": date_str},
289
+
)
290
+
md_path.write_text(md_text, encoding="utf-8")
291
+
292
+
created_files.append(str(md_path))
293
+
segments.append((day, seg_key))
294
+
manifest_entries.append(
295
+
{
296
+
"id": f"document-{index}",
297
+
"title": title,
298
+
"date": day,
299
+
"type": "document",
300
+
"preview": text[:200],
301
+
"meta": {
302
+
"page_count": page_count,
303
+
"extraction_method": extraction_method,
304
+
},
305
+
"segments": [{"day": day, "key": seg_key}],
306
+
}
307
+
)
308
+
309
+
if facet:
310
+
try:
311
+
resolved = seed_entities(
312
+
facet, day, _extract_entities(text, title)
313
+
)
314
+
entities_seeded += len(resolved)
315
+
except Exception as exc:
316
+
errors.append(
317
+
f"Failed to seed entities for {pdf_path.name}: {exc}"
318
+
)
319
+
except Exception as exc:
320
+
errors.append(f"Failed to process {pdf_path.name}: {exc}")
321
+
322
+
if progress_callback:
323
+
earliest = None
324
+
latest = None
325
+
if timestamps:
326
+
earliest = dt.datetime.fromtimestamp(min(timestamps)).strftime(
327
+
"%Y%m%d"
328
+
)
329
+
latest = dt.datetime.fromtimestamp(max(timestamps)).strftime(
330
+
"%Y%m%d"
331
+
)
332
+
progress_callback(
333
+
index + 1,
334
+
len(pdfs),
335
+
earliest_date=earliest,
336
+
latest_date=latest,
337
+
entities_found=entities_seeded,
338
+
)
339
+
340
+
write_content_manifest(import_id, manifest_entries)
341
+
342
+
if timestamps:
343
+
earliest = dt.datetime.fromtimestamp(min(timestamps)).strftime("%Y%m%d")
344
+
latest = dt.datetime.fromtimestamp(max(timestamps)).strftime("%Y%m%d")
345
+
date_range: tuple[str, str] | None = (earliest, latest)
346
+
else:
347
+
date_range = None
348
+
349
+
return ImportResult(
350
+
entries_written=len(segments),
351
+
entities_seeded=entities_seeded,
352
+
files_created=created_files,
353
+
errors=errors,
354
+
summary=(
355
+
f"Imported {len(segments)} PDF documents across "
356
+
f"{len({day for day, _ in segments})} days into {len(segments)} segments"
357
+
),
358
+
segments=segments,
359
+
date_range=date_range,
360
+
)
361
+
362
+
363
+
importer = DocumentImporter()
+1
think/importers/file_importer.py
+1
think/importers/file_importer.py
+36
uv.lock
+36
uv.lock
···
2173
2173
]
2174
2174
2175
2175
[[package]]
2176
+
name = "pdf2image"
2177
+
version = "1.17.0"
2178
+
source = { registry = "https://pypi.org/simple" }
2179
+
dependencies = [
2180
+
{ name = "pillow" },
2181
+
]
2182
+
sdist = { url = "https://files.pythonhosted.org/packages/00/d8/b280f01045555dc257b8153c00dee3bc75830f91a744cd5f84ef3a0a64b1/pdf2image-1.17.0.tar.gz", hash = "sha256:eaa959bc116b420dd7ec415fcae49b98100dda3dd18cd2fdfa86d09f112f6d57", size = 12811, upload-time = "2024-01-07T20:33:01.965Z" }
2183
+
wheels = [
2184
+
{ url = "https://files.pythonhosted.org/packages/62/33/61766ae033518957f877ab246f87ca30a85b778ebaad65b7f74fa7e52988/pdf2image-1.17.0-py3-none-any.whl", hash = "sha256:ecdd58d7afb810dffe21ef2b1bbc057ef434dabbac6c33778a38a3f7744a27e2", size = 11618, upload-time = "2024-01-07T20:32:59.957Z" },
2185
+
]
2186
+
2187
+
[[package]]
2176
2188
name = "pillow"
2177
2189
version = "12.1.0"
2178
2190
source = { registry = "https://pypi.org/simple" }
···
2653
2665
sdist = { url = "https://files.pythonhosted.org/packages/10/45/8340de1c752bfda2da912ea0fa8c9a432f7de3f6315e82f1c0847811dff6/pypdf-6.7.0.tar.gz", hash = "sha256:eb95e244d9f434e6cfd157272283339ef586e593be64ee699c620f756d5c3f7e", size = 5299947, upload-time = "2026-02-08T14:47:11.897Z" }
2654
2666
wheels = [
2655
2667
{ url = "https://files.pythonhosted.org/packages/ed/f1/c92e75a0eb18bb10845e792054ded113010de958b6d4998e201c029417bb/pypdf-6.7.0-py3-none-any.whl", hash = "sha256:62e85036d50839cbdf45b8067c2c1a1b925517514d7cba4cbe8755a6c2829bc9", size = 330557, upload-time = "2026-02-08T14:47:10.111Z" },
2668
+
]
2669
+
2670
+
[[package]]
2671
+
name = "pytesseract"
2672
+
version = "0.3.13"
2673
+
source = { registry = "https://pypi.org/simple" }
2674
+
dependencies = [
2675
+
{ name = "packaging" },
2676
+
{ name = "pillow" },
2677
+
]
2678
+
sdist = { url = "https://files.pythonhosted.org/packages/9f/a6/7d679b83c285974a7cb94d739b461fa7e7a9b17a3abfd7bf6cbc5c2394b0/pytesseract-0.3.13.tar.gz", hash = "sha256:4bf5f880c99406f52a3cfc2633e42d9dc67615e69d8a509d74867d3baddb5db9", size = 17689, upload-time = "2024-08-16T02:33:56.762Z" }
2679
+
wheels = [
2680
+
{ url = "https://files.pythonhosted.org/packages/7a/33/8312d7ce74670c9d39a532b2c246a853861120486be9443eebf048043637/pytesseract-0.3.13-py3-none-any.whl", hash = "sha256:7a99c6c2ac598360693d83a416e36e0b33a67638bb9d77fdcac094a3589d4b34", size = 14705, upload-time = "2024-08-16T02:36:10.09Z" },
2656
2681
]
2657
2682
2658
2683
[[package]]
···
3573
3598
{ name = "openai" },
3574
3599
{ name = "openai-agents" },
3575
3600
{ name = "opencv-python" },
3601
+
{ name = "pdf2image" },
3576
3602
{ name = "pillow" },
3577
3603
{ name = "platformdirs" },
3578
3604
{ name = "playwright" },
···
3582
3608
{ name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" },
3583
3609
{ name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" },
3584
3610
{ name = "pypdf" },
3611
+
{ name = "pytesseract" },
3585
3612
{ name = "pytest" },
3586
3613
{ name = "pytest-asyncio" },
3587
3614
{ name = "pytest-cov" },
···
3622
3649
{ name = "openai", specifier = ">=1.2.0" },
3623
3650
{ name = "openai-agents", specifier = ">=0.1.0" },
3624
3651
{ name = "opencv-python" },
3652
+
{ name = "pdf2image" },
3625
3653
{ name = "pillow" },
3626
3654
{ name = "platformdirs" },
3627
3655
{ name = "playwright", specifier = ">=1.40.0" },
···
3631
3659
{ name = "pyobjc-framework-cocoa", marker = "sys_platform == 'darwin'" },
3632
3660
{ name = "pyobjc-framework-quartz", marker = "sys_platform == 'darwin'" },
3633
3661
{ name = "pypdf" },
3662
+
{ name = "pytesseract" },
3634
3663
{ name = "pytest" },
3635
3664
{ name = "pytest-asyncio" },
3636
3665
{ name = "pytest-cov" },
···
3951
3980
{ url = "https://files.pythonhosted.org/packages/0f/8b/4b61d6e13f7108f36910df9ab4b58fd389cc2520d54d81b88660804aad99/torch-2.10.0-2-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:418997cb02d0a0f1497cf6a09f63166f9f5df9f3e16c8a716ab76a72127c714f", size = 79423467, upload-time = "2026-02-10T21:44:48.711Z" },
3952
3981
{ url = "https://files.pythonhosted.org/packages/d3/54/a2ba279afcca44bbd320d4e73675b282fcee3d81400ea1b53934efca6462/torch-2.10.0-2-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:13ec4add8c3faaed8d13e0574f5cd4a323c11655546f91fbe6afa77b57423574", size = 79498202, upload-time = "2026-02-10T21:44:52.603Z" },
3953
3982
{ url = "https://files.pythonhosted.org/packages/ec/23/2c9fe0c9c27f7f6cb865abcea8a4568f29f00acaeadfc6a37f6801f84cb4/torch-2.10.0-2-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:e521c9f030a3774ed770a9c011751fb47c4d12029a3d6522116e48431f2ff89e", size = 79498254, upload-time = "2026-02-10T21:44:44.095Z" },
3983
+
{ url = "https://files.pythonhosted.org/packages/16/ee/efbd56687be60ef9af0c9c0ebe106964c07400eade5b0af8902a1d8cd58c/torch-2.10.0-3-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:a1ff626b884f8c4e897c4c33782bdacdff842a165fee79817b1dd549fdda1321", size = 915510070, upload-time = "2026-03-11T14:16:39.386Z" },
3984
+
{ url = "https://files.pythonhosted.org/packages/36/ab/7b562f1808d3f65414cd80a4f7d4bb00979d9355616c034c171249e1a303/torch-2.10.0-3-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:ac5bdcbb074384c66fa160c15b1ead77839e3fe7ed117d667249afce0acabfac", size = 915518691, upload-time = "2026-03-11T14:15:43.147Z" },
3985
+
{ url = "https://files.pythonhosted.org/packages/b3/7a/abada41517ce0011775f0f4eacc79659bc9bc6c361e6bfe6f7052a6b9363/torch-2.10.0-3-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:98c01b8bb5e3240426dcde1446eed6f40c778091c8544767ef1168fc663a05a6", size = 915622781, upload-time = "2026-03-11T14:17:11.354Z" },
3986
+
{ url = "https://files.pythonhosted.org/packages/ab/c6/4dfe238342ffdcec5aef1c96c457548762d33c40b45a1ab7033bb26d2ff2/torch-2.10.0-3-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:80b1b5bfe38eb0e9f5ff09f206dcac0a87aadd084230d4a36eea5ec5232c115b", size = 915627275, upload-time = "2026-03-11T14:16:11.325Z" },
3987
+
{ url = "https://files.pythonhosted.org/packages/d8/f0/72bf18847f58f877a6a8acf60614b14935e2f156d942483af1ffc081aea0/torch-2.10.0-3-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:46b3574d93a2a8134b3f5475cfb98e2eb46771794c57015f6ad1fb795ec25e49", size = 915523474, upload-time = "2026-03-11T14:17:44.422Z" },
3988
+
{ url = "https://files.pythonhosted.org/packages/f4/39/590742415c3030551944edc2ddc273ea1fdfe8ffb2780992e824f1ebee98/torch-2.10.0-3-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b1d5e2aba4eb7f8e87fbe04f86442887f9167a35f092afe4c237dfcaaef6e328", size = 915632474, upload-time = "2026-03-11T14:15:13.666Z" },
3989
+
{ url = "https://files.pythonhosted.org/packages/b6/8e/34949484f764dde5b222b7fe3fede43e4a6f0da9d7f8c370bb617d629ee2/torch-2.10.0-3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:0228d20b06701c05a8f978357f657817a4a63984b0c90745def81c18aedfa591", size = 915523882, upload-time = "2026-03-11T14:14:46.311Z" },
3954
3990
{ url = "https://files.pythonhosted.org/packages/0c/1a/c61f36cfd446170ec27b3a4984f072fd06dab6b5d7ce27e11adb35d6c838/torch-2.10.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:5276fa790a666ee8becaffff8acb711922252521b28fbce5db7db5cf9cb2026d", size = 145992962, upload-time = "2026-01-21T16:24:14.04Z" },
3955
3991
{ url = "https://files.pythonhosted.org/packages/b5/60/6662535354191e2d1555296045b63e4279e5a9dbad49acf55a5d38655a39/torch-2.10.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:aaf663927bcd490ae971469a624c322202a2a1e68936eb952535ca4cd3b90444", size = 915599237, upload-time = "2026-01-21T16:23:25.497Z" },
3956
3992
{ url = "https://files.pythonhosted.org/packages/40/b8/66bbe96f0d79be2b5c697b2e0b187ed792a15c6c4b8904613454651db848/torch-2.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:a4be6a2a190b32ff5c8002a0977a25ea60e64f7ba46b1be37093c141d9c49aeb", size = 113720931, upload-time = "2026-01-21T16:24:23.743Z" },