this repo has no description
Fix code review findings: image paths, cache keys, and stdout warning
- HIGH: Fix broken image links pointing to deleted temp directories
- enhance_markdown_with_image_paths() now rewrites actual ![]() refs
- New normalize_image_paths_for_cache() stores relative paths in cache
- Bump EXTRACTOR_VERSION to 3.3.0 to invalidate old caches
- MEDIUM: Fix cache key collision for 64KB-128KB files
- Hash entire content for files <= 128KB instead of just first chunk
- MEDIUM: Suppress PyMuPDF warning polluting --stdout output
- Set PYMUPDF_SUGGEST_LAYOUT_ANALYZER=0 before any pymupdf imports
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
+63
-10
+8
-1
scripts/extractor.py
+8
-1
scripts/extractor.py
···
4
4
- Accurate mode: IBM Docling with TableFormer AI (better for complex/borderless tables)
5
5
"""
6
6
7
+
import os
7
8
import sys
8
9
from pathlib import Path
10
+
11
+
# Suppress PyMuPDF's "Consider using pymupdf_layout" recommendation
12
+
# This prints to stdout and pollutes --stdout output
13
+
os.environ.setdefault("PYMUPDF_SUGGEST_LAYOUT_ANALYZER", "0")
9
14
10
15
# Version for cache invalidation - increment when extraction logic changes
11
16
# Format: major.minor.patch
···
13
18
# Image extraction includes nested XObjects (full=True)
14
19
# 3.2.0: Fast mode now includes image references in markdown (write_images=True)
15
20
# Cache keys now include no_images flag to avoid contamination
16
-
EXTRACTOR_VERSION = "3.2.0"
21
+
# 3.3.0: Image paths in cached markdown now use relative 'images/' prefix
22
+
# (fixes broken temp directory references in cached output)
23
+
EXTRACTOR_VERSION = "3.3.0"
17
24
18
25
19
26
def check_docling_models():
+55
-9
scripts/pdf_to_md.py
+55
-9
scripts/pdf_to_md.py
···
32
32
from pathlib import Path
33
33
from datetime import datetime
34
34
35
+
# Suppress PyMuPDF's "Consider using pymupdf_layout" recommendation
36
+
# This must be set before any pymupdf imports to take effect
37
+
os.environ.setdefault("PYMUPDF_SUGGEST_LAYOUT_ANALYZER", "0")
38
+
35
39
# Cache directory
36
40
CACHE_DIR = Path.home() / ".cache" / "pdf-to-markdown"
37
41
···
59
63
stat = p.stat()
60
64
file_size = stat.st_size
61
65
62
-
# Hash first 64KB + last 64KB for fast content-based identity
66
+
# Hash content for cache key identity
67
+
# For files <= 128KB: hash entire content (avoids collision for similar templates)
68
+
# For larger files: hash first 64KB + last 64KB for speed
63
69
chunk_size = 65536 # 64KB
64
70
hasher = hashlib.sha256()
65
71
66
72
with open(p, "rb") as f:
67
-
# Read first chunk
68
-
hasher.update(f.read(chunk_size))
69
-
70
-
# Read last chunk (if file is large enough)
71
-
if file_size > chunk_size * 2:
73
+
if file_size <= chunk_size * 2:
74
+
# Small/medium files: hash entire content
75
+
hasher.update(f.read())
76
+
else:
77
+
# Large files: hash first + last chunks
78
+
hasher.update(f.read(chunk_size))
72
79
f.seek(-chunk_size, 2) # Seek from end
73
80
hasher.update(f.read(chunk_size))
74
81
···
210
217
return full_md, image_dir, total_pages
211
218
212
219
220
+
def normalize_image_paths_for_cache(markdown: str, source_image_dir: Path) -> str:
221
+
"""
222
+
Normalize image paths in markdown to use relative 'images/' prefix.
223
+
224
+
This ensures cached markdown has stable paths that work when loaded later,
225
+
regardless of the original temp directory used during extraction.
226
+
"""
227
+
if not source_image_dir:
228
+
return markdown
229
+
230
+
source_image_dir = Path(source_image_dir)
231
+
232
+
def normalize_ref(match):
233
+
alt_text = match.group(1)
234
+
filename_raw = match.group(2)
235
+
filename = Path(filename_raw).name
236
+
237
+
# Check if file exists in source directory
238
+
if (source_image_dir / filename).exists():
239
+
# Rewrite to relative path for cache storage
240
+
return f""
241
+
return match.group(0)
242
+
243
+
pattern = r"!\[([^\]]*)\]\(([^)]+)\)"
244
+
return re.sub(pattern, normalize_ref, markdown)
245
+
246
+
213
247
def save_to_cache(
214
248
cache_key: str,
215
249
markdown: str,
···
224
258
225
259
Uses temp files + os.replace() to ensure cache integrity even if
226
260
the process is interrupted mid-write (power loss, Ctrl+C, etc.).
261
+
262
+
Image paths in markdown are normalized to 'images/<filename>' before
263
+
caching so they work correctly when loaded later.
227
264
"""
228
265
from extractor import EXTRACTOR_VERSION
229
266
230
267
cache_dir = get_cache_dir(cache_key)
231
268
cache_dir.mkdir(parents=True, exist_ok=True)
269
+
270
+
# Normalize image paths before caching (convert temp paths to relative)
271
+
if image_dir:
272
+
markdown = normalize_image_paths_for_cache(markdown, image_dir)
232
273
233
274
# Build metadata
234
275
p = Path(pdf_path).resolve()
···
656
697
657
698
def enhance_markdown_with_image_paths(markdown, image_dir):
658
699
"""
659
-
Enhance image references in markdown with full absolute paths.
700
+
Rewrite image references in markdown to point to the actual image location.
701
+
702
+
This fixes broken references that point to temp directories by rewriting
703
+
the actual  to use the durable image_dir location.
660
704
"""
661
705
if not image_dir:
662
706
return markdown
···
683
727
except:
684
728
dims = "?"
685
729
686
-
return f"\n\n**[Image: {filename} ({dims}, {size_kb}KB) → {full_path}]**"
730
+
# FIXED: Rewrite the actual image reference to the correct path
731
+
return f"\n\n**[Image: {filename} ({dims}, {size_kb}KB)]**"
687
732
except:
688
-
return f"\n\n**[Image: {filename} → {full_path}]**"
733
+
# FIXED: Rewrite even on metadata failure
734
+
return f"\n\n**[Image: {filename}]**"
689
735
690
736
return match.group(0)
691
737