A 5e storytelling engine with an LLM DM
Fix SRD chunking so recall actually finds class features and conditions
The DM's recall tool was basically useless for anything that lived inside
a large file — class features like Sneak Attack, conditions like Grappled,
and core mechanics like Opportunity Attack all got diluted into massive
single-file embeddings. Named entities in their own files (spells,
monsters) searched fine, but subsections didn't.
The chunker now cascades through ## → ### → #### → **Bold** headings
instead of stopping at ###, and the threshold dropped from 4K to 1.5K
chars. This takes the SRD from 905 to 3,090 chunks — every class feature,
glossary definition, and combat rule gets its own embedding.
Also fixes the SQLite check_same_thread error that was breaking recall
in live play (the MCP server runs tools on a different thread than the
one that created the connection).
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+238
-28
+4
-1
pyproject.toml
+4
-1
pyproject.toml
···
48
48
"--cov=tests",
49
49
"--cov-report=term-missing:skip-covered",
50
50
"--cov-branch",
51
+
"-m", "not slow",
51
52
]
53
+
markers = ["slow: slow tests that build real search indices"]
52
54
53
55
[tool.mypy]
54
56
python_version = "3.12"
···
64
66
[tool.coverage.run]
65
67
source = ["src/storied"]
66
68
branch = true
69
+
omit = ["src/storied/cli.py", "src/storied/srd/*"]
67
70
68
71
[tool.coverage.report]
69
-
fail_under = 70
72
+
fail_under = 80
+47
-27
src/storied/search.py
+47
-27
src/storied/search.py
···
17
17
18
18
EMBED_MODEL = "BAAI/bge-small-en-v1.5"
19
19
EMBED_DIM = 384
20
-
CHUNK_CHAR_THRESHOLD = 4_000
20
+
CHUNK_CHAR_THRESHOLD = 1_500
21
+
22
+
# Splitting patterns, tried in order from coarsest to finest.
23
+
_SPLIT_PATTERNS = [
24
+
r"\n(?=## )", # ## headings
25
+
r"\n(?=### )", # ### headings
26
+
r"\n(?=#### )", # #### headings (class features, combat sub-topics)
27
+
r"\n(?=\*\*[A-Z])", # bold definitions (glossary entries, level features)
28
+
]
21
29
22
30
23
31
@dataclass
···
42
50
return 0.5 ** (age / half_life)
43
51
44
52
53
+
def _split_oversized(
54
+
sections: list[str], threshold: int, patterns: list[str],
55
+
) -> list[str]:
56
+
"""Recursively split sections that exceed threshold using finer patterns."""
57
+
if not patterns:
58
+
return sections
59
+
60
+
pattern, *remaining = patterns
61
+
result: list[str] = []
62
+
63
+
for section in sections:
64
+
if len(section) <= threshold:
65
+
result.append(section)
66
+
continue
67
+
68
+
parts = re.split(pattern, section)
69
+
if len(parts) <= 1:
70
+
result.extend(_split_oversized([section], threshold, remaining))
71
+
else:
72
+
result.extend(
73
+
_split_oversized(
74
+
[p for p in parts if p.strip()], threshold, remaining,
75
+
)
76
+
)
77
+
78
+
return result
79
+
80
+
45
81
def chunk_document(path: Path, content: str) -> list[tuple[int, str]]:
46
82
"""Split a document into indexed chunks for embedding.
47
83
48
-
Small files return a single chunk. Larger files split on ## headers,
49
-
each prefixed with the document title for context.
84
+
Uses cascading splits (## → ### → #### → **Bold**) to break large
85
+
documents into chunks small enough for useful embeddings. Each chunk
86
+
gets the document title prepended for context.
50
87
51
88
Returns list of (chunk_index, chunk_text) pairs.
52
89
"""
53
90
if not content.strip():
54
91
return [(0, path.stem)]
55
92
56
-
# Extract title from first # heading
57
93
title_match = re.match(r"^#\s+(.+)", content)
58
94
title = title_match.group(1).strip() if title_match else path.stem
59
95
60
96
if len(content) < CHUNK_CHAR_THRESHOLD:
61
97
return [(0, content)]
62
98
63
-
# Split on ## headers
64
-
sections = re.split(r"\n(?=## )", content)
65
-
66
-
# If only one section (no ## headers), return as single chunk
67
-
if len(sections) <= 1:
68
-
return [(0, content)]
99
+
raw_sections = _split_oversized(
100
+
[content], CHUNK_CHAR_THRESHOLD, _SPLIT_PATTERNS,
101
+
)
69
102
70
103
chunks: list[tuple[int, str]] = []
71
-
chunk_idx = 0
72
-
73
-
for section in sections:
104
+
for idx, section in enumerate(raw_sections):
74
105
section = section.strip()
75
106
if not section:
76
107
continue
77
108
78
-
# Prepend title to non-first chunks for context
79
-
if chunk_idx > 0:
109
+
if idx > 0 and not section.startswith(f"# {title}"):
80
110
text = f"# {title}\n\n{section}"
81
111
else:
82
112
text = section
83
113
84
-
# Sub-split oversized sections on ### headers
85
-
if len(text) > CHUNK_CHAR_THRESHOLD:
86
-
subsections = re.split(r"\n(?=### )", text)
87
-
for sub in subsections:
88
-
sub = sub.strip()
89
-
if sub:
90
-
chunks.append((chunk_idx, sub))
91
-
chunk_idx += 1
92
-
else:
93
-
chunks.append((chunk_idx, text))
94
-
chunk_idx += 1
114
+
chunks.append((idx, text))
95
115
96
116
return chunks if chunks else [(0, content)]
97
117
···
110
130
111
131
def _connect(db_path: str) -> sqlite3.Connection:
112
132
"""Open a SQLite connection with sqlite-vec loaded."""
113
-
conn = sqlite3.connect(db_path)
133
+
conn = sqlite3.connect(db_path, check_same_thread=False)
114
134
conn.enable_load_extension(True)
115
135
sqlite_vec.load(conn)
116
136
conn.enable_load_extension(False)
+66
tests/test_search.py
+66
tests/test_search.py
···
99
99
assert len(chunks) == 1
100
100
assert "some content" in chunks[0][1]
101
101
102
+
def test_splits_on_h4_headings(self):
103
+
content = "# Rogue\n\n" + "\n".join(
104
+
f"#### **Feature {i}**\n\n{'x ' * 400}\n" for i in range(5)
105
+
)
106
+
chunks = chunk_document(Path("rogue.md"), content)
107
+
assert len(chunks) >= 5
108
+
109
+
def test_splits_on_bold_definitions(self):
110
+
content = "# Rules Glossary\n\n## Definitions\n\n" + "\n".join(
111
+
f"**Term {i}**\nDefinition of term {i}. {'y ' * 400}\n"
112
+
for i in range(5)
113
+
)
114
+
chunks = chunk_document(Path("glossary.md"), content)
115
+
assert len(chunks) >= 5
116
+
117
+
def test_class_features_split_by_level(self):
118
+
content = "# Fighter\n\n#### **Fighter Class Features**\n\n"
119
+
content += "**Level 1: Fighting Style**\n" + "Choose a style. " * 100 + "\n\n"
120
+
content += "**Level 1: Second Wind**\n" + "Heal yourself. " * 100 + "\n\n"
121
+
content += "**Level 2: Action Surge**\n" + "Extra action. " * 100 + "\n"
122
+
chunks = chunk_document(Path("fighter.md"), content)
123
+
texts = [t for _, t in chunks]
124
+
assert any("Second Wind" in t for t in texts)
125
+
assert any("Action Surge" in t for t in texts)
126
+
127
+
def test_oversized_with_no_splittable_headings(self):
128
+
content = "# Blob\n\n" + "word " * 1_000
129
+
chunks = chunk_document(Path("blob.md"), content)
130
+
assert len(chunks) >= 1
131
+
assert "word" in chunks[0][1]
132
+
133
+
def test_chunks_get_title_context(self):
134
+
content = "# Rogue\n\n" + "\n".join(
135
+
f"#### **Section {i}**\n\n{'z ' * 400}\n" for i in range(5)
136
+
)
137
+
chunks = chunk_document(Path("rogue.md"), content)
138
+
for _, text in chunks[1:]:
139
+
assert text.startswith("# Rogue")
140
+
102
141
103
142
# --- Age Decay ---
104
143
···
346
385
assert seeded.stats()["by_source"]["srd"] == 1
347
386
assert seeded.stats()["by_source"]["world"] == 1
348
387
seeded.close()
388
+
389
+
390
+
class TestThreadSafety:
391
+
"""Tests for cross-thread access (MCP server runs on a background thread)."""
392
+
393
+
def test_search_from_different_thread(self, index: VectorIndex):
394
+
index.upsert("world:npcs/vex.md:0", "Captain Vex, harbor master",
395
+
{"source": "world", "content_type": "npcs",
396
+
"path": "/tmp/vex.md", "title": "Captain Vex"})
397
+
398
+
import threading
399
+
400
+
results: list[list[SearchHit]] = []
401
+
error: list[Exception] = []
402
+
403
+
def search_on_thread():
404
+
try:
405
+
results.append(index.search("harbor master"))
406
+
except Exception as e:
407
+
error.append(e)
408
+
409
+
t = threading.Thread(target=search_on_thread)
410
+
t.start()
411
+
t.join()
412
+
413
+
assert not error, f"Cross-thread search failed: {error[0]}"
414
+
assert len(results[0]) == 1
+121
tests/test_srd_recall.py
+121
tests/test_srd_recall.py
···
1
+
"""Empirical tests: can recall find key SRD content?
2
+
3
+
These test against the real SRD files to verify that chunking and search
4
+
produce useful results for the queries a DM actually needs.
5
+
6
+
Slow (~60s) because they build a real embedding index. Run with:
7
+
pytest tests/test_srd_recall.py -v
8
+
"""
9
+
10
+
from pathlib import Path
11
+
12
+
import pytest
13
+
14
+
from storied.search import VectorIndex
15
+
16
+
SRD_DIR = Path("rules/srd-5.2.1/sections")
17
+
18
+
pytestmark = pytest.mark.slow
19
+
20
+
21
+
@pytest.fixture(scope="module")
22
+
def srd_index(tmp_path_factory: pytest.TempPathFactory) -> VectorIndex:
23
+
"""Build a real SRD index (once per module, reused across tests)."""
24
+
if not SRD_DIR.exists():
25
+
pytest.skip("SRD not available")
26
+
27
+
db_path = tmp_path_factory.mktemp("search") / "srd.db"
28
+
index = VectorIndex(db_path)
29
+
count = index.reindex_directory(SRD_DIR, source="srd")
30
+
assert count > 0, "No documents indexed"
31
+
yield index
32
+
index.close()
33
+
34
+
35
+
def _top_titles(index: VectorIndex, query: str, n: int = 5) -> list[str]:
36
+
"""Return titles of top N results for a query."""
37
+
hits = index.search(query, limit=n)
38
+
return [h.doc_id.split(":")[1] for h in hits]
39
+
40
+
41
+
def _assert_any_hit_contains(
42
+
index: VectorIndex, query: str, needle: str, top_n: int = 3,
43
+
):
44
+
"""Assert that at least one of the top N results contains needle."""
45
+
hits = index.search(query, limit=top_n)
46
+
snippets = [h.snippet for h in hits]
47
+
doc_ids = [h.doc_id for h in hits]
48
+
assert any(
49
+
needle.lower() in s.lower() for s in snippets
50
+
) or any(
51
+
needle.lower() in d.lower() for d in doc_ids
52
+
), (
53
+
f"'{needle}' not found in top {top_n} for query '{query}'.\n"
54
+
f"Got: {list(zip(doc_ids, [s[:80] for s in snippets]))}"
55
+
)
56
+
57
+
58
+
# --- Class Features ---
59
+
60
+
class TestClassFeatureRecall:
61
+
62
+
def test_sneak_attack(self, srd_index: VectorIndex):
63
+
_assert_any_hit_contains(srd_index, "sneak attack", "Sneak Attack")
64
+
65
+
def test_second_wind(self, srd_index: VectorIndex):
66
+
_assert_any_hit_contains(srd_index, "second wind", "Second Wind")
67
+
68
+
def test_action_surge(self, srd_index: VectorIndex):
69
+
_assert_any_hit_contains(srd_index, "action surge", "Action Surge")
70
+
71
+
def test_cunning_action(self, srd_index: VectorIndex):
72
+
_assert_any_hit_contains(srd_index, "cunning action", "Cunning Action")
73
+
74
+
def test_uncanny_dodge(self, srd_index: VectorIndex):
75
+
_assert_any_hit_contains(srd_index, "uncanny dodge", "Uncanny Dodge")
76
+
77
+
def test_wild_shape(self, srd_index: VectorIndex):
78
+
_assert_any_hit_contains(srd_index, "wild shape", "Wild Shape")
79
+
80
+
def test_lay_on_hands(self, srd_index: VectorIndex):
81
+
_assert_any_hit_contains(srd_index, "lay on hands", "Lay on Hands")
82
+
83
+
84
+
# --- Conditions ---
85
+
86
+
class TestConditionRecall:
87
+
88
+
def test_grappled(self, srd_index: VectorIndex):
89
+
_assert_any_hit_contains(srd_index, "grappled condition", "Grappled")
90
+
91
+
def test_frightened(self, srd_index: VectorIndex):
92
+
_assert_any_hit_contains(srd_index, "frightened condition", "Frightened")
93
+
94
+
def test_prone(self, srd_index: VectorIndex):
95
+
_assert_any_hit_contains(srd_index, "prone condition", "Prone")
96
+
97
+
def test_paralyzed(self, srd_index: VectorIndex):
98
+
_assert_any_hit_contains(srd_index, "paralyzed condition", "Paralyzed")
99
+
100
+
101
+
# --- Core Mechanics ---
102
+
103
+
class TestCoreMechanicsRecall:
104
+
105
+
def test_opportunity_attack(self, srd_index: VectorIndex):
106
+
_assert_any_hit_contains(
107
+
srd_index, "opportunity attack", "Opportunity Attack",
108
+
)
109
+
110
+
def test_death_saving_throw(self, srd_index: VectorIndex):
111
+
_assert_any_hit_contains(
112
+
srd_index, "death saving throw", "Death",
113
+
)
114
+
115
+
def test_short_rest(self, srd_index: VectorIndex):
116
+
_assert_any_hit_contains(srd_index, "short rest", "Short Rest")
117
+
118
+
def test_concentration(self, srd_index: VectorIndex):
119
+
_assert_any_hit_contains(
120
+
srd_index, "concentration spell", "Concentration",
121
+
)