this repo has no description
Consolidate dataset under data/, init DVC, track all 14 splits as v1
- Copied CROHME+MathWriting raster splits from eff-mer with clean names
- Renamed data/typeset -> data/typeset_train for consistency
- Unified RASTER_ROOT+TYPESET_ROOT into single DATA_ROOT in src/data.py
- typeset_train now in TRAIN_SPLITS directly; train.py simplified accordingly
- DVC initialized; all 14 splits tracked as .dvc pointer files
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
+118
-29
.dvc/config
.dvc/config
This is a binary file and will not be displayed.
+3
.dvcignore
+3
.dvcignore
+14
data/.gitignore
+14
data/.gitignore
···
1
+
/crohme_gen_2019
2
+
/crohme_gen_2023
3
+
/crohme_gen_syntactic
4
+
/crohme_real_train
5
+
/crohme_val
6
+
/crohme_test
7
+
/mathwriting_train
8
+
/mathwriting_synthetic
9
+
/mathwriting_symbols
10
+
/mathwriting_val
11
+
/mathwriting_test
12
+
/typeset_train
13
+
/typeset_val
14
+
/typeset_test
+6
data/crohme_gen_2019.dvc
+6
data/crohme_gen_2019.dvc
+6
data/crohme_gen_2023.dvc
+6
data/crohme_gen_2023.dvc
+6
data/crohme_gen_syntactic.dvc
+6
data/crohme_gen_syntactic.dvc
+6
data/crohme_real_train.dvc
+6
data/crohme_real_train.dvc
+6
data/crohme_test.dvc
+6
data/crohme_test.dvc
+6
data/crohme_val.dvc
+6
data/crohme_val.dvc
+6
data/mathwriting_symbols.dvc
+6
data/mathwriting_symbols.dvc
+6
data/mathwriting_synthetic.dvc
+6
data/mathwriting_synthetic.dvc
+6
data/mathwriting_test.dvc
+6
data/mathwriting_test.dvc
+6
data/mathwriting_train.dvc
+6
data/mathwriting_train.dvc
+6
data/mathwriting_val.dvc
+6
data/mathwriting_val.dvc
+6
data/typeset_test.dvc
+6
data/typeset_test.dvc
+6
data/typeset_train.dvc
+6
data/typeset_train.dvc
+6
data/typeset_val.dvc
+6
data/typeset_val.dvc
+12
-15
src/data.py
+12
-15
src/data.py
···
1
1
"""
2
2
Data loading for Gemma 4 vision fine-tuning.
3
3
4
-
Manifests live in ../eff-mer/data/raster/<split>/manifest.jsonl.
4
+
Manifests live in data/<split>/manifest.jsonl.
5
5
Each line: {"image": "images/xxx.png", "latex": "...", "typst": "..."}
6
6
7
7
Images are loaded lazily via Dataset.set_transform -- never all in memory.
···
18
18
from PIL import Image, ImageFilter
19
19
import torchvision.transforms.functional as TF
20
20
21
-
RASTER_ROOT = Path(__file__).parent.parent / "../eff-mer/data/raster"
22
-
TYPESET_ROOT = Path(__file__).parent.parent / "data"
21
+
DATA_ROOT = Path(__file__).parent.parent / "data"
23
22
24
23
TRAIN_SPLITS = [
25
-
"crohme_gen_LaTeX_data_CROHME_2019",
26
-
"crohme_gen_LaTeX_data_CROHME_2023_corpus",
27
-
"crohme_gen_syntactic_data",
24
+
"crohme_gen_2019",
25
+
"crohme_gen_2023",
26
+
"crohme_gen_syntactic",
28
27
"crohme_real_train",
29
-
"mathwriting_Train",
30
-
"mathwriting_Synthetic",
31
-
"mathwriting_Symbols",
28
+
"mathwriting_train",
29
+
"mathwriting_synthetic",
30
+
"mathwriting_symbols",
31
+
"typeset_train",
32
32
]
33
-
VAL_SPLITS = ["mathwriting_Val"]
34
-
TEST_SPLITS = ["mathwriting_Test"]
35
-
TYPESET_SPLITS = ["typeset"] # uv run generate-typeset
36
-
TYPESET_VAL_SPLITS = ["typeset_val"] # uv run generate-typeset --out data/typeset_val --count 1000 --seed 43
37
-
TYPESET_TEST_SPLITS = ["typeset_test"] # uv run generate-typeset --out data/typeset_test --count 500 --seed 44
33
+
VAL_SPLITS = ["mathwriting_val", "typeset_val"]
34
+
TEST_SPLITS = ["mathwriting_test", "typeset_test"]
38
35
39
36
PROMPT = "Transcribe this mathematical expression to Typst math notation."
40
37
BASE_MODEL = "unsloth/gemma-4-E2B-it"
···
57
54
58
55
59
56
def load_records(split_names: list[str], dedupe: bool = True,
60
-
root: Path = RASTER_ROOT) -> list[dict]:
57
+
root: Path = DATA_ROOT) -> list[dict]:
61
58
"""
62
59
Load records from manifests.
63
60
+2
-13
src/train.py
+2
-13
src/train.py
···
8
8
from unsloth.trainer import UnslothVisionDataCollator
9
9
from trl import SFTTrainer, SFTConfig
10
10
11
-
from .data import (BASE_MODEL, TRAIN_SPLITS, VAL_SPLITS, TYPESET_SPLITS,
12
-
TYPESET_VAL_SPLITS, TYPESET_ROOT, load_records, make_dataset)
11
+
from .data import (BASE_MODEL, TRAIN_SPLITS, VAL_SPLITS, load_records, make_dataset)
13
12
14
13
15
14
def main() -> None:
···
32
31
)
33
32
34
33
train_records = load_records(TRAIN_SPLITS, dedupe=True)
35
-
typeset_manifest = TYPESET_ROOT / "typeset" / "manifest.jsonl"
36
-
if typeset_manifest.exists():
37
-
typeset_records = load_records(TYPESET_SPLITS, dedupe=False, root=TYPESET_ROOT)
38
-
train_records += typeset_records
39
-
print(f"Typeset train: {len(typeset_records):,} records mixed in")
40
-
val_records = load_records(VAL_SPLITS, dedupe=False)
41
-
typeset_val_manifest = TYPESET_ROOT / "typeset_val" / "manifest.jsonl"
42
-
if typeset_val_manifest.exists():
43
-
typeset_val_records = load_records(TYPESET_VAL_SPLITS, dedupe=False, root=TYPESET_ROOT)
44
-
val_records += typeset_val_records
45
-
print(f"Typeset val: {len(typeset_val_records):,} records mixed in")
34
+
val_records = load_records(VAL_SPLITS, dedupe=False)
46
35
47
36
import random as _random
48
37
_rng = _random.Random(42)