this repo has no description
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat: SVG overlay cloze boxes via typst frame walking

- Remove typst-rendered yellow highlight from #blank[]; blanks now render
as plain text in the authoring preamble
- Add frame walker in tala-typst: after compilation, recurse the layout
frame tree matching per-glyph source spans against blank content spans
to produce pixel-space bounding boxes (blank_boxes in RenderResult)
- tala-typst render() gains blank_spans parameter; update example and tests
- Add blank_boxes_located test: verifies 2 non-zero boxes found for 2-blank fixture
- Editor parses source via tala-format to extract blank content spans,
passes them to render(), normalizes boxes to [0,1] in PreviewData
- SVG overlay on preview image: yellow semi-transparent rects for existing
blanks, blue rects for user-drawn boxes, dashed live drag preview
- Draw Cloze toggle button; draw-capture div handles click-drag-release;
IoU matching identifies which existing blank a drawn box targets

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

+403 -20
+1
Cargo.lock
··· 7601 7601 "base64", 7602 7602 "dioxus", 7603 7603 "image", 7604 + "tala-format", 7604 7605 "tala-typst", 7605 7606 "tokio", 7606 7607 ]
+1 -1
crates/tala-typst/examples/dump_render.rs
··· 12 12 let mut images = Vec::new(); 13 13 14 14 for (label, preamble, fragment) in cases { 15 - let r = tala_typst::render(&tmp, fragment, preamble).unwrap(); 15 + let r = tala_typst::render(&tmp, fragment, preamble, &[]).unwrap(); 16 16 let straight = premul_to_straight(&r.rgba); 17 17 let img = image::RgbaImage::from_raw(r.width, r.height, straight).unwrap(); 18 18 println!("{label}: {}x{}", r.width, r.height);
+130 -9
crates/tala-typst/src/lib.rs
··· 1 + use std::ops::Range; 1 2 use std::path::{Path, PathBuf}; 2 3 use std::sync::OnceLock; 3 4 ··· 26 27 pub rgba: Vec<u8>, 27 28 pub width: u32, 28 29 pub height: u32, 30 + /// Pixel-space [x, y, w, h] bounding boxes for each blank, in input order. 31 + /// Populated only when `blank_spans` is non-empty. Zero-rect if a blank 32 + /// produced no glyphs (e.g. empty content or parse mismatch). 33 + pub blank_boxes: Vec<[f32; 4]>, 29 34 } 30 35 31 36 // ── Entry point ─────────────────────────────────────────────────────────────── ··· 33 38 /// Render a fragment of typst markup (e.g. a single `#card(...)` call) with the 34 39 /// given preamble. `deck_dir` is the deck root; image files are resolved 35 40 /// relative to `deck_dir/images/`. 36 - pub fn render(deck_dir: &Path, fragment: &str, preamble: Preamble) -> Result<RenderResult, Error> { 37 - let source_text = format!("{}\n{fragment}", preamble.as_str()); 41 + /// 42 + /// `blank_spans` are byte ranges (relative to `fragment`) of the *content* of 43 + /// each `#blank[]` call, in document order. Pass `&[]` when not needed. 44 + pub fn render( 45 + deck_dir: &Path, 46 + fragment: &str, 47 + preamble: Preamble, 48 + blank_spans: &[Range<usize>], 49 + ) -> Result<RenderResult, Error> { 50 + let preamble_str = preamble.as_str(); 51 + let source_text = format!("{preamble_str}\n{fragment}"); 38 52 let world = TalaWorld::new(deck_dir, source_text); 39 53 40 54 let warned = typst::compile::<typst::layout::PagedDocument>(&world); ··· 43 57 })?; 44 58 45 59 let page = document.pages.first().ok_or(Error::NoOutput)?; 60 + 61 + // Locate blanks in the frame before rasterising (frame coords are in pt). 62 + let blank_boxes = if blank_spans.is_empty() { 63 + vec![] 64 + } else { 65 + // Spans from tala-format are relative to `fragment`; shift to full source. 66 + let offset = preamble_str.len() + 1; // +1 for the '\n' separator 67 + let shifted: Vec<Range<usize>> = blank_spans 68 + .iter() 69 + .map(|s| (s.start + offset)..(s.end + offset)) 70 + .collect(); 71 + collect_blank_boxes(&page.frame, &world.source, &shifted, 2.0) 72 + }; 73 + 46 74 let pixmap = typst_render_page(page, 2.0); // 2 px/pt ≈ 144 dpi 47 75 48 76 Ok(RenderResult { 49 77 rgba: pixmap.data().to_vec(), 50 78 width: pixmap.width(), 51 79 height: pixmap.height(), 80 + blank_boxes, 52 81 }) 53 82 } 54 83 84 + // ── Frame walking ───────────────────────────────────────────────────────────── 85 + 86 + /// Walk the frame tree and return one pixel-space bounding box per blank span. 87 + fn collect_blank_boxes( 88 + frame: &typst::layout::Frame, 89 + source: &Source, 90 + blank_spans: &[Range<usize>], 91 + scale: f32, 92 + ) -> Vec<[f32; 4]> { 93 + let mut boxes: Vec<Option<[f32; 4]>> = vec![None; blank_spans.len()]; 94 + walk_frame(frame, source, blank_spans, scale, 0.0, 0.0, &mut boxes); 95 + boxes 96 + .into_iter() 97 + .map(|b| b.unwrap_or([0.0, 0.0, 0.0, 0.0])) 98 + .collect() 99 + } 100 + 101 + fn walk_frame( 102 + frame: &typst::layout::Frame, 103 + source: &Source, 104 + blank_spans: &[Range<usize>], 105 + scale: f32, 106 + ox: f32, 107 + oy: f32, 108 + boxes: &mut Vec<Option<[f32; 4]>>, 109 + ) { 110 + for (pos, item) in frame.items() { 111 + let ax = ox + pos.x.to_pt() as f32; 112 + let ay = oy + pos.y.to_pt() as f32; 113 + match item { 114 + typst::layout::FrameItem::Group(g) => { 115 + walk_frame(&g.frame, source, blank_spans, scale, ax, ay, boxes); 116 + } 117 + typst::layout::FrameItem::Text(text) => { 118 + let fs = text.size.to_pt() as f32; 119 + let mut cx = ax; 120 + for glyph in &text.glyphs { 121 + let gw = glyph.x_advance.at(text.size).to_pt() as f32; 122 + if let Some(sr) = source.range(glyph.span.0) { 123 + for (i, span) in blank_spans.iter().enumerate() { 124 + if sr.start >= span.start && sr.end <= span.end { 125 + // Approximate glyph box: baseline is at ay, 126 + // cap-height ≈ 0.8*fs above, descender ≈ 0.2*fs below. 127 + let px = cx * scale; 128 + let py = (ay - fs * 0.8) * scale; 129 + let pw = gw * scale; 130 + let ph = fs * scale; 131 + boxes[i] = Some(match boxes[i] { 132 + None => [px, py, pw, ph], 133 + Some(b) => { 134 + let x0 = b[0].min(px); 135 + let y0 = b[1].min(py); 136 + let x1 = (b[0] + b[2]).max(px + pw); 137 + let y1 = (b[1] + b[3]).max(py + ph); 138 + [x0, y0, x1 - x0, y1 - y0] 139 + } 140 + }); 141 + } 142 + } 143 + } 144 + cx += gw; 145 + } 146 + } 147 + _ => {} 148 + } 149 + } 150 + } 151 + 55 152 // ── Preamble strings ────────────────────────────────────────────────────────── 56 153 57 154 impl Preamble { 58 155 fn as_str(self) -> &'static str { 59 156 match self { 60 - Preamble::Authoring => PREAMBLE_AUTHORING, 157 + Preamble::Authoring => PREAMBLE_AUTHORING, 61 158 Preamble::ReviewFront => PREAMBLE_REVIEW_FRONT, 62 159 Preamble::ReviewCloze => PREAMBLE_REVIEW_CLOZE, 63 160 } ··· 65 162 } 66 163 67 164 /// Show both sides of a FrontBack card separated by a rule; blank contents 68 - /// visible; cloze body visible. 165 + /// visible as plain text (SVG overlay in the editor provides the highlight). 69 166 const PREAMBLE_AUTHORING: &str = r#" 70 167 #set page(width: 300pt, height: auto, margin: 10pt) 71 168 #let card(id: "", dir: "fwd", ..args) = { ··· 76 173 sides.at(1) 77 174 } 78 175 } 79 - #let blank(..args) = box(fill: rgb(255, 220, 50), inset: (x: 3pt, y: 2pt), radius: 2pt, baseline: 20%)[#args.pos().at(0, default: [])] 176 + #let blank(..args) = args.pos().at(0, default: []) 80 177 #let cloze(id: "", ..args) = args.pos().at(0, default: []) 81 178 #let img_cloze(id: "", src: "") = image("images/" + src + ".png") 82 179 #let img(name) = image("images/" + name + ".png") ··· 257 354 #[test] 258 355 fn render_authoring_front_back() { 259 356 let tmp = std::env::temp_dir(); 260 - let result = render(&tmp, FRONT_BACK, Preamble::Authoring).unwrap(); 357 + let result = render(&tmp, FRONT_BACK, Preamble::Authoring, &[]).unwrap(); 261 358 assert!(result.width > 0 && result.height > 0); 262 359 assert_eq!(result.rgba.len() as u32, result.width * result.height * 4); 263 360 } ··· 265 362 #[test] 266 363 fn render_authoring_cloze() { 267 364 let tmp = std::env::temp_dir(); 268 - let result = render(&tmp, CLOZE, Preamble::Authoring).unwrap(); 365 + let result = render(&tmp, CLOZE, Preamble::Authoring, &[]).unwrap(); 269 366 assert!(result.width > 0); 270 367 } 271 368 272 369 #[test] 273 370 fn render_review_front_only() { 274 371 let tmp = std::env::temp_dir(); 275 - let result = render(&tmp, FRONT_BACK, Preamble::ReviewFront).unwrap(); 372 + let result = render(&tmp, FRONT_BACK, Preamble::ReviewFront, &[]).unwrap(); 276 373 assert!(result.width > 0); 277 374 } 278 375 279 376 #[test] 280 377 fn render_review_cloze_blanks_hidden() { 281 378 let tmp = std::env::temp_dir(); 282 - let result = render(&tmp, CLOZE, Preamble::ReviewCloze).unwrap(); 379 + let result = render(&tmp, CLOZE, Preamble::ReviewCloze, &[]).unwrap(); 283 380 assert!(result.width > 0); 381 + } 382 + 383 + #[test] 384 + fn blank_boxes_located() { 385 + let tmp = std::env::temp_dir(); 386 + // Find the byte ranges of "inductance" and "capacitance" in CLOZE. 387 + let inductance_start = CLOZE.find("inductance").unwrap(); 388 + let inductance_end = inductance_start + "inductance".len(); 389 + let capacitance_start = CLOZE.find("capacitance").unwrap(); 390 + let capacitance_end = capacitance_start + "capacitance".len(); 391 + let spans = [ 392 + inductance_start..inductance_end, 393 + capacitance_start..capacitance_end, 394 + ]; 395 + let result = render(&tmp, CLOZE, Preamble::Authoring, &spans).unwrap(); 396 + assert_eq!(result.blank_boxes.len(), 2); 397 + // Both boxes should have non-zero width (glyphs found). 398 + assert!(result.blank_boxes[0][2] > 0.0, "inductance box has no width"); 399 + assert!(result.blank_boxes[1][2] > 0.0, "capacitance box has no width"); 400 + // Inductance should be to the left of capacitance (same line, earlier in text). 401 + assert!( 402 + result.blank_boxes[0][0] < result.blank_boxes[1][0], 403 + "inductance should be left of capacitance" 404 + ); 284 405 } 285 406 }
+1
crates/tala/Cargo.toml
··· 9 9 [dependencies] 10 10 dioxus = { version = "0.7.1", features = ["router"] } 11 11 tala-typst = { path = "../tala-typst" } 12 + tala-format = { path = "../tala-format" } 12 13 image = { workspace = true } 13 14 base64 = "0.22" 14 15 tokio = { version = "1", features = ["time"] }
+45 -1
crates/tala/assets/main.css
··· 107 107 #preview-pane { 108 108 flex: 1; 109 109 display: flex; 110 + flex-direction: column; 110 111 align-items: center; 111 - justify-content: center; 112 112 background: #13151c; 113 113 padding: 24px; 114 114 overflow: auto; 115 115 } 116 116 117 + .preview-toolbar { 118 + display: flex; 119 + gap: 8px; 120 + padding-bottom: 8px; 121 + flex-shrink: 0; 122 + align-self: flex-start; 123 + } 124 + 125 + .preview-container { 126 + position: relative; 127 + display: inline-block; 128 + line-height: 0; 129 + } 130 + 131 + .cloze-overlay { 132 + position: absolute; 133 + top: 0; 134 + left: 0; 135 + width: 100%; 136 + height: 100%; 137 + pointer-events: none; 138 + overflow: visible; 139 + } 140 + 141 + .draw-capture { 142 + position: absolute; 143 + inset: 0; 144 + cursor: crosshair; 145 + } 146 + 147 + .btn { 148 + background: #1e2130; 149 + color: #8892aa; 150 + border: 1px solid #2a2f45; 151 + border-radius: 4px; 152 + font-size: 12px; 153 + padding: 4px 10px; 154 + cursor: pointer; 155 + } 156 + 157 + .btn:hover { color: #fff; background: #252a3d; } 158 + .btn.active { color: #fff; background: #3a4070; border-color: #5060a0; } 159 + 117 160 .card-preview { 161 + display: block; 118 162 max-width: 100%; 119 163 border-radius: 4px; 120 164 box-shadow: 0 4px 24px rgba(0,0,0,0.5);
+225 -9
crates/tala/src/main.rs
··· 4 4 use base64::Engine as _; 5 5 use dioxus::prelude::*; 6 6 use image::RgbaImage; 7 + use tala_format::CardKind; 7 8 8 9 const FAVICON: Asset = asset!("/assets/favicon.ico"); 9 10 const MAIN_CSS: Asset = asset!("/assets/main.css"); ··· 115 116 } 116 117 } 117 118 119 + struct PreviewData { 120 + b64: String, 121 + img_w: u32, 122 + img_h: u32, 123 + /// Normalized [x, y, w, h] in [0.0, 1.0] for each blank, in document order. 124 + blank_rects: Vec<[f64; 4]>, 125 + } 126 + 118 127 #[component] 119 128 fn Editor() -> Element { 120 129 let mut source = use_signal(|| { ··· 125 134 let preview = use_resource(move || async move { 126 135 let text = source.read().clone(); 127 136 tokio::time::sleep(Duration::from_millis(300)).await; 128 - tokio::task::spawn_blocking(move || render_to_png(&text)) 137 + tokio::task::spawn_blocking(move || render_preview(&text)) 129 138 .await 130 139 .unwrap_or_else(|e| Err(e.to_string())) 131 140 }); 132 141 142 + // Blank rects synced from the latest preview result (needed in event closures). 143 + let mut blank_rects_sig = use_signal(|| Vec::<[f64; 4]>::new()); 144 + use_effect(move || { 145 + if let Some(Ok(data)) = &*preview.read() { 146 + blank_rects_sig.set(data.blank_rects.clone()); 147 + } 148 + }); 149 + 150 + // Draw mode state. 151 + let mut draw_mode = use_signal(|| false); 152 + let mut drag_start = use_signal(|| Option::<(f64, f64)>::None); 153 + let mut drag_current = use_signal(|| Option::<(f64, f64)>::None); 154 + // (normalized rect, matched blank index) 155 + let mut drawn_boxes = use_signal(|| Vec::<([f64; 4], Option<usize>)>::new()); 156 + // CSS pixel dimensions of the draw-capture div (updated on mousedown). 157 + let cap_size = use_signal(|| (1.0f64, 1.0f64)); 158 + 133 159 rsx! { 134 160 div { id: "editor", 135 161 div { id: "source-pane", ··· 145 171 onmousedown: move |_| { document::eval(DIVIDER_DRAG_JS); }, 146 172 } 147 173 div { id: "preview-pane", 174 + div { class: "preview-toolbar", 175 + button { 176 + class: if *draw_mode.read() { "btn active" } else { "btn" }, 177 + onclick: move |_| { 178 + let m = *draw_mode.read(); 179 + draw_mode.set(!m); 180 + drag_start.set(None); 181 + drag_current.set(None); 182 + }, 183 + if *draw_mode.read() { "Exit Draw" } else { "Draw Cloze" } 184 + } 185 + } 148 186 match &*preview.read() { 149 187 None => rsx! { span { class: "status", "Rendering…" } }, 150 - Some(Ok(b64)) => { 151 - let src = format!("data:image/png;base64,{b64}"); 152 - rsx! { img { class: "card-preview", src: "{src}" } } 153 - } 154 188 Some(Err(msg)) => rsx! { pre { class: "render-error", "{msg}" } }, 189 + Some(Ok(data)) => { 190 + let src = format!("data:image/png;base64,{}", data.b64); 191 + let vb = format!("0 0 {} {}", data.img_w, data.img_h); 192 + let iw = data.img_w as f64; 193 + let ih = data.img_h as f64; 194 + let blank_rects = data.blank_rects.clone(); 195 + let drawn = drawn_boxes.read().clone(); 196 + let ds = *drag_start.read(); 197 + let dc = *drag_current.read(); 198 + let in_draw = *draw_mode.read(); 199 + rsx! { 200 + div { class: "preview-container", 201 + img { class: "card-preview", src: "{src}" } 202 + svg { 203 + class: "cloze-overlay", 204 + view_box: "{vb}", 205 + // Existing blank overlays 206 + for rect in &blank_rects { 207 + rect { 208 + x: "{rect[0] * iw}", 209 + y: "{rect[1] * ih}", 210 + width: "{rect[2] * iw}", 211 + height: "{rect[3] * ih}", 212 + fill: "rgba(255,220,50,0.35)", 213 + stroke: "rgb(200,150,0)", 214 + stroke_width: "1.5", 215 + rx: "3", 216 + } 217 + } 218 + // User-drawn boxes 219 + for (rect, _) in &drawn { 220 + rect { 221 + x: "{rect[0] * iw}", 222 + y: "{rect[1] * ih}", 223 + width: "{rect[2] * iw}", 224 + height: "{rect[3] * ih}", 225 + fill: "rgba(100,180,255,0.3)", 226 + stroke: "rgb(70,130,220)", 227 + stroke_width: "1.5", 228 + rx: "3", 229 + } 230 + } 231 + // Live drag preview 232 + if let (Some((sx, sy)), Some((cx, cy))) = (ds, dc) { 233 + { 234 + let rx = sx.min(cx) * iw; 235 + let ry = sy.min(cy) * ih; 236 + let rw = (cx - sx).abs() * iw; 237 + let rh = (cy - sy).abs() * ih; 238 + rsx! { 239 + rect { 240 + x: "{rx}", y: "{ry}", 241 + width: "{rw}", height: "{rh}", 242 + fill: "rgba(100,180,255,0.1)", 243 + stroke: "rgb(70,130,220)", 244 + stroke_width: "1.5", 245 + stroke_dasharray: "4 2", 246 + rx: "3", 247 + } 248 + } 249 + } 250 + } 251 + } 252 + if in_draw { 253 + div { 254 + class: "draw-capture", 255 + onmousedown: move |e| { 256 + let ox = e.data().element_coordinates().x; 257 + let oy = e.data().element_coordinates().y; 258 + // Read cap_size via JS on each drag start for accuracy. 259 + let mut ds = drag_start; 260 + let mut dc = drag_current; 261 + let mut cs = cap_size; 262 + spawn(async move { 263 + let mut eval = document::eval(r#" 264 + var el = document.querySelector('.draw-capture'); 265 + var r = el ? el.getBoundingClientRect() : {width:1,height:1}; 266 + dioxus.send([r.width, r.height]); 267 + "#); 268 + if let Ok(val) = eval.recv::<[f64; 2]>().await { 269 + let w = val[0].max(1.0); 270 + let h = val[1].max(1.0); 271 + cs.set((w, h)); 272 + ds.set(Some((ox / w, oy / h))); 273 + dc.set(Some((ox / w, oy / h))); 274 + } 275 + }); 276 + }, 277 + onmousemove: move |e| { 278 + if drag_start.read().is_some() { 279 + let (w, h) = *cap_size.read(); 280 + let nx = e.data().element_coordinates().x / w; 281 + let ny = e.data().element_coordinates().y / h; 282 + drag_current.set(Some((nx.clamp(0.0, 1.0), ny.clamp(0.0, 1.0)))); 283 + } 284 + }, 285 + onmouseleave: move |_| { 286 + drag_start.set(None); 287 + drag_current.set(None); 288 + }, 289 + onmouseup: move |e| { 290 + let Some((sx, sy)) = *drag_start.read() else { return; }; 291 + let (w, h) = *cap_size.read(); 292 + let nx = (e.data().element_coordinates().x / w).clamp(0.0, 1.0); 293 + let ny = (e.data().element_coordinates().y / h).clamp(0.0, 1.0); 294 + let rx = sx.min(nx); 295 + let ry = sy.min(ny); 296 + let rw = (nx - sx).abs(); 297 + let rh = (ny - sy).abs(); 298 + drag_start.set(None); 299 + drag_current.set(None); 300 + if rw * rh < 0.001 { return; } 301 + let drawn_rect = [rx, ry, rw, rh]; 302 + let blanks = blank_rects_sig.read().clone(); 303 + let best = blanks 304 + .iter() 305 + .enumerate() 306 + .map(|(i, &b)| (i, iou(drawn_rect, b))) 307 + .filter(|&(_, s)| s > 0.05) 308 + .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap()) 309 + .map(|(i, _)| i); 310 + drawn_boxes.write().push((drawn_rect, best)); 311 + }, 312 + } 313 + } 314 + } 315 + } 316 + } 155 317 } 156 318 } 157 319 } 158 320 } 159 321 } 160 322 161 - fn render_to_png(source: &str) -> Result<String, String> { 323 + fn iou(a: [f64; 4], b: [f64; 4]) -> f64 { 324 + let ix = (a[0] + a[2]).min(b[0] + b[2]) - a[0].max(b[0]); 325 + let iy = (a[1] + a[3]).min(b[1] + b[3]) - a[1].max(b[1]); 326 + if ix <= 0.0 || iy <= 0.0 { 327 + return 0.0; 328 + } 329 + let inter = ix * iy; 330 + inter / (a[2] * a[3] + b[2] * b[3] - inter) 331 + } 332 + 333 + fn render_preview(source: &str) -> Result<PreviewData, String> { 162 334 let deck_dir = PathBuf::from(std::env::temp_dir()); 163 - let result = tala_typst::render(&deck_dir, source, tala_typst::Preamble::Authoring) 164 - .map_err(|e| e.to_string())?; 335 + 336 + // Collect content_spans of all blanks in document order. 337 + let cards = tala_format::parse_cards(source); 338 + let blank_spans: Vec<std::ops::Range<usize>> = cards 339 + .into_iter() 340 + .filter_map(|c| { 341 + if let CardKind::Cloze { blanks, .. } = c.kind { 342 + Some(blanks) 343 + } else { 344 + None 345 + } 346 + }) 347 + .flatten() 348 + .map(|b| b.content_span) 349 + .collect(); 165 350 351 + let result = tala_typst::render( 352 + &deck_dir, 353 + source, 354 + tala_typst::Preamble::Authoring, 355 + &blank_spans, 356 + ) 357 + .map_err(|e| e.to_string())?; 358 + 359 + // Normalize blank boxes to [0,1]. 360 + let iw = result.width as f64; 361 + let ih = result.height as f64; 362 + let blank_rects = result 363 + .blank_boxes 364 + .iter() 365 + .filter(|b| b[2] > 0.0) // skip zero-width (not found) 366 + .map(|b| { 367 + [ 368 + b[0] as f64 / iw, 369 + b[1] as f64 / ih, 370 + b[2] as f64 / iw, 371 + b[3] as f64 / ih, 372 + ] 373 + }) 374 + .collect(); 375 + 376 + // Premultiplied → straight alpha. 166 377 let straight: Vec<u8> = result 167 378 .rgba 168 379 .chunks_exact(4) ··· 190 401 .write_to(&mut buf, image::ImageFormat::Png) 191 402 .map_err(|e| e.to_string())?; 192 403 193 - Ok(base64::engine::general_purpose::STANDARD.encode(buf.into_inner())) 404 + Ok(PreviewData { 405 + b64: base64::engine::general_purpose::STANDARD.encode(buf.into_inner()), 406 + img_w: result.width, 407 + img_h: result.height, 408 + blank_rects, 409 + }) 194 410 }