feat: draw-to-blank phase 2 -- insert #blank[] from drawn box

+110 -5

crates/tala-typst/src/lib.rs

··· 4 4 5 5 use typst::diag::{FileError, FileResult}; 6 6 use typst::foundations::{Bytes, Datetime}; 7 - use typst::syntax::{FileId, Source, VirtualPath}; 7 + use typst::syntax::{FileId, LinkedNode, Source, SyntaxKind, VirtualPath}; 8 8 use typst::text::{Font, FontBook}; 9 9 use typst::utils::LazyHash; 10 10 use typst::{Library, LibraryExt}; ··· 31 31 /// Populated only when `blank_spans` is non-empty. Zero-rect if a blank 32 32 /// produced no glyphs (e.g. empty content or parse mismatch). 33 33 pub blank_boxes: Vec<[f32; 4]>, 34 + /// Pixel-space `[x, y, w, h]`, fragment-relative byte range, and math flag 35 + /// for every glyph rendered from the fragment (preamble glyphs excluded). 36 + pub glyph_map: Vec<([f32; 4], Range<usize>, bool)>, 34 37 } 35 38 36 39 // ── Entry point ─────────────────────────────────────────────────────────────── ··· 58 61 59 62 let page = document.pages.first().ok_or(Error::NoOutput)?; 60 63 61 - // Locate blanks in the frame before rasterising (frame coords are in pt). 64 + // Spans from tala-format are relative to `fragment`; shift to full source. 65 + let preamble_offset = preamble_str.len() + 1; // +1 for the '\n' separator 66 + 62 67 let blank_boxes = if blank_spans.is_empty() { 63 68 vec![] 64 69 } else { 65 - // Spans from tala-format are relative to `fragment`; shift to full source. 66 - let offset = preamble_str.len() + 1; // +1 for the '\n' separator 67 70 let shifted: Vec<Range<usize>> = blank_spans 68 71 .iter() 69 - .map(|s| (s.start + offset)..(s.end + offset)) 72 + .map(|s| (s.start + preamble_offset)..(s.end + preamble_offset)) 70 73 .collect(); 71 74 collect_blank_boxes(&page.frame, &world.source, &shifted, 2.0) 72 75 }; 73 76 77 + let math_spans = collect_math_spans(&world.source, preamble_offset); 78 + let glyph_map = 79 + collect_glyph_map(&page.frame, &world.source, preamble_offset, 2.0, &math_spans); 80 + 74 81 let pixmap = typst_render_page(page, 2.0); // 2 px/pt ≈ 144 dpi 75 82 76 83 Ok(RenderResult { ··· 78 85 width: pixmap.width(), 79 86 height: pixmap.height(), 80 87 blank_boxes, 88 + glyph_map, 81 89 }) 82 90 } 83 91 ··· 139 147 } 140 148 }); 141 149 } 150 + } 151 + } 152 + cx += gw; 153 + } 154 + } 155 + _ => {} 156 + } 157 + } 158 + } 159 + 160 + // ── Glyph map (all fragment glyphs with source ranges and math flag) ────────── 161 + 162 + /// Collect byte ranges of all `Equation` nodes in `source` that originate from 163 + /// the fragment (i.e. start at or after `preamble_offset`), returned as 164 + /// fragment-relative ranges. 165 + fn collect_math_spans(source: &Source, preamble_offset: usize) -> Vec<Range<usize>> { 166 + let linked = LinkedNode::new(source.root()); 167 + let mut spans = Vec::new(); 168 + collect_equation_spans(linked, &mut spans, preamble_offset); 169 + spans 170 + } 171 + 172 + fn collect_equation_spans( 173 + node: LinkedNode<'_>, 174 + spans: &mut Vec<Range<usize>>, 175 + preamble_offset: usize, 176 + ) { 177 + if node.kind() == SyntaxKind::Equation { 178 + let r = node.range(); 179 + if r.end > preamble_offset { 180 + let start = r.start.saturating_sub(preamble_offset); 181 + let end = r.end - preamble_offset; 182 + spans.push(start..end); 183 + } 184 + return; // don't recurse into nested equations 185 + } 186 + for child in node.children() { 187 + collect_equation_spans(child, spans, preamble_offset); 188 + } 189 + } 190 + 191 + /// Walk the frame tree and return one entry per fragment glyph: 192 + /// `(pixel_rect [x,y,w,h], fragment-relative source range, is_in_math)`. 193 + fn collect_glyph_map( 194 + frame: &typst::layout::Frame, 195 + source: &Source, 196 + preamble_offset: usize, 197 + scale: f32, 198 + math_spans: &[Range<usize>], 199 + ) -> Vec<([f32; 4], Range<usize>, bool)> { 200 + let mut map = Vec::new(); 201 + walk_frame_for_glyphs(frame, source, preamble_offset, scale, 0.0, 0.0, math_spans, &mut map); 202 + map 203 + } 204 + 205 + fn walk_frame_for_glyphs( 206 + frame: &typst::layout::Frame, 207 + source: &Source, 208 + preamble_offset: usize, 209 + scale: f32, 210 + ox: f32, 211 + oy: f32, 212 + math_spans: &[Range<usize>], 213 + map: &mut Vec<([f32; 4], Range<usize>, bool)>, 214 + ) { 215 + for (pos, item) in frame.items() { 216 + let ax = ox + pos.x.to_pt() as f32; 217 + let ay = oy + pos.y.to_pt() as f32; 218 + match item { 219 + typst::layout::FrameItem::Group(g) => { 220 + walk_frame_for_glyphs( 221 + &g.frame, source, preamble_offset, scale, ax, ay, math_spans, map, 222 + ); 223 + } 224 + typst::layout::FrameItem::Text(text) => { 225 + let fs = text.size.to_pt() as f32; 226 + let src_text = source.text(); 227 + let mut cx = ax; 228 + for glyph in &text.glyphs { 229 + let gw = glyph.x_advance.at(text.size).to_pt() as f32; 230 + if let Some(sr) = source.range(glyph.span.0) { 231 + // glyph.span.1 is the byte offset of this specific character 232 + // within the span node -- crucial for getting per-char ranges. 233 + let char_abs = sr.start + glyph.span.1 as usize; 234 + if char_abs >= preamble_offset { 235 + let frag_start = char_abs - preamble_offset; 236 + let char_len = src_text[char_abs..] 237 + .chars() 238 + .next() 239 + .map_or(1, |c| c.len_utf8()); 240 + let frag_end = frag_start + char_len; 241 + let in_math = math_spans 242 + .iter() 243 + .any(|ms| frag_start < ms.end && frag_end > ms.start); 244 + let px = cx * scale; 245 + let py = (ay - fs * 0.8) * scale; 246 + map.push(([px, py, gw * scale, fs * scale], frag_start..frag_end, in_math)); 142 247 } 143 248 } 144 249 cx += gw;

+6

crates/tala/assets/main.css

··· 116 116 117 117 .preview-toolbar { 118 118 display: flex; 119 + align-items: center; 119 120 gap: 8px; 120 121 padding-bottom: 8px; 121 122 flex-shrink: 0; 122 123 align-self: flex-start; 124 + } 125 + 126 + .insert-error { 127 + color: #e06c75; 128 + font-size: 12px; 123 129 } 124 130 125 131 .preview-container {

+69 -23

crates/tala/src/main.rs

··· 122 122 img_h: u32, 123 123 /// Normalized [x, y, w, h] in [0.0, 1.0] for each blank, in document order. 124 124 blank_rects: Vec<[f64; 4]>, 125 + /// Normalized rect, fragment-relative byte range, is_in_math for every glyph. 126 + glyph_map: Vec<([f64; 4], std::ops::Range<usize>, bool)>, 125 127 } 126 128 127 129 #[component] ··· 139 141 .unwrap_or_else(|e| Err(e.to_string())) 140 142 }); 141 143 142 - // Blank rects synced from the latest preview result (needed in event closures). 144 + // Blank rects and glyph map synced from latest preview (needed in event closures). 143 145 let mut blank_rects_sig = use_signal(|| Vec::<[f64; 4]>::new()); 146 + let mut glyph_map_sig = 147 + use_signal(|| Vec::<([f64; 4], std::ops::Range<usize>, bool)>::new()); 144 148 use_effect(move || { 145 149 if let Some(Ok(data)) = &*preview.read() { 146 150 blank_rects_sig.set(data.blank_rects.clone()); 151 + glyph_map_sig.set(data.glyph_map.clone()); 147 152 } 148 153 }); 149 154 ··· 151 156 let mut draw_mode = use_signal(|| false); 152 157 let mut drag_start = use_signal(|| Option::<(f64, f64)>::None); 153 158 let mut drag_current = use_signal(|| Option::<(f64, f64)>::None); 154 - // (normalized rect, matched blank index) 155 - let mut drawn_boxes = use_signal(|| Vec::<([f64; 4], Option<usize>)>::new()); 159 + // Blue error boxes: drawn boxes that didn't map to insertable text. 160 + let mut drawn_boxes = use_signal(|| Vec::<[f64; 4]>::new()); 156 161 // CSS pixel dimensions of the draw-capture div (updated on mousedown). 157 162 let cap_size = use_signal(|| (1.0f64, 1.0f64)); 163 + // Insertion error shown near the toolbar. 164 + let mut insert_error = use_signal(|| Option::<String>::None); 158 165 159 166 rsx! { 160 167 div { id: "editor", ··· 179 186 draw_mode.set(!m); 180 187 drag_start.set(None); 181 188 drag_current.set(None); 189 + drawn_boxes.write().clear(); 190 + insert_error.set(None); 182 191 }, 183 192 if *draw_mode.read() { "Exit Draw" } else { "Draw Cloze" } 193 + } 194 + if let Some(err) = &*insert_error.read() { 195 + span { class: "insert-error", "{err}" } 184 196 } 185 197 } 186 198 match &*preview.read() { ··· 215 227 rx: "3", 216 228 } 217 229 } 218 - // User-drawn boxes 219 - for (rect, _) in &drawn { 230 + // Error boxes (drawn but no text found / math) 231 + for rect in &drawn { 220 232 rect { 221 233 x: "{rect[0] * iw}", 222 234 y: "{rect[1] * ih}", 223 235 width: "{rect[2] * iw}", 224 236 height: "{rect[3] * ih}", 225 - fill: "rgba(100,180,255,0.3)", 226 - stroke: "rgb(70,130,220)", 237 + fill: "rgba(255,100,100,0.2)", 238 + stroke: "rgb(200,60,60)", 227 239 stroke_width: "1.5", 228 240 rx: "3", 229 241 } ··· 299 311 drag_current.set(None); 300 312 if rw * rh < 0.001 { return; } 301 313 let drawn_rect = [rx, ry, rw, rh]; 302 - let blanks = blank_rects_sig.read().clone(); 303 - let best = blanks 314 + 315 + // Find glyphs whose rects overlap the drawn box. 316 + let glyphs = glyph_map_sig.read().clone(); 317 + let hits: Vec<_> = glyphs 304 318 .iter() 305 - .enumerate() 306 - .map(|(i, &b)| (i, iou(drawn_rect, b))) 307 - .filter(|&(_, s)| s > 0.05) 308 - .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap()) 309 - .map(|(i, _)| i); 310 - drawn_boxes.write().push((drawn_rect, best)); 319 + .filter(|(gr, _, _)| rects_overlap(drawn_rect, *gr)) 320 + .cloned() 321 + .collect(); 322 + 323 + if hits.is_empty() { 324 + insert_error.set(Some("No text found in drawn region.".into())); 325 + drawn_boxes.write().push(drawn_rect); 326 + return; 327 + } 328 + if hits.iter().any(|(_, _, in_math)| *in_math) { 329 + insert_error.set(Some("Math regions not supported yet — draw over plain text only.".into())); 330 + drawn_boxes.write().push(drawn_rect); 331 + return; 332 + } 333 + let min_start = hits.iter().map(|(_, r, _)| r.start).min().unwrap(); 334 + let max_end = hits.iter().map(|(_, r, _)| r.end ).max().unwrap(); 335 + let cur = source.read().clone(); 336 + source.set(insert_blank_wrap(&cur, min_start, max_end)); 337 + insert_error.set(None); 311 338 }, 312 339 } 313 340 } ··· 320 347 } 321 348 } 322 349 323 - fn iou(a: [f64; 4], b: [f64; 4]) -> f64 { 324 - let ix = (a[0] + a[2]).min(b[0] + b[2]) - a[0].max(b[0]); 325 - let iy = (a[1] + a[3]).min(b[1] + b[3]) - a[1].max(b[1]); 326 - if ix <= 0.0 || iy <= 0.0 { 327 - return 0.0; 328 - } 329 - let inter = ix * iy; 330 - inter / (a[2] * a[3] + b[2] * b[3] - inter) 350 + /// True if two normalized [x, y, w, h] rects have any overlap. 351 + fn rects_overlap(a: [f64; 4], b: [f64; 4]) -> bool { 352 + a[0] < b[0] + b[2] && a[0] + a[2] > b[0] && a[1] < b[1] + b[3] && a[1] + a[3] > b[1] 331 353 } 354 + 355 + /// Wrap `source[start..end]` in `#blank[...]`. 356 + fn insert_blank_wrap(source: &str, start: usize, end: usize) -> String { 357 + format!("{}#blank[{}]{}", &source[..start], &source[start..end], &source[end..]) 358 + } 359 + 332 360 333 361 fn render_preview(source: &str) -> Result<PreviewData, String> { 334 362 let deck_dir = PathBuf::from(std::env::temp_dir()); ··· 401 429 .write_to(&mut buf, image::ImageFormat::Png) 402 430 .map_err(|e| e.to_string())?; 403 431 432 + let glyph_map = result 433 + .glyph_map 434 + .iter() 435 + .map(|(rect, range, in_math)| { 436 + ( 437 + [ 438 + rect[0] as f64 / iw, 439 + rect[1] as f64 / ih, 440 + rect[2] as f64 / iw, 441 + rect[3] as f64 / ih, 442 + ], 443 + range.clone(), 444 + *in_math, 445 + ) 446 + }) 447 + .collect(); 448 + 404 449 Ok(PreviewData { 405 450 b64: base64::engine::general_purpose::STANDARD.encode(buf.into_inner()), 406 451 img_w: result.width, 407 452 img_h: result.height, 408 453 blank_rects, 454 + glyph_map, 409 455 }) 410 456 }

Configure Feed

Configure Feed