A fork of pulp-os for the xteink4 adding custom apps
2
fork

Configure Feed

Select the types of activity you want to include in your feed.

fix: missing chars

hans 20038aab 54972a82

+197 -20
+112 -19
smol-epub/src/html_strip.rs
··· 53 53 #[repr(u8)] 54 54 enum Phase { 55 55 Text, 56 + Utf8Cont, 56 57 AfterLt, 57 58 TagName, 58 59 TagBody, ··· 114 115 trailing_nl: u8, // deferred newlines; flushed before next visible byte; capped at 2 115 116 has_output: bool, // true once any visible char emitted; suppresses leading whitespace 116 117 118 + // UTF-8 multi-byte accumulator (used in Utf8Cont phase) 119 + utf8_acc: u32, 120 + utf8_remaining: u8, 121 + 117 122 // deferred open-style markers; open-tag markers (bold on, heading on, etc.) 118 123 // appear AFTER paragraph-break newlines and BEFORE text. 119 124 // close-tag markers go to `pending` immediately (before paragraph newlines). ··· 154 159 last_was_space: true, 155 160 trailing_nl: 0, 156 161 has_output: false, 162 + utf8_acc: 0, 163 + utf8_remaining: 0, 157 164 deferred: [0u8; DEFERRED_CAP], 158 165 deferred_len: 0, 159 166 pending: [0u8; PENDING_CAP], ··· 226 233 self.phase = Phase::Entity; 227 234 } else if is_html_ws(b) { 228 235 self.queue_ws(); 236 + } else if b >= 0xC0 { 237 + // UTF-8 lead byte: start multi-byte accumulation 238 + if b < 0xE0 { 239 + self.utf8_acc = (b as u32) & 0x1F; 240 + self.utf8_remaining = 1; 241 + } else if b < 0xF0 { 242 + self.utf8_acc = (b as u32) & 0x0F; 243 + self.utf8_remaining = 2; 244 + } else { 245 + self.utf8_acc = (b as u32) & 0x07; 246 + self.utf8_remaining = 3; 247 + } 248 + self.phase = Phase::Utf8Cont; 249 + } else if b >= 0x80 { 250 + // stray continuation byte; skip silently 229 251 } else { 230 252 self.queue_text(b); 253 + } 254 + } 255 + 256 + // UTF-8 continuation bytes; accumulate then map to ASCII 257 + Phase::Utf8Cont => { 258 + if b & 0xC0 == 0x80 { 259 + self.utf8_acc = (self.utf8_acc << 6) | (b as u32 & 0x3F); 260 + self.utf8_remaining -= 1; 261 + if self.utf8_remaining == 0 { 262 + if let Some(ascii) = codepoint_to_byte(self.utf8_acc) { 263 + if is_html_ws(ascii) { 264 + self.queue_ws(); 265 + } else { 266 + self.queue_text(ascii); 267 + } 268 + } 269 + self.phase = Phase::Text; 270 + } 271 + } else { 272 + // broken sequence; emit replacement and reprocess byte 273 + self.queue_text(b'?'); 274 + self.phase = Phase::Text; 275 + advance = false; 231 276 } 232 277 } 233 278 ··· 914 959 last_was_space = true; 915 960 trailing_nl = 0; 916 961 } 962 + } else if b >= 0xC0 { 963 + // UTF-8 multi-byte sequence; decode and replace with ASCII approx 964 + let (cp, seq_len) = decode_utf8_char(buf, r, len); 965 + if let Some(ascii) = codepoint_to_byte(cp) { 966 + if is_html_ws(ascii) { 967 + if !last_was_space { 968 + buf[w] = b' '; 969 + w += 1; 970 + last_was_space = true; 971 + trailing_nl = 0; 972 + } 973 + } else { 974 + buf[w] = ascii; 975 + w += 1; 976 + last_was_space = false; 977 + trailing_nl = 0; 978 + } 979 + } 980 + r += seq_len; 981 + continue; 982 + } else if b >= 0x80 { 983 + // stray continuation byte; skip 984 + r += 1; 985 + continue; 917 986 } else { 918 987 buf[w] = b; 919 988 w += 1; ··· 1079 1148 match cp { 1080 1149 0 => None, 1081 1150 0x0001..=0x007F => Some(cp as u8), 1082 - 0x00A0 => Some(b' '), // nbsp 1083 - 0x00AD => Some(b'-'), // soft hyphen 1084 - 0x2013 | 0x2014 => Some(b'-'), 1085 - 0x2018..=0x201A => Some(b'\''), 1086 - 0x201C..=0x201E => Some(b'"'), 1087 - 0x2022 => Some(b'*'), 1088 - 0x2026 => Some(b'.'), 1089 - _ => Some(b'?'), // unicode placeholder 1151 + 0x00A0 => Some(b' '), // nbsp 1152 + 0x00AB | 0x00BB => Some(b'"'), // « » 1153 + 0x00AD => Some(b'-'), // soft hyphen 1154 + 0x00B7 => Some(b'.'), // middle dot 1155 + 0x00D7 => Some(b'x'), // multiplication sign 1156 + 0x00F7 => Some(b'/'), // division sign 1157 + 0x2010..=0x2015 => Some(b'-'), // hyphens, dashes (figure dash, horiz bar) 1158 + 0x2018..=0x201B => Some(b'\''), // single quotes (left, right, low-9, reversed-9) 1159 + 0x201C..=0x201F => Some(b'"'), // double quotes (left, right, low-9, reversed-9) 1160 + 0x2022 => Some(b'*'), // bullet 1161 + 0x2026 => Some(b'.'), // horizontal ellipsis 1162 + 0x2032 => Some(b'\''), // prime 1163 + 0x2033 => Some(b'"'), // double prime 1164 + 0x2039 | 0x203A => Some(b'\''), // single guillemets 1165 + 0x2212 => Some(b'-'), // minus sign 1166 + _ => Some(b'?'), // unmapped codepoint 1090 1167 } 1168 + } 1169 + 1170 + /// Decode one UTF-8 character starting at `buf[pos]` (which must be a lead byte >= 0xC0). 1171 + /// Returns `(codepoint, byte_length)`. On malformed input returns `(0xFFFD, 1)`. 1172 + fn decode_utf8_char(buf: &[u8], pos: usize, len: usize) -> (u32, usize) { 1173 + let b0 = buf[pos]; 1174 + let (mut cp, expected) = if b0 < 0xE0 { 1175 + ((b0 as u32) & 0x1F, 2) 1176 + } else if b0 < 0xF0 { 1177 + ((b0 as u32) & 0x0F, 3) 1178 + } else { 1179 + ((b0 as u32) & 0x07, 4) 1180 + }; 1181 + if pos + expected > len { 1182 + return (0xFFFD, len - pos); // truncated sequence; consume remaining 1183 + } 1184 + for i in 1..expected { 1185 + let cont = buf[pos + i]; 1186 + if cont & 0xC0 != 0x80 { 1187 + return (0xFFFD, i); // broken: stop before bad byte 1188 + } 1189 + cp = (cp << 6) | (cont as u32 & 0x3F); 1190 + } 1191 + (cp, expected) 1091 1192 } 1092 1193 1093 1194 // in-place entity decoding; separate from resolve_entity ··· 1145 1246 } 1146 1247 1147 1248 fn codepoint_to_decoded_inplace(cp: u32) -> DecodedInplace { 1148 - match cp { 1149 - 0 => DecodedInplace::None, 1150 - 0x0001..=0x007F => DecodedInplace::Byte(cp as u8), 1151 - 0x00A0 => DecodedInplace::Byte(b' '), 1152 - 0x00AD => DecodedInplace::Byte(b'-'), 1153 - 0x2013 | 0x2014 => DecodedInplace::Byte(b'-'), 1154 - 0x2018..=0x201A => DecodedInplace::Byte(b'\''), 1155 - 0x201C..=0x201E => DecodedInplace::Byte(b'"'), 1156 - 0x2022 => DecodedInplace::Byte(b'*'), 1157 - 0x2026 => DecodedInplace::Byte(b'.'), 1158 - _ => DecodedInplace::Byte(b'?'), 1249 + match codepoint_to_byte(cp) { 1250 + Some(b) => DecodedInplace::Byte(b), 1251 + None => DecodedInplace::None, 1159 1252 } 1160 1253 } 1161 1254
+85 -1
src/apps/reader.rs
··· 1266 1266 1267 1267 // helpers 1268 1268 1269 + /// Decode one UTF-8 character starting at `buf[pos]` (a lead byte >= 0xC0) 1270 + /// and map the codepoint to a printable ASCII replacement. 1271 + /// Returns `(ascii_byte, byte_length_consumed)`. 1272 + fn decode_utf8_to_ascii(buf: &[u8], pos: usize) -> (u8, usize) { 1273 + let b0 = buf[pos]; 1274 + let (mut cp, expected) = if b0 < 0xE0 { 1275 + ((b0 as u32) & 0x1F, 2) 1276 + } else if b0 < 0xF0 { 1277 + ((b0 as u32) & 0x0F, 3) 1278 + } else { 1279 + ((b0 as u32) & 0x07, 4) 1280 + }; 1281 + let len = buf.len(); 1282 + if pos + expected > len { 1283 + return (b'?', len - pos); 1284 + } 1285 + for i in 1..expected { 1286 + let cont = buf[pos + i]; 1287 + if cont & 0xC0 != 0x80 { 1288 + return (b'?', i); 1289 + } 1290 + cp = (cp << 6) | (cont as u32 & 0x3F); 1291 + } 1292 + let ascii = match cp { 1293 + 0x00A0 => b' ', // non-breaking space 1294 + 0x00AB | 0x00BB => b'"', // « » 1295 + 0x00AD => b'-', // soft hyphen 1296 + 0x00B7 => b'.', // middle dot 1297 + 0x00D7 => b'x', // multiplication sign 1298 + 0x00F7 => b'/', // division sign 1299 + 0x2010..=0x2015 => b'-', // hyphens, en-dash, em-dash, etc. 1300 + 0x2018..=0x201B => b'\'', // single curly quotes 1301 + 0x201C..=0x201F => b'"', // double curly quotes 1302 + 0x2022 => b'*', // bullet 1303 + 0x2026 => b'.', // horizontal ellipsis 1304 + 0x2032 => b'\'', // prime 1305 + 0x2033 => b'"', // double prime 1306 + 0x2039 | 0x203A => b'\'', // single guillemets 1307 + 0x2212 => b'-', // minus sign 1308 + _ => b'?', 1309 + }; 1310 + (ascii, expected) 1311 + } 1312 + 1269 1313 fn trim_trailing_cr(buf: &[u8], start: usize, end: usize) -> usize { 1270 1314 if end > start && buf[end - 1] == b'\r' { 1271 1315 end - 1 ··· 1420 1464 if lc >= max_l { 1421 1465 return (ls, lc); 1422 1466 } 1467 + i += 1; 1468 + continue; 1469 + } 1470 + 1471 + // UTF-8 multi-byte: decode entire sequence, use replacement char advance 1472 + if b >= 0xC0 { 1473 + let (repl, seq_len) = decode_utf8_to_ascii(buf, i); 1474 + let sty = current_style(bold, italic, heading); 1475 + let adv = fonts.advance(repl as char, sty) as u32; 1476 + px += adv; 1477 + if px > max_w { 1478 + if sp > ls { 1479 + emit!(ls, sp); 1480 + px -= sp_px; 1481 + ls = sp; 1482 + } else { 1483 + emit!(ls, i); 1484 + ls = i; 1485 + px = adv; 1486 + } 1487 + sp = ls; 1488 + sp_px = 0; 1489 + if lc >= max_l { 1490 + return (ls, lc); 1491 + } 1492 + } 1493 + i += seq_len; 1494 + continue; 1495 + } 1496 + if b >= 0x80 { 1497 + // stray continuation byte; skip without affecting layout 1423 1498 i += 1; 1424 1499 continue; 1425 1500 } ··· 2300 2375 j += 2; 2301 2376 continue; 2302 2377 } 2378 + // UTF-8 lead byte: decode full sequence, render ASCII replacement 2379 + if b >= 0xC0 { 2380 + let (repl, seq_len) = decode_utf8_to_ascii(line, j); 2381 + if (bitmap::FIRST_CHAR..=bitmap::LAST_CHAR).contains(&repl) { 2382 + cx += fs.draw_char(strip, repl as char, sty, cx, baseline) as i32; 2383 + } 2384 + j += seq_len; 2385 + continue; 2386 + } 2303 2387 if !(bitmap::FIRST_CHAR..=bitmap::LAST_CHAR).contains(&b) { 2304 2388 j += 1; 2305 - continue; // non-printable 2389 + continue; // stray continuation byte or control char 2306 2390 } 2307 2391 cx += fs.draw_char(strip, b as char, sty, cx, baseline) as i32; 2308 2392 j += 1;