+32

include/internal.h

reviewed

··· 227 227 228 228 typedef struct { 229 229 ant_offset_t len; 230 230 + uint8_t is_ascii; 230 231 char bytes[]; 231 232 } ant_flat_string_t; 233 233 + 234 234 + enum { 235 235 + STR_ASCII_UNKNOWN = 0, 236 236 + STR_ASCII_YES = 1, 237 237 + STR_ASCII_NO = 2, 238 238 + }; 232 239 233 240 typedef struct { 234 241 ant_offset_t len; ··· 436 443 ant_value_t k = js_mkstr(js, name, len); 437 444 if (is_err(k)) return k; 438 445 return mkprop(js, obj, k, fn, ANT_PROP_ATTR_WRITABLE | ANT_PROP_ATTR_CONFIGURABLE); 446 446 + } 447 447 + 448 448 + static inline ant_flat_string_t *str_flat_from_bytes(const char *str) { 449 449 + return (ant_flat_string_t *)((char *)str - offsetof(ant_flat_string_t, bytes)); 450 450 + } 451 451 + 452 452 + static inline uint8_t str_detect_ascii_bytes(const char *str, size_t len) { 453 453 + const unsigned char *s = (const unsigned char *)str; 454 454 + for (size_t i = 0; i < len; i++) { 455 455 + if (s[i] >= 0x80) return STR_ASCII_NO; 456 456 + } 457 457 + return STR_ASCII_YES; 458 458 + } 459 459 + 460 460 + static inline void str_set_ascii_state(const char *str, uint8_t state) { 461 461 + ant_flat_string_t *flat = str_flat_from_bytes(str); 462 462 + flat->is_ascii = state; 463 463 + } 464 464 + 465 465 + static inline bool str_is_ascii(const char *str) { 466 466 + ant_flat_string_t *flat = str_flat_from_bytes(str); 467 467 + if (flat->is_ascii == STR_ASCII_UNKNOWN) { 468 468 + flat->is_ascii = str_detect_ascii_bytes(flat->bytes, (size_t)flat->len); 469 469 + } 470 470 + return flat->is_ascii == STR_ASCII_YES; 439 471 } 440 472 441 473 static inline void js_set_module_default(ant_t *js, ant_value_t lib, ant_value_t ctor_fn, const char *name) {

+58 -8

src/ant.c

reviewed

··· 2307 2307 2308 2308 flat->len = (ant_offset_t)len; 2309 2309 if (ptr && len > 0) memcpy(flat->bytes, ptr, len); 2310 2310 + 2310 2311 flat->bytes[len] = '\0'; 2312 2312 + flat->is_ascii = (ptr || len == 0) 2313 2313 + ? str_detect_ascii_bytes(flat->bytes, len) 2314 2314 + : STR_ASCII_UNKNOWN; 2315 2315 + 2311 2316 return mkval(T_STR, (uintptr_t)flat); 2312 2317 } 2313 2318 ··· 2324 2329 2325 2330 flat->len = (ant_offset_t)len; 2326 2331 if (ptr && len > 0) memcpy(flat->bytes, ptr, len); 2332 2332 + 2327 2333 flat->bytes[len] = '\0'; 2334 2334 + flat->is_ascii = (ptr || len == 0) 2335 2335 + ? str_detect_ascii_bytes(flat->bytes, len) 2336 2336 + : STR_ASCII_UNKNOWN; 2337 2337 + 2328 2338 return mkval(T_STR, (uintptr_t)flat); 2329 2339 } 2330 2340 ··· 8832 8842 } 8833 8843 8834 8844 static ant_value_t builtin_string_toLowerCase(ant_t *js, ant_value_t *args, int nargs) { 8835 8835 - (void) args; (void) nargs; 8836 8845 ant_value_t str = to_string_val(js, js->this_val); 8837 8846 if (vtype(str) != T_STR) return js_mkerr(js, "toLowerCase called on non-string"); 8838 8847 ··· 8845 8854 8846 8855 ant_offset_t out_len = 0; 8847 8856 utf8proc_ssize_t pos = 0; 8857 8857 + 8848 8858 while (pos < src_len) { 8849 8859 utf8proc_int32_t cp; 8850 8860 utf8proc_ssize_t n = utf8_next(src + pos, src_len - pos, &cp); ··· 8856 8866 8857 8867 ant_value_t result = js_mkstr(js, NULL, out_len); 8858 8868 if (is_err(result)) return result; 8869 8869 + 8859 8870 ant_offset_t result_len, result_off = vstr(js, result, &result_len); 8860 8871 char *result_ptr = (char *)(uintptr_t)(result_off); 8872 8872 + uint8_t ascii_state = STR_ASCII_YES; 8861 8873 8862 8874 pos = 0; 8863 8875 ant_offset_t wpos = 0; 8876 8876 + 8864 8877 while (pos < src_len) { 8865 8878 utf8proc_int32_t cp; 8866 8879 utf8proc_ssize_t n = utf8_next(src + pos, src_len - pos, &cp); 8867 8867 - if (cp < 0) { result_ptr[wpos++] = (char)src[pos]; pos++; continue; } 8868 8868 - wpos += (ant_offset_t)utf8proc_encode_char(utf8proc_tolower(cp), (utf8proc_uint8_t *)(result_ptr + wpos)); 8880 8880 + 8881 8881 + if (cp < 0) { 8882 8882 + unsigned char byte = src[pos]; 8883 8883 + if (byte >= 0x80) ascii_state = STR_ASCII_NO; 8884 8884 + result_ptr[wpos++] = (char)byte; 8885 8885 + pos++; continue; 8886 8886 + } 8887 8887 + 8888 8888 + utf8proc_int32_t mapped = utf8proc_tolower(cp); 8889 8889 + if (mapped >= 0x80) ascii_state = STR_ASCII_NO; 8890 8890 + 8891 8891 + wpos += (ant_offset_t)utf8proc_encode_char(mapped, (utf8proc_uint8_t *)(result_ptr + wpos)); 8869 8892 pos += n; 8870 8893 } 8871 8871 - 8894 8894 + 8895 8895 + str_set_ascii_state(result_ptr, ascii_state); 8872 8896 return result; 8873 8897 } 8874 8898 8875 8899 static ant_value_t builtin_string_toUpperCase(ant_t *js, ant_value_t *args, int nargs) { 8876 8876 - (void) args; (void) nargs; 8877 8900 ant_value_t str = to_string_val(js, js->this_val); 8878 8901 if (vtype(str) != T_STR) return js_mkerr(js, "toUpperCase called on non-string"); 8879 8902 ··· 8886 8909 8887 8910 ant_offset_t out_len = 0; 8888 8911 utf8proc_ssize_t pos = 0; 8912 8912 + 8889 8913 while (pos < src_len) { 8890 8914 utf8proc_int32_t cp; 8891 8915 utf8proc_ssize_t n = utf8_next(src + pos, src_len - pos, &cp); ··· 8897 8921 8898 8922 ant_value_t result = js_mkstr(js, NULL, out_len); 8899 8923 if (is_err(result)) return result; 8924 8924 + 8900 8925 ant_offset_t result_len, result_off = vstr(js, result, &result_len); 8901 8926 char *result_ptr = (char *)(uintptr_t)(result_off); 8927 8927 + uint8_t ascii_state = STR_ASCII_YES; 8902 8928 8903 8929 pos = 0; 8904 8930 ant_offset_t wpos = 0; 8931 8931 + 8905 8932 while (pos < src_len) { 8906 8933 utf8proc_int32_t cp; 8907 8934 utf8proc_ssize_t n = utf8_next(src + pos, src_len - pos, &cp); 8908 8908 - if (cp < 0) { result_ptr[wpos++] = (char)src[pos]; pos++; continue; } 8909 8909 - wpos += (ant_offset_t)utf8proc_encode_char(utf8proc_toupper(cp), (utf8proc_uint8_t *)(result_ptr + wpos)); 8935 8935 + 8936 8936 + if (cp < 0) { 8937 8937 + unsigned char byte = src[pos]; 8938 8938 + if (byte >= 0x80) ascii_state = STR_ASCII_NO; 8939 8939 + result_ptr[wpos++] = (char)byte; 8940 8940 + pos++; continue; 8941 8941 + } 8942 8942 + 8943 8943 + utf8proc_int32_t mapped = utf8proc_toupper(cp); 8944 8944 + if (mapped >= 0x80) ascii_state = STR_ASCII_NO; 8945 8945 + 8946 8946 + wpos += (ant_offset_t)utf8proc_encode_char(mapped, (utf8proc_uint8_t *)(result_ptr + wpos)); 8910 8947 pos += n; 8911 8948 } 8912 8912 - 8949 8949 + 8950 8950 + str_set_ascii_state(result_ptr, ascii_state); 8913 8951 return result; 8914 8952 } 8915 8953 ··· 8979 9017 for (ant_offset_t i = 0; i < count; i++) { 8980 9018 memcpy(result_ptr + i * str_len, str_ptr, str_len); 8981 9019 } 9020 9020 + str_set_ascii_state( 9021 9021 + result_ptr, 9022 9022 + str_is_ascii(str_ptr) ? STR_ASCII_YES : STR_ASCII_NO 9023 9023 + ); 8982 9024 8983 9025 return result; 8984 9026 } ··· 9035 9077 pos += rem_bytes; 9036 9078 } 9037 9079 memcpy(result_ptr + pos, str_ptr, (size_t)str_len); 9080 9080 + str_set_ascii_state( 9081 9081 + result_ptr, 9082 9082 + (str_is_ascii(pad_str) && str_is_ascii(str_ptr)) ? STR_ASCII_YES : STR_ASCII_NO 9083 9083 + ); 9038 9084 9039 9085 return result; 9040 9086 } ··· 9091 9137 memcpy(result_ptr + pos, pad_str, rem_bytes); 9092 9138 pos += rem_bytes; 9093 9139 } 9140 9140 + str_set_ascii_state( 9141 9141 + result_ptr, 9142 9142 + (str_is_ascii(str_ptr) && str_is_ascii(pad_str)) ? STR_ASCII_YES : STR_ASCII_NO 9143 9143 + ); 9094 9144 9095 9145 return result; 9096 9146 }

+273 -135

src/utf8.c

reviewed

··· 1 1 #include "utf8.h" 2 2 #include "utils.h" 3 3 + #include "internal.h" 4 4 + #include "gc/objects.h" 5 5 + 3 6 #include <stdlib.h> 4 7 #include <string.h> 5 8 #include <stdbool.h> 9 9 + #include <stddef.h> 10 10 + 11 11 + typedef struct { 12 12 + uint64_t epoch; 13 13 + const char *str; 14 14 + size_t byte_len; 15 15 + size_t byte_pos; 16 16 + size_t utf16_pos; 17 17 + } utf16_scan_cache_t; 18 18 + 19 19 + typedef struct { 20 20 + const char *str; 21 21 + size_t byte_len; 22 22 + const unsigned char *start; 23 23 + const unsigned char *end; 24 24 + const unsigned char *p; 25 25 + size_t utf16_pos; 26 26 + } utf16_scan_cursor_t; 27 27 + 28 28 + static _Thread_local utf16_scan_cache_t utf16_scan_cache = { 0 }; 29 29 + 30 30 + static inline void utf16_scan_cache_sync_epoch(void) { 31 31 + uint64_t epoch = gc_get_epoch(); 32 32 + if (utf16_scan_cache.epoch == epoch) return; 33 33 + utf16_scan_cache = (utf16_scan_cache_t){ .epoch = epoch }; 34 34 + } 35 35 + 36 36 + static inline void utf16_scan_cursor_init( 37 37 + utf16_scan_cursor_t *cursor, 38 38 + const char *str, 39 39 + size_t byte_len 40 40 + ) { 41 41 + utf16_scan_cache_sync_epoch(); 42 42 + cursor->str = str; 43 43 + cursor->byte_len = byte_len; 44 44 + cursor->start = (const unsigned char *)str; 45 45 + cursor->end = cursor->start + byte_len; 46 46 + cursor->p = cursor->start; 47 47 + cursor->utf16_pos = 0; 48 48 + } 49 49 + 50 50 + static inline bool utf16_scan_cache_matches(const utf16_scan_cursor_t *cursor) { 51 51 + return utf16_scan_cache.str == cursor->str 52 52 + && utf16_scan_cache.byte_pos <= cursor->byte_len; 53 53 + } 54 54 + 55 55 + static inline void utf16_scan_cursor_resume_cached(utf16_scan_cursor_t *cursor) { 56 56 + if (!utf16_scan_cache_matches(cursor)) return; 57 57 + cursor->p = cursor->start + utf16_scan_cache.byte_pos; 58 58 + cursor->utf16_pos = utf16_scan_cache.utf16_pos; 59 59 + } 60 60 + 61 61 + static inline void utf16_scan_cursor_resume_utf16( 62 62 + utf16_scan_cursor_t *cursor, 63 63 + size_t target_utf16 64 64 + ) { 65 65 + if (!utf16_scan_cache_matches(cursor)) return; 66 66 + if (target_utf16 < utf16_scan_cache.utf16_pos) return; 67 67 + cursor->p = cursor->start + utf16_scan_cache.byte_pos; 68 68 + cursor->utf16_pos = utf16_scan_cache.utf16_pos; 69 69 + } 70 70 + 71 71 + static inline void utf16_scan_cursor_resume_byte( 72 72 + utf16_scan_cursor_t *cursor, 73 73 + size_t target_byte 74 74 + ) { 75 75 + if (!utf16_scan_cache_matches(cursor)) return; 76 76 + if (target_byte < utf16_scan_cache.byte_pos) return; 77 77 + cursor->p = cursor->start + utf16_scan_cache.byte_pos; 78 78 + cursor->utf16_pos = utf16_scan_cache.utf16_pos; 79 79 + } 80 80 + 81 81 + static inline void utf16_scan_cursor_store(const utf16_scan_cursor_t *cursor) { 82 82 + utf16_scan_cache.str = cursor->str; 83 83 + utf16_scan_cache.byte_len = cursor->byte_len; 84 84 + utf16_scan_cache.byte_pos = (size_t)(cursor->p - cursor->start); 85 85 + utf16_scan_cache.utf16_pos = cursor->utf16_pos; 86 86 + } 87 87 + 88 88 + static inline void utf16_scan_decode( 89 89 + const unsigned char *p, 90 90 + const unsigned char *end, 91 91 + size_t *slen_out, 92 92 + size_t *units_out, 93 93 + uint32_t *cp_out 94 94 + ) { 95 95 + unsigned char c = *p; 96 96 + if (c < 0x80) { 97 97 + if (cp_out) *cp_out = c; 98 98 + *slen_out = 1; 99 99 + *units_out = 1; 100 100 + return; 101 101 + } 102 102 + 103 103 + if ((c & 0xE0) == 0xC0) { 104 104 + if (cp_out && p + 1 < end) { 105 105 + *cp_out = ((uint32_t)(c & 0x1F) << 6) | (uint32_t)(p[1] & 0x3F); 106 106 + *slen_out = 2; 107 107 + *units_out = 1; 108 108 + return; 109 109 + } 110 110 + if (!cp_out) { 111 111 + *slen_out = 2; 112 112 + *units_out = 1; 113 113 + return; 114 114 + } 115 115 + } else if ((c & 0xF0) == 0xE0) { 116 116 + if (cp_out && p + 2 < end) { 117 117 + *cp_out = ((uint32_t)(c & 0x0F) << 12) 118 118 + | ((uint32_t)(p[1] & 0x3F) << 6) 119 119 + | (uint32_t)(p[2] & 0x3F); 120 120 + *slen_out = 3; 121 121 + *units_out = 1; 122 122 + return; 123 123 + } 124 124 + if (!cp_out) { 125 125 + *slen_out = 3; 126 126 + *units_out = 1; 127 127 + return; 128 128 + } 129 129 + } else if ((c & 0xF8) == 0xF0) { 130 130 + if (cp_out && p + 3 < end) { 131 131 + *cp_out = ((uint32_t)(c & 0x07) << 18) 132 132 + | ((uint32_t)(p[1] & 0x3F) << 12) 133 133 + | ((uint32_t)(p[2] & 0x3F) << 6) 134 134 + | (uint32_t)(p[3] & 0x3F); 135 135 + *slen_out = 4; 136 136 + *units_out = 2; 137 137 + return; 138 138 + } 139 139 + if (!cp_out) { 140 140 + *slen_out = 4; 141 141 + *units_out = 2; 142 142 + return; 143 143 + } 144 144 + } 145 145 + 146 146 + if (cp_out) *cp_out = c; 147 147 + *slen_out = 1; 148 148 + *units_out = 1; 149 149 + } 150 150 + 151 151 + static inline bool utf16_scan_cursor_advance( 152 152 + utf16_scan_cursor_t *cursor, 153 153 + const unsigned char *bound_end 154 154 + ) { 155 155 + size_t slen, units; 156 156 + const unsigned char *next; 157 157 + 158 158 + utf16_scan_decode(cursor->p, cursor->end, &slen, &units, NULL); 159 159 + next = cursor->p + slen; 160 160 + cursor->utf16_pos += units; 161 161 + if (next > bound_end) { 162 162 + cursor->p = bound_end; 163 163 + return false; 164 164 + } 165 165 + cursor->p = next; 166 166 + return true; 167 167 + } 6 168 7 169 static uint32_t utf8_decode(const unsigned char *buf, size_t len, int *seq_len) { 8 170 if (len == 0) { *seq_len = 0; return 0; } ··· 133 295 } 134 296 135 297 size_t utf16_strlen(const char *str, size_t byte_len) { 136 136 - const unsigned char *p = (const unsigned char *)str; 137 137 - const unsigned char *end = p + byte_len; 138 138 - 139 139 - size_t i = 0; 140 140 - for (; i + 8 <= byte_len; i += 8) { 141 141 - uint64_t chunk; 142 142 - memcpy(&chunk, p + i, 8); 143 143 - if (chunk & 0x8080808080808080ULL) goto slow_path; 144 144 - } 145 145 - for (; i < byte_len; i++) { 146 146 - if (p[i] & 0x80) goto slow_path; 147 147 - } 148 148 - return byte_len; 149 149 - 150 150 - slow_path:; 151 151 - size_t count = i; 152 152 - p += i; 153 153 - while (p < end) { 154 154 - unsigned char c = *p; 155 155 - if ((c & 0xC0) != 0x80) { 156 156 - count++; 157 157 - if ((c & 0xF8) == 0xF0) count++; 158 158 - } 159 159 - p++; 298 298 + if (str_is_ascii(str)) return byte_len; 299 299 + 300 300 + utf16_scan_cursor_t cursor; 301 301 + utf16_scan_cursor_init(&cursor, str, byte_len); 302 302 + utf16_scan_cursor_resume_cached(&cursor); 303 303 + 304 304 + while (cursor.p < cursor.end) { 305 305 + utf16_scan_cursor_advance(&cursor, cursor.end); 160 306 } 161 161 - return count; 307 307 + 308 308 + utf16_scan_cursor_store(&cursor); 309 309 + return cursor.utf16_pos; 162 310 } 163 311 164 312 int utf16_index_to_byte_offset( ··· 167 315 size_t utf16_idx, 168 316 size_t *out_char_bytes 169 317 ) { 170 170 - const unsigned char *p = (const unsigned char *)str; 171 171 - const unsigned char *end = p + byte_len; 172 172 - size_t utf16_pos = 0; 318 318 + if (str_is_ascii(str)) { 319 319 + if (utf16_idx > byte_len) return -1; 320 320 + if (out_char_bytes) *out_char_bytes = (utf16_idx < byte_len) ? 1 : 0; 321 321 + return (int)utf16_idx; 322 322 + } 323 323 + 324 324 + utf16_scan_cursor_t cursor; 325 325 + utf16_scan_cursor_init(&cursor, str, byte_len); 326 326 + utf16_scan_cursor_resume_utf16(&cursor, utf16_idx); 173 327 174 174 - while (p < end && utf16_pos < utf16_idx) { 175 175 - unsigned char c = *p; 176 176 - if (c < 0x80) { p++; utf16_pos++; } 177 177 - else if ((c & 0xE0) == 0xC0) { p += 2; utf16_pos++; } 178 178 - else if ((c & 0xF0) == 0xE0) { p += 3; utf16_pos++; } 179 179 - else if ((c & 0xF8) == 0xF0) { p += 4; utf16_pos += 2; } 180 180 - else { p++; utf16_pos++; } 181 181 - if (p > end) p = end; 328 328 + while (cursor.p < cursor.end && cursor.utf16_pos < utf16_idx) { 329 329 + utf16_scan_cursor_advance(&cursor, cursor.end); 182 330 } 183 331 184 184 - if (p >= end) { 185 185 - if (utf16_pos == utf16_idx) { 332 332 + if (cursor.p >= cursor.end) { 333 333 + if (cursor.utf16_pos == utf16_idx) { 186 334 if (out_char_bytes) *out_char_bytes = 0; 335 335 + utf16_scan_cursor_store(&cursor); 187 336 return (int)byte_len; 188 188 - } return -1; 337 337 + } 338 338 + utf16_scan_cursor_store(&cursor); 339 339 + return -1; 189 340 } 190 341 191 191 - unsigned char c = *p; 192 192 - size_t slen = (c < 0x80) 193 193 - ? 1 : ((c & 0xE0) == 0xC0) 194 194 - ? 2 : ((c & 0xF0) == 0xE0) 195 195 - ? 3 : ((c & 0xF8) == 0xF0) 196 196 - ? 4 : 1; 342 342 + size_t slen, units; 343 343 + utf16_scan_decode(cursor.p, cursor.end, &slen, &units, NULL); 197 344 198 345 if (out_char_bytes) *out_char_bytes = slen; 199 199 - return (int)(p - (const unsigned char *)str); 346 346 + utf16_scan_cursor_store(&cursor); 347 347 + return (int)(cursor.p - cursor.start); 200 348 } 201 349 202 350 int utf16_range_to_byte_range( ··· 207 355 size_t *byte_start, 208 356 size_t *byte_end 209 357 ) { 210 210 - const unsigned char *p = (const unsigned char *)str; 211 211 - const unsigned char *end = p + byte_len; 212 212 - 213 213 - size_t utf16_pos = 0; 358 358 + if (str_is_ascii(str)) { 359 359 + *byte_start = (utf16_start <= byte_len) ? utf16_start : byte_len; 360 360 + *byte_end = (utf16_end <= byte_len) ? utf16_end : byte_len; 361 361 + return 0; 362 362 + } 363 363 + 364 364 + utf16_scan_cursor_t cursor; 365 365 + utf16_scan_cursor_init(&cursor, str, byte_len); 366 366 + utf16_scan_cursor_resume_utf16(&cursor, utf16_start); 367 367 + 214 368 size_t b_start = 0, b_end = byte_len; 215 369 int found_start = 0, found_end = 0; 216 370 217 217 - while (p < end) { 218 218 - if (utf16_pos == utf16_start) { b_start = p - (const unsigned char *)str; found_start = 1; } 219 219 - if (utf16_pos == utf16_end) { b_end = p - (const unsigned char *)str; found_end = 1; break; } 220 220 - 221 221 - unsigned char c = *p; 222 222 - if (c < 0x80) { p++; utf16_pos++; } 223 223 - else if ((c & 0xE0) == 0xC0) { p += 2; utf16_pos++; } 224 224 - else if ((c & 0xF0) == 0xE0) { p += 3; utf16_pos++; } 225 225 - else if ((c & 0xF8) == 0xF0) { p += 4; utf16_pos += 2; } 226 226 - else { p++; utf16_pos++; } 227 227 - if (p > end) p = end; 371 371 + while (cursor.p < cursor.end) { 372 372 + if (cursor.utf16_pos == utf16_start) { 373 373 + b_start = (size_t)(cursor.p - cursor.start); 374 374 + found_start = 1; 375 375 + } 376 376 + if (cursor.utf16_pos == utf16_end) { 377 377 + b_end = (size_t)(cursor.p - cursor.start); 378 378 + found_end = 1; 379 379 + break; 380 380 + } 381 381 + utf16_scan_cursor_advance(&cursor, cursor.end); 228 382 } 229 383 230 230 - if (!found_start && utf16_start >= utf16_pos) b_start = byte_len; 231 231 - if (!found_end && utf16_end >= utf16_pos) b_end = byte_len; 384 384 + if (!found_start && utf16_start >= cursor.utf16_pos) b_start = byte_len; 385 385 + if (!found_end && utf16_end >= cursor.utf16_pos) b_end = byte_len; 232 386 233 387 *byte_start = b_start; 234 388 *byte_end = b_end; 389 389 + utf16_scan_cursor_store(&cursor); 235 390 236 391 return 0; 237 392 } 238 393 239 394 size_t byte_offset_to_utf16(const char *str, size_t byte_off) { 240 240 - const unsigned char *p = (const unsigned char *)str; 241 241 - const unsigned char *end = p + byte_off; 242 242 - size_t utf16_pos = 0; 395 395 + if (str_is_ascii(str)) return byte_off; 396 396 + 397 397 + utf16_scan_cursor_t cursor; 398 398 + const unsigned char *bound_end; 399 399 + bool ended_on_boundary = true; 400 400 + 401 401 + utf16_scan_cursor_init(&cursor, str, byte_off); 402 402 + utf16_scan_cursor_resume_byte(&cursor, byte_off); 403 403 + bound_end = cursor.start + byte_off; 243 404 244 244 - while (p < end) { 245 245 - unsigned char c = *p; 246 246 - if (c < 0x80) { p++; utf16_pos++; } 247 247 - else if ((c & 0xE0) == 0xC0) { p += 2; utf16_pos++; } 248 248 - else if ((c & 0xF0) == 0xE0) { p += 3; utf16_pos++; } 249 249 - else if ((c & 0xF8) == 0xF0) { p += 4; utf16_pos += 2; } 250 250 - else { p++; utf16_pos++; } 251 251 - if (p > end) p = end; 405 405 + while (cursor.p < bound_end) { 406 406 + if (!utf16_scan_cursor_advance(&cursor, bound_end)) { 407 407 + ended_on_boundary = false; 408 408 + break; 409 409 + } 252 410 } 253 253 - return utf16_pos; 411 411 + 412 412 + if (ended_on_boundary) utf16_scan_cursor_store(&cursor); 413 413 + return cursor.utf16_pos; 254 414 } 255 415 256 416 uint32_t utf16_code_unit_at(const char *str, size_t byte_len, size_t utf16_idx) { 257 257 - const unsigned char *p = (const unsigned char *)str; 258 258 - const unsigned char *end = p + byte_len; 259 259 - size_t utf16_pos = 0; 417 417 + if (str_is_ascii(str)) { 418 418 + if (utf16_idx >= byte_len) return 0xFFFFFFFF; 419 419 + return (unsigned char)str[utf16_idx]; 420 420 + } 421 421 + 422 422 + utf16_scan_cursor_t cursor; 423 423 + utf16_scan_cursor_init(&cursor, str, byte_len); 424 424 + utf16_scan_cursor_resume_utf16(&cursor, utf16_idx); 260 425 261 261 - while (p < end) { 262 262 - unsigned char c = *p; 263 263 - size_t units, slen; 426 426 + while (cursor.p < cursor.end) { 427 427 + size_t slen, units; 264 428 uint32_t cp; 265 429 266 266 - if (c < 0x80) { cp = c; slen = 1; units = 1; } 267 267 - else if ((c & 0xE0) == 0xC0 && p + 1 < end) { 268 268 - cp = ((c & 0x1F) << 6) 269 269 - | (p[1] & 0x3F); 270 270 - slen = 2; units = 1; 271 271 - } 272 272 - else if ((c & 0xF0) == 0xE0 && p + 2 < end) { 273 273 - cp = ((c & 0x0F) << 12) 274 274 - | ((p[1] & 0x3F) << 6) 275 275 - | (p[2] & 0x3F); 276 276 - slen = 3; units = 1; 277 277 - } 278 278 - else if ((c & 0xF8) == 0xF0 && p + 3 < end) { 279 279 - cp = ((c & 0x07) << 18) 280 280 - | ((p[1] & 0x3F) << 12) 281 281 - | ((p[2] & 0x3F) << 6) 282 282 - | (p[3] & 0x3F); 283 283 - slen = 4; units = 2; 284 284 - } 285 285 - else { cp = c; slen = 1; units = 1; } 430 430 + utf16_scan_decode(cursor.p, cursor.end, &slen, &units, &cp); 286 431 287 287 - if (utf16_pos == utf16_idx) { 432 432 + if (cursor.utf16_pos == utf16_idx) { 433 433 + utf16_scan_cursor_store(&cursor); 288 434 if (units == 2) return 0xD800 + ((cp - 0x10000) >> 10); 289 435 return cp; 290 436 } 291 291 - if (units == 2 && utf16_pos + 1 == utf16_idx) { 437 437 + if (units == 2 && cursor.utf16_pos + 1 == utf16_idx) { 438 438 + utf16_scan_cursor_store(&cursor); 292 439 return 0xDC00 + ((cp - 0x10000) & 0x3FF); 293 440 } 294 294 - p += slen; 295 295 - utf16_pos += units; 441 441 + cursor.p += slen; 442 442 + cursor.utf16_pos += units; 296 443 } 297 444 445 445 + utf16_scan_cursor_store(&cursor); 298 446 return 0xFFFFFFFF; 299 447 } 300 448 ··· 384 532 } 385 533 386 534 uint32_t utf16_codepoint_at(const char *str, size_t byte_len, size_t utf16_idx) { 387 387 - const unsigned char *p = (const unsigned char *)str; 388 388 - const unsigned char *end = p + byte_len; 389 389 - size_t utf16_pos = 0; 535 535 + if (str_is_ascii(str)) { 536 536 + if (utf16_idx >= byte_len) return 0xFFFFFFFF; 537 537 + return (unsigned char)str[utf16_idx]; 538 538 + } 539 539 + 540 540 + utf16_scan_cursor_t cursor; 541 541 + utf16_scan_cursor_init(&cursor, str, byte_len); 542 542 + utf16_scan_cursor_resume_utf16(&cursor, utf16_idx); 390 543 391 391 - while (p < end) { 392 392 - unsigned char c = *p; 393 393 - size_t units, slen; 544 544 + while (cursor.p < cursor.end) { 545 545 + size_t slen, units; 394 546 uint32_t cp; 395 547 396 396 - if (c < 0x80) { cp = c; slen = 1; units = 1; } 397 397 - else if ((c & 0xE0) == 0xC0 && p + 1 < end) { 398 398 - cp = ((c & 0x1F) << 6) 399 399 - | (p[1] & 0x3F); 400 400 - slen = 2; units = 1; 401 401 - } 402 402 - else if ((c & 0xF0) == 0xE0 && p + 2 < end) { 403 403 - cp = ((c & 0x0F) << 12) 404 404 - | ((p[1] & 0x3F) << 6) 405 405 - | (p[2] & 0x3F); 406 406 - slen = 3; units = 1; 548 548 + utf16_scan_decode(cursor.p, cursor.end, &slen, &units, &cp); 549 549 + 550 550 + if (cursor.utf16_pos == utf16_idx) { 551 551 + utf16_scan_cursor_store(&cursor); 552 552 + return cp; 407 553 } 408 408 - else if ((c & 0xF8) == 0xF0 && p + 3 < end) { 409 409 - cp = ((c & 0x07) << 18) 410 410 - | ((p[1] & 0x3F) << 12) 411 411 - | ((p[2] & 0x3F) << 6) 412 412 - | (p[3] & 0x3F); 413 413 - slen = 4; units = 2; 414 414 - } 415 415 - else { cp = c; slen = 1; units = 1; } 416 416 - 417 417 - if (utf16_pos == utf16_idx) return cp; 418 418 - if (units == 2 && utf16_pos + 1 == utf16_idx) { 554 554 + if (units == 2 && cursor.utf16_pos + 1 == utf16_idx) { 555 555 + utf16_scan_cursor_store(&cursor); 419 556 return 0xDC00 + ((cp - 0x10000) & 0x3FF); 420 557 } 421 558 422 422 - p += slen; 423 423 - utf16_pos += units; 559 559 + cursor.p += slen; 560 560 + cursor.utf16_pos += units; 424 561 } 425 562 563 563 + utf16_scan_cursor_store(&cursor); 426 564 return 0xFFFFFFFF; 427 565 }