refactor(lexicon-doc): faster UTF-8 and grapheme length validation

+5

.changeset/lovely-singers-jam.md

··· 1 + --- 2 + "@atcute/lexicon-doc": patch 3 + --- 4 + 5 + faster UTF-8 and grapheme length validation

+9 -9

packages/lexicons/lexicon-doc/lib/builder.test.ts

··· 305 305 }), 306 306 ], 307 307 }), 308 - ).toThrow(/com\.example\.test#main\/const:.*can't be shorter than minimum length/); 308 + ).toThrow(/com\.example\.test#main\/const:.*length \d+, must be 1000 <= len <= Infinity/); 309 309 }); 310 310 311 311 test('throws when token default value violates maxLength during build', () => { ··· 326 326 }), 327 327 ], 328 328 }), 329 - ).toThrow(/com\.example\.test#main\/default:.*can't be longer than maximum length/); 329 + ).toThrow(/com\.example\.test#main\/default:.*length \d+, must be 0 <= len <= 5/); 330 330 }); 331 331 332 332 test('throws when token enum value violates minLength during build', () => { ··· 347 347 }), 348 348 ], 349 349 }), 350 - ).toThrow(/com\.example\.test#main\/enum\/0:.*can't be shorter than minimum length/); 350 + ).toThrow(/com\.example\.test#main\/enum\/0:.*length \d+, must be 1000 <= len <= Infinity/); 351 351 }); 352 352 353 353 test('throws when token knownValues violates maxLength during build', () => { ··· 368 368 }), 369 369 ], 370 370 }), 371 - ).toThrow(/com\.example\.test#main\/knownValues\/0:.*can't be longer than maximum length/); 371 + ).toThrow(/com\.example\.test#main\/knownValues\/0:.*length \d+, must be 0 <= len <= 5/); 372 372 }); 373 373 374 374 test('throws when token const value does not match format during build', () => { ··· 501 501 502 502 test('throws when default is shorter than minLength', () => { 503 503 expect(() => string({ minLength: 10, default: 'hi' })).toThrow( 504 - 'string/default: value ("hi") can\'t be shorter than minimum length (10)', 504 + 'string/default: value ("hi") length 2, must be 10 <= len <= Infinity', 505 505 ); 506 506 }); 507 507 508 508 test('throws when default is longer than maxLength', () => { 509 509 expect(() => string({ maxLength: 5, default: 'hello world' })).toThrow( 510 - 'string/default: value ("hello world") can\'t be longer than maximum length (5)', 510 + 'string/default: value ("hello world") length 11, must be 0 <= len <= 5', 511 511 ); 512 512 }); 513 513 ··· 581 581 582 582 test('throws when enum value is shorter than minLength', () => { 583 583 expect(() => string({ minLength: 5, enum: ['hi', 'hello', 'world'] })).toThrow( 584 - 'string/enum[0]: value ("hi") can\'t be shorter than minimum length (5)', 584 + 'string/enum[0]: value ("hi") length 2, must be 5 <= len <= Infinity', 585 585 ); 586 586 }); 587 587 588 588 test('throws when enum value is longer than maxLength', () => { 589 589 expect(() => string({ maxLength: 5, enum: ['hi', 'hello', 'worlds'] })).toThrow( 590 - 'string/enum[2]: value ("worlds") can\'t be longer than maximum length (5)', 590 + 'string/enum[2]: value ("worlds") length 6, must be 0 <= len <= 5', 591 591 ); 592 592 }); 593 593 594 594 test('throws when knownValues value is shorter than minLength', () => { 595 595 expect(() => string({ minLength: 5, knownValues: ['hi', 'hello', 'world'] })).toThrow( 596 - 'string/knownValues[0]: value ("hi") can\'t be shorter than minimum length (5)', 596 + 'string/knownValues[0]: value ("hi") length 2, must be 5 <= len <= Infinity', 597 597 ); 598 598 }); 599 599

+82 -226

packages/lexicons/lexicon-doc/lib/builder.ts

··· 1 1 import { type Nsid } from '@atcute/lexicons/syntax'; 2 - 3 - import { isWithinGraphemeBounds, isWithinUtf8Bounds } from './internal/utils.js'; 2 + import { getUtf8Length, isUtf8LengthInRange } from '@atcute/uint8array'; 3 + import { getGraphemeLength, isGraphemeLengthInRange } from '@atcute/util-text'; 4 4 import { DELIMITED_MIME_TYPE_RE, KEY_RE, MIME_TYPE_RE, validateStringFormat } from './internal/validation.js'; 5 5 import type * as t from './types.js'; 6 6 import { formatLexiconRef, type ParsedLexiconRef } from './utils/refs.js'; ··· 250 250 } 251 251 } 252 252 253 - { 254 - const bound = isWithinUtf8Bounds(defaultValue, minLength, maxLength); 255 - 256 - if (bound === 'min') { 257 - throw new Error( 258 - `string/default: value (${JSON.stringify(defaultValue)}) can't be shorter than minimum length (${minLength})`, 259 - ); 260 - } 261 - 262 - if (bound === 'max') { 263 - throw new Error( 264 - `string/default: value (${JSON.stringify(defaultValue)}) can't be longer than maximum length (${maxLength})`, 265 - ); 266 - } 253 + if (!isUtf8LengthInRange(defaultValue, minLength, maxLength)) { 254 + const len = getUtf8Length(defaultValue); 255 + throw new Error( 256 + `string/default: value (${JSON.stringify(defaultValue)}) length ${len}, must be ${minLength} <= len <= ${maxLength}`, 257 + ); 267 258 } 268 259 269 - { 270 - const bound = isWithinGraphemeBounds(defaultValue, minGraphemes, maxGraphemes); 271 - 272 - if (bound === 'min') { 273 - throw new Error( 274 - `string/default: value (${JSON.stringify(defaultValue)}) can't be shorter than minimum graphemes (${minGraphemes})`, 275 - ); 276 - } 277 - 278 - if (bound === 'max') { 279 - throw new Error( 280 - `string/default: value (${JSON.stringify(defaultValue)}) can't be longer than maximum graphemes (${maxGraphemes})`, 281 - ); 282 - } 260 + if (!isGraphemeLengthInRange(defaultValue, minGraphemes, maxGraphemes)) { 261 + const len = getGraphemeLength(defaultValue); 262 + throw new Error( 263 + `string/default: value (${JSON.stringify(defaultValue)}) grapheme length ${len}, must be ${minGraphemes} <= len <= ${maxGraphemes}`, 264 + ); 283 265 } 284 266 285 267 if (format !== undefined && !validateStringFormat(defaultValue, format)) { ··· 299 281 } 300 282 301 283 if (typeof constValue === 'string') { 302 - { 303 - const bound = isWithinUtf8Bounds(constValue, minLength, maxLength); 304 - 305 - if (bound === 'min') { 306 - throw new Error( 307 - `string/const: value (${JSON.stringify(constValue)}) can't be shorter than minimum length (${minLength})`, 308 - ); 309 - } 310 - 311 - if (bound === 'max') { 312 - throw new Error( 313 - `string/const: value (${JSON.stringify(constValue)}) can't be longer than maximum length (${maxLength})`, 314 - ); 315 - } 284 + if (!isUtf8LengthInRange(constValue, minLength, maxLength)) { 285 + const len = getUtf8Length(constValue); 286 + throw new Error( 287 + `string/const: value (${JSON.stringify(constValue)}) length ${len}, must be ${minLength} <= len <= ${maxLength}`, 288 + ); 316 289 } 317 290 318 - { 319 - const bound = isWithinGraphemeBounds(constValue, minGraphemes, maxGraphemes); 320 - 321 - if (bound === 'min') { 322 - throw new Error( 323 - `string/const: value (${JSON.stringify(constValue)}) can't be shorter than minimum graphemes (${minGraphemes})`, 324 - ); 325 - } 326 - 327 - if (bound === 'max') { 328 - throw new Error( 329 - `string/const: value (${JSON.stringify(constValue)}) can't be longer than maximum graphemes (${maxGraphemes})`, 330 - ); 331 - } 291 + if (!isGraphemeLengthInRange(constValue, minGraphemes, maxGraphemes)) { 292 + const len = getGraphemeLength(constValue); 293 + throw new Error( 294 + `string/const: value (${JSON.stringify(constValue)}) grapheme length ${len}, must be ${minGraphemes} <= len <= ${maxGraphemes}`, 295 + ); 332 296 } 333 297 334 298 if (format !== undefined && !validateStringFormat(constValue, format)) { ··· 348 312 const enumValue = enumValues[idx]; 349 313 350 314 if (typeof enumValue === 'string') { 351 - { 352 - const bound = isWithinUtf8Bounds(enumValue, minLength, maxLength); 353 - 354 - if (bound === 'min') { 355 - throw new Error( 356 - `string/enum[${idx}]: value (${JSON.stringify(enumValue)}) can't be shorter than minimum length (${minLength})`, 357 - ); 358 - } 359 - 360 - if (bound === 'max') { 361 - throw new Error( 362 - `string/enum[${idx}]: value (${JSON.stringify(enumValue)}) can't be longer than maximum length (${maxLength})`, 363 - ); 364 - } 315 + if (!isUtf8LengthInRange(enumValue, minLength, maxLength)) { 316 + const len = getUtf8Length(enumValue); 317 + throw new Error( 318 + `string/enum[${idx}]: value (${JSON.stringify(enumValue)}) length ${len}, must be ${minLength} <= len <= ${maxLength}`, 319 + ); 365 320 } 366 321 367 - { 368 - const bound = isWithinGraphemeBounds(enumValue, minGraphemes, maxGraphemes); 369 - 370 - if (bound === 'min') { 371 - throw new Error( 372 - `string/enum[${idx}]: value (${JSON.stringify(enumValue)}) can't have fewer graphemes than minimum graphemes (${minGraphemes})`, 373 - ); 374 - } 375 - 376 - if (bound === 'max') { 377 - throw new Error( 378 - `string/enum[${idx}]: value (${JSON.stringify(enumValue)}) can't have more graphemes than maximum graphemes (${maxGraphemes})`, 379 - ); 380 - } 322 + if (!isGraphemeLengthInRange(enumValue, minGraphemes, maxGraphemes)) { 323 + const graphemeLen = getGraphemeLength(enumValue); 324 + throw new Error( 325 + `string/enum[${idx}]: value (${JSON.stringify(enumValue)}) grapheme length ${graphemeLen}, must be ${minGraphemes} <= len <= ${maxGraphemes}`, 326 + ); 381 327 } 382 328 383 329 if (format !== undefined && !validateStringFormat(enumValue, format)) { ··· 394 340 const knownValue = knownValues[idx]; 395 341 396 342 if (typeof knownValue === 'string') { 397 - { 398 - const bound = isWithinUtf8Bounds(knownValue, minLength, maxLength); 399 - 400 - if (bound === 'min') { 401 - throw new Error( 402 - `string/knownValues[${idx}]: value (${JSON.stringify(knownValue)}) can't be shorter than minimum length (${minLength})`, 403 - ); 404 - } 405 - 406 - if (bound === 'max') { 407 - throw new Error( 408 - `string/knownValues[${idx}]: value (${JSON.stringify(knownValue)}) can't be longer than maximum length (${maxLength})`, 409 - ); 410 - } 343 + if (!isUtf8LengthInRange(knownValue, minLength, maxLength)) { 344 + const len = getUtf8Length(knownValue); 345 + throw new Error( 346 + `string/knownValues[${idx}]: value (${JSON.stringify(knownValue)}) length ${len}, must be ${minLength} <= len <= ${maxLength}`, 347 + ); 411 348 } 412 349 413 - { 414 - const bound = isWithinGraphemeBounds(knownValue, minGraphemes, maxGraphemes); 415 - 416 - if (bound === 'min') { 417 - throw new Error( 418 - `string/knownValues[${idx}]: value (${JSON.stringify(knownValue)}) can't have fewer graphemes than minimum graphemes (${minGraphemes})`, 419 - ); 420 - } 421 - 422 - if (bound === 'max') { 423 - throw new Error( 424 - `string/knownValues[${idx}]: value (${JSON.stringify(knownValue)}) can't have more graphemes than maximum graphemes (${maxGraphemes})`, 425 - ); 426 - } 350 + if (!isGraphemeLengthInRange(knownValue, minGraphemes, maxGraphemes)) { 351 + const graphemeLen = getGraphemeLength(knownValue); 352 + throw new Error( 353 + `string/knownValues[${idx}]: value (${JSON.stringify(knownValue)}) grapheme length ${graphemeLen}, must be ${minGraphemes} <= len <= ${maxGraphemes}`, 354 + ); 427 355 } 428 356 429 357 if (format !== undefined && !validateStringFormat(knownValue, format)) { ··· 495 423 496 424 // validate resolved const value 497 425 if (builtConstValue !== undefined && typeof constValue !== 'string') { 498 - { 499 - const bound = isWithinUtf8Bounds(builtConstValue, minLength, maxLength); 500 - 501 - if (bound === 'min') { 502 - throw new Error( 503 - `${ctx.dotPath}/const: value (${JSON.stringify(builtConstValue)}) can't be shorter than minimum length (${minLength})`, 504 - ); 505 - } 506 - 507 - if (bound === 'max') { 508 - throw new Error( 509 - `${ctx.dotPath}/const: value (${JSON.stringify(builtConstValue)}) can't be longer than maximum length (${maxLength})`, 510 - ); 511 - } 426 + if (!isUtf8LengthInRange(builtConstValue, minLength, maxLength)) { 427 + const len = getUtf8Length(builtConstValue); 428 + throw new Error( 429 + `${ctx.dotPath}/const: value (${JSON.stringify(builtConstValue)}) length ${len}, must be ${minLength} <= len <= ${maxLength}`, 430 + ); 512 431 } 513 432 514 - { 515 - const bound = isWithinGraphemeBounds(builtConstValue, minGraphemes, maxGraphemes); 516 - 517 - if (bound === 'min') { 518 - throw new Error( 519 - `${ctx.dotPath}/const: value (${JSON.stringify(builtConstValue)}) can't be shorter than minimum graphemes (${minGraphemes})`, 520 - ); 521 - } 522 - 523 - if (bound === 'max') { 524 - throw new Error( 525 - `${ctx.dotPath}/const: value (${JSON.stringify(builtConstValue)}) can't be longer than maximum graphemes (${maxGraphemes})`, 526 - ); 527 - } 433 + if (!isGraphemeLengthInRange(builtConstValue, minGraphemes, maxGraphemes)) { 434 + const len = getGraphemeLength(builtConstValue); 435 + throw new Error( 436 + `${ctx.dotPath}/const: value (${JSON.stringify(builtConstValue)}) grapheme length ${len}, must be ${minGraphemes} <= len <= ${maxGraphemes}`, 437 + ); 528 438 } 529 439 530 440 if (format !== undefined && !validateStringFormat(builtConstValue, format)) { ··· 544 454 throw new Error(`${ctx.dotPath}/default: value must be one of the enum values`); 545 455 } 546 456 547 - { 548 - const bound = isWithinUtf8Bounds(builtDefaultValue, minLength, maxLength); 549 - 550 - if (bound === 'min') { 551 - throw new Error( 552 - `${ctx.dotPath}/default: value (${JSON.stringify(builtDefaultValue)}) can't be shorter than minimum length (${minLength})`, 553 - ); 554 - } 555 - 556 - if (bound === 'max') { 557 - throw new Error( 558 - `${ctx.dotPath}/default: value (${JSON.stringify(builtDefaultValue)}) can't be longer than maximum length (${maxLength})`, 559 - ); 560 - } 457 + if (!isUtf8LengthInRange(builtDefaultValue, minLength, maxLength)) { 458 + const len = getUtf8Length(builtDefaultValue); 459 + throw new Error( 460 + `${ctx.dotPath}/default: value (${JSON.stringify(builtDefaultValue)}) length ${len}, must be ${minLength} <= len <= ${maxLength}`, 461 + ); 561 462 } 562 463 563 - { 564 - const bound = isWithinGraphemeBounds(builtDefaultValue, minGraphemes, maxGraphemes); 565 - 566 - if (bound === 'min') { 567 - throw new Error( 568 - `${ctx.dotPath}/default: value (${JSON.stringify(builtDefaultValue)}) can't be shorter than minimum graphemes (${minGraphemes})`, 569 - ); 570 - } 571 - 572 - if (bound === 'max') { 573 - throw new Error( 574 - `${ctx.dotPath}/default: value (${JSON.stringify(builtDefaultValue)}) can't be longer than maximum graphemes (${maxGraphemes})`, 575 - ); 576 - } 464 + if (!isGraphemeLengthInRange(builtDefaultValue, minGraphemes, maxGraphemes)) { 465 + const len = getGraphemeLength(builtDefaultValue); 466 + throw new Error( 467 + `${ctx.dotPath}/default: value (${JSON.stringify(builtDefaultValue)}) grapheme length ${len}, must be ${minGraphemes} <= len <= ${maxGraphemes}`, 468 + ); 577 469 } 578 470 579 471 if (format !== undefined && !validateStringFormat(builtDefaultValue, format)) { ··· 590 482 const builtEnumValue = builtEnumValues[idx]; 591 483 592 484 if (typeof enumValue !== 'string') { 593 - { 594 - const bound = isWithinUtf8Bounds(builtEnumValue, minLength, maxLength); 595 - 596 - if (bound === 'min') { 597 - throw new Error( 598 - `${ctx.dotPath}/enum/${idx}: value (${JSON.stringify(builtEnumValue)}) can't be shorter than minimum length (${minLength})`, 599 - ); 600 - } 601 - 602 - if (bound === 'max') { 603 - throw new Error( 604 - `${ctx.dotPath}/enum/${idx}: value (${JSON.stringify(builtEnumValue)}) can't be longer than maximum length (${maxLength})`, 605 - ); 606 - } 485 + if (!isUtf8LengthInRange(builtEnumValue, minLength, maxLength)) { 486 + const len = getUtf8Length(builtEnumValue); 487 + throw new Error( 488 + `${ctx.dotPath}/enum/${idx}: value (${JSON.stringify(builtEnumValue)}) length ${len}, must be ${minLength} <= len <= ${maxLength}`, 489 + ); 607 490 } 608 491 609 - { 610 - const bound = isWithinGraphemeBounds(builtEnumValue, minGraphemes, maxGraphemes); 611 - 612 - if (bound === 'min') { 613 - throw new Error( 614 - `${ctx.dotPath}/enum/${idx}: value (${JSON.stringify(builtEnumValue)}) can't have fewer graphemes than minimum graphemes (${minGraphemes})`, 615 - ); 616 - } 617 - 618 - if (bound === 'max') { 619 - throw new Error( 620 - `${ctx.dotPath}/enum/${idx}: value (${JSON.stringify(builtEnumValue)}) can't have more graphemes than maximum graphemes (${maxGraphemes})`, 621 - ); 622 - } 492 + if (!isGraphemeLengthInRange(builtEnumValue, minGraphemes, maxGraphemes)) { 493 + const graphemeLen = getGraphemeLength(builtEnumValue); 494 + throw new Error( 495 + `${ctx.dotPath}/enum/${idx}: value (${JSON.stringify(builtEnumValue)}) grapheme length ${graphemeLen}, must be ${minGraphemes} <= len <= ${maxGraphemes}`, 496 + ); 623 497 } 624 498 625 499 if (format !== undefined && !validateStringFormat(builtEnumValue, format)) { ··· 638 512 const builtKnownValue = builtKnownValues[idx]; 639 513 640 514 if (typeof knownValue !== 'string') { 641 - { 642 - const bound = isWithinUtf8Bounds(builtKnownValue, minLength, maxLength); 643 - 644 - if (bound === 'min') { 645 - throw new Error( 646 - `${ctx.dotPath}/knownValues/${idx}: value (${JSON.stringify(builtKnownValue)}) can't be shorter than minimum length (${minLength})`, 647 - ); 648 - } 649 - 650 - if (bound === 'max') { 651 - throw new Error( 652 - `${ctx.dotPath}/knownValues/${idx}: value (${JSON.stringify(builtKnownValue)}) can't be longer than maximum length (${maxLength})`, 653 - ); 654 - } 515 + if (!isUtf8LengthInRange(builtKnownValue, minLength, maxLength)) { 516 + const len = getUtf8Length(builtKnownValue); 517 + throw new Error( 518 + `${ctx.dotPath}/knownValues/${idx}: value (${JSON.stringify(builtKnownValue)}) length ${len}, must be ${minLength} <= len <= ${maxLength}`, 519 + ); 655 520 } 656 521 657 - { 658 - const bound = isWithinGraphemeBounds(builtKnownValue, minGraphemes, maxGraphemes); 659 - 660 - if (bound === 'min') { 661 - throw new Error( 662 - `${ctx.dotPath}/knownValues/${idx}: value (${JSON.stringify(builtKnownValue)}) can't have fewer graphemes than minimum graphemes (${minGraphemes})`, 663 - ); 664 - } 665 - 666 - if (bound === 'max') { 667 - throw new Error( 668 - `${ctx.dotPath}/knownValues/${idx}: value (${JSON.stringify(builtKnownValue)}) can't have more graphemes than maximum graphemes (${maxGraphemes})`, 669 - ); 670 - } 522 + if (!isGraphemeLengthInRange(builtKnownValue, minGraphemes, maxGraphemes)) { 523 + const graphemeLen = getGraphemeLength(builtKnownValue); 524 + throw new Error( 525 + `${ctx.dotPath}/knownValues/${idx}: value (${JSON.stringify(builtKnownValue)}) grapheme length ${graphemeLen}, must be ${minGraphemes} <= len <= ${maxGraphemes}`, 526 + ); 671 527 } 672 528 673 529 if (format !== undefined && !validateStringFormat(builtKnownValue, format)) {

-59

packages/lexicons/lexicon-doc/lib/internal/utils.ts

··· 1 - import { getUtf8Length } from '@atcute/uint8array'; 2 - import { getGraphemeLength } from '@atcute/util-text'; 3 - 4 - export const isWithinUtf8Bounds = (input: string, min = 0, max = Infinity): 'max' | 'min' | undefined => { 5 - const utf16Len = input.length; 6 - const maybeUtf8Len = utf16Len * 3; 7 - 8 - // fail early if estimated upper bound is too small 9 - if (maybeUtf8Len < min) { 10 - return 'min'; 11 - } 12 - 13 - // skip if UTF-16 length already satisfies both constraints 14 - if (utf16Len >= min && maybeUtf8Len <= max) { 15 - return undefined; 16 - } 17 - 18 - const utf8Len = getUtf8Length(input); 19 - 20 - if (utf8Len < min) { 21 - return 'min'; 22 - } 23 - 24 - if (utf8Len > max) { 25 - return 'max'; 26 - } 27 - 28 - return undefined; 29 - }; 30 - 31 - export const isWithinGraphemeBounds = (input: string, min = 0, max = Infinity): 'max' | 'min' | undefined => { 32 - // grapheme conversion is expensive, so we're going to do some safe naive 33 - // checks where we assume 1 UTF-16 character = 1 grapheme. 34 - 35 - const utf16Len = input.length; 36 - 37 - // fail early if UTF-16 length is too small 38 - if (utf16Len < min) { 39 - return 'min'; 40 - } 41 - 42 - // if there is no minimum bounds, we can safely skip when UTF-16 is 43 - // within the maximum bounds. 44 - if (min === 0 && utf16Len <= max) { 45 - return undefined; 46 - } 47 - 48 - const graphemeLen = getGraphemeLength(input); 49 - 50 - if (graphemeLen < min) { 51 - return 'min'; 52 - } 53 - 54 - if (graphemeLen > max) { 55 - return 'max'; 56 - } 57 - 58 - return undefined; 59 - };

+50 -130

packages/lexicons/lexicon-doc/lib/refinements.ts

···············

Configure Feed

Configure Feed