Implement JS lexer/tokenizer (ECMAScript 2024)

+1896

2 changed files

expand all

crates

src

lexer.rs

lib.rs

+1894

crates/js/src/lexer.rs

··· 1 + //! JavaScript lexer/tokenizer conforming to ECMAScript 2024. 2 + //! 3 + //! Converts JavaScript source text into a stream of [`Token`]s, each annotated 4 + //! with its [`Span`] (byte offset, line, column). 5 + 6 + use std::fmt; 7 + 8 + /// A position in the source text. 9 + #[derive(Debug, Clone, Copy, PartialEq, Eq)] 10 + pub struct SourcePos { 11 + /// 1-based line number. 12 + pub line: u32, 13 + /// 1-based column (in bytes from the start of the line). 14 + pub col: u32, 15 + } 16 + 17 + /// A span covering a range of source text. 18 + #[derive(Debug, Clone, Copy, PartialEq, Eq)] 19 + pub struct Span { 20 + pub start: SourcePos, 21 + pub end: SourcePos, 22 + } 23 + 24 + /// A token produced by the lexer. 25 + #[derive(Debug, Clone, PartialEq)] 26 + pub struct Token { 27 + pub kind: TokenKind, 28 + pub span: Span, 29 + /// Whether at least one newline preceded this token (for ASI). 30 + pub preceded_by_newline: bool, 31 + } 32 + 33 + /// Every distinct token kind the lexer can produce. 34 + #[derive(Debug, Clone, PartialEq)] 35 + pub enum TokenKind { 36 + // ── Literals ────────────────────────────────────────────── 37 + /// Numeric literal (the parsed `f64` value). 38 + Number(f64), 39 + /// String literal (the decoded content, without quotes). 40 + String(std::string::String), 41 + /// Regular expression literal: pattern and flags. 42 + RegExp { 43 + pattern: std::string::String, 44 + flags: std::string::String, 45 + }, 46 + /// Template literal with no substitutions (full string content). 47 + TemplateFull(std::string::String), 48 + /// Opening part of a template literal (before the first `${`). 49 + TemplateHead(std::string::String), 50 + /// Middle part of a template literal (between `}` and next `${`). 51 + TemplateMiddle(std::string::String), 52 + /// Closing part of a template literal (after the last `}`). 53 + TemplateTail(std::string::String), 54 + 55 + // ── Identifiers & Keywords ─────────────────────────────── 56 + Identifier(std::string::String), 57 + 58 + // Keywords 59 + Await, 60 + Break, 61 + Case, 62 + Catch, 63 + Class, 64 + Const, 65 + Continue, 66 + Debugger, 67 + Default, 68 + Delete, 69 + Do, 70 + Else, 71 + Export, 72 + Extends, 73 + Finally, 74 + For, 75 + Function, 76 + If, 77 + Import, 78 + In, 79 + Instanceof, 80 + Let, 81 + New, 82 + Of, 83 + Return, 84 + Static, 85 + Super, 86 + Switch, 87 + This, 88 + Throw, 89 + Try, 90 + Typeof, 91 + Var, 92 + Void, 93 + While, 94 + With, 95 + Yield, 96 + Async, 97 + 98 + // Literal keywords 99 + True, 100 + False, 101 + Null, 102 + 103 + // ── Punctuators ────────────────────────────────────────── 104 + // Grouping 105 + LParen, // ( 106 + RParen, // ) 107 + LBracket, // [ 108 + RBracket, // ] 109 + LBrace, // { 110 + RBrace, // } 111 + 112 + // Delimiters 113 + Semicolon, // ; 114 + Comma, // , 115 + Colon, // : 116 + Dot, // . 117 + Ellipsis, // ... 118 + 119 + // Arrow 120 + Arrow, // => 121 + 122 + // Optional chaining 123 + QuestionDot, // ?. 124 + 125 + // Ternary 126 + Question, // ? 127 + 128 + // Assignment 129 + Assign, // = 130 + PlusAssign, // += 131 + MinusAssign, // -= 132 + StarAssign, // *= 133 + SlashAssign, // /= 134 + PercentAssign, // %= 135 + ExpAssign, // **= 136 + AmpAssign, // &= 137 + PipeAssign, // |= 138 + CaretAssign, // ^= 139 + ShlAssign, // <<= 140 + ShrAssign, // >>= 141 + UshrAssign, // >>>= 142 + AndAssign, // &&= 143 + OrAssign, // ||= 144 + NullishAssign, // ??= 145 + 146 + // Comparison 147 + Eq, // == 148 + Ne, // != 149 + StrictEq, // === 150 + StrictNe, // !== 151 + Lt, // < 152 + Gt, // > 153 + Le, // <= 154 + Ge, // >= 155 + 156 + // Arithmetic 157 + Plus, // + 158 + Minus, // - 159 + Star, // * 160 + Slash, // / 161 + Percent, // % 162 + Exp, // ** 163 + 164 + // Increment / Decrement 165 + PlusPlus, // ++ 166 + MinusMinus, // -- 167 + 168 + // Bitwise 169 + Amp, // & 170 + Pipe, // | 171 + Caret, // ^ 172 + Tilde, // ~ 173 + Shl, // << 174 + Shr, // >> 175 + Ushr, // >>> 176 + 177 + // Logical 178 + And, // && 179 + Or, // || 180 + Not, // ! 181 + Nullish, // ?? 182 + 183 + // ── Special ────────────────────────────────────────────── 184 + /// End of input. 185 + Eof, 186 + } 187 + 188 + impl fmt::Display for TokenKind { 189 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 190 + match self { 191 + TokenKind::Number(n) => write!(f, "{}", n), 192 + TokenKind::String(s) => write!(f, "\"{}\"", s), 193 + TokenKind::RegExp { pattern, flags } => write!(f, "/{}/{}", pattern, flags), 194 + TokenKind::TemplateFull(s) => write!(f, "`{}`", s), 195 + TokenKind::TemplateHead(s) => write!(f, "`{}${{", s), 196 + TokenKind::TemplateMiddle(s) => write!(f, "}}{}${{", s), 197 + TokenKind::TemplateTail(s) => write!(f, "}}{}`", s), 198 + TokenKind::Identifier(s) => write!(f, "{}", s), 199 + TokenKind::Await => write!(f, "await"), 200 + TokenKind::Break => write!(f, "break"), 201 + TokenKind::Case => write!(f, "case"), 202 + TokenKind::Catch => write!(f, "catch"), 203 + TokenKind::Class => write!(f, "class"), 204 + TokenKind::Const => write!(f, "const"), 205 + TokenKind::Continue => write!(f, "continue"), 206 + TokenKind::Debugger => write!(f, "debugger"), 207 + TokenKind::Default => write!(f, "default"), 208 + TokenKind::Delete => write!(f, "delete"), 209 + TokenKind::Do => write!(f, "do"), 210 + TokenKind::Else => write!(f, "else"), 211 + TokenKind::Export => write!(f, "export"), 212 + TokenKind::Extends => write!(f, "extends"), 213 + TokenKind::Finally => write!(f, "finally"), 214 + TokenKind::For => write!(f, "for"), 215 + TokenKind::Function => write!(f, "function"), 216 + TokenKind::If => write!(f, "if"), 217 + TokenKind::Import => write!(f, "import"), 218 + TokenKind::In => write!(f, "in"), 219 + TokenKind::Instanceof => write!(f, "instanceof"), 220 + TokenKind::Let => write!(f, "let"), 221 + TokenKind::New => write!(f, "new"), 222 + TokenKind::Of => write!(f, "of"), 223 + TokenKind::Return => write!(f, "return"), 224 + TokenKind::Static => write!(f, "static"), 225 + TokenKind::Super => write!(f, "super"), 226 + TokenKind::Switch => write!(f, "switch"), 227 + TokenKind::This => write!(f, "this"), 228 + TokenKind::Throw => write!(f, "throw"), 229 + TokenKind::Try => write!(f, "try"), 230 + TokenKind::Typeof => write!(f, "typeof"), 231 + TokenKind::Var => write!(f, "var"), 232 + TokenKind::Void => write!(f, "void"), 233 + TokenKind::While => write!(f, "while"), 234 + TokenKind::With => write!(f, "with"), 235 + TokenKind::Yield => write!(f, "yield"), 236 + TokenKind::Async => write!(f, "async"), 237 + TokenKind::True => write!(f, "true"), 238 + TokenKind::False => write!(f, "false"), 239 + TokenKind::Null => write!(f, "null"), 240 + TokenKind::LParen => write!(f, "("), 241 + TokenKind::RParen => write!(f, ")"), 242 + TokenKind::LBracket => write!(f, "["), 243 + TokenKind::RBracket => write!(f, "]"), 244 + TokenKind::LBrace => write!(f, "{{"), 245 + TokenKind::RBrace => write!(f, "}}"), 246 + TokenKind::Semicolon => write!(f, ";"), 247 + TokenKind::Comma => write!(f, ","), 248 + TokenKind::Colon => write!(f, ":"), 249 + TokenKind::Dot => write!(f, "."), 250 + TokenKind::Ellipsis => write!(f, "..."), 251 + TokenKind::Arrow => write!(f, "=>"), 252 + TokenKind::QuestionDot => write!(f, "?."), 253 + TokenKind::Question => write!(f, "?"), 254 + TokenKind::Assign => write!(f, "="), 255 + TokenKind::PlusAssign => write!(f, "+="), 256 + TokenKind::MinusAssign => write!(f, "-="), 257 + TokenKind::StarAssign => write!(f, "*="), 258 + TokenKind::SlashAssign => write!(f, "/="), 259 + TokenKind::PercentAssign => write!(f, "%="), 260 + TokenKind::ExpAssign => write!(f, "**="), 261 + TokenKind::AmpAssign => write!(f, "&="), 262 + TokenKind::PipeAssign => write!(f, "|="), 263 + TokenKind::CaretAssign => write!(f, "^="), 264 + TokenKind::ShlAssign => write!(f, "<<="), 265 + TokenKind::ShrAssign => write!(f, ">>="), 266 + TokenKind::UshrAssign => write!(f, ">>>="), 267 + TokenKind::AndAssign => write!(f, "&&="), 268 + TokenKind::OrAssign => write!(f, "||="), 269 + TokenKind::NullishAssign => write!(f, "??="), 270 + TokenKind::Eq => write!(f, "=="), 271 + TokenKind::Ne => write!(f, "!="), 272 + TokenKind::StrictEq => write!(f, "==="), 273 + TokenKind::StrictNe => write!(f, "!=="), 274 + TokenKind::Lt => write!(f, "<"), 275 + TokenKind::Gt => write!(f, ">"), 276 + TokenKind::Le => write!(f, "<="), 277 + TokenKind::Ge => write!(f, ">="), 278 + TokenKind::Plus => write!(f, "+"), 279 + TokenKind::Minus => write!(f, "-"), 280 + TokenKind::Star => write!(f, "*"), 281 + TokenKind::Slash => write!(f, "/"), 282 + TokenKind::Percent => write!(f, "%"), 283 + TokenKind::Exp => write!(f, "**"), 284 + TokenKind::PlusPlus => write!(f, "++"), 285 + TokenKind::MinusMinus => write!(f, "--"), 286 + TokenKind::Amp => write!(f, "&"), 287 + TokenKind::Pipe => write!(f, "|"), 288 + TokenKind::Caret => write!(f, "^"), 289 + TokenKind::Tilde => write!(f, "~"), 290 + TokenKind::Shl => write!(f, "<<"), 291 + TokenKind::Shr => write!(f, ">>"), 292 + TokenKind::Ushr => write!(f, ">>>"), 293 + TokenKind::And => write!(f, "&&"), 294 + TokenKind::Or => write!(f, "||"), 295 + TokenKind::Not => write!(f, "!"), 296 + TokenKind::Nullish => write!(f, "??"), 297 + TokenKind::Eof => write!(f, "<EOF>"), 298 + } 299 + } 300 + } 301 + 302 + /// The lexer converts JavaScript source text into tokens. 303 + pub struct Lexer<'a> { 304 + source: &'a [u8], 305 + /// Current byte offset into `source`. 306 + pos: usize, 307 + /// Current 1-based line number. 308 + line: u32, 309 + /// Current 1-based column (byte offset from line start). 310 + col: u32, 311 + /// Whether we have crossed at least one newline since the last token. 312 + saw_newline: bool, 313 + /// Nesting depth for template literal `${...}` expressions. 314 + /// When > 0, a `}` at the matching depth resumes template scanning. 315 + template_depth: u32, 316 + /// Stack tracking brace depth at each template nesting level. 317 + /// When we enter `${`, we push the current brace depth. 318 + template_brace_stack: Vec<u32>, 319 + /// Current brace depth (incremented on `{`, decremented on `}`). 320 + brace_depth: u32, 321 + /// Tracks whether the previous token could end an expression. 322 + /// Used to disambiguate `/` as division vs RegExp. 323 + prev_token_is_expr_end: bool, 324 + } 325 + 326 + impl<'a> Lexer<'a> { 327 + /// Create a new lexer for the given source text. 328 + pub fn new(source: &'a str) -> Self { 329 + Self { 330 + source: source.as_bytes(), 331 + pos: 0, 332 + line: 1, 333 + col: 1, 334 + saw_newline: false, 335 + template_depth: 0, 336 + template_brace_stack: Vec::new(), 337 + brace_depth: 0, 338 + prev_token_is_expr_end: false, 339 + } 340 + } 341 + 342 + /// Tokenize the entire source and return all tokens (including final `Eof`). 343 + pub fn tokenize(source: &str) -> Result<Vec<Token>, LexError> { 344 + let mut lexer = Lexer::new(source); 345 + let mut tokens = Vec::new(); 346 + loop { 347 + let tok = lexer.next_token()?; 348 + let is_eof = tok.kind == TokenKind::Eof; 349 + tokens.push(tok); 350 + if is_eof { 351 + break; 352 + } 353 + } 354 + Ok(tokens) 355 + } 356 + 357 + // ── Helpers ────────────────────────────────────────────── 358 + 359 + fn current_pos(&self) -> SourcePos { 360 + SourcePos { 361 + line: self.line, 362 + col: self.col, 363 + } 364 + } 365 + 366 + fn peek(&self) -> Option<u8> { 367 + self.source.get(self.pos).copied() 368 + } 369 + 370 + fn peek_at(&self, offset: usize) -> Option<u8> { 371 + self.source.get(self.pos + offset).copied() 372 + } 373 + 374 + fn advance(&mut self) -> Option<u8> { 375 + let b = self.source.get(self.pos).copied()?; 376 + self.pos += 1; 377 + if b == b'\n' { 378 + self.line += 1; 379 + self.col = 1; 380 + self.saw_newline = true; 381 + } else { 382 + self.col += 1; 383 + } 384 + Some(b) 385 + } 386 + 387 + fn advance_if(&mut self, expected: u8) -> bool { 388 + if self.peek() == Some(expected) { 389 + self.advance(); 390 + true 391 + } else { 392 + false 393 + } 394 + } 395 + 396 + fn slice(&self, start: usize, end: usize) -> &'a str { 397 + // Safety: we only slice at positions we've already walked over, 398 + // and we trust the input to be valid UTF-8 at identifier/keyword 399 + // boundaries. In practice this is safe because the lexer only 400 + // slices ASCII-compatible byte sequences. 401 + std::str::from_utf8(&self.source[start..end]).unwrap_or("") 402 + } 403 + 404 + // ── Whitespace & Comments ──────────────────────────────── 405 + 406 + fn skip_whitespace_and_comments(&mut self) -> Result<(), LexError> { 407 + loop { 408 + match self.peek() { 409 + Some(b' ' | b'\t' | b'\r' | b'\n') => { 410 + self.advance(); 411 + } 412 + // Unicode BOM / non-breaking spaces 413 + Some(0xC2) if self.peek_at(1) == Some(0xA0) => { 414 + // U+00A0 non-breaking space (2-byte UTF-8) 415 + self.advance(); 416 + self.advance(); 417 + } 418 + Some(0xEF) if self.peek_at(1) == Some(0xBB) && self.peek_at(2) == Some(0xBF) => { 419 + // BOM U+FEFF 420 + self.advance(); 421 + self.advance(); 422 + self.advance(); 423 + } 424 + Some(b'/') => { 425 + match self.peek_at(1) { 426 + Some(b'/') => { 427 + // single-line comment 428 + self.advance(); // / 429 + self.advance(); // / 430 + while let Some(b) = self.peek() { 431 + if b == b'\n' { 432 + break; 433 + } 434 + self.advance(); 435 + } 436 + } 437 + Some(b'*') => { 438 + // multi-line comment 439 + let start = self.current_pos(); 440 + self.advance(); // / 441 + self.advance(); // * 442 + let mut closed = false; 443 + while let Some(b) = self.advance() { 444 + if b == b'*' && self.peek() == Some(b'/') { 445 + self.advance(); // / 446 + closed = true; 447 + break; 448 + } 449 + } 450 + if !closed { 451 + return Err(LexError { 452 + message: "unterminated block comment".into(), 453 + pos: start, 454 + }); 455 + } 456 + } 457 + _ => break, 458 + } 459 + } 460 + _ => break, 461 + } 462 + } 463 + Ok(()) 464 + } 465 + 466 + // ── Main dispatch ──────────────────────────────────────── 467 + 468 + /// Produce the next token. 469 + pub fn next_token(&mut self) -> Result<Token, LexError> { 470 + self.saw_newline = false; 471 + self.skip_whitespace_and_comments()?; 472 + 473 + let start = self.current_pos(); 474 + 475 + let Some(b) = self.peek() else { 476 + return Ok(Token { 477 + kind: TokenKind::Eof, 478 + span: Span { 479 + start, 480 + end: self.current_pos(), 481 + }, 482 + preceded_by_newline: self.saw_newline, 483 + }); 484 + }; 485 + 486 + // If we're inside a template `${...}` and hit the matching `}`, 487 + // resume template scanning. 488 + if b == b'}' 489 + && !self.template_brace_stack.is_empty() 490 + && self.brace_depth == *self.template_brace_stack.last().unwrap() 491 + { 492 + self.template_brace_stack.pop(); 493 + self.template_depth -= 1; 494 + self.advance(); // consume } 495 + return self.scan_template_continuation(start); 496 + } 497 + 498 + let kind = match b { 499 + b'`' => { 500 + self.advance(); 501 + return self.scan_template_start(start); 502 + } 503 + 504 + b'0'..=b'9' => self.scan_number()?, 505 + b'.' if matches!(self.peek_at(1), Some(b'0'..=b'9')) => self.scan_number()?, 506 + 507 + b'"' | b'\'' => self.scan_string()?, 508 + 509 + b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'$' => self.scan_identifier_or_keyword(), 510 + // UTF-8 multi-byte identifier start 511 + 0xC0..=0xF7 if is_unicode_id_start(self.source, self.pos) => { 512 + self.scan_identifier_or_keyword() 513 + } 514 + 515 + b'/' if !self.prev_token_is_expr_end => self.scan_regexp()?, 516 + 517 + _ => self.scan_punctuator()?, 518 + }; 519 + 520 + let end = self.current_pos(); 521 + let preceded_by_newline = self.saw_newline; 522 + 523 + // Track whether this token ends an expression (for `/` disambiguation). 524 + self.prev_token_is_expr_end = token_is_expr_end(&kind); 525 + 526 + Ok(Token { 527 + kind, 528 + span: Span { start, end }, 529 + preceded_by_newline, 530 + }) 531 + } 532 + 533 + // ── Numbers ────────────────────────────────────────────── 534 + 535 + fn scan_number(&mut self) -> Result<TokenKind, LexError> { 536 + let start = self.pos; 537 + 538 + if self.peek() == Some(b'0') { 539 + match self.peek_at(1) { 540 + Some(b'x' | b'X') => return self.scan_hex_number(), 541 + Some(b'o' | b'O') => return self.scan_octal_number(), 542 + Some(b'b' | b'B') => return self.scan_binary_number(), 543 + _ => {} 544 + } 545 + } 546 + 547 + // Decimal integer or float 548 + self.eat_decimal_digits(); 549 + 550 + if self.peek() == Some(b'.') { 551 + // Could be `1..toString()` — only consume `.` if followed by a digit 552 + // or if this is a leading dot (start already has a digit, so peek is safe). 553 + // Actually, `1.` is a valid numeric literal (= 1.0), and `1.e2` = 100. 554 + // We consume the dot always unless it's `..` (spread). 555 + if self.peek_at(1) != Some(b'.') { 556 + self.advance(); // . 557 + self.eat_decimal_digits(); 558 + } 559 + } 560 + 561 + // Exponent 562 + if matches!(self.peek(), Some(b'e' | b'E')) { 563 + self.advance(); 564 + if matches!(self.peek(), Some(b'+' | b'-')) { 565 + self.advance(); 566 + } 567 + if !matches!(self.peek(), Some(b'0'..=b'9')) { 568 + return Err(LexError { 569 + message: "expected digit after exponent".into(), 570 + pos: self.current_pos(), 571 + }); 572 + } 573 + self.eat_decimal_digits(); 574 + } 575 + 576 + // BigInt suffix `n` — we tokenize it but store as f64 (for now) 577 + self.advance_if(b'n'); 578 + 579 + let text = self.slice(start, self.pos); 580 + let value = parse_decimal(text); 581 + Ok(TokenKind::Number(value)) 582 + } 583 + 584 + fn scan_hex_number(&mut self) -> Result<TokenKind, LexError> { 585 + self.advance(); // 0 586 + self.advance(); // x/X 587 + let digit_start = self.pos; 588 + self.eat_hex_digits(); 589 + if self.pos == digit_start { 590 + return Err(LexError { 591 + message: "expected hex digit after 0x".into(), 592 + pos: self.current_pos(), 593 + }); 594 + } 595 + self.advance_if(b'n'); 596 + let text = self.slice(digit_start, self.pos); 597 + let text = text.trim_end_matches('n'); 598 + let value = u64_from_hex(text) as f64; 599 + Ok(TokenKind::Number(value)) 600 + } 601 + 602 + fn scan_octal_number(&mut self) -> Result<TokenKind, LexError> { 603 + self.advance(); // 0 604 + self.advance(); // o/O 605 + let digit_start = self.pos; 606 + while matches!(self.peek(), Some(b'0'..=b'7' | b'_')) { 607 + self.advance(); 608 + } 609 + if self.pos == digit_start { 610 + return Err(LexError { 611 + message: "expected octal digit after 0o".into(), 612 + pos: self.current_pos(), 613 + }); 614 + } 615 + self.advance_if(b'n'); 616 + let text = self.slice(digit_start, self.pos).trim_end_matches('n'); 617 + let value = u64_from_octal(text) as f64; 618 + Ok(TokenKind::Number(value)) 619 + } 620 + 621 + fn scan_binary_number(&mut self) -> Result<TokenKind, LexError> { 622 + self.advance(); // 0 623 + self.advance(); // b/B 624 + let digit_start = self.pos; 625 + while matches!(self.peek(), Some(b'0' | b'1' | b'_')) { 626 + self.advance(); 627 + } 628 + if self.pos == digit_start { 629 + return Err(LexError { 630 + message: "expected binary digit after 0b".into(), 631 + pos: self.current_pos(), 632 + }); 633 + } 634 + self.advance_if(b'n'); 635 + let text = self.slice(digit_start, self.pos).trim_end_matches('n'); 636 + let value = u64_from_binary(text) as f64; 637 + Ok(TokenKind::Number(value)) 638 + } 639 + 640 + fn eat_decimal_digits(&mut self) { 641 + while matches!(self.peek(), Some(b'0'..=b'9' | b'_')) { 642 + self.advance(); 643 + } 644 + } 645 + 646 + fn eat_hex_digits(&mut self) { 647 + while matches!( 648 + self.peek(), 649 + Some(b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' | b'_') 650 + ) { 651 + self.advance(); 652 + } 653 + } 654 + 655 + // ── Strings ────────────────────────────────────────────── 656 + 657 + fn scan_string(&mut self) -> Result<TokenKind, LexError> { 658 + let quote = self.advance().unwrap(); // opening quote 659 + let start_pos = self.current_pos(); 660 + let mut value = std::string::String::new(); 661 + 662 + loop { 663 + match self.peek() { 664 + None | Some(b'\n') => { 665 + return Err(LexError { 666 + message: "unterminated string literal".into(), 667 + pos: start_pos, 668 + }); 669 + } 670 + Some(b) if b == quote => { 671 + self.advance(); 672 + break; 673 + } 674 + Some(b'\\') => { 675 + self.advance(); // backslash 676 + let ch = self.scan_escape_sequence()?; 677 + value.push(ch); 678 + } 679 + Some(_) => { 680 + let ch = self.advance_char(); 681 + value.push(ch); 682 + } 683 + } 684 + } 685 + 686 + Ok(TokenKind::String(value)) 687 + } 688 + 689 + fn scan_escape_sequence(&mut self) -> Result<char, LexError> { 690 + let pos = self.current_pos(); 691 + match self.advance() { 692 + Some(b'n') => Ok('\n'), 693 + Some(b'r') => Ok('\r'), 694 + Some(b't') => Ok('\t'), 695 + Some(b'b') => Ok('\u{0008}'), 696 + Some(b'f') => Ok('\u{000C}'), 697 + Some(b'v') => Ok('\u{000B}'), 698 + Some(b'0') if !matches!(self.peek(), Some(b'0'..=b'9')) => Ok('\0'), 699 + Some(b'\\') => Ok('\\'), 700 + Some(b'\'') => Ok('\''), 701 + Some(b'"') => Ok('"'), 702 + Some(b'`') => Ok('`'), 703 + Some(b'\n') => Ok('\n'), 704 + Some(b'\r') => { 705 + self.advance_if(b'\n'); 706 + Ok('\n') 707 + } 708 + Some(b'x') => { 709 + let hi = self.advance().and_then(hex_digit_val).ok_or(LexError { 710 + message: "invalid hex escape".into(), 711 + pos, 712 + })?; 713 + let lo = self.advance().and_then(hex_digit_val).ok_or(LexError { 714 + message: "invalid hex escape".into(), 715 + pos, 716 + })?; 717 + let code = (hi << 4) | lo; 718 + Ok(code as char) 719 + } 720 + Some(b'u') => self.scan_unicode_escape(pos), 721 + Some(b) => { 722 + // identity escape 723 + Ok(b as char) 724 + } 725 + None => Err(LexError { 726 + message: "unexpected end of input in escape sequence".into(), 727 + pos, 728 + }), 729 + } 730 + } 731 + 732 + fn scan_unicode_escape(&mut self, pos: SourcePos) -> Result<char, LexError> { 733 + if self.advance_if(b'{') { 734 + // \u{XXXXX} 735 + let mut code: u32 = 0; 736 + let mut count = 0; 737 + while let Some(b) = self.peek() { 738 + if b == b'}' { 739 + break; 740 + } 741 + let d = hex_digit_val(self.advance().unwrap()).ok_or(LexError { 742 + message: "invalid unicode escape".into(), 743 + pos, 744 + })?; 745 + code = code * 16 + d as u32; 746 + count += 1; 747 + if code > 0x10FFFF { 748 + return Err(LexError { 749 + message: "unicode escape out of range".into(), 750 + pos, 751 + }); 752 + } 753 + } 754 + if count == 0 || !self.advance_if(b'}') { 755 + return Err(LexError { 756 + message: "invalid unicode escape".into(), 757 + pos, 758 + }); 759 + } 760 + char::from_u32(code).ok_or(LexError { 761 + message: "invalid unicode code point".into(), 762 + pos, 763 + }) 764 + } else { 765 + // \uXXXX 766 + let mut code: u32 = 0; 767 + for _ in 0..4 { 768 + let d = self.advance().and_then(hex_digit_val).ok_or(LexError { 769 + message: "invalid unicode escape".into(), 770 + pos, 771 + })?; 772 + code = code * 16 + d as u32; 773 + } 774 + char::from_u32(code).ok_or(LexError { 775 + message: "invalid unicode code point".into(), 776 + pos, 777 + }) 778 + } 779 + } 780 + 781 + /// Advance one full UTF-8 character and return it. 782 + fn advance_char(&mut self) -> char { 783 + let start = self.pos; 784 + let b = self.advance().unwrap(); 785 + if b < 0x80 { 786 + return b as char; 787 + } 788 + // multi-byte: determine length 789 + let len = if b >= 0xF0 { 790 + 4 791 + } else if b >= 0xE0 { 792 + 3 793 + } else { 794 + 2 795 + }; 796 + for _ in 1..len { 797 + self.advance(); 798 + } 799 + let s = std::str::from_utf8(&self.source[start..self.pos]).unwrap_or("\u{FFFD}"); 800 + s.chars().next().unwrap_or('\u{FFFD}') 801 + } 802 + 803 + // ── Template Literals ──────────────────────────────────── 804 + 805 + fn scan_template_start(&mut self, start: SourcePos) -> Result<Token, LexError> { 806 + let mut value = std::string::String::new(); 807 + loop { 808 + match self.peek() { 809 + None => { 810 + return Err(LexError { 811 + message: "unterminated template literal".into(), 812 + pos: start, 813 + }); 814 + } 815 + Some(b'`') => { 816 + self.advance(); 817 + let end = self.current_pos(); 818 + let kind = TokenKind::TemplateFull(value); 819 + self.prev_token_is_expr_end = true; 820 + return Ok(Token { 821 + kind, 822 + span: Span { start, end }, 823 + preceded_by_newline: self.saw_newline, 824 + }); 825 + } 826 + Some(b'$') if self.peek_at(1) == Some(b'{') => { 827 + self.advance(); // $ 828 + self.advance(); // { 829 + self.template_depth += 1; 830 + self.template_brace_stack.push(self.brace_depth); 831 + let end = self.current_pos(); 832 + let kind = TokenKind::TemplateHead(value); 833 + self.prev_token_is_expr_end = false; 834 + return Ok(Token { 835 + kind, 836 + span: Span { start, end }, 837 + preceded_by_newline: self.saw_newline, 838 + }); 839 + } 840 + Some(b'\\') => { 841 + self.advance(); 842 + let ch = self.scan_escape_sequence()?; 843 + value.push(ch); 844 + } 845 + Some(_) => { 846 + let ch = self.advance_char(); 847 + value.push(ch); 848 + } 849 + } 850 + } 851 + } 852 + 853 + fn scan_template_continuation(&mut self, start: SourcePos) -> Result<Token, LexError> { 854 + let mut value = std::string::String::new(); 855 + loop { 856 + match self.peek() { 857 + None => { 858 + return Err(LexError { 859 + message: "unterminated template literal".into(), 860 + pos: start, 861 + }); 862 + } 863 + Some(b'`') => { 864 + self.advance(); 865 + let end = self.current_pos(); 866 + let kind = TokenKind::TemplateTail(value); 867 + self.prev_token_is_expr_end = true; 868 + return Ok(Token { 869 + kind, 870 + span: Span { start, end }, 871 + preceded_by_newline: self.saw_newline, 872 + }); 873 + } 874 + Some(b'$') if self.peek_at(1) == Some(b'{') => { 875 + self.advance(); // $ 876 + self.advance(); // { 877 + self.template_depth += 1; 878 + self.template_brace_stack.push(self.brace_depth); 879 + let end = self.current_pos(); 880 + let kind = TokenKind::TemplateMiddle(value); 881 + self.prev_token_is_expr_end = false; 882 + return Ok(Token { 883 + kind, 884 + span: Span { start, end }, 885 + preceded_by_newline: self.saw_newline, 886 + }); 887 + } 888 + Some(b'\\') => { 889 + self.advance(); 890 + let ch = self.scan_escape_sequence()?; 891 + value.push(ch); 892 + } 893 + Some(_) => { 894 + let ch = self.advance_char(); 895 + value.push(ch); 896 + } 897 + } 898 + } 899 + } 900 + 901 + // ── Identifiers & Keywords ─────────────────────────────── 902 + 903 + fn scan_identifier_or_keyword(&mut self) -> TokenKind { 904 + let start = self.pos; 905 + 906 + // Consume the first character (which we already validated) 907 + self.advance_char(); 908 + 909 + // Consume continue characters 910 + while self.pos < self.source.len() { 911 + let b = self.source[self.pos]; 912 + match b { 913 + b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' | b'_' | b'$' => { 914 + self.advance(); 915 + } 916 + 0xC0..=0xF7 if is_unicode_id_continue(self.source, self.pos) => { 917 + self.advance_char(); 918 + } 919 + _ => break, 920 + } 921 + } 922 + 923 + let text = self.slice(start, self.pos); 924 + keyword_or_ident(text) 925 + } 926 + 927 + // ── Regular Expressions ────────────────────────────────── 928 + 929 + fn scan_regexp(&mut self) -> Result<TokenKind, LexError> { 930 + let start_pos = self.current_pos(); 931 + self.advance(); // opening / 932 + 933 + let mut pattern = std::string::String::new(); 934 + let mut in_class = false; 935 + 936 + loop { 937 + match self.peek() { 938 + None | Some(b'\n') => { 939 + return Err(LexError { 940 + message: "unterminated regexp literal".into(), 941 + pos: start_pos, 942 + }); 943 + } 944 + Some(b'/') if !in_class => { 945 + self.advance(); 946 + break; 947 + } 948 + Some(b'[') => { 949 + in_class = true; 950 + pattern.push('['); 951 + self.advance(); 952 + } 953 + Some(b']') if in_class => { 954 + in_class = false; 955 + pattern.push(']'); 956 + self.advance(); 957 + } 958 + Some(b'\\') => { 959 + self.advance(); 960 + pattern.push('\\'); 961 + if let Some(b2) = self.peek() { 962 + if b2 != b'\n' { 963 + pattern.push(b2 as char); 964 + self.advance(); 965 + } 966 + } 967 + } 968 + Some(b) => { 969 + pattern.push(b as char); 970 + self.advance(); 971 + } 972 + } 973 + } 974 + 975 + // Flags 976 + let mut flags = std::string::String::new(); 977 + while matches!( 978 + self.peek(), 979 + Some(b'g' | b'i' | b'm' | b's' | b'u' | b'v' | b'y' | b'd') 980 + ) { 981 + flags.push(self.advance().unwrap() as char); 982 + } 983 + 984 + Ok(TokenKind::RegExp { pattern, flags }) 985 + } 986 + 987 + // ── Punctuators ────────────────────────────────────────── 988 + 989 + fn scan_punctuator(&mut self) -> Result<TokenKind, LexError> { 990 + let pos = self.current_pos(); 991 + let b = self.advance().unwrap(); 992 + 993 + let kind = match b { 994 + b'(' => TokenKind::LParen, 995 + b')' => TokenKind::RParen, 996 + b'[' => TokenKind::LBracket, 997 + b']' => TokenKind::RBracket, 998 + b'{' => { 999 + self.brace_depth += 1; 1000 + TokenKind::LBrace 1001 + } 1002 + b'}' => { 1003 + self.brace_depth = self.brace_depth.saturating_sub(1); 1004 + TokenKind::RBrace 1005 + } 1006 + b';' => TokenKind::Semicolon, 1007 + b',' => TokenKind::Comma, 1008 + b':' => TokenKind::Colon, 1009 + b'~' => TokenKind::Tilde, 1010 + 1011 + b'.' => { 1012 + if self.peek() == Some(b'.') && self.peek_at(1) == Some(b'.') { 1013 + self.advance(); 1014 + self.advance(); 1015 + TokenKind::Ellipsis 1016 + } else { 1017 + TokenKind::Dot 1018 + } 1019 + } 1020 + 1021 + b'?' => { 1022 + if self.advance_if(b'?') { 1023 + if self.advance_if(b'=') { 1024 + TokenKind::NullishAssign 1025 + } else { 1026 + TokenKind::Nullish 1027 + } 1028 + } else if self.peek() == Some(b'.') && !matches!(self.peek_at(1), Some(b'0'..=b'9')) 1029 + { 1030 + self.advance(); 1031 + TokenKind::QuestionDot 1032 + } else { 1033 + TokenKind::Question 1034 + } 1035 + } 1036 + 1037 + b'+' => { 1038 + if self.advance_if(b'+') { 1039 + TokenKind::PlusPlus 1040 + } else if self.advance_if(b'=') { 1041 + TokenKind::PlusAssign 1042 + } else { 1043 + TokenKind::Plus 1044 + } 1045 + } 1046 + 1047 + b'-' => { 1048 + if self.advance_if(b'-') { 1049 + TokenKind::MinusMinus 1050 + } else if self.advance_if(b'=') { 1051 + TokenKind::MinusAssign 1052 + } else { 1053 + TokenKind::Minus 1054 + } 1055 + } 1056 + 1057 + b'*' => { 1058 + if self.advance_if(b'*') { 1059 + if self.advance_if(b'=') { 1060 + TokenKind::ExpAssign 1061 + } else { 1062 + TokenKind::Exp 1063 + } 1064 + } else if self.advance_if(b'=') { 1065 + TokenKind::StarAssign 1066 + } else { 1067 + TokenKind::Star 1068 + } 1069 + } 1070 + 1071 + b'/' => { 1072 + // We only get here for division (regexp was handled earlier) 1073 + if self.advance_if(b'=') { 1074 + TokenKind::SlashAssign 1075 + } else { 1076 + TokenKind::Slash 1077 + } 1078 + } 1079 + 1080 + b'%' => { 1081 + if self.advance_if(b'=') { 1082 + TokenKind::PercentAssign 1083 + } else { 1084 + TokenKind::Percent 1085 + } 1086 + } 1087 + 1088 + b'=' => { 1089 + if self.advance_if(b'=') { 1090 + if self.advance_if(b'=') { 1091 + TokenKind::StrictEq 1092 + } else { 1093 + TokenKind::Eq 1094 + } 1095 + } else if self.advance_if(b'>') { 1096 + TokenKind::Arrow 1097 + } else { 1098 + TokenKind::Assign 1099 + } 1100 + } 1101 + 1102 + b'!' => { 1103 + if self.advance_if(b'=') { 1104 + if self.advance_if(b'=') { 1105 + TokenKind::StrictNe 1106 + } else { 1107 + TokenKind::Ne 1108 + } 1109 + } else { 1110 + TokenKind::Not 1111 + } 1112 + } 1113 + 1114 + b'<' => { 1115 + if self.advance_if(b'<') { 1116 + if self.advance_if(b'=') { 1117 + TokenKind::ShlAssign 1118 + } else { 1119 + TokenKind::Shl 1120 + } 1121 + } else if self.advance_if(b'=') { 1122 + TokenKind::Le 1123 + } else { 1124 + TokenKind::Lt 1125 + } 1126 + } 1127 + 1128 + b'>' => { 1129 + if self.advance_if(b'>') { 1130 + if self.advance_if(b'>') { 1131 + if self.advance_if(b'=') { 1132 + TokenKind::UshrAssign 1133 + } else { 1134 + TokenKind::Ushr 1135 + } 1136 + } else if self.advance_if(b'=') { 1137 + TokenKind::ShrAssign 1138 + } else { 1139 + TokenKind::Shr 1140 + } 1141 + } else if self.advance_if(b'=') { 1142 + TokenKind::Ge 1143 + } else { 1144 + TokenKind::Gt 1145 + } 1146 + } 1147 + 1148 + b'&' => { 1149 + if self.advance_if(b'&') { 1150 + if self.advance_if(b'=') { 1151 + TokenKind::AndAssign 1152 + } else { 1153 + TokenKind::And 1154 + } 1155 + } else if self.advance_if(b'=') { 1156 + TokenKind::AmpAssign 1157 + } else { 1158 + TokenKind::Amp 1159 + } 1160 + } 1161 + 1162 + b'|' => { 1163 + if self.advance_if(b'|') { 1164 + if self.advance_if(b'=') { 1165 + TokenKind::OrAssign 1166 + } else { 1167 + TokenKind::Or 1168 + } 1169 + } else if self.advance_if(b'=') { 1170 + TokenKind::PipeAssign 1171 + } else { 1172 + TokenKind::Pipe 1173 + } 1174 + } 1175 + 1176 + b'^' => { 1177 + if self.advance_if(b'=') { 1178 + TokenKind::CaretAssign 1179 + } else { 1180 + TokenKind::Caret 1181 + } 1182 + } 1183 + 1184 + _ => { 1185 + return Err(LexError { 1186 + message: format!("unexpected character: {:?}", b as char), 1187 + pos, 1188 + }); 1189 + } 1190 + }; 1191 + 1192 + Ok(kind) 1193 + } 1194 + } 1195 + 1196 + // ── Keyword lookup ─────────────────────────────────────────── 1197 + 1198 + fn keyword_or_ident(s: &str) -> TokenKind { 1199 + match s { 1200 + "await" => TokenKind::Await, 1201 + "break" => TokenKind::Break, 1202 + "case" => TokenKind::Case, 1203 + "catch" => TokenKind::Catch, 1204 + "class" => TokenKind::Class, 1205 + "const" => TokenKind::Const, 1206 + "continue" => TokenKind::Continue, 1207 + "debugger" => TokenKind::Debugger, 1208 + "default" => TokenKind::Default, 1209 + "delete" => TokenKind::Delete, 1210 + "do" => TokenKind::Do, 1211 + "else" => TokenKind::Else, 1212 + "export" => TokenKind::Export, 1213 + "extends" => TokenKind::Extends, 1214 + "finally" => TokenKind::Finally, 1215 + "for" => TokenKind::For, 1216 + "function" => TokenKind::Function, 1217 + "if" => TokenKind::If, 1218 + "import" => TokenKind::Import, 1219 + "in" => TokenKind::In, 1220 + "instanceof" => TokenKind::Instanceof, 1221 + "let" => TokenKind::Let, 1222 + "new" => TokenKind::New, 1223 + "of" => TokenKind::Of, 1224 + "return" => TokenKind::Return, 1225 + "static" => TokenKind::Static, 1226 + "super" => TokenKind::Super, 1227 + "switch" => TokenKind::Switch, 1228 + "this" => TokenKind::This, 1229 + "throw" => TokenKind::Throw, 1230 + "try" => TokenKind::Try, 1231 + "typeof" => TokenKind::Typeof, 1232 + "var" => TokenKind::Var, 1233 + "void" => TokenKind::Void, 1234 + "while" => TokenKind::While, 1235 + "with" => TokenKind::With, 1236 + "yield" => TokenKind::Yield, 1237 + "async" => TokenKind::Async, 1238 + "true" => TokenKind::True, 1239 + "false" => TokenKind::False, 1240 + "null" => TokenKind::Null, 1241 + _ => TokenKind::Identifier(s.to_owned()), 1242 + } 1243 + } 1244 + 1245 + // ── Expression-end tracking ────────────────────────────────── 1246 + 1247 + /// Returns `true` if a token of this kind could end an expression. 1248 + /// Used to decide whether a following `/` is division or a RegExp literal. 1249 + fn token_is_expr_end(kind: &TokenKind) -> bool { 1250 + matches!( 1251 + kind, 1252 + TokenKind::Identifier(_) 1253 + | TokenKind::Number(_) 1254 + | TokenKind::String(_) 1255 + | TokenKind::TemplateFull(_) 1256 + | TokenKind::TemplateTail(_) 1257 + | TokenKind::True 1258 + | TokenKind::False 1259 + | TokenKind::Null 1260 + | TokenKind::This 1261 + | TokenKind::Super 1262 + | TokenKind::RParen 1263 + | TokenKind::RBracket 1264 + | TokenKind::RBrace 1265 + | TokenKind::PlusPlus 1266 + | TokenKind::MinusMinus 1267 + | TokenKind::RegExp { .. } 1268 + ) 1269 + } 1270 + 1271 + // ── Unicode helpers ────────────────────────────────────────── 1272 + 1273 + /// Check if the byte sequence at `pos` starts a valid Unicode identifier start character. 1274 + fn is_unicode_id_start(source: &[u8], pos: usize) -> bool { 1275 + let s = std::str::from_utf8(&source[pos..]).unwrap_or(""); 1276 + if let Some(ch) = s.chars().next() { 1277 + ch.is_alphabetic() || ch == '_' || ch == '$' 1278 + } else { 1279 + false 1280 + } 1281 + } 1282 + 1283 + /// Check if the byte sequence at `pos` starts a valid Unicode identifier continue character. 1284 + fn is_unicode_id_continue(source: &[u8], pos: usize) -> bool { 1285 + let s = std::str::from_utf8(&source[pos..]).unwrap_or(""); 1286 + if let Some(ch) = s.chars().next() { 1287 + ch.is_alphanumeric() || ch == '_' || ch == '$' || ch == '\u{200C}' || ch == '\u{200D}' 1288 + } else { 1289 + false 1290 + } 1291 + } 1292 + 1293 + // ── Numeric parsing helpers ────────────────────────────────── 1294 + 1295 + fn hex_digit_val(b: u8) -> Option<u8> { 1296 + match b { 1297 + b'0'..=b'9' => Some(b - b'0'), 1298 + b'a'..=b'f' => Some(b - b'a' + 10), 1299 + b'A'..=b'F' => Some(b - b'A' + 10), 1300 + _ => None, 1301 + } 1302 + } 1303 + 1304 + fn parse_decimal(s: &str) -> f64 { 1305 + let s = s.replace('_', ""); 1306 + let s = s.trim_end_matches('n'); 1307 + // Use manual parsing for basic decimal and float 1308 + if let Ok(v) = s.parse::<f64>() { 1309 + return v; 1310 + } 1311 + 0.0 1312 + } 1313 + 1314 + fn u64_from_hex(s: &str) -> u64 { 1315 + let mut result: u64 = 0; 1316 + for b in s.bytes() { 1317 + if b == b'_' { 1318 + continue; 1319 + } 1320 + let d = hex_digit_val(b).unwrap_or(0) as u64; 1321 + result = result.wrapping_mul(16).wrapping_add(d); 1322 + } 1323 + result 1324 + } 1325 + 1326 + fn u64_from_octal(s: &str) -> u64 { 1327 + let mut result: u64 = 0; 1328 + for b in s.bytes() { 1329 + if b == b'_' { 1330 + continue; 1331 + } 1332 + let d = (b - b'0') as u64; 1333 + result = result.wrapping_mul(8).wrapping_add(d); 1334 + } 1335 + result 1336 + } 1337 + 1338 + fn u64_from_binary(s: &str) -> u64 { 1339 + let mut result: u64 = 0; 1340 + for b in s.bytes() { 1341 + if b == b'_' { 1342 + continue; 1343 + } 1344 + let d = (b - b'0') as u64; 1345 + result = result.wrapping_mul(2).wrapping_add(d); 1346 + } 1347 + result 1348 + } 1349 + 1350 + // ── Error type ─────────────────────────────────────────────── 1351 + 1352 + /// An error produced during lexing. 1353 + #[derive(Debug, Clone, PartialEq, Eq)] 1354 + pub struct LexError { 1355 + pub message: std::string::String, 1356 + pub pos: SourcePos, 1357 + } 1358 + 1359 + impl fmt::Display for LexError { 1360 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 1361 + write!( 1362 + f, 1363 + "LexError at {}:{}: {}", 1364 + self.pos.line, self.pos.col, self.message 1365 + ) 1366 + } 1367 + } 1368 + 1369 + // ── Tests ──────────────────────────────────────────────────── 1370 + 1371 + #[cfg(test)] 1372 + mod tests { 1373 + use super::*; 1374 + 1375 + fn kinds(src: &str) -> Vec<TokenKind> { 1376 + Lexer::tokenize(src) 1377 + .unwrap() 1378 + .into_iter() 1379 + .map(|t| t.kind) 1380 + .collect() 1381 + } 1382 + 1383 + fn kind(src: &str) -> TokenKind { 1384 + let tokens = Lexer::tokenize(src).unwrap(); 1385 + assert!(tokens.len() >= 2, "expected at least one token + Eof"); 1386 + tokens[0].kind.clone() 1387 + } 1388 + 1389 + // ── Keywords ────────────────────────────────────────── 1390 + 1391 + #[test] 1392 + fn test_keywords() { 1393 + assert_eq!(kind("var"), TokenKind::Var); 1394 + assert_eq!(kind("let"), TokenKind::Let); 1395 + assert_eq!(kind("const"), TokenKind::Const); 1396 + assert_eq!(kind("function"), TokenKind::Function); 1397 + assert_eq!(kind("class"), TokenKind::Class); 1398 + assert_eq!(kind("if"), TokenKind::If); 1399 + assert_eq!(kind("else"), TokenKind::Else); 1400 + assert_eq!(kind("for"), TokenKind::For); 1401 + assert_eq!(kind("while"), TokenKind::While); 1402 + assert_eq!(kind("do"), TokenKind::Do); 1403 + assert_eq!(kind("switch"), TokenKind::Switch); 1404 + assert_eq!(kind("case"), TokenKind::Case); 1405 + assert_eq!(kind("break"), TokenKind::Break); 1406 + assert_eq!(kind("continue"), TokenKind::Continue); 1407 + assert_eq!(kind("return"), TokenKind::Return); 1408 + assert_eq!(kind("throw"), TokenKind::Throw); 1409 + assert_eq!(kind("try"), TokenKind::Try); 1410 + assert_eq!(kind("catch"), TokenKind::Catch); 1411 + assert_eq!(kind("finally"), TokenKind::Finally); 1412 + assert_eq!(kind("new"), TokenKind::New); 1413 + assert_eq!(kind("delete"), TokenKind::Delete); 1414 + assert_eq!(kind("typeof"), TokenKind::Typeof); 1415 + assert_eq!(kind("instanceof"), TokenKind::Instanceof); 1416 + assert_eq!(kind("void"), TokenKind::Void); 1417 + assert_eq!(kind("in"), TokenKind::In); 1418 + assert_eq!(kind("of"), TokenKind::Of); 1419 + assert_eq!(kind("import"), TokenKind::Import); 1420 + assert_eq!(kind("export"), TokenKind::Export); 1421 + assert_eq!(kind("default"), TokenKind::Default); 1422 + assert_eq!(kind("async"), TokenKind::Async); 1423 + assert_eq!(kind("await"), TokenKind::Await); 1424 + assert_eq!(kind("yield"), TokenKind::Yield); 1425 + assert_eq!(kind("this"), TokenKind::This); 1426 + assert_eq!(kind("super"), TokenKind::Super); 1427 + assert_eq!(kind("extends"), TokenKind::Extends); 1428 + assert_eq!(kind("static"), TokenKind::Static); 1429 + assert_eq!(kind("debugger"), TokenKind::Debugger); 1430 + assert_eq!(kind("with"), TokenKind::With); 1431 + } 1432 + 1433 + #[test] 1434 + fn test_literal_keywords() { 1435 + assert_eq!(kind("true"), TokenKind::True); 1436 + assert_eq!(kind("false"), TokenKind::False); 1437 + assert_eq!(kind("null"), TokenKind::Null); 1438 + } 1439 + 1440 + // ── Identifiers ────────────────────────────────────── 1441 + 1442 + #[test] 1443 + fn test_identifiers() { 1444 + assert_eq!(kind("foo"), TokenKind::Identifier("foo".into())); 1445 + assert_eq!(kind("_bar"), TokenKind::Identifier("_bar".into())); 1446 + assert_eq!(kind("$baz"), TokenKind::Identifier("$baz".into())); 1447 + assert_eq!(kind("abc123"), TokenKind::Identifier("abc123".into())); 1448 + assert_eq!(kind("camelCase"), TokenKind::Identifier("camelCase".into())); 1449 + } 1450 + 1451 + #[test] 1452 + fn test_unicode_identifiers() { 1453 + assert_eq!(kind("café"), TokenKind::Identifier("café".into())); 1454 + } 1455 + 1456 + // ── Numbers ────────────────────────────────────────── 1457 + 1458 + #[test] 1459 + fn test_integers() { 1460 + assert_eq!(kind("0"), TokenKind::Number(0.0)); 1461 + assert_eq!(kind("42"), TokenKind::Number(42.0)); 1462 + assert_eq!(kind("123456"), TokenKind::Number(123456.0)); 1463 + } 1464 + 1465 + #[test] 1466 + fn test_floats() { 1467 + assert_eq!(kind("3.14"), TokenKind::Number(3.14)); 1468 + assert_eq!(kind("0.5"), TokenKind::Number(0.5)); 1469 + assert_eq!(kind(".5"), TokenKind::Number(0.5)); 1470 + assert_eq!(kind("1."), TokenKind::Number(1.0)); 1471 + } 1472 + 1473 + #[test] 1474 + fn test_exponents() { 1475 + assert_eq!(kind("1e2"), TokenKind::Number(100.0)); 1476 + assert_eq!(kind("1E2"), TokenKind::Number(100.0)); 1477 + assert_eq!(kind("1e+2"), TokenKind::Number(100.0)); 1478 + assert_eq!(kind("1e-2"), TokenKind::Number(0.01)); 1479 + assert_eq!(kind("2.5e3"), TokenKind::Number(2500.0)); 1480 + } 1481 + 1482 + #[test] 1483 + fn test_hex() { 1484 + assert_eq!(kind("0xFF"), TokenKind::Number(255.0)); 1485 + assert_eq!(kind("0x0"), TokenKind::Number(0.0)); 1486 + assert_eq!(kind("0xDEAD"), TokenKind::Number(0xDEAD as f64)); 1487 + } 1488 + 1489 + #[test] 1490 + fn test_octal() { 1491 + assert_eq!(kind("0o77"), TokenKind::Number(63.0)); 1492 + assert_eq!(kind("0O10"), TokenKind::Number(8.0)); 1493 + } 1494 + 1495 + #[test] 1496 + fn test_binary() { 1497 + assert_eq!(kind("0b1010"), TokenKind::Number(10.0)); 1498 + assert_eq!(kind("0B11"), TokenKind::Number(3.0)); 1499 + } 1500 + 1501 + #[test] 1502 + fn test_numeric_separators() { 1503 + assert_eq!(kind("1_000"), TokenKind::Number(1000.0)); 1504 + assert_eq!(kind("0xFF_FF"), TokenKind::Number(65535.0)); 1505 + assert_eq!(kind("0b1010_0101"), TokenKind::Number(165.0)); 1506 + } 1507 + 1508 + // ── Strings ────────────────────────────────────────── 1509 + 1510 + #[test] 1511 + fn test_double_quoted_string() { 1512 + assert_eq!(kind(r#""hello""#), TokenKind::String("hello".into())); 1513 + } 1514 + 1515 + #[test] 1516 + fn test_single_quoted_string() { 1517 + assert_eq!(kind("'world'"), TokenKind::String("world".into())); 1518 + } 1519 + 1520 + #[test] 1521 + fn test_string_escapes() { 1522 + assert_eq!(kind(r#""\n\t\r""#), TokenKind::String("\n\t\r".into())); 1523 + assert_eq!(kind(r#""\\""#), TokenKind::String("\\".into())); 1524 + assert_eq!(kind(r#""\"""#), TokenKind::String("\"".into())); 1525 + } 1526 + 1527 + #[test] 1528 + fn test_string_hex_escape() { 1529 + assert_eq!(kind(r#""\x41""#), TokenKind::String("A".into())); 1530 + } 1531 + 1532 + #[test] 1533 + fn test_string_unicode_escape() { 1534 + assert_eq!(kind(r#""\u0041""#), TokenKind::String("A".into())); 1535 + assert_eq!( 1536 + kind(r#""\u{1F600}""#), 1537 + TokenKind::String("\u{1F600}".into()) 1538 + ); 1539 + } 1540 + 1541 + #[test] 1542 + fn test_empty_string() { 1543 + assert_eq!(kind(r#""""#), TokenKind::String("".into())); 1544 + assert_eq!(kind("''"), TokenKind::String("".into())); 1545 + } 1546 + 1547 + // ── Template Literals ──────────────────────────────── 1548 + 1549 + #[test] 1550 + fn test_template_no_substitution() { 1551 + assert_eq!(kind("`hello`"), TokenKind::TemplateFull("hello".into())); 1552 + } 1553 + 1554 + #[test] 1555 + fn test_template_with_substitution() { 1556 + let tokens = Lexer::tokenize("`hello ${name}!`").unwrap(); 1557 + let k: Vec<_> = tokens.iter().map(|t| &t.kind).collect(); 1558 + assert_eq!(k[0], &TokenKind::TemplateHead("hello ".into())); 1559 + assert_eq!(k[1], &TokenKind::Identifier("name".into())); 1560 + assert_eq!(k[2], &TokenKind::TemplateTail("!".into())); 1561 + } 1562 + 1563 + #[test] 1564 + fn test_template_multiple_substitutions() { 1565 + let tokens = Lexer::tokenize("`a${1}b${2}c`").unwrap(); 1566 + let k: Vec<_> = tokens.iter().map(|t| &t.kind).collect(); 1567 + assert_eq!(k[0], &TokenKind::TemplateHead("a".into())); 1568 + assert_eq!(k[1], &TokenKind::Number(1.0)); 1569 + assert_eq!(k[2], &TokenKind::TemplateMiddle("b".into())); 1570 + assert_eq!(k[3], &TokenKind::Number(2.0)); 1571 + assert_eq!(k[4], &TokenKind::TemplateTail("c".into())); 1572 + } 1573 + 1574 + #[test] 1575 + fn test_template_with_nested_braces() { 1576 + // `${({a:1})}` — the object literal inside ${ } has its own braces 1577 + let tokens = Lexer::tokenize("`${({a:1})}`").unwrap(); 1578 + let k: Vec<_> = tokens.iter().map(|t| &t.kind).collect(); 1579 + assert_eq!(k[0], &TokenKind::TemplateHead("".into())); 1580 + assert_eq!(k[1], &TokenKind::LParen); 1581 + assert_eq!(k[2], &TokenKind::LBrace); 1582 + assert_eq!(k[3], &TokenKind::Identifier("a".into())); 1583 + assert_eq!(k[4], &TokenKind::Colon); 1584 + assert_eq!(k[5], &TokenKind::Number(1.0)); 1585 + assert_eq!(k[6], &TokenKind::RBrace); 1586 + assert_eq!(k[7], &TokenKind::RParen); 1587 + assert_eq!(k[8], &TokenKind::TemplateTail("".into())); 1588 + } 1589 + 1590 + // ── Regular Expressions ────────────────────────────── 1591 + 1592 + #[test] 1593 + fn test_regexp_basic() { 1594 + let tokens = Lexer::tokenize("x = /foo/gi").unwrap(); 1595 + let k: Vec<_> = tokens.iter().map(|t| &t.kind).collect(); 1596 + assert_eq!( 1597 + k[2], 1598 + &TokenKind::RegExp { 1599 + pattern: "foo".into(), 1600 + flags: "gi".into() 1601 + } 1602 + ); 1603 + } 1604 + 1605 + #[test] 1606 + fn test_regexp_with_class() { 1607 + // /[a-z]/ — the `/` inside the character class is not the end 1608 + let tokens = Lexer::tokenize("x = /[a/b]/").unwrap(); 1609 + let k: Vec<_> = tokens.iter().map(|t| &t.kind).collect(); 1610 + assert_eq!( 1611 + k[2], 1612 + &TokenKind::RegExp { 1613 + pattern: "[a/b]".into(), 1614 + flags: "".into() 1615 + } 1616 + ); 1617 + } 1618 + 1619 + #[test] 1620 + fn test_regexp_vs_division() { 1621 + // After an identifier, `/` is division 1622 + let tokens = Lexer::tokenize("a / b").unwrap(); 1623 + let k: Vec<_> = tokens.iter().map(|t| &t.kind).collect(); 1624 + assert_eq!(k[1], &TokenKind::Slash); 1625 + } 1626 + 1627 + // ── Punctuators ────────────────────────────────────── 1628 + 1629 + #[test] 1630 + fn test_simple_punctuators() { 1631 + assert_eq!(kind("("), TokenKind::LParen); 1632 + assert_eq!(kind(")"), TokenKind::RParen); 1633 + assert_eq!(kind("["), TokenKind::LBracket); 1634 + assert_eq!(kind("]"), TokenKind::RBracket); 1635 + assert_eq!(kind("{"), TokenKind::LBrace); 1636 + assert_eq!(kind("}"), TokenKind::RBrace); 1637 + assert_eq!(kind(";"), TokenKind::Semicolon); 1638 + assert_eq!(kind(","), TokenKind::Comma); 1639 + assert_eq!(kind(":"), TokenKind::Colon); 1640 + assert_eq!(kind("~"), TokenKind::Tilde); 1641 + } 1642 + 1643 + #[test] 1644 + fn test_dot_and_ellipsis() { 1645 + assert_eq!(kind("."), TokenKind::Dot); 1646 + assert_eq!(kind("..."), TokenKind::Ellipsis); 1647 + } 1648 + 1649 + #[test] 1650 + fn test_arrow() { 1651 + assert_eq!(kind("=>"), TokenKind::Arrow); 1652 + } 1653 + 1654 + #[test] 1655 + fn test_optional_chaining() { 1656 + assert_eq!(kind("?."), TokenKind::QuestionDot); 1657 + } 1658 + 1659 + #[test] 1660 + fn test_comparison_operators() { 1661 + assert_eq!(kind("=="), TokenKind::Eq); 1662 + assert_eq!(kind("!="), TokenKind::Ne); 1663 + assert_eq!(kind("==="), TokenKind::StrictEq); 1664 + assert_eq!(kind("!=="), TokenKind::StrictNe); 1665 + assert_eq!(kind("<"), TokenKind::Lt); 1666 + assert_eq!(kind(">"), TokenKind::Gt); 1667 + assert_eq!(kind("<="), TokenKind::Le); 1668 + assert_eq!(kind(">="), TokenKind::Ge); 1669 + } 1670 + 1671 + #[test] 1672 + fn test_arithmetic_operators() { 1673 + assert_eq!(kind("+"), TokenKind::Plus); 1674 + assert_eq!(kind("-"), TokenKind::Minus); 1675 + assert_eq!(kind("*"), TokenKind::Star); 1676 + assert_eq!(kind("%"), TokenKind::Percent); 1677 + assert_eq!(kind("**"), TokenKind::Exp); 1678 + assert_eq!(kind("++"), TokenKind::PlusPlus); 1679 + assert_eq!(kind("--"), TokenKind::MinusMinus); 1680 + } 1681 + 1682 + #[test] 1683 + fn test_bitwise_operators() { 1684 + assert_eq!(kind("&"), TokenKind::Amp); 1685 + assert_eq!(kind("|"), TokenKind::Pipe); 1686 + assert_eq!(kind("^"), TokenKind::Caret); 1687 + assert_eq!(kind("<<"), TokenKind::Shl); 1688 + assert_eq!(kind(">>"), TokenKind::Shr); 1689 + assert_eq!(kind(">>>"), TokenKind::Ushr); 1690 + } 1691 + 1692 + #[test] 1693 + fn test_logical_operators() { 1694 + assert_eq!(kind("&&"), TokenKind::And); 1695 + assert_eq!(kind("||"), TokenKind::Or); 1696 + assert_eq!(kind("!"), TokenKind::Not); 1697 + assert_eq!(kind("??"), TokenKind::Nullish); 1698 + } 1699 + 1700 + #[test] 1701 + fn test_assignment_operators() { 1702 + assert_eq!(kind("="), TokenKind::Assign); 1703 + assert_eq!(kind("+="), TokenKind::PlusAssign); 1704 + assert_eq!(kind("-="), TokenKind::MinusAssign); 1705 + assert_eq!(kind("*="), TokenKind::StarAssign); 1706 + assert_eq!(kind("%="), TokenKind::PercentAssign); 1707 + assert_eq!(kind("**="), TokenKind::ExpAssign); 1708 + assert_eq!(kind("&="), TokenKind::AmpAssign); 1709 + assert_eq!(kind("|="), TokenKind::PipeAssign); 1710 + assert_eq!(kind("^="), TokenKind::CaretAssign); 1711 + assert_eq!(kind("<<="), TokenKind::ShlAssign); 1712 + assert_eq!(kind(">>="), TokenKind::ShrAssign); 1713 + assert_eq!(kind(">>>="), TokenKind::UshrAssign); 1714 + assert_eq!(kind("&&="), TokenKind::AndAssign); 1715 + assert_eq!(kind("||="), TokenKind::OrAssign); 1716 + assert_eq!(kind("??="), TokenKind::NullishAssign); 1717 + } 1718 + 1719 + // ── Comments ───────────────────────────────────────── 1720 + 1721 + #[test] 1722 + fn test_single_line_comment() { 1723 + let tokens = kinds("a // comment\nb"); 1724 + assert_eq!(tokens.len(), 3); // a, b, Eof 1725 + assert_eq!(tokens[0], TokenKind::Identifier("a".into())); 1726 + assert_eq!(tokens[1], TokenKind::Identifier("b".into())); 1727 + } 1728 + 1729 + #[test] 1730 + fn test_multi_line_comment() { 1731 + let tokens = kinds("a /* comment */ b"); 1732 + assert_eq!(tokens.len(), 3); 1733 + assert_eq!(tokens[0], TokenKind::Identifier("a".into())); 1734 + assert_eq!(tokens[1], TokenKind::Identifier("b".into())); 1735 + } 1736 + 1737 + // ── Source positions ───────────────────────────────── 1738 + 1739 + #[test] 1740 + fn test_source_positions() { 1741 + let tokens = Lexer::tokenize("let x = 42").unwrap(); 1742 + // `let` at line 1, col 1 1743 + assert_eq!(tokens[0].span.start, SourcePos { line: 1, col: 1 }); 1744 + // `x` at line 1, col 5 1745 + assert_eq!(tokens[1].span.start, SourcePos { line: 1, col: 5 }); 1746 + // `=` at line 1, col 7 1747 + assert_eq!(tokens[2].span.start, SourcePos { line: 1, col: 7 }); 1748 + // `42` at line 1, col 9 1749 + assert_eq!(tokens[3].span.start, SourcePos { line: 1, col: 9 }); 1750 + } 1751 + 1752 + #[test] 1753 + fn test_multiline_positions() { 1754 + let tokens = Lexer::tokenize("a\nb\nc").unwrap(); 1755 + assert_eq!(tokens[0].span.start, SourcePos { line: 1, col: 1 }); 1756 + assert_eq!(tokens[1].span.start, SourcePos { line: 2, col: 1 }); 1757 + assert_eq!(tokens[2].span.start, SourcePos { line: 3, col: 1 }); 1758 + } 1759 + 1760 + // ── Newline tracking (ASI) ─────────────────────────── 1761 + 1762 + #[test] 1763 + fn test_preceded_by_newline() { 1764 + let tokens = Lexer::tokenize("a\nb").unwrap(); 1765 + assert!(!tokens[0].preceded_by_newline); // `a` 1766 + assert!(tokens[1].preceded_by_newline); // `b` 1767 + } 1768 + 1769 + // ── Error cases ────────────────────────────────────── 1770 + 1771 + #[test] 1772 + fn test_unterminated_string() { 1773 + assert!(Lexer::tokenize("\"hello").is_err()); 1774 + } 1775 + 1776 + #[test] 1777 + fn test_unterminated_block_comment() { 1778 + assert!(Lexer::tokenize("/* oops").is_err()); 1779 + } 1780 + 1781 + #[test] 1782 + fn test_unterminated_template() { 1783 + assert!(Lexer::tokenize("`hello").is_err()); 1784 + } 1785 + 1786 + #[test] 1787 + fn test_bad_hex_literal() { 1788 + assert!(Lexer::tokenize("0x").is_err()); 1789 + } 1790 + 1791 + // ── Full statement tokenization ────────────────────── 1792 + 1793 + #[test] 1794 + fn test_full_statement() { 1795 + let tokens = kinds("const x = 42 + y;"); 1796 + assert_eq!( 1797 + tokens, 1798 + vec![ 1799 + TokenKind::Const, 1800 + TokenKind::Identifier("x".into()), 1801 + TokenKind::Assign, 1802 + TokenKind::Number(42.0), 1803 + TokenKind::Plus, 1804 + TokenKind::Identifier("y".into()), 1805 + TokenKind::Semicolon, 1806 + TokenKind::Eof, 1807 + ] 1808 + ); 1809 + } 1810 + 1811 + #[test] 1812 + fn test_arrow_function() { 1813 + let tokens = kinds("(x) => x + 1"); 1814 + assert_eq!( 1815 + tokens, 1816 + vec![ 1817 + TokenKind::LParen, 1818 + TokenKind::Identifier("x".into()), 1819 + TokenKind::RParen, 1820 + TokenKind::Arrow, 1821 + TokenKind::Identifier("x".into()), 1822 + TokenKind::Plus, 1823 + TokenKind::Number(1.0), 1824 + TokenKind::Eof, 1825 + ] 1826 + ); 1827 + } 1828 + 1829 + #[test] 1830 + fn test_complex_expression() { 1831 + let tokens = kinds("a?.b ?? c !== d"); 1832 + assert_eq!( 1833 + tokens, 1834 + vec![ 1835 + TokenKind::Identifier("a".into()), 1836 + TokenKind::QuestionDot, 1837 + TokenKind::Identifier("b".into()), 1838 + TokenKind::Nullish, 1839 + TokenKind::Identifier("c".into()), 1840 + TokenKind::StrictNe, 1841 + TokenKind::Identifier("d".into()), 1842 + TokenKind::Eof, 1843 + ] 1844 + ); 1845 + } 1846 + 1847 + #[test] 1848 + fn test_division_after_paren() { 1849 + // `(a) / b` — the `/` after `)` should be division, not regexp 1850 + let tokens = kinds("(a) / b"); 1851 + assert_eq!( 1852 + tokens, 1853 + vec![ 1854 + TokenKind::LParen, 1855 + TokenKind::Identifier("a".into()), 1856 + TokenKind::RParen, 1857 + TokenKind::Slash, 1858 + TokenKind::Identifier("b".into()), 1859 + TokenKind::Eof, 1860 + ] 1861 + ); 1862 + } 1863 + 1864 + #[test] 1865 + fn test_slash_assign() { 1866 + let tokens = kinds("a /= b"); 1867 + assert_eq!( 1868 + tokens, 1869 + vec![ 1870 + TokenKind::Identifier("a".into()), 1871 + TokenKind::SlashAssign, 1872 + TokenKind::Identifier("b".into()), 1873 + TokenKind::Eof, 1874 + ] 1875 + ); 1876 + } 1877 + 1878 + #[test] 1879 + fn test_regexp_after_assign() { 1880 + let tokens = kinds("x = /test/g"); 1881 + assert_eq!( 1882 + tokens, 1883 + vec![ 1884 + TokenKind::Identifier("x".into()), 1885 + TokenKind::Assign, 1886 + TokenKind::RegExp { 1887 + pattern: "test".into(), 1888 + flags: "g".into() 1889 + }, 1890 + TokenKind::Eof, 1891 + ] 1892 + ); 1893 + } 1894 + }

crates/js/src/lib.rs

··· 1 1 //! JavaScript engine — lexer, parser, bytecode, register VM, GC, JIT (AArch64). 2 2 3 + pub mod lexer; 4 + 3 5 use std::fmt; 4 6 5 7 /// An error produced by the JavaScript engine.

Configure Feed

Configure Feed