we (web engine): Experimental web browser project to understand the limits of Claude
Merge js-lexer: fix line continuation escape in string and template literals
+38
-23
+38
-23
crates/js/src/lexer.rs
+38
-23
crates/js/src/lexer.rs
···
673
673
}
674
674
Some(b'\\') => {
675
675
self.advance(); // backslash
676
-
let ch = self.scan_escape_sequence()?;
677
-
value.push(ch);
676
+
if let Some(ch) = self.scan_escape_sequence()? {
677
+
value.push(ch);
678
+
}
678
679
}
679
680
Some(_) => {
680
681
let ch = self.advance_char();
···
686
687
Ok(TokenKind::String(value))
687
688
}
688
689
689
-
fn scan_escape_sequence(&mut self) -> Result<char, LexError> {
690
+
/// Scan an escape sequence after the backslash has been consumed.
691
+
/// Returns `None` for line continuations (`\<newline>`), which produce no character.
692
+
fn scan_escape_sequence(&mut self) -> Result<Option<char>, LexError> {
690
693
let pos = self.current_pos();
691
694
match self.advance() {
692
-
Some(b'n') => Ok('\n'),
693
-
Some(b'r') => Ok('\r'),
694
-
Some(b't') => Ok('\t'),
695
-
Some(b'b') => Ok('\u{0008}'),
696
-
Some(b'f') => Ok('\u{000C}'),
697
-
Some(b'v') => Ok('\u{000B}'),
698
-
Some(b'0') if !matches!(self.peek(), Some(b'0'..=b'9')) => Ok('\0'),
699
-
Some(b'\\') => Ok('\\'),
700
-
Some(b'\'') => Ok('\''),
701
-
Some(b'"') => Ok('"'),
702
-
Some(b'`') => Ok('`'),
703
-
Some(b'\n') => Ok('\n'),
695
+
Some(b'n') => Ok(Some('\n')),
696
+
Some(b'r') => Ok(Some('\r')),
697
+
Some(b't') => Ok(Some('\t')),
698
+
Some(b'b') => Ok(Some('\u{0008}')),
699
+
Some(b'f') => Ok(Some('\u{000C}')),
700
+
Some(b'v') => Ok(Some('\u{000B}')),
701
+
Some(b'0') if !matches!(self.peek(), Some(b'0'..=b'9')) => Ok(Some('\0')),
702
+
Some(b'\\') => Ok(Some('\\')),
703
+
Some(b'\'') => Ok(Some('\'')),
704
+
Some(b'"') => Ok(Some('"')),
705
+
Some(b'`') => Ok(Some('`')),
706
+
// Line continuation: \<newline> produces no character
707
+
Some(b'\n') => Ok(None),
704
708
Some(b'\r') => {
705
709
self.advance_if(b'\n');
706
-
Ok('\n')
710
+
Ok(None)
707
711
}
708
712
Some(b'x') => {
709
713
let hi = self.advance().and_then(hex_digit_val).ok_or(LexError {
···
715
719
pos,
716
720
})?;
717
721
let code = (hi << 4) | lo;
718
-
Ok(code as char)
722
+
Ok(Some(code as char))
719
723
}
720
-
Some(b'u') => self.scan_unicode_escape(pos),
724
+
Some(b'u') => self.scan_unicode_escape(pos).map(Some),
721
725
Some(b) => {
722
726
// identity escape
723
-
Ok(b as char)
727
+
Ok(Some(b as char))
724
728
}
725
729
None => Err(LexError {
726
730
message: "unexpected end of input in escape sequence".into(),
···
839
843
}
840
844
Some(b'\\') => {
841
845
self.advance();
842
-
let ch = self.scan_escape_sequence()?;
843
-
value.push(ch);
846
+
if let Some(ch) = self.scan_escape_sequence()? {
847
+
value.push(ch);
848
+
}
844
849
}
845
850
Some(_) => {
846
851
let ch = self.advance_char();
···
887
892
}
888
893
Some(b'\\') => {
889
894
self.advance();
890
-
let ch = self.scan_escape_sequence()?;
891
-
value.push(ch);
895
+
if let Some(ch) = self.scan_escape_sequence()? {
896
+
value.push(ch);
897
+
}
892
898
}
893
899
Some(_) => {
894
900
let ch = self.advance_char();
···
1535
1541
assert_eq!(
1536
1542
kind(r#""\u{1F600}""#),
1537
1543
TokenKind::String("\u{1F600}".into())
1544
+
);
1545
+
}
1546
+
1547
+
#[test]
1548
+
fn test_string_line_continuation() {
1549
+
// \<newline> is a line continuation producing no character
1550
+
assert_eq!(
1551
+
kind("\"line1\\\nline2\""),
1552
+
TokenKind::String("line1line2".into())
1538
1553
);
1539
1554
}
1540
1555