Compiler experimentation.
Add basic lexer
+108
-2
+93
src/lex.rs
+93
src/lex.rs
···
1
+
use std::{iter::Peekable, str::CharIndices};
2
+
3
+
#[derive(Debug, Clone, Copy, PartialEq)]
4
+
pub enum Token<'a> {
5
+
Ident(&'a str),
6
+
Num(u64),
7
+
Colon,
8
+
Equals,
9
+
}
10
+
11
+
struct Lexer<'a> {
12
+
code: &'a str,
13
+
iter: Peekable<CharIndices<'a>>,
14
+
}
15
+
16
+
impl<'a> Lexer<'a> {
17
+
fn new(code: &'a str) -> Self {
18
+
Self {
19
+
code,
20
+
iter: code.char_indices().peekable(),
21
+
}
22
+
}
23
+
24
+
fn ident(&mut self, i: usize) -> Token<'a> {
25
+
let start = i;
26
+
let mut end = start;
27
+
while let Some((j, x)) = self.iter.peek() {
28
+
if !x.is_alphanumeric() {
29
+
end = *j;
30
+
break;
31
+
}
32
+
self.iter.next();
33
+
}
34
+
Token::Ident(&self.code[start..end])
35
+
}
36
+
37
+
fn num(&mut self, i: usize) -> Token<'a> {
38
+
let start = i;
39
+
let mut end = start;
40
+
while let Some((j, x)) = self.iter.peek() {
41
+
if !x.is_ascii_digit() {
42
+
end = *j;
43
+
break;
44
+
}
45
+
self.iter.next();
46
+
}
47
+
let num: u64 = self.code[start..end]
48
+
.parse()
49
+
.expect("failed to parse number");
50
+
Token::Num(num)
51
+
}
52
+
}
53
+
54
+
impl<'a> Iterator for Lexer<'a> {
55
+
type Item = Token<'a>;
56
+
57
+
fn next(&mut self) -> Option<Self::Item> {
58
+
use Token::*;
59
+
60
+
loop {
61
+
match self.iter.peek().cloned() {
62
+
Some((_, ':')) => {
63
+
self.iter.next();
64
+
break Some(Colon);
65
+
}
66
+
Some((_, '=')) => {
67
+
self.iter.next();
68
+
break Some(Equals);
69
+
}
70
+
Some((i, a)) if a.is_alphabetic() => {
71
+
self.iter.next();
72
+
break Some(self.ident(i));
73
+
}
74
+
Some((i, a)) if a.is_ascii_digit() => {
75
+
self.iter.next();
76
+
break Some(self.num(i));
77
+
}
78
+
Some((_, w)) if w.is_whitespace() => {
79
+
self.iter.next();
80
+
}
81
+
Some((i, t)) => {
82
+
panic!("unexpected token {t:?} at position {i}");
83
+
}
84
+
None => break None,
85
+
}
86
+
}
87
+
}
88
+
}
89
+
90
+
/// Run the lexer on some source code, producing an iterator of tokens.
91
+
pub fn lex<'a>(code: &'a str) -> impl Iterator<Item = Token<'a>> {
92
+
Lexer::new(code)
93
+
}
+15
-2
src/lib.rs
+15
-2
src/lib.rs
···
1
+
use crate::lex::Token;
2
+
mod lex;
3
+
1
4
/// Interpret a program, returning the result.
2
-
pub fn interpret(_input: &str) -> i32 {
3
-
0
5
+
pub fn interpret(input: &str) -> u64 {
6
+
use Token::*;
7
+
let mut tokens = lex::lex(input);
8
+
assert_eq!(Some(Ident("main")), tokens.next());
9
+
assert_eq!(Some(Colon), tokens.next());
10
+
assert_eq!(Some(Ident("I32")), tokens.next());
11
+
assert_eq!(Some(Ident("main")), tokens.next());
12
+
assert_eq!(Some(Equals), tokens.next());
13
+
let Some(Num(out)) = tokens.next() else {
14
+
panic!("expected main = <NUMBER>");
15
+
};
16
+
out
4
17
}