Embedded programming language for Zig
Initial Commit
Lexer written
+462
+1
.envrc
+1
.envrc
···
1
+
use flake
+65
Cargo.lock
+65
Cargo.lock
···
1
+
# This file is automatically @generated by Cargo.
2
+
# It is not intended for manual editing.
3
+
version = 4
4
+
5
+
[[package]]
6
+
name = "proc-macro2"
7
+
version = "1.0.101"
8
+
source = "registry+https://github.com/rust-lang/crates.io-index"
9
+
checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"
10
+
dependencies = [
11
+
"unicode-ident",
12
+
]
13
+
14
+
[[package]]
15
+
name = "quote"
16
+
version = "1.0.41"
17
+
source = "registry+https://github.com/rust-lang/crates.io-index"
18
+
checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1"
19
+
dependencies = [
20
+
"proc-macro2",
21
+
]
22
+
23
+
[[package]]
24
+
name = "syn"
25
+
version = "2.0.106"
26
+
source = "registry+https://github.com/rust-lang/crates.io-index"
27
+
checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6"
28
+
dependencies = [
29
+
"proc-macro2",
30
+
"quote",
31
+
"unicode-ident",
32
+
]
33
+
34
+
[[package]]
35
+
name = "thiserror"
36
+
version = "2.0.17"
37
+
source = "registry+https://github.com/rust-lang/crates.io-index"
38
+
checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
39
+
dependencies = [
40
+
"thiserror-impl",
41
+
]
42
+
43
+
[[package]]
44
+
name = "thiserror-impl"
45
+
version = "2.0.17"
46
+
source = "registry+https://github.com/rust-lang/crates.io-index"
47
+
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
48
+
dependencies = [
49
+
"proc-macro2",
50
+
"quote",
51
+
"syn",
52
+
]
53
+
54
+
[[package]]
55
+
name = "unicode-ident"
56
+
version = "1.0.19"
57
+
source = "registry+https://github.com/rust-lang/crates.io-index"
58
+
checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d"
59
+
60
+
[[package]]
61
+
name = "zeta"
62
+
version = "0.1.0"
63
+
dependencies = [
64
+
"thiserror",
65
+
]
+7
Cargo.toml
+7
Cargo.toml
+85
flake.lock
+85
flake.lock
···
1
+
{
2
+
"nodes": {
3
+
"crane": {
4
+
"locked": {
5
+
"lastModified": 1759893430,
6
+
"narHash": "sha256-yAy4otLYm9iZ+NtQwTMEbqHwswSFUbhn7x826RR6djw=",
7
+
"owner": "ipetkov",
8
+
"repo": "crane",
9
+
"rev": "1979a2524cb8c801520bd94c38bb3d5692419d93",
10
+
"type": "github"
11
+
},
12
+
"original": {
13
+
"owner": "ipetkov",
14
+
"repo": "crane",
15
+
"type": "github"
16
+
}
17
+
},
18
+
"flake-parts": {
19
+
"inputs": {
20
+
"nixpkgs-lib": [
21
+
"nixpkgs"
22
+
]
23
+
},
24
+
"locked": {
25
+
"lastModified": 1759362264,
26
+
"narHash": "sha256-wfG0S7pltlYyZTM+qqlhJ7GMw2fTF4mLKCIVhLii/4M=",
27
+
"owner": "hercules-ci",
28
+
"repo": "flake-parts",
29
+
"rev": "758cf7296bee11f1706a574c77d072b8a7baa881",
30
+
"type": "github"
31
+
},
32
+
"original": {
33
+
"owner": "hercules-ci",
34
+
"repo": "flake-parts",
35
+
"type": "github"
36
+
}
37
+
},
38
+
"nixpkgs": {
39
+
"locked": {
40
+
"lastModified": 1760164275,
41
+
"narHash": "sha256-gKl2Gtro/LNf8P+4L3S2RsZ0G390ccd5MyXYrTdMCFE=",
42
+
"owner": "nixos",
43
+
"repo": "nixpkgs",
44
+
"rev": "362791944032cb532aabbeed7887a441496d5e6e",
45
+
"type": "github"
46
+
},
47
+
"original": {
48
+
"owner": "nixos",
49
+
"ref": "nixpkgs-unstable",
50
+
"repo": "nixpkgs",
51
+
"type": "github"
52
+
}
53
+
},
54
+
"root": {
55
+
"inputs": {
56
+
"crane": "crane",
57
+
"flake-parts": "flake-parts",
58
+
"nixpkgs": "nixpkgs",
59
+
"rust-overlay": "rust-overlay"
60
+
}
61
+
},
62
+
"rust-overlay": {
63
+
"inputs": {
64
+
"nixpkgs": [
65
+
"nixpkgs"
66
+
]
67
+
},
68
+
"locked": {
69
+
"lastModified": 1760323082,
70
+
"narHash": "sha256-SKhC9tyt+gVgQHnZGMVPSdptlDYNqApT56JF5t8RwBY=",
71
+
"owner": "oxalica",
72
+
"repo": "rust-overlay",
73
+
"rev": "c73e6874fe8dce0bab82c0387b510875f1eff9f8",
74
+
"type": "github"
75
+
},
76
+
"original": {
77
+
"owner": "oxalica",
78
+
"repo": "rust-overlay",
79
+
"type": "github"
80
+
}
81
+
}
82
+
},
83
+
"root": "root",
84
+
"version": 7
85
+
}
+61
flake.nix
+61
flake.nix
···
1
+
{
2
+
description = "Rust template";
3
+
4
+
inputs = {
5
+
nixpkgs.url = "github:nixos/nixpkgs?ref=nixpkgs-unstable";
6
+
flake-parts = {
7
+
url = "github:hercules-ci/flake-parts";
8
+
inputs.nixpkgs-lib.follows = "nixpkgs";
9
+
};
10
+
crane.url = "github:ipetkov/crane";
11
+
rust-overlay = {
12
+
url = "github:oxalica/rust-overlay";
13
+
inputs.nixpkgs.follows = "nixpkgs";
14
+
};
15
+
};
16
+
17
+
outputs = {
18
+
self,
19
+
nixpkgs,
20
+
crane,
21
+
rust-overlay,
22
+
flake-parts,
23
+
...
24
+
} @ inputs:
25
+
flake-parts.lib.mkFlake {inherit inputs;} {
26
+
imports = [];
27
+
28
+
systems = [
29
+
"x86_64-linux"
30
+
];
31
+
32
+
perSystem = {
33
+
# pkgs,
34
+
system,
35
+
...
36
+
}: let
37
+
overlays = [(import rust-overlay)];
38
+
pkgs = import nixpkgs {
39
+
inherit system overlays;
40
+
};
41
+
craneLib = (crane.mkLib pkgs).overrideToolchain (
42
+
p: p.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml
43
+
);
44
+
45
+
nativeBuildInputs = [];
46
+
buildInputs = [];
47
+
in {
48
+
formatter = pkgs.alejandra;
49
+
50
+
packages.default = craneLib.buildPackage {
51
+
src = craneLib.cleanCargoSource ./.;
52
+
53
+
inherit buildInputs nativeBuildInputs;
54
+
};
55
+
56
+
devShells.default = craneLib.devShell {
57
+
packages = buildInputs ++ nativeBuildInputs;
58
+
};
59
+
};
60
+
};
61
+
}
+3
rust-toolchain.toml
+3
rust-toolchain.toml
+225
src/lexer.rs
+225
src/lexer.rs
···
1
+
#[derive(Debug)]
2
+
pub enum TokenValue {
3
+
LParen,
4
+
RParen,
5
+
Quote,
6
+
Atom,
7
+
Num(String),
8
+
Sym(String),
9
+
Str(String),
10
+
}
11
+
12
+
#[derive(Debug)]
13
+
pub struct Token {
14
+
line: LineInfo,
15
+
value: TokenValue,
16
+
}
17
+
18
+
#[derive(Clone, Copy, Debug)]
19
+
pub struct LineInfo(usize, usize);
20
+
21
+
impl LineInfo {
22
+
pub fn line(self) -> usize {
23
+
self.0
24
+
}
25
+
26
+
pub fn column(self) -> usize {
27
+
self.1
28
+
}
29
+
}
30
+
31
+
impl std::fmt::Display for LineInfo {
32
+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
33
+
write!(f, "{}:{}", self.0 + 1, self.1)
34
+
}
35
+
}
36
+
37
+
#[derive(Debug, thiserror::Error)]
38
+
pub enum LexError {
39
+
#[error("unexpected token near {0}: `{1}`")]
40
+
UnexpectedToken(LineInfo, char),
41
+
42
+
#[error("failed to parse float near {0}: too much decimal points")]
43
+
ExtraDecimalPoint(LineInfo),
44
+
45
+
#[error("invalid number near {0}")]
46
+
InvalidNumber(LineInfo),
47
+
48
+
#[error("invalid symbol near {0}, invalid character: {1}")]
49
+
InvalidSymbol(LineInfo, char),
50
+
51
+
#[error("invalid escape character near {0}: `{1}`")]
52
+
InvalidEscape(LineInfo, char),
53
+
54
+
#[error("string broken with newline near {0}")]
55
+
StringBreak(LineInfo)
56
+
}
57
+
58
+
pub mod chars {
59
+
pub const LPAREN: char = '(';
60
+
pub const RPAREN: char = ')';
61
+
pub const QUOTE: char = '\'';
62
+
pub const ATOM: char = ':';
63
+
pub const STR_DELIM: char = '"';
64
+
}
65
+
66
+
macro_rules! lex_pat {
67
+
(ws) => {
68
+
' '|'\n'|'\r'
69
+
};
70
+
(schar) => {
71
+
chars::LPAREN|chars::RPAREN|chars::QUOTE|chars::ATOM
72
+
};
73
+
(symbol) => {
74
+
'a'..='z'|'A'..='Z'|'-'|'_'|'+'|'='|'*'|'#'|'!'|'$'|'%'
75
+
};
76
+
(symbol_rest) => {
77
+
lex_pat!(symbol) | lex_pat!(num)
78
+
};
79
+
(num) => {
80
+
'0'..='9'
81
+
};
82
+
}
83
+
84
+
pub fn lex(buf: &str) -> Result<Vec<Token>, LexError> {
85
+
let mut tokens = vec![];
86
+
let mut chars = buf.chars().peekable();
87
+
let mut line = LineInfo(0, 0);
88
+
89
+
while let Some(c) = chars.next() {
90
+
match c {
91
+
chars::LPAREN => tokens.push(Token {
92
+
line,
93
+
value: TokenValue::LParen,
94
+
}),
95
+
chars::RPAREN => tokens.push(Token {
96
+
line,
97
+
value: TokenValue::RParen,
98
+
}),
99
+
chars::QUOTE => tokens.push(Token {
100
+
line,
101
+
value: TokenValue::Quote,
102
+
}),
103
+
chars::ATOM => tokens.push(Token {
104
+
line,
105
+
value: TokenValue::Atom,
106
+
}),
107
+
108
+
lex_pat!(num) => {
109
+
let mut buf = String::from(c);
110
+
let mut has_decimal = false;
111
+
112
+
while let Some(c) = chars.peek() {
113
+
if matches!(c, lex_pat!(num)) {
114
+
buf.push(chars.next().unwrap());
115
+
} else if *c == '.' {
116
+
if has_decimal {
117
+
return Err(LexError::ExtraDecimalPoint(line));
118
+
}
119
+
has_decimal = true;
120
+
buf.push(chars.next().unwrap());
121
+
} else {
122
+
break;
123
+
}
124
+
}
125
+
126
+
tokens.push(Token {
127
+
line,
128
+
value: TokenValue::Num(buf),
129
+
});
130
+
}
131
+
lex_pat!(symbol) => {
132
+
let mut buf = String::from(c);
133
+
134
+
while let Some(c) = chars.peek() {
135
+
if matches!(c, lex_pat!(symbol_rest)) {
136
+
buf.push(chars.next().unwrap());
137
+
} else if matches!(*c, lex_pat!(ws) | lex_pat!(schar)) {
138
+
break;
139
+
} else {
140
+
return Err(LexError::InvalidSymbol(line, *c));
141
+
}
142
+
}
143
+
144
+
tokens.push(Token {
145
+
line,
146
+
value: TokenValue::Sym(buf),
147
+
});
148
+
}
149
+
lex_pat!(ws) => {
150
+
if c == '\n' {
151
+
line.0 += 1;
152
+
line.1 = 0;
153
+
} else {
154
+
line.1 += 1;
155
+
}
156
+
}
157
+
chars::STR_DELIM => {
158
+
let mut buf = String::new();
159
+
let mut in_escape = false;
160
+
161
+
while let Some(c) = chars.peek() {
162
+
if *c == '\n' {
163
+
return Err(LexError::StringBreak(line));
164
+
}
165
+
166
+
if *c == chars::STR_DELIM && !in_escape {
167
+
chars.next().unwrap();
168
+
break;
169
+
}
170
+
171
+
if in_escape {
172
+
let c = chars.next().unwrap();
173
+
174
+
match c {
175
+
'\\' => buf.push('\\'),
176
+
'n' => buf.push('\n'),
177
+
'r' => buf.push('\r'),
178
+
't' => buf.push('\t'),
179
+
'a' => buf.push('\x07'),
180
+
'b' => buf.push('\x08'),
181
+
chars::STR_DELIM => buf.push(chars::STR_DELIM),
182
+
_ => return Err(LexError::InvalidEscape(line, c)),
183
+
}
184
+
185
+
in_escape = false;
186
+
} else if *c == '\\' {
187
+
in_escape = true;
188
+
chars.next().unwrap();
189
+
} else {
190
+
buf.push(chars.next().unwrap());
191
+
}
192
+
}
193
+
194
+
tokens.push(Token {
195
+
line,
196
+
value: TokenValue::Str(buf),
197
+
});
198
+
}
199
+
_ => {
200
+
return Err(LexError::UnexpectedToken(line, c));
201
+
}
202
+
}
203
+
}
204
+
205
+
Ok(tokens)
206
+
}
207
+
208
+
#[cfg(test)]
209
+
mod tests {
210
+
use super::*;
211
+
212
+
#[test]
213
+
fn lexer() -> Result<(), LexError> {
214
+
let code = r#" (symbol 3.14 'quoted_symbol :atomized_symbol32 "my string") "#;
215
+
let tokens = lex(code)?;
216
+
217
+
for t in tokens {
218
+
eprintln!("{:?}", t);
219
+
}
220
+
221
+
Ok(())
222
+
}
223
+
}
224
+
225
+
// TODO: write the parser :3