Embedded programming language for Zig
Some progress on the lexer
+208
-1
+203
src/lexer.zig
+203
src/lexer.zig
···
1
+
const std = @import("std");
2
+
const Allocator = std.mem.Allocator;
3
+
4
+
const mitochondria = @import("mitochondria");
5
+
const SliceIterator = mitochondria.SliceIterator;
6
+
const StringBuilder = mitochondria.StringBuilder;
7
+
8
+
pub const TokenValue = union(enum) {
9
+
lparen,
10
+
rparen,
11
+
quote,
12
+
num: []const u8,
13
+
sym: []const u8,
14
+
str: []const u8,
15
+
16
+
pub fn deinit(self: *@This(), allocator: Allocator) void {
17
+
switch (self.*) {
18
+
.str => |str| {
19
+
allocator.free(str);
20
+
},
21
+
else => {},
22
+
}
23
+
}
24
+
};
25
+
26
+
pub const LineInfo = struct {
27
+
file: ?[]const u8 = null,
28
+
line: usize,
29
+
column: usize,
30
+
31
+
pub fn format(self: *const LineInfo, writer: *std.Io.Writer) !void {
32
+
if (self.file) |p| {
33
+
try writer.print("{s}:", .{p});
34
+
}
35
+
try writer.print("{}:{}", .{ self.line, self.column });
36
+
}
37
+
};
38
+
39
+
test "line_info" {
40
+
std.debug.print("at: {f}\n", .{LineInfo{ .file = "src/lexer.zig", .line = 3, .column = 6 }});
41
+
}
42
+
43
+
pub const Token = struct {
44
+
line: LineInfo,
45
+
value: TokenValue,
46
+
};
47
+
48
+
pub const LexResult = union(enum) {
49
+
pub const Exception = union(enum) {
50
+
unexpected_token: struct { LineInfo, u8 },
51
+
extra_decimal_point: struct { LineInfo },
52
+
invalid_number: struct { LineInfo },
53
+
invalid_symbol_char: struct { LineInfo, u8 },
54
+
invalid_escape: struct { LineInfo, u8 },
55
+
string_break: struct { LineInfo },
56
+
57
+
pub fn format(self: *const @This(), writer: *std.Io.Writer) !void {
58
+
_ = self;
59
+
_ = writer;
60
+
}
61
+
};
62
+
63
+
exception: Exception,
64
+
success: struct { allocator: Allocator, tokens: []Token },
65
+
66
+
pub fn deinit(self: *@This()) void {
67
+
switch (self.*) {
68
+
.success => |s| {
69
+
for (s.tokens) |tok| {
70
+
tok.value.deinit(s.allocator);
71
+
}
72
+
73
+
s.allocator.free(s.tokens);
74
+
},
75
+
else => {},
76
+
}
77
+
}
78
+
};
79
+
80
+
fn exception(c: LexResult.Exception) LexResult {
81
+
return LexResult{ .exception = c };
82
+
}
83
+
84
+
pub const CHARS = struct {
85
+
pub const LPAREN: u8 = '(';
86
+
pub const RPAREN: u8 = ')';
87
+
pub const QUOTE: u8 = '\'';
88
+
pub const STRDELIM: u8 = '"';
89
+
pub const COMMENT: u8 = ';';
90
+
};
91
+
92
+
fn sym_compat(c: u8) bool {
93
+
return switch (c) {
94
+
// zig fmt: off
95
+
'a'...'z', 'A'...'Z',
96
+
'!', '@', '#', '$', '%', '^', '&', '*', '_', '-', '+', '=', '[', '{',
97
+
']', '}', '|', '~', '`', ',', '.', '<', '>', '/', '?', ':',
98
+
=> true,
99
+
// zig fmt: on
100
+
else => false,
101
+
};
102
+
}
103
+
104
+
fn num_compat(c: u8) bool {
105
+
return switch (c) {
106
+
'0'...'9' => true,
107
+
else => false,
108
+
};
109
+
}
110
+
111
+
pub fn lex(allocator: Allocator, buf: []const u8) !LexResult {
112
+
var tokens = try std.ArrayList(Token).initCapacity(allocator, 1000);
113
+
defer tokens.deinit(allocator);
114
+
var iter = SliceIterator(u8).init(buf);
115
+
var line = LineInfo{ .line = 1, .column = 0 };
116
+
117
+
while (iter.next()) |c| : (line.column += 1) {
118
+
switch (c) {
119
+
CHARS.LPAREN => try tokens.append(Token{
120
+
.line = line,
121
+
.value = .lparen,
122
+
}),
123
+
CHARS.RPAREN => try tokens.append(Token{
124
+
.line = line,
125
+
.value = .rparen,
126
+
}),
127
+
CHARS.QUOTE => try tokens.append(Token{
128
+
.line = line,
129
+
.value = .quote,
130
+
}),
131
+
CHARS.COMMENT => {
132
+
while (iter.next()) |nc| {
133
+
if (nc == '\n') {
134
+
line.line += 1;
135
+
line.column = 0;
136
+
break;
137
+
}
138
+
}
139
+
},
140
+
CHARS.STRDELIM => {
141
+
var sb = try StringBuilder.init(allocator, 15);
142
+
defer sb.deinit();
143
+
144
+
var in_escape = false;
145
+
146
+
while (iter.peek()) |nc| : (line.column += 1) {
147
+
if (nc == '\n') {
148
+
return exception(.{ .string_break = struct { line } });
149
+
}
150
+
151
+
if (nc == CHARS.STRDELIM and !in_escape) {
152
+
_ = iter.next();
153
+
break;
154
+
}
155
+
156
+
if (in_escape) {
157
+
switch (nc) {
158
+
'\\' => try sb.append("\\"),
159
+
'n' => try sb.append("\n"),
160
+
'r' => try sb.append("\r"),
161
+
't' => try sb.append("\t"),
162
+
'a' => try sb.append("\x07"),
163
+
'b' => try sb.append("\x08"),
164
+
CHARS.STRDELIM => try sb.append("\""),
165
+
else => {},
166
+
}
167
+
168
+
_ = iter.next();
169
+
in_escape = false;
170
+
} else if (nc == '\\') {
171
+
in_escape = true;
172
+
_ = iter.next();
173
+
} else {
174
+
try sb.append(&.{nc});
175
+
}
176
+
}
177
+
178
+
try tokens.append(allocator, Token{
179
+
.line = line,
180
+
.value = TokenValue{
181
+
.str = sb.build(allocator),
182
+
},
183
+
});
184
+
},
185
+
'0'...'9' => {
186
+
var sb = try StringBuilder.init(allocator, 15);
187
+
defer sb.deinit();
188
+
189
+
const has_decimal = false;
190
+
_ = has_decimal;
191
+
},
192
+
'\n' => {
193
+
line.line += 1;
194
+
line.column = 0;
195
+
},
196
+
else => return exception(.{ .unexpected_token = struct { line, c } }),
197
+
}
198
+
}
199
+
200
+
return .{
201
+
.success = .{ .allocator = allocator, .tokens = tokens.toOwnedSlice(allocator) },
202
+
};
203
+
}
src/parser.zig
src/parser.zig
This is a binary file and will not be displayed.
+4
src/root.zig
+4
src/root.zig
···
1
1
pub const types = @import("./types.zig");
2
2
pub const env = @import("./env.zig");
3
3
pub const std = @import("./std.zig");
4
+
pub const lexer = @import("./lexer.zig");
5
+
pub const parser = @import("./parser.zig");
4
6
5
7
test {
6
8
_ = types;
7
9
_ = env;
8
10
_ = std;
11
+
_ = lexer;
12
+
_ = parser;
9
13
}
+1
-1
src/types.zig
+1
-1
src/types.zig
···
123
123
124
124
/// A value that lives in a `Scope`
125
125
///
126
-
/// The lifetime of any value is bounded to the scope by an arena allocator.
126
+
/// The lifetime of any value created by the `init*` functions is bounded to the scope by an arena allocator.
127
127
pub const Value = union(enum) {
128
128
const Self = @This();
129
129