Tholp's bespoke website generator
Refactor tokens to be chars instead of strings
+349
-303
-1
src/macros/simple_blocks.rs
-1
src/macros/simple_blocks.rs
+11
-6
src/macros/template.rs
+11
-6
src/macros/template.rs
···
3
3
project::ProjectContext,
4
4
reservednames::{RESERVED_NAMES_HTML, RESERVED_NAMES_MISC},
5
5
stringtools::{find_pattern, split_to_tokens, WhitespaceChecks},
6
-
types::{SkidContext, Token},
6
+
types::{IsScoped, SkidContext, Token},
7
7
};
8
8
9
9
use super::MACRO_LIST;
···
31
31
allows_trailing_args: trailing,
32
32
}
33
33
}
34
-
35
34
pub fn expand(
36
35
&self,
37
36
//_file: &mut InputFile,
38
37
origin_index: usize,
39
38
origin_line: usize,
40
-
project_context: &mut ProjectContext,
39
+
proj_context: &mut ProjectContext,
41
40
args: &Vec<String>,
42
41
scope: &[Token],
43
42
) -> Vec<Token> {
···
45
44
46
45
if !self.allows_trailing_args && args.len() != self.args.len() {
47
46
error_skid(
48
-
project_context,
47
+
proj_context,
49
48
origin_index,
50
49
origin_line,
51
50
&format!(
···
59
58
}
60
59
if self.allows_trailing_args && args.len() < self.args.len() {
61
60
error_skid(
62
-
project_context,
61
+
proj_context,
63
62
origin_index,
64
63
origin_line,
65
64
&format!(
···
75
74
let mut output = self.tokens.clone();
76
75
77
76
for tok in &mut output {
78
-
tok.origin_file = origin_index;
77
+
tok.origin_index = origin_index;
79
78
}
80
79
81
80
let mut args_index: usize = 0;
···
126
125
}
127
126
128
127
output
128
+
}
129
+
}
130
+
131
+
impl IsScoped for SkidTemplate {
132
+
fn is_scoped(&self) -> bool {
133
+
self.has_scope
129
134
}
130
135
}
131
136
+234
-215
src/main.rs
+234
-215
src/main.rs
···
8
8
mod types;
9
9
10
10
use crate::{
11
-
args::ProgramArgs,
12
-
closures::CLOSURE_LIST,
13
-
macros::template::SkidTemplate,
14
-
project::FileGroup,
15
11
reservednames::RESERVED_NAMES_MISC,
16
-
types::{Expand, SkidContext},
12
+
stringtools::TokenTools,
13
+
types::{IsScoped, MacroExpand, SkidContext},
17
14
};
18
-
use clap::Parser;
15
+
19
16
use console::*;
20
17
use macros::MACRO_LIST;
21
18
use markdown::{CompileOptions, Constructs, Options, ParseOptions};
···
25
22
env,
26
23
fs::{self},
27
24
path::PathBuf,
28
-
task::Context,
29
25
};
30
-
use stringtools::{collect_arguments, collect_block, split_to_tokens, trim_whitespace_tokens};
26
+
use stringtools::{collect_arguments, collect_block, split_to_tokens};
31
27
use types::{InputFile, Token};
32
28
33
29
// really need to change this whole thing to work with characters rather than
···
36
32
' ', '\n', '\t', '(', ')', '{', '}', '[', ']', '<', '>', '\\', '\'', '\"', ';', '?', '^', '-',
37
33
];
38
34
35
+
#[derive(PartialEq)]
36
+
enum EphemeralType {
37
+
Normal,
38
+
Ephemeral,
39
+
InverseEphemeral,
40
+
}
41
+
39
42
fn main() {
40
43
// let args = ProgramArgs::parse();
41
44
···
73
76
// }
74
77
75
78
for group in &mut project.filegroups {
79
+
if !group.process {
80
+
continue;
81
+
}
76
82
for infile in &mut group.files {
77
83
let contents =
78
84
fs::read_to_string(&infile.file_input).expect("File unreadable or missing");
79
-
infile.tokens =
85
+
let tokens =
80
86
split_to_tokens(contents, project.context.index_of_file(&infile.file_input));
81
87
82
-
let mut skid_context = SkidContext::new();
83
-
process_skid(
84
-
&mut infile.tokens,
85
-
project.context.index_of_file(&infile.file_input),
86
-
&mut project.context,
87
-
&mut skid_context,
88
+
let mut skid_context =
89
+
SkidContext::new(project.context.index_of_file(&infile.file_input));
90
+
write_file(
91
+
infile,
92
+
group.convert_html,
93
+
&process_skid(&tokens, &mut project.context, &mut skid_context),
88
94
);
89
95
}
90
96
}
91
97
}
92
98
93
-
fn process_skid(
94
-
tokens_in: &mut [Token],
95
-
file_index: usize,
96
-
context: &mut ProjectContext,
99
+
fn find_and_run_macro(
100
+
tokens_in: &[Token],
101
+
proj_context: &mut ProjectContext,
97
102
skid_context: &mut SkidContext,
98
-
) -> Vec<Token> {
99
-
//}, context: &mut ProjectContext) {
100
-
//println!("{}\n {}", f.filename_out, contents);
103
+
) -> Option<(Vec<Token>, usize)> {
104
+
// (Output, to be consumed size)
101
105
102
-
//file.tokens = strings_to_tokens(split_keep_delimiters(contents), file.filename_input.clone());
106
+
// At this point we think its a macro (starts with ! or &) so check which, we have the rest of the file
107
+
let ephemeral_type: EphemeralType;
108
+
if tokens_in.len() < 2 {
109
+
return None;
110
+
}
111
+
112
+
if tokens_in[0] == '!' && tokens_in[1] == '&' {
113
+
ephemeral_type = EphemeralType::InverseEphemeral;
114
+
} else if tokens_in[0] == '!' {
115
+
ephemeral_type = EphemeralType::Normal;
116
+
} else if tokens_in[0] == '&' {
117
+
ephemeral_type = EphemeralType::Ephemeral;
118
+
} else {
119
+
return None;
120
+
}
103
121
104
-
//let mut escaped = false;
105
-
let mut tokens = tokens_in.to_vec();
106
-
let mut starting_template_count = skid_context.templates.len();
122
+
let mut chars_consumed = if ephemeral_type == EphemeralType::InverseEphemeral {
123
+
2
124
+
} else {
125
+
1
126
+
};
127
+
// Look for name
128
+
let mut symbol: String = "".into();
129
+
for tok in &tokens_in[chars_consumed..] {
130
+
if tok.contents.is_whitespace() || DELIMITERS.contains(&tok.contents) {
131
+
break;
132
+
}
133
+
symbol.push(tok.contents);
134
+
chars_consumed += 1;
135
+
}
107
136
108
-
let mut working_index = 0;
137
+
if symbol.is_empty() {
138
+
return None;
139
+
}
109
140
110
-
while working_index < tokens.len() {
111
-
//look for macros or blocks
112
-
//println!(">\"{}\"<", tokens[working_index].contents);
141
+
let args;
142
+
let block;
113
143
114
-
if tokens[working_index].contents.len() == 0 {
115
-
working_index += 1;
116
-
continue;
144
+
{
145
+
let mut expander: &dyn IsScoped = &MACRO_LIST[0]; // assinging because it complains about possibly being empty later even if not the case
146
+
let mut found = false;
147
+
// Check if its a macro
148
+
for m in MACRO_LIST {
149
+
if m.symbol == symbol {
150
+
found = true;
151
+
expander = m;
152
+
break;
153
+
}
117
154
}
118
155
119
-
if tokens[working_index].contents == "\\" {
120
-
tokens[working_index].contents = "".into();
121
-
working_index += 2;
122
-
//println!("Hit backslash");
123
-
continue;
156
+
// Not a macro check templates
157
+
if !found {
158
+
for t in &skid_context.templates {
159
+
if t.symbol == symbol {
160
+
found = true;
161
+
expander = t;
162
+
break;
163
+
}
164
+
}
124
165
}
125
166
126
-
let mut matched_macro: bool = false;
127
-
if tokens[working_index].contents.starts_with(['!', '&']) {
128
-
let mut prefix_len = 1;
129
-
let mut symbol = tokens[working_index].contents.clone();
130
-
symbol = symbol.trim().to_string();
131
-
132
-
if symbol.len() > 2 {
133
-
let mut ephemeral = false;
134
-
let same_file = tokens[working_index].origin_file != file_index;
167
+
// Not a template either, see if its reserved or not to see if we should say something
168
+
if !found {
169
+
let name = symbol.to_lowercase();
170
+
let mut dont_error = false;
135
171
136
-
// Inversely Ephemeral
137
-
if symbol.starts_with("!&") {
138
-
prefix_len = 2;
139
-
ephemeral = !same_file;
172
+
for reserved in RESERVED_NAMES_HTML {
173
+
if name.starts_with(reserved) {
174
+
dont_error = true;
175
+
break;
140
176
}
141
-
// Ephemeral
142
-
else if symbol.starts_with("&") {
143
-
ephemeral = same_file;
177
+
}
178
+
179
+
if !dont_error {
180
+
for reserved in RESERVED_NAMES_MISC {
181
+
if name.starts_with(reserved) {
182
+
dont_error = true;
183
+
break;
184
+
}
144
185
}
186
+
}
145
187
146
-
// Check if its a macro
147
-
for m in MACRO_LIST {
148
-
if &symbol[prefix_len..] == m.symbol {
149
-
matched_macro = true;
150
-
//println!("Found a macro ({})", m.symbol);
188
+
if !dont_error {
189
+
warn_skid(
190
+
&proj_context,
191
+
tokens_in[0].origin_index,
192
+
tokens_in[0].origin_line,
193
+
&format!("No such macro or defined template \"{symbol}\""),
194
+
);
195
+
}
196
+
return None;
197
+
}
151
198
152
-
let (args, args_tokcount) = collect_arguments(&tokens[working_index..]);
153
-
let expansion: Vec<Token>;
154
-
let block_tokcount: usize;
155
-
if m.has_scope {
156
-
//println!("is scoped.");
199
+
let args_result = collect_arguments(&tokens_in[chars_consumed..]);
200
+
if args_result.is_none() {
201
+
error_skid(
202
+
proj_context,
203
+
tokens_in[0].origin_index,
204
+
tokens_in[0].origin_line,
205
+
&format!("Didnt find any arguments for macro \"{symbol}\"."),
206
+
);
207
+
return None;
208
+
}
157
209
158
-
let block_opt =
159
-
collect_block(&tokens[(working_index + args_tokcount)..]);
160
-
if block_opt.is_none() {
161
-
error_skid(
162
-
context,
163
-
tokens[working_index].template_origin,
164
-
tokens[working_index].line_number,
165
-
&"Malformed Block".into(),
166
-
);
167
-
}
168
-
let block: Vec<Token>;
169
-
(block, block_tokcount) = block_opt.unwrap();
210
+
let consumed_by_args;
211
+
(args, consumed_by_args) = args_result.unwrap();
212
+
chars_consumed += consumed_by_args;
170
213
171
-
if ephemeral {
172
-
expansion = Vec::new();
173
-
} else {
174
-
expansion = m.expand(
175
-
tokens[working_index].origin_file,
176
-
tokens[working_index].line_number,
177
-
context,
178
-
skid_context,
179
-
&args,
180
-
&block,
181
-
);
182
-
}
183
-
} else {
184
-
block_tokcount = 0;
214
+
if expander.is_scoped() {
215
+
let block_result = collect_block(&tokens_in[chars_consumed..]);
216
+
if block_result.is_none() {
217
+
error_skid(
218
+
proj_context,
219
+
tokens_in[0].origin_index,
220
+
tokens_in[0].origin_line,
221
+
&format!("Didnt find a block for macro \"{symbol}\"."),
222
+
);
223
+
return None;
224
+
}
225
+
let consumed_by_block;
226
+
(block, consumed_by_block) = block_result.unwrap();
227
+
chars_consumed += consumed_by_block;
228
+
} else {
229
+
block = Vec::new();
230
+
}
231
+
}
185
232
186
-
if ephemeral {
187
-
expansion = Vec::new();
188
-
} else {
189
-
expansion = m.expand(
190
-
tokens[working_index].origin_file,
191
-
tokens[working_index].line_number,
192
-
context,
193
-
skid_context,
194
-
&args,
195
-
&Vec::new()[..],
196
-
);
197
-
}
198
-
}
199
-
200
-
let trimmed = trim_whitespace_tokens(&expansion);
201
-
202
-
tokens.remove(working_index);
203
-
tokens.splice(
204
-
working_index..(working_index + args_tokcount + block_tokcount - 1),
205
-
trimmed.iter().cloned(),
206
-
);
207
-
if expansion.len() == 0 && working_index > 0 {
208
-
working_index -= 1;
209
-
}
210
-
}
211
-
}
233
+
let return_empty: bool;
212
234
213
-
// check for templates
214
-
// todo maybe deduplicate this
215
-
for t in &skid_context.templates {
216
-
if &symbol[prefix_len..] == t.symbol {
217
-
matched_macro = true;
218
-
//println!("Found a macro ({})", m.symbol);
235
+
match ephemeral_type {
236
+
EphemeralType::Normal => return_empty = false,
237
+
EphemeralType::Ephemeral => {
238
+
return_empty = skid_context.file_index != tokens_in[0].origin_index
239
+
}
240
+
EphemeralType::InverseEphemeral => {
241
+
return_empty = skid_context.file_index == tokens_in[0].origin_index
242
+
}
243
+
}
219
244
220
-
let (args, args_tokcount) = collect_arguments(&tokens[working_index..]);
221
-
let expansion: Vec<Token>;
222
-
let block_tokcount: usize;
245
+
if return_empty {
246
+
return Some((Vec::new(), chars_consumed));
247
+
} else {
248
+
// we have to find it again because of borrower
249
+
for m in MACRO_LIST {
250
+
if m.symbol == symbol {
251
+
return Some((
252
+
m.expand(
253
+
tokens_in[0].origin_index,
254
+
tokens_in[0].origin_line,
255
+
proj_context,
256
+
skid_context,
257
+
&args,
258
+
&block,
259
+
)
260
+
.trim_whitespace()
261
+
.to_vec(),
262
+
chars_consumed,
263
+
));
264
+
}
265
+
}
266
+
let mut i = 0;
267
+
while i < skid_context.templates.len() {
268
+
if skid_context.templates[i].symbol == symbol {
269
+
return Some((
270
+
skid_context.templates[i]
271
+
.expand(
272
+
tokens_in[0].origin_index,
273
+
tokens_in[0].origin_line,
274
+
proj_context,
275
+
&args,
276
+
&block,
277
+
)
278
+
.trim_whitespace()
279
+
.to_vec(),
280
+
chars_consumed,
281
+
));
282
+
}
283
+
i += 1;
284
+
}
285
+
}
286
+
None
287
+
}
223
288
224
-
if t.has_scope {
225
-
//println!("is scoped.");
226
-
let block: Vec<Token>;
227
-
let block_opt =
228
-
collect_block(&tokens[(working_index + args_tokcount)..]);
229
-
if block_opt.is_none() {
230
-
error_skid(
231
-
context,
232
-
tokens[working_index].template_origin,
233
-
tokens[working_index].line_number,
234
-
&"Malformed Block".into(),
235
-
);
236
-
}
289
+
fn process_skid(
290
+
tokens_in: &[Token],
291
+
proj_context: &mut ProjectContext,
292
+
skid_context: &mut SkidContext,
293
+
) -> Vec<Token> {
294
+
//}, context: &mut ProjectContext) {
295
+
//println!("{}\n {}", f.filename_out, contents);
237
296
238
-
(block, block_tokcount) = block_opt.unwrap();
297
+
//file.tokens = strings_to_tokens(split_keep_delimiters(contents), file.filename_input.clone());
239
298
240
-
if ephemeral {
241
-
expansion = Vec::new();
242
-
} else {
243
-
expansion = t.expand(
244
-
//file,
245
-
tokens[working_index].origin_file,
246
-
tokens[working_index].line_number,
247
-
context,
248
-
&args,
249
-
&block,
250
-
);
251
-
}
252
-
} else {
253
-
block_tokcount = 0;
299
+
//let mut escaped = false;
300
+
let mut tokens = tokens_in.to_vec();
301
+
let starting_template_count = skid_context.templates.len();
254
302
255
-
if ephemeral {
256
-
expansion = Vec::new();
257
-
} else {
258
-
expansion = t.expand(
259
-
//file,
260
-
tokens[working_index].origin_file,
261
-
tokens[working_index].line_number,
262
-
context,
263
-
&args,
264
-
&Vec::new()[..],
265
-
);
266
-
}
267
-
}
303
+
let mut escaped = false;
304
+
let mut working_index = 0;
268
305
269
-
let trimmed = trim_whitespace_tokens(&expansion);
306
+
while working_index < tokens.len() {
307
+
if tokens[working_index] == '\\' && !escaped {
308
+
tokens[working_index].contents = '\0'; // skip over this later when outputting to avoid shifting memory rn
309
+
escaped = true;
310
+
working_index += 1;
270
311
271
-
tokens.remove(working_index);
272
-
tokens.splice(
273
-
working_index..(working_index + args_tokcount + block_tokcount - 1),
274
-
trimmed.iter().cloned(),
275
-
);
276
-
if expansion.len() == 0 && working_index > 0 {
277
-
working_index -= 1;
278
-
}
279
-
}
280
-
}
312
+
// bit of a hack for reverse ephemeral escaping behavior to be the same as previously
313
+
if tokens.len() > working_index + 1
314
+
&& tokens[working_index] == '!'
315
+
&& tokens[working_index + 1] == '&'
316
+
{
317
+
working_index += 1;
281
318
}
282
-
if !matched_macro {
283
-
let name = tokens[working_index].contents.trim().to_lowercase();
284
-
let mut dont_error = name.len() <= 1;
285
-
{
286
-
if !dont_error {
287
-
for reserved in RESERVED_NAMES_HTML {
288
-
if name[1..].starts_with(reserved) {
289
-
dont_error = true;
290
-
break;
291
-
}
292
-
}
293
-
}
319
+
continue;
320
+
}
294
321
295
-
if !dont_error {
296
-
for reserved in RESERVED_NAMES_MISC {
297
-
if name[1..].starts_with(reserved) {
298
-
dont_error = true;
299
-
break;
300
-
}
301
-
}
302
-
}
303
-
}
304
-
if !dont_error {
305
-
warn_skid(
306
-
context,
307
-
tokens[working_index].origin_file,
308
-
tokens[working_index].line_number,
309
-
&format!(
310
-
"Token written as a function but no such function exists \"{}\"",
311
-
tokens[working_index].contents.trim()
312
-
),
313
-
);
314
-
}
322
+
if (tokens[working_index] == '!' || tokens[working_index] == '&') && !escaped {
323
+
let expansion =
324
+
find_and_run_macro(&tokens[working_index..], proj_context, skid_context);
325
+
if expansion.is_some() {
326
+
tokens.splice(
327
+
working_index..working_index + expansion.as_ref().unwrap().1,
328
+
expansion.unwrap().0,
329
+
);
330
+
continue;
315
331
}
316
332
}
317
333
···
324
340
// }
325
341
// }
326
342
327
-
if !matched_macro {
328
-
working_index += 1;
329
-
}
343
+
working_index += 1;
344
+
345
+
escaped = false;
330
346
}
331
347
skid_context.templates.truncate(starting_template_count);
348
+
349
+
tokens.retain(|t| t.contents != '\0');
350
+
332
351
return tokens;
333
352
}
334
353
335
-
fn write_file(file: InputFile, convert_html: bool) {
354
+
fn write_file(file: &InputFile, convert_html: bool, tokens: &[Token]) {
336
355
//println!("{:?}", tokens);
337
356
let mut skid_output: String = "".to_string();
338
-
for t in &file.tokens {
339
-
skid_output += &t.contents;
357
+
for t in tokens {
358
+
skid_output.push(t.contents);
340
359
}
341
360
342
361
let mut folder = file.file_skidout.clone();
-2
src/project.rs
-2
src/project.rs
···
18
18
pub post_insert: PathBuf,
19
19
pub process: bool,
20
20
pub convert_html: bool,
21
-
pub output_extention: String,
22
21
}
23
22
24
23
pub struct ProjectContext {
···
128
127
post_insert: post_insert.into(),
129
128
process,
130
129
convert_html,
131
-
output_extention: extention.into(),
132
130
};
133
131
134
132
if filegroup_def.contains_key("files") {
+55
-46
src/stringtools.rs
+55
-46
src/stringtools.rs
···
1
1
use super::DELIMITERS;
2
2
use crate::types::Token;
3
3
4
-
pub fn collect_arguments(tokens: &[Token]) -> (Vec<String>, usize) {
5
-
// Arguments vec and number of tokens consumed
4
+
pub fn collect_arguments(tokens: &[Token]) -> Option<(Vec<String>, usize)> {
5
+
// Returns arguments vec and number of tokens to be consumed
6
6
//let mut output = Vec::new();
7
-
let mut split_tokens = Vec::new();
8
-
for tok in tokens {
9
-
for s in split_keep_delimiters(tok.contents.clone()) {
10
-
split_tokens.push(s);
11
-
}
12
-
}
13
7
14
8
let mut quoted: bool = false;
9
+
let mut escaped: bool = false;
15
10
let mut entered: bool = false;
16
11
let mut arg = "".to_string();
17
12
let mut args: Vec<String> = Vec::new();
18
13
19
14
let mut in_token_count = 0;
15
+
let mut exited_cleanly = false;
20
16
21
-
for tok in split_tokens {
22
-
in_token_count += 1; // This could be a problem if it something got split above..
23
-
if tok.starts_with([' ', '\t']) && !quoted {
17
+
for tok in tokens {
18
+
let c = tok.contents;
19
+
20
+
in_token_count += 1;
21
+
if c.is_whitespace() && !entered {
24
22
continue;
25
23
}
26
24
27
-
if !entered && tok.starts_with('(') {
25
+
if !entered && c == '(' {
28
26
entered = true;
29
27
continue;
30
28
}
31
29
32
30
if !entered {
33
-
continue;
31
+
break;
34
32
}
35
33
36
-
if !quoted && tok.starts_with(')') {
34
+
if !quoted && tok.contents == ')' {
35
+
exited_cleanly = true;
36
+
if !arg.is_empty() {
37
+
args.push(arg.clone());
38
+
arg.clear();
39
+
}
37
40
break;
38
41
}
39
42
40
-
for c in tok.chars() {
41
-
if c == '\"' {
42
-
quoted = !quoted;
43
-
continue;
44
-
}
43
+
if c == '\"' && !escaped {
44
+
quoted = !quoted;
45
+
continue;
46
+
}
45
47
46
-
arg.push(c);
48
+
if c == '\\' && !escaped {
49
+
escaped = true;
50
+
continue;
47
51
}
48
52
49
-
if !quoted {
50
-
args.push(arg.clone());
51
-
arg.clear();
53
+
if c.is_whitespace() && !quoted {
54
+
if !arg.is_empty() {
55
+
args.push(arg.clone());
56
+
arg.clear();
57
+
}
58
+
continue;
52
59
}
60
+
arg.push(c);
53
61
}
54
62
55
-
return (args, in_token_count);
63
+
if !entered || !exited_cleanly {
64
+
return None;
65
+
}
66
+
return Some((args, in_token_count));
56
67
}
57
68
58
69
pub fn collect_block(tokens: &[Token]) -> Option<(Vec<Token>, usize)> {
···
67
78
68
79
// We dont really care about doing anything that in the block right now
69
80
// maybe have the Token struct contain scope level later?
70
-
let mut escaped_tok: Token = Token::new("\\".into(), 0, 0);
81
+
let mut escaped_tok: Token = Token::new('\\', 0, 0);
71
82
for tok in tokens {
72
83
tokens_consumed += 1;
73
84
if !entered {
74
-
if tok.contents.is_only_whitespace() {
85
+
if tok.contents.is_whitespace() {
75
86
continue;
76
87
}
77
-
if tok.contents != "{"
88
+
if tok.contents != '{'
78
89
// Expected block start, got garbage
79
90
{
80
91
// println!("Expected block start, got {}",tok.contents);
···
90
101
let mut escaped_used = false;
91
102
92
103
// Scope Start
93
-
if tok.contents == "{" && !escaped {
104
+
if tok.contents == '{' && !escaped {
94
105
entering_bracket_count += 1;
95
106
96
107
if entering_bracket_count == 3 {
···
107
118
}
108
119
}
109
120
// Scope End
110
-
if tok.contents == "}" && !escaped {
121
+
if tok.contents == '}' && !escaped {
111
122
exiting_bracket_count += 1;
112
123
if exiting_bracket_count == 3 {
113
124
scope_count -= 1;
···
128
139
block.push(escaped_tok.clone());
129
140
}
130
141
131
-
if tok.contents == "\\" {
142
+
if tok.contents == '\\' {
132
143
escaped = true;
133
144
escaped_tok = tok.clone();
134
145
} else {
···
183
194
let mut line_count = 1;
184
195
185
196
for str in in_strings {
186
-
if str.len() == 0 {
187
-
continue;
188
-
}
189
-
190
-
let current_line = line_count;
191
-
for char in str.chars() {
192
-
if char == '\n' {
193
-
line_count += 1;
197
+
for c in str.chars() {
198
+
let current_line = line_count;
199
+
for char in str.chars() {
200
+
if char == '\n' {
201
+
line_count += 1;
202
+
}
194
203
}
204
+
let token: Token = Token::new(c, origin_file, current_line);
205
+
tokens.push(token);
195
206
}
196
-
let token: Token = Token::new(str, origin_file, current_line);
197
-
tokens.push(token);
198
207
}
199
208
200
209
return tokens;
···
226
235
return strings_to_tokens(new_split, origin_file);
227
236
}
228
237
229
-
pub fn next_nonwhitespace_token(tokens: &Vec<Token>, index: usize) -> (bool, usize) {
238
+
pub fn next_nonwhitespace_token(tokens: &Vec<Token>, index: usize) -> Option<usize> {
230
239
while index < tokens.len() {
231
-
if tokens[index].contents.is_only_whitespace() {
240
+
if tokens[index].contents.is_whitespace() {
232
241
continue;
233
242
}
234
-
return (true, index);
243
+
return Some(index);
235
244
}
236
-
return (false, 0);
245
+
return None;
237
246
}
238
247
239
248
//trim whitespace from the ends
···
241
250
let mut start: usize = 0;
242
251
let mut end: usize = tokens.len();
243
252
for tok in tokens {
244
-
if !tok.contents.is_only_whitespace() {
253
+
if !tok.contents.is_whitespace() {
245
254
break;
246
255
}
247
256
start = start + 1;
248
257
}
249
258
250
259
for tok in tokens.iter().rev() {
251
-
if !tok.contents.is_only_whitespace() {
260
+
if !tok.contents.is_whitespace() {
252
261
break;
253
262
}
254
263
end = end - 1;
+49
-33
src/types.rs
+49
-33
src/types.rs
···
1
-
use boa_engine::Context;
2
1
use std::path::PathBuf;
3
2
4
3
use crate::{
···
8
7
};
9
8
10
9
pub struct Token {
11
-
pub contents: String,
12
-
pub origin_file: usize,
10
+
//pub contents: String,
11
+
pub contents: char,
12
+
pub origin_index: usize,
13
13
pub template_origin: usize,
14
-
pub line_number: usize,
14
+
pub origin_line: usize,
15
15
pub section_name_index: usize,
16
16
}
17
17
18
+
impl PartialEq<char> for Token {
19
+
fn eq(&self, other: &char) -> bool {
20
+
self.contents == *other
21
+
}
22
+
}
23
+
18
24
pub struct InputFile {
19
25
pub file_input: PathBuf,
20
26
pub file_skidout: PathBuf,
21
27
pub file_out: PathBuf,
22
-
pub tokens: Vec<Token>,
23
28
}
24
29
25
30
pub struct SkidContext {
26
31
pub templates: Vec<SkidTemplate>,
32
+
pub file_index: usize,
27
33
}
28
34
29
35
impl SkidContext {
30
-
pub fn new() -> SkidContext {
36
+
pub fn new(file_index: usize) -> SkidContext {
31
37
SkidContext {
32
38
templates: Vec::new(),
39
+
file_index,
33
40
}
34
41
}
35
42
}
···
53
60
pub max_args: usize,
54
61
}
55
62
56
-
pub trait Expand {
63
+
pub trait MacroExpand {
57
64
fn expand(
58
65
&self,
59
66
origin_index: usize,
···
63
70
args: &Vec<String>,
64
71
scope: &[Token],
65
72
) -> Vec<Token>;
73
+
}
66
74
67
-
fn default() -> Macro;
75
+
pub trait IsScoped {
76
+
fn is_scoped(&self) -> bool;
68
77
}
69
78
70
-
impl Expand for Macro {
79
+
impl Macro {
80
+
fn default() -> Macro {
81
+
Macro {
82
+
symbol: "default_symbol",
83
+
expansion: macro_comment,
84
+
has_scope: true,
85
+
min_args: 0,
86
+
max_args: usize::max_value(),
87
+
}
88
+
}
89
+
}
90
+
91
+
impl MacroExpand for Macro {
71
92
fn expand(
72
93
&self,
73
94
origin_index: usize,
74
95
origin_line: usize,
75
-
context: &mut ProjectContext,
96
+
proj_context: &mut ProjectContext,
76
97
skid_context: &mut SkidContext,
77
98
args: &Vec<String>,
78
-
scope: &[Token],
99
+
block: &[Token],
79
100
) -> Vec<Token> {
80
101
if (args.len() > self.max_args) || (args.len() < self.min_args) {
81
-
error_skid(context, origin_index, origin_line, &format!("Macro \'{}\' was given a number of arguments ({}) not in its acceptable range ({}-{})",
102
+
error_skid(proj_context, origin_index, origin_line, &format!("Macro \'{}\' was given a number of arguments ({}) not in its acceptable range ({}-{})",
82
103
self.symbol, args.len(), self.min_args, if self.max_args == usize::max_value() {"No Limit".to_string()} else {format!("{}", self.max_args)}));
83
104
Vec::new()
84
105
} else {
85
106
(self.expansion)(
86
107
origin_index,
87
108
origin_line,
88
-
context,
109
+
proj_context,
89
110
skid_context,
90
111
args,
91
-
scope,
112
+
block,
92
113
)
93
114
}
94
115
}
116
+
}
95
117
96
-
fn default() -> Macro {
97
-
Macro {
98
-
symbol: "default_symbol",
99
-
expansion: macro_comment,
100
-
has_scope: true,
101
-
min_args: 0,
102
-
max_args: usize::max_value(),
103
-
}
118
+
impl IsScoped for Macro {
119
+
fn is_scoped(&self) -> bool {
120
+
self.has_scope
104
121
}
105
122
}
106
123
···
110
127
file_input: "".into(),
111
128
file_skidout: "".into(),
112
129
file_out: "".into(),
113
-
tokens: Vec::new(),
114
130
}
115
131
}
116
132
}
117
133
118
134
impl Token {
119
-
pub fn new(contents: String, origin_file: usize, line_number: usize) -> Token {
135
+
pub fn new(contents: char, origin_file: usize, line_number: usize) -> Token {
120
136
Token {
121
137
contents: contents,
122
-
origin_file: origin_file,
138
+
origin_index: origin_file,
123
139
template_origin: origin_file,
124
-
line_number: line_number,
140
+
origin_line: line_number,
125
141
section_name_index: 0,
126
142
}
127
143
}
128
144
}
129
145
130
-
impl ToString for Token {
131
-
fn to_string(&self) -> String {
132
-
return self.contents.clone();
133
-
}
134
-
}
146
+
// impl ToString for Token {
147
+
// fn to_string(&self) -> String {
148
+
// return self.contents.clone();
149
+
// }
150
+
// }
135
151
136
152
impl Clone for Token {
137
153
fn clone(&self) -> Self {
138
154
let mut t = Token::new(
139
155
self.contents.clone(),
140
-
self.origin_file.clone(),
141
-
self.line_number,
156
+
self.origin_index.clone(),
157
+
self.origin_line,
142
158
);
143
159
t.template_origin = self.template_origin;
144
160
return t;