MIRROR: javascript for 🐜's, a tiny runtime with big ambitions
feat: token streaming cache
+225
-1
+4
include/internal.h
+4
include/internal.h
+221
-1
src/ant.c
+221
-1
src/ant.c
···
259
259
bool is_destruct;
260
260
} parsed_param_t;
261
261
262
+
typedef struct cached_token {
263
+
uint8_t tok;
264
+
uint8_t had_newline;
265
+
jsoff_t toff;
266
+
jsoff_t tlen;
267
+
jsoff_t pos;
268
+
jsval_t tval;
269
+
} cached_token_t;
270
+
271
+
typedef struct token_stream {
272
+
cached_token_t *tokens;
273
+
int count;
274
+
int capacity;
275
+
} token_stream_t;
276
+
262
277
typedef struct parsed_func {
263
278
uint64_t code_hash;
264
279
jsoff_t body_start;
···
270
285
jsoff_t rest_param_start;
271
286
jsoff_t rest_param_len;
272
287
UT_array *params;
288
+
token_stream_t *tokens;
273
289
UT_hash_handle hh;
274
290
} parsed_func_t;
275
291
···
610
626
const char *code;
611
627
jsoff_t clen, pos;
612
628
uint8_t tok, consumed;
629
+
int token_stream_pos;
613
630
} js_parse_state_t;
614
631
615
632
#define JS_SAVE_STATE(js, state) do { \
···
618
635
(state).pos = (js)->pos; \
619
636
(state).tok = (js)->tok; \
620
637
(state).consumed = (js)->consumed; \
638
+
(state).token_stream_pos = (js)->token_stream_pos; \
621
639
} while(0)
622
640
623
641
#define JS_RESTORE_STATE(js, state) do { \
···
626
644
(js)->pos = (state).pos; \
627
645
(js)->tok = (state).tok; \
628
646
(js)->consumed = (state).consumed; \
647
+
(js)->token_stream_pos = (state).token_stream_pos; \
629
648
} while(0)
630
649
631
650
static size_t strstring(struct js *js, jsval_t value, char *buf, size_t len);
···
714
733
static jsval_t proxy_has(struct js *js, jsval_t proxy, const char *key, size_t key_len);
715
734
static jsval_t proxy_delete(struct js *js, jsval_t proxy, const char *key, size_t key_len);
716
735
736
+
static uint8_t next_raw(struct js *js);
717
737
static inline bool push_this(jsval_t this_value);
718
738
static inline jsval_t pop_this(void);
719
739
···
4336
4356
return js->tok;
4337
4357
}
4338
4358
4339
-
static uint8_t next(struct js *js) {
4359
+
static token_stream_t *token_stream_create(void) {
4360
+
token_stream_t *ts = (token_stream_t *)malloc(sizeof(token_stream_t));
4361
+
if (!ts) return NULL;
4362
+
ts->capacity = 64;
4363
+
ts->tokens = (cached_token_t *)malloc(ts->capacity * sizeof(cached_token_t));
4364
+
if (!ts->tokens) { free(ts); return NULL; }
4365
+
ts->count = 0;
4366
+
return ts;
4367
+
}
4368
+
4369
+
static bool token_stream_push(token_stream_t *ts, struct js *js) {
4370
+
if (ts->count >= ts->capacity) {
4371
+
int new_cap = ts->capacity * 2;
4372
+
cached_token_t *new_tokens = (cached_token_t *)realloc(ts->tokens, new_cap * sizeof(cached_token_t));
4373
+
if (!new_tokens) return false;
4374
+
ts->tokens = new_tokens;
4375
+
ts->capacity = new_cap;
4376
+
}
4377
+
cached_token_t *ct = &ts->tokens[ts->count++];
4378
+
ct->tok = js->tok;
4379
+
ct->had_newline = js->had_newline ? 1 : 0;
4380
+
ct->toff = js->toff;
4381
+
ct->tlen = js->tlen;
4382
+
ct->pos = js->pos;
4383
+
ct->tval = js->tval;
4384
+
return true;
4385
+
}
4386
+
4387
+
static inline bool token_stream_next(struct js *js) {
4388
+
token_stream_t *ts = (token_stream_t *)js->token_stream;
4389
+
if (js->token_stream_pos >= ts->count) {
4390
+
js->tok = TOK_EOF;
4391
+
return false;
4392
+
}
4393
+
cached_token_t *ct = &ts->tokens[js->token_stream_pos++];
4394
+
js->tok = ct->tok;
4395
+
js->had_newline = ct->had_newline;
4396
+
js->toff = ct->toff;
4397
+
js->tlen = ct->tlen;
4398
+
js->pos = ct->pos;
4399
+
js->tval = ct->tval;
4400
+
return true;
4401
+
}
4402
+
4403
+
static inline void token_stream_sync_pos(struct js *js) {
4404
+
if (!js->token_stream || js->code != js->token_stream_code) return;
4405
+
token_stream_t *ts = (token_stream_t *)js->token_stream;
4406
+
jsoff_t target = js->pos;
4407
+
4408
+
int lo = 0, hi = ts->count;
4409
+
while (lo < hi) {
4410
+
int mid = lo + (hi - lo) / 2;
4411
+
if (ts->tokens[mid].toff < target) lo = mid + 1;
4412
+
else hi = mid;
4413
+
}
4414
+
js->token_stream_pos = lo;
4415
+
}
4416
+
4417
+
static token_stream_t *tokenize_body(struct js *js, const char *code, jsoff_t len) {
4418
+
token_stream_t *ts = token_stream_create();
4419
+
if (!ts) return NULL;
4420
+
4421
+
const char *saved_code = js->code;
4422
+
jsoff_t saved_clen = js->clen;
4423
+
jsoff_t saved_pos = js->pos;
4424
+
uint8_t saved_tok = js->tok;
4425
+
uint8_t saved_consumed = js->consumed;
4426
+
4427
+
js->code = code;
4428
+
js->clen = len;
4429
+
js->pos = 0;
4430
+
js->consumed = 1;
4431
+
4432
+
while (next_raw(js) != TOK_EOF) {
4433
+
if (!token_stream_push(ts, js)) {
4434
+
free(ts->tokens);
4435
+
free(ts);
4436
+
ts = NULL;
4437
+
break;
4438
+
}
4439
+
js->consumed = 1;
4440
+
}
4441
+
4442
+
if (ts) {
4443
+
cached_token_t eof_tok = {
4444
+
.tok = TOK_EOF,
4445
+
.had_newline = 0,
4446
+
.toff = len,
4447
+
.tlen = 0,
4448
+
.pos = len,
4449
+
.tval = 0
4450
+
};
4451
+
if (ts->count < ts->capacity ||
4452
+
(ts->count >= ts->capacity &&
4453
+
(ts->tokens = realloc(ts->tokens, (ts->capacity + 1) * sizeof(cached_token_t))) != NULL)) {
4454
+
ts->tokens[ts->count++] = eof_tok;
4455
+
}
4456
+
}
4457
+
4458
+
js->code = saved_code;
4459
+
js->clen = saved_clen;
4460
+
js->pos = saved_pos;
4461
+
js->tok = saved_tok;
4462
+
js->consumed = saved_consumed;
4463
+
4464
+
return ts;
4465
+
}
4466
+
4467
+
static uint8_t next_raw(struct js *js) {
4340
4468
if (likely(js->consumed == 0)) return js->tok;
4341
4469
4342
4470
js->consumed = 0;
···
4403
4531
return js->tok;
4404
4532
}
4405
4533
4534
+
static uint8_t next(struct js *js) {
4535
+
if (likely(js->consumed == 0)) return js->tok;
4536
+
4537
+
if (js->token_stream && js->code == js->token_stream_code) {
4538
+
token_stream_t *ts = (token_stream_t *)js->token_stream;
4539
+
int idx = js->token_stream_pos;
4540
+
if (unlikely(idx > 0 && idx <= ts->count && js->pos != ts->tokens[idx - 1].pos)) {
4541
+
token_stream_sync_pos(js);
4542
+
}
4543
+
js->consumed = 0;
4544
+
token_stream_next(js);
4545
+
return js->tok;
4546
+
}
4547
+
4548
+
return next_raw(js);
4549
+
}
4550
+
4406
4551
static inline uint8_t lookahead(struct js *js) {
4407
4552
uint8_t old = js->tok, tok = 0;
4408
4553
uint8_t old_consumed = js->consumed;
4409
4554
jsoff_t pos = js->pos;
4555
+
int stream_pos = js->token_stream_pos;
4410
4556
4411
4557
js->consumed = 1;
4412
4558
tok = next(js);
4413
4559
js->pos = pos;
4414
4560
js->tok = old;
4415
4561
js->consumed = old_consumed;
4562
+
js->token_stream_pos = stream_pos;
4416
4563
4417
4564
return tok;
4418
4565
}
···
4421
4568
jsoff_t pos = js->pos, toff = js->toff, tlen = js->tlen;
4422
4569
uint8_t tok = js->tok, consumed = js->consumed;
4423
4570
bool had_newline = js->had_newline;
4571
+
int stream_pos = js->token_stream_pos;
4424
4572
4425
4573
int depth = 0;
4426
4574
uint8_t t = next(js);
···
4436
4584
4437
4585
js->pos = pos; js->toff = toff; js->tlen = tlen;
4438
4586
js->tok = tok; js->consumed = consumed; js->had_newline = had_newline;
4587
+
js->token_stream_pos = stream_pos;
4439
4588
4440
4589
return bare;
4441
4590
}
···
4535
4684
jsoff_t pos = js->pos, toff = js->toff, tlen = js->tlen;
4536
4685
uint8_t tok = js->tok, consumed = js->consumed;
4537
4686
bool had_newline = js->had_newline;
4687
+
int stream_pos = js->token_stream_pos;
4538
4688
4539
4689
bool needs = false;
4540
4690
int depth = 1;
···
4558
4708
4559
4709
js->pos = pos; js->toff = toff; js->tlen = tlen;
4560
4710
js->tok = tok; js->consumed = consumed; js->had_newline = had_newline;
4711
+
js->token_stream_pos = stream_pos;
4561
4712
4562
4713
return needs;
4563
4714
}
···
6104
6255
pf->body_len = (fnlen > fnpos) ? (fnlen - fnpos - (is_block ? 1 : 0)) : 0;
6105
6256
pf->is_expr = !is_block;
6106
6257
pf->is_strict = is_strict_function_body(&fn[fnpos], pf->body_len);
6258
+
pf->tokens = NULL;
6107
6259
6108
6260
HASH_ADD(hh, func_parse_cache, code_hash, sizeof(pf->code_hash), pf);
6109
6261
return pf;
···
6330
6482
6331
6483
js->flags = F_CALL | (func_strict ? F_STRICT : 0);
6332
6484
jsval_t res;
6485
+
6486
+
void *saved_token_stream = js->token_stream;
6487
+
int saved_token_stream_pos = js->token_stream_pos;
6488
+
const char *saved_token_stream_code = js->token_stream_code;
6489
+
6490
+
if (!pf->is_expr && !pf->tokens && pf->body_len > 0) {
6491
+
pf->tokens = tokenize_body(js, &fn[pf->body_start], pf->body_len);
6492
+
}
6493
+
6494
+
if (pf->tokens && !pf->is_expr) {
6495
+
js->token_stream = pf->tokens;
6496
+
js->token_stream_pos = 0;
6497
+
js->token_stream_code = &fn[pf->body_start];
6498
+
} else {
6499
+
js->token_stream = NULL;
6500
+
js->token_stream_code = NULL;
6501
+
}
6502
+
6333
6503
if (pf->is_expr) {
6334
6504
res = js_eval_str(js, &fn[pf->body_start], pf->body_len);
6335
6505
res = resolveprop(js, res);
···
6337
6507
res = js_eval(js, &fn[pf->body_start], pf->body_len);
6338
6508
if (!is_err(res) && !(js->flags & F_RETURN)) res = js_mkundef();
6339
6509
}
6510
+
6511
+
js->token_stream = saved_token_stream;
6512
+
js->token_stream_pos = saved_token_stream_pos;
6513
+
js->token_stream_code = saved_token_stream_code;
6514
+
6340
6515
js->skip_func_hoist = false;
6341
6516
if (global_scope_stack && utarray_len(global_scope_stack) > 0) utarray_pop_back(global_scope_stack);
6342
6517
···
7736
7911
uint8_t saved_tok = js->tok;
7737
7912
uint8_t saved_consumed = js->consumed;
7738
7913
uint8_t saved_flags = js->flags;
7914
+
int saved_stream_pos = js->token_stream_pos;
7739
7915
7740
7916
js->flags |= F_NOEXEC;
7741
7917
js->consumed = 1;
···
7752
7928
js->tok = saved_tok;
7753
7929
js->consumed = saved_consumed;
7754
7930
js->flags = saved_flags;
7931
+
js->token_stream_pos = saved_stream_pos;
7755
7932
7756
7933
if (is_destruct) {
7757
7934
return js_arr_destruct_assign(js);
···
7988
8165
uint8_t saved_consumed = js->consumed;
7989
8166
jsoff_t saved_toff = js->toff;
7990
8167
jsoff_t saved_tlen = js->tlen;
8168
+
int saved_stream_pos = js->token_stream_pos;
7991
8169
7992
8170
js->consumed = 1;
7993
8171
uint8_t peek = next(js);
···
8041
8219
js->consumed = saved_consumed;
8042
8220
js->toff = saved_toff;
8043
8221
js->tlen = saved_tlen;
8222
+
js->token_stream_pos = saved_stream_pos;
8044
8223
8045
8224
id_off = saved_toff;
8046
8225
id_len = saved_tlen;
···
8609
8788
uint8_t saved_tok = js->tok;
8610
8789
uint8_t saved_consumed = js->consumed;
8611
8790
uint8_t saved_flags = js->flags;
8791
+
int saved_stream_pos = js->token_stream_pos;
8612
8792
int paren_depth = 1;
8613
8793
js->flags |= F_NOEXEC;
8614
8794
while (paren_depth > 0 && next(js) != TOK_EOF) {
···
8622
8802
js->tok = saved_tok;
8623
8803
js->consumed = saved_consumed;
8624
8804
js->flags = saved_flags;
8805
+
js->token_stream_pos = saved_stream_pos;
8625
8806
if (is_arrow) {
8626
8807
js->flags |= F_NOEXEC;
8627
8808
while (next(js) != TOK_RPAREN && next(js) != TOK_EOF) {
···
8650
8831
uint8_t saved_tok = js->tok;
8651
8832
uint8_t saved_consumed = js->consumed;
8652
8833
uint8_t saved_flags = js->flags;
8834
+
int saved_stream_pos = js->token_stream_pos;
8653
8835
8654
8836
int paren_depth = 1;
8655
8837
bool could_be_arrow = true;
···
8669
8851
js->tok = saved_tok;
8670
8852
js->consumed = saved_consumed;
8671
8853
js->flags = saved_flags;
8854
+
js->token_stream_pos = saved_stream_pos;
8672
8855
8673
8856
if (is_arrow) {
8674
8857
js->flags |= F_NOEXEC;
···
8756
8939
jsoff_t saved_pos = js->pos;
8757
8940
uint8_t saved_tok = js->tok;
8758
8941
uint8_t saved_consumed = js->consumed;
8942
+
int saved_stream_pos = js->token_stream_pos;
8759
8943
res = lookup(js, &js->code[id_off], id_len);
8760
8944
if (is_err(res)) {
8761
8945
if (!(js->flags & F_STRICT) && !(js->flags & F_NOEXEC)) {
8762
8946
js->pos = saved_pos;
8763
8947
js->tok = saved_tok;
8764
8948
js->consumed = saved_consumed;
8949
+
js->token_stream_pos = saved_stream_pos;
8765
8950
uint8_t next_tok = next(js);
8766
8951
if (next_tok == TOK_ASSIGN) {
8767
8952
js->flags &= (uint8_t)~F_THROW;
···
9037
9222
jsoff_t id_pos = js->pos;
9038
9223
uint8_t id_tok = js->tok;
9039
9224
jsoff_t id_toff = js->toff, id_tlen = js->tlen;
9225
+
int id_stream_pos = js->token_stream_pos;
9040
9226
js->consumed = 1;
9041
9227
uint8_t after = next(js);
9042
9228
if (after != TOK_DOT && after != TOK_LBRACKET && after != TOK_OPTIONAL_CHAIN) {
9043
9229
return js_mkerr_typed(js, JS_ERR_SYNTAX, "cannot delete unqualified identifier in strict mode");
9044
9230
}
9045
9231
js->pos = id_pos; js->tok = id_tok; js->toff = id_toff; js->tlen = id_tlen; js->consumed = 0;
9232
+
js->token_stream_pos = id_stream_pos;
9046
9233
}
9047
9234
9048
9235
js_parse_state_t saved_state;
···
9233
9420
js->consumed = 1;
9234
9421
jsoff_t saved_pos = js->pos;
9235
9422
uint8_t saved_tok = js->tok, saved_consumed = js->consumed, saved_flags = js->flags;
9423
+
int saved_stream_pos = js->token_stream_pos;
9236
9424
bool bare = is_typeof_bare_ident(js);
9237
9425
jsval_t operand = js_unary(js);
9238
9426
if (is_err(operand)) {
9239
9427
if (!bare || get_error_type(js) != JS_ERR_REFERENCE) return operand;
9240
9428
js->pos = saved_pos; js->tok = saved_tok; js->consumed = saved_consumed;
9429
+
js->token_stream_pos = saved_stream_pos;
9241
9430
js->flags = (saved_flags & ~F_THROW) | F_NOEXEC;
9242
9431
js_unary(js);
9243
9432
js->flags = saved_flags & ~F_THROW;
···
10151
10340
jsoff_t saved_pos = js->pos;
10152
10341
uint8_t saved_tok = js->tok;
10153
10342
uint8_t saved_consumed = js->consumed;
10343
+
int saved_stream_pos = js->token_stream_pos;
10154
10344
10155
10345
js->consumed = 1;
10156
10346
next(js);
···
10159
10349
js->pos = saved_pos;
10160
10350
js->tok = saved_tok;
10161
10351
js->consumed = saved_consumed;
10352
+
js->token_stream_pos = saved_stream_pos;
10162
10353
10163
10354
if (ctx->needs_scope) ctx->loop_scope_handle = js_root(js, js_mkscope(js));
10164
10355
}
···
11788
11979
jsoff_t saved_toff = js->toff;
11789
11980
jsoff_t saved_tlen = js->tlen;
11790
11981
uint8_t saved_tok = js->tok;
11982
+
int saved_stream_pos = js->token_stream_pos;
11791
11983
js->consumed = 1;
11792
11984
if (next(js) == TOK_IDENTIFIER) {
11793
11985
is_getter_method = is_get;
···
11797
11989
js->toff = saved_toff;
11798
11990
js->tlen = saved_tlen;
11799
11991
js->tok = saved_tok;
11992
+
js->token_stream_pos = saved_stream_pos;
11800
11993
js->consumed = 0;
11801
11994
}
11802
11995
}
···
22330
22523
js->clen = (jsoff_t) len;
22331
22524
js->pos = 0;
22332
22525
22526
+
if (js->token_stream && js->code == js->token_stream_code) {
22527
+
js->token_stream_pos = 0;
22528
+
}
22529
+
22333
22530
uint8_t saved_tok = js->tok;
22334
22531
jsoff_t saved_pos = js->pos;
22335
22532
uint8_t saved_consumed = js->consumed;
22533
+
int saved_stream_pos = js->token_stream_pos;
22336
22534
js->consumed = 1;
22337
22535
22338
22536
bool is_strict = inherit_strict;
···
22350
22548
js->tok = saved_tok;
22351
22549
js->pos = saved_pos;
22352
22550
js->consumed = saved_consumed;
22551
+
js->token_stream_pos = saved_stream_pos;
22353
22552
22354
22553
if (is_strict) {
22355
22554
mkscope(js);
···
22516
22715
js->flags = F_CALL | (pf->is_strict ? F_STRICT : 0);
22517
22716
jsval_t res;
22518
22717
22718
+
void *saved_token_stream = js->token_stream;
22719
+
int saved_token_stream_pos = js->token_stream_pos;
22720
+
const char *saved_token_stream_code = js->token_stream_code;
22721
+
22722
+
if (!pf->is_expr && !pf->tokens && pf->body_len > 0) {
22723
+
pf->tokens = tokenize_body(js, &fn[pf->body_start], pf->body_len);
22724
+
}
22725
+
22726
+
if (pf->tokens && !pf->is_expr) {
22727
+
js->token_stream = pf->tokens;
22728
+
js->token_stream_pos = 0;
22729
+
js->token_stream_code = &fn[pf->body_start];
22730
+
} else {
22731
+
js->token_stream = NULL;
22732
+
js->token_stream_code = NULL;
22733
+
}
22734
+
22519
22735
if (pf->is_expr) {
22520
22736
res = js_eval_str(js, &fn[pf->body_start], pf->body_len);
22521
22737
res = resolveprop(js, res);
···
22523
22739
res = js_eval(js, &fn[pf->body_start], pf->body_len);
22524
22740
if (!is_err(res) && !(js->flags & F_RETURN)) res = js_mkundef();
22525
22741
}
22742
+
22743
+
js->token_stream = saved_token_stream;
22744
+
js->token_stream_pos = saved_token_stream_pos;
22745
+
js->token_stream_code = saved_token_stream_code;
22526
22746
22527
22747
JS_RESTORE_STATE(js, saved_state);
22528
22748
js->flags = caller_flags;