themackabu.com / ant
MIRROR: javascript for 🐜's, a tiny runtime with big ambitions
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

add method inlining

theMackabu 39de90df b46ad342

+986 -18
+24 -4
src/gc/gc.c
··· 87 87 } 88 88 } 89 89 90 + static bool gc_rope_candidate_is_valid(ant_rope_heap_t *rope) { 91 + if (!rope || !gc_ropes_mark(rope)) return false; 92 + if (rope->depth >= ROPE_MAX_DEPTH) return false; 93 + if (rope->len >= ROPE_FLATTEN_THRESHOLD) return false; 94 + return true; 95 + } 96 + 97 + static bool gc_builder_candidate_is_valid(ant_string_builder_t *builder) { 98 + if (!builder || !gc_ropes_mark(builder)) return false; 99 + if (builder->tail_len > STR_BUILDER_TAIL_CAP) return false; 100 + if (builder->ascii_state > STR_ASCII_NO) return false; 101 + if (!builder->head && builder->chunk_tail) return false; 102 + if (builder->head && !gc_ropes_mark(builder->head)) return false; 103 + if (builder->chunk_tail && !gc_ropes_mark(builder->chunk_tail)) return false; 104 + return true; 105 + } 106 + 90 107 static void gc_mark_str(ant_t *js, ant_value_t v) { 91 108 static const void *dispatch[] = { 92 109 [STR_HEAP_TAG_FLAT] = &&l_flat, ··· 107 124 108 125 l_rope: { 109 126 ant_rope_heap_t *rope = (ant_rope_heap_t *)(data & ~STR_HEAP_TAG_MASK); 110 - if (!rope || !gc_ropes_mark(rope)) return; 127 + if (!gc_rope_candidate_is_valid(rope)) return; 111 128 gc_mark_str(js, rope->left); 112 129 gc_mark_str(js, rope->right); 113 130 gc_mark_str(js, rope->cached); ··· 116 133 117 134 l_builder: { 118 135 ant_string_builder_t *builder = (ant_string_builder_t *)(data & ~STR_HEAP_TAG_MASK); 119 - if (!builder || !gc_ropes_mark(builder)) return; 136 + if (!gc_builder_candidate_is_valid(builder)) return; 120 137 gc_mark_value(js, builder->cached); 121 - for (ant_builder_chunk_t *chunk = builder->head; chunk; chunk = chunk->next) { 122 - if (gc_ropes_mark(chunk)) gc_mark_value(js, chunk->value); 138 + for (ant_builder_chunk_t *chunk = builder->head; chunk;) { 139 + if (!gc_ropes_mark(chunk)) return; 140 + ant_builder_chunk_t *next = chunk->next; 141 + gc_mark_value(js, chunk->value); 142 + chunk = next; 123 143 } 124 144 return; 125 145 }
+962 -14
src/silver/swarm.c
··· 138 138 MIR_reg_t *regs; 139 139 MIR_reg_t *d_regs; 140 140 sv_func_t **known_func; 141 + struct jit_method_ic_info *method_ic; 141 142 uint8_t *slot_type; 142 143 uint64_t *known_const; 143 144 bool *has_const; ··· 145 146 int max; 146 147 } jit_vstack_t; 147 148 149 + typedef struct jit_method_ic_info { 150 + sv_ic_entry_t *ic; 151 + ant_shape_t *receiver_shape; 152 + sv_func_t *callee; 153 + bool valid; 154 + } jit_method_ic_info_t; 155 + 156 + static bool jit_atom_matches_shape_prop(sv_atom_t *atom, 157 + const ant_shape_prop_t *prop) { 158 + if (!atom || !prop || prop->type != ANT_SHAPE_KEY_STRING || !prop->key.interned) 159 + return false; 160 + if (prop->key.interned == atom->str) return true; 161 + return strncmp(prop->key.interned, atom->str, atom->len) == 0 && 162 + prop->key.interned[atom->len] == '\0'; 163 + } 164 + 148 165 static MIR_reg_t vstack_push(jit_vstack_t *vs) { 149 166 if (vs->known_func) vs->known_func[vs->sp] = NULL; 167 + if (vs->method_ic) vs->method_ic[vs->sp] = (jit_method_ic_info_t){0}; 150 168 if (vs->slot_type) vs->slot_type[vs->sp] = 0; 151 169 if (vs->has_const) vs->has_const[vs->sp] = false; 152 170 return vs->regs[vs->sp++]; ··· 154 172 155 173 static MIR_reg_t vstack_push_const(jit_vstack_t *vs, uint64_t val) { 156 174 if (vs->known_func) vs->known_func[vs->sp] = NULL; 175 + if (vs->method_ic) vs->method_ic[vs->sp] = (jit_method_ic_info_t){0}; 157 176 if (vs->slot_type) vs->slot_type[vs->sp] = 0; 158 177 if (vs->has_const) { vs->has_const[vs->sp] = true; vs->known_const[vs->sp] = val; } 159 178 return vs->regs[vs->sp++]; ··· 871 890 return true; 872 891 } 873 892 893 + static bool jit_ic_try_resolve_method( 894 + sv_func_t *func, 895 + uint16_t ic_idx, 896 + sv_atom_t *atom, 897 + jit_method_ic_info_t *out 898 + ) { 899 + if (out) *out = (jit_method_ic_info_t){0}; 900 + if (!func || !func->ic_slots || !atom || is_length_key(atom->str, atom->len)) 901 + return false; 902 + if (ic_idx == UINT16_MAX || ic_idx >= func->ic_count) return false; 903 + 904 + sv_ic_entry_t *ic = &func->ic_slots[ic_idx]; 905 + if (!sv_gf_ic_active(ic->cached_aux)) return false; 906 + if (ic->epoch != ant_ic_epoch_counter) return false; 907 + if (!ic->cached_shape || !ic->cached_holder) return false; 908 + if (ic->cached_index >= ic->cached_holder->prop_count) return false; 909 + 910 + ant_shape_t *prop_shape = ic->cached_is_own 911 + ? ic->cached_shape 912 + : ic->cached_holder->shape; 913 + if (!prop_shape) return false; 914 + 915 + const ant_shape_prop_t *prop = ant_shape_prop_at(prop_shape, ic->cached_index); 916 + if (!prop) return false; 917 + if (!jit_atom_matches_shape_prop(atom, prop)) return false; 918 + if (prop->has_getter || prop->has_setter) return false; 919 + 920 + ant_value_t value = ant_object_prop_get_unchecked(ic->cached_holder, ic->cached_index); 921 + if (vtype(value) != T_FUNC) return false; 922 + sv_closure_t *closure = js_func_closure(value); 923 + if (!closure || !closure->func) return false; 924 + if (closure->func->is_arrow) return false; 925 + if (closure->call_flags & SV_CALL_HAS_BOUND_ARGS) 926 + return false; 927 + if (vtype(closure->bound_this) != T_UNDEF) return false; 928 + 929 + if (out) { 930 + out->ic = ic; 931 + out->receiver_shape = ic->cached_shape; 932 + out->callee = closure->func; 933 + out->valid = true; 934 + } 935 + return true; 936 + } 937 + 938 + static bool jit_ic_direct_this_slot( 939 + sv_func_t *func, 940 + uint16_t ic_idx, 941 + sv_atom_t *atom, 942 + ant_shape_t *receiver_shape, 943 + bool require_writable, 944 + sv_ic_entry_t **out_ic, 945 + uint32_t *out_index 946 + ) { 947 + if (out_ic) *out_ic = NULL; 948 + if (out_index) *out_index = 0; 949 + if (!func || !func->ic_slots || !atom || !receiver_shape) return false; 950 + if (ic_idx == UINT16_MAX || ic_idx >= func->ic_count) return false; 951 + if (is_length_key(atom->str, atom->len)) return false; 952 + 953 + sv_ic_entry_t *ic = &func->ic_slots[ic_idx]; 954 + if (!require_writable && !sv_gf_ic_active(ic->cached_aux)) return false; 955 + if (ic->epoch != ant_ic_epoch_counter) return false; 956 + if (!require_writable && !ic->cached_is_own) return false; 957 + if (ic->cached_shape != receiver_shape) return false; 958 + if (!ic->cached_holder || ic->cached_holder->shape != receiver_shape) return false; 959 + if (ic->cached_index >= ic->cached_holder->prop_count) return false; 960 + 961 + const ant_shape_prop_t *prop = ant_shape_prop_at(receiver_shape, ic->cached_index); 962 + if (!prop) return false; 963 + if (!jit_atom_matches_shape_prop(atom, prop)) return false; 964 + if (prop->has_getter || prop->has_setter) return false; 965 + if (require_writable && (prop->attrs & ANT_PROP_ATTR_WRITABLE) == 0) 966 + return false; 967 + 968 + if (out_ic) *out_ic = ic; 969 + if (out_index) *out_index = ic->cached_index; 970 + return true; 971 + } 972 + 973 + static bool jit_shape_direct_this_slot( 974 + sv_atom_t *atom, 975 + ant_shape_t *receiver_shape, 976 + bool require_writable, 977 + uint32_t *out_index 978 + ) { 979 + if (out_index) *out_index = 0; 980 + if (!atom || !receiver_shape || is_length_key(atom->str, atom->len)) 981 + return false; 982 + 983 + int32_t slot = ant_shape_lookup_interned(receiver_shape, atom->str); 984 + if (slot < 0) { 985 + uint32_t count = ant_shape_count(receiver_shape); 986 + for (uint32_t i = 0; i < count; i++) { 987 + const ant_shape_prop_t *candidate = ant_shape_prop_at(receiver_shape, i); 988 + if (jit_atom_matches_shape_prop(atom, candidate)) { 989 + slot = (int32_t)i; 990 + break; 991 + } 992 + } 993 + } 994 + if (slot < 0) return false; 995 + 996 + const ant_shape_prop_t *prop = ant_shape_prop_at(receiver_shape, (uint32_t)slot); 997 + if (!prop) return false; 998 + if (!jit_atom_matches_shape_prop(atom, prop)) return false; 999 + if (prop->has_getter || prop->has_setter) return false; 1000 + if (require_writable && (prop->attrs & ANT_PROP_ATTR_WRITABLE) == 0) 1001 + return false; 1002 + 1003 + if (out_index) *out_index = (uint32_t)slot; 1004 + return true; 1005 + } 1006 + 1007 + static void mir_emit_ic_epoch_guard( 1008 + MIR_context_t ctx, 1009 + MIR_item_t fn, 1010 + sv_ic_entry_t *ic, 1011 + MIR_reg_t r_global_epoch, 1012 + MIR_label_t slow, 1013 + const char *name 1014 + ) { 1015 + if (!ic || !r_global_epoch) { 1016 + MIR_append_insn(ctx, fn, 1017 + MIR_new_insn(ctx, MIR_JMP, MIR_new_label_op(ctx, slow))); 1018 + return; 1019 + } 1020 + 1021 + char rn_ic[48], rn_epoch[48], rn_cur_epoch[48]; 1022 + snprintf(rn_ic, sizeof(rn_ic), "%s_ic", name); 1023 + snprintf(rn_epoch, sizeof(rn_epoch), "%s_epoch", name); 1024 + snprintf(rn_cur_epoch, sizeof(rn_cur_epoch), "%s_cur_epoch", name); 1025 + 1026 + MIR_reg_t r_ic = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_ic); 1027 + MIR_reg_t r_epoch = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_epoch); 1028 + MIR_reg_t r_cur_epoch = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_cur_epoch); 1029 + 1030 + MIR_append_insn(ctx, fn, 1031 + MIR_new_insn(ctx, MIR_MOV, 1032 + MIR_new_reg_op(ctx, r_ic), 1033 + MIR_new_uint_op(ctx, (uint64_t)(uintptr_t)ic))); 1034 + MIR_append_insn(ctx, fn, 1035 + MIR_new_insn(ctx, MIR_MOV, 1036 + MIR_new_reg_op(ctx, r_epoch), 1037 + MIR_new_mem_op(ctx, MIR_T_U32, 1038 + (MIR_disp_t)offsetof(sv_ic_entry_t, epoch), r_ic, 0, 1))); 1039 + MIR_append_insn(ctx, fn, 1040 + MIR_new_insn(ctx, MIR_MOV, 1041 + MIR_new_reg_op(ctx, r_cur_epoch), 1042 + MIR_new_mem_op(ctx, MIR_T_U32, 0, r_global_epoch, 0, 1))); 1043 + MIR_append_insn(ctx, fn, 1044 + MIR_new_insn(ctx, MIR_BNE, 1045 + MIR_new_label_op(ctx, slow), 1046 + MIR_new_reg_op(ctx, r_epoch), 1047 + MIR_new_reg_op(ctx, r_cur_epoch))); 1048 + } 1049 + 1050 + static void mir_emit_ic_active_epoch_guard( 1051 + MIR_context_t ctx, 1052 + MIR_item_t fn, 1053 + sv_ic_entry_t *ic, 1054 + MIR_reg_t r_global_epoch, 1055 + MIR_label_t slow, 1056 + const char *name 1057 + ) { 1058 + if (!ic || !r_global_epoch) { 1059 + MIR_append_insn(ctx, fn, 1060 + MIR_new_insn(ctx, MIR_JMP, MIR_new_label_op(ctx, slow))); 1061 + return; 1062 + } 1063 + 1064 + char rn_ic[48], rn_aux[48], rn_epoch[48], rn_cur_epoch[48]; 1065 + snprintf(rn_ic, sizeof(rn_ic), "%s_ic", name); 1066 + snprintf(rn_aux, sizeof(rn_aux), "%s_aux", name); 1067 + snprintf(rn_epoch, sizeof(rn_epoch), "%s_epoch", name); 1068 + snprintf(rn_cur_epoch, sizeof(rn_cur_epoch), "%s_cur_epoch", name); 1069 + 1070 + MIR_reg_t r_ic = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_ic); 1071 + MIR_reg_t r_aux = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_aux); 1072 + MIR_reg_t r_epoch = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_epoch); 1073 + MIR_reg_t r_cur_epoch = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_cur_epoch); 1074 + 1075 + MIR_append_insn(ctx, fn, 1076 + MIR_new_insn(ctx, MIR_MOV, 1077 + MIR_new_reg_op(ctx, r_ic), 1078 + MIR_new_uint_op(ctx, (uint64_t)(uintptr_t)ic))); 1079 + MIR_append_insn(ctx, fn, 1080 + MIR_new_insn(ctx, MIR_MOV, 1081 + MIR_new_reg_op(ctx, r_aux), 1082 + MIR_new_mem_op(ctx, MIR_T_I64, 1083 + (MIR_disp_t)offsetof(sv_ic_entry_t, cached_aux), r_ic, 0, 1))); 1084 + MIR_append_insn(ctx, fn, 1085 + MIR_new_insn(ctx, MIR_AND, 1086 + MIR_new_reg_op(ctx, r_aux), 1087 + MIR_new_reg_op(ctx, r_aux), 1088 + MIR_new_uint_op(ctx, (uint64_t)SV_GF_IC_AUX_ACTIVE_BIT))); 1089 + MIR_append_insn(ctx, fn, 1090 + MIR_new_insn(ctx, MIR_BEQ, 1091 + MIR_new_label_op(ctx, slow), 1092 + MIR_new_reg_op(ctx, r_aux), 1093 + MIR_new_int_op(ctx, 0))); 1094 + MIR_append_insn(ctx, fn, 1095 + MIR_new_insn(ctx, MIR_MOV, 1096 + MIR_new_reg_op(ctx, r_epoch), 1097 + MIR_new_mem_op(ctx, MIR_T_U32, 1098 + (MIR_disp_t)offsetof(sv_ic_entry_t, epoch), r_ic, 0, 1))); 1099 + MIR_append_insn(ctx, fn, 1100 + MIR_new_insn(ctx, MIR_MOV, 1101 + MIR_new_reg_op(ctx, r_cur_epoch), 1102 + MIR_new_mem_op(ctx, MIR_T_U32, 0, r_global_epoch, 0, 1))); 1103 + MIR_append_insn(ctx, fn, 1104 + MIR_new_insn(ctx, MIR_BNE, 1105 + MIR_new_label_op(ctx, slow), 1106 + MIR_new_reg_op(ctx, r_epoch), 1107 + MIR_new_reg_op(ctx, r_cur_epoch))); 1108 + } 1109 + 1110 + static void mir_emit_method_receiver_shape_guard( 1111 + MIR_context_t ctx, 1112 + MIR_item_t fn, 1113 + jit_method_ic_info_t *mi, 1114 + MIR_reg_t receiver, 1115 + MIR_reg_t r_obj_ptr, 1116 + MIR_reg_t r_shape, 1117 + MIR_reg_t r_tag, 1118 + MIR_reg_t r_global_epoch, 1119 + MIR_label_t slow, 1120 + int id 1121 + ) { 1122 + char guard_name[48]; 1123 + snprintf(guard_name, sizeof(guard_name), "minl%d_recv", id); 1124 + mir_emit_ic_active_epoch_guard(ctx, fn, mi ? mi->ic : NULL, 1125 + r_global_epoch, slow, guard_name); 1126 + 1127 + mir_emit_value_to_objptr_or_jmp(ctx, fn, receiver, r_obj_ptr, r_tag, slow); 1128 + 1129 + char rn_ic[48], rn_expected_shape[48]; 1130 + snprintf(rn_ic, sizeof(rn_ic), "minl%d_recv_shape_ic", id); 1131 + snprintf(rn_expected_shape, sizeof(rn_expected_shape), "minl%d_recv_shape", id); 1132 + MIR_reg_t r_ic = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_ic); 1133 + MIR_reg_t r_expected_shape = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_expected_shape); 1134 + MIR_append_insn(ctx, fn, 1135 + MIR_new_insn(ctx, MIR_MOV, 1136 + MIR_new_reg_op(ctx, r_ic), 1137 + MIR_new_uint_op(ctx, (uint64_t)(uintptr_t)(mi ? mi->ic : NULL)))); 1138 + MIR_append_insn(ctx, fn, 1139 + MIR_new_insn(ctx, MIR_MOV, 1140 + MIR_new_reg_op(ctx, r_expected_shape), 1141 + MIR_new_mem_op(ctx, MIR_T_P, 1142 + (MIR_disp_t)offsetof(sv_ic_entry_t, cached_shape), r_ic, 0, 1))); 1143 + MIR_append_insn(ctx, fn, 1144 + MIR_new_insn(ctx, MIR_MOV, 1145 + MIR_new_reg_op(ctx, r_shape), 1146 + MIR_new_mem_op(ctx, MIR_T_P, 1147 + (MIR_disp_t)offsetof(ant_object_t, shape), r_obj_ptr, 0, 1))); 1148 + MIR_append_insn(ctx, fn, 1149 + MIR_new_insn(ctx, MIR_BNE, 1150 + MIR_new_label_op(ctx, slow), 1151 + MIR_new_reg_op(ctx, r_shape), 1152 + MIR_new_reg_op(ctx, r_expected_shape))); 1153 + } 1154 + 1155 + static void mir_emit_this_slot_load( 1156 + MIR_context_t ctx, 1157 + MIR_item_t fn, 1158 + MIR_reg_t dst, 1159 + MIR_reg_t r_obj_ptr, 1160 + uint32_t prop_index, 1161 + MIR_label_t slow, 1162 + int id, 1163 + int bc_off 1164 + ) { 1165 + char rn_idx[48], rn_pc[48], rn_il[48], rn_ovf[48], rn_ovi[48]; 1166 + snprintf(rn_idx, sizeof(rn_idx), "inl%d_gf_idx_%d", id, bc_off); 1167 + snprintf(rn_pc, sizeof(rn_pc), "inl%d_gf_pc_%d", id, bc_off); 1168 + snprintf(rn_il, sizeof(rn_il), "inl%d_gf_il_%d", id, bc_off); 1169 + snprintf(rn_ovf, sizeof(rn_ovf), "inl%d_gf_ovf_%d", id, bc_off); 1170 + snprintf(rn_ovi, sizeof(rn_ovi), "inl%d_gf_ovi_%d", id, bc_off); 1171 + 1172 + MIR_reg_t r_idx = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_idx); 1173 + MIR_reg_t r_prop_count = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_pc); 1174 + MIR_reg_t r_inobj_limit = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_il); 1175 + MIR_reg_t r_overflow = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_ovf); 1176 + MIR_reg_t r_overflow_idx = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_ovi); 1177 + MIR_label_t load_overflow = MIR_new_label(ctx); 1178 + MIR_label_t done = MIR_new_label(ctx); 1179 + 1180 + MIR_append_insn(ctx, fn, 1181 + MIR_new_insn(ctx, MIR_MOV, 1182 + MIR_new_reg_op(ctx, r_idx), 1183 + MIR_new_uint_op(ctx, (uint64_t)prop_index))); 1184 + MIR_append_insn(ctx, fn, 1185 + MIR_new_insn(ctx, MIR_MOV, 1186 + MIR_new_reg_op(ctx, r_prop_count), 1187 + MIR_new_mem_op(ctx, MIR_T_U32, 1188 + (MIR_disp_t)offsetof(ant_object_t, prop_count), r_obj_ptr, 0, 1))); 1189 + MIR_append_insn(ctx, fn, 1190 + MIR_new_insn(ctx, MIR_UBGE, 1191 + MIR_new_label_op(ctx, slow), 1192 + MIR_new_reg_op(ctx, r_idx), 1193 + MIR_new_reg_op(ctx, r_prop_count))); 1194 + MIR_append_insn(ctx, fn, 1195 + MIR_new_insn(ctx, MIR_MOV, 1196 + MIR_new_reg_op(ctx, r_inobj_limit), 1197 + MIR_new_mem_op(ctx, MIR_T_U8, 1198 + (MIR_disp_t)offsetof(ant_object_t, inobj_limit), r_obj_ptr, 0, 1))); 1199 + MIR_append_insn(ctx, fn, 1200 + MIR_new_insn(ctx, MIR_UBGE, 1201 + MIR_new_label_op(ctx, load_overflow), 1202 + MIR_new_reg_op(ctx, r_idx), 1203 + MIR_new_reg_op(ctx, r_inobj_limit))); 1204 + MIR_append_insn(ctx, fn, 1205 + MIR_new_insn(ctx, MIR_MOV, 1206 + MIR_new_reg_op(ctx, dst), 1207 + MIR_new_mem_op(ctx, MIR_T_I64, 1208 + (MIR_disp_t)offsetof(ant_object_t, inobj), r_obj_ptr, r_idx, 8))); 1209 + MIR_append_insn(ctx, fn, 1210 + MIR_new_insn(ctx, MIR_JMP, MIR_new_label_op(ctx, done))); 1211 + 1212 + MIR_append_insn(ctx, fn, load_overflow); 1213 + MIR_append_insn(ctx, fn, 1214 + MIR_new_insn(ctx, MIR_MOV, 1215 + MIR_new_reg_op(ctx, r_overflow), 1216 + MIR_new_mem_op(ctx, MIR_T_P, 1217 + (MIR_disp_t)offsetof(ant_object_t, overflow_prop), r_obj_ptr, 0, 1))); 1218 + MIR_append_insn(ctx, fn, 1219 + MIR_new_insn(ctx, MIR_BEQ, 1220 + MIR_new_label_op(ctx, slow), 1221 + MIR_new_reg_op(ctx, r_overflow), 1222 + MIR_new_int_op(ctx, 0))); 1223 + MIR_append_insn(ctx, fn, 1224 + MIR_new_insn(ctx, MIR_SUB, 1225 + MIR_new_reg_op(ctx, r_overflow_idx), 1226 + MIR_new_reg_op(ctx, r_idx), 1227 + MIR_new_reg_op(ctx, r_inobj_limit))); 1228 + MIR_append_insn(ctx, fn, 1229 + MIR_new_insn(ctx, MIR_MOV, 1230 + MIR_new_reg_op(ctx, dst), 1231 + MIR_new_mem_op(ctx, MIR_T_I64, 0, r_overflow, r_overflow_idx, 8))); 1232 + MIR_append_insn(ctx, fn, done); 1233 + } 1234 + 1235 + static void mir_emit_this_slot_store_num( 1236 + MIR_context_t ctx, 1237 + MIR_item_t fn, 1238 + MIR_reg_t val, 1239 + MIR_reg_t r_obj_ptr, 1240 + uint32_t prop_index, 1241 + MIR_label_t slow, 1242 + MIR_reg_t r_bool, 1243 + int id, 1244 + int bc_off 1245 + ) { 1246 + mir_emit_is_num_guard(ctx, fn, r_bool, val, slow); 1247 + 1248 + char rn_idx[48], rn_pc[48], rn_il[48], rn_ovf[48], rn_ovi[48]; 1249 + snprintf(rn_idx, sizeof(rn_idx), "inl%d_pf_idx_%d", id, bc_off); 1250 + snprintf(rn_pc, sizeof(rn_pc), "inl%d_pf_pc_%d", id, bc_off); 1251 + snprintf(rn_il, sizeof(rn_il), "inl%d_pf_il_%d", id, bc_off); 1252 + snprintf(rn_ovf, sizeof(rn_ovf), "inl%d_pf_ovf_%d", id, bc_off); 1253 + snprintf(rn_ovi, sizeof(rn_ovi), "inl%d_pf_ovi_%d", id, bc_off); 1254 + 1255 + MIR_reg_t r_idx = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_idx); 1256 + MIR_reg_t r_prop_count = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_pc); 1257 + MIR_reg_t r_inobj_limit = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_il); 1258 + MIR_reg_t r_overflow = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_ovf); 1259 + MIR_reg_t r_overflow_idx = MIR_new_func_reg(ctx, fn->u.func, MIR_T_I64, rn_ovi); 1260 + MIR_label_t store_overflow = MIR_new_label(ctx); 1261 + MIR_label_t done = MIR_new_label(ctx); 1262 + 1263 + MIR_append_insn(ctx, fn, 1264 + MIR_new_insn(ctx, MIR_MOV, 1265 + MIR_new_reg_op(ctx, r_idx), 1266 + MIR_new_uint_op(ctx, (uint64_t)prop_index))); 1267 + MIR_append_insn(ctx, fn, 1268 + MIR_new_insn(ctx, MIR_MOV, 1269 + MIR_new_reg_op(ctx, r_prop_count), 1270 + MIR_new_mem_op(ctx, MIR_T_U32, 1271 + (MIR_disp_t)offsetof(ant_object_t, prop_count), r_obj_ptr, 0, 1))); 1272 + MIR_append_insn(ctx, fn, 1273 + MIR_new_insn(ctx, MIR_UBGE, 1274 + MIR_new_label_op(ctx, slow), 1275 + MIR_new_reg_op(ctx, r_idx), 1276 + MIR_new_reg_op(ctx, r_prop_count))); 1277 + MIR_append_insn(ctx, fn, 1278 + MIR_new_insn(ctx, MIR_MOV, 1279 + MIR_new_reg_op(ctx, r_inobj_limit), 1280 + MIR_new_mem_op(ctx, MIR_T_U8, 1281 + (MIR_disp_t)offsetof(ant_object_t, inobj_limit), r_obj_ptr, 0, 1))); 1282 + MIR_append_insn(ctx, fn, 1283 + MIR_new_insn(ctx, MIR_UBGE, 1284 + MIR_new_label_op(ctx, store_overflow), 1285 + MIR_new_reg_op(ctx, r_idx), 1286 + MIR_new_reg_op(ctx, r_inobj_limit))); 1287 + MIR_append_insn(ctx, fn, 1288 + MIR_new_insn(ctx, MIR_MOV, 1289 + MIR_new_mem_op(ctx, MIR_T_I64, 1290 + (MIR_disp_t)offsetof(ant_object_t, inobj), r_obj_ptr, r_idx, 8), 1291 + MIR_new_reg_op(ctx, val))); 1292 + MIR_append_insn(ctx, fn, 1293 + MIR_new_insn(ctx, MIR_JMP, MIR_new_label_op(ctx, done))); 1294 + 1295 + MIR_append_insn(ctx, fn, store_overflow); 1296 + MIR_append_insn(ctx, fn, 1297 + MIR_new_insn(ctx, MIR_MOV, 1298 + MIR_new_reg_op(ctx, r_overflow), 1299 + MIR_new_mem_op(ctx, MIR_T_P, 1300 + (MIR_disp_t)offsetof(ant_object_t, overflow_prop), r_obj_ptr, 0, 1))); 1301 + MIR_append_insn(ctx, fn, 1302 + MIR_new_insn(ctx, MIR_BEQ, 1303 + MIR_new_label_op(ctx, slow), 1304 + MIR_new_reg_op(ctx, r_overflow), 1305 + MIR_new_int_op(ctx, 0))); 1306 + MIR_append_insn(ctx, fn, 1307 + MIR_new_insn(ctx, MIR_SUB, 1308 + MIR_new_reg_op(ctx, r_overflow_idx), 1309 + MIR_new_reg_op(ctx, r_idx), 1310 + MIR_new_reg_op(ctx, r_inobj_limit))); 1311 + MIR_append_insn(ctx, fn, 1312 + MIR_new_insn(ctx, MIR_MOV, 1313 + MIR_new_mem_op(ctx, MIR_T_I64, 0, r_overflow, r_overflow_idx, 8), 1314 + MIR_new_reg_op(ctx, val))); 1315 + MIR_append_insn(ctx, fn, done); 1316 + } 1317 + 874 1318 875 1319 #define MAX_OSR_ENTRIES 64 876 1320 ··· 1019 1463 } 1020 1464 1021 1465 1022 - #define JIT_INLINE_MAX_BYTECODE 128 1466 + #define JIT_INLINE_MAX_BYTECODE 64 1467 + #define JIT_INLINE_MAX_DEPTH 3 1468 + #define JIT_INLINE_BUDGET 512 1023 1469 1024 1470 static bool jit_inlineable(sv_func_t *f) { 1025 1471 if (!f) return false; ··· 1040 1486 case OP_GET_LOCAL: case OP_GET_LOCAL8: 1041 1487 case OP_PUT_LOCAL: case OP_PUT_LOCAL8: 1042 1488 case OP_SET_LOCAL: case OP_SET_LOCAL8: 1489 + case OP_SET_LOCAL_UNDEF: 1043 1490 case OP_GET_UPVAL: 1044 1491 case OP_POP: case OP_DUP: case OP_DUP2: 1045 1492 case OP_INSERT2: case OP_INSERT3: ··· 1053 1500 case OP_JMP_TRUE8: case OP_JMP_FALSE8: 1054 1501 case OP_JMP_TRUE_PEEK: case OP_JMP_FALSE_PEEK: 1055 1502 case OP_RETURN: case OP_RETURN_UNDEF: 1056 - case OP_GET_FIELD: case OP_GET_FIELD2: case OP_GET_GLOBAL: 1503 + case OP_GET_FIELD: case OP_GET_FIELD2: case OP_PUT_FIELD: 1504 + case OP_GET_GLOBAL: 1057 1505 case OP_NOP: case OP_LINE_NUM: case OP_COL_NUM: case OP_LABEL: 1058 1506 break; 1059 1507 case OP_SPECIAL_OBJ: ··· 1132 1580 MIR_reg_t r_bool, MIR_reg_t *p_d_slot, int id, 1133 1581 MIR_reg_t r_inl_closure, MIR_reg_t r_inl_this, 1134 1582 MIR_reg_t r_inl_new_target, MIR_reg_t r_inl_super, 1583 + ant_shape_t *guarded_this_shape, 1584 + MIR_reg_t r_guarded_this_obj, MIR_reg_t r_guarded_this_shape, 1585 + MIR_reg_t r_global_epoch, 1135 1586 MIR_reg_t r_vm, MIR_reg_t r_js, 1136 1587 MIR_item_t helper2_proto, MIR_item_t imp_seq, 1137 1588 MIR_item_t imp_sne, MIR_item_t imp_eq, MIR_item_t imp_ne, 1138 1589 MIR_item_t gf_proto, MIR_item_t imp_get_field, 1590 + MIR_item_t put_field_proto, MIR_item_t imp_put_field, 1139 1591 MIR_item_t gg_proto, MIR_item_t imp_gg, 1140 1592 MIR_item_t special_obj_proto, MIR_item_t imp_special_obj 1141 1593 ) { 1142 1594 int inl_max_stack = callee->max_stack > 0 ? callee->max_stack : 4; 1143 1595 MIR_reg_t inl_vs[inl_max_stack]; 1596 + uint8_t inl_slot_type[inl_max_stack]; 1597 + bool inl_is_this[inl_max_stack]; 1598 + memset(inl_slot_type, SLOT_BOXED, sizeof(inl_slot_type)); 1599 + memset(inl_is_this, 0, sizeof(inl_is_this)); 1600 + #define INL_RESET_SLOT(i) do { inl_slot_type[(i)] = SLOT_BOXED; inl_is_this[(i)] = false; } while (0) 1601 + #define INL_MARK_NUM(i) do { inl_slot_type[(i)] = SLOT_NUM; inl_is_this[(i)] = false; } while (0) 1144 1602 for (int i = 0; i < inl_max_stack; i++) { 1145 1603 char rn[32]; snprintf(rn, sizeof(rn), "inl%d_s%d", id, i); 1146 1604 inl_vs[i] = MIR_new_func_reg(ctx, jit_func->u.func, MIR_JSVAL, rn); ··· 1179 1637 switch (op) { 1180 1638 case OP_GET_ARG: { 1181 1639 uint16_t idx = sv_get_u16(ip + 1); 1640 + int dst_i = isp; 1182 1641 MIR_reg_t dst = inl_vs[isp++]; 1642 + INL_RESET_SLOT(dst_i); 1183 1643 if ((int)idx < caller_argc) 1184 1644 MIR_append_insn(ctx, jit_func, 1185 1645 MIR_new_insn(ctx, MIR_MOV, ··· 1193 1653 case OP_CONST_I8: { 1194 1654 double d = (double)(int8_t)sv_get_i8(ip + 1); 1195 1655 union { double d; uint64_t u; } u = {d}; 1656 + int dst_i = isp; 1196 1657 mir_load_imm(ctx, jit_func, inl_vs[isp++], u.u); 1658 + INL_MARK_NUM(dst_i); 1197 1659 break; 1198 1660 } 1199 1661 case OP_CONST: { 1200 1662 uint32_t idx = sv_get_u32(ip + 1); 1201 1663 if (idx >= (uint32_t)callee->const_count) return false; 1202 1664 ant_value_t cv = callee->constants[idx]; 1665 + int dst_i = isp; 1203 1666 MIR_reg_t dst = inl_vs[isp++]; 1667 + INL_RESET_SLOT(dst_i); 1204 1668 if (jit_const_is_heap(cv)) 1205 1669 mir_load_const_slot(ctx, jit_func, dst, &callee->constants[idx]); 1206 - else 1670 + else { 1207 1671 mir_load_imm(ctx, jit_func, dst, cv); 1672 + if (vtype(cv) == T_NUM) INL_MARK_NUM(dst_i); 1673 + } 1208 1674 break; 1209 1675 } 1210 1676 case OP_CONST8: { 1211 1677 uint8_t idx = sv_get_u8(ip + 1); 1212 1678 if (idx >= (uint8_t)callee->const_count) return false; 1213 1679 ant_value_t cv = callee->constants[idx]; 1680 + int dst_i = isp; 1214 1681 MIR_reg_t dst = inl_vs[isp++]; 1682 + INL_RESET_SLOT(dst_i); 1215 1683 if (jit_const_is_heap(cv)) 1216 1684 mir_load_const_slot(ctx, jit_func, dst, &callee->constants[idx]); 1217 - else 1685 + else { 1218 1686 mir_load_imm(ctx, jit_func, dst, cv); 1687 + if (vtype(cv) == T_NUM) INL_MARK_NUM(dst_i); 1688 + } 1219 1689 break; 1220 1690 } 1221 - case OP_UNDEF: mir_load_imm(ctx, jit_func, inl_vs[isp++], mkval(T_UNDEF, 0)); break; 1222 - case OP_NULL: mir_load_imm(ctx, jit_func, inl_vs[isp++], mkval(T_NULL, 0)); break; 1223 - case OP_TRUE: mir_load_imm(ctx, jit_func, inl_vs[isp++], js_true); break; 1224 - case OP_FALSE: mir_load_imm(ctx, jit_func, inl_vs[isp++], js_false); break; 1691 + case OP_UNDEF: { int dst_i = isp; mir_load_imm(ctx, jit_func, inl_vs[isp++], mkval(T_UNDEF, 0)); INL_RESET_SLOT(dst_i); break; } 1692 + case OP_NULL: { int dst_i = isp; mir_load_imm(ctx, jit_func, inl_vs[isp++], mkval(T_NULL, 0)); INL_RESET_SLOT(dst_i); break; } 1693 + case OP_TRUE: { int dst_i = isp; mir_load_imm(ctx, jit_func, inl_vs[isp++], js_true); INL_RESET_SLOT(dst_i); break; } 1694 + case OP_FALSE: { int dst_i = isp; mir_load_imm(ctx, jit_func, inl_vs[isp++], js_false); INL_RESET_SLOT(dst_i); break; } 1225 1695 1226 1696 case OP_THIS: { 1697 + int dst_i = isp; 1227 1698 MIR_append_insn(ctx, jit_func, 1228 1699 MIR_new_insn(ctx, MIR_MOV, 1229 1700 MIR_new_reg_op(ctx, inl_vs[isp++]), 1230 1701 MIR_new_reg_op(ctx, r_inl_this))); 1702 + INL_RESET_SLOT(dst_i); 1703 + inl_is_this[dst_i] = true; 1231 1704 break; 1232 1705 } 1233 1706 1234 1707 case OP_GET_LOCAL: { 1235 1708 uint16_t idx = sv_get_u16(ip + 1); 1236 1709 if (idx >= (uint16_t)inl_n_locals) return false; 1710 + int dst_i = isp; 1237 1711 MIR_append_insn(ctx, jit_func, 1238 1712 MIR_new_insn(ctx, MIR_MOV, 1239 1713 MIR_new_reg_op(ctx, inl_vs[isp++]), 1240 1714 MIR_new_reg_op(ctx, inl_locals[idx]))); 1715 + INL_RESET_SLOT(dst_i); 1241 1716 break; 1242 1717 } 1243 1718 case OP_GET_LOCAL8: { 1244 1719 uint8_t idx = sv_get_u8(ip + 1); 1245 1720 if (idx >= (uint8_t)inl_n_locals) return false; 1721 + int dst_i = isp; 1246 1722 MIR_append_insn(ctx, jit_func, 1247 1723 MIR_new_insn(ctx, MIR_MOV, 1248 1724 MIR_new_reg_op(ctx, inl_vs[isp++]), 1249 1725 MIR_new_reg_op(ctx, inl_locals[idx]))); 1726 + INL_RESET_SLOT(dst_i); 1250 1727 break; 1251 1728 } 1252 1729 case OP_PUT_LOCAL: { ··· 1285 1762 MIR_new_reg_op(ctx, inl_vs[isp - 1]))); 1286 1763 break; 1287 1764 } 1765 + case OP_SET_LOCAL_UNDEF: { 1766 + uint16_t idx = sv_get_u16(ip + 1); 1767 + if (idx >= (uint16_t)inl_n_locals) return false; 1768 + mir_load_imm(ctx, jit_func, inl_locals[idx], mkval(T_UNDEF, 0)); 1769 + break; 1770 + } 1288 1771 1289 1772 case OP_GET_UPVAL: { 1290 1773 if (!r_inl_closure) return false; ··· 1298 1781 MIR_reg_t r_uvs = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_uvs); 1299 1782 MIR_reg_t r_uv = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_uv); 1300 1783 MIR_reg_t r_loc = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_loc); 1784 + int dst_i = isp; 1301 1785 MIR_reg_t dst = inl_vs[isp++]; 1786 + INL_RESET_SLOT(dst_i); 1302 1787 1303 1788 MIR_append_insn(ctx, jit_func, 1304 1789 MIR_new_insn(ctx, MIR_MOV, ··· 1327 1812 1328 1813 case OP_POP: isp--; break; 1329 1814 case OP_DUP: { 1815 + inl_slot_type[isp] = inl_slot_type[isp - 1]; 1816 + inl_is_this[isp] = inl_is_this[isp - 1]; 1330 1817 MIR_append_insn(ctx, jit_func, 1331 1818 MIR_new_insn(ctx, MIR_MOV, 1332 1819 MIR_new_reg_op(ctx, inl_vs[isp]), ··· 1338 1825 if (isp < 2) return false; 1339 1826 MIR_reg_t ra = inl_vs[isp - 2]; 1340 1827 MIR_reg_t rb = inl_vs[isp - 1]; 1828 + inl_slot_type[isp] = inl_slot_type[isp - 2]; 1829 + inl_is_this[isp] = inl_is_this[isp - 2]; 1830 + inl_slot_type[isp + 1] = inl_slot_type[isp - 1]; 1831 + inl_is_this[isp + 1] = inl_is_this[isp - 1]; 1341 1832 MIR_append_insn(ctx, jit_func, 1342 1833 MIR_new_insn(ctx, MIR_MOV, 1343 1834 MIR_new_reg_op(ctx, inl_vs[isp]), ··· 1352 1843 1353 1844 case OP_INSERT2: { 1354 1845 if (isp < 2) return false; 1846 + uint8_t a_type = inl_slot_type[isp - 1]; 1847 + bool a_is_this = inl_is_this[isp - 1]; 1848 + uint8_t obj_type = inl_slot_type[isp - 2]; 1849 + bool obj_is_this = inl_is_this[isp - 2]; 1355 1850 char tn[32]; snprintf(tn, sizeof(tn), "inl%d_ins2t", id); 1356 1851 MIR_reg_t r_t = MIR_new_func_reg(ctx, jit_func->u.func, MIR_JSVAL, tn); 1357 1852 MIR_reg_t r_a = inl_vs[isp - 1]; ··· 1372 1867 MIR_new_insn(ctx, MIR_MOV, 1373 1868 MIR_new_reg_op(ctx, inl_vs[isp]), 1374 1869 MIR_new_reg_op(ctx, r_t))); 1870 + inl_slot_type[isp - 2] = a_type; 1871 + inl_is_this[isp - 2] = a_is_this; 1872 + inl_slot_type[isp - 1] = obj_type; 1873 + inl_is_this[isp - 1] = obj_is_this; 1874 + inl_slot_type[isp] = a_type; 1875 + inl_is_this[isp] = a_is_this; 1375 1876 isp++; 1376 1877 break; 1377 1878 } 1378 1879 case OP_INSERT3: { 1379 1880 if (isp < 3) return false; 1881 + uint8_t a_type = inl_slot_type[isp - 1]; 1882 + bool a_is_this = inl_is_this[isp - 1]; 1883 + uint8_t prop_type = inl_slot_type[isp - 2]; 1884 + bool prop_is_this = inl_is_this[isp - 2]; 1885 + uint8_t obj_type = inl_slot_type[isp - 3]; 1886 + bool obj_is_this = inl_is_this[isp - 3]; 1380 1887 char tn[32]; snprintf(tn, sizeof(tn), "inl%d_ins3t", id); 1381 1888 MIR_reg_t r_t = MIR_new_func_reg(ctx, jit_func->u.func, MIR_JSVAL, tn); 1382 1889 MIR_reg_t r_a = inl_vs[isp - 1]; ··· 1402 1909 MIR_new_insn(ctx, MIR_MOV, 1403 1910 MIR_new_reg_op(ctx, inl_vs[isp]), 1404 1911 MIR_new_reg_op(ctx, r_t))); 1912 + inl_slot_type[isp - 3] = a_type; 1913 + inl_is_this[isp - 3] = a_is_this; 1914 + inl_slot_type[isp - 2] = obj_type; 1915 + inl_is_this[isp - 2] = obj_is_this; 1916 + inl_slot_type[isp - 1] = prop_type; 1917 + inl_is_this[isp - 1] = prop_is_this; 1918 + inl_slot_type[isp] = a_type; 1919 + inl_is_this[isp] = a_is_this; 1405 1920 isp++; 1406 1921 break; 1407 1922 } ··· 1410 1925 case OP_ADD_NUM: case OP_SUB_NUM: case OP_MUL_NUM: case OP_DIV_NUM: { 1411 1926 MIR_reg_t rr = inl_vs[--isp]; 1412 1927 MIR_reg_t rl = inl_vs[--isp]; 1928 + int dst_i = isp; 1413 1929 MIR_reg_t rd = inl_vs[isp++]; 1414 1930 1415 1931 if (!(op == OP_ADD_NUM || op == OP_SUB_NUM || ··· 1455 1971 MIR_new_reg_op(ctx, fd1), 1456 1972 MIR_new_reg_op(ctx, fd2))); 1457 1973 mir_d_to_i64(ctx, jit_func, rd, fd3, *p_d_slot); 1974 + INL_MARK_NUM(dst_i); 1458 1975 break; 1459 1976 } 1460 1977 ··· 1473 1990 1474 1991 MIR_reg_t rr = inl_vs[--isp]; 1475 1992 MIR_reg_t rl = inl_vs[--isp]; 1993 + int dst_i = isp; 1476 1994 MIR_reg_t rd = inl_vs[isp++]; 1477 1995 1478 1996 int mn = inl_arith++; ··· 1514 2032 MIR_new_reg_op(ctx, fd1), 1515 2033 MIR_new_reg_op(ctx, fd4))); 1516 2034 mir_d_to_i64(ctx, jit_func, rd, fd5, *p_d_slot); 2035 + INL_MARK_NUM(dst_i); 1517 2036 break; 1518 2037 } 1519 2038 ··· 1542 2061 MIR_new_reg_op(ctx, fd2), 1543 2062 MIR_new_reg_op(ctx, fd1))); 1544 2063 mir_d_to_i64(ctx, jit_func, rs, fd2, *p_d_slot); 2064 + INL_MARK_NUM(isp - 1); 1545 2065 break; 1546 2066 } 1547 2067 1548 2068 case OP_LT: case OP_LE: case OP_GT: case OP_GE: { 1549 2069 MIR_reg_t rr = inl_vs[--isp]; 1550 2070 MIR_reg_t rl = inl_vs[--isp]; 2071 + int dst_i = isp; 1551 2072 MIR_reg_t rd = inl_vs[isp++]; 1552 2073 1553 2074 mir_emit_is_num_guard(ctx, jit_func, r_bool, rl, slow); ··· 1594 2115 MIR_new_reg_op(ctx, rd), 1595 2116 MIR_new_uint_op(ctx, js_false), 1596 2117 MIR_new_reg_op(ctx, r_tmp))); 2118 + INL_RESET_SLOT(dst_i); 1597 2119 break; 1598 2120 } 1599 2121 1600 2122 case OP_SEQ: { 1601 2123 MIR_reg_t rr = inl_vs[--isp]; 1602 2124 MIR_reg_t rl = inl_vs[--isp]; 2125 + int dst_i = isp; 1603 2126 MIR_reg_t rd = inl_vs[isp++]; 2127 + INL_RESET_SLOT(dst_i); 1604 2128 mir_call_helper2(ctx, jit_func, rd, 1605 2129 helper2_proto, imp_seq, 1606 2130 r_vm, r_js, rl, rr); ··· 1609 2133 case OP_SNE: { 1610 2134 MIR_reg_t rr = inl_vs[--isp]; 1611 2135 MIR_reg_t rl = inl_vs[--isp]; 2136 + int dst_i = isp; 1612 2137 MIR_reg_t rd = inl_vs[isp++]; 2138 + INL_RESET_SLOT(dst_i); 1613 2139 mir_call_helper2(ctx, jit_func, rd, 1614 2140 helper2_proto, imp_sne, 1615 2141 r_vm, r_js, rl, rr); ··· 1618 2144 case OP_EQ: { 1619 2145 MIR_reg_t rr = inl_vs[--isp]; 1620 2146 MIR_reg_t rl = inl_vs[--isp]; 2147 + int dst_i = isp; 1621 2148 MIR_reg_t rd = inl_vs[isp++]; 2149 + INL_RESET_SLOT(dst_i); 1622 2150 mir_call_helper2(ctx, jit_func, rd, 1623 2151 helper2_proto, imp_eq, 1624 2152 r_vm, r_js, rl, rr); ··· 1627 2155 case OP_NE: { 1628 2156 MIR_reg_t rr = inl_vs[--isp]; 1629 2157 MIR_reg_t rl = inl_vs[--isp]; 2158 + int dst_i = isp; 1630 2159 MIR_reg_t rd = inl_vs[isp++]; 2160 + INL_RESET_SLOT(dst_i); 1631 2161 mir_call_helper2(ctx, jit_func, rd, 1632 2162 helper2_proto, imp_ne, 1633 2163 r_vm, r_js, rl, rr); ··· 1720 2250 uint32_t idx = sv_get_u32(ip + 1); 1721 2251 if (idx >= (uint32_t)callee->atom_count) return false; 1722 2252 sv_atom_t *atom = &callee->atoms[idx]; 1723 - MIR_reg_t obj = inl_vs[--isp]; 2253 + int obj_i = --isp; 2254 + bool obj_is_this = inl_is_this[obj_i]; 2255 + MIR_reg_t obj = inl_vs[obj_i]; 2256 + int dst_i = isp; 1724 2257 MIR_reg_t dst = inl_vs[isp++]; 2258 + INL_RESET_SLOT(dst_i); 2259 + uint16_t ic_idx = sv_get_u16(ip + 5); 2260 + sv_ic_entry_t *slot_ic = NULL; 2261 + uint32_t slot_idx = 0; 2262 + if (obj_is_this && guarded_this_shape && r_guarded_this_obj && 2263 + jit_ic_direct_this_slot(callee, ic_idx, atom, guarded_this_shape, 2264 + false, &slot_ic, &slot_idx)) { 2265 + char guard_name[48], rn_ic[48], rn_ic_shape[48]; 2266 + snprintf(guard_name, sizeof(guard_name), "inl%d_gf_epoch_%d", id, inl_bc_off); 2267 + snprintf(rn_ic, sizeof(rn_ic), "inl%d_gf_ic_%d", id, inl_bc_off); 2268 + snprintf(rn_ic_shape, sizeof(rn_ic_shape), "inl%d_gf_shape_%d", id, inl_bc_off); 2269 + mir_emit_ic_active_epoch_guard(ctx, jit_func, slot_ic, r_global_epoch, 2270 + slow, guard_name); 2271 + MIR_reg_t r_ic = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_ic); 2272 + MIR_reg_t r_ic_shape = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_ic_shape); 2273 + MIR_append_insn(ctx, jit_func, 2274 + MIR_new_insn(ctx, MIR_MOV, 2275 + MIR_new_reg_op(ctx, r_ic), 2276 + MIR_new_uint_op(ctx, (uint64_t)(uintptr_t)slot_ic))); 2277 + MIR_append_insn(ctx, jit_func, 2278 + MIR_new_insn(ctx, MIR_MOV, 2279 + MIR_new_reg_op(ctx, r_ic_shape), 2280 + MIR_new_mem_op(ctx, MIR_T_P, 2281 + (MIR_disp_t)offsetof(sv_ic_entry_t, cached_shape), r_ic, 0, 1))); 2282 + MIR_append_insn(ctx, jit_func, 2283 + MIR_new_insn(ctx, MIR_BNE, 2284 + MIR_new_label_op(ctx, slow), 2285 + MIR_new_reg_op(ctx, r_guarded_this_shape), 2286 + MIR_new_reg_op(ctx, r_ic_shape))); 2287 + mir_emit_this_slot_load(ctx, jit_func, dst, r_guarded_this_obj, 2288 + slot_idx, slow, id, inl_bc_off); 2289 + break; 2290 + } 1725 2291 MIR_append_insn(ctx, jit_func, 1726 2292 MIR_new_call_insn(ctx, 10, 1727 2293 MIR_new_ref_op(ctx, gf_proto), ··· 1750 2316 uint32_t idx = sv_get_u32(ip + 1); 1751 2317 if (idx >= (uint32_t)callee->atom_count) return false; 1752 2318 sv_atom_t *atom = &callee->atoms[idx]; 2319 + bool obj_is_this = inl_is_this[isp - 1]; 1753 2320 MIR_reg_t obj = inl_vs[isp - 1]; 2321 + int dst_i = isp; 1754 2322 MIR_reg_t dst = inl_vs[isp++]; 2323 + INL_RESET_SLOT(dst_i); 2324 + uint16_t ic_idx = sv_get_u16(ip + 5); 2325 + sv_ic_entry_t *slot_ic = NULL; 2326 + uint32_t slot_idx = 0; 2327 + if (obj_is_this && guarded_this_shape && r_guarded_this_obj && 2328 + jit_ic_direct_this_slot(callee, ic_idx, atom, guarded_this_shape, 2329 + false, &slot_ic, &slot_idx)) { 2330 + char guard_name[48], rn_ic[48], rn_ic_shape[48]; 2331 + snprintf(guard_name, sizeof(guard_name), "inl%d_gf2_epoch_%d", id, inl_bc_off); 2332 + snprintf(rn_ic, sizeof(rn_ic), "inl%d_gf2_ic_%d", id, inl_bc_off); 2333 + snprintf(rn_ic_shape, sizeof(rn_ic_shape), "inl%d_gf2_shape_%d", id, inl_bc_off); 2334 + mir_emit_ic_active_epoch_guard(ctx, jit_func, slot_ic, r_global_epoch, 2335 + slow, guard_name); 2336 + MIR_reg_t r_ic = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_ic); 2337 + MIR_reg_t r_ic_shape = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_ic_shape); 2338 + MIR_append_insn(ctx, jit_func, 2339 + MIR_new_insn(ctx, MIR_MOV, 2340 + MIR_new_reg_op(ctx, r_ic), 2341 + MIR_new_uint_op(ctx, (uint64_t)(uintptr_t)slot_ic))); 2342 + MIR_append_insn(ctx, jit_func, 2343 + MIR_new_insn(ctx, MIR_MOV, 2344 + MIR_new_reg_op(ctx, r_ic_shape), 2345 + MIR_new_mem_op(ctx, MIR_T_P, 2346 + (MIR_disp_t)offsetof(sv_ic_entry_t, cached_shape), r_ic, 0, 1))); 2347 + MIR_append_insn(ctx, jit_func, 2348 + MIR_new_insn(ctx, MIR_BNE, 2349 + MIR_new_label_op(ctx, slow), 2350 + MIR_new_reg_op(ctx, r_guarded_this_shape), 2351 + MIR_new_reg_op(ctx, r_ic_shape))); 2352 + mir_emit_this_slot_load(ctx, jit_func, dst, r_guarded_this_obj, 2353 + slot_idx, slow, id, inl_bc_off); 2354 + break; 2355 + } 1755 2356 MIR_append_insn(ctx, jit_func, 1756 2357 MIR_new_call_insn(ctx, 10, 1757 2358 MIR_new_ref_op(ctx, gf_proto), ··· 1776 2377 MIR_new_uint_op(ctx, JIT_ERR_TAG))); 1777 2378 break; 1778 2379 } 2380 + case OP_PUT_FIELD: { 2381 + uint32_t idx = sv_get_u32(ip + 1); 2382 + if (idx >= (uint32_t)callee->atom_count) return false; 2383 + sv_atom_t *atom = &callee->atoms[idx]; 2384 + int val_i = --isp; 2385 + MIR_reg_t val = inl_vs[val_i]; 2386 + bool val_is_num = inl_slot_type[val_i] == SLOT_NUM; 2387 + int obj_i = --isp; 2388 + MIR_reg_t obj = inl_vs[obj_i]; 2389 + bool obj_is_this = inl_is_this[obj_i]; 2390 + uint16_t ic_idx = sv_get_u16(ip + 5); 2391 + sv_ic_entry_t *slot_ic = NULL; 2392 + uint32_t slot_idx = 0; 2393 + bool direct_slot = false; 2394 + if (obj_is_this && val_is_num && guarded_this_shape && r_guarded_this_obj) { 2395 + direct_slot = jit_ic_direct_this_slot(callee, ic_idx, atom, guarded_this_shape, 2396 + true, &slot_ic, &slot_idx); 2397 + if (!direct_slot) 2398 + direct_slot = jit_shape_direct_this_slot(atom, guarded_this_shape, 2399 + true, &slot_idx); 2400 + } 2401 + if (direct_slot) { 2402 + char guard_name[48], rn_ic[48], rn_ic_shape[48]; 2403 + snprintf(guard_name, sizeof(guard_name), "inl%d_pf_epoch_%d", id, inl_bc_off); 2404 + snprintf(rn_ic, sizeof(rn_ic), "inl%d_pf_ic_%d", id, inl_bc_off); 2405 + snprintf(rn_ic_shape, sizeof(rn_ic_shape), "inl%d_pf_shape_%d", id, inl_bc_off); 2406 + if (slot_ic) { 2407 + mir_emit_ic_epoch_guard(ctx, jit_func, slot_ic, r_global_epoch, 2408 + slow, guard_name); 2409 + MIR_reg_t r_ic = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_ic); 2410 + MIR_reg_t r_ic_shape = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_ic_shape); 2411 + MIR_append_insn(ctx, jit_func, 2412 + MIR_new_insn(ctx, MIR_MOV, 2413 + MIR_new_reg_op(ctx, r_ic), 2414 + MIR_new_uint_op(ctx, (uint64_t)(uintptr_t)slot_ic))); 2415 + MIR_append_insn(ctx, jit_func, 2416 + MIR_new_insn(ctx, MIR_MOV, 2417 + MIR_new_reg_op(ctx, r_ic_shape), 2418 + MIR_new_mem_op(ctx, MIR_T_P, 2419 + (MIR_disp_t)offsetof(sv_ic_entry_t, cached_shape), r_ic, 0, 1))); 2420 + MIR_append_insn(ctx, jit_func, 2421 + MIR_new_insn(ctx, MIR_BNE, 2422 + MIR_new_label_op(ctx, slow), 2423 + MIR_new_reg_op(ctx, r_guarded_this_shape), 2424 + MIR_new_reg_op(ctx, r_ic_shape))); 2425 + } 2426 + mir_emit_this_slot_store_num(ctx, jit_func, val, r_guarded_this_obj, 2427 + slot_idx, slow, r_bool, id, inl_bc_off); 2428 + break; 2429 + } 2430 + MIR_append_insn(ctx, jit_func, 2431 + MIR_new_call_insn(ctx, 9, 2432 + MIR_new_ref_op(ctx, put_field_proto), 2433 + MIR_new_ref_op(ctx, imp_put_field), 2434 + MIR_new_reg_op(ctx, result), 2435 + MIR_new_reg_op(ctx, r_vm), 2436 + MIR_new_reg_op(ctx, r_js), 2437 + MIR_new_reg_op(ctx, obj), 2438 + MIR_new_reg_op(ctx, val), 2439 + MIR_new_uint_op(ctx, (uint64_t)(uintptr_t)atom->str), 2440 + MIR_new_uint_op(ctx, (uint64_t)atom->len))); 2441 + MIR_append_insn(ctx, jit_func, 2442 + MIR_new_insn(ctx, MIR_URSH, 2443 + MIR_new_reg_op(ctx, r_bool), 2444 + MIR_new_reg_op(ctx, result), 2445 + MIR_new_int_op(ctx, NANBOX_TYPE_SHIFT))); 2446 + MIR_append_insn(ctx, jit_func, 2447 + MIR_new_insn(ctx, MIR_BEQ, 2448 + MIR_new_label_op(ctx, slow), 2449 + MIR_new_reg_op(ctx, r_bool), 2450 + MIR_new_uint_op(ctx, JIT_ERR_TAG))); 2451 + break; 2452 + } 1779 2453 case OP_GET_GLOBAL: { 1780 2454 uint32_t idx = sv_get_u32(ip + 1); 1781 2455 if (idx >= (uint32_t)callee->atom_count) return false; 1782 2456 sv_atom_t *atom = &callee->atoms[idx]; 2457 + int dst_i = isp; 1783 2458 MIR_reg_t dst = inl_vs[isp++]; 2459 + INL_RESET_SLOT(dst_i); 1784 2460 MIR_append_insn(ctx, jit_func, 1785 2461 MIR_new_call_insn(ctx, 7, 1786 2462 MIR_new_ref_op(ctx, gg_proto), ··· 1811 2487 1812 2488 case OP_SPECIAL_OBJ: { 1813 2489 uint8_t which = sv_get_u8(ip + 1); 2490 + int dst_i = isp; 1814 2491 MIR_reg_t dst = inl_vs[isp++]; 2492 + INL_RESET_SLOT(dst_i); 1815 2493 if (which == 1) { 1816 2494 MIR_append_insn(ctx, jit_func, 1817 2495 MIR_new_insn(ctx, MIR_MOV, ··· 1845 2523 } 1846 2524 ip += sz; 1847 2525 } 2526 + #undef INL_MARK_NUM 2527 + #undef INL_RESET_SLOT 1848 2528 return true; 1849 2529 } 1850 2530 ··· 2514 3194 vs.max = func->max_stack > 0 ? func->max_stack : 32; 2515 3195 vs.regs = calloc((size_t)vs.max, sizeof(MIR_reg_t)); 2516 3196 vs.known_func = calloc((size_t)vs.max, sizeof(sv_func_t *)); 3197 + vs.method_ic = calloc((size_t)vs.max, sizeof(jit_method_ic_info_t)); 2517 3198 vs.d_regs = calloc((size_t)vs.max, sizeof(MIR_reg_t)); 2518 3199 vs.slot_type = calloc((size_t)vs.max, sizeof(uint8_t)); 2519 3200 vs.known_const = calloc((size_t)vs.max, sizeof(uint64_t)); 2520 3201 vs.has_const = calloc((size_t)vs.max, sizeof(bool)); 2521 - if (!vs.regs || !vs.known_func || !vs.d_regs || !vs.slot_type || !vs.known_const || !vs.has_const) { 2522 - free(vs.regs); free(vs.known_func); free(vs.d_regs); free(vs.slot_type); 3202 + if (!vs.regs || !vs.known_func || !vs.method_ic || !vs.d_regs || !vs.slot_type || !vs.known_const || !vs.has_const) { 3203 + free(vs.regs); free(vs.known_func); free(vs.method_ic); free(vs.d_regs); free(vs.slot_type); 2523 3204 free(vs.known_const); free(vs.has_const); 2524 3205 MIR_finish_func(ctx); MIR_finish_module(ctx); func->jit_compiling = false; return NULL; 2525 3206 } ··· 2546 3227 known_func_locals = calloc((size_t)n_locals, sizeof(sv_func_t *)); 2547 3228 known_type_locals = calloc((size_t)n_locals, sizeof(uint8_t)); 2548 3229 if (!local_regs || !local_d_regs || !known_func_locals || !known_type_locals) { 2549 - free(vs.regs); free(vs.known_func); free(vs.d_regs); free(vs.slot_type); 3230 + free(vs.regs); free(vs.known_func); free(vs.method_ic); free(vs.d_regs); free(vs.slot_type); 2550 3231 free(vs.known_const); free(vs.has_const); 2551 3232 free(local_regs); free(local_d_regs); free(known_func_locals); free(known_type_locals); 2552 3233 MIR_finish_func(ctx); MIR_finish_module(ctx); return NULL; ··· 2668 3349 int slotbuf_count = use_unified_slotbuf ? (param_count + n_locals) : param_count; 2669 3350 2670 3351 if (has_captured_params && needs_bailout) { 2671 - free(vs.regs); free(vs.known_func); free(vs.d_regs); free(vs.slot_type); 3352 + free(vs.regs); free(vs.known_func); free(vs.method_ic); free(vs.d_regs); free(vs.slot_type); 2672 3353 free(vs.known_const); free(vs.has_const); 2673 3354 free(local_regs); free(local_d_regs); free(known_func_locals); free(known_type_locals); 2674 3355 free(captured_params); free(captured_locals); ··· 2814 3495 int call_n = 0; 2815 3496 int upval_n = 0; 2816 3497 int arith_n = 0; 3498 + int inline_depth = 0; 3499 + int inline_budget = JIT_INLINE_BUDGET; 2817 3500 2818 3501 while (ip < end) { 2819 3502 int bc_off = (int)(ip - func->code); ··· 2828 3511 vs.sp = lm.entries[i].sp; 2829 3512 if (vs.slot_type) 2830 3513 memset(vs.slot_type, SLOT_BOXED, (size_t)vs.max); 3514 + if (vs.known_func) 3515 + memset(vs.known_func, 0, (size_t)vs.max * sizeof(sv_func_t *)); 3516 + if (vs.method_ic) 3517 + memset(vs.method_ic, 0, (size_t)vs.max * sizeof(jit_method_ic_info_t)); 2831 3518 if (vs.has_const) 2832 3519 memset(vs.has_const, 0, (size_t)vs.max * sizeof(bool)); 2833 3520 if (known_type_locals && local_d_regs) { ··· 3261 3948 3262 3949 case OP_DUP: { 3263 3950 sv_func_t *kf = vs.known_func[vs.sp - 1]; 3951 + jit_method_ic_info_t mi = vs.method_ic ? vs.method_ic[vs.sp - 1] : (jit_method_ic_info_t){0}; 3264 3952 bool kc = vs.has_const && vs.has_const[vs.sp - 1]; 3265 3953 uint64_t kcv = kc ? vs.known_const[vs.sp - 1] : 0; 3266 3954 vstack_ensure_boxed(&vs, vs.sp - 1, ctx, jit_func, r_d_slot); 3267 3955 MIR_reg_t top = vstack_top(&vs); 3268 3956 MIR_reg_t dst = vstack_push(&vs); 3269 3957 vs.known_func[vs.sp - 1] = kf; 3958 + if (vs.method_ic) vs.method_ic[vs.sp - 1] = mi; 3270 3959 if (kc && vs.has_const) { vs.has_const[vs.sp - 1] = true; vs.known_const[vs.sp - 1] = kcv; } 3271 3960 MIR_append_insn(ctx, jit_func, 3272 3961 MIR_new_insn(ctx, MIR_MOV, ··· 3281 3970 vstack_ensure_boxed(&vs, vs.sp - 2, ctx, jit_func, r_d_slot); 3282 3971 MIR_reg_t ra = vs.regs[vs.sp - 2]; 3283 3972 MIR_reg_t rb = vs.regs[vs.sp - 1]; 3973 + sv_func_t *kf_a = vs.known_func ? vs.known_func[vs.sp - 2] : NULL; 3974 + sv_func_t *kf_b = vs.known_func ? vs.known_func[vs.sp - 1] : NULL; 3975 + jit_method_ic_info_t mi_a = vs.method_ic ? vs.method_ic[vs.sp - 2] : (jit_method_ic_info_t){0}; 3976 + jit_method_ic_info_t mi_b = vs.method_ic ? vs.method_ic[vs.sp - 1] : (jit_method_ic_info_t){0}; 3284 3977 MIR_reg_t da = vstack_push(&vs); 3285 3978 MIR_reg_t db = vstack_push(&vs); 3979 + if (vs.known_func) { 3980 + vs.known_func[vs.sp - 2] = kf_a; 3981 + vs.known_func[vs.sp - 1] = kf_b; 3982 + } 3983 + if (vs.method_ic) { 3984 + vs.method_ic[vs.sp - 2] = mi_a; 3985 + vs.method_ic[vs.sp - 1] = mi_b; 3986 + } 3286 3987 MIR_append_insn(ctx, jit_func, 3287 3988 MIR_new_insn(ctx, MIR_MOV, 3288 3989 MIR_new_reg_op(ctx, da), ··· 4713 5414 if (!inline_callee) 4714 5415 inline_callee = sv_tfb_get_call_target(func, bc_off); 4715 5416 bool speculative = (inline_callee && !vs.known_func[vs.sp - call_argc - 1]); 4716 - if (inline_callee && jit_inlineable(inline_callee) 5417 + if (inline_callee && inline_callee != func && 5418 + inline_depth < JIT_INLINE_MAX_DEPTH && 5419 + inline_budget >= inline_callee->code_len && 5420 + jit_inlineable(inline_callee) 4717 5421 && jit_inline_body_feasible(inline_callee)) { 4718 5422 int cn = call_n++; 4719 5423 ··· 4806 5510 r_call_res, inl_slow, inl_join, 4807 5511 r_bool, &r_d_slot, cn, 4808 5512 r_inl_cl, r_inl_this, r_inl_new_target, r_inl_super, 5513 + NULL, 0, 0, r_ic_epoch_val, 4809 5514 r_vm, r_js, 4810 5515 helper2_proto, imp_seq, imp_sne, imp_eq, imp_ne, 4811 5516 gf_proto, imp_get_field, 5517 + put_field_proto, imp_put_field, 4812 5518 gg_proto, imp_gg, 4813 5519 special_obj_proto, imp_special_obj); 4814 5520 4815 5521 if (inlined) { 5522 + inline_budget -= inline_callee->code_len; 4816 5523 MIR_append_insn(ctx, jit_func, inl_slow); 4817 5524 for (int i = 0; i < (int)call_argc; i++) 4818 5525 MIR_append_insn(ctx, jit_func, ··· 5878 6585 MIR_new_ret_insn(ctx, 1, MIR_new_reg_op(ctx, dst))); 5879 6586 } 5880 6587 MIR_append_insn(ctx, jit_func, no_err); 6588 + if (vs.known_func && vs.method_ic) { 6589 + jit_method_ic_info_t mi = {0}; 6590 + if (jit_ic_try_resolve_method(func, ic_idx, atom, &mi)) { 6591 + vs.known_func[vs.sp - 1] = mi.callee; 6592 + vs.method_ic[vs.sp - 1] = mi; 6593 + } 6594 + } 5881 6595 break; 5882 6596 } 5883 6597 ··· 5935 6649 MIR_new_ret_insn(ctx, 1, MIR_new_reg_op(ctx, dst))); 5936 6650 } 5937 6651 MIR_append_insn(ctx, jit_func, no_err); 6652 + if (vs.known_func && vs.method_ic) { 6653 + jit_method_ic_info_t mi = {0}; 6654 + if (jit_ic_try_resolve_method(func, ic_idx, atom, &mi)) { 6655 + vs.known_func[vs.sp - 1] = mi.callee; 6656 + vs.method_ic[vs.sp - 1] = mi; 6657 + } 6658 + } 5938 6659 break; 5939 6660 } 5940 6661 ··· 7605 8326 uint16_t call_argc = sv_get_u16(ip + 1); 7606 8327 if (call_argc > 16 || vs.sp < (int)call_argc + 2) { ok = false; break; } 7607 8328 8329 + if (!is_tail && vs.method_ic) { 8330 + int callee_slot = vs.sp - (int)call_argc - 1; 8331 + jit_method_ic_info_t mi = vs.method_ic[callee_slot]; 8332 + sv_func_t *inline_callee = mi.valid ? mi.callee : NULL; 8333 + if (inline_callee && inline_callee != func && 8334 + inline_depth < JIT_INLINE_MAX_DEPTH && 8335 + inline_budget >= inline_callee->code_len && 8336 + jit_inlineable(inline_callee) && 8337 + jit_inline_body_feasible(inline_callee)) { 8338 + int cn = call_n++; 8339 + 8340 + MIR_reg_t inl_arg_regs[call_argc > 0 ? call_argc : 1]; 8341 + for (int i = (int)call_argc - 1; i >= 0; i--) 8342 + inl_arg_regs[i] = vstack_pop(&vs); 8343 + 8344 + MIR_reg_t r_call_func = vstack_pop(&vs); 8345 + MIR_reg_t r_call_this = vstack_pop(&vs); 8346 + MIR_reg_t r_call_res = vstack_push(&vs); 8347 + 8348 + MIR_label_t inl_slow = MIR_new_label(ctx); 8349 + MIR_label_t inl_join = MIR_new_label(ctx); 8350 + 8351 + char rn_cl[32], rn_fn[32], rn_sup[32], rn_this[32]; 8352 + char rn_nt[32], rn_flags[32], rn_bound[32], rn_recv_obj[32]; 8353 + char rn_recv_shape[32], rn_recv_tag[32]; 8354 + snprintf(rn_cl, sizeof(rn_cl), "minl%d_cl", cn); 8355 + snprintf(rn_fn, sizeof(rn_fn), "minl%d_fn", cn); 8356 + snprintf(rn_sup, sizeof(rn_sup), "minl%d_sup", cn); 8357 + snprintf(rn_this, sizeof(rn_this), "minl%d_this", cn); 8358 + snprintf(rn_nt, sizeof(rn_nt), "minl%d_nt", cn); 8359 + snprintf(rn_flags, sizeof(rn_flags), "minl%d_flags", cn); 8360 + snprintf(rn_bound, sizeof(rn_bound), "minl%d_bound", cn); 8361 + snprintf(rn_recv_obj, sizeof(rn_recv_obj), "minl%d_recv_obj", cn); 8362 + snprintf(rn_recv_shape, sizeof(rn_recv_shape), "minl%d_recv_shape_reg", cn); 8363 + snprintf(rn_recv_tag, sizeof(rn_recv_tag), "minl%d_recv_tag", cn); 8364 + 8365 + MIR_reg_t r_inl_cl = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_cl); 8366 + MIR_reg_t r_inl_fn = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_fn); 8367 + MIR_reg_t r_inl_super = MIR_new_func_reg(ctx, jit_func->u.func, MIR_JSVAL, rn_sup); 8368 + MIR_reg_t r_inl_this = MIR_new_func_reg(ctx, jit_func->u.func, MIR_JSVAL, rn_this); 8369 + MIR_reg_t r_inl_new_target = MIR_new_func_reg(ctx, jit_func->u.func, MIR_JSVAL, rn_nt); 8370 + MIR_reg_t r_inl_flags = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_flags); 8371 + MIR_reg_t r_inl_bound = MIR_new_func_reg(ctx, jit_func->u.func, MIR_JSVAL, rn_bound); 8372 + MIR_reg_t r_recv_obj = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_recv_obj); 8373 + MIR_reg_t r_recv_shape = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_recv_shape); 8374 + MIR_reg_t r_recv_tag = MIR_new_func_reg(ctx, jit_func->u.func, MIR_T_I64, rn_recv_tag); 8375 + 8376 + mir_emit_method_receiver_shape_guard( 8377 + ctx, jit_func, &mi, r_call_this, r_recv_obj, r_recv_shape, 8378 + r_recv_tag, r_ic_epoch_val, inl_slow, cn); 8379 + 8380 + mir_emit_get_closure(ctx, jit_func, r_inl_cl, r_call_func, 8381 + r_bool, inl_slow); 8382 + MIR_append_insn(ctx, jit_func, 8383 + MIR_new_insn(ctx, MIR_MOV, 8384 + MIR_new_reg_op(ctx, r_inl_fn), 8385 + MIR_new_mem_op(ctx, MIR_T_P, 8386 + (MIR_disp_t)offsetof(sv_closure_t, func), 8387 + r_inl_cl, 0, 1))); 8388 + MIR_append_insn(ctx, jit_func, 8389 + MIR_new_insn(ctx, MIR_BNE, 8390 + MIR_new_label_op(ctx, inl_slow), 8391 + MIR_new_reg_op(ctx, r_inl_fn), 8392 + MIR_new_uint_op(ctx, (uint64_t)(uintptr_t)inline_callee))); 8393 + MIR_append_insn(ctx, jit_func, 8394 + MIR_new_insn(ctx, MIR_MOV, 8395 + MIR_new_reg_op(ctx, r_inl_flags), 8396 + MIR_new_mem_op(ctx, MIR_T_U32, 8397 + (MIR_disp_t)offsetof(sv_closure_t, call_flags), 8398 + r_inl_cl, 0, 1))); 8399 + MIR_append_insn(ctx, jit_func, 8400 + MIR_new_insn(ctx, MIR_AND, 8401 + MIR_new_reg_op(ctx, r_inl_flags), 8402 + MIR_new_reg_op(ctx, r_inl_flags), 8403 + MIR_new_uint_op(ctx, SV_CALL_HAS_BOUND_ARGS | SV_CALL_IS_ARROW))); 8404 + MIR_append_insn(ctx, jit_func, 8405 + MIR_new_insn(ctx, MIR_BNE, 8406 + MIR_new_label_op(ctx, inl_slow), 8407 + MIR_new_reg_op(ctx, r_inl_flags), 8408 + MIR_new_int_op(ctx, 0))); 8409 + MIR_append_insn(ctx, jit_func, 8410 + MIR_new_insn(ctx, MIR_MOV, 8411 + MIR_new_reg_op(ctx, r_inl_bound), 8412 + MIR_new_mem_op(ctx, MIR_JSVAL, 8413 + (MIR_disp_t)offsetof(sv_closure_t, bound_this), 8414 + r_inl_cl, 0, 1))); 8415 + MIR_append_insn(ctx, jit_func, 8416 + MIR_new_insn(ctx, MIR_BNE, 8417 + MIR_new_label_op(ctx, inl_slow), 8418 + MIR_new_reg_op(ctx, r_inl_bound), 8419 + MIR_new_uint_op(ctx, mkval(T_UNDEF, 0)))); 8420 + MIR_append_insn(ctx, jit_func, 8421 + MIR_new_insn(ctx, MIR_MOV, 8422 + MIR_new_reg_op(ctx, r_inl_super), 8423 + MIR_new_mem_op(ctx, MIR_T_I64, 8424 + (MIR_disp_t)offsetof(sv_closure_t, super_val), 8425 + r_inl_cl, 0, 1))); 8426 + mir_load_imm(ctx, jit_func, r_inl_new_target, mkval(T_UNDEF, 0)); 8427 + MIR_append_insn(ctx, jit_func, 8428 + MIR_new_insn(ctx, MIR_MOV, 8429 + MIR_new_reg_op(ctx, r_inl_this), 8430 + MIR_new_reg_op(ctx, r_call_this))); 8431 + 8432 + bool inlined = jit_emit_inline_body( 8433 + ctx, jit_func, inline_callee, 8434 + inl_arg_regs, (int)call_argc, 8435 + r_call_res, inl_slow, inl_join, 8436 + r_bool, &r_d_slot, cn, 8437 + r_inl_cl, r_inl_this, r_inl_new_target, r_inl_super, 8438 + mi.receiver_shape, r_recv_obj, r_recv_shape, r_ic_epoch_val, 8439 + r_vm, r_js, 8440 + helper2_proto, imp_seq, imp_sne, imp_eq, imp_ne, 8441 + gf_proto, imp_get_field, 8442 + put_field_proto, imp_put_field, 8443 + gg_proto, imp_gg, 8444 + special_obj_proto, imp_special_obj); 8445 + 8446 + if (!inlined) { ok = false; break; } 8447 + inline_budget -= inline_callee->code_len; 8448 + 8449 + MIR_append_insn(ctx, jit_func, inl_slow); 8450 + for (int i = 0; i < (int)call_argc; i++) 8451 + MIR_append_insn(ctx, jit_func, 8452 + MIR_new_insn(ctx, MIR_MOV, 8453 + MIR_new_mem_op(ctx, MIR_JSVAL, 8454 + (MIR_disp_t)(i * (int)sizeof(ant_value_t)), 8455 + r_args_buf, 0, 1), 8456 + MIR_new_reg_op(ctx, inl_arg_regs[i]))); 8457 + 8458 + MIR_label_t slow_super = MIR_new_label(ctx); 8459 + MIR_label_t slow_done = MIR_new_label(ctx); 8460 + MIR_append_insn(ctx, jit_func, 8461 + MIR_new_insn(ctx, MIR_BEQ, 8462 + MIR_new_label_op(ctx, slow_super), 8463 + MIR_new_reg_op(ctx, r_call_func), 8464 + MIR_new_reg_op(ctx, r_super_val))); 8465 + MIR_append_insn(ctx, jit_func, 8466 + MIR_new_call_insn(ctx, 9, 8467 + MIR_new_ref_op(ctx, call_proto), 8468 + MIR_new_ref_op(ctx, imp_call), 8469 + MIR_new_reg_op(ctx, r_call_res), 8470 + MIR_new_reg_op(ctx, r_vm), 8471 + MIR_new_reg_op(ctx, r_js), 8472 + MIR_new_reg_op(ctx, r_call_func), 8473 + MIR_new_reg_op(ctx, r_call_this), 8474 + MIR_new_reg_op(ctx, r_args_buf), 8475 + MIR_new_int_op(ctx, (int64_t)call_argc))); 8476 + MIR_append_insn(ctx, jit_func, 8477 + MIR_new_insn(ctx, MIR_JMP, MIR_new_label_op(ctx, slow_done))); 8478 + 8479 + MIR_append_insn(ctx, jit_func, slow_super); 8480 + MIR_append_insn(ctx, jit_func, 8481 + MIR_new_insn(ctx, MIR_MOV, 8482 + MIR_new_mem_op(ctx, MIR_T_I64, 0, r_call_out_this, 0, 1), 8483 + MIR_new_reg_op(ctx, r_call_this))); 8484 + MIR_append_insn(ctx, jit_func, 8485 + MIR_new_call_insn(ctx, 12, 8486 + MIR_new_ref_op(ctx, call_method_proto), 8487 + MIR_new_ref_op(ctx, imp_call_method), 8488 + MIR_new_reg_op(ctx, r_call_res), 8489 + MIR_new_reg_op(ctx, r_vm), 8490 + MIR_new_reg_op(ctx, r_js), 8491 + MIR_new_reg_op(ctx, r_call_func), 8492 + MIR_new_reg_op(ctx, r_call_this), 8493 + MIR_new_reg_op(ctx, r_args_buf), 8494 + MIR_new_int_op(ctx, (int64_t)call_argc), 8495 + MIR_new_reg_op(ctx, r_super_val), 8496 + MIR_new_reg_op(ctx, r_new_target), 8497 + MIR_new_reg_op(ctx, r_call_out_this))); 8498 + MIR_append_insn(ctx, jit_func, 8499 + MIR_new_insn(ctx, MIR_MOV, 8500 + MIR_new_reg_op(ctx, r_this_curr), 8501 + MIR_new_mem_op(ctx, MIR_T_I64, 0, r_call_out_this, 0, 1))); 8502 + MIR_append_insn(ctx, jit_func, slow_done); 8503 + 8504 + MIR_append_insn(ctx, jit_func, inl_join); 8505 + if (has_captures) { 8506 + for (int i = 0; i < n_locals; i++) 8507 + if (captured_locals[i]) 8508 + MIR_append_insn(ctx, jit_func, 8509 + MIR_new_insn(ctx, MIR_MOV, 8510 + MIR_new_reg_op(ctx, local_regs[i]), 8511 + MIR_new_mem_op(ctx, MIR_T_I64, 8512 + (MIR_disp_t)(i * (int)sizeof(ant_value_t)), r_lbuf, 0, 1))); 8513 + } 8514 + if (jit_try_depth > 0) { 8515 + jit_try_entry_t *h = &jit_try_stack[jit_try_depth - 1]; 8516 + MIR_label_t no_err = MIR_new_label(ctx); 8517 + MIR_append_insn(ctx, jit_func, 8518 + MIR_new_insn(ctx, MIR_URSH, 8519 + MIR_new_reg_op(ctx, r_bool), 8520 + MIR_new_reg_op(ctx, r_call_res), 8521 + MIR_new_int_op(ctx, NANBOX_TYPE_SHIFT))); 8522 + MIR_append_insn(ctx, jit_func, 8523 + MIR_new_insn(ctx, MIR_BNE, 8524 + MIR_new_label_op(ctx, no_err), 8525 + MIR_new_reg_op(ctx, r_bool), 8526 + MIR_new_uint_op(ctx, JIT_ERR_TAG))); 8527 + MIR_append_insn(ctx, jit_func, 8528 + MIR_new_insn(ctx, MIR_MOV, 8529 + MIR_new_reg_op(ctx, vs.regs[h->saved_sp]), 8530 + MIR_new_reg_op(ctx, r_call_res))); 8531 + MIR_append_insn(ctx, jit_func, 8532 + MIR_new_insn(ctx, MIR_JMP, 8533 + MIR_new_label_op(ctx, h->catch_label))); 8534 + MIR_append_insn(ctx, jit_func, no_err); 8535 + } else { 8536 + MIR_label_t no_err = MIR_new_label(ctx); 8537 + MIR_append_insn(ctx, jit_func, 8538 + MIR_new_insn(ctx, MIR_URSH, 8539 + MIR_new_reg_op(ctx, r_bool), 8540 + MIR_new_reg_op(ctx, r_call_res), 8541 + MIR_new_int_op(ctx, NANBOX_TYPE_SHIFT))); 8542 + MIR_append_insn(ctx, jit_func, 8543 + MIR_new_insn(ctx, MIR_BNE, 8544 + MIR_new_label_op(ctx, no_err), 8545 + MIR_new_reg_op(ctx, r_bool), 8546 + MIR_new_uint_op(ctx, JIT_ERR_TAG))); 8547 + MIR_append_insn(ctx, jit_func, 8548 + MIR_new_ret_insn(ctx, 1, MIR_new_reg_op(ctx, r_call_res))); 8549 + MIR_append_insn(ctx, jit_func, no_err); 8550 + } 8551 + break; 8552 + } 8553 + } 8554 + 7608 8555 int cn = call_n++; 7609 8556 7610 8557 char rn_arr[32], rn_ccl[32], rn_cfn[32], rn_jptr[32], rn_sup[32]; ··· 8204 9151 free(vs.regs); 8205 9152 free(vs.d_regs); 8206 9153 free(vs.known_func); 9154 + free(vs.method_ic); 8207 9155 free(vs.slot_type); 8208 9156 free(vs.known_const); 8209 9157 free(vs.has_const);