/home/travis/build/MoarVM/MoarVM/src/spesh/deopt.c
Line | Count | Source (jump to first uncovered line) |
1 | | #include "moar.h" |
2 | | |
3 | | /* In some cases, we may have specialized bytecode "on the stack" and need to |
4 | | * back out of it, because some assumption it made has been invalidated. This |
5 | | * file contains implementations of those various forms of de-opt. */ |
6 | | |
7 | | #define MVM_LOG_DEOPTS 0 |
8 | | |
9 | | /* Uninlining can invalidate what the dynlex cache points to, so we'll |
10 | | * clear it in various caches. */ |
11 | 45.2k | MVM_STATIC_INLINE void clear_dynlex_cache(MVMThreadContext *tc, MVMFrame *f) { |
12 | 45.2k | MVMFrameExtra *e = f->extra; |
13 | 45.2k | if (e) { |
14 | 5.55k | e->dynlex_cache_name = NULL; |
15 | 5.55k | e->dynlex_cache_reg = NULL; |
16 | 5.55k | } |
17 | 45.2k | } |
18 | | |
19 | | /* If we have to deopt inside of a frame containing inlines, and we're in |
20 | | * an inlined frame at the point we hit deopt, we need to undo the inlining |
21 | | * by switching all levels of inlined frame out for a bunch of frames that |
22 | | * are running the de-optimized code. We may, of course, be in the original, |
23 | | * non-inline, bit of the code - in which case we've nothing to do. */ |
24 | | static void uninline(MVMThreadContext *tc, MVMFrame *f, MVMSpeshCandidate *cand, |
25 | 27.0k | MVMint32 offset, MVMint32 deopt_offset, MVMFrame *callee) { |
26 | 27.0k | MVMFrame *last_uninlined = NULL; |
27 | 27.0k | MVMuint16 last_res_reg; |
28 | 27.0k | MVMReturnType last_res_type; |
29 | 27.0k | MVMuint32 last_return_deopt_idx; |
30 | 27.0k | MVMint32 i; |
31 | 91.2k | for (i = 0; i < cand->num_inlines; i++) { |
32 | 64.1k | if (offset > cand->inlines[i].start && offset <= cand->inlines[i].end) { |
33 | 24 | /* Create the frame. */ |
34 | 24 | MVMCode *ucode = (MVMCode *)f->work[cand->inlines[i].code_ref_reg].o; |
35 | 24 | MVMStaticFrame *usf = cand->inlines[i].sf; |
36 | 24 | MVMFrame *uf; |
37 | 24 | if (REPR(ucode)->ID != MVM_REPR_ID_MVMCode) |
38 | 0 | MVM_panic(1, "Deopt: did not find code object when uninlining"); |
39 | 24 | MVMROOT4(tc, f, callee, last_uninlined, usf, { |
40 | 24 | uf = MVM_frame_create_for_deopt(tc, usf, ucode); |
41 | 24 | }); |
42 | 24 | #if MVM_LOG_DEOPTS |
43 | | fprintf(stderr, "Recreated frame '%s' (cuid '%s')\n", |
44 | | MVM_string_utf8_encode_C_string(tc, usf->body.name), |
45 | | MVM_string_utf8_encode_C_string(tc, usf->body.cuuid)); |
46 | | #endif |
47 | 24 | |
48 | 24 | /* Copy the locals and lexicals into place. */ |
49 | 24 | if (usf->body.num_locals) |
50 | 24 | memcpy(uf->work, f->work + cand->inlines[i].locals_start, |
51 | 24 | usf->body.num_locals * sizeof(MVMRegister)); |
52 | 24 | if (usf->body.num_lexicals) |
53 | 0 | memcpy(uf->env, f->env + cand->inlines[i].lexicals_start, |
54 | 0 | usf->body.num_lexicals * sizeof(MVMRegister)); |
55 | 24 | |
56 | 24 | /* Store the named argument used bit field, since if we deopt in |
57 | 24 | * argument handling code we may have missed some. */ |
58 | 24 | if (cand->inlines[i].deopt_named_used_bit_field) |
59 | 0 | uf->params.named_used.bit_field = cand->inlines[i].deopt_named_used_bit_field; |
60 | 24 | |
61 | 24 | /* Did we already uninline a frame? */ |
62 | 24 | if (last_uninlined) { |
63 | 0 | /* Yes; multi-level un-inline. Switch it back to deopt'd |
64 | 0 | * code. */ |
65 | 0 | uf->effective_spesh_slots = NULL; |
66 | 0 | uf->spesh_cand = NULL; |
67 | 0 |
|
68 | 0 | /* Set up the return location. */ |
69 | 0 | uf->return_address = usf->body.bytecode + |
70 | 0 | cand->deopts[2 * last_return_deopt_idx]; |
71 | 0 |
|
72 | 0 | /* Set result type and register. */ |
73 | 0 | uf->return_type = last_res_type; |
74 | 0 | if (last_res_type == MVM_RETURN_VOID) |
75 | 0 | uf->return_value = NULL; |
76 | 0 | else |
77 | 0 | uf->return_value = uf->work + last_res_reg; |
78 | 0 |
|
79 | 0 | /* Set up last uninlined's caller to us. */ |
80 | 0 | MVM_ASSERT_NOT_FROMSPACE(tc, uf); |
81 | 0 | MVM_ASSIGN_REF(tc, &(last_uninlined->header), last_uninlined->caller, uf); |
82 | 0 | } |
83 | 24 | else { |
84 | 24 | /* First uninlined frame. Are we in the middle of the call |
85 | 24 | * stack (and thus in deopt_all)? */ |
86 | 24 | if (callee) { |
87 | 0 | /* Tweak the callee's caller to the uninlined frame, not |
88 | 0 | * the frame holding the inlinings. */ |
89 | 0 | MVM_ASSERT_NOT_FROMSPACE(tc, uf); |
90 | 0 | MVM_ASSIGN_REF(tc, &(callee->header), callee->caller, uf); |
91 | 0 |
|
92 | 0 | /* Copy over the return location. */ |
93 | 0 | uf->return_address = usf->body.bytecode + deopt_offset; |
94 | 0 |
|
95 | 0 | /* Set result type and register. */ |
96 | 0 | uf->return_type = f->return_type; |
97 | 0 | if (uf->return_type == MVM_RETURN_VOID) { |
98 | 0 | uf->return_value = NULL; |
99 | 0 | } |
100 | 0 | else { |
101 | 0 | MVMuint16 orig_reg = (MVMuint16)(f->return_value - f->work); |
102 | 0 | MVMuint16 ret_reg = orig_reg - cand->inlines[i].locals_start; |
103 | 0 | uf->return_value = uf->work + ret_reg; |
104 | 0 | } |
105 | 0 | } |
106 | 24 | else { |
107 | 24 | /* No, it's the deopt_one case, so this is where we'll point |
108 | 24 | * the interpreter. */ |
109 | 24 | tc->cur_frame = uf; |
110 | 24 | tc->current_frame_nr = uf->sequence_nr; |
111 | 24 | *(tc->interp_cur_op) = usf->body.bytecode + deopt_offset; |
112 | 24 | *(tc->interp_bytecode_start) = usf->body.bytecode; |
113 | 24 | *(tc->interp_reg_base) = uf->work; |
114 | 24 | *(tc->interp_cu) = usf->body.cu; |
115 | 24 | } |
116 | 24 | } |
117 | 24 | |
118 | 24 | /* Update tracking variables for last uninline. */ |
119 | 24 | last_uninlined = uf; |
120 | 24 | last_res_reg = cand->inlines[i].res_reg; |
121 | 24 | last_res_type = cand->inlines[i].res_type; |
122 | 24 | last_return_deopt_idx = cand->inlines[i].return_deopt_idx; |
123 | 24 | } |
124 | 64.1k | } |
125 | 27.0k | if (last_uninlined) { |
126 | 24 | /* Set return address, which we need to resolve to the deopt'd one. */ |
127 | 24 | f->return_address = f->static_info->body.bytecode + |
128 | 24 | cand->deopts[2 * last_return_deopt_idx]; |
129 | 24 | |
130 | 24 | /* Set result type and register. */ |
131 | 24 | f->return_type = last_res_type; |
132 | 24 | if (last_res_type == MVM_RETURN_VOID) |
133 | 0 | f->return_value = NULL; |
134 | 24 | else |
135 | 24 | f->return_value = f->work + last_res_reg; |
136 | 24 | |
137 | 24 | /* Set up inliner as the caller, given we now have a direct inline. */ |
138 | 24 | MVM_ASSERT_NOT_FROMSPACE(tc, f); |
139 | 24 | MVM_ASSIGN_REF(tc, &(last_uninlined->header), last_uninlined->caller, f); |
140 | 24 | } |
141 | 27.0k | else { |
142 | 27.0k | /* Weren't in an inline after all. What kind of deopt? */ |
143 | 27.0k | if (callee) { |
144 | 1.02k | /* Deopt all. Move return address. */ |
145 | 1.02k | f->return_address = f->static_info->body.bytecode + deopt_offset; |
146 | 1.02k | } |
147 | 26.0k | else { |
148 | 26.0k | /* Deopt one. Move interpreter. */ |
149 | 26.0k | *(tc->interp_cur_op) = f->static_info->body.bytecode + deopt_offset; |
150 | 26.0k | *(tc->interp_bytecode_start) = f->static_info->body.bytecode; |
151 | 26.0k | } |
152 | 27.0k | } |
153 | 27.0k | } |
154 | | |
155 | 30.9k | static void deopt_named_args_used(MVMThreadContext *tc, MVMFrame *f) { |
156 | 30.9k | if (f->spesh_cand->deopt_named_used_bit_field) |
157 | 119 | f->params.named_used.bit_field = f->spesh_cand->deopt_named_used_bit_field; |
158 | 30.9k | } |
159 | | |
160 | 28.7k | static void deopt_frame(MVMThreadContext *tc, MVMFrame *f, MVMint32 deopt_offset, MVMint32 deopt_target) { |
161 | 28.7k | /* Found it; are we in an inline? */ |
162 | 28.7k | MVMSpeshInline *inlines = f->spesh_cand->inlines; |
163 | 28.7k | deopt_named_args_used(tc, f); |
164 | 28.7k | if (inlines) { |
165 | 26.0k | /* Yes, going to have to re-create the frames; uninline |
166 | 26.0k | * moves the interpreter, so we can just tweak the last |
167 | 26.0k | * frame. For the moment, uninlining creates its frames |
168 | 26.0k | * on the heap, so we'll force the current call stack to |
169 | 26.0k | * the heap to preserve the "no heap -> stack pointers" |
170 | 26.0k | * invariant. */ |
171 | 26.0k | f = MVM_frame_force_to_heap(tc, f); |
172 | 26.0k | MVMROOT(tc, f, { |
173 | 26.0k | uninline(tc, f, f->spesh_cand, deopt_offset, deopt_target, NULL); |
174 | 26.0k | }); |
175 | 26.0k | f->effective_spesh_slots = NULL; |
176 | 26.0k | f->spesh_cand = NULL; |
177 | 26.0k | #if MVM_LOG_DEOPTS |
178 | | fprintf(stderr, "Completed deopt_one in '%s' (cuid '%s') with uninlining\n", |
179 | | MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.name), |
180 | | MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.cuuid)); |
181 | | #endif |
182 | 26.0k | } |
183 | 2.74k | else { |
184 | 2.74k | /* No inlining; simple case. Switch back to the original code. */ |
185 | 2.74k | *(tc->interp_cur_op) = f->static_info->body.bytecode + deopt_target; |
186 | 2.74k | *(tc->interp_bytecode_start) = f->static_info->body.bytecode; |
187 | 2.74k | f->effective_spesh_slots = NULL; |
188 | 2.74k | f->spesh_cand = NULL; |
189 | 2.74k | #if MVM_LOG_DEOPTS |
190 | | fprintf(stderr, "Completed deopt_one in '%s' (cuid '%s')\n", |
191 | | MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.name), |
192 | | MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.cuuid)); |
193 | | #endif |
194 | 2.74k | } |
195 | 28.7k | |
196 | 28.7k | } |
197 | | |
198 | | /* De-optimizes the currently executing frame, provided it is specialized and |
199 | | * at a valid de-optimization point. Typically used when a guard fails. */ |
200 | 1.22k | void MVM_spesh_deopt_one(MVMThreadContext *tc, MVMuint32 deopt_target) { |
201 | 1.22k | MVMFrame *f = tc->cur_frame; |
202 | 1.22k | if (tc->instance->profiling) |
203 | 0 | MVM_profiler_log_deopt_one(tc); |
204 | 1.22k | #if MVM_LOG_DEOPTS |
205 | | fprintf(stderr, "Deopt one requested by interpreter in frame '%s' (cuid '%s')\n", |
206 | | MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.name), |
207 | | MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.cuuid)); |
208 | | #endif |
209 | 1.22k | clear_dynlex_cache(tc, f); |
210 | 1.22k | if (f->spesh_cand) { |
211 | 1.22k | MVMuint32 deopt_offset = *(tc->interp_cur_op) - f->spesh_cand->bytecode; |
212 | 1.22k | #if MVM_LOG_DEOPTS |
213 | | fprintf(stderr, "Will deopt %u -> %u\n", deopt_offset, deopt_target); |
214 | | #endif |
215 | 1.22k | deopt_frame(tc, tc->cur_frame, deopt_offset, deopt_target); |
216 | 1.22k | } |
217 | 0 | else { |
218 | 0 | MVM_oops(tc, "deopt_one failed for %s (%s)", |
219 | 0 | MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.name), |
220 | 0 | MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.cuuid)); |
221 | 0 | } |
222 | 1.22k | |
223 | 1.22k | MVM_CHECK_CALLER_CHAIN(tc, tc->cur_frame); |
224 | 1.22k | } |
225 | | |
226 | | /* De-optimizes the current frame by directly specifying the addresses */ |
227 | | void MVM_spesh_deopt_one_direct(MVMThreadContext *tc, MVMuint32 deopt_offset, |
228 | 27.5k | MVMuint32 deopt_target) { |
229 | 27.5k | MVMFrame *f = tc->cur_frame; |
230 | 27.5k | #if MVM_LOG_DEOPTS |
231 | | fprintf(stderr, "Deopt one requested by JIT in frame '%s' (cuid '%s') (%u -> %u)\n", |
232 | | MVM_string_utf8_encode_C_string(tc, f->static_info->body.name), |
233 | | MVM_string_utf8_encode_C_string(tc, f->static_info->body.cuuid), |
234 | | deopt_offset, deopt_target); |
235 | | #endif |
236 | 27.5k | if (tc->instance->profiling) |
237 | 0 | MVM_profiler_log_deopt_one(tc); |
238 | 27.5k | clear_dynlex_cache(tc, f); |
239 | 27.5k | deopt_frame(tc, tc->cur_frame, deopt_offset, deopt_target); |
240 | 27.5k | |
241 | 27.5k | MVM_CHECK_CALLER_CHAIN(tc, tc->cur_frame); |
242 | 27.5k | } |
243 | | |
244 | | /* De-optimizes all specialized frames on the call stack. Used when a change |
245 | | * is made the could invalidate all kinds of assumptions all over the place |
246 | | * (such as a mix-in). */ |
247 | 2.45k | void MVM_spesh_deopt_all(MVMThreadContext *tc) { |
248 | 2.45k | /* Walk frames looking for any callers in specialized bytecode. */ |
249 | 2.45k | MVMFrame *l = MVM_frame_force_to_heap(tc, tc->cur_frame); |
250 | 2.45k | MVMFrame *f = tc->cur_frame->caller; |
251 | 2.45k | #if MVM_LOG_DEOPTS |
252 | | fprintf(stderr, "Deopt all requested in frame '%s' (cuid '%s')\n", |
253 | | MVM_string_utf8_encode_C_string(tc, l->static_info->body.name), |
254 | | MVM_string_utf8_encode_C_string(tc, l->static_info->body.cuuid)); |
255 | | #endif |
256 | 2.45k | if (tc->instance->profiling) |
257 | 0 | MVM_profiler_log_deopt_all(tc); |
258 | 2.45k | |
259 | 16.7k | while (f) { |
260 | 16.4k | clear_dynlex_cache(tc, f); |
261 | 16.4k | if (f->spesh_cand) { |
262 | 3.23k | /* Found one. Is it JITted code? */ |
263 | 3.23k | if (f->spesh_cand->jitcode) { |
264 | 3.23k | MVMJitCode *jitcode = f->spesh_cand->jitcode; |
265 | 3.23k | MVMint32 idx = MVM_jit_code_get_active_deopt_idx(tc, jitcode, f); |
266 | 3.23k | if (idx < jitcode->num_deopts) { |
267 | 2.16k | /* Resolve offset and target. */ |
268 | 2.16k | MVMint32 deopt_idx = jitcode->deopts[idx].idx; |
269 | 2.16k | MVMint32 deopt_offset = f->spesh_cand->deopts[2 * deopt_idx + 1]; |
270 | 2.16k | MVMint32 deopt_target = f->spesh_cand->deopts[2 * deopt_idx]; |
271 | 2.16k | #if MVM_LOG_DEOPTS |
272 | | fprintf(stderr, "Found deopt label for JIT (%d) (label %d idx %d)\n", i, |
273 | | deopts[i].label, deopts[i].idx); |
274 | | #endif |
275 | 2.16k | |
276 | 2.16k | /* Re-create any frames needed if we're in an inline; if not, |
277 | 2.16k | * just update return address. */ |
278 | 2.16k | if (f->spesh_cand->inlines) { |
279 | 1.02k | MVMROOT2(tc, f, l, { |
280 | 1.02k | uninline(tc, f, f->spesh_cand, deopt_offset, deopt_target, l); |
281 | 1.02k | }); |
282 | 1.02k | } |
283 | 1.13k | else { |
284 | 1.13k | f->return_address = f->static_info->body.bytecode + deopt_target; |
285 | 1.13k | } |
286 | 2.16k | |
287 | 2.16k | /* No spesh cand/slots needed now. */ |
288 | 2.16k | deopt_named_args_used(tc, f); |
289 | 2.16k | f->effective_spesh_slots = NULL; |
290 | 2.16k | f->spesh_cand = NULL; |
291 | 2.16k | f->jit_entry_label = NULL; |
292 | 2.16k | |
293 | 2.16k | break; |
294 | 2.16k | } |
295 | 3.23k | #if MVM_LOG_DEOPTS |
296 | | if (i == num_deopts) |
297 | | fprintf(stderr, "JIT: can't find deopt all idx\n"); |
298 | | #endif |
299 | 3.23k | } |
300 | 3.23k | |
301 | 0 | else { |
302 | 0 | /* Not JITted; see if we can find the return address in the deopt table. */ |
303 | 0 | MVMint32 ret_offset = f->return_address - f->spesh_cand->bytecode; |
304 | 0 | MVMint32 n = f->spesh_cand->num_deopts * 2; |
305 | 0 | MVMint32 i; |
306 | 0 | for (i = 0; i < n; i += 2) { |
307 | 0 | if (f->spesh_cand->deopts[i + 1] == ret_offset) { |
308 | 0 | /* Re-create any frames needed if we're in an inline; if not, |
309 | 0 | * just update return address. */ |
310 | 0 | if (f->spesh_cand->inlines) { |
311 | 0 | MVMROOT2(tc, f, l, { |
312 | 0 | uninline(tc, f, f->spesh_cand, ret_offset, f->spesh_cand->deopts[i], l); |
313 | 0 | }); |
314 | 0 | } |
315 | 0 | else { |
316 | 0 | f->return_address = f->static_info->body.bytecode + f->spesh_cand->deopts[i]; |
317 | 0 | } |
318 | 0 |
|
319 | 0 | /* No spesh cand/slots needed now. */ |
320 | 0 | f->effective_spesh_slots = NULL; |
321 | 0 | f->spesh_cand = NULL; |
322 | 0 |
|
323 | 0 | break; |
324 | 0 | } |
325 | 0 | } |
326 | 0 | #if MVM_LOG_DEOPTS |
327 | | if (i == n) |
328 | | fprintf(stderr, "Interpreter: can't find deopt all idx\n"); |
329 | | #endif |
330 | 0 | } |
331 | 3.23k | } |
332 | 14.3k | l = f; |
333 | 14.3k | f = f->caller; |
334 | 14.3k | } |
335 | 2.45k | |
336 | 2.45k | MVM_CHECK_CALLER_CHAIN(tc, tc->cur_frame); |
337 | 2.45k | } |