Coverage Report

Created: 2018-07-03 15:31

/home/travis/build/MoarVM/MoarVM/src/spesh/deopt.c
Line
Count
Source (jump to first uncovered line)
1
#include "moar.h"
2
3
/* In some cases, we may have specialized bytecode "on the stack" and need to
4
 * back out of it, because some assumption it made has been invalidated. This
5
 * file contains implementations of those various forms of de-opt. */
6
7
#define MVM_LOG_DEOPTS 0
8
9
/* Uninlining can invalidate what the dynlex cache points to, so we'll
10
 * clear it in various caches. */
11
45.2k
MVM_STATIC_INLINE void clear_dynlex_cache(MVMThreadContext *tc, MVMFrame *f) {
12
45.2k
    MVMFrameExtra *e = f->extra;
13
45.2k
    if (e) {
14
5.55k
        e->dynlex_cache_name = NULL;
15
5.55k
        e->dynlex_cache_reg = NULL;
16
5.55k
    }
17
45.2k
}
18
19
/* If we have to deopt inside of a frame containing inlines, and we're in
20
 * an inlined frame at the point we hit deopt, we need to undo the inlining
21
 * by switching all levels of inlined frame out for a bunch of frames that
22
 * are running the de-optimized code. We may, of course, be in the original,
23
 * non-inline, bit of the code - in which case we've nothing to do. */
24
static void uninline(MVMThreadContext *tc, MVMFrame *f, MVMSpeshCandidate *cand,
25
27.0k
                     MVMint32 offset, MVMint32 deopt_offset, MVMFrame *callee) {
26
27.0k
    MVMFrame      *last_uninlined = NULL;
27
27.0k
    MVMuint16      last_res_reg;
28
27.0k
    MVMReturnType  last_res_type;
29
27.0k
    MVMuint32      last_return_deopt_idx;
30
27.0k
    MVMint32 i;
31
91.2k
    for (i = 0; i < cand->num_inlines; i++) {
32
64.1k
        if (offset > cand->inlines[i].start && offset <= cand->inlines[i].end) {
33
24
            /* Create the frame. */
34
24
            MVMCode        *ucode = (MVMCode *)f->work[cand->inlines[i].code_ref_reg].o;
35
24
            MVMStaticFrame *usf   = cand->inlines[i].sf;
36
24
            MVMFrame       *uf;
37
24
            if (REPR(ucode)->ID != MVM_REPR_ID_MVMCode)
38
0
                MVM_panic(1, "Deopt: did not find code object when uninlining");
39
24
            MVMROOT4(tc, f, callee, last_uninlined, usf, {
40
24
                uf = MVM_frame_create_for_deopt(tc, usf, ucode);
41
24
            });
42
24
#if MVM_LOG_DEOPTS
43
            fprintf(stderr, "Recreated frame '%s' (cuid '%s')\n",
44
                MVM_string_utf8_encode_C_string(tc, usf->body.name),
45
                MVM_string_utf8_encode_C_string(tc, usf->body.cuuid));
46
#endif
47
24
48
24
            /* Copy the locals and lexicals into place. */
49
24
            if (usf->body.num_locals)
50
24
                memcpy(uf->work, f->work + cand->inlines[i].locals_start,
51
24
                    usf->body.num_locals * sizeof(MVMRegister));
52
24
            if (usf->body.num_lexicals)
53
0
                memcpy(uf->env, f->env + cand->inlines[i].lexicals_start,
54
0
                    usf->body.num_lexicals * sizeof(MVMRegister));
55
24
56
24
            /* Store the named argument used bit field, since if we deopt in
57
24
             * argument handling code we may have missed some. */
58
24
            if (cand->inlines[i].deopt_named_used_bit_field)
59
0
                uf->params.named_used.bit_field = cand->inlines[i].deopt_named_used_bit_field;
60
24
61
24
            /* Did we already uninline a frame? */
62
24
            if (last_uninlined) {
63
0
                /* Yes; multi-level un-inline. Switch it back to deopt'd
64
0
                 * code. */
65
0
                uf->effective_spesh_slots = NULL;
66
0
                uf->spesh_cand            = NULL;
67
0
68
0
                /* Set up the return location. */
69
0
                uf->return_address = usf->body.bytecode +
70
0
                    cand->deopts[2 * last_return_deopt_idx];
71
0
72
0
                /* Set result type and register. */
73
0
                uf->return_type = last_res_type;
74
0
                if (last_res_type == MVM_RETURN_VOID)
75
0
                    uf->return_value = NULL;
76
0
                else
77
0
                    uf->return_value = uf->work + last_res_reg;
78
0
79
0
                /* Set up last uninlined's caller to us. */
80
0
                MVM_ASSERT_NOT_FROMSPACE(tc, uf);
81
0
                MVM_ASSIGN_REF(tc, &(last_uninlined->header), last_uninlined->caller, uf);
82
0
            }
83
24
            else {
84
24
                /* First uninlined frame. Are we in the middle of the call
85
24
                 * stack (and thus in deopt_all)? */
86
24
                if (callee) {
87
0
                    /* Tweak the callee's caller to the uninlined frame, not
88
0
                     * the frame holding the inlinings. */
89
0
                    MVM_ASSERT_NOT_FROMSPACE(tc, uf);
90
0
                    MVM_ASSIGN_REF(tc, &(callee->header), callee->caller, uf);
91
0
92
0
                    /* Copy over the return location. */
93
0
                    uf->return_address = usf->body.bytecode + deopt_offset;
94
0
95
0
                    /* Set result type and register. */
96
0
                    uf->return_type = f->return_type;
97
0
                    if (uf->return_type == MVM_RETURN_VOID) {
98
0
                        uf->return_value = NULL;
99
0
                    }
100
0
                    else {
101
0
                        MVMuint16 orig_reg = (MVMuint16)(f->return_value - f->work);
102
0
                        MVMuint16 ret_reg  = orig_reg - cand->inlines[i].locals_start;
103
0
                        uf->return_value = uf->work + ret_reg;
104
0
                    }
105
0
                }
106
24
                else {
107
24
                    /* No, it's the deopt_one case, so this is where we'll point
108
24
                     * the interpreter. */
109
24
                    tc->cur_frame                = uf;
110
24
                    tc->current_frame_nr         = uf->sequence_nr;
111
24
                    *(tc->interp_cur_op)         = usf->body.bytecode + deopt_offset;
112
24
                    *(tc->interp_bytecode_start) = usf->body.bytecode;
113
24
                    *(tc->interp_reg_base)       = uf->work;
114
24
                    *(tc->interp_cu)             = usf->body.cu;
115
24
                }
116
24
            }
117
24
118
24
            /* Update tracking variables for last uninline. */
119
24
            last_uninlined        = uf;
120
24
            last_res_reg          = cand->inlines[i].res_reg;
121
24
            last_res_type         = cand->inlines[i].res_type;
122
24
            last_return_deopt_idx = cand->inlines[i].return_deopt_idx;
123
24
        }
124
64.1k
    }
125
27.0k
    if (last_uninlined) {
126
24
        /* Set return address, which we need to resolve to the deopt'd one. */
127
24
        f->return_address = f->static_info->body.bytecode +
128
24
            cand->deopts[2 * last_return_deopt_idx];
129
24
130
24
        /* Set result type and register. */
131
24
        f->return_type = last_res_type;
132
24
        if (last_res_type == MVM_RETURN_VOID)
133
0
            f->return_value = NULL;
134
24
        else
135
24
            f->return_value = f->work + last_res_reg;
136
24
137
24
        /* Set up inliner as the caller, given we now have a direct inline. */
138
24
        MVM_ASSERT_NOT_FROMSPACE(tc, f);
139
24
        MVM_ASSIGN_REF(tc, &(last_uninlined->header), last_uninlined->caller, f);
140
24
    }
141
27.0k
    else {
142
27.0k
        /* Weren't in an inline after all. What kind of deopt? */
143
27.0k
        if (callee) {
144
1.02k
            /* Deopt all. Move return address. */
145
1.02k
            f->return_address = f->static_info->body.bytecode + deopt_offset;
146
1.02k
        }
147
26.0k
        else {
148
26.0k
            /* Deopt one. Move interpreter. */
149
26.0k
            *(tc->interp_cur_op)         = f->static_info->body.bytecode + deopt_offset;
150
26.0k
            *(tc->interp_bytecode_start) = f->static_info->body.bytecode;
151
26.0k
        }
152
27.0k
    }
153
27.0k
}
154
155
30.9k
static void deopt_named_args_used(MVMThreadContext *tc, MVMFrame *f) {
156
30.9k
    if (f->spesh_cand->deopt_named_used_bit_field)
157
119
        f->params.named_used.bit_field = f->spesh_cand->deopt_named_used_bit_field;
158
30.9k
}
159
160
28.7k
static void deopt_frame(MVMThreadContext *tc, MVMFrame *f, MVMint32 deopt_offset, MVMint32 deopt_target) {
161
28.7k
    /* Found it; are we in an inline? */
162
28.7k
    MVMSpeshInline *inlines = f->spesh_cand->inlines;
163
28.7k
    deopt_named_args_used(tc, f);
164
28.7k
    if (inlines) {
165
26.0k
        /* Yes, going to have to re-create the frames; uninline
166
26.0k
         * moves the interpreter, so we can just tweak the last
167
26.0k
         * frame. For the moment, uninlining creates its frames
168
26.0k
         * on the heap, so we'll force the current call stack to
169
26.0k
         * the heap to preserve the "no heap -> stack pointers"
170
26.0k
         * invariant. */
171
26.0k
        f = MVM_frame_force_to_heap(tc, f);
172
26.0k
        MVMROOT(tc, f, {
173
26.0k
            uninline(tc, f, f->spesh_cand, deopt_offset, deopt_target, NULL);
174
26.0k
        });
175
26.0k
        f->effective_spesh_slots = NULL;
176
26.0k
        f->spesh_cand            = NULL;
177
26.0k
#if MVM_LOG_DEOPTS
178
        fprintf(stderr, "Completed deopt_one in '%s' (cuid '%s') with uninlining\n",
179
          MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.name),
180
          MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.cuuid));
181
#endif
182
26.0k
    }
183
2.74k
    else {
184
2.74k
        /* No inlining; simple case. Switch back to the original code. */
185
2.74k
        *(tc->interp_cur_op)         = f->static_info->body.bytecode + deopt_target;
186
2.74k
        *(tc->interp_bytecode_start) = f->static_info->body.bytecode;
187
2.74k
        f->effective_spesh_slots     = NULL;
188
2.74k
        f->spesh_cand                = NULL;
189
2.74k
#if MVM_LOG_DEOPTS
190
        fprintf(stderr, "Completed deopt_one in '%s' (cuid '%s')\n",
191
          MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.name),
192
          MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.cuuid));
193
#endif
194
2.74k
    }
195
28.7k
196
28.7k
}
197
198
/* De-optimizes the currently executing frame, provided it is specialized and
199
 * at a valid de-optimization point. Typically used when a guard fails. */
200
1.22k
void MVM_spesh_deopt_one(MVMThreadContext *tc, MVMuint32 deopt_target) {
201
1.22k
    MVMFrame *f = tc->cur_frame;
202
1.22k
    if (tc->instance->profiling)
203
0
        MVM_profiler_log_deopt_one(tc);
204
1.22k
#if MVM_LOG_DEOPTS
205
    fprintf(stderr, "Deopt one requested by interpreter in frame '%s' (cuid '%s')\n",
206
        MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.name),
207
        MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.cuuid));
208
#endif
209
1.22k
    clear_dynlex_cache(tc, f);
210
1.22k
    if (f->spesh_cand) {
211
1.22k
        MVMuint32 deopt_offset = *(tc->interp_cur_op) - f->spesh_cand->bytecode;
212
1.22k
#if MVM_LOG_DEOPTS
213
    fprintf(stderr, "Will deopt %u -> %u\n", deopt_offset, deopt_target);
214
#endif
215
1.22k
        deopt_frame(tc, tc->cur_frame, deopt_offset, deopt_target);
216
1.22k
    }
217
0
    else {
218
0
        MVM_oops(tc, "deopt_one failed for %s (%s)",
219
0
            MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.name),
220
0
            MVM_string_utf8_encode_C_string(tc, tc->cur_frame->static_info->body.cuuid));
221
0
    }
222
1.22k
223
1.22k
    MVM_CHECK_CALLER_CHAIN(tc, tc->cur_frame);
224
1.22k
}
225
226
/* De-optimizes the current frame by directly specifying the addresses */
227
void MVM_spesh_deopt_one_direct(MVMThreadContext *tc, MVMuint32 deopt_offset,
228
27.5k
                                MVMuint32 deopt_target) {
229
27.5k
    MVMFrame *f = tc->cur_frame;
230
27.5k
#if MVM_LOG_DEOPTS
231
    fprintf(stderr, "Deopt one requested by JIT in frame '%s' (cuid '%s') (%u -> %u)\n",
232
        MVM_string_utf8_encode_C_string(tc, f->static_info->body.name),
233
        MVM_string_utf8_encode_C_string(tc, f->static_info->body.cuuid),
234
        deopt_offset, deopt_target);
235
#endif
236
27.5k
    if (tc->instance->profiling)
237
0
        MVM_profiler_log_deopt_one(tc);
238
27.5k
    clear_dynlex_cache(tc, f);
239
27.5k
    deopt_frame(tc, tc->cur_frame, deopt_offset, deopt_target);
240
27.5k
241
27.5k
    MVM_CHECK_CALLER_CHAIN(tc, tc->cur_frame);
242
27.5k
}
243
244
/* De-optimizes all specialized frames on the call stack. Used when a change
245
 * is made the could invalidate all kinds of assumptions all over the place
246
 * (such as a mix-in). */
247
2.45k
void MVM_spesh_deopt_all(MVMThreadContext *tc) {
248
2.45k
    /* Walk frames looking for any callers in specialized bytecode. */
249
2.45k
    MVMFrame *l = MVM_frame_force_to_heap(tc, tc->cur_frame);
250
2.45k
    MVMFrame *f = tc->cur_frame->caller;
251
2.45k
#if MVM_LOG_DEOPTS
252
    fprintf(stderr, "Deopt all requested in frame '%s' (cuid '%s')\n",
253
        MVM_string_utf8_encode_C_string(tc, l->static_info->body.name),
254
        MVM_string_utf8_encode_C_string(tc, l->static_info->body.cuuid));
255
#endif
256
2.45k
    if (tc->instance->profiling)
257
0
        MVM_profiler_log_deopt_all(tc);
258
2.45k
259
16.7k
    while (f) {
260
16.4k
        clear_dynlex_cache(tc, f);
261
16.4k
        if (f->spesh_cand) {
262
3.23k
            /* Found one. Is it JITted code? */
263
3.23k
            if (f->spesh_cand->jitcode) {
264
3.23k
                MVMJitCode *jitcode = f->spesh_cand->jitcode;
265
3.23k
                MVMint32 idx = MVM_jit_code_get_active_deopt_idx(tc, jitcode, f);
266
3.23k
                if (idx < jitcode->num_deopts) {
267
2.16k
                    /* Resolve offset and target. */
268
2.16k
                    MVMint32 deopt_idx    = jitcode->deopts[idx].idx;
269
2.16k
                    MVMint32 deopt_offset = f->spesh_cand->deopts[2 * deopt_idx + 1];
270
2.16k
                    MVMint32 deopt_target = f->spesh_cand->deopts[2 * deopt_idx];
271
2.16k
#if MVM_LOG_DEOPTS
272
                    fprintf(stderr, "Found deopt label for JIT (%d) (label %d idx %d)\n", i,
273
                            deopts[i].label, deopts[i].idx);
274
#endif
275
2.16k
276
2.16k
                    /* Re-create any frames needed if we're in an inline; if not,
277
2.16k
                     * just update return address. */
278
2.16k
                    if (f->spesh_cand->inlines) {
279
1.02k
                        MVMROOT2(tc, f, l, {
280
1.02k
                            uninline(tc, f, f->spesh_cand, deopt_offset, deopt_target, l);
281
1.02k
                        });
282
1.02k
                    }
283
1.13k
                    else {
284
1.13k
                        f->return_address = f->static_info->body.bytecode + deopt_target;
285
1.13k
                    }
286
2.16k
287
2.16k
                    /* No spesh cand/slots needed now. */
288
2.16k
                    deopt_named_args_used(tc, f);
289
2.16k
                    f->effective_spesh_slots = NULL;
290
2.16k
                    f->spesh_cand            = NULL;
291
2.16k
                    f->jit_entry_label       = NULL;
292
2.16k
293
2.16k
                    break;
294
2.16k
                }
295
3.23k
#if MVM_LOG_DEOPTS
296
                if (i == num_deopts)
297
                    fprintf(stderr, "JIT: can't find deopt all idx\n");
298
#endif
299
3.23k
            }
300
3.23k
301
0
            else {
302
0
                /* Not JITted; see if we can find the return address in the deopt table. */
303
0
                MVMint32 ret_offset = f->return_address - f->spesh_cand->bytecode;
304
0
                MVMint32 n = f->spesh_cand->num_deopts * 2;
305
0
                MVMint32 i;
306
0
                for (i = 0; i < n; i += 2) {
307
0
                    if (f->spesh_cand->deopts[i + 1] == ret_offset) {
308
0
                        /* Re-create any frames needed if we're in an inline; if not,
309
0
                        * just update return address. */
310
0
                        if (f->spesh_cand->inlines) {
311
0
                            MVMROOT2(tc, f, l, {
312
0
                                uninline(tc, f, f->spesh_cand, ret_offset, f->spesh_cand->deopts[i], l);
313
0
                            });
314
0
                        }
315
0
                        else {
316
0
                            f->return_address = f->static_info->body.bytecode + f->spesh_cand->deopts[i];
317
0
                        }
318
0
319
0
                        /* No spesh cand/slots needed now. */
320
0
                        f->effective_spesh_slots = NULL;
321
0
                        f->spesh_cand            = NULL;
322
0
323
0
                        break;
324
0
                    }
325
0
                }
326
0
#if MVM_LOG_DEOPTS
327
                if (i == n)
328
                    fprintf(stderr, "Interpreter: can't find deopt all idx\n");
329
#endif
330
0
            }
331
3.23k
        }
332
14.3k
        l = f;
333
14.3k
        f = f->caller;
334
14.3k
    }
335
2.45k
336
2.45k
    MVM_CHECK_CALLER_CHAIN(tc, tc->cur_frame);
337
2.45k
}