/home/travis/build/MoarVM/MoarVM/src/spesh/inline.c
Line | Count | Source (jump to first uncovered line) |
1 | | #include "moar.h" |
2 | | |
3 | | /* Ensures that a given compilation unit has access to the specified extop. */ |
4 | | static void demand_extop(MVMThreadContext *tc, MVMCompUnit *target_cu, MVMCompUnit *source_cu, |
5 | 0 | const MVMOpInfo *info) { |
6 | 0 | MVMExtOpRecord *extops; |
7 | 0 | MVMuint16 i, num_extops; |
8 | 0 |
|
9 | 0 | uv_mutex_lock(target_cu->body.inline_tweak_mutex); |
10 | 0 |
|
11 | 0 | /* See if the target compunit already has the extop. */ |
12 | 0 | extops = target_cu->body.extops; |
13 | 0 | num_extops = target_cu->body.num_extops; |
14 | 0 | for (i = 0; i < num_extops; i++) |
15 | 0 | if (extops[i].info == info) { |
16 | 0 | uv_mutex_unlock(target_cu->body.inline_tweak_mutex); |
17 | 0 | return; |
18 | 0 | } |
19 | 0 |
|
20 | 0 | /* If not, need to add it. Locate it in the source CU. */ |
21 | 0 | extops = source_cu->body.extops; |
22 | 0 | num_extops = source_cu->body.num_extops; |
23 | 0 | for (i = 0; i < num_extops; i++) { |
24 | 0 | if (extops[i].info == info) { |
25 | 0 | MVMuint32 orig_size = target_cu->body.num_extops * sizeof(MVMExtOpRecord); |
26 | 0 | MVMuint32 new_size = (target_cu->body.num_extops + 1) * sizeof(MVMExtOpRecord); |
27 | 0 | MVMExtOpRecord *new_extops = MVM_fixed_size_alloc(tc, |
28 | 0 | tc->instance->fsa, new_size); |
29 | 0 | memcpy(new_extops, target_cu->body.extops, orig_size); |
30 | 0 | memcpy(&new_extops[target_cu->body.num_extops], &extops[i], sizeof(MVMExtOpRecord)); |
31 | 0 | if (target_cu->body.extops) |
32 | 0 | MVM_fixed_size_free_at_safepoint(tc, tc->instance->fsa, orig_size, |
33 | 0 | target_cu->body.extops); |
34 | 0 | target_cu->body.extops = new_extops; |
35 | 0 | target_cu->body.num_extops++; |
36 | 0 | uv_mutex_unlock(target_cu->body.inline_tweak_mutex); |
37 | 0 | return; |
38 | 0 | } |
39 | 0 | } |
40 | 0 |
|
41 | 0 | /* Didn't find it; should be impossible. */ |
42 | 0 | uv_mutex_unlock(target_cu->body.inline_tweak_mutex); |
43 | 0 | MVM_oops(tc, "Spesh: inline failed to find source CU extop entry"); |
44 | 0 | } |
45 | | |
46 | | /* Sees if it will be possible to inline the target code ref, given we could |
47 | | * already identify a spesh candidate. Returns NULL if no inlining is possible |
48 | | * or a graph ready to be merged if it will be possible. */ |
49 | | MVMSpeshGraph * MVM_spesh_inline_try_get_graph(MVMThreadContext *tc, MVMSpeshGraph *inliner, |
50 | | MVMStaticFrame *target_sf, |
51 | | MVMSpeshCandidate *cand, |
52 | | MVMSpeshIns *invoke_ins, |
53 | 13.7k | char **no_inline_reason) { |
54 | 13.7k | MVMSpeshGraph *ig; |
55 | 13.7k | MVMSpeshBB *bb; |
56 | 13.7k | MVMint32 same_hll; |
57 | 13.7k | |
58 | 13.7k | /* Check inlining is enabled. */ |
59 | 13.7k | if (!tc->instance->spesh_inline_enabled) { |
60 | 0 | *no_inline_reason = "inlining is disabled"; |
61 | 0 | return NULL; |
62 | 0 | } |
63 | 13.7k | |
64 | 13.7k | /* Check frame is not marked as not being allowed to be inlined. */ |
65 | 13.7k | if (target_sf->body.no_inline) { |
66 | 0 | *no_inline_reason = "the frame is marked as no-inline"; |
67 | 0 | return NULL; |
68 | 0 | } |
69 | 13.7k | |
70 | 13.7k | /* Check bytecode size is within the inline limit. */ |
71 | 13.7k | if (cand->bytecode_size > MVM_SPESH_MAX_INLINE_SIZE) { |
72 | 3.74k | *no_inline_reason = "bytecode is too large to inline"; |
73 | 3.74k | return NULL; |
74 | 3.74k | } |
75 | 13.7k | |
76 | 13.7k | /* Ensure that this isn't a recursive inlining. */ |
77 | 10.0k | if (target_sf == inliner->sf) { |
78 | 1 | *no_inline_reason = "recursive calls cannot be inlined"; |
79 | 1 | return NULL; |
80 | 1 | } |
81 | 10.0k | |
82 | 10.0k | /* Ensure it has no state vars (these need the setup code in frame |
83 | 10.0k | * invoke). */ |
84 | 10.0k | if (target_sf->body.has_state_vars) { |
85 | 0 | *no_inline_reason = "cannot inline code that declares a state variable"; |
86 | 0 | return NULL; |
87 | 0 | } |
88 | 10.0k | |
89 | 10.0k | /* Ensure it's not a thunk (need to skip over those in exception search). */ |
90 | 10.0k | if (target_sf->body.is_thunk) { |
91 | 0 | *no_inline_reason = "cannot inline code marked as a thunk"; |
92 | 0 | return NULL; |
93 | 0 | } |
94 | 10.0k | |
95 | 10.0k | /* If they're from the same HLL, we'll need to watch out for ops that are |
96 | 10.0k | * HLL sensitive. */ |
97 | 10.0k | same_hll = target_sf->body.cu->body.hll_config == inliner->sf->body.cu->body.hll_config; |
98 | 10.0k | |
99 | 10.0k | /* Build graph from the already-specialized bytecode. */ |
100 | 10.0k | ig = MVM_spesh_graph_create_from_cand(tc, target_sf, cand, 0); |
101 | 10.0k | |
102 | 10.0k | /* Traverse graph, looking for anything that might prevent inlining and |
103 | 10.0k | * also building usage counts up. */ |
104 | 10.0k | bb = ig->entry; |
105 | 47.1k | while (bb) { |
106 | 39.5k | MVMSpeshIns *ins = bb->first_ins; |
107 | 212k | while (ins) { |
108 | 175k | /* Track usages. */ |
109 | 175k | MVMint32 opcode = ins->info->opcode; |
110 | 175k | MVMint32 is_phi = opcode == MVM_SSA_PHI; |
111 | 175k | MVMuint8 i; |
112 | 561k | for (i = 0; i < ins->info->num_operands; i++) |
113 | 385k | if ((is_phi && i > 0) |
114 | 336k | || (!is_phi && (ins->info->operands[i] & MVM_operand_rw_mask) == MVM_operand_read_reg)) |
115 | 155k | ig->facts[ins->operands[i].reg.orig][ins->operands[i].reg.i].usages++; |
116 | 175k | if (opcode == MVM_OP_inc_i || opcode == MVM_OP_inc_u || |
117 | 175k | opcode == MVM_OP_dec_i || opcode == MVM_OP_dec_u) |
118 | 0 | ig->facts[ins->operands[0].reg.orig][ins->operands[0].reg.i - 1].usages++; |
119 | 175k | |
120 | 175k | /* Instruction may be marked directly as not being inlinable, in |
121 | 175k | * which case we're done. */ |
122 | 175k | if (!is_phi && ins->info->no_inline) { |
123 | 2.42k | *no_inline_reason = "target has a :noinline instruction"; |
124 | 2.42k | goto not_inlinable; |
125 | 2.42k | } |
126 | 175k | |
127 | 175k | /* If we don't have the same HLL and there's a :useshll op, we |
128 | 175k | * cannot inline. */ |
129 | 173k | if (!same_hll && ins->info->uses_hll) { |
130 | 0 | *no_inline_reason = "target has a :useshll instruction and HLLs are different"; |
131 | 0 | goto not_inlinable; |
132 | 0 | } |
133 | 173k | |
134 | 173k | /* If we have an invoke_o, but a return_[ins] that would require |
135 | 173k | * boxing, we can't inline if it's not the same HLL. */ |
136 | 173k | if (!same_hll && invoke_ins->info->opcode == MVM_OP_invoke_o) { |
137 | 0 | switch (ins->info->opcode) { |
138 | 0 | case MVM_OP_return_i: |
139 | 0 | case MVM_OP_return_n: |
140 | 0 | case MVM_OP_return_s: |
141 | 0 | *no_inline_reason = "target needs a return boxing and HLLs are different"; |
142 | 0 | goto not_inlinable; |
143 | 0 | } |
144 | 0 | } |
145 | 173k | |
146 | 173k | /* If we have lexical bind, make sure it's within the frame. */ |
147 | 173k | if (ins->info->opcode == MVM_OP_bindlex) { |
148 | 102 | if (ins->operands[0].lex.outers > 0) { |
149 | 50 | *no_inline_reason = "target has bind to outer lexical"; |
150 | 50 | goto not_inlinable; |
151 | 50 | } |
152 | 102 | } |
153 | 173k | |
154 | 173k | /* Check we don't have too many args for inlining to work out. */ |
155 | 173k | else if (ins->info->opcode == MVM_OP_sp_getarg_o || |
156 | 161k | ins->info->opcode == MVM_OP_sp_getarg_i || |
157 | 160k | ins->info->opcode == MVM_OP_sp_getarg_n || |
158 | 160k | ins->info->opcode == MVM_OP_sp_getarg_s) { |
159 | 14.2k | if (ins->operands[1].lit_i16 >= MAX_ARGS_FOR_OPT) { |
160 | 1 | *no_inline_reason = "too many arguments to inline"; |
161 | 1 | goto not_inlinable; |
162 | 1 | } |
163 | 14.2k | } |
164 | 173k | |
165 | 173k | /* Ext-ops need special care in inter-comp-unit inlines. */ |
166 | 173k | if (ins->info->opcode == (MVMuint16)-1) { |
167 | 0 | MVMCompUnit *target_cu = inliner->sf->body.cu; |
168 | 0 | MVMCompUnit *source_cu = target_sf->body.cu; |
169 | 0 | if (source_cu != target_cu) |
170 | 0 | demand_extop(tc, target_cu, source_cu, ins->info); |
171 | 0 | } |
172 | 173k | |
173 | 173k | ins = ins->next; |
174 | 173k | } |
175 | 37.0k | bb = bb->linear_next; |
176 | 37.0k | } |
177 | 10.0k | |
178 | 10.0k | /* If we found nothing we can't inline, inlining is fine. */ |
179 | 7.57k | return ig; |
180 | 10.0k | |
181 | 10.0k | /* If we can't find a way to inline, we end up here. */ |
182 | 2.47k | not_inlinable: |
183 | 2.47k | MVM_free(ig->spesh_slots); |
184 | 2.47k | MVM_spesh_graph_destroy(tc, ig); |
185 | 2.47k | return NULL; |
186 | 10.0k | } |
187 | | |
188 | | /* Finds the deopt index of the return. */ |
189 | 7.57k | static MVMint32 return_deopt_idx(MVMThreadContext *tc, MVMSpeshIns *invoke_ins) { |
190 | 7.57k | MVMSpeshAnn *ann = invoke_ins->annotations; |
191 | 7.57k | while (ann) { |
192 | 7.57k | if (ann->type == MVM_SPESH_ANN_DEOPT_ALL_INS) |
193 | 7.57k | return ann->data.deopt_idx; |
194 | 0 | ann = ann->next; |
195 | 0 | } |
196 | 0 | MVM_oops(tc, "Spesh inline: return_deopt_idx failed"); |
197 | 0 | } |
198 | | |
199 | | /* The following routines fix references to per-compilation-unit things |
200 | | * that would be broken by inlining. */ |
201 | | static void fix_callsite(MVMThreadContext *tc, MVMSpeshGraph *inliner, |
202 | 837 | MVMSpeshGraph *inlinee, MVMSpeshOperand *to_fix) { |
203 | 837 | to_fix->callsite_idx = MVM_cu_callsite_add(tc, inliner->sf->body.cu, |
204 | 837 | inlinee->sf->body.cu->body.callsites[to_fix->callsite_idx]); |
205 | 837 | } |
206 | | static void fix_coderef(MVMThreadContext *tc, MVMSpeshGraph *inliner, |
207 | 0 | MVMSpeshGraph *inlinee, MVMSpeshIns *to_fix) { |
208 | 0 | MVM_oops(tc, "Spesh inline: fix_coderef NYI"); |
209 | 0 | } |
210 | | static void fix_const_str(MVMThreadContext *tc, MVMSpeshGraph *inliner, |
211 | 1.78k | MVMSpeshGraph *inlinee, MVMSpeshIns *to_fix) { |
212 | 1.78k | MVMCompUnit *cu = inlinee->sf->body.cu; |
213 | 1.78k | MVMuint16 sslot; |
214 | 1.78k | MVMSpeshOperand dest = to_fix->operands[0]; |
215 | 1.78k | MVMuint16 str_idx = to_fix->operands[1].lit_str_idx; |
216 | 1.78k | |
217 | 1.78k | sslot = MVM_spesh_add_spesh_slot_try_reuse(tc, inlinee, (MVMCollectable *)cu); |
218 | 1.78k | |
219 | 1.78k | /*fprintf(stderr, "fixed up string get to use spesh slot %u\n", sslot);*/ |
220 | 1.78k | |
221 | 1.78k | to_fix->info = MVM_op_get_op(MVM_OP_sp_getstringfrom); |
222 | 1.78k | to_fix->operands = MVM_spesh_alloc(tc, inliner, 3 * sizeof(MVMSpeshOperand)); |
223 | 1.78k | to_fix->operands[0] = dest; |
224 | 1.78k | to_fix->operands[1].lit_i16 = sslot; |
225 | 1.78k | to_fix->operands[2].lit_str_idx = str_idx; |
226 | 1.78k | } |
227 | | static void fix_str(MVMThreadContext *tc, MVMSpeshGraph *inliner, |
228 | 304 | MVMSpeshGraph *inlinee, MVMSpeshOperand *to_fix) { |
229 | 304 | to_fix->lit_str_idx = MVM_cu_string_add(tc, inliner->sf->body.cu, |
230 | 304 | MVM_cu_string(tc, inlinee->sf->body.cu, to_fix->lit_str_idx)); |
231 | 304 | } |
232 | | static void fix_wval(MVMThreadContext *tc, MVMSpeshGraph *inliner, |
233 | 3.75k | MVMSpeshGraph *inlinee, MVMSpeshIns *to_fix) { |
234 | 3.75k | /* We have three different ways to make a wval refer to the right |
235 | 3.75k | * object after an inline: |
236 | 3.75k | * - If a simple change of the dep parameter lets us refer to the |
237 | 3.75k | * same SC in the inliner's dependency list, just change that. |
238 | 3.75k | * - If the object had already been deserialized, we can just |
239 | 3.75k | * stash it away in a spesh slot. |
240 | 3.75k | * - If the object hadn't been deserialized yet, an indirect lookup |
241 | 3.75k | * via a spesh slot holding the actual SC is possible. |
242 | 3.75k | * |
243 | 3.75k | * The last option is needed because deserializing objects causes |
244 | 3.75k | * allocations, which lead to GC, which is not supported inside |
245 | 3.75k | * the spesh process. |
246 | 3.75k | */ |
247 | 3.75k | MVMCompUnit *targetcu = inliner->sf->body.cu; |
248 | 3.75k | MVMCompUnit *sourcecu = inlinee->sf->body.cu; |
249 | 3.75k | MVMint16 dep = to_fix->operands[1].lit_i16; |
250 | 3.75k | MVMint64 idx = to_fix->info->opcode == MVM_OP_wval |
251 | 3.75k | ? to_fix->operands[2].lit_i16 |
252 | 0 | : to_fix->operands[2].lit_i64; |
253 | 3.75k | if (dep >= 0 && dep < sourcecu->body.num_scs) { |
254 | 3.75k | MVMSerializationContext *sc = MVM_sc_get_sc(tc, sourcecu, dep); |
255 | 3.75k | if (sc) { |
256 | 3.75k | MVMuint16 otherdep; |
257 | 16.2k | for (otherdep = 0; otherdep < targetcu->body.num_scs; otherdep++) { |
258 | 15.8k | MVMSerializationContext *othersc = MVM_sc_get_sc(tc, targetcu, otherdep); |
259 | 15.8k | if (sc == othersc) { |
260 | 3.31k | to_fix->operands[1].lit_i16 = otherdep; |
261 | 3.31k | return; |
262 | 3.31k | } |
263 | 15.8k | } |
264 | 440 | if (MVM_sc_is_object_immediately_available(tc, sc, idx)) { |
265 | 439 | MVMint16 ss = MVM_spesh_add_spesh_slot_try_reuse(tc, inliner, (MVMCollectable *)MVM_sc_get_object(tc, sc, idx)); |
266 | 439 | to_fix->info = MVM_op_get_op(MVM_OP_sp_getspeshslot); |
267 | 439 | to_fix->operands[1].lit_i16 = ss; |
268 | 439 | } |
269 | 1 | else { |
270 | 1 | MVMint16 ss = MVM_spesh_add_spesh_slot_try_reuse(tc, inliner, (MVMCollectable *)sc); |
271 | 1 | to_fix->info = MVM_op_get_op(MVM_OP_sp_getwvalfrom); |
272 | 1 | to_fix->operands[1].lit_i16 = ss; |
273 | 1 | to_fix->operands[2].lit_i64 = idx; |
274 | 1 | } |
275 | 440 | } |
276 | 0 | else { |
277 | 0 | MVM_oops(tc, |
278 | 0 | "Spesh inline: SC not yet resolved; lookup failed"); |
279 | 0 | } |
280 | 3.75k | } |
281 | 0 | else { |
282 | 0 | MVM_oops(tc, |
283 | 0 | "Spesh inline: invalid SC index %d found", dep); |
284 | 0 | } |
285 | 3.75k | } |
286 | | |
287 | | /* Resizes the handlers table, making a copy if needed. */ |
288 | 7.57k | static void resize_handlers_table(MVMThreadContext *tc, MVMSpeshGraph *inliner, MVMuint32 new_handler_count) { |
289 | 7.57k | if (inliner->handlers == inliner->sf->body.handlers) { |
290 | 3.28k | /* Original handlers table; need a copy. */ |
291 | 3.28k | MVMFrameHandler *new_handlers = MVM_malloc(new_handler_count * sizeof(MVMFrameHandler)); |
292 | 3.28k | if (inliner->handlers) |
293 | 725 | memcpy(new_handlers, inliner->handlers, |
294 | 725 | inliner->num_handlers * sizeof(MVMFrameHandler)); |
295 | 3.28k | inliner->handlers = new_handlers; |
296 | 3.28k | } |
297 | 4.29k | else { |
298 | 4.29k | /* Probably already did some inlines into this frame; resize. */ |
299 | 4.29k | inliner->handlers = MVM_realloc(inliner->handlers, |
300 | 4.29k | new_handler_count * sizeof(MVMFrameHandler)); |
301 | 4.29k | } |
302 | 7.57k | } |
303 | | |
304 | | /* Rewrites a lexical lookup to an outer to be done via. a register holding |
305 | | * the outer coderef. */ |
306 | | static void rewrite_outer_lookup(MVMThreadContext *tc, MVMSpeshGraph *g, |
307 | | MVMSpeshIns *ins, MVMuint16 num_locals, |
308 | 370 | MVMuint16 op, MVMSpeshOperand code_ref_reg) { |
309 | 370 | MVMSpeshOperand *new_operands = MVM_spesh_alloc(tc, g, 4 * sizeof(MVMSpeshOperand)); |
310 | 370 | new_operands[0] = ins->operands[0]; |
311 | 370 | new_operands[0].reg.orig += num_locals; |
312 | 370 | new_operands[1].lit_ui16 = ins->operands[1].lex.idx; |
313 | 370 | new_operands[2].lit_ui16 = ins->operands[1].lex.outers; |
314 | 370 | new_operands[3] = code_ref_reg; |
315 | 370 | ins->info = MVM_op_get_op(op); |
316 | 370 | ins->operands = new_operands; |
317 | 370 | } |
318 | | |
319 | | /* Merges the inlinee's spesh graph into the inliner. */ |
320 | | MVMSpeshBB * merge_graph(MVMThreadContext *tc, MVMSpeshGraph *inliner, |
321 | | MVMSpeshGraph *inlinee, MVMStaticFrame *inlinee_sf, |
322 | | MVMSpeshBB *invoke_bb, MVMSpeshIns *invoke_ins, |
323 | | MVMSpeshOperand code_ref_reg, |
324 | 7.57k | MVMuint32 *inline_boundary_handler) { |
325 | 7.57k | MVMSpeshFacts **merged_facts; |
326 | 7.57k | MVMuint16 *merged_fact_counts; |
327 | 7.57k | MVMint32 i, orig_inlines, total_inlines, orig_deopt_addrs; |
328 | 7.57k | MVMuint32 total_handlers = inliner->num_handlers + inlinee->num_handlers + 1; |
329 | 7.57k | MVMSpeshBB *inlinee_first_bb = NULL, *inlinee_last_bb = NULL; |
330 | 7.57k | |
331 | 7.57k | /* If the inliner and inlinee are from different compilation units, we |
332 | 7.57k | * potentially have to fix up extra things. */ |
333 | 7.57k | MVMint32 same_comp_unit = inliner->sf->body.cu == inlinee->sf->body.cu; |
334 | 7.57k | |
335 | 7.57k | /* Renumber the locals, lexicals, and basic blocks of the inlinee; also |
336 | 7.57k | * re-write any indexes in annotations that need it. */ |
337 | 7.57k | MVMSpeshBB *bb = inlinee->entry; |
338 | 41.1k | while (bb) { |
339 | 33.6k | MVMSpeshIns *ins = bb->first_ins; |
340 | 192k | while (ins) { |
341 | 158k | MVMuint16 opcode = ins->info->opcode; |
342 | 158k | MVMSpeshAnn *ann = ins->annotations; |
343 | 211k | while (ann) { |
344 | 53.3k | switch (ann->type) { |
345 | 46.7k | case MVM_SPESH_ANN_DEOPT_INLINE: |
346 | 46.7k | ann->data.deopt_idx += inliner->num_deopt_addrs; |
347 | 46.7k | break; |
348 | 1.93k | case MVM_SPESH_ANN_INLINE_START: |
349 | 1.93k | case MVM_SPESH_ANN_INLINE_END: |
350 | 1.93k | ann->data.inline_idx += inliner->num_inlines; |
351 | 1.93k | break; |
352 | 53.3k | } |
353 | 53.3k | ann = ann->next; |
354 | 53.3k | } |
355 | 158k | |
356 | 158k | if (opcode == MVM_SSA_PHI) { |
357 | 89.3k | for (i = 0; i < ins->info->num_operands; i++) |
358 | 69.3k | ins->operands[i].reg.orig += inliner->num_locals; |
359 | 20.0k | } |
360 | 138k | else if (opcode == MVM_OP_sp_getlex_o && ins->operands[1].lex.outers > 0) { |
361 | 255 | rewrite_outer_lookup(tc, inliner, ins, inliner->num_locals, |
362 | 255 | MVM_OP_sp_getlexvia_o, code_ref_reg); |
363 | 255 | } |
364 | 138k | else if (opcode == MVM_OP_sp_getlex_ins && ins->operands[1].lex.outers > 0) { |
365 | 115 | rewrite_outer_lookup(tc, inliner, ins, inliner->num_locals, |
366 | 115 | MVM_OP_sp_getlexvia_ins, code_ref_reg); |
367 | 115 | } |
368 | 138k | else if (opcode == MVM_OP_getlex && ins->operands[1].lex.outers > 0) { |
369 | 0 | MVMuint16 outers = ins->operands[1].lex.outers; |
370 | 0 | MVMStaticFrame *outer = inlinee_sf; |
371 | 0 | while (outers--) |
372 | 0 | outer = outer->body.outer; |
373 | 0 | if (outer->body.lexical_types[ins->operands[1].lex.idx] == MVM_reg_obj) |
374 | 0 | rewrite_outer_lookup(tc, inliner, ins, inliner->num_locals, |
375 | 0 | MVM_OP_sp_getlexvia_o, code_ref_reg); |
376 | 0 | else |
377 | 0 | rewrite_outer_lookup(tc, inliner, ins, inliner->num_locals, |
378 | 0 | MVM_OP_sp_getlexvia_ins, code_ref_reg); |
379 | 0 | } |
380 | 138k | else { |
381 | 138k | if (!same_comp_unit) { |
382 | 91.1k | if (ins->info->opcode == MVM_OP_const_s) { |
383 | 1.78k | fix_const_str(tc, inliner, inlinee, ins); |
384 | 1.78k | } |
385 | 91.1k | } |
386 | 138k | |
387 | 428k | for (i = 0; i < ins->info->num_operands; i++) { |
388 | 290k | MVMuint8 flags = ins->info->operands[i]; |
389 | 290k | switch (flags & MVM_operand_rw_mask) { |
390 | 200k | case MVM_operand_read_reg: |
391 | 200k | case MVM_operand_write_reg: |
392 | 200k | ins->operands[i].reg.orig += inliner->num_locals; |
393 | 200k | break; |
394 | 53 | case MVM_operand_read_lex: |
395 | 53 | case MVM_operand_write_lex: |
396 | 53 | ins->operands[i].lex.idx += inliner->num_lexicals; |
397 | 53 | break; |
398 | 90.1k | default: { |
399 | 90.1k | MVMuint32 type = flags & MVM_operand_type_mask; |
400 | 90.1k | if (type == MVM_operand_spesh_slot) { |
401 | 23.6k | ins->operands[i].lit_i16 += inliner->num_spesh_slots; |
402 | 23.6k | } |
403 | 66.4k | else if (type == MVM_operand_coderef) { |
404 | 0 | if (!same_comp_unit) |
405 | 0 | /* NYI, but apparently this never happens?! */ |
406 | 0 | fix_coderef(tc, inliner, inlinee, ins); |
407 | 0 | } |
408 | 66.4k | else if (type == MVM_operand_str) { |
409 | 587 | if (!same_comp_unit) |
410 | 304 | fix_str(tc, inliner, inlinee, &(ins->operands[i])); |
411 | 587 | } |
412 | 65.8k | else if (type == MVM_operand_callsite) { |
413 | 2.11k | if (!same_comp_unit) |
414 | 837 | fix_callsite(tc, inliner, inlinee, &(ins->operands[i])); |
415 | 2.11k | } |
416 | 90.1k | break; |
417 | 53 | } |
418 | 290k | } |
419 | 290k | } |
420 | 138k | } |
421 | 158k | |
422 | 158k | ins = ins->next; |
423 | 158k | } |
424 | 33.6k | bb->idx += inliner->num_bbs - 1; /* -1 as we won't include entry */ |
425 | 33.6k | bb->inlined = 1; |
426 | 33.6k | if (!bb->linear_next) |
427 | 7.57k | inlinee_last_bb = bb; |
428 | 33.6k | bb = bb->linear_next; |
429 | 33.6k | } |
430 | 7.57k | |
431 | 7.57k | bb = invoke_bb->linear_next; |
432 | 7.57k | invoke_bb->linear_next = inlinee_first_bb = inlinee->entry->linear_next; |
433 | 7.57k | inlinee_last_bb->linear_next = bb; |
434 | 7.57k | |
435 | 7.57k | bb = NULL; |
436 | 7.57k | |
437 | 7.57k | /* Make all of the inlinee's entry block's successors (except the linear |
438 | 7.57k | * next) also be successors of the inliner's entry block; this keeps any |
439 | 7.57k | * exception handlers alive in the graph. */ |
440 | 7.57k | while (inlinee->entry->num_succ > 1) { |
441 | 0 | MVMSpeshBB *move = inlinee->entry->succ[0] == inlinee->entry->linear_next |
442 | 0 | ? inlinee->entry->succ[1] |
443 | 0 | : inlinee->entry->succ[0]; |
444 | 0 | MVM_spesh_manipulate_remove_successor(tc, inlinee->entry, move); |
445 | 0 | MVM_spesh_manipulate_add_successor(tc, inliner, inliner->entry, move); |
446 | 0 | } |
447 | 7.57k | |
448 | 7.57k | /* Merge facts. */ |
449 | 7.57k | merged_facts = MVM_spesh_alloc(tc, inliner, |
450 | 7.57k | (inliner->num_locals + inlinee->num_locals) * sizeof(MVMSpeshFacts *)); |
451 | 7.57k | memcpy(merged_facts, inliner->facts, |
452 | 7.57k | inliner->num_locals * sizeof(MVMSpeshFacts *)); |
453 | 7.57k | memcpy(merged_facts + inliner->num_locals, inlinee->facts, |
454 | 7.57k | inlinee->num_locals * sizeof(MVMSpeshFacts *)); |
455 | 7.57k | inliner->facts = merged_facts; |
456 | 7.57k | merged_fact_counts = MVM_spesh_alloc(tc, inliner, |
457 | 7.57k | (inliner->num_locals + inlinee->num_locals) * sizeof(MVMuint16)); |
458 | 7.57k | memcpy(merged_fact_counts, inliner->fact_counts, |
459 | 7.57k | inliner->num_locals * sizeof(MVMuint16)); |
460 | 7.57k | memcpy(merged_fact_counts + inliner->num_locals, inlinee->fact_counts, |
461 | 7.57k | inlinee->num_locals * sizeof(MVMuint16)); |
462 | 7.57k | inliner->fact_counts = merged_fact_counts; |
463 | 7.57k | |
464 | 7.57k | /* Copy over spesh slots. */ |
465 | 33.4k | for (i = 0; i < inlinee->num_spesh_slots; i++) |
466 | 25.8k | MVM_spesh_add_spesh_slot(tc, inliner, inlinee->spesh_slots[i]); |
467 | 7.57k | |
468 | 7.57k | /* If they are from separate compilation units, make another pass through |
469 | 7.57k | * to fix up on wvals. Note we can't do this in the first pass as we must |
470 | 7.57k | * not modify the spesh slots once we've got started with the rewrites. |
471 | 7.57k | * Now we've resolved all that, we're good to map wvals elsewhere into |
472 | 7.57k | * some extra spesh slots. */ |
473 | 7.57k | if (!same_comp_unit) { |
474 | 4.86k | bb = inlinee->entry; |
475 | 20.2k | while (1) { |
476 | 20.2k | MVMSpeshIns *ins = bb->first_ins; |
477 | 123k | while (ins) { |
478 | 103k | MVMuint16 opcode = ins->info->opcode; |
479 | 103k | if (opcode == MVM_OP_wval || opcode == MVM_OP_wval_wide) |
480 | 3.75k | fix_wval(tc, inliner, inlinee, ins); |
481 | 103k | ins = ins->next; |
482 | 103k | } |
483 | 20.2k | if (bb == inlinee_last_bb) break; |
484 | 15.3k | bb = bb->linear_next; |
485 | 15.3k | } |
486 | 4.86k | } |
487 | 7.57k | |
488 | 7.57k | /* Merge de-opt tables, if needed. */ |
489 | 7.57k | orig_deopt_addrs = inliner->num_deopt_addrs; |
490 | 7.57k | if (inlinee->num_deopt_addrs) { |
491 | 7.56k | assert(inlinee->deopt_addrs != inliner->deopt_addrs); |
492 | 7.56k | inliner->alloc_deopt_addrs += inlinee->alloc_deopt_addrs; |
493 | 7.56k | if (inliner->deopt_addrs) |
494 | 7.56k | inliner->deopt_addrs = MVM_realloc(inliner->deopt_addrs, |
495 | 7.56k | inliner->alloc_deopt_addrs * sizeof(MVMint32) * 2); |
496 | 7.56k | else |
497 | 0 | inliner->deopt_addrs = MVM_malloc(inliner->alloc_deopt_addrs * sizeof(MVMint32) * 2); |
498 | 7.56k | memcpy(inliner->deopt_addrs + inliner->num_deopt_addrs * 2, |
499 | 7.56k | inlinee->deopt_addrs, inlinee->alloc_deopt_addrs * sizeof(MVMint32) * 2); |
500 | 7.56k | inliner->num_deopt_addrs += inlinee->num_deopt_addrs; |
501 | 7.56k | } |
502 | 7.57k | |
503 | 7.57k | /* Merge inlines table, and add us an entry too. */ |
504 | 7.57k | orig_inlines = inliner->num_inlines; |
505 | 7.57k | total_inlines = inliner->num_inlines + inlinee->num_inlines + 1; |
506 | 7.57k | inliner->inlines = inliner->num_inlines |
507 | 4.29k | ? MVM_realloc(inliner->inlines, total_inlines * sizeof(MVMSpeshInline)) |
508 | 3.28k | : MVM_malloc(total_inlines * sizeof(MVMSpeshInline)); |
509 | 7.57k | if (inlinee->num_inlines) |
510 | 843 | memcpy(inliner->inlines + inliner->num_inlines, inlinee->inlines, |
511 | 843 | inlinee->num_inlines * sizeof(MVMSpeshInline)); |
512 | 8.54k | for (i = inliner->num_inlines; i < total_inlines - 1; i++) { |
513 | 968 | inliner->inlines[i].code_ref_reg += inliner->num_locals; |
514 | 968 | inliner->inlines[i].locals_start += inliner->num_locals; |
515 | 968 | inliner->inlines[i].lexicals_start += inliner->num_lexicals; |
516 | 968 | inliner->inlines[i].return_deopt_idx += orig_deopt_addrs; |
517 | 968 | } |
518 | 7.57k | inliner->inlines[total_inlines - 1].sf = inlinee_sf; |
519 | 7.57k | inliner->inlines[total_inlines - 1].code_ref_reg = code_ref_reg.reg.orig; |
520 | 7.57k | inliner->inlines[total_inlines - 1].g = inlinee; |
521 | 7.57k | inliner->inlines[total_inlines - 1].locals_start = inliner->num_locals; |
522 | 7.57k | inliner->inlines[total_inlines - 1].lexicals_start = inliner->num_lexicals; |
523 | 7.57k | inliner->inlines[total_inlines - 1].num_locals = inlinee->num_locals; |
524 | 7.57k | switch (invoke_ins->info->opcode) { |
525 | 632 | case MVM_OP_invoke_v: |
526 | 632 | inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_VOID; |
527 | 632 | break; |
528 | 6.94k | case MVM_OP_invoke_o: |
529 | 6.94k | inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig; |
530 | 6.94k | inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_OBJ; |
531 | 6.94k | break; |
532 | 0 | case MVM_OP_invoke_i: |
533 | 0 | inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig; |
534 | 0 | inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_INT; |
535 | 0 | break; |
536 | 0 | case MVM_OP_invoke_n: |
537 | 0 | inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig; |
538 | 0 | inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_NUM; |
539 | 0 | break; |
540 | 0 | case MVM_OP_invoke_s: |
541 | 0 | inliner->inlines[total_inlines - 1].res_reg = invoke_ins->operands[0].reg.orig; |
542 | 0 | inliner->inlines[total_inlines - 1].res_type = MVM_RETURN_STR; |
543 | 0 | break; |
544 | 0 | default: |
545 | 0 | MVM_oops(tc, "Spesh inline: unknown invoke instruction"); |
546 | 7.57k | } |
547 | 7.57k | inliner->inlines[total_inlines - 1].return_deopt_idx = return_deopt_idx(tc, invoke_ins); |
548 | 7.57k | inliner->inlines[total_inlines - 1].unreachable = 0; |
549 | 7.57k | inliner->inlines[total_inlines - 1].deopt_named_used_bit_field = |
550 | 7.57k | inlinee->deopt_named_used_bit_field; |
551 | 7.57k | inliner->num_inlines = total_inlines; |
552 | 7.57k | |
553 | 7.57k | /* Create/update per-specialization local and lexical type maps. */ |
554 | 7.57k | if (!inliner->local_types && inliner->num_locals) { |
555 | 927 | MVMint32 local_types_size = inliner->num_locals * sizeof(MVMuint16); |
556 | 927 | inliner->local_types = MVM_malloc(local_types_size); |
557 | 927 | memcpy(inliner->local_types, inliner->sf->body.local_types, local_types_size); |
558 | 927 | } |
559 | 7.57k | inliner->local_types = MVM_realloc(inliner->local_types, |
560 | 7.57k | (inliner->num_locals + inlinee->num_locals) * sizeof(MVMuint16)); |
561 | 7.57k | if (inlinee->num_locals) |
562 | 7.57k | memcpy(inliner->local_types + inliner->num_locals, |
563 | 5.86k | inlinee->local_types ? inlinee->local_types : inlinee->sf->body.local_types, |
564 | 7.57k | inlinee->num_locals * sizeof(MVMuint16)); |
565 | 7.57k | if (!inliner->lexical_types && inliner->num_lexicals) { |
566 | 953 | MVMint32 lexical_types_size = inliner->num_lexicals * sizeof(MVMuint16); |
567 | 953 | inliner->lexical_types = MVM_malloc(lexical_types_size); |
568 | 953 | memcpy(inliner->lexical_types, inliner->sf->body.lexical_types, lexical_types_size); |
569 | 953 | } |
570 | 7.57k | inliner->lexical_types = MVM_realloc(inliner->lexical_types, |
571 | 7.57k | (inliner->num_lexicals + inlinee->num_lexicals) * sizeof(MVMuint16)); |
572 | 7.57k | if (inlinee->num_lexicals) |
573 | 18 | memcpy(inliner->lexical_types + inliner->num_lexicals, |
574 | 18 | inlinee->lexical_types ? inlinee->lexical_types : inlinee->sf->body.lexical_types, |
575 | 18 | inlinee->num_lexicals * sizeof(MVMuint16)); |
576 | 7.57k | |
577 | 7.57k | /* Merge unreachable handlers array if needed. */ |
578 | 7.57k | if (inliner->unreachable_handlers || inlinee->unreachable_handlers) { |
579 | 0 | MVMint8 *new_uh = MVM_spesh_alloc(tc, inliner, total_handlers); |
580 | 0 | if (inlinee->unreachable_handlers) |
581 | 0 | memcpy(new_uh, inlinee->unreachable_handlers, |
582 | 0 | inlinee->num_handlers); |
583 | 0 | new_uh[inlinee->num_handlers] = 0; |
584 | 0 | if (inliner->unreachable_handlers) |
585 | 0 | memcpy(new_uh + inlinee->num_handlers + 1, inliner->unreachable_handlers, inliner->num_handlers); |
586 | 0 | inliner->unreachable_handlers = new_uh; |
587 | 0 | } |
588 | 7.57k | |
589 | 7.57k | /* Merge handlers from inlinee. */ |
590 | 7.57k | resize_handlers_table(tc, inliner, total_handlers); |
591 | 7.57k | |
592 | 7.57k | if (inliner->num_handlers > 0) |
593 | 5.01k | memmove(inliner->handlers + inlinee->num_handlers + 1, inliner->handlers, |
594 | 5.01k | inliner->num_handlers * sizeof(MVMFrameHandler)); |
595 | 7.57k | |
596 | 7.57k | if (inlinee->num_handlers > 0) { |
597 | 876 | memcpy(inliner->handlers, inlinee->handlers, |
598 | 876 | inlinee->num_handlers * sizeof(MVMFrameHandler)); |
599 | 876 | |
600 | 2.05k | for (i = 0; i < inlinee->num_handlers; i++) { |
601 | 1.18k | inliner->handlers[i].block_reg += inliner->num_locals; |
602 | 1.18k | inliner->handlers[i].label_reg += inliner->num_locals; |
603 | 1.18k | if (inliner->handlers[i].inlinee == -1) |
604 | 212 | inliner->handlers[i].inlinee = total_inlines - 1; |
605 | 1.18k | else |
606 | 971 | inliner->handlers[i].inlinee += orig_inlines; |
607 | 1.18k | } |
608 | 876 | } |
609 | 7.57k | |
610 | 7.57k | /* Adjust indexes in inliner's frame handler annotations */ |
611 | 7.57k | bb = inliner->entry; |
612 | 1.30M | while (bb) { |
613 | 1.30M | if (bb == inlinee_first_bb) /* No need to adjust inlinee's annotations */ |
614 | 7.57k | bb = inlinee_last_bb->linear_next; |
615 | 1.30M | if (bb) { |
616 | 1.30M | MVMSpeshIns *ins = bb->first_ins; |
617 | 7.52M | while (ins) { |
618 | 6.22M | MVMSpeshAnn *ann = ins->annotations; |
619 | 7.65M | while (ann) { |
620 | 1.42M | switch (ann->type) { |
621 | 136k | case MVM_SPESH_ANN_FH_START: |
622 | 136k | case MVM_SPESH_ANN_FH_END: |
623 | 136k | case MVM_SPESH_ANN_FH_GOTO: |
624 | 136k | ann->data.frame_handler_index += inlinee->num_handlers + 1; |
625 | 1.42M | } |
626 | 1.42M | ann = ann->next; |
627 | 1.42M | } |
628 | 6.22M | ins = ins->next; |
629 | 6.22M | } |
630 | 1.30M | bb = bb->linear_next; |
631 | 1.30M | } |
632 | 1.30M | } |
633 | 7.57k | |
634 | 7.57k | /* Insert inline boundary entry into the handlers table. */ |
635 | 7.57k | *inline_boundary_handler = inlinee->num_handlers; |
636 | 7.57k | inliner->handlers[*inline_boundary_handler].category_mask = MVM_EX_INLINE_BOUNDARY; |
637 | 7.57k | inliner->handlers[*inline_boundary_handler].action = 0; |
638 | 7.57k | inliner->handlers[*inline_boundary_handler].inlinee = total_inlines - 1; |
639 | 7.57k | |
640 | 7.57k | /* Update total locals, lexicals, basic blocks, and handlers of the |
641 | 7.57k | * inliner. */ |
642 | 7.57k | inliner->num_bbs += inlinee->num_bbs - 1; |
643 | 7.57k | inliner->num_locals += inlinee->num_locals; |
644 | 7.57k | inliner->num_lexicals += inlinee->num_lexicals; |
645 | 7.57k | inliner->num_handlers += inlinee->num_handlers + 1; |
646 | 7.57k | |
647 | 7.57k | return inlinee_last_bb; |
648 | 7.57k | } |
649 | | |
650 | | /* Tweak the successor of a BB, also updating the target BBs pred. */ |
651 | 15.1k | static void tweak_succ(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshBB *bb, MVMSpeshBB *new_succ) { |
652 | 15.1k | if (bb->num_succ == 0) { |
653 | 7.57k | /* It had no successors, so we'll add one. */ |
654 | 7.57k | bb->succ = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshBB *)); |
655 | 7.57k | bb->num_succ = 1; |
656 | 7.57k | bb->succ[0] = new_succ; |
657 | 7.57k | } |
658 | 7.57k | else { |
659 | 7.57k | /* Otherwise, we can assume that the first successor is the one to |
660 | 7.57k | * update; others will be there as a result of control handlers, but |
661 | 7.57k | * these are always added last. */ |
662 | 7.57k | bb->succ[0] = new_succ; |
663 | 7.57k | } |
664 | 15.1k | if (new_succ->num_pred == 0) { |
665 | 0 | new_succ->pred = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshBB *)); |
666 | 0 | new_succ->num_pred = 1; |
667 | 0 | new_succ->pred[0] = bb; |
668 | 0 | } |
669 | 15.1k | else { |
670 | 15.1k | MVMint32 found = 0; |
671 | 15.1k | MVMint32 i; |
672 | 15.4k | for (i = 0; i < new_succ->num_pred; i++) |
673 | 15.4k | if (new_succ->pred[i]->idx + 1 == new_succ->idx) { |
674 | 15.1k | new_succ->pred[i] = bb; |
675 | 15.1k | found = 1; |
676 | 15.1k | break; |
677 | 15.1k | } |
678 | 15.1k | if (!found) |
679 | 0 | MVM_oops(tc, |
680 | 0 | "Spesh inline: could not find appropriate pred to update\n"); |
681 | 15.1k | } |
682 | 15.1k | } |
683 | | |
684 | | /* Finds return instructions and re-writes them into gotos, doing any needed |
685 | | * boxing or unboxing. */ |
686 | 5.09k | static void return_to_set(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshIns *return_ins, MVMSpeshOperand target) { |
687 | 5.09k | MVMSpeshOperand *operands = MVM_spesh_alloc(tc, g, 2 * sizeof(MVMSpeshOperand)); |
688 | 5.09k | operands[0] = target; |
689 | 5.09k | operands[1] = return_ins->operands[0]; |
690 | 5.09k | return_ins->info = MVM_op_get_op(MVM_OP_set); |
691 | 5.09k | return_ins->operands = operands; |
692 | 5.09k | } |
693 | | |
694 | | static void return_to_box(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshBB *return_bb, |
695 | | MVMSpeshIns *return_ins, MVMSpeshOperand target, |
696 | 1.85k | MVMuint16 box_type_op, MVMuint16 box_op) { |
697 | 1.85k | /* Create and insert boxing instruction after current return instruction. */ |
698 | 1.85k | MVMSpeshIns *box_ins = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshIns)); |
699 | 1.85k | MVMSpeshOperand *box_operands = MVM_spesh_alloc(tc, g, 3 * sizeof(MVMSpeshOperand)); |
700 | 1.85k | box_ins->info = MVM_op_get_op(box_op); |
701 | 1.85k | box_ins->operands = box_operands; |
702 | 1.85k | box_operands[0] = target; |
703 | 1.85k | box_operands[1] = return_ins->operands[0]; |
704 | 1.85k | box_operands[2] = target; |
705 | 1.85k | MVM_spesh_manipulate_insert_ins(tc, return_bb, return_ins, box_ins); |
706 | 1.85k | |
707 | 1.85k | /* Now turn return instruction node into lookup of appropriate box |
708 | 1.85k | * type. */ |
709 | 1.85k | return_ins->info = MVM_op_get_op(box_type_op); |
710 | 1.85k | return_ins->operands[0] = target; |
711 | 1.85k | } |
712 | | |
713 | | static void rewrite_int_return(MVMThreadContext *tc, MVMSpeshGraph *g, |
714 | | MVMSpeshBB *return_bb, MVMSpeshIns *return_ins, |
715 | 1.90k | MVMSpeshBB *invoke_bb, MVMSpeshIns *invoke_ins) { |
716 | 1.90k | switch (invoke_ins->info->opcode) { |
717 | 631 | case MVM_OP_invoke_v: |
718 | 631 | MVM_spesh_manipulate_delete_ins(tc, g, return_bb, return_ins); |
719 | 631 | break; |
720 | 0 | case MVM_OP_invoke_i: |
721 | 0 | return_to_set(tc, g, return_ins, invoke_ins->operands[0]); |
722 | 0 | break; |
723 | 1.27k | case MVM_OP_invoke_o: |
724 | 1.27k | return_to_box(tc, g, return_bb, return_ins, invoke_ins->operands[0], |
725 | 1.27k | MVM_OP_hllboxtype_i, MVM_OP_box_i); |
726 | 1.27k | break; |
727 | 0 | default: |
728 | 0 | MVM_oops(tc, |
729 | 0 | "Spesh inline: unhandled case of return_i"); |
730 | 1.90k | } |
731 | 1.90k | } |
732 | | static void rewrite_num_return(MVMThreadContext *tc, MVMSpeshGraph *g, |
733 | | MVMSpeshBB *return_bb, MVMSpeshIns *return_ins, |
734 | 1 | MVMSpeshBB *invoke_bb, MVMSpeshIns *invoke_ins) { |
735 | 1 | switch (invoke_ins->info->opcode) { |
736 | 0 | case MVM_OP_invoke_v: |
737 | 0 | MVM_spesh_manipulate_delete_ins(tc, g, return_bb, return_ins); |
738 | 0 | break; |
739 | 0 | case MVM_OP_invoke_n: |
740 | 0 | return_to_set(tc, g, return_ins, invoke_ins->operands[0]); |
741 | 0 | break; |
742 | 1 | case MVM_OP_invoke_o: |
743 | 1 | return_to_box(tc, g, return_bb, return_ins, invoke_ins->operands[0], |
744 | 1 | MVM_OP_hllboxtype_n, MVM_OP_box_n); |
745 | 1 | break; |
746 | 0 | default: |
747 | 0 | MVM_oops(tc, |
748 | 0 | "Spesh inline: unhandled case of return_n"); |
749 | 1 | } |
750 | 1 | } |
751 | | |
752 | | static void rewrite_str_return(MVMThreadContext *tc, MVMSpeshGraph *g, |
753 | | MVMSpeshBB *return_bb, MVMSpeshIns *return_ins, |
754 | 571 | MVMSpeshBB *invoke_bb, MVMSpeshIns *invoke_ins) { |
755 | 571 | switch (invoke_ins->info->opcode) { |
756 | 0 | case MVM_OP_invoke_v: |
757 | 0 | MVM_spesh_manipulate_delete_ins(tc, g, return_bb, return_ins); |
758 | 0 | break; |
759 | 0 | case MVM_OP_invoke_s: |
760 | 0 | return_to_set(tc, g, return_ins, invoke_ins->operands[0]); |
761 | 0 | break; |
762 | 571 | case MVM_OP_invoke_o: |
763 | 571 | return_to_box(tc, g, return_bb, return_ins, invoke_ins->operands[0], |
764 | 571 | MVM_OP_hllboxtype_s, MVM_OP_box_s); |
765 | 571 | break; |
766 | 0 | default: |
767 | 0 | MVM_oops(tc, |
768 | 0 | "Spesh inline: unhandled case of return_s"); |
769 | 571 | } |
770 | 571 | } |
771 | | |
772 | | static void rewrite_obj_return(MVMThreadContext *tc, MVMSpeshGraph *g, |
773 | | MVMSpeshBB *return_bb, MVMSpeshIns *return_ins, |
774 | 5.09k | MVMSpeshBB *invoke_bb, MVMSpeshIns *invoke_ins) { |
775 | 5.09k | switch (invoke_ins->info->opcode) { |
776 | 1 | case MVM_OP_invoke_v: |
777 | 1 | MVM_spesh_manipulate_delete_ins(tc, g, return_bb, return_ins); |
778 | 1 | break; |
779 | 5.09k | case MVM_OP_invoke_o: |
780 | 5.09k | return_to_set(tc, g, return_ins, invoke_ins->operands[0]); |
781 | 5.09k | break; |
782 | 0 | default: |
783 | 0 | MVM_oops(tc, |
784 | 0 | "Spesh inline: unhandled case of return_o"); |
785 | 5.09k | } |
786 | 5.09k | } |
787 | | |
788 | | static void rewrite_returns(MVMThreadContext *tc, MVMSpeshGraph *inliner, |
789 | | MVMSpeshGraph *inlinee, MVMSpeshBB *invoke_bb, |
790 | 7.57k | MVMSpeshIns *invoke_ins, MVMSpeshBB *inlinee_last_bb) { |
791 | 7.57k | /* Locate return instructions. */ |
792 | 7.57k | MVMSpeshBB *bb = inlinee->entry; |
793 | 33.6k | while (bb) { |
794 | 33.6k | MVMSpeshIns *ins = bb->first_ins; |
795 | 201k | while (ins) { |
796 | 167k | MVMuint16 opcode = ins->info->opcode; |
797 | 167k | switch (opcode) { |
798 | 0 | case MVM_OP_return: |
799 | 0 | if (invoke_ins->info->opcode == MVM_OP_invoke_v) { |
800 | 0 | MVM_spesh_manipulate_insert_goto(tc, inliner, bb, ins, |
801 | 0 | invoke_bb->succ[0]); |
802 | 0 | tweak_succ(tc, inliner, bb, invoke_bb->succ[0]); |
803 | 0 | } |
804 | 0 | else { |
805 | 0 | MVM_oops(tc, |
806 | 0 | "Spesh inline: return_v/invoke_[!v] mismatch"); |
807 | 0 | } |
808 | 0 | break; |
809 | 1.90k | case MVM_OP_return_i: |
810 | 1.90k | MVM_spesh_manipulate_insert_goto(tc, inliner, bb, ins, |
811 | 1.90k | invoke_bb->succ[0]); |
812 | 1.90k | tweak_succ(tc, inliner, bb, invoke_bb->succ[0]); |
813 | 1.90k | rewrite_int_return(tc, inliner, bb, ins, invoke_bb, invoke_ins); |
814 | 1.90k | break; |
815 | 1 | case MVM_OP_return_n: |
816 | 1 | MVM_spesh_manipulate_insert_goto(tc, inliner, bb, ins, |
817 | 1 | invoke_bb->succ[0]); |
818 | 1 | tweak_succ(tc, inliner, bb, invoke_bb->succ[0]); |
819 | 1 | rewrite_num_return(tc, inliner, bb, ins, invoke_bb, invoke_ins); |
820 | 1 | break; |
821 | 571 | case MVM_OP_return_s: |
822 | 571 | MVM_spesh_manipulate_insert_goto(tc, inliner, bb, ins, |
823 | 571 | invoke_bb->succ[0]); |
824 | 571 | tweak_succ(tc, inliner, bb, invoke_bb->succ[0]); |
825 | 571 | rewrite_str_return(tc, inliner, bb, ins, invoke_bb, invoke_ins); |
826 | 571 | break; |
827 | 5.09k | case MVM_OP_return_o: |
828 | 5.09k | MVM_spesh_manipulate_insert_goto(tc, inliner, bb, ins, |
829 | 5.09k | invoke_bb->succ[0]); |
830 | 5.09k | tweak_succ(tc, inliner, bb, invoke_bb->succ[0]); |
831 | 5.09k | rewrite_obj_return(tc, inliner, bb, ins, invoke_bb, invoke_ins); |
832 | 5.09k | break; |
833 | 167k | } |
834 | 167k | ins = ins->next; |
835 | 167k | } |
836 | 33.6k | if (bb == inlinee_last_bb) break; |
837 | 26.0k | bb = bb->linear_next; |
838 | 26.0k | } |
839 | 7.57k | } |
840 | | |
841 | | /* Re-writes argument passing and parameter taking instructions to simple |
842 | | * register set operations. */ |
843 | | static void rewrite_args(MVMThreadContext *tc, MVMSpeshGraph *inliner, |
844 | | MVMSpeshGraph *inlinee, MVMSpeshBB *invoke_bb, |
845 | 7.57k | MVMSpeshCallInfo *call_info, MVMSpeshBB *inlinee_last_bb) { |
846 | 7.57k | /* Look for param-taking instructions. Track what arg instructions we |
847 | 7.57k | * use in the process. */ |
848 | 7.57k | MVMSpeshBB *bb = inlinee->entry; |
849 | 33.6k | while (bb) { |
850 | 33.6k | MVMSpeshIns *ins = bb->first_ins; |
851 | 200k | while (ins) { |
852 | 167k | MVMuint16 opcode = ins->info->opcode; |
853 | 167k | MVMSpeshIns *next = ins->next; |
854 | 167k | switch (opcode) { |
855 | 11.8k | case MVM_OP_sp_getarg_o: |
856 | 11.8k | case MVM_OP_sp_getarg_i: |
857 | 11.8k | case MVM_OP_sp_getarg_n: |
858 | 11.8k | case MVM_OP_sp_getarg_s: { |
859 | 11.8k | MVMuint16 idx = ins->operands[1].lit_i16; |
860 | 11.8k | MVMSpeshIns *arg_ins = call_info->arg_ins[idx]; |
861 | 11.8k | switch (arg_ins->info->opcode) { |
862 | 11.8k | case MVM_OP_arg_i: |
863 | 11.8k | case MVM_OP_arg_n: |
864 | 11.8k | case MVM_OP_arg_s: |
865 | 11.8k | case MVM_OP_arg_o: |
866 | 11.8k | /* Receiver just becomes a set instruction; delete the |
867 | 11.8k | * argument passing instruction. */ |
868 | 11.8k | ins->info = MVM_op_get_op(MVM_OP_set); |
869 | 11.8k | ins->operands[1] = arg_ins->operands[1]; |
870 | 11.8k | MVM_spesh_get_facts(tc, inliner, ins->operands[1])->usages++; |
871 | 11.8k | MVM_spesh_manipulate_delete_ins(tc, inliner, |
872 | 11.8k | call_info->prepargs_bb, arg_ins); |
873 | 11.8k | break; |
874 | 0 | case MVM_OP_argconst_i: |
875 | 0 | arg_ins->info = MVM_op_get_op(MVM_OP_const_i64); |
876 | 0 | arg_ins->operands[0] = ins->operands[0]; |
877 | 0 | MVM_spesh_manipulate_delete_ins(tc, inliner, bb, ins); |
878 | 0 | MVM_spesh_get_facts(tc, inliner, arg_ins->operands[0])->usages++; |
879 | 0 | break; |
880 | 0 | case MVM_OP_argconst_n: |
881 | 0 | arg_ins->info = MVM_op_get_op(MVM_OP_const_n64); |
882 | 0 | arg_ins->operands[0] = ins->operands[0]; |
883 | 0 | MVM_spesh_manipulate_delete_ins(tc, inliner, bb, ins); |
884 | 0 | MVM_spesh_get_facts(tc, inliner, arg_ins->operands[0])->usages++; |
885 | 0 | break; |
886 | 0 | case MVM_OP_argconst_s: |
887 | 0 | arg_ins->info = MVM_op_get_op(MVM_OP_const_s); |
888 | 0 | arg_ins->operands[0] = ins->operands[0]; |
889 | 0 | MVM_spesh_manipulate_delete_ins(tc, inliner, bb, ins); |
890 | 0 | MVM_spesh_get_facts(tc, inliner, arg_ins->operands[0])->usages++; |
891 | 0 | break; |
892 | 0 | default: |
893 | 0 | MVM_oops(tc, |
894 | 0 | "Spesh inline: unhandled arg instruction %d", |
895 | 0 | arg_ins->info->opcode); |
896 | 11.8k | } |
897 | 11.8k | break; |
898 | 11.8k | } |
899 | 167k | } |
900 | 167k | ins = next; |
901 | 167k | } |
902 | 33.6k | if (bb == inlinee_last_bb) break; |
903 | 26.0k | bb = bb->linear_next; |
904 | 26.0k | } |
905 | 7.57k | |
906 | 7.57k | { |
907 | 7.57k | MVMSpeshIns *arg_ins = call_info->prepargs_ins->next; |
908 | 7.57k | /* If there's some args that are not fetched by our inlinee, |
909 | 7.57k | * we have to kick them out, as arg_* ops are only valid between |
910 | 7.57k | * a prepargs and invoke_* op. */ |
911 | 15.7k | while (arg_ins) { |
912 | 8.14k | MVMuint16 opcode = arg_ins->info->opcode; |
913 | 8.14k | MVMSpeshIns *next = arg_ins->next; |
914 | 8.14k | switch (opcode) { |
915 | 569 | case MVM_OP_arg_i: |
916 | 569 | case MVM_OP_arg_n: |
917 | 569 | case MVM_OP_arg_s: |
918 | 569 | case MVM_OP_arg_o: |
919 | 569 | case MVM_OP_argconst_i: |
920 | 569 | case MVM_OP_argconst_n: |
921 | 569 | case MVM_OP_argconst_s: |
922 | 569 | MVM_spesh_manipulate_delete_ins(tc, inliner, call_info->prepargs_bb, arg_ins); |
923 | 569 | break; |
924 | 0 | case MVM_OP_set: |
925 | 0 | break; |
926 | 7.57k | case MVM_OP_invoke_i: |
927 | 7.57k | case MVM_OP_invoke_n: |
928 | 7.57k | case MVM_OP_invoke_s: |
929 | 7.57k | case MVM_OP_invoke_o: |
930 | 7.57k | case MVM_OP_invoke_v: |
931 | 7.57k | default: |
932 | 7.57k | next = NULL; |
933 | 8.14k | } |
934 | 8.14k | arg_ins = next; |
935 | 8.14k | } |
936 | 7.57k | } |
937 | 7.57k | |
938 | 7.57k | /* Delete the prepargs instruction. */ |
939 | 7.57k | MVM_spesh_manipulate_delete_ins(tc, inliner, invoke_bb, call_info->prepargs_ins); |
940 | 7.57k | } |
941 | | |
942 | | /* Annotates first and last instruction in post-processed inlinee with start |
943 | | * and end inline annotations. */ |
944 | | static void annotate_inline_start_end(MVMThreadContext *tc, MVMSpeshGraph *inliner, |
945 | | MVMSpeshGraph *inlinee, MVMint32 idx, |
946 | | MVMSpeshBB *inlinee_last_bb, |
947 | 7.57k | MVMuint32 inline_boundary_handler) { |
948 | 7.57k | /* Annotate first instruction as an inline start. */ |
949 | 7.57k | MVMSpeshAnn *start_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); |
950 | 7.57k | MVMSpeshAnn *end_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); |
951 | 7.57k | MVMSpeshBB *bb = inlinee->entry->succ[0]; |
952 | 7.57k | start_ann->next = bb->first_ins->annotations; |
953 | 7.57k | start_ann->type = MVM_SPESH_ANN_INLINE_START; |
954 | 7.57k | start_ann->data.inline_idx = idx; |
955 | 7.57k | bb->first_ins->annotations = start_ann; |
956 | 7.57k | |
957 | 7.57k | /* Insert annotation for handler boundary indicator fixup. */ |
958 | 7.57k | start_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); |
959 | 7.57k | start_ann->next = bb->first_ins->annotations; |
960 | 7.57k | start_ann->type = MVM_SPESH_ANN_FH_START; |
961 | 7.57k | start_ann->data.frame_handler_index = inline_boundary_handler; |
962 | 7.57k | bb->first_ins->annotations = start_ann; |
963 | 7.57k | |
964 | 7.57k | /* Now look for last instruction and annotate it. */ |
965 | 7.57k | end_ann->next = inlinee_last_bb->last_ins->annotations; |
966 | 7.57k | end_ann->type = MVM_SPESH_ANN_INLINE_END; |
967 | 7.57k | end_ann->data.inline_idx = idx; |
968 | 7.57k | inlinee_last_bb->last_ins->annotations = end_ann; |
969 | 7.57k | |
970 | 7.57k | /* Insert annotation for handler boundary fixup; we add the end |
971 | 7.57k | * one that is needed and also a dummy goto one to keep things |
972 | 7.57k | * that want all three happy. */ |
973 | 7.57k | end_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); |
974 | 7.57k | end_ann->next = inlinee_last_bb->last_ins->annotations; |
975 | 7.57k | end_ann->type = MVM_SPESH_ANN_FH_END; |
976 | 7.57k | end_ann->data.frame_handler_index = inline_boundary_handler; |
977 | 7.57k | inlinee_last_bb->last_ins->annotations = end_ann; |
978 | 7.57k | end_ann = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshAnn)); |
979 | 7.57k | end_ann->next = inlinee_last_bb->last_ins->annotations; |
980 | 7.57k | end_ann->type = MVM_SPESH_ANN_FH_GOTO; |
981 | 7.57k | end_ann->data.frame_handler_index = inline_boundary_handler; |
982 | 7.57k | inlinee_last_bb->last_ins->annotations = end_ann; |
983 | 7.57k | |
984 | 7.57k | return; |
985 | 7.57k | } |
986 | | |
987 | | /* Drives the overall inlining process. */ |
988 | | void MVM_spesh_inline(MVMThreadContext *tc, MVMSpeshGraph *inliner, |
989 | | MVMSpeshCallInfo *call_info, MVMSpeshBB *invoke_bb, |
990 | | MVMSpeshIns *invoke_ins, MVMSpeshGraph *inlinee, |
991 | 7.57k | MVMStaticFrame *inlinee_sf, MVMSpeshOperand code_ref_reg) { |
992 | 7.57k | /* Merge inlinee's graph into the inliner. */ |
993 | 7.57k | MVMuint32 inline_boundary_handler; |
994 | 7.57k | MVMSpeshBB *inlinee_last_bb = merge_graph(tc, inliner, inlinee, inlinee_sf, |
995 | 7.57k | invoke_bb, invoke_ins, code_ref_reg, &inline_boundary_handler); |
996 | 7.57k | |
997 | 7.57k | /* If we're profiling, note it's an inline. */ |
998 | 7.57k | if (inlinee->entry->linear_next->first_ins->info->opcode == MVM_OP_prof_enterspesh) { |
999 | 0 | MVMSpeshIns *profenter = inlinee->entry->linear_next->first_ins; |
1000 | 0 | profenter->info = MVM_op_get_op(MVM_OP_prof_enterinline); |
1001 | 0 | profenter->operands = MVM_spesh_alloc(tc, inliner, sizeof(MVMSpeshOperand)); |
1002 | 0 | profenter->operands[0].lit_i16 = MVM_spesh_add_spesh_slot(tc, inliner, |
1003 | 0 | (MVMCollectable *)inlinee->sf); |
1004 | 0 | } |
1005 | 7.57k | |
1006 | 7.57k | /* Re-write returns to a set and goto. */ |
1007 | 7.57k | rewrite_returns(tc, inliner, inlinee, invoke_bb, invoke_ins, inlinee_last_bb); |
1008 | 7.57k | |
1009 | 7.57k | /* Re-write the argument passing instructions to poke values into the |
1010 | 7.57k | * appropriate slots. */ |
1011 | 7.57k | rewrite_args(tc, inliner, inlinee, invoke_bb, call_info, inlinee_last_bb); |
1012 | 7.57k | |
1013 | 7.57k | /* Annotate first and last instruction with inline table annotations; also |
1014 | 7.57k | * add annotations for fixing up the handlers table inline boundary |
1015 | 7.57k | * indicators. */ |
1016 | 7.57k | annotate_inline_start_end(tc, inliner, inlinee, inliner->num_inlines - 1, |
1017 | 7.57k | inlinee_last_bb, inline_boundary_handler); |
1018 | 7.57k | |
1019 | 7.57k | /* Finally, turn the invoke instruction into a goto. */ |
1020 | 7.57k | invoke_ins->info = MVM_op_get_op(MVM_OP_goto); |
1021 | 7.57k | invoke_ins->operands[0].ins_bb = inlinee->entry->linear_next; |
1022 | 7.57k | tweak_succ(tc, inliner, invoke_bb, inlinee->entry->linear_next); |
1023 | 7.57k | } |