/home/travis/build/MoarVM/MoarVM/src/spesh/facts.c
Line | Count | Source (jump to first uncovered line) |
1 | | #include "moar.h" |
2 | | |
3 | | /* The code in this file walks the spesh graph, recording facts we discover |
4 | | * about each version of each local variable, and propagating the info as it |
5 | | * can. */ |
6 | | |
7 | | /* Copies facts from one var to another. */ |
8 | | static void copy_facts(MVMThreadContext *tc, MVMSpeshGraph *g, MVMuint16 to_orig, |
9 | 219k | MVMuint16 to_i, MVMuint16 from_orig, MVMuint16 from_i) { |
10 | 219k | MVMSpeshFacts *tfacts = &g->facts[to_orig][to_i]; |
11 | 219k | MVMSpeshFacts *ffacts = &g->facts[from_orig][from_i]; |
12 | 219k | tfacts->flags = ffacts->flags; |
13 | 219k | tfacts->type = ffacts->type; |
14 | 219k | tfacts->decont_type = ffacts->decont_type; |
15 | 219k | tfacts->value = ffacts->value; |
16 | 219k | tfacts->log_guard = ffacts->log_guard; |
17 | 219k | } |
18 | | |
19 | | /* Called when one set of facts depend on another, allowing any log guard |
20 | | * that is to thank to be marked used as needed later on. */ |
21 | | void MVM_spesh_facts_depend(MVMThreadContext *tc, MVMSpeshGraph *g, |
22 | 18.7k | MVMSpeshFacts *target, MVMSpeshFacts *source) { |
23 | 18.7k | if (source->flags & MVM_SPESH_FACT_FROM_LOG_GUARD) { |
24 | 2.84k | target->flags |= MVM_SPESH_FACT_FROM_LOG_GUARD; |
25 | 2.84k | target->log_guard = source->log_guard; |
26 | 2.84k | } |
27 | 18.7k | } |
28 | | |
29 | | /* Handles object-creating instructions. */ |
30 | | static void create_facts(MVMThreadContext *tc, MVMSpeshGraph *g, MVMuint16 obj_orig, |
31 | 14.1k | MVMuint16 obj_i, MVMuint16 type_orig, MVMuint16 type_i) { |
32 | 14.1k | MVMSpeshFacts *type_facts = &(g->facts[type_orig][type_i]); |
33 | 14.1k | MVMSpeshFacts *obj_facts = &(g->facts[obj_orig][obj_i]); |
34 | 14.1k | |
35 | 14.1k | /* The type is carried. */ |
36 | 14.1k | if (type_facts->flags & MVM_SPESH_FACT_KNOWN_TYPE) { |
37 | 14.0k | obj_facts->type = type_facts->type; |
38 | 14.0k | obj_facts->flags |= MVM_SPESH_FACT_KNOWN_TYPE; |
39 | 14.0k | MVM_spesh_facts_depend(tc, g, obj_facts, type_facts); |
40 | 14.0k | } |
41 | 14.1k | |
42 | 14.1k | /* We know it's a concrete object. */ |
43 | 14.1k | obj_facts->flags |= MVM_SPESH_FACT_CONCRETE; |
44 | 14.1k | |
45 | 14.1k | /* If we know the type object, then we can check to see if |
46 | 14.1k | * it's a container type. */ |
47 | 14.1k | if (type_facts->flags & MVM_SPESH_FACT_KNOWN_TYPE) { |
48 | 14.0k | MVMObject *type = type_facts->type; |
49 | 14.0k | if (type && !STABLE(type)->container_spec) |
50 | 14.0k | obj_facts->flags |= MVM_SPESH_FACT_DECONTED; |
51 | 14.0k | } |
52 | 14.1k | } |
53 | | |
54 | | static void create_facts_with_type(MVMThreadContext *tc, MVMSpeshGraph *g, |
55 | | MVMuint16 obj_orig, MVMuint16 obj_i, |
56 | 0 | MVMObject *type) { |
57 | 0 | MVMSpeshFacts *obj_facts = &(g->facts[obj_orig][obj_i]); |
58 | 0 |
|
59 | 0 | /* The type is carried. */ |
60 | 0 | obj_facts->type = type; |
61 | 0 | obj_facts->flags |= MVM_SPESH_FACT_KNOWN_TYPE; |
62 | 0 |
|
63 | 0 | /* We know it's a concrete object. */ |
64 | 0 | obj_facts->flags |= MVM_SPESH_FACT_CONCRETE; |
65 | 0 |
|
66 | 0 | /* If we know the type object, then we can check to see if |
67 | 0 | * it's a container type. */ |
68 | 0 | if (type && !STABLE(type)->container_spec) |
69 | 0 | obj_facts->flags |= MVM_SPESH_FACT_DECONTED; |
70 | 0 | } |
71 | | |
72 | | /* Adds facts from knowing the exact value being put into an object local. */ |
73 | | static void object_facts(MVMThreadContext *tc, MVMSpeshGraph *g, MVMuint16 tgt_orig, |
74 | 65.9k | MVMuint16 tgt_i, MVMObject *obj) { |
75 | 65.9k | /* Ensure it's non-null. */ |
76 | 65.9k | if (!obj) |
77 | 2.94k | return; |
78 | 65.9k | |
79 | 65.9k | /* Set the value itself. */ |
80 | 63.0k | g->facts[tgt_orig][tgt_i].value.o = obj; |
81 | 63.0k | g->facts[tgt_orig][tgt_i].flags |= MVM_SPESH_FACT_KNOWN_VALUE; |
82 | 63.0k | |
83 | 63.0k | /* We also know the type. */ |
84 | 63.0k | g->facts[tgt_orig][tgt_i].type = STABLE(obj)->WHAT; |
85 | 63.0k | g->facts[tgt_orig][tgt_i].flags |= MVM_SPESH_FACT_KNOWN_TYPE; |
86 | 63.0k | |
87 | 63.0k | /* Set concreteness and decontainerized flags. */ |
88 | 63.0k | if (IS_CONCRETE(obj)) { |
89 | 321 | g->facts[tgt_orig][tgt_i].flags |= MVM_SPESH_FACT_CONCRETE; |
90 | 321 | if (!STABLE(obj)->container_spec) |
91 | 321 | g->facts[tgt_orig][tgt_i].flags |= MVM_SPESH_FACT_DECONTED; |
92 | 321 | } |
93 | 62.7k | else { |
94 | 62.7k | g->facts[tgt_orig][tgt_i].flags |= MVM_SPESH_FACT_TYPEOBJ | MVM_SPESH_FACT_DECONTED; |
95 | 62.7k | } |
96 | 63.0k | } |
97 | | void MVM_spesh_facts_object_facts(MVMThreadContext *tc, MVMSpeshGraph *g, |
98 | 6 | MVMSpeshOperand tgt, MVMObject *obj) { |
99 | 6 | object_facts(tc, g, tgt.reg.orig, tgt.reg.i, obj); |
100 | 6 | } |
101 | | |
102 | | /* Propagates information relating to decontainerization. */ |
103 | | static void decont_facts(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshIns *ins, |
104 | | MVMuint16 out_orig, MVMuint16 out_i, MVMuint16 in_orig, |
105 | 117k | MVMuint16 in_i) { |
106 | 117k | MVMSpeshFacts *out_facts = &(g->facts[out_orig][out_i]); |
107 | 117k | MVMSpeshFacts *in_facts = &(g->facts[in_orig][in_i]); |
108 | 117k | |
109 | 117k | /* If we know the original is decontainerized already, just copy its |
110 | 117k | * info. */ |
111 | 117k | MVMint32 in_flags = in_facts->flags; |
112 | 117k | if (in_flags & MVM_SPESH_FACT_DECONTED) |
113 | 60.7k | copy_facts(tc, g, out_orig, out_i, in_orig, in_i); |
114 | 117k | |
115 | 117k | /* We know the result is decontainerized. */ |
116 | 117k | out_facts->flags |= MVM_SPESH_FACT_DECONTED; |
117 | 117k | |
118 | 117k | /* We may also know the original was containerized, and have some facts |
119 | 117k | * about its contents. */ |
120 | 117k | if (in_flags & MVM_SPESH_FACT_KNOWN_DECONT_TYPE) { |
121 | 0 | out_facts->type = in_facts->decont_type; |
122 | 0 | out_facts->flags |= MVM_SPESH_FACT_KNOWN_TYPE; |
123 | 0 | } |
124 | 117k | if (in_flags & MVM_SPESH_FACT_DECONT_CONCRETE) |
125 | 0 | out_facts->flags |= MVM_SPESH_FACT_CONCRETE; |
126 | 117k | else if (in_flags & MVM_SPESH_FACT_DECONT_TYPEOBJ) |
127 | 0 | out_facts->flags |= MVM_SPESH_FACT_TYPEOBJ; |
128 | 117k | if (in_flags & (MVM_SPESH_FACT_KNOWN_DECONT_TYPE | |
129 | 117k | MVM_SPESH_FACT_DECONT_CONCRETE | |
130 | 117k | MVM_SPESH_FACT_DECONT_TYPEOBJ)) |
131 | 0 | MVM_spesh_facts_depend(tc, g, out_facts, in_facts); |
132 | 117k | } |
133 | | |
134 | | /* Looks up a wval and adds information based on it. */ |
135 | | static void wval_facts(MVMThreadContext *tc, MVMSpeshGraph *g, MVMuint16 tgt_orig, |
136 | 52.6k | MVMuint16 tgt_i, MVMuint16 dep, MVMint64 idx) { |
137 | 52.6k | MVMCompUnit *cu = g->sf->body.cu; |
138 | 52.6k | if (dep < cu->body.num_scs) { |
139 | 52.6k | MVMSerializationContext *sc = MVM_sc_get_sc(tc, cu, dep); |
140 | 52.6k | if (sc) |
141 | 52.6k | object_facts(tc, g, tgt_orig, tgt_i, MVM_sc_try_get_object(tc, sc, idx)); |
142 | 52.6k | } |
143 | 52.6k | } |
144 | | |
145 | | /* Let's figure out what exact type of iter we'll get from an iter op */ |
146 | | static void iter_facts(MVMThreadContext *tc, MVMSpeshGraph *g, |
147 | | MVMuint16 out_orig, MVMuint16 out_i, |
148 | 1.84k | MVMuint16 in_orig, MVMuint16 in_i) { |
149 | 1.84k | MVMSpeshFacts *out_facts = &(g->facts[out_orig][out_i]); |
150 | 1.84k | MVMSpeshFacts *in_facts = &(g->facts[in_orig][in_i]); |
151 | 1.84k | |
152 | 1.84k | if (in_facts->flags & MVM_SPESH_FACT_KNOWN_TYPE) { |
153 | 875 | switch (REPR(in_facts->type)->ID) { |
154 | 527 | case MVM_REPR_ID_VMArray: |
155 | 527 | out_facts->type = g->sf->body.cu->body.hll_config->array_iterator_type; |
156 | 527 | out_facts->flags |= MVM_SPESH_FACT_ARRAY_ITER; |
157 | 527 | break; |
158 | 340 | case MVM_REPR_ID_MVMHash: |
159 | 340 | case MVM_REPR_ID_MVMContext: |
160 | 340 | out_facts->type = g->sf->body.cu->body.hll_config->hash_iterator_type; |
161 | 340 | out_facts->flags |= MVM_SPESH_FACT_HASH_ITER; |
162 | 340 | break; |
163 | 8 | default: |
164 | 8 | return; |
165 | 875 | } |
166 | 867 | out_facts->flags |= MVM_SPESH_FACT_KNOWN_TYPE | MVM_SPESH_FACT_CONCRETE; |
167 | 867 | } |
168 | 1.84k | |
169 | 1.84k | } |
170 | | |
171 | | /* constant ops on literals give us a specialize-time-known value */ |
172 | 97.6k | static void literal_facts(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshIns *ins) { |
173 | 97.6k | MVMSpeshFacts *tgt_facts = &g->facts[ins->operands[0].reg.orig][ins->operands[0].reg.i]; |
174 | 97.6k | switch (ins->info->opcode) { |
175 | 0 | case MVM_OP_const_i64: |
176 | 0 | tgt_facts->value.i = ins->operands[1].lit_i64; |
177 | 0 | break; |
178 | 0 | case MVM_OP_const_i32: |
179 | 0 | tgt_facts->value.i = ins->operands[1].lit_i32; |
180 | 0 | break; |
181 | 0 | case MVM_OP_const_i16: |
182 | 0 | tgt_facts->value.i = ins->operands[1].lit_i16; |
183 | 0 | break; |
184 | 0 | case MVM_OP_const_i8: |
185 | 0 | tgt_facts->value.i = ins->operands[1].lit_i8; |
186 | 0 | break; |
187 | 0 | case MVM_OP_const_n32: |
188 | 0 | tgt_facts->value.n = ins->operands[1].lit_n32; |
189 | 0 | break; |
190 | 12 | case MVM_OP_const_n64: |
191 | 12 | tgt_facts->value.n = ins->operands[1].lit_n64; |
192 | 12 | break; |
193 | 15 | case MVM_OP_const_i64_32: |
194 | 15 | tgt_facts->value.i = ins->operands[1].lit_i32; |
195 | 15 | break; |
196 | 37.0k | case MVM_OP_const_i64_16: |
197 | 37.0k | tgt_facts->value.i = ins->operands[1].lit_i16; |
198 | 37.0k | break; |
199 | 60.5k | case MVM_OP_const_s: |
200 | 60.5k | tgt_facts->value.s = MVM_cu_string(tc, g->sf->body.cu, |
201 | 60.5k | ins->operands[1].lit_str_idx); |
202 | 60.5k | break; |
203 | 0 | default: |
204 | 0 | return; |
205 | 97.6k | } |
206 | 97.6k | tgt_facts->flags |= MVM_SPESH_FACT_KNOWN_VALUE; |
207 | 97.6k | } |
208 | | |
209 | | /* Discover facts from extops. */ |
210 | 0 | static void discover_extop(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshIns *ins) { |
211 | 0 | MVMExtOpRecord *extops = g->sf->body.cu->body.extops; |
212 | 0 | MVMuint16 num_extops = g->sf->body.cu->body.num_extops; |
213 | 0 | MVMuint16 i; |
214 | 0 | for (i = 0; i < num_extops; i++) { |
215 | 0 | if (extops[i].info == ins->info) { |
216 | 0 | /* Found op; call its discovery function, if any. */ |
217 | 0 | if (extops[i].discover) |
218 | 0 | extops[i].discover(tc, g, ins); |
219 | 0 | return; |
220 | 0 | } |
221 | 0 | } |
222 | 0 | } |
223 | | |
224 | | /* Considers logged types and, if they are stable, adds facts and a guard. */ |
225 | | static void log_facts(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshBB *bb, |
226 | | MVMSpeshIns *ins, MVMSpeshPlanned *p, |
227 | 63.9k | MVMSpeshAnn *deopt_one_ann, MVMSpeshAnn *logged_ann) { |
228 | 63.9k | /* See if we have stable type information. For now, we need consistent |
229 | 63.9k | * types, since a mis-match will force a deopt. In the future we may be |
230 | 63.9k | * able to do Basic Block Versioning inspired tricks, like producing two |
231 | 63.9k | * different code paths ahead when there are a small number of options. */ |
232 | 63.9k | MVMObject *agg_type = NULL; |
233 | 63.9k | MVMuint32 agg_type_count = 0; |
234 | 63.9k | MVMuint32 agg_type_object = 0; |
235 | 63.9k | MVMuint32 agg_concrete = 0; |
236 | 63.9k | MVMuint32 i; |
237 | 121k | for (i = 0; i < p->num_type_stats; i++) { |
238 | 60.8k | MVMSpeshStatsByType *ts = p->type_stats[i]; |
239 | 60.8k | MVMuint32 j; |
240 | 718k | for (j = 0; j < ts->num_by_offset; j++) { |
241 | 695k | if (ts->by_offset[j].bytecode_offset == logged_ann->data.bytecode_offset) { |
242 | 37.5k | /* Go over the logged types. */ |
243 | 37.5k | MVMuint32 num_types = ts->by_offset[j].num_types; |
244 | 37.5k | MVMuint32 k; |
245 | 71.4k | for (k = 0; k < num_types; k++) { |
246 | 37.3k | /* If it's inconsistent with the aggregated type so far, |
247 | 37.3k | * then first check if the type we're now seeing is either |
248 | 37.3k | * massively more popular or massively less popular. If |
249 | 37.3k | * massively less, disregard this one. If massively more, |
250 | 37.3k | * disregard the previous one. Otherwise, tot up the type |
251 | 37.3k | * object vs. concrete. */ |
252 | 37.3k | MVMObject *cur_type = ts->by_offset[j].types[k].type; |
253 | 37.3k | MVMuint32 count = ts->by_offset[j].types[k].count; |
254 | 37.3k | if (agg_type) { |
255 | 3.52k | if (agg_type != cur_type) { |
256 | 3.52k | if (count > 100 * agg_type_count) { |
257 | 15 | /* This one is hugely more popular. */ |
258 | 15 | agg_type = cur_type; |
259 | 15 | agg_type_count = 0; |
260 | 15 | agg_concrete = 0; |
261 | 15 | agg_type_object = 0; |
262 | 15 | } |
263 | 3.50k | else if (agg_type_count > 100 * count) { |
264 | 28 | /* This one is hugely less popular. */ |
265 | 28 | continue; |
266 | 28 | } |
267 | 3.48k | else { |
268 | 3.48k | /* Unstable types. */ |
269 | 3.48k | return; |
270 | 3.48k | } |
271 | 3.52k | } |
272 | 3.52k | } |
273 | 33.8k | else { |
274 | 33.8k | agg_type = cur_type; |
275 | 33.8k | } |
276 | 33.8k | agg_type_count += count; |
277 | 33.8k | if (ts->by_offset[j].types[k].type_concrete) |
278 | 29.5k | agg_concrete++; |
279 | 33.8k | else |
280 | 4.37k | agg_type_object++; |
281 | 33.8k | } |
282 | 37.5k | |
283 | 37.5k | /* No need to consider searching after this offset. */ |
284 | 34.0k | break; |
285 | 37.5k | } |
286 | 695k | } |
287 | 60.8k | } |
288 | 60.4k | if (agg_type) { |
289 | 30.3k | MVMSpeshIns *guard; |
290 | 30.3k | MVMSpeshAnn *ann; |
291 | 30.3k | MVMuint16 guard_op; |
292 | 30.3k | |
293 | 30.3k | /* Add facts and choose guard op. */ |
294 | 30.3k | MVMSpeshFacts *facts = &g->facts[ins->operands[0].reg.orig][ins->operands[0].reg.i]; |
295 | 30.3k | facts->type = agg_type; |
296 | 30.3k | facts->flags |= MVM_SPESH_FACT_KNOWN_TYPE; |
297 | 30.3k | if (agg_concrete && !agg_type_object) { |
298 | 27.7k | facts->flags |= MVM_SPESH_FACT_CONCRETE; |
299 | 27.7k | if (!agg_type->st->container_spec) |
300 | 27.7k | facts->flags |= MVM_SPESH_FACT_DECONTED; |
301 | 27.7k | guard_op = MVM_OP_sp_guardconc; |
302 | 27.7k | } |
303 | 2.61k | else if (agg_type_object && !agg_concrete) { |
304 | 2.61k | facts->flags |= MVM_SPESH_FACT_TYPEOBJ | MVM_SPESH_FACT_DECONTED; |
305 | 2.61k | guard_op = MVM_OP_sp_guardtype; |
306 | 2.61k | } |
307 | 0 | else { |
308 | 0 | if (!agg_type->st->container_spec) |
309 | 0 | facts->flags |= MVM_SPESH_FACT_DECONTED; |
310 | 0 | guard_op = MVM_OP_sp_guard; |
311 | 0 | } |
312 | 30.3k | |
313 | 30.3k | /* Insert guard instruction. */ |
314 | 30.3k | guard = MVM_spesh_alloc(tc, g, sizeof(MVMSpeshIns)); |
315 | 30.3k | guard->info = MVM_op_get_op(guard_op); |
316 | 30.3k | guard->operands = MVM_spesh_alloc(tc, g, 3 * sizeof(MVMSpeshOperand)); |
317 | 30.3k | guard->operands[0] = ins->operands[0]; |
318 | 30.3k | guard->operands[1].lit_i16 = MVM_spesh_add_spesh_slot_try_reuse(tc, g, |
319 | 30.3k | (MVMCollectable *)agg_type->st); |
320 | 30.3k | guard->operands[2].lit_ui32 = g->deopt_addrs[2 * deopt_one_ann->data.deopt_idx]; |
321 | 30.3k | if (ins->next) |
322 | 16.2k | MVM_spesh_manipulate_insert_ins(tc, bb, ins, guard); |
323 | 30.3k | else |
324 | 14.1k | MVM_spesh_manipulate_insert_ins(tc, bb->linear_next, NULL, guard); |
325 | 30.3k | |
326 | 30.3k | /* Move deopt annotation to the guard instruction. */ |
327 | 30.3k | ann = ins->annotations; |
328 | 30.3k | if (ann == deopt_one_ann) { |
329 | 16.1k | ins->annotations = ann->next; |
330 | 16.1k | } |
331 | 14.2k | else { |
332 | 14.3k | while (ann) { |
333 | 14.3k | if (ann->next == deopt_one_ann) { |
334 | 14.2k | ann->next = deopt_one_ann->next; |
335 | 14.2k | break; |
336 | 14.2k | } |
337 | 126 | ann = ann->next; |
338 | 126 | } |
339 | 14.2k | } |
340 | 30.3k | deopt_one_ann->next = NULL; |
341 | 30.3k | guard->annotations = deopt_one_ann; |
342 | 30.3k | |
343 | 30.3k | /* Add entry in log guards table, and mark facts as depending on it. */ |
344 | 30.3k | if (g->num_log_guards % 16 == 0) { |
345 | 7.84k | MVMSpeshLogGuard *orig_log_guards = g->log_guards; |
346 | 7.84k | g->log_guards = MVM_spesh_alloc(tc, g, |
347 | 7.84k | (g->num_log_guards + 16) * sizeof(MVMSpeshLogGuard)); |
348 | 7.84k | if (orig_log_guards) |
349 | 161 | memcpy(g->log_guards, orig_log_guards, |
350 | 161 | g->num_log_guards * sizeof(MVMSpeshLogGuard)); |
351 | 7.84k | } |
352 | 30.3k | g->log_guards[g->num_log_guards].ins = guard; |
353 | 16.2k | g->log_guards[g->num_log_guards].bb = ins->next ? bb : bb->linear_next; |
354 | 30.3k | facts->flags |= MVM_SPESH_FACT_FROM_LOG_GUARD; |
355 | 30.3k | facts->log_guard = g->num_log_guards; |
356 | 30.3k | g->num_log_guards++; |
357 | 30.3k | } |
358 | 60.4k | } |
359 | | |
360 | | /* Considers a spesh plugin's logged data. If it gives a consistent result, |
361 | | * then replaces this instruction with a spesh slot that resolves to the |
362 | | * result, and prepends guards as specified by the plugin. */ |
363 | | static void plugin_facts(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshBB *bb, |
364 | | MVMSpeshIns *ins, MVMSpeshPlanned *p, |
365 | 6 | MVMSpeshAnn *deopt_one_ann, MVMSpeshAnn *logged_ann) { |
366 | 6 | /* Search for a stable guard index. (Later, we can stack up a set of |
367 | 6 | * conditions or some such if there's no stable or really popular one. */ |
368 | 6 | MVMint32 agg_guard_index = -1; |
369 | 6 | MVMuint32 agg_guard_index_count = 0; |
370 | 6 | MVMuint32 i; |
371 | 12 | for (i = 0; i < p->num_type_stats; i++) { |
372 | 6 | MVMSpeshStatsByType *ts = p->type_stats[i]; |
373 | 6 | MVMuint32 j; |
374 | 12 | for (j = 0; j < ts->num_by_offset; j++) { |
375 | 12 | if (ts->by_offset[j].bytecode_offset == logged_ann->data.bytecode_offset) { |
376 | 6 | /* Go over the guard indexes. */ |
377 | 6 | MVMuint32 num_plugin_guards = ts->by_offset[j].num_plugin_guards; |
378 | 6 | MVMuint32 k; |
379 | 12 | for (k = 0; k < num_plugin_guards; k++) { |
380 | 6 | /* If it's inconsistent with the aggregated guard index, |
381 | 6 | * then first check if the index we're now seeing is either |
382 | 6 | * massively more popular or massively less popular. If |
383 | 6 | * massively less, disregard this one. If massively more, |
384 | 6 | * disregard the previous one. */ |
385 | 6 | MVMuint32 cur_guard_index = ts->by_offset[j].plugin_guards[k].guard_index; |
386 | 6 | MVMuint32 count = ts->by_offset[j].plugin_guards[k].count; |
387 | 6 | if (agg_guard_index >= 0) { |
388 | 0 | if (agg_guard_index != cur_guard_index) { |
389 | 0 | if (count > 100 * agg_guard_index_count) { |
390 | 0 | /* This one is hugely more popular. */ |
391 | 0 | agg_guard_index = cur_guard_index; |
392 | 0 | agg_guard_index_count = 0; |
393 | 0 | } |
394 | 0 | else if (agg_guard_index_count > 100 * count) { |
395 | 0 | /* This one is hugely less popular. */ |
396 | 0 | continue; |
397 | 0 | } |
398 | 0 | else { |
399 | 0 | /* Unstable guard indexes. */ |
400 | 0 | return; |
401 | 0 | } |
402 | 0 | } |
403 | 0 | } |
404 | 6 | else { |
405 | 6 | agg_guard_index = cur_guard_index; |
406 | 6 | } |
407 | 6 | agg_guard_index_count += count; |
408 | 6 | } |
409 | 6 | |
410 | 6 | /* No need to consider searching after this offset. */ |
411 | 6 | break; |
412 | 6 | } |
413 | 12 | } |
414 | 6 | } |
415 | 6 | |
416 | 6 | /* If we picked a guard index, insert the guards and rewrite the resolve |
417 | 6 | * instruction. If not, just rewrite the resolve instruction into the |
418 | 6 | * spesh version of itself including the index. */ |
419 | 6 | if (agg_guard_index != -1) { |
420 | 6 | MVM_spesh_plugin_rewrite_resolve(tc, g, bb, ins, logged_ann->data.bytecode_offset, |
421 | 6 | agg_guard_index); |
422 | 6 | } |
423 | 0 | else { |
424 | 0 | MVMSpeshOperand *new_operands = MVM_spesh_alloc(tc, g, 4 * sizeof(MVMSpeshOperand)); |
425 | 0 | new_operands[0] = ins->operands[0]; |
426 | 0 | new_operands[1] = ins->operands[1]; |
427 | 0 | new_operands[2].lit_ui32 = logged_ann->data.bytecode_offset; |
428 | 0 | new_operands[3].lit_i16 = MVM_spesh_add_spesh_slot_try_reuse(tc, g, |
429 | 0 | (MVMCollectable *)g->sf); |
430 | 0 | ins->info = MVM_op_get_op(MVM_OP_sp_speshresolve); |
431 | 0 | ins->operands = new_operands; |
432 | 0 | } |
433 | 6 | } |
434 | | |
435 | | /* Visits the blocks in dominator tree order, recursively. */ |
436 | | static void add_bb_facts(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshBB *bb, |
437 | 481k | MVMSpeshPlanned *p, MVMint32 cur_deopt_idx) { |
438 | 481k | MVMint32 i, is_phi; |
439 | 481k | |
440 | 481k | /* Look for instructions that provide or propagate facts. */ |
441 | 481k | MVMSpeshIns *ins = bb->first_ins; |
442 | 2.78M | while (ins) { |
443 | 2.30M | /* See if there's deopt and logged annotations. Sync cur_deopt_idx |
444 | 2.30M | * and, for logged+deopt-one, add logged facts and guards. */ |
445 | 2.30M | MVMSpeshAnn *ann = ins->annotations; |
446 | 2.30M | MVMSpeshAnn *ann_deopt_one = NULL; |
447 | 2.30M | MVMSpeshAnn *ann_logged = NULL; |
448 | 2.30M | MVMint32 is_deopt_ins = 0; |
449 | 2.76M | while (ann) { |
450 | 461k | switch (ann->type) { |
451 | 247k | case MVM_SPESH_ANN_DEOPT_ONE_INS: |
452 | 247k | ann_deopt_one = ann; |
453 | 247k | cur_deopt_idx = ann->data.deopt_idx; |
454 | 247k | is_deopt_ins = 1; |
455 | 247k | break; |
456 | 35.3k | case MVM_SPESH_ANN_DEOPT_ALL_INS: |
457 | 35.3k | cur_deopt_idx = ann->data.deopt_idx; |
458 | 35.3k | break; |
459 | 83.6k | case MVM_SPESH_ANN_LOGGED: |
460 | 83.6k | ann_logged = ann; |
461 | 461k | } |
462 | 461k | ann = ann->next; |
463 | 461k | } |
464 | 2.30M | if (ann_deopt_one && ann_logged) { |
465 | 63.9k | if (ins->info->opcode == MVM_OP_speshresolve) |
466 | 6 | plugin_facts(tc, g, bb, ins, p, ann_deopt_one, ann_logged); |
467 | 63.9k | else |
468 | 63.9k | log_facts(tc, g, bb, ins, p, ann_deopt_one, ann_logged); |
469 | 63.9k | } |
470 | 2.30M | |
471 | 2.30M | /* Look through operands for reads and writes. */ |
472 | 2.30M | is_phi = ins->info->opcode == MVM_SSA_PHI; |
473 | 23.6M | for (i = 0; i < ins->info->num_operands; i++) { |
474 | 21.3M | /* Reads need usage tracking; if the read is after a deopt point |
475 | 21.3M | * relative to the write then give it an extra usage bump. */ |
476 | 21.3M | if ((is_phi && i > 0) |
477 | 18.6M | || (!is_phi && (ins->info->operands[i] & MVM_operand_rw_mask) == MVM_operand_read_reg)) { |
478 | 18.6M | MVMSpeshFacts *facts = &(g->facts[ins->operands[i].reg.orig][ins->operands[i].reg.i]); |
479 | 10.1M | facts->usages += facts->deopt_idx == cur_deopt_idx ? 1 : 2; |
480 | 18.6M | } |
481 | 21.3M | |
482 | 21.3M | /* Writes need the current deopt index and the writing instruction |
483 | 21.3M | * to be specified. A write that's on a deopt instruction bumps |
484 | 21.3M | * the usage too. */ |
485 | 21.3M | if ((is_phi && i == 0) |
486 | 20.3M | || (!is_phi && (ins->info->operands[i] & MVM_operand_rw_mask) == MVM_operand_write_reg)) { |
487 | 1.90M | MVMSpeshFacts *facts = &(g->facts[ins->operands[i].reg.orig][ins->operands[i].reg.i]); |
488 | 1.90M | facts->deopt_idx = cur_deopt_idx; |
489 | 1.90M | facts->writer = ins; |
490 | 1.90M | if (is_deopt_ins) |
491 | 181k | facts->usages++; |
492 | 1.90M | } |
493 | 21.3M | } |
494 | 2.30M | |
495 | 2.30M | /* Look for ops that are fact-interesting. */ |
496 | 2.30M | switch (ins->info->opcode) { |
497 | 10.2k | case MVM_OP_inc_i: |
498 | 10.2k | case MVM_OP_inc_u: |
499 | 10.2k | case MVM_OP_dec_i: |
500 | 10.2k | case MVM_OP_dec_u: |
501 | 10.2k | /* These all read as well as write a value, so bump usages. */ |
502 | 10.2k | g->facts[ins->operands[0].reg.orig][ins->operands[0].reg.i - 1].usages++; |
503 | 10.2k | break; |
504 | 157k | case MVM_OP_set: |
505 | 157k | copy_facts(tc, g, |
506 | 157k | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
507 | 157k | ins->operands[1].reg.orig, ins->operands[1].reg.i); |
508 | 157k | break; |
509 | 6.09k | case MVM_OP_create: |
510 | 6.09k | create_facts(tc, g, |
511 | 6.09k | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
512 | 6.09k | ins->operands[1].reg.orig, ins->operands[1].reg.i); |
513 | 6.09k | break; |
514 | 371 | case MVM_OP_clone: |
515 | 371 | copy_facts(tc, g, |
516 | 371 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
517 | 371 | ins->operands[1].reg.orig, ins->operands[1].reg.i); |
518 | 371 | break; |
519 | 8.08k | case MVM_OP_box_s: |
520 | 8.08k | case MVM_OP_box_i: |
521 | 8.08k | case MVM_OP_box_n: { |
522 | 8.08k | MVMSpeshFacts *target_facts = &(g->facts[ins->operands[0].reg.orig][ins->operands[0].reg.i]); |
523 | 8.08k | create_facts(tc, g, |
524 | 8.08k | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
525 | 8.08k | ins->operands[2].reg.orig, ins->operands[2].reg.i); |
526 | 8.08k | target_facts->flags |= MVM_SPESH_FACT_KNOWN_BOX_SRC; |
527 | 8.08k | break; |
528 | 8.08k | } |
529 | 0 | case MVM_OP_add_I: |
530 | 0 | case MVM_OP_sub_I: |
531 | 0 | case MVM_OP_mul_I: |
532 | 0 | case MVM_OP_div_I: |
533 | 0 | case MVM_OP_mod_I: |
534 | 0 | create_facts(tc, g, |
535 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
536 | 0 | ins->operands[3].reg.orig, ins->operands[3].reg.i); |
537 | 0 | break; |
538 | 0 | case MVM_OP_neg_I: |
539 | 0 | case MVM_OP_abs_I: |
540 | 0 | create_facts(tc, g, |
541 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
542 | 0 | ins->operands[2].reg.orig, ins->operands[2].reg.i); |
543 | 0 | break; |
544 | 0 | case MVM_OP_bootint: |
545 | 0 | object_facts(tc, g, |
546 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
547 | 0 | tc->instance->boot_types.BOOTInt); |
548 | 0 | break; |
549 | 0 | case MVM_OP_bootnum: |
550 | 0 | object_facts(tc, g, |
551 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
552 | 0 | tc->instance->boot_types.BOOTNum); |
553 | 0 | break; |
554 | 0 | case MVM_OP_bootstr: |
555 | 0 | object_facts(tc, g, |
556 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
557 | 0 | tc->instance->boot_types.BOOTStr); |
558 | 0 | break; |
559 | 0 | case MVM_OP_bootarray: |
560 | 0 | object_facts(tc, g, |
561 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
562 | 0 | tc->instance->boot_types.BOOTArray); |
563 | 0 | break; |
564 | 444 | case MVM_OP_bootintarray: |
565 | 444 | object_facts(tc, g, |
566 | 444 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
567 | 444 | tc->instance->boot_types.BOOTIntArray); |
568 | 444 | break; |
569 | 0 | case MVM_OP_bootnumarray: |
570 | 0 | object_facts(tc, g, |
571 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
572 | 0 | tc->instance->boot_types.BOOTNumArray); |
573 | 0 | break; |
574 | 44 | case MVM_OP_bootstrarray: |
575 | 44 | object_facts(tc, g, |
576 | 44 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
577 | 44 | tc->instance->boot_types.BOOTStrArray); |
578 | 44 | break; |
579 | 0 | case MVM_OP_boothash: |
580 | 0 | object_facts(tc, g, |
581 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
582 | 0 | tc->instance->boot_types.BOOTHash); |
583 | 0 | break; |
584 | 5.29k | case MVM_OP_hllboxtype_i: |
585 | 5.29k | object_facts(tc, g, |
586 | 5.29k | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
587 | 5.29k | g->sf->body.cu->body.hll_config->int_box_type); |
588 | 5.29k | break; |
589 | 403 | case MVM_OP_hllboxtype_n: |
590 | 403 | object_facts(tc, g, |
591 | 403 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
592 | 403 | g->sf->body.cu->body.hll_config->num_box_type); |
593 | 403 | break; |
594 | 2.39k | case MVM_OP_hllboxtype_s: |
595 | 2.39k | object_facts(tc, g, |
596 | 2.39k | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
597 | 2.39k | g->sf->body.cu->body.hll_config->str_box_type); |
598 | 2.39k | break; |
599 | 3.42k | case MVM_OP_hlllist: |
600 | 3.42k | object_facts(tc, g, |
601 | 3.42k | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
602 | 3.42k | g->sf->body.cu->body.hll_config->slurpy_array_type); |
603 | 3.42k | break; |
604 | 1.32k | case MVM_OP_hllhash: |
605 | 1.32k | object_facts(tc, g, |
606 | 1.32k | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
607 | 1.32k | g->sf->body.cu->body.hll_config->slurpy_hash_type); |
608 | 1.32k | break; |
609 | 117k | case MVM_OP_decont: |
610 | 117k | decont_facts(tc, g, ins, |
611 | 117k | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
612 | 117k | ins->operands[1].reg.orig, ins->operands[1].reg.i); |
613 | 117k | break; |
614 | 52.6k | case MVM_OP_wval: |
615 | 52.6k | wval_facts(tc, g, |
616 | 52.6k | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
617 | 52.6k | ins->operands[1].lit_i16, ins->operands[2].lit_i16); |
618 | 52.6k | break; |
619 | 0 | case MVM_OP_wval_wide: |
620 | 0 | wval_facts(tc, g, |
621 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
622 | 0 | ins->operands[1].lit_i16, ins->operands[2].lit_i64); |
623 | 0 | break; |
624 | 1.84k | case MVM_OP_iter: |
625 | 1.84k | iter_facts(tc, g, |
626 | 1.84k | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
627 | 1.84k | ins->operands[1].reg.orig, ins->operands[1].reg.i); |
628 | 1.84k | break; |
629 | 0 | case MVM_OP_newexception: |
630 | 0 | create_facts_with_type(tc, g, |
631 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
632 | 0 | tc->instance->boot_types.BOOTException); |
633 | 0 | break; |
634 | 0 | case MVM_OP_getlexref_i: |
635 | 0 | case MVM_OP_getlexref_i32: |
636 | 0 | case MVM_OP_getlexref_i16: |
637 | 0 | case MVM_OP_getlexref_i8: |
638 | 0 | case MVM_OP_getlexref_u32: |
639 | 0 | case MVM_OP_getlexref_u16: |
640 | 0 | case MVM_OP_getlexref_u8: |
641 | 0 | create_facts_with_type(tc, g, |
642 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
643 | 0 | g->sf->body.cu->body.hll_config->int_lex_ref); |
644 | 0 | break; |
645 | 0 | case MVM_OP_getlexref_n: |
646 | 0 | case MVM_OP_getlexref_n32: |
647 | 0 | create_facts_with_type(tc, g, |
648 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
649 | 0 | g->sf->body.cu->body.hll_config->num_lex_ref); |
650 | 0 | break; |
651 | 0 | case MVM_OP_getlexref_s: |
652 | 0 | create_facts_with_type(tc, g, |
653 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
654 | 0 | g->sf->body.cu->body.hll_config->str_lex_ref); |
655 | 0 | break; |
656 | 0 | case MVM_OP_getattrref_i: |
657 | 0 | case MVM_OP_getattrsref_i: |
658 | 0 | create_facts_with_type(tc, g, |
659 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
660 | 0 | g->sf->body.cu->body.hll_config->int_attr_ref); |
661 | 0 | break; |
662 | 0 | case MVM_OP_getattrref_n: |
663 | 0 | case MVM_OP_getattrsref_n: |
664 | 0 | create_facts_with_type(tc, g, |
665 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
666 | 0 | g->sf->body.cu->body.hll_config->num_attr_ref); |
667 | 0 | break; |
668 | 0 | case MVM_OP_getattrref_s: |
669 | 0 | case MVM_OP_getattrsref_s: |
670 | 0 | create_facts_with_type(tc, g, |
671 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
672 | 0 | g->sf->body.cu->body.hll_config->str_attr_ref); |
673 | 0 | break; |
674 | 0 | case MVM_OP_atposref_i: |
675 | 0 | create_facts_with_type(tc, g, |
676 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
677 | 0 | g->sf->body.cu->body.hll_config->int_pos_ref); |
678 | 0 | break; |
679 | 0 | case MVM_OP_atposref_n: |
680 | 0 | create_facts_with_type(tc, g, |
681 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
682 | 0 | g->sf->body.cu->body.hll_config->num_pos_ref); |
683 | 0 | break; |
684 | 0 | case MVM_OP_atposref_s: |
685 | 0 | create_facts_with_type(tc, g, |
686 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
687 | 0 | g->sf->body.cu->body.hll_config->str_pos_ref); |
688 | 0 | break; |
689 | 0 |
|
690 | 97.6k | case MVM_OP_const_i64: |
691 | 97.6k | case MVM_OP_const_i32: |
692 | 97.6k | case MVM_OP_const_i16: |
693 | 97.6k | case MVM_OP_const_i8: |
694 | 97.6k | case MVM_OP_const_n64: |
695 | 97.6k | case MVM_OP_const_n32: |
696 | 97.6k | case MVM_OP_const_i64_32: |
697 | 97.6k | case MVM_OP_const_i64_16: |
698 | 97.6k | case MVM_OP_const_s: |
699 | 97.6k | literal_facts(tc, g, ins); |
700 | 97.6k | break; |
701 | 8 | case MVM_OP_encode: |
702 | 8 | create_facts(tc, g, |
703 | 8 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
704 | 8 | ins->operands[3].reg.orig, ins->operands[3].reg.i); |
705 | 8 | break; |
706 | 0 | case MVM_OP_encoderep: |
707 | 0 | create_facts(tc, g, |
708 | 0 | ins->operands[0].reg.orig, ins->operands[0].reg.i, |
709 | 0 | ins->operands[4].reg.orig, ins->operands[4].reg.i); |
710 | 0 | break; |
711 | 0 | case MVM_OP_cas_o: |
712 | 0 | case MVM_OP_atomicload_o: { |
713 | 0 | MVMSpeshOperand result = ins->operands[0]; |
714 | 0 | g->facts[result.reg.orig][result.reg.i].flags |= MVM_SPESH_FACT_DECONTED; |
715 | 0 | break; |
716 | 0 | } |
717 | 1.84M | default: |
718 | 1.84M | if (ins->info->opcode == (MVMuint16)-1) |
719 | 0 | discover_extop(tc, g, ins); |
720 | 2.30M | } |
721 | 2.30M | ins = ins->next; |
722 | 2.30M | } |
723 | 481k | |
724 | 481k | /* Visit children. */ |
725 | 943k | for (i = 0; i < bb->num_children; i++) |
726 | 462k | add_bb_facts(tc, g, bb->children[i], p, cur_deopt_idx); |
727 | 481k | } |
728 | | |
729 | | /* Exception handlers that use a block to store the handler must not have the |
730 | | * instructions that install the block eliminated. This tweaks the usage of |
731 | | * them. */ |
732 | 18.6k | static void tweak_block_handler_usage(MVMThreadContext *tc, MVMSpeshGraph *g) { |
733 | 18.6k | MVMint32 i; |
734 | 27.8k | for (i = 0; i < g->sf->body.num_handlers; i++) { |
735 | 9.19k | if (g->sf->body.handlers[i].action == MVM_EX_ACTION_INVOKE) |
736 | 15 | g->facts[g->sf->body.handlers[i].block_reg][1].usages++; |
737 | 9.19k | } |
738 | 18.6k | } |
739 | | |
740 | | /* Kicks off fact discovery from the top of the (dominator) tree. */ |
741 | 18.6k | void MVM_spesh_facts_discover(MVMThreadContext *tc, MVMSpeshGraph *g, MVMSpeshPlanned *p) { |
742 | 18.6k | add_bb_facts(tc, g, g->entry, p, -1); |
743 | 18.6k | tweak_block_handler_usage(tc, g); |
744 | 18.6k | } |