/home/travis/build/MoarVM/MoarVM/src/spesh/codegen.c

Source (jump to first uncovered line)
#include "moar.h"

/* Here we turn a spesh tree back into MoarVM bytecode, after optimizations
 * have been applied to it. */

/* Writer state. */
typedef struct {
    /* Bytecode output buffer. */
    MVMuint8  *bytecode;
    MVMuint32  bytecode_pos;
    MVMuint32  bytecode_alloc;

    /* Offsets where basic blocks are. */
    MVMint32 *bb_offsets;

    /* Fixups we need to do by basic block. */
    MVMint32    *fixup_locations;
    MVMSpeshBB **fixup_bbs;
    MVMint32     num_fixups;
    MVMint32     alloc_fixups;

    /* Copied frame handlers (which we'll update offsets of). */
    MVMFrameHandler *handlers;
} SpeshWriterState;

/* Write functions; all native endian. */
static void ensure_space(SpeshWriterState *ws, int bytes) {
    if (ws->bytecode_pos + bytes >= ws->bytecode_alloc) {
        ws->bytecode_alloc *= 2;
        ws->bytecode = MVM_realloc(ws->bytecode, ws->bytecode_alloc);
    }
}
static void write_int64(SpeshWriterState *ws, MVMuint64 value) {
    ensure_space(ws, 8);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 8);
    ws->bytecode_pos += 8;
}
static void write_int32(SpeshWriterState *ws, MVMuint32 value) {
    ensure_space(ws, 4);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 4);
    ws->bytecode_pos += 4;
}
static void write_int16(SpeshWriterState *ws, MVMuint16 value) {
    ensure_space(ws, 2);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 2);
    ws->bytecode_pos += 2;
}
static void write_int8(SpeshWriterState *ws, MVMuint8 value) {
    ensure_space(ws, 1);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 1);
    ws->bytecode_pos++;
}
static void write_num32(SpeshWriterState *ws, MVMnum32 value) {
    ensure_space(ws, 4);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 4);
    ws->bytecode_pos += 4;
}
static void write_num64(SpeshWriterState *ws, MVMnum64 value) {
    ensure_space(ws, 8);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 8);
    ws->bytecode_pos += 8;
}

/* Writes instructions within a basic block boundary. */
static void write_instructions(MVMThreadContext *tc, MVMSpeshGraph *g, SpeshWriterState *ws, MVMSpeshBB *bb) {
    MVMSpeshIns *ins = bb->first_ins;
    while (ins) {
        MVMint32 i;

        /* Process any annotations. */
        MVMSpeshAnn *ann              = ins->annotations;
        MVMSpeshAnn *deopt_one_ann    = NULL;
        MVMSpeshAnn *deopt_all_ann    = NULL;
        MVMSpeshAnn *deopt_inline_ann = NULL;
        while (ann) {
            switch (ann->type) {
            case MVM_SPESH_ANN_FH_START:
                ws->handlers[ann->data.frame_handler_index].start_offset =
                    ws->bytecode_pos;
                break;
            case MVM_SPESH_ANN_FH_END:
                ws->handlers[ann->data.frame_handler_index].end_offset =
                    ws->bytecode_pos;
                break;
            case MVM_SPESH_ANN_FH_GOTO:
                ws->handlers[ann->data.frame_handler_index].goto_offset =
                    ws->bytecode_pos;
                break;
            case MVM_SPESH_ANN_DEOPT_ONE_INS:
                deopt_one_ann = ann;
                break;
            case MVM_SPESH_ANN_DEOPT_ALL_INS:
                deopt_all_ann = ann;
                break;
            case MVM_SPESH_ANN_INLINE_START:
                g->inlines[ann->data.inline_idx].start = ws->bytecode_pos;
                break;
            case MVM_SPESH_ANN_INLINE_END:
                g->inlines[ann->data.inline_idx].end = ws->bytecode_pos;
                break;
            case MVM_SPESH_ANN_DEOPT_INLINE:
                deopt_inline_ann = ann;
                break;
            case MVM_SPESH_ANN_DEOPT_OSR:
                g->deopt_addrs[2 * ann->data.deopt_idx + 1] = ws->bytecode_pos;
                break;
            }
            ann = ann->next;
        }

        if (ins->info->opcode != MVM_SSA_PHI) {
            /* Real instruction, not a phi. Emit opcode. */
            if (ins->info->opcode == (MVMuint16)-1) {
                /* Ext op; resolve. */
                MVMExtOpRecord *extops     = g->sf->body.cu->body.extops;
                MVMuint16       num_extops = g->sf->body.cu->body.num_extops;
                MVMint32        found      = 0;
                for (i = 0; i < num_extops; i++) {
                    if (extops[i].info == ins->info) {
                        write_int16(ws, MVM_OP_EXT_BASE + i);
                        found = 1;
                        break;
                    }
                }
                if (!found)
                    MVM_oops(tc, "Spesh: failed to resolve extop in code-gen");
            }
            else {
                /* Core op. */
                write_int16(ws, ins->info->opcode);
            }

            /* Write out operands. */
            for (i = 0; i < ins->info->num_operands; i++) {
                MVMuint8 flags = ins->info->operands[i];
                MVMuint8 rw    = flags & MVM_operand_rw_mask;
                switch (rw) {
                case MVM_operand_read_reg:
                case MVM_operand_write_reg:
                    write_int16(ws, ins->operands[i].reg.orig);
                    break;
                case MVM_operand_read_lex:
                case MVM_operand_write_lex:
                    write_int16(ws, ins->operands[i].lex.idx);
                    write_int16(ws, ins->operands[i].lex.outers);
                    break;
                case MVM_operand_literal: {
                    MVMuint8 type = flags & MVM_operand_type_mask;
                    switch (type) {
                    case MVM_operand_int8:
                        write_int8(ws, ins->operands[i].lit_i8);
                        break;
                    case MVM_operand_int16:
                        write_int16(ws, ins->operands[i].lit_i16);
                        break;
                    case MVM_operand_int32:
                        write_int32(ws, ins->operands[i].lit_i32);
                        break;
                    case MVM_operand_int64:
                        write_int64(ws, ins->operands[i].lit_i64);
                        break;
                    case MVM_operand_num32:
                        write_num32(ws, ins->operands[i].lit_n32);
                        break;
                    case MVM_operand_num64:
                        write_num64(ws, ins->operands[i].lit_n64);
                        break;
                    case MVM_operand_callsite:
                        write_int16(ws, ins->operands[i].callsite_idx);
                        break;
                    case MVM_operand_coderef:
                        write_int16(ws, ins->operands[i].coderef_idx);
                        break;
                    case MVM_operand_str:
                        write_int32(ws, ins->operands[i].lit_str_idx);
                        break;
                    case MVM_operand_ins: {
                        MVMint32 offset = ws->bb_offsets[ins->operands[i].ins_bb->idx];
                        if (offset >= 0) {
                            /* Already know where it is, so just write it. */
                            write_int32(ws, offset);
                        }
                        else {
                            /* Need to fix it up. */
                            if (ws->num_fixups == ws->alloc_fixups) {
                                ws->alloc_fixups *= 2;
                                ws->fixup_locations = MVM_realloc(ws->fixup_locations,
                                    ws->alloc_fixups * sizeof(MVMint32));
                                ws->fixup_bbs = MVM_realloc(ws->fixup_bbs,
                                    ws->alloc_fixups * sizeof(MVMSpeshBB *));
                            }
                            ws->fixup_locations[ws->num_fixups] = ws->bytecode_pos;
                            ws->fixup_bbs[ws->num_fixups]       = ins->operands[i].ins_bb;
                            write_int32(ws, 0);
                            ws->num_fixups++;
                        }
                        break;
                    }
                    case MVM_operand_spesh_slot:
                        write_int16(ws, ins->operands[i].lit_i16);
                        break;
                    default:
                        MVM_oops(tc,
                            "Spesh: unknown operand type %d in codegen (op %s)",
                            (int)type, ins->info->name);
                    }
                    }
                    break;
                default:
                    MVM_oops(tc, "Spesh: unknown operand type in codegen");
                }
            }
        }

        /* If there was a deopt point annotation, update table. */
        if (deopt_one_ann)
            g->deopt_addrs[2 * deopt_one_ann->data.deopt_idx + 1] = ws->bytecode_pos;
        if (deopt_all_ann)
            g->deopt_addrs[2 * deopt_all_ann->data.deopt_idx + 1] = ws->bytecode_pos;
        if (deopt_inline_ann)
            g->deopt_addrs[2 * deopt_inline_ann->data.deopt_idx + 1] = ws->bytecode_pos;

        ins = ins->next;
    }
}

/* Generate bytecode from a spesh graph. */
MVMSpeshCode * MVM_spesh_codegen(MVMThreadContext *tc, MVMSpeshGraph *g) {
    MVMSpeshCode *res;
    MVMSpeshBB   *bb;
    MVMint32      i, hanlen;

    /* Initialize writer state. */
    SpeshWriterState *ws     = MVM_malloc(sizeof(SpeshWriterState));
    ws->bytecode_pos    = 0;
    ws->bytecode_alloc  = 1024;
    ws->bytecode        = MVM_malloc(ws->bytecode_alloc);
    ws->bb_offsets      = MVM_malloc(g->num_bbs * sizeof(MVMint32));
    ws->num_fixups      = 0;
    ws->alloc_fixups    = 64;
    ws->fixup_locations = MVM_malloc(ws->alloc_fixups * sizeof(MVMint32));
    ws->fixup_bbs       = MVM_malloc(ws->alloc_fixups * sizeof(MVMSpeshBB *));
    for (i = 0; i < g->num_bbs; i++)
        ws->bb_offsets[i] = -1;

    /* Create copy of handlers, and -1 all offsets so we can catch missing
     * updates. */
    hanlen = g->num_handlers * sizeof(MVMFrameHandler);
    if (hanlen) {
        ws->handlers = MVM_malloc(hanlen);
        memcpy(ws->handlers, g->handlers, hanlen);
        for (i = 0; i < g->num_handlers; i++) {
            ws->handlers[i].start_offset = -1;
            ws->handlers[i].end_offset   = -1;
            ws->handlers[i].goto_offset  = -1;
        }
    }
    else {
        ws->handlers = NULL;
    }

    /* -1 all the deopt targets, so we'll easily catch those that don't get
     * mapped if we try to use them. Same for inlines. */
    for (i = 0; i < g->num_deopt_addrs; i++)
        g->deopt_addrs[i * 2 + 1] = -1;
    for (i = 0; i < g->num_inlines; i++) {
        g->inlines[i].start = -1;
        g->inlines[i].end = -1;
    }

    /* Write out each of the basic blocks, in linear order. Skip the first,
     * dummy, block. */
    bb = g->entry->linear_next;
    while (bb) {
        ws->bb_offsets[bb->idx] = ws->bytecode_pos;
        write_instructions(tc, g, ws, bb);
        bb = bb->linear_next;
    }

    /* Fixup labels we were too early for. */
    for (i = 0; i < ws->num_fixups; i++)
        *((MVMuint32 *)(ws->bytecode + ws->fixup_locations[i])) =
            ws->bb_offsets[ws->fixup_bbs[i]->idx];

    /* Ensure all handlers got fixed up. */
    for (i = 0; i < g->num_handlers; i++) {
        if (ws->handlers[i].start_offset == -1 ||
            ws->handlers[i].end_offset   == -1 ||
            ws->handlers[i].goto_offset  == -1)
            MVM_oops(tc, "Spesh: failed to fix up handlers (%d, %d, %d)",
                (int)ws->handlers[i].start_offset,
                (int)ws->handlers[i].end_offset,
                (int)ws->handlers[i].goto_offset);
    }

    /* Ensure all inlines got fixed up. */
    for (i = 0; i < g->num_inlines; i++)
        if (g->inlines[i].start == -1 || g->inlines[i].end == -1)
            MVM_oops(tc, "Spesh: failed to fix up inline %d", i);

    /* Produce result data structure. */
    res                = MVM_malloc(sizeof(MVMSpeshCode));
    res->bytecode      = ws->bytecode;
    res->bytecode_size = ws->bytecode_pos;
    res->handlers      = ws->handlers;

    /* Cleanup. */
    MVM_free(ws->bb_offsets);
    MVM_free(ws->fixup_locations);
    MVM_free(ws->fixup_bbs);
    MVM_free(ws);

    return res;
}

Coverage Report

Created: 2017-04-15 07:07

Line	Count	Source (jump to first uncovered line)
1		#include "moar.h"
2
3		/* Here we turn a spesh tree back into MoarVM bytecode, after optimizations
4		* have been applied to it. */
5
6		/* Writer state. */
7		typedef struct {
8		/* Bytecode output buffer. */
9		MVMuint8 *bytecode;
10		MVMuint32 bytecode_pos;
11		MVMuint32 bytecode_alloc;
12
13		/* Offsets where basic blocks are. */
14		MVMint32 *bb_offsets;
15
16		/* Fixups we need to do by basic block. */
17		MVMint32 *fixup_locations;
18		MVMSpeshBB **fixup_bbs;
19		MVMint32 num_fixups;
20		MVMint32 alloc_fixups;
21
22		/* Copied frame handlers (which we'll update offsets of). */
23		MVMFrameHandler *handlers;
24		} SpeshWriterState;
25
26		/* Write functions; all native endian. */
27	7.57M	static void ensure_space(SpeshWriterState *ws, int bytes) {
28	7.57M	if (ws->bytecode_pos + bytes >= ws->bytecode_alloc) {
29	5.87k	ws->bytecode_alloc *= 2;
30	5.87k	ws->bytecode = MVM_realloc(ws->bytecode, ws->bytecode_alloc);
31	5.87k	}
32	7.57M	}
33	10.1k	static void write_int64(SpeshWriterState *ws, MVMuint64 value) {
34	10.1k	ensure_space(ws, 8);
35	10.1k	memcpy(ws->bytecode + ws->bytecode_pos, &value, 8);
36	10.1k	ws->bytecode_pos += 8;
37	10.1k	}
38	503k	static void write_int32(SpeshWriterState *ws, MVMuint32 value) {
39	503k	ensure_space(ws, 4);
40	503k	memcpy(ws->bytecode + ws->bytecode_pos, &value, 4);
41	503k	ws->bytecode_pos += 4;
42	503k	}
43	7.05M	static void write_int16(SpeshWriterState *ws, MVMuint16 value) {
44	7.05M	ensure_space(ws, 2);
45	7.05M	memcpy(ws->bytecode + ws->bytecode_pos, &value, 2);
46	7.05M	ws->bytecode_pos += 2;
47	7.05M	}
48	0	static void write_int8(SpeshWriterState *ws, MVMuint8 value) {
49	0	ensure_space(ws, 1);
50	0	memcpy(ws->bytecode + ws->bytecode_pos, &value, 1);
51	0	ws->bytecode_pos++;
52	0	}
53	0	static void write_num32(SpeshWriterState *ws, MVMnum32 value) {
54	0	ensure_space(ws, 4);
55	0	memcpy(ws->bytecode + ws->bytecode_pos, &value, 4);
56	0	ws->bytecode_pos += 4;
57	0	}
58	1.45k	static void write_num64(SpeshWriterState *ws, MVMnum64 value) {
59	1.45k	ensure_space(ws, 8);
60	1.45k	memcpy(ws->bytecode + ws->bytecode_pos, &value, 8);
61	1.45k	ws->bytecode_pos += 8;
62	1.45k	}
63
64		/* Writes instructions within a basic block boundary. */
65	549k	static void write_instructions(MVMThreadContext tc, MVMSpeshGraph g, SpeshWriterState ws, MVMSpeshBB bb) {
66	549k	MVMSpeshIns *ins = bb->first_ins;
67	3.99M	while (ins) {
68	3.44M	MVMint32 i;
69	3.44M
70	3.44M	/* Process any annotations. */
71	3.44M	MVMSpeshAnn *ann = ins->annotations;
72	3.44M	MVMSpeshAnn *deopt_one_ann = NULL;
73	3.44M	MVMSpeshAnn *deopt_all_ann = NULL;
74	3.44M	MVMSpeshAnn *deopt_inline_ann = NULL;
75	3.71M	while (ann) {
76	277k	switch (ann->type) {
77	10.5k	case MVM_SPESH_ANN_FH_START:
78	10.5k	ws->handlers[ann->data.frame_handler_index].start_offset =
79	10.5k	ws->bytecode_pos;
80	10.5k	break;
81	10.5k	case MVM_SPESH_ANN_FH_END:
82	10.5k	ws->handlers[ann->data.frame_handler_index].end_offset =
83	10.5k	ws->bytecode_pos;
84	10.5k	break;
85	10.5k	case MVM_SPESH_ANN_FH_GOTO:
86	10.5k	ws->handlers[ann->data.frame_handler_index].goto_offset =
87	10.5k	ws->bytecode_pos;
88	10.5k	break;
89	155k	case MVM_SPESH_ANN_DEOPT_ONE_INS:
90	155k	deopt_one_ann = ann;
91	155k	break;
92	69.1k	case MVM_SPESH_ANN_DEOPT_ALL_INS:
93	69.1k	deopt_all_ann = ann;
94	69.1k	break;
95	4.25k	case MVM_SPESH_ANN_INLINE_START:
96	4.25k	g->inlines[ann->data.inline_idx].start = ws->bytecode_pos;
97	4.25k	break;
98	4.25k	case MVM_SPESH_ANN_INLINE_END:
99	4.25k	g->inlines[ann->data.inline_idx].end = ws->bytecode_pos;
100	4.25k	break;
101	4.70k	case MVM_SPESH_ANN_DEOPT_INLINE:
102	4.70k	deopt_inline_ann = ann;
103	4.70k	break;
104	7.57k	case MVM_SPESH_ANN_DEOPT_OSR:
105	7.57k	g->deopt_addrs[2 * ann->data.deopt_idx + 1] = ws->bytecode_pos;
106	7.57k	break;
107	277k	}
108	277k	ann = ann->next;
109	277k	}
110	3.44M
111	3.44M	if (ins->info->opcode != MVM_SSA_PHI) {
112	2.45M	/* Real instruction, not a phi. Emit opcode. */
113	2.45M	if (ins->info->opcode == (MVMuint16)-1) {
114	0	/* Ext op; resolve. */
115	0	MVMExtOpRecord *extops = g->sf->body.cu->body.extops;
116	0	MVMuint16 num_extops = g->sf->body.cu->body.num_extops;
117	0	MVMint32 found = 0;
118	0	for (i = 0; i < num_extops; i++) {
119	0	if (extops[i].info == ins->info) {
120	0	write_int16(ws, MVM_OP_EXT_BASE + i);
121	0	found = 1;
122	0	break;
123	0	}
124	0	}
125	0	if (!found)
126	0	MVM_oops(tc, "Spesh: failed to resolve extop in code-gen");
127	0	}
128	2.45M	else {
129	2.45M	/* Core op. */
130	2.45M	write_int16(ws, ins->info->opcode);
131	2.45M	}
132	2.45M
133	2.45M	/* Write out operands. */
134	7.52M	for (i = 0; i < ins->info->num_operands; i++) {
135	5.06M	MVMuint8 flags = ins->info->operands[i];
136	5.06M	MVMuint8 rw = flags & MVM_operand_rw_mask;
137	5.06M	switch (rw) {
138	3.66M	case MVM_operand_read_reg:
139	3.66M	case MVM_operand_write_reg:
140	3.66M	write_int16(ws, ins->operands[i].reg.orig);
141	3.66M	break;
142	48.7k	case MVM_operand_read_lex:
143	48.7k	case MVM_operand_write_lex:
144	48.7k	write_int16(ws, ins->operands[i].lex.idx);
145	48.7k	write_int16(ws, ins->operands[i].lex.outers);
146	48.7k	break;
147	1.35M	case MVM_operand_literal: {
148	1.35M	MVMuint8 type = flags & MVM_operand_type_mask;
149	1.35M	switch (type) {
150	0	case MVM_operand_int8:
151	0	write_int8(ws, ins->operands[i].lit_i8);
152	0	break;
153	681k	case MVM_operand_int16:
154	681k	write_int16(ws, ins->operands[i].lit_i16);
155	681k	break;
156	14	case MVM_operand_int32:
157	14	write_int32(ws, ins->operands[i].lit_i32);
158	14	break;
159	10.1k	case MVM_operand_int64:
160	10.1k	write_int64(ws, ins->operands[i].lit_i64);
161	10.1k	break;
162	0	case MVM_operand_num32:
163	0	write_num32(ws, ins->operands[i].lit_n32);
164	0	break;
165	1.45k	case MVM_operand_num64:
166	1.45k	write_num64(ws, ins->operands[i].lit_n64);
167	1.45k	break;
168	66.1k	case MVM_operand_callsite:
169	66.1k	write_int16(ws, ins->operands[i].callsite_idx);
170	66.1k	break;
171	1.56k	case MVM_operand_coderef:
172	1.56k	write_int16(ws, ins->operands[i].coderef_idx);
173	1.56k	break;
174	211k	case MVM_operand_str:
175	211k	write_int32(ws, ins->operands[i].lit_str_idx);
176	211k	break;
177	291k	case MVM_operand_ins: {
178	291k	MVMint32 offset = ws->bb_offsets[ins->operands[i].ins_bb->idx];
179	291k	if (offset >= 0) {
180	23.4k	/* Already know where it is, so just write it. */
181	23.4k	write_int32(ws, offset);
182	23.4k	}
183	267k	else {
184	267k	/* Need to fix it up. */
185	267k	if (ws->num_fixups == ws->alloc_fixups) {
186	1.30k	ws->alloc_fixups *= 2;
187	1.30k	ws->fixup_locations = MVM_realloc(ws->fixup_locations,
188	1.30k	ws->alloc_fixups * sizeof(MVMint32));
189	1.30k	ws->fixup_bbs = MVM_realloc(ws->fixup_bbs,
190	1.30k	ws->alloc_fixups * sizeof(MVMSpeshBB *));
191	1.30k	}
192	267k	ws->fixup_locations[ws->num_fixups] = ws->bytecode_pos;
193	267k	ws->fixup_bbs[ws->num_fixups] = ins->operands[i].ins_bb;
194	267k	write_int32(ws, 0);
195	267k	ws->num_fixups++;
196	267k	}
197	291k	break;
198	0	}
199	91.4k	case MVM_operand_spesh_slot:
200	91.4k	write_int16(ws, ins->operands[i].lit_i16);
201	91.4k	break;
202	0	default:
203	0	MVM_oops(tc,
204	0	"Spesh: unknown operand type %d in codegen (op %s)",
205	0	(int)type, ins->info->name);
206	1.35M	}
207	1.35M	}
208	1.35M	break;
209	0	default:
210	0	MVM_oops(tc, "Spesh: unknown operand type in codegen");
211	5.06M	}
212	5.06M	}
213	2.45M	}
214	3.44M
215	3.44M	/* If there was a deopt point annotation, update table. */
216	3.44M	if (deopt_one_ann)
217	155k	g->deopt_addrs[2 * deopt_one_ann->data.deopt_idx + 1] = ws->bytecode_pos;
218	3.44M	if (deopt_all_ann)
219	69.1k	g->deopt_addrs[2 * deopt_all_ann->data.deopt_idx + 1] = ws->bytecode_pos;
220	3.44M	if (deopt_inline_ann)
221	4.06k	g->deopt_addrs[2 * deopt_inline_ann->data.deopt_idx + 1] = ws->bytecode_pos;
222	3.44M
223	3.44M	ins = ins->next;
224	3.44M	}
225	549k	}
226
227		/* Generate bytecode from a spesh graph. */
228	31.0k	MVMSpeshCode * MVM_spesh_codegen(MVMThreadContext tc, MVMSpeshGraph g) {
229	31.0k	MVMSpeshCode *res;
230	31.0k	MVMSpeshBB *bb;
231	31.0k	MVMint32 i, hanlen;
232	31.0k
233	31.0k	/* Initialize writer state. */
234	31.0k	SpeshWriterState *ws = MVM_malloc(sizeof(SpeshWriterState));
235	31.0k	ws->bytecode_pos = 0;
236	31.0k	ws->bytecode_alloc = 1024;
237	31.0k	ws->bytecode = MVM_malloc(ws->bytecode_alloc);
238	31.0k	ws->bb_offsets = MVM_malloc(g->num_bbs * sizeof(MVMint32));
239	31.0k	ws->num_fixups = 0;
240	31.0k	ws->alloc_fixups = 64;
241	31.0k	ws->fixup_locations = MVM_malloc(ws->alloc_fixups * sizeof(MVMint32));
242	31.0k	ws->fixup_bbs = MVM_malloc(ws->alloc_fixups * sizeof(MVMSpeshBB *));
243	612k	for (i = 0; i < g->num_bbs; i++)
244	581k	ws->bb_offsets[i] = -1;
245	31.0k
246	31.0k	/* Create copy of handlers, and -1 all offsets so we can catch missing
247	31.0k	* updates. */
248	31.0k	hanlen = g->num_handlers * sizeof(MVMFrameHandler);
249	31.0k	if (hanlen) {
250	3.55k	ws->handlers = MVM_malloc(hanlen);
251	3.55k	memcpy(ws->handlers, g->handlers, hanlen);
252	14.1k	for (i = 0; i < g->num_handlers; i++) {
253	10.5k	ws->handlers[i].start_offset = -1;
254	10.5k	ws->handlers[i].end_offset = -1;
255	10.5k	ws->handlers[i].goto_offset = -1;
256	10.5k	}
257	3.55k	}
258	27.4k	else {
259	27.4k	ws->handlers = NULL;
260	27.4k	}
261	31.0k
262	31.0k	/* -1 all the deopt targets, so we'll easily catch those that don't get
263	31.0k	* mapped if we try to use them. Same for inlines. */
264	273k	for (i = 0; i < g->num_deopt_addrs; i++)
265	242k	g->deopt_addrs[i * 2 + 1] = -1;
266	35.3k	for (i = 0; i < g->num_inlines; i++) {
267	4.25k	g->inlines[i].start = -1;
268	4.25k	g->inlines[i].end = -1;
269	4.25k	}
270	31.0k
271	31.0k	/* Write out each of the basic blocks, in linear order. Skip the first,
272	31.0k	* dummy, block. */
273	31.0k	bb = g->entry->linear_next;
274	581k	while (bb) {
275	549k	ws->bb_offsets[bb->idx] = ws->bytecode_pos;
276	549k	write_instructions(tc, g, ws, bb);
277	549k	bb = bb->linear_next;
278	549k	}
279	31.0k
280	31.0k	/* Fixup labels we were too early for. */
281	298k	for (i = 0; i < ws->num_fixups; i++)
282	267k	((MVMuint32 )(ws->bytecode + ws->fixup_locations[i])) =
283	267k	ws->bb_offsets[ws->fixup_bbs[i]->idx];
284	31.0k
285	31.0k	/* Ensure all handlers got fixed up. */
286	41.6k	for (i = 0; i < g->num_handlers; i++) {
287	10.5k	if (ws->handlers[i].start_offset == -1 \|\|
288	10.5k	ws->handlers[i].end_offset == -1 \|\|
289	10.5k	ws->handlers[i].goto_offset == -1)
290	0	MVM_oops(tc, "Spesh: failed to fix up handlers (%d, %d, %d)",
291	0	(int)ws->handlers[i].start_offset,
292	0	(int)ws->handlers[i].end_offset,
293	0	(int)ws->handlers[i].goto_offset);
294	10.5k	}
295	31.0k
296	31.0k	/* Ensure all inlines got fixed up. */
297	35.3k	for (i = 0; i < g->num_inlines; i++)
298	4.25k	if (g->inlines[i].start == -1 \|\| g->inlines[i].end == -1)
299	0	MVM_oops(tc, "Spesh: failed to fix up inline %d", i);
300	31.0k
301	31.0k	/* Produce result data structure. */
302	31.0k	res = MVM_malloc(sizeof(MVMSpeshCode));
303	31.0k	res->bytecode = ws->bytecode;
304	31.0k	res->bytecode_size = ws->bytecode_pos;
305	31.0k	res->handlers = ws->handlers;
306	31.0k
307	31.0k	/* Cleanup. */
308	31.0k	MVM_free(ws->bb_offsets);
309	31.0k	MVM_free(ws->fixup_locations);
310	31.0k	MVM_free(ws->fixup_bbs);
311	31.0k	MVM_free(ws);
312	31.0k
313	31.0k	return res;
314	31.0k	}