/home/travis/build/MoarVM/MoarVM/src/spesh/codegen.c

Source (jump to first uncovered line)
#include "moar.h"

/* Here we turn a spesh tree back into MoarVM bytecode, after optimizations
 * have been applied to it. */

/* Writer state. */
typedef struct {
    /* Bytecode output buffer. */
    MVMuint8  *bytecode;
    MVMuint32  bytecode_pos;
    MVMuint32  bytecode_alloc;

    /* Offsets where basic blocks are. */
    MVMint32 *bb_offsets;

    /* Fixups we need to do by basic block. */
    MVMint32    *fixup_locations;
    MVMSpeshBB **fixup_bbs;
    MVMint32     num_fixups;
    MVMint32     alloc_fixups;

    /* Copied frame handlers (which we'll update offsets of). */
    MVMFrameHandler *handlers;
} SpeshWriterState;

/* Write functions; all native endian. */
static void ensure_space(SpeshWriterState *ws, int bytes) {
    if (ws->bytecode_pos + bytes >= ws->bytecode_alloc) {
        ws->bytecode_alloc *= 2;
        ws->bytecode = MVM_realloc(ws->bytecode, ws->bytecode_alloc);
    }
}
static void write_int64(SpeshWriterState *ws, MVMuint64 value) {
    ensure_space(ws, 8);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 8);
    ws->bytecode_pos += 8;
}
static void write_int32(SpeshWriterState *ws, MVMuint32 value) {
    ensure_space(ws, 4);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 4);
    ws->bytecode_pos += 4;
}
static void write_int16(SpeshWriterState *ws, MVMuint16 value) {
    ensure_space(ws, 2);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 2);
    ws->bytecode_pos += 2;
}
static void write_int8(SpeshWriterState *ws, MVMuint8 value) {
    ensure_space(ws, 1);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 1);
    ws->bytecode_pos++;
}
static void write_num32(SpeshWriterState *ws, MVMnum32 value) {
    ensure_space(ws, 4);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 4);
    ws->bytecode_pos += 4;
}
static void write_num64(SpeshWriterState *ws, MVMnum64 value) {
    ensure_space(ws, 8);
    memcpy(ws->bytecode + ws->bytecode_pos, &value, 8);
    ws->bytecode_pos += 8;
}

/* Writes instructions within a basic block boundary. */
static void write_instructions(MVMThreadContext *tc, MVMSpeshGraph *g, SpeshWriterState *ws, MVMSpeshBB *bb) {
    MVMSpeshIns *ins = bb->first_ins;
    while (ins) {
        MVMint32 i;

        /* Process any annotations. */
        MVMSpeshAnn *ann              = ins->annotations;
        MVMSpeshAnn *deopt_one_ann    = NULL;
        MVMSpeshAnn *deopt_all_ann    = NULL;
        MVMSpeshAnn *deopt_inline_ann = NULL;
        while (ann) {
            switch (ann->type) {
            case MVM_SPESH_ANN_FH_START:
                ws->handlers[ann->data.frame_handler_index].start_offset =
                    ws->bytecode_pos;
                break;
            case MVM_SPESH_ANN_FH_END:
                ws->handlers[ann->data.frame_handler_index].end_offset =
                    ws->bytecode_pos;
                break;
            case MVM_SPESH_ANN_FH_GOTO:
                ws->handlers[ann->data.frame_handler_index].goto_offset =
                    ws->bytecode_pos;
                break;
            case MVM_SPESH_ANN_DEOPT_ONE_INS:
                deopt_one_ann = ann;
                break;
            case MVM_SPESH_ANN_DEOPT_ALL_INS:
                deopt_all_ann = ann;
                break;
            case MVM_SPESH_ANN_INLINE_START:
                g->inlines[ann->data.inline_idx].start = ws->bytecode_pos;
                break;
            case MVM_SPESH_ANN_INLINE_END:
                g->inlines[ann->data.inline_idx].end = ws->bytecode_pos;
                break;
            case MVM_SPESH_ANN_DEOPT_INLINE:
                deopt_inline_ann = ann;
                break;
            case MVM_SPESH_ANN_DEOPT_OSR:
                g->deopt_addrs[2 * ann->data.deopt_idx + 1] = ws->bytecode_pos;
                break;
            }
            ann = ann->next;
        }

        if (ins->info->opcode != MVM_SSA_PHI) {
            /* Real instruction, not a phi. Emit opcode. */
            if (ins->info->opcode == (MVMuint16)-1) {
                /* Ext op; resolve. */
                MVMExtOpRecord *extops     = g->sf->body.cu->body.extops;
                MVMuint16       num_extops = g->sf->body.cu->body.num_extops;
                MVMint32        found      = 0;
                for (i = 0; i < num_extops; i++) {
                    if (extops[i].info == ins->info) {
                        write_int16(ws, MVM_OP_EXT_BASE + i);
                        found = 1;
                        break;
                    }
                }
                if (!found)
                    MVM_oops(tc, "Spesh: failed to resolve extop in code-gen");
            }
            else {
                /* Core op. */
                write_int16(ws, ins->info->opcode);
            }

            /* Write out operands. */
            for (i = 0; i < ins->info->num_operands; i++) {
                MVMuint8 flags = ins->info->operands[i];
                MVMuint8 rw    = flags & MVM_operand_rw_mask;
                switch (rw) {
                case MVM_operand_read_reg:
                case MVM_operand_write_reg:
                    write_int16(ws, ins->operands[i].reg.orig);
                    break;
                case MVM_operand_read_lex:
                case MVM_operand_write_lex:
                    write_int16(ws, ins->operands[i].lex.idx);
                    write_int16(ws, ins->operands[i].lex.outers);
                    break;
                case MVM_operand_literal: {
                    MVMuint8 type = flags & MVM_operand_type_mask;
                    switch (type) {
                    case MVM_operand_int8:
                        write_int8(ws, ins->operands[i].lit_i8);
                        break;
                    case MVM_operand_int16:
                        write_int16(ws, ins->operands[i].lit_i16);
                        break;
                    case MVM_operand_int32:
                        write_int32(ws, ins->operands[i].lit_i32);
                        break;
                    case MVM_operand_uint32:
                        write_int32(ws, ins->operands[i].lit_ui32);
                        break;
                    case MVM_operand_int64:
                        write_int64(ws, ins->operands[i].lit_i64);
                        break;
                    case MVM_operand_num32:
                        write_num32(ws, ins->operands[i].lit_n32);
                        break;
                    case MVM_operand_num64:
                        write_num64(ws, ins->operands[i].lit_n64);
                        break;
                    case MVM_operand_callsite:
                        write_int16(ws, ins->operands[i].callsite_idx);
                        break;
                    case MVM_operand_coderef:
                        write_int16(ws, ins->operands[i].coderef_idx);
                        break;
                    case MVM_operand_str:
                        write_int32(ws, ins->operands[i].lit_str_idx);
                        break;
                    case MVM_operand_ins: {
                        MVMint32 bb_idx = ins->operands[i].ins_bb->idx;
                        MVMint32 offset;
                        if (bb_idx >= g->num_bbs)
                            MVM_panic(1, "Spesh codegen: out of range BB index %d", bb_idx);
                        offset = ws->bb_offsets[bb_idx];
                        if (offset >= 0) {
                            /* Already know where it is, so just write it. */
                            write_int32(ws, offset);
                        }
                        else {
                            /* Need to fix it up. */
                            if (ws->num_fixups == ws->alloc_fixups) {
                                ws->alloc_fixups *= 2;
                                ws->fixup_locations = MVM_realloc(ws->fixup_locations,
                                    ws->alloc_fixups * sizeof(MVMint32));
                                ws->fixup_bbs = MVM_realloc(ws->fixup_bbs,
                                    ws->alloc_fixups * sizeof(MVMSpeshBB *));
                            }
                            ws->fixup_locations[ws->num_fixups] = ws->bytecode_pos;
                            ws->fixup_bbs[ws->num_fixups]       = ins->operands[i].ins_bb;
                            write_int32(ws, 0);
                            ws->num_fixups++;
                        }
                        break;
                    }
                    case MVM_operand_spesh_slot:
                        write_int16(ws, ins->operands[i].lit_i16);
                        break;
                    default:
                        MVM_oops(tc,
                            "Spesh: unknown operand type %d in codegen (op %s)",
                            (int)type, ins->info->name);
                    }
                    }
                    break;
                default:
                    MVM_oops(tc, "Spesh: unknown operand type in codegen");
                }
            }
        }

        /* If there was a deopt point annotation, update table. */
        if (deopt_one_ann)
            g->deopt_addrs[2 * deopt_one_ann->data.deopt_idx + 1] = ws->bytecode_pos;
        if (deopt_all_ann)
            g->deopt_addrs[2 * deopt_all_ann->data.deopt_idx + 1] = ws->bytecode_pos;
        if (deopt_inline_ann)
            g->deopt_addrs[2 * deopt_inline_ann->data.deopt_idx + 1] = ws->bytecode_pos;

        ins = ins->next;
    }
}

/* Generate bytecode from a spesh graph. */
MVMSpeshCode * MVM_spesh_codegen(MVMThreadContext *tc, MVMSpeshGraph *g) {
    MVMSpeshCode *res;
    MVMSpeshBB   *bb;
    MVMint32      i, hanlen;

    /* Initialize writer state. */
    SpeshWriterState *ws     = MVM_malloc(sizeof(SpeshWriterState));
    ws->bytecode_pos    = 0;
    ws->bytecode_alloc  = 1024;
    ws->bytecode        = MVM_malloc(ws->bytecode_alloc);
    ws->bb_offsets      = MVM_malloc(g->num_bbs * sizeof(MVMint32));
    ws->num_fixups      = 0;
    ws->alloc_fixups    = 64;
    ws->fixup_locations = MVM_malloc(ws->alloc_fixups * sizeof(MVMint32));
    ws->fixup_bbs       = MVM_malloc(ws->alloc_fixups * sizeof(MVMSpeshBB *));
    for (i = 0; i < g->num_bbs; i++)
        ws->bb_offsets[i] = -1;

    /* Create copy of handlers, and -1 all offsets so we can catch missing
     * updates. */
    hanlen = g->num_handlers * sizeof(MVMFrameHandler);
    if (hanlen) {
        ws->handlers = MVM_malloc(hanlen);
        memcpy(ws->handlers, g->handlers, hanlen);
        for (i = 0; i < g->num_handlers; i++) {
            ws->handlers[i].start_offset = -1;
            ws->handlers[i].end_offset   = -1;
            ws->handlers[i].goto_offset  = -1;
        }
    }
    else {
        ws->handlers = NULL;
    }

    /* -1 all the deopt targets, so we'll easily catch those that don't get
     * mapped if we try to use them. Same for inlines. */
    for (i = 0; i < g->num_deopt_addrs; i++)
        g->deopt_addrs[i * 2 + 1] = -1;
    for (i = 0; i < g->num_inlines; i++) {
        g->inlines[i].start = -1;
        g->inlines[i].end = -1;
    }

    /* Write out each of the basic blocks, in linear order. Skip the first,
     * dummy, block. */
    bb = g->entry->linear_next;
    while (bb) {
        ws->bb_offsets[bb->idx] = ws->bytecode_pos;
        write_instructions(tc, g, ws, bb);
        bb = bb->linear_next;
    }

    /* Fixup labels we were too early for. */
    for (i = 0; i < ws->num_fixups; i++)
        *((MVMuint32 *)(ws->bytecode + ws->fixup_locations[i])) =
            ws->bb_offsets[ws->fixup_bbs[i]->idx];

    /* Ensure all handlers that are reachable got fixed up. */
    for (i = 0; i < g->num_handlers; i++) {
        if (g->unreachable_handlers && g->unreachable_handlers[i]) {
            ws->handlers[i].start_offset = -1;
            ws->handlers[i].end_offset = -1;
            ws->handlers[i].goto_offset = -1;
        }
        else if (ws->handlers[i].start_offset == -1 ||
                 ws->handlers[i].end_offset   == -1 ||
                 ws->handlers[i].goto_offset  == -1) {
            MVM_oops(tc, "Spesh: failed to fix up handler %d in %s (%d, %d, %d)",
                i,
                MVM_string_utf8_maybe_encode_C_string(tc, g->sf->body.name),
                (int)ws->handlers[i].start_offset,
                (int)ws->handlers[i].end_offset,
                (int)ws->handlers[i].goto_offset);
        }
    }

    /* Ensure all inlines got fixed up. */
    for (i = 0; i < g->num_inlines; i++) {
        if (g->inlines[i].unreachable) {
            g->inlines[i].start = -1;
            g->inlines[i].end = -1;
        }
        else {
            if (g->inlines[i].start == -1 || g->inlines[i].end == -1)
                MVM_oops(tc, "Spesh: failed to fix up inline %d %p (%s) %d %d",
                    i,
                    g->inlines[i].g,
                    MVM_string_utf8_maybe_encode_C_string(tc, g->inlines[i].sf->body.name),
                    g->inlines[i].start,
                    g->inlines[i].end
                );
        }
    }

    /* Produce result data structure. */
    res                = MVM_malloc(sizeof(MVMSpeshCode));
    res->bytecode      = ws->bytecode;
    res->bytecode_size = ws->bytecode_pos;
    res->handlers      = ws->handlers;

    /* Cleanup. */
    MVM_free(ws->bb_offsets);
    MVM_free(ws->fixup_locations);
    MVM_free(ws->fixup_bbs);
    MVM_free(ws);

    return res;
}

Coverage Report

Created: 2018-07-03 15:31

Line	Count	Source (jump to first uncovered line)
1		#include "moar.h"
2
3		/* Here we turn a spesh tree back into MoarVM bytecode, after optimizations
4		* have been applied to it. */
5
6		/* Writer state. */
7		typedef struct {
8		/* Bytecode output buffer. */
9		MVMuint8 *bytecode;
10		MVMuint32 bytecode_pos;
11		MVMuint32 bytecode_alloc;
12
13		/* Offsets where basic blocks are. */
14		MVMint32 *bb_offsets;
15
16		/* Fixups we need to do by basic block. */
17		MVMint32 *fixup_locations;
18		MVMSpeshBB **fixup_bbs;
19		MVMint32 num_fixups;
20		MVMint32 alloc_fixups;
21
22		/* Copied frame handlers (which we'll update offsets of). */
23		MVMFrameHandler *handlers;
24		} SpeshWriterState;
25
26		/* Write functions; all native endian. */
27	3.59M	static void ensure_space(SpeshWriterState *ws, int bytes) {
28	3.59M	if (ws->bytecode_pos + bytes >= ws->bytecode_alloc) {
29	3.41k	ws->bytecode_alloc *= 2;
30	3.41k	ws->bytecode = MVM_realloc(ws->bytecode, ws->bytecode_alloc);
31	3.41k	}
32	3.59M	}
33	2.53k	static void write_int64(SpeshWriterState *ws, MVMuint64 value) {
34	2.53k	ensure_space(ws, 8);
35	2.53k	memcpy(ws->bytecode + ws->bytecode_pos, &value, 8);
36	2.53k	ws->bytecode_pos += 8;
37	2.53k	}
38	255k	static void write_int32(SpeshWriterState *ws, MVMuint32 value) {
39	255k	ensure_space(ws, 4);
40	255k	memcpy(ws->bytecode + ws->bytecode_pos, &value, 4);
41	255k	ws->bytecode_pos += 4;
42	255k	}
43	3.33M	static void write_int16(SpeshWriterState *ws, MVMuint16 value) {
44	3.33M	ensure_space(ws, 2);
45	3.33M	memcpy(ws->bytecode + ws->bytecode_pos, &value, 2);
46	3.33M	ws->bytecode_pos += 2;
47	3.33M	}
48	0	static void write_int8(SpeshWriterState *ws, MVMuint8 value) {
49	0	ensure_space(ws, 1);
50	0	memcpy(ws->bytecode + ws->bytecode_pos, &value, 1);
51	0	ws->bytecode_pos++;
52	0	}
53	0	static void write_num32(SpeshWriterState *ws, MVMnum32 value) {
54	0	ensure_space(ws, 4);
55	0	memcpy(ws->bytecode + ws->bytecode_pos, &value, 4);
56	0	ws->bytecode_pos += 4;
57	0	}
58	1.51k	static void write_num64(SpeshWriterState *ws, MVMnum64 value) {
59	1.51k	ensure_space(ws, 8);
60	1.51k	memcpy(ws->bytecode + ws->bytecode_pos, &value, 8);
61	1.51k	ws->bytecode_pos += 8;
62	1.51k	}
63
64		/* Writes instructions within a basic block boundary. */
65	310k	static void write_instructions(MVMThreadContext tc, MVMSpeshGraph g, SpeshWriterState ws, MVMSpeshBB bb) {
66	310k	MVMSpeshIns *ins = bb->first_ins;
67	2.13M	while (ins) {
68	1.82M	MVMint32 i;
69	1.82M
70	1.82M	/* Process any annotations. */
71	1.82M	MVMSpeshAnn *ann = ins->annotations;
72	1.82M	MVMSpeshAnn *deopt_one_ann = NULL;
73	1.82M	MVMSpeshAnn *deopt_all_ann = NULL;
74	1.82M	MVMSpeshAnn *deopt_inline_ann = NULL;
75	2.27M	while (ann) {
76	444k	switch (ann->type) {
77	17.6k	case MVM_SPESH_ANN_FH_START:
78	17.6k	ws->handlers[ann->data.frame_handler_index].start_offset =
79	17.6k	ws->bytecode_pos;
80	17.6k	break;
81	17.6k	case MVM_SPESH_ANN_FH_END:
82	17.6k	ws->handlers[ann->data.frame_handler_index].end_offset =
83	17.6k	ws->bytecode_pos;
84	17.6k	break;
85	17.6k	case MVM_SPESH_ANN_FH_GOTO:
86	17.6k	ws->handlers[ann->data.frame_handler_index].goto_offset =
87	17.6k	ws->bytecode_pos;
88	17.6k	break;
89	219k	case MVM_SPESH_ANN_DEOPT_ONE_INS:
90	219k	deopt_one_ann = ann;
91	219k	break;
92	25.6k	case MVM_SPESH_ANN_DEOPT_ALL_INS:
93	25.6k	deopt_all_ann = ann;
94	25.6k	break;
95	8.50k	case MVM_SPESH_ANN_INLINE_START:
96	8.50k	g->inlines[ann->data.inline_idx].start = ws->bytecode_pos;
97	8.50k	break;
98	8.50k	case MVM_SPESH_ANN_INLINE_END:
99	8.50k	g->inlines[ann->data.inline_idx].end = ws->bytecode_pos;
100	8.50k	break;
101	46.5k	case MVM_SPESH_ANN_DEOPT_INLINE:
102	46.5k	deopt_inline_ann = ann;
103	46.5k	break;
104	4.98k	case MVM_SPESH_ANN_DEOPT_OSR:
105	4.98k	g->deopt_addrs[2 * ann->data.deopt_idx + 1] = ws->bytecode_pos;
106	4.98k	break;
107	444k	}
108	444k	ann = ann->next;
109	444k	}
110	1.82M
111	1.82M	if (ins->info->opcode != MVM_SSA_PHI) {
112	1.13M	/* Real instruction, not a phi. Emit opcode. */
113	1.13M	if (ins->info->opcode == (MVMuint16)-1) {
114	0	/* Ext op; resolve. */
115	0	MVMExtOpRecord *extops = g->sf->body.cu->body.extops;
116	0	MVMuint16 num_extops = g->sf->body.cu->body.num_extops;
117	0	MVMint32 found = 0;
118	0	for (i = 0; i < num_extops; i++) {
119	0	if (extops[i].info == ins->info) {
120	0	write_int16(ws, MVM_OP_EXT_BASE + i);
121	0	found = 1;
122	0	break;
123	0	}
124	0	}
125	0	if (!found)
126	0	MVM_oops(tc, "Spesh: failed to resolve extop in code-gen");
127	0	}
128	1.13M	else {
129	1.13M	/* Core op. */
130	1.13M	write_int16(ws, ins->info->opcode);
131	1.13M	}
132	1.13M
133	1.13M	/* Write out operands. */
134	3.58M	for (i = 0; i < ins->info->num_operands; i++) {
135	2.44M	MVMuint8 flags = ins->info->operands[i];
136	2.44M	MVMuint8 rw = flags & MVM_operand_rw_mask;
137	2.44M	switch (rw) {
138	1.73M	case MVM_operand_read_reg:
139	1.73M	case MVM_operand_write_reg:
140	1.73M	write_int16(ws, ins->operands[i].reg.orig);
141	1.73M	break;
142	13.3k	case MVM_operand_read_lex:
143	13.3k	case MVM_operand_write_lex:
144	13.3k	write_int16(ws, ins->operands[i].lex.idx);
145	13.3k	write_int16(ws, ins->operands[i].lex.outers);
146	13.3k	break;
147	700k	case MVM_operand_literal: {
148	700k	MVMuint8 type = flags & MVM_operand_type_mask;
149	700k	switch (type) {
150	0	case MVM_operand_int8:
151	0	write_int8(ws, ins->operands[i].lit_i8);
152	0	break;
153	288k	case MVM_operand_int16:
154	288k	write_int16(ws, ins->operands[i].lit_i16);
155	288k	break;
156	1.79k	case MVM_operand_int32:
157	1.79k	write_int32(ws, ins->operands[i].lit_i32);
158	1.79k	break;
159	37.8k	case MVM_operand_uint32:
160	37.8k	write_int32(ws, ins->operands[i].lit_ui32);
161	37.8k	break;
162	2.53k	case MVM_operand_int64:
163	2.53k	write_int64(ws, ins->operands[i].lit_i64);
164	2.53k	break;
165	0	case MVM_operand_num32:
166	0	write_num32(ws, ins->operands[i].lit_n32);
167	0	break;
168	1.51k	case MVM_operand_num64:
169	1.51k	write_num64(ws, ins->operands[i].lit_n64);
170	1.51k	break;
171	27.7k	case MVM_operand_callsite:
172	27.7k	write_int16(ws, ins->operands[i].callsite_idx);
173	27.7k	break;
174	496	case MVM_operand_coderef:
175	496	write_int16(ws, ins->operands[i].coderef_idx);
176	496	break;
177	72.8k	case MVM_operand_str:
178	72.8k	write_int32(ws, ins->operands[i].lit_str_idx);
179	72.8k	break;
180	143k	case MVM_operand_ins: {
181	143k	MVMint32 bb_idx = ins->operands[i].ins_bb->idx;
182	143k	MVMint32 offset;
183	143k	if (bb_idx >= g->num_bbs)
184	0	MVM_panic(1, "Spesh codegen: out of range BB index %d", bb_idx);
185	143k	offset = ws->bb_offsets[bb_idx];
186	143k	if (offset >= 0) {
187	14.2k	/* Already know where it is, so just write it. */
188	14.2k	write_int32(ws, offset);
189	14.2k	}
190	129k	else {
191	129k	/* Need to fix it up. */
192	129k	if (ws->num_fixups == ws->alloc_fixups) {
193	706	ws->alloc_fixups *= 2;
194	706	ws->fixup_locations = MVM_realloc(ws->fixup_locations,
195	706	ws->alloc_fixups * sizeof(MVMint32));
196	706	ws->fixup_bbs = MVM_realloc(ws->fixup_bbs,
197	706	ws->alloc_fixups * sizeof(MVMSpeshBB *));
198	706	}
199	129k	ws->fixup_locations[ws->num_fixups] = ws->bytecode_pos;
200	129k	ws->fixup_bbs[ws->num_fixups] = ins->operands[i].ins_bb;
201	129k	write_int32(ws, 0);
202	129k	ws->num_fixups++;
203	129k	}
204	143k	break;
205	0	}
206	123k	case MVM_operand_spesh_slot:
207	123k	write_int16(ws, ins->operands[i].lit_i16);
208	123k	break;
209	0	default:
210	0	MVM_oops(tc,
211	0	"Spesh: unknown operand type %d in codegen (op %s)",
212	0	(int)type, ins->info->name);
213	700k	}
214	700k	}
215	700k	break;
216	0	default:
217	0	MVM_oops(tc, "Spesh: unknown operand type in codegen");
218	2.44M	}
219	2.44M	}
220	1.13M	}
221	1.82M
222	1.82M	/* If there was a deopt point annotation, update table. */
223	1.82M	if (deopt_one_ann)
224	213k	g->deopt_addrs[2 * deopt_one_ann->data.deopt_idx + 1] = ws->bytecode_pos;
225	1.82M	if (deopt_all_ann)
226	25.6k	g->deopt_addrs[2 * deopt_all_ann->data.deopt_idx + 1] = ws->bytecode_pos;
227	1.82M	if (deopt_inline_ann)
228	44.9k	g->deopt_addrs[2 * deopt_inline_ann->data.deopt_idx + 1] = ws->bytecode_pos;
229	1.82M
230	1.82M	ins = ins->next;
231	1.82M	}
232	310k	}
233
234		/* Generate bytecode from a spesh graph. */
235	11.1k	MVMSpeshCode * MVM_spesh_codegen(MVMThreadContext tc, MVMSpeshGraph g) {
236	11.1k	MVMSpeshCode *res;
237	11.1k	MVMSpeshBB *bb;
238	11.1k	MVMint32 i, hanlen;
239	11.1k
240	11.1k	/* Initialize writer state. */
241	11.1k	SpeshWriterState *ws = MVM_malloc(sizeof(SpeshWriterState));
242	11.1k	ws->bytecode_pos = 0;
243	11.1k	ws->bytecode_alloc = 1024;
244	11.1k	ws->bytecode = MVM_malloc(ws->bytecode_alloc);
245	11.1k	ws->bb_offsets = MVM_malloc(g->num_bbs * sizeof(MVMint32));
246	11.1k	ws->num_fixups = 0;
247	11.1k	ws->alloc_fixups = 64;
248	11.1k	ws->fixup_locations = MVM_malloc(ws->alloc_fixups * sizeof(MVMint32));
249	11.1k	ws->fixup_bbs = MVM_malloc(ws->alloc_fixups * sizeof(MVMSpeshBB *));
250	332k	for (i = 0; i < g->num_bbs; i++)
251	321k	ws->bb_offsets[i] = -1;
252	11.1k
253	11.1k	/* Create copy of handlers, and -1 all offsets so we can catch missing
254	11.1k	* updates. */
255	11.1k	hanlen = g->num_handlers * sizeof(MVMFrameHandler);
256	11.1k	if (hanlen) {
257	4.13k	ws->handlers = MVM_malloc(hanlen);
258	4.13k	memcpy(ws->handlers, g->handlers, hanlen);
259	21.7k	for (i = 0; i < g->num_handlers; i++) {
260	17.6k	ws->handlers[i].start_offset = -1;
261	17.6k	ws->handlers[i].end_offset = -1;
262	17.6k	ws->handlers[i].goto_offset = -1;
263	17.6k	}
264	4.13k	}
265	6.96k	else {
266	6.96k	ws->handlers = NULL;
267	6.96k	}
268	11.1k
269	11.1k	/* -1 all the deopt targets, so we'll easily catch those that don't get
270	11.1k	* mapped if we try to use them. Same for inlines. */
271	331k	for (i = 0; i < g->num_deopt_addrs; i++)
272	319k	g->deopt_addrs[i * 2 + 1] = -1;
273	19.6k	for (i = 0; i < g->num_inlines; i++) {
274	8.51k	g->inlines[i].start = -1;
275	8.51k	g->inlines[i].end = -1;
276	8.51k	}
277	11.1k
278	11.1k	/* Write out each of the basic blocks, in linear order. Skip the first,
279	11.1k	* dummy, block. */
280	11.1k	bb = g->entry->linear_next;
281	321k	while (bb) {
282	310k	ws->bb_offsets[bb->idx] = ws->bytecode_pos;
283	310k	write_instructions(tc, g, ws, bb);
284	310k	bb = bb->linear_next;
285	310k	}
286	11.1k
287	11.1k	/* Fixup labels we were too early for. */
288	140k	for (i = 0; i < ws->num_fixups; i++)
289	129k	((MVMuint32 )(ws->bytecode + ws->fixup_locations[i])) =
290	129k	ws->bb_offsets[ws->fixup_bbs[i]->idx];
291	11.1k
292	11.1k	/* Ensure all handlers that are reachable got fixed up. */
293	28.7k	for (i = 0; i < g->num_handlers; i++) {
294	17.6k	if (g->unreachable_handlers && g->unreachable_handlers[i]) {
295	0	ws->handlers[i].start_offset = -1;
296	0	ws->handlers[i].end_offset = -1;
297	0	ws->handlers[i].goto_offset = -1;
298	0	}
299	17.6k	else if (ws->handlers[i].start_offset == -1 \|\|
300	17.6k	ws->handlers[i].end_offset == -1 \|\|
301	17.6k	ws->handlers[i].goto_offset == -1) {
302	0	MVM_oops(tc, "Spesh: failed to fix up handler %d in %s (%d, %d, %d)",
303	0	i,
304	0	MVM_string_utf8_maybe_encode_C_string(tc, g->sf->body.name),
305	0	(int)ws->handlers[i].start_offset,
306	0	(int)ws->handlers[i].end_offset,
307	0	(int)ws->handlers[i].goto_offset);
308	0	}
309	17.6k	}
310	11.1k
311	11.1k	/* Ensure all inlines got fixed up. */
312	19.6k	for (i = 0; i < g->num_inlines; i++) {
313	8.50k	if (g->inlines[i].unreachable) {
314	0	g->inlines[i].start = -1;
315	0	g->inlines[i].end = -1;
316	0	}
317	8.50k	else {
318	8.50k	if (g->inlines[i].start == -1 \|\| g->inlines[i].end == -1)
319	0	MVM_oops(tc, "Spesh: failed to fix up inline %d %p (%s) %d %d",
320	0	i,
321	0	g->inlines[i].g,
322	0	MVM_string_utf8_maybe_encode_C_string(tc, g->inlines[i].sf->body.name),
323	0	g->inlines[i].start,
324	0	g->inlines[i].end
325	0	);
326	8.50k	}
327	8.50k	}
328	11.1k
329	11.1k	/* Produce result data structure. */
330	11.1k	res = MVM_malloc(sizeof(MVMSpeshCode));
331	11.1k	res->bytecode = ws->bytecode;
332	11.1k	res->bytecode_size = ws->bytecode_pos;
333	11.1k	res->handlers = ws->handlers;
334	11.1k
335	11.1k	/* Cleanup. */
336	11.1k	MVM_free(ws->bb_offsets);
337	11.1k	MVM_free(ws->fixup_locations);
338	11.1k	MVM_free(ws->fixup_bbs);
339	11.1k	MVM_free(ws);
340	11.1k
341	11.1k	return res;
342	11.1k	}