#include "moar.h"

#include "math.h"

#include "platform/sys.h"

#include "platform/time.h"

static void jg_append_node(MVMJitGraph *jg, MVMJitNode *node) {

    if (jg->last_node) {

        jg->last_node->next = node;

        jg->last_node = node;

    } else {

        jg->first_node = node;

        jg->last_node = node;

    }

    node->next = NULL;

}

static void jg_append_primitive(MVMThreadContext *tc, MVMJitGraph *jg,

                                MVMSpeshIns * ins) {

    MVMJitNode * node = MVM_spesh_alloc(tc, jg->sg, sizeof(MVMJitNode));

    node->type = MVM_JIT_NODE_PRIMITIVE;

    node->u.prim.ins = ins;

    jg_append_node(jg, node);

}

static void jg_append_call_c(MVMThreadContext *tc, MVMJitGraph *jg,

                              void * func_ptr, MVMint16 num_args,

                              MVMJitCallArg *call_args,

                              MVMJitRVMode rv_mode, MVMint16 rv_idx) {

    MVMJitNode * node = MVM_spesh_alloc(tc, jg->sg, sizeof(MVMJitNode));

    size_t args_size =  num_args * sizeof(MVMJitCallArg);

    node->type             = MVM_JIT_NODE_CALL_C;

    node->u.call.func_ptr  = func_ptr;

    node->u.call.num_args  = num_args;

    node->u.call.has_vargs = 0; /* don't support them yet */

    /* Call argument array is typically stack allocated,

     * so they need to be copied */

    node->u.call.args      = MVM_spesh_alloc(tc, jg->sg, args_size);

    memcpy(node->u.call.args, call_args, args_size);

    node->u.call.rv_mode   = rv_mode;

    node->u.call.rv_idx    = rv_idx;

    jg_append_node(jg, node);

}

static void add_deopt_idx(MVMThreadContext *tc, MVMJitGraph *jg, MVMint32 label_name, MVMint32 deopt_idx) {

    MVMJitDeopt deopt;

    deopt.label = label_name;

    deopt.idx   = deopt_idx;

    MVM_VECTOR_PUSH(jg->deopts, deopt);

}

static void jg_append_branch(MVMThreadContext *tc, MVMJitGraph *jg,

                              MVMint32 name, MVMSpeshIns *ins) {

    MVMJitNode * node = MVM_spesh_alloc(tc, jg->sg, sizeof(MVMJitNode));

    node->type = MVM_JIT_NODE_BRANCH;

    if (ins == NULL) {

        node->u.branch.ins = NULL;

        node->u.branch.dest = name;

    }

    else {

        MVMSpeshBB *bb;

        node->u.branch.ins = ins;

        if (ins->info->opcode == MVM_OP_goto) {

            bb = ins->operands[0].ins_bb;

        }

        else if (ins->info->opcode == MVM_OP_indexat ||

                 ins->info->opcode == MVM_OP_indexnat) {

            bb = ins->operands[3].ins_bb;

        }

        else {

            bb = ins->operands[1].ins_bb;

        }

        node->u.branch.dest = MVM_jit_label_before_bb(tc, jg, bb);

    }

    jg_append_node(jg, node);

}

static void jg_append_label(MVMThreadContext *tc, MVMJitGraph *jg, MVMint32 name) {

    MVMJitNode *node;

    /* does this label already exist? */

    MVM_VECTOR_ENSURE_SIZE(jg->label_nodes, name);

    if (jg->label_nodes[name] != NULL)

        return;

    node = MVM_spesh_alloc(tc, jg->sg, sizeof(MVMJitNode));

    node->type = MVM_JIT_NODE_LABEL;

    node->u.label.name = name;

    jg_append_node(jg, node);

    jg->label_nodes[name] = node;

    MVM_jit_log(tc, "append label: %d\n", node->u.label.name);

}

static void * op_to_func(MVMThreadContext *tc, MVMint16 opcode) {

    switch(opcode) {

    case MVM_OP_checkarity: return MVM_args_checkarity;

    case MVM_OP_say: return MVM_string_say;

    case MVM_OP_print: return MVM_string_print;

    case MVM_OP_isnull: return MVM_is_null;

    case MVM_OP_capturelex: return MVM_frame_capturelex;

    case MVM_OP_captureinnerlex: return MVM_frame_capture_inner;

    case MVM_OP_takeclosure: return MVM_frame_takeclosure;

    case MVM_OP_usecapture: return MVM_args_use_capture;

    case MVM_OP_savecapture: return MVM_args_save_capture;

    case MVM_OP_captureposprimspec: return MVM_capture_pos_primspec;

    case MVM_OP_return: return MVM_args_assert_void_return_ok;

    case MVM_OP_return_i: return MVM_args_set_result_int;

    case MVM_OP_return_s: return MVM_args_set_result_str;

    case MVM_OP_return_o: return MVM_args_set_result_obj;

    case MVM_OP_return_n: return MVM_args_set_result_num;

    case MVM_OP_coerce_is: return MVM_coerce_i_s;

    case MVM_OP_coerce_ns: return MVM_coerce_n_s;

    case MVM_OP_coerce_si: return MVM_coerce_s_i;

    case MVM_OP_coerce_sn: return MVM_coerce_s_n;

    case MVM_OP_coerce_In: return MVM_bigint_to_num;

    case MVM_OP_coerce_nI: return MVM_bigint_from_num;

    case MVM_OP_coerce_II: return MVM_bigint_from_bigint;

    case MVM_OP_iterkey_s: return MVM_iterkey_s;

    case MVM_OP_iter: return MVM_iter;

    case MVM_OP_iterval: return MVM_iterval;

    case MVM_OP_die: return MVM_exception_die;

    case MVM_OP_throwdyn:

    case MVM_OP_throwlex:

    case MVM_OP_throwlexotic:

    case MVM_OP_rethrow: return MVM_exception_throwobj;

    case MVM_OP_throwcatdyn:

    case MVM_OP_throwcatlex:

    case MVM_OP_throwcatlexotic: return MVM_exception_throwcat;

    case MVM_OP_throwpayloadlex: return MVM_exception_throwpayload;

    case MVM_OP_bindexpayload: return MVM_bind_exception_payload;

    case MVM_OP_getexpayload: return MVM_get_exception_payload;

    case MVM_OP_resume: return MVM_exception_resume;

    case MVM_OP_continuationreset: return MVM_continuation_reset;

    case MVM_OP_continuationcontrol: return MVM_continuation_control;

    case MVM_OP_continuationinvoke: return MVM_continuation_invoke;

    case MVM_OP_smrt_numify: return MVM_coerce_smart_numify;

    case MVM_OP_smrt_strify: return MVM_coerce_smart_stringify;

    case MVM_OP_gethow: return MVM_6model_get_how_obj;

    case MVM_OP_box_i: return MVM_box_int;

    case MVM_OP_box_s: return MVM_box_str;

    case MVM_OP_box_n: return MVM_box_num;

    case MVM_OP_unbox_i: return MVM_repr_get_int;

    case MVM_OP_unbox_s: return MVM_repr_get_str;

    case MVM_OP_unbox_n: return MVM_repr_get_num;

    case MVM_OP_istrue: case MVM_OP_isfalse: return MVM_coerce_istrue;

    case MVM_OP_istype: return MVM_6model_istype;

    case MVM_OP_isint: case MVM_OP_isnum: case MVM_OP_isstr: /* continued */

    case MVM_OP_islist: case MVM_OP_ishash: return MVM_repr_compare_repr_id;

    case MVM_OP_wval: case MVM_OP_wval_wide: return MVM_sc_get_sc_object;

    case MVM_OP_scgetobjidx: return MVM_sc_find_object_idx_jit;

    case MVM_OP_getdynlex: return MVM_frame_getdynlex;

    case MVM_OP_binddynlex: return MVM_frame_binddynlex;

    case MVM_OP_getlexouter: return MVM_frame_find_lexical_by_name_outer;

    case MVM_OP_getlexcaller: return MVM_frame_find_lexical_by_name_rel_caller;

    case MVM_OP_findmeth: case MVM_OP_findmeth_s: return MVM_6model_find_method;

    case MVM_OP_tryfindmeth: case MVM_OP_tryfindmeth_s: return MVM_6model_find_method;

    case MVM_OP_multicacheadd: return MVM_multi_cache_add;

    case MVM_OP_multicachefind: return MVM_multi_cache_find;

    case MVM_OP_can: case MVM_OP_can_s: return MVM_6model_can_method;

    case MVM_OP_push_i: return MVM_repr_push_i;

    case MVM_OP_push_n: return MVM_repr_push_n;

    case MVM_OP_push_s: return MVM_repr_push_s;

    case MVM_OP_push_o: return MVM_repr_push_o;

    case MVM_OP_unshift_i: return MVM_repr_unshift_i;

    case MVM_OP_unshift_n: return MVM_repr_unshift_n;

    case MVM_OP_unshift_s: return MVM_repr_unshift_s;

    case MVM_OP_unshift_o: return MVM_repr_unshift_o;

    case MVM_OP_pop_i: return MVM_repr_pop_i;

    case MVM_OP_pop_n: return MVM_repr_pop_n;

    case MVM_OP_pop_s: return MVM_repr_pop_s;

    case MVM_OP_pop_o: return MVM_repr_pop_o;

    case MVM_OP_shift_i: return MVM_repr_shift_i;

    case MVM_OP_shift_n: return MVM_repr_shift_n;

    case MVM_OP_shift_s: return MVM_repr_shift_s;

    case MVM_OP_shift_o: return MVM_repr_shift_o;

    case MVM_OP_setelemspos: return MVM_repr_pos_set_elems;

    case MVM_OP_slice:  return MVM_repr_pos_slice;

    case MVM_OP_splice: return MVM_repr_pos_splice;

    case MVM_OP_existskey: return MVM_repr_exists_key;

    case MVM_OP_deletekey: return MVM_repr_delete_key;

    case MVM_OP_atpos_i: return MVM_repr_at_pos_i;

    case MVM_OP_atpos_n: return MVM_repr_at_pos_n;

    case MVM_OP_atpos_s: return MVM_repr_at_pos_s;

    case MVM_OP_atpos_o: return MVM_repr_at_pos_o;

    case MVM_OP_existspos: return MVM_repr_exists_pos;

    case MVM_OP_atkey_i: return MVM_repr_at_key_i;

    case MVM_OP_atkey_n: return MVM_repr_at_key_n;

    case MVM_OP_atkey_s: return MVM_repr_at_key_s;

    case MVM_OP_atkey_o: return MVM_repr_at_key_o;

    case MVM_OP_bindpos_i: return MVM_repr_bind_pos_i;

    case MVM_OP_bindpos_n: return MVM_repr_bind_pos_n;

    case MVM_OP_bindpos_s: return MVM_repr_bind_pos_s;

    case MVM_OP_bindpos_o: return MVM_repr_bind_pos_o;

    case MVM_OP_bindkey_i: return MVM_repr_bind_key_i;

    case MVM_OP_bindkey_n: return MVM_repr_bind_key_n;

    case MVM_OP_bindkey_s: return MVM_repr_bind_key_s;

    case MVM_OP_bindkey_o: return MVM_repr_bind_key_o;

    case MVM_OP_getattr_s: return MVM_repr_get_attr_s;

    case MVM_OP_getattr_n: return MVM_repr_get_attr_n;

    case MVM_OP_getattr_i: return MVM_repr_get_attr_i;

    case MVM_OP_getattr_o: return MVM_repr_get_attr_o;

    case MVM_OP_getattrs_s: return MVM_repr_get_attr_s;

    case MVM_OP_getattrs_n: return MVM_repr_get_attr_n;

    case MVM_OP_getattrs_i: return MVM_repr_get_attr_i;

    case MVM_OP_getattrs_o: return MVM_repr_get_attr_o;

    case MVM_OP_attrinited: return MVM_repr_attribute_inited;

    case MVM_OP_bindattr_i: case MVM_OP_bindattr_n: case MVM_OP_bindattr_s: case MVM_OP_bindattr_o: return MVM_repr_bind_attr_inso;

    case MVM_OP_bindattrs_i: case MVM_OP_bindattrs_n: case MVM_OP_bindattrs_s: case MVM_OP_bindattrs_o: return MVM_repr_bind_attr_inso;

    case MVM_OP_hintfor: return MVM_repr_hint_for;

    case MVM_OP_gt_s: case MVM_OP_ge_s: case MVM_OP_lt_s: case MVM_OP_le_s: case MVM_OP_cmp_s: return MVM_string_compare;

    case MVM_OP_queuepoll: return MVM_concblockingqueue_jit_poll;

    case MVM_OP_open_fh: return MVM_file_open_fh;

    case MVM_OP_close_fh: return MVM_io_close;

    case MVM_OP_eof_fh: return MVM_io_eof;

    case MVM_OP_write_fhb: return MVM_io_write_bytes;

    case MVM_OP_read_fhb: return MVM_io_read_bytes;

    case MVM_OP_encode: return MVM_string_encode_to_buf;

    case MVM_OP_decoderaddbytes: return MVM_decoder_add_bytes;

    case MVM_OP_decodertakeline: return MVM_decoder_take_line;

    case MVM_OP_elems: return MVM_repr_elems;

    case MVM_OP_concat_s: return MVM_string_concatenate;

    case MVM_OP_repeat_s: return MVM_string_repeat;

    case MVM_OP_flip: return MVM_string_flip;

    case MVM_OP_split: return MVM_string_split;

    case MVM_OP_escape: return MVM_string_escape;

    case MVM_OP_uc: return MVM_string_uc;

    case MVM_OP_tc: return MVM_string_tc;

    case MVM_OP_lc: return MVM_string_lc;

    case MVM_OP_fc: return MVM_string_fc;

    case MVM_OP_eq_s: return MVM_string_equal;

    case MVM_OP_eqat_s: return MVM_string_equal_at;

    case MVM_OP_eqatic_s: return MVM_string_equal_at_ignore_case;

    case MVM_OP_eqatim_s: return MVM_string_equal_at_ignore_mark;

    case MVM_OP_eqaticim_s: return MVM_string_equal_at_ignore_case_ignore_mark;

    case MVM_OP_chars: case MVM_OP_graphs_s: return MVM_string_graphs;

    case MVM_OP_chr: return MVM_string_chr;

    case MVM_OP_codes_s: return MVM_string_codes;

    case MVM_OP_getcp_s: return MVM_string_get_grapheme_at;

    case MVM_OP_index_s: return MVM_string_index;

    case MVM_OP_substr_s: return MVM_string_substring;

    case MVM_OP_join: return MVM_string_join;

    case MVM_OP_replace: return MVM_string_replace;

    case MVM_OP_iscclass: return MVM_string_is_cclass;

    case MVM_OP_findcclass: return MVM_string_find_cclass;

    case MVM_OP_findnotcclass: return MVM_string_find_not_cclass;

    case MVM_OP_nfarunalt: return MVM_nfa_run_alt;

    case MVM_OP_nfarunproto: return MVM_nfa_run_proto;

    case MVM_OP_nfafromstatelist: return MVM_nfa_from_statelist;

    case MVM_OP_hllize: return MVM_hll_map;

    case MVM_OP_gethllsym: return MVM_hll_sym_get;

    case MVM_OP_clone: return MVM_repr_clone;

    case MVM_OP_create: return MVM_repr_alloc_init;

    case MVM_OP_getcodeobj: return MVM_frame_get_code_object;

    case MVM_OP_isbig_I: return MVM_bigint_is_big;

    case MVM_OP_cmp_I: return MVM_bigint_cmp;

    case MVM_OP_add_I: return MVM_bigint_add;

    case MVM_OP_sub_I: return MVM_bigint_sub;

    case MVM_OP_mul_I: return MVM_bigint_mul;

    case MVM_OP_div_I: return MVM_bigint_div;

    case MVM_OP_bor_I: return MVM_bigint_or;

    case MVM_OP_band_I: return MVM_bigint_and;

    case MVM_OP_bxor_I: return MVM_bigint_xor;

    case MVM_OP_mod_I: return MVM_bigint_mod;

    case MVM_OP_lcm_I: return MVM_bigint_lcm;

    case MVM_OP_gcd_I: return MVM_bigint_gcd;

    case MVM_OP_bool_I: return MVM_bigint_bool;

    case MVM_OP_isprime_I: return MVM_bigint_is_prime;

    case MVM_OP_brshift_I: return MVM_bigint_shr;

    case MVM_OP_blshift_I: return MVM_bigint_shl;

    case MVM_OP_bnot_I: return MVM_bigint_not;

    case MVM_OP_div_In: return MVM_bigint_div_num;

    case MVM_OP_coerce_Is: case MVM_OP_base_I: return MVM_bigint_to_str;

    case MVM_OP_radix: return MVM_radix;

    case MVM_OP_radix_I: return MVM_bigint_radix;

    case MVM_OP_sqrt_n: return sqrt;

    case MVM_OP_sin_n: return sin;

    case MVM_OP_cos_n: return cos;

    case MVM_OP_tan_n: return tan;

    case MVM_OP_asin_n: return asin;

    case MVM_OP_acos_n: return acos;

    case MVM_OP_atan_n: return atan;

    case MVM_OP_atan2_n: return atan2;

    case MVM_OP_floor_n: return floor;

    case MVM_OP_pow_I: return MVM_bigint_pow;

    case MVM_OP_rand_I: return MVM_bigint_rand;

    case MVM_OP_pow_n: return pow;

    case MVM_OP_time_n: return MVM_proc_time_n;

    case MVM_OP_randscale_n: return MVM_proc_randscale_n;

    case MVM_OP_isnanorinf: return MVM_num_isnanorinf;

    case MVM_OP_nativecallcast: return MVM_nativecall_cast;

    case MVM_OP_nativecallinvoke: return MVM_nativecall_invoke;

    case MVM_OP_nativeinvoke_o: return MVM_nativecall_invoke_jit;

    case MVM_OP_typeparameterized: return MVM_6model_parametric_type_parameterized;

    case MVM_OP_typeparameters: return MVM_6model_parametric_type_parameters;

    case MVM_OP_typeparameterat: return MVM_6model_parametric_type_parameter_at;

    case MVM_OP_objectid: return MVM_gc_object_id;

    case MVM_OP_iscont_i: return MVM_6model_container_iscont_i;

    case MVM_OP_iscont_n: return MVM_6model_container_iscont_n;

    case MVM_OP_iscont_s: return MVM_6model_container_iscont_s;

    case MVM_OP_isrwcont: return MVM_6model_container_iscont_rw;

    case MVM_OP_assign_i: return MVM_6model_container_assign_i;

    case MVM_OP_assign_n: return MVM_6model_container_assign_n;

    case MVM_OP_assign_s: return MVM_6model_container_assign_s;

    case MVM_OP_decont_i: return MVM_6model_container_decont_i;

    case MVM_OP_decont_n: return MVM_6model_container_decont_n;

    case MVM_OP_decont_s: return MVM_6model_container_decont_s;

    case MVM_OP_getrusage: return MVM_proc_getrusage;

    case MVM_OP_cpucores: return MVM_platform_cpu_count;

    case MVM_OP_getsignals: return MVM_io_get_signals;

    case MVM_OP_sleep: return MVM_platform_sleep;

    case MVM_OP_getlexref_i32: case MVM_OP_getlexref_i16: case MVM_OP_getlexref_i8: case MVM_OP_getlexref_i: return MVM_nativeref_lex_i;

    case MVM_OP_getlexref_n32: case MVM_OP_getlexref_n: return MVM_nativeref_lex_n;

    case MVM_OP_getlexref_s: return MVM_nativeref_lex_s;

    case MVM_OP_getattrref_i: return MVM_nativeref_attr_i;

    case MVM_OP_getattrref_n: return MVM_nativeref_attr_n;

    case MVM_OP_getattrref_s: return MVM_nativeref_attr_s;

    case MVM_OP_getattrsref_i: return MVM_nativeref_attr_i;

    case MVM_OP_getattrsref_n: return MVM_nativeref_attr_n;

    case MVM_OP_getattrsref_s: return MVM_nativeref_attr_s;

    case MVM_OP_atposref_i: return MVM_nativeref_pos_i;

    case MVM_OP_atposref_n: return MVM_nativeref_pos_n;

    case MVM_OP_atposref_s: return MVM_nativeref_pos_s;

    case MVM_OP_indexingoptimized: return MVM_string_indexing_optimized;

    case MVM_OP_sp_boolify_iter: return MVM_iter_istrue;

    case MVM_OP_prof_allocated: return MVM_profile_log_allocated;

    case MVM_OP_prof_exit: return MVM_profile_log_exit;

    case MVM_OP_sp_resolvecode: return MVM_frame_resolve_invokee_spesh;

    case MVM_OP_cas_o: return MVM_6model_container_cas;

    case MVM_OP_cas_i: return MVM_6model_container_cas_i;

    case MVM_OP_atomicinc_i: return MVM_6model_container_atomic_inc;

    case MVM_OP_atomicdec_i: return MVM_6model_container_atomic_dec;

    case MVM_OP_atomicadd_i: return MVM_6model_container_atomic_add;

    case MVM_OP_atomicload_o: return MVM_6model_container_atomic_load;

    case MVM_OP_atomicload_i: return MVM_6model_container_atomic_load_i;

    case MVM_OP_atomicstore_o: return MVM_6model_container_atomic_store;

    case MVM_OP_atomicstore_i: return MVM_6model_container_atomic_store_i;

    case MVM_OP_lock: return MVM_reentrantmutex_lock_checked;

    case MVM_OP_unlock: return MVM_reentrantmutex_unlock_checked;

    case MVM_OP_getexcategory: return MVM_get_exception_category;

    case MVM_OP_bindexcategory: return MVM_bind_exception_category;

    case MVM_OP_exreturnafterunwind: return MVM_exception_returnafterunwind;

    case MVM_OP_breakpoint: return MVM_debugserver_breakpoint_check;

    case MVM_OP_sp_getstringfrom: return MVM_cu_string;

    case MVM_OP_encoderepconf: return MVM_string_encode_to_buf_config;

    default:

        MVM_oops(tc, "JIT: No function for op %d in op_to_func (%s)", opcode, MVM_op_get_op(opcode)->name);

    }

}

static void jg_append_guard(MVMThreadContext *tc, MVMJitGraph *jg,

                             MVMSpeshIns *ins, MVMuint32 target_operand) {

    MVMSpeshAnn   *ann = ins->annotations;

    MVMJitNode   *node = MVM_spesh_alloc(tc, jg->sg, sizeof(MVMJitNode));

    MVMint32 deopt_idx;

    node->type = MVM_JIT_NODE_GUARD;

    node->u.guard.ins = ins;

    while (ann) {

        if (ann->type == MVM_SPESH_ANN_DEOPT_ONE_INS ||

            ann->type == MVM_SPESH_ANN_DEOPT_INLINE) {

            deopt_idx = ann->data.deopt_idx;

            break;

        }

        ann = ann->next;

    }

    if (!ann) {

        MVM_oops(tc, "Can't find deopt idx annotation on spesh ins <%s>",

            ins->info->name);

    }

    node->u.guard.deopt_target = ins->operands[target_operand].lit_ui32;

    node->u.guard.deopt_offset = jg->sg->deopt_addrs[2 * deopt_idx + 1];

    jg_append_node(jg, node);

}

static MVMint32 consume_invoke(MVMThreadContext *tc, MVMJitGraph *jg,

                               MVMSpeshIterator *iter, MVMSpeshIns *ins) {

    MVMCompUnit       *cu = iter->graph->sf->body.cu;

    MVMint16 callsite_idx = ins->operands[0].callsite_idx;

    MVMCallsite       *cs = cu->body.callsites[callsite_idx];

    MVMSpeshIns **arg_ins = MVM_spesh_alloc(tc, iter->graph, sizeof(MVMSpeshIns*) * cs->arg_count);

    MVMint16            i = 0;

    MVMJitNode      *node;

    MVMint32      reentry_label;

    MVMReturnType return_type;

    MVMint16      return_register;

    MVMint16      code_register_or_name;

    MVMint16      spesh_cand_or_sf_slot;

    MVMint16      is_fast;

    MVMint16      is_resolve = 0;

    MVMuint32     resolve_offset = 0;

    while ((ins = ins->next)) {

        switch(ins->info->opcode) {

        case MVM_OP_arg_i:

        case MVM_OP_arg_n:

        case MVM_OP_arg_s:

        case MVM_OP_arg_o:

        case MVM_OP_argconst_i:

        case MVM_OP_argconst_n:

        case MVM_OP_argconst_s:

            MVM_jit_log(tc, "Invoke arg: <%s>\n", ins->info->name);

            arg_ins[i++] = ins;

            break;

        case MVM_OP_invoke_v:

            return_type           = MVM_RETURN_VOID;

            return_register       = -1;

            code_register_or_name = ins->operands[0].reg.orig;

            spesh_cand_or_sf_slot = -1;

            is_fast               = 0;

            goto checkargs;

        case MVM_OP_invoke_i:

            return_type           = MVM_RETURN_INT;

            return_register       = ins->operands[0].reg.orig;

            code_register_or_name = ins->operands[1].reg.orig;

            spesh_cand_or_sf_slot = -1;

            is_fast               = 0;

            goto checkargs;

        case MVM_OP_invoke_n:

            return_type           = MVM_RETURN_NUM;

            return_register       = ins->operands[0].reg.orig;

            code_register_or_name = ins->operands[1].reg.orig;

            spesh_cand_or_sf_slot = -1;

            is_fast               = 0;

            goto checkargs;

        case MVM_OP_invoke_s:

            return_type           = MVM_RETURN_STR;

            return_register       = ins->operands[0].reg.orig;

            code_register_or_name = ins->operands[1].reg.orig;

            spesh_cand_or_sf_slot = -1;

            is_fast               = 0;

            goto checkargs;

        case MVM_OP_invoke_o:

            return_type           = MVM_RETURN_OBJ;

            return_register       = ins->operands[0].reg.orig;

            code_register_or_name = ins->operands[1].reg.orig;

            spesh_cand_or_sf_slot = -1;

            is_fast               = 0;

            goto checkargs;

        case MVM_OP_nativeinvoke_o: {

            MVMint16 dst     = ins->operands[0].reg.orig;

            MVMint16 site    = ins->operands[1].reg.orig;

            MVMint16 restype = ins->operands[2].reg.orig;

            MVMNativeCallBody *body;

            MVMJitGraph *nc_jg;

            MVMObject *nc_site;

            MVMSpeshFacts *object_facts = MVM_spesh_get_facts(tc, iter->graph, ins->operands[1]);

            MVM_jit_log(tc, "Invoke instruction: <%s>\n", ins->info->name);

            if (!(object_facts->flags & MVM_SPESH_FACT_KNOWN_VALUE)) {

                MVM_jit_log(tc, "BAIL: op <%s> (Can't find nc_site value on spesh ins)\n", ins->info->name);

                return 0;

            }

            body = MVM_nativecall_get_nc_body(tc, object_facts->value.o);

            nc_jg = MVM_nativecall_jit_graph_for_caller_code(tc, iter->graph, body, restype, dst, arg_ins);

            if (nc_jg == NULL)

                return 0;

            jg->last_node->next = nc_jg->first_node;

            jg->last_node = nc_jg->last_node;

            goto success;

        }

        case MVM_OP_sp_fastinvoke_v:

            return_type           = MVM_RETURN_VOID;

            return_register       = -1;

            code_register_or_name = ins->operands[0].reg.orig;

            spesh_cand_or_sf_slot = ins->operands[1].lit_i16;

            is_fast               = 1;

            goto checkargs;

        case MVM_OP_sp_fastinvoke_o:

            return_type           = MVM_RETURN_OBJ;

            return_register       = ins->operands[0].reg.orig;;

            code_register_or_name = ins->operands[1].reg.orig;

            spesh_cand_or_sf_slot = ins->operands[2].lit_i16;

            is_fast               = 1;

            goto checkargs;

        case MVM_OP_sp_fastinvoke_s:

            return_type           = MVM_RETURN_STR;

            return_register       = ins->operands[0].reg.orig;;

            code_register_or_name = ins->operands[1].reg.orig;

            spesh_cand_or_sf_slot = ins->operands[2].lit_i16;

            is_fast               = 1;

            goto checkargs;

        case MVM_OP_sp_fastinvoke_i:

            return_type           = MVM_RETURN_INT;

            return_register       = ins->operands[0].reg.orig;;

            code_register_or_name = ins->operands[1].reg.orig;

            spesh_cand_or_sf_slot = ins->operands[2].lit_i16;

            is_fast               = 1;

            goto checkargs;

        case MVM_OP_sp_fastinvoke_n:

            return_type           = MVM_RETURN_NUM;

            return_register       = ins->operands[0].reg.orig;;

            code_register_or_name = ins->operands[1].reg.orig;

            spesh_cand_or_sf_slot = ins->operands[2].lit_i16;

            is_fast               = 1;

            goto checkargs;

        case MVM_OP_sp_speshresolve:

            return_type           = MVM_RETURN_OBJ;

            return_register       = ins->operands[0].reg.orig;;

            code_register_or_name = ins->operands[1].lit_ui32;

            resolve_offset        = ins->operands[2].lit_ui32;

            spesh_cand_or_sf_slot = ins->operands[3].lit_i16;

            is_fast               = 0;

            is_resolve            = 1;

            goto checkargs;

        default:

            MVM_jit_log(tc, "Unexpected opcode in invoke sequence: <%s>\n",

                        ins->info->name);

            return 0;

        }

    }

 checkargs:

    if (!ins || i < cs->arg_count) {

        MVM_jit_log(tc, "Could not find invoke opcode or enough arguments\n"

                    "BAIL: op <%s>, expected args: %d, num of args: %d\n",

                    ins? ins->info->name : "NULL", i, cs->arg_count);

        return 0;

    }

    MVM_jit_log(tc, "Invoke instruction: <%s>\n", ins->info->name);

    /* get label /after/ current (invoke) ins, where we'll need to reenter the JIT */

    reentry_label = MVM_jit_label_after_ins(tc, jg, iter->bb, ins);

    /* create invoke node */

    node = MVM_spesh_alloc(tc, jg->sg, sizeof(MVMJitNode));

    node->type                           = MVM_JIT_NODE_INVOKE;

    node->u.invoke.callsite_idx          = callsite_idx;

    node->u.invoke.arg_count             = cs->arg_count;

    node->u.invoke.arg_ins               = arg_ins;

    node->u.invoke.return_type           = return_type;

    node->u.invoke.return_register       = return_register;

    node->u.invoke.code_register_or_name = code_register_or_name;

    node->u.invoke.spesh_cand_or_sf_slot = spesh_cand_or_sf_slot;

    node->u.invoke.resolve_offset        = resolve_offset;

    node->u.invoke.reentry_label         = reentry_label;

    node->u.invoke.is_fast               = is_fast;

    node->u.invoke.is_resolve            = is_resolve;

    jg_append_node(jg, node);

    /* append reentry label */

    jg_append_label(tc, jg, reentry_label);

  success:

    /* move forward to invoke ins */

    iter->ins = ins;

    return 1;

}

static void jg_append_control(MVMThreadContext *tc, MVMJitGraph *jg,

                              MVMSpeshIns *ins, MVMJitControlType ctrl) {

    MVMJitNode *node = MVM_spesh_alloc(tc, jg->sg, sizeof(MVMJitNode));

    node->type = MVM_JIT_NODE_CONTROL;

    node->u.control.ins  = ins;

    node->u.control.type = ctrl;

    jg_append_node(jg, node);

}

static MVMint32 consume_jumplist(MVMThreadContext *tc, MVMJitGraph *jg,

                                 MVMSpeshIterator *iter, MVMSpeshIns *ins) {

    MVMint64 num_labels  = ins->operands[0].lit_i64;

    MVMint16 idx_reg     = ins->operands[1].reg.orig;

    MVMint32 *in_labels  = MVM_spesh_alloc(tc, jg->sg, sizeof(MVMint32) * num_labels);

    MVMint32 *out_labels = MVM_spesh_alloc(tc, jg->sg, sizeof(MVMint32) * num_labels);

    MVMSpeshBB *bb       = iter->bb;

    MVMJitNode *node;

    MVMint64 i;

    for (i = 0; i < num_labels; i++) {

        bb = bb->linear_next; /* take the next basic block */

        if (!bb || bb->first_ins != bb->last_ins) return 0; /*  which must exist */

        ins = bb->first_ins;  /*  and it's first and only entry */

        if (ins->info->opcode != MVM_OP_goto)  /* which must be a goto */

            return 0;

        in_labels[i]  = MVM_jit_label_before_bb(tc, jg, bb);

        out_labels[i] = MVM_jit_label_before_bb(tc, jg, ins->operands[0].ins_bb);

    }

    /* build the node */

    node = MVM_spesh_alloc(tc, jg->sg, sizeof(MVMJitNode));

    node->type = MVM_JIT_NODE_JUMPLIST;

    node->u.jumplist.num_labels = num_labels;

    node->u.jumplist.reg = idx_reg;

    node->u.jumplist.in_labels = in_labels;

    node->u.jumplist.out_labels = out_labels;

    jg_append_node(jg, node);

    /* set iterator bb and ins to the end of our jumplist */

    iter->bb = bb;

    iter->ins = ins;

    return 1;

}

static MVMString* wrap_MVM_cu_string(MVMThreadContext *tc, MVMCompUnit *cu, MVMuint32 idx) {

    return MVM_cu_string(tc, cu, idx);

}

static MVMint32 jg_add_data_node(MVMThreadContext *tc, MVMJitGraph *jg, void *data, size_t size) {

    MVMJitNode *node = MVM_spesh_alloc(tc, jg->sg, sizeof(MVMJitNode));

    MVMint32 label   = MVM_jit_label_for_obj(tc, jg, data);

    node->type         = MVM_JIT_NODE_DATA;

    node->u.data.data  = data;

    node->u.data.size  = size;

    node->u.data.label = label;

    jg_append_node(jg, node);

    return label;

}

static MVMuint16 * try_fake_extop_regs(MVMThreadContext *tc, MVMSpeshGraph *sg, MVMSpeshIns *ins, size_t *bufsize) {

    MVMuint16 *regs = MVM_spesh_alloc(tc, sg, (*bufsize = (ins->info->num_operands * sizeof(MVMuint16))));

    MVMuint16 i;

    for (i = 0; i < ins->info->num_operands; i++) {

        switch (ins->info->operands[i] & MVM_operand_rw_mask) {

        case MVM_operand_read_reg:

        case MVM_operand_write_reg:

            regs[i] = ins->operands[i].reg.orig;

            break;

        default:

            MVM_free(regs);

            return NULL;

        }

    }

    return regs;

}

static void log_inline(MVMThreadContext *tc, MVMSpeshGraph *sg, MVMint32 inline_idx, MVMint32 is_entry) {

    MVMStaticFrame *sf = sg->inlines[inline_idx].sf;

    char *name         = MVM_string_utf8_encode_C_string(tc, sf->body.name);

    char *cuuid        = MVM_string_utf8_encode_C_string(tc, sf->body.cuuid);

    MVM_jit_log(tc, "%s inline %d (name: %s, cuuid: %s)\n", is_entry ? "Entering" : "Leaving",

                inline_idx, name, cuuid);

    MVM_free(name);

    MVM_free(cuuid);

}

static void before_ins(MVMThreadContext *tc, MVMJitGraph *jg,

                       MVMSpeshIterator *iter, MVMSpeshIns  *ins) {

    MVMSpeshBB   *bb = iter->bb;

    MVMSpeshAnn *ann = ins->annotations;

    MVMint32 has_label = 0, label;

    /* Search annotations for stuff that may need a label. */

    while (ann) {

        switch(ann->type) {

        case MVM_SPESH_ANN_DEOPT_OSR: {

            /* get label before our instruction */

            label = MVM_jit_label_before_ins(tc, jg, bb, ins);

            add_deopt_idx(tc, jg, label, ann->data.deopt_idx);

            has_label = 1;

            break;

        }

        case MVM_SPESH_ANN_FH_START: {

            label = MVM_jit_label_before_ins(tc, jg, bb, ins);

            jg->handlers[ann->data.frame_handler_index].start_label = label;

            has_label = 1;

            /* Load the current position into the jit entry label, so that

             * when throwing we'll know which handler to use */

            break;

        }

        case MVM_SPESH_ANN_FH_END: {

            label = MVM_jit_label_before_ins(tc, jg, bb, ins);

            jg->handlers[ann->data.frame_handler_index].end_label = label;

            /* Same as above. Note that the dynamic label control

             * actually loads a position a few bytes away from the

             * label appended above. This is in this case intentional

             * because the frame handler end is exclusive; once it is

             * passed we should not use the same handler again.  If we

             * loaded the exact same position, we would not be able to

             * distinguish between the end of the basic block to which

             * the handler applies and the start of the basic block to

             * which it doesn't. */

            has_label = 1;

            break;

        }

        case MVM_SPESH_ANN_FH_GOTO: {

            label = MVM_jit_label_before_ins(tc, jg, bb, ins);

            jg->handlers[ann->data.frame_handler_index].goto_label = label;

            has_label = 1;

            break;

        }

        case MVM_SPESH_ANN_INLINE_START: {

            label = MVM_jit_label_before_ins(tc, jg, bb, ins);

            jg->inlines[ann->data.inline_idx].start_label = label;

            if (tc->instance->jit_log_fh)

                log_inline(tc, jg->sg, ann->data.inline_idx, 1);

            has_label = 1;

            break;

        }

        } /* switch */

        ann = ann->next;

    }

    if (has_label) {

        jg_append_label(tc, jg, label);

    }

}

static void after_ins(MVMThreadContext *tc, MVMJitGraph *jg,

                      MVMSpeshIterator *iter, MVMSpeshIns *ins) {

    MVMSpeshBB   *bb = iter->bb;

    MVMSpeshAnn *ann = ins->annotations;

    /* This order of processing is necessary to ensure that a label

     * calculated by one of the control guards as well as the labels

     * calculated below point to the exact same instruction. This is a

     * relatively fragile construction! One could argue that the

     * control guards should in fact use the same (dynamic) labels. */

    while (ann) {

        if (ann->type == MVM_SPESH_ANN_INLINE_END) {

            MVMint32 label = MVM_jit_label_after_ins(tc, jg, bb, ins);

            jg_append_label(tc, jg, label);

            jg->inlines[ann->data.inline_idx].end_label = label;

            if (tc->instance->jit_log_fh)

                log_inline(tc, jg->sg, ann->data.inline_idx, 0);

        } else if (ann->type == MVM_SPESH_ANN_DEOPT_ALL_INS) {

            /* An underlying assumption here is that this instruction

             * will in fact set the jit_entry_label to a correct

             * value. This is clearly true for invoking ops as well

             * as invokish ops, and in fact there is no other way

             * to get a deopt_all_ins annotation. Still, be warned. */

            MVMint32 label = MVM_jit_label_after_ins(tc, jg, bb, ins);

            jg_append_label(tc, jg, label);

            add_deopt_idx(tc, jg, label, ann->data.deopt_idx);

        }

        ann = ann->next;

    }

}

static void jg_sc_wb(MVMThreadContext *tc, MVMJitGraph *jg, MVMSpeshOperand check) {

    MVMJitCallArg args[] = { { MVM_JIT_INTERP_VAR,  MVM_JIT_INTERP_TC },

                             { MVM_JIT_REG_VAL,     check.reg.orig } };

    jg_append_call_c(tc, jg, &MVM_SC_WB_OBJ, 2, args, MVM_JIT_RV_VOID, -1);

}

static MVMint32 consume_reprop(MVMThreadContext *tc, MVMJitGraph *jg,

                               MVMSpeshIterator *iterator, MVMSpeshIns *ins) {

    MVMint16 op = ins->info->opcode;

    MVMSpeshOperand type_operand;

    MVMSpeshFacts *type_facts = 0;

    MVMint32 alternative = 0;

    switch (op) {

        case MVM_OP_unshift_i:

        case MVM_OP_unshift_n:

        case MVM_OP_unshift_s:

        case MVM_OP_unshift_o:

        case MVM_OP_bindkey_i:

        case MVM_OP_bindkey_n:

        case MVM_OP_bindkey_s:

        case MVM_OP_bindkey_o:

        case MVM_OP_bindpos_i:

        case MVM_OP_bindpos_n:

        case MVM_OP_bindpos_s:

        case MVM_OP_bindpos_o:

        case MVM_OP_bindattr_i:

        case MVM_OP_bindattr_n:

        case MVM_OP_bindattr_s:

        case MVM_OP_bindattr_o:

        case MVM_OP_bindattrs_i:

        case MVM_OP_bindattrs_n:

        case MVM_OP_bindattrs_s:

        case MVM_OP_bindattrs_o:

        case MVM_OP_push_i:

        case MVM_OP_push_n:

        case MVM_OP_push_s:

        case MVM_OP_push_o:

        case MVM_OP_deletekey:

        case MVM_OP_setelemspos:

        case MVM_OP_splice:

            type_operand = ins->operands[0];

            break;

        case MVM_OP_atpos_i:

        case MVM_OP_atpos_n:

        case MVM_OP_atpos_s:

        case MVM_OP_atpos_o:

        case MVM_OP_atkey_i:

        case MVM_OP_atkey_n:

        case MVM_OP_atkey_s:

        case MVM_OP_atkey_o:

        case MVM_OP_elems:

        case MVM_OP_shift_i:

        case MVM_OP_shift_n:

        case MVM_OP_shift_s:

        case MVM_OP_shift_o:

        case MVM_OP_pop_i:

        case MVM_OP_pop_n:

        case MVM_OP_pop_s:

        case MVM_OP_pop_o:

        case MVM_OP_existskey:

        case MVM_OP_existspos:

        case MVM_OP_getattr_i:

        case MVM_OP_getattr_n:

        case MVM_OP_getattr_s:

        case MVM_OP_getattr_o:

        case MVM_OP_getattrs_i:

        case MVM_OP_getattrs_n:

        case MVM_OP_getattrs_s:

        case MVM_OP_getattrs_o:

        case MVM_OP_attrinited:

        case MVM_OP_hintfor:

        case MVM_OP_slice:

            type_operand = ins->operands[1];

            break;

        case MVM_OP_box_i:

        case MVM_OP_box_n:

        case MVM_OP_box_s:

            type_operand = ins->operands[2];

            break;

        default:

            MVM_jit_log(tc, "devirt: couldn't figure out type operand for op %s\n", ins->info->name);

            return 0;

    }

    type_facts = MVM_spesh_get_facts(tc, jg->sg, type_operand);

    if (type_facts && type_facts->flags & MVM_SPESH_FACT_KNOWN_TYPE && type_facts->type &&

            type_facts->flags & MVM_SPESH_FACT_CONCRETE) {

        switch(op) {

            case MVM_OP_atkey_i:

            case MVM_OP_atkey_n:

            case MVM_OP_atkey_s:

            case MVM_OP_atkey_o:

                alternative = 1;

            case MVM_OP_atpos_i:

            case MVM_OP_atpos_n:

            case MVM_OP_atpos_s:

            case MVM_OP_atpos_o: {

                /* atpos_i             w(int64) r(obj) r(int64) */

                /* atkey_i             w(int64) r(obj) r(str)*/

                /*void (*at_pos) (MVMThreadContext *tc, MVMSTable *st,

                 *    MVMObject *root, void *data, MVMint64 index,

                 *    MVMRegister *result, MVMuint16 kind);*/

                /*REPR(obj)->pos_funcs.at_pos(tc, STABLE(obj), obj, OBJECT_BODY(obj),

                 *  idx, &value, MVM_reg_int64);*/

                MVMint32 dst      = ins->operands[0].reg.orig;

                MVMint32 invocant = ins->operands[1].reg.orig;

                MVMint32 value    = ins->operands[2].reg.orig;

                void *function = alternative

                    ? (void *)((MVMObject*)type_facts->type)->st->REPR->ass_funcs.at_key

                    : (void *)((MVMObject*)type_facts->type)->st->REPR->pos_funcs.at_pos;

                MVMJitCallArg args[] = { { MVM_JIT_INTERP_VAR,  MVM_JIT_INTERP_TC },

                                         { MVM_JIT_REG_STABLE,  invocant },

                                         { MVM_JIT_REG_VAL,     invocant },

                                         { MVM_JIT_REG_OBJBODY, invocant },

                                         { MVM_JIT_REG_VAL,  value },

                                         { MVM_JIT_REG_ADDR, dst },

                                         { MVM_JIT_LITERAL,

                                             op == MVM_OP_atpos_i || op == MVM_OP_atkey_i ? MVM_reg_int64 :

                                             op == MVM_OP_atpos_n || op == MVM_OP_atkey_n ? MVM_reg_num64 :

                                             op == MVM_OP_atpos_s || op == MVM_OP_atkey_s ? MVM_reg_str :