/* ** Trace recorder (bytecode -> SSA IR). ** Copyright (C) 2005-2022 Mike Pall. See Copyright Notice in luajit.h */ #define lj_record_c #define LUA_CORE #include "lj_obj.h" #if LJ_HASJIT #include "lj_err.h" #include "lj_str.h" #include "lj_tab.h" #include "lj_meta.h" #include "lj_frame.h" #if LJ_HASFFI #include "lj_ctype.h" #endif #include "lj_bc.h" #include "lj_ff.h" #if LJ_HASPROFILE #include "lj_debug.h" #endif #include "lj_ir.h" #include "lj_jit.h" #include "lj_ircall.h" #include "lj_iropt.h" #include "lj_trace.h" #include "lj_record.h" #include "lj_ffrecord.h" #include "lj_snap.h" #include "lj_dispatch.h" #include "lj_vm.h" #include "lj_prng.h" /* Some local macros to save typing. Undef'd at the end. */ #define IR(ref) (&J->cur.ir[(ref)]) /* Pass IR on to next optimization in chain (FOLD). */ #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J)) /* Emit raw IR without passing through optimizations. */ #define emitir_raw(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_ir_emit(J)) /* -- Sanity checks ------------------------------------------------------- */ #ifdef LUA_USE_ASSERT /* Sanity check the whole IR -- sloooow. */ static void rec_check_ir(jit_State *J) { IRRef i, nins = J->cur.nins, nk = J->cur.nk; lj_assertJ(nk <= REF_BIAS && nins >= REF_BIAS && nins < 65536, "inconsistent IR layout"); for (i = nk; i < nins; i++) { IRIns *ir = IR(i); uint32_t mode = lj_ir_mode[ir->o]; IRRef op1 = ir->op1; IRRef op2 = ir->op2; const char *err = NULL; switch (irm_op1(mode)) { case IRMnone: if (op1 != 0) err = "IRMnone op1 used"; break; case IRMref: if (op1 < nk || (i >= REF_BIAS ? op1 >= i : op1 <= i)) err = "IRMref op1 out of range"; break; case IRMlit: break; case IRMcst: if (i >= REF_BIAS) { err = "constant in IR range"; break; } if (irt_is64(ir->t) && ir->o != IR_KNULL) i++; continue; } switch (irm_op2(mode)) { case IRMnone: if (op2) err = "IRMnone op2 used"; break; case IRMref: if (op2 < nk || (i >= REF_BIAS ? op2 >= i : op2 <= i)) err = "IRMref op2 out of range"; break; case IRMlit: break; case IRMcst: err = "IRMcst op2"; break; } if (!err && ir->prev) { if (ir->prev < nk || (i >= REF_BIAS ? ir->prev >= i : ir->prev <= i)) err = "chain out of range"; else if (ir->o != IR_NOP && IR(ir->prev)->o != ir->o) err = "chain to different op"; } lj_assertJ(!err, "bad IR %04d op %d(%04d,%04d): %s", i-REF_BIAS, ir->o, irm_op1(mode) == IRMref ? op1-REF_BIAS : op1, irm_op2(mode) == IRMref ? op2-REF_BIAS : op2, err); } } /* Compare stack slots and frames of the recorder and the VM. */ static void rec_check_slots(jit_State *J) { BCReg s, nslots = J->baseslot + J->maxslot; int32_t depth = 0; cTValue *base = J->L->base - J->baseslot; lj_assertJ(J->baseslot >= 1+LJ_FR2, "bad baseslot"); lj_assertJ(J->baseslot == 1+LJ_FR2 || (J->slot[J->baseslot-1] & TREF_FRAME), "baseslot does not point to frame"); lj_assertJ(nslots <= LJ_MAX_JSLOTS, "slot overflow"); for (s = 0; s < nslots; s++) { TRef tr = J->slot[s]; if (tr) { cTValue *tv = &base[s]; IRRef ref = tref_ref(tr); IRIns *ir = NULL; /* Silence compiler. */ if (!LJ_FR2 || ref || !(tr & (TREF_FRAME | TREF_CONT))) { lj_assertJ(ref >= J->cur.nk && ref < J->cur.nins, "slot %d ref %04d out of range", s, ref - REF_BIAS); ir = IR(ref); lj_assertJ(irt_t(ir->t) == tref_t(tr), "slot %d IR type mismatch", s); } if (s == 0) { lj_assertJ(tref_isfunc(tr), "frame slot 0 is not a function"); #if LJ_FR2 } else if (s == 1) { lj_assertJ((tr & ~TREF_FRAME) == 0, "bad frame slot 1"); #endif } else if ((tr & TREF_FRAME)) { GCfunc *fn = gco2func(frame_gc(tv)); BCReg delta = (BCReg)(tv - frame_prev(tv)); #if LJ_FR2 lj_assertJ(!ref || ir_knum(ir)->u64 == tv->u64, "frame slot %d PC mismatch", s); tr = J->slot[s-1]; ir = IR(tref_ref(tr)); #endif lj_assertJ(tref_isfunc(tr), "frame slot %d is not a function", s-LJ_FR2); lj_assertJ(!tref_isk(tr) || fn == ir_kfunc(ir), "frame slot %d function mismatch", s-LJ_FR2); lj_assertJ(s > delta + LJ_FR2 ? (J->slot[s-delta] & TREF_FRAME) : (s == delta + LJ_FR2), "frame slot %d broken chain", s-LJ_FR2); depth++; } else if ((tr & TREF_CONT)) { #if LJ_FR2 lj_assertJ(!ref || ir_knum(ir)->u64 == tv->u64, "cont slot %d continuation mismatch", s); #else lj_assertJ(ir_kptr(ir) == gcrefp(tv->gcr, void), "cont slot %d continuation mismatch", s); #endif lj_assertJ((J->slot[s+1+LJ_FR2] & TREF_FRAME), "cont slot %d not followed by frame", s); depth++; } else if ((tr & TREF_KEYINDEX)) { lj_assertJ(tref_isint(tr), "keyindex slot %d bad type %d", s, tref_type(tr)); } else { /* Number repr. may differ, but other types must be the same. */ lj_assertJ(tvisnumber(tv) ? tref_isnumber(tr) : itype2irt(tv) == tref_type(tr), "slot %d type mismatch: stack type %d vs IR type %d", s, itypemap(tv), tref_type(tr)); if (tref_isk(tr)) { /* Compare constants. */ TValue tvk; lj_ir_kvalue(J->L, &tvk, ir); lj_assertJ((tvisnum(&tvk) && tvisnan(&tvk)) ? (tvisnum(tv) && tvisnan(tv)) : lj_obj_equal(tv, &tvk), "slot %d const mismatch: stack %016llx vs IR %016llx", s, tv->u64, tvk.u64); } } } } lj_assertJ(J->framedepth == depth, "frame depth mismatch %d vs %d", J->framedepth, depth); } #endif /* -- Type handling and specialization ------------------------------------ */ /* Note: these functions return tagged references (TRef). */ /* Specialize a slot to a specific type. Note: slot can be negative! */ static TRef sloadt(jit_State *J, int32_t slot, IRType t, int mode) { /* Caller may set IRT_GUARD in t. */ TRef ref = emitir_raw(IRT(IR_SLOAD, t), (int32_t)J->baseslot+slot, mode); J->base[slot] = ref; return ref; } /* Specialize a slot to the runtime type. Note: slot can be negative! */ static TRef sload(jit_State *J, int32_t slot) { IRType t = itype2irt(&J->L->base[slot]); TRef ref = emitir_raw(IRTG(IR_SLOAD, t), (int32_t)J->baseslot+slot, IRSLOAD_TYPECHECK); if (irtype_ispri(t)) ref = TREF_PRI(t); /* Canonicalize primitive refs. */ J->base[slot] = ref; return ref; } /* Get TRef from slot. Load slot and specialize if not done already. */ #define getslot(J, s) (J->base[(s)] ? J->base[(s)] : sload(J, (int32_t)(s))) /* Get TRef for current function. */ static TRef getcurrf(jit_State *J) { if (J->base[-1-LJ_FR2]) return J->base[-1-LJ_FR2]; /* Non-base frame functions ought to be loaded already. */ lj_assertJ(J->baseslot == 1+LJ_FR2, "bad baseslot"); return sloadt(J, -1-LJ_FR2, IRT_FUNC, IRSLOAD_READONLY); } /* Compare for raw object equality. ** Returns 0 if the objects are the same. ** Returns 1 if they are different, but the same type. ** Returns 2 for two different types. ** Comparisons between primitives always return 1 -- no caller cares about it. */ int lj_record_objcmp(jit_State *J, TRef a, TRef b, cTValue *av, cTValue *bv) { int diff = !lj_obj_equal(av, bv); if (!tref_isk2(a, b)) { /* Shortcut, also handles primitives. */ IRType ta = tref_isinteger(a) ? IRT_INT : tref_type(a); IRType tb = tref_isinteger(b) ? IRT_INT : tref_type(b); if (ta != tb) { /* Widen mixed number/int comparisons to number/number comparison. */ if (ta == IRT_INT && tb == IRT_NUM) { a = emitir(IRTN(IR_CONV), a, IRCONV_NUM_INT); ta = IRT_NUM; } else if (ta == IRT_NUM && tb == IRT_INT) { b = emitir(IRTN(IR_CONV), b, IRCONV_NUM_INT); } else { return 2; /* Two different types are never equal. */ } } emitir(IRTG(diff ? IR_NE : IR_EQ, ta), a, b); } return diff; } /* Constify a value. Returns 0 for non-representable object types. */ TRef lj_record_constify(jit_State *J, cTValue *o) { if (tvisgcv(o)) return lj_ir_kgc(J, gcV(o), itype2irt(o)); else if (tvisint(o)) return lj_ir_kint(J, intV(o)); else if (tvisnum(o)) return lj_ir_knumint(J, numV(o)); else if (tvisbool(o)) return TREF_PRI(itype2irt(o)); else return 0; /* Can't represent lightuserdata (pointless). */ } /* Emit a VLOAD with the correct type. */ TRef lj_record_vload(jit_State *J, TRef ref, MSize idx, IRType t) { TRef tr = emitir(IRTG(IR_VLOAD, t), ref, idx); if (irtype_ispri(t)) tr = TREF_PRI(t); /* Canonicalize primitives. */ return tr; } /* -- Record loop ops ----------------------------------------------------- */ /* Loop event. */ typedef enum { LOOPEV_LEAVE, /* Loop is left or not entered. */ LOOPEV_ENTERLO, /* Loop is entered with a low iteration count left. */ LOOPEV_ENTER /* Loop is entered. */ } LoopEvent; /* Canonicalize slots: convert integers to numbers. */ static void canonicalize_slots(jit_State *J) { BCReg s; if (LJ_DUALNUM) return; for (s = J->baseslot+J->maxslot-1; s >= 1; s--) { TRef tr = J->slot[s]; if (tref_isinteger(tr) && !(tr & TREF_KEYINDEX)) { IRIns *ir = IR(tref_ref(tr)); if (!(ir->o == IR_SLOAD && (ir->op2 & (IRSLOAD_READONLY)))) J->slot[s] = emitir(IRTN(IR_CONV), tr, IRCONV_NUM_INT); } } } /* Stop recording. */ void lj_record_stop(jit_State *J, TraceLink linktype, TraceNo lnk) { #ifdef LUAJIT_ENABLE_TABLE_BUMP if (J->retryrec) lj_trace_err(J, LJ_TRERR_RETRY); #endif lj_trace_end(J); J->cur.linktype = (uint8_t)linktype; J->cur.link = (uint16_t)lnk; /* Looping back at the same stack level? */ if (lnk == J->cur.traceno && J->framedepth + J->retdepth == 0) { if ((J->flags & JIT_F_OPT_LOOP)) /* Shall we try to create a loop? */ goto nocanon; /* Do not canonicalize or we lose the narrowing. */ if (J->cur.root) /* Otherwise ensure we always link to the root trace. */ J->cur.link = J->cur.root; } canonicalize_slots(J); nocanon: /* Note: all loop ops must set J->pc to the following instruction! */ lj_snap_add(J); /* Add loop snapshot. */ J->needsnap = 0; J->mergesnap = 1; /* In case recording continues. */ } /* Search bytecode backwards for a int/num constant slot initializer. */ static TRef find_kinit(jit_State *J, const BCIns *endpc, BCReg slot, IRType t) { /* This algorithm is rather simplistic and assumes quite a bit about ** how the bytecode is generated. It works fine for FORI initializers, ** but it won't necessarily work in other cases (e.g. iterator arguments). ** It doesn't do anything fancy, either (like backpropagating MOVs). */ const BCIns *pc, *startpc = proto_bc(J->pt); for (pc = endpc-1; pc > startpc; pc--) { BCIns ins = *pc; BCOp op = bc_op(ins); /* First try to find the last instruction that stores to this slot. */ if (bcmode_a(op) == BCMbase && bc_a(ins) <= slot) { return 0; /* Multiple results, e.g. from a CALL or KNIL. */ } else if (bcmode_a(op) == BCMdst && bc_a(ins) == slot) { if (op == BC_KSHORT || op == BC_KNUM) { /* Found const. initializer. */ /* Now try to verify there's no forward jump across it. */ const BCIns *kpc = pc; for (; pc > startpc; pc--) if (bc_op(*pc) == BC_JMP) { const BCIns *target = pc+bc_j(*pc)+1; if (target > kpc && target <= endpc) return 0; /* Conditional assignment. */ } if (op == BC_KSHORT) { int32_t k = (int32_t)(int16_t)bc_d(ins); return t == IRT_INT ? lj_ir_kint(J, k) : lj_ir_knum(J, (lua_Number)k); } else { cTValue *tv = proto_knumtv(J->pt, bc_d(ins)); if (t == IRT_INT) { int32_t k = numberVint(tv); if (tvisint(tv) || numV(tv) == (lua_Number)k) /* -0 is ok here. */ return lj_ir_kint(J, k); return 0; /* Type mismatch. */ } else { return lj_ir_knum(J, numberVnum(tv)); } } } return 0; /* Non-constant initializer. */ } } return 0; /* No assignment to this slot found? */ } /* Load and optionally convert a FORI argument from a slot. */ static TRef fori_load(jit_State *J, BCReg slot, IRType t, int mode) { int conv = (tvisint(&J->L->base[slot]) != (t==IRT_INT)) ? IRSLOAD_CONVERT : 0; return sloadt(J, (int32_t)slot, t + (((mode & IRSLOAD_TYPECHECK) || (conv && t == IRT_INT && !(mode >> 16))) ? IRT_GUARD : 0), mode + conv); } /* Peek before FORI to find a const initializer. Otherwise load from slot. */ static TRef fori_arg(jit_State *J, const BCIns *fori, BCReg slot, IRType t, int mode) { TRef tr = J->base[slot]; if (!tr) { tr = find_kinit(J, fori, slot, t); if (!tr) tr = fori_load(J, slot, t, mode); } return tr; } /* Return the direction of the FOR loop iterator. ** It's important to exactly reproduce the semantics of the interpreter. */ static int rec_for_direction(cTValue *o) { return (tvisint(o) ? intV(o) : (int32_t)o->u32.hi) >= 0; } /* Simulate the runtime behavior of the FOR loop iterator. */ static LoopEvent rec_for_iter(IROp *op, cTValue *o, int isforl) { lua_Number stopv = numberVnum(&o[FORL_STOP]); lua_Number idxv = numberVnum(&o[FORL_IDX]); lua_Number stepv = numberVnum(&o[FORL_STEP]); if (isforl) idxv += stepv; if (rec_for_direction(&o[FORL_STEP])) { if (idxv <= stopv) { *op = IR_LE; return idxv + 2*stepv > stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER; } *op = IR_GT; return LOOPEV_LEAVE; } else { if (stopv <= idxv) { *op = IR_GE; return idxv + 2*stepv < stopv ? LOOPEV_ENTERLO : LOOPEV_ENTER; } *op = IR_LT; return LOOPEV_LEAVE; } } /* Record checks for FOR loop overflow and step direction. */ static void rec_for_check(jit_State *J, IRType t, int dir, TRef stop, TRef step, int init) { if (!tref_isk(step)) { /* Non-constant step: need a guard for the direction. */ TRef zero = (t == IRT_INT) ? lj_ir_kint(J, 0) : lj_ir_knum_zero(J); emitir(IRTG(dir ? IR_GE : IR_LT, t), step, zero); /* Add hoistable overflow checks for a narrowed FORL index. */ if (init && t == IRT_INT) { if (tref_isk(stop)) { /* Constant stop: optimize check away or to a range check for step. */ int32_t k = IR(tref_ref(stop))->i; if (dir) { if (k > 0) emitir(IRTGI(IR_LE), step, lj_ir_kint(J, (int32_t)0x7fffffff-k)); } else { if (k < 0) emitir(IRTGI(IR_GE), step, lj_ir_kint(J, (int32_t)0x80000000-k)); } } else { /* Stop+step variable: need full overflow check. */ TRef tr = emitir(IRTGI(IR_ADDOV), step, stop); emitir(IRTI(IR_USE), tr, 0); /* ADDOV is weak. Avoid dead result. */ } } } else if (init && t == IRT_INT && !tref_isk(stop)) { /* Constant step: optimize overflow check to a range check for stop. */ int32_t k = IR(tref_ref(step))->i; k = (int32_t)(dir ? 0x7fffffff : 0x80000000) - k; emitir(IRTGI(dir ? IR_LE : IR_GE), stop, lj_ir_kint(J, k)); } } /* Record a FORL instruction. */ static void rec_for_loop(jit_State *J, const BCIns *fori, ScEvEntry *scev, int init) { BCReg ra = bc_a(*fori); cTValue *tv = &J->L->base[ra]; TRef idx = J->base[ra+FORL_IDX]; IRType t = idx ? tref_type(idx) : (init || LJ_DUALNUM) ? lj_opt_narrow_forl(J, tv) : IRT_NUM; int mode = IRSLOAD_INHERIT + ((!LJ_DUALNUM || tvisint(tv) == (t == IRT_INT)) ? IRSLOAD_READONLY : 0); TRef stop = fori_arg(J, fori, ra+FORL_STOP, t, mode); TRef step = fori_arg(J, fori, ra+FORL_STEP, t, mode); int tc, dir = rec_for_direction(&tv[FORL_STEP]); lj_assertJ(bc_op(*fori) == BC_FORI || bc_op(*fori) == BC_JFORI, "bad bytecode %d instead of FORI/JFORI", bc_op(*fori)); scev->t.irt = t; scev->dir = dir; scev->stop = tref_ref(stop); scev->step = tref_ref(step); rec_for_check(J, t, dir, stop, step, init); scev->start = tref_ref(find_kinit(J, fori, ra+FORL_IDX, IRT_INT)); tc = (LJ_DUALNUM && !(scev->start && irref_isk(scev->stop) && irref_isk(scev->step) && tvisint(&tv[FORL_IDX]) == (t == IRT_INT))) ? IRSLOAD_TYPECHECK : 0; if (tc) { J->base[ra+FORL_STOP] = stop; J->base[ra+FORL_STEP] = step; } if (!idx) idx = fori_load(J, ra+FORL_IDX, t, IRSLOAD_INHERIT + tc + (J->scev.start << 16)); if (!init) J->base[ra+FORL_IDX] = idx = emitir(IRT(IR_ADD, t), idx, step); J->base[ra+FORL_EXT] = idx; scev->idx = tref_ref(idx); setmref(scev->pc, fori); J->maxslot = ra+FORL_EXT+1; } /* Record FORL/JFORL or FORI/JFORI. */ static LoopEvent rec_for(jit_State *J, const BCIns *fori, int isforl) { BCReg ra = bc_a(*fori); TValue *tv = &J->L->base[ra]; TRef *tr = &J->base[ra]; IROp op; LoopEvent ev; TRef stop; IRType t; if (isforl) { /* Handle FORL/JFORL opcodes. */ TRef idx = tr[FORL_IDX]; if (mref(J->scev.pc, const BCIns) == fori && tref_ref(idx) == J->scev.idx) { t = J->scev.t.irt; stop = J->scev.stop; idx = emitir(IRT(IR_ADD, t), idx, J->scev.step); tr[FORL_EXT] = tr[FORL_IDX] = idx; } else { ScEvEntry scev; rec_for_loop(J, fori, &scev, 0); t = scev.t.irt; stop = scev.stop; } } else { /* Handle FORI/JFORI opcodes. */ BCReg i; lj_meta_for(J->L, tv); t = (LJ_DUALNUM || tref_isint(tr[FORL_IDX])) ? lj_opt_narrow_forl(J, tv) : IRT_NUM; for (i = FORL_IDX; i <= FORL_STEP; i++) { if (!tr[i]) sload(J, ra+i); lj_assertJ(tref_isnumber_str(tr[i]), "bad FORI argument type"); if (tref_isstr(tr[i])) tr[i] = emitir(IRTG(IR_STRTO, IRT_NUM), tr[i], 0); if (t == IRT_INT) { if (!tref_isinteger(tr[i])) tr[i] = emitir(IRTGI(IR_CONV), tr[i], IRCONV_INT_NUM|IRCONV_CHECK); } else { if (!tref_isnum(tr[i])) tr[i] = emitir(IRTN(IR_CONV), tr[i], IRCONV_NUM_INT); } } tr[FORL_EXT] = tr[FORL_IDX]; stop = tr[FORL_STOP]; rec_for_check(J, t, rec_for_direction(&tv[FORL_STEP]), stop, tr[FORL_STEP], 1); } ev = rec_for_iter(&op, tv, isforl); if (ev == LOOPEV_LEAVE) { J->maxslot = ra+FORL_EXT+1; J->pc = fori+1; } else { J->maxslot = ra; J->pc = fori+bc_j(*fori)+1; } lj_snap_add(J); emitir(IRTG(op, t), tr[FORL_IDX], stop); if (ev == LOOPEV_LEAVE) { J->maxslot = ra; J->pc = fori+bc_j(*fori)+1; } else { J->maxslot = ra+FORL_EXT+1; J->pc = fori+1; } J->needsnap = 1; return ev; } /* Record ITERL/JITERL. */ static LoopEvent rec_iterl(jit_State *J, const BCIns iterins) { BCReg ra = bc_a(iterins); if (!tref_isnil(getslot(J, ra))) { /* Looping back? */ J->base[ra-1] = J->base[ra]; /* Copy result of ITERC to control var. */ J->maxslot = ra-1+bc_b(J->pc[-1]); J->pc += bc_j(iterins)+1; return LOOPEV_ENTER; } else { J->maxslot = ra-3; J->pc++; return LOOPEV_LEAVE; } } /* Record LOOP/JLOOP. Now, that was easy. */ static LoopEvent rec_loop(jit_State *J, BCReg ra, int skip) { if (ra < J->maxslot) J->maxslot = ra; J->pc += skip; return LOOPEV_ENTER; } /* Check if a loop repeatedly failed to trace because it didn't loop back. */ static int innerloopleft(jit_State *J, const BCIns *pc) { ptrdiff_t i; for (i = 0; i < PENALTY_SLOTS; i++) if (mref(J->penalty[i].pc, const BCIns) == pc) { if ((J->penalty[i].reason == LJ_TRERR_LLEAVE || J->penalty[i].reason == LJ_TRERR_LINNER) && J->penalty[i].val >= 2*PENALTY_MIN) return 1; break; } return 0; } /* Handle the case when an interpreted loop op is hit. */ static void rec_loop_interp(jit_State *J, const BCIns *pc, LoopEvent ev) { if (J->parent == 0 && J->exitno == 0) { if (pc == J->startpc && J->framedepth + J->retdepth == 0) { if (bc_op(J->cur.startins) == BC_ITERN) return; /* See rec_itern(). */ /* Same loop? */ if (ev == LOOPEV_LEAVE) /* Must loop back to form a root trace. */ lj_trace_err(J, LJ_TRERR_LLEAVE); lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Looping trace. */ } else if (ev != LOOPEV_LEAVE) { /* Entering inner loop? */ /* It's usually better to abort here and wait until the inner loop ** is traced. But if the inner loop repeatedly didn't loop back, ** this indicates a low trip count. In this case try unrolling ** an inner loop even in a root trace. But it's better to be a bit ** more conservative here and only do it for very short loops. */ if (bc_j(*pc) != -1 && !innerloopleft(J, pc)) lj_trace_err(J, LJ_TRERR_LINNER); /* Root trace hit an inner loop. */ if ((ev != LOOPEV_ENTERLO && J->loopref && J->cur.nins - J->loopref > 24) || --J->loopunroll < 0) lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */ J->loopref = J->cur.nins; } } else if (ev != LOOPEV_LEAVE) { /* Side trace enters an inner loop. */ J->loopref = J->cur.nins; if (--J->loopunroll < 0) lj_trace_err(J, LJ_TRERR_LUNROLL); /* Limit loop unrolling. */ } /* Side trace continues across a loop that's left or not entered. */ } /* Handle the case when an already compiled loop op is hit. */ static void rec_loop_jit(jit_State *J, TraceNo lnk, LoopEvent ev) { if (J->parent == 0 && J->exitno == 0) { /* Root trace hit an inner loop. */ /* Better let the inner loop spawn a side trace back here. */ lj_trace_err(J, LJ_TRERR_LINNER); } else if (ev != LOOPEV_LEAVE) { /* Side trace enters a compiled loop. */ J->instunroll = 0; /* Cannot continue across a compiled loop op. */ if (J->pc == J->startpc && J->framedepth + J->retdepth == 0) lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Form extra loop. */ else lj_record_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the loop. */ } /* Side trace continues across a loop that's left or not entered. */ } /* Record ITERN. */ static LoopEvent rec_itern(jit_State *J, BCReg ra, BCReg rb) { #if LJ_BE /* YAGNI: Disabled on big-endian due to issues with lj_vm_next, ** IR_HIOP, RID_RETLO/RID_RETHI and ra_destpair. */ UNUSED(ra); UNUSED(rb); setintV(&J->errinfo, (int32_t)BC_ITERN); lj_trace_err_info(J, LJ_TRERR_NYIBC); #else RecordIndex ix; /* Since ITERN is recorded at the start, we need our own loop detection. */ if (J->pc == J->startpc && J->framedepth + J->retdepth == 0 && J->parent == 0 && J->exitno == 0) { IRRef ref = REF_FIRST + LJ_HASPROFILE; #ifdef LUAJIT_ENABLE_CHECKHOOK ref += 3; #endif if (J->cur.nins > ref || (LJ_HASPROFILE && J->cur.nins == ref && J->cur.ir[ref-1].o != IR_PROF)) { J->instunroll = 0; /* Cannot continue unrolling across an ITERN. */ lj_record_stop(J, LJ_TRLINK_LOOP, J->cur.traceno); /* Looping trace. */ return LOOPEV_ENTER; } } J->maxslot = ra; lj_snap_add(J); /* Required to make JLOOP the first ins in a side-trace. */ ix.tab = getslot(J, ra-2); ix.key = J->base[ra-1] ? J->base[ra-1] : sloadt(J, (int32_t)(ra-1), IRT_GUARD|IRT_INT, IRSLOAD_TYPECHECK|IRSLOAD_KEYINDEX); copyTV(J->L, &ix.tabv, &J->L->base[ra-2]); copyTV(J->L, &ix.keyv, &J->L->base[ra-1]); ix.idxchain = (rb < 3); /* Omit value type check, if unused. */ ix.mobj = 1; /* We need the next index, too. */ J->maxslot = ra + lj_record_next(J, &ix); J->needsnap = 1; if (!tref_isnil(ix.key)) { /* Looping back? */ J->base[ra-1] = ix.mobj | TREF_KEYINDEX; /* Control var has next index. */ J->base[ra] = ix.key; J->base[ra+1] = ix.val; J->pc += bc_j(J->pc[1])+2; return LOOPEV_ENTER; } else { J->maxslot = ra-3; J->pc += 2; return LOOPEV_LEAVE; } #endif } /* Record ISNEXT. */ static void rec_isnext(jit_State *J, BCReg ra) { cTValue *b = &J->L->base[ra-3]; if (tvisfunc(b) && funcV(b)->c.ffid == FF_next && tvistab(b+1) && tvisnil(b+2)) { /* These checks are folded away for a compiled pairs(). */ TRef func = getslot(J, ra-3); TRef trid = emitir(IRT(IR_FLOAD, IRT_U8), func, IRFL_FUNC_FFID); emitir(IRTGI(IR_EQ), trid, lj_ir_kint(J, FF_next)); (void)getslot(J, ra-2); /* Type check for table. */ (void)getslot(J, ra-1); /* Type check for nil key. */ J->base[ra-1] = lj_ir_kint(J, 0) | TREF_KEYINDEX; J->maxslot = ra; } else { /* Abort trace. Interpreter will despecialize bytecode. */ lj_trace_err(J, LJ_TRERR_RECERR); } } /* -- Record profiler hook checks ----------------------------------------- */ #if LJ_HASPROFILE /* Need to insert profiler hook check? */ static int rec_profile_need(jit_State *J, GCproto *pt, const BCIns *pc) { GCproto *ppt; lj_assertJ(J->prof_mode == 'f' || J->prof_mode == 'l', "bad profiler mode %c", J->prof_mode); if (!pt) return 0; ppt = J->prev_pt; J->prev_pt = pt; if (pt != ppt && ppt) { J->prev_line = -1; return 1; } if (J->prof_mode == 'l') { BCLine line = lj_debug_line(pt, proto_bcpos(pt, pc)); BCLine pline = J->prev_line; J->prev_line = line; if (pline != line) return 1; } return 0; } static void rec_profile_ins(jit_State *J, const BCIns *pc) { if (J->prof_mode && rec_profile_need(J, J->pt, pc)) { emitir(IRTG(IR_PROF, IRT_NIL), 0, 0); lj_snap_add(J); } } static void rec_profile_ret(jit_State *J) { if (J->prof_mode == 'f') { emitir(IRTG(IR_PROF, IRT_NIL), 0, 0); J->prev_pt = NULL; lj_snap_add(J); } } #endif /* -- Record calls and returns -------------------------------------------- */ /* Specialize to the runtime value of the called function or its prototype. */ static TRef rec_call_specialize(jit_State *J, GCfunc *fn, TRef tr) { TRef kfunc; if (isluafunc(fn)) { GCproto *pt = funcproto(fn); /* Too many closures created? Probably not a monomorphic function. */ if (pt->flags >= PROTO_CLC_POLY) { /* Specialize to prototype instead. */ TRef trpt = emitir(IRT(IR_FLOAD, IRT_PGC), tr, IRFL_FUNC_PC); emitir(IRTG(IR_EQ, IRT_PGC), trpt, lj_ir_kptr(J, proto_bc(pt))); (void)lj_ir_kgc(J, obj2gco(pt), IRT_PROTO); /* Prevent GC of proto. */ return tr; } } else { /* Don't specialize to non-monomorphic builtins. */ switch (fn->c.ffid) { case FF_coroutine_wrap_aux: case FF_string_gmatch_aux: /* NYI: io_file_iter doesn't have an ffid, yet. */ { /* Specialize to the ffid. */ TRef trid = emitir(IRT(IR_FLOAD, IRT_U8), tr, IRFL_FUNC_FFID); emitir(IRTGI(IR_EQ), trid, lj_ir_kint(J, fn->c.ffid)); } return tr; default: /* NYI: don't specialize to non-monomorphic C functions. */ break; } } /* Otherwise specialize to the function (closure) value itself. */ kfunc = lj_ir_kfunc(J, fn); emitir(IRTG(IR_EQ, IRT_FUNC), tr, kfunc); return kfunc; } /* Record call setup. */ static void rec_call_setup(jit_State *J, BCReg func, ptrdiff_t nargs) { RecordIndex ix; TValue *functv = &J->L->base[func]; TRef kfunc, *fbase = &J->base[func]; ptrdiff_t i; (void)getslot(J, func); /* Ensure func has a reference. */ for (i = 1; i <= nargs; i++) (void)getslot(J, func+LJ_FR2+i); /* Ensure all args have a reference. */ if (!tref_isfunc(fbase[0])) { /* Resolve __call metamethod. */ ix.tab = fbase[0]; copyTV(J->L, &ix.tabv, functv); if (!lj_record_mm_lookup(J, &ix, MM_call) || !tref_isfunc(ix.mobj)) lj_trace_err(J, LJ_TRERR_NOMM); for (i = ++nargs; i > LJ_FR2; i--) /* Shift arguments up. */ fbase[i+LJ_FR2] = fbase[i+LJ_FR2-1]; #if LJ_FR2 fbase[2] = fbase[0]; #endif fbase[0] = ix.mobj; /* Replace function. */ functv = &ix.mobjv; } kfunc = rec_call_specialize(J, funcV(functv), fbase[0]); #if LJ_FR2 fbase[0] = kfunc; fbase[1] = TREF_FRAME; #else fbase[0] = kfunc | TREF_FRAME; #endif J->maxslot = (BCReg)nargs; } /* Record call. */ void lj_record_call(jit_State *J, BCReg func, ptrdiff_t nargs) { rec_call_setup(J, func, nargs); /* Bump frame. */ J->framedepth++; J->base += func+1+LJ_FR2; J->baseslot += func+1+LJ_FR2; if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS) lj_trace_err(J, LJ_TRERR_STACKOV); } /* Record tail call. */ void lj_record_tailcall(jit_State *J, BCReg func, ptrdiff_t nargs) { rec_call_setup(J, func, nargs); if (frame_isvarg(J->L->base - 1)) { BCReg cbase = (BCReg)frame_delta(J->L->base - 1); if (--J->framedepth < 0) lj_trace_err(J, LJ_TRERR_NYIRETL); J->baseslot -= (BCReg)cbase; J->base -= cbase; func += cbase; } /* Move func + args down. */ if (LJ_FR2 && J->baseslot == 2) J->base[func+1] = TREF_FRAME; memmove(&J->base[-1-LJ_FR2], &J->base[func], sizeof(TRef)*(J->maxslot+1+LJ_FR2)); /* Note: the new TREF_FRAME is now at J->base[-1] (even for slot #0). */ /* Tailcalls can form a loop, so count towards the loop unroll limit. */ if (++J->tailcalled > J->loopunroll) lj_trace_err(J, LJ_TRERR_LUNROLL); } /* Check unroll limits for down-recursion. */ static int check_downrec_unroll(jit_State *J, GCproto *pt) { IRRef ptref; for (ptref = J->chain[IR_KGC]; ptref; ptref = IR(ptref)->prev) if (ir_kgc(IR(ptref)) == obj2gco(pt)) { int count = 0; IRRef ref; for (ref = J->chain[IR_RETF]; ref; ref = IR(ref)->prev) if (IR(ref)->op1 == ptref) count++; if (count) { if (J->pc == J->startpc) { if (count + J->tailcalled > J->param[JIT_P_recunroll]) return 1; } else { lj_trace_err(J, LJ_TRERR_DOWNREC); } } } return 0; } static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot); /* Record return. */ void lj_record_ret(jit_State *J, BCReg rbase, ptrdiff_t gotresults) { TValue *frame = J->L->base - 1; ptrdiff_t i; for (i = 0; i < gotresults; i++) (void)getslot(J, rbase+i); /* Ensure all results have a reference. */ while (frame_ispcall(frame)) { /* Immediately resolve pcall() returns. */ BCReg cbase = (BCReg)frame_delta(frame); if (--J->framedepth <= 0) lj_trace_err(J, LJ_TRERR_NYIRETL); lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return"); gotresults++; rbase += cbase; J->baseslot -= (BCReg)cbase; J->base -= cbase; J->base[--rbase] = TREF_TRUE; /* Prepend true to results. */ frame = frame_prevd(frame); J->needsnap = 1; /* Stop catching on-trace errors. */ } /* Return to lower frame via interpreter for unhandled cases. */ if (J->framedepth == 0 && J->pt && bc_isret(bc_op(*J->pc)) && (!frame_islua(frame) || (J->parent == 0 && J->exitno == 0 && !bc_isret(bc_op(J->cur.startins))))) { /* NYI: specialize to frame type and return directly, not via RET*. */ for (i = 0; i < (ptrdiff_t)rbase; i++) J->base[i] = 0; /* Purge dead slots. */ J->maxslot = rbase + (BCReg)gotresults; lj_record_stop(J, LJ_TRLINK_RETURN, 0); /* Return to interpreter. */ return; } if (frame_isvarg(frame)) { BCReg cbase = (BCReg)frame_delta(frame); if (--J->framedepth < 0) /* NYI: return of vararg func to lower frame. */ lj_trace_err(J, LJ_TRERR_NYIRETL); lj_assertJ(J->baseslot > 1+LJ_FR2, "bad baseslot for return"); rbase += cbase; J->baseslot -= (BCReg)cbase; J->base -= cbase; frame = frame_prevd(frame); } if (frame_islua(frame)) { /* Return to Lua frame. */ BCIns callins = *(frame_pc(frame)-1); ptrdiff_t nresults = bc_b(callins) ? (ptrdiff_t)bc_b(callins)-1 :gotresults; BCReg cbase = bc_a(callins); GCproto *pt = funcproto(frame_func(frame - (cbase+1+LJ_FR2))); if ((pt->flags & PROTO_NOJIT)) lj_trace_err(J, LJ_TRERR_CJITOFF); if (J->framedepth == 0 && J->pt && frame == J->L->base - 1) { if (check_downrec_unroll(J, pt)) { J->maxslot = (BCReg)(rbase + gotresults); lj_snap_purge(J); lj_record_stop(J, LJ_TRLINK_DOWNREC, J->cur.traceno); /* Down-rec. */ return; } lj_snap_add(J); } for (i = 0; i < nresults; i++) /* Adjust results. */ J->base[i-1-LJ_FR2] = i < gotresults ? J->base[rbase+i] : TREF_NIL; J->maxslot = cbase+(BCReg)nresults; if (J->framedepth > 0) { /* Return to a frame that is part of the trace. */ J->framedepth--; lj_assertJ(J->baseslot > cbase+1+LJ_FR2, "bad baseslot for return"); J->baseslot -= cbase+1+LJ_FR2; J->base -= cbase+1+LJ_FR2; } else if (J->parent == 0 && J->exitno == 0 && !bc_isret(bc_op(J->cur.startins))) { /* Return to lower frame would leave the loop in a root trace. */ lj_trace_err(J, LJ_TRERR_LLEAVE); } else if (J->needsnap) { /* Tailcalled to ff with side-effects. */ lj_trace_err(J, LJ_TRERR_NYIRETL); /* No way to insert snapshot here. */ } else { /* Return to lower frame. Guard for the target we return to. */ TRef trpt = lj_ir_kgc(J, obj2gco(pt), IRT_PROTO); TRef trpc = lj_ir_kptr(J, (void *)frame_pc(frame)); emitir(IRTG(IR_RETF, IRT_PGC), trpt, trpc); J->retdepth++; J->needsnap = 1; lj_assertJ(J->baseslot == 1+LJ_FR2, "bad baseslot for return"); /* Shift result slots up and clear the slots of the new frame below. */ memmove(J->base + cbase, J->base-1-LJ_FR2, sizeof(TRef)*nresults); memset(J->base-1-LJ_FR2, 0, sizeof(TRef)*(cbase+1+LJ_FR2)); } } else if (frame_iscont(frame)) { /* Return to continuation frame. */ ASMFunction cont = frame_contf(frame); BCReg cbase = (BCReg)frame_delta(frame); if ((J->framedepth -= 2) < 0) lj_trace_err(J, LJ_TRERR_NYIRETL); J->baseslot -= (BCReg)cbase; J->base -= cbase; J->maxslot = cbase-(2<base[dst] = gotresults ? J->base[cbase+rbase] : TREF_NIL; if (dst >= J->maxslot) { J->maxslot = dst+1; } } else if (cont == lj_cont_nop) { /* Nothing to do here. */ } else if (cont == lj_cont_cat) { BCReg bslot = bc_b(*(frame_contpc(frame)-1)); TRef tr = gotresults ? J->base[cbase+rbase] : TREF_NIL; if (bslot != J->maxslot) { /* Concatenate the remainder. */ TValue *b = J->L->base, save; /* Simulate lower frame and result. */ /* Can't handle MM_concat + CALLT + fast func side-effects. */ if (J->postproc != LJ_POST_NONE) lj_trace_err(J, LJ_TRERR_NYIRETL); J->base[J->maxslot] = tr; copyTV(J->L, &save, b-(2<L, b-(2<L->base = b - cbase; tr = rec_cat(J, bslot, cbase-(2<L->base + cbase; /* Undo. */ J->L->base = b; copyTV(J->L, b-(2<base[dst] = tr; if (dst >= J->maxslot) { J->maxslot = dst+1; } } /* Otherwise continue with another __concat call. */ } else { /* Result type already specialized. */ lj_assertJ(cont == lj_cont_condf || cont == lj_cont_condt, "bad continuation type"); } } else { lj_trace_err(J, LJ_TRERR_NYIRETL); /* NYI: handle return to C frame. */ } lj_assertJ(J->baseslot >= 1+LJ_FR2, "bad baseslot for return"); } /* -- Metamethod handling ------------------------------------------------- */ /* Prepare to record call to metamethod. */ static BCReg rec_mm_prep(jit_State *J, ASMFunction cont) { BCReg s, top = cont == lj_cont_cat ? J->maxslot : curr_proto(J->L)->framesize; #if LJ_FR2 J->base[top] = lj_ir_k64(J, IR_KNUM, u64ptr(contptr(cont))); J->base[top+1] = TREF_CONT; #else J->base[top] = lj_ir_kptr(J, contptr(cont)) | TREF_CONT; #endif J->framedepth++; for (s = J->maxslot; s < top; s++) J->base[s] = 0; /* Clear frame gap to avoid resurrecting previous refs. */ return top+1+LJ_FR2; } /* Record metamethod lookup. */ int lj_record_mm_lookup(jit_State *J, RecordIndex *ix, MMS mm) { RecordIndex mix; GCtab *mt; if (tref_istab(ix->tab)) { mt = tabref(tabV(&ix->tabv)->metatable); mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META); } else if (tref_isudata(ix->tab)) { int udtype = udataV(&ix->tabv)->udtype; mt = tabref(udataV(&ix->tabv)->metatable); /* The metatables of special userdata objects are treated as immutable. */ if (udtype != UDTYPE_USERDATA) { cTValue *mo; if (LJ_HASFFI && udtype == UDTYPE_FFI_CLIB) { /* Specialize to the C library namespace object. */ emitir(IRTG(IR_EQ, IRT_PGC), ix->tab, lj_ir_kptr(J, udataV(&ix->tabv))); } else { /* Specialize to the type of userdata. */ TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), ix->tab, IRFL_UDATA_UDTYPE); emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, udtype)); } immutable_mt: mo = lj_tab_getstr(mt, mmname_str(J2G(J), mm)); if (!mo || tvisnil(mo)) return 0; /* No metamethod. */ /* Treat metamethod or index table as immutable, too. */ if (!(tvisfunc(mo) || tvistab(mo))) lj_trace_err(J, LJ_TRERR_BADTYPE); copyTV(J->L, &ix->mobjv, mo); ix->mobj = lj_ir_kgc(J, gcV(mo), tvisfunc(mo) ? IRT_FUNC : IRT_TAB); ix->mtv = mt; ix->mt = TREF_NIL; /* Dummy value for comparison semantics. */ return 1; /* Got metamethod or index table. */ } mix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_UDATA_META); } else { /* Specialize to base metatable. Must flush mcode in lua_setmetatable(). */ mt = tabref(basemt_obj(J2G(J), &ix->tabv)); if (mt == NULL) { ix->mt = TREF_NIL; return 0; /* No metamethod. */ } /* The cdata metatable is treated as immutable. */ if (LJ_HASFFI && tref_iscdata(ix->tab)) goto immutable_mt; ix->mt = mix.tab = lj_ir_ggfload(J, IRT_TAB, GG_OFS(g.gcroot[GCROOT_BASEMT+itypemap(&ix->tabv)])); goto nocheck; } ix->mt = mt ? mix.tab : TREF_NIL; emitir(IRTG(mt ? IR_NE : IR_EQ, IRT_TAB), mix.tab, lj_ir_knull(J, IRT_TAB)); nocheck: if (mt) { GCstr *mmstr = mmname_str(J2G(J), mm); cTValue *mo = lj_tab_getstr(mt, mmstr); if (mo && !tvisnil(mo)) copyTV(J->L, &ix->mobjv, mo); ix->mtv = mt; settabV(J->L, &mix.tabv, mt); setstrV(J->L, &mix.keyv, mmstr); mix.key = lj_ir_kstr(J, mmstr); mix.val = 0; mix.idxchain = 0; ix->mobj = lj_record_idx(J, &mix); return !tref_isnil(ix->mobj); /* 1 if metamethod found, 0 if not. */ } return 0; /* No metamethod. */ } /* Record call to arithmetic metamethod. */ static TRef rec_mm_arith(jit_State *J, RecordIndex *ix, MMS mm) { /* Set up metamethod call first to save ix->tab and ix->tabv. */ BCReg func = rec_mm_prep(J, mm == MM_concat ? lj_cont_cat : lj_cont_ra); TRef *base = J->base + func; TValue *basev = J->L->base + func; base[1+LJ_FR2] = ix->tab; base[2+LJ_FR2] = ix->key; copyTV(J->L, basev+1+LJ_FR2, &ix->tabv); copyTV(J->L, basev+2+LJ_FR2, &ix->keyv); if (!lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */ if (mm != MM_unm) { ix->tab = ix->key; copyTV(J->L, &ix->tabv, &ix->keyv); if (lj_record_mm_lookup(J, ix, mm)) /* Lookup mm on 2nd operand. */ goto ok; } lj_trace_err(J, LJ_TRERR_NOMM); } ok: base[0] = ix->mobj; #if LJ_FR2 base[1] = 0; #endif copyTV(J->L, basev+0, &ix->mobjv); lj_record_call(J, func, 2); return 0; /* No result yet. */ } /* Record call to __len metamethod. */ static TRef rec_mm_len(jit_State *J, TRef tr, TValue *tv) { RecordIndex ix; ix.tab = tr; copyTV(J->L, &ix.tabv, tv); if (lj_record_mm_lookup(J, &ix, MM_len)) { BCReg func = rec_mm_prep(J, lj_cont_ra); TRef *base = J->base + func; TValue *basev = J->L->base + func; base[0] = ix.mobj; copyTV(J->L, basev+0, &ix.mobjv); base += LJ_FR2; basev += LJ_FR2; base[1] = tr; copyTV(J->L, basev+1, tv); #if LJ_52 base[2] = tr; copyTV(J->L, basev+2, tv); #else base[2] = TREF_NIL; setnilV(basev+2); #endif lj_record_call(J, func, 2); } else { if (LJ_52 && tref_istab(tr)) return emitir(IRTI(IR_ALEN), tr, TREF_NIL); lj_trace_err(J, LJ_TRERR_NOMM); } return 0; /* No result yet. */ } /* Call a comparison metamethod. */ static void rec_mm_callcomp(jit_State *J, RecordIndex *ix, int op) { BCReg func = rec_mm_prep(J, (op&1) ? lj_cont_condf : lj_cont_condt); TRef *base = J->base + func + LJ_FR2; TValue *tv = J->L->base + func + LJ_FR2; base[-LJ_FR2] = ix->mobj; base[1] = ix->val; base[2] = ix->key; copyTV(J->L, tv-LJ_FR2, &ix->mobjv); copyTV(J->L, tv+1, &ix->valv); copyTV(J->L, tv+2, &ix->keyv); lj_record_call(J, func, 2); } /* Record call to equality comparison metamethod (for tab and udata only). */ static void rec_mm_equal(jit_State *J, RecordIndex *ix, int op) { ix->tab = ix->val; copyTV(J->L, &ix->tabv, &ix->valv); if (lj_record_mm_lookup(J, ix, MM_eq)) { /* Lookup mm on 1st operand. */ cTValue *bv; TRef mo1 = ix->mobj; TValue mo1v; copyTV(J->L, &mo1v, &ix->mobjv); /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */ bv = &ix->keyv; if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) { TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META); emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) { TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META); emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); } else { /* Lookup metamethod on 2nd operand and compare both. */ ix->tab = ix->key; copyTV(J->L, &ix->tabv, bv); if (!lj_record_mm_lookup(J, ix, MM_eq) || lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv)) return; } rec_mm_callcomp(J, ix, op); } } /* Record call to ordered comparison metamethods (for arbitrary objects). */ static void rec_mm_comp(jit_State *J, RecordIndex *ix, int op) { ix->tab = ix->val; copyTV(J->L, &ix->tabv, &ix->valv); while (1) { MMS mm = (op & 2) ? MM_le : MM_lt; /* Try __le + __lt or only __lt. */ #if LJ_52 if (!lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */ ix->tab = ix->key; copyTV(J->L, &ix->tabv, &ix->keyv); if (!lj_record_mm_lookup(J, ix, mm)) /* Lookup mm on 2nd operand. */ goto nomatch; } rec_mm_callcomp(J, ix, op); return; #else if (lj_record_mm_lookup(J, ix, mm)) { /* Lookup mm on 1st operand. */ cTValue *bv; TRef mo1 = ix->mobj; TValue mo1v; copyTV(J->L, &mo1v, &ix->mobjv); /* Avoid the 2nd lookup and the objcmp if the metatables are equal. */ bv = &ix->keyv; if (tvistab(bv) && tabref(tabV(bv)->metatable) == ix->mtv) { TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_TAB_META); emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); } else if (tvisudata(bv) && tabref(udataV(bv)->metatable) == ix->mtv) { TRef mt2 = emitir(IRT(IR_FLOAD, IRT_TAB), ix->key, IRFL_UDATA_META); emitir(IRTG(IR_EQ, IRT_TAB), mt2, ix->mt); } else { /* Lookup metamethod on 2nd operand and compare both. */ ix->tab = ix->key; copyTV(J->L, &ix->tabv, bv); if (!lj_record_mm_lookup(J, ix, mm) || lj_record_objcmp(J, mo1, ix->mobj, &mo1v, &ix->mobjv)) goto nomatch; } rec_mm_callcomp(J, ix, op); return; } #endif nomatch: /* Lookup failed. Retry with __lt and swapped operands. */ if (!(op & 2)) break; /* Already at __lt. Interpreter will throw. */ ix->tab = ix->key; ix->key = ix->val; ix->val = ix->tab; copyTV(J->L, &ix->tabv, &ix->keyv); copyTV(J->L, &ix->keyv, &ix->valv); copyTV(J->L, &ix->valv, &ix->tabv); op ^= 3; } } #if LJ_HASFFI /* Setup call to cdata comparison metamethod. */ static void rec_mm_comp_cdata(jit_State *J, RecordIndex *ix, int op, MMS mm) { lj_snap_add(J); if (tref_iscdata(ix->val)) { ix->tab = ix->val; copyTV(J->L, &ix->tabv, &ix->valv); } else { lj_assertJ(tref_iscdata(ix->key), "cdata expected"); ix->tab = ix->key; copyTV(J->L, &ix->tabv, &ix->keyv); } lj_record_mm_lookup(J, ix, mm); rec_mm_callcomp(J, ix, op); } #endif /* -- Indexed access ------------------------------------------------------ */ #ifdef LUAJIT_ENABLE_TABLE_BUMP /* Bump table allocations in bytecode when they grow during recording. */ static void rec_idx_bump(jit_State *J, RecordIndex *ix) { RBCHashEntry *rbc = &J->rbchash[(ix->tab & (RBCHASH_SLOTS-1))]; if (tref_ref(ix->tab) == rbc->ref) { const BCIns *pc = mref(rbc->pc, const BCIns); GCtab *tb = tabV(&ix->tabv); uint32_t nhbits; IRIns *ir; if (!tvisnil(&ix->keyv)) (void)lj_tab_set(J->L, tb, &ix->keyv); /* Grow table right now. */ nhbits = tb->hmask > 0 ? lj_fls(tb->hmask)+1 : 0; ir = IR(tref_ref(ix->tab)); if (ir->o == IR_TNEW) { uint32_t ah = bc_d(*pc); uint32_t asize = ah & 0x7ff, hbits = ah >> 11; if (nhbits > hbits) hbits = nhbits; if (tb->asize > asize) { asize = tb->asize <= 0x7ff ? tb->asize : 0x7ff; } if ((asize | (hbits<<11)) != ah) { /* Has the size changed? */ /* Patch bytecode, but continue recording (for more patching). */ setbc_d(pc, (asize | (hbits<<11))); /* Patching TNEW operands is only safe if the trace is aborted. */ ir->op1 = asize; ir->op2 = hbits; J->retryrec = 1; /* Abort the trace at the end of recording. */ } } else if (ir->o == IR_TDUP) { GCtab *tpl = gco2tab(proto_kgc(&gcref(rbc->pt)->pt, ~(ptrdiff_t)bc_d(*pc))); /* Grow template table, but preserve keys with nil values. */ if ((tb->asize > tpl->asize && (1u << nhbits)-1 == tpl->hmask) || (tb->asize == tpl->asize && (1u << nhbits)-1 > tpl->hmask)) { Node *node = noderef(tpl->node); uint32_t i, hmask = tpl->hmask, asize; TValue *array; for (i = 0; i <= hmask; i++) { if (!tvisnil(&node[i].key) && tvisnil(&node[i].val)) settabV(J->L, &node[i].val, tpl); } if (!tvisnil(&ix->keyv) && tref_isk(ix->key)) { TValue *o = lj_tab_set(J->L, tpl, &ix->keyv); if (tvisnil(o)) settabV(J->L, o, tpl); } lj_tab_resize(J->L, tpl, tb->asize, nhbits); node = noderef(tpl->node); hmask = tpl->hmask; for (i = 0; i <= hmask; i++) { /* This is safe, since template tables only hold immutable values. */ if (tvistab(&node[i].val)) setnilV(&node[i].val); } /* The shape of the table may have changed. Clean up array part, too. */ asize = tpl->asize; array = tvref(tpl->array); for (i = 0; i < asize; i++) { if (tvistab(&array[i])) setnilV(&array[i]); } J->retryrec = 1; /* Abort the trace at the end of recording. */ } } } } #endif /* Record bounds-check. */ static void rec_idx_abc(jit_State *J, TRef asizeref, TRef ikey, uint32_t asize) { /* Try to emit invariant bounds checks. */ if ((J->flags & (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) == (JIT_F_OPT_LOOP|JIT_F_OPT_ABC)) { IRRef ref = tref_ref(ikey); IRIns *ir = IR(ref); int32_t ofs = 0; IRRef ofsref = 0; /* Handle constant offsets. */ if (ir->o == IR_ADD && irref_isk(ir->op2)) { ofsref = ir->op2; ofs = IR(ofsref)->i; ref = ir->op1; ir = IR(ref); } /* Got scalar evolution analysis results for this reference? */ if (ref == J->scev.idx) { int32_t stop; lj_assertJ(irt_isint(J->scev.t) && ir->o == IR_SLOAD, "only int SCEV supported"); stop = numberVint(&(J->L->base - J->baseslot)[ir->op1 + FORL_STOP]); /* Runtime value for stop of loop is within bounds? */ if ((uint64_t)stop + ofs < (uint64_t)asize) { /* Emit invariant bounds check for stop. */ emitir(IRTG(IR_ABC, IRT_P32), asizeref, ofs == 0 ? J->scev.stop : emitir(IRTI(IR_ADD), J->scev.stop, ofsref)); /* Emit invariant bounds check for start, if not const or negative. */ if (!(J->scev.dir && J->scev.start && (int64_t)IR(J->scev.start)->i + ofs >= 0)) emitir(IRTG(IR_ABC, IRT_P32), asizeref, ikey); return; } } } emitir(IRTGI(IR_ABC), asizeref, ikey); /* Emit regular bounds check. */ } /* Record indexed key lookup. */ static TRef rec_idx_key(jit_State *J, RecordIndex *ix, IRRef *rbref, IRType1 *rbguard) { TRef key; GCtab *t = tabV(&ix->tabv); ix->oldv = lj_tab_get(J->L, t, &ix->keyv); /* Lookup previous value. */ *rbref = 0; rbguard->irt = 0; /* Integer keys are looked up in the array part first. */ key = ix->key; if (tref_isnumber(key)) { int32_t k = numberVint(&ix->keyv); if (!tvisint(&ix->keyv) && numV(&ix->keyv) != (lua_Number)k) k = LJ_MAX_ASIZE; if ((MSize)k < LJ_MAX_ASIZE) { /* Potential array key? */ TRef ikey = lj_opt_narrow_index(J, key); TRef asizeref = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE); if ((MSize)k < t->asize) { /* Currently an array key? */ TRef arrayref; rec_idx_abc(J, asizeref, ikey, t->asize); arrayref = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_ARRAY); return emitir(IRT(IR_AREF, IRT_PGC), arrayref, ikey); } else { /* Currently not in array (may be an array extension)? */ emitir(IRTGI(IR_ULE), asizeref, ikey); /* Inv. bounds check. */ if (k == 0 && tref_isk(key)) key = lj_ir_knum_zero(J); /* Canonicalize 0 or +-0.0 to +0.0. */ /* And continue with the hash lookup. */ } } else if (!tref_isk(key)) { /* We can rule out const numbers which failed the integerness test ** above. But all other numbers are potential array keys. */ if (t->asize == 0) { /* True sparse tables have an empty array part. */ /* Guard that the array part stays empty. */ TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_ASIZE); emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0)); } else { lj_trace_err(J, LJ_TRERR_NYITMIX); } } } /* Otherwise the key is located in the hash part. */ if (t->hmask == 0) { /* Shortcut for empty hash part. */ /* Guard that the hash part stays empty. */ TRef tmp = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK); emitir(IRTGI(IR_EQ), tmp, lj_ir_kint(J, 0)); return lj_ir_kkptr(J, niltvg(J2G(J))); } if (tref_isinteger(key)) /* Hash keys are based on numbers, not ints. */ key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT); if (tref_isk(key)) { /* Optimize lookup of constant hash keys. */ GCSize hslot = (GCSize)((char *)ix->oldv-(char *)&noderef(t->node)[0].val); if (hslot <= t->hmask*(GCSize)sizeof(Node) && hslot <= 65535*(GCSize)sizeof(Node)) { TRef node, kslot, hm; *rbref = J->cur.nins; /* Mark possible rollback point. */ *rbguard = J->guardemit; hm = emitir(IRTI(IR_FLOAD), ix->tab, IRFL_TAB_HMASK); emitir(IRTGI(IR_EQ), hm, lj_ir_kint(J, (int32_t)t->hmask)); node = emitir(IRT(IR_FLOAD, IRT_PGC), ix->tab, IRFL_TAB_NODE); kslot = lj_ir_kslot(J, key, (IRRef)(hslot / sizeof(Node))); return emitir(IRTG(IR_HREFK, IRT_PGC), node, kslot); } } /* Fall back to a regular hash lookup. */ return emitir(IRT(IR_HREF, IRT_PGC), ix->tab, key); } /* Determine whether a key is NOT one of the fast metamethod names. */ static int nommstr(jit_State *J, TRef key) { if (tref_isstr(key)) { if (tref_isk(key)) { GCstr *str = ir_kstr(IR(tref_ref(key))); uint32_t mm; for (mm = 0; mm <= MM_FAST; mm++) if (mmname_str(J2G(J), mm) == str) return 0; /* MUST be one the fast metamethod names. */ } else { return 0; /* Variable string key MAY be a metamethod name. */ } } return 1; /* CANNOT be a metamethod name. */ } /* Record indexed load/store. */ TRef lj_record_idx(jit_State *J, RecordIndex *ix) { TRef xref; IROp xrefop, loadop; IRRef rbref; IRType1 rbguard; cTValue *oldv; while (!tref_istab(ix->tab)) { /* Handle non-table lookup. */ /* Never call raw lj_record_idx() on non-table. */ lj_assertJ(ix->idxchain != 0, "bad usage"); if (!lj_record_mm_lookup(J, ix, ix->val ? MM_newindex : MM_index)) lj_trace_err(J, LJ_TRERR_NOMM); handlemm: if (tref_isfunc(ix->mobj)) { /* Handle metamethod call. */ BCReg func = rec_mm_prep(J, ix->val ? lj_cont_nop : lj_cont_ra); TRef *base = J->base + func + LJ_FR2; TValue *tv = J->L->base + func + LJ_FR2; base[-LJ_FR2] = ix->mobj; base[1] = ix->tab; base[2] = ix->key; setfuncV(J->L, tv-LJ_FR2, funcV(&ix->mobjv)); copyTV(J->L, tv+1, &ix->tabv); copyTV(J->L, tv+2, &ix->keyv); if (ix->val) { base[3] = ix->val; copyTV(J->L, tv+3, &ix->valv); lj_record_call(J, func, 3); /* mobj(tab, key, val) */ return 0; } else { lj_record_call(J, func, 2); /* res = mobj(tab, key) */ return 0; /* No result yet. */ } } #if LJ_HASBUFFER /* The index table of buffer objects is treated as immutable. */ if (ix->mt == TREF_NIL && !ix->val && tref_isudata(ix->tab) && udataV(&ix->tabv)->udtype == UDTYPE_BUFFER && tref_istab(ix->mobj) && tref_isstr(ix->key) && tref_isk(ix->key)) { cTValue *val = lj_tab_getstr(tabV(&ix->mobjv), strV(&ix->keyv)); TRef tr = lj_record_constify(J, val); if (tr) return tr; /* Specialize to the value, i.e. a method. */ } #endif /* Otherwise retry lookup with metaobject. */ ix->tab = ix->mobj; copyTV(J->L, &ix->tabv, &ix->mobjv); if (--ix->idxchain == 0) lj_trace_err(J, LJ_TRERR_IDXLOOP); } /* First catch nil and NaN keys for tables. */ if (tvisnil(&ix->keyv) || (tvisnum(&ix->keyv) && tvisnan(&ix->keyv))) { if (ix->val) /* Better fail early. */ lj_trace_err(J, LJ_TRERR_STORENN); if (tref_isk(ix->key)) { if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_index)) goto handlemm; return TREF_NIL; } } /* Record the key lookup. */ xref = rec_idx_key(J, ix, &rbref, &rbguard); xrefop = IR(tref_ref(xref))->o; loadop = xrefop == IR_AREF ? IR_ALOAD : IR_HLOAD; /* The lj_meta_tset() inconsistency is gone, but better play safe. */ oldv = xrefop == IR_KKPTR ? (cTValue *)ir_kptr(IR(tref_ref(xref))) : ix->oldv; if (ix->val == 0) { /* Indexed load */ IRType t = itype2irt(oldv); TRef res; if (oldv == niltvg(J2G(J))) { emitir(IRTG(IR_EQ, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J)))); res = TREF_NIL; } else { res = emitir(IRTG(loadop, t), xref, 0); } if (tref_ref(res) < rbref) { /* HREFK + load forwarded? */ lj_ir_rollback(J, rbref); /* Rollback to eliminate hmask guard. */ J->guardemit = rbguard; } if (t == IRT_NIL && ix->idxchain && lj_record_mm_lookup(J, ix, MM_index)) goto handlemm; if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitives. */ return res; } else { /* Indexed store. */ GCtab *mt = tabref(tabV(&ix->tabv)->metatable); int keybarrier = tref_isgcv(ix->key) && !tref_isnil(ix->val); if (tref_ref(xref) < rbref) { /* HREFK forwarded? */ lj_ir_rollback(J, rbref); /* Rollback to eliminate hmask guard. */ J->guardemit = rbguard; } if (tvisnil(oldv)) { /* Previous value was nil? */ /* Need to duplicate the hasmm check for the early guards. */ int hasmm = 0; if (ix->idxchain && mt) { cTValue *mo = lj_tab_getstr(mt, mmname_str(J2G(J), MM_newindex)); hasmm = mo && !tvisnil(mo); } if (hasmm) emitir(IRTG(loadop, IRT_NIL), xref, 0); /* Guard for nil value. */ else if (xrefop == IR_HREF) emitir(IRTG(oldv == niltvg(J2G(J)) ? IR_EQ : IR_NE, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J)))); if (ix->idxchain && lj_record_mm_lookup(J, ix, MM_newindex)) { lj_assertJ(hasmm, "inconsistent metamethod handling"); goto handlemm; } lj_assertJ(!hasmm, "inconsistent metamethod handling"); if (oldv == niltvg(J2G(J))) { /* Need to insert a new key. */ TRef key = ix->key; if (tref_isinteger(key)) /* NEWREF needs a TValue as a key. */ key = emitir(IRTN(IR_CONV), key, IRCONV_NUM_INT); xref = emitir(IRT(IR_NEWREF, IRT_PGC), ix->tab, key); keybarrier = 0; /* NEWREF already takes care of the key barrier. */ #ifdef LUAJIT_ENABLE_TABLE_BUMP if ((J->flags & JIT_F_OPT_SINK)) /* Avoid a separate flag. */ rec_idx_bump(J, ix); #endif } } else if (!lj_opt_fwd_wasnonnil(J, loadop, tref_ref(xref))) { /* Cannot derive that the previous value was non-nil, must do checks. */ if (xrefop == IR_HREF) /* Guard against store to niltv. */ emitir(IRTG(IR_NE, IRT_PGC), xref, lj_ir_kkptr(J, niltvg(J2G(J)))); if (ix->idxchain) { /* Metamethod lookup required? */ /* A check for NULL metatable is cheaper (hoistable) than a load. */ if (!mt) { TRef mtref = emitir(IRT(IR_FLOAD, IRT_TAB), ix->tab, IRFL_TAB_META); emitir(IRTG(IR_EQ, IRT_TAB), mtref, lj_ir_knull(J, IRT_TAB)); } else { IRType t = itype2irt(oldv); emitir(IRTG(loadop, t), xref, 0); /* Guard for non-nil value. */ } } } else { keybarrier = 0; /* Previous non-nil value kept the key alive. */ } /* Convert int to number before storing. */ if (!LJ_DUALNUM && tref_isinteger(ix->val)) ix->val = emitir(IRTN(IR_CONV), ix->val, IRCONV_NUM_INT); emitir(IRT(loadop+IRDELTA_L2S, tref_type(ix->val)), xref, ix->val); if (keybarrier || tref_isgcv(ix->val)) emitir(IRT(IR_TBAR, IRT_NIL), ix->tab, 0); /* Invalidate neg. metamethod cache for stores with certain string keys. */ if (!nommstr(J, ix->key)) { TRef fref = emitir(IRT(IR_FREF, IRT_PGC), ix->tab, IRFL_TAB_NOMM); emitir(IRT(IR_FSTORE, IRT_U8), fref, lj_ir_kint(J, 0)); } J->needsnap = 1; return 0; } } /* Determine result type of table traversal. */ static IRType rec_next_types(GCtab *t, uint32_t idx) { for (; idx < t->asize; idx++) { cTValue *a = arrayslot(t, idx); if (LJ_LIKELY(!tvisnil(a))) return (LJ_DUALNUM ? IRT_INT : IRT_NUM) + (itype2irt(a) << 8); } idx -= t->asize; for (; idx <= t->hmask; idx++) { Node *n = &noderef(t->node)[idx]; if (!tvisnil(&n->val)) return itype2irt(&n->key) + (itype2irt(&n->val) << 8); } return IRT_NIL + (IRT_NIL << 8); } /* Record a table traversal step aka next(). */ int lj_record_next(jit_State *J, RecordIndex *ix) { IRType t, tkey, tval; TRef trvk; t = rec_next_types(tabV(&ix->tabv), ix->keyv.u32.lo); tkey = (t & 0xff); tval = (t >> 8); trvk = lj_ir_call(J, IRCALL_lj_vm_next, ix->tab, ix->key); if (ix->mobj || tkey == IRT_NIL) { TRef idx = emitir(IRTI(IR_HIOP), trvk, trvk); /* Always check for invalid key from next() for nil result. */ if (!ix->mobj) emitir(IRTGI(IR_NE), idx, lj_ir_kint(J, -1)); ix->mobj = idx; } ix->key = lj_record_vload(J, trvk, 1, tkey); if (tkey == IRT_NIL || ix->idxchain) { /* Omit value type check. */ ix->val = TREF_NIL; return 1; } else { /* Need value. */ ix->val = lj_record_vload(J, trvk, 0, tval); return 2; } } static void rec_tsetm(jit_State *J, BCReg ra, BCReg rn, int32_t i) { RecordIndex ix; cTValue *basev = J->L->base; GCtab *t = tabV(&basev[ra-1]); settabV(J->L, &ix.tabv, t); ix.tab = getslot(J, ra-1); ix.idxchain = 0; #ifdef LUAJIT_ENABLE_TABLE_BUMP if ((J->flags & JIT_F_OPT_SINK)) { if (t->asize < i+rn-ra) lj_tab_reasize(J->L, t, i+rn-ra); setnilV(&ix.keyv); rec_idx_bump(J, &ix); } #endif for (; ra < rn; i++, ra++) { setintV(&ix.keyv, i); ix.key = lj_ir_kint(J, i); copyTV(J->L, &ix.valv, &basev[ra]); ix.val = getslot(J, ra); lj_record_idx(J, &ix); } } /* -- Upvalue access ------------------------------------------------------ */ /* Check whether upvalue is immutable and ok to constify. */ static int rec_upvalue_constify(jit_State *J, GCupval *uvp) { if (uvp->immutable) { cTValue *o = uvval(uvp); /* Don't constify objects that may retain large amounts of memory. */ #if LJ_HASFFI if (tviscdata(o)) { GCcdata *cd = cdataV(o); if (!cdataisv(cd) && !(cd->marked & LJ_GC_CDATA_FIN)) { CType *ct = ctype_raw(ctype_ctsG(J2G(J)), cd->ctypeid); if (!ctype_hassize(ct->info) || ct->size <= 16) return 1; } return 0; } #else UNUSED(J); #endif if (!(tvistab(o) || tvisudata(o) || tvisthread(o))) return 1; } return 0; } /* Record upvalue load/store. */ static TRef rec_upvalue(jit_State *J, uint32_t uv, TRef val) { GCupval *uvp = &gcref(J->fn->l.uvptr[uv])->uv; TRef fn = getcurrf(J); IRRef uref; int needbarrier = 0; if (rec_upvalue_constify(J, uvp)) { /* Try to constify immutable upvalue. */ TRef tr, kfunc; lj_assertJ(val == 0, "bad usage"); if (!tref_isk(fn)) { /* Late specialization of current function. */ if (J->pt->flags >= PROTO_CLC_POLY) goto noconstify; kfunc = lj_ir_kfunc(J, J->fn); emitir(IRTG(IR_EQ, IRT_FUNC), fn, kfunc); #if LJ_FR2 J->base[-2] = kfunc; #else J->base[-1] = kfunc | TREF_FRAME; #endif fn = kfunc; } tr = lj_record_constify(J, uvval(uvp)); if (tr) return tr; } noconstify: /* Note: this effectively limits LJ_MAX_UPVAL to 127. */ uv = (uv << 8) | (hashrot(uvp->dhash, uvp->dhash + HASH_BIAS) & 0xff); if (!uvp->closed) { uref = tref_ref(emitir(IRTG(IR_UREFO, IRT_PGC), fn, uv)); /* In current stack? */ if (uvval(uvp) >= tvref(J->L->stack) && uvval(uvp) < tvref(J->L->maxstack)) { int32_t slot = (int32_t)(uvval(uvp) - (J->L->base - J->baseslot)); if (slot >= 0) { /* Aliases an SSA slot? */ emitir(IRTG(IR_EQ, IRT_PGC), REF_BASE, emitir(IRT(IR_ADD, IRT_PGC), uref, lj_ir_kint(J, (slot - 1 - LJ_FR2) * -8))); slot -= (int32_t)J->baseslot; /* Note: slot number may be negative! */ if (val == 0) { return getslot(J, slot); } else { J->base[slot] = val; if (slot >= (int32_t)J->maxslot) J->maxslot = (BCReg)(slot+1); return 0; } } } emitir(IRTG(IR_UGT, IRT_PGC), emitir(IRT(IR_SUB, IRT_PGC), uref, REF_BASE), lj_ir_kint(J, (J->baseslot + J->maxslot) * 8)); } else { needbarrier = 1; uref = tref_ref(emitir(IRTG(IR_UREFC, IRT_PGC), fn, uv)); } if (val == 0) { /* Upvalue load */ IRType t = itype2irt(uvval(uvp)); TRef res = emitir(IRTG(IR_ULOAD, t), uref, 0); if (irtype_ispri(t)) res = TREF_PRI(t); /* Canonicalize primitive refs. */ return res; } else { /* Upvalue store. */ /* Convert int to number before storing. */ if (!LJ_DUALNUM && tref_isinteger(val)) val = emitir(IRTN(IR_CONV), val, IRCONV_NUM_INT); emitir(IRT(IR_USTORE, tref_type(val)), uref, val); if (needbarrier && tref_isgcv(val)) emitir(IRT(IR_OBAR, IRT_NIL), uref, val); J->needsnap = 1; return 0; } } /* -- Record calls to Lua functions --------------------------------------- */ /* Check unroll limits for calls. */ static void check_call_unroll(jit_State *J, TraceNo lnk) { cTValue *frame = J->L->base - 1; void *pc = mref(frame_func(frame)->l.pc, void); int32_t depth = J->framedepth; int32_t count = 0; if ((J->pt->flags & PROTO_VARARG)) depth--; /* Vararg frame still missing. */ for (; depth > 0; depth--) { /* Count frames with same prototype. */ if (frame_iscont(frame)) depth--; frame = frame_prev(frame); if (mref(frame_func(frame)->l.pc, void) == pc) count++; } if (J->pc == J->startpc) { if (count + J->tailcalled > J->param[JIT_P_recunroll]) { J->pc++; if (J->framedepth + J->retdepth == 0) lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Tail-rec. */ else lj_record_stop(J, LJ_TRLINK_UPREC, J->cur.traceno); /* Up-recursion. */ } } else { if (count > J->param[JIT_P_callunroll]) { if (lnk) { /* Possible tail- or up-recursion. */ lj_trace_flush(J, lnk); /* Flush trace that only returns. */ /* Set a small, pseudo-random hotcount for a quick retry of JFUNC*. */ hotcount_set(J2GG(J), J->pc+1, lj_prng_u64(&J2G(J)->prng) & 15u); } lj_trace_err(J, LJ_TRERR_CUNROLL); } } } /* Record Lua function setup. */ static void rec_func_setup(jit_State *J) { GCproto *pt = J->pt; BCReg s, numparams = pt->numparams; if ((pt->flags & PROTO_NOJIT)) lj_trace_err(J, LJ_TRERR_CJITOFF); if (J->baseslot + pt->framesize >= LJ_MAX_JSLOTS) lj_trace_err(J, LJ_TRERR_STACKOV); /* Fill up missing parameters with nil. */ for (s = J->maxslot; s < numparams; s++) J->base[s] = TREF_NIL; /* The remaining slots should never be read before they are written. */ J->maxslot = numparams; } /* Record Lua vararg function setup. */ static void rec_func_vararg(jit_State *J) { GCproto *pt = J->pt; BCReg s, fixargs, vframe = J->maxslot+1+LJ_FR2; lj_assertJ((pt->flags & PROTO_VARARG), "FUNCV in non-vararg function"); if (J->baseslot + vframe + pt->framesize >= LJ_MAX_JSLOTS) lj_trace_err(J, LJ_TRERR_STACKOV); J->base[vframe-1-LJ_FR2] = J->base[-1-LJ_FR2]; /* Copy function up. */ #if LJ_FR2 J->base[vframe-1] = TREF_FRAME; #endif /* Copy fixarg slots up and set their original slots to nil. */ fixargs = pt->numparams < J->maxslot ? pt->numparams : J->maxslot; for (s = 0; s < fixargs; s++) { J->base[vframe+s] = J->base[s]; J->base[s] = TREF_NIL; } J->maxslot = fixargs; J->framedepth++; J->base += vframe; J->baseslot += vframe; } /* Record entry to a Lua function. */ static void rec_func_lua(jit_State *J) { rec_func_setup(J); check_call_unroll(J, 0); } /* Record entry to an already compiled function. */ static void rec_func_jit(jit_State *J, TraceNo lnk) { GCtrace *T; rec_func_setup(J); T = traceref(J, lnk); if (T->linktype == LJ_TRLINK_RETURN) { /* Trace returns to interpreter? */ check_call_unroll(J, lnk); /* Temporarily unpatch JFUNC* to continue recording across function. */ J->patchins = *J->pc; J->patchpc = (BCIns *)J->pc; *J->patchpc = T->startins; return; } J->instunroll = 0; /* Cannot continue across a compiled function. */ if (J->pc == J->startpc && J->framedepth + J->retdepth == 0) lj_record_stop(J, LJ_TRLINK_TAILREC, J->cur.traceno); /* Extra tail-rec. */ else lj_record_stop(J, LJ_TRLINK_ROOT, lnk); /* Link to the function. */ } /* -- Vararg handling ----------------------------------------------------- */ /* Detect y = select(x, ...) idiom. */ static int select_detect(jit_State *J) { BCIns ins = J->pc[1]; if (bc_op(ins) == BC_CALLM && bc_b(ins) == 2 && bc_c(ins) == 1) { cTValue *func = &J->L->base[bc_a(ins)]; if (tvisfunc(func) && funcV(func)->c.ffid == FF_select) { TRef kfunc = lj_ir_kfunc(J, funcV(func)); emitir(IRTG(IR_EQ, IRT_FUNC), getslot(J, bc_a(ins)), kfunc); return 1; } } return 0; } /* Record vararg instruction. */ static void rec_varg(jit_State *J, BCReg dst, ptrdiff_t nresults) { int32_t numparams = J->pt->numparams; ptrdiff_t nvararg = frame_delta(J->L->base-1) - numparams - 1 - LJ_FR2; lj_assertJ(frame_isvarg(J->L->base-1), "VARG in non-vararg frame"); if (LJ_FR2 && dst > J->maxslot) J->base[dst-1] = 0; /* Prevent resurrection of unrelated slot. */ if (J->framedepth > 0) { /* Simple case: varargs defined on-trace. */ ptrdiff_t i; if (nvararg < 0) nvararg = 0; if (nresults == -1) { nresults = nvararg; J->maxslot = dst + (BCReg)nvararg; } else if (dst + nresults > J->maxslot) { J->maxslot = dst + (BCReg)nresults; } for (i = 0; i < nresults; i++) J->base[dst+i] = i < nvararg ? getslot(J, i - nvararg - 1 - LJ_FR2) : TREF_NIL; } else { /* Unknown number of varargs passed to trace. */ TRef fr = emitir(IRTI(IR_SLOAD), LJ_FR2, IRSLOAD_READONLY|IRSLOAD_FRAME); int32_t frofs = 8*(1+LJ_FR2+numparams)+FRAME_VARG; if (nresults >= 0) { /* Known fixed number of results. */ ptrdiff_t i; if (nvararg > 0) { ptrdiff_t nload = nvararg >= nresults ? nresults : nvararg; TRef vbase; if (nvararg >= nresults) emitir(IRTGI(IR_GE), fr, lj_ir_kint(J, frofs+8*(int32_t)nresults)); else emitir(IRTGI(IR_EQ), fr, lj_ir_kint(J, (int32_t)frame_ftsz(J->L->base-1))); vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr); vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase, lj_ir_kint(J, frofs-8*(1+LJ_FR2))); for (i = 0; i < nload; i++) { IRType t = itype2irt(&J->L->base[i-1-LJ_FR2-nvararg]); J->base[dst+i] = lj_record_vload(J, vbase, (MSize)i, t); } } else { emitir(IRTGI(IR_LE), fr, lj_ir_kint(J, frofs)); nvararg = 0; } for (i = nvararg; i < nresults; i++) J->base[dst+i] = TREF_NIL; if (dst + (BCReg)nresults > J->maxslot) J->maxslot = dst + (BCReg)nresults; } else if (select_detect(J)) { /* y = select(x, ...) */ TRef tridx = J->base[dst-1]; TRef tr = TREF_NIL; ptrdiff_t idx = lj_ffrecord_select_mode(J, tridx, &J->L->base[dst-1]); if (idx < 0) goto nyivarg; if (idx != 0 && !tref_isinteger(tridx)) tridx = emitir(IRTGI(IR_CONV), tridx, IRCONV_INT_NUM|IRCONV_INDEX); if (idx != 0 && tref_isk(tridx)) { emitir(IRTGI(idx <= nvararg ? IR_GE : IR_LT), fr, lj_ir_kint(J, frofs+8*(int32_t)idx)); frofs -= 8; /* Bias for 1-based index. */ } else if (idx <= nvararg) { /* Compute size. */ TRef tmp = emitir(IRTI(IR_ADD), fr, lj_ir_kint(J, -frofs)); if (numparams) emitir(IRTGI(IR_GE), tmp, lj_ir_kint(J, 0)); tr = emitir(IRTI(IR_BSHR), tmp, lj_ir_kint(J, 3)); if (idx != 0) { tridx = emitir(IRTI(IR_ADD), tridx, lj_ir_kint(J, -1)); rec_idx_abc(J, tr, tridx, (uint32_t)nvararg); } } else { TRef tmp = lj_ir_kint(J, frofs); if (idx != 0) { TRef tmp2 = emitir(IRTI(IR_BSHL), tridx, lj_ir_kint(J, 3)); tmp = emitir(IRTI(IR_ADD), tmp2, tmp); } else { tr = lj_ir_kint(J, 0); } emitir(IRTGI(IR_LT), fr, tmp); } if (idx != 0 && idx <= nvararg) { IRType t; TRef aref, vbase = emitir(IRT(IR_SUB, IRT_IGC), REF_BASE, fr); vbase = emitir(IRT(IR_ADD, IRT_PGC), vbase, lj_ir_kint(J, frofs-(8<L->base[idx-2-LJ_FR2-nvararg]); aref = emitir(IRT(IR_AREF, IRT_PGC), vbase, tridx); tr = lj_record_vload(J, aref, 0, t); } J->base[dst-2-LJ_FR2] = tr; J->maxslot = dst-1-LJ_FR2; J->bcskip = 2; /* Skip CALLM + select. */ } else { nyivarg: setintV(&J->errinfo, BC_VARG); lj_trace_err_info(J, LJ_TRERR_NYIBC); } } if (J->baseslot + J->maxslot >= LJ_MAX_JSLOTS) lj_trace_err(J, LJ_TRERR_STACKOV); } /* -- Record allocations -------------------------------------------------- */ static TRef rec_tnew(jit_State *J, uint32_t ah) { uint32_t asize = ah & 0x7ff; uint32_t hbits = ah >> 11; TRef tr; if (asize == 0x7ff) asize = 0x801; tr = emitir(IRTG(IR_TNEW, IRT_TAB), asize, hbits); #ifdef LUAJIT_ENABLE_TABLE_BUMP J->rbchash[(tr & (RBCHASH_SLOTS-1))].ref = tref_ref(tr); setmref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pc, J->pc); setgcref(J->rbchash[(tr & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt)); #endif return tr; } /* -- Concatenation ------------------------------------------------------- */ static TRef rec_cat(jit_State *J, BCReg baseslot, BCReg topslot) { TRef *top = &J->base[topslot]; TValue savetv[5+LJ_FR2]; BCReg s; RecordIndex ix; lj_assertJ(baseslot < topslot, "bad CAT arg"); for (s = baseslot; s <= topslot; s++) (void)getslot(J, s); /* Ensure all arguments have a reference. */ if (tref_isnumber_str(top[0]) && tref_isnumber_str(top[-1])) { TRef tr, hdr, *trp, *xbase, *base = &J->base[baseslot]; /* First convert numbers to strings. */ for (trp = top; trp >= base; trp--) { if (tref_isnumber(*trp)) *trp = emitir(IRT(IR_TOSTR, IRT_STR), *trp, tref_isnum(*trp) ? IRTOSTR_NUM : IRTOSTR_INT); else if (!tref_isstr(*trp)) break; } xbase = ++trp; tr = hdr = emitir(IRT(IR_BUFHDR, IRT_PGC), lj_ir_kptr(J, &J2G(J)->tmpbuf), IRBUFHDR_RESET); do { tr = emitir(IRTG(IR_BUFPUT, IRT_PGC), tr, *trp++); } while (trp <= top); tr = emitir(IRTG(IR_BUFSTR, IRT_STR), tr, hdr); J->maxslot = (BCReg)(xbase - J->base); if (xbase == base) return tr; /* Return simple concatenation result. */ /* Pass partial result. */ topslot = J->maxslot--; *xbase = tr; top = xbase; setstrV(J->L, &ix.keyv, &J2G(J)->strempty); /* Simulate string result. */ } else { J->maxslot = topslot-1; copyTV(J->L, &ix.keyv, &J->L->base[topslot]); } copyTV(J->L, &ix.tabv, &J->L->base[topslot-1]); ix.tab = top[-1]; ix.key = top[0]; memcpy(savetv, &J->L->base[topslot-1], sizeof(savetv)); /* Save slots. */ rec_mm_arith(J, &ix, MM_concat); /* Call __concat metamethod. */ memcpy(&J->L->base[topslot-1], savetv, sizeof(savetv)); /* Restore slots. */ return 0; /* No result yet. */ } /* -- Record bytecode ops ------------------------------------------------- */ /* Prepare for comparison. */ static void rec_comp_prep(jit_State *J) { /* Prevent merging with snapshot #0 (GC exit) since we fixup the PC. */ if (J->cur.nsnap == 1 && J->cur.snap[0].ref == J->cur.nins) emitir_raw(IRT(IR_NOP, IRT_NIL), 0, 0); lj_snap_add(J); } /* Fixup comparison. */ static void rec_comp_fixup(jit_State *J, const BCIns *pc, int cond) { BCIns jmpins = pc[1]; const BCIns *npc = pc + 2 + (cond ? bc_j(jmpins) : 0); SnapShot *snap = &J->cur.snap[J->cur.nsnap-1]; /* Set PC to opposite target to avoid re-recording the comp. in side trace. */ #if LJ_FR2 SnapEntry *flink = &J->cur.snapmap[snap->mapofs + snap->nent]; uint64_t pcbase; memcpy(&pcbase, flink, sizeof(uint64_t)); pcbase = (pcbase & 0xff) | (u64ptr(npc) << 8); memcpy(flink, &pcbase, sizeof(uint64_t)); #else J->cur.snapmap[snap->mapofs + snap->nent] = SNAP_MKPC(npc); #endif J->needsnap = 1; if (bc_a(jmpins) < J->maxslot) J->maxslot = bc_a(jmpins); lj_snap_shrink(J); /* Shrink last snapshot if possible. */ } /* Record the next bytecode instruction (_before_ it's executed). */ void lj_record_ins(jit_State *J) { cTValue *lbase; RecordIndex ix; const BCIns *pc; BCIns ins; BCOp op; TRef ra, rb, rc; /* Perform post-processing action before recording the next instruction. */ if (LJ_UNLIKELY(J->postproc != LJ_POST_NONE)) { switch (J->postproc) { case LJ_POST_FIXCOMP: /* Fixup comparison. */ pc = (const BCIns *)(uintptr_t)J2G(J)->tmptv.u64; rec_comp_fixup(J, pc, (!tvistruecond(&J2G(J)->tmptv2) ^ (bc_op(*pc)&1))); /* fallthrough */ case LJ_POST_FIXGUARD: /* Fixup and emit pending guard. */ case LJ_POST_FIXGUARDSNAP: /* Fixup and emit pending guard and snapshot. */ if (!tvistruecond(&J2G(J)->tmptv2)) { J->fold.ins.o ^= 1; /* Flip guard to opposite. */ if (J->postproc == LJ_POST_FIXGUARDSNAP) { SnapShot *snap = &J->cur.snap[J->cur.nsnap-1]; J->cur.snapmap[snap->mapofs+snap->nent-1]--; /* False -> true. */ } } lj_opt_fold(J); /* Emit pending guard. */ /* fallthrough */ case LJ_POST_FIXBOOL: if (!tvistruecond(&J2G(J)->tmptv2)) { BCReg s; TValue *tv = J->L->base; for (s = 0; s < J->maxslot; s++) /* Fixup stack slot (if any). */ if (J->base[s] == TREF_TRUE && tvisfalse(&tv[s])) { J->base[s] = TREF_FALSE; break; } } break; case LJ_POST_FIXCONST: { BCReg s; TValue *tv = J->L->base; for (s = 0; s < J->maxslot; s++) /* Constify stack slots (if any). */ if (J->base[s] == TREF_NIL && !tvisnil(&tv[s])) J->base[s] = lj_record_constify(J, &tv[s]); } break; case LJ_POST_FFRETRY: /* Suppress recording of retried fast function. */ if (bc_op(*J->pc) >= BC__MAX) return; break; default: lj_assertJ(0, "bad post-processing mode"); break; } J->postproc = LJ_POST_NONE; } /* Need snapshot before recording next bytecode (e.g. after a store). */ if (J->needsnap) { J->needsnap = 0; if (J->pt) lj_snap_purge(J); lj_snap_add(J); J->mergesnap = 1; } /* Skip some bytecodes. */ if (LJ_UNLIKELY(J->bcskip > 0)) { J->bcskip--; return; } /* Record only closed loops for root traces. */ pc = J->pc; if (J->framedepth == 0 && (MSize)((char *)pc - (char *)J->bc_min) >= J->bc_extent) lj_trace_err(J, LJ_TRERR_LLEAVE); #ifdef LUA_USE_ASSERT rec_check_slots(J); rec_check_ir(J); #endif #if LJ_HASPROFILE rec_profile_ins(J, pc); #endif /* Keep a copy of the runtime values of var/num/str operands. */ #define rav (&ix.valv) #define rbv (&ix.tabv) #define rcv (&ix.keyv) lbase = J->L->base; ins = *pc; op = bc_op(ins); ra = bc_a(ins); ix.val = 0; switch (bcmode_a(op)) { case BCMvar: copyTV(J->L, rav, &lbase[ra]); ix.val = ra = getslot(J, ra); break; default: break; /* Handled later. */ } rb = bc_b(ins); rc = bc_c(ins); switch (bcmode_b(op)) { case BCMnone: rb = 0; rc = bc_d(ins); break; /* Upgrade rc to 'rd'. */ case BCMvar: copyTV(J->L, rbv, &lbase[rb]); ix.tab = rb = getslot(J, rb); break; default: break; /* Handled later. */ } switch (bcmode_c(op)) { case BCMvar: copyTV(J->L, rcv, &lbase[rc]); ix.key = rc = getslot(J, rc); break; case BCMpri: setpriV(rcv, ~rc); ix.key = rc = TREF_PRI(IRT_NIL+rc); break; case BCMnum: { cTValue *tv = proto_knumtv(J->pt, rc); copyTV(J->L, rcv, tv); ix.key = rc = tvisint(tv) ? lj_ir_kint(J, intV(tv)) : tv->u32.hi == LJ_KEYINDEX ? (lj_ir_kint(J, 0) | TREF_KEYINDEX) : lj_ir_knumint(J, numV(tv)); } break; case BCMstr: { GCstr *s = gco2str(proto_kgc(J->pt, ~(ptrdiff_t)rc)); setstrV(J->L, rcv, s); ix.key = rc = lj_ir_kstr(J, s); } break; default: break; /* Handled later. */ } switch (op) { /* -- Comparison ops ---------------------------------------------------- */ case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT: #if LJ_HASFFI if (tref_iscdata(ra) || tref_iscdata(rc)) { rec_mm_comp_cdata(J, &ix, op, ((int)op & 2) ? MM_le : MM_lt); break; } #endif /* Emit nothing for two numeric or string consts. */ if (!(tref_isk2(ra,rc) && tref_isnumber_str(ra) && tref_isnumber_str(rc))) { IRType ta = tref_isinteger(ra) ? IRT_INT : tref_type(ra); IRType tc = tref_isinteger(rc) ? IRT_INT : tref_type(rc); int irop; if (ta != tc) { /* Widen mixed number/int comparisons to number/number comparison. */ if (ta == IRT_INT && tc == IRT_NUM) { ra = emitir(IRTN(IR_CONV), ra, IRCONV_NUM_INT); ta = IRT_NUM; } else if (ta == IRT_NUM && tc == IRT_INT) { rc = emitir(IRTN(IR_CONV), rc, IRCONV_NUM_INT); } else if (LJ_52) { ta = IRT_NIL; /* Force metamethod for different types. */ } else if (!((ta == IRT_FALSE || ta == IRT_TRUE) && (tc == IRT_FALSE || tc == IRT_TRUE))) { break; /* Interpreter will throw for two different types. */ } } rec_comp_prep(J); irop = (int)op - (int)BC_ISLT + (int)IR_LT; if (ta == IRT_NUM) { if ((irop & 1)) irop ^= 4; /* ISGE/ISGT are unordered. */ if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop)) irop ^= 5; } else if (ta == IRT_INT) { if (!lj_ir_numcmp(numberVnum(rav), numberVnum(rcv), (IROp)irop)) irop ^= 1; } else if (ta == IRT_STR) { if (!lj_ir_strcmp(strV(rav), strV(rcv), (IROp)irop)) irop ^= 1; ra = lj_ir_call(J, IRCALL_lj_str_cmp, ra, rc); rc = lj_ir_kint(J, 0); ta = IRT_INT; } else { rec_mm_comp(J, &ix, (int)op); break; } emitir(IRTG(irop, ta), ra, rc); rec_comp_fixup(J, J->pc, ((int)op ^ irop) & 1); } break; case BC_ISEQV: case BC_ISNEV: case BC_ISEQS: case BC_ISNES: case BC_ISEQN: case BC_ISNEN: case BC_ISEQP: case BC_ISNEP: #if LJ_HASFFI if (tref_iscdata(ra) || tref_iscdata(rc)) { rec_mm_comp_cdata(J, &ix, op, MM_eq); break; } #endif /* Emit nothing for two non-table, non-udata consts. */ if (!(tref_isk2(ra, rc) && !(tref_istab(ra) || tref_isudata(ra)))) { int diff; rec_comp_prep(J); diff = lj_record_objcmp(J, ra, rc, rav, rcv); if (diff == 2 || !(tref_istab(ra) || tref_isudata(ra))) rec_comp_fixup(J, J->pc, ((int)op & 1) == !diff); else if (diff == 1) /* Only check __eq if different, but same type. */ rec_mm_equal(J, &ix, (int)op); } break; /* -- Unary test and copy ops ------------------------------------------- */ case BC_ISTC: case BC_ISFC: if ((op & 1) == tref_istruecond(rc)) rc = 0; /* Don't store if condition is not true. */ /* fallthrough */ case BC_IST: case BC_ISF: /* Type specialization suffices. */ if (bc_a(pc[1]) < J->maxslot) J->maxslot = bc_a(pc[1]); /* Shrink used slots. */ break; case BC_ISTYPE: case BC_ISNUM: /* These coercions need to correspond with lj_meta_istype(). */ if (LJ_DUALNUM && rc == ~LJ_TNUMX+1) ra = lj_opt_narrow_toint(J, ra); else if (rc == ~LJ_TNUMX+2) ra = lj_ir_tonum(J, ra); else if (rc == ~LJ_TSTR+1) ra = lj_ir_tostr(J, ra); /* else: type specialization suffices. */ J->base[bc_a(ins)] = ra; break; /* -- Unary ops --------------------------------------------------------- */ case BC_NOT: /* Type specialization already forces const result. */ rc = tref_istruecond(rc) ? TREF_FALSE : TREF_TRUE; break; case BC_LEN: if (tref_isstr(rc)) rc = emitir(IRTI(IR_FLOAD), rc, IRFL_STR_LEN); else if (!LJ_52 && tref_istab(rc)) rc = emitir(IRTI(IR_ALEN), rc, TREF_NIL); else rc = rec_mm_len(J, rc, rcv); break; /* -- Arithmetic ops ---------------------------------------------------- */ case BC_UNM: if (tref_isnumber_str(rc)) { rc = lj_opt_narrow_unm(J, rc, rcv); } else { ix.tab = rc; copyTV(J->L, &ix.tabv, rcv); rc = rec_mm_arith(J, &ix, MM_unm); } break; case BC_ADDNV: case BC_SUBNV: case BC_MULNV: case BC_DIVNV: case BC_MODNV: /* Swap rb/rc and rbv/rcv. rav is temp. */ ix.tab = rc; ix.key = rc = rb; rb = ix.tab; copyTV(J->L, rav, rbv); copyTV(J->L, rbv, rcv); copyTV(J->L, rcv, rav); if (op == BC_MODNV) goto recmod; /* fallthrough */ case BC_ADDVN: case BC_SUBVN: case BC_MULVN: case BC_DIVVN: case BC_ADDVV: case BC_SUBVV: case BC_MULVV: case BC_DIVVV: { MMS mm = bcmode_mm(op); if (tref_isnumber_str(rb) && tref_isnumber_str(rc)) rc = lj_opt_narrow_arith(J, rb, rc, rbv, rcv, (int)mm - (int)MM_add + (int)IR_ADD); else rc = rec_mm_arith(J, &ix, mm); break; } case BC_MODVN: case BC_MODVV: recmod: if (tref_isnumber_str(rb) && tref_isnumber_str(rc)) rc = lj_opt_narrow_mod(J, rb, rc, rbv, rcv); else rc = rec_mm_arith(J, &ix, MM_mod); break; case BC_POW: if (tref_isnumber_str(rb) && tref_isnumber_str(rc)) rc = lj_opt_narrow_arith(J, rb, rc, rbv, rcv, IR_POW); else rc = rec_mm_arith(J, &ix, MM_pow); break; /* -- Miscellaneous ops ------------------------------------------------- */ case BC_CAT: rc = rec_cat(J, rb, rc); break; /* -- Constant and move ops --------------------------------------------- */ case BC_MOV: /* Clear gap of method call to avoid resurrecting previous refs. */ if (ra > J->maxslot) { #if LJ_FR2 memset(J->base + J->maxslot, 0, (ra - J->maxslot) * sizeof(TRef)); #else J->base[ra-1] = 0; #endif } break; case BC_KSTR: case BC_KNUM: case BC_KPRI: break; case BC_KSHORT: rc = lj_ir_kint(J, (int32_t)(int16_t)rc); break; case BC_KNIL: if (LJ_FR2 && ra > J->maxslot) J->base[ra-1] = 0; while (ra <= rc) J->base[ra++] = TREF_NIL; if (rc >= J->maxslot) J->maxslot = rc+1; break; #if LJ_HASFFI case BC_KCDATA: rc = lj_ir_kgc(J, proto_kgc(J->pt, ~(ptrdiff_t)rc), IRT_CDATA); break; #endif /* -- Upvalue and function ops ------------------------------------------ */ case BC_UGET: rc = rec_upvalue(J, rc, 0); break; case BC_USETV: case BC_USETS: case BC_USETN: case BC_USETP: rec_upvalue(J, ra, rc); break; /* -- Table ops --------------------------------------------------------- */ case BC_GGET: case BC_GSET: settabV(J->L, &ix.tabv, tabref(J->fn->l.env)); ix.tab = emitir(IRT(IR_FLOAD, IRT_TAB), getcurrf(J), IRFL_FUNC_ENV); ix.idxchain = LJ_MAX_IDXCHAIN; rc = lj_record_idx(J, &ix); break; case BC_TGETB: case BC_TSETB: setintV(&ix.keyv, (int32_t)rc); ix.key = lj_ir_kint(J, (int32_t)rc); /* fallthrough */ case BC_TGETV: case BC_TGETS: case BC_TSETV: case BC_TSETS: ix.idxchain = LJ_MAX_IDXCHAIN; rc = lj_record_idx(J, &ix); break; case BC_TGETR: case BC_TSETR: ix.idxchain = 0; rc = lj_record_idx(J, &ix); break; case BC_TSETM: rec_tsetm(J, ra, (BCReg)(J->L->top - J->L->base), (int32_t)rcv->u32.lo); break; case BC_TNEW: rc = rec_tnew(J, rc); break; case BC_TDUP: rc = emitir(IRTG(IR_TDUP, IRT_TAB), lj_ir_ktab(J, gco2tab(proto_kgc(J->pt, ~(ptrdiff_t)rc))), 0); #ifdef LUAJIT_ENABLE_TABLE_BUMP J->rbchash[(rc & (RBCHASH_SLOTS-1))].ref = tref_ref(rc); setmref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pc, pc); setgcref(J->rbchash[(rc & (RBCHASH_SLOTS-1))].pt, obj2gco(J->pt)); #endif break; /* -- Calls and vararg handling ----------------------------------------- */ case BC_ITERC: J->base[ra] = getslot(J, ra-3); J->base[ra+1+LJ_FR2] = getslot(J, ra-2); J->base[ra+2+LJ_FR2] = getslot(J, ra-1); { /* Do the actual copy now because lj_record_call needs the values. */ TValue *b = &J->L->base[ra]; copyTV(J->L, b, b-3); copyTV(J->L, b+1+LJ_FR2, b-2); copyTV(J->L, b+2+LJ_FR2, b-1); } lj_record_call(J, ra, (ptrdiff_t)rc-1); break; /* L->top is set to L->base+ra+rc+NARGS-1+1. See lj_dispatch_ins(). */ case BC_CALLM: rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2; /* fallthrough */ case BC_CALL: lj_record_call(J, ra, (ptrdiff_t)rc-1); break; case BC_CALLMT: rc = (BCReg)(J->L->top - J->L->base) - ra - LJ_FR2; /* fallthrough */ case BC_CALLT: lj_record_tailcall(J, ra, (ptrdiff_t)rc-1); break; case BC_VARG: rec_varg(J, ra, (ptrdiff_t)rb-1); break; /* -- Returns ----------------------------------------------------------- */ case BC_RETM: /* L->top is set to L->base+ra+rc+NRESULTS-1, see lj_dispatch_ins(). */ rc = (BCReg)(J->L->top - J->L->base) - ra + 1; /* fallthrough */ case BC_RET: case BC_RET0: case BC_RET1: #if LJ_HASPROFILE rec_profile_ret(J); #endif lj_record_ret(J, ra, (ptrdiff_t)rc-1); break; /* -- Loops and branches ------------------------------------------------ */ case BC_FORI: if (rec_for(J, pc, 0) != LOOPEV_LEAVE) J->loopref = J->cur.nins; break; case BC_JFORI: lj_assertJ(bc_op(pc[(ptrdiff_t)rc-BCBIAS_J]) == BC_JFORL, "JFORI does not point to JFORL"); if (rec_for(J, pc, 0) != LOOPEV_LEAVE) /* Link to existing loop. */ lj_record_stop(J, LJ_TRLINK_ROOT, bc_d(pc[(ptrdiff_t)rc-BCBIAS_J])); /* Continue tracing if the loop is not entered. */ break; case BC_FORL: rec_loop_interp(J, pc, rec_for(J, pc+((ptrdiff_t)rc-BCBIAS_J), 1)); break; case BC_ITERL: rec_loop_interp(J, pc, rec_iterl(J, *pc)); break; case BC_ITERN: rec_loop_interp(J, pc, rec_itern(J, ra, rb)); break; case BC_LOOP: rec_loop_interp(J, pc, rec_loop(J, ra, 1)); break; case BC_JFORL: rec_loop_jit(J, rc, rec_for(J, pc+bc_j(traceref(J, rc)->startins), 1)); break; case BC_JITERL: rec_loop_jit(J, rc, rec_iterl(J, traceref(J, rc)->startins)); break; case BC_JLOOP: rec_loop_jit(J, rc, rec_loop(J, ra, !bc_isret(bc_op(traceref(J, rc)->startins)) && bc_op(traceref(J, rc)->startins) != BC_ITERN)); break; case BC_IFORL: case BC_IITERL: case BC_ILOOP: case BC_IFUNCF: case BC_IFUNCV: lj_trace_err(J, LJ_TRERR_BLACKL); break; case BC_JMP: if (ra < J->maxslot) J->maxslot = ra; /* Shrink used slots. */ break; case BC_ISNEXT: rec_isnext(J, ra); break; /* -- Function headers -------------------------------------------------- */ case BC_FUNCF: rec_func_lua(J); break; case BC_JFUNCF: rec_func_jit(J, rc); break; case BC_FUNCV: rec_func_vararg(J); rec_func_lua(J); break; case BC_JFUNCV: /* Cannot happen. No hotcall counting for varag funcs. */ lj_assertJ(0, "unsupported vararg hotcall"); break; case BC_FUNCC: case BC_FUNCCW: lj_ffrecord_func(J); break; default: if (op >= BC__MAX) { lj_ffrecord_func(J); break; } /* fallthrough */ case BC_UCLO: case BC_FNEW: setintV(&J->errinfo, (int32_t)op); lj_trace_err_info(J, LJ_TRERR_NYIBC); break; } /* rc == 0 if we have no result yet, e.g. pending __index metamethod call. */ if (bcmode_a(op) == BCMdst && rc) { J->base[ra] = rc; if (ra >= J->maxslot) { #if LJ_FR2 if (ra > J->maxslot) J->base[ra-1] = 0; #endif J->maxslot = ra+1; } } #undef rav #undef rbv #undef rcv /* Limit the number of recorded IR instructions and constants. */ if (J->cur.nins > REF_FIRST+(IRRef)J->param[JIT_P_maxrecord] || J->cur.nk < REF_BIAS-(IRRef)J->param[JIT_P_maxirconst]) lj_trace_err(J, LJ_TRERR_TRACEOV); } /* -- Recording setup ----------------------------------------------------- */ /* Setup recording for a root trace started by a hot loop. */ static const BCIns *rec_setup_root(jit_State *J) { /* Determine the next PC and the bytecode range for the loop. */ const BCIns *pcj, *pc = J->pc; BCIns ins = *pc; BCReg ra = bc_a(ins); switch (bc_op(ins)) { case BC_FORL: J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns); pc += 1+bc_j(ins); J->bc_min = pc; break; case BC_ITERL: if (bc_op(pc[-1]) == BC_JLOOP) lj_trace_err(J, LJ_TRERR_LINNER); lj_assertJ(bc_op(pc[-1]) == BC_ITERC, "no ITERC before ITERL"); J->maxslot = ra + bc_b(pc[-1]) - 1; J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns); pc += 1+bc_j(ins); lj_assertJ(bc_op(pc[-1]) == BC_JMP, "ITERL does not point to JMP+1"); J->bc_min = pc; break; case BC_ITERN: lj_assertJ(bc_op(pc[1]) == BC_ITERL, "no ITERL after ITERN"); J->maxslot = ra; J->bc_extent = (MSize)(-bc_j(pc[1]))*sizeof(BCIns); J->bc_min = pc+2 + bc_j(pc[1]); J->state = LJ_TRACE_RECORD_1ST; /* Record the first ITERN, too. */ break; case BC_LOOP: /* Only check BC range for real loops, but not for "repeat until true". */ pcj = pc + bc_j(ins); ins = *pcj; if (bc_op(ins) == BC_JMP && bc_j(ins) < 0) { J->bc_min = pcj+1 + bc_j(ins); J->bc_extent = (MSize)(-bc_j(ins))*sizeof(BCIns); } J->maxslot = ra; pc++; break; case BC_RET: case BC_RET0: case BC_RET1: /* No bytecode range check for down-recursive root traces. */ J->maxslot = ra + bc_d(ins) - 1; break; case BC_FUNCF: /* No bytecode range check for root traces started by a hot call. */ J->maxslot = J->pt->numparams; pc++; break; case BC_CALLM: case BC_CALL: case BC_ITERC: /* No bytecode range check for stitched traces. */ pc++; break; default: lj_assertJ(0, "bad root trace start bytecode %d", bc_op(ins)); break; } return pc; } /* Setup for recording a new trace. */ void lj_record_setup(jit_State *J) { uint32_t i; /* Initialize state related to current trace. */ memset(J->slot, 0, sizeof(J->slot)); memset(J->chain, 0, sizeof(J->chain)); #ifdef LUAJIT_ENABLE_TABLE_BUMP memset(J->rbchash, 0, sizeof(J->rbchash)); #endif memset(J->bpropcache, 0, sizeof(J->bpropcache)); J->scev.idx = REF_NIL; setmref(J->scev.pc, NULL); J->baseslot = 1+LJ_FR2; /* Invoking function is at base[-1-LJ_FR2]. */ J->base = J->slot + J->baseslot; J->maxslot = 0; J->framedepth = 0; J->retdepth = 0; J->instunroll = J->param[JIT_P_instunroll]; J->loopunroll = J->param[JIT_P_loopunroll]; J->tailcalled = 0; J->loopref = 0; J->bc_min = NULL; /* Means no limit. */ J->bc_extent = ~(MSize)0; /* Emit instructions for fixed references. Also triggers initial IR alloc. */ emitir_raw(IRT(IR_BASE, IRT_PGC), J->parent, J->exitno); for (i = 0; i <= 2; i++) { IRIns *ir = IR(REF_NIL-i); ir->i = 0; ir->t.irt = (uint8_t)(IRT_NIL+i); ir->o = IR_KPRI; ir->prev = 0; } J->cur.nk = REF_TRUE; J->startpc = J->pc; setmref(J->cur.startpc, J->pc); if (J->parent) { /* Side trace. */ GCtrace *T = traceref(J, J->parent); TraceNo root = T->root ? T->root : J->parent; J->cur.root = (uint16_t)root; J->cur.startins = BCINS_AD(BC_JMP, 0, 0); /* Check whether we could at least potentially form an extra loop. */ if (J->exitno == 0 && T->snap[0].nent == 0) { /* We can narrow a FORL for some side traces, too. */ if (J->pc > proto_bc(J->pt) && bc_op(J->pc[-1]) == BC_JFORI && bc_d(J->pc[bc_j(J->pc[-1])-1]) == root) { lj_snap_add(J); rec_for_loop(J, J->pc-1, &J->scev, 1); goto sidecheck; } } else { J->startpc = NULL; /* Prevent forming an extra loop. */ } lj_snap_replay(J, T); sidecheck: if ((traceref(J, J->cur.root)->nchild >= J->param[JIT_P_maxside] || T->snap[J->exitno].count >= J->param[JIT_P_hotexit] + J->param[JIT_P_tryside])) { if (bc_op(*J->pc) == BC_JLOOP) { BCIns startins = traceref(J, bc_d(*J->pc))->startins; if (bc_op(startins) == BC_ITERN) rec_itern(J, bc_a(startins), bc_b(startins)); } lj_record_stop(J, LJ_TRLINK_INTERP, 0); } } else { /* Root trace. */ J->cur.root = 0; J->cur.startins = *J->pc; J->pc = rec_setup_root(J); /* Note: the loop instruction itself is recorded at the end and not ** at the start! So snapshot #0 needs to point to the *next* instruction. ** The one exception is BC_ITERN, which sets LJ_TRACE_RECORD_1ST. */ lj_snap_add(J); if (bc_op(J->cur.startins) == BC_FORL) rec_for_loop(J, J->pc-1, &J->scev, 1); else if (bc_op(J->cur.startins) == BC_ITERC) J->startpc = NULL; if (1 + J->pt->framesize >= LJ_MAX_JSLOTS) lj_trace_err(J, LJ_TRERR_STACKOV); } #if LJ_HASPROFILE J->prev_pt = NULL; J->prev_line = -1; #endif #ifdef LUAJIT_ENABLE_CHECKHOOK /* Regularly check for instruction/line hooks from compiled code and ** exit to the interpreter if the hooks are set. ** ** This is a compile-time option and disabled by default, since the ** hook checks may be quite expensive in tight loops. ** ** Note this is only useful if hooks are *not* set most of the time. ** Use this only if you want to *asynchronously* interrupt the execution. ** ** You can set the instruction hook via lua_sethook() with a count of 1 ** from a signal handler or another native thread. Please have a look ** at the first few functions in luajit.c for an example (Ctrl-C handler). */ { TRef tr = emitir(IRT(IR_XLOAD, IRT_U8), lj_ir_kptr(J, &J2G(J)->hookmask), IRXLOAD_VOLATILE); tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (LUA_MASKLINE|LUA_MASKCOUNT))); emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, 0)); } #endif } #undef IR #undef emitir_raw #undef emitir #endif