diff --git a/hphp/runtime/vm/translator/annotation.cpp b/hphp/runtime/vm/translator/annotation.cpp index 6b8463eff..6a985234d 100644 --- a/hphp/runtime/vm/translator/annotation.cpp +++ b/hphp/runtime/vm/translator/annotation.cpp @@ -14,10 +14,10 @@ +----------------------------------------------------------------------+ */ -#include "hphp/util/base.h" +#include "hphp/runtime/vm/translator/annotation.h" #include "hphp/runtime/vm/translator/translator.h" #include "hphp/runtime/vm/translator/translator-inline.h" -#include "hphp/runtime/vm/translator/annotation.h" +#include "hphp/util/base.h" namespace HPHP { namespace Transl { @@ -88,7 +88,6 @@ static void recordFunc(NormalizedInstruction& i, cr.m_type = Function; cr.m_func = func; s_callDB.insert(std::make_pair(sk, cr)); - i.directCall = true; } static void recordActRecPush(NormalizedInstruction& i, diff --git a/hphp/runtime/vm/translator/hopt/irtranslator.cpp b/hphp/runtime/vm/translator/hopt/irtranslator.cpp index 78acba083..a5aaea4e0 100644 --- a/hphp/runtime/vm/translator/hopt/irtranslator.cpp +++ b/hphp/runtime/vm/translator/hopt/irtranslator.cpp @@ -191,7 +191,14 @@ TranslatorX64::irTranslateLtGtOp(const Tracelet& t, assert(i.inputs[0]->outerType() != KindOfRef); assert(i.inputs[1]->outerType() != KindOfRef); - HHIR_UNIMPLEMENTED_WHEN((!i.isNative()), LtGtOp); + DataType leftType = i.inputs[0]->outerType(); + DataType rightType = i.inputs[1]->outerType(); + bool ok = TypeConstraint::equivDataTypes(leftType, rightType) && + (i.inputs[0]->isNull() || + leftType == KindOfBoolean || + i.inputs[0]->isInt()); + + HHIR_UNIMPLEMENTED_WHEN(!ok, LtGtOp); switch (op) { case OpLt : HHIR_EMIT(Lt); case OpLte : HHIR_EMIT(Lte); @@ -442,7 +449,8 @@ TranslatorX64::irTranslateAdd(const Tracelet& t, const NormalizedInstruction& i) { assert(i.inputs.size() == 2); - if (planInstrAdd_Array(i)) { + if (i.inputs[0]->valueType() == KindOfArray && + i.inputs[1]->valueType() == KindOfArray) { HHIR_EMIT(ArrayAdd); return; } @@ -759,6 +767,12 @@ TranslatorX64::irTranslateUnsetG(const Tracelet& t, HHIR_EMIT(UnsetG, gblName); } +void +TranslatorX64::irTranslateUnsetN(const Tracelet& t, + const NormalizedInstruction& i) { + HHIR_EMIT(UnsetN); +} + void TranslatorX64::irTranslateCGetS(const Tracelet& t, const NormalizedInstruction& i) { const int kPropNameIdx = 1; @@ -911,13 +925,6 @@ TranslatorX64::irTranslateUnsetL(const Tracelet& t, HHIR_EMIT(UnsetL, i.inputs[0]->location.offset); } -void -TranslatorX64::irTranslateReqLit(const Tracelet& t, - const NormalizedInstruction& i, - InclOpFlags flags) { - HHIR_UNIMPLEMENTED(ReqLit); -} - void TranslatorX64::irTranslateReqDoc(const Tracelet& t, const NormalizedInstruction& i) { @@ -1037,6 +1044,18 @@ TranslatorX64::irTranslateFPushFuncD(const Tracelet& t, HHIR_EMIT(FPushFuncD, (i.imm[0].u_IVA), (i.imm[1].u_SA)); } +void +TranslatorX64::irTranslateBPassC(const Tracelet& t, + const NormalizedInstruction& i) { + // No-op +} + +void +TranslatorX64::irTranslateBPassV(const Tracelet& t, + const NormalizedInstruction& i) { + // No-op +} + void TranslatorX64::irTranslateFPassCOp(const Tracelet& t, const NormalizedInstruction& i) { @@ -1537,54 +1556,8 @@ TranslatorX64::irTranslateInstrDefault(const Tracelet& t, }; const Opcode op = i.op(); - // Add to this switch the bytecodes that the IR handles but the base - // translator does not analyze and translate - switch (op) { - case OpLateBoundCls: - irTranslateLateBoundCls(t, i); - break; - case OpEmptyS: - irTranslateEmptyS(t, i); - break; - case OpEmptyG: - irTranslateEmptyG(t, i); - break; - case OpVGetS: - irTranslateVGetS(t, i); - break; - case OpIssetS: - irTranslateIssetS(t, i); - break; - case OpIssetG: - irTranslateIssetG(t, i); - break; - case OpUnsetN: - m_hhbcTrans->emitUnsetN(); - break; - case OpUnsetG: - irTranslateUnsetG(t, i); - break; - case OpBindG: - irTranslateBindG(t, i); - break; - case OpIterFree: - irTranslateIterFree(t, i); - break; - case OpBPassC: - case OpBPassV: - // OpBPass* instructions are no-ops - break; - case OpFPassV: - irTranslateFPassV(t, i); - break; - case OpBindS: - irTranslateBindS(t, i); - break; - default: - // GO: if you hit this, check opNames[op] and add support for it - HHIR_UNIMPLEMENTED_OP(opNames[op]); - assert(false); - } + HHIR_UNIMPLEMENTED_OP(opNames[op]); + assert(false); } void @@ -1693,14 +1666,7 @@ void TranslatorX64::irTranslateInstr(const Tracelet& t, m_hhbcTrans->setBcOff(i.source.offset(), i.breaksTracelet && !m_hhbcTrans->isInlining()); - if (!i.grouped) { - emitVariantGuards(t, i); - const NormalizedInstruction* n = &i; - while (n->next && n->next->grouped) { - n = n->next; - emitVariantGuards(t, *n); - } - } + emitVariantGuards(t, i); if (i.guardedThis) { // Task #2067635: This should really generate an AssertThis diff --git a/hphp/runtime/vm/translator/translator-x64.cpp b/hphp/runtime/vm/translator/translator-x64.cpp index c418e2408..237fb61eb 100644 --- a/hphp/runtime/vm/translator/translator-x64.cpp +++ b/hphp/runtime/vm/translator/translator-x64.cpp @@ -161,15 +161,6 @@ static const int kLocalsToInitializeInline = 9; // instruction pointers. static const uint64_t kUninitializedRIP = 0xba5eba11acc01ade; -static int -localOffset(int loc) { - PhysReg base; - int offset; - locToRegDisp(Location(Location::Local, loc), &base, &offset); - assert(base == rVmFp); - return offset; -} - // Return the SrcKey for the operation that should follow the supplied // NormalizedInstruction. (This might not be the next SrcKey in the // unit if we merged some instructions or otherwise modified them @@ -345,55 +336,6 @@ TranslatorX64::emitCopyToStack(X64Assembler& a, m_regMap.invalidate(ni.outStack->location); } -/* - * Emit code that does the same thing as tvSet(). - * - * The `oldType' and `oldData' registers are used for temporary - * storage and unconditionally destroyed. - * `toPtr' will be destroyed iff the cell we're storing to is - * KindOfRef. - * The variant check will not be performed if toOffset is nonzero, so - * only pass a nonzero offset if you know the destination is not - * KindOfRef. - * `from' will not be modified. - */ -void TranslatorX64::emitTvSetRegSafe(const NormalizedInstruction& i, - PhysReg from, - DataType fromType, - PhysReg toPtr, - int toOffset, - PhysReg oldType, - PhysReg oldData, - bool incRefFrom) { - assert(!i.isNative()); - assert(!i.isSimple()); - assert(fromType != KindOfRef); - - if (toOffset == 0) { - emitDerefIfVariant(a, toPtr); - } - - emitLoadTVType(a, toPtr[toOffset + TVOFF(m_type)], r32(oldType)); - a. loadq (toPtr[toOffset + TVOFF(m_data)], oldData); - emitStoreTypedValue(a, fromType, from, toOffset, toPtr); - if (incRefFrom) { - emitIncRef(from, fromType); - } - emitDecRefGenericReg(oldData, oldType); -} - -void TranslatorX64::emitTvSet(const NormalizedInstruction& i, - PhysReg from, - DataType fromType, - PhysReg toPtr, - int toOffset, - bool incRefFrom) { - ScratchReg oldType(m_regMap); - ScratchReg oldData(m_regMap); - emitTvSetRegSafe(i, from, fromType, toPtr, toOffset, - r(oldType), r(oldData), incRefFrom); -} - // Logical register move: ensures the value in src will be in dest // after execution, but might do so in strange ways. Do not count on // being able to smash dest to a different register in the future, e.g. @@ -822,9 +764,6 @@ TranslatorX64::recordCallImpl(X64Assembler& a, stackOff += getStackDelta(i); } stackOff += adjust; - assert(i.checkedInputs || - (reentrant && !i.isSimple()) || - (!reentrant && !i.isNative())); Offset pcOff = sk.offset() - curFunc()->base(); SKTRACE(2, sk, "record%sCall pcOff %d\n", reentrant ? "Reentrant" : "", int(pcOff)); @@ -3367,119 +3306,6 @@ TranslatorX64::poison(PhysReg dest) { } } -/** - * Spill all dirty registers, mark all registers as 'free' in the - * register file, and update rVmSp to point to the top of stack at - * the end of the tracelet. - */ -void -TranslatorX64::syncOutputs(const Tracelet& t) { - syncOutputs(t.m_stackChange); -} - -/** - * Same as above, except that it sets rVmSp to point to the top of - * stack at the beginning of the specified instruction. - */ -void -TranslatorX64::syncOutputs(const NormalizedInstruction& i) { - syncOutputs(i.stackOff); -} - -void -TranslatorX64::syncOutputs(int stackOff) { - SpaceRecorder sr("_SyncOuts", *m_spillFillCode); - TCA start = m_spillFillCode->code.frontier; - // Mark all stack locations above the top of stack as dead - m_regMap.scrubStackEntries(stackOff); - // Spill all dirty registers - m_regMap.cleanAll(); - if (stackOff != 0) { - TRACE(1, "syncOutputs: rVmSp + %d\n", stackOff); - // t.stackChange is in negative Cells, not bytes. - m_spillFillCode->add_imm32_reg64(-cellsToBytes(stackOff), rVmSp); - } - // All registers have been smashed for realz, yo - m_regMap.smashRegs(kAllRegs); - recordBCInstr(OpSyncOutputs, *m_spillFillCode, start); -} - -/* - * getBinaryStackInputs -- - * - * Helper for a common pattern of instruction, where two items are popped - * and one is pushed. The second item on the stack at the beginning of - * the instruction is both a source and destination. - */ -static void -getBinaryStackInputs(RegAlloc& regmap, const NormalizedInstruction& i, - PhysReg& rsrc, PhysReg& rsrcdest) { - assert(i.inputs.size() == 2); - assert(i.outStack && !i.outLocal); - rsrcdest = regmap.getReg(i.outStack->location); - rsrc = regmap.getReg(i.inputs[0]->location); - assert(regmap.getReg(i.inputs[1]->location) == rsrcdest); -} - -// emitBox -- -// Leave a boxed version of input in RAX. Destroys the register -// mapping. -void -TranslatorX64::emitBox(DataType t, PhysReg rSrc) { - if (false) { // typecheck - RefData* retval = tvBoxHelper(KindOfArray, 0xdeadbeef01ul); - (void)retval; - } - // tvBoxHelper will set the refcount of the inner cell to 1 - // for us. Because the inner cell now holds a reference to the - // original value, we don't need to perform a decRef. - EMIT_CALL(a, tvBoxHelper, IMM(t), R(rSrc)); -} - -// emitUnboxTopOfStack -- -// Unbox the known-to-be Variant on top of stack in place. -void -TranslatorX64::emitUnboxTopOfStack(const NormalizedInstruction& i) { - const vector& inputs = i.inputs; - - assert(inputs.size() == 1); - assert(i.outStack && !i.outLocal); - assert(inputs[0]->isStack()); - assert(i.outStack && i.outStack->location == inputs[0]->location); - DataType outType = inputs[0]->rtt.innerType(); - assert(outType != KindOfInvalid); - assert(outType == i.outStack->outerType()); - PhysReg rSrc = getReg(inputs[0]->location); - // Detach the register rSrc from the input location. We must - // do this dance because the input and output location are the - // same and we want to have separate registers for the input - // and output. - m_regMap.invalidate(inputs[0]->location); - ScratchReg rSrcScratch(m_regMap, rSrc); - // This call to allocOutputRegs will allocate a new register - // for the output location - m_regMap.allocOutputRegs(i); - PhysReg rDest = getReg(i.outStack->location); - emitDerefRef(a, rSrc, rDest); - emitIncRef(rDest, outType); - // decRef the var on the evaluation stack - emitDecRef(i, rSrc, KindOfRef); -} - -// setOpOpToOpcodeOp -- -// The SetOp opcode space has nothing to do with the bytecode opcode -// space. Reasonable people like it that way, so translate them here. -static Opcode -setOpOpToOpcodeOp(SetOpOp soo) { - switch(soo) { -#define SETOP_OP(_soo, _bc) case SetOp##_soo: return _bc; - SETOP_OPS -#undef SETOP_OP - default: assert(false); - } - return -1; -} - void TranslatorX64::getInputsIntoXMMRegs(const NormalizedInstruction& ni, PhysReg lr, PhysReg rr, @@ -3733,51 +3559,6 @@ TranslatorX64::syncWork() { Stats::inc(Stats::TC_Sync); } -void -TranslatorX64::emitInterpOne(const Tracelet& t, - const NormalizedInstruction& ni) { - // Write any dirty values to memory - m_regMap.cleanAll(); - // Call into the appropriate interpOne method. Note that this call will - // preserve the callee-saved registers including rVmFp and rVmSp. - if (false) { /* typecheck */ - UNUSED VMExecutionContext* ec = interpOnePopC((ActRec*)vmfp(), vmsp(), 0); - } - void* func = interpOneEntryPoints[ni.op()]; - TRACE(3, "ip %p of unit %p -> interpOne @%p\n", ni.pc(), ni.unit(), func); - EMIT_CALL(a, func, - R(rVmFp), - RPLUS(rVmSp, -int32_t(cellsToBytes(ni.stackOff))), - IMM(ni.source.offset())); - // The interpreter may have written to memory, so we need to invalidate - // all locations - m_regMap.reset(); - // The interpOne method returned a pointer to the current - // ExecutionContext in rax, so we can read the 'm_*' fields - // by adding the appropriate offset to rax and dereferencing. - - // If this instruction ends the tracelet, we have some extra work to do. - if (ni.breaksTracelet) { - // Read the 'm_fp' and 'm_stack.m_top' fields into the rVmFp and - // rVmSp registers. - a. load_reg64_disp_reg64(rax, offsetof(VMExecutionContext, m_fp), - rVmFp); - a. load_reg64_disp_reg64(rax, offsetof(VMExecutionContext, m_stack) + - Stack::topOfStackOffset(), rVmSp); - if (opcodeChangesPC(ni.op())) { - // If interpreting this instruction can potentially set PC to point - // to something other than the next instruction in the bytecode, so - // we need to emit a service request to figure out where to go next - TCA stubDest = emitServiceReq(REQ_RESUME, 0ull); - a. jmp(stubDest); - } else { - // If this instruction always advances PC to the next instruction in - // the bytecode, then we know what SrcKey to bind to - emitBindJmp(nextSrcKey(t, ni)); - } - } -} - // could be static but used in hopt/codegen.cpp void raiseUndefVariable(StringData* nm) { raise_notice(Strings::UNDEFINED_VARIABLE, nm->data()); @@ -3794,1460 +3575,6 @@ mathEquivTypes(RuntimeType lt, RuntimeType rt) { (lt.isDouble() && rt.isInt()); } -static TXFlags -planBinaryArithOp(const NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - if (mathEquivTypes(i.inputs[0]->rtt, i.inputs[1]->rtt)) { - auto op = i.op(); - return nativePlan(op == OpMul || op == OpAdd || op == OpSub); - } - return nativePlan(i.inputs[0]->isInt() && i.inputs[1]->isInt()); -} - -void -TranslatorX64::analyzeBinaryArithOp(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = planBinaryArithOp(i); -} - -void -TranslatorX64::translateBinaryArithOp(const Tracelet& t, - const NormalizedInstruction& i) { - const Opcode op = i.op(); - assert(op == OpSub || op == OpMul || op == OpBitAnd || - op == OpBitOr || op == OpBitXor); - assert(planBinaryArithOp(i)); - assert(i.inputs.size() == 2); - - binaryArithCell(i, op, *i.inputs[0], *i.outStack); -} - -static inline bool sameDataTypes(DataType t1, DataType t2) { - return TypeConstraint::equivDataTypes(t1, t2); -} - -static TXFlags -planSameOp_SameTypes(const NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - const RuntimeType& left = i.inputs[0]->rtt; - const RuntimeType& right = i.inputs[1]->rtt; - DataType leftType = left.outerType(); - DataType rightType = right.outerType(); - return nativePlan(sameDataTypes(leftType, rightType) && - (left.isNull() || leftType == KindOfBoolean || - left.isInt() || left.isString())); -} - -static TXFlags -planSameOp_DifferentTypes(const NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - DataType leftType = i.inputs[0]->outerType(); - DataType rightType = i.inputs[1]->outerType(); - if (!sameDataTypes(leftType, rightType)) { - if (IS_REFCOUNTED_TYPE(leftType) || IS_REFCOUNTED_TYPE(rightType)) { - // For dissimilar datatypes, we might call out to handle a refcount. - return Supported; - } - return Native; - } - return Interp; -} - -void -TranslatorX64::analyzeSameOp(Tracelet& t, NormalizedInstruction& i) { - assert(!(planSameOp_SameTypes(i) && planSameOp_DifferentTypes(i))); - i.m_txFlags = TXFlags(planSameOp_SameTypes(i) | planSameOp_DifferentTypes(i)); - i.manuallyAllocInputs = true; -} - -void -TranslatorX64::translateSameOp(const Tracelet& t, - const NormalizedInstruction& i) { - const Opcode op = i.op(); - assert(op == OpSame || op == OpNSame); - const vector& inputs = i.inputs; - bool instrNeg = (op == OpNSame); - assert(inputs.size() == 2); - assert(i.outStack && !i.outLocal); - DataType leftType = i.inputs[0]->outerType(); - DataType rightType DEBUG_ONLY = i.inputs[1]->outerType(); - assert(leftType != KindOfRef); - assert(rightType != KindOfRef); - - if (planSameOp_DifferentTypes(i)) { - // Some easy cases: when the valueTypes do not match, - // NSame -> true and Same -> false. - SKTRACE(1, i.source, "different types %d %d\n", - leftType, rightType); - emitDecRefInput(a, i, 0); - emitDecRefInput(a, i, 1); - m_regMap.allocOutputRegs(i); - emitImmReg(a, instrNeg, getReg(i.outStack->location)); - return; // Done - } - - assert(planSameOp_SameTypes(i)); - - if (IS_NULL_TYPE(leftType)) { - m_regMap.allocOutputRegs(i); - // null === null is always true - SKTRACE(2, i.source, "straightening null/null comparison\n"); - emitImmReg(a, !instrNeg, getReg(i.outStack->location)); - return; // Done - } - if (IS_STRING_TYPE(leftType)) { - int args[2]; - args[0] = 0; - args[1] = 1; - allocInputsForCall(i, args); - EMIT_CALL(a, same_str_str, - V(inputs[0]->location), - V(inputs[1]->location)); - if (instrNeg) { - a. xor_imm32_reg32(1, rax); - } - m_regMap.bind(rax, i.outStack->location, i.outStack->outerType(), - RegInfo::DIRTY); - return; // Done - } - m_regMap.allocInputRegs(i); - PhysReg src, srcdest; - getBinaryStackInputs(m_regMap, i, src, srcdest); - m_regMap.allocOutputRegs(i); - assert(getReg(i.outStack->location) == srcdest); - a. cmp_reg64_reg64(src, srcdest); - if (op == OpSame) { - a. sete(rbyte(srcdest)); - } else { - a. setne(rbyte(srcdest)); - } - a. movzbl (rbyte(srcdest), r32(srcdest)); -} - -static bool -trivialEquivType(const RuntimeType& rtt) { - DataType t = rtt.valueType(); - return t == KindOfUninit || t == KindOfNull || t == KindOfBoolean || - rtt.isInt() || rtt.isString(); -} - -static void -emitConvertToBool(X64Assembler &a, PhysReg src, PhysReg dest, bool instrNeg) { - a. test_reg64_reg64(src, src); - if (instrNeg) { - a. setz(rbyte(dest)); - } else { - a. setnz(rbyte(dest)); - } - a. movzbl (rbyte(dest), r32(dest)); -} - -void -TranslatorX64::analyzeEqOp(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - RuntimeType < = i.inputs[0]->rtt; - RuntimeType &rt = i.inputs[1]->rtt; - i.m_txFlags = nativePlan(trivialEquivType(lt) && - trivialEquivType(rt)); - if (!i.m_txFlags) { - i.m_txFlags = nativePlan(mathEquivTypes(lt, rt)); - } - if (i.isNative() && - IS_NULL_TYPE(lt.outerType()) && - IS_NULL_TYPE(rt.outerType())) { - i.manuallyAllocInputs = true; - } -} - -void -TranslatorX64::fpEq(const NormalizedInstruction& ni, - PhysReg lr, PhysReg rr) { - getInputsIntoXMMRegs(ni, lr, rr, xmm0, xmm1); - m_regMap.allocOutputRegs(ni); - a. ucomisd_xmm_xmm(xmm0, xmm1); - semiLikelyIfBlock(CC_P, a, [&] { - // PF means unordered; treat it as !eq. Or 1 into anything at all - // to clear ZF. - a. or_imm32_reg64(1, reg::rAsm); - }); -} - -void -TranslatorX64::translateEqOp(const Tracelet& t, - const NormalizedInstruction& i) { - const Opcode op = i.op(); - assert(op == OpEq || op == OpNeq); - assert(i.isNative()); - const vector& inputs = i.inputs; - bool instrNeg = (op == OpNeq); - assert(inputs.size() == 2); - assert(i.outStack && !i.outLocal); - DataType leftType = i.inputs[0]->outerType(); - DataType rightType = i.inputs[1]->outerType(); - assert(leftType != KindOfRef); - assert(rightType != KindOfRef); - - ConditionCode ccBranch = CC_E; - if (instrNeg) ccBranch = ccNegate(ccBranch); - - // Inputless case. - if (IS_NULL_TYPE(leftType) && IS_NULL_TYPE(rightType)) { - assert(i.manuallyAllocInputs); - // null == null is always true - bool result = !instrNeg; - SKTRACE(2, i.source, "straightening null/null comparison\n"); - if (i.changesPC) { - fuseBranchAfterStaticBool(a, t, i, result); - } else { - m_regMap.allocOutputRegs(i); - emitImmReg(a, result, getReg(i.outStack->location)); - } - return; // Done - } - - if (IS_STRING_TYPE(leftType) || IS_STRING_TYPE(rightType)) { - void* fptr = nullptr; - bool leftIsString = false; - bool eqNullStr = false; - switch (leftType) { - STRINGCASE(): { - leftIsString = true; - switch (rightType) { - STRINGCASE(): fptr = (void*)eq_str_str; break; - case KindOfInt64: fptr = (void*)eq_int_str; break; - case KindOfBoolean: fptr = (void*)eq_bool_str; break; - NULLCASE(): fptr = (void*)eq_null_str; eqNullStr = true; break; - default: assert(false); break; - } - } break; - case KindOfInt64: fptr = (void*)eq_int_str; break; - case KindOfBoolean: fptr = (void*)eq_bool_str; break; - NULLCASE(): fptr = (void*)eq_null_str; eqNullStr = true; break; - default: assert(false); break; - } - if (eqNullStr) { - assert(fptr == (void*)eq_null_str); - EMIT_CALL(a, fptr, - V(inputs[leftIsString ? 0 : 1]->location)); - } else { - assert(fptr != nullptr); - EMIT_CALL(a, fptr, - V(inputs[leftIsString ? 1 : 0]->location), - V(inputs[leftIsString ? 0 : 1]->location)); - } - if (i.changesPC) { - fuseBranchSync(t, i); - prepareForTestAndSmash(a, kTestImmRegLen, kAlignJccAndJmp); - a. testb (1, al); - fuseBranchAfterBool(t, i, ccNegate(ccBranch)); - return; - } - m_regMap.bind(rax, i.outStack->location, i.outStack->outerType(), - RegInfo::DIRTY); - if (instrNeg) { - a. xor_imm32_reg32(1, rax); - } - return; - } - - m_regMap.allocOutputRegs(i); - PhysReg src, srcdest; - getBinaryStackInputs(m_regMap, i, src, srcdest); - assert(getReg(i.outStack->location) == srcdest); - if (i.changesPC) { - fuseBranchSync(t, i); - } - if (IS_NULL_TYPE(leftType) || IS_NULL_TYPE(rightType)) { - prepareForTestAndSmash(a, kTestRegRegLen, kAlignJccAndJmp); - if (IS_NULL_TYPE(leftType)) { - a. test_reg64_reg64(srcdest, srcdest); - } else { - assert(IS_NULL_TYPE(rightType)); - a. test_reg64_reg64(src, src); - } - } else if (leftType == KindOfBoolean || - rightType == KindOfBoolean) { - // OK to destroy src and srcdest in-place; their stack locations are - // blown away by this instruction. - if (leftType != KindOfBoolean) - emitConvertToBool(a, src, src, false); - if (rightType != KindOfBoolean) - emitConvertToBool(a, srcdest, srcdest, false); - a. cmp_reg64_reg64(src, srcdest); - } else if (leftType == KindOfDouble || rightType == KindOfDouble) { - fpEq(i, src, srcdest); - } else { - a. cmp_reg64_reg64(src, srcdest); - } - if (i.changesPC) { - fuseBranchAfterBool(t, i, ccBranch); - return; - } - if (instrNeg) { - a. setnz (rbyte(srcdest)); - } else { - a. setz (rbyte(srcdest)); - } - a. movzbl (rbyte(srcdest), r32(srcdest)); -} - -void -TranslatorX64::analyzeLtGtOp(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - const RuntimeType& left = i.inputs[0]->rtt; - DataType leftType = left.outerType(); - DataType rightType = i.inputs[1]->outerType(); - i.m_txFlags = nativePlan(sameDataTypes(leftType, rightType) && - (left.isNull() || - leftType == KindOfBoolean || - left.isInt())); - if (i.isNative() && IS_NULL_TYPE(left.outerType())) { - // No inputs. w00t. - i.manuallyAllocInputs = true; - } -} - -void -TranslatorX64::translateLtGtOp(const Tracelet& t, - const NormalizedInstruction& i) { - const Opcode op = i.op(); - assert(op == OpLt || op == OpLte || op == OpGt || op == OpGte); - assert(i.inputs.size() == 2); - assert(i.outStack && !i.outLocal); - assert(i.inputs[0]->outerType() != KindOfRef); - assert(i.inputs[1]->outerType() != KindOfRef); - assert(i.isNative()); - - bool fEquals = (op == OpLte || op == OpGte); - bool fLessThan = (op == OpLt || op == OpLte); - - m_regMap.allocOutputRegs(i); - if (IS_NULL_TYPE(i.inputs[0]->outerType())) { - assert(IS_NULL_TYPE(i.inputs[1]->outerType())); - // null < null is always false, null <= null is always true - SKTRACE(2, i.source, "straightening null/null comparison\n"); - PhysReg rOut = getReg(i.outStack->location); - bool resultIsTrue = (op == OpLte || op == OpGte); - if (i.changesPC) { - fuseBranchAfterStaticBool(a, t, i, resultIsTrue); - } else { - emitImmReg(a, resultIsTrue, rOut); - } - return; - } - PhysReg src, srcdest; - getBinaryStackInputs(m_regMap, i, src, srcdest); - assert(getReg(i.outStack->location) == srcdest); - if (i.changesPC) { - fuseBranchSync(t, i); - } - a. cmp_reg64_reg64(src, srcdest); - static const ConditionCode opToCc[2][2] = { - // !fEquals fEquals - { CC_G, CC_GE }, // !fLessThan - { CC_L, CC_LE }, // fLessThan - }; - ConditionCode cc = opToCc[fLessThan][fEquals]; - if (i.changesPC) { - // Fuse the coming branch. - fuseBranchAfterBool(t, i, cc); - return; - } - a. setcc(cc, rbyte(srcdest)); - a. movzbl (rbyte(srcdest), r32(srcdest)); -} - -static TXFlags -planUnaryBooleanOp(const NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - RuntimeType& rtt = i.inputs[0]->rtt; - DataType inType = rtt.valueType(); - if (inType == KindOfArray) { - return Supported; - } - if (rtt.isString()) { - return Simple; - } - return nativePlan(rtt.isNull() || - inType == KindOfBoolean || rtt.isInt()); -} - -void -TranslatorX64::analyzeUnaryBooleanOp(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = planUnaryBooleanOp(i); -} - -void -TranslatorX64::translateUnaryBooleanOp(const Tracelet& t, - const NormalizedInstruction& i) { - const Opcode op = i.op(); - assert(op == OpCastBool || op == OpEmptyL); - const vector& inputs = i.inputs; - assert(inputs.size() == 1); - assert(i.outStack && !i.outLocal); - bool instrNeg = (op == OpEmptyL); - DataType inType = inputs[0]->valueType(); - const Location& inLoc = inputs[0]->location; - bool boxedForm = (inputs[0]->outerType() == KindOfRef); - - - switch (inType) { - NULLCASE(): { - m_regMap.allocOutputRegs(i); - PhysReg outReg = getReg(i.outStack->location); - emitImmReg(a, instrNeg, outReg); - assert(i.isNative()); - } break; - case KindOfBoolean: { - if (op == OpCastBool) { - // Casting bool to bool is a nop. CastBool's input must be - // a cell on the stack as per the bytecode specification. - assert(inputs[0]->isStack()); - assert(inputs[0]->outerType() != KindOfRef); - assert(inputs[0]->location.space == Location::Stack); - assert(i.isNative()); - break; - } - m_regMap.allocOutputRegs(i); - PhysReg reg = getReg(inLoc); - PhysReg outReg = getReg(i.outStack->location); - if (boxedForm) { - emitDerefRef(a, reg, outReg); - } else { - emitMovRegReg(a, reg, outReg); - } - if (instrNeg) { - a. xor_imm32_reg32(1, outReg); - } - } break; - case KindOfInt64: { - m_regMap.allocOutputRegs(i); - PhysReg reg = getReg(inLoc); - PhysReg outReg = getReg(i.outStack->location); - ScratchReg scratch(m_regMap); - if (boxedForm) { - emitDerefRef(a, reg, r(scratch)); - emitConvertToBool(a, r(scratch), outReg, instrNeg); - } else { - emitConvertToBool(a, reg, outReg, instrNeg); - } - } break; - STRINGCASE(): - case KindOfArray: { - bool doDecRef = (inputs[0]->isStack()); - void* fptr = IS_STRING_TYPE(inType) ? - (doDecRef ? (void*)str_to_bool : (void*)str0_to_bool) : - (doDecRef ? (void*)arr_to_bool : (void*)arr0_to_bool); - if (boxedForm) { - EMIT_CALL(a, fptr, DEREF(inLoc)); - } else { - EMIT_CALL(a, fptr, V(inLoc)); - } - if (!IS_STRING_TYPE(inType)) { - recordReentrantCall(i); - } - if (instrNeg) { - a. xor_imm32_reg32(1, rax); - } - m_regMap.bind(rax, i.outStack->location, i.outStack->outerType(), - RegInfo::DIRTY); - } break; - default: { - assert(false); - } break; - } -} - -void -TranslatorX64::analyzeBranchOp(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = Supported; -} - -// Helper for decoding dests of branch-like instructions at the end of -// a basic block. -static void branchDests(const Tracelet& t, - const NormalizedInstruction& i, - SrcKey* outTaken, SrcKey* outNotTaken, - int immIdx = 0) { - *outNotTaken = nextSrcKey(t, i); - int dest = i.imm[immIdx].u_BA; - *outTaken = SrcKey(curFunc(), i.offset() + dest); -} - -void TranslatorX64::branchWithFlagsSet(const Tracelet& t, - const NormalizedInstruction& i, - ConditionCode cc) { - assert(i.op() == OpJmpNZ || i.op() == OpJmpZ); - // not_taken - SrcKey taken, notTaken; - branchDests(t, i, &taken, ¬Taken); - TRACE(3, "branchWithFlagsSet %d %d cc%02x jmp%sz\n", - taken.offset(), notTaken.offset(), cc, - i.isJmpNZ() ? "n" : ""); - emitCondJmp(taken, notTaken, cc); -} - -void TranslatorX64::fuseBranchAfterStaticBool(Asm& a, - const Tracelet& t, - const NormalizedInstruction& i, - bool resultIsTrue, - bool doSync) { - assert(i.breaksTracelet); - assert(i.next); - NormalizedInstruction &nexti = *i.next; - if (doSync) { - fuseBranchSync(t, i); - } else { - assert(m_regMap.branchSynced()); - } - bool isTaken = (resultIsTrue == nexti.isJmpNZ()); - SrcKey taken, notTaken; - branchDests(t, nexti, &taken, ¬Taken); - if (isTaken) { - emitBindJmp(a, taken); - } else { - emitBindJmp(a, notTaken); - } -} - -void TranslatorX64::fuseBranchAfterHelper(const Tracelet& t, - const NormalizedInstruction& i) { - fuseBranchSync(t, i); - a.test_reg64_reg64(rax, rax); - fuseBranchAfterBool(t, i, CC_NZ); -} - -void TranslatorX64::fuseBranchSync(const Tracelet& t, - const NormalizedInstruction& i) { - assert(!m_regMap.branchSynced()); - // Don't bother sync'ing the output of this instruction. - m_regMap.scrubStackEntries(i.outStack->location.offset); - syncOutputs(t); - m_regMap.setBranchSynced(); -} - -void TranslatorX64::fuseBranchAfterBool(const Tracelet& t, - const NormalizedInstruction& i, - ConditionCode cc) { - assert(m_regMap.branchSynced() && i.breaksTracelet && i.next); - NormalizedInstruction &nexti = *i.next; - if (!i.next->isJmpNZ()) cc = ccNegate(cc); - branchWithFlagsSet(t, nexti, cc); -} - -/* - * Fusing "half" of a branch is useful in situations where you would - * otherwise emit a jcc to or over a fuseStaticBranch. Pass in the - * condition code and whether that CC means the branch is taken or - * not. For example, if %rax == 0 means that your branch is not taken - * (but %rax != 0 means you have to do more checks), do something like - * this: - * - * a.test_reg64_reg64(rax, rax); - * fuseHalfBranchAfterBool(t, i, CC_Z, false); - * // ...more comparisons - */ -void TranslatorX64::fuseHalfBranchAfterBool(const Tracelet& t, - const NormalizedInstruction& i, - ConditionCode cc, - bool taken) { - assert(m_regMap.branchSynced() && i.breaksTracelet && i.next); - SrcKey destTaken, destNotTaken; - branchDests(t, *i.next, &destTaken, &destNotTaken); - if (!i.next->isJmpNZ()) taken = !taken; - emitBindJcc(a, cc, taken ? destTaken : destNotTaken); -} - -void -TranslatorX64::translateBranchOp(const Tracelet& t, - const NormalizedInstruction& i) { - DEBUG_ONLY const Opcode op = i.op(); - assert(op == OpJmpZ || op == OpJmpNZ); - - bool isZ = !i.isJmpNZ(); - assert(i.inputs.size() == 1); - assert(!i.outStack && !i.outLocal && !i.outStack2 && !i.outStack3); - const DynLocation& in = *i.inputs[0]; - const RuntimeType& rtt = in.rtt; - const Location& inLoc = in.location; - DataType inputType = rtt.outerType(); - PhysReg src = getReg(inLoc); - /* - * Careful. We're operating with intimate knowledge of the - * constraints of the register allocator from here out. - */ - if (rtt.isString() || inputType == KindOfArray) { - // str_to_bool and arr_to_bool will decRef for us - bool inStr = rtt.isString(); - void* fptr = inStr ? (void*)str_to_bool : (void*)arr_to_bool; - EMIT_CALL(a, fptr, V(inLoc)); - if (!inStr) recordReentrantCall(i); - src = rax; - ScratchReg sr(m_regMap, rax); - syncOutputs(t); - } else if (inputType != KindOfUninit && - inputType != KindOfNull && - inputType != KindOfBoolean && - !rtt.isInt()) { - // input might be in-flight - m_regMap.cleanLoc(inLoc); - // Cast to a bool. - if (false) { - TypedValue *tv = nullptr; - int64_t ret = tv_to_bool(tv); - if (ret) { - printf("zoot"); - } - } - TRACE(2, Trace::prettyNode("tv_to_bool", inLoc) + string("\n")); - // tv_to_bool will decRef for us if appropriate - EMIT_CALL(a, tv_to_bool, A(inLoc)); - recordReentrantCall(i); - src = rax; - ScratchReg sr(m_regMap, rax); - syncOutputs(t); - } else { - syncOutputs(t); - } - - // not_taken - SrcKey taken, notTaken; - branchDests(t, i, &taken, ¬Taken); - - // Since null always evaluates to false, we can emit an - // unconditional jump. OpJmpNZ will never take the branch - // while OpJmpZ will always take the branch. - if (IS_NULL_TYPE(inputType)) { - TRACE(1, "branch on Null -> always Z\n"); - emitBindJmp(isZ ? taken : notTaken); - return; - } - prepareForTestAndSmash(a, kTestRegRegLen, kAlignJccAndJmp); - a. test_reg64_reg64(src, src); - branchWithFlagsSet(t, i, isZ ? CC_Z : CC_NZ); -} - -void -TranslatorX64::analyzeCGetL(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - const RuntimeType& type = i.inputs[0]->rtt; - i.m_txFlags = (type.isUninit() || GuardType(type).mayBeUninit()) ? - Supported : Native; -} - -void -TranslatorX64::translateCGetL(const Tracelet& t, - const NormalizedInstruction& i) { - const DEBUG_ONLY Opcode op = i.op(); - assert(op == OpFPassL || OpCGetL); - const vector& inputs = i.inputs; - assert(inputs.size() == 1); - assert(inputs[0]->isLocal()); - DataType outType = inputs[0]->valueType(); - assert(outType != KindOfInvalid); - - if (GuardType(outType).isRelaxed()) { - assert(outType == KindOfUncountedInit); - PhysReg locBase, stackBase; - int locDisp, stackDisp; - locToRegDisp(inputs[0]->location, &locBase, &locDisp); - locToRegDisp(i.outStack->location, &stackBase, &stackDisp); - if (i.manuallyAllocInputs && !m_regMap.hasReg(inputs[0]->location)) { - emitCopyToAligned(a, locBase, locDisp, stackBase, stackDisp); - } else { - ScratchReg rTmp(m_regMap); - PhysReg localReg = getReg(inputs[0]->location); - a. storeq (localReg, stackBase[stackDisp + TVOFF(m_data)]); - emitLoadTVType(a, locBase[locDisp + TVOFF(m_type)], r(rTmp)); - emitStoreTVType(a, r(rTmp), stackBase[stackDisp + TVOFF(m_type)]); - } - return; - } - - // Check for use of an undefined local. - if (inputs[0]->rtt.isUninit()) { - assert(!i.outStack || i.outStack->outerType() == KindOfNull); - outType = KindOfNull; - assert(inputs[0]->location.offset < curFunc()->numLocals()); - const StringData* name = local_name(inputs[0]->location); - EMIT_CALL(a, raiseUndefVariable, IMM((uintptr_t)name)); - recordReentrantCall(i); - if (i.outStack) { - m_regMap.allocOutputRegs(i); - } - return; - } - - /* - * we can merge a CGetL with a following InstanceOfD - * to avoid the incRef/decRef on the result (see - * analyzeSecondPass). - * - * outStack will be clear in that case. - */ - if (!i.outStack) return; - assert(outType == i.outStack->outerType()); - m_regMap.allocOutputRegs(i); - if (IS_NULL_TYPE(outType)) return; - PhysReg dest = getReg(i.outStack->location); - - if (i.manuallyAllocInputs && !m_regMap.hasReg(inputs[0]->location)) { - fill(inputs[0]->location, dest); - } else { - PhysReg localReg = getReg(inputs[0]->location); - emitMovRegReg(a, localReg, dest); - } - if (inputs[0]->isRef()) { - emitDerefRef(a, dest, dest); - } - assert(outType != KindOfStaticString); - emitIncRef(dest, outType); -} - -void -TranslatorX64::analyzeCGetL2(Tracelet& t, - NormalizedInstruction& ni) { - const int locIdx = 1; - assert(ni.inputs.size() == 2); - ni.m_txFlags = ni.inputs[locIdx]->rtt.isUninit() ? Supported : Native; -} - -void -TranslatorX64::translateCGetL2(const Tracelet& t, - const NormalizedInstruction& ni) { - const int stackIdx = 0; - const int locIdx = 1; - - // Note: even if it's an undefined local we need to move a few - // values around to have outputs end up in the right place. - const bool undefinedLocal = ni.inputs[locIdx]->rtt.isUninit(); - - if (undefinedLocal) { - assert(ni.outStack->valueType() == KindOfNull); - assert(ni.inputs[locIdx]->location.offset < curFunc()->numLocals()); - const StringData* name = local_name(ni.inputs[locIdx]->location); - - EMIT_CALL(a, raiseUndefVariable, IMM((uintptr_t)name)); - recordReentrantCall(ni); - - m_regMap.allocInputRegs(ni); - } - - m_regMap.allocOutputRegs(ni); - const PhysReg stackIn = getReg(ni.inputs[stackIdx]->location); - const PhysReg localIn = getReg(ni.inputs[locIdx]->location); - const PhysReg stackOut = getReg(ni.outStack2->location); - assert(ni.inputs[stackIdx]->location.isStack()); - assert(ni.inputs[locIdx]->location.isLocal()); - - /* - * These registers overlap a bit, so we can swap a few bindings to - * avoid a move. - */ - assert(stackIn == getReg(ni.outStack->location) && localIn != stackOut); - m_regMap.swapRegisters(stackIn, stackOut); - const PhysReg cellOut = getReg(ni.outStack->location); - assert(cellOut != stackIn); - if (ni.inputs[locIdx]->isRef()) { - emitDerefRef(a, localIn, cellOut); - } else if (!undefinedLocal) { - emitMovRegReg(a, localIn, cellOut); - } - emitIncRef(cellOut, ni.inputs[locIdx]->valueType()); -} - -void -TranslatorX64::analyzeVGetL(Tracelet& t, - NormalizedInstruction& i) { - i.m_txFlags = Native; -} - -void -TranslatorX64::translateVGetL(const Tracelet& t, - const NormalizedInstruction& i) { - const DEBUG_ONLY Opcode op = i.op(); - assert(op == OpVGetL || op == OpFPassL); - const vector& inputs = i.inputs; - assert(inputs.size() == 1); - assert(i.outStack); - assert(inputs[0]->isLocal()); - assert(i.outStack->rtt.outerType() == KindOfRef); - - PhysReg localReg = getReg(inputs[0]->location); - PhysReg dest; - if (inputs[0]->rtt.outerType() != KindOfRef) { - emitBox(inputs[0]->rtt.outerType(), localReg); - m_regMap.bind(rax, inputs[0]->location, KindOfRef, - RegInfo::DIRTY); - m_regMap.allocOutputRegs(i); - dest = getReg(i.outStack->location); - emitMovRegReg(a, rax, dest); - } else { - m_regMap.allocOutputRegs(i); - dest = getReg(i.outStack->location); - emitMovRegReg(a, localReg, dest); - } - emitIncRef(dest, KindOfRef); -} - -static bool -isSupportedInstrVGetG(const NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - return (i.inputs[0]->rtt.isString()); -} - -void -TranslatorX64::analyzeVGetG(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = simplePlan(isSupportedInstrVGetG(i)); -} - -static TypedValue* lookupAddBoxedGlobal(StringData* name) { - VarEnv* ve = g_vmContext->m_globalVarEnv; - TypedValue* r = ve->lookupAdd(name); - if (r->m_type != KindOfRef) { - tvBox(r); - } - decRefStr(name); - return r; -} - -void -TranslatorX64::translateVGetG(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - assert(i.outStack); - assert(i.outStack->isRef()); - assert(i.inputs[0]->location == i.outStack->location); - - using namespace TargetCache; - const StringData* maybeName = i.inputs[0]->rtt.valueString(); - if (!maybeName) { - EMIT_CALL(a, lookupAddBoxedGlobal, V(i.inputs[0]->location)); - recordCall(i); - } else { - CacheHandle ch = BoxedGlobalCache::alloc(maybeName); - - if (false) { // typecheck - StringData *key = nullptr; - TypedValue UNUSED *glob = BoxedGlobalCache::lookupCreate(ch, key); - } - SKTRACE(1, i.source, "ch %d\n", ch); - EMIT_CALL(a, BoxedGlobalCache::lookupCreate, - IMM(ch), - V(i.inputs[0]->location)); - recordCall(i); - } - m_regMap.bind(rax, i.outStack->location, KindOfRef, RegInfo::DIRTY); - emitIncRefGeneric(rax, 0); - emitDeref(a, rax, rax); -} - -void -TranslatorX64::analyzeAssignToLocalOp(Tracelet& t, - NormalizedInstruction& ni) { - const int locIdx = 1; - ni.m_txFlags = planHingesOnRefcounting(ni.inputs[locIdx]->outerType()); -} - - -void -TranslatorX64::translateAssignToLocalOp(const Tracelet& t, - const NormalizedInstruction& ni) { - const int rhsIdx = 0; - const int locIdx = 1; - const Opcode op = ni.op(); - assert(op == OpSetL || op == OpBindL); - assert(ni.inputs.size() == 2); - assert((op == OpBindL) == - (ni.inputs[rhsIdx]->outerType() == KindOfRef)); - - assert(!ni.outStack || ni.inputs[locIdx]->location != ni.outStack->location); - assert(ni.outLocal); - assert(ni.inputs[locIdx]->location == ni.outLocal->location); - assert(ni.inputs[rhsIdx]->isStack()); - - const DataType oldLocalType = ni.inputs[locIdx]->outerType(); - const DataType rhsType = ni.inputs[rhsIdx]->outerType(); - bool rhsTypeRelaxed = GuardType(rhsType).isRelaxed(); - bool locTypeRelaxed = GuardType(oldLocalType).isRelaxed(); - - m_regMap.allocOutputRegs(ni); - - const PhysReg rhsReg = getReg(ni.inputs[rhsIdx]->location); - const PhysReg localReg = getReg(ni.outLocal->location); - assert(localReg != rhsReg); - - LazyScratchReg oldLocalReg(m_regMap); - DataType decRefType; - - // For SetL, when the local is boxed, we need to change the - // type/value of the inner cell. If we're doing BindL, we don't - // want to affect the old inner cell in any case (except to decref - // it). - const bool affectInnerCell = op == OpSetL && - oldLocalType == KindOfRef; - if (affectInnerCell) { - assert(rhsType != KindOfRef); - decRefType = ni.inputs[locIdx]->rtt.innerType(); - bool useOldType = (locTypeRelaxed && GuardType(decRefType).isCounted()) || - (!locTypeRelaxed && IS_REFCOUNTED_TYPE(decRefType)); - if (useOldType) { - oldLocalReg.alloc(); - emitDerefRef(a, localReg, r(oldLocalReg)); - } - if (rhsTypeRelaxed) { - PhysReg base; - int disp; - ScratchReg rTmp(m_regMap); - locToRegDisp(ni.inputs[rhsIdx]->location, &base, &disp); - size_t typeOff = RefData::tvOffset() + TVOFF(m_type); - size_t dataOff = RefData::tvOffset() + TVOFF(m_data); - emitLoadTVType(a, base[disp + TVOFF(m_type)], r(rTmp)); - a. storeq (rhsReg, localReg[dataOff]); - emitStoreTVType(a, r(rTmp), localReg[typeOff]); - } else { - emitStoreToRefData(a, rhsType, rhsReg, 0, localReg); - } - } else if (rhsTypeRelaxed) { - PhysReg rhsBase; - int rhsDisp; - locToRegDisp(ni.inputs[rhsIdx]->location, &rhsBase, &rhsDisp); - PhysReg locBase; - int locDisp; - locToRegDisp(ni.inputs[locIdx]->location, &locBase, &locDisp); - ScratchReg rTmp(m_regMap); - a. storeq(rhsReg, locBase[locDisp + TVOFF(m_data)]); - emitLoadTVType(a, rhsBase[rhsDisp + TVOFF(m_type)], r(rTmp)); - emitStoreTVType(a, r(rTmp), locBase[locDisp + TVOFF(m_type)]); - m_regMap.swapRegisters(rhsReg, localReg); - decRefType = oldLocalType; - m_regMap.markAsClean(ni.inputs[locIdx]->location); - } else { - /* - * Instead of emitting a mov, just swap the locations these two - * registers are mapped to. - * - * TODO: this might not be the best idea now that the register - * allocator has some awareness about what is a local. (Maybe we - * should just xchg.) - */ - m_regMap.swapRegisters(rhsReg, localReg); - decRefType = oldLocalType; - } - - // If we're giving stack output, it's important to incref before - // calling a possible destructor, since the destructor could have - // access to the local if it is a var. - if (ni.outStack) { - if (rhsTypeRelaxed) { - if (GuardType(rhsType).isCounted()) { - PhysReg base; - int disp; - locToRegDisp(ni.inputs[rhsIdx]->location, &base, &disp); - emitIncRefGeneric(base, disp); // forces static check - } - } else { - emitIncRef(rhsReg, rhsType); - } - } else { - SKTRACE(3, ni.source, "hoisting Pop* into current instr\n"); - } - - if (locTypeRelaxed) { - if (GuardType(decRefType).isCounted()) { - emitDecRef(ni, oldLocalReg.isAllocated() ? r(oldLocalReg) : localReg, - decRefType); - } - } else { - emitDecRef(ni, oldLocalReg.isAllocated() ? r(oldLocalReg) : localReg, - decRefType); - } - - if (ni.outStack && !IS_NULL_TYPE(ni.outStack->outerType())) { - assert(!rhsTypeRelaxed); - PhysReg stackReg = getReg(ni.outStack->location); - emitMovRegReg(a, rhsReg, stackReg); - } -} - -static void -planPop(NormalizedInstruction& i) { - DataType type = i.inputs[0]->outerType(); - i.m_txFlags = - (type == KindOfInvalid || IS_REFCOUNTED_TYPE(type)) ? Supported : Native; - i.manuallyAllocInputs = true; -} - -void TranslatorX64::analyzePopC(Tracelet& t, NormalizedInstruction& i) { - planPop(i); -} - -void TranslatorX64::analyzePopV(Tracelet& t, NormalizedInstruction& i) { - planPop(i); -} - -void TranslatorX64::analyzePopR(Tracelet& t, NormalizedInstruction& i) { - planPop(i); -} - -void -TranslatorX64::translatePopC(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - assert(!i.outStack && !i.outLocal); - if (i.inputs[0]->rtt.isVagueValue()) { - PhysReg base; - int disp; - locToRegDisp(i.inputs[0]->location, &base, &disp); - emitDecRefGeneric(i, base, disp); - } else { - emitDecRefInput(a, i, 0); - } -} - -void -TranslatorX64::translatePopV(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs[0]->rtt.isVagueValue() || - i.inputs[0]->isRef()); - translatePopC(t, i); -} - -void -TranslatorX64::translatePopR(const Tracelet& t, - const NormalizedInstruction& i) { - translatePopC(t, i); -} - -void -TranslatorX64::translateUnboxR(const Tracelet& t, - const NormalizedInstruction& i) { - assert(!i.inputs[0]->rtt.isVagueValue()); - - // If the value on the top of a stack is a var, unbox it and - // leave it on the top of the stack. - if (i.inputs[0]->isRef()) { - emitUnboxTopOfStack(i); - } -} - -void -TranslatorX64::translateNull(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outLocal); - if (i.outStack) { - assert(i.outStack->outerType() == KindOfNull); - - // We have to mark the output register as dirty to ensure that - // the type gets spilled at the end of the tracelet - m_regMap.allocOutputRegs(i); - } - /* nop */ -} - -void -TranslatorX64::translateNullUninit(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outLocal); - if (i.outStack) { - assert(i.outStack->outerType() == KindOfUninit); - // We have to mark the output register as dirty to ensure that - // the type gets spilled at the end of the tracelet - m_regMap.allocOutputRegs(i); - } -} - -void -TranslatorX64::translateTrue(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outLocal); - if (i.outStack) { - m_regMap.allocOutputRegs(i); - PhysReg rdest = getReg(i.outStack->location); - emitImmReg(a, 1, rdest); - } -} - -void -TranslatorX64::translateFalse(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outLocal); - if (i.outStack) { - m_regMap.allocOutputRegs(i); - PhysReg dest = getReg(i.outStack->location); - emitImmReg(a, false, dest); - } -} - -void -TranslatorX64::translateInt(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outLocal); - if (i.outStack) { - assert(i.outStack->isInt()); - m_regMap.allocOutputRegs(i); - PhysReg dest = getReg(i.outStack->location); - uint64_t srcImm = i.imm[0].u_I64A; - emitImmReg(a, srcImm, dest); - } -} - -void -TranslatorX64::translateDouble(const Tracelet& t, - const NormalizedInstruction& i) { - if (i.outStack) { - assert(i.outStack->isDouble()); - m_regMap.allocOutputRegs(i); - emitImmReg(a, i.imm[0].u_I64A, getReg(i.outStack->location)); - } -} - -void -TranslatorX64::translateString(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outLocal); - if (!i.outStack) return; - assert(Translator::typeIsString(i.outStack->outerType())); - m_regMap.allocOutputRegs(i); - PhysReg dest = getReg(i.outStack->location); - uint64_t srcImm = (uintptr_t)curUnit()->lookupLitstrId(i.imm[0].u_SA); - // XXX: can simplify the lookup here by just fishing it out of the - // output's valueString(). - // We are guaranteed that the string is static, so we do not need to - // increment the refcount - assert(((StringData*)srcImm)->isStatic()); - SKTRACE(2, i.source, "Litstr %d -> %p \"%s\"\n", - i.imm[0].u_SA, (StringData*)srcImm, - Util::escapeStringForCPP(((StringData*)srcImm)->data()).c_str()); - emitImmReg(a, srcImm, dest); -} - -void -TranslatorX64::translateArray(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outLocal); - if (i.outStack) { - assert(i.outStack->outerType() == KindOfArray); - m_regMap.allocOutputRegs(i); - ArrayData* ad = curUnit()->lookupArrayId(i.imm[0].u_AA); - PhysReg r = getReg(i.outStack->location); - emitImmReg(a, uint64_t(ad), r); - // We are guaranteed that the array is static, so we do not need to - // increment the refcount - assert(ad->isStatic()); - } -} - -void -TranslatorX64::translateNewArray(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(i.outStack && !i.outLocal); - assert(i.outStack->outerType() == KindOfArray); - int capacity = i.imm[0].u_IVA; - if (capacity == 0) { - m_regMap.allocOutputRegs(i); - PhysReg r = getReg(i.outStack->location); - emitImmReg(a, uint64_t(HphpArray::GetStaticEmptyArray()), r); - // We are guaranteed that the new array is static, so we do not need to - // increment the refcount - assert(HphpArray::GetStaticEmptyArray()->isStatic()); - } else { - // create an empty array with a nonzero capacity - if (false) { - ArrayData* a = new_array(42); - printf("%p", a); // use ret - } - EMIT_CALL(a, new_array, IMM(capacity)); - m_regMap.bind(rax, i.outStack->location, KindOfArray, RegInfo::DIRTY); - } -} - -void TranslatorX64::analyzeNewTuple(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = Simple; // the array constructors are not re-entrant. - i.manuallyAllocInputs = true; // all values passed via stack. -} - -void TranslatorX64::translateNewTuple(const Tracelet& t, - const NormalizedInstruction& i) { - int arity = i.imm[0].u_IVA; - assert(arity > 0 && i.inputs.size() == unsigned(arity)); - assert(i.outStack && !i.outLocal); - for (int j = 0; j < arity; j++) { - assert(i.inputs[j]->outerType() != KindOfRef); - assert(i.inputs[j]->isStack()); - } - - // We pass the values by address, so we need to sync them back to memory - for (int j = 0; j < arity; j++) { - m_regMap.cleanLoc(i.inputs[j]->location); - } - if (false) { - TypedValue* rhs = 0; - ArrayData* ret = new_tuple(arity, rhs); - printf("%p", ret); // use ret - } - EMIT_CALL(a, new_tuple, IMM(arity), A(i.inputs[0]->location)); - // new_tuple() returns the up-to-date array pointer in rax. Therefore, we - // can bind rax to the result location and mark it as dirty. - m_regMap.bind(rax, i.inputs[arity-1]->location, KindOfArray, RegInfo::DIRTY); -} - -void -TranslatorX64::translateNewCol(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(i.outStack && !i.outLocal); - assert(i.outStack->outerType() == KindOfObject); - int cType = i.imm[0].u_IVA; - int nElms = i.imm[1].u_IVA; - void* fptr = nullptr; - switch (cType) { - case Collection::VectorType: fptr = (void*)newVectorHelper; break; - case Collection::MapType: fptr = (void*)newMapHelper; break; - case Collection::StableMapType: fptr = (void*)newStableMapHelper; break; - case Collection::SetType: fptr = (void*)newSetHelper; break; - case Collection::PairType: fptr = (void*)newPairHelper; break; - default: assert(false); break; - } - if (false) { - ObjectData* obj1 UNUSED = newVectorHelper(42); - ObjectData* obj2 UNUSED = newMapHelper(42); - ObjectData* obj3 UNUSED = newStableMapHelper(42); - ObjectData* obj4 UNUSED = newSetHelper(42); - ObjectData* obj5 UNUSED = newPairHelper(); - } - if (cType == Collection::PairType) { - // newPairHelper does not take any arguments, since Pairs always - // have exactly two elements - EMIT_CALL(a, fptr); - } else { - EMIT_CALL(a, fptr, IMM(nElms)); - } - m_regMap.bind(rax, i.outStack->location, KindOfObject, RegInfo::DIRTY); -} - -void -TranslatorX64::analyzeNop(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = Native; -} - -void -TranslatorX64::translateNop(const Tracelet& t, - const NormalizedInstruction& i) { -} - -void -TranslatorX64::analyzeAddElemC(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = supportedPlan(i.inputs[2]->outerType() == KindOfArray && - (i.inputs[1]->isInt() || - i.inputs[1]->isString())); -} - -void -TranslatorX64::translateAddElemC(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.outStack && !i.outLocal); - assert(i.inputs.size() >= 3); - const DynLocation& arr = *i.inputs[2]; - const DynLocation& key = *i.inputs[1]; - const DynLocation& val = *i.inputs[0]; - assert(!arr.isRef()); // not handling variants. - assert(!key.isRef()); - assert(!val.isRef()); - - const Location& arrLoc = arr.location; - const Location& keyLoc = key.location; - const Location& valLoc = val.location; - - assert(arrLoc.isStack()); - assert(keyLoc.isStack()); - assert(arrLoc.isStack()); - - // We will need to pass the rhs by address, so we need to sync it to memory - m_regMap.cleanLoc(valLoc); - - // The array_setm helpers will decRef any old value that is - // overwritten if appropriate. If copy-on-write occurs, it will also - // incRef the new array and decRef the old array for us. Finally, - // some of the array_setm helpers will decRef the key if it is a - // string (for cases where the key is not a local), while others do - // not (for cases where the key is a local). - assert(key.rtt.isInt() || key.rtt.isString()); - if (false) { // type-check - RefData* ref = nullptr; - TypedValue* rhs = nullptr; - StringData* strkey = nullptr; - ArrayData* arr = nullptr; - ArrayData* ret; - ret = array_setm_ik1_v0(ref, arr, 12, rhs); - printf("%p", ret); // use ret - ret = array_setm_sk1_v0(ref, arr, strkey, rhs); - printf("%p", ret); // use ret - } - // Otherwise, we pass the rhs by address - void* fptr = key.rtt.isString() ? (void*)array_setm_sk1_v0 : - (void*)array_setm_ik1_v0; - EMIT_CALL(a, fptr, - IMM(0), - V(arrLoc), - V(keyLoc), - A(valLoc)); - recordReentrantCall(i); - // The array value may have changed, so we need to invalidate any - // register we have associated with arrLoc - m_regMap.invalidate(arrLoc); - // The array_setm helper returns the up-to-date array pointer in rax. - // Therefore, we can bind rax to arrLoc and mark it as dirty. - m_regMap.bind(rax, arrLoc, KindOfArray, RegInfo::DIRTY); -} - -void -TranslatorX64::analyzeAddNewElemC(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - i.m_txFlags = supportedPlan(i.inputs[1]->outerType() == KindOfArray); -} - -void -TranslatorX64::translateAddNewElemC(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - assert(i.outStack && !i.outLocal); - assert(i.inputs[0]->outerType() != KindOfRef); - assert(i.inputs[1]->outerType() != KindOfRef); - assert(i.inputs[0]->isStack()); - assert(i.inputs[1]->isStack()); - - Location arrLoc = i.inputs[1]->location; - Location valLoc = i.inputs[0]->location; - - // We pass the rhs by address, so we need to sync it back to memory - m_regMap.cleanLoc(valLoc); - - // The array_setm_wki_v0 helper will decRef the value if it cannot - // be stored; otherwise the value is moved (neither incref'd or decref'd). - // Copy-on-write is expected not to occur since AddNewElemC is used - // for array initialization. - if (false) { // type-check - TypedValue* rhs = nullptr; - ArrayData* arr = nullptr; - ArrayData* ret; - ret = array_setm_wk1_v0(arr, rhs); - printf("%p", ret); // use ret - } - EMIT_CALL(a, array_setm_wk1_v0, - V(arrLoc), A(valLoc)); - recordReentrantCall(i); - // The array value may have changed, so we need to invalidate any - // register we have associated with arrLoc - m_regMap.invalidate(arrLoc); - // The array_setm helper returns the up-to-date array pointer in rax. - // Therefore, we can bind rax to arrLoc and mark it as dirty. - m_regMap.bind(rax, arrLoc, KindOfArray, RegInfo::DIRTY); -} - -void -TranslatorX64::analyzeColAddNewElemC(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - i.m_txFlags = supportedPlan(i.inputs[1]->outerType() == KindOfObject); -} - -void -TranslatorX64::translateColAddNewElemC(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - assert(i.outStack && !i.outLocal); - assert(i.inputs[0]->outerType() != KindOfRef); - assert(i.inputs[1]->outerType() != KindOfRef); - assert(i.inputs[0]->isStack()); - assert(i.inputs[1]->isStack()); - Location collLoc = i.inputs[1]->location; - Location valLoc = i.inputs[0]->location; - m_regMap.cleanLoc(valLoc); - if (false) { // type-check - TypedValue* rhs = nullptr; - ObjectData* coll = nullptr; - collection_setm_wk1_v0(coll, rhs); - } - EMIT_RCALL(a, i, collection_setm_wk1_v0, - V(collLoc), - A(valLoc)); -} - -void -TranslatorX64::analyzeColAddElemC(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = supportedPlan(i.inputs[2]->outerType() == KindOfObject && - (i.inputs[1]->isInt() || - i.inputs[1]->isString())); -} - -void -TranslatorX64::translateColAddElemC(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.outStack && !i.outLocal); - assert(i.inputs.size() >= 3); - const DynLocation& coll = *i.inputs[2]; - const DynLocation& key = *i.inputs[1]; - const DynLocation& val = *i.inputs[0]; - assert(!coll.isRef()); // not handling variants. - assert(!key.isRef()); - assert(!val.isRef()); - const Location& collLoc = coll.location; - const Location& keyLoc = key.location; - const Location& valLoc = val.location; - assert(collLoc.isStack()); - assert(keyLoc.isStack()); - assert(collLoc.isStack()); - m_regMap.cleanLoc(valLoc); - assert(key.rtt.isInt() || key.rtt.isString()); - if (false) { // type-check - TypedValue* rhs = nullptr; - StringData* strkey = nullptr; - ObjectData* coll = nullptr; - collection_setm_ik1_v0(coll, 12, rhs); - collection_setm_sk1_v0(coll, strkey, rhs); - } - void* fptr = key.rtt.isString() ? (void*)collection_setm_sk1_v0 : - (void*)collection_setm_ik1_v0; - EMIT_RCALL(a, i, fptr, - V(collLoc), - V(keyLoc), - A(valLoc)); -} - -static int64_t undefCns(const TypedValue* tv, const StringData* nm, Cell* c1) { - assert(tv->m_type == KindOfUninit); - TypedValue *cns = nullptr; - if (UNLIKELY(tv->m_data.pref != nullptr)) { - ClassInfo::ConstantInfo* ci = - (ClassInfo::ConstantInfo*)(void*)tv->m_data.pref; - cns = const_cast(ci->getDeferredValue()).asTypedValue(); - tvReadCell(cns, c1); - } else { - if (UNLIKELY(TargetCache::s_constants != nullptr)) { - cns = TargetCache::s_constants->HphpArray::nvGet(nm); - } - if (!cns) { - cns = Unit::loadCns(const_cast(nm)); - } - if (UNLIKELY(!cns)) { - raise_notice(Strings::UNDEFINED_CONSTANT, nm->data(), nm->data()); - c1->m_data.pstr = const_cast(nm); - c1->m_type = BitwiseKindOfString; - } else { - c1->m_type = cns->m_type; - c1->m_data = cns->m_data; - } - } - return c1->m_type; -} - void TranslatorX64::emitSideExit(Asm& a, const NormalizedInstruction& i, bool next) { const NormalizedInstruction& dest = next ? *i.next : i; @@ -5272,443 +3599,6 @@ void TranslatorX64::emitSideExit(Asm& a, const NormalizedInstruction& i, emitBindJmp(a, dest.source, REQ_BIND_SIDE_EXIT); } -void -TranslatorX64::translateCns(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(i.outStack && !i.outLocal); - - // OK to burn "name" into TC: it was merged into the static string - // table, so as long as this code is reachable, so should the string - // be. - DataType outType = i.outStack->valueType(); - StringData* name = curUnit()->lookupLitstrId(i.imm[0].u_SA); - const TypedValue* tv = Unit::lookupPersistentCns(name); - bool checkDefined = false; - if (tv) { - // KindOfUninit is for a small number of "dynamic" - // system constants - checkDefined = tv->m_type == KindOfUninit; - } else { - if (outType != KindOfInvalid && - !RuntimeOption::RepoAuthoritative) { - PreConstDepMap::accessor acc; - tv = findUniquePreConst(acc, name); - if (tv != nullptr) { - checkDefined = true; - acc->second.srcKeys.insert(t.m_sk); - Stats::emitInc(a, Stats::Tx64_CnsFast); - } else { - // We had a unique value while analyzing but don't anymore. This - // should be rare so just punt to keep things simple. - punt(); - } - } - } - using namespace TargetCache; - if (tv && tvIsStatic(tv)) { - ScratchReg ret(m_regMap); - boost::scoped_ptr astubsRet; - m_regMap.invalidate(i.outStack->location); - if (checkDefined) { - CacheHandle ch = StringData::GetCnsHandle(name); - assert(ch); - emitCmpTVType(a, KindOfUninit, rVmTl[ch + TVOFF(m_type)]); - if (!i.next) astubsRet.reset(new DiamondReturn); - { - // If we get to the optimistic translation and the constant - // isn't defined, our tracelet is ruined because the type may - // not be what we expect. If we were expecting KindOfString we - // could theoretically keep going here since that's the type - // of an undefined constant expression, but it should be rare - // enough that it's not worth the complexity. - UnlikelyIfBlock ifZero(CC_Z, a, astubs, astubsRet.get()); - Stats::emitInc(astubs, Stats::Tx64_CnsFast, -1); - EMIT_CALL(astubs, undefCns, - RPLUS(rVmTl, ch), - IMM((uintptr_t)name), - A(i.outStack->location)); - recordReentrantStubCall(i); - if (i.next) { - emitMovRegReg(astubs, rax, r(ret)); - ifZero.reconcileEarly(); - astubs.cmp_imm32_reg64(outType, r(ret)); - astubs.je(a.code.frontier); - // Now we're definitely exiting. - // Save it, and thaw - RegAlloc save = m_regMap; - m_regMap.defrost(); - emitSideExit(astubs, i, true); - m_regMap = save; - } else { - // DiamondReturn will take care of branching - // to the return, below - } - } - } else { - // Its type and value are known at compile-time. - assert(tv->m_type == outType || - (IS_STRING_TYPE(tv->m_type) && IS_STRING_TYPE(outType))); - // tv is static; no need to incref - } - m_regMap.allocOutputRegs(i); - PhysReg r = getReg(i.outStack->location); - a. movq (tv->m_data.num, r); - return; - } - - Stats::emitInc(a, Stats::Tx64_CnsSlow); - CacheHandle ch = StringData::DefCnsHandle(name, false); - TRACE(2, "Cns: %s -> ch %" PRId64 "\n", name->data(), ch); - // Load the constant out of the thread-private tl_targetCaches. - ScratchReg cns(m_regMap); - a. lea_reg64_disp_reg64(rVmTl, ch, r(cns)); - emitCmpTVType(a, KindOfUninit, r(cns)[TVOFF(m_type)]); - DiamondReturn astubsRet; - int stackDest = 0 - int(sizeof(Cell)); // popped - pushed - { - // It's tempting to dedup these, but not obvious we really can; - // at least stackDest and tmp are specific to the translation - // context. - UnlikelyIfBlock ifb(CC_Z, a, astubs, &astubsRet); - EMIT_CALL(astubs, undefCns, - R(r(cns)), - IMM((uintptr_t)name), - A(i.outStack->location)); - recordReentrantStubCall(i); - m_regMap.invalidate(i.outStack->location); - } - - // Bitwise copy to output area. - emitCopyToStack(a, i, r(cns), stackDest); - m_regMap.invalidate(i.outStack->location); -} - -void -TranslatorX64::analyzeDefCns(Tracelet& t, - NormalizedInstruction& i) { - StringData* name = curUnit()->lookupLitstrId(i.imm[0].u_SA); - /* don't bother to translate if it names a builtin constant */ - i.m_txFlags = supportedPlan(!Unit::lookupPersistentCns(name)); -} - -void -TranslatorX64::translateDefCns(const Tracelet& t, - const NormalizedInstruction& i) { - using namespace TargetCache; - - StringData* name = curUnit()->lookupLitstrId(i.imm[0].u_SA); - CacheHandle ch = StringData::DefCnsHandle(name, false); - - if (false) { - TypedValue *value = 0; - Unit::defCnsHelper(ch, value, name); - } - - TRACE(2, "DefCns: %s -> ch %" PRId64 "\n", name->data(), ch); - - m_regMap.cleanLoc(i.inputs[0]->location); - EMIT_CALL(a, Unit::defCnsHelper, - IMM(ch), A(i.inputs[0]->location), - IMM((uint64_t)name)); - - recordReentrantCall(i); - m_regMap.bind(rax, i.outStack->location, i.outStack->outerType(), - RegInfo::DIRTY); -} - -void -TranslatorX64::translateClsCnsD(const Tracelet& t, - const NormalizedInstruction& i) { - using namespace TargetCache; - const NamedEntityPair& namedEntityPair = - curUnit()->lookupNamedEntityPairId(i.imm[1].u_SA); - assert(namedEntityPair.second); - const StringData *clsName = namedEntityPair.first; - assert(clsName->isStatic()); - StringData* cnsName = curUnit()->lookupLitstrId(i.imm[0].u_SA); - assert(cnsName->isStatic()); - StringData* fullName = StringData::GetStaticString( - Util::toLower(clsName->data()) + "::" + cnsName->data()); - - Stats::emitInc(a, Stats::TgtCache_ClsCnsHit); - CacheHandle ch = allocClassConstant(fullName); - ScratchReg cns(m_regMap); - a.lea_reg64_disp_reg64(rVmTl, ch, r(cns)); - emitCmpTVType(a, 0, r(cns)[TVOFF(m_type)]); - { - UnlikelyIfBlock ifNull(CC_Z, a, astubs); - - if (false) { // typecheck - TypedValue* tv = nullptr; - UNUSED TypedValue* ret = - TargetCache::lookupClassConstant(tv, namedEntityPair.second, - namedEntityPair.first, cnsName); - } - - EMIT_CALL(astubs, TCA(TargetCache::lookupClassConstant), - R(cns), - IMM(uintptr_t(namedEntityPair.second)), - IMM(uintptr_t(namedEntityPair.first)), - IMM(uintptr_t(cnsName))); - recordReentrantStubCall(i); - // DiamondGuard will restore cns's SCRATCH state but not its - // contents. lookupClassConstant returns the value we want. - emitMovRegReg(astubs, rax, r(cns)); - } - int stackDest = 0 - int(sizeof(Cell)); // 0 popped - 1 pushed - emitCopyToStack(a, i, r(cns), stackDest); -} - -void -TranslatorX64::analyzeConcat(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - const RuntimeType& r = i.inputs[0]->rtt; - const RuntimeType& l = i.inputs[1]->rtt; - // The concat translation isn't reentrant; objects that override - // __toString() can cause reentry. - i.m_txFlags = supportedPlan(r.valueType() != KindOfObject && - l.valueType() != KindOfObject); -} - -void -TranslatorX64::translateConcat(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - const DynLocation& r = *i.inputs[0]; - const DynLocation& l = *i.inputs[1]; - // We have specialized helpers for concatenating two strings, a - // string and an int, and an int an a string. - void* fptr = nullptr; - if (l.rtt.isString() && r.rtt.isString()) { - fptr = (void*)concat_ss; - } else if (l.rtt.isString() && r.rtt.isInt()) { - fptr = (void*)concat_si; - } else if (l.rtt.isInt() && r.rtt.isString()) { - fptr = (void*)concat_is; - } - if (fptr) { - // If we have a specialized helper, use it - if (false) { // type check - StringData* v1 = nullptr; - StringData* v2 = nullptr; - StringData* retval = concat_ss(v1, v2); - printf("%p", retval); // use retval - } - - // The concat helper will decRef the inputs and incRef the output - // for us if appropriate - EMIT_RCALL(a, i, fptr, V(l.location), V(r.location)); - assert(i.outStack->rtt.isString()); - m_regMap.bind(rax, i.outStack->location, i.outStack->outerType(), - RegInfo::DIRTY); - - } else { - // Otherwise, use the generic concat helper - if (false) { // type check - uint64_t v1 = 0, v2 = 0; - DataType t1 = KindOfUninit, t2 = KindOfUninit; - StringData *retval = concat_tv(t1, v1, t2, v2); - printf("%p", retval); // use retval - } - // concat will decRef the two inputs and incRef the output - // for us if appropriate - EMIT_RCALL(a, i, concat_tv, - IMM(l.valueType()), V(l.location), - IMM(r.valueType()), V(r.location)); - assert(i.outStack->isString()); - m_regMap.bind(rax, i.outStack->location, i.outStack->outerType(), - RegInfo::DIRTY); - } -} - -TXFlags -planInstrAdd_Int(const NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - return nativePlan(i.inputs[0]->isInt() && i.inputs[1]->isInt()); -} - -TXFlags -planInstrAdd_Array(const NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - return supportedPlan(i.inputs[0]->valueType() == KindOfArray && - i.inputs[1]->valueType() == KindOfArray); -} - -TXFlags -planInstrAdd_Double(const NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - return nativePlan(i.inputs[0]->isDouble() && i.inputs[1]->isDouble()); -} - -void -TranslatorX64::analyzeAdd(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = TXFlags(planInstrAdd_Int(i) | planInstrAdd_Array(i) | - planInstrAdd_Double(i)); -} - -void -TranslatorX64::translateAdd(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - - if (planInstrAdd_Array(i)) { - // Handle adding two arrays - assert(i.outStack->outerType() == KindOfArray); - if (false) { // type check - ArrayData* v = nullptr; - v = array_add(v, v); - } - // The array_add helper will decRef the inputs and incRef the output - // for us if appropriate - EMIT_CALL(a, array_add, - V(i.inputs[1]->location), - V(i.inputs[0]->location)); - recordReentrantCall(i); - m_regMap.bind(rax, i.outStack->location, i.outStack->outerType(), - RegInfo::DIRTY); - return; - } - - assert(planInstrAdd_Int(i) | planInstrAdd_Double(i)); - binaryArithCell(i, OpAdd, *i.inputs[0], *i.outStack); -} - -void -TranslatorX64::analyzeXor(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = nativePlan((i.inputs[0]->outerType() == KindOfBoolean || - i.inputs[0]->isInt()) && - (i.inputs[1]->outerType() == KindOfBoolean || - i.inputs[1]->isInt())); -} - -static inline void -emitIntToCCBool(X64Assembler &a, PhysReg srcdest, PhysReg scratch, - int CC) { - /* - * test %srcdest, %srcdest - * set %scratchL - * movzbq %scratchL, %srcdest - */ - a. test_reg64_reg64(srcdest, srcdest); - a. setcc (CC, rbyte(scratch)); - a. movzbl (rbyte(scratch), r32(srcdest)); -} - -static inline void -emitIntToBool(X64Assembler &a, PhysReg srcdest, PhysReg scratch) { - emitIntToCCBool(a, srcdest, scratch, CC_NZ); -} - -static inline void -emitIntToNegBool(X64Assembler &a, PhysReg srcdest, PhysReg scratch) { - emitIntToCCBool(a, srcdest, scratch, CC_Z); -} - -void -TranslatorX64::translateXor(const Tracelet& t, - const NormalizedInstruction& i) { - PhysReg src, srcdest; - getBinaryStackInputs(m_regMap, i, src, srcdest); - m_regMap.allocOutputRegs(i); - ScratchReg scr(m_regMap); - if (i.inputs[0]->isInt()) { - emitIntToBool(a, src, r(scr)); - } - if (i.inputs[1]->isInt()) { - emitIntToBool(a, srcdest, r(scr)); - } - a. xor_reg64_reg64(src, srcdest); -} - -void -TranslatorX64::analyzeMod(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = nativePlan(false); -} - -void TranslatorX64::translateMod(const Tracelet& t, const NormalizedInstruction& i) { - not_reached(); -} - -void -TranslatorX64::analyzeNot(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - i.m_txFlags = nativePlan(i.inputs[0]->isInt() || - i.inputs[0]->outerType() == KindOfBoolean); -} - -void -TranslatorX64::translateNot(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.isNative()); - assert(i.outStack && !i.outLocal); - assert(!i.inputs[0]->isRef()); - m_regMap.allocOutputRegs(i); - PhysReg srcdest = m_regMap.getReg(i.outStack->location); - ScratchReg scr(m_regMap); - emitIntToNegBool(a, srcdest, r(scr)); -} - -void -TranslatorX64::analyzeBitNot(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = nativePlan(i.inputs[0]->isInt()); -} - -void -TranslatorX64::translateBitNot(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.outStack && !i.outLocal); - m_regMap.allocOutputRegs(i); - PhysReg srcdest = m_regMap.getReg(i.outStack->location); - a. not (srcdest); -} - -#define TRIVIAL_CAST(Type) \ -void \ -TranslatorX64::analyzeCast## Type(Tracelet& t, NormalizedInstruction& i) { \ - i.m_txFlags = nativePlan(i.inputs[0]->is## Type()); \ -} \ - \ -void \ -TranslatorX64::translateCast## Type(const Tracelet& t, \ - const NormalizedInstruction& i) { \ - assert(i.inputs.size() == 1); \ - assert(i.outStack && !i.outLocal); \ - assert(i.inputs[0]->is## Type()); \ - \ - /* nop */ \ -} - -TRIVIAL_CAST(Int) -TRIVIAL_CAST(Array) -TRIVIAL_CAST(Object) -#undef TRIVIAL_CAST - -void -TranslatorX64::analyzeCastString(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = - i.inputs[0]->isArray() || i.inputs[0]->isObject() ? Supported : - i.inputs[0]->isInt() ? Simple : - Native; - i.funcd = nullptr; -} - -static void toStringError(StringData *cls) { - raise_error("Method __toString() must return a string value"); -} - -static const StringData* stringDataFromInt(int64_t n) { - StringData* s = buildStringData(n); - s->incRefCount(); - return s; -} - -static const StringData* stringDataFromDouble(int64_t n) { - StringData* s = buildStringData(*(double*)&n); - s->incRefCount(); - return s; -} - // returns the prologue address to execute uint64_t TranslatorX64::toStringHelper(ObjectData *obj) { // caller must set r15 to the new ActRec @@ -5736,845 +3626,6 @@ uint64_t TranslatorX64::toStringHelper(ObjectData *obj) { return (uint64_t)toString->getPrologue(0); } -void -TranslatorX64::translateCastString(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - assert(i.outStack && !i.outLocal); - - if (i.inputs[0]->isNull()) { - m_regMap.allocOutputRegs(i); - PhysReg dest = m_regMap.getReg(i.outStack->location); - a. mov_imm64_reg((uint64_t)empty_string.get(), dest); - } else if (i.inputs[0]->isBoolean()) { - static StringData* s_1 = StringData::GetStaticString("1"); - m_regMap.allocOutputRegs(i); - PhysReg dest = m_regMap.getReg(i.outStack->location); - a. cmp_imm32_reg64(0, dest); - a. mov_imm64_reg((uint64_t)empty_string.get(), dest); - ScratchReg scratch(m_regMap); - a. mov_imm64_reg((intptr_t)s_1, r(scratch)); - a. cmov_reg64_reg64(CC_NZ, r(scratch), dest); - } else if (i.inputs[0]->isInt()) { - EMIT_CALL(a, stringDataFromInt, V(i.inputs[0]->location)); - m_regMap.bind(rax, i.outStack->location, i.outStack->outerType(), - RegInfo::DIRTY); - } else if (i.inputs[0]->isDouble()) { - EMIT_CALL(a, stringDataFromDouble, V(i.inputs[0]->location)); - m_regMap.bind(rax, i.outStack->location, i.outStack->outerType(), - RegInfo::DIRTY); - } else if (i.inputs[0]->isString()) { - // nop - } else if (i.inputs[0]->isArray()) { - static StringData* s_array = StringData::GetStaticString("Array"); - m_regMap.allocOutputRegs(i); - PhysReg dest = m_regMap.getReg(i.outStack->location); - emitDecRef(i, dest, KindOfArray); - a. mov_imm64_reg((uint64_t)s_array, dest); - } else if (i.inputs[0]->isObject()) { - m_regMap.scrubStackEntries(i.stackOff - 1); - m_regMap.cleanAll(); - int delta = i.stackOff + kNumActRecCells - 1; - if (delta) { - a. add_imm64_reg64(-cellsToBytes(delta), rVmSp); - } - a. store_reg64_disp_reg64(rVmFp, AROFF(m_savedRbp), rVmSp); - a. store_imm32_disp_reg(nextSrcKey(t, i).offset() - curFunc()->base(), - AROFF(m_soff), rVmSp); - PhysReg obj = m_regMap.getReg(i.inputs[0]->location); - if (obj != argNumToRegName[0]) { - a. mov_reg64_reg64(obj, argNumToRegName[0]); - } - m_regMap.smashRegs(kAllRegs); - a. mov_reg64_reg64(rVmSp, rStashedAR); - - EMIT_CALL(a, TCA(toStringHelper)); - recordReentrantCall(i); - // call to the address returned by toStringHelper - a. call(reg::rax); - if (i.stackOff != 0) { - a. add_imm64_reg64(cellsToBytes(i.stackOff), rVmSp); - } - - PhysReg base; - int disp; - locToRegDisp(i.outStack->location, &base, &disp); - emitStringCheck(a, base, disp + TVOFF(m_type)); - { - UnlikelyIfBlock ifNotString(CC_Z, a, astubs); - EMIT_CALL(astubs, toStringError, IMM(0)); - recordReentrantStubCall(i); - } - } else { - NOT_REACHED(); - } -} - -void -TranslatorX64::analyzeCastDouble(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = nativePlan(i.inputs[0]->valueType() == KindOfDouble); -} - -void -TranslatorX64::translateCastDouble(const Tracelet& t, - const NormalizedInstruction& i) { - // nop. -} - -void -TranslatorX64::analyzePrint(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - const RuntimeType& rtt = i.inputs[0]->rtt; - DataType type = rtt.outerType(); - i.m_txFlags = simplePlan( - type == KindOfUninit || - type == KindOfNull || - type == KindOfBoolean || - rtt.isInt() || - rtt.isString()); -} - -void -TranslatorX64::translatePrint(const Tracelet& t, - const NormalizedInstruction& i) { - const vector& inputs = i.inputs; - assert(inputs.size() == 1); - assert(!i.outLocal); - assert(!i.outStack || i.outStack->isInt()); - Location loc = inputs[0]->location; - DataType type = inputs[0]->outerType(); - switch (type) { - STRINGCASE(): EMIT_CALL(a, print_string, V(loc)); break; - case KindOfInt64: EMIT_CALL(a, print_int, V(loc)); break; - case KindOfBoolean: EMIT_CALL(a, print_boolean, V(loc)); break; - NULLCASE(): /* do nothing */ break; - default: { - // Translation is only supported for Null, Boolean, Int, and String - assert(false); - break; - } - } - m_regMap.allocOutputRegs(i); - if (i.outStack) { - PhysReg outReg = getReg(i.outStack->location); - emitImmReg(a, 1, outReg); - } -} - -void -TranslatorX64::translateJmp(const Tracelet& t, - const NormalizedInstruction& i) { - assert(!i.outStack && !i.outLocal); - if (i.breaksTracelet) { - syncOutputs(t); - } - - // Check the surprise page on all backwards jumps - if (i.imm[0].u_BA < 0 && !i.noSurprise) { - if (trustSigSegv) { - const uint64_t stackMask = - ~(cellsToBytes(RuntimeOption::EvalVMStackElms) - 1); - a.mov_reg64_reg64(rVmSp, rAsm); - a.and_imm64_reg64(stackMask, rAsm); - TCA surpriseLoad = a.code.frontier; - a.load_reg64_disp_reg64(rAsm, 0, rAsm); - - if (!m_segvStubs.insert(SignalStubMap::value_type(surpriseLoad, - astubs.code.frontier))) - NOT_REACHED(); - /* - * Note that it is safe not to register unwind information here, - * because we just called syncOutputs so all registers are - * already clean. - */ - astubs.call((TCA)&EventHook::CheckSurprise); - recordStubCall(i); - astubs.jmp(a.code.frontier); - } else { - emitTestSurpriseFlags(a); - { - UnlikelyIfBlock ifSurprise(CC_NZ, a, astubs); - astubs.call((TCA)&EventHook::CheckSurprise); - recordStubCall(i); - } - } - } - if (i.breaksTracelet) { - SrcKey sk(curFunc(), i.offset() + i.imm[0].u_BA); - emitBindJmp(sk); - } -} - -void -TranslatorX64::analyzeSwitch(Tracelet& t, - NormalizedInstruction& i) { - RuntimeType& rtt = i.inputs[0]->rtt; - assert(rtt.outerType() != KindOfRef); - switch (rtt.outerType()) { - NULLCASE(): - case KindOfBoolean: - case KindOfInt64: - i.m_txFlags = Native; - break; - - case KindOfDouble: - i.m_txFlags = Simple; - break; - - STRINGCASE(): - case KindOfObject: - case KindOfArray: - i.m_txFlags = Supported; - break; - - default: - not_reached(); - } -} - -template -static int64_t switchBoundsCheck(T v, int64_t base, int64_t nTargets) { - // I'm relying on gcc to be smart enough to optimize away the next - // two lines when T is int64. - if (int64_t(v) == v) { - int64_t ival = v; - if (ival >= base && ival < (base + nTargets)) { - return ival - base; - } - } - return nTargets + 1; -} - -int64_t switchDoubleHelper(int64_t val, int64_t base, int64_t nTargets) { - union { - int64_t intbits; - double dblval; - } u; - u.intbits = val; - return switchBoundsCheck(u.dblval, base, nTargets); -} - -int64_t switchStringHelper(StringData* s, int64_t base, int64_t nTargets) { - int64_t ival; - double dval; - switch (s->isNumericWithVal(ival, dval, 1)) { - case KindOfNull: - ival = switchBoundsCheck(0, base, nTargets); - break; - - case KindOfDouble: - ival = switchBoundsCheck(dval, base, nTargets); - break; - - case KindOfInt64: - ival = switchBoundsCheck(ival, base, nTargets); - break; - - default: - not_reached(); - } - decRefStr(s); - return ival; -} - -int64_t switchObjHelper(ObjectData* o, int64_t base, int64_t nTargets) { - int64_t ival = o->o_toInt64(); - decRefObj(o); - return switchBoundsCheck(ival, base, nTargets); -} - -void -TranslatorX64::translateSwitch(const Tracelet& t, - const NormalizedInstruction& i) { - int64_t base = i.imm[1].u_I64A; - bool bounded = i.imm[2].u_IVA; - const ImmVector& iv = i.immVec; - int nTargets = bounded ? iv.size() - 2 : iv.size(); - int jmptabSize = nTargets; - assert(nTargets > 0); - PhysReg valReg = getReg(i.inputs[0]->location); - DataType inType = i.inputs[0]->outerType(); - assert(IMPLIES(inType != KindOfInt64, bounded)); - assert(IMPLIES(bounded, iv.size() > 2)); - syncOutputs(t); // this will mark valReg as FREE but it still has - // its old value - - SrcKey defaultSk(curFunc(), i.offset() + iv.vec32()[iv.size() - 1]); - SrcKey zeroSk(curFunc(), 0); - if (0 >= base && 0 < (base + nTargets)) { - zeroSk.m_offset = i.offset() + iv.vec32()[0 - base]; - } else { - zeroSk.m_offset = defaultSk.m_offset; - } - - switch (i.inputs[0]->outerType()) { - NULLCASE(): { - emitBindJmp(zeroSk); - return; - } - - case KindOfBoolean: { - SrcKey nonzeroSk(curFunc(), i.offset() + iv.vec32()[iv.size() - 2]); - a.test_reg64_reg64(valReg, valReg); - emitCondJmp(nonzeroSk, zeroSk, CC_NZ); - return; - } - - case KindOfInt64: - // No special treatment needed - break; - - case KindOfDouble: - STRINGCASE(): - case KindOfObject: { - // switch(Double|String|Obj)Helper do bounds-checking for us, so - // we need to make sure the default case is in the jump table, - // and don't emit our own bounds-checking code - jmptabSize = iv.size(); - bounded = false; - if (false) { - StringData* s = nullptr; - ObjectData* o = nullptr; - switchDoubleHelper(0.0, 0, 0); - switchStringHelper(s, 0, 0); - switchObjHelper(o, 0, 0); - } - EMIT_CALL(a, - inType == KindOfDouble ? (TCA)switchDoubleHelper : - (IS_STRING_TYPE(inType) ? (TCA)switchStringHelper : - (TCA)switchObjHelper), - R(valReg), IMM(base), IMM(nTargets)); - recordCall(i); - valReg = rax; - break; - } - - case KindOfArray: - emitDecRef(a, i, valReg, KindOfArray); - emitBindJmp(defaultSk); - return; - - default: - not_reached(); - } - - if (bounded) { - if (base) { - a.sub_imm64_reg64(base, valReg); - } - a.cmp_imm64_reg64(nTargets, valReg); - prepareForSmash(a, kJmpccLen); - TCA defaultStub = - emitServiceReq(REQ_BIND_JMPCC_SECOND, 3, - a.code.frontier, defaultSk.m_offset, CC_AE); - // Unsigned comparison: check for < 0 and >= nTargets at the same time - a.jae(defaultStub); - } - - TCA* jmptab = m_globalData.alloc(sizeof(TCA), jmptabSize); - TCA afterLea = a.code.frontier + kLeaRipLen; - ptrdiff_t diff = (TCA)jmptab - afterLea; - assert(deltaFits(diff, sz::dword)); - a. lea (rip[diff], rAsm); - assert(a.code.frontier == afterLea); - a. jmp (rAsm[valReg*8]); - - for (int idx = 0; idx < jmptabSize; ++idx) { - SrcKey sk(curFunc(), i.offset() + iv.vec32()[idx]); - jmptab[idx] = emitServiceReq(SRFlags::None, REQ_BIND_ADDR, 2ull, - &jmptab[idx], uint64_t(sk.offset())); - } -} - -void -TranslatorX64::analyzeSSwitch(Tracelet& t, - NormalizedInstruction& i) { - i.m_txFlags = Supported; -} - -static TCA sswitchHelperSlow(TypedValue* val, const StringData** strs, - int cases, TCA* jmptab) { - if (val->m_type == KindOfRef) val = val->m_data.pref->tv(); - for (int i = 0; i < cases; ++i) { - if (tvAsCVarRef(val).equal(strs[i])) return jmptab[i]; - } - // default case - return jmptab[cases]; -} - -typedef FixedStringMap SSwitchMap; - -HOT_FUNC_VM -TCA sswitchHelperFast(const StringData* val, SSwitchMap* table, - TCA* def) { - TCA* dest = table->find(val); - if (dest) { - return *dest; - } else { - return *def; - } -} - -void -TranslatorX64::translateSSwitch(const Tracelet& t, - const NormalizedInstruction& ni) { - DynLocation& input = *ni.inputs[0]; - Location& inLoc = input.location; - const ImmVector& iv = ni.immVec; - const StrVecItem* strvec = iv.strvec(); - int targets = iv.size(); - assert(targets > 1); - unsigned cases = targets - 1; - const Unit* u = curUnit(); - std::vector strings; - for (unsigned i = 0; i < cases; ++i) { - strings.push_back(u->lookupLitstrId(strvec[i].str)); - } - - // We support the fast path if the input is a string and none of the - // cases are numeric strings - bool fastPath = IS_STRING_TYPE(input.valueType()); - for (auto s : strings) { - if (s->isNumeric()) { - fastPath = false; - break; - } - } - - auto bindAddr = [&](TCA& dest, Offset o) { - SrcKey sk(curFunc(), ni.offset() + o); - dest = emitServiceReq(SRFlags::None, REQ_BIND_ADDR, 2ull, - &dest, uint64_t(sk.offset())); - }; - if (fastPath) { - Stats::emitInc(a, Stats::Tx64_StringSwitchFast); - - SSwitchMap* table = m_globalData.alloc(kX64CacheLineSize); - table->init(cases); - TCA* def = m_globalData.alloc(sizeof(TCA), 1); - for (unsigned i = 0; i < cases; ++i) { - table->add(strings[i], nullptr); - TCA* addr = table->find(strings[i]); - assert(addr && *addr == nullptr); - bindAddr(*addr, strvec[i].dest); - } - bindAddr(*def, strvec[targets-1].dest); - - EMIT_RCALL(a, ni, sswitchHelperFast, - input.isRef() ? DEREF(inLoc) : V(inLoc), - IMM(int64_t(table)), IMM(int64_t(def))); - } else { - Stats::emitInc(a, Stats::Tx64_StringSwitchSlow); - const StringData** strtab = m_globalData.alloc( - sizeof(const StringData*), cases); - memcpy(strtab, &strings[0], sizeof(const StringData*) * cases); - - // Build the jump table. - TCA* jmptab = m_globalData.alloc(sizeof(TCA), targets); - for (int i = 0; i < targets; ++i) { - bindAddr(jmptab[i], strvec[i].dest); - } - - m_regMap.cleanLoc(inLoc); - EMIT_RCALL(a, ni, sswitchHelperSlow, - A(inLoc), IMM(int64_t(strtab)), IMM(cases), IMM(int64_t(jmptab))); - } - ScratchReg holdRax(m_regMap, rax); - m_regMap.allocInputReg(ni, 0); - emitDecRef(a, ni, getReg(inLoc), input.outerType()); - syncOutputs(t); - a.jmp(rax); -} - -void -TranslatorX64::analyzeRetC(Tracelet& t, - NormalizedInstruction& i) { - i.manuallyAllocInputs = true; - i.m_txFlags = Supported; -} - -void -TranslatorX64::analyzeRetV(Tracelet& t, - NormalizedInstruction& i) { - analyzeRetC(t, i); -} - -void TranslatorX64::emitReturnVal( - Asm& a, const NormalizedInstruction& i, - PhysReg dstBase, int dstOffset, PhysReg thisBase, int thisOffset, - PhysReg scratch) { - - if (!i.grouped) return; - TypedValue tv; - tvWriteUninit(&tv); - tv.m_data.num = 0; // to keep the compiler happy - - auto moveRetValIfNeeded = [&] { - if (thisBase != dstBase || - thisOffset != (dstOffset + TVOFF(m_data))) { - a. loadq(thisBase[thisOffset], scratch); - a. storeq(scratch, dstBase[dstOffset + TVOFF(m_data)]); - } - }; - /* - * We suppressed the write of the (literal) return value - * to the stack. Figure out what it was. - */ - NormalizedInstruction* prev = i.prev; - assert(!prev->outStack); - switch (prev->op()) { - case OpNull: - tv.m_type = KindOfNull; - break; - case OpTrue: - case OpFalse: - tv.m_type = KindOfBoolean; - tv.m_data.num = prev->op() == OpTrue; - break; - case OpInt: - tv.m_type = KindOfInt64; - tv.m_data.num = prev->imm[0].u_I64A; - break; - case OpDouble: - tv.m_type = KindOfDouble; - tv.m_data.dbl = prev->imm[0].u_DA; - break; - case OpString: - tv.m_type = BitwiseKindOfString; - tv.m_data.pstr = curUnit()->lookupLitstrId(prev->imm[0].u_SA); - break; - case OpArray: - tv.m_type = KindOfArray; - tv.m_data.parr = curUnit()->lookupArrayId(prev->imm[0].u_AA); - break; - case OpThis: { - moveRetValIfNeeded(); - emitStoreTVType(a, KindOfObject, dstBase[dstOffset + TVOFF(m_type)]); - return; - } - case OpBareThis: { - assert(curFunc()->cls()); - moveRetValIfNeeded(); - a. mov_imm32_reg32(KindOfNull, scratch); - a. testb(1, thisBase[thisOffset]); - { - JccBlock noThis(a); - a. mov_imm32_reg32(KindOfObject, scratch); - } - emitStoreTVType(a, scratch, dstBase[dstOffset + TVOFF(m_type)]); - return; - } - default: - not_reached(); - } - - emitStoreTVType(a, tv.m_type, r64(dstBase)[dstOffset + TVOFF(m_type)]); - if (tv.m_type != KindOfNull) { - emitStoreImm(a, tv.m_data.num, - dstBase, dstOffset + TVOFF(m_data), sz::qword); - } - -} - -void TranslatorX64::emitDecRefThis(const ScratchReg& rTmp) { - // If we grouped a $this into the ret we're returning this, but we - // didn't incRef it, so we dont have to decRef here. - const bool mergedThis = m_curNI->wasGroupedWith(OpThis, OpBareThis); - if (mergedThis) { - return; - } - - /* - * In both of these cases we need to write back a null pointer to - * the this field in the ActRec, just for the case that a local - * might do debug_backtrace and access a freed object. - * - * In the case of mergedThis it's safe not to do this, because we - * are returning a reference on $this from the function so it will - * still be alive in any case. - */ - - // If this is a instance method called on an object or if it is a - // pseudomain, we need to decRef $this (if there is one) - if (curFunc()->isMethod() && !curFunc()->isStatic()) { - // This assert is weaker than it looks; it only checks the invocation - // we happen to be translating for. The runtime "assert" is the - // unconditional dereference of m_this we emit; if the frame has - // neither this nor a class, then m_this will be null and we'll - // SEGV. - assert(curFrame()->hasThis() || curFrame()->hasClass()); - // m_this and m_cls share a slot in the ActRec, so we check the - // lowest bit (0 -> m_this, 1 -> m_cls) - a. load_reg64_disp_reg64(rVmFp, AROFF(m_this), r(rTmp)); - a. store_imm64_disp_reg64(0, AROFF(m_this), rVmFp); - if (m_curNI->guardedThis) { - emitDecRef(*m_curNI, r(rTmp), KindOfObject); - } else { - a. testb(1, rbyte(rTmp)); - { - JccBlock ifZero(a); - emitDecRef(a, *m_curNI, r(rTmp), KindOfObject); - } - } - } else if (curFunc()->isPseudoMain()) { - a. load_reg64_disp_reg64(rVmFp, AROFF(m_this), r(rTmp)); - a. store_imm64_disp_reg64(0, AROFF(m_this), rVmFp); - a. shrq(1, r(rTmp)); // sets c (from bit 0) and z - FreezeRegs ice(m_regMap); - { - // tests for Not Zero and Not Carry - UnlikelyIfBlock ifRealThis(CC_NBE, a, astubs); - astubs. shlq(1, r(rTmp)); - emitDecRef(astubs, *m_curNI, r(rTmp), KindOfObject); - } - } -} - -/* - * If this function can possibly use variadic arguments or shared - * variable environment, we need to check for it and clear them if - * they exist. - */ -void TranslatorX64::emitVVRet(const ScratchReg& rTmp, - Label& extraArgsReturn, - Label& varEnvReturn) { - if (!(curFunc()->attrs() & AttrMayUseVV)) return; - SKTRACE(2, m_curNI->source, "emitting mayUseVV in UnlikelyIf\n"); - - a. load_reg64_disp_reg64(rVmFp, AROFF(m_varEnv), r(rTmp)); - a. test_reg64_reg64(r(rTmp), r(rTmp)); - { - // TODO: maybe this should be a semi-likely block when there - // is a varenv at translation time. - UnlikelyIfBlock varEnvCheck(CC_NZ, a, astubs); - auto& a = astubs; - - a. test_imm32_reg32(ActRec::kExtraArgsBit, r(rTmp)); - jccBlock(a, [&] { - guardDiamond(a, [&] { - EMIT_RCALL( - a, *m_curNI, - TCA(static_cast(ExtraArgs::deallocate)), - R(rVmFp) - ); - }); - extraArgsReturn.jmp(a); - }); - - m_regMap.cleanAll(); - EMIT_RCALL( - a, *m_curNI, - TCA(getMethodPtr(&VarEnv::detach)), - R(rTmp), - R(rVmFp) - ); - - if (!m_curNI->inlineReturn) { - // If it's not inline, the return we're about to jump to expects - // the helper has adjusted rVmSp already. - a.lea_reg64_disp_reg64(rVmFp, AROFF(m_r), rVmSp); - } - varEnvReturn.jmp(a); - } -} - -void TranslatorX64::emitInlineReturn(Location retvalSrcLoc, - int retvalSrcDisp) { - SKTRACE(2, m_curNI->source, "emitting specialized inline return\n"); - - assert(int(m_curNI->inputs.size()) == curFunc()->numLocals() - - int(m_curNI->nonRefCountedLocals.count())); - for (int k = m_curNI->inputs.size() - 1; k >= 0; --k) { - assert(m_curNI->inputs[k]->location.space == Location::Local); - DataType t = m_curNI->inputs[k]->outerType(); - if (GuardType(t).isCounted()) { - PhysReg reg = m_regMap.allocReg(m_curNI->inputs[k]->location, t, - RegInfo::CLEAN); - // We currently need to zero the type just in case the event - // hook throws (see #2088495), or a destructor captures a - // backtrace - PhysReg base; - int disp; - locToRegDisp(m_curNI->inputs[k]->location, &base, &disp); - emitStoreTVType(a, KindOfUninit, base[disp + TVOFF(m_type)]); - - emitDecRef(*m_curNI, reg, t); - } - } - - // Register map is officially out of commission now. - m_regMap.scrubLoc(retvalSrcLoc); - m_regMap.smashRegs(kAllRegs); -} - -void TranslatorX64::emitGenericReturn(bool noThis, int retvalSrcDisp) { - SKTRACE(2, m_curNI->source, "emitting generic return\n"); - assert(m_curNI->inputs.size() == 0); - - m_regMap.cleanAll(); - m_regMap.smashRegs(kAllRegs); - - if (m_curNI->grouped) { - /* - * What a pain: EventHook::onFunctionExit needs access - * to the return value - so we have to write it to the - * stack anyway. We still win for OpThis, and - * OpBareThis, since we dont have to do any refCounting - */ - ScratchReg s(m_regMap); - emitReturnVal(a, *m_curNI, - rVmSp, retvalSrcDisp, rVmFp, AROFF(m_this), r(s)); - } - - // Custom calling convention: the argument is in rVmSp. - int numLocals = curFunc()->numLocals(); - assert(numLocals > 0); - a. subq(0x8, rsp); // For parity. Callee will do retq $0x8. - a. lea(rVmFp[-numLocals * sizeof(TypedValue)], rVmSp); - if (numLocals > kNumFreeLocalsHelpers) { - emitCall(a, m_freeManyLocalsHelper); - } else { - emitCall(a, m_freeLocalsHelpers[numLocals - 1]); - } - recordReentrantCall(a, *m_curNI); -} - -void -TranslatorX64::translateRetC(const Tracelet& t, - const NormalizedInstruction& i) { - if (i.skipSync) assert(i.grouped); - - int stackAdjustment = t.m_stackChange; - if (i.skipSync) { - SKTRACE(2, i.source, "i.skipSync\n"); - - /* - * getting here means there was nothing to do between - * a previous reqXXX and this ret. Any spill code we generate - * here would be broken (because the rbx is wrong), so - * verify that we don't generate anything... - */ - TCA s DEBUG_ONLY = a.code.frontier; - syncOutputs(0); - assert(s == a.code.frontier); - stackAdjustment = 0; - } else { - /* - * no need to syncOutputs here... we're going to update - * rbx at the end of this function anyway, and we may want - * to use enregistered locals on the fast path below - */ - m_regMap.scrubStackEntries(t.m_stackChange); - m_regMap.cleanAll(); // TODO(#1339331): don't. - } - - const bool noThis = !curFunc()->isPseudoMain() && - (!curFunc()->isMethod() || curFunc()->isStatic()); - - /* - * figure out where to put the return value, and where to get it from - */ - assert(i.stackOff == t.m_stackChange); - const Location retvalSrcLoc(Location::Stack, stackAdjustment - 1); - - const Func *callee = curFunc(); - assert(callee); - int nLocalCells = - callee == nullptr ? 0 : // This happens for returns from pseudo-main. - callee->numSlotsInFrame(); - int retvalSrcDisp = cellsToBytes(-stackAdjustment); - assert(cellsToBytes(locPhysicalOffset(retvalSrcLoc)) == retvalSrcDisp); - - Label varEnvReturn; - Label extraArgsReturn; - { - ScratchReg rTmp(m_regMap); - emitDecRefThis(rTmp); - emitVVRet(rTmp, extraArgsReturn, varEnvReturn); - } -asm_label(a, extraArgsReturn); - if (m_curNI->inlineReturn) { - emitInlineReturn(retvalSrcLoc, retvalSrcDisp); - } else { - emitGenericReturn(noThis, retvalSrcDisp); - } - assert(m_regMap.empty()); - - // The (1 + nLocalCells) skips 1 slot for the return value. - int retvalDestDisp = cellsToBytes(1 + nLocalCells - stackAdjustment) + - AROFF(m_r); - - if (!m_curNI->inlineReturn) { - // Compensate for rVmSp already being adjusted by the helper in - // emitFrameRelease. - retvalSrcDisp -= sizeof(ActRec) + - cellsToBytes(nLocalCells - stackAdjustment); - retvalDestDisp = 0; - } - -asm_label(a, varEnvReturn); - emitTestSurpriseFlags(a); - { - UnlikelyIfBlock ifTracer(CC_NZ, a, astubs); - if (m_curNI->grouped) { - // We need to drop the return value on the stack for the event - // hook, same as in emitGenericReturn. - ScratchReg s(m_regMap); - emitReturnVal(astubs, *m_curNI, - rVmSp, retvalSrcDisp, rVmFp, AROFF(m_this), r(s)); - } - astubs.mov_reg64_reg64(rVmFp, argNumToRegName[0]); - emitCall(astubs, (TCA)&EventHook::onFunctionExit, true); - recordReentrantStubCall(*m_curNI); - } - - /* - * We're officially between tracelets now, and the normal register - * allocator is not being used. - */ - RegSet scratchRegs = kScratchCrossTraceRegs; - DumbScratchReg rRetAddr(scratchRegs); - - /* - * Having gotten everything we care about out of the current frame - * pointer, smash the return address type and value over it. We don't - * care about reference counts: as long as this runs to completion, we're - * refcount-neutral. - */ - if (i.grouped) { - DumbScratchReg s(scratchRegs); - emitReturnVal(a, i, rVmSp, retvalDestDisp, - rVmSp, retvalDestDisp - AROFF(m_r) + AROFF(m_this), - r(s)); - } else { - emitCopyToAligned(a, rVmSp, retvalSrcDisp, rVmSp, retvalDestDisp); - } - - /* - * Now update the principal hardware registers. - * - * Stack pointer has to skip over all the locals as well as the - * activation record. - */ - if (m_curNI->inlineReturn) { - // If we're not freeing inline, the helper took care of this. - a. lea_reg64_disp_reg64(rVmFp, AROFF(m_r), rVmSp); - } - a. load_reg64_disp_reg64(rVmFp, AROFF(m_savedRip), r(rRetAddr)); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_savedRbp), rVmFp); - emitRB(a, RBTypeFuncExit, curFunc()->fullName()->data(), RegSet(r(rRetAddr))); - // push the return address and do a ret - a. push(r(rRetAddr)); - a. ret(); - translator_not_reached(a); -} - -void -TranslatorX64::translateRetV(const Tracelet& t, - const NormalizedInstruction& i) { - translateRetC(t, i); -} - /* This is somewhat hacky. It decides which helpers/builtins should use * eager vmreganchor based on profile information. Using * eager vmreganchor for all helper calls is a perf regression. */ @@ -6585,19 +3636,17 @@ bool TranslatorX64::eagerRecord(const Func* func) { "func_num_args", "array_filter", "array_map", - "WaitHandle::join", }; - assert(func->info()); - - const StringData* name = func->isMethod() ? - func->fullName() : func->info()->name.get(); for (int i = 0; i < sizeof(list)/sizeof(list[0]); i++) { - if (!strcmp(name->data(), list[i])) { + if (!strcmp(func->name()->data(), list[i])) { return true; } } - + if (func->cls() && !strcmp(func->cls()->name()->data(), "WaitHandle") + && !strcmp(func->name()->data(), "join")) { + return true; + } return false; } @@ -6641,7 +3690,7 @@ int32_t TranslatorX64::emitNativeImpl(const Func* func, * contains only a single opcode (NativeImpl), and there are no * non-argument locals. */ - assert(func->numIterators() == 0 && func->info()); + assert(func->numIterators() == 0 && func->isBuiltin()); assert(func->numLocals() == func->numParams()); assert(*func->getEntry() == OpNativeImpl); assert(instrLen(func->getEntry()) == func->past() - func->base()); @@ -6681,18 +3730,6 @@ int32_t TranslatorX64::emitNativeImpl(const Func* func, return sizeof(ActRec) + cellsToBytes(nLocalCells-1); } -void -TranslatorX64::translateNativeImpl(const Tracelet& t, - const NormalizedInstruction& ni) { - /* - * We assume that NativeImpl is the only instruction in the trace, - * and the only instruction for the implementation of the function. - */ - assert(ni.stackOff == 0); - assert(m_regMap.empty()); - emitNativeImpl(curFunc(), true); -} - // emitClsLocalIndex -- // emitStringToClass -- // emitStringToKnownClass -- @@ -6761,154 +3798,6 @@ TranslatorX64::emitKnownClassCheck(const NormalizedInstruction& i, } } -void -TranslatorX64::emitStringToKnownClass(const NormalizedInstruction& i, - const StringData* clsName) { - ScratchReg cls(m_regMap); - emitKnownClassCheck(i, clsName, r(cls)); - m_regMap.bindScratch(cls, i.outStack->location, KindOfClass, RegInfo::DIRTY); -} - -void -TranslatorX64::emitStringToClass(const NormalizedInstruction& i) { - using namespace TargetCache; - if (!i.inputs[kEmitClsLocalIdx]->rtt.valueString()) { - // Handle the case where we don't know the name of the class - // at translation time - const Location& in = i.inputs[kEmitClsLocalIdx]->location; - const Location& out = i.outStack->location; - CacheHandle ch = ClassCache::alloc(); - if (false) { - StringData *name = nullptr; - const UNUSED Class* cls = ClassCache::lookup(ch, name); - } - TRACE(1, "ClassCache @ %d\n", int(ch)); - if (i.inputs[kEmitClsLocalIdx]->rtt.isRef()) { - EMIT_CALL(a, ClassCache::lookup, - IMM(ch), - DEREF(in)); - } else { - EMIT_CALL(a, ClassCache::lookup, - IMM(ch), - V(in)); - } - recordReentrantCall(i); - m_regMap.bind(rax, out, KindOfClass, RegInfo::DIRTY); - return; - } - // We know the name of the class at translation time; use the - // target cache associated with the name of the class - const StringData* clsName = i.inputs[kEmitClsLocalIdx]->rtt.valueString(); - emitStringToKnownClass(i, clsName); -} - -void -TranslatorX64::emitObjToClass(const NormalizedInstruction& i) { - m_regMap.allocOutputRegs(i); - const Location& in = i.inputs[kEmitClsLocalIdx]->location; - const Location& out = i.outStack->location; - PhysReg src = getReg(in); - ScratchReg tmp(m_regMap); - if (i.inputs[kEmitClsLocalIdx]->rtt.isRef()) { - emitDerefRef(a, src, r(tmp)); - src = r(tmp); - } - assert(i.outStack->valueType() == KindOfClass); - a. load_reg64_disp_reg64(src, ObjectData::getVMClassOffset(), getReg(out)); -} - -void -TranslatorX64::emitClsAndPals(const NormalizedInstruction& ni) { - if (ni.inputs[kEmitClsLocalIdx]->isString()) { - emitStringToClass(ni); - } else { - emitObjToClass(ni); - } -} - -void -TranslatorX64::analyzeAGetC(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - assert(i.outStack && !i.outLocal); - assert(i.outStack->valueType() == KindOfClass); - const RuntimeType& rtt = i.inputs[0]->rtt; - assert(!rtt.isRef()); - i.m_txFlags = supportedPlan(rtt.isString() || - rtt.valueType() == KindOfObject); - if (rtt.isString() && rtt.valueString()) i.manuallyAllocInputs = true; -} - -void TranslatorX64::translateAGetC(const Tracelet& t, - const NormalizedInstruction& ni) { - if (ni.outStack) { - emitClsAndPals(ni); - } -} - -void TranslatorX64::analyzeAGetL(Tracelet& t, - NormalizedInstruction& ni) { - assert(ni.inputs.size() == 1); - assert(ni.inputs[0]->isLocal()); - const RuntimeType& rtt = ni.inputs[0]->rtt; - ni.m_txFlags = supportedPlan(rtt.isString() || - rtt.valueType() == KindOfObject); -} - -void TranslatorX64::translateAGetL(const Tracelet& t, - const NormalizedInstruction& ni) { - emitClsAndPals(ni); -} - -void TranslatorX64::translateSelf(const Tracelet& t, - const NormalizedInstruction& i) { - m_regMap.allocOutputRegs(i); - PhysReg tmp = getReg(i.outStack->location); - assert(curFunc()->cls()); - emitImmReg(a, (int64_t)curFunc()->cls(), tmp); -} - -void TranslatorX64::translateParent(const Tracelet& t, - const NormalizedInstruction& i) { - m_regMap.allocOutputRegs(i); - PhysReg tmp = getReg(i.outStack->location); - assert(curFunc()->cls() && curFunc()->cls()->parent()); - emitImmReg(a, (int64_t)curFunc()->cls()->parent(), tmp); -} - -void TranslatorX64::analyzeSelf(Tracelet& t,NormalizedInstruction& i) { - Class* clss = curClass(); - if (clss == nullptr) { - i.m_txFlags = Interp; - return; - } - i.m_txFlags = Supported; -} - -void TranslatorX64::analyzeParent(Tracelet& t,NormalizedInstruction& i) { - Class* clss = curClass(); - if (clss == nullptr) { - i.m_txFlags = Interp; - return; - } - if (clss->parent() == nullptr) { - // clss has no parent; interpret to throw fatal - i.m_txFlags = Interp; - return; - } - i.m_txFlags = Supported; -} - -void TranslatorX64::translateDup(const Tracelet& t, - const NormalizedInstruction& ni) { - assert(ni.inputs.size() == 1); - assert(ni.outStack); - assert(!ni.inputs[0]->rtt.isRef()); - m_regMap.allocOutputRegs(ni); - PhysReg outR = getReg(ni.outStack->location); - emitMovRegReg(a, getReg(ni.inputs[0]->location), outR); - emitIncRef(outR, ni.inputs[0]->outerType()); -} - typedef std::map ParamMap; /* * mapContParams determines if every named local in origFunc has a @@ -6948,564 +3837,6 @@ void TranslatorX64::emitCallFillCont(X64Assembler& a, R(rax)); } -void TranslatorX64::translateCreateCont(const Tracelet& t, - const NormalizedInstruction& i) { - bool getArgs = i.imm[0].u_IVA; - const StringData* genName = curUnit()->lookupLitstrId(i.imm[1].u_SA); - const Func* origFunc = curFunc(); - const Func* genFunc = origFunc->getGeneratorBody(genName); - - if (false) { - ActRec* fp = nullptr; - UNUSED c_Continuation* cont = - VMExecutionContext::createContinuation(fp, getArgs, origFunc, - genFunc); - VMExecutionContext::createContinuation(fp, getArgs, origFunc, - genFunc); - } - - // Even callee-saved regs need to be clean, because - // createContinuation will read all locals. - m_regMap.cleanAll(); - auto helper = origFunc->isMethod() ? - VMExecutionContext::createContinuation : - VMExecutionContext::createContinuation; - EMIT_CALL(a, - (TCA)helper, - R(rVmFp), - IMM(getArgs), - IMM((intptr_t)origFunc), - IMM((intptr_t)genFunc)); - ScratchReg holdRax(m_regMap, rax); - - int origLocals = origFunc->numLocals(); - int genLocals = genFunc->numLocals(); - ContParamMap params; - if (origLocals <= kMaxInlineContLocals && - mapContParams(params, origFunc, genFunc)) { - ScratchReg rAsm(m_regMap); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_varEnv), r(rAsm)); - a. test_reg64_reg64(r(rAsm), r(rAsm)); - DiamondReturn astubsRet; - { - UnlikelyIfBlock ifVarEnv(CC_NZ, a, astubs, &astubsRet); - Stats::emitInc(astubs, Stats::Tx64_ContCreateSlow); - emitCallFillCont(astubs, origFunc, genFunc); - } - // fillContinuationVars returned the continuation in rax and - // DiamondGuard marked rax as scratch again, so it's safe to keep - // using it - Stats::emitInc(a, Stats::Tx64_ContCreateFast); - static const StringData* thisStr = StringData::GetStaticString("this"); - Id thisId = kInvalidId; - bool fillThis = origFunc->isNonClosureMethod() && !origFunc->isStatic() && - ((thisId = genFunc->lookupVarId(thisStr)) != kInvalidId) && - (origFunc->lookupVarId(thisStr) == kInvalidId); - ScratchReg rDest(m_regMap); - if (origLocals > 0 || fillThis) { - a. load_reg64_disp_reg32(rax, CONTOFF(m_localsOffset), r(rDest)); - a. add_reg64_reg64(rax, r(rDest)); - } - for (int i = 0; i < origLocals; ++i) { - assert(mapContains(params, i)); - int destOff = cellsToBytes(genLocals - params[i] - 1); - emitCopyTo(a, rVmFp, localOffset(i), r(rDest), destOff, r(rAsm)); - emitIncRefGenericRegSafe(r(rDest), destOff, r(rAsm)); - } - - // Deal with a potential $this local in the generator body - if (fillThis) { - assert(thisId != kInvalidId); - a. load_reg64_disp_reg64(rax, CONTOFF(m_obj), r(rAsm)); - a. test_reg64_reg64(r(rAsm), r(rAsm)); - { - JccBlock ifObj(a); - const int thisOff = cellsToBytes(genLocals - thisId - 1); - // We don't have to check for a static refcount since we - // know it's an Object - a. incl(r(rAsm)[FAST_REFCOUNT_OFFSET]); - a. storeq(r(rAsm), r(rDest)[thisOff + TVOFF(m_data)]); - emitStoreTVType(a, KindOfObject, r(rDest)[thisOff + TVOFF(m_type)]); - } - } - } else { - Stats::emitInc(a, Stats::Tx64_ContCreateSlow); - emitCallFillCont(a, origFunc, genFunc); - } - m_regMap.bindScratch(holdRax, i.outStack->location, KindOfObject, - RegInfo::DIRTY); -} - -void TranslatorX64::translateUnpackCont(const Tracelet& t, - const NormalizedInstruction& i) { - m_regMap.allocOutputRegs(i); - const int contIdx = 0; - assert(i.inputs.size() == 1); - assert(i.inputs[contIdx]->location == Location(Location::Local, 0)); - assert(i.outStack->outerType() == KindOfInt64); - - ScratchReg rAsm(m_regMap); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_varEnv), r(rAsm)); - a. test_reg64_reg64(r(rAsm), r(rAsm)); - { - UnlikelyIfBlock hasVars(CC_NZ, a, astubs); - Stats::emitInc(astubs, Stats::Tx64_ContUnpackSlow); - if (false) { - ActRec* fp = nullptr; - VMExecutionContext::unpackContVarEnvLinkage(fp); - } - EMIT_CALL(astubs, - VMExecutionContext::unpackContVarEnvLinkage, - R(rVmFp)); - // helper can't reenter - } - Stats::emitInc(a, Stats::Tx64_ContUnpackFast); - - PhysReg rCont = getReg(i.inputs[contIdx]->location); - PhysReg rLabel = getReg(i.outStack->location); - - a. load_reg64_disp_reg64(rCont, CONTOFF(m_label), rLabel); -} - -void TranslatorX64::emitCallPack(X64Assembler& a, - const NormalizedInstruction& i) { - if (false) { - ActRec* fp = nullptr; - VMExecutionContext::packContVarEnvLinkage(fp); - } - EMIT_CALL(a, - VMExecutionContext::packContVarEnvLinkage, - R(rVmFp)); - recordCall(a, i); -} - -void TranslatorX64::translatePackCont(const Tracelet& t, - const NormalizedInstruction& i) { - const int valIdx = 0; - const int contIdx = 1; - - ScratchReg rAsm(m_regMap); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_varEnv), r(rAsm)); - a. test_reg64_reg64(r(rAsm), r(rAsm)); - { - // TODO: Task #1132976: We can probably prove that this is impossible in - // most cases using information from hphpc - UnlikelyIfBlock varEnv(CC_NZ, a, astubs); - Stats::emitInc(astubs, Stats::Tx64_ContPackSlow); - emitCallPack(astubs, i); - } - Stats::emitInc(a, Stats::Tx64_ContPackFast); - - PhysReg rCont = getReg(i.inputs[contIdx]->location); - PhysReg rValue = getReg(i.inputs[valIdx]->location); - - // We're moving our reference to the value from the stack to the - // continuation object, so we don't have to incRef or decRef - emitTvSet(i, rValue, i.inputs[valIdx]->outerType(), - rCont, CONTOFF(m_value), false); - - emitImmReg(a, i.imm[0].u_IVA, r(rAsm)); - a. store_reg64_disp_reg64(r(rAsm), CONTOFF(m_label), rCont); -} - -static void continuationRaiseHelper(c_Continuation* cont) { - cont->t_raised(); - not_reached(); -} - -void TranslatorX64::emitContRaiseCheck(X64Assembler& a, - const NormalizedInstruction& i) { - const int contIdx = 0; - assert(i.inputs[contIdx]->location == Location(Location::Local, 0)); - PhysReg rCont = getReg(i.inputs[contIdx]->location); - a. testb(0x1, rCont[CONTOFF(m_should_throw)]); - { - UnlikelyIfBlock ifThrow(CC_NZ, a, astubs); - if (false) { - c_Continuation* c = nullptr; - continuationRaiseHelper(c); - } - EMIT_CALL(astubs, - continuationRaiseHelper, - R(rCont)); - recordReentrantStubCall(i); - translator_not_reached(astubs); - } -} - -void TranslatorX64::translateContReceive(const Tracelet& t, - const NormalizedInstruction& i) { - const int contIdx = 0; - emitContRaiseCheck(a, i); - PhysReg rCont = getReg(i.inputs[contIdx]->location); - ScratchReg rAsm(m_regMap); - emitLea(a, rCont, CONTOFF(m_received), r(rAsm)); - emitCopyToStack(a, i, r(rAsm), -1 * (int)sizeof(Cell)); - emitStoreUninitNull(a, CONTOFF(m_received), rCont); -} - -void TranslatorX64::translateContEnter(const Tracelet& t, - const NormalizedInstruction& i) { - // We're about to execute the generator body, which uses regs - syncOutputs(i); - - a. loadq (rVmFp[AROFF(m_this)], rStashedAR); - a. loadq (rStashedAR[CONTOFF(m_arPtr)], rStashedAR); - - // Frame linkage. - int32_t returnOffset = nextSrcKey(t, i).offset() - curFunc()->base(); - a. storel (returnOffset, rStashedAR[AROFF(m_soff)]); - a. storeq (rVmFp, rStashedAR[AROFF(m_savedRbp)]); - - // We're between tracelets; hardcode the register - a. loadq (rStashedAR[AROFF(m_func)], rax); - a. loadq (rax[Func::prologueTableOff() + sizeof(TCA)], rax); - - a. call (rax); -} - -void TranslatorX64::emitContExit() { - emitTestSurpriseFlags(a); - { - UnlikelyIfBlock ifTracer(CC_NZ, a, astubs); - astubs.mov_reg64_reg64(rVmFp, argNumToRegName[0]); - emitCall(astubs, (TCA)&EventHook::onFunctionExit, true); - recordReentrantStubCall(*m_curNI); - } - a. push (rVmFp[AROFF(m_savedRip)]); - a. loadq (rVmFp[AROFF(m_savedRbp)], rVmFp); - a. ret (); -} - -void TranslatorX64::translateContExit(const Tracelet& t, - const NormalizedInstruction& i) { - syncOutputs(i); - emitContExit(); -} - -void TranslatorX64::translateContRetC(const Tracelet& t, - const NormalizedInstruction& i) { - PhysReg valueReg = getReg(i.inputs[0]->location); - PhysReg contReg = getReg(i.inputs[1]->location); - a. store_imm8_disp_reg(0x1, CONTOFF(m_done), contReg); - - // m_value = $1 - emitTvSet(i, valueReg, i.inputs[0]->outerType(), - contReg, CONTOFF(m_value), false); - - // transfer control - syncOutputs(i.stackOff - 1); - emitContExit(); -} - -static void contPreNextThrowHelper(c_Continuation* c) { - c->preNext(); - not_reached(); -} - -void TranslatorX64::emitContPreNext(const NormalizedInstruction& i, - ScratchReg& rCont) { - const Offset doneOffset = CONTOFF(m_done); - static_assert((doneOffset + 1) == CONTOFF(m_running), - "m_done should immediately precede m_running"); - // Check m_done and m_running at the same time - a. test_imm32_disp_reg32(0x0101, doneOffset, r(rCont)); - { - UnlikelyIfBlock ifThrow(CC_NZ, a, astubs); - EMIT_CALL(astubs, contPreNextThrowHelper, R(rCont)); - recordReentrantStubCall(i); - translator_not_reached(astubs); - } - - // ++m_index - a. incq(r(rCont)[CONTOFF(m_index)]); - // m_running = true - a. store_imm8_disp_reg(0x1, CONTOFF(m_running), r(rCont)); -} - -void TranslatorX64::translateContNext(const Tracelet& t, - const NormalizedInstruction& i) { - ScratchReg rCont(m_regMap); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_this), r(rCont)); - emitContPreNext(i, rCont); - - // m_received.setNull() - emitTvSet(i, InvalidReg, KindOfNull, r(rCont), CONTOFF(m_received), false); -} - -static void contNextCheckThrowHelper(c_Continuation* cont) { - cont->startedCheck(); - not_reached(); -} - -void TranslatorX64::emitContStartedCheck(const NormalizedInstruction& i, - ScratchReg& cont) { - // if (m_index < 0) - a. cmpq (0x0, r(cont)[CONTOFF(m_index)]); - { - UnlikelyIfBlock whoops(CC_L, a, astubs); - EMIT_CALL(astubs, contNextCheckThrowHelper, r(cont)); - recordReentrantStubCall(i); - translator_not_reached(astubs); - } -} - -template -void TranslatorX64::translateContSendImpl(const NormalizedInstruction& i) { - const int valIdx = 0; - assert(i.inputs[valIdx]->location == Location(Location::Local, 0)); - - ScratchReg rCont(m_regMap); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_this), r(rCont)); - emitContStartedCheck(i, rCont); - emitContPreNext(i, rCont); - - // m_received = value - PhysReg valReg = getReg(i.inputs[valIdx]->location); - DataType valType = i.inputs[valIdx]->outerType(); - emitTvSet(i, valReg, valType, r(rCont), CONTOFF(m_received), true); - - // m_should_throw = true (maybe) - if (raise) { - a. store_imm8_disp_reg(0x1, CONTOFF(m_should_throw), r(rCont)); - } -} - -void TranslatorX64::translateContSend(const Tracelet& t, - const NormalizedInstruction& i) { - translateContSendImpl(i); -} - -void TranslatorX64::translateContRaise(const Tracelet& t, - const NormalizedInstruction& i) { - translateContSendImpl(i); -} - -void TranslatorX64::translateContValid(const Tracelet& t, - const NormalizedInstruction& i) { - ScratchReg rCont(m_regMap); - a. loadq (rVmFp[AROFF(m_this)], r64(rCont)); - - m_regMap.allocOutputRegs(i); - PhysReg validReg = getReg(i.outStack->location); - // !m_done - a. loadzbl (r(rCont)[CONTOFF(m_done)], r32(validReg)); - a. xorl (0x1, r32(validReg)); -} - -void TranslatorX64::translateContCurrent(const Tracelet& t, - const NormalizedInstruction& i) { - ScratchReg rCont(m_regMap); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_this), r(rCont)); - emitContStartedCheck(i, rCont); - - emitLea(a, r(rCont), CONTOFF(m_value), r(rCont)); - emitIncRefGeneric(r(rCont), 0); - emitCopyToStack(a, i, r(rCont), -1 * (int)sizeof(Cell)); -} - -void TranslatorX64::translateContStopped(const Tracelet& t, - const NormalizedInstruction& i) { - ScratchReg rCont(m_regMap); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_this), r(rCont)); - a. store_imm8_disp_reg(0x0, CONTOFF(m_running), r(rCont)); -} - -void TranslatorX64::analyzeStrlen(Tracelet& t, - NormalizedInstruction& i) { - switch (i.inputs[0]->rtt.valueType()) { - STRINGCASE() : - // May have to destroy a StringData, but can't reenter - i.m_txFlags = Simple; - break; - NULLCASE() : - case KindOfBoolean: - case KindOfArray: - case KindOfInt64: - case KindOfDouble: - case KindOfObject: - i.m_txFlags = Interp; - break; - default: - not_reached(); - } -} - -void TranslatorX64::translateStrlen(const Tracelet& t, - const NormalizedInstruction& i) { - PhysReg rInput = getReg(i.inputs[0]->location); - DataType inType = i.inputs[0]->rtt.valueType(); - - switch (inType) { - NULLCASE(): { - m_regMap.allocOutputRegs(i); - PhysReg rOutput = getReg(i.outStack->location); - a. xor_reg64_reg64(rOutput, rOutput); - break; - } - case KindOfBoolean: - m_regMap.allocOutputRegs(i); - // Nothing. strlen(true) == 1, strlen(false) == 0 - break; - STRINGCASE(): { - ScratchReg rAsm(m_regMap); - a. load_reg64_disp_reg32(rInput, StringData::sizeOffset(), r(rAsm)); - emitDecRef(a, i, rInput, inType); - m_regMap.bindScratch(rAsm, i.outStack->location, KindOfInt64, - RegInfo::DIRTY); - assert(m_regMap.regIsFree(rInput)); - break; - } - case KindOfArray: - case KindOfInt64: - case KindOfDouble: - case KindOfObject: - default: - not_reached(); - } -} -void TranslatorX64::translateIncStat(const Tracelet& t, - const NormalizedInstruction& i) { - int32_t counter = i.imm[0].u_IVA; - int32_t value = i.imm[1].u_IVA; - Stats::emitInc(a, Stats::StatCounter(counter), value); -} - -static void analyzeClassExistsImpl(NormalizedInstruction& i) { - const int nameIdx = 1; - const int autoIdx = 0; - assert(!i.inputs[nameIdx]->isRef() && !i.inputs[autoIdx]->isRef()); - i.m_txFlags = supportedPlan(i.inputs[nameIdx]->isString() && - i.inputs[autoIdx]->isBoolean()); - i.fuseBranch = (i.m_txFlags & Supported) && - i.inputs[nameIdx]->rtt.valueString() && - i.inputs[autoIdx]->rtt.valueBoolean() != RuntimeType::UnknownBool; -} - -void TranslatorX64::analyzeClassExists(Tracelet& t, - NormalizedInstruction& i) { - analyzeClassExistsImpl(i); -} - -void TranslatorX64::analyzeInterfaceExists(Tracelet& t, - NormalizedInstruction& i) { - analyzeClassExistsImpl(i); -} - -void TranslatorX64::analyzeTraitExists(Tracelet& t, - NormalizedInstruction& i) { - analyzeClassExistsImpl(i); -} - -static int64_t classExistsSlow(const StringData* name, bool autoload, - Attr typeAttr) { - bool ret = Unit::classExists(name, autoload, typeAttr); - // XXX: do we need to decref this during an exception? - decRefStr(const_cast(name)); - return ret; -} - -void TranslatorX64::translateClassExistsImpl(const Tracelet& t, - const NormalizedInstruction& i, - Attr typeAttr) { - const int nameIdx = 1; - const int autoIdx = 0; - const StringData* name = i.inputs[nameIdx]->rtt.valueString(); - assert(IMPLIES(name, name->isStatic())); - const int autoload = i.inputs[autoIdx]->rtt.valueBoolean(); - - ScratchReg scratch(m_regMap); - if (name != nullptr && autoload != RuntimeType::UnknownBool) { - assert(i.fuseBranch); - const Attr attrNotClass = Attr(AttrTrait | AttrInterface); - const bool isClass = typeAttr == AttrNone; - using namespace TargetCache; - Stats::emitInc(a, Stats::Tx64_ClassExistsFast); - CacheHandle ch = allocKnownClass(name); - - { - DiamondReturn astubsRet; - a. load_reg64_disp_reg64(rVmTl, ch, r(scratch)); - a. test_reg64_reg64(r(scratch), r(scratch)); - if (autoload) { - UnlikelyIfBlock ifNull(CC_Z, a, astubs, &astubsRet); - if (false) { - Class** c = nullptr; - UNUSED Class* ret = lookupKnownClass(c, name, false); - } - Stats::emitInc(astubs, Stats::TgtCache_ClassExistsMiss); - // If the class exists after autoloading, the helper will - // return the Class's flags. Otherwise, it will return a set - // of flags such that our flag check at the join point below - // will fail. - EMIT_CALL(astubs, (lookupKnownClass_func_t)lookupKnownClass, - RPLUS(rVmTl, ch), - IMM((uintptr_t)name), - IMM(isClass)); - recordReentrantStubCall(i); - emitMovRegReg(astubs, rax, r(scratch)); - } else { - UnlikelyIfBlock ifNull(CC_Z, a, astubs, &astubsRet); - // This isn't really a traditional slow path, count as a hit - Stats::emitInc(astubs, Stats::TgtCache_ClassExistsHit); - // Provide flags so the check back in a fails - emitImmReg(astubs, isClass ? attrNotClass : AttrNone, r(scratch)); - } - // If we don't take the slow/NULL path, load the Class's attrs - // into *scratch to prepare for the flag check. - Stats::emitInc(a, Stats::TgtCache_ClassExistsHit); - a. load_reg64_disp_reg64(r(scratch), Class::preClassOff(), - r(scratch)); - a. load_reg64_disp_reg32(r(scratch), PreClass::attrsOffset(), - r(scratch)); - } - - if (i.changesPC) { - fuseBranchSync(t, i); - } - prepareForTestAndSmash(a, kTestImmRegLen, kAlignJccAndJmp); - a. test_imm32_reg32(isClass ? attrNotClass : typeAttr, r(scratch)); - ConditionCode cc = isClass ? CC_Z : CC_NZ; - if (i.changesPC) { - fuseBranchAfterBool(t, i, cc); - } else { - a. setcc(cc, rbyte(scratch)); - a. movzbl(rbyte(scratch), r32(scratch)); - m_regMap.bindScratch(scratch, i.outStack->location, KindOfBoolean, - RegInfo::DIRTY); - } - } else { - assert(!i.fuseBranch); - Stats::emitInc(a, Stats::Tx64_ClassExistsSlow); - if (false) { - UNUSED bool ret = false; - ret = classExistsSlow(name, ret, typeAttr); - } - EMIT_CALL(a, classExistsSlow, - V(i.inputs[nameIdx]->location), - V(i.inputs[autoIdx]->location), - IMM(typeAttr)); - recordReentrantCall(i); - // Our helper decrefs the string - m_regMap.bind(rax, i.outStack->location, KindOfBoolean, RegInfo::DIRTY); - } -} - -void TranslatorX64::translateClassExists(const Tracelet& t, - const NormalizedInstruction& i) { - translateClassExistsImpl(t, i, AttrNone); -} - -void TranslatorX64::translateInterfaceExists(const Tracelet& t, - const NormalizedInstruction& i) { - translateClassExistsImpl(t, i, AttrInterface); -} - -void TranslatorX64::translateTraitExists(const Tracelet& t, - const NormalizedInstruction& i) { - translateClassExistsImpl(t, i, AttrTrait); -} - // Helper function for static property access. This function emits code // which leaves a pointer to the static property for clsInput::$propInput in // register scr. We destroy scr early on, yet do not consume inputs until @@ -7566,1209 +3897,6 @@ void TranslatorX64::emitStaticPropInlineLookup(const NormalizedInstruction& i, } } -void TranslatorX64::analyzeCGetS(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - assert(i.inputs[0]->valueType() == KindOfClass); - assert(i.outStack); - const Class* cls = i.inputs[0]->rtt.valueClass(); - const StringData* propName = i.inputs[1]->rtt.valueString(); - i.m_txFlags = supportedPlan(cls && propName && curFunc()->cls() == cls); - i.manuallyAllocInputs = true; -} - -void TranslatorX64::translateCGetS(const Tracelet& t, - const NormalizedInstruction& i) { - const int kClassIdx = 0; - const int kPropIdx = 1; - - ScratchReg sprop(m_regMap); - emitStaticPropInlineLookup(i, kClassIdx, *i.inputs[kPropIdx], r(sprop)); - emitDerefIfVariant(a, r(sprop)); - emitIncRefGeneric(r(sprop), 0); - // Finally copy the thing to the stack - int stackDest = 2 * sizeof(Cell) - sizeof(Cell); // popped - pushed - emitCopyToStack(a, i, r(sprop), stackDest); -} - -void TranslatorX64::analyzeSetS(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 3); - assert(i.inputs[1]->valueType() == KindOfClass); - assert(i.outStack); - const Class* cls = i.inputs[1]->rtt.valueClass(); - const StringData* propName = i.inputs[2]->rtt.valueString(); - // Might be able to broaden this: if cls is an ancestor of the current context, - // the context is Fixed, and the property is not private - // Also if the m_hoistable in cls is set to AlwaysHoistable, defined in - // the same unit as context, and the property is public - i.m_txFlags = supportedPlan(cls && propName && curFunc()->cls() == cls); - i.manuallyAllocInputs = true; -} - -void TranslatorX64::translateSetS(const Tracelet& t, - const NormalizedInstruction& i) { - const int kClassIdx = 1; - - ScratchReg sprop(m_regMap); - const RuntimeType& rhsType = i.inputs[0]->rtt; - emitStaticPropInlineLookup(i, kClassIdx, *i.inputs[2], r(sprop)); - - assert(m_regMap.getInfo(r(sprop))->m_state == RegInfo::SCRATCH); - assert(!rhsType.isRef()); - - m_regMap.allocInputReg(i, 0); - m_regMap.allocOutputRegs(i); - PhysReg rhsReg = getReg(i.inputs[0]->location); - PhysReg outReg = getReg(i.outStack->location); - emitTvSet(i, rhsReg, rhsType.outerType(), r(sprop)); - assert(i.inputs[2]->location == i.outStack->location); - emitMovRegReg(a, rhsReg, outReg); -} - -void TranslatorX64::analyzeSetG(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - i.m_txFlags = supportedPlan( - i.inputs[1]->isString() && - !i.inputs[0]->isRef() - ); - if (i.m_txFlags) i.manuallyAllocInputs = true; -} - -void TranslatorX64::translateSetG(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.outStack && !i.outLocal); - assert(i.inputs.size() == 2); - assert(i.inputs[1]->isString()); - assert(i.inputs[1]->location == i.outStack->location); - - const DataType type = i.inputs[0]->rtt.outerType(); - - /* - * Grab the global from the target cache; rax will get a pointer to - * the TypedValue in the globals array, maybe newly created as a - * null. - */ - emitGetGlobal(i, 1, true /* allowCreate */); - ScratchReg raxSaver(m_regMap, rax); - m_regMap.allocInputReg(i, 0); - PhysReg src = getReg(i.inputs[0]->location); - m_regMap.allocOutputRegs(i); - PhysReg out = getReg(i.outStack->location); - - emitTvSet(i, src, type, rax); - emitMovRegReg(a, src, out); -} - -static TypedValue* lookupGlobal(StringData* name) { - VarEnv* ve = g_vmContext->m_globalVarEnv; - TypedValue* r = ve->lookup(name); - // If the global didn't exist, we need to leave name un-decref'd for - // the caller to raise warnings. - if (r) { - decRefStr(name); - if (r->m_type == KindOfRef) r = r->m_data.pref->tv(); - } - return r; -} - -static TypedValue* lookupAddGlobal(StringData* name) { - VarEnv* ve = g_vmContext->m_globalVarEnv; - TypedValue* r = ve->lookupAdd(name); - if (r->m_type == KindOfRef) r = r->m_data.pref->tv(); - decRefStr(name); - return r; -} - -/* - * Look up a global in the TargetCache with the name - * i.inputs[nameIdx]. If `allowCreate' is true, also creates it. If - * we don't create the global, the input name is not decref'd yet. - */ -void -TranslatorX64::emitGetGlobal(const NormalizedInstruction& i, int nameIdx, - bool allowCreate) { - using namespace TargetCache; - assert(i.inputs.size() > size_t(nameIdx)); - assert(i.inputs[nameIdx]->isString()); - - const StringData *maybeName = i.inputs[nameIdx]->rtt.valueString(); - if (!maybeName) { - m_regMap.allocInputReg(i, nameIdx, argNumToRegName[0]); - // Always do a lookup when there's no statically-known name. - // There's not much we can really cache here right now anyway. - EMIT_CALL(a, allowCreate ? lookupAddGlobal : lookupGlobal, - V(i.inputs[nameIdx]->location)); - recordCall(i); - return; - } - - CacheHandle ch = GlobalCache::alloc(maybeName); - if (false) { // typecheck - StringData* UNUSED key = nullptr; - TypedValue* UNUSED glob = GlobalCache::lookup(ch, key); - TypedValue* UNUSED glob2 = GlobalCache::lookupCreate(ch, key); - } - SKTRACE(1, i.source, "ch %d\n", ch); - EMIT_CALL(a, allowCreate ? GlobalCache::lookupCreate - : GlobalCache::lookup, - IMM(ch), - IMM((uint64_t)maybeName)); - recordCall(i); -} - -static bool -isSupportedInstrCGetG(const NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - return (i.inputs[0]->rtt.isString()); -} - -void -TranslatorX64::analyzeCGetG(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = simplePlan(isSupportedInstrCGetG(i)); - if (i.m_txFlags) i.manuallyAllocInputs = true; -} - -void -TranslatorX64::translateCGetG(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.outStack && !i.outLocal); - assert(i.inputs.size() == 1); - assert(i.inputs[0]->isString()); - - emitGetGlobal(i, 0, false /* allowCreate */); - ScratchReg raxHolder(m_regMap, rax); - - // If non-null, rax now points to the in-memory location of the - // object of unknown type. lookup() has already decref'd the name. - a. test_reg64_reg64(rax, rax); - DiamondReturn astubsRet; - { - UnlikelyIfBlock ifNotRax(CC_Z, a, astubs, &astubsRet); - if (!i.inputs[0]->rtt.valueString()) { - m_regMap.allocInputReg(i, 0); - PhysReg reg = getReg(i.inputs[0]->location); - emitDecRef(astubs, i, reg, BitwiseKindOfString); - } - // TODO: if (MoreWarnings) raise a undefined variable warning. - // (Note: when changing this remember to change the Simple flag to - // Supported in analyze.) - emitStoreNull(astubs, vstackOffset(i, 0), rVmSp); - m_regMap.invalidate(i.outStack->location); - } - - emitCopyToStack(a, i, rax, 0); - emitIncRefGeneric(rax, 0); - m_regMap.invalidate(i.outStack->location); -} - -void TranslatorX64::analyzeFPassL(Tracelet& t, - NormalizedInstruction& ni) { - if (ni.preppedByRef) { - analyzeVGetL(t, ni); - } else { - analyzeCGetL(t, ni); - } -} - -void TranslatorX64::translateFPassL(const Tracelet& t, - const NormalizedInstruction& ni) { - if (ni.preppedByRef) { - translateVGetL(t, ni); - } else { - translateCGetL(t, ni); - } -} - -void TranslatorX64::analyzeFPassS(Tracelet& t, - NormalizedInstruction& ni) { - if (ni.preppedByRef) { - // We need a VGetS translation. - ni.m_txFlags = Interp; - } else { - analyzeCGetS(t, ni); - } -} - -void TranslatorX64::translateFPassS(const Tracelet& t, - const NormalizedInstruction& ni) { - if (ni.preppedByRef) { - assert(false); - } else { - translateCGetS(t, ni); - } -} - -void TranslatorX64::analyzeFPassG(Tracelet& t, - NormalizedInstruction& ni) { - if (ni.preppedByRef) { - analyzeVGetG(t, ni); - } else { - analyzeCGetG(t, ni); - } -} - -void TranslatorX64::translateFPassG(const Tracelet& t, - const NormalizedInstruction& ni) { - if (ni.preppedByRef) { - translateVGetG(t, ni); - } else { - translateCGetG(t, ni); - } -} - -void TranslatorX64::analyzeCheckTypeOp(Tracelet& t, - NormalizedInstruction& ni) { - assert(ni.inputs.size() == 1); - - if (ni.op() == OpIsObjectL || ni.op() == OpIsObjectC) { - // is_object is weird because it's supposed to return false for - // things where ObjectData::isResource() is true. For now we only - // translate when it is not an object. - if (ni.inputs[0]->valueType() == KindOfObject) { - ni.m_txFlags = Interp; - return; - } - } - - if (ni.inputs[0]->isLocal()) { - ni.manuallyAllocInputs = true; - if (ni.op() != OpIssetL && ni.inputs[0]->rtt.isUninit()) { - ni.m_txFlags = Supported; - } else { - ni.m_txFlags = Native; - } - return; - } - - ni.m_txFlags = planHingesOnRefcounting(ni.inputs[0]->valueType()); -} - -static bool checkTypeHelper(Opcode op, DataType dt) { - switch (op) { - case OpIssetL: return !IS_NULL_TYPE(dt); - case OpIsNullL: case OpIsNullC: return IS_NULL_TYPE(dt); - case OpIsStringL: case OpIsStringC: return IS_STRING_TYPE(dt); - case OpIsArrayL: case OpIsArrayC: return IS_ARRAY_TYPE(dt); - case OpIsIntL: case OpIsIntC: return IS_INT_TYPE(dt); - case OpIsBoolL: case OpIsBoolC: return IS_BOOL_TYPE(dt); - case OpIsDoubleL: case OpIsDoubleC: return IS_DOUBLE_TYPE(dt); - - case OpIsObjectL: case OpIsObjectC: - // Note: this is because we refused to translate if it was - // actually an object for now. (We'd need to emit some kind of - // call to ObjectData::isResource or something.) - return 0; - } - assert(false); - NOT_REACHED(); -} - -static void warnNullThis() { raise_notice(Strings::WARN_NULL_THIS); } - -void -TranslatorX64::translateCheckTypeOp(const Tracelet& t, - const NormalizedInstruction& ni) { - assert(ni.inputs.size() == 1); - assert(ni.outStack); - - bool isType; - - if (ni.wasGroupedWith(OpThis, OpBareThis)) { - assert(ni.op() == OpIsNullC); - if (ni.prev->op() == OpThis) { - isType = false; - } else { - if (ni.changesPC) { - fuseBranchSync(t, ni); - a. testb(1, rVmFp[AROFF(m_this)]); - if (ni.prev->imm[0].u_OA) { - UnlikelyIfBlock nullThis(CC_NZ, a, astubs); - EMIT_CALL(astubs, warnNullThis); - recordReentrantStubCall(ni); - nullThis.reconcileEarly(); - astubs.testb(1, rVmFp[AROFF(m_this)]); - } - fuseBranchAfterBool(t, ni, ni.invertCond ? CC_Z : CC_NZ); - } else { - m_regMap.allocOutputRegs(ni); - PhysReg res = getReg(ni.outStack->location); - a. testb(1, rVmFp[AROFF(m_this)]); - a. setcc(ni.invertCond ? CC_Z : CC_NZ, rbyte(res)); - if (ni.prev->imm[0].u_OA) { - UnlikelyIfBlock nullThis(CC_NZ, a, astubs); - EMIT_CALL(astubs, warnNullThis); - recordReentrantStubCall(ni); - } - a. movzbl (rbyte(res), r32(res)); - } - return; - } - } else { - const DataType dt = ni.inputs[0]->valueType(); - const bool isLocalOp = ni.inputs[0]->isLocal(); - - isType = checkTypeHelper(ni.op(), dt) != ni.invertCond; - if (!isLocalOp) { - emitDecRef(ni, getReg(ni.inputs[0]->location), dt); - } - if (isLocalOp && - ni.op() != OpIssetL && - ni.inputs[0]->rtt.isUninit()) { - const StringData* name = local_name(ni.inputs[0]->location); - assert(name->isStatic()); - EMIT_CALL(a, raiseUndefVariable, IMM((uintptr_t)name)); - recordReentrantCall(ni); - } - } - - m_regMap.allocOutputRegs(ni); - if (ni.changesPC) { - // Don't bother driving an output reg. Just take the branch - // where it leads. - Stats::emitInc(a, Stats::Tx64_FusedTypeCheck); - fuseBranchAfterStaticBool(a, t, ni, isType); - return; - } - Stats::emitInc(a, Stats::Tx64_UnfusedTypeCheck); - emitImmReg(a, isType, getReg(ni.outStack->location)); -} - -static void badArray() { - throw_bad_type_exception("array_key_exists expects an array or an object; " - "false returned."); -} - -static void badKey() { - raise_warning("Array key should be either a string or an integer"); -} - -static inline int64_t ak_exist_string_helper(StringData* key, ArrayData* arr) { - int64_t n; - if (key->isStrictlyInteger(n)) { - return arr->exists(n); - } - return arr->exists(StrNR(key)); -} - -static int64_t ak_exist_string(StringData* key, ArrayData* arr) { - int64_t res = ak_exist_string_helper(key, arr); - decRefArr(arr); - decRefStr(key); - return res; -} - -static int64_t ak_exist_int(int64_t key, ArrayData* arr) { - bool res = arr->exists(key); - decRefArr(arr); - return res; -} - -static int64_t ak_exist_string_obj(StringData* key, ObjectData* obj) { - if (obj->isCollection()) { - return collectionOffsetContains(obj, key); - } - CArrRef arr = obj->o_toArray(); - int64_t res = ak_exist_string_helper(key, arr.get()); - decRefObj(obj); - decRefStr(key); - return res; -} - -static int64_t ak_exist_int_obj(int64_t key, ObjectData* obj) { - if (obj->isCollection()) { - return collectionOffsetContains(obj, key); - } - CArrRef arr = obj->o_toArray(); - bool res = arr.get()->exists(key); - decRefObj(obj); - return res; -} - -void -TranslatorX64::analyzeAKExists(Tracelet& t, NormalizedInstruction& i) { - const int keyIx = 1; - const int arrIx = 0; - - const DataType dta = i.inputs[arrIx]->valueType(); - const DataType dtk = i.inputs[keyIx]->valueType(); - - bool reentrant = (dta != KindOfArray && dta != KindOfObject) || - (!IS_STRING_TYPE(dtk) && dtk != KindOfInt64 && dtk != KindOfNull); - - i.m_txFlags = reentrant ? Supported : Simple; - i.manuallyAllocInputs = true; -} - -void -TranslatorX64::translateAKExists(const Tracelet& t, - const NormalizedInstruction& ni) { - assert(ni.inputs.size() == 2); - assert(ni.outStack); - - const int keyIx = 1; - const int arrIx = 0; - - const DataType dta = ni.inputs[arrIx]->valueType(); - const DataType dtk = ni.inputs[keyIx]->valueType(); - TCA string_func = (TCA)ak_exist_string; - TCA int_func = (TCA)ak_exist_int; - - int result = -1; - int args[2]; - args[keyIx] = 0; - args[arrIx] = 1; - switch (dta) { - case KindOfObject: - string_func = (TCA)ak_exist_string_obj; - int_func = (TCA)ak_exist_int_obj; - case KindOfArray: - switch (dtk) { - case BitwiseKindOfString: - case KindOfStaticString: - case KindOfInt64: { - allocInputsForCall(ni, args); - PhysReg rk = getReg(ni.inputs[keyIx]->location); - PhysReg ra = getReg(ni.inputs[arrIx]->location); - m_regMap.scrubStackEntries(ni.outStack->location.offset); - EMIT_CALL(a, dtk == KindOfInt64 ? int_func : string_func, - R(rk), R(ra)); - recordCall(ni); - break; - } - case KindOfNull: - if (dta == KindOfArray) { - args[keyIx] = ArgDontAllocate; - allocInputsForCall(ni, args); - PhysReg ra = getReg(ni.inputs[arrIx]->location); - m_regMap.scrubStackEntries(ni.outStack->location.offset); - EMIT_CALL(a, string_func, - IMM((uint64_t)empty_string.get()), R(ra)); - recordCall(ni); - } else { - result = ni.invertCond; - } - break; - default: - EMIT_CALL(a, badKey); - recordReentrantCall(ni); - result = ni.invertCond; - break; - } - break; - default: - EMIT_CALL(a, badArray); - recordReentrantCall(ni); - result = ni.invertCond; - break; - } - - if (result >= 0) { - if (ni.changesPC) { - fuseBranchAfterStaticBool(a, t, ni, result); - return; - } else { - m_regMap.allocOutputRegs(ni); - emitImmReg(a, result, getReg(ni.outStack->location)); - } - } else { - ScratchReg res(m_regMap, rax); - if (ni.changesPC) { - fuseBranchSync(t, ni); - prepareForTestAndSmash(a, kTestRegRegLen, kAlignJccAndJmp); - a. test_reg64_reg64(r(res), r(res)); - fuseBranchAfterBool(t, ni, ni.invertCond ? CC_Z : CC_NZ); - } else { - if (ni.invertCond) { - a. xor_imm32_reg64(1, r(res)); - } - m_regMap.bindScratch(res, ni.outStack->location, KindOfBoolean, - RegInfo::DIRTY); - } - } -} - -void -TranslatorX64::analyzeSetOpL(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 2); - const SetOpOp subOp = SetOpOp(i.imm[1].u_OA); - Opcode arithOp = setOpOpToOpcodeOp(subOp); - i.m_txFlags = nativePlan(i.inputs[0]->isInt() && - i.inputs[1]->valueType() == KindOfInt64 && - (arithOp == OpAdd || arithOp == OpSub || - arithOp == OpMul || - arithOp == OpBitAnd || arithOp == OpBitOr || - arithOp == OpBitXor)); - if (!i.m_txFlags) { - i.m_txFlags = nativePlan(mathEquivTypes(i.inputs[0]->rtt, - i.inputs[1]->rtt) && - (arithOp == OpAdd || arithOp == OpSub || - arithOp == OpMul)); - return; - } -} - -void -TranslatorX64::translateSetOpL(const Tracelet& t, - const NormalizedInstruction& i) { - const vector& inputs = i.inputs; - assert(inputs.size() >= 2); - assert(i.outStack && i.outLocal); - const int valIdx = 0; - const int localIdx = 1; - assert(inputs[localIdx]->isLocal()); - assert(inputs[valIdx]->isStack()); - assert(inputs[valIdx]->outerType() != KindOfRef); - - const SetOpOp subOp = SetOpOp(i.imm[1].u_OA); - Opcode arithOp = setOpOpToOpcodeOp(subOp); - m_regMap.allocOutputRegs(i); - binaryArithLocal(i, arithOp, *inputs[valIdx], *inputs[localIdx], - *i.outStack); -} - -void -TranslatorX64::analyzeIncDecL(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = nativePlan(i.inputs[0]->isInt()); -} - -void -TranslatorX64::translateIncDecL(const Tracelet& t, - const NormalizedInstruction& i) { - const vector& inputs = i.inputs; - assert(inputs.size() == 1); - assert(i.outLocal); - assert(inputs[0]->isLocal()); - const IncDecOp oplet = IncDecOp(i.imm[1].u_OA); - assert(oplet == PreInc || oplet == PostInc || oplet == PreDec || - oplet == PostDec); - assert(inputs[0]->isInt() && (!i.outStack || i.outStack->isInt())); - bool post = (oplet == PostInc || oplet == PostDec); - bool pre = !post; - bool inc = (oplet == PostInc || oplet == PreInc); - - m_regMap.allocOutputRegs(i); - PhysReg localVal = getReg(inputs[0]->location); - if (i.outStack && post) { // $a++, $a-- - PhysReg output = getReg(i.outStack->location); - emitMovRegReg(a, localVal, output); - } - if (inc) { - a. incq(localVal); - } else { - a. decq(localVal); - } - if (i.outStack && pre) { // --$a, ++$a - PhysReg output = getReg(i.outStack->location); - emitMovRegReg(a, localVal, output); - } -} - -void -TranslatorX64::translateUnsetL(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - assert(!i.outStack && i.outLocal); - const int locIdx = 0; - const DynLocation& localDl = *i.inputs[locIdx]; - assert(localDl.isLocal()); - - // We have to mark the output register as dirty to ensure that - // the type gets spilled at the tend of the tracelet - m_regMap.allocOutputRegs(i); - - DataType type = localDl.outerType(); - // decRef the value that currently lives in the local if appropriate. - emitDecRef(i, getReg(localDl.location), type); -} - - -void -TranslatorX64::analyzeReqLit(Tracelet& t, NormalizedInstruction& i, - InclOpFlags flags) { - assert(i.inputs.size() == 1); - Eval::PhpFile* efile = g_vmContext->lookupIncludeRoot( - (StringData*)i.inputs[0]->rtt.valueString(), - flags, nullptr); - i.m_txFlags = supportedPlan(i.inputs[0]->isString() && - i.inputs[0]->rtt.valueString() != nullptr && - efile && - (RuntimeOption::RepoAuthoritative || - RuntimeOption::ServerStatCache)); - if (efile && efile->unit()->getMainReturn()->m_type != KindOfUninit) { - i.outStack->rtt = RuntimeType(efile->unit()->getMainReturn()->m_type); - } - - // We don't need the reference lookupIncludeRoot made for us. - if (efile) efile->decRef(); - i.manuallyAllocInputs = true; -} - -void -TranslatorX64::analyzeReqDoc(Tracelet& t, NormalizedInstruction& i) { - analyzeReqLit(t, i, InclOpDocRoot); -} - -void -TranslatorX64::translateReqLit(const Tracelet& t, - const NormalizedInstruction& i, - InclOpFlags flags) { - bool local = flags & InclOpLocal; - StringData *s = const_cast(i.inputs[0]->rtt.valueString()); - HPHP::Eval::PhpFile* efile = - g_vmContext->lookupIncludeRoot(s, flags, nullptr); - /* - * lookupIncludeRoot increments the refcount for us. This reference is - * going to be burned into the translation cache. We will remove it only - * when the file changes (via invalidateFile), and we're sure that no - * outstanding requests are using the old code (via the Treadmill - * module). - */ - TRACE(1, "lookupIncludeRoot: %s -> %p c %d\n", s->data(), efile, - efile->getRef()); - /* - * Remember that this tracelet (not just this instruction) now depends on the - * contents of the required file. - */ - m_srcDB.recordDependency(efile, t.m_sk); - Unit *unit = efile->unit(); - Func *func = unit->getMain(local ? nullptr : curClass()); - - const Offset after = nextSrcKey(t, i).offset(); - TRACE(1, "requireHelper: efile %p offset %d%s\n", efile, after, - i.skipSync ? " [skipsync]" : ""); - - if (i.skipSync) { - /* - * getting here means there was nothing to do between - * the previous req and this one. Any spill code we generate - * here would be broken (because the rbx is wrong), so - * verify that we don't generate anything... - */ - TCA s DEBUG_ONLY = a.code.frontier; - syncOutputs(0); - assert(s == a.code.frontier); - } else { - syncOutputs(i); - } - ReqLitStaticArgs* args = m_globalData.alloc(); - emitImmReg(a, (uint64_t)args, argNumToRegName[0]); - emitCall(a, (TCA)reqLitHelper, true); - - args->m_efile = efile; - args->m_pseudoMain = emitServiceReq(SRFlags::None, REQ_BIND_REQUIRE, 3, - uint64_t(args), - uint64_t(func), uint64_t(func->base())); - args->m_pcOff = after; - args->m_local = local; - - if (i.breaksTracelet) { - SrcKey fallThru(curFunc(), after); - emitBindJmp(fallThru); - } else { - /* - * When we get here, rVmSp points to the actual top of stack, - * but the rest of this tracelet assumes that rVmSp is set to - * the top of the stack at the beginning of the tracelet, so we - * have to fix it up here. - * - */ - if (!i.outStack) { - /* as a special case, if we're followed by a pop, and - we return a non-refcounted type, and then followed - by another require, we can avoid the add here and the sub - in the following require - */ - } else { - int delta = i.stackOff + getStackDelta(i); - if (delta != 0) { - // i.stackOff is in negative Cells, not bytes. - a. add_imm64_reg64(cellsToBytes(delta), rVmSp); - } - } - } -} - -void -TranslatorX64::translateReqDoc(const Tracelet& t, - const NormalizedInstruction& i) { - translateReqLit(t, i, InclOpDocRoot); -} - -TCA -TranslatorX64::emitNativeTrampoline(TCA helperAddr) { - auto& a = atrampolines; - - if (!a.code.canEmit(m_trampolineSize)) { - // not enough space to emit a trampoline, so just return the - // helper address and emitCall will the emit the right sequence - // to call it indirectly - TRACE(1, "Ran out of space to emit a trampoline for %p\n", helperAddr); - assert(false); - return helperAddr; - } - uint32_t index = m_numNativeTrampolines++; - TCA trampAddr = a.code.frontier; - if (Stats::enabled()) { - Stats::emitInc(a, &Stats::tl_helper_counters[0], index); - char* name = Util::getNativeFunctionName(helperAddr); - const size_t limit = 50; - if (strlen(name) > limit) { - name[limit] = '\0'; - } - Stats::helperNames[index] = name; - } - - /* - * For stubs that take arguments in rAsm, we need to make sure - * we're not damaging its contents here. (If !jmpDeltaFits, the jmp - * opcode will need to movabs the address into rAsm before - * jumping.) - */ - auto UNUSED stubUsingRScratch = [&](TCA tca) { - return tca == m_dtorGenericStubRegs; - }; - - assert(IMPLIES(stubUsingRScratch(helperAddr), a.jmpDeltaFits(helperAddr))); - a. jmp (helperAddr); - a. ud2 (); - - trampolineMap[helperAddr] = trampAddr; - if (m_trampolineSize == 0) { - m_trampolineSize = a.code.frontier - trampAddr; - assert(m_trampolineSize >= kMinPerTrampolineSize); - } - recordBCInstr(OpNativeTrampoline, a, trampAddr); - return trampAddr; -} - -TCA -TranslatorX64::getNativeTrampoline(TCA helperAddr) { - if (!RuntimeOption::EvalJitTrampolines && !Stats::enabled()) { - return helperAddr; - } - TCA trampAddr = (TCA)mapGet(trampolineMap, helperAddr); - if (trampAddr) { - return trampAddr; - } - return emitNativeTrampoline(helperAddr); -} - -void TranslatorX64::analyzeDefCls(Tracelet& t, - NormalizedInstruction& i) { - i.m_txFlags = Supported; -} - -static void defClsHelper(PreClass *preClass) { - assert(tl_regState == REGSTATE_DIRTY); - tl_regState = REGSTATE_CLEAN; - Unit::defClass(preClass); - - /* - * m_defClsHelper sync'd the registers for us already. This means - * if an exception propagates we want to leave things as - * REGSTATE_CLEAN, since we're still in sync. Only set it to dirty - * if we are actually returning to run in the TC again. - */ - tl_regState = REGSTATE_DIRTY; -} - -void TranslatorX64::translateDefCls(const Tracelet& t, - const NormalizedInstruction& i) { - int cid = i.imm[0].u_IVA; - const Opcode* after = curUnit()->at(i.source.offset()); - PreClass* c = curFunc()->unit()->lookupPreClassId(cid); - - assert(m_defClsHelper); - - /* - compute the corrected stack ptr as a pseudo-param to m_defClsHelper - which it will store in g_vmContext, in case of fatals, or __autoload - */ - m_regMap.cleanReg(rax); - m_regMap.smashReg(rax); - ScratchReg offset(m_regMap, rax); - emitLea(a, rVmSp, -cellsToBytes(i.stackOff), rax); - - EMIT_CALL(a, m_defClsHelper, IMM((uint64_t)c), IMM((uint64_t)after)); -} - -void TranslatorX64::analyzeDefFunc(Tracelet& t, - NormalizedInstruction& i) { - i.m_txFlags = Supported; -} - -void defFuncHelper(Func *f) { - f->setCached(); -} - -void TranslatorX64::translateDefFunc(const Tracelet& t, - const NormalizedInstruction& i) { - int fid = i.imm[0].u_IVA; - Func* f = curFunc()->unit()->lookupFuncId(fid); - - EMIT_CALL(a, defFuncHelper, IMM((uint64_t)f)); - recordReentrantCall(i); -} - -void -TranslatorX64::analyzeFPushFunc(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() >= 1); - // The input might be an object implementing __invoke() - i.m_txFlags = simplePlan(i.inputs[0]->isString()); -} - -void -TranslatorX64::translateFPushFunc(const Tracelet& t, - const NormalizedInstruction& i) { - using namespace TargetCache; - CacheHandle ch = FuncCache::alloc(); - assert(i.inputs.size() == 1); - Location& inLoc = i.inputs[0]->location; - - m_regMap.allocOutputRegs(i); - m_regMap.scrubStackRange(i.stackOff - 1, - i.stackOff - 1 + kNumActRecCells); - // Popped one cell, pushed an actrec - int startOfActRec = int(sizeof(Cell)) - int(sizeof(ActRec)); - size_t funcOff = AROFF(m_func) + startOfActRec; - size_t thisOff = AROFF(m_this) + startOfActRec; - if (false) { // typecheck - StackStringData sd("foo"); - const UNUSED Func* f = FuncCache::lookup(ch, &sd); - } - SKTRACE(1, i.source, "ch %d\n", ch); - EMIT_CALL(a, FuncCache::lookup, IMM(ch), V(inLoc)); - recordCall(i); - emitVStackStore(a, i, rax, funcOff, sz::qword); - emitVStackStoreImm(a, i, 0, thisOff, sz::qword, &m_regMap); - emitPushAR(i, nullptr, sizeof(Cell) /* bytesPopped */); -} - -void -TranslatorX64::analyzeFPushClsMethodD(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = supportedPlan(true); -} - -void -TranslatorX64::translateFPushClsMethodD(const Tracelet& t, - const NormalizedInstruction& i) { - using namespace TargetCache; - const StringData* meth = curUnit()->lookupLitstrId(i.imm[1].u_SA); - const NamedEntityPair& np = curUnit()->lookupNamedEntityPairId(i.imm[2].u_SA); - const StringData* cls = np.first; - assert(meth && meth->isStatic() && - cls && cls->isStatic()); - assert(i.inputs.size() == 0); - - const Class* baseClass = Unit::lookupUniqueClass(np.second); - bool magicCall = false; - const Func* func = lookupImmutableMethod(baseClass, meth, magicCall, - true /* staticLookup */); - - m_regMap.scrubStackRange(i.stackOff, - i.stackOff + kNumActRecCells); - - int startOfActRec = -int(sizeof(ActRec)); - SKTRACE(2, i.source, "FPushClsMethodD %s :: %s\n", - cls->data(), meth->data()); - - size_t clsOff = AROFF(m_cls) + startOfActRec; - if (func) { - emitKnownClassCheck(i, cls, reg::noreg); - Stats::emitInc(a, Stats::TgtCache_StaticMethodBypass); - emitPushAR(i, func, 0 /*bytesPopped*/, - false /* isCtor */, false /* clearThis */, - magicCall ? uintptr_t(meth) | ActRec::kInvNameBit : 0); - - setupActRecClsForStaticCall(i, func, baseClass, clsOff, false); - } else { - Stats::emitInc(a, Stats::TgtCache_StaticMethodHit); - CacheHandle ch = StaticMethodCache::alloc(cls, meth, getContextName()); - ScratchReg rFunc(m_regMap); - // Unconditionally set rCls; if we miss, the miss path will clean it up for - // us. The fill path has already |'ed in the necessary 1. - ScratchReg rCls(m_regMap); - a. load_reg64_disp_reg64(rVmTl, - ch + offsetof(StaticMethodCache, m_cls), - r(rCls)); - emitVStackStore(a, i, r(rCls), clsOff); - TCA stubsSkipRet; - a. load_reg64_disp_reg64(rVmTl, ch, r(rFunc)); - a. test_reg64_reg64(r(rFunc), r(rFunc)); - { - UnlikelyIfBlock miss(CC_Z, a, astubs); - if (false) { // typecheck - const UNUSED Func* f = StaticMethodCache::lookup(ch, np.second, - cls, meth); - } - EMIT_CALL(astubs, - StaticMethodCache::lookup, - IMM(ch), - IMM(int64_t(np.second)), - IMM(int64_t(cls)), - IMM(int64_t(meth))); - recordReentrantStubCall(i); - emitMovRegReg(astubs, rax, r(rFunc)); - // NULL return means our work is done; see also - // translateFPushClsMethodF. - miss.reconcileEarly(); - astubs.test_reg64_reg64(r(rFunc), r(rFunc)); - stubsSkipRet = astubs.code.frontier; - astubs.jz(a.code.frontier); // 1f to be patched later - } - - { - FreezeRegs ice(m_regMap); - emitPushAR(i, nullptr); - size_t funcOff = AROFF(m_func) + startOfActRec; - emitVStackStore(a, i, r(rFunc), funcOff, sz::qword); - } - // 1: - astubs.patchJcc(stubsSkipRet, a.code.frontier); - } -} - -void -TranslatorX64::analyzeFPushClsMethodF(Tracelet& t, - NormalizedInstruction& i) { - assert(i.inputs[0]->valueType() == KindOfClass); - i.m_txFlags = supportedPlan( - i.inputs[1]->isString() && - i.inputs[1]->rtt.valueString() != nullptr && // We know the method name - i.inputs[0]->valueType() == KindOfClass && - i.inputs[0]->rtt.valueClass() != nullptr // We know the class name - ); -} - -void -TranslatorX64::translateFPushClsMethodF(const Tracelet& t, - const NormalizedInstruction& i) { - using namespace TargetCache; - assert(!curFunc()->isPseudoMain()); - assert(curFunc()->cls() != nullptr); // self:: and parent:: should only - // appear in methods - DynLocation* clsLoc = i.inputs[0]; - DynLocation* nameLoc = i.inputs[1]; - const StringData* name = nameLoc->rtt.valueString(); - assert(name && name->isStatic()); - - // Even though we know the Class* at compile time, it's not - // guaranteed to be the same between requests. The name, however, is - // fixed, so we can use that. - const Class* cls = clsLoc->rtt.valueClass(); - assert(cls); - bool magicCall = false; - const Func* func = lookupImmutableMethod(cls, name, magicCall, - true /* staticLookup */); - - const int bytesPopped = 2 * sizeof(Cell); // [A C] popped - const int startOfActRec = -int(sizeof(ActRec)) + bytesPopped; - const Offset clsOff = startOfActRec + AROFF(m_cls); - - UNUSED ActRec* fp = curFrame(); - assert(!fp->hasThis() || fp->getThis()->instanceof(cls)); - if (func) { - Stats::emitInc(a, Stats::TgtCache_StaticMethodFBypass); - emitPushAR(i, func, bytesPopped, - false /* isCtor */, false /* clearThis */, - magicCall ? uintptr_t(name) | 1 : 0 /* varEnvInvName */); - - setupActRecClsForStaticCall(i, func, cls, clsOff, true); - m_regMap.scrubStackRange(i.stackOff - 2, - i.stackOff - 2 + kNumActRecCells); - } else { - const StringData* clsName = cls->name(); - CacheHandle ch = StaticMethodFCache::alloc(clsName, name, getContextName()); - - Stats::emitInc(a, Stats::TgtCache_StaticMethodFHit); - TCA stubsSkipRet; - ScratchReg rFunc(m_regMap); - a. load_reg64_disp_reg64(rVmTl, ch, r(rFunc)); - a. test_reg64_reg64(r(rFunc), r(rFunc)); - { - UnlikelyIfBlock miss(CC_Z, a, astubs); - if (false) { // typecheck - const UNUSED Func* f = StaticMethodFCache::lookup(ch, cls, name); - } - EMIT_CALL(astubs, - StaticMethodFCache::lookup, - IMM(ch), - V(clsLoc->location), - V(nameLoc->location)); - recordReentrantStubCall(i); - emitMovRegReg(astubs, rax, r(rFunc)); - // if rax == NULL, the helper interpreted the entire - // instruction for us. Skip over the rest of the emitted code in - // a, but we don't want to skip the branch spill/fill code. - miss.reconcileEarly(); - astubs.test_reg64_reg64(r(rFunc), r(rFunc)); - stubsSkipRet = astubs.code.frontier; - astubs.jz(a.code.frontier); // to be patched later - } - - const Offset funcOff = startOfActRec + AROFF(m_func); - m_regMap.scrubStackRange(i.stackOff - 2, - i.stackOff - 2 + kNumActRecCells); - { - FreezeRegs ice(m_regMap); - emitPushAR(i, nullptr, bytesPopped); - emitVStackStore(a, i, r(rFunc), funcOff); - - // We know we're in a method so we don't have to worry about - // rVmFp->m_cls being NULL. We just have to figure out if it's a - // Class* or $this, and whether or not we should pass along $this or - // its class. - PhysReg rCls = r(rFunc); // no need to allocate another scratch - a. load_reg64_disp_reg64(rVmFp, AROFF(m_cls), rCls); - a. testb(1, rbyte(rCls)); - { - JccBlock ifThis(a); - // rCls is holding $this. Should we pass it to the callee? - a. cmp_imm32_disp_reg32(1, ch + offsetof(StaticMethodFCache, m_static), - rVmTl); - { - IfElseBlock ifStatic(a); - // We're calling a static method. Load (this->m_cls | 0x1) into rCls. - a.load_reg64_disp_reg64(rCls, ObjectData::getVMClassOffset(), rCls); - a.or_imm32_reg64(1, rCls); - - ifStatic.Else(); - // We're calling an instance method. incRef $this. - emitIncRef(rCls, KindOfObject); - } - } - emitVStackStore(a, i, rCls, clsOff); - } - - astubs.patchJcc(stubsSkipRet, a.code.frontier); - // No need to decref our inputs: one was KindOfClass and the other's - // a static string. - } -} - -void -TranslatorX64::analyzeFPushObjMethodD(Tracelet& t, - NormalizedInstruction &i) { - DynLocation* objLoc = i.inputs[0]; - i.m_txFlags = supportedPlan(objLoc->valueType() == KindOfObject); -} - -void -TranslatorX64::translateFPushObjMethodD(const Tracelet &t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - Location& objLoc = i.inputs[0]->location; - assert(i.inputs[0]->valueType() == KindOfObject); - int id = i.imm[1].u_IVA; - const StringData* name = curUnit()->lookupLitstrId(id); - - const Class* baseClass = i.inputs[0]->rtt.valueClass(); - bool magicCall = false; - const Func* func = lookupImmutableMethod(baseClass, name, magicCall, - false /* staticLookup */); - m_regMap.scrubStackRange(i.stackOff - 1, - i.stackOff - 1 + kNumActRecCells); - // Popped one cell, pushed an actrec - int startOfActRec = int(sizeof(Cell)) - int(sizeof(ActRec)); - size_t thisOff = AROFF(m_this) + startOfActRec; - size_t funcOff = AROFF(m_func) + startOfActRec; - emitPushAR(i, func, sizeof(Cell) /*bytesPopped*/, - false /* isCtor */, false /* clearThis */, - func && magicCall ? uintptr_t(name) | 1 : 0 /* varEnvInvName */); - - if (!func) { - if (baseClass && !(baseClass->attrs() & AttrInterface)) { - MethodLookup::LookupResult res = - g_vmContext->lookupObjMethod(func, baseClass, name, false); - if ((res == MethodLookup::MethodFoundWithThis || - res == MethodLookup::MethodFoundNoThis) && - !func->isAbstract()) { - /* - * if we found the func in baseClass, then either: - * - its private, and this is always going to be the - * called function, or - * - any derived class must have a func that matches in - * staticness, and is at least as accessible (and in - * particular, you can't override a public/protected - * method with a private method) - */ - if (func->attrs() & AttrPrivate) { - emitVStackStoreImm(a, i, uintptr_t(func), funcOff, sz::qword); - } else { - Offset methodsOff = Class::getMethodsOffset(); - Offset vecOff = methodsOff + Class::MethodMap::vecOff(); - ScratchReg scratch(m_regMap); - // get the object's class into *scratch - a. load_reg64_disp_reg64(getReg(objLoc), - ObjectData::getVMClassOffset(), - r(scratch)); - if (res == MethodLookup::MethodFoundNoThis) { - emitDecRef(a, i, getReg(objLoc), KindOfObject); - a. lea_reg64_disp_reg64(r(scratch), 1, getReg(objLoc)); - } - emitVStackStore(a, i, getReg(objLoc), thisOff, sz::qword); - - // get the method vector into *scratch - a. load_reg64_disp_reg64(r(scratch), vecOff, r(scratch)); - // get the func - a. load_reg64_disp_reg64(r(scratch), - func->methodSlot() * sizeof(Func*), - r(scratch)); - emitVStackStore(a, i, r(scratch), funcOff, sz::qword); - Stats::emitInc(a, Stats::TgtCache_MethodFast); - return; - } - } else { - func = nullptr; - } - } - } - - if (func) { - if (func->attrs() & AttrStatic && !func->isClosureBody()) { - if (func->attrs() & AttrPrivate) { - emitVStackStoreImm(a, i, uintptr_t(curFunc()->cls()) | 1, - thisOff, sz::qword); - } else { - ScratchReg scratch(m_regMap); - a. load_reg64_disp_reg64(getReg(objLoc), - ObjectData::getVMClassOffset(), - r(scratch)); - a. or_imm32_reg64(1, r(scratch)); - emitVStackStore(a, i, r(scratch), thisOff, sz::qword); - } - emitDecRef(a, i, getReg(objLoc), KindOfObject); - } else { - emitVStackStore(a, i, getReg(objLoc), thisOff, sz::qword); - } - Stats::emitInc(a, Stats::TgtCache_MethodBypass); - } else { - emitVStackStore(a, i, getReg(objLoc), thisOff, sz::qword); - using namespace TargetCache; - CacheHandle ch = MethodCache::alloc(); - if (false) { // typecheck - ActRec* ar = nullptr; - MethodCache::lookup(ch, ar, name); - } - int arOff = vstackOffset(i, startOfActRec); - SKTRACE(1, i.source, "ch %d\n", ch); - EMIT_CALL(a, MethodCache::lookup, IMM(ch), - RPLUS(rVmSp, arOff), IMM(uint64_t(name))); - recordReentrantCall(i); - } -} - -static inline ALWAYS_INLINE Class* getKnownClass(Class** classCache, - const StringData* clsName) { - Class* cls = *classCache; - if (UNLIKELY(cls == nullptr)) { - // lookupKnownClass does its own VMRegAnchor'ing. - cls = TargetCache::lookupKnownClass(classCache, clsName, true); - assert(*classCache && *classCache == cls); - } - assert(cls); - return cls; -} - Instance* HOT_FUNC_VM newInstanceHelper(Class* cls, int numArgs, ActRec* ar, ActRec* prevAr) { @@ -8797,443 +3925,6 @@ newInstanceHelper(Class* cls, int numArgs, ActRec* ar, ActRec* prevAr) { return ret; } -void TranslatorX64::translateFPushCtor(const Tracelet& t, - const NormalizedInstruction& i) { - int numArgs = i.imm[0].u_IVA; - int arOff = vstackOffset(i, -int(sizeof(ActRec))); - m_regMap.scrubStackRange(i.stackOff, i.stackOff + kNumActRecCells); - EMIT_CALL(a, newInstanceHelper, - V(i.inputs[0]->location), - IMM(numArgs), - RPLUS(rVmSp, arOff), - R(rVmFp)); - recordReentrantCall(i); - - m_regMap.bind(rax, i.outStack->location, KindOfObject, RegInfo::DIRTY); -} - -static Instance* -HOT_FUNC_VM -newInstanceHelperNoCtor(Class* cls) { - Instance* ret = newInstance(cls); - ret->incRefCount(); - return ret; -} - -Instance* -HOT_FUNC_VM -newInstanceHelperNoCtorCached(Class** classCache, const StringData* clsName) { - Class* cls = getKnownClass(classCache, clsName); - return newInstanceHelperNoCtor(cls); -} - -Instance* -HOT_FUNC_VM -newInstanceHelperCached(Class** classCache, - const StringData* clsName, int numArgs, - ActRec* ar, ActRec* prevAr) { - Class* cls = getKnownClass(classCache, clsName); - return newInstanceHelper(cls, numArgs, ar, prevAr); -} - -void TranslatorX64::translateFPushCtorD(const Tracelet& t, - const NormalizedInstruction& i) { - using namespace TargetCache; - int numArgs = i.imm[0].u_IVA; - const StringData* clsName = curUnit()->lookupLitstrId(i.imm[1].u_SA); - Class* cls = Unit::lookupUniqueClass(clsName); - bool fastPath = !RuntimeOption::EnableObjDestructCall && - classIsPersistent(cls) && - !(cls->attrs() & (AttrAbstract | AttrInterface | AttrTrait)) && - (cls->getCtor()->attrs() & AttrPublic); - int arOff = -int(sizeof(ActRec)) - cellsToBytes(1); - m_regMap.scrubStackRange(i.stackOff, i.stackOff + kNumActRecCells + 1); - - LazyScratchReg clsCache(m_regMap); - if (fastPath) { - emitFPushCtorDFast(i, cls, arOff); - } else { - CacheHandle classCh = allocKnownClass(clsName); - clsCache.alloc(); - a. lea_reg64_disp_reg64(rVmTl, classCh, r(clsCache)); - - if (i.noCtor) { - Stats::emitInc(a, Stats::Tx64_NewInstanceNoCtor); - EMIT_RCALL(a, i, newInstanceHelperNoCtorCached, - R(clsCache), IMM(uintptr_t(clsName))); - } else { - arOff = vstackOffset(i, arOff); - Stats::emitInc(a, Stats::Tx64_NewInstanceGeneric); - EMIT_RCALL(a, i, newInstanceHelperCached, - R(clsCache), - IMM(uintptr_t(clsName)), - IMM(numArgs), - RPLUS(rVmSp, arOff), // ActRec - R(rVmFp)); // prevAR - } - } - m_regMap.bind(rax, i.outStack->location, KindOfObject, RegInfo::DIRTY); -} - -void -TranslatorX64::emitFPushCtorDFast(const NormalizedInstruction& i, - Class* cls, int arOff) { - size_t size = Instance::sizeForNProps(cls->numDeclProperties()); - int allocator = object_alloc_size_to_index(size); - if (i.noCtor) { - Stats::emitInc(a, Stats::Tx64_NewInstanceNoCtorFast); - } else { - Stats::emitInc(a, Stats::Tx64_NewInstanceFast); - } - - // First, make sure our property init vectors are all set up - bool props = cls->pinitVec().size() > 0; - bool sprops = cls->numStaticProperties() > 0; - assert((props || sprops) == cls->needInitialization()); - if (cls->needInitialization()) { - if (props) { - cls->initPropHandle(); - Stats::emitInc(a, Stats::Tx64_NewInstancePropCheck); - a.test_imm64_disp_reg64(-1, cls->propHandle(), rVmTl); - { - UnlikelyIfBlock ifZero(CC_Z, a, astubs); - Stats::emitInc(a, Stats::Tx64_NewInstancePropInit); - EMIT_RCALL(astubs, i, getMethodPtr(&Class::initProps), - IMM(int64_t(cls))); - } - } - if (sprops) { - cls->initSPropHandle(); - Stats::emitInc(a, Stats::Tx64_NewInstanceSPropCheck); - a.test_imm64_disp_reg64(-1, cls->sPropHandle(), rVmTl); - { - UnlikelyIfBlock ifZero(CC_Z, a, astubs); - Stats::emitInc(a, Stats::Tx64_NewInstanceSPropInit); - EMIT_RCALL(astubs, i, getMethodPtr(&Class::initSProps), - IMM(int64_t(cls))); - } - } - } - - // Next, allocate the object - if (cls->instanceCtor()) { - EMIT_RCALL(a, i, cls->instanceCtor(), IMM(int64_t(cls))); - } else { - assert(allocator != -1); - EMIT_RCALL(a, i, getMethodPtr(&Instance::newInstanceRaw), - IMM(int64_t(cls)), IMM(allocator)); - } - ScratchReg holdRax(m_regMap, rax); - - // Set the attributes, if any - int odAttrs = cls->getODAttrs(); - if (odAttrs) { - // o_attribute is 16 bits but the fact that we're or-ing a mask makes - // it ok - assert(!(odAttrs & 0xffff0000)); - a.or_imm32_disp_reg32(odAttrs, ObjectData::attributeOff(), rax); - } - - // Initialize the properties - size_t nProps = cls->numDeclProperties(); - if (nProps > 0) { - ScratchReg propVec(m_regMap); - a.lea_reg64_disp_reg64(rax, - sizeof(ObjectData) + cls->builtinPropSize(), - r(propVec)); - a.push(rax); - a.sub_imm32_reg64(8, rsp); // rsp alignment to keep memcpy happy - if (cls->pinitVec().size() == 0) { - // Fast case: copy from a known address in the Class - EMIT_CALL(a, memcpy, - R(propVec), - IMM(int64_t(&cls->declPropInit()[0])), - IMM(cellsToBytes(nProps))); - } else { - // Slower case: we have to load the src address from the targetcache - ScratchReg propData(m_regMap); - // Load the Class's propInitVec from the targetcache - a.load_reg64_disp_reg64(rVmTl, cls->propHandle(), r(propData)); - // propData holds the PropInitVec. We want &(*propData)[0] - a.load_reg64_disp_reg64(r(propData), Class::PropInitVec::dataOff(), - r(propData)); - if (!cls->hasDeepInitProps()) { - EMIT_CALL(a, memcpy, - R(propVec), - R(propData), - IMM(cellsToBytes(nProps))); - } else { - EMIT_CALL(a, deepInitHelper, - R(propVec), - R(propData), - IMM(nProps)); - } - } - a.add_imm32_reg64(8, rsp); - a.pop(rax); - } - if (cls->callsCustomInstanceInit()) { - // callCustomInstanceInit returns the instance in rax - if (false) { - UNUSED Instance* ret = ret->callCustomInstanceInit(); - } - EMIT_RCALL(a, i, - getMethodPtr(&Instance::callCustomInstanceInit), - R(rax)); - } - - // We're done with what Instance's constructor would've done. Set up the - // ActRec if needed. - if (i.noCtor) { - // If we're not running the constructor, just incref the object once and - // don't set up the ActRec. - a.incl(rax[FAST_REFCOUNT_OFFSET]); - return; - } else { - // Incref the object twice: once for the stack and once for $this in the - // ActRec. - a.add_imm32_disp_reg32(2, FAST_REFCOUNT_OFFSET, rax); - } - emitVStackStore(a, i, rVmFp, arOff + AROFF(m_savedRbp)); - emitVStackStoreImm(a, i, int64_t(cls->getCtor()), arOff + AROFF(m_func)); - emitVStackStoreImm(a, i, ActRec::encodeNumArgs(i.imm[0].u_IVA, true), - arOff + AROFF(m_numArgsAndCtorFlag), sz::dword); - emitVStackStoreImm(a, i, 0, arOff + AROFF(m_varEnv)); - emitVStackStore(a, i, rax, arOff + AROFF(m_this)); -} - -void -TranslatorX64::translateCreateCl(const Tracelet& t, - const NormalizedInstruction& i) { - int getArgs = i.imm[0].u_IVA; - const StringData* clsName = curUnit()->lookupLitstrId(i.imm[1].u_SA); - - LazyScratchReg clsCache(m_regMap); - - TargetCache::CacheHandle classCh = TargetCache::allocKnownClass(clsName); - clsCache.alloc(); - a. lea (rVmTl[classCh], r(clsCache)); - EMIT_RCALL(a, i, - newInstanceHelperNoCtorCached, - R(clsCache), - IMM(uintptr_t(clsName))); - - for (auto& input : i.inputs) { - m_regMap.cleanLoc(input->location); - } - - EMIT_RCALL(a, i, - getMethodPtr(&c_Closure::init), - R(rax), - IMM(getArgs), - R(rVmFp), - RPLUS(rVmSp, vstackOffset(i, 0))); - - m_regMap.bind(rax, i.outStack->location, KindOfObject, RegInfo::DIRTY); -} - - -static void fatalNullThis() { - raise_error(Strings::FATAL_NULL_THIS); -} - -void -TranslatorX64::emitThisCheck(const NormalizedInstruction& i, - PhysReg reg) { - if (curFunc()->cls() == nullptr) { // Non-class - a.test_reg64_reg64(reg, reg); - a.jz(astubs.code.frontier); // jz if_null - } - - a. testb(1, rbyte(reg)); - { - UnlikelyIfBlock ifThisNull(CC_NZ, a, astubs); - // if_null: - EMIT_CALL(astubs, fatalNullThis); - recordReentrantStubCall(i); - } -} - -void -TranslatorX64::translateThis(const Tracelet &t, - const NormalizedInstruction &i) { - if (!i.outStack) { - assert(i.next && i.next->grouped); - return; - } - - assert(!i.outLocal); - assert(curFunc()->isPseudoMain() || curFunc()->cls() || - curFunc()->isClosureBody()); - m_regMap.allocOutputRegs(i); - PhysReg out = getReg(i.outStack->location); - a. loadq(rVmFp[AROFF(m_this)], out); - - if (!i.guardedThis) { - emitThisCheck(i, out); - } - emitIncRef(out, KindOfObject); -} - -void -TranslatorX64::translateBareThis(const Tracelet &t, - const NormalizedInstruction &i) { - if (!i.outStack) { - assert(i.next && i.next->grouped); - return; - } - assert(!i.outLocal); - assert(curFunc()->cls() || curFunc()->isClosureBody()); - ScratchReg outScratch(m_regMap); - PhysReg out = r(outScratch); - PhysReg base; - int offset; - locToRegDisp(i.outStack->location, &base, &offset); - if (i.outStack->rtt.isVagueValue()) { - m_regMap.scrubLoc(i.outStack->location); - } - a. load_reg64_disp_reg64(rVmFp, AROFF(m_this), out); - a. testb(1, rbyte(out)); - DiamondReturn astubsRet; - { - UnlikelyIfBlock ifThisNull(CC_NZ, a, astubs, &astubsRet); - emitStoreTVType(astubs, KindOfNull, base[offset + TVOFF(m_type)]); - if (i.imm[0].u_OA) { - EMIT_CALL(astubs, warnNullThis); - recordReentrantStubCall(i); - } - if (i.next && !i.outStack->rtt.isVagueValue()) { - // To handle the case where we predict that - // the bare this will have type Object. - // Using the normal type prediction mechanism - // would require writing the object to the stack - // anyway. - // This is currently dead, however - I couldnt - // find a win. - emitSideExit(astubs, i, true); - astubsRet.kill(); - } - } - emitIncRef(out, KindOfObject); - if (i.outStack->rtt.isVagueValue()) { - emitStoreTVType(a, KindOfObject, base[offset + TVOFF(m_type)]); - a. storeq(out, base[TVOFF(m_data) + offset]); - } else { - assert(i.outStack->isObject()); - m_regMap.bindScratch(outScratch, i.outStack->location, KindOfObject, - RegInfo::DIRTY); - } -} - -void -TranslatorX64::translateCheckThis(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 1 && - i.inputs[0]->location == Location(Location::This)); - if (i.guardedThis) return; - emitThisCheck(i, getReg(i.inputs[0]->location)); -} - -void -TranslatorX64::translateInitThisLoc(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.outLocal && !i.outStack); - assert(curFunc()->isPseudoMain() || curFunc()->cls()); - - PhysReg base; - int offset; - locToRegDisp(i.outLocal->location, &base, &offset); - assert(base == rVmFp); - - ScratchReg thiz(m_regMap); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_this), r(thiz)); - if (curFunc()->cls() == nullptr) { - // If we're in a pseudomain, m_this could be NULL - a. testq (r(thiz), r(thiz)); - a. jz (astubs.code.frontier); // jz if_null - } - // Ok, it's not NULL but it might be a Class which should be treated - // equivalently - a. testb(1, rbyte(thiz)); - a. jnz(astubs.code.frontier); // jnz if_null - - // We have a valid $this! - emitStoreTVType(a, KindOfObject, base[offset + TVOFF(m_type)]); - a. storeq(r(thiz), base[offset + TVOFF(m_data)]); - emitIncRef(r(thiz), KindOfObject); - - // if_null: - emitStoreUninitNull(astubs, offset, base); - astubs.jmp(a.code.frontier); - - m_regMap.invalidate(i.outLocal->location); -} - -void -TranslatorX64::analyzeFPushFuncD(Tracelet& t, NormalizedInstruction& i) { - Id funcId = i.imm[1].u_SA; - const NamedEntity* ne = curUnit()->lookupNamedEntityId(funcId); - const Func* func = Unit::lookupFunc(ne); - i.m_txFlags = supportedPlan(func != nullptr); -} - -void -TranslatorX64::translateFPushFuncD(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outStack && !i.outLocal); - Id funcId = i.imm[1].u_SA; - const NamedEntityPair& nep = curUnit()->lookupNamedEntityPairId(funcId); - const StringData* name = nep.first; - const Func* func = Unit::lookupFunc(nep.second); - - // Translation is only supported if function lookup succeeds - func->validate(); - if (Trace::enabled && !func) { - TRACE(1, "Attempt to invoke undefined function %s\n", name->data()); - } - - // Inform the register allocator that we just annihilated a range of - // possibly-dirty stack entries. - m_regMap.scrubStackRange(i.stackOff, - i.stackOff + kNumActRecCells); - - size_t thisOff = AROFF(m_this) - sizeof(ActRec); - bool funcCanChange = !func->isNameBindingImmutable(curUnit()); - if (funcCanChange) { - // Look it up in a FuncCache. - using namespace TargetCache; - CacheHandle ch = allocFixedFunction(nep.second, false); - size_t funcOff = AROFF(m_func) - sizeof(ActRec); - size_t funcCacheOff = ch + offsetof(FixedFuncCache, m_func); - - SKTRACE(1, i.source, "ch %d\n", ch); - - Stats::emitInc(a, Stats::TgtCache_FuncDHit); - ScratchReg scratch(m_regMap); - a.load_reg64_disp_reg64(rVmTl, funcCacheOff, r(scratch)); - a.test_reg64_reg64(r(scratch), r(scratch)); - { - UnlikelyIfBlock ifNull(CC_Z, a, astubs); - - if (false) { // typecheck - StackStringData sd("foo"); - FixedFuncCache::lookupUnknownFunc(&sd); - } - - EMIT_CALL(astubs, TCA(FixedFuncCache::lookupUnknownFunc), - IMM(uintptr_t(name))); - recordReentrantStubCall(i); - emitMovRegReg(astubs, rax, r(scratch)); - } - emitVStackStore(a, i, r(scratch), funcOff, sz::qword); - } - // delay writing the ActRec until after calling lookupUnknownFunc - // since it can re-enter and overwrite anything we had written... - emitVStackStoreImm(a, i, 0, thisOff, sz::qword, &m_regMap); - emitPushAR(i, funcCanChange ? nullptr : func, 0, false, false); -} - const Func* TranslatorX64::findCuf(const NormalizedInstruction& ni, Class*& cls, StringData*& invName, bool& forward) { @@ -9308,1258 +3999,128 @@ TranslatorX64::findCuf(const NormalizedInstruction& ni, return f; } -void -TranslatorX64::analyzeFPushCufOp(Tracelet& t, - NormalizedInstruction& ni) { - Class* cls = nullptr; - StringData* invName = nullptr; - bool forward = false; - const Func* func = findCuf(ni, cls, invName, forward); - ni.m_txFlags = supportedPlan(func != nullptr); - ni.manuallyAllocInputs = true; -} +TCA +TranslatorX64::emitNativeTrampoline(TCA helperAddr) { + auto& a = atrampolines; -void -TranslatorX64::setupActRecClsForStaticCall(const NormalizedInstruction &i, - const Func* func, const Class* cls, - size_t clsOff, bool forward) { - if (forward) { - ScratchReg rClsScratch(m_regMap); - PhysReg rCls = r(rClsScratch); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_cls), rCls); - if (!(curFunc()->attrs() & AttrStatic)) { - assert(curFunc()->cls() && - curFunc()->cls()->classof(cls)); - /* the context is non-static, so we have to deal - with passing in $this or getClass($this) */ - a. testb(1, rbyte(rCls)); - { - JccBlock ifThis(a); - // rCls is holding a real $this. - if (func->attrs() & AttrStatic) { - // but we're a static method, so pass getClass($this)|1 - a.load_reg64_disp_reg64(rCls, ObjectData::getVMClassOffset(), rCls); - a.or_imm32_reg64(1, rCls); - } else { - // We should pass $this to the callee - emitIncRef(rCls, KindOfObject); - } - } - } - emitVStackStore(a, i, rCls, clsOff); - } else { - if (!(func->attrs() & AttrStatic) && - !(curFunc()->attrs() & AttrStatic) && - curFunc()->cls() && - curFunc()->cls()->classof(cls)) { - /* might be a non-static call */ - ScratchReg rClsScratch(m_regMap); - PhysReg rCls = r(rClsScratch); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_cls), rCls); - a. testb(1, rbyte(rCls)); - { - IfElseBlock ifThis(a); - // rCls is holding $this. We should pass it to the callee - emitIncRef(rCls, KindOfObject); - emitVStackStore(a, i, rCls, clsOff); - ifThis.Else(); - emitVStackStoreImm(a, i, uintptr_t(cls)|1, clsOff); - } - } else { - emitVStackStoreImm(a, i, uintptr_t(cls)|1, clsOff); - } - } -} - -int64_t checkClass(TargetCache::CacheHandle ch, StringData* clsName, - ActRec *ar) { - VMRegAnchor _; - AutoloadHandler::s_instance->invokeHandler(clsName->data()); - if (*(Class**)TargetCache::handleToPtr(ch)) return true; - ar->m_func = SystemLib::s_nullFunc; - if (ar->hasThis()) { - // cannot hit zero, we just inc'ed it - ar->getThis()->decRefCount(); - } - ar->setThis(0); - return false; -} - -static const Func* autoloadMissingFunc(const StringData* funcName, - TargetCache::CacheHandle ch, - bool safe) { - VMRegAnchor _; - AutoloadHandler::s_instance->autoloadFunc( - const_cast(funcName)); - Func* toCall = *(Func**)TargetCache::handleToPtr(ch); - /* toCall could be a different function due to renaming */ - if (toCall) { - return toCall; - } - if (!safe) { - throw_invalid_argument("function: method '%s' not found", - funcName->data()); - } - return SystemLib::s_nullFunc; -} - -void -TranslatorX64::translateFPushCufOp(const Tracelet& t, - const NormalizedInstruction& ni) { - Class* cls = nullptr; - StringData* invName = nullptr; - bool forward = false; - const Func* func = findCuf(ni, cls, invName, forward); - assert(func); - - int numPopped = ni.op() == OpFPushCufSafe ? 0 : 1; - m_regMap.scrubStackRange(ni.stackOff - numPopped, - ni.stackOff - numPopped + kNumActRecCells); - - int startOfActRec = int(numPopped * sizeof(Cell)) - int(sizeof(ActRec)); - - emitPushAR(ni, cls ? func : nullptr, numPopped * sizeof(Cell), - false /* isCtor */, false /* clearThis */, - invName ? uintptr_t(invName) | ActRec::kInvNameBit : 0); - - bool safe = (ni.op() == OpFPushCufSafe); - size_t clsOff = AROFF(m_cls) + startOfActRec; - size_t funcOff = AROFF(m_func) + startOfActRec; - LazyScratchReg flag(m_regMap); - if (safe) { - flag.alloc(); - emitImmReg(a, true, r(flag)); - } - if (cls) { - setupActRecClsForStaticCall(ni, func, cls, clsOff, forward); - TargetCache::CacheHandle ch = cls->m_cachedOffset; - if (!TargetCache::isPersistentHandle(ch)) { - a. cmp_imm32_disp_reg32(0, ch, rVmTl); - { - UnlikelyIfBlock ifNull(CC_Z, a, astubs); - if (false) { - checkClass(0, nullptr, nullptr); - } - EMIT_CALL(astubs, TCA(checkClass), - IMM(ch), IMM(uintptr_t(cls->name())), - RPLUS(rVmSp, vstackOffset(ni, startOfActRec))); - recordReentrantStubCall(ni, true); - if (safe) { - astubs. mov_reg64_reg64(rax, r(flag)); - } - } - } - } else { - TargetCache::CacheHandle ch = func->getCachedOffset(); - if (TargetCache::isPersistentHandle(ch)) { - emitVStackStoreImm(a, ni, uintptr_t(func), funcOff, sz::qword); - emitVStackStoreImm(a, ni, 0, clsOff, sz::qword, &m_regMap); - } else { - ScratchReg funcReg(m_regMap); - a. load_reg64_disp_reg64(rVmTl, ch, r(funcReg)); - emitVStackStore(a, ni, r(funcReg), funcOff); - emitVStackStoreImm(a, ni, 0, clsOff, sz::qword, &m_regMap); - a. test_reg64_reg64(r(funcReg), r(funcReg)); - { - UnlikelyIfBlock ifNull(CC_Z, a, astubs); - EMIT_CALL(astubs, TCA(autoloadMissingFunc), - IMM(uintptr_t(func->name())), - IMM(ch), - IMM(safe)); - recordReentrantStubCall(ni, true); - emitVStackStore(astubs, ni, rax, funcOff); - if (safe) { - astubs.xorq(r(flag), r(flag)); - astubs.cmpq(SystemLib::s_nullFunc, rax); - astubs.setne(rbyte(flag)); - } - } + if (!a.code.canEmit(m_trampolineSize)) { + // not enough space to emit a trampoline, so just return the + // helper address and emitCall will the emit the right sequence + // to call it indirectly + TRACE(1, "Ran out of space to emit a trampoline for %p\n", helperAddr); + assert(false); + return helperAddr; + } + uint32_t index = m_numNativeTrampolines++; + TCA trampAddr = a.code.frontier; + if (Stats::enabled()) { + Stats::emitInc(a, &Stats::tl_helper_counters[0], index); + char* name = Util::getNativeFunctionName(helperAddr); + const size_t limit = 50; + if (strlen(name) > limit) { + name[limit] = '\0'; } + Stats::helperNames[index] = name; } - if (safe) { - DynLocation* outFlag = ni.outStack2; - DynLocation* outDef = ni.outStack; - - DynLocation* inDef = ni.inputs[0]; - if (!m_regMap.hasReg(inDef->location)) { - m_regMap.scrubStackRange(ni.stackOff - 2, ni.stackOff - 2); - PhysReg base1, base2; - int disp1, disp2; - locToRegDisp(inDef->location, &base1, &disp1); - locToRegDisp(outDef->location, &base2, &disp2); - ScratchReg tmp(m_regMap); - a. load_reg64_disp_reg64(base1, TVOFF(m_data) + disp1, r(tmp)); - a. store_reg64_disp_reg64(r(tmp), TVOFF(m_data) + disp2, base2); - if (!inDef->rtt.isVagueValue()) { - emitStoreTVType(a, inDef->outerType(), base2[disp2 + TVOFF(m_type)]); - } else { - emitLoadTVType(a, base1[TVOFF(m_type) + disp1], r(tmp)); - emitStoreTVType(a, r(tmp), base2[disp2 + TVOFF(m_type)]); - } - } else { - PhysReg reg = m_regMap.getReg(inDef->location); - m_regMap.scrubStackRange(ni.stackOff - 1, ni.stackOff - 1); - m_regMap.bind(reg, outDef->location, inDef->rtt.outerType(), - RegInfo::DIRTY); - } - m_regMap.bindScratch(flag, outFlag->location, KindOfBoolean, - RegInfo::DIRTY); - } -} - -void -TranslatorX64::analyzeFPassCOp(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = nativePlan(!i.preppedByRef); -} - -void -TranslatorX64::translateFPassCOp(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outStack && !i.outLocal); - assert(!i.preppedByRef); -} - -void -TranslatorX64::translateFPassR(const Tracelet& t, - const NormalizedInstruction& i) { /* - * Like FPassC, FPassR is able to cheat on boxing if the current - * parameter is pass by reference but we have a cell: the box would refer - * to exactly one datum (the value currently on the stack). - * - * However, if the callee wants a cell and we have a variant we must - * unbox; otherwise we might accidentally make callee changes to its - * parameter globally visible. + * For stubs that take arguments in rAsm, we need to make sure + * we're not damaging its contents here. (If !jmpDeltaFits, the jmp + * opcode will need to movabs the address into rAsm before + * jumping.) */ - assert(!i.inputs[0]->rtt.isVagueValue()); + auto UNUSED stubUsingRScratch = [&](TCA tca) { + return tca == m_dtorGenericStubRegs; + }; - assert(i.inputs.size() == 1); - const RuntimeType& inRtt = i.inputs[0]->rtt; - if (inRtt.isRef() && !i.preppedByRef) { - emitUnboxTopOfStack(i); + assert(IMPLIES(stubUsingRScratch(helperAddr), a.jmpDeltaFits(helperAddr))); + a. jmp (helperAddr); + a. ud2 (); + + trampolineMap[helperAddr] = trampAddr; + if (m_trampolineSize == 0) { + m_trampolineSize = a.code.frontier - trampAddr; + assert(m_trampolineSize >= kMinPerTrampolineSize); } + recordBCInstr(OpNativeTrampoline, a, trampAddr); + return trampAddr; } -void -TranslatorX64::translateFCall(const Tracelet& t, - const NormalizedInstruction& i) { - int numArgs = i.imm[0].u_IVA; - const Opcode* after = curUnit()->at(nextSrcKey(t, i).offset()); - const Func* srcFunc = curFunc(); - - // Sync all dirty registers and adjust rVmSp to point to the - // top of stack at the beginning of the current instruction - syncOutputs(i); - - // We are "between" tracelets and don't use the register map - // anymore. (Note that the currently executing trace may actually - // continue past the FCall, but it will have to resume with a fresh - // register map.) - RegSet scratchRegs = kScratchCrossTraceRegs; - DumbScratchReg retIPReg(scratchRegs); - - // Caller-specific fields: return addresses and the frame pointer - // offset. - assert(sizeof(Cell) == 1 << 4); - - // The kooky offset here a) gets us to the current ActRec, - // and b) accesses m_soff. - int32_t callOffsetInUnit = srcFunc->unit()->offsetOf(after - srcFunc->base()); - a. storel (callOffsetInUnit, - rVmSp[cellsToBytes(numArgs) + AROFF(m_soff)]); - - int32_t adjust = emitBindCall(i.source, i.funcd, numArgs); - - if (i.breaksTracelet) { - if (adjust) { - a. addq (adjust, rVmSp); - } - SrcKey fallThru(curFunc(), after); - emitBindJmp(fallThru); - } else { - /* - * Before returning, the callee restored rVmSp to point to the - * current top of stack but the rest of this tracelet assumes that - * rVmSp is set to the top of the stack at the beginning of the - * tracelet, so we have to fix it up here. - * - * TODO: in the case of an inlined NativeImpl, we're essentially - * emitting two adds to rVmSp in a row, which we can combine ... - */ - int delta = cellsToBytes(i.stackOff + getStackDelta(i)) + adjust; - if (delta != 0) { - // i.stackOff is in negative Cells, not bytes. - a. addq (delta, rVmSp); - } +TCA +TranslatorX64::getNativeTrampoline(TCA helperAddr) { + if (!RuntimeOption::EvalJitTrampolines && !Stats::enabled()) { + return helperAddr; } + TCA trampAddr = (TCA)mapGet(trampolineMap, helperAddr); + if (trampAddr) { + return trampAddr; + } + return emitNativeTrampoline(helperAddr); } -void TranslatorX64::analyzeFCallArray(Tracelet& t, - NormalizedInstruction& i) { - i.m_txFlags = Supported; +static void defClsHelper(PreClass *preClass) { + assert(tl_regState == REGSTATE_DIRTY); + tl_regState = REGSTATE_CLEAN; + Unit::defClass(preClass); + + /* + * m_defClsHelper sync'd the registers for us already. This means + * if an exception propagates we want to leave things as + * REGSTATE_CLEAN, since we're still in sync. Only set it to dirty + * if we are actually returning to run in the TC again. + */ + tl_regState = REGSTATE_DIRTY; } -void TranslatorX64::translateFCallArray(const Tracelet& t, - const NormalizedInstruction& i) { - const Offset after = nextSrcKey(t, i).offset(); - - syncOutputs(i); - - emitImmReg(a, (uint64_t)i.offset(), argNumToRegName[0]); - emitImmReg(a, (uint64_t)after, argNumToRegName[1]); - emitCall(a, (TCA)fCallArrayHelper, true); - - if (i.breaksTracelet) { - SrcKey fallThru(curFunc(), after); - emitBindJmp(fallThru); - } else { - /* - * When we get here, rVmSp points to the actual top of stack, - * but the rest of this tracelet assumes that rVmSp is set to - * the top of the stack at the beginning of the tracelet, so we - * have to fix it up here. - * - */ - assert(i.outStack); - int delta = i.stackOff + getStackDelta(i); - if (delta != 0) { - // i.stackOff is in negative Cells, not bytes. - a. add_imm64_reg64(cellsToBytes(delta), rVmSp); +template +static int64_t switchBoundsCheck(T v, int64_t base, int64_t nTargets) { + // I'm relying on gcc to be smart enough to optimize away the next + // two lines when T is int64. + if (int64_t(v) == v) { + int64_t ival = v; + if (ival >= base && ival < (base + nTargets)) { + return ival - base; } } + return nTargets + 1; } -void TranslatorX64::analyzeFCallBuiltin(Tracelet& t, - NormalizedInstruction& i) { - Id funcId = i.imm[2].u_SA; - const NamedEntity* ne = curUnit()->lookupNamedEntityId(funcId); - const Func* func = Unit::lookupFunc(ne); - i.m_txFlags = supportedPlan(func != nullptr); +int64_t switchDoubleHelper(int64_t val, int64_t base, int64_t nTargets) { + union { + int64_t intbits; + double dblval; + } u; + u.intbits = val; + return switchBoundsCheck(u.dblval, base, nTargets); } -void TranslatorX64::translateFCallBuiltin(const Tracelet& t, - const NormalizedInstruction& ni) { - int numArgs = ni.imm[0].u_IVA; - int numNonDefault = ni.imm[1].u_IVA; - Id funcId = ni.imm[2].u_SA; - const NamedEntity* ne = curUnit()->lookupNamedEntityId(funcId); - const Func* func = Unit::lookupFunc(ne); - PhysReg base; - int disp; - assert(ni.outStack); - assert(numArgs == func->numParams()); - assert(numArgs <= kMaxBuiltinArgs); - - func->validate(); - - // Sync all dirty registers - m_regMap.scrubStackEntries(ni.stackOff); - m_regMap.cleanAll(); - - // Emit typecasts if needed - for (int i = 0; i < numNonDefault; i++) { - const Func::ParamInfo& pi = func->params()[i]; - const Location& in = ni.inputs[numArgs - i - 1]->location; - RuntimeType& rtt = ni.inputs[numArgs - i - 1]->rtt; - -#define CSE(type) case KindOf ## type : do { \ - if (!rtt.is ## type ()) { \ - EMIT_CALL(a, tvCastTo ## type ## InPlace, A(in)); \ - recordCall(ni); \ - } \ -} while(0); break; - - switch (pi.builtinType()) { - CSE(Boolean) - case KindOfInt64 : { - if (!rtt.isInt()) { - EMIT_CALL(a, tvCastToInt64InPlace, A(in), IMM(10)); - recordCall(ni); - } - } break; - CSE(Double) - CSE(Array) - CSE(Object) - case BitwiseKindOfString : { - if (!rtt.isString()) { - EMIT_CALL(a, tvCastToStringInPlace, A(in)); - recordCall(ni); - } - } break; - case KindOfUnknown: break; - default: not_reached(); - } - } -#undef CSE - - int refReturn = 0; - PhysReg returnBase = rsp; - int returnOffset = offsetof(MInstrState, tvBuiltinReturn); - - auto returnType = func->returnType(); - if (isCppByRef(returnType)) { - if (isSmartPtrRef(returnType)) returnOffset += TVOFF(m_data); - emitLea(a, returnBase, returnOffset, argNumToRegName[0]); - refReturn = 1; - } - - // Load args into registers - for (int i = 0; i < numArgs; i++) { - const Func::ParamInfo& pi = func->params()[i]; - locToRegDisp(ni.inputs[numArgs - i - 1]->location, &base, &disp); - auto argReg = argNumToRegName[i + refReturn]; - switch (pi.builtinType()) { - case KindOfDouble: - assert(false); - case KindOfBoolean: - case KindOfInt64: - // pass by value - a. loadq (base[disp + TVOFF(m_data.num)], argReg); - break; - STRINGCASE(): - case KindOfArray: - case KindOfObject: - // pass ptr to TV.m_data as String&, Array&, or Object& - emitLea(a, base, disp + TVOFF(m_data), argReg); - break; - default: - // pass ptr to TV as Variant& - emitLea(a, base, disp, argReg); - break; - } - } - // Call builtin - BuiltinFunction nativeFuncPtr = func->nativeFuncPtr(); - if (eagerRecord(func)) { - recordEagerCall(a, ni); - } - emitCall(a, (TCA)nativeFuncPtr, true); - recordReentrantCall(ni); - - // Bind return value to a scratch reg so that decref helpers - // don't throw it away - ScratchReg ret(m_regMap, rax); - - // Decref and free arguments - for (int i = 0; i < numNonDefault; i++) { - const Func::ParamInfo& pi = func->params()[i]; - locToRegDisp(ni.inputs[numArgs - i - 1]->location, &base, &disp); - if (pi.builtinType() == KindOfUnknown) { - emitDecRefGeneric(ni, base, disp); - } else if (IS_REFCOUNTED_TYPE(pi.builtinType())) { - a. loadq (base[disp + TVOFF(m_data)], rAsm); - emitDecRef(ni, rAsm, pi.builtinType()); - } - } - - // invalidate return value - m_regMap.invalidate(ni.outStack->location); - - // copy return value - locToRegDisp(ni.outStack->location, &base, &disp); - - switch (returnType) { - // For bool return value, get the %al byte - case KindOfBoolean: - a. movzbl (al, eax); // sign extend byte->qword - emitStoreTypedValue(a, returnType, rax, disp, base, true); +int64_t switchStringHelper(StringData* s, int64_t base, int64_t nTargets) { + int64_t ival; + double dval; + switch (s->isNumericWithVal(ival, dval, 1)) { + case KindOfNull: + ival = switchBoundsCheck(0, base, nTargets); break; - case KindOfNull: /* void return type */ + + case KindOfDouble: + ival = switchBoundsCheck(dval, base, nTargets); + break; + case KindOfInt64: - emitStoreTypedValue(a, returnType, rax, disp, base, true); + ival = switchBoundsCheck(ival, base, nTargets); break; - STRINGCASE(): - case KindOfArray: - case KindOfObject: - // returnOffset already has TVOFF(m_data) added if necessary. - a. loadq (returnBase[returnOffset], rax); - a. testq (rax, rax); - { - IfElseBlock ifNotZero(a); - emitStoreTypedValue(a, returnType, rax, disp, base, true); - ifNotZero.Else(); - emitStoreTVType(a, KindOfNull, base[disp + TVOFF(m_type)]); - } - break; - case KindOfUnknown: // return type was Variant - emitLea(a, returnBase, returnOffset, rax); - emitCmpTVType(a, KindOfUninit, rax[TVOFF(m_type)]); - { - IfElseBlock ifNotUninit(a); - // copy 16-byte TypedValue - emitCopyToAligned(a, rax, 0, base, disp); - - ifNotUninit.Else(); - // result was KindOfUninit; convert to KindOfNull - emitStoreTVType(a, KindOfNull, base[disp + TVOFF(m_type)]); - } - break; default: not_reached(); } + decRefStr(s); + return ival; } -template -static TypedValue* -staticLocHelper(StringData* name, ActRec* fp, TypedValue* sp, - TargetCache::CacheHandle ch) { - if (UseTC) { - Stats::inc(Stats::TgtCache_StaticMiss); - Stats::inc(Stats::TgtCache_StaticHit, -1); - } - HphpArray* map = get_static_locals(fp); - TypedValue* retval = map->nvGet(name); // Local to num - if (!retval) { - // Read the initial value off the stack. - TypedValue tv = *sp; - map->nvSet(name, &tv, false); - retval = map->nvGet(name); - } - assert(retval); - if (retval->m_type != KindOfRef) { - tvBox(retval); - } - assert(retval->m_type == KindOfRef); - if (UseTC) { - TypedValue** chTv = (TypedValue**)TargetCache::handleToPtr(ch); - assert(*chTv == nullptr); - return (*chTv = retval); - } else { - return retval; - } -} - -void -TranslatorX64::emitCallStaticLocHelper(X64Assembler& as, - const NormalizedInstruction& i, - ScratchReg& output, - TargetCache::CacheHandle ch) { - // The helper is going to read the value from memory, so record it. We - // could also pass type/value as parameters, but this is hopefully a - // rare path. - m_regMap.cleanLoc(i.inputs[0]->location); - if (false) { // typecheck - StringData* sd = nullptr; - ActRec* fp = nullptr; - TypedValue* sp = nullptr; - sp = staticLocHelper(sd, fp, sp, ch); - sp = staticLocHelper(sd, fp, sp, ch); - } - const StringData* name = curFunc()->unit()->lookupLitstrId(i.imm[1].u_SA); - assert(name->isStatic()); - if (ch) { - EMIT_CALL(as, (TCA)staticLocHelper, IMM(uintptr_t(name)), R(rVmFp), - RPLUS(rVmSp, -cellsToBytes(i.stackOff)), IMM(ch)); - } else { - EMIT_CALL(as, (TCA)staticLocHelper, IMM(uintptr_t(name)), R(rVmFp), - RPLUS(rVmSp, -cellsToBytes(i.stackOff))); - } - recordCall(as, i); - emitMovRegReg(as, rax, r(output)); -} - -void -TranslatorX64::translateStaticLocInit(const Tracelet& t, - const NormalizedInstruction& i) { - using namespace TargetCache; - ScratchReg output(m_regMap); - const Location& outLoc = i.outLocal->location; - - // Closures and generators from closures don't satisfy the "one - // static per source location" rule that the inline fastpath - // requires - if (!curFunc()->isClosureBody() && - !curFunc()->isGeneratorFromClosure()) { - // Miss path explicitly decrements. - Stats::emitInc(a, Stats::TgtCache_StaticHit); - Stats::emitInc(a, Stats::Tx64_StaticLocFast); - - CacheHandle ch = allocStatic(); - assert(ch); - a. load_reg64_disp_reg64(rVmTl, ch, r(output)); - a. test_reg64_reg64(r(output), r(output)); - { - UnlikelyIfBlock fooey(CC_Z, a, astubs); - emitCallStaticLocHelper(astubs, i, output, ch); - } - } else { - Stats::emitInc(a, Stats::Tx64_StaticLocSlow); - emitCallStaticLocHelper(a, i, output, 0); - } - // Now we've got the outer variant in *output. Get the address of the - // inner cell, since that's the enregistered representation of a variant. - emitDeref(a, r(output), r(output)); - emitIncRef(r(output), KindOfRef); - // Turn output into the local we just initialized. - m_regMap.bindScratch(output, outLoc, KindOfRef, RegInfo::DIRTY); -} - -void -TranslatorX64::analyzeVerifyParamType(Tracelet& t, NormalizedInstruction& i) { - int param = i.imm[0].u_IVA; - const TypeConstraint& tc = curFunc()->params()[param].typeConstraint(); - if (!tc.isObjectOrTypedef()) { - // We are actually using the translation-time value of this local as a - // prediction; if the param check failed at compile-time, we predict it - // will continue failing. - bool compileTimeCheck = tc.check(frame_local(curFrame(), param), curFunc()); - i.m_txFlags = nativePlan(compileTimeCheck); - i.manuallyAllocInputs = true; - } else { - bool trace = i.inputs[0]->isObject() || - (i.inputs[0]->isNull() && tc.nullable()); - i.m_txFlags = supportedPlan(trace); - } -} - -/* - * This function will happily give you a Class* to a Class that hasn't been - * defined in your request yet. Make sure code using it is tolerant of that. - */ -static void -emitClassToReg(X64Assembler& a, const StringData* name, PhysReg r) { - if (!name) { - emitImmReg(a, 0, r); - return; - } - - Class* cls = Unit::lookupClass(name); - if (classIsUniqueOrCtxParent(cls)) { - emitImmReg(a, int64_t(cls), r); - } else { - TargetCache::CacheHandle ch = TargetCache::allocKnownClass(name); - a. load_reg64_disp_reg64(rVmTl, ch, r); - } -} - -static void -VerifyParamCallable(ObjectData* obj, int param) { - TypedValue tv; - tvWriteObject(obj, &tv); - if (!UNLIKELY(f_is_callable(tvAsCVarRef(&tv)))) { - VerifyParamTypeFail(param); - } - tvDecRef(&tv); -} - -void -TranslatorX64::translateVerifyParamType(const Tracelet& t, - const NormalizedInstruction& i) { - int param = i.imm[0].u_IVA; - const TypeConstraint& tc = curFunc()->params()[param].typeConstraint(); - // not quite a nop. The guards should have verified that the m_type field - // is compatible, but for objects we need to go one step further and - // ensure that we're dealing with the right class. - // NULL inputs only get traced when constraint is nullable. - assert(i.inputs.size() == 1); - if (!i.inputs[0]->isObject()) return; // nop. - - // Get the input's class from ObjectData->m_cls - const Location& in = i.inputs[0]->location; - PhysReg src = getReg(in); - ScratchReg inCls(m_regMap); - if (i.inputs[0]->rtt.isRef()) { - emitDerefRef(a, src, r(inCls)); - src = r(inCls); - } - a. load_reg64_disp_reg64(src, ObjectData::getVMClassOffset(), r(inCls)); - - ScratchReg cls(m_regMap); - // Constraint may not be in the class-hierarchy of the method being traced, - // look up the class handle and emit code to put the Class* into a reg. - bool isSpecial = tc.isSelf() || tc.isParent() || tc.isCallable(); - const Class* constraint = nullptr; - const StringData* clsName; - if (!isSpecial) { - clsName = tc.typeName(); - constraint = Unit::lookupUniqueClass(clsName); - } else { - if (tc.isSelf()) { - tc.selfToClass(curFunc(), &constraint); - } else if (tc.isParent()) { - tc.parentToClass(curFunc(), &constraint); - } else { - assert(tc.isCallable()); - EMIT_RCALL(a, i, VerifyParamCallable, R(src), IMM(param)); - return; - } - clsName = constraint ? constraint->preClass()->name() : nullptr; - } - Class::initInstanceBits(); - bool haveBit = Class::haveInstanceBit(clsName); - // See the first big comment in emitInstanceCheck for the contract here - if (!haveBit || !classIsUniqueOrCtxParent(constraint)) { - emitClassToReg(a, clsName, r(cls)); - } - - if (haveBit || classIsUniqueNormalClass(constraint)) { - LazyScratchReg dummy(m_regMap); - Stats::emitInc(a, Stats::Tx64_VerifyParamTypeFast); - emitInstanceCheck(t, i, clsName, constraint, inCls, cls, dummy); - } else { - // Compare this class to the incoming object's class. If the - // typehint's class is not present, can not be an instance, unless - // this is a typedef. The slow path handles that case. - Stats::emitInc(a, Stats::Tx64_VerifyParamTypeSlowShortcut); - a. cmp_reg64_reg64(r(inCls), r(cls)); - { - JccBlock subclassCheck(a); - // Call helper since ObjectData::instanceof is a member function - if (false) { - VerifyParamTypeSlow(constraint, constraint, param, &tc); - } - EMIT_RCALL(a, i, VerifyParamTypeSlow, R(inCls), R(cls), - IMM(param), - IMM(uintptr_t(&tc))); - } - } -} - -void -TranslatorX64::analyzeInstanceOfD(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - assert(i.outStack && !i.outLocal); - i.m_txFlags = planHingesOnRefcounting(i.inputs[0]->outerType()); -} - -// Helpers for InstanceOfD. They return uint64_t so the translated -// code calling them doesn't have to zero-extend the lower byte. -static uint64_t -InstanceOfDSlow(const Class* cls, const Class* constraint) { - Stats::inc(Stats::Tx64_InstanceOfDSlow); - return constraint && cls->classof(constraint); -} - -static uint64_t -InstanceOfDSlowInterface(const Class* cls, const Class* parent) { - Stats::inc(Stats::Tx64_InstanceOfDInterface); - return parent && cls->classof(parent); -} - -void -TranslatorX64::translateInstanceOfD(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 1); - assert(i.outStack && !i.outLocal); - - DynLocation* input0 = i.inputs[0]; - bool input0IsLoc = input0->isLocal(); - DataType type = input0->valueType(); - PhysReg srcReg; - LazyScratchReg result(m_regMap); - LazyScratchReg srcScratch(m_regMap); - TCA patchAddr = nullptr; - boost::scoped_ptr retFromNullThis; - - if (!i.changesPC) { - result.alloc(); - } else { - Stats::emitInc(a, Stats::Tx64_InstanceOfDFused); - } - - if (i.wasGroupedWith(OpThis, OpBareThis)) { - assert(curFunc()->cls()); - srcScratch.alloc(); - srcReg = r(srcScratch); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_this), srcReg); - if (i.prev->op() == OpThis) { - assert(i.prev->guardedThis); - } else { - if (i.prev->imm[0].u_OA) { - // Warn on null $this - if (!i.changesPC) { - retFromNullThis.reset(new DiamondReturn); - } - a. testb(1, rbyte(srcReg)); - { - UnlikelyIfBlock ifNull(CC_NZ, a, astubs, retFromNullThis.get()); - EMIT_RCALL(astubs, i, warnNullThis); - if (i.changesPC) { - fuseBranchAfterStaticBool(astubs, t, i, false); - } else { - emitImmReg(astubs, false, r(result)); - } - } - } else { - if (!i.changesPC) { - emitImmReg(a, false, r(result)); - } - a. testb(1, rbyte(srcReg)); - if (i.changesPC) { - JccBlock ifNull(a); - fuseBranchAfterStaticBool(a, t, i, false); - } else { - patchAddr = a.code.frontier; - a. jcc(CC_NZ, patchAddr); - } - } - } - input0IsLoc = true; // we dont want a decRef - type = KindOfObject; - } else { - srcReg = getReg(input0->location); - } - - const StringData* clsName = curUnit()->lookupLitstrId(i.imm[0].u_SA); - - if (type != KindOfObject) { - // Handle cases where the left hand side is not an object. If the - // left hand side is an array and the right hand side is an interface - // that supports arrays, we return true, otherwise we return false. - bool res = (type == KindOfArray && interface_supports_array(clsName)); - Stats::emitInc(a, Stats::Tx64_InstanceOfDBypass); - if (!input0IsLoc) { - assert(!input0->isRef()); - emitDecRef(i, srcReg, type); - } - if (i.changesPC) { - fuseBranchAfterStaticBool(a, t, i, res); - assert(!patchAddr); - return; - } else { - emitImmReg(a, res, r(result)); - } - } else { - // Get the input's class from ObjectData->m_cls - ScratchReg inCls(m_regMap); - PhysReg baseReg = srcReg; - if (input0->rtt.isRef()) { - assert(input0IsLoc); - emitDerefRef(a, srcReg, r(inCls)); - baseReg = r(inCls); - } - a. load_reg64_disp_reg64(baseReg, ObjectData::getVMClassOffset(), - r(inCls)); - if (!input0IsLoc) { - emitDecRef(i, srcReg, type); - } - - Class* maybeCls = Unit::lookupUniqueClass(clsName); - - // maybeInterface is just used as a hint: If it's a trait/interface now but - // a class at runtime, InstanceOfDSlowInterface will still do the right - // thing but more slowly. fastPath is guaranteed to be correct. - Class::initInstanceBits(); - bool haveBit = Class::haveInstanceBit(clsName); - bool maybeInterface = maybeCls && !haveBit && - (maybeCls->attrs() & (AttrTrait | AttrInterface)); - bool fastPath = !maybeInterface && - (classIsUniqueNormalClass(maybeCls) || haveBit); - auto afterHelper = [&] { - if (i.changesPC) fuseBranchAfterHelper(t, i); - else emitMovRegReg(a, rax, r(result)); - }; - - ScratchReg cls(m_regMap); - // See the first big comment in emitInstanceCheck for the contract here - if (!haveBit || !classIsUniqueOrCtxParent(maybeCls)) { - emitClassToReg(a, clsName, r(cls)); - } - if (maybeInterface) { - EMIT_CALL(a, InstanceOfDSlowInterface, R(inCls), R(cls)); - afterHelper(); - } else if (fastPath) { - Stats::emitInc(a, Stats::Tx64_InstanceOfDFast); - emitInstanceCheck(t, i, clsName, maybeCls, inCls, cls, result); - } else { - EMIT_CALL(a, InstanceOfDSlow, R(inCls), R(cls)); - afterHelper(); - } - if (i.changesPC) { - assert(!patchAddr && !retFromNullThis); - return; - } - } - - assert(!patchAddr || !retFromNullThis); - assert(IMPLIES(retFromNullThis, !i.changesPC)); - if (patchAddr) { - a. patchJcc(patchAddr, a.code.frontier); - } else { - retFromNullThis.reset(); - } - - // Bind result and destination - assert(!i.changesPC); - m_regMap.bindScratch(result, i.outStack->location, i.outStack->outerType(), - RegInfo::DIRTY); -} - -void -TranslatorX64::emitInstanceCheck(const Tracelet& t, - const NormalizedInstruction& i, - const StringData* clsName, - const Class* klass, - const ScratchReg& inCls, - const ScratchReg& cls, - const LazyScratchReg& result) { - LazyScratchReg one(m_regMap); - bool verifying = i.op() == OpVerifyParamType; - bool haveBit = Class::haveInstanceBit(clsName); - assert(IMPLIES(verifying, !i.changesPC)); - - TCA equalJe = nullptr; - TCA parentJmp = nullptr; - TCA parentFailJe = nullptr; - - if (i.changesPC) { - fuseBranchSync(t, i); - } else if (!verifying) { - one.alloc(); - emitImmReg(a, 1, r(one)); - } - std::unique_ptr ice; - if (!verifying) ice.reset(new FreezeRegs(m_regMap)); - - if (haveBit) { - Stats::emitInc(a, verifying ? Stats::Tx64_VerifyParamTypeBit - : Stats::Tx64_InstanceOfDBit); - translatorAssert(a, CC_NZ, "Class instance bits must be initialized", [&]{ - a.testb(0x1, r(inCls)[Class::instanceBitsOff()]); - }); - } - - // Are the Class*s the exact same class? If we have a bit then this is the - // only part of the translation that needs a pointer to the class. If the - // class is also unique (or a parent class of the current context), we can - // burn its value into the translation, so it won't be in *cls and we use an - // immediate. - if (haveBit && classIsUniqueOrCtxParent(klass)) { - a. cmp_imm64_reg64(int64_t(klass), r(inCls)); - } else { - a. cmp_reg64_reg64(r(inCls), r(cls)); - } - { - std::unique_ptr> ifElse; - if (verifying) { - equalJe = a.code.frontier; - a. je8(equalJe); - } else { - Stats::emitInc(a, Stats::Tx64_InstanceOfDEqual, 1, CC_E); - if (i.changesPC) { - fuseHalfBranchAfterBool(t, i, CC_E, true); - } else { - ifElse.reset(new IfElseBlock(a)); - a. mov_reg64_reg64(r(one), r(result)); - ifElse->Else(); - } - } - - // Default to false and override if all the checks succeed - if (!i.changesPC && !verifying) { - emitImmReg(a, 0, r(result)); - } - - int offset; - uint8_t mask; - if (Class::getInstanceBitMask(clsName, offset, mask)) { - // We don't need to check that the parent class exists: if it doesn't - // exist then it's impossible for this object to be an instance of it, - // and the corresponding bit won't be set. - a. testb((int8_t)mask, r(inCls)[offset]); - if (verifying) { - { - UnlikelyIfBlock fail(CC_Z, a, astubs); - EMIT_RCALL(astubs, i, VerifyParamTypeFail, IMM(i.imm[0].u_IVA)); - } - a.patchJcc8(equalJe, a.code.frontier); - } else if (i.changesPC) { - fuseBranchAfterBool(t, i, CC_NZ); - } else { - a.cmov_reg64_reg64(CC_NZ, r(one), r(result)); - } - return; - } - - assert(klass); - // Is our inheritence hierarchy no shorter than the candidate? - unsigned parentVecLen = klass->classVecLen(); - a. cmp_imm32_disp_reg32(parentVecLen, Class::classVecLenOff(), - r(inCls)); - { - JccBlock veclen(a); - - // Is the spot in our inheritance hierarchy corresponding to the - // candidate equal to the candidate? *cls might still be NULL here - // (meaning the class isn't defined yet) but that's ok: if it is null - // the cmp will always fail. - int offset = Class::classVecOff() + sizeof(Class*) * (parentVecLen-1); - a.cmp_reg64_disp_reg64(r(cls), offset, r(inCls)); - if (verifying) { - parentFailJe = a.code.frontier; - a.jne8(parentFailJe); - parentJmp = a.code.frontier; - a.jmp8(parentJmp); - } else { - Stats::emitInc(a, Stats::Tx64_InstanceOfDFinalTrue, 1, CC_E); - Stats::emitInc(a, Stats::Tx64_InstanceOfDFinalFalse, 1, CC_NE); - if (i.changesPC) { - // The decision is done here but if we fallthrough it's to the - // failure case, so it's ok to only bind half the branch. - fuseHalfBranchAfterBool(t, i, CC_E, true); - } else { - a.cmov_reg64_reg64(CC_E, r(one), r(result)); - } - } - } - - // If execution makes it here the check has failed - if (i.changesPC) { - fuseBranchAfterStaticBool(a, t, i, false, false); - } else if (verifying) { - a.patchJcc8(parentFailJe, a.code.frontier); - stubBlock(a, astubs, [&]{ - EMIT_RCALL(astubs, i, VerifyParamTypeFail, IMM(i.imm[0].u_IVA)); - }); - } - } - - if (verifying) { - a.patchJcc8(equalJe, a.code.frontier); - Stats::emitInc(a, Stats::Tx64_VerifyParamTypeEqual); - a.patchJmp8(parentJmp, a.code.frontier); - Stats::emitInc(a, Stats::Tx64_VerifyParamTypePass); - } -} - -static void translatorAssertFail(const char* msg) { - VMExecutionContext::PrintTCCallerInfo(); - std::cerr << "Failed assertion in translated code: " << msg << std::endl; - not_reached(); -} - -template -void TranslatorX64::translatorAssert(X64Assembler& a, ConditionCode cc, - const char* msg, L setup) { - if (!debug) return; - setup(); - TCA jmp = a.code.frontier; - a.jcc8(cc, jmp); - emitImmReg(a, int64_t(msg), rdi); - a.call((TCA)translatorAssertFail); - recordCall(a, *m_curNI); - a.patchJcc8(jmp, a.code.frontier); -} - -// note: this is ok for all the iterkey/itervalue stuff too -void -TranslatorX64::analyzeIterInit(Tracelet& t, NormalizedInstruction& ni) { - DataType inType = ni.inputs[0]->valueType(); - ni.m_txFlags = supportedPlan(inType == KindOfArray || inType == KindOfObject); -} - -void -TranslatorX64::analyzeIterInitK(Tracelet& t, NormalizedInstruction& ni) { - DataType inType = ni.inputs[0]->valueType(); - ni.m_txFlags = supportedPlan(inType == KindOfArray || inType == KindOfObject); -} - -void -TranslatorX64::analyzeWIterInit(Tracelet& t, NormalizedInstruction& ni) { - DataType inType = ni.inputs[0]->valueType(); - ni.m_txFlags = supportedPlan(inType == KindOfArray || inType == KindOfObject); -} - -void -TranslatorX64::analyzeWIterInitK(Tracelet& t, NormalizedInstruction& ni) { - DataType inType = ni.inputs[0]->valueType(); - ni.m_txFlags = supportedPlan(inType == KindOfArray || inType == KindOfObject); -} - -void TranslatorX64::translateBasicIterInit(const Tracelet& t, - const NormalizedInstruction& ni) { - const int kValIdx = 0; - DynLocation* in = ni.inputs[kValIdx]; - assert(in->outerType() != KindOfRef); - SKTRACE(1, ni.source, "IterInit: committed to translation\n"); - PhysReg src = getReg(in->location); - SrcKey taken, notTaken; - branchDests(t, ni, &taken, ¬Taken, 1 /* immIdx */); - Location iterLoc(Location::Iter, ni.imm[0].u_IVA); - switch (in->valueType()) { - case KindOfArray: { - if (false) { // typecheck - Iter *dest = nullptr; - HphpArray *arr = nullptr; - TypedValue *val = nullptr; - TypedValue *key = nullptr; - new_iter_array(dest, arr, val); - new_iter_array_key(dest, arr, val, key); - new_iter_array_key(dest, arr, val, key); - } - if (ni.outLocal2 || ni.op() == OpWIterInit) { - EMIT_RCALL(a, ni, - ni.op() == OpIterInitK ? - (TCA)new_iter_array_key : (TCA)new_iter_array_key, - A(iterLoc), R(src), A(ni.outLocal->location), - ni.outLocal2 ? A(ni.outLocal2->location) : IMM(0)); - } else { - EMIT_RCALL(a, ni, new_iter_array, A(iterLoc), R(src), - A(ni.outLocal->location)); - } - break; - } - case KindOfObject: { - if (false) { // typecheck - Iter *dest = nullptr; - ObjectData *obj = nullptr; - Class *ctx = nullptr; - TypedValue *val = nullptr; - TypedValue *key = nullptr; - new_iter_object(dest, obj, ctx, val, key); - } - Class* ctx = arGetContextClass(curFrame()); - m_regMap.scrubLoc(in->location); - EMIT_CALL(a, new_iter_object, A(iterLoc), R(src), - IMM((uintptr_t)ctx), - A(ni.outLocal->location), - ni.outLocal2 ? A(ni.outLocal2->location) : IMM(0)); - recordReentrantCall(a, ni, false, -1); - break; - } - default: not_reached(); - } - syncOutputs(t); // Ends BB - // If a new iterator is created, new_iter_* will not adjust the refcount of - // the input. If a new iterator is not created, new_iter_* will decRef the - // input for us. new_iter_* returns 0 if an iterator was not created, - // otherwise it returns 1. - prepareForTestAndSmash(a, kTestRegRegLen, kAlignJccAndJmp); - a. test_reg64_reg64(rax, rax); - emitCondJmp(taken, notTaken, CC_Z); -} - -void TranslatorX64::translateIterInit(const Tracelet& t, - const NormalizedInstruction& ni) { - assert(ni.inputs.size() == 1); - assert(ni.outLocal); - assert(!ni.outStack && !ni.outLocal2); - translateBasicIterInit(t, ni); -} - -void TranslatorX64::translateIterInitK(const Tracelet& t, - const NormalizedInstruction& ni) { - assert(ni.inputs.size() == 1); - assert(ni.outLocal && ni.outLocal2); - assert(!ni.outStack); - translateBasicIterInit(t, ni); -} - -void TranslatorX64::translateWIterInit(const Tracelet& t, - const NormalizedInstruction& ni) { - assert(ni.inputs.size() == 1); - assert(ni.outLocal); - assert(!ni.outStack && !ni.outLocal2); - translateBasicIterInit(t, ni); -} - -void TranslatorX64::translateWIterInitK(const Tracelet& t, - const NormalizedInstruction& ni) { - assert(ni.inputs.size() == 1); - assert(ni.outLocal && ni.outLocal2); - assert(!ni.outStack); - translateBasicIterInit(t, ni); -} - -void -TranslatorX64::analyzeIterNext(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - i.m_txFlags = Supported; -} - -void -TranslatorX64::analyzeIterNextK(Tracelet& t, NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - i.m_txFlags = Supported; -} - -void -TranslatorX64::analyzeWIterNext(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = Supported; -} - -void -TranslatorX64::analyzeWIterNextK(Tracelet& t, NormalizedInstruction& i) { - i.m_txFlags = Supported; -} - - -void -TranslatorX64::translateBasicIterNext(const Tracelet& t, - const NormalizedInstruction& i) { - if (false) { // type check - Iter* it = nullptr; - TypedValue* val = nullptr; - TypedValue* key = nullptr; - int64_t ret = iter_next(it, val); - ret = iter_next_key(it, val, key); - ret = iter_next_key(it, val, key); - if (ret) printf("\n"); - } - m_regMap.cleanAll(); // input might be in-flight - // If the iterator reaches the end, iter_next will handle - // freeing the iterator and it will decRef the array - Location iterLoc(Location::Iter, i.imm[0].u_IVA); - if (i.outLocal2 || i.op() == OpWIterNext) { - EMIT_CALL(a, - i.op() == OpIterNextK ? - (TCA)iter_next_key : (TCA)iter_next_key, - A(iterLoc), - A(i.outLocal->location), - i.op() == OpWIterNext ? IMM(0) : A(i.outLocal2->location)); - } else { - EMIT_CALL(a, iter_next, A(iterLoc), A(i.outLocal->location)); - } - recordReentrantCall(a, i); - ScratchReg raxScratch(m_regMap, rax); - - // syncOutputs before we handle the branch. - syncOutputs(t); - SrcKey taken, notTaken; - branchDests(t, i, &taken, ¬Taken, 1 /* destImmIdx */); - - prepareForTestAndSmash(a, kTestRegRegLen, kAlignJccAndJmp); - a. test_reg64_reg64(rax, rax); - emitCondJmp(taken, notTaken, CC_NZ); -} - -void -TranslatorX64::translateIterNext(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outStack && !i.outLocal2); - assert(i.outLocal); - translateBasicIterNext(t, i); -} - -void -TranslatorX64::translateIterNextK(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outStack); - assert(i.outLocal && i.outLocal2); - translateBasicIterNext(t, i); -} - -void -TranslatorX64::translateWIterNext(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outStack && !i.outLocal2); - assert(i.outLocal); - translateBasicIterNext(t, i); -} - -void -TranslatorX64::translateWIterNextK(const Tracelet& t, - const NormalizedInstruction& i) { - assert(i.inputs.size() == 0); - assert(!i.outStack); - assert(i.outLocal && i.outLocal2); - translateBasicIterNext(t, i); +int64_t switchObjHelper(ObjectData* o, int64_t base, int64_t nTargets) { + int64_t ival = o->o_toInt64(); + decRefObj(o); + return switchBoundsCheck(ival, base, nTargets); } // PSEUDOINSTR_DISPATCH is a switch() fragment that routes opcodes to their @@ -10616,28 +4177,6 @@ TranslatorX64::translateWIterNextK(const Tracelet& t, case OpIsDoubleC: \ func(CheckTypeOp, t, i) -void -TranslatorX64::analyzeInstr(Tracelet& t, - NormalizedInstruction& i) { - const Opcode op = i.op(); - switch (op) { -#define CASE(iNm) \ - case Op ## iNm: { \ - analyze ## iNm(t, i); \ - } break; -#define ANALYZE(a, b, c) analyze ## a(b, c); break; - INSTRS - PSEUDOINSTR_DISPATCH(ANALYZE) - -#undef ANALYZE -#undef CASE - default: { - assert(i.m_txFlags == Interp); - } - } - SKTRACE(1, i.source, "translation plan: %x\n", i.m_txFlags); -} - bool TranslatorX64::dontGuardAnyInputs(Opcode op) { switch (op) { @@ -10652,48 +4191,6 @@ TranslatorX64::dontGuardAnyInputs(Opcode op) { #undef CASE } -void TranslatorX64::emitOneGuard(const Tracelet& t, - const NormalizedInstruction& i, - PhysReg reg, int disp, DataType type, - TCA &sideExit) { - bool isFirstInstr = (&i == t.m_instrStream.first); - bool regsClean = !m_regMap.hasDirtyRegs(i.stackOff); - ConditionCode cc = IS_STRING_TYPE(type) ? CC_Z : CC_NZ; - emitTypeCheck(a, type, reg, disp); - if (isFirstInstr) { - SrcRec& srcRec = *getSrcRec(t.m_sk); - // If it's the first instruction, we haven't made any forward - // progress yet, so this is really a tracelet-level guard rather - // than a side exit. If we tried to "side exit", we'd come right - // back to this check! - // - // We need to record this as a fallback branch. - emitFallbackJmp(srcRec, cc); - } else if (!sideExit || regsClean) { - if (regsClean) { - // If we have no dirty regs and no stack offset at our destination, we - // can do this with a single jnz. If the destination has a translation - // already we'd emit an unlikely backwards jne, so use semiLikelyIfBlock - // in that case. - if (i.stackOff == 0 && !lookupTranslation(i.source)) { - Stats::emitInc(a, Stats::Tx64_OneGuardShort); - emitBindJcc(a, cc, i.source, REQ_BIND_SIDE_EXIT); - } else { - Stats::emitInc(a, Stats::Tx64_OneGuardLong); - semiLikelyIfBlock(cc, a, [&]{ - emitSideExit(a, i, false /*next*/); - }); - } - } else { - UnlikelyIfBlock ifFail(cc, a, astubs); - sideExit = astubs.code.frontier; - emitSideExit(astubs, i, false /*next*/); - } - } else { - a.jcc(cc, sideExit); - } -} - // Emit necessary guards for variants and pseudo-main locals before instr i. // For HHIR, this only inserts guards for pseudo-main locals. Variants are // guarded in a different way. @@ -10702,11 +4199,6 @@ TranslatorX64::emitVariantGuards(const Tracelet& t, const NormalizedInstruction& i) { bool pseudoMain = Translator::liveFrameIsPseudoMain(); bool isFirstInstr = (&i == t.m_instrStream.first); - const NormalizedInstruction *base = &i; - while (base->grouped) { - base = base->prev; - assert(base); - } for (size_t in = 0; in < i.inputs.size(); ++in) { DynLocation* input = i.inputs[in]; if (!input->isValue()) continue; @@ -10738,62 +4230,6 @@ TranslatorX64::emitVariantGuards(const Tracelet& t, } } -void -TranslatorX64::emitPredictionGuards(const NormalizedInstruction& i) { - if (!i.outputPredicted || i.breaksTracelet) return; - NormalizedInstruction::OutputUse u = i.getOutputUsage(i.outStack); - - switch (u) { - case NormalizedInstruction::OutputUsed: - break; - case NormalizedInstruction::OutputUnused: - return; - case NormalizedInstruction::OutputInferred: - Stats::emitInc(a, Stats::TC_TypePredOverridden); - return; - case NormalizedInstruction::OutputDoesntCare: - Stats::emitInc(a, Stats::TC_TypePredUnneeded); - return; - } - - assert(i.outStack); - PhysReg base; - int disp; - locToRegDisp(i.outStack->location, &base, &disp); - assert(base == rVmSp); - TRACE(1, "PREDGUARD: %p dt %d offset %d voffset %" PRId64 "\n", - a.code.frontier, i.outStack->outerType(), disp, - i.outStack->location.offset); - DataType type = i.outStack->outerType(); - emitTypeCheck(a, type, rVmSp, disp); - ConditionCode cc = IS_STRING_TYPE(type) ? CC_Z : CC_NZ; - { - UnlikelyIfBlock branchToSideExit(cc, a, astubs); - Stats::emitInc(astubs, Stats::TC_TypePredMiss); - emitSideExit(astubs, i, true); - } - Stats::emitInc(a, Stats::TC_TypePredHit); -} - -void -TranslatorX64::translateInstrWork(const Tracelet& t, - const NormalizedInstruction& i) { - const Opcode op = i.op(); - switch (op) { -#define CASE(iNm) \ - case Op ## iNm: \ - translate ## iNm(t, i); \ - break; -#define TRANSLATE(a, b, c) translate ## a(b, c); break; - INSTRS - PSEUDOINSTR_DISPATCH(TRANSLATE) -#undef TRANSLATE -#undef CASE - default: - assert(false); - } -} - void TranslatorX64::translateInstr(const Tracelet& t, const NormalizedInstruction& i) { @@ -11834,103 +5270,6 @@ bool TranslatorX64::dumpTCData() { return true; } -#define NATIVE_OP(X) PLAN(X, Native) -#define SUPPORTED_OP(X) PLAN(X, Supported) -#define SIMPLE_OP(X) PLAN(X, Simple) -#define INTERP_OP(X) PLAN(X, Interp) \ - void TranslatorX64::translate##X(const Tracelet&, \ - const NormalizedInstruction&) { \ - not_reached(); \ - } - -#define SUPPORTED_OPS() \ - /* - * Translations with no callouts to C++ whatsoever. - */ \ - NATIVE_OP(Null) \ - NATIVE_OP(NullUninit) \ - NATIVE_OP(True) \ - NATIVE_OP(False) \ - NATIVE_OP(Int) \ - NATIVE_OP(Double) \ - NATIVE_OP(String) \ - NATIVE_OP(Array) \ - NATIVE_OP(NewArray) \ - NATIVE_OP(InitThisLoc) \ - NATIVE_OP(Dup) \ - NATIVE_OP(ContEnter) \ - NATIVE_OP(ContValid) \ - NATIVE_OP(ContStopped) \ - NATIVE_OP(IncStat) \ - /* - * Translations with non-reentrant helpers. - */ \ - SIMPLE_OP(Jmp) \ - SIMPLE_OP(UnpackCont) \ - SIMPLE_OP(CreateCont) \ - SIMPLE_OP(NewCol) \ - SIMPLE_OP(FCall) \ - /* - * Translations with a reentrant helper. - * - * TODO: neither UnboxR nor FPassR can actually call destructors. - */ \ - SUPPORTED_OP(ContExit) \ - SUPPORTED_OP(UnboxR) \ - SUPPORTED_OP(FPassR) \ - SUPPORTED_OP(NativeImpl) \ - SUPPORTED_OP(UnsetL) \ - SUPPORTED_OP(Cns) \ - SUPPORTED_OP(ClsCnsD) \ - SUPPORTED_OP(This) \ - SUPPORTED_OP(BareThis) \ - SUPPORTED_OP(CheckThis) \ - SUPPORTED_OP(PackCont) \ - SUPPORTED_OP(ContReceive) \ - SUPPORTED_OP(ContRetC) \ - SUPPORTED_OP(ContNext) \ - SUPPORTED_OP(ContSend) \ - SUPPORTED_OP(ContRaise) \ - SUPPORTED_OP(ContCurrent) \ - SUPPORTED_OP(FPushCtor) \ - SUPPORTED_OP(FPushCtorD) \ - SUPPORTED_OP(CreateCl) \ - SUPPORTED_OP(StaticLocInit) \ - /* - * Always-interp instructions, - */ \ - INTERP_OP(ContHandle) \ - INTERP_OP(SetWithRefLM) \ - INTERP_OP(SetWithRefRM) \ - INTERP_OP(ArrayIdx) \ - INTERP_OP(CGetM) \ - INTERP_OP(FPassM) \ - INTERP_OP(VGetM) \ - INTERP_OP(IssetM) \ - INTERP_OP(EmptyM) \ - INTERP_OP(SetM) \ - INTERP_OP(SetOpM) \ - INTERP_OP(IncDecM) \ - INTERP_OP(BindM) \ - INTERP_OP(UnsetM) \ - INTERP_OP(FPushCufIter) \ - INTERP_OP(CIterFree) - -// Define the trivial analyze methods -#define PLAN(Op, Spt) \ -void \ -TranslatorX64::analyze ## Op(Tracelet& t, NormalizedInstruction& i) { \ - i.m_txFlags = Spt; \ -} - -SUPPORTED_OPS() - -#undef NATIVE_OP -#undef SUPPORTED_OP -#undef SIMPLE_OP -#undef INTERP_OP -#undef SUPPORTED_OPS - void TranslatorX64::invalidateSrcKey(SrcKey sk) { assert(!RuntimeOption::RepoAuthoritative); assert(s_writeLease.amOwner()); diff --git a/hphp/runtime/vm/translator/translator-x64.h b/hphp/runtime/vm/translator/translator-x64.h index c0f8c25a2..4314ca9eb 100644 --- a/hphp/runtime/vm/translator/translator-x64.h +++ b/hphp/runtime/vm/translator/translator-x64.h @@ -298,11 +298,6 @@ private: PhysReg src, int off, PhysReg tmpReg); - void emitTvSetRegSafe(const NormalizedInstruction&, PhysReg from, - DataType fromType, PhysReg toPtr, int toOffset, PhysReg tmp1, PhysReg tmp2, - bool incRefFrom); - void emitTvSet(const NormalizedInstruction&, PhysReg from, - DataType fromType, PhysReg toPtr, int toOffset = 0, bool incRefFrom = true); void emitThisCheck(const NormalizedInstruction& i, PhysReg reg); void emitPushAR(const NormalizedInstruction& i, const Func* func, @@ -557,6 +552,21 @@ private: CASE(ArrayIdx) \ CASE(FPushCufIter) \ CASE(CIterFree) \ + CASE(LateBoundCls) \ + CASE(IssetS) \ + CASE(IssetG) \ + CASE(UnsetG) \ + CASE(EmptyS) \ + CASE(EmptyG) \ + CASE(VGetS) \ + CASE(BindS) \ + CASE(BindG) \ + CASE(IterFree) \ + CASE(FPassV) \ + CASE(UnsetN) \ + CASE(BPassC) \ + CASE(BPassV) \ + // These are instruction-like functions which cover more than one // opcode. #define PSEUDOINSTRS \ @@ -571,56 +581,6 @@ private: CASE(FPassCOp) \ CASE(CheckTypeOp) -#define PAIR(nm) \ - void analyze ## nm(Tracelet& t, NormalizedInstruction& i); \ - void translate ## nm(const Tracelet& t, const NormalizedInstruction& i); -#define CASE PAIR - -INSTRS -PSEUDOINSTRS - -#undef CASE -#undef PAIR - - - void branchWithFlagsSet(const Tracelet& t, const NormalizedInstruction& i, - ConditionCode cc); - void fuseBranchSync(const Tracelet& t, const NormalizedInstruction& i); - void fuseBranchAfterBool(const Tracelet& t, const NormalizedInstruction& i, - ConditionCode cc); - void fuseHalfBranchAfterBool(const Tracelet& t, - const NormalizedInstruction& i, - ConditionCode cc, bool taken); - void fuseBranchAfterStaticBool(Asm& a, const Tracelet& t, - const NormalizedInstruction& i, - bool resultIsTrue, bool doSync = true); - void emitReturnVal(Asm& a, const NormalizedInstruction& i, - PhysReg dstBase, int dstOffset, - PhysReg thisBase, int thisOffset, - PhysReg scratch); - void fuseBranchAfterHelper(const Tracelet& t, - const NormalizedInstruction& i); - void translateSetMArray(const Tracelet &t, const NormalizedInstruction& i); - void emitGetGlobal(const NormalizedInstruction& i, int nameIdx, - bool allowCreate); - void emitArrayElem(const NormalizedInstruction& i, - const DynLocation* baseInput, - PhysReg baseReg, - const DynLocation* keyIn, - const Location& outLoc); - void translateIssetMFast(const Tracelet& t, - const NormalizedInstruction& ni); - void setupActRecClsForStaticCall(const NormalizedInstruction& i, - const Func* func, const Class* cls, - size_t clsOff, bool forward); - void emitInstanceCheck(const Tracelet& t, const NormalizedInstruction& i, - const StringData* clsName, - const Class* maybeCls, - const ScratchReg& inCls, - const ScratchReg& cls, - const LazyScratchReg& result); - void emitFPushCtorDFast(const NormalizedInstruction& i, Class* cls, - int arOff); template void translatorAssert(X64Assembler& a, ConditionCode cc, const char* msg, L setup); @@ -691,9 +651,6 @@ PSEUDOINSTRS Asm& getAsm() { return a; } void emitChainTo(SrcKey dest, bool isCall = false); - void syncOutputs(const Tracelet& t); - void syncOutputs(const NormalizedInstruction& i); - void syncOutputs(int stackOff); static bool isPseudoEvent(const char* event); void getPerfCounters(Array& ret); @@ -713,7 +670,6 @@ private: void poison(PhysReg dest); public: - void analyzeInstr(Tracelet& t, NormalizedInstruction& i); bool acquireWriteLease(bool blocking) { return s_writeLease.acquire(blocking); } @@ -723,14 +679,9 @@ public: void emitGuardChecks(Asm& a, SrcKey, const ChangeMap&, const RefDeps&, SrcRec&); - void emitOneGuard(const Tracelet& t, - const NormalizedInstruction& i, - PhysReg reg, int disp, DataType type, - TCA &sideExit); void irEmitResolvedDeps(const ChangeMap& resolvedDeps); void emitVariantGuards(const Tracelet& t, const NormalizedInstruction& i); - void emitPredictionGuards(const NormalizedInstruction& i); Debug::DebugInfo* getDebugInfo() { return &m_debugInfo; } @@ -837,11 +788,8 @@ private: TCA emitRetFromInterpretedGeneratorFrame(); TCA emitGearTrigger(Asm& a, SrcKey sk, TransID transId); void emitPopRetIntoActRec(Asm& a); - void emitBox(DataType t, PhysReg rToBox); - void emitUnboxTopOfStack(const NormalizedInstruction& ni); int32_t emitBindCall(SrcKey srcKey, const Func* funcd, int numArgs); void emitCondJmp(SrcKey skTrue, SrcKey skFalse, ConditionCode cc); - void emitInterpOne(const Tracelet& t, const NormalizedInstruction& i); bool handleServiceRequest(TReqInfo&, TCA& start, SrcKey& sk); void recordGdbTranslation(SrcKey sk, const Func* f, @@ -878,10 +826,6 @@ private: int numArgs); void emitIncCounter(TCA start, int cntOfs); - void analyzeReqLit(Tracelet& t, NormalizedInstruction& i, - InclOpFlags flags); - void translateReqLit(const Tracelet& t, const NormalizedInstruction& i, - InclOpFlags flags); struct ReqLitStaticArgs { HPHP::Eval::PhpFile* m_efile; TCA m_pseudoMain; @@ -897,11 +841,6 @@ private: // Utility function shared with IR code static uint64_t packBitVec(const vector& bits, unsigned i); - void translateBasicIterInit(const Tracelet& t, - const NormalizedInstruction& ni); - void translateBasicIterNext(const Tracelet& t, - const NormalizedInstruction& ni); - public: /* * enterTC is the main entry point for the translator from the @@ -992,18 +931,6 @@ PSEUDOINSTRS #undef CASE #undef DECLARE_FUNC - // Helper functions not covered by macros above - void irTranslateIssetS(const Tracelet& t, const NormalizedInstruction& i); - void irTranslateIssetG(const Tracelet& t, const NormalizedInstruction& i); - void irTranslateUnsetG(const Tracelet& t, const NormalizedInstruction& i); - void irTranslateEmptyS(const Tracelet& t, const NormalizedInstruction& i); - void irTranslateEmptyG(const Tracelet& t, const NormalizedInstruction& i); - void irTranslateVGetS(const Tracelet& t, const NormalizedInstruction& i); - void irTranslateBindS(const Tracelet& t, const NormalizedInstruction& i); - void irTranslateBindG(const Tracelet& t, const NormalizedInstruction& i); - void irTranslateIterFree(const Tracelet& t, const NormalizedInstruction& i); - void irTranslateLateBoundCls(const Tracelet&, const NormalizedInstruction&i); - void irTranslateFPassV(const Tracelet& t, const NormalizedInstruction& i); void irTranslateReqLit(const Tracelet& t, const NormalizedInstruction& i, InclOpFlags flags); diff --git a/hphp/runtime/vm/translator/translator.cpp b/hphp/runtime/vm/translator/translator.cpp index 2c6fa483f..23cf9baf3 100644 --- a/hphp/runtime/vm/translator/translator.cpp +++ b/hphp/runtime/vm/translator/translator.cpp @@ -1456,265 +1456,6 @@ int getStackDelta(const NormalizedInstruction& ni) { return delta; } -/* - * analyzeSecondPass -- - * - * Whole-tracelet analysis pass, after we've set up the dataflow - * graph. Modifies the instruction stream, and further annotates - * individual instructions. - */ -void Translator::analyzeSecondPass(Tracelet& t) { - assert(t.m_instrStream.last); - NormalizedInstruction* next; - for (NormalizedInstruction* ni = t.m_instrStream.first; ni; ni = next) { - const Opcode op = ni->op(); - next = ni->next; - - if (op == OpNop) { - t.m_instrStream.remove(ni); - continue; - } - - if (op == OpCGetL) { - /* - * If the local isn't more broadly useful in this tracelet, don't bother - * allocating space for it. - */ - const Location& local = ni->inputs[0]->location; - - bool seen = false; - for (NormalizedInstruction* cur = ni->next; cur; cur = cur->next) { - if ((ni->m_txFlags & Native) != Native) break; - for (unsigned dli = 0; dli < cur->inputs.size(); ++dli) { - Location& loc = cur->inputs[dli]->location; - if (loc == local) { - SKTRACE(1, ni->source, "CGetL: loading input\n"); - seen = true; - break; - } - } - } - - ni->manuallyAllocInputs = !seen && !ni->inputs[0]->rtt.isUninit(); - SKTRACE(1, ni->source, "CGetL: manuallyAllocInputs: %d\n", - ni->manuallyAllocInputs); - continue; - } - - NormalizedInstruction* prev = ni->prev; - if (!prev || !(prev->m_txFlags & Supported)) continue; - const Opcode prevOp = prev->op(); - - if (!ni->next && - (op == OpJmpZ || op == OpJmpNZ)) { - if (prevOp == OpNot && (ni->m_txFlags & Supported)) { - ni->invertCond = !ni->invertCond; - ni->inputs[0] = prev->inputs[0]; - assert(!prev->deadLocs.size()); - t.m_instrStream.remove(prev); - next = ni; - continue; - } - if (prevOp == OpGt || prevOp == OpLt || - prevOp == OpGte || prevOp == OpLte || - prevOp == OpEq || prevOp == OpNeq || - prevOp == OpIssetL || prevOp == OpAKExists || - isTypePred(prevOp) || prevOp == OpInstanceOfD || - prev->fuseBranch) { - prev->breaksTracelet = true; - prev->changesPC = true; // Dont generate generic glue. - // Leave prev->next linked. The translator will end up needing it. The - // breaksTracelet annotation here will prevent us from really translating the - // Jmp*. - continue; - } - } - - if (op == OpFPushClsMethodF && ni->directCall && - prevOp == OpAGetC && - prev->prev && prev->prev->op() == OpString && - prev->prev->prev && prev->prev->prev->op() == OpString) { - /* - * We have a fully determined OpFPushClsMethodF. We dont - * need to put the class and method name strings, or the - * Class* into registers. - */ - prev->outStack = nullptr; - prev->prev->outStack = nullptr; - prev->prev->prev->outStack = nullptr; - } - - if (RuntimeOption::RepoAuthoritative && - prevOp == OpFPushCtorD && - !prev->noCtor && - prev->imm[0].u_IVA == 0 && - op == OpFCall && (ni->m_txFlags & Supported) && - ni->next && (ni->next->m_txFlags & Supported) && - ni->next->op() == OpPopR) { - /* new obj with a ctor that takes no args */ - const NamedEntityPair& np = - curUnit()->lookupNamedEntityPairId(prev->imm[1].u_SA); - const Class* cls = Unit::lookupUniqueClass(np.second); - if (cls && (cls->attrs() & AttrUnique) && - Func::isSpecial(cls->getCtor()->name())) { - /* its the generated 86ctor, so no need to call it */ - next = next->next; - t.m_instrStream.remove(ni->next); - t.m_instrStream.remove(ni); - prev->noCtor = 1; - SKTRACE(1, prev->source, "FPushCtorD: killing ctor for %s in %s\n", - np.first->data(), curFunc()->fullName()->data()); - continue; - } - } - - /* - * If this is a Pop instruction and the previous instruction pushed a - * single return value cell on the stack, we can roll the pop into the - * previous instruction. - * - * TODO: SetG/SetS? - */ - const bool isPop = op == OpPopC || op == OpPopV; - const bool isOptimizable = prevOp == OpSetL || - prevOp == OpBindL || - prevOp == OpIncDecL || - prevOp == OpPrint || - prevOp == OpSetM || - prevOp == OpSetOpM || - prevOp == OpIncDecM; - - if (isPop && isOptimizable) { - // If one of these instructions already has a null outStack, we - // already hoisted a pop into it. - const bool alreadyHoisted = !prev->outStack; - - if (!alreadyHoisted) { - prev->outStack = nullptr; - SKTRACE(3, ni->source, "hoisting Pop instruction in analysis\n"); - for (unsigned i = 0; i < ni->deadLocs.size(); ++i) { - prev->deadLocs.push_back(ni->deadLocs[i]); - } - t.m_instrStream.remove(ni); - if ((prevOp == OpSetM || prevOp == OpSetOpM || prevOp == OpIncDecM) && - prev->prev && prev->prev->op() == OpCGetL && - prev->prev->inputs[0]->outerType() != KindOfUninit) { - assert(prev->prev->outStack); - prev->prev->outStack = 0; - prev->prev->manuallyAllocInputs = true; - prev->prev->ignoreInnerType = true; - prev->inputs[0] = prev->prev->inputs[0]; - prev->grouped = true; - } - continue; - } - } - - /* - * A Not instruction following an Is* instruction can - * be folded. - */ - if (op == OpNot) { - switch (prevOp) { - case OpAKExists: - case OpIssetL: - case OpIsNullL: case OpIsNullC: - case OpIsBoolL: case OpIsBoolC: - case OpIsIntL: case OpIsIntC: - case OpIsDoubleL: case OpIsDoubleC: - case OpIsStringL: case OpIsStringC: - case OpIsArrayL: case OpIsArrayC: - case OpIsObjectL: case OpIsObjectC: - prev->invertCond = !prev->invertCond; - prev->outStack = ni->outStack; - SKTRACE(3, ni->source, "folding Not instruction in analysis\n"); - assert(!ni->deadLocs.size()); - t.m_instrStream.remove(ni); - continue; - } - } - - if (op == OpInstanceOfD && prevOp == OpCGetL && - (ni->m_txFlags & Supported)) { - assert(prev->outStack); - ni->inputs[0] = prev->inputs[0]; - /* - the CGetL becomes a no-op (other - than checking for UninitNull), but - we mark the InstanceOfD as grouped to - avoid breaking the tracelet between the - two. - */ - prev->ignoreInnerType = true; - prev->outStack = 0; - prev->manuallyAllocInputs = true; - ni->grouped = true; - } - - if ((op == OpInstanceOfD || op == OpIsNullC) && - (ni->m_txFlags & Supported) && - (prevOp == OpThis || prevOp == OpBareThis)) { - prev->outStack = 0; - ni->grouped = true; - ni->manuallyAllocInputs = true; - } - - /* - * TODO: #1181258 this should mostly be subsumed by the IR. - * Remove this once the IR is seen to be handling it. - */ - NormalizedInstruction* pp = nullptr; - if (prevOp == OpString && - (ni->m_txFlags & Supported)) { - switch (op) { - case OpReqDoc: - /* Dont waste a register on the string */ - prev->outStack = nullptr; - pp = prev->prev; - } - } - - if (op == OpRetC && (ni->m_txFlags & Supported) && - (prevOp == OpString || - prevOp == OpInt || - prevOp == OpNull || - prevOp == OpTrue || - prevOp == OpFalse || - prevOp == OpDouble || - prevOp == OpArray || - prevOp == OpThis || - prevOp == OpBareThis)) { - assert(!ni->outStack); - ni->grouped = true; - prev->outStack = nullptr; - pp = prev->prev; - } - - if (pp && pp->op() == OpPopC && - pp->m_txFlags == Native) { - NormalizedInstruction* ppp = prev->prev->prev; - if (ppp && (ppp->m_txFlags & Supported)) { - switch (ppp->op()) { - case OpReqDoc: - /* - We have a require+pop followed by a require or a scalar ret, - where the pop doesnt have to do any work (the pop is Native). - There is no need to inc/dec rbx between the two (since - there will be no code between them) - */ - ppp->outStack = nullptr; - ni->skipSync = true; - break; - - default: - // do nothing - break; - } - } - } - } -} - static NormalizedInstruction* findInputSrc(NormalizedInstruction* ni, DynLocation* dl) { while (ni != nullptr) { @@ -3065,17 +2806,6 @@ void Translator::constrainOperandType(GuardType& relxType, } } -void Translator::reanalizeConsumers(Tracelet& tclet, DynLocation* depDynLoc) { - for (auto& instr : tclet.m_instrs) { - for (size_t i = 0; i < instr.inputs.size(); i++) { - if (instr.inputs[i] == depDynLoc) { - analyzeInstr(tclet, instr); - } - } - } -} - - /* * This method looks at all the uses of the tracelet dependencies in the * instruction stream and tries to relax the type associated with each location. @@ -3125,7 +2855,6 @@ void Translator::relaxDeps(Tracelet& tclet, TraceletContext& tctxt) { assert(relxType.getOuterType() != KindOfInvalid); deps[loc->location]->rtt = RuntimeType(relxType.getOuterType(), relxType.getInnerType()); - reanalizeConsumers(tclet, loc); } } } @@ -3626,7 +3355,6 @@ std::unique_ptr Translator::analyze(SrcKey sk, annotate(ni); } - analyzeInstr(t, *ni); if (ni->op() == OpFCall) { analyzeCallee(tas, t, ni); } @@ -3635,52 +3363,6 @@ std::unique_ptr Translator::analyze(SrcKey sk, tas.recordDelete(l); } - if (debug) { - // The interpreter has lots of nice sanity assertions in debug mode - // that the translator doesn't exercise. As a cross-check on the - // translator's correctness, this is a debug-only facility for - // sending a random selection of instructions through the - // interpreter. - // - // Set stress_txInterpPct to a value between 1 and 100. If you want - // to reproduce a failing case, look for the seed in the log and set - // stress_txInterpSeed accordingly. - if (!dbgTranslateCoin) { - dbgTranslateCoin = new BiasedCoin(Trace::stress_txInterpPct, - Trace::stress_txInterpSeed); - TRACE(1, "BiasedCoin(stress_txInterpPct,stress_txInterpSeed): " - "pct %f, seed %d\n", - dbgTranslateCoin->getPercent(), dbgTranslateCoin->getSeed()); - } - assert(dbgTranslateCoin); - if (dbgTranslateCoin->flip()) { - SKTRACE(3, ni->source, "stress interp\n"); - ni->m_txFlags = Interp; - } - - if ((ni->op()) > Trace::moduleLevel(Trace::tmp0) && - (ni->op()) < Trace::moduleLevel(Trace::tmp1)) { - ni->m_txFlags = Interp; - } - } - - Op txOpBisectLowOp = (Op)moduleLevel(Trace::txOpBisectLow), - txOpBisectHighOp = (Op)moduleLevel(Trace::txOpBisectHigh); - if (txOpBisectLowOp > OpLowInvalid && - txOpBisectHighOp > OpLowInvalid && - txOpBisectHighOp < OpHighInvalid) { - // If the user specified an operation bisection interval [Low, High] - // that is strictly included in (OpLowInvalid, OpHighInvalid), then - // only support the operations in that interval. Since the default - // value of moduleLevel is 0 and OpLowInvalid is also 0, this ensures - // that bisection is disabled by default. - static_assert(OpLowInvalid >= 0, - "OpLowInvalid must be nonnegative"); - if (ni->op() < txOpBisectLowOp || - ni->op() > txOpBisectHighOp) - ni->m_txFlags = Interp; - } - // Check if we need to break the tracelet. // // If we've gotten this far, it mostly boils down to control-flow @@ -3696,7 +3378,7 @@ std::unique_ptr Translator::analyze(SrcKey sk, sk = SrcKey(curFunc(), sk.m_offset + ni->imm[0].u_IA); goto head; // don't advance sk } else if (opcodeBreaksBB(ni->op()) || - (ni->m_txFlags == Interp && opcodeChangesPC(ni->op()))) { + (dontGuardAnyInputs(ni->op()) && opcodeChangesPC(ni->op()))) { SKTRACE(1, sk, "BB broken\n"); sk.advance(unit); goto breakBB; diff --git a/hphp/runtime/vm/translator/translator.h b/hphp/runtime/vm/translator/translator.h index 97e96957b..a5329830b 100644 --- a/hphp/runtime/vm/translator/translator.h +++ b/hphp/runtime/vm/translator/translator.h @@ -317,27 +317,10 @@ class NormalizedInstruction { bool fuseBranch:1; bool preppedByRef:1; // For FPass*; indicates parameter reffiness bool manuallyAllocInputs:1; - bool invertCond:1; bool outputPredicted:1; bool outputPredictionStatic:1; bool ignoreInnerType:1; - /* - * skipSync indicates that a previous instruction that should have - * adjusted the stack (eg FCall, Req*) didnt, because it could see - * that the next one was going to immediately adjust it again - * (ie at this point, rVmSp holds the "correct" value, rather - * than the value it had at the beginning of the tracelet) - */ - bool skipSync:1; - - /* - * grouped indicates that the tracelet should not be broken - * (eg by a side exit) between the preceding instruction and - * this one - */ - bool grouped:1; - /* * guardedThis indicates that we know that ar->m_this is * a valid $this. eg: @@ -358,12 +341,6 @@ class NormalizedInstruction { */ bool noSurprise:1; - /* - noCtor is set on FPushCtorD to say that the ctor is - going to be skipped (so dont setup an actrec) - */ - bool noCtor:1; - /* * instruction is statically known to have no effect, e.g. unboxing a Cell */ @@ -375,11 +352,6 @@ class NormalizedInstruction { */ bool interp:1; - /* - * This is an FPush* that will be directly bound to a Func* - */ - bool directCall:1; - /* * Indicates that a RetC/RetV should generate inlined return code * rather than calling the shared stub. @@ -392,7 +364,6 @@ class NormalizedInstruction { boost::dynamic_bitset<> nonRefCountedLocals; ArgUnion constImm; - TXFlags m_txFlags; Op op() const; Op mInstrOp() const; @@ -410,40 +381,17 @@ class NormalizedInstruction { , outStack3(nullptr) , checkedInputs(0) , hasConstImm(false) - , invertCond(false) , ignoreInnerType(false) - , skipSync(false) - , grouped(false) , guardedThis(false) , guardedCls(false) , noSurprise(false) - , noCtor(false) , noOp(false) , interp(false) - , directCall(false) , inlineReturn(false) - , m_txFlags(Interp) { memset(imm, 0, sizeof(imm)); } - bool isJmpNZ() const { - assert(op() == OpJmpNZ || op() == OpJmpZ); - return (op() == OpJmpNZ) != invertCond; - } - - bool isSupported() const { - return (m_txFlags & Supported) == Supported; - } - - bool isSimple() const { - return (m_txFlags & Simple) == Simple; - } - - bool isNative() const { - return (m_txFlags & Native) == Native; - } - void markInputInferred(int i) { if (i < 32) checkedInputs |= 1u << i; } @@ -452,15 +400,6 @@ class NormalizedInstruction { return i < 32 && ((checkedInputs >> i) & 1); } - bool wasGroupedWith(Op op) const { - return grouped && prev->op() == op; - } - - template - bool wasGroupedWith(Op op, OpTypes... ops) const { - return wasGroupedWith(op) || wasGroupedWith(ops...); - } - enum OutputUse { OutputUsed, OutputUnused, @@ -845,7 +784,6 @@ private: int& currentStackOffset, bool& varEnvTaint); void relaxDeps(Tracelet& tclet, TraceletContext& tctxt); - void reanalizeConsumers(Tracelet& tclet, DynLocation* depDynLoc); DataTypeCategory getOperandConstraintCategory(NormalizedInstruction* instr, size_t opndIdx); GuardType getOperandConstraintType(NormalizedInstruction* instr, @@ -905,7 +843,6 @@ public: */ virtual void requestInit() = 0; virtual void requestExit() = 0; - virtual void analyzeInstr(Tracelet& t, NormalizedInstruction& i) = 0; virtual TCA funcPrologue(Func* f, int nArgs, ActRec* ar = nullptr) = 0; virtual TCA getCallToExit() = 0; virtual TCA getRetFromInterpretedFrame() = 0;