diff --git a/hphp/doc/ir.specification b/hphp/doc/ir.specification index 0791b946d..81fe7eed9 100644 --- a/hphp/doc/ir.specification +++ b/hphp/doc/ir.specification @@ -326,15 +326,21 @@ Instruction set 1. Checks and Asserts -D:T = GuardType S0:Gen -> L +D:T = CheckType S0:Gen -> L Check that the type of the src S0 is T, and if so copy it to D. If - S0 is not type T, branch to the exit trace label L. + S0 is not type T, branch to the label L. -GuardLoc S0:FramePtr -> L +GuardLoc S0:FramePtr + + Guard that type of the given localId on the frame S0 is a subtype of + the type T; if not, make a fallback jump. (A jump to a service + request that chains to a retranslation for this tracelet.) + +CheckLoc S0:FramePtr Check that type of the given localId on the frame S0 is T; if not, - branch to the exit trace label L. + branch to the label L. AssertLoc S0:FramePtr @@ -348,10 +354,20 @@ OverrideLoc S0:FramePtr Overrides tracked information about the type of a local in frame S0. This is used to fix tracked state after InterpOne instructions. -D:StkPtr = GuardStk S0:StkPtr -> L +D:StkPtr = GuardStk S0:StkPtr + + Guard that the type of the cell on the stack pointed to by S0 at + offset (in cells) is T. If not, make a fallback jump. (A jump to a + service request that chains to a retranslation for this tracelet.) + + Returns a new StkPtr that represents the same stack but with the + knowledge that the slot at the index S1 has type T. + +D:StkPtr = CheckStk S0:StkPtr -> L Check that the type of the cell on the stack pointed to by S0 at - offset (in cells) is T; if not, branch to the exit trace label L. + offset (in cells) is T; if not, branch to the label L. + Returns a new StkPtr that represents the same stack but with the knowledge that the slot at the index S1 has type T. @@ -377,6 +393,9 @@ CheckInitMem S0:PtrToGen S1:ConstInt -> L GuardRefs + Perform reffiness guard checks. If any fail, make a fallback jump. + (Jump to a service request that will chain to a retranslation of + this tracelet.) 2. Arithmetic @@ -1034,36 +1053,61 @@ D:StkPtr = SpillStack S0:StkP S1:ConstInt, S2... 11. Trace exits -ExitTrace S0:ConstFunc S1:ConstInt S2:StkPtr S3:StkPtr +SyncABIRegs S0:FramePtr S1:StkPtr - Unconditional exit to bytecode offset S1 in function S0. S2 and - S3 are the VM stack and frame pointers, respectively. If given, - toSmash points to a jump or branch instruction to patch later. + Ensures the cross-tracelet ABI registers are in a consistent state + in preparation for an instruction that may leave the trace. -ExitTraceCc S0:ConstFunc S1:ConstInt S2:StkPtr S3:StkPtr - S4:ConstInt + In practice this instruction currently doesn't do anything except + act as an essential instruction that prevents the current stack from + being dce'd. - Exit conditionally to bytecode offset S1 or S4 in function S0. - S4 is bytecode address to exit to if the condition is false. - S2 and S3 are the VM stack and frame pointer, respectively. +ReqBindJmp -ExitGuardFailure S0:ConstFunc S1:ConstInt S2:StkPtr S3:StkPtr + Emit a jump to a REQ_BIND_JMP service request to the target offset + bcOff. - A start-of-trace guard failed before the trace began executing. - Runtime may retranslate the bytecode using HHIR. S1 is the - bytecode offset, S2 and S3 are the VM stack and frame pointer. +ReqBindJmpNoIR -ExitSlow S0:ConstFunc S1:ConstInt S2:StkPtr S3:StkPtr + Emit a jump to a REQ_BIND_JMP_NO_IR service request to the target + offset bcOff. This is used for unsupported situations, where we + need to leave the trace and want the "retranslation" to occur + without using HHIR (i.e. interp). - Like ExitTrace, but with a runtime hint to not use HHIR at this func/pc - position in the future. Runtime may start a new trace using Tx64 or - interpreter. +ReqRetranslate -ExitSlowNoProgress S0:ConstFunc, S1:ConstInt, S2:StkPtr, S3:StkPtr + Emit a jump to a service request that will chain to a retranslation + of this tracelet. - Like ExitGuardFailure, with the "slow" hint from ExitSlow. - In this case we can retranslate or interprete the whole tracelet since - it didn't make any progress. + This instruction is used in exit traces for a type prediction that + occurs at the first bytecode offset of a tracelet. + +ReqBindJmpGt +ReqBindJmpGte +ReqBindJmpLt +ReqBindJmpLte +ReqBindJmpEq +ReqBindJmpNeq +ReqBindJmpSame +ReqBindJmpNSame +ReqBindJmpInstanceOfBitmask +ReqBindJmpNInstanceOfBitmask +ReqBindJmpZero +ReqBindJmpNZero + + Test the condition based on the Jmp* op of similar name, and then + emit a pair of smashable jumps to a REQ_BIND_JMPCC_FIRST service + request. + +SideExitGuardLoc S0:FramePtr + + Test the type of a local, and if it fails jump to a side exit + service request. + +D:StkPtr = SideExitGuardStk S0:StkPtr + + Test the type of a stack cell, and if it fails jump to a side exit + service request. 12. Refcounting and copies diff --git a/hphp/runtime/vm/translator/hopt/cfg.h b/hphp/runtime/vm/translator/hopt/cfg.h index 300d108a6..576bd2763 100644 --- a/hphp/runtime/vm/translator/hopt/cfg.h +++ b/hphp/runtime/vm/translator/hopt/cfg.h @@ -32,9 +32,9 @@ namespace HPHP { namespace JIT { */ template struct PostorderSort { - PostorderSort(Visitor &visitor, unsigned num_blocks) : - m_visited(num_blocks), m_visitor(visitor) { - } + PostorderSort(Visitor &visitor, unsigned num_blocks) + : m_visited(num_blocks), m_visitor(visitor) + {} void walk(Block* block) { assert(!block->empty()); @@ -49,6 +49,7 @@ struct PostorderSort { if (taken) walk(taken); m_visitor(block); } + private: boost::dynamic_bitset<> m_visited; Visitor &m_visitor; @@ -64,8 +65,36 @@ void postorderWalk(Visitor visitor, unsigned num_blocks, Block* head) { } /* - * Compute the postorder number of each immediate dominator of each block, - * using the postorder numbers assigned by sortCfg(). + * Compute a reverse postorder list of the basic blocks reachable from + * the first block in trace. + * + * Post: isRPOSorted(return value) + */ +BlockList sortCfg(Trace*, const IRFactory&); + +/* + * Returns: true if the supplied block list is sorted in reverse post + * order. + */ +bool isRPOSorted(const BlockList&); + +/* + * Compute predecessors for all blocks in a list, using a list + * produced by sortCfg(). + * + * TODO(#2272599): we want to track this on the fly instead of + * recomputing it. + * + * Pre: isRPOSorted(blocks) + */ +typedef smart::vector> PredVector; +PredVector computePredecessors(const BlockList& blocks); + +/* + * Compute the postorder number of each immediate dominator of each + * block, using a list produced by sortCfg(). + * + * Pre: isRPOSorted(blocks) */ typedef std::vector IdomVector; IdomVector findDominators(const BlockList& blocks); @@ -87,12 +116,6 @@ DomChildren findDomChildren(const BlockList& blocks); */ bool dominates(const Block* b1, const Block* b2, const IdomVector& idoms); -/* - * Compute a reverse postorder list of the basic blocks reachable from - * the first block in trace. - */ -BlockList sortCfg(Trace*, const IRFactory&); - /* * Return true if trace has internal control flow (IE it has a branch * to itself somewhere. diff --git a/hphp/runtime/vm/translator/hopt/codegen.cpp b/hphp/runtime/vm/translator/hopt/codegen.cpp index 36fc755b1..1f88a72f4 100644 --- a/hphp/runtime/vm/translator/hopt/codegen.cpp +++ b/hphp/runtime/vm/translator/hopt/codegen.cpp @@ -490,39 +490,8 @@ Address CodeGenerator::emitFwdJmp(Block* target) { return emitFwdJmp(m_as, target); } -// Patch with service request REQ_BIND_JMPCC_FIRST/SECOND -Address CodeGenerator::emitSmashableFwdJccAtEnd(ConditionCode cc, Block* target, - IRInstruction* toSmash) { - Address start = m_as.code.frontier; - if (toSmash) { - m_tx64->prepareForSmash(m_as, TranslatorX64::kJmpLen + - TranslatorX64::kJmpccLen); - Address tcaJcc = emitFwdJcc(cc, target); - emitFwdJmp(target); - toSmash->setTCA(tcaJcc); - } else { - emitFwdJcc(cc, target); - } - return start; -} - -void CodeGenerator::emitJccDirectExit(IRInstruction* inst, - ConditionCode cc) { - if (cc == CC_None) return; - auto* toSmash = inst->getTCA() == kIRDirectJccJmpActive ? inst : nullptr; - emitSmashableFwdJccAtEnd(cc, inst->getTaken(), toSmash); -} - -// Patch with service request REQ_BIND_JCC -Address CodeGenerator::emitSmashableFwdJcc(ConditionCode cc, Block* target, - IRInstruction* toSmash) { - Address start = m_as.code.frontier; - assert(toSmash); - - m_tx64->prepareForSmash(m_as, TranslatorX64::kJmpccLen); - Address tcaJcc = emitFwdJcc(cc, target); - toSmash->setTCA(tcaJcc); - return start; +void emitLoadImm(CodeGenerator::Asm& as, int64_t val, PhysReg dstReg) { + as.emitImmReg(val, dstReg); } static void @@ -697,13 +666,44 @@ void CodeGenerator::doubleCmp(X64Assembler& a, RegXMM xmmReg0, RegXMM xmmReg1) { asm_label(a, notPF); } -void CodeGenerator::cgJcc(IRInstruction* inst) { - SSATmp* src1 = inst->getSrc(0); - SSATmp* src2 = inst->getSrc(1); - Opcode opc = inst->op(); - ConditionCode cc = queryJmpToCC(opc); - Type src1Type = src1->type(); - Type src2Type = src2->type(); +static ConditionCode opToConditionCode(Opcode opc) { + using namespace HPHP::Transl; + + switch (opc) { + case JmpGt: return CC_G; + case JmpGte: return CC_GE; + case JmpLt: return CC_L; + case JmpLte: return CC_LE; + case JmpEq: return CC_E; + case JmpNeq: return CC_NE; + case JmpSame: return CC_E; + case JmpNSame: return CC_NE; + case JmpInstanceOfBitmask: return CC_NZ; + case JmpNInstanceOfBitmask: return CC_Z; + case JmpIsType: return CC_NZ; + case JmpIsNType: return CC_Z; + case JmpZero: return CC_Z; + case JmpNZero: return CC_NZ; + case ReqBindJmpGt: return CC_G; + case ReqBindJmpGte: return CC_GE; + case ReqBindJmpLt: return CC_L; + case ReqBindJmpLte: return CC_LE; + case ReqBindJmpEq: return CC_E; + case ReqBindJmpNeq: return CC_NE; + case ReqBindJmpSame: return CC_E; + case ReqBindJmpNSame: return CC_NE; + case ReqBindJmpInstanceOfBitmask: return CC_NZ; + case ReqBindJmpNInstanceOfBitmask: return CC_Z; + case ReqBindJmpZero: return CC_Z; + case ReqBindJmpNZero: return CC_NZ; + default: + always_assert(0); + } +} + +void CodeGenerator::emitCompare(SSATmp* src1, SSATmp* src2) { + auto const src1Type = src1->type(); + auto const src2Type = src2->type(); // can't generate CMP instructions correctly for anything that isn't // a bool or a numeric, and we can't mix bool/numerics because @@ -713,7 +713,7 @@ void CodeGenerator::cgJcc(IRInstruction* inst) { (src2Type == Type::Int || src2Type == Type::Dbl)) || (src1Type == Type::Bool && src2Type == Type::Bool) || (src1Type == Type::Cls && src2Type == Type::Cls))) { - CG_PUNT(cgJcc); + CG_PUNT(emitCompare); } if (src1Type == Type::Dbl || src2Type == Type::Dbl) { PhysReg srcReg1 = prepXMMReg(src1, m_as, m_regs, rXMMScratch0); @@ -721,9 +721,6 @@ void CodeGenerator::cgJcc(IRInstruction* inst) { assert(srcReg1 != rXMMScratch1 && srcReg2 != rXMMScratch0); doubleCmp(m_as, srcReg1, srcReg2); } else { - if (src1Type == Type::Cls && src2Type == Type::Cls) { - assert(opc == JmpSame || opc == JmpNSame); - } auto srcReg1 = m_regs[src1].getReg(); auto srcReg2 = m_regs[src2].getReg(); @@ -751,18 +748,65 @@ void CodeGenerator::cgJcc(IRInstruction* inst) { } } } - - emitJccDirectExit(inst, cc); } -void CodeGenerator::cgJmpGt (IRInstruction* inst) { cgJcc(inst); } -void CodeGenerator::cgJmpGte (IRInstruction* inst) { cgJcc(inst); } -void CodeGenerator::cgJmpLt (IRInstruction* inst) { cgJcc(inst); } -void CodeGenerator::cgJmpLte (IRInstruction* inst) { cgJcc(inst); } -void CodeGenerator::cgJmpEq (IRInstruction* inst) { cgJcc(inst); } -void CodeGenerator::cgJmpNeq (IRInstruction* inst) { cgJcc(inst); } -void CodeGenerator::cgJmpSame (IRInstruction* inst) { cgJcc(inst); } -void CodeGenerator::cgJmpNSame(IRInstruction* inst) { cgJcc(inst); } +void CodeGenerator::emitReqBindJcc(ConditionCode cc, + const ReqBindJccData* extra) { + auto& a = m_as; + assert(&m_as != &m_astubs && + "ReqBindJcc only makes sense outside of astubs"); + + prepareForTestAndSmash(a, 0, kAlignJccAndJmp); + auto const patchAddr = a.code.frontier; + + auto const jccStub = m_astubs.code.frontier; + { + auto& a = m_astubs; + // TODO(#2404398): move the setcc into the generic stub code so we + // don't need SRFlags::Persistent. + a. setcc (cc, rbyte(serviceReqArgRegs[4])); + m_tx64->emitServiceReq( + SRFlags::Persistent, + REQ_BIND_JMPCC_FIRST, + 4ull, + patchAddr, + uint64_t(extra->taken), + uint64_t(extra->notTaken), + uint64_t(cc) + ); + } + + a. jcc (cc, jccStub); + a. jmp (jccStub); +} + +void CodeGenerator::cgJcc(IRInstruction* inst) { + emitCompare(inst->getSrc(0), inst->getSrc(1)); + emitFwdJcc(opToConditionCode(inst->op()), inst->getTaken()); +} + +void CodeGenerator::cgReqBindJcc(IRInstruction* inst) { + // TODO(#2404427): prepareForTestAndSmash? + emitCompare(inst->getSrc(0), inst->getSrc(1)); + emitReqBindJcc(opToConditionCode(inst->op()), + inst->getExtra()); +} + +#define X(x) \ + void CodeGenerator::cgReqBind##x(IRInstruction* i) { cgReqBindJcc(i); } \ + void CodeGenerator::cg##x (IRInstruction* i) { cgJcc(i); } + +X(JmpGt); +X(JmpGte); +X(JmpLt); +X(JmpLte); +X(JmpEq); +X(JmpNeq); +X(JmpSame); +X(JmpNSame); + +#undef X + /** * Once the arg sources and dests are all assigned; emit moves and exchanges to @@ -1636,17 +1680,6 @@ ConditionCode CodeGenerator::emitTypeTest(Type type, OpndType src, return negate ? ccNegate(cc) : cc; } -template -ConditionCode CodeGenerator::emitTypeTest(IRInstruction* inst, OpndType src, - bool negate) { - return emitTypeTest(inst->getTypeParam(), src, negate); -} - -void CodeGenerator::emitSetCc(IRInstruction* inst, ConditionCode cc) { - if (cc == CC_None) return; - m_as.setcc(cc, rbyte(m_regs[inst->getDst()].getReg())); -} - ConditionCode CodeGenerator::emitIsTypeTest(IRInstruction* inst, bool negate) { auto const src = inst->getSrc(0); @@ -1677,7 +1710,7 @@ ConditionCode CodeGenerator::emitIsTypeTest(IRInstruction* inst, bool negate) { if (src->isA(Type::PtrToGen)) { PhysReg base = m_regs[src].getReg(); - return emitTypeTest(inst, base[TVOFF(m_type)], negate); + return emitTypeTest(inst->getTypeParam(), base[TVOFF(m_type)], negate); } assert(src->isA(Type::Gen)); assert(!src->isConst()); @@ -1686,18 +1719,31 @@ ConditionCode CodeGenerator::emitIsTypeTest(IRInstruction* inst, bool negate) { if (srcReg == InvalidReg) { CG_PUNT(IsType-KnownType); } - return emitTypeTest(inst, srcReg, negate); + return emitTypeTest(inst->getTypeParam(), srcReg, negate); } -template -void CodeGenerator::emitGuardType(OpndType src, IRInstruction* inst) { - emitGuardOrFwdJcc(inst, emitTypeTest(inst, src, true)); +template +void CodeGenerator::emitTypeCheck(Type type, MemLoc mem, Block* taken) { + auto const negate = true; + auto const cc = emitTypeTest(type, mem, negate); + if (cc == CC_None) return; + emitFwdJcc(cc, taken); } -void CodeGenerator::cgGuardTypeCell(PhysReg baseReg, - int64_t offset, - IRInstruction* inst) { - emitGuardType(baseReg[offset + TVOFF(m_type)], inst); +template +void CodeGenerator::emitTypeGuard(Type type, MemLoc mem) { + auto const negate = true; + auto const cc = emitTypeTest(type, mem, negate); + if (cc == CC_None) return; + + auto const destSK = SrcKey(getCurFunc(), m_curTrace->getBcOff()); + auto const destSR = m_tx64->getSrcRec(destSK); + m_tx64->emitFallbackCondJmp(m_as, *destSR, cc); +} + +void CodeGenerator::emitSetCc(IRInstruction* inst, ConditionCode cc) { + if (cc == CC_None) return; + m_as.setcc(cc, rbyte(m_regs[inst->getDst()].getReg())); } void CodeGenerator::cgIsTypeMemCommon(IRInstruction* inst, bool negate) { @@ -1709,7 +1755,7 @@ void CodeGenerator::cgIsTypeCommon(IRInstruction* inst, bool negate) { } void CodeGenerator::cgJmpIsTypeCommon(IRInstruction* inst, bool negate) { - emitJccDirectExit(inst, emitIsTypeTest(inst, negate)); + emitFwdJcc(emitIsTypeTest(inst, negate), inst->getTaken()); } void CodeGenerator::cgIsType(IRInstruction* inst) { @@ -1720,6 +1766,8 @@ void CodeGenerator::cgIsNType(IRInstruction* inst) { cgIsTypeCommon(inst, true); } +// TODO(#2404341): remove JmpIs{N,}Type + void CodeGenerator::cgJmpIsType(IRInstruction* inst) { cgJmpIsTypeCommon(inst, false); } @@ -1787,12 +1835,24 @@ void CodeGenerator::cgNInstanceOfBitmask(IRInstruction* inst) { void CodeGenerator::cgJmpInstanceOfBitmask(IRInstruction* inst) { emitInstanceBitmaskCheck(inst); - emitJccDirectExit(inst, CC_NZ); + emitFwdJcc(CC_NZ, inst->getTaken()); } void CodeGenerator::cgJmpNInstanceOfBitmask(IRInstruction* inst) { emitInstanceBitmaskCheck(inst); - emitJccDirectExit(inst, CC_Z); + emitFwdJcc(CC_Z, inst->getTaken()); +} + +void CodeGenerator::cgReqBindJmpInstanceOfBitmask(IRInstruction* inst) { + emitInstanceBitmaskCheck(inst); + emitReqBindJcc(opToConditionCode(inst->op()), + inst->getExtra()); +} + +void CodeGenerator::cgReqBindJmpNInstanceOfBitmask(IRInstruction* inst) { + emitInstanceBitmaskCheck(inst); + emitReqBindJcc(opToConditionCode(inst->op()), + inst->getExtra()); } /* @@ -2228,14 +2288,14 @@ void CodeGenerator::cgJmpSwitchDest(IRInstruction* inst) { m_as. subq(data->base, indexReg); } m_as. cmpq(data->cases - 2, indexReg); - m_tx64->prepareForSmash(m_as, TranslatorX64::kJmpccLen); + prepareForSmash(m_as, kJmpccLen); TCA def = m_tx64->emitServiceReq(REQ_BIND_JMPCC_SECOND, 3, m_as.code.frontier, data->defaultOff, CC_AE); m_as. jae(def); } TCA* table = m_tx64->m_globalData.alloc(sizeof(TCA), data->cases); - TCA afterLea = m_as.code.frontier + TranslatorX64::kLeaRipLen; + TCA afterLea = m_as.code.frontier + kLeaRipLen; ptrdiff_t diff = (TCA)table - afterLea; assert(deltaFits(diff, sz::dword)); m_as. lea(rip[diff], m_rScratch); @@ -2471,151 +2531,60 @@ void CodeGenerator::cgStLocNT(IRInstruction* inst) { false /* store type */); } -void CodeGenerator::cgSyncVMRegs(IRInstruction* inst) { +void CodeGenerator::cgSyncABIRegs(IRInstruction* inst) { emitMovRegReg(m_as, m_regs[inst->getSrc(0)].getReg(), rVmFp); emitMovRegReg(m_as, m_regs[inst->getSrc(1)].getReg(), rVmSp); } -void CodeGenerator::cgExitTrace(IRInstruction* inst) { - SSATmp* func = inst->getSrc(0); - SSATmp* pc = inst->getSrc(1); - SSATmp* sp = inst->getSrc(2); - SSATmp* fp = inst->getSrc(3); - SSATmp* notTakenPC = nullptr; - IRInstruction* toSmash = nullptr; - assert(pc->isConst() && inst->getNumSrcs() <= 6); +void CodeGenerator::cgReqBindJmp(IRInstruction* inst) { + m_tx64->emitBindJmp( + m_as, + SrcKey(getCurFunc(), inst->getExtra()->offset) + ); +} - TraceExitType::ExitType exitType = getExitType(inst->op()); - if (exitType == TraceExitType::Normal && inst->getExtra()) { - // Unconditional trace exit - toSmash = inst->getExtra()->toSmash; - assert(toSmash); - } else if (exitType == TraceExitType::NormalCc) { - // Exit at trace end which is the target of a conditional branch - notTakenPC = inst->getSrc(4); - assert(notTakenPC->isConst()); - if (inst->getExtra()) { - toSmash = inst->getExtra()->toSmash; - assert(toSmash); - } +static void emitExitNoIRStats(Asm& a, + TranslatorX64* tx64, + const Func* func, + SrcKey dest) { + if (RuntimeOption::EnableInstructionCounts || + HPHP::Trace::moduleEnabled(HPHP::Trace::stats, 3)) { + Stats::emitInc(a, + Stats::opcodeToIRPreStatCounter( + Op(*func->unit()->at(dest.m_offset))), + -1, + Transl::CC_None, + true); } - using namespace HPHP::Transl; - Asm& a = m_as; // Note: m_as is the same as m_atubs for Exit Traces, - // unless exit trace was moved to end of main trace - - emitMovRegReg(a, m_regs[sp].getReg(), rVmSp); - emitMovRegReg(a, m_regs[fp].getReg(), rVmFp); - - // Get the SrcKey for the dest - SrcKey destSK(func->getValFunc(), pc->getValInt()); - - switch (exitType) { - case TraceExitType::NormalCc: - if (toSmash) { - TCA smashAddr = toSmash->getTCA(); - if (smashAddr == kIRDirectJmpInactive) { - // The jump in the main trace has been optimized away - // this exit trace is no longer needed - break; - } - // Patch the original jcc;jmp, don't emit another - IRInstruction* jcc = toSmash; - Opcode opc = jcc->op(); - ConditionCode cc = queryJmpToCC(opc); - uint64_t taken = pc->getValInt(); - uint64_t notTaken = notTakenPC->getValInt(); - - m_astubs.setcc(cc, rbyte(serviceReqArgRegs[4])); - m_tx64->emitServiceReq(SRFlags::Persistent, - REQ_BIND_JMPCC_FIRST, - 4ull, - smashAddr, - taken, - notTaken, - uint64_t(cc)); - } else { - // NormalCc exit but not optimized to jcc directly to destination - m_tx64->emitBindJmp(a, destSK, REQ_BIND_JMP); - } - break; - case TraceExitType::Normal: - { - TCA smashAddr = toSmash ? toSmash->getTCA() : nullptr; - if (smashAddr) { - assert(smashAddr != kIRDirectJmpInactive); - if (smashAddr != kIRDirectJccJmpActive) { - // kIRDirectJccJmpActive only needs NormalCc exit in astubs - - m_tx64->emitServiceReq(SRFlags::Persistent, - REQ_BIND_JMP, 2, - smashAddr, - uint64_t(destSK.offset())); - - } - } else { - assert(smashAddr == kIRDirectJmpInactive); - m_tx64->emitBindJmp(a, destSK, REQ_BIND_JMP); - } - } - break; - case TraceExitType::Slow: - case TraceExitType::SlowNoProgress: - if (RuntimeOption::EnableInstructionCounts || - HPHP::Trace::moduleEnabled(HPHP::Trace::stats, 3)) { - Stats::emitInc(m_as, - Stats::opcodeToIRPostStatCounter( - Op(*getCurFunc()->unit()->at(destSK.m_offset))), - -1, - Transl::CC_None, - true); - } - - if (HPHP::Trace::moduleEnabled(HPHP::Trace::punt, 1)) { - Op op = (Op)*func->getValFunc()->unit()->at(destSK.m_offset); - std::string name = folly::format( - "exitSlow{}-{}", - exitType == TraceExitType::SlowNoProgress ? "-np" : "", - opcodeToName(op)).str(); - m_tx64->emitRecordPunt(a, name); - } - if (RuntimeOption::EvalHHIRDisableTx64) { - // Emit a service request to interpret a single instruction before - // creating a new translation - m_tx64->emitServiceReq(SRFlags::Persistent, - REQ_INTERPRET, - 2ull, uint64_t(destSK.offset()), 1); - } else { - if (exitType == TraceExitType::Slow) { - m_tx64->emitBindJmp(a, destSK, REQ_BIND_JMP_NO_IR); - } else { // SlowNoProgress - m_tx64->emitReqRetransNoIR(a, destSK); - } - } - break; - - case TraceExitType::GuardFailure: { - SrcRec* destSR = m_tx64->getSrcRec(destSK); - m_tx64->emitFallbackUncondJmp(a, *destSR); - break; - } + if (HPHP::Trace::moduleEnabled(HPHP::Trace::punt, 1)) { + auto const op = Op(*func->unit()->at(dest.m_offset)); + auto const name = folly::format( + "exitSlow-{}", + opcodeToName(op) + ).str(); + tx64->emitRecordPunt(a, name); } } -void CodeGenerator::cgExitTraceCc(IRInstruction* inst) { - cgExitTrace(inst); +void CodeGenerator::cgReqBindJmpNoIR(IRInstruction* inst) { + auto const dest = SrcKey(getCurFunc(), + inst->getExtra()->offset); + emitExitNoIRStats(m_as, m_tx64, getCurFunc(), dest); + m_tx64->emitBindJmp(m_as, dest, REQ_BIND_JMP_NO_IR); } -void CodeGenerator::cgExitSlow(IRInstruction* inst) { - cgExitTrace(inst); +void CodeGenerator::cgReqRetranslateNoIR(IRInstruction* inst) { + auto const dest = SrcKey(getCurFunc(), + inst->getExtra()->offset); + emitExitNoIRStats(m_as, m_tx64, getCurFunc(), dest); + m_tx64->emitReqRetransNoIR(m_as, dest); } -void CodeGenerator::cgExitSlowNoProgress(IRInstruction* inst) { - cgExitTrace(inst); -} - -void CodeGenerator::cgExitGuardFailure(IRInstruction* inst) { - cgExitTrace(inst); +void CodeGenerator::cgReqRetranslate(IRInstruction* inst) { + auto const destSK = SrcKey(getCurFunc(), m_curTrace->getBcOff()); + auto const destSR = m_tx64->getSrcRec(destSK); + m_tx64->emitFallbackUncondJmp(m_as, *destSR); } static void emitAssertFlagsNonNegative(CodeGenerator::Asm& as) { @@ -3834,12 +3803,12 @@ void CodeGenerator::cgLoadTypedValue(PhysReg base, if (typeDstReg != InvalidReg) { emitLoadTVType(m_as, base[off + TVOFF(m_type)], typeDstReg); if (label) { - // Check type needed - emitGuardType(typeDstReg, inst); + emitTypeCheck(inst->getTypeParam(), typeDstReg, inst->getTaken()); } } else if (label) { - // Check type needed - cgGuardTypeCell(base, off, inst); + emitTypeCheck(inst->getTypeParam(), + base[off + TVOFF(m_type)], + inst->getTaken()); } // Load value if it's not dead @@ -3892,23 +3861,6 @@ void CodeGenerator::cgStore(PhysReg base, } } -void CodeGenerator::emitGuardOrFwdJcc(IRInstruction* inst, ConditionCode cc) { - if (cc == CC_None) return; - Block* label = inst->getTaken(); - if (inst && inst->getTCA() == kIRDirectGuardActive) { - if (dumpIREnabled(kCodeGenLevel)) { - m_tx64->prepareForSmash(m_as, TranslatorX64::kJmpccLen); - inst->setTCA(m_as.code.frontier); - } - // Get the SrcKey for the dest - SrcKey destSK(getCurFunc(), m_curTrace->getBcOff()); - SrcRec* destSR = m_tx64->getSrcRec(destSK); - m_tx64->emitFallbackCondJmp(m_as, *destSR, cc); - } else { - emitFwdJcc(cc, label); - } -} - void CodeGenerator::cgLoad(PhysReg base, int64_t off, IRInstruction* inst) { @@ -3918,7 +3870,9 @@ void CodeGenerator::cgLoad(PhysReg base, } Block* label = inst->getTaken(); if (label != NULL) { - cgGuardTypeCell(base, off, inst); + emitTypeCheck(inst->getTypeParam(), + base[off + TVOFF(m_type)], + inst->getTaken()); } if (type.isNull()) return; // these are constants auto dstReg = m_regs[inst->getDst()].getReg(); @@ -4000,15 +3954,55 @@ void CodeGenerator::cgLdStack(IRInstruction* inst) { } void CodeGenerator::cgGuardStk(IRInstruction* inst) { - cgGuardTypeCell(m_regs[inst->getSrc(0)].getReg(), - cellsToBytes(inst->getExtra()->offset), - inst); + auto const rSP = m_regs[inst->getSrc(0)].getReg(); + auto const off = cellsToBytes(inst->getExtra()->offset) + + TVOFF(m_type); + emitTypeGuard(inst->getTypeParam(), rSP[off]); +} + +void CodeGenerator::cgCheckStk(IRInstruction* inst) { + auto const rbase = m_regs[inst->getSrc(0)].getReg(); + auto const off = cellsToBytes(inst->getExtra()->offset) + + TVOFF(m_type); + emitTypeCheck(inst->getTypeParam(), rbase[off], inst->getTaken()); } void CodeGenerator::cgGuardLoc(IRInstruction* inst) { - cgGuardTypeCell(m_regs[inst->getSrc(0)].getReg(), - getLocalOffset(inst->getExtra()->locId), - inst); + auto const rFP = m_regs[inst->getSrc(0)].getReg(); + auto const off = getLocalOffset(inst->getExtra()->locId) + + TVOFF(m_type); + emitTypeGuard(inst->getTypeParam(), rFP[off]); +} + +void CodeGenerator::cgCheckLoc(IRInstruction* inst) { + auto const rbase = m_regs[inst->getSrc(0)].getReg(); + auto const off = getLocalOffset(inst->getExtra()->locId) + + TVOFF(m_type); + emitTypeCheck(inst->getTypeParam(), rbase[off], inst->getTaken()); +} + +template +void CodeGenerator::emitSideExitGuard(Type type, MemLoc mem, Offset taken) { + auto const cc = emitTypeTest(type, mem, true /* negate */); + auto const sk = SrcKey(getCurFunc(), taken); + if (cc == CC_None) return; + m_tx64->emitBindJcc(m_as, cc, sk, REQ_BIND_SIDE_EXIT); +} + +void CodeGenerator::cgSideExitGuardLoc(IRInstruction* inst) { + auto const fp = m_regs[inst->getSrc(0)].getReg(); + auto const extra = inst->getExtra(); + emitSideExitGuard(inst->getTypeParam(), + fp[getLocalOffset(extra->checkedSlot) + TVOFF(m_type)], + extra->taken); +} + +void CodeGenerator::cgSideExitGuardStk(IRInstruction* inst) { + auto const sp = m_regs[inst->getSrc(0)].getReg(); + auto const extra = inst->getExtra(); + emitSideExitGuard(inst->getTypeParam(), + sp[cellsToBytes(extra->checkedSlot) + TVOFF(m_type)], + extra->taken); } void CodeGenerator::cgDefMIStateBase(IRInstruction* inst) { @@ -4016,17 +4010,16 @@ void CodeGenerator::cgDefMIStateBase(IRInstruction* inst) { assert(m_regs[inst->getDst()].getReg() == rsp); } -void CodeGenerator::cgGuardType(IRInstruction* inst) { - Type type = inst->getTypeParam(); - SSATmp* src = inst->getSrc(0); - auto srcTypeReg = m_regs[src].getReg(1); - assert(srcTypeReg != InvalidReg); +void CodeGenerator::cgCheckType(IRInstruction* inst) { + auto const src = inst->getSrc(0); + auto const rType = m_regs[src].getReg(1); + + auto const cc = emitTypeTest(inst->getTypeParam(), rType, true); + if (cc == CC_None) return; - ConditionCode cc; - cc = emitTypeTest(type, srcTypeReg, true); emitFwdJcc(cc, inst->getTaken()); - auto dstReg = m_regs[inst->getDst()].getReg(); + auto const dstReg = m_regs[inst->getDst()].getReg(); if (dstReg != InvalidReg) { emitMovRegReg(m_as, m_regs[src].getReg(0), dstReg); } @@ -4041,7 +4034,6 @@ void CodeGenerator::cgGuardRefs(IRInstruction* inst) { SSATmp* firstBitNumTmp = inst->getSrc(3); SSATmp* mask64Tmp = inst->getSrc(4); SSATmp* vals64Tmp = inst->getSrc(5); - Block* exitLabel = inst->getTaken(); // Get values in place assert(funcPtrTmp->type() == Type::Func); @@ -4071,6 +4063,9 @@ void CodeGenerator::cgGuardRefs(IRInstruction* inst) { assert(vals64Reg != InvalidReg); int64_t vals64 = vals64Tmp->getValInt(); + auto const destSK = SrcKey(getCurFunc(), m_curTrace->getBcOff()); + auto const destSR = m_tx64->getSrcRec(destSK); + auto thenBody = [&] { auto bitsValReg = m_rScratch; // Load the bit values in bitValReg: @@ -4082,7 +4077,7 @@ void CodeGenerator::cgGuardRefs(IRInstruction* inst) { // If bitsValReg != vals64Reg, then goto Exit m_as.cmp_reg64_reg64(bitsValReg, vals64Reg); - emitFwdJcc(CC_NE, exitLabel); + m_tx64->emitFallbackCondJmp(m_as, *destSR, CC_NE); }; // If few enough args... @@ -4090,18 +4085,18 @@ void CodeGenerator::cgGuardRefs(IRInstruction* inst) { if (vals64 == 0 && mask64 == 0) { ifThen(m_as, CC_NL, thenBody); } else if (vals64 != 0 && vals64 != mask64) { - emitFwdJcc(CC_L, exitLabel); + m_tx64->emitFallbackCondJmp(m_as, *destSR, CC_L); thenBody(); } else if (vals64 != 0) { ifThenElse(CC_NL, thenBody, /* else */ [&] { // If not special builtin... m_as.testl(AttrVariadicByRef, funcPtrReg[Func::attrsOff()]); - emitFwdJcc(CC_Z, exitLabel); + m_tx64->emitFallbackCondJmp(m_as, *destSR, CC_Z); }); } else { ifThenElse(CC_NL, thenBody, /* else */ [&] { m_as.testl(AttrVariadicByRef, funcPtrReg[Func::attrsOff()]); - emitFwdJcc(CC_NZ, exitLabel); + m_tx64->emitFallbackCondJmp(m_as, *destSR, CC_NZ); }); } } @@ -4632,45 +4627,48 @@ void CodeGenerator::cgLdGblAddrDef(IRInstruction* inst) { ArgGroup(m_regs).ssa(inst->getSrc(0))); } -void CodeGenerator::cgJmpZeroHelper(IRInstruction* inst, - ConditionCode cc) { - SSATmp* src = inst->getSrc(0); +void CodeGenerator::emitTestZero(SSATmp* src) { + auto& a = m_as; + auto reg = m_regs[src].getReg(); - auto srcReg = m_regs[src].getReg(); - if (src->isConst()) { - bool valIsZero = src->getValRawInt() == 0; - if ((cc == CC_Z && valIsZero) || - (cc == CC_NZ && !valIsZero)) { - // assert(false) here after new simplifier pass, t2019643 - // For now, materialize the test condition and use a Jcc - m_as.xor_reg64_reg64(m_rScratch, m_rScratch); - m_as.test_reg64_reg64(m_rScratch, m_rScratch); - cc = CC_Z; - // Update the instr opcode since cgExitTrace uses it - // to determine correct cc for service request. - inst->setOpcode(JmpZero); - } else { - // Fall through to next bytecode, disable DirectJmp - inst->setTCA(kIRDirectJmpInactive); - return; - } - } else { - if (src->type() == Type::Bool) { - m_as.testb(Reg8(int(srcReg)), Reg8(int(srcReg))); - } else { - m_as.test_reg64_reg64(srcReg, srcReg); - } + /* + * If src is const, normally a earlier optimization pass should have + * converted the thing testing this condition into something + * unconditional. So rather than supporting constants efficiently + * here, we just materialize the value into a register. + */ + if (reg == InvalidReg) { + reg = m_rScratch; + a. movq (src->getValBits(), reg); } - emitJccDirectExit(inst, cc); + if (src->isA(Type::Bool)) { + a. testb (rbyte(reg), rbyte(reg)); + } else { + a. testq (reg, reg); + } } void CodeGenerator::cgJmpZero(IRInstruction* inst) { - cgJmpZeroHelper(inst, CC_Z); + emitTestZero(inst->getSrc(0)); + emitFwdJcc(CC_Z, inst->getTaken()); } void CodeGenerator::cgJmpNZero(IRInstruction* inst) { - cgJmpZeroHelper(inst, CC_NZ); + emitTestZero(inst->getSrc(0)); + emitFwdJcc(CC_NZ, inst->getTaken()); +} + +void CodeGenerator::cgReqBindJmpZero(IRInstruction* inst) { + // TODO(#2404427): prepareForTestAndSmash? + emitTestZero(inst->getSrc(0)); + emitReqBindJcc(CC_Z, inst->getExtra()); +} + +void CodeGenerator::cgReqBindJmpNZero(IRInstruction* inst) { + // TODO(#2404427): prepareForTestAndSmash? + emitTestZero(inst->getSrc(0)); + emitReqBindJcc(CC_NZ, inst->getExtra()); } void CodeGenerator::cgJmp_(IRInstruction* inst) { @@ -4784,7 +4782,7 @@ void CodeGenerator::cgBoxPtr(IRInstruction* inst) { auto base = m_regs[addr].getReg(); auto dstReg = m_regs[dst].getReg(); emitMovRegReg(m_as, base, dstReg); - ConditionCode cc = emitTypeTest(Type::BoxedCell, base[TVOFF(m_type)], true); + auto const cc = emitTypeTest(Type::BoxedCell, base[TVOFF(m_type)], true); ifThen(m_as, cc, [&] { cgCallHelper(m_as, (TCA)tvBox, dstReg, kNoSyncPoint, ArgGroup(m_regs).ssa(addr)); diff --git a/hphp/runtime/vm/translator/hopt/codegen.h b/hphp/runtime/vm/translator/hopt/codegen.h index 01749a724..3e99e7ba5 100644 --- a/hphp/runtime/vm/translator/hopt/codegen.h +++ b/hphp/runtime/vm/translator/hopt/codegen.h @@ -193,13 +193,10 @@ private: template ConditionCode emitTypeTest(Type type, OpndType src, bool negate); - template - ConditionCode emitTypeTest(IRInstruction* inst, OpndType src, bool negate); - - template - void emitGuardType(OpndType src, IRInstruction* instr); - - void cgGuardTypeCell(PhysReg baseReg,int64_t offset,IRInstruction* instr); + template + void emitTypeCheck(Type type, MemLoc src, Block* taken); + template + void emitTypeGuard(Type type, MemLoc mem); void cgStMemWork(IRInstruction* inst, bool genStoreType); void cgStRefWork(IRInstruction* inst, bool genStoreType); @@ -242,8 +239,8 @@ private: void cgLoadTypedValue(PhysReg base, int64_t off, IRInstruction* inst); - void cgNegate(IRInstruction* inst); // helper - void cgJcc(IRInstruction* inst); // helper + void cgJcc(IRInstruction* inst); // helper + void cgReqBindJcc(IRInstruction* inst); // helper void cgOpCmpHelper( IRInstruction* inst, void (Asm::*setter)(Reg8), @@ -253,7 +250,13 @@ private: int64_t (*obj_cmp_obj)(ObjectData*, ObjectData*), int64_t (*obj_cmp_int)(ObjectData*, int64_t), int64_t (*arr_cmp_arr)(ArrayData*, ArrayData*)); - void cgJmpZeroHelper(IRInstruction* inst, ConditionCode cc); + + template + void emitSideExitGuard(Type type, MemLoc mem, Offset taken); + void emitReqBindJcc(ConditionCode cc, const ReqBindJccData*); + + void emitCompare(SSATmp*, SSATmp*); + void emitTestZero(SSATmp*); bool emitIncDecHelper(SSATmp* dst, SSATmp* src1, SSATmp* src2, void(Asm::*emitFunc)(Reg64)); bool emitInc(SSATmp* dst, SSATmp* src1, SSATmp* src2); @@ -307,12 +310,6 @@ private: Address emitFwdJcc(Asm& a, ConditionCode cc, Block* target); Address emitFwdJmp(Asm& as, Block* target); Address emitFwdJmp(Block* target); - Address emitSmashableFwdJccAtEnd(ConditionCode cc, Block* target, - IRInstruction* toSmash); - void emitJccDirectExit(IRInstruction*, ConditionCode); - Address emitSmashableFwdJcc(ConditionCode cc, Block* target, - IRInstruction* toSmash); - void emitGuardOrFwdJcc(IRInstruction* inst, ConditionCode cc); void emitContVarEnvHelperCall(SSATmp* fp, TCA helper); const Func* getCurFunc() const; Class* getCurClass() const { return getCurFunc()->cls(); } diff --git a/hphp/runtime/vm/translator/hopt/dce.cpp b/hphp/runtime/vm/translator/hopt/dce.cpp index f5f6434e0..445a97172 100644 --- a/hphp/runtime/vm/translator/hopt/dce.cpp +++ b/hphp/runtime/vm/translator/hopt/dce.cpp @@ -482,7 +482,7 @@ void optimizeActRecs(Trace* trace, DceState& state, IRFactory* factory, // Assuming that the 'consumer' instruction consumes 'src', trace back through // src's instruction to the real origin of the value. Currently this traces -// through GuardType and DefLabel. +// through CheckType and DefLabel. void consumeIncRef(const IRInstruction* consumer, const SSATmp* src, DceState& state, SSACache& ssas, SSASet visitedSrcs) { assert(!visitedSrcs.count(src) && "Cycle detected in dataflow graph"); @@ -497,11 +497,11 @@ void consumeIncRef(const IRInstruction* consumer, const SSATmp* src, const IRInstruction* srcInst = src->inst(); visitedSrcs.insert(src); - if (srcInst->op() == GuardType && + if (srcInst->op() == CheckType && srcInst->getTypeParam().maybeCounted()) { - // srcInst is a GuardType that guards to a refcounted type. We need to - // trace through to its source. If the GuardType guards to a non-refcounted - // type then the reference is consumed by GuardType itself. + // srcInst is a CheckType that guards to a refcounted type. We need to + // trace through to its source. If the CheckType guards to a non-refcounted + // type then the reference is consumed by CheckType itself. consumeIncRef(consumer, srcInst->getSrc(0), state, ssas, visitedSrcs); } else if (srcInst->op() == DefLabel) { // srcInst is a DefLabel that may be a join node. We need to find diff --git a/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp b/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp index 668911849..c78b1d227 100644 --- a/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp +++ b/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp @@ -1350,29 +1350,25 @@ void HhbcTranslator::emitJmp(int32_t offset, } SSATmp* HhbcTranslator::emitJmpCondHelper(int32_t offset, - bool negate, - SSATmp* src) { - Trace* target = nullptr; - if (m_lastBcOff) { - // Spill everything on main trace if all paths will exit - spillStack(); - target = getExitTrace(offset, getBcOffNextTrace()); - } else { - target = getExitTrace(offset); - } + bool negate, + SSATmp* src) { + // Spill everything on main trace if all paths will exit. + if (m_lastBcOff) spillStack(); + + auto const target = getExitTrace(offset); auto const boolSrc = gen(ConvCellToBool, src); gen(DecRef, src); return gen(negate ? JmpZero : JmpNZero, target, boolSrc); } -void HhbcTranslator::emitJmpZ(int32_t offset) { - SSATmp* src = popC(); - emitJmpCondHelper(offset, true, src); +void HhbcTranslator::emitJmpZ(Offset taken) { + auto const src = popC(); + emitJmpCondHelper(taken, true, src); } -void HhbcTranslator::emitJmpNZ(int32_t offset) { - SSATmp* src = popC(); - emitJmpCondHelper(offset, false, src); +void HhbcTranslator::emitJmpNZ(Offset taken) { + auto const src = popC(); + emitJmpCondHelper(taken, false, src); } void HhbcTranslator::emitCmp(Opcode opc) { @@ -1584,8 +1580,7 @@ void HhbcTranslator::emitFPushCtor(int32_t numParams) { emitFPushCtorCommon(cls, obj, nullptr, numParams); } -bool -canInstantiateClass(const Class* cls) { +static bool canInstantiateClass(const Class* cls) { return cls && !(cls->attrs() & (AttrAbstract | AttrInterface | AttrTrait)); } @@ -2188,7 +2183,7 @@ void HhbcTranslator::emitSwitch(const ImmVector& iv, data.targets = &targets[0]; auto const stack = spillStack(); - gen(SyncVMRegs, m_tb->getFp(), stack); + gen(SyncABIRegs, m_tb->getFp(), stack); gen(JmpSwitchDest, data, index); m_hasExit = true; @@ -2215,8 +2210,7 @@ void HhbcTranslator::emitSSwitch(const ImmVector& iv) { // The slow path can throw exceptions and reenter the VM. if (!fastPath) exceptionBarrier(); - SSATmp* const testVal = popC(); - assert(bcOff() != -1); + auto const testVal = popC(); std::vector cases(numCases); for (int i = 0; i < numCases; ++i) { @@ -2237,7 +2231,7 @@ void HhbcTranslator::emitSSwitch(const ImmVector& iv) { testVal); gen(DecRef, testVal); auto const stack = spillStack(); - gen(SyncVMRegs, m_tb->getFp(), stack); + gen(SyncABIRegs, m_tb->getFp(), stack); gen(JmpIndirect, dest); m_hasExit = true; } @@ -2255,11 +2249,11 @@ void HhbcTranslator::setThisAvailable() { } void HhbcTranslator::guardTypeLocal(uint32_t locId, Type type) { - checkTypeLocal(locId, type); + gen(GuardLoc, type, LocalId(locId), m_tb->getFp()); } void HhbcTranslator::checkTypeLocal(uint32_t locId, Type type) { - gen(GuardLoc, type, getExitTrace(), LocalId(locId), m_tb->getFp()); + gen(CheckLoc, type, LocalId(locId), getExitTrace(), m_tb->getFp()); } void HhbcTranslator::assertTypeLocal(uint32_t locId, Type type) { @@ -2270,35 +2264,28 @@ void HhbcTranslator::overrideTypeLocal(uint32_t locId, Type type) { gen(OverrideLoc, type, LocalId(locId), m_tb->getFp()); } -Trace* HhbcTranslator::guardTypeStack(uint32_t stackIndex, - Type type, - Trace* nextTrace) { +void HhbcTranslator::guardTypeStack(uint32_t stackIndex, Type type) { + // Should not generate guards for class; instead assert their type if (type.subtypeOf(Type::Cls)) { - // Should not generate guards for class; instead assert their type assertTypeStack(stackIndex, type); - return nextTrace; + return; } - if (nextTrace == nullptr) { - nextTrace = getGuardExit(); - } - gen(GuardStk, type, nextTrace, StackOffset(stackIndex), m_tb->getSp()); - return nextTrace; + gen(GuardStk, type, StackOffset(stackIndex), m_tb->getSp()); } -void HhbcTranslator::checkTypeTopOfStack(Type type, - Offset nextByteCode) { +void HhbcTranslator::checkTypeTopOfStack(Type type, Offset nextByteCode) { Trace* exitTrace = getExitTrace(nextByteCode); SSATmp* tmp = m_evalStack.top(); if (!tmp) { FTRACE(1, "checkTypeTopOfStack: no tmp: {}\n", type.toString()); - gen(GuardStk, type, exitTrace, StackOffset(0), m_tb->getSp()); + gen(CheckStk, type, exitTrace, StackOffset(0), m_tb->getSp()); push(pop(type)); } else { - FTRACE(1, "checkTypeTopOfStack: generating GuardType for {}\n", + FTRACE(1, "checkTypeTopOfStack: generating CheckType for {}\n", type.toString()); m_evalStack.pop(); - tmp = gen(GuardType, type, exitTrace, tmp); + tmp = gen(CheckType, type, exitTrace, tmp); push(tmp); } } @@ -2319,14 +2306,9 @@ void HhbcTranslator::assertTypeStack(uint32_t stackIndex, Type type) { refineType(tmp, type); } -Trace* HhbcTranslator::guardRefs(int64_t entryArDelta, - const vector& mask, - const vector& vals, - Trace* exitTrace) { - if (exitTrace == nullptr) { - exitTrace = getGuardExit(); - } - +void HhbcTranslator::guardRefs(int64_t entryArDelta, + const vector& mask, + const vector& vals) { int32_t actRecOff = cellsToBytes(entryArDelta); SSATmp* funcPtr = gen(LdARFuncPtr, m_tb->getSp(), cns(actRecOff)); SSATmp* nParams = gen( @@ -2347,7 +2329,6 @@ Trace* HhbcTranslator::guardRefs(int64_t entryArDelta, gen( GuardRefs, - exitTrace, funcPtr, nParams, bitsPtr, @@ -2356,8 +2337,6 @@ Trace* HhbcTranslator::guardRefs(int64_t entryArDelta, m_tb->genLdConst(vals64) ); } - - return exitTrace; } void HhbcTranslator::emitVerifyParamType(int32_t paramId) { @@ -2936,23 +2915,15 @@ void HhbcTranslator::emitMod() { // Exit path spills an additional false auto exitSpillValues = getSpillValues(); exitSpillValues.push_back(cns(false)); - // Generate an exit for the rare case that r is zero - auto exit = - m_tb->ifThenExit( - getCurFunc(), - m_stackDeficit, - exitSpillValues, - [&](IRFactory* irf, Trace* t) { - // Dividing by zero. Interpreting will raise a notice and - // produce the boolean false. Punch out here and resume after - // the Mod instruction; this should be rare. - m_tb->genFor(t, RaiseWarning, - cns(StringData::GetStaticString( - Strings::DIVISION_BY_ZERO))); - }, - getNextSrcKey().offset() /* exitBcOff */, - bcOff() - ); + + // Generate an exit for the rare case that r is zero. Interpreting + // will raise a notice and produce the boolean false. Punch out + // here and resume after the Mod instruction; this should be rare. + auto const exit = getExitTraceWarn( + getNextSrcKey().offset(), + exitSpillValues, + StringData::GetStaticString(Strings::DIVISION_BY_ZERO) + ); gen(JmpZero, exit, r); push(gen(OpMod, l, r)); } @@ -3031,14 +3002,6 @@ void HhbcTranslator::emitInterpOneOrPunt(Type type, int numPopped, } } -Trace* HhbcTranslator::getGuardExit() { - assert(bcOff() == -1 || bcOff() == m_startBcOff); - assert(!isInlining()); - // stack better be empty since we're at the start of the trace - assert((m_evalStack.numCells() - m_stackDeficit) == 0); - return m_exitGuardFailureTrace; -} - /* * Get SSATmps representing all the information on the virtual eval * stack in preparation for a spill or exit trace. @@ -3055,56 +3018,82 @@ std::vector HhbcTranslator::getSpillValues() const { return ret; } +Trace* HhbcTranslator::getExitTrace(Offset targetBcOff /* = -1 */) { + auto spillValues = getSpillValues(); + return getExitTrace(targetBcOff, spillValues); +} + +Trace* HhbcTranslator::getExitTrace(Offset targetBcOff, + std::vector& spillValues) { + if (targetBcOff == -1) targetBcOff = bcOff(); + return getExitTraceImpl(targetBcOff, ExitFlag::None, spillValues, nullptr); +} + +Trace* HhbcTranslator::getExitTraceWarn(Offset targetBcOff, + std::vector& spillValues, + const StringData* warning) { + assert(targetBcOff != -1); + return getExitTraceImpl(targetBcOff, ExitFlag::None, spillValues, warning); +} + /* * Generates an exit trace which will continue execution without HHIR. * This should be used in situations that HHIR cannot handle -- ideally * only in slow paths. */ Trace* HhbcTranslator::getExitSlowTrace() { - auto stackValues = getSpillValues(); - return m_tb->getExitSlowTrace(bcOff(), - m_stackDeficit, - stackValues.size(), - stackValues.size() ? &stackValues[0] : 0); + auto spillValues = getSpillValues(); + return getExitTraceImpl(bcOff(), ExitFlag::NoIR, spillValues, nullptr); } -/* - * Generates an exit trace for the given targetBcOff - * (defaults to the current offset). - * The exit trace returned will be linked to a translation starting at - * targetBcOff, which will be a retranslation of the same tracelet if this - * exit is taken before executing any bytecode instruction of the current - * tracelet. - */ -Trace* HhbcTranslator::getExitTrace(Offset targetBcOff, - const std::vector& stackValues) { - if (targetBcOff == -1) { - targetBcOff = bcOff() != -1 ? bcOff() : m_startBcOff; - } - if (targetBcOff == m_startBcOff) { - return m_exitGuardFailureTrace; +Trace* HhbcTranslator::getExitTraceImpl(Offset targetBcOff, + ExitFlag flag, + std::vector& stackValues, + const StringData* warning) { + auto const exit = m_tb->makeExitTrace(targetBcOff); + + MarkerData exitMarker; + exitMarker.bcOff = targetBcOff; + exitMarker.stackOff = m_tb->getSpOffset() + + stackValues.size() - m_stackDeficit; + exitMarker.func = getCurFunc(); + genFor(exit, Marker, exitMarker); + + if (warning) { + genFor(exit, RaiseWarning, cns(warning)); } - return m_tb->genExitTrace(targetBcOff, - m_stackDeficit, - stackValues.size(), - stackValues.size() ? &stackValues[0] : nullptr, - TraceExitType::Normal); -} + auto const stack = [&]{ + // TODO(#2404447) move this conditional to the simplifier? + if (m_stackDeficit != 0 || !stackValues.empty()) { + stackValues.insert( + stackValues.begin(), + { m_tb->getSp(), cns(int64_t(m_stackDeficit)) } + ); + return genFor(exit, + SpillStack, std::make_pair(stackValues.size(), &stackValues[0]) + ); + } + return m_tb->getSp(); + }(); -/* - * Generates a trace exit that can be the target of a conditional - * control flow instruction at the current bytecode offset. - */ -Trace* HhbcTranslator::getExitTrace(uint32_t targetBcOff, - uint32_t notTakenBcOff) { - std::vector stackValues = getSpillValues(); - return m_tb->genExitTrace(targetBcOff, - m_stackDeficit, - stackValues.size(), - stackValues.size() ? &stackValues[0] : nullptr, - TraceExitType::NormalCc, - notTakenBcOff); + genFor(exit, SyncABIRegs, m_tb->getFp(), stack); + + if (flag == ExitFlag::NoIR) { + genFor(exit, + targetBcOff == m_startBcOff ? ReqRetranslateNoIR : ReqBindJmpNoIR, + BCOffset(targetBcOff) + ); + return exit; + } + + if (bcOff() == m_startBcOff && targetBcOff == m_startBcOff) { + genFor(exit, ReqRetranslate); + } else { + genFor(exit, ReqBindJmp, BCOffset(targetBcOff)); + } + + return exit; } SSATmp* HhbcTranslator::spillStack() { @@ -3239,9 +3228,10 @@ SSATmp* HhbcTranslator::stLocNRC(uint32_t id, Trace* exit, SSATmp* newVal) { return stLocImpl(id, exit, newVal, doRefCount); } -void HhbcTranslator::end(int nextPc) { +void HhbcTranslator::end() { if (m_hasExit) return; + auto const nextPc = m_nextTraceBcOff; if (nextPc >= getCurFunc()->past()) { // We have fallen off the end of the func's bytecodes. This happens // when the function's bytecodes end with an unconditional @@ -3254,12 +3244,14 @@ void HhbcTranslator::end(int nextPc) { return; } setBcOff(nextPc, true); - spillStack(); - m_tb->genTraceEnd(nextPc); + auto const sp = spillStack(); + gen(SyncABIRegs, m_tb->getFp(), sp); + gen(ReqBindJmp, BCOffset(nextPc)); } + void HhbcTranslator::checkStrictlyInteger( - SSATmp*& key, KeyType& keyType, bool& checkForInt) { + SSATmp*& key, KeyType& keyType, bool& checkForInt) { checkForInt = false; if (key->isA(Type::Int)) { keyType = IntKey; diff --git a/hphp/runtime/vm/translator/hopt/hhbctranslator.h b/hphp/runtime/vm/translator/hopt/hhbctranslator.h index c7b728de2..9016622c9 100644 --- a/hphp/runtime/vm/translator/hopt/hhbctranslator.h +++ b/hphp/runtime/vm/translator/hopt/hhbctranslator.h @@ -101,26 +101,49 @@ private: */ struct HhbcTranslator { HhbcTranslator(IRFactory& irFactory, - Offset bcStartOffset, + Offset startOffset, + Offset nextTraceOffset, uint32_t initialSpOffsetFromFp, const Func* func) : m_irFactory(irFactory) - , m_tb(new TraceBuilder(bcStartOffset, + , m_tb(new TraceBuilder(startOffset, initialSpOffsetFromFp, m_irFactory, func)) - , m_bcStateStack {BcState(-1, func)} - , m_startBcOff(bcStartOffset) + , m_bcStateStack {BcState(startOffset, func)} + , m_startBcOff(startOffset) + , m_nextTraceBcOff(nextTraceOffset) , m_lastBcOff(false) , m_hasExit(false) , m_stackDeficit(0) - , m_exitGuardFailureTrace(m_tb->genExitGuardFailure(bcStartOffset)) - {} + { + emitMarker(); + } - void end(int nextBcOff); + // Accessors. Trace* getTrace() const { return m_tb->getTrace(); } TraceBuilder* getTraceBuilder() const { return m_tb.get(); } + // In between each emit* call, irtranslator indicates the new + // bytecode offset (or whether we're finished) using this API. + void setBcOff(Offset newOff, bool lastBcOff); + void end(); + + // Tracelet guards. + void guardTypeStack(uint32_t stackIndex, Type type); + void guardTypeLocal(uint32_t locId, Type type); + void guardRefs(int64_t entryArDelta, + const vector& mask, + const vector& vals); + + // Interface to irtranslator for predicted and inferred types. + void assertTypeLocal(uint32_t localIndex, Type type); + void assertTypeStack(uint32_t stackIndex, Type type); + void checkTypeLocal(uint32_t localIndex, Type type); + void checkTypeTopOfStack(Type type, Offset nextByteCode); + void overrideTypeLocal(uint32_t localIndex, Type type); + + // Inlining-related functions. void beginInlining(unsigned numArgs, const Func* target, Offset returnBcOffset); @@ -131,9 +154,14 @@ struct HhbcTranslator { void profileSmallFunctionShape(const std::string& str); void profileFailedInlShape(const std::string& str); - void setBcOff(Offset newOff, bool lastBcOff); - void setBcOffNextTrace(Offset bcOff) { m_bcOffNextTrace = bcOff; } - uint32_t getBcOffNextTrace() { return m_bcOffNextTrace; } + // Other public functions for irtranslator. + void setThisAvailable(); + void emitInterpOne(Type type, int numPopped, int numExtraPushed = 0); + void emitInterpOneCF(int numPopped); + + /* + * An emit* function for each HHBC opcode. + */ void emitPrint(); void emitThis(); @@ -207,8 +235,8 @@ struct HhbcTranslator { void emitPopR(); void emitDup(); void emitUnboxR(); - void emitJmpZ(int32_t offset); - void emitJmpNZ(int32_t offset); + void emitJmpZ(Offset taken); + void emitJmpNZ(Offset taken); void emitJmp(int32_t offset, bool breakTracelet, bool noSurprise); void emitGt() { emitCmp(OpGt); } void emitGte() { emitCmp(OpGte); } @@ -352,29 +380,6 @@ struct HhbcTranslator { void emitIncTransCounter(); void emitArrayIdx(); - // tracelet guards - Trace* guardTypeStack(uint32_t stackIndex, - Type type, - Trace* nextTrace = nullptr); - void guardTypeLocal(uint32_t locId, Type type); - Trace* guardRefs(int64_t entryArDelta, - const vector& mask, - const vector& vals, - Trace* exitTrace = nullptr); - - // Interface to irtranslator for predicted and inferred types. - void assertTypeLocal(uint32_t localIndex, Type type); - void assertTypeStack(uint32_t stackIndex, Type type); - void checkTypeLocal(uint32_t localIndex, Type type); - void checkTypeTopOfStack(Type type, Offset nextByteCode); - void overrideTypeLocal(uint32_t localIndex, Type type); - void setThisAvailable(); - void emitInterpOne(Type type, int numPopped, int numExtraPushed = 0); - void emitInterpOneCF(int numPopped); - - void checkStrictlyInteger(SSATmp*& key, KeyType& keyType, - bool& checkForInt); - private: /* * VectorTranslator is responsible for translating one of the vector @@ -502,6 +507,11 @@ private: // tracebuilder forwarding utilities return m_tb->gen(std::forward(args)...); } + template + SSATmp* genFor(Trace* trace, Args&&... args) { + return m_tb->genFor(trace, std::forward(args)...); + } + private: /* * Emit helpers. @@ -536,6 +546,8 @@ private: bool checkSupportedGblName(const StringData* gblName, HPHP::JIT::Type resultType, int stkIndex); + void checkStrictlyInteger(SSATmp*& key, KeyType& keyType, + bool& checkForInt); SSATmp* emitLdClsPropAddrOrExit(const StringData* propName, Block* block); SSATmp* emitLdClsPropAddr(const StringData* propName) { return emitLdClsPropAddrOrExit(propName, nullptr); @@ -556,31 +568,45 @@ private: SSATmp* emitJmpCondHelper(int32_t offset, bool negate, SSATmp* src); SSATmp* emitIncDec(bool pre, bool inc, SSATmp* src); SSATmp* getMemberAddr(const char* vectorDesc, Trace* exitTrace); - Trace* getExitTrace(Offset targetBcOff, - const std::vector& spillValues); - Trace* getExitTrace(Offset targetBcOff = -1) { - return getExitTrace(targetBcOff, getSpillValues()); - } - Trace* getExitTrace(uint32_t targetBcOff, uint32_t notTakenBcOff); - Trace* getExitSlowTrace(); - Trace* getGuardExit(); void emitInterpOneOrPunt(Type type, int numPopped, int numExtraPushed = 0); void emitBinaryArith(Opcode); template SSATmp* emitIterInitCommon(int offset, Lambda genFunc); void emitMarker(); + // Exit trace creation routines. + Trace* getExitTrace(Offset targetBcOff = -1); + Trace* getExitTrace(Offset targetBcOff, + std::vector& spillValues); + Trace* getExitTraceWarn(Offset targetBcOff, + std::vector& spillValues, + const StringData* warning); + Trace* getExitSlowTrace(); + + enum class ExitFlag { + None, + NoIR, + }; + Trace* getExitTraceImpl(Offset targetBcOff, + ExitFlag noIRExit, + std::vector& spillValues, + const StringData* warning); + /* * Accessors for the current function being compiled and its * class and unit. */ - const Func* getCurFunc() const { return m_bcStateStack.back().func; } - Class* getCurClass() const { return getCurFunc()->cls(); } - Unit* getCurUnit() const { return getCurFunc()->unit(); } - Offset bcOff() const { return m_bcStateStack.back().bcOff; } + const Func* getCurFunc() const { return m_bcStateStack.back().func; } + Class* getCurClass() const { return getCurFunc()->cls(); } + Unit* getCurUnit() const { return getCurFunc()->unit(); } + Offset bcOff() const { return m_bcStateStack.back().bcOff; } + SrcKey getCurSrcKey() const { return SrcKey(getCurFunc(), bcOff()); } - SrcKey getCurSrcKey() const { return SrcKey(getCurFunc(), bcOff()); } - SrcKey getNextSrcKey() const { + /* + * Return the SrcKey for the next HHBC (whether it is in this + * tracelet or not). + */ + SrcKey getNextSrcKey() const { SrcKey srcKey(getCurFunc(), bcOff()); srcKey.advance(getCurFunc()->unit()); return srcKey; @@ -645,15 +671,24 @@ private: }; private: - IRFactory& m_irFactory; - std::unique_ptr - m_tb; - std::vector - m_bcStateStack; - Offset m_startBcOff; - Offset m_bcOffNextTrace; - bool m_lastBcOff; - bool m_hasExit; + IRFactory& m_irFactory; + std::unique_ptr const m_tb; + + std::vector m_bcStateStack; + + // The first HHBC offset for this tracelet, and the offset for the + // next Traclet. + const Offset m_startBcOff; + const Offset m_nextTraceBcOff; + + // True if we're on the last HHBC opcode that will be emitted for + // this tracelet. + bool m_lastBcOff; + + // True if we've emitted an instruction that already handled + // end-of-tracelet duties. (E.g. emitRetC, etc.) If it's not true, + // we'll create a generic ReqBindJmp instruction after we're done. + bool m_hasExit; /* * Tracking of the state of the virtual execution stack: @@ -668,18 +703,15 @@ private: * m_stackDeficit represents the number of cells we've popped off * the virtual stack since the last sync. */ - uint32_t m_stackDeficit; - EvalStack m_evalStack; + uint32_t m_stackDeficit; + EvalStack m_evalStack; /* * The FPI stack is used for inlining---when we start inlining at an * FCall, we look in here to find a definition of the StkPtr,offset * that can be used after the inlined callee "returns". */ - std::stack> - m_fpiStack; - - Trace* const m_exitGuardFailureTrace; + std::stack> m_fpiStack; }; ////////////////////////////////////////////////////////////////////// diff --git a/hphp/runtime/vm/translator/hopt/ir.cpp b/hphp/runtime/vm/translator/hopt/ir.cpp index 7ddb15356..23c9cd6a4 100644 --- a/hphp/runtime/vm/translator/hopt/ir.cpp +++ b/hphp/runtime/vm/translator/hopt/ir.cpp @@ -300,7 +300,6 @@ IRInstruction::IRInstruction(Arena& arena, const IRInstruction* inst, Id id) , m_dst(nullptr) , m_taken(nullptr) , m_block(nullptr) - , m_tca(nullptr) , m_extra(inst->m_extra ? cloneExtra(op(), inst->m_extra, arena) : nullptr) { @@ -360,11 +359,11 @@ bool IRInstruction::hasMemEffects() const { bool IRInstruction::canCSE() const { auto canCSE = opcodeHasFlags(op(), CanCSE); // Make sure that instructions that are CSE'able can't produce a reference - // count or consume reference counts. GuardType is special because it can + // count or consume reference counts. CheckType is special because it can // refine a maybeCounted type to a notCounted type, so it logically consumes // and produces a reference without doing any work. - assert(!canCSE || !producesReference() || m_op == GuardType); - assert(!canCSE || !consumesReferences() || m_op == GuardType); + assert(!canCSE || !producesReference() || m_op == CheckType); + assert(!canCSE || !consumesReferences() || m_op == CheckType); return canCSE && !mayReenterHelper(); } @@ -376,9 +375,9 @@ bool IRInstruction::consumesReference(int srcNo) const { if (!consumesReferences()) { return false; } - // GuardType consumes a reference if we're guarding from a maybeCounted type + // CheckType consumes a reference if we're guarding from a maybeCounted type // to a notCounted type. - if (m_op == GuardType) { + if (m_op == CheckType) { assert(srcNo == 0); return getSrc(0)->type().maybeCounted() && getTypeParam().notCounted(); } @@ -454,7 +453,7 @@ bool IRInstruction::isPassthrough() const { SSATmp* IRInstruction::getPassthroughValue() const { assert(isPassthrough()); - assert(m_op == IncRef || m_op == GuardType || m_op == Mov); + assert(m_op == IncRef || m_op == CheckType || m_op == Mov); return getSrc(0); } @@ -548,9 +547,122 @@ const StringData* findClassName(SSATmp* cls) { return nullptr; } +bool isQueryOp(Opcode opc) { + switch (opc) { + case OpGt: + case OpGte: + case OpLt: + case OpLte: + case OpEq: + case OpNeq: + case OpSame: + case OpNSame: + case InstanceOfBitmask: + case NInstanceOfBitmask: + case IsType: + case IsNType: + return true; + default: + return false; + } +} + +bool isCmpOp(Opcode opc) { + switch (opc) { + case OpGt: + case OpGte: + case OpLt: + case OpLte: + case OpEq: + case OpNeq: + case OpSame: + case OpNSame: + return true; + default: + return false; + } +} + +bool isQueryJmpOp(Opcode opc) { + switch (opc) { + case JmpGt: + case JmpGte: + case JmpLt: + case JmpLte: + case JmpEq: + case JmpNeq: + case JmpSame: + case JmpNSame: + case JmpInstanceOfBitmask: + case JmpNInstanceOfBitmask: + case JmpIsType: + case JmpIsNType: + case JmpZero: + case JmpNZero: + return true; + default: + return false; + } +} + +Opcode queryToJmpOp(Opcode opc) { + assert(isQueryOp(opc)); + switch (opc) { + case OpGt: return JmpGt; + case OpGte: return JmpGte; + case OpLt: return JmpLt; + case OpLte: return JmpLte; + case OpEq: return JmpEq; + case OpNeq: return JmpNeq; + case OpSame: return JmpSame; + case OpNSame: return JmpNSame; + case InstanceOfBitmask: return JmpInstanceOfBitmask; + case NInstanceOfBitmask: return JmpNInstanceOfBitmask; + case IsType: return JmpIsType; + case IsNType: return JmpIsNType; + default: always_assert(0); + } +} + +Opcode queryJmpToQueryOp(Opcode opc) { + assert(isQueryJmpOp(opc)); + switch (opc) { + case JmpGt: return OpGt; + case JmpGte: return OpGte; + case JmpLt: return OpLt; + case JmpLte: return OpLte; + case JmpEq: return OpEq; + case JmpNeq: return OpNeq; + case JmpSame: return OpSame; + case JmpNSame: return OpNSame; + case JmpInstanceOfBitmask: return InstanceOfBitmask; + case JmpNInstanceOfBitmask: return NInstanceOfBitmask; + case JmpIsType: return IsType; + case JmpIsNType: return IsNType; + default: always_assert(0); + } +} + +Opcode jmpToReqBindJmp(Opcode opc) { + switch (opc) { + case JmpGt: return ReqBindJmpGt; + case JmpGte: return ReqBindJmpGte; + case JmpLt: return ReqBindJmpLt; + case JmpLte: return ReqBindJmpLte; + case JmpEq: return ReqBindJmpEq; + case JmpNeq: return ReqBindJmpNeq; + case JmpSame: return ReqBindJmpSame; + case JmpNSame: return ReqBindJmpNSame; + case JmpInstanceOfBitmask: return ReqBindJmpInstanceOfBitmask; + case JmpNInstanceOfBitmask: return ReqBindJmpNInstanceOfBitmask; + case JmpZero: return ReqBindJmpZero; + case JmpNZero: return ReqBindJmpNZero; + default: always_assert(0); + } +} + Opcode negateQueryOp(Opcode opc) { assert(isQueryOp(opc)); - switch (opc) { case OpGt: return OpLte; case OpGte: return OpLt; @@ -568,16 +680,20 @@ Opcode negateQueryOp(Opcode opc) { } } -Opcode queryCommuteTable[] = { - OpLt, // OpGt - OpLte, // OpGte - OpGt, // OpLt - OpGte, // OpLte - OpEq, // OpEq - OpNeq, // OpNeq - OpSame, // OpSame - OpNSame // OpNSame -}; +Opcode commuteQueryOp(Opcode opc) { + assert(isQueryOp(opc)); + switch (opc) { + case OpGt: return OpLt; + case OpGte: return OpLte; + case OpLt: return OpGt; + case OpLte: return OpGte; + case OpEq: return OpEq; + case OpNeq: return OpNeq; + case OpSame: return OpSame; + case OpNSame: return OpNSame; + default: always_assert(0); + } +} // Objects compared with strings may involve calling a user-defined // __toString function. @@ -589,15 +705,6 @@ bool cmpOpTypesMayReenter(Opcode op, Type t0, Type t1) { (t0.isString() || t1.isString())); } -TraceExitType::ExitType getExitType(Opcode opc) { - assert(opc >= ExitTrace && opc <= ExitGuardFailure); - return TraceExitType::ExitType(opc - ExitTrace); -} - -Opcode getExitOpcode(TraceExitType::ExitType type) { - return (Opcode)(ExitTrace + type); -} - bool isRefCounted(SSATmp* tmp) { if (tmp->type().notCounted()) { return false; @@ -620,7 +727,6 @@ void IRInstruction::convertToNop() { m_numDsts = nop.m_numDsts; m_dst = nop.m_dst; m_taken = nullptr; - m_tca = nop.m_tca; m_extra = nullptr; } @@ -654,7 +760,6 @@ void IRInstruction::become(IRFactory* factory, IRInstruction* other) { m_op = other->m_op; m_typeParam = other->m_typeParam; m_taken = other->m_taken; - m_tca = other->m_tca; m_numSrcs = other->m_numSrcs; m_extra = other->m_extra ? cloneExtra(m_op, other->m_extra, arena) : nullptr; m_srcs = new (arena) SSATmp*[m_numSrcs]; @@ -871,10 +976,6 @@ TCA SSATmp::getValTCA() const { return m_inst->getExtra()->as(); } -std::string ExitData::show() const { - return folly::to(toSmash->getId()); -} - std::string SSATmp::toString() const { std::ostringstream out; print(out, this); @@ -893,37 +994,59 @@ int32_t spillValueCells(IRInstruction* spillStack) { return numSrcs - 2; } -/** - * Return a list of blocks in reverse postorder - */ BlockList sortCfg(Trace* trace, const IRFactory& factory) { assert(trace->isMain()); BlockList blocks; unsigned next_id = 0; - postorderWalk([&](Block* block) { + postorderWalk( + [&](Block* block) { block->setPostId(next_id++); blocks.push_front(block); - }, factory.numBlocks(), trace->front()); + }, + factory.numBlocks(), + trace->front() + ); + assert(blocks.size() <= factory.numBlocks()); + assert(next_id <= factory.numBlocks()); return blocks; } +bool isRPOSorted(const BlockList& blocks) { + int id = 0; + for (auto it = blocks.rbegin(); it != blocks.rend(); ++it) { + if ((*it)->postId() != id++) return false; + } + return true; +} + +PredVector computePredecessors(const BlockList& blocks) { + assert(isRPOSorted(blocks)); + + PredVector ret(blocks.size()); + + for (auto& block : blocks) { + if (auto succ = block->getNext()) { + ret[succ->postId()].push_back(block); + } + if (auto succ = block->getTaken()) { + ret[succ->postId()].push_back(block); + } + } + + return ret; +} + /* * Find the immediate dominator of each block using Cooper, Harvey, and * Kennedy's "A Simple, Fast Dominance Algorithm", returned as a vector * of postorder ids, indexed by postorder id. */ IdomVector findDominators(const BlockList& blocks) { - // compute predecessors of each block - int num_blocks = blocks.size(); - std::forward_list preds[num_blocks]; - for (Block* block : blocks) { - if (Block* succ = block->getNext()) { - preds[succ->postId()].push_front(block->postId()); - } - if (Block* succ = block->getTaken()) { - preds[succ->postId()].push_front(block->postId()); - } - } + assert(isRPOSorted(blocks)); + + auto const num_blocks = blocks.size(); + auto const preds = computePredecessors(blocks); + // Calculate immediate dominators with the iterative two-finger algorithm. // When it terminates, idom[post-id] will contain the post-id of the // immediate dominator of each block. idom[start] will be -1. This is @@ -940,10 +1063,11 @@ IdomVector findDominators(const BlockList& blocks) { int b = (*it)->postId(); // new_idom = any already-processed predecessor auto pred_it = preds[b].begin(); - int new_idom = *pred_it; - while (idom[new_idom] == -1) new_idom = *(++pred_it); + int new_idom = (*pred_it)->postId(); + while (idom[new_idom] == -1) new_idom = (*++pred_it)->postId(); // for all other already-processed predecessors p of b - for (int p : preds[b]) { + for (auto pred : preds[b]) { + auto p = pred->postId(); if (p != new_idom && idom[p] != -1) { // find earliest common predecessor of p and new_idom // (higher postIds are earlier in flow and in dom-tree). diff --git a/hphp/runtime/vm/translator/hopt/ir.h b/hphp/runtime/vm/translator/hopt/ir.h index 62c7e1198..fe2ec1b22 100644 --- a/hphp/runtime/vm/translator/hopt/ir.h +++ b/hphp/runtime/vm/translator/hopt/ir.h @@ -69,16 +69,6 @@ class FailedIRGen : public std::exception { file(_file), line(_line), func(_func) { } }; -// Flags to identify if a branch should go to a patchable jmp in astubs -// happens when instructions have been moved off the main trace to the exit path. -static const TCA kIRDirectJmpInactive = nullptr; -// Fixup Jcc;Jmp branches out of trace using REQ_BIND_JMPCC_FIRST/SECOND -static const TCA kIRDirectJccJmpActive = (TCA)0x01; -// Optimize Jcc exit from trace when fall through path stays in trace -static const TCA kIRDirectJccActive = (TCA)0x02; -// Optimize guard exit from beginning of trace -static const TCA kIRDirectGuardActive = (TCA)0x03; - #define SPUNT(instr) do { \ throw FailedIRGen(__FILE__, __LINE__, instr); \ } while(0) @@ -164,9 +154,11 @@ static const TCA kIRDirectGuardActive = (TCA)0x03; #define IR_OPCODES \ /* name dstinfo srcinfo flags */ \ -O(GuardType, DParam, S(Gen), C|E|CRc|PRc|P) \ +O(CheckType, DParam, S(Gen), C|E|CRc|PRc|P) \ O(GuardLoc, ND, S(FramePtr), E) \ O(GuardStk, D(StkPtr), S(StkPtr), E) \ +O(CheckLoc, ND, S(FramePtr), E) \ +O(CheckStk, D(StkPtr), S(StkPtr), E) \ O(CastStk, D(StkPtr), S(StkPtr), Mem|N|Er) \ O(AssertStk, D(StkPtr), S(StkPtr), E) \ O(GuardRefs, ND, SUnk, E) \ @@ -223,8 +215,7 @@ O(ExtendsClass, D(Bool), S(Cls) C(Cls), C) \ O(InstanceOf, D(Bool), S(Cls) S(Cls) C(Bool), C|N) \ O(IsTypeMem, D(Bool), S(PtrToGen), NA) \ O(IsNTypeMem, D(Bool), S(PtrToGen), NA) \ - \ - /* TODO(#2058842): order currently matters for the 'query ops' here */ \ +/* name dstinfo srcinfo flags */ \ O(OpGt, D(Bool), S(Gen) S(Gen), C|N) \ O(OpGte, D(Bool), S(Gen) S(Gen), C|N) \ O(OpLt, D(Bool), S(Gen) S(Gen), C|N) \ @@ -250,12 +241,25 @@ O(JmpInstanceOfBitmask, D(None), S(Cls) CStr, E) \ O(JmpNInstanceOfBitmask, D(None), S(Cls) CStr, E) \ O(JmpIsType, D(None), SUnk, E) \ O(JmpIsNType, D(None), SUnk, E) \ - /* TODO(#2058842) keep preceeding conditional branches contiguous */ \ - \ /* name dstinfo srcinfo flags */ \ O(JmpZero, D(None), SNum, E) \ O(JmpNZero, D(None), SNum, E) \ O(Jmp_, D(None), SUnk, T|E) \ +O(ReqBindJmpGt, ND, S(Gen) S(Gen), T|E) \ +O(ReqBindJmpGte, ND, S(Gen) S(Gen), T|E) \ +O(ReqBindJmpLt, ND, S(Gen) S(Gen), T|E) \ +O(ReqBindJmpLte, ND, S(Gen) S(Gen), T|E) \ +O(ReqBindJmpEq, ND, S(Gen) S(Gen), T|E) \ +O(ReqBindJmpNeq, ND, S(Gen) S(Gen), T|E) \ +O(ReqBindJmpSame, ND, S(Gen) S(Gen), T|E) \ +O(ReqBindJmpNSame, ND, S(Gen) S(Gen), T|E) \ +O(ReqBindJmpInstanceOfBitmask, ND, S(Cls) CStr, T|E) \ +O(ReqBindJmpNInstanceOfBitmask, ND, S(Cls) CStr, T|E) \ +O(ReqBindJmpZero, ND, SNum, T|E) \ +O(ReqBindJmpNZero, ND, SNum, T|E) \ +O(SideExitGuardLoc, ND, S(FramePtr), E) \ +O(SideExitGuardStk, D(StkPtr), S(StkPtr), E) \ +/* name dstinfo srcinfo flags */ \ O(JmpIndirect, ND, S(TCA), T|E) \ O(ExitWhenSurprised, ND, NA, E) \ O(ExitOnVarEnv, ND, S(FramePtr), E) \ @@ -358,12 +362,11 @@ O(SpillFrame, D(StkPtr), S(StkPtr) \ S(Func,FuncCls,FuncCtx,Null) \ S(Ctx,Cls,InitNull), CRc) \ O(ExceptionBarrier, D(StkPtr), S(StkPtr), E) \ -O(ExitTrace, ND, SUnk, T|E) \ -O(ExitTraceCc, ND, SUnk, T|E) \ -O(ExitSlow, ND, SUnk, T|E) \ -O(ExitSlowNoProgress, ND, SUnk, T|E) \ -O(ExitGuardFailure, ND, SUnk, T|E) \ -O(SyncVMRegs, ND, S(FramePtr) S(StkPtr), E) \ +O(ReqBindJmp, ND, NA, T|E) \ +O(ReqBindJmpNoIR, ND, NA, T|E) \ +O(ReqRetranslateNoIR, ND, NA, T|E) \ +O(ReqRetranslate, ND, NA, T|E) \ +O(SyncABIRegs, ND, S(FramePtr) S(StkPtr), E) \ O(Mov, DofS(0), SUnk, C|P) \ O(LdAddr, DofS(0), SUnk, C) \ O(IncRef, DofS(0), S(Gen), Mem|PRc|P) \ @@ -743,14 +746,29 @@ struct EdgeData : IRExtraData { }; /* - * ExitData contains the address of a jmp instruction we can smash later - * if we start a new tracelet at this exit point. + * Information for the REQ_BIND_JMPCC stubs we create when a tracelet + * ends with conditional jumps. */ -struct ExitData : IRExtraData { - explicit ExitData(IRInstruction* toSmash) : toSmash(toSmash) {} - IRInstruction* toSmash; +struct ReqBindJccData : IRExtraData { + Offset taken; + Offset notTaken; - std::string show() const; + std::string show() const { + return folly::to(taken, ',', notTaken); + } +}; + +/* + * Information for a conditional side exit based on a type check of a + * local or stack cell. + */ +struct SideExitGuardData : IRExtraData { + uint32_t checkedSlot; + Offset taken; + + std::string show() const { + return folly::to(checkedSlot, ',', taken); + } }; /* @@ -779,6 +797,15 @@ struct StackOffset : IRExtraData { int32_t offset; }; +/* + * Bytecode offsets. + */ +struct BCOffset : IRExtraData { + explicit BCOffset(Offset offset) : offset(offset) {} + std::string show() const { return folly::to(offset); } + Offset offset; +}; + /* * DefInlineFP is present when we need to create a frame for inlining. * This instruction also carries some metadata used by tracebuilder to @@ -808,7 +835,6 @@ struct CallArrayData : IRExtraData { Offset pc, after; }; - ////////////////////////////////////////////////////////////////////// #define X(op, data) \ @@ -817,37 +843,54 @@ struct CallArrayData : IRExtraData { static_assert(boost::has_trivial_destructor::value, \ "IR extra data type must be trivially destructible") -X(JmpSwitchDest, JmpSwitchData); -X(LdSSwitchDestFast, LdSSwitchData); -X(LdSSwitchDestSlow, LdSSwitchData); -X(Marker, MarkerData); -X(RaiseUninitLoc, LocalId); -X(GuardLoc, LocalId); -X(AssertLoc, LocalId); -X(OverrideLoc, LocalId); -X(LdLocAddr, LocalId); -X(DecRefLoc, LocalId); -X(LdLoc, LocalId); -X(StLoc, LocalId); -X(StLocNT, LocalId); -X(DefConst, ConstData); -X(LdConst, ConstData); -X(Jmp_, EdgeData); -X(ExitTrace, ExitData); -X(ExitTraceCc, ExitData); -X(SpillFrame, ActRecInfo); -X(GuardStk, StackOffset); -X(CastStk, StackOffset); -X(AssertStk, StackOffset); -X(ReDefSP, StackOffset); -X(ReDefGeneratorSP, StackOffset); -X(DefSP, StackOffset); -X(LdStack, StackOffset); -X(LdStackAddr, StackOffset); -X(DecRefStack, StackOffset); -X(DefInlineFP, DefInlineFPData); -X(InlineCreateCont, CreateContData); -X(CallArray, CallArrayData); +X(JmpSwitchDest, JmpSwitchData); +X(LdSSwitchDestFast, LdSSwitchData); +X(LdSSwitchDestSlow, LdSSwitchData); +X(Marker, MarkerData); +X(RaiseUninitLoc, LocalId); +X(GuardLoc, LocalId); +X(CheckLoc, LocalId); +X(AssertLoc, LocalId); +X(OverrideLoc, LocalId); +X(LdLocAddr, LocalId); +X(DecRefLoc, LocalId); +X(LdLoc, LocalId); +X(StLoc, LocalId); +X(StLocNT, LocalId); +X(DefConst, ConstData); +X(LdConst, ConstData); +X(Jmp_, EdgeData); +X(SpillFrame, ActRecInfo); +X(GuardStk, StackOffset); +X(CheckStk, StackOffset); +X(CastStk, StackOffset); +X(AssertStk, StackOffset); +X(ReDefSP, StackOffset); +X(ReDefGeneratorSP, StackOffset); +X(DefSP, StackOffset); +X(LdStack, StackOffset); +X(LdStackAddr, StackOffset); +X(DecRefStack, StackOffset); +X(DefInlineFP, DefInlineFPData); +X(ReqBindJmp, BCOffset); +X(ReqBindJmpNoIR, BCOffset); +X(ReqRetranslateNoIR, BCOffset); +X(InlineCreateCont, CreateContData); +X(CallArray, CallArrayData); +X(ReqBindJmpGt, ReqBindJccData); +X(ReqBindJmpGte, ReqBindJccData); +X(ReqBindJmpLt, ReqBindJccData); +X(ReqBindJmpLte, ReqBindJccData); +X(ReqBindJmpEq, ReqBindJccData); +X(ReqBindJmpNeq, ReqBindJccData); +X(ReqBindJmpSame, ReqBindJccData); +X(ReqBindJmpNSame, ReqBindJccData); +X(ReqBindJmpInstanceOfBitmask, ReqBindJccData); +X(ReqBindJmpNInstanceOfBitmask, ReqBindJccData); +X(ReqBindJmpZero, ReqBindJccData); +X(ReqBindJmpNZero, ReqBindJccData); +X(SideExitGuardLoc, SideExitGuardData); +X(SideExitGuardStk, SideExitGuardData); #undef X @@ -885,103 +928,57 @@ std::string showExtra(Opcode opc, const IRExtraData* data); ////////////////////////////////////////////////////////////////////// -inline bool isCmpOp(Opcode opc) { - return (opc >= OpGt && opc <= OpNSame); -} +/* + * A "query op" is any instruction returning Type::Bool that is both + * branch-fusable and negateable. + */ +bool isQueryOp(Opcode opc); -// A "query op" is any instruction returning Type::Bool that is both -// branch-fusable and negateable. -inline bool isQueryOp(Opcode opc) { - return (opc >= OpGt && opc <= IsNType); -} +/* + * A "cmp ops" is query op that takes exactly two arguments of type + * Gen. + */ +bool isCmpOp(Opcode opc); -inline Opcode queryToJmpOp(Opcode opc) { - assert(isQueryOp(opc)); - return (Opcode)(JmpGt + (opc - OpGt)); -} +/* + * A "query jump op" is a conditional jump instruction that + * corresponds to one of the query op instructions. + */ +bool isQueryJmpOp(Opcode opc); -inline bool isQueryJmpOp(Opcode opc) { - switch (opc) { - case JmpGt: - case JmpGte: - case JmpLt: - case JmpLte: - case JmpEq: - case JmpNeq: - case JmpSame: - case JmpNSame: - case JmpInstanceOfBitmask: - case JmpNInstanceOfBitmask: - case JmpIsType: - case JmpIsNType: - case JmpZero: - case JmpNZero: - return true; - default: - return false; - } -} +/* + * Translate a query op into a conditional jump that does the same + * test (a "query jump op"). + * + * Pre: isQueryOp(opc) + */ +Opcode queryToJmpOp(Opcode opc); -inline Opcode queryJmpToQueryOp(Opcode opc) { - assert(isQueryJmpOp(opc)); - assert(opc != JmpZero && opc != JmpNZero); - return Opcode(OpGt + (opc - JmpGt)); -} +/* + * Translate a "query jump op" to a query op. + * + * Pre: isQueryJmpOp(opc); + */ +Opcode queryJmpToQueryOp(Opcode opc); -inline ConditionCode queryJmpToCC(Opcode opc) { - assert(isQueryJmpOp(opc)); - - using namespace HPHP::Transl; - - switch (opc) { - case JmpGt: return CC_G; - case JmpGte: return CC_GE; - case JmpLt: return CC_L; - case JmpLte: return CC_LE; - case JmpEq: return CC_E; - case JmpNeq: return CC_NE; - case JmpSame: return CC_E; - case JmpNSame: return CC_NE; - case JmpInstanceOfBitmask: return CC_NZ; - case JmpNInstanceOfBitmask: return CC_Z; - case JmpIsType: return CC_NZ; - case JmpIsNType: return CC_Z; - case JmpZero: return CC_Z; - case JmpNZero: return CC_NZ; - default: - not_reached(); - } -} +/* + * Convert a jump operation to its corresponding conditional + * ReqBindJmp. + * + * Pre: opc is a conditional jump. + */ +Opcode jmpToReqBindJmp(Opcode opc); /* * Return the opcode that corresponds to negation of opc. */ Opcode negateQueryOp(Opcode opc); -extern Opcode queryCommuteTable[]; - -inline Opcode commuteQueryOp(Opcode opc) { - assert(opc >= OpGt && opc <= OpNSame); - return queryCommuteTable[opc - OpGt]; -} - -namespace TraceExitType { -// Must update in sync with ExitTrace entries in OPC table above -enum ExitType { - Normal, - NormalCc, - Slow, - SlowNoProgress, - GuardFailure, -}; -} - -TraceExitType::ExitType getExitType(Opcode opc); -Opcode getExitOpcode(TraceExitType::ExitType); - -inline bool isExitSlow(TraceExitType::ExitType t) { - return t == TraceExitType::Slow || t == TraceExitType::SlowNoProgress; -} +/* + * Return the opcode that corresponds to commuting the arguments of + * opc. + */ +Opcode commuteQueryOp(Opcode opc); const char* opcodeName(Opcode opcode); @@ -1655,7 +1652,6 @@ struct IRInstruction { , m_dst(nullptr) , m_taken(nullptr) , m_block(nullptr) - , m_tca(nullptr) , m_extra(nullptr) {} @@ -1824,9 +1820,6 @@ struct IRInstruction { m_dst = newDsts; } - TCA getTCA() const { return m_tca; } - void setTCA(TCA newTCA) { m_tca = newTCA; } - /* * Instruction id is stable and useful as an array index. */ @@ -1906,7 +1899,6 @@ private: SSATmp* m_dst; // if HasDest or NaryDest Block* m_taken; // for branches, guards, and jmp Block* m_block; // block that owns this instruction - TCA m_tca; IRExtraData* m_extra; public: boost::intrusive::list_member_hook<> m_listNode; // for InstructionList @@ -2090,7 +2082,7 @@ private: typedef folly::Range TcaRange; -/** +/* * A Block refers to a basic block: single-entry, single-exit, list of * instructions. The instruction list is an intrusive list, so each * instruction can only be in one block at a time. Likewise, a block @@ -2108,8 +2100,13 @@ struct Block : boost::noncopyable { enum Hint { Neither, Likely, Unlikely }; Block(unsigned id, const Func* func, IRInstruction* label) - : m_trace(nullptr), m_func(func), m_next(nullptr), m_id(id) - , m_preds(nullptr), m_hint(Neither) { + : m_trace(nullptr) + , m_func(func) + , m_next(nullptr) + , m_id(id) + , m_preds(nullptr) + , m_hint(Neither) + { push_back(label); } diff --git a/hphp/runtime/vm/translator/hopt/irtranslator.cpp b/hphp/runtime/vm/translator/hopt/irtranslator.cpp index d6e661a1e..c89447818 100644 --- a/hphp/runtime/vm/translator/hopt/irtranslator.cpp +++ b/hphp/runtime/vm/translator/hopt/irtranslator.cpp @@ -122,22 +122,26 @@ TranslatorX64::irCheckType(X64Assembler& a, // items out of BBs we truncate; they don't need guards. if (rtt.isVagueValue()) return; - if (l.space == Location::Stack) { - // tx64LocPhysicalOffset returns: - // negative offsets for locals accessed via rVmFp - // positive offsets for stack values, relative to rVmSp - uint32_t stackOffset = tx64LocPhysicalOffset(l); - m_hhbcTrans->guardTypeStack(stackOffset, JIT::Type::fromRuntimeType(rtt)); - } else if (l.space == Location::Local){ - // Convert negative offset to a positive offset for convenience - m_hhbcTrans->guardTypeLocal(l.offset, JIT::Type::fromRuntimeType(rtt)); - } else if (l.space == Location::Iter) { - assert(false); // should not happen - } else { - HHIR_UNIMPLEMENTED(Invalid_space); - } + switch (l.space) { + case Location::Stack: + { + uint32_t stackOffset = tx64LocPhysicalOffset(l); + m_hhbcTrans->guardTypeStack(stackOffset, + JIT::Type::fromRuntimeType(rtt)); + } + break; - return; + case Location::Local: + m_hhbcTrans->guardTypeLocal(l.offset, JIT::Type::fromRuntimeType(rtt)); + break; + + case Location::Iter: + case Location::Invalid: + case Location::Litstr: + case Location::Litint: + case Location::This: + assert(false); // should not happen + } } void @@ -228,6 +232,8 @@ TranslatorX64::irTranslateBranchOp(const Tracelet& t, const NormalizedInstruction& i) { const Opcode op = i.op(); assert(op == OpJmpZ || op == OpJmpNZ); + assert(!i.next); + if (op == OpJmpZ) { HHIR_EMIT(JmpZ, i.offset() + i.imm[0].u_BA); } else { @@ -1758,7 +1764,6 @@ TranslatorX64::irTranslateTracelet(Tracelet& t, emitRB(a, RBTypeTraceletBody, t.m_sk); Stats::emitInc(a, Stats::Instr_TC, t.m_numOpcodes); recordBCInstr(OpTraceletGuard, a, start); - m_hhbcTrans->setBcOffNextTrace(t.m_nextSk.offset()); // Profiling on function entry. if (m_curTrace->m_sk.offset() == curFunc()->base()) { @@ -1820,7 +1825,7 @@ TranslatorX64::irTranslateTracelet(Tracelet& t, if (ni->breaksTracelet) break; } - hhirTraceEnd(t.m_nextSk.offset()); + hhirTraceEnd(); if (transResult != Retry) { try { transResult = Success; @@ -1892,7 +1897,8 @@ TranslatorX64::irTranslateTracelet(Tracelet& t, return transResult; } -void TranslatorX64::hhirTraceStart(Offset bcStartOffset) { +void TranslatorX64::hhirTraceStart(Offset bcStartOffset, + Offset nextTraceletOffset) { assert(!m_irFactory); assert(m_useHHIR); @@ -1908,12 +1914,12 @@ void TranslatorX64::hhirTraceStart(Offset bcStartOffset) { m_useHHIR = true; m_irFactory.reset(new JIT::IRFactory()); m_hhbcTrans.reset(new JIT::HhbcTranslator( - *m_irFactory, bcStartOffset, fp - vmsp(), curFunc())); + *m_irFactory, bcStartOffset, nextTraceletOffset, fp - vmsp(), curFunc())); } -void TranslatorX64::hhirTraceEnd(Offset bcSuccOffset) { +void TranslatorX64::hhirTraceEnd() { assert(m_useHHIR); - m_hhbcTrans->end(bcSuccOffset); + m_hhbcTrans->end(); FTRACE(1, "{}{:-^40}{}\n", color(ANSI_COLOR_BLACK, ANSI_BGCOLOR_GREEN), "", diff --git a/hphp/runtime/vm/translator/hopt/jumpsopts.cpp b/hphp/runtime/vm/translator/hopt/jumpsopts.cpp index 96255321c..feee695b2 100644 --- a/hphp/runtime/vm/translator/hopt/jumpsopts.cpp +++ b/hphp/runtime/vm/translator/hopt/jumpsopts.cpp @@ -14,23 +14,26 @@ +----------------------------------------------------------------------+ */ +#include + +#include + #include "hphp/runtime/vm/translator/hopt/ir.h" #include "hphp/runtime/vm/translator/hopt/opt.h" #include "hphp/runtime/vm/translator/hopt/irfactory.h" namespace HPHP { namespace JIT { -// These are the conditional branches supported for direct branch -// to their target trace at TraceExit, TraceExitType::NormalCc -static bool jccCanBeDirectExit(Opcode opc) { - return isQueryJmpOp(opc) && (opc != JmpIsType) && (opc != JmpIsNType); - // TODO(#2053369): JmpIsType, etc -} +TRACE_SET_MOD(hhir); + +namespace { + +////////////////////////////////////////////////////////////////////// // If main trace ends with an unconditional jump, and the target is not // reached by any other branch, then copy the target of the jump to the // end of the trace -static void elimUnconditionalJump(Trace* trace, IRFactory* irFactory) { +void elimUnconditionalJump(Trace* trace, IRFactory* irFactory) { boost::dynamic_bitset<> isJoin(irFactory->numBlocks()); boost::dynamic_bitset<> havePred(irFactory->numBlocks()); for (Block* block : trace->getBlocks()) { @@ -55,140 +58,177 @@ static void elimUnconditionalJump(Trace* trace, IRFactory* irFactory) { } } -/** - * If main trace ends with a conditional jump with no side-effects on exit, - * hook it to the exitTrace and make it a TraceExitType::NormalCc. - * - * This function essentially looks for the following code pattern: - * - * Main Trace: - * ---------- - * L1: // jccBlock - * ... - * Jcc ... -> L3 - * L2: // lastBlock - * DefLabel - * [Marker] - * ExitTrace - * - * Exit Trace: - * ---------- - * L3: // targetBlock - * DefLabel - * [Marker] - * ExitTraceCc - * - * If the pattern is found, Jcc's dst operand is linked to the ExitTrace and - * ExitTraceCc instructions and it's flagged with kIRDirectJccJmpActive. This - * then triggers CodeGenerator to emit a REQ_BIND_JMPCC_FIRST service request. +Block* findMainExitBlock(Trace* trace, IRFactory* irFactory) { + assert(trace->isMain()); + auto const back = trace->back(); + + /* + * We require the invariant that the main trace exit comes last in + * the main trace block list. Right now this is always the case, + * but this assertion is here in case we want to make changes that + * affect this ordering. (If we do want to change it, we could use + * something like the assert below to find the main exit.) + */ + if (debug) { + auto const sorted = sortCfg(trace, *irFactory); + auto it = sorted.rbegin(); + while (it != sorted.rend() && !(*it)->isMain()) { + ++it; + } + assert(it != sorted.rend()); + assert(*it == back && "jumpopts invariant violated"); + } + + return back; +} + +/* + * Utility class for pattern matching the instructions in a Block, + * ignoring markers and the label. * + * To use, create a BlockMatcher and call match with a variable-length + * list of opcode ids. */ -static void hoistConditionalJumps(Trace* trace, IRFactory* irFactory) { - IRInstruction* exitInst = nullptr; - IRInstruction* exitCcInst = nullptr; - Opcode opc = OpAdd; - // Normally Jcc comes before a Marker - auto& blocks = trace->getBlocks(); - if (blocks.size() < 2) return; - auto it = blocks.end(); - Block* lastBlock = *(--it); - Block* jccBlock = *(--it); +struct BlockMatcher { + explicit BlockMatcher(Block* block) + : m_block(block) + , m_it(block->skipLabel()) + {} - IRInstruction& jccInst = *(jccBlock->back()); - if (!jccCanBeDirectExit(jccInst.op())) return; + bool match() { return true; } - for (auto it = lastBlock->skipLabel(), end = lastBlock->end(); it != end; - it++) { - IRInstruction& inst = *it; - opc = inst.op(); - if (opc == ExitTrace) { - exitInst = &inst; - break; - } - if (opc != Marker) { - // Found real instruction on the last block - return; - } + template + bool match(Opcode op, Opcodes... opcs) { + while (m_it != m_block->end() && m_it->op() == Marker) ++m_it; + if (m_it == m_block->end()) return false; + auto const cur = m_it->op(); + ++m_it; + return cur == op && match(opcs...); } - if (exitInst) { - Block* targetBlock = jccInst.getTaken(); - auto targetInstIter = targetBlock->skipLabel(); - // Check for a NormalCc exit with no side effects - for (auto it = targetInstIter, end = targetBlock->end(); it != end; ++it) { - IRInstruction* instr = &*it; - // Extend to support ExitSlow, ExitSlowNoProgress, ... - Opcode opc = instr->op(); - if (opc == ExitTraceCc) { - exitCcInst = instr; - break; - } else if (opc != Marker) { - // Do not optimize if there are other instructions - break; - } - } +private: + Block* m_block; + Block::const_iterator m_it; +}; - if (exitCcInst) { - // Found both exits, link them to Jcc for codegen - ExitData* exitData = new (irFactory->arena()) ExitData(&jccInst); - exitCcInst->setExtra(exitData); - exitInst->setExtra(exitData); - // Set flag so Jcc and exits know this is active - jccInst.setTCA(kIRDirectJccJmpActive); - } - } +/* + * Returns whether the supplied block is a "normal" trace exit. + * + * That is, it does nothing other than sync ABI registers and bind to + * the next tracelet. + */ +bool isNormalExit(Block* block) { + return BlockMatcher(block).match(SyncABIRegs, ReqBindJmp); } -// If main trace starts with guards, have them generate a patchable jump -// to the anchor trace -static void hoistGuardJumps(Trace* trace, IRFactory* irFactory) { - Block* guardLabel = nullptr; - // Check the beginning of the trace for guards - for (Block* block : trace->getBlocks()) { - for (IRInstruction& instr : *block) { - IRInstruction* inst = &instr; - Opcode opc = inst->op(); - if (inst->getTaken() && - (opc == LdLoc || opc == LdStack || - opc == GuardLoc || opc == GuardStk)) { - Block* exitLabel = inst->getTaken(); - // Find the GuardFailure's label and confirm this branches there - if (!guardLabel && exitLabel->getTrace() != trace) { - auto instIter = exitLabel->skipLabel(); - // Confirm this is a GuardExit - for (auto it = instIter, end = exitLabel->end(); it != end; ++it) { - Opcode op = it->op(); - if (op == Marker) { - continue; - } - if (op == ExitGuardFailure) { - guardLabel = exitLabel; - } - // Do not optimize if other instructions are on exit trace - break; - } - } - if (exitLabel == guardLabel) { - inst->setTCA(kIRDirectGuardActive); - continue; - } - return; // terminate search - } - if (opc == Marker || opc == DefLabel || opc == DefSP || opc == DefFP || - opc == LdStack) { - continue; - } - return; // terminate search - } - } +// Returns whether `opc' is a within-tracelet conditional jump that +// can be folded into a ReqBindJmpFoo instruction. +bool jccCanBeDirectExit(Opcode opc) { + return isQueryJmpOp(opc) && (opc != JmpIsType) && (opc != JmpIsNType); + // TODO(#2404341) } +/* + * If main trace ends with a conditional jump with no side-effects on + * exit, followed by the normal ReqBindJmp sequence, convert the whole + * thing into a conditional ReqBindJmp. + * + * This leads to more efficient code because the service request stubs + * will patch jumps in the main trace instead of off-trace. + */ +void optimizeCondTraceExit(Trace* trace, IRFactory* irFactory) { + FTRACE(5, "CondExit:vvvvvvvvvvvvvvvvvvvvv\n"); + SCOPE_EXIT { FTRACE(5, "CondExit:^^^^^^^^^^^^^^^^^^^^^\n"); }; + + auto const sortedBlocks = sortCfg(trace, *irFactory); + auto const preds = computePredecessors(sortedBlocks); + + auto const mainExit = findMainExitBlock(trace, irFactory); + if (!isNormalExit(mainExit)) return; + + auto const mainPreds = preds[mainExit->postId()]; + if (mainPreds.size() != 1) return; + + auto const jccBlock = mainPreds[0]; + if (!jccCanBeDirectExit(jccBlock->back()->op())) return; + FTRACE(5, "previous block ends with jccCanBeDirectExit ({})\n", + opcodeName(jccBlock->back()->op())); + + auto const jccInst = jccBlock->back(); + auto const jccExitTrace = jccInst->getTaken(); + if (!isNormalExit(jccExitTrace)) return; + FTRACE(5, "exit trace is side-effect free\n"); + + auto const newOpcode = jmpToReqBindJmp(jccBlock->back()->op()); + ReqBindJccData data; + data.taken = jccExitTrace->back()->getExtra()->offset; + data.notTaken = mainExit->back()->getExtra()->offset; + + FTRACE(5, "replacing {} with {}\n", jccInst->getId(), opcodeName(newOpcode)); + irFactory->replace( + mainExit->back(), + newOpcode, + data, + std::make_pair(jccInst->getNumSrcs(), jccInst->getSrcs().begin()) + ); + + jccInst->convertToNop(); +} + +/* + * Look for CheckStk/CheckLoc instructions in the main trace that + * branch to "normal exits". We can optimize these into the + * SideExitGuard* instructions that can be patched in place. + */ +void optimizeSideExits(Trace* trace, IRFactory* irFactory) { + FTRACE(5, "SideExit:vvvvvvvvvvvvvvvvvvvvv\n"); + SCOPE_EXIT { FTRACE(5, "SideExit:^^^^^^^^^^^^^^^^^^^^^\n"); }; + + forEachInst(trace, [&] (IRInstruction* inst) { + if (inst->op() != CheckStk && inst->op() != CheckLoc) return; + auto const exitBlock = inst->getTaken(); + if (!isNormalExit(exitBlock)) return; + + auto const syncABI = &*boost::prior(exitBlock->backIter()); + assert(syncABI->op() == SyncABIRegs); + + FTRACE(5, "converting jump ({}) to side exit\n", + inst->getId()); + + auto const isStack = inst->op() == CheckStk; + auto const fp = syncABI->getSrc(0); + auto const sp = syncABI->getSrc(1); + + SideExitGuardData data; + data.checkedSlot = isStack + ? inst->getExtra()->offset + : inst->getExtra()->locId; + data.taken = exitBlock->back()->getExtra()->offset; + + auto const block = inst->getBlock(); + block->insert(block->iteratorTo(inst), + irFactory->cloneInstruction(syncABI)); + + irFactory->replace( + inst, + isStack ? SideExitGuardStk : SideExitGuardLoc, + inst->getTypeParam(), + data, + isStack ? sp : fp + ); + }); +} + +} + +////////////////////////////////////////////////////////////////////// + void optimizeJumps(Trace* trace, IRFactory* irFactory) { elimUnconditionalJump(trace, irFactory); if (RuntimeOption::EvalHHIRDirectExit) { - hoistConditionalJumps(trace, irFactory); - hoistGuardJumps(trace, irFactory); + optimizeCondTraceExit(trace, irFactory); + optimizeSideExits(trace, irFactory); } } diff --git a/hphp/runtime/vm/translator/hopt/linearscan.cpp b/hphp/runtime/vm/translator/hopt/linearscan.cpp index cff37fb48..dfbaa0f7b 100644 --- a/hphp/runtime/vm/translator/hopt/linearscan.cpp +++ b/hphp/runtime/vm/translator/hopt/linearscan.cpp @@ -405,9 +405,11 @@ void LinearScan::allocRegToInstruction(InstructionList::iterator it) { opc == RetAdjustStack || opc == InterpOne || opc == GenericRetDecRefs || + opc == CheckStk || opc == GuardStk || opc == AssertStk || opc == CastStk || + opc == SideExitGuardStk || VectorEffects::supported(opc)); allocRegToTmp(&m_regs[int(rVmSp)], &dst, 0); continue; diff --git a/hphp/runtime/vm/translator/hopt/opt.cpp b/hphp/runtime/vm/translator/hopt/opt.cpp index 7838ec431..b61fe3c37 100644 --- a/hphp/runtime/vm/translator/hopt/opt.cpp +++ b/hphp/runtime/vm/translator/hopt/opt.cpp @@ -106,33 +106,34 @@ void optimizeTrace(Trace* trace, TraceBuilder* traceBuilder) { if (debug) forEachTraceInst(trace, assertOperandTypes); }; - if (RuntimeOption::EvalHHIRMemOpt) { + auto dcePass = [&](const char* which) { + if (!RuntimeOption::EvalHHIRDeadCodeElim) return; + eliminateDeadCode(trace, irFactory); + finishPass(folly::format("after {} DCE", which).str().c_str()); + }; + + if (false && RuntimeOption::EvalHHIRMemOpt) { optimizeMemoryAccesses(trace, irFactory); finishPass("after MemeLim"); } - if (RuntimeOption::EvalHHIRDeadCodeElim) { - eliminateDeadCode(trace, irFactory); - finishPass("after DCE"); - } + dcePass("initial"); if (RuntimeOption::EvalHHIRExtraOptPass && (RuntimeOption::EvalHHIRCse || RuntimeOption::EvalHHIRSimplification)) { traceBuilder->reoptimize(); - finishPass("after CSE/Simplification"); + finishPass("after reoptimize"); // Cleanup any dead code left around by CSE/Simplification // Ideally, this would be controlled by a flag returned // by optimzeTrace indicating whether DCE is necessary - if (RuntimeOption::EvalHHIRDeadCodeElim) { - eliminateDeadCode(trace, irFactory); - finishPass("after DCE"); - } + dcePass("reoptimize"); } if (RuntimeOption::EvalHHIRJumpOpts) { optimizeJumps(trace, irFactory); finishPass("jump opts"); + dcePass("jump opts"); } if (RuntimeOption::EvalHHIRGenerateAsserts) { diff --git a/hphp/runtime/vm/translator/hopt/print.cpp b/hphp/runtime/vm/translator/hopt/print.cpp index 502a18bc2..5526a2343 100644 --- a/hphp/runtime/vm/translator/hopt/print.cpp +++ b/hphp/runtime/vm/translator/hopt/print.cpp @@ -153,22 +153,6 @@ void print(std::ostream& ostream, const IRInstruction* inst, ostream << punc(" -> "); printLabel(ostream, taken); } - - if (TCA tca = inst->getTCA()) { - ostream << punc(", "); - if (tca == kIRDirectJccJmpActive) { - ostream << "JccJmp_Exit "; - } - else if (tca == kIRDirectJccActive) { - ostream << "Jcc_Exit "; - } - else if (tca == kIRDirectGuardActive) { - ostream << "Guard_Exit "; - } - else { - ostream << (void*)tca; - } - } } void print(const IRInstruction* inst) { diff --git a/hphp/runtime/vm/translator/hopt/simplifier.cpp b/hphp/runtime/vm/translator/hopt/simplifier.cpp index 2902f7e98..cb11f3951 100644 --- a/hphp/runtime/vm/translator/hopt/simplifier.cpp +++ b/hphp/runtime/vm/translator/hopt/simplifier.cpp @@ -49,10 +49,15 @@ StackValueInfo getStackValue(SSATmp* sp, uint32_t index) { case ExceptionBarrier: return getStackValue(inst->getSrc(0), index); + case SideExitGuardStk: + always_assert(0 && "simplifier is not tested for running after jumpopts"); + case AssertStk: // fallthrough case CastStk: // fallthrough + case CheckStk: + // fallthrough case GuardStk: // We don't have a value, but we may know the type due to guarding // on it. @@ -305,7 +310,7 @@ SSATmp* Simplifier::simplify(IRInstruction* inst) { case DecRefNZOrBranch: case DecRefNZ: return simplifyDecRef(inst); case IncRef: return simplifyIncRef(inst); - case GuardType: return simplifyGuardType(inst); + case CheckType: return simplifyCheckType(inst); case LdCls: return simplifyLdCls(inst); case LdThis: return simplifyLdThis(inst); @@ -500,7 +505,7 @@ SSATmp* Simplifier::simplifyLdCls(IRInstruction* inst) { return nullptr; } -SSATmp* Simplifier::simplifyGuardType(IRInstruction* inst) { +SSATmp* Simplifier::simplifyCheckType(IRInstruction* inst) { Type type = inst->getTypeParam(); SSATmp* src = inst->getSrc(0); Type srcType = src->type(); @@ -1595,9 +1600,9 @@ SSATmp* Simplifier::simplifyCondJmp(IRInstruction* inst) { inst->op() == JmpZero ? negateQueryOp(srcOpcode) : srcOpcode), - srcInst->getTypeParam(), // if it had a type param - inst->getTaken(), - std::make_pair(ssas.size(), ssas.begin()) + srcInst->getTypeParam(), // if it had a type param + inst->getTaken(), + std::make_pair(ssas.size(), ssas.begin()) ); } diff --git a/hphp/runtime/vm/translator/hopt/simplifier.h b/hphp/runtime/vm/translator/hopt/simplifier.h index b68cc5c5d..e871a5da5 100644 --- a/hphp/runtime/vm/translator/hopt/simplifier.h +++ b/hphp/runtime/vm/translator/hopt/simplifier.h @@ -106,7 +106,7 @@ private: SSATmp* simplifyPrint(IRInstruction* inst); SSATmp* simplifyDecRef(IRInstruction* inst); SSATmp* simplifyIncRef(IRInstruction* inst); - SSATmp* simplifyGuardType(IRInstruction* inst); + SSATmp* simplifyCheckType(IRInstruction* inst); SSATmp* simplifyLdThis(IRInstruction*); SSATmp* simplifyLdCls(IRInstruction* inst); SSATmp* simplifyLdClsPropAddr(IRInstruction*); diff --git a/hphp/runtime/vm/translator/hopt/tracebuilder.cpp b/hphp/runtime/vm/translator/hopt/tracebuilder.cpp index 78b470836..83238fd64 100644 --- a/hphp/runtime/vm/translator/hopt/tracebuilder.cpp +++ b/hphp/runtime/vm/translator/hopt/tracebuilder.cpp @@ -88,115 +88,6 @@ bool TraceBuilder::isValueAvailable(SSATmp* tmp) const { } } -/* - * Code generation support for side exits. - * There are 3 types of side exits as defined by the ExitType enum: - * (1) Normal: Conditional or unconditional program branches - * that take you out of the trace. - * (2) Slow: branches to slow paths to handle rare and slow cases - * such as null check failures, warnings, fatals, or type guard - * failures in the middle of a trace. - * (3) GuardFailure: branches due to guard failures at the beginning - * of a trace. - */ - -Trace* TraceBuilder::genExitGuardFailure(uint32_t bcOff) { - Trace* trace = makeExitTrace(bcOff); - - MarkerData marker; - marker.bcOff = bcOff; - marker.stackOff = m_spOffset; - marker.func = m_curFunc->getValFunc(); - gen(Marker, marker); // goes on main trace - - SSATmp* pc = cns((int64_t)bcOff); - // TODO change exit trace to a control flow instruction that - // takes sp, fp, and a Marker as the target label instruction - trace->back()->push_back( - m_irFactory.gen(getExitOpcode(TraceExitType::GuardFailure), - m_curFunc, - pc, - m_spValue, - m_fpValue)); - return trace; -} - -/* - * getExitSlowTrace generates a target exit trace for - * TraceExitType::Slow branches. - */ -Trace* TraceBuilder::getExitSlowTrace(uint32_t bcOff, - int32_t stackDeficit, - uint32_t numOpnds, - SSATmp** opnds) { - // this is a newly created check with no label - TraceExitType::ExitType exitType = - bcOff == m_initialBcOff ? TraceExitType::SlowNoProgress - : TraceExitType::Slow; - return genExitTrace(bcOff, stackDeficit, numOpnds, opnds, exitType); - -} - -void TraceBuilder::genTraceEnd(uint32_t nextPc, - TraceExitType::ExitType exitType /* = Normal */) { - gen(getExitOpcode(TraceExitType::Normal), - m_curFunc, - cns(nextPc), - m_spValue, - m_fpValue); -} - -Trace* TraceBuilder::genExitTrace(uint32_t bcOff, - int32_t stackDeficit, - uint32_t numOpnds, - SSATmp* const* opnds, - TraceExitType::ExitType exitType, - uint32_t notTakenBcOff, - std::function - beforeExit) { - Trace* exitTrace = makeExitTrace(bcOff); - - MarkerData marker; - marker.bcOff = bcOff; - marker.stackOff = m_spOffset + numOpnds - stackDeficit; - marker.func = m_curFunc->getValFunc(); - exitTrace->back()->push_back(m_irFactory.gen(Marker, marker)); - - if (beforeExit) { - beforeExit(&m_irFactory, exitTrace); - } - SSATmp* sp = m_spValue; - if (numOpnds != 0 || stackDeficit != 0) { - SSATmp* srcs[numOpnds + 2]; - srcs[0] = m_spValue; - srcs[1] = cns(stackDeficit); - std::copy(opnds, opnds + numOpnds, srcs + 2); - - SSATmp** decayedPtr = srcs; - auto* spillInst = m_irFactory.gen( - SpillStack, - std::make_pair(numOpnds + 2, decayedPtr) - ); - sp = spillInst->getDst(); - exitTrace->back()->push_back(spillInst); - } - SSATmp* pc = cns(int64_t(bcOff)); - if (exitType == TraceExitType::NormalCc) { - assert(notTakenBcOff != 0); - SSATmp* notTakenPC = cns(notTakenBcOff); - genFor(exitTrace, getExitOpcode(exitType), - m_curFunc, - pc, sp, m_fpValue, - notTakenPC); - } else { - assert(notTakenBcOff == 0); - genFor(exitTrace, getExitOpcode(exitType), - m_curFunc, - pc, sp, m_fpValue); - } - return exitTrace; -} - SSATmp* TraceBuilder::genDefUninit() { return gen(DefConst, Type::Uninit, ConstData(0)); } @@ -318,6 +209,7 @@ void TraceBuilder::updateTrackedState(IRInstruction* inst) { case AssertStk: case CastStk: + case CheckStk: case GuardStk: case ExceptionBarrier: m_spValue = inst->getDst(); @@ -625,19 +517,21 @@ SSATmp* TraceBuilder::cseLookup(IRInstruction* inst) { ////////////////////////////////////////////////////////////////////// -SSATmp* TraceBuilder::preOptimizeGuardLoc(IRInstruction* inst) { - auto const locId = inst->getExtra()->locId; +SSATmp* TraceBuilder::preOptimizeCheckLoc(IRInstruction* inst) { + auto const locId = inst->getExtra()->locId; if (auto const prevValue = getLocalValue(locId)) { + always_assert(false && "WTF"); return gen( - GuardType, inst->getTypeParam(), inst->getTaken(), prevValue + CheckType, inst->getTypeParam(), inst->getTaken(), prevValue ); } auto const prevType = getLocalType(locId); if (prevType != Type::None) { - // It doesn't make sense to be guarding on something that's deemed - // to fail. + always_assert(false && "WTF2"); + // It doesn't make sense to be checking something that's deemed to + // fail. assert(prevType == inst->getTypeParam()); inst->convertToNop(); } @@ -802,7 +696,7 @@ SSATmp* TraceBuilder::preOptimizeStLoc(IRInstruction* inst) { SSATmp* TraceBuilder::preOptimize(IRInstruction* inst) { #define X(op) case op: return preOptimize##op(inst) switch (inst->op()) { - X(GuardLoc); + X(CheckLoc); X(AssertLoc); X(LdThis); X(LdCtx); diff --git a/hphp/runtime/vm/translator/hopt/tracebuilder.h b/hphp/runtime/vm/translator/hopt/tracebuilder.h index a1cb79ac3..d83ab723d 100644 --- a/hphp/runtime/vm/translator/hopt/tracebuilder.h +++ b/hphp/runtime/vm/translator/hopt/tracebuilder.h @@ -103,10 +103,6 @@ struct TraceBuilder { SSATmp* getSp() const { return m_spValue; } SSATmp* getFp() const { return m_fpValue; } - Trace* makeExitTrace(uint32_t bcOff) { - return m_trace->addExitTrace(makeTrace(m_curFunc->getValFunc(), - bcOff)); - } bool isThisAvailable() const { return m_thisIsAvailable; } @@ -136,12 +132,16 @@ struct TraceBuilder { /* * Create an IRInstruction, similar to gen(), except link it into * the Trace t instead of the current main trace. + * + * Also does not run optimization passes. + * + * TODO(#2404447): run simplifier? */ template - IRInstruction* genFor(Trace* t, Args... args) { + SSATmp* genFor(Trace* t, Args... args) { auto instr = m_irFactory.gen(args...); t->back()->push_back(instr); - return instr; + return instr->getDst(); } ////////////////////////////////////////////////////////////////////// @@ -176,8 +176,6 @@ struct TraceBuilder { ////////////////////////////////////////////////////////////////////// // control flow - typedef std::function ExitTraceCallback; - // hint the execution frequency of the current block void hint(Block::Hint h) const { m_trace->back()->setHint(h); @@ -225,22 +223,6 @@ struct TraceBuilder { appendBlock(done_block); } - /* - * ifThenExit produces a conditional exit with user-supplied logic - * if the exit is taken. - */ - Trace* ifThenExit(const Func* func, - int stackDeficit, - const std::vector &stackValues, - ExitTraceCallback exit, - Offset exitBcOff, - Offset bcOff) { - return genExitTrace(exitBcOff, stackDeficit, - stackValues.size(), - stackValues.size() ? &stackValues[0] : nullptr, - TraceExitType::NormalCc, bcOff /* notTakenOff */, exit); - } - /* * ifElse generates if-then-else blocks with an empty 'then' block * that do not produce values. Code emitted in the next lambda will @@ -257,37 +239,15 @@ struct TraceBuilder { appendBlock(done_block); } - Trace* getExitSlowTrace(uint32_t bcOff, - int32_t stackDeficit, - uint32_t numOpnds, - SSATmp** opnds); - /* - * Generates a trace exit that can be the target of a conditional - * or unconditional control flow instruction from the main trace. - * - * Lifetime of the returned pointer is managed by the trace this - * TraceBuilder is generating. + * Create a new "exit trace". This is a Trace that is assumed to be + * a cold path, which always exits the tracelet without control flow + * rejoining the main line. */ - Trace* genExitTrace(uint32_t bcOff, - int32_t stackDeficit, - uint32_t numOpnds, - SSATmp* const* opnds, - TraceExitType::ExitType, - uint32_t notTakenBcOff = 0, - ExitTraceCallback beforeExit = ExitTraceCallback()); - - /* - * Generates a target exit trace for GuardFailure exits. - * - * Lifetime of the returned pointer is managed by the trace this - * TraceBuilder is generating. - */ - Trace* genExitGuardFailure(uint32_t off); - - // generates the ExitTrace instruction at the end of a trace - void genTraceEnd(uint32_t nextPc, - TraceExitType::ExitType exitType = TraceExitType::Normal); + Trace* makeExitTrace(uint32_t bcOff) { + return m_trace->addExitTrace(makeTrace(m_curFunc->getValFunc(), + bcOff)); + } private: // RAII disable of CSE; only restores if it used to be on. Used for @@ -323,7 +283,7 @@ private: }; private: - SSATmp* preOptimizeGuardLoc(IRInstruction*); + SSATmp* preOptimizeCheckLoc(IRInstruction*); SSATmp* preOptimizeAssertLoc(IRInstruction*); SSATmp* preOptimizeLdThis(IRInstruction*); SSATmp* preOptimizeLdCtx(IRInstruction*); diff --git a/hphp/runtime/vm/translator/hopt/vectortranslator.cpp b/hphp/runtime/vm/translator/hopt/vectortranslator.cpp index 197114c50..6b7f0228c 100644 --- a/hphp/runtime/vm/translator/hopt/vectortranslator.cpp +++ b/hphp/runtime/vm/translator/hopt/vectortranslator.cpp @@ -2014,7 +2014,7 @@ void HhbcTranslator::VectorTranslator::emitMPost() { assert(spillValues.back() == m_predictedResult); spillValues.back() = m_result; - gen(GuardType, + gen(CheckType, m_predictedResult->type(), m_ht.getExitTrace(m_ht.getNextSrcKey().offset(), spillValues), m_result); diff --git a/hphp/runtime/vm/translator/srcdb.cpp b/hphp/runtime/vm/translator/srcdb.cpp index b32d15b1e..cc6f1e2f9 100644 --- a/hphp/runtime/vm/translator/srcdb.cpp +++ b/hphp/runtime/vm/translator/srcdb.cpp @@ -190,10 +190,8 @@ void SrcRec::patch(IncomingBranch branch, TCA dest) { case IncomingBranch::JCC: { // patch destination, but preserve the condition code - int32_t delta = safe_cast((dest - branch.toSmash()) - - TranslatorX64::kJmpccLen); - int32_t* addr = (int32_t*)(branch.toSmash() + - TranslatorX64::kJmpccLen - 4); + int32_t delta = safe_cast((dest - branch.toSmash()) - kJmpccLen); + int32_t* addr = (int32_t*)(branch.toSmash() + kJmpccLen - 4); atomic_release_store(addr, delta); break; } diff --git a/hphp/runtime/vm/translator/translator-x64.cpp b/hphp/runtime/vm/translator/translator-x64.cpp index 9f590508a..221cfa43b 100644 --- a/hphp/runtime/vm/translator/translator-x64.cpp +++ b/hphp/runtime/vm/translator/translator-x64.cpp @@ -1479,8 +1479,7 @@ TranslatorX64::translate(SrcKey sk, bool align, bool allowIR) { * Returns true if the given current frontier can have an nBytes-long * instruction written without any risk of cache-tearing. */ -bool -TranslatorX64::isSmashable(Address frontier, int nBytes, int offset /* = 0 */) { +bool isSmashable(Address frontier, int nBytes, int offset /* = 0 */) { assert(nBytes <= int(kX64CacheLineSize)); uintptr_t iFrontier = uintptr_t(frontier) + offset; uintptr_t lastByte = uintptr_t(frontier) + nBytes - 1; @@ -1491,31 +1490,31 @@ TranslatorX64::isSmashable(Address frontier, int nBytes, int offset /* = 0 */) { * Call before emitting a test-jcc sequence. Inserts a nop gap such that after * writing a testBytes-long instruction, the frontier will be smashable. */ -void -TranslatorX64::prepareForTestAndSmash(int testBytes, TestAndSmashFlags flags) { - if (flags == kAlignJcc) { - prepareForSmash(testBytes + kJmpccLen, testBytes); +void prepareForTestAndSmash(Asm& a, int testBytes, TestAndSmashFlags flags) { + switch (flags) { + case kAlignJcc: + prepareForSmash(a, testBytes + kJmpccLen, testBytes); assert(isSmashable(a.code.frontier + testBytes, kJmpccLen)); - } else if (flags == kAlignJccImmediate) { - prepareForSmash(testBytes + kJmpccLen, + break; + case kAlignJccImmediate: + prepareForSmash(a, + testBytes + kJmpccLen, testBytes + kJmpccLen - kJmpImmBytes); assert(isSmashable(a.code.frontier + testBytes, kJmpccLen, kJmpccLen - kJmpImmBytes)); - } else if (flags == kAlignJccAndJmp) { + break; + case kAlignJccAndJmp: // Ensure that the entire jcc, and the entire jmp are smashable // (but we dont need them both to be in the same cache line) - prepareForSmash(testBytes + kJmpccLen, testBytes); - prepareForSmash(testBytes + kJmpccLen + kJmpLen, testBytes + kJmpccLen); + prepareForSmash(a, testBytes + kJmpccLen, testBytes); + prepareForSmash(a, testBytes + kJmpccLen + kJmpLen, testBytes + kJmpccLen); assert(isSmashable(a.code.frontier + testBytes, kJmpccLen)); assert(isSmashable(a.code.frontier + testBytes + kJmpccLen, kJmpLen)); - } else { - not_reached(); + break; } } -void -TranslatorX64::prepareForSmash(X64Assembler& a, int nBytes, - int offset /* = 0 */) { +void prepareForSmash(X64Assembler& a, int nBytes, int offset /* = 0 */) { if (!isSmashable(a.code.frontier, nBytes, offset)) { int gapSize = (~(uintptr_t(a.code.frontier) + offset) & kX64CacheLineMask) + 1; @@ -1524,11 +1523,6 @@ TranslatorX64::prepareForSmash(X64Assembler& a, int nBytes, } } -void -TranslatorX64::prepareForSmash(int nBytes, int offset /* = 0 */) { - prepareForSmash(a, nBytes, offset); -} - void TranslatorX64::smash(X64Assembler &a, TCA src, TCA dest, bool isCall) { assert(canWrite()); @@ -1847,7 +1841,7 @@ funcPrologToGuardImm(TCA prolog) { kFuncGuardShortLen - kFuncCmpImm)); // We padded these so the immediate would fit inside a cache line assert(((uintptr_t(retval) ^ (uintptr_t(retval + 1) - 1)) & - ~(TranslatorX64::kX64CacheLineSize - 1)) == 0); + ~(kX64CacheLineSize - 1)) == 0); return retval; } @@ -2364,7 +2358,7 @@ TranslatorX64::emitBindCallHelper(SrcKey srcKey, // Func we encounter as a decent prediction. Make space to burn in a // TCA. ReqBindCall* req = m_globalData.alloc(); - prepareForSmash(kCallLen); + prepareForSmash(a, kCallLen); TCA toSmash = a.code.frontier; a. call(astubs.code.frontier); @@ -2391,7 +2385,7 @@ TranslatorX64::emitCondJmp(SrcKey skTaken, SrcKey skNotTaken, // reserve space for a smashable jnz/jmp pair; both initially point // to our stub. - prepareForTestAndSmash(0, kAlignJccAndJmp); + prepareForTestAndSmash(a, 0, kAlignJccAndJmp); TCA old = a.code.frontier; TCA stub = astubs.code.frontier; @@ -2705,7 +2699,7 @@ TranslatorX64::emitFallbackCondJmp(Asm& as, SrcRec& dest, ConditionCode cc) { dest.emitFallbackJump(as.code.frontier, cc); } -void TranslatorX64::emitReqRetransNoIR(Asm& as, SrcKey& sk) { +void TranslatorX64::emitReqRetransNoIR(Asm& as, const SrcKey& sk) { prepareForSmash(as, kJmpLen); TCA toSmash = as.code.frontier; if (&as == &astubs) { @@ -2835,7 +2829,7 @@ TranslatorX64::checkRefs(X64Assembler& a, // Other than these builtins, we need to have all by value // args in this case. - prepareForTestAndSmash(kTestRegRegLen, kAlignJccImmediate); + prepareForTestAndSmash(a, kTestRegRegLen, kAlignJccImmediate); a. test_reg64_reg64(r(rExpectedBits), r(rExpectedBits)); emitFallbackJmp(fail); @@ -4175,7 +4169,7 @@ TranslatorX64::translateEqOp(const Tracelet& t, } if (i.changesPC) { fuseBranchSync(t, i); - prepareForTestAndSmash(kTestImmRegLen, kAlignJccAndJmp); + prepareForTestAndSmash(a, kTestImmRegLen, kAlignJccAndJmp); a. testb (1, al); fuseBranchAfterBool(t, i, ccNegate(ccBranch)); return; @@ -4196,7 +4190,7 @@ TranslatorX64::translateEqOp(const Tracelet& t, fuseBranchSync(t, i); } if (IS_NULL_TYPE(leftType) || IS_NULL_TYPE(rightType)) { - prepareForTestAndSmash(kTestRegRegLen, kAlignJccAndJmp); + prepareForTestAndSmash(a, kTestRegRegLen, kAlignJccAndJmp); if (IS_NULL_TYPE(leftType)) { a. test_reg64_reg64(srcdest, srcdest); } else { @@ -4560,7 +4554,7 @@ TranslatorX64::translateBranchOp(const Tracelet& t, emitBindJmp(isZ ? taken : notTaken); return; } - prepareForTestAndSmash(kTestRegRegLen, kAlignJccAndJmp); + prepareForTestAndSmash(a, kTestRegRegLen, kAlignJccAndJmp); a. test_reg64_reg64(src, src); branchWithFlagsSet(t, i, isZ ? CC_Z : CC_NZ); } @@ -7623,7 +7617,7 @@ void TranslatorX64::translateClassExistsImpl(const Tracelet& t, if (i.changesPC) { fuseBranchSync(t, i); } - prepareForTestAndSmash(kTestImmRegLen, kAlignJccAndJmp); + prepareForTestAndSmash(a, kTestImmRegLen, kAlignJccAndJmp); a. test_imm32_reg32(isClass ? attrNotClass : typeAttr, r(scratch)); ConditionCode cc = isClass ? CC_Z : CC_NZ; if (i.changesPC) { @@ -8233,7 +8227,7 @@ TranslatorX64::translateAKExists(const Tracelet& t, ScratchReg res(m_regMap, rax); if (ni.changesPC) { fuseBranchSync(t, ni); - prepareForTestAndSmash(kTestRegRegLen, kAlignJccAndJmp); + prepareForTestAndSmash(a, kTestRegRegLen, kAlignJccAndJmp); a. test_reg64_reg64(r(res), r(res)); fuseBranchAfterBool(t, ni, ni.invertCond ? CC_Z : CC_NZ); } else { @@ -10567,7 +10561,7 @@ void TranslatorX64::translateBasicIterInit(const Tracelet& t, // the input. If a new iterator is not created, new_iter_* will decRef the // input for us. new_iter_* returns 0 if an iterator was not created, // otherwise it returns 1. - prepareForTestAndSmash(kTestRegRegLen, kAlignJccAndJmp); + prepareForTestAndSmash(a, kTestRegRegLen, kAlignJccAndJmp); a. test_reg64_reg64(rax, rax); emitCondJmp(taken, notTaken, CC_Z); } @@ -10630,7 +10624,7 @@ TranslatorX64::translateBasicIterNext(const Tracelet& t, SrcKey taken, notTaken; branchDests(t, i, &taken, ¬Taken, 1 /* destImmIdx */); - prepareForTestAndSmash(kTestRegRegLen, kAlignJccAndJmp); + prepareForTestAndSmash(a, kTestRegRegLen, kAlignJccAndJmp); a. test_reg64_reg64(rax, rax); emitCondJmp(taken, notTaken, CC_NZ); } @@ -11191,7 +11185,7 @@ TranslatorX64::translateTracelet(SrcKey sk, bool considerHHIR/*=true*/, if (m_useHHIR) { TranslateTraceletResult result; do { - hhirTraceStart(sk.offset()); + hhirTraceStart(sk.offset(), t.m_nextSk.offset()); SKTRACE(1, sk, "retrying irTranslateTracelet\n"); result = irTranslateTracelet(t, start, stubStart, &bcMapping); if (result == Retry) { @@ -12088,7 +12082,7 @@ void TranslatorX64::addDbgGuardImpl(SrcKey sk, SrcRec& srcRec) { } // Emit a jump to the actual code TCA realCode = srcRec.getTopTranslation(); - prepareForSmash(kJmpLen); + prepareForSmash(a, kJmpLen); TCA dbgBranchGuardSrc = a.code.frontier; a. jmp(realCode); // Add it to srcRec diff --git a/hphp/runtime/vm/translator/translator-x64.h b/hphp/runtime/vm/translator/translator-x64.h index fe0fa39fc..2ece6c1ef 100644 --- a/hphp/runtime/vm/translator/translator-x64.h +++ b/hphp/runtime/vm/translator/translator-x64.h @@ -101,6 +101,32 @@ struct Label; static const int kNumFreeLocalsHelpers = 9; +typedef X64Assembler Asm; + +constexpr size_t kJmpTargetAlign = 16; +constexpr size_t kNonFallthroughAlign = 64; +constexpr int kJmpLen = 5; +constexpr int kCallLen = 5; +constexpr int kJmpccLen = 6; +constexpr int kJmpImmBytes = 4; +constexpr int kJcc8Len = 3; +constexpr int kLeaRipLen = 7; +constexpr int kTestRegRegLen = 3; +constexpr int kTestImmRegLen = 5; // only for rax -- special encoding +// Cache alignment is required for mutable instructions to make sure +// mutations don't "tear" on remote cpus. +constexpr size_t kX64CacheLineSize = 64; +constexpr size_t kX64CacheLineMask = kX64CacheLineSize - 1; + +enum TestAndSmashFlags { + kAlignJccImmediate, + kAlignJcc, + kAlignJccAndJmp +}; +void prepareForTestAndSmash(Asm&, int testBytes, TestAndSmashFlags flags); +void prepareForSmash(Asm&, int nBytes, int offset = 0); +bool isSmashable(Address frontier, int nBytes, int offset = 0); + class TranslatorX64 : public Translator , SpillFill , boost::noncopyable { @@ -195,9 +221,9 @@ class TranslatorX64 : public Translator std::string m_lastHHIRPunt; std::string m_lastHHIRDump; - void hhirTraceStart(Offset bcStartOffset); + void hhirTraceStart(Offset bcStartOffset, Offset nextTraceOffset); void hhirTraceCodeGen(vector* bcMap); - void hhirTraceEnd(Offset bcSuccOffset); + void hhirTraceEnd(); void hhirTraceFree(); @@ -857,33 +883,9 @@ private: void emitGenericReturn(bool noThis, int retvalSrcDisp); void dumpStack(const char* msg, int offset) const; - public: - static const size_t kJmpTargetAlign = 16; - static const size_t kNonFallthroughAlign = 64; - static const int kJmpLen = 5; - static const int kCallLen = 5; - static const int kJmpccLen = 6; - static const int kJmpImmBytes = 4; - static const int kJcc8Len = 3; - static const int kLeaRipLen = 7; - static const int kTestRegRegLen = 3; - static const int kTestImmRegLen = 5; // only for rax -- special encoding - // Cache alignment is required for mutable instructions to make sure - // mutations don't "tear" on remote cpus. - static const size_t kX64CacheLineSize = 64; - static const size_t kX64CacheLineMask = kX64CacheLineSize - 1; private: void moveToAlign(Asm &aa, const size_t alignment = kJmpTargetAlign, const bool unreachable = true); - enum TestAndSmashFlags { - kAlignJccImmediate, - kAlignJcc, - kAlignJccAndJmp - }; - void prepareForTestAndSmash(int testBytes, TestAndSmashFlags flags); - void prepareForSmash(Asm &a, int nBytes, int offset = 0); - void prepareForSmash(int nBytes, int offset = 0); - static bool isSmashable(Address frontier, int nBytes, int offset = 0); static void smash(Asm &a, TCA src, TCA dest, bool isCall); static void smashJmp(Asm &a, TCA src, TCA dest) { smash(a, src, dest, false); @@ -1077,11 +1079,9 @@ private: virtual bool addDbgGuard(const Func* func, Offset offset); void addDbgGuardImpl(SrcKey sk, SrcRec& sr); -private: // Only for HackIR - void emitReqRetransNoIR(Asm& as, SrcKey& sk); - void emitRecordPunt(Asm& as, const std::string& name); - public: // Only for HackIR + void emitReqRetransNoIR(Asm& as, const SrcKey& sk); + void emitRecordPunt(Asm& as, const std::string& name); #define DECLARE_FUNC(nm) \ void irTranslate ## nm(const Tracelet& t, \ const NormalizedInstruction& i); diff --git a/hphp/test/slow/apc/1824.php b/hphp/test/slow/apc/1824.php index d460da265..6d5fa4d83 100644 --- a/hphp/test/slow/apc/1824.php +++ b/hphp/test/slow/apc/1824.php @@ -1,5 +1,16 @@ cache_gen = $x->gen('a', 'b'); foreach ($x->cache_gen as $v) { var_dump($v); } apc_store('key', $x); $y = apc_fetch('key'); print_r($y->cache_gen); \ No newline at end of file +class A { + public function gen($a, $b) { + yield $a; + yield $b; + } +} + +$x = new A; +$x->cache_gen = $x->gen('a', 'b'); +foreach ($x->cache_gen as $v) { var_dump($v); } +apc_store('key', $x); +$y = apc_fetch('key'); +print_r($y->cache_gen);