diff --git a/hphp/doc/ir.specification b/hphp/doc/ir.specification index f69d2d380..b60deb0ef 100644 --- a/hphp/doc/ir.specification +++ b/hphp/doc/ir.specification @@ -153,6 +153,7 @@ above PHP-facing types. PtrToT Exists for all T in Gen. Represents a TypedValue* None No value, KindOfInvalid Bottom No value, {}. Subtype of every other type + Top Supertype of every other type Cls Class* Func Func* VarEnv VarEnv* @@ -166,6 +167,7 @@ above PHP-facing types. StkPtr Pointer into VM execution stack FramePtr Pointer to a frame on the VM execution stack TCA Machine code address + Nullptr Represents a null pointer In the instruction descriptions below, we also use a "Const" prefix on several types loosly, which means the operand must be the requested @@ -327,7 +329,7 @@ Instruction set 1. Checks and Asserts -D:T = CheckType S0:Gen -> L +D:T = CheckType S0:{Gen|Nullptr} -> L Check that the type of the src S0 is T, and if so copy it to D. If S0 is not type T, branch to the label L. @@ -410,6 +412,10 @@ GuardRefs S0:FuncPtr S1:Int S2:Int S3:Int S4:Int S5:Int S6:Int If any of the checks fail, make a fallback jump. (Jump to a service request that will chain to a retranslation of this tracelet.) +D:CountedStr = AssertNonNull S0:{Nullptr|CountedStr} + + Returns S0, with Nullptr removed from its type. This instruction currently + supports a very limited range of types but can be expanded if needed. 2. Arithmetic @@ -810,12 +816,11 @@ D:PtrToGen = LdGblAddrDef S0:Str D:PtrToGen = LdClsPropAddr S0:Cls S1:Str S2:ConstCls [ -> L ] - Loads a pointer to a static class property. S0 points to the class, - S1 is the property name, and S2 is the class representing the - context of the code accessing the property. If class S0 does not - have a visible and accessible static property named S1, then this - instruction will either (1) throw a fatal error if the optional - label L is not present, or (2) jump to L if it is present. + Loads a pointer to a static class property. S0 points to the class, S1 is the + property name, and S2 is the class representing the context of the code + accessing the property. If class S0 does not have a visible and accessible + static property named S1, then this instruction will either (1) jump to L if + it is present and not a catch trace, or (2) throw a fatal error. D:PtrToGen = LdClsPropAddrCached S0:Cls S1:ConstStr S2:ConstStr S3:ConstCls [ -> L ] @@ -977,7 +982,7 @@ NativeImpl = S0:ConstFunc S1:FramePtr Execute a call to the native builtin specified by the func in S0. -D:T = CallBuiltin S0:FuncPtr S1...SN +D:T = CallBuiltin S0:FuncPtr S1:StkPtr S2...SN Call builtin function with N-1 arguments. Operands: @@ -1627,18 +1632,37 @@ D:PtrToCell = DefMIStateBase All of the remaining opcodes in this section are simple wrappers around helper functions (specified in S0) to perform the corresponding vector operation. If S1 is a ConstCls it represents the context class -for the operation. D:Vector indicates that the return value of the -opcode depends on the input types: +for the operation. -SetElem/SetOpElem/IncDecElem: -- If the base is not a subtype of Type::Arr or Type::Obj, the - operation will fail with a warning and the return value is - null. Otherwise the return value is the same as the value input. +SetElem, SetProp, and SetNewElem are used to implement part of the SetM hhbc +opcode, which almost always pushes its first stack input or a CountedStr as its +stack result. The combinations of input types that cause SetM to push anything +other than those two values are vanishingly rare in correct PHP programs, so +these three instructions have been optimized for the common cases. SetNewElem +and SetProp have no destination, allowing the compiler to predict that the +SetM's output will be the same as its input (and optimize accordingly). If that +turns out to not be the case at runtime, the instruction will throw an +InvalidSetMException. The exception will hold a Cell containing the value the +SetM should push on the stack instead of its input value. The runtime is +responsible for catching this exception, finishing execution of the SetM +instruction, pushing the value from the exception on the stack, and proceeding +as appropriate (most likely with a side exit to the next bytecode instruction, +since it has pushed an unexpected type onto the stack). -SetProp/SetOpProp/IncDecProp: -- If the base is not a subtype of Type::Obj, the operation will fail - with a warning and the return value is null. Otherwise the return - value is the same as the value input. +SetElem is similar to SetProp and SetNewElem but can also be used for setting +characters within strings. When given a string base and a valid offset, SetElem +returns a string representation of the newly inserted character. In all other +cases it returns nullptr or throws an InvalidSetMException. It will throw this +exception when it detects invalid input types, or when trying to set a string +offset that would grow the string beyond the maximum supported size. + +The input types that will cause the errors described above are listed here: + +SetNewElem will fail if the base is not a subtype of {Null|Str|Arr|Obj} and not + Bool. +SetElem has the same base constraint as SetNewElem. In addition, the key must + not be a subtype of {Arr|Obj}. +SetProp will fail if the base is not a subtype of {Obj|Null}. Any instructions that take a pointer to an MInstrState struct use the various fields of that struct for holding intermediate values. @@ -1677,9 +1701,8 @@ D:StkPtr = BindPropStk S0:ConstTCA S1:ConstCls S2:{Obj|PtrtoGen} S3:Gen Bind property with key S3 in base S2 to the reference in S4. S5 should point to an MInstrState struct. -D:Vector = SetProp S0:ConstTCA S1:ConstCls S2:{Obj|PtrToGen} S3:Gen S4:Cell -D:Vector, D:StkPtr = SetPropStk S0:ConstTCA S1:ConstCls S2:{Obj|PtrToGen} S3:Gen - S4:Cell +SetProp S0:ConstTCA S1:ConstCls S2:{Obj|PtrToGen} S3:Gen S4:Cell +D:StkPtr = SetPropStk S0:ConstTCA S1:ConstCls S2:{Obj|PtrToGen} S3:Gen S4:Cell Set property with key S3 in S2 to S4. @@ -1755,8 +1778,8 @@ ArraySet S0:ConstTCA S1:Arr S2:{Int|Str} S3:Cell S4:BoxedArr operation, it will be replaced with the new Array resulting from the set operation. -D:Vector = SetElem S0:ConstTCA S1:PtrToGen S2:Gen S3:Cell -D:Vector, D:StkPtr = SetElemStk S0:ConstTCA S1:PtrToGen S2:Gen S3:Cell +D:{CountedStr|Nullptr} = SetElem S0:ConstTCA S1:PtrToGen S2:Gen S3:Cell +D:{CountedStr|Nullptr} D:StkPtr = SetElemStk S0:ConstTCA S1:PtrToGen S2:Gen S3:Cell Set element with key S2 in S1 to S3. @@ -1784,8 +1807,8 @@ D:Cell, D:StkPtr = IncDecElemStk S0:ConstTCA S1:PtrToGen S2:Gen S3:PtrToCell Increment/decrement element with key S2 in base S1. S3 should point to an MInstrState struct. -D:Vector = SetNewElem S0:PtrToGen S1:Cell -D:Vector, D:StkPtr = SetNewElemStk S0:PtrToGen S1:Cell +SetNewElem S0:PtrToGen S1:Cell +D:StkPtr = SetNewElemStk S0:PtrToGen S1:Cell Append the value in S1 to S0. @@ -1812,3 +1835,23 @@ D:Bool = EmptyElem S0:ConstTCA S1:PtrToGen S2:Gen S3:PtrToCell Returns true iff the element at key S2 in S1 is set and not equal (as defined by the hhbc Eq instruction) to false. + +20. Exception/unwinding support + +BeginCatch + + Marks the beginning of a catch region. Exact behavior is implementation and + architecture specific. + +EndCatch S0:StkPtr + + Marks the end of a catch region and returns control to the unwinder if + appropriate. Exact behavior is implementation and architecture specific. + +D:Cell = LdUnwinderValue + + Loads the value contained by the current unwinder exception. + +DeleteUnwinderException + + Deletes the current unwinder exception. diff --git a/hphp/runtime/base/stats.h b/hphp/runtime/base/stats.h index 17eaf562b..96842cde7 100644 --- a/hphp/runtime/base/stats.h +++ b/hphp/runtime/base/stats.h @@ -134,6 +134,10 @@ namespace Stats { STAT(TC_TypePredMiss) \ STAT(TC_TypePredUnneeded) \ STAT(TC_TypePredOverridden) \ + STAT(TC_CatchTrace) \ + STAT(TC_CatchSideExit) \ + STAT(TC_SetMStrGuess_Hit) \ + STAT(TC_SetMStrGuess_Miss) \ /* Fixup */ \ STAT(Fixup_Find) \ STAT(Fixup_Probe) \ diff --git a/hphp/runtime/vm/bytecode.cpp b/hphp/runtime/vm/bytecode.cpp index 87b48df1f..a11fd9e00 100644 --- a/hphp/runtime/vm/bytecode.cpp +++ b/hphp/runtime/vm/bytecode.cpp @@ -3720,16 +3720,17 @@ inline void OPTBLD_INLINE VMExecutionContext::setHelperPost( } // NOTE: currently the only instructions using this that have return - // values on the stack also have more inputs than the K-vector, so + // values on the stack also have more inputs than the -vector, so // mdepth > 0. They also always return the original top value of // the stack. if (mdepth > 0) { assert(mdepth == 1 && - "We don't really support mdepth > 1 in setHelperPost"); + "We don't really support mdepth > 1 in setHelperPost"); - TypedValue* retSrc = m_stack.topTV(); if (ndiscard > 0) { + TypedValue* retSrc = m_stack.topTV(); TypedValue* dest = m_stack.indTV(ndiscard + mdepth - 1); + assert(dest != retSrc); memcpy(dest, retSrc, sizeof *dest); } } @@ -5229,9 +5230,15 @@ inline void OPTBLD_INLINE VMExecutionContext::iopSetM(PC& pc) { case MEL: case MEC: case MET: - case MEI: - SetElem(base, curMember, c1); + case MEI: { + StringData* result = SetElem(base, curMember, c1); + if (result) { + tvRefcountedDecRefCell(c1); + c1->m_type = KindOfString; + c1->m_data.pstr = result; + } break; + } case MPL: case MPC: case MPT: { @@ -5268,8 +5275,8 @@ inline void OPTBLD_INLINE VMExecutionContext::iopSetWithRefRM(PC& pc) { TypedValue* from = m_stack.top(); tvAsVariant(base) = withRefBind(tvAsVariant(from)); } + setHelperPost<0>(SETHELPERPOST_ARGS); m_stack.popTV(); - setHelperPost<1>(SETHELPERPOST_ARGS); } inline void OPTBLD_INLINE VMExecutionContext::iopSetOpL(PC& pc) { diff --git a/hphp/runtime/vm/debug/debug.h b/hphp/runtime/vm/debug/debug.h index b755ab677..1463a8a53 100644 --- a/hphp/runtime/vm/debug/debug.h +++ b/hphp/runtime/vm/debug/debug.h @@ -24,14 +24,12 @@ namespace HPHP { namespace Debug { -using namespace HPHP::Transl; - class DebugInfo { public: DebugInfo(); ~DebugInfo(); - void recordTracelet(TCRange range, + void recordTracelet(TCRange range, const Func* func, const Opcode *instr, bool exit, bool inPrologue); diff --git a/hphp/runtime/vm/member_operations.h b/hphp/runtime/vm/member_operations.h index 86b03e088..64e3e7ee8 100644 --- a/hphp/runtime/vm/member_operations.h +++ b/hphp/runtime/vm/member_operations.h @@ -27,6 +27,28 @@ namespace HPHP { +class InvalidSetMException : public std::runtime_error { + public: + InvalidSetMException() + : std::runtime_error("Empty InvalidSetMException") + , m_tv(HPHP::tv(KindOfNull, 0LL)) + {} + + explicit InvalidSetMException(const TypedValue& value) + : std::runtime_error(folly::format("InvalidSetMException containing {}", + value.pretty()).str()) + , m_tv(value) + {} + + ~InvalidSetMException() noexcept {} + + const TypedValue& tv() const { return m_tv; }; + private: + /* m_tv will contain a TypedValue with a reference destined for the + * VM eval stack. */ + const TypedValue m_tv; +}; + // When MoreWarnings is set to true, the VM will raise more warnings // on SetOpM, IncDecM and CGetG, intended to match Zend. const bool MoreWarnings = @@ -447,7 +469,7 @@ inline TypedValue* ElemU(TypedValue& tvScratch, TypedValue& tvRef, // $result = ($base[] = ...); inline TypedValue* NewElem(TypedValue& tvScratch, TypedValue& tvRef, - TypedValue* base) { + TypedValue* base) { TypedValue* result; DataType type; opPre(base, type); @@ -516,9 +538,12 @@ inline void SetElemEmptyish(TypedValue* base, TypedValue* key, template inline void SetElemNumberish(Cell* value) { raise_warning(Strings::CANNOT_USE_SCALAR_AS_ARRAY); + // XXX Flip? if (setResult) { tvRefcountedDecRefCell((TypedValue*)value); tvWriteNull((TypedValue*)value); + } else { + throw InvalidSetMException(tv(KindOfNull, 0LL)); } } @@ -605,17 +630,35 @@ inline void SetElemArray(TypedValue* base, TypedValue* key, if (setResult) { tvRefcountedDecRef(value); tvWriteNull(value); + } else { + throw InvalidSetMException(tv(KindOfNull, 0LL)); } } arrayRefShuffle(a, newData, base); } +template +inline int64_t castKeyToInt(TypedValue* key) { + TypedValue scratch; + initScratchKey(scratch, key); + if (key->m_type == KindOfInt64) { + return key->m_data.num; + } else { + return tvCastToInt64(key); + } +} + +template<> +inline int64_t castKeyToInt(TypedValue* key) { + return keyAsRaw(key); +} + // SetElem() leaves the result in 'value', rather than returning it as in // SetOpElem(), because doing so avoids a dup operation that SetOpElem() can't // get around. template -inline void SetElem(TypedValue* base, TypedValue* key, Cell* value) { +inline StringData* SetElem(TypedValue* base, TypedValue* key, Cell* value) { TypedValue scratch; DataType type; opPre(base, type); @@ -642,23 +685,20 @@ inline void SetElem(TypedValue* base, TypedValue* key, Cell* value) { } case KindOfStaticString: case KindOfString: { - initScratchKey(scratch, key); int baseLen = base->m_data.pstr->size(); if (baseLen == 0) { SetElemEmptyish(base, key, value); } else { // Convert key to string offset. - int64_t x; - if (LIKELY(key->m_type == KindOfInt64)) { - x = key->m_data.num; - } else { - x = tvCastToInt64(key); - } - if (x < 0 || x >= StringData::MaxSize) { - // Andrei: can't use PRId64 here because of order of inclusion - // issues + int64_t x = castKeyToInt(key); + if (UNLIKELY(x < 0 || x >= StringData::MaxSize)) { + // Can't use PRId64 here because of order of inclusion issues raise_warning("Illegal string offset: %lld", (long long)x); - break; + if (!setResult) { + throw InvalidSetMException(*value); + } else { + break; + } } // Compute how long the resulting string will be. Type needs // to agree with x. @@ -706,14 +746,10 @@ inline void SetElem(TypedValue* base, TypedValue* key, Cell* value) { base->m_data.pstr = sd; base->m_type = KindOfString; } - if (setResult) { - // Push y onto the stack. - tvRefcountedDecRef(value); - StringData* sd = NEW(StringData)(y, strlen(y), CopyString); - sd->incRefCount(); - value->m_data.pstr = sd; - value->m_type = KindOfString; - } + + StringData* sd = NEW(StringData)(y, strlen(y), CopyString); + sd->incRefCount(); + return sd; } break; } @@ -732,6 +768,8 @@ inline void SetElem(TypedValue* base, TypedValue* key, Cell* value) { } default: not_reached(); } + + return nullptr; } inline void SetNewElemEmptyish(TypedValue* base, Cell* value) { @@ -745,6 +783,8 @@ inline void SetNewElemNumberish(Cell* value) { if (setResult) { tvRefcountedDecRefCell((TypedValue*)value); tvWriteNull((TypedValue*)value); + } else { + throw InvalidSetMException(tv(KindOfNull, 0LL)); } } template @@ -1469,11 +1509,13 @@ inline bool IssetEmptyProp(Class* ctx, TypedValue* base, template inline void SetPropNull(Cell* val) { + raise_warning("Cannot access property on non-object"); if (setResult) { tvRefcountedDecRefCell(val); tvWriteNull(val); + } else { + throw InvalidSetMException(tv(KindOfNull, 0LL)); } - raise_warning("Cannot access property on non-object"); } inline void SetPropStdclass(TypedValue* base, TypedValue* key, Cell* val) { diff --git a/hphp/runtime/vm/translator/hopt/block.h b/hphp/runtime/vm/translator/hopt/block.h index 14daf6ed7..c6126123d 100644 --- a/hphp/runtime/vm/translator/hopt/block.h +++ b/hphp/runtime/vm/translator/hopt/block.h @@ -49,6 +49,13 @@ struct Block : boost::noncopyable { , m_hint(Neither) {} + const IRInstruction* beginCatch() const { + auto it = begin(); + ++it; + assert(it->op() == BeginCatch); + return &*it; + } + uint32_t id() const { return m_id; } Trace* trace() const { return m_trace; } void setTrace(Trace* t) { m_trace = t; } @@ -112,21 +119,30 @@ struct Block : boost::noncopyable { void setPostId(unsigned id) { m_postid = id; } /* - * Insert inst after this block's Marker, return an - * iterator to the newly inserted instruction. + * Insert inst after this block's optional DefLabel, optional + * BeginCatch, and Marker, return an iterator to the newly inserted + * instruction. * * Pre: the block contains a Marker after the optional DefLabel. */ iterator prepend(IRInstruction* inst) { - auto it = skipLabel(); + auto it = skipHeader(); assert(it->op() == Marker); return insert(++it, inst); } - // return iterator to first instruction after the (optional) label - iterator skipLabel() { + // return iterator to first instruction after the DefLabel (if + // present) and BeginCatch (if present). + iterator skipHeader() { auto it = begin(); - return it->op() == DefLabel ? ++it : it; + auto e = end(); + if (it != e && it->op() == DefLabel) ++it; + if (it != e && it->op() == BeginCatch) ++it; + return it; + } + + const_iterator skipHeader() const { + return const_cast(this)->skipHeader(); } // return iterator to last instruction diff --git a/hphp/runtime/vm/translator/hopt/check.cpp b/hphp/runtime/vm/translator/hopt/check.cpp index fd116466a..bdb36a9c4 100644 --- a/hphp/runtime/vm/translator/hopt/check.cpp +++ b/hphp/runtime/vm/translator/hopt/check.cpp @@ -62,48 +62,64 @@ DEBUG_ONLY static int numBlockParams(Block* b) { } /* - * Check one block for being well formed. It must: - * 1. Optionally start with DefLabel. DefLabel may not appear anywhere else. - * 2. Have either no other instructions, or else at least one Marker - * following the DefLabel before any other instructions. - * 3. If any instruction is isBlockEnd(), it must be last. - * 4. If the last instruction isTerminal(), block->next must be null. - * 5. If the block starts with a DefLabel with >=1 destinations, all the - * incoming must be from blocks listed in this block's trace. + * Check one block for being well formed. Invariants verified: + * 1. The block begins with an optional DefLabel, followed by an optional + * BeginCatch, followed by either a Marker or no more instructions. + * 2. DefLabel and BeginCatch may not appear anywhere in a block other than + * where specified in #1. + * 3. If the optional BeginCatch is present, the block must belong to an exit + * trace and must be the first block in its Trace's block list. + * 4. If any instruction is isBlockEnd(), it must be last. + * 5. If the last instruction isTerminal(), block->next must be null. + * 6. If the DefLabel produces a value, all of its incoming edges must be from + * blocks listed in the block list for this block's Trace. + * 7. Any path from this block to a Block that expects values must be + * from a Jmp_ instruciton. */ bool checkBlock(Block* b) { - if (b->skipLabel() != b->end()) { - assert(b->skipLabel()->op() == Marker); + auto it = b->begin(); + auto end = b->end(); + if (it == end) return true; + + // Invariant #1 + if (it->op() == DefLabel) ++it; + + // Invariant #1, #3 + if (it != end && it->op() == BeginCatch) { + ++it; + assert(!b->trace()->isMain()); + assert(b == b->trace()->front()); } - if (Block* next DEBUG_ONLY = b->next()) { - // cannot fall-through to join block expecting values - assert(numBlockParams(next) == 0); + + // Invariant #3 + if (it == end) return true; + assert(it->op() == Marker); + + // Invariants #2, #4 + if (++it == end) return true; + if (b->back()->isBlockEnd()) --end; + while (it != end && it->op() == Marker) ++it; + for (DEBUG_ONLY IRInstruction& inst : folly::makeRange(it, end)) { + assert(inst.op() != DefLabel); + assert(inst.op() != BeginCatch); + assert(!inst.isBlockEnd()); } - if (b->empty()) { - return true; + for (DEBUG_ONLY IRInstruction& inst : *b) { + assert(inst.block() == b); } - if (b->back()->isTerminal()) { - assert(!b->next()); - } - if (Block* taken DEBUG_ONLY = b->taken()) { + + // Invariant #5 + assert(IMPLIES(b->back()->isTerminal(), !b->next())); + + // Invariant #7 + if (b->taken()) { // only Jmp_ can branch to a join block expecting values. DEBUG_ONLY IRInstruction* branch = b->back(); DEBUG_ONLY auto numArgs = branch->op() == Jmp_ ? branch->numSrcs() : 0; - assert(numBlockParams(taken) == numArgs); - } - - { - auto i = b->skipLabel(), e = b->end(); - if (b->back()->isBlockEnd()) --e; - for (DEBUG_ONLY IRInstruction& inst : folly::makeRange(i, e)) { - assert(inst.op() != DefLabel); - assert(!inst.isBlockEnd()); - } - for (DEBUG_ONLY IRInstruction& inst : *b) { - assert(inst.block() == b); - } + assert(numBlockParams(b->taken()) == numArgs); } + // Invariant #6 if (b->front()->op() == DefLabel) { for (int i = 0; i < b->front()->numDsts(); ++i) { auto const traceBlocks = b->trace()->blocks(); @@ -117,6 +133,21 @@ bool checkBlock(Block* b) { return true; } +/* + * Check that every catch trace has at most one incoming branch and a single + * block. + */ +bool checkCatchTraces(Trace* trace, const IRFactory& irFactory) { + forEachTraceBlock(trace, [&](Block* b) { + auto trace = b->trace(); + if (trace->isCatch()) { + assert(trace->blocks().size() == 1); + assert(b->preds().size() <= 1); + } + }); + return true; +} + } const Edge* takenEdge(IRInstruction* inst) { @@ -167,6 +198,8 @@ bool checkCfg(Trace* trace, const IRFactory& factory) { } } + checkCatchTraces(trace, factory); + // visit dom tree in preorder, checking all tmps auto const children = findDomChildren(blocks); StateVector defined0(&factory, false); diff --git a/hphp/runtime/vm/translator/hopt/codegen.cpp b/hphp/runtime/vm/translator/hopt/codegen.cpp index c7141118b..d34d516a7 100644 --- a/hphp/runtime/vm/translator/hopt/codegen.cpp +++ b/hphp/runtime/vm/translator/hopt/codegen.cpp @@ -17,6 +17,7 @@ #include "hphp/runtime/vm/translator/hopt/codegen.h" #include +#include #include "folly/ScopeGuard.h" #include "folly/Format.h" @@ -318,6 +319,12 @@ Address CodeGenerator::cgInst(IRInstruction* inst) { Opcode opc = inst->op(); auto const start = m_as.code.frontier; m_rScratch = selectScratchReg(inst); + if (inst->taken() && inst->taken()->trace()->isCatch()) { + m_state.catchTrace = inst->taken()->trace(); + } else { + m_state.catchTrace = nullptr; + } + switch (opc) { #define O(name, dsts, srcs, flags) \ case name: FTRACE(7, "cg" #name "\n"); \ @@ -778,6 +785,58 @@ void CodeGenerator::emitReqBindJcc(ConditionCode cc, a. jmp (jccStub); } +void CodeGenerator::cgAssertNonNull(IRInstruction* inst) { + auto srcReg = m_regs[inst->src(0)].getReg(); + auto dstReg = m_regs[inst->dst()].getReg(); + if (RuntimeOption::EvalHHIRGenerateAsserts) { + Label nonNull; + m_as.testq (srcReg, srcReg); + m_as.jne8 (nonNull); + m_as.ud2(); + asm_label(m_as, nonNull); + } + emitMovRegReg(m_as, srcReg, dstReg); +} + +void CodeGenerator::cgLdUnwinderValue(IRInstruction* inst) { + cgLoad(rVmTl, TargetCache::kUnwinderTvOff, inst); +} + +void CodeGenerator::cgBeginCatch(IRInstruction* inst) { + auto const& info = m_state.catches[inst->block()]; + assert(info.afterCall); + + m_tx64->registerCatchTrace(info.afterCall, m_as.code.frontier); + + Stats::emitInc(m_as, Stats::TC_CatchTrace); + + // We want to restore state as though the call had completed + // successfully, so skip over any stack arguments and pop any + // saved registers. + if (info.rspOffset) { + m_as.subq(info.rspOffset, rsp); + } + PhysRegSaverParity::emitPops(m_as, info.savedRegs); +} + +void CodeGenerator::cgEndCatch(IRInstruction* inst) { + m_as.cmpb (0, rVmTl[TargetCache::kUnwinderSideExitOff]); + unlikelyIfBlock(CC_E, + [&](Asm& as) { // doSideExit == false, so call _Unwind_Resume + as.loadq(rVmTl[TargetCache::kUnwinderScratchOff], rdi); + as.call ((TCA)_Unwind_Resume); // pass control back to the unwinder + as.ud2(); + }); + + // doSideExit == true, so fall through to the side exit code + Stats::emitInc(m_as, Stats::TC_CatchSideExit); +} + +void CodeGenerator::cgDeleteUnwinderException(IRInstruction* inst) { + m_as.loadq(rVmTl[TargetCache::kUnwinderScratchOff], rdi); + m_as.call ((TCA)_Unwind_DeleteException); +} + void CodeGenerator::cgJcc(IRInstruction* inst) { emitCompare(inst->src(0), inst->src(1)); emitFwdJcc(opToConditionCode(inst->op()), inst->taken()); @@ -1073,6 +1132,14 @@ void CodeGenerator::cgCallHelper(Asm& a, recordSyncPoint(a, sync); } + if (m_state.catchTrace) { + auto& info = m_state.catches[m_state.catchTrace->front()]; + assert(!info.afterCall); + info.afterCall = a.code.frontier; + info.savedRegs = toSave; + info.rspOffset = regSaver.rspAdjustment(); + } + // copy the call result to the destination register(s) if (destType == DestType::TV) { // rax contains m_type and m_aux but we're expecting just the @@ -3525,8 +3592,8 @@ void CodeGenerator::cgCastStk(IRInstruction *inst) { void CodeGenerator::cgCallBuiltin(IRInstruction* inst) { SSATmp* f = inst->src(0); - auto args = inst->srcs().subpiece(1); - int32_t numArgs = args.size(); + auto args = inst->srcs().subpiece(2); + int32_t numArgs = args.size(); SSATmp* dst = inst->dst(); auto dstReg = m_regs[dst].getReg(0); auto dstType = m_regs[dst].getReg(1); @@ -4133,18 +4200,28 @@ void CodeGenerator::cgDefMIStateBase(IRInstruction* inst) { void CodeGenerator::cgCheckType(IRInstruction* inst) { auto const src = inst->src(0); + auto const t = inst->typeParam(); auto const rData = m_regs[src].getReg(0); auto const rType = m_regs[src].getReg(1); - emitTypeTest(inst->typeParam(), rType, rData, - [&](ConditionCode cc) { - emitFwdJcc(ccNegate(cc), inst->taken()); + auto doJcc = [&](ConditionCode cc) { + emitFwdJcc(ccNegate(cc), inst->taken()); + }; - auto const dstReg = m_regs[inst->dst()].getReg(); - if (dstReg != InvalidReg) { - emitMovRegReg(m_as, m_regs[src].getReg(0), dstReg); - } - }); + if (t.equals(Type::Nullptr)) { + if (!src->type().equals(Type::Nullptr | Type::CountedStr)) { + CG_PUNT(CheckType-Nullptr-UnsupportedType); + } + m_as.testq (rData, rData); + doJcc(CC_E); + } else { + emitTypeTest(inst->typeParam(), rType, rData, doJcc); + } + + auto const dstReg = m_regs[inst->dst()].getReg(); + if (dstReg != InvalidReg) { + emitMovRegReg(m_as, rData, dstReg); + } } void CodeGenerator::cgCheckTypeMem(IRInstruction* inst) { @@ -4502,11 +4579,15 @@ void CodeGenerator::cgLdClsPropAddrCached(IRInstruction* inst) { } void CodeGenerator::cgLdClsPropAddr(IRInstruction* inst) { - SSATmp* dst = inst->dst(); - SSATmp* cls = inst->src(0); - SSATmp* prop = inst->src(1); - SSATmp* cxt = inst->src(2); - Block* target = inst->taken(); + SSATmp* dst = inst->dst(); + SSATmp* cls = inst->src(0); + SSATmp* prop = inst->src(1); + SSATmp* ctx = inst->src(2); + Block* target = inst->taken(); + // If our label is a catch trace we pretend we don't have one, to + // avoid emitting a jmp to it or calling the wrong helper. + if (target && target->trace()->isCatch()) target = nullptr; + auto dstReg = m_regs[dst].getReg(); if (dstReg == InvalidReg && target) { // result is unused but this instruction was not eliminated @@ -4518,7 +4599,7 @@ void CodeGenerator::cgLdClsPropAddr(IRInstruction* inst) { : (TCA)SPropCache::lookupSProp, // raise on error dstReg, kSyncPoint, // could re-enter to initialize properties - ArgGroup(m_regs).ssa(cls).ssa(prop).ssa(cxt)); + ArgGroup(m_regs).ssa(cls).ssa(prop).ssa(ctx)); if (target) { m_as.testq(dstReg, dstReg); emitFwdJcc(m_as, CC_Z, target); @@ -4804,12 +4885,13 @@ void CodeGenerator::cgJmp_(IRInstruction* inst) { if (unsigned n = inst->numSrcs()) { // Parallel-copy sources to the label's destination registers. // TODO: t2040286: this only works if all destinations fit in registers. - SrcRange srcs = inst->srcs(); - DstRange dsts = target->front()->dsts(); + auto srcs = inst->srcs(); + auto dsts = target->front()->dsts(); ArgGroup args(m_regs); for (unsigned i = 0, j = 0; i < n; i++) { assert(srcs[i]->type().subtypeOf(dsts[i].type())); - SSATmp *dst = &dsts[i], *src = srcs[i]; + auto dst = &dsts[i]; + auto src = srcs[i]; // Currently, full XMM registers cannot be assigned to SSATmps // passed from to Jmp_ to DefLabel. If this changes, it'll require // teaching shuffleArgs() how to handle full XMM values. @@ -5290,6 +5372,7 @@ void CodeGenerator::cgBlock(Block* block, vector* bcMap) { IRInstruction* inst = &instr; if (inst->op() == Marker) { m_state.lastMarker = inst->extra(); + FTRACE(7, "lastMarker is now {}\n", inst->extra()->show()); if (m_tx64 && m_tx64->isTransDBEnabled() && bcMap) { bcMap->push_back((TransBCMapping){Offset(m_state.lastMarker->bcOff), m_as.code.frontier, diff --git a/hphp/runtime/vm/translator/hopt/codegen.h b/hphp/runtime/vm/translator/hopt/codegen.h index f2ddc179c..edb4d90ac 100644 --- a/hphp/runtime/vm/translator/hopt/codegen.h +++ b/hphp/runtime/vm/translator/hopt/codegen.h @@ -84,6 +84,8 @@ struct CodegenState { , liveRegs(liveRegs) , lifetime(lifetime) , asmInfo(asmInfo) + , catches(factory, CatchInfo()) + , catchTrace(nullptr) {} // Each block has a list of addresses to patch, and an address if @@ -113,6 +115,14 @@ struct CodegenState { // Output: start/end ranges of machine code addresses of each instruction. AsmInfo* asmInfo; + + // Used to pass information about the state of the world at native + // calls between cgCallHelper and cgBeginCatch. + StateVector catches; + + // If non-null, represents the catch trace for the current + // instruction, to be registered with the unwinder. + Trace* catchTrace; }; constexpr Reg64 rCgGP (reg::r11); diff --git a/hphp/runtime/vm/translator/hopt/dce.cpp b/hphp/runtime/vm/translator/hopt/dce.cpp index ed1ee8921..07d3c8293 100644 --- a/hphp/runtime/vm/translator/hopt/dce.cpp +++ b/hphp/runtime/vm/translator/hopt/dce.cpp @@ -122,6 +122,10 @@ void removeDeadInstructions(Trace* trace, const DceState& state) { auto cur = it; ++it; Block* block = *cur; block->remove_if([&] (const IRInstruction& inst) { + ONTRACE(7, + if (state[inst].isDead()) { + FTRACE(3, "Removing dead instruction {}\n", inst.toString()); + }); return state[inst].isDead(); }); // Marker and DefLabel instructions are marked live in reachable blocks @@ -204,7 +208,7 @@ BlockList removeUnreachable(Trace* trace, IRFactory* factory) { WorkList initInstructions(const BlockList& blocks, DceState& state) { - TRACE(5, "DCE:vvvvvvvvvvvvvvvvvvvv\n"); + TRACE(5, "DCE(initInstructions):vvvvvvvvvvvvvvvvvvvv\n"); // mark reachable, essential, instructions live and enqueue them WorkList wl; for (Block* block : blocks) { diff --git a/hphp/runtime/vm/translator/hopt/extradata.h b/hphp/runtime/vm/translator/hopt/extradata.h index aea66a343..af27d9334 100644 --- a/hphp/runtime/vm/translator/hopt/extradata.h +++ b/hphp/runtime/vm/translator/hopt/extradata.h @@ -112,6 +112,13 @@ struct MarkerData : IRExtraData { uint32_t bcOff; // the bytecode offset in unit int32_t stackOff; // stack off from start of trace const Func* func; // which func are we in + + std::string show() const { + return folly::format("--- bc {}, spOff {} ({})", + bcOff, + stackOff, + func->fullName()->data()).str(); + } }; struct LocalId : IRExtraData { diff --git a/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp b/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp index 0a2ede348..ae77ada6d 100644 --- a/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp +++ b/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp @@ -268,18 +268,22 @@ bool HhbcTranslator::isInlining() const { return m_bcStateStack.size() > 1; } -void HhbcTranslator::emitMarker() { +IRInstruction* HhbcTranslator::makeMarker(Offset bcOff) { int32_t stackOff = m_tb->getSpOffset() + m_evalStack.numCells() - m_stackDeficit; FTRACE(2, "emitMarker: bc {} sp {} fn {}\n", - bcOff(), stackOff, getCurFunc()->fullName()->data()); + bcOff, stackOff, getCurFunc()->fullName()->data()); MarkerData marker; - marker.bcOff = bcOff(); + marker.bcOff = bcOff; marker.func = getCurFunc(); marker.stackOff = stackOff; - gen(Marker, marker); + return m_irFactory.gen(Marker, marker); +} + +void HhbcTranslator::emitMarker() { + m_tb->add(makeMarker(bcOff())); } void HhbcTranslator::profileFunctionEntry(const char* category) { @@ -518,7 +522,6 @@ void HhbcTranslator::emitCns(uint32_t id) { not_reached(); } } else { - spillStack(); // do this on main trace so we update stack tracking once. SSATmp* c1 = gen(LdCns, cnsType, cnsNameTmp); result = m_tb->cond( getCurFunc(), @@ -530,7 +533,7 @@ void HhbcTranslator::emitCns(uint32_t id) { }, [&] { // Taken: miss in TC, do lookup & init m_tb->hint(Block::Unlikely); - return gen(LookupCns, cnsType, cnsNameTmp); + return gen(LookupCns, getCatchTrace(), cnsType, cnsNameTmp); } ); } @@ -639,8 +642,9 @@ void HhbcTranslator::emitCGetL(int32_t id) { void HhbcTranslator::emitCGetL2(int32_t id) { Trace* exitTrace = getExitTrace(); + Trace* catchTrace = getCatchTrace(); SSATmp* oldTop = pop(Type::Gen); - pushIncRef(ldLocInnerWarn(id, exitTrace)); + pushIncRef(ldLocInnerWarn(id, exitTrace, catchTrace)); push(oldTop); } @@ -827,7 +831,6 @@ SSATmp* HhbcTranslator::emitIterInitCommon(int offset, Lambda genFunc) { if (!type.isArray() && type != Type::Obj) { PUNT(IterInit); } - spillStack(); SSATmp* res = genFunc(src); return emitJmpCondHelper(offset, true, res); } @@ -1024,7 +1027,10 @@ void HhbcTranslator::emitCreateCont(bool getArgs, } void HhbcTranslator::emitContEnter(int32_t returnBcOffset) { - spillStack(); + // The stack should always be clean here; this only appears in generated + // methods we control. + assert(m_evalStack.size() == 0); + assert(m_stackDeficit == 0); assert(getCurClass()); SSATmp* cont = gen(LdThis, m_tb->getFp()); @@ -1293,7 +1299,7 @@ SSATmp* HhbcTranslator::emitLdClsPropAddrOrExit(const StringData* propName, return emitLdClsPropAddrCached(propName, block); } - if (!block) exceptionBarrier(); + if (!block) block = getCatchTrace()->front(); SSATmp* clsTmp = popA(); SSATmp* prop = getStrName(propName); @@ -1466,13 +1472,14 @@ void HhbcTranslator::emitJmpNZ(Offset taken) { } void HhbcTranslator::emitCmp(Opcode opc) { + Trace* catchTrace = nullptr; if (cmpOpTypesMayReenter(opc, topC(0)->type(), topC(1)->type())) { - exceptionBarrier(); + catchTrace = getCatchTrace(); } // src2 opc src1 SSATmp* src1 = popC(); SSATmp* src2 = popC(); - push(gen(opc, src2, src1)); + push(gen(opc, catchTrace, src2, src1)); gen(DecRef, src2); gen(DecRef, src1); } @@ -1494,7 +1501,6 @@ void HhbcTranslator::emitClsCnsD(int32_t cnsNameStrId, int32_t clsNameStrId) { // if-then-else // todo: t2068502: refine the type? hhbc spec says null|bool|int|dbl|str // and, str should always be static-str. - exceptionBarrier(); // do on main trace so we update stack tracking once. Type cnsType = Type::Cell; SSATmp* c1 = gen(LdClsCns, cnsType, cnsNameTmp, clsNameTmp); SSATmp* result = m_tb->cond(getCurFunc(), @@ -1506,7 +1512,8 @@ void HhbcTranslator::emitClsCnsD(int32_t cnsNameStrId, int32_t clsNameStrId) { }, [&] { // Taken: miss in TC, do lookup & init m_tb->hint(Block::Unlikely); - return gen(LookupClsCns, cnsType, cnsNameTmp, clsNameTmp); + return gen(LookupClsCns, getCatchTrace(), + cnsType, cnsNameTmp, clsNameTmp); } ); push(result); @@ -1663,13 +1670,14 @@ void HhbcTranslator::emitFPushActRec(SSATmp* func, void HhbcTranslator::emitFPushCtorCommon(SSATmp* cls, SSATmp* obj, const Func* func, - int32_t numParams) { + int32_t numParams, + Trace* catchTrace) { push(obj); SSATmp* fn = nullptr; if (func) { fn = cns(func); } else { - fn = gen(LdClsCtor, cls); + fn = gen(LdClsCtor, catchTrace, cls); } SSATmp* obj2 = gen(IncRef, obj); int32_t numArgsAndCtorFlag = ActRec::encodeNumArgs(numParams, true); @@ -1677,10 +1685,10 @@ void HhbcTranslator::emitFPushCtorCommon(SSATmp* cls, } void HhbcTranslator::emitFPushCtor(int32_t numParams) { + Trace* catchTrace = getCatchTrace(); SSATmp* cls = popA(); - exceptionBarrier(); SSATmp* obj = gen(IncRef, gen(AllocObj, cls)); - emitFPushCtorCommon(cls, obj, nullptr, numParams); + emitFPushCtorCommon(cls, obj, nullptr, numParams, catchTrace); } static bool canInstantiateClass(const Class* cls) { @@ -1690,7 +1698,10 @@ static bool canInstantiateClass(const Class* cls) { void HhbcTranslator::emitFPushCtorD(int32_t numParams, int32_t classNameStrId) { const StringData* className = lookupStringId(classNameStrId); - exceptionBarrier(); + // The code generated for the catch trace depends on the environment at the + // call so we can't share them between instructions. + Trace* catchTrace1 = getCatchTrace(); + Trace* catchTrace2 = getCatchTrace(); const Class* cls = Unit::lookupUniqueClass(className); bool uniqueCls = classIsUnique(cls); @@ -1716,7 +1727,7 @@ void HhbcTranslator::emitFPushCtorD(int32_t numParams, int32_t classNameStrId) { if (persistentCls) { clss = cns(cls); } else { - clss = gen(LdClsCached, cns(className)); + clss = gen(LdClsCached, catchTrace1, cns(className)); } SSATmp* obj = nullptr; @@ -1726,7 +1737,7 @@ void HhbcTranslator::emitFPushCtorD(int32_t numParams, int32_t classNameStrId) { obj = gen(IncRef, gen(AllocObj, clss)); } - emitFPushCtorCommon(clss, obj, func, numParams); + emitFPushCtorCommon(clss, obj, func, numParams, catchTrace2); } /* @@ -1767,11 +1778,12 @@ void HhbcTranslator::emitFPushFuncD(int32_t numParams, int32_t funcId) { const bool immutable = func->isNameBindingImmutable(getCurUnit()); + Trace* catchTrace = nullptr; if (!immutable) { - exceptionBarrier(); // LdFuncCached can reenter + catchTrace = getCatchTrace(); // LdFuncCached can throw } SSATmp* ssaFunc = immutable ? cns(func) - : gen(LdFuncCached, cns(name)); + : gen(LdFuncCached, catchTrace, cns(name)); emitFPushActRec(ssaFunc, m_tb->genDefInitNull(), numParams, @@ -1795,8 +1807,7 @@ void HhbcTranslator::emitFPushFunc(int32_t numParams) { } void HhbcTranslator::emitFPushFunc(int32_t numParams, SSATmp* funcName) { - exceptionBarrier(); // LdFunc can reenter - emitFPushActRec(gen(LdFunc, funcName), + emitFPushActRec(gen(LdFunc, getCatchTrace(), funcName), m_tb->genDefInitNull(), numParams, nullptr); @@ -2019,9 +2030,11 @@ void HhbcTranslator::emitFCallBuiltin(uint32_t numArgs, // 1. some of the arguments may be passed by reference, for which // case we will pass a stack address. // 2. type conversions of the arguments (using tvCast* helpers) - // may throw an exception, so we need to have the VM stack - // in a clean state at that point. - exceptionBarrier(); + // may throw an exception, so we either need to have the VM stack + // in a clean state at that point or give each helper a catch + // trace. Since we have to spillstack anyway, the catch trace + // would be overkill. + spillStack(); // Convert types if needed. for (int i = 0; i < numNonDefault; i++) { @@ -2046,30 +2059,31 @@ void HhbcTranslator::emitFCallBuiltin(uint32_t numArgs, } // Pass arguments for CallBuiltin. - SSATmp* args[numArgs + 1]; + const int argsSize = numArgs + 2; + SSATmp* args[argsSize]; args[0] = cns(callee); + args[1] = m_tb->getSp(); for (int i = numArgs - 1; i >= 0; i--) { const Func::ParamInfo& pi = callee->params()[i]; switch (pi.builtinType()) { case KindOfBoolean: case KindOfInt64: - args[i + 1] = top(Type::fromDataType(pi.builtinType(), KindOfInvalid), + args[i + 2] = top(Type::fromDataType(pi.builtinType(), KindOfInvalid), numArgs - i - 1); break; case KindOfDouble: assert(false); default: - args[i + 1] = ldStackAddr(numArgs - i - 1); + args[i + 2] = ldStackAddr(numArgs - i - 1); break; } } // Generate call and set return type - SSATmp** decayedPtr = args; auto const ret = gen( CallBuiltin, Type::fromDataTypeWithRef(callee->returnType(), (callee->attrs() & ClassInfo::IsReference)), - std::make_pair(numArgs + 1, decayedPtr) + std::make_pair(argsSize, (SSATmp**)&args) ); // Decref and free args @@ -2263,12 +2277,12 @@ void HhbcTranslator::emitSwitch(const ImmVector& iv, switchVal, ssabase, ssatargets); } else if (type.subtypeOf(Type::Obj)) { // switchObjHelper can throw exceptions and reenter the VM + Trace* catchTrace = nullptr; if (type.subtypeOf(Type::Obj)) { - exceptionBarrier(); + catchTrace = getCatchTrace(); } bounded = false; - index = gen(LdSwitchObjIndex, - switchVal, ssabase, ssatargets); + index = gen(LdSwitchObjIndex, catchTrace, switchVal, ssabase, ssatargets); } else if (type.subtypeOf(Type::Arr)) { gen(DecRef, switchVal); gen(Jmp_, getExitTrace(defaultOff)); @@ -2315,8 +2329,9 @@ void HhbcTranslator::emitSSwitch(const ImmVector& iv) { } ); + Trace* catchTrace = nullptr; // The slow path can throw exceptions and reenter the VM. - if (!fastPath) exceptionBarrier(); + if (!fastPath) catchTrace = getCatchTrace(); auto const testVal = popC(); @@ -2334,9 +2349,10 @@ void HhbcTranslator::emitSSwitch(const ImmVector& iv) { data.defaultOff = bcOff() + iv.strvec()[iv.size() - 1].dest; SSATmp* dest = gen(fastPath ? LdSSwitchDestFast - : LdSSwitchDestSlow, - data, - testVal); + : LdSSwitchDestSlow, + catchTrace, + data, + testVal); gen(DecRef, testVal); auto const stack = spillStack(); gen(SyncABIRegs, m_tb->getFp(), stack); @@ -2458,9 +2474,8 @@ void HhbcTranslator::emitVerifyParamType(int32_t paramId) { return; } if (tc.isCallable()) { - exceptionBarrier(); locVal = gen(Unbox, getExitTrace(), locVal); - gen(VerifyParamCallable, locVal, cns(paramId)); + gen(VerifyParamCallable, getCatchTrace(), locVal, cns(paramId)); return; } @@ -2471,8 +2486,7 @@ void HhbcTranslator::emitVerifyParamType(int32_t paramId) { locVal = gen(LdRef, locVal->type().innerType(), getExitTrace(), locVal); } if (!tc.checkPrimitive(locType.toDataType())) { - exceptionBarrier(); - gen(VerifyParamFail, cns(paramId)); + gen(VerifyParamFail, getCatchTrace(), cns(paramId)); return; } return; @@ -2507,8 +2521,7 @@ void HhbcTranslator::emitVerifyParamType(int32_t paramId) { } else { // The hint was self or parent and there's no corresponding // class for the current func. This typehint will always fail. - exceptionBarrier(); - gen(VerifyParamFail, cns(paramId)); + gen(VerifyParamFail, getCatchTrace(), cns(paramId)); return; } } @@ -2528,23 +2541,22 @@ void HhbcTranslator::emitVerifyParamType(int32_t paramId) { SSATmp* isInstance = haveBit ? gen(InstanceOfBitmask, objClass, cns(clsName)) : gen(ExtendsClass, objClass, constraint); - exceptionBarrier(); m_tb->ifThen(getCurFunc(), [&](Block* taken) { gen(JmpZero, taken, isInstance); }, [&] { // taken: the param type does not match m_tb->hint(Block::Unlikely); - gen(VerifyParamFail, cns(paramId)); + gen(VerifyParamFail, getCatchTrace(), cns(paramId)); } ); } else { - exceptionBarrier(); gen(VerifyParamCls, - objClass, - constraint, - cns(paramId), - cns(uintptr_t(&tc))); + getCatchTrace(), + objClass, + constraint, + cns(paramId), + cns(uintptr_t(&tc))); } } @@ -2651,6 +2663,7 @@ void HhbcTranslator::emitCastBool() { } void HhbcTranslator::emitCastDouble() { + Trace* catchTrace = getCatchTrace(); SSATmp* src = popC(); Type fromType = src->type(); if (fromType.isDbl()) { @@ -2667,15 +2680,14 @@ void HhbcTranslator::emitCastDouble() { } else if (fromType.isString()) { push(gen(ConvStrToDbl, src)); } else if (fromType.isObj()) { - exceptionBarrier(); - push(gen(ConvObjToDbl, src)); + push(gen(ConvObjToDbl, catchTrace, src)); } else { - exceptionBarrier(); // may throw - push(gen(ConvCellToDbl, src)); + push(gen(ConvCellToDbl, catchTrace, src)); } } void HhbcTranslator::emitCastInt() { + Trace* catchTrace = getCatchTrace(); SSATmp* src = popC(); Type fromType = src->type(); if (fromType.isInt()) { @@ -2693,11 +2705,9 @@ void HhbcTranslator::emitCastInt() { push(gen(ConvStrToInt, src)); gen(DecRef, src); } else if (fromType.isObj()) { - exceptionBarrier(); - push(gen(ConvObjToInt, src)); + push(gen(ConvObjToInt, catchTrace, src)); } else { - exceptionBarrier(); - push(gen(ConvCellToInt, src)); + push(gen(ConvCellToInt, catchTrace, src)); } } @@ -2712,6 +2722,7 @@ void HhbcTranslator::emitCastObject() { } void HhbcTranslator::emitCastString() { + Trace* catchTrace = getCatchTrace(); SSATmp* src = popC(); Type fromType = src->type(); if (fromType.isString()) { @@ -2728,11 +2739,9 @@ void HhbcTranslator::emitCastString() { } else if (fromType.isInt()) { push(gen(ConvIntToStr, src)); } else if (fromType.isObj()) { - exceptionBarrier(); - push(gen(ConvObjToStr, src)); + push(gen(ConvObjToStr, catchTrace, src)); } else { - exceptionBarrier(); - push(gen(ConvCellToStr, src)); + push(gen(ConvCellToStr, catchTrace, src)); } } @@ -2778,8 +2787,7 @@ void HhbcTranslator::emitBindMem(SSATmp* ptr, SSATmp* src) { gen(StMem, ptr, cns(0), src); if (isRefCounted(src) && src->type().canRunDtor()) { Block* exitBlock = getExitTrace(nextSrcKey().offset())->front(); - Block::iterator markerInst = exitBlock->skipLabel(); - exitBlock->insert(++markerInst, m_irFactory.gen(DecRef, prevValue)); + exitBlock->prepend(m_irFactory.gen(DecRef, prevValue)); gen(DecRefNZOrBranch, exitBlock, prevValue); } else { gen(DecRef, prevValue); @@ -3076,7 +3084,8 @@ void HhbcTranslator::emitXor() { */ void HhbcTranslator::emitInterpOne(Type type, int numPopped, int numExtraPushed) { - exceptionBarrier(); + // We're calling into the interpreter so we want the stack synced to memory. + SSATmp* sp = spillStack(); // discard the top elements of the stack, which are consumed by this instr discard(numPopped); assert(numPopped == m_stackDeficit); @@ -3085,7 +3094,7 @@ void HhbcTranslator::emitInterpOne(Type type, int numPopped, InterpOne, type, m_tb->getFp(), - m_tb->getSp(), + sp, cns(bcOff()), cns(numPopped - numPushed) ); @@ -3093,18 +3102,20 @@ void HhbcTranslator::emitInterpOne(Type type, int numPopped, } void HhbcTranslator::emitInterpOneCF(int numPopped) { - exceptionBarrier(); + // We're calling into the interpreter so we want the stack synced to memory. + SSATmp* sp = spillStack(); // discard the top elements of the stack, which are consumed by this instr discard(numPopped); assert(numPopped == m_stackDeficit); - gen(InterpOneCF, m_tb->getFp(), m_tb->getSp(), cns(bcOff())); + gen(InterpOneCF, m_tb->getFp(), sp, cns(bcOff())); m_stackDeficit = 0; m_hasExit = true; } /* * Get SSATmps representing all the information on the virtual eval - * stack in preparation for a spill or exit trace. + * stack in preparation for a spill or exit trace. Top of stack will + * be at index 0. * * Doesn't actually remove these values from the eval stack. */ @@ -3196,15 +3207,45 @@ Trace* HhbcTranslator::getExitTraceImpl(Offset targetBcOff, return exit; } +/* + * Create a catch trace for the current state of the eval stack. This is a + * trace intended to be invoked by the unwinder while unwinding a frame + * containing a call to C++ from translated code. When attached to an + * instruction as its taken field, code will be generated and the trace will be + * registered with the unwinder automatically. + */ +Trace* HhbcTranslator::getCatchTrace() { + auto exit = m_tb->makeExitTrace(bcOff()); + assert(exit->blocks().size() == 1); + + genFor(exit, BeginCatch); + exit->front()->push_back(makeMarker(bcOff())); + auto sp = emitSpillStack(exit, m_tb->getSp(), getSpillValues()); + genFor(exit, EndCatch, sp); + + assert(exit->blocks().size() == 1); + return exit; +} + +SSATmp* HhbcTranslator::emitSpillStack(Trace* t, SSATmp* sp, + const std::vector& spillVals) { + std::vector ssaArgs{ sp, cns(int64_t(m_stackDeficit)) }; + ssaArgs.insert(ssaArgs.end(), spillVals.begin(), spillVals.end()); + + auto args = std::make_pair(ssaArgs.size(), &ssaArgs[0]); + if (t->isMain()) { + return gen(SpillStack, args); + } else { + return genFor(t, SpillStack, args); + } +} + SSATmp* HhbcTranslator::spillStack() { - auto ssaArgs = getSpillValues(); - ssaArgs.insert( - ssaArgs.begin(), - { m_tb->getSp(), cns(int64_t(m_stackDeficit)) } - ); + auto newSp = + emitSpillStack(m_tb->trace(), m_tb->getSp(), getSpillValues()); m_evalStack.clear(); m_stackDeficit = 0; - return gen(SpillStack, std::make_pair(ssaArgs.size(), &ssaArgs[0])); + return newSp; } void HhbcTranslator::exceptionBarrier() { @@ -3260,15 +3301,18 @@ SSATmp* HhbcTranslator::ldLocInner(uint32_t locId, Trace* exitTrace) { } /* - * This is a wrapper to ldLocInner that also emits the RaiseUninitLoc - * if the local is uninitialized + * This is a wrapper to ldLocInner that also emits the RaiseUninitLoc if the + * local is uninitialized. The catchTrace argument may be provided if the + * caller requires the catch trace to be generated at a point earlier than when + * it calls this function. */ -SSATmp* HhbcTranslator::ldLocInnerWarn(uint32_t id, Trace* target) { +SSATmp* HhbcTranslator::ldLocInnerWarn(uint32_t id, Trace* target, + Trace* catchTrace /* = nullptr */) { + if (!catchTrace) catchTrace = getCatchTrace(); auto const locVal = ldLocInner(id, target); if (locVal->type().subtypeOf(Type::Uninit)) { - exceptionBarrier(); - gen(RaiseUninitLoc, cns(getCurFunc()->localVarName(id))); + gen(RaiseUninitLoc, catchTrace, cns(getCurFunc()->localVarName(id))); return m_tb->genDefInitNull(); } diff --git a/hphp/runtime/vm/translator/hopt/hhbctranslator.h b/hphp/runtime/vm/translator/hopt/hhbctranslator.h index 1ff1472aa..a35766249 100644 --- a/hphp/runtime/vm/translator/hopt/hhbctranslator.h +++ b/hphp/runtime/vm/translator/hopt/hhbctranslator.h @@ -261,7 +261,8 @@ struct HhbcTranslator { void emitFPushCtorCommon(SSATmp* cls, SSATmp* obj, const Func* func, - int32_t numParams); + int32_t numParams, + Trace* catchTrace); void emitCreateCl(int32_t numParams, int32_t classNameStrId); void emitFCallArray(const Offset pcOffset, const Offset after); void emitFCall(uint32_t numParams, @@ -394,6 +395,7 @@ private: void emitMPre(); void emitFinalMOp(); void emitMPost(); + void emitSideExits(SSATmp* catchSp, int nStack); void emitMTrace(); // Bases @@ -455,13 +457,12 @@ private: * if appropriate. */ template - SSATmp* genStk(Opcode op, Srcs... srcs); + SSATmp* genStk(Opcode op, Trace* taken, Srcs... srcs); /* Various predicates about the current instruction */ bool isSimpleArrayOp(); bool isSimpleBase(); bool isSingleMember(); - bool usePredictedResult(); bool generateMVal() const; bool needFirstRatchet() const; @@ -482,6 +483,7 @@ private: const Transl::NormalizedInstruction& m_ni; HhbcTranslator& m_ht; TraceBuilder& m_tb; + IRFactory& m_irf; const MInstrInfo& m_mii; hphp_hash_map m_stackInputs; @@ -490,10 +492,29 @@ private: bool m_needMIS; + /* The base for any accesses to the current MInstrState. */ SSATmp* m_misBase; + + /* The value of the base for the next member operation. Starts as the base + * for the whole instruction and is updated as the translator makes + * progress. */ SSATmp* m_base; + + /* The result of the vector instruction. nullptr if the current instruction + * doesn't produce a result. */ SSATmp* m_result; - SSATmp* m_predictedResult; + + /* If set, contains a value of type CountedStr|Nullptr. If a runtime test + * determines that the value is not Nullptr, we incorrectly predicted the + * output type of the instruction and must side exit. */ + SSATmp* m_strTestResult; + + /* If set, contains the catch trace for the final set operation of this + * instruction. The operations that set this member may need to return an + * unexpected type, in which case they'll throw an InvalidSetMException. To + * handle this, emitMPost adds code to the catch trace to fish the correct + * value out of the exception and side exit. */ + Trace* m_failedSetTrace; }; private: // tracebuilder forwarding utilities @@ -571,6 +592,7 @@ private: void emitBinaryArith(Opcode); template SSATmp* emitIterInitCommon(int offset, Lambda genFunc); + IRInstruction* makeMarker(Offset bcOff); void emitMarker(); // Exit trace creation routines. @@ -581,6 +603,7 @@ private: std::vector& spillValues, const StringData* warning); Trace* getExitSlowTrace(); + Trace* getCatchTrace(); enum class ExitFlag { None, @@ -636,6 +659,8 @@ private: SSATmp* popF() { return pop(Type::Gen); } SSATmp* topC(uint32_t i = 0) { return top(Type::Cell, i); } std::vector getSpillValues() const; + SSATmp* emitSpillStack(Trace* t, SSATmp* sp, + const std::vector& spillVals); SSATmp* spillStack(); void exceptionBarrier(); SSATmp* ldStackAddr(int32_t offset); @@ -650,7 +675,8 @@ private: SSATmp* ldLoc(uint32_t id); SSATmp* ldLocAddr(uint32_t id); SSATmp* ldLocInner(uint32_t id, Trace* exitTrace); - SSATmp* ldLocInnerWarn(uint32_t id, Trace* target); + SSATmp* ldLocInnerWarn(uint32_t id, Trace* target, + Trace* catchTrace = nullptr); SSATmp* stLoc(uint32_t id, Trace* exitTrace, SSATmp* newVal); SSATmp* stLocNRC(uint32_t id, Trace* exitTrace, SSATmp* newVal); SSATmp* stLocImpl(uint32_t id, Trace*, SSATmp* newVal, bool doRefCount); diff --git a/hphp/runtime/vm/translator/hopt/ir.cpp b/hphp/runtime/vm/translator/hopt/ir.cpp index a261fe947..d5b0ef237 100644 --- a/hphp/runtime/vm/translator/hopt/ir.cpp +++ b/hphp/runtime/vm/translator/hopt/ir.cpp @@ -108,10 +108,11 @@ namespace { #define DParam HasDest #define DArith HasDest #define DMulti NaryDest -#define DVector HasDest +#define DSetElem HasDest #define DStk(x) ModifiesStack|(x) #define DPtrToParam HasDest #define DBuiltin HasDest +#define DSubtract(n,t) HasDest struct { const char* name; @@ -151,10 +152,11 @@ struct { #undef DParam #undef DArith #undef DMulti -#undef DVector +#undef DSetElem #undef DStk #undef DPtrToParam #undef DBuiltin +#undef DSubtract ////////////////////////////////////////////////////////////////////// @@ -408,7 +410,7 @@ bool IRInstruction::mayModifyRefs() const { // count. Therefore, its MayModifyRefs should be false. if (opc == DecRef) { auto type = src(0)->type(); - if (isControlFlowInstruction()) { + if (isControlFlow()) { // If the decref has a target label, then it exits if the destructor // has to be called, so it does not have any side effects on the main // trace. @@ -437,7 +439,7 @@ bool IRInstruction::isEssential() const { return true; } } - return isControlFlowInstruction() || + return isControlFlow() || opcodeHasFlags(opc, Essential) || mayReenterHelper(); } @@ -778,10 +780,9 @@ Opcode commuteQueryOp(Opcode opc) { // __toString function. bool cmpOpTypesMayReenter(Opcode op, Type t0, Type t1) { if (op == OpNSame || op == OpSame) return false; - assert(t0 != Type::Gen && t1 != Type::Gen); - return (t0 == Type::Cell || t1 == Type::Cell) || - ((t0 == Type::Obj || t1 == Type::Obj) && - (t0.isString() || t1.isString())); + assert(!t0.equals(Type::Gen) && !t1.equals(Type::Gen)); + return (t0.maybe(Type::Obj) && t1.maybe(Type::Str)) || + (t0.maybe(Type::Str) && t1.maybe(Type::Obj)); } bool isRefCounted(SSATmp* tmp) { @@ -810,19 +811,20 @@ void IRInstruction::convertToNop() { } void IRInstruction::convertToJmp() { - assert(isControlFlowInstruction()); - assert(block()->back() == this); + assert(isControlFlow()); + assert(IMPLIES(block(), block()->back() == this)); m_op = Jmp_; m_typeParam = Type::None; m_numSrcs = 0; m_numDsts = 0; m_srcs = nullptr; m_dst = nullptr; - block()->setNext(nullptr); + // Instructions in the simplifier don't have blocks yet. + if (block()) block()->setNext(nullptr); } void IRInstruction::convertToMov() { - assert(!isControlFlowInstruction()); + assert(!isControlFlow()); m_op = Mov; m_typeParam = Type::None; assert(m_numSrcs == 1); @@ -916,9 +918,12 @@ std::string IRInstruction::toString() const { return str.str(); } -int SSATmp::numNeededRegs() const { - auto t = type(); - if (t.subtypeOfAny(Type::None, Type::Null, Type::ActRec, Type::RetAddr)) { +namespace { +int typeNeededRegs(Type t) { + assert(!t.equals(Type::Bottom)); + + if (t.subtypeOfAny(Type::None, Type::Null, Type::ActRec, Type::RetAddr, + Type::Nullptr)) { // These don't need a register because their values are static or unused. // // RetAddr doesn't take any register because currently we only target x86, @@ -926,6 +931,9 @@ int SSATmp::numNeededRegs() const { // moved to a machine-specific section once we target other architectures. return 0; } + if (t.maybe(Type::Nullptr)) { + return typeNeededRegs(t - Type::Nullptr); + } if (t.subtypeOf(Type::Ctx) || t.isPtr()) { // Ctx and PtrTo* may be statically unknown but always need just 1 register. return 1; @@ -944,6 +952,11 @@ int SSATmp::numNeededRegs() const { assert(t.subtypeOf(Type::Gen)); return t.needsReg() ? 2 : 1; } +} + +int SSATmp::numNeededRegs() const { + return typeNeededRegs(type()); +} bool SSATmp::getValBool() const { assert(isConst()); diff --git a/hphp/runtime/vm/translator/hopt/ir.h b/hphp/runtime/vm/translator/hopt/ir.h index eb35f7901..531556916 100644 --- a/hphp/runtime/vm/translator/hopt/ir.h +++ b/hphp/runtime/vm/translator/hopt/ir.h @@ -98,13 +98,15 @@ class FailedIRGen : public std::exception { * DParam single dst has type of the instruction's type parameter * DArith single dst has a type based on arithmetic type rules * DMulti multiple dests. type and number depend on instruction - * DVector single dst depends on semantics of the vector instruction + * DSetElem single dst is a subset of CountedStr|Nullptr depending on + * sources * DStk(x) up to two dests. x should be another D* macro and indicates * the type of the first dest, if any. the second (or first, * depending on the presence of a primary destination), will be * of type Type::StkPtr. implies ModifiesStack. * DBuiltin single dst for CallBuiltin. This can return complex data * types such as (Type::Str | Type::Null) + * DSubtract(N,t) single dest has type of src N with t removed * * srcinfo: * @@ -153,7 +155,7 @@ class FailedIRGen : public std::exception { #define IR_OPCODES \ /* name dstinfo srcinfo flags */ \ -O(CheckType, DParam, S(Gen), C|E|CRc|PRc|P) \ +O(CheckType, DParam, S(Gen,Nullptr), C|E|CRc|PRc|P) \ O(CheckTypeMem, ND, S(PtrToGen), E) \ O(GuardLoc, ND, S(FramePtr), E) \ O(GuardStk, D(StkPtr), S(StkPtr), E) \ @@ -169,6 +171,10 @@ O(GuardRefs, ND, S(Func) \ S(Int), E) \ O(AssertLoc, ND, S(FramePtr), E) \ O(OverrideLoc, ND, S(FramePtr), E) \ +O(BeginCatch, ND, NA, E|Mem) \ +O(EndCatch, ND, S(StkPtr), E|Mem) \ +O(LdUnwinderValue, DParam, NA, PRc) \ +O(DeleteUnwinderException, ND, NA, N|E|Mem) \ O(OpAdd, DArith, S(Int,Dbl) S(Int,Dbl), C) \ O(OpSub, DArith, S(Int,Dbl) S(Int,Dbl), C) \ O(OpMul, DArith, S(Int,Dbl) S(Int,Dbl), C) \ @@ -269,10 +275,11 @@ O(JmpIndirect, ND, S(TCA), T|E) \ O(ExitWhenSurprised, ND, NA, E) \ O(ExitOnVarEnv, ND, S(FramePtr), E) \ O(ReleaseVVOrExit, ND, S(FramePtr), N|E) \ -O(RaiseError, ND, S(Str), E|N|Mem|Refs|T) \ +O(RaiseError, ND, S(Str), E|N|Mem|Refs|T|Er) \ O(RaiseWarning, ND, S(Str), E|N|Mem|Refs|Er) \ O(CheckInit, ND, S(Gen), NF) \ O(CheckInitMem, ND, S(PtrToGen) C(Int), NF) \ +O(AssertNonNull, DSubtract(0, Nullptr), S(Nullptr,CountedStr), NF) \ O(Unbox, DUnbox(0), S(Gen), NF) \ O(Box, DBox(0), S(Init), E|N|Mem|CRc|PRc) \ O(UnboxPtr, D(PtrToCell), S(PtrToGen), NF) \ @@ -505,7 +512,7 @@ O_STK(BindProp, ND, C(TCA) \ S(Cell) \ S(BoxedCell) \ S(PtrToCell),VProp|E|N|Mem|Refs|Er) \ -O_STK(SetProp, DVector, C(TCA) \ +O_STK(SetProp, ND, C(TCA) \ C(Cls) \ S(Obj,PtrToGen) \ S(Cell) \ @@ -570,7 +577,7 @@ O(ArraySetRef, ND, C(TCA) \ S(Int,Str) \ S(Cell) \ S(BoxedArr),E|N|PRc|CRc|Refs|Mem|K) \ -O_STK(SetElem, DVector, C(TCA) \ +O_STK(SetElem, DSetElem, C(TCA) \ S(PtrToGen) \ S(Cell) \ S(Cell), VElem|E|N|Mem|Refs|Er) \ @@ -591,7 +598,7 @@ O_STK(IncDecElem, D(Cell), C(TCA) \ S(PtrToGen) \ S(Cell) \ S(PtrToCell),VElem|E|N|Mem|Refs|Er) \ -O_STK(SetNewElem, DVector, S(PtrToGen) \ +O_STK(SetNewElem, ND, S(PtrToGen) \ S(Cell), VElem|E|N|Mem|Refs|Er) \ O_STK(SetWithRefNewElem, ND, C(TCA) \ S(PtrToGen) \ @@ -899,15 +906,32 @@ struct VectorEffects { VectorEffects(Opcode opc, const std::vector& srcs); Type baseType; - Type valType; bool baseTypeChanged; bool baseValChanged; - bool valTypeChanged; private: void init(Opcode op, const Type base, const Type key, const Type val); }; +struct CatchInfo { + /* afterCall is the address after the call instruction that this catch trace + * belongs to. It's the key used to look up catch traces by the + * unwinder, since it's the value of %rip during unwinding. */ + TCA afterCall; + + /* savedRegs contains the caller-saved registers that were pushed onto the + * C++ stack at the time of the call. The catch trace will pop these + * registers (in the same order as PhysRegSaver's destructor) before doing + * any real work to restore the register state from before the call. */ + RegSet savedRegs; + + /* rspOffset is the number of bytes pushed on the C++ stack after the + * registers in savedRegs were saved, typically from function calls with >6 + * arguments. The catch trace will adjust rsp by this amount before popping + * anything in savedRegs. */ + Offset rspOffset; +}; + typedef folly::Range TcaRange; /** diff --git a/hphp/runtime/vm/translator/hopt/irinstruction.h b/hphp/runtime/vm/translator/hopt/irinstruction.h index cbc7c438a..d244a4018 100644 --- a/hphp/runtime/vm/translator/hopt/irinstruction.h +++ b/hphp/runtime/vm/translator/hopt/irinstruction.h @@ -243,7 +243,7 @@ struct IRInstruction { else m_taken.setTo(b); } - bool isControlFlowInstruction() const { return bool(m_taken.to()); } + bool isControlFlow() const { return bool(m_taken.to()); } bool isBlockEnd() const { return m_taken.to() || isTerminal(); } bool isLoad() const; bool stores(uint32_t srcIdx) const; diff --git a/hphp/runtime/vm/translator/hopt/irtranslator.cpp b/hphp/runtime/vm/translator/hopt/irtranslator.cpp index 36ed4d606..78acba083 100644 --- a/hphp/runtime/vm/translator/hopt/irtranslator.cpp +++ b/hphp/runtime/vm/translator/hopt/irtranslator.cpp @@ -1977,8 +1977,8 @@ void TranslatorX64::hhirTraceCodeGen(vector* bcMap) { auto finishPass = [&](const char* msg, int level, const RegAllocInfo* regs = nullptr, const LifetimeInfo* lifetime = nullptr) { - assert(checkCfg(trace, *m_irFactory)); dumpTrace(level, trace, msg, regs, lifetime); + assert(checkCfg(trace, *m_irFactory)); }; finishPass(" after initial translation ", kIRLevel); diff --git a/hphp/runtime/vm/translator/hopt/jumpopts.cpp b/hphp/runtime/vm/translator/hopt/jumpopts.cpp index af9927465..22d9f7bdc 100644 --- a/hphp/runtime/vm/translator/hopt/jumpopts.cpp +++ b/hphp/runtime/vm/translator/hopt/jumpopts.cpp @@ -53,7 +53,7 @@ void elimUnconditionalJump(Trace* trace, IRFactory* irFactory) { IRInstruction& jmp = *lastInst; if (jmp.op() == Jmp_ && !isJoin[jmp.taken()->id()]) { Block* target = jmp.taken(); - lastBlock->splice(lastInst, target, target->skipLabel(), target->end()); + lastBlock->splice(lastInst, target, target->skipHeader(), target->end()); lastBlock->erase(lastInst); // delete the jmp } } @@ -92,7 +92,7 @@ Block* findMainExitBlock(Trace* trace, IRFactory* irFactory) { struct BlockMatcher { explicit BlockMatcher(Block* block) : m_block(block) - , m_it(block->skipLabel()) + , m_it(block->skipHeader()) {} bool match() { return true; } diff --git a/hphp/runtime/vm/translator/hopt/mutation.cpp b/hphp/runtime/vm/translator/hopt/mutation.cpp index 662aaf9b9..311c46e03 100644 --- a/hphp/runtime/vm/translator/hopt/mutation.cpp +++ b/hphp/runtime/vm/translator/hopt/mutation.cpp @@ -42,9 +42,9 @@ void cloneToBlock(const BlockList& rpoBlocks, } }; - auto targetIt = target->skipLabel(); + auto targetIt = target->skipHeader(); for (auto it = first; it != last; ++it) { - assert(!it->isControlFlowInstruction()); + assert(!it->isControlFlow()); FTRACE(5, "cloneToBlock({}): {}\n", target->id(), it->toString()); auto const newInst = irFactory->cloneInstruction(&*it); @@ -79,10 +79,10 @@ void moveToBlock(Block::iterator const first, auto const srcBlock = first->block(); - auto targetIt = target->skipLabel(); + auto targetIt = target->skipHeader(); for (auto it = first; it != last;) { auto const inst = &*it; - assert(!inst->isControlFlowInstruction()); + assert(!inst->isControlFlow()); FTRACE(5, "moveToBlock({}): {}\n", target->id(), diff --git a/hphp/runtime/vm/translator/hopt/nativecalls.cpp b/hphp/runtime/vm/translator/hopt/nativecalls.cpp index 65e302ffe..8d0844496 100644 --- a/hphp/runtime/vm/translator/hopt/nativecalls.cpp +++ b/hphp/runtime/vm/translator/hopt/nativecalls.cpp @@ -195,7 +195,7 @@ static CallMap s_callMap({ {{SSA, 1}, {SSA, 2}, {VecKeyS, 3}, {SSA, 4}}}, {BindProp, {FSSA, 0}, DNone, SSync, {{SSA, 1}, {SSA, 2}, {TV, 3}, {SSA, 4}, {SSA, 5}}}, - {SetProp, {FSSA, 0}, DTV, SSync, + {SetProp, {FSSA, 0}, DNone, SSync, {{SSA, 1}, {SSA, 2}, {TV, 3}, {TV, 4}}}, {UnsetProp, {FSSA, 0}, DNone, SSync, {{SSA, 1}, {SSA, 2}, {TV, 3}}}, @@ -227,7 +227,7 @@ static CallMap s_callMap({ {{SSA, 1}, {SSA, 2}, {TV, 3}}}, {ArraySetRef, {FSSA, 0}, DSSA, SSync, {{SSA, 1}, {SSA, 2}, {TV, 3}, {SSA, 4}}}, - {SetElem, {FSSA, 0}, DTV, SSync, + {SetElem, {FSSA, 0}, DSSA, SSync, {{SSA, 1}, {VecKeyIS, 2}, {TV, 3}}}, {UnsetElem, {FSSA, 0}, DNone, SSync, {{SSA, 1}, {VecKeyIS, 2}}}, @@ -235,7 +235,7 @@ static CallMap s_callMap({ {{SSA, 1}, {TV, 2}, {TV, 3}, {SSA, 4}}}, {IncDecElem, {FSSA, 0}, DTV, SSync, {{SSA, 1}, {TV, 2}, {SSA, 3}}}, - {SetNewElem, (TCA)setNewElem, DTV, SSync, {{SSA, 0}, {TV, 1}}}, + {SetNewElem, (TCA)setNewElem, DNone, SSync, {{SSA, 0}, {TV, 1}}}, {SetWithRefNewElem, {FSSA, 0}, DNone, SSync, {{SSA, 1}, {SSA, 2}, {SSA, 3}}}, {BindNewElem, (TCA)bindNewElemIR, DNone, SSync, diff --git a/hphp/runtime/vm/translator/hopt/predictionopts.cpp b/hphp/runtime/vm/translator/hopt/predictionopts.cpp index 557c12d86..a351e7006 100644 --- a/hphp/runtime/vm/translator/hopt/predictionopts.cpp +++ b/hphp/runtime/vm/translator/hopt/predictionopts.cpp @@ -123,7 +123,7 @@ void optimizePredictions(Trace* const trace, IRFactory* const irFactory) { // Clone the instructions to the exit before specializing. cloneToBlock(rpoSort, irFactory, sinkFirst, sinkLast, exit); - exit->insert(exit->skipLabel(), irFactory->cloneInstruction(lastMarker)); + exit->insert(exit->skipHeader(), irFactory->cloneInstruction(lastMarker)); /* * Specialize the LdMem left on the main trace after cloning the @@ -145,7 +145,7 @@ void optimizePredictions(Trace* const trace, IRFactory* const irFactory) { // Move the fallthrough case to specialized. moveToBlock(sinkFirst, boost::next(sinkLast), specialized); - specialized->insert(specialized->skipLabel(), + specialized->insert(specialized->skipHeader(), irFactory->cloneInstruction(lastMarker)); reflowTypes(specialized, rpoSort); diff --git a/hphp/runtime/vm/translator/hopt/print.cpp b/hphp/runtime/vm/translator/hopt/print.cpp index ee482fd12..ecea00d9c 100644 --- a/hphp/runtime/vm/translator/hopt/print.cpp +++ b/hphp/runtime/vm/translator/hopt/print.cpp @@ -132,10 +132,7 @@ void print(std::ostream& ostream, const IRInstruction* inst, if (inst->op() == Marker) { auto* marker = inst->extra(); ostream << color(ANSI_COLOR_BLUE) - << folly::format("--- bc {}, spOff {} ({})", - marker->bcOff, - marker->stackOff, - marker->func->fullName()->data()) + << marker->show() << color(ANSI_COLOR_END); return; } diff --git a/hphp/runtime/vm/translator/hopt/simplifier.cpp b/hphp/runtime/vm/translator/hopt/simplifier.cpp index 4314c7bfa..d72233876 100644 --- a/hphp/runtime/vm/translator/hopt/simplifier.cpp +++ b/hphp/runtime/vm/translator/hopt/simplifier.cpp @@ -264,7 +264,7 @@ SSATmp* Simplifier::simplify(IRInstruction* inst) { case OpNeq: case OpSame: case OpNSame: - return simplifyCmp(opc, src1, src2); + return simplifyCmp(opc, inst, src1, src2); case Concat: return simplifyConcat(src1, src2); case Mov: return simplifyMov(src1); @@ -323,6 +323,7 @@ SSATmp* Simplifier::simplify(IRInstruction* inst) { case DecRefNZ: return simplifyDecRef(inst); case IncRef: return simplifyIncRef(inst); case CheckType: return simplifyCheckType(inst); + case AssertNonNull:return simplifyAssertNonNull(inst); case LdCls: return simplifyLdCls(inst); case LdThis: return simplifyLdThis(inst); @@ -512,7 +513,7 @@ SSATmp* Simplifier::simplifyQueryJmp(IRInstruction* inst) { SSATmp* src2 = inst->src(1); Opcode opc = inst->op(); // reuse the logic in simplifyCmp. - SSATmp* newCmp = simplifyCmp(queryJmpToQueryOp(opc), src1, src2); + SSATmp* newCmp = simplifyCmp(queryJmpToQueryOp(opc), nullptr, src1, src2); if (!newCmp) return nullptr; SSATmp* newQueryJmp = makeInstruction( @@ -932,7 +933,11 @@ static typename std::common_type::type cmpOp(Opcode opName, T a, U b) { } } -SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { +SSATmp* Simplifier::simplifyCmp(Opcode opName, IRInstruction* inst, + SSATmp* src1, SSATmp* src2) { + auto newInst = [inst, this](Opcode op, SSATmp* src1, SSATmp* src2) { + return gen(op, inst ? inst->taken() : (Block*)nullptr, src1, src2); + }; // --------------------------------------------------------------------- // Perform some execution optimizations immediately // --------------------------------------------------------------------- @@ -985,9 +990,9 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { } // Type is neither a string nor an object - simplify to OpEq/OpNeq if (opName == OpSame) { - return gen(OpEq, src1, src2); + return newInst(OpEq, src1, src2); } - return gen(OpNeq, src1, src2); + return newInst(OpNeq, src1, src2); } // --------------------------------------------------------------------- @@ -1047,9 +1052,9 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { // E.g. `some-int > false` is equivalent to `some-int == true` if (opName != OpEq) { if (cmpOp(opName, false, b)) { - return gen(OpEq, src1, cns(false)); + return newInst(OpEq, src1, cns(false)); } else { - return gen(OpEq, src1, cns(true)); + return newInst(OpEq, src1, cns(true)); } } } @@ -1062,7 +1067,7 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { if (src1->type() == src2->type() || (src1->type().isString() && src2->type().isString())) { if (src1->isConst() && !src2->isConst()) { - return gen(commuteQueryOp(opName), src2, src1); + return newInst(commuteQueryOp(opName), src2, src1); } return nullptr; } @@ -1074,32 +1079,32 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { // nulls get canonicalized to the right if (src1->type().isNull()) { - return gen(commuteQueryOp(opName), src2, src1); + return newInst(commuteQueryOp(opName), src2, src1); } // case 1: null cmp string. Convert null to "" if (src1->type().isString() && src2->type().isNull()) { - return gen(opName, src1, cns(StringData::GetStaticString(""))); + return newInst(opName, src1, cns(StringData::GetStaticString(""))); } // case 2a: null cmp anything. Convert null to false if (src2->type().isNull()) { - return gen(opName, src1, cns(false)); + return newInst(opName, src1, cns(false)); } // bools get canonicalized to the right if (src1->type() == Type::Bool) { - return gen(commuteQueryOp(opName), src2, src1); + return newInst(commuteQueryOp(opName), src2, src1); } // case 2b: bool cmp anything. Convert anything to bool if (src2->type() == Type::Bool) { if (src1->isConst()) { if (src1->type() == Type::Int) { - return gen(opName, cns(bool(src1->getValInt())), src2); + return newInst(opName, cns(bool(src1->getValInt())), src2); } else if (src1->type().isString()) { auto str = src1->getValStr(); - return gen(opName, cns(str->toBoolean()), src2); + return newInst(opName, cns(str->toBoolean()), src2); } } @@ -1109,14 +1114,14 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { always_assert(opName == OpEq); if (src2->getValBool()) { - return gen(OpNeq, src1, cns(0)); + return newInst(OpNeq, src1, cns(0)); } else { - return gen(OpEq, src1, cns(0)); + return newInst(OpEq, src1, cns(0)); } } // Nothing fancy to do - perform juggling as normal. - return gen(opName, gen(ConvCellToBool, src1), src2); + return newInst(opName, gen(ConvCellToBool, src1), src2); } // From here on, we must be careful of how Type::Obj gets dealt with, @@ -1127,12 +1132,12 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { // strings get canonicalized to the left if (src2->type().isString()) { - return gen(commuteQueryOp(opName), src2, src1); + return newInst(commuteQueryOp(opName), src2, src1); } // ints get canonicalized to the right if (src1->type() == Type::Int) { - return gen(commuteQueryOp(opName), src2, src1); + return newInst(commuteQueryOp(opName), src2, src1); } // case 4: number/string/resource cmp. Convert to number (int OR double) @@ -1146,12 +1151,12 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { int64_t si; double sd; auto st = str->isNumericWithVal(si, sd, true /* allow errors */); if (st == KindOfDouble) { - return gen(opName, cns(sd), src2); + return newInst(opName, cns(sd), src2); } if (st == KindOfNull) { si = 0; } - return gen(opName, cns(si), src2); + return newInst(opName, cns(si), src2); } // case 5: array cmp array. No juggling to do @@ -1672,6 +1677,15 @@ SSATmp* Simplifier::simplifyDecRefStack(IRInstruction* inst) { return nullptr; } +SSATmp* Simplifier::simplifyAssertNonNull(IRInstruction* inst) { + auto t = inst->typeParam(); + assert(t.maybe(Type::Nullptr)); + if (t.subtypeOf(Type::Nullptr)) { + return inst->src(0); + } + return nullptr; +} + ////////////////////////////////////////////////////////////////////// }} diff --git a/hphp/runtime/vm/translator/hopt/simplifier.h b/hphp/runtime/vm/translator/hopt/simplifier.h index c7e9215cd..8feaea1c4 100644 --- a/hphp/runtime/vm/translator/hopt/simplifier.h +++ b/hphp/runtime/vm/translator/hopt/simplifier.h @@ -115,7 +115,8 @@ private: SSATmp* simplifyGetCtxFwdCall(IRInstruction* inst); SSATmp* simplifySpillStack(IRInstruction* inst); SSATmp* simplifyCall(IRInstruction* inst); - SSATmp* simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2); + SSATmp* simplifyCmp(Opcode opName, IRInstruction* inst, + SSATmp* src1, SSATmp* src2); SSATmp* simplifyCondJmp(IRInstruction*); SSATmp* simplifyQueryJmp(IRInstruction*); SSATmp* simplifyExitOnVarEnv(IRInstruction*); @@ -127,6 +128,7 @@ private: SSATmp* simplifyDecRefLoc(IRInstruction*); SSATmp* simplifyLdLoc(IRInstruction*); SSATmp* simplifyStRef(IRInstruction*); + SSATmp* simplifyAssertNonNull(IRInstruction*); private: // tracebuilder forwarders template SSATmp* cns(Args&&...); diff --git a/hphp/runtime/vm/translator/hopt/test/type.cpp b/hphp/runtime/vm/translator/hopt/test/type.cpp index 3ce3f5663..d9d8d2f59 100644 --- a/hphp/runtime/vm/translator/hopt/test/type.cpp +++ b/hphp/runtime/vm/translator/hopt/test/type.cpp @@ -189,6 +189,7 @@ TEST(Type, CanRunDtor) { expectTrue(Type::Ctx); expectTrue(Type::Obj | Type::Func); expectTrue(Type::Init); + expectTrue(Type::Top); for (Type t : types) { EXPECT_FALSE(t.canRunDtor()) << t.toString() << ".canRunDtor == false"; @@ -202,4 +203,14 @@ TEST(Type, UnionOf) { EXPECT_EQ(Type::Gen, Type::unionOf(Type::Cell, Type::BoxedInt)); } +TEST(Type, Top) { + for (auto t : allTypes()) { + EXPECT_TRUE(t.subtypeOf(Type::Top)); + } + for (auto t : allTypes()) { + if (t.equals(Type::Top)) continue; + EXPECT_FALSE(Type::Top.subtypeOf(t)); + } +} + } } diff --git a/hphp/runtime/vm/translator/hopt/test/vector.cpp b/hphp/runtime/vm/translator/hopt/test/vector.cpp index a6ad6a1f3..cfacc025d 100644 --- a/hphp/runtime/vm/translator/hopt/test/vector.cpp +++ b/hphp/runtime/vm/translator/hopt/test/vector.cpp @@ -30,69 +30,53 @@ namespace HPHP { namespace JIT { TEST(VectorEffects, Basic) { VectorEffects elem(SetElem, Type::PtrToArr, Type::Int, Type::Str); EXPECT_TEQ(Type::PtrToArr, elem.baseType); - EXPECT_TEQ(Type::Str, elem.valType); EXPECT_FALSE(elem.baseTypeChanged); EXPECT_TRUE(elem.baseValChanged); - EXPECT_FALSE(elem.valTypeChanged); VectorEffects prop(SetProp, Type::Obj, Type::StaticStr, Type::Str); EXPECT_TEQ(Type::Obj, prop.baseType); - EXPECT_TEQ(Type::Str, prop.valType); EXPECT_FALSE(prop.baseTypeChanged); EXPECT_FALSE(prop.baseValChanged); - EXPECT_FALSE(prop.valTypeChanged); } TEST(VectorEffects, BadArrayKey) { VectorEffects ve(SetElem, Type::PtrToArr, Type::Arr, Type::Int); EXPECT_TEQ(Type::PtrToArr, ve.baseType); - EXPECT_TEQ(Type::InitNull, ve.valType); EXPECT_FALSE(ve.baseTypeChanged); EXPECT_TRUE(ve.baseValChanged); - EXPECT_TRUE(ve.valTypeChanged); } TEST(VectorEffects, NonObjProp) { VectorEffects ve(SetProp, Type::PtrToInt, Type::Str, Type::Dbl); EXPECT_TEQ(Type::PtrToInt, ve.baseType); - EXPECT_TEQ(Type::InitNull, ve.valType); EXPECT_FALSE(ve.baseTypeChanged); EXPECT_FALSE(ve.baseValChanged); - EXPECT_TRUE(ve.valTypeChanged); } TEST(VectorEffects, NonArrElem) { VectorEffects ve(SetElem, Type::PtrToDbl, Type::Int, Type::Obj); EXPECT_TEQ(Type::PtrToDbl, ve.baseType); - EXPECT_TEQ(Type::InitNull, ve.valType); EXPECT_FALSE(ve.baseTypeChanged); EXPECT_FALSE(ve.baseValChanged); - EXPECT_TRUE(ve.valTypeChanged); } TEST(VectorEffects, PromoteNull) { VectorEffects elem(SetElem, Type::PtrToNull, Type::Int, Type::Str); EXPECT_TEQ(Type::PtrToArr, elem.baseType); - EXPECT_TEQ(Type::Str, elem.valType); EXPECT_TRUE(elem.baseTypeChanged); EXPECT_TRUE(elem.baseValChanged); - EXPECT_FALSE(elem.valTypeChanged); VectorEffects prop(SetProp, Type::PtrToUninit, Type::StaticStr, Type::Str); EXPECT_TEQ(Type::PtrToObj, prop.baseType); - EXPECT_TEQ(Type::Str, prop.valType); EXPECT_TRUE(prop.baseTypeChanged); EXPECT_TRUE(prop.baseValChanged); - EXPECT_FALSE(prop.valTypeChanged); } TEST(VectorEffects, UnknownBase) { VectorEffects ve(SetElem, Type::PtrToCell, Type::Int, Type::Obj); EXPECT_TEQ(Type::PtrToCell, ve.baseType); - EXPECT_TEQ(Type::Obj|Type::InitNull, ve.valType); EXPECT_FALSE(ve.baseTypeChanged); EXPECT_TRUE(ve.baseValChanged); - EXPECT_TRUE(ve.valTypeChanged); } } } diff --git a/hphp/runtime/vm/translator/hopt/trace.h b/hphp/runtime/vm/translator/hopt/trace.h index 53f28313b..c53a8ca23 100644 --- a/hphp/runtime/vm/translator/hopt/trace.h +++ b/hphp/runtime/vm/translator/hopt/trace.h @@ -95,6 +95,15 @@ struct Trace : private boost::noncopyable { return exit; } bool isMain() const { return m_main == nullptr; } + bool isCatch() const { + auto it = front()->skipHeader(); + if (it == front()->end()) return false; + + assert(it->op() == Marker); + if (it == front()->begin()) return false; + + return (--it)->op() == BeginCatch; + } void setMain(Trace* t) { assert(m_main == nullptr); m_main = t; diff --git a/hphp/runtime/vm/translator/hopt/tracebuilder.cpp b/hphp/runtime/vm/translator/hopt/tracebuilder.cpp index 7ff9f8445..fb0076ec5 100644 --- a/hphp/runtime/vm/translator/hopt/tracebuilder.cpp +++ b/hphp/runtime/vm/translator/hopt/tracebuilder.cpp @@ -785,13 +785,13 @@ SSATmp* TraceBuilder::optimizeWork(IRInstruction* inst) { return nullptr; } -SSATmp* TraceBuilder::optimizeInst(IRInstruction* inst) { +SSATmp* TraceBuilder::optimizeInst(IRInstruction* inst, CloneFlag doClone) { if (SSATmp* tmp = optimizeWork(inst)) { return tmp; } // Couldn't CSE or simplify the instruction; clone it and append. if (inst->op() != Nop) { - inst = inst->clone(&m_irFactory); + if (doClone == CloneFlag::Yes) inst = inst->clone(&m_irFactory); appendInstruction(inst); // returns nullptr if instruction has no dest, returns the first // (possibly only) dest otherwise diff --git a/hphp/runtime/vm/translator/hopt/tracebuilder.h b/hphp/runtime/vm/translator/hopt/tracebuilder.h index 04ef089b7..2e52da733 100644 --- a/hphp/runtime/vm/translator/hopt/tracebuilder.h +++ b/hphp/runtime/vm/translator/hopt/tracebuilder.h @@ -125,7 +125,9 @@ struct TraceBuilder { template SSATmp* gen(Args&&... args) { return makeInstruction( - [this] (IRInstruction* inst) { return optimizeInst(inst); }, + [this] (IRInstruction* inst) { + return optimizeInst(inst, CloneFlag::Yes); + }, std::forward(args)... ); } @@ -134,17 +136,25 @@ struct TraceBuilder { * Create an IRInstruction, similar to gen(), except link it into * the Trace t instead of the current main trace. * - * Also does not run optimization passes. + * Does not run optimization passes. * * TODO(#2404447): run simplifier? */ template - SSATmp* genFor(Trace* t, Args... args) { - auto instr = m_irFactory.gen(args...); + SSATmp* genFor(Trace* t, Args&&... args) { + auto instr = m_irFactory.gen(std::forward(args)...); t->back()->push_back(instr); return instr->dst(); } + /* + * Add an already created instruction, running it through the normal + * optimization passes and updating tracked state. + */ + SSATmp* add(IRInstruction* inst) { + return optimizeInst(inst, CloneFlag::No); + } + ////////////////////////////////////////////////////////////////////// // constants @@ -300,7 +310,8 @@ private: SSATmp* preOptimize(IRInstruction* inst); SSATmp* optimizeWork(IRInstruction* inst); - SSATmp* optimizeInst(IRInstruction* inst); + enum class CloneFlag { Yes, No }; + SSATmp* optimizeInst(IRInstruction* inst, CloneFlag doclone); private: static void appendInstruction(IRInstruction* inst, Block* block); diff --git a/hphp/runtime/vm/translator/hopt/type.cpp b/hphp/runtime/vm/translator/hopt/type.cpp index 7089ba306..b438a9154 100644 --- a/hphp/runtime/vm/translator/hopt/type.cpp +++ b/hphp/runtime/vm/translator/hopt/type.cpp @@ -44,8 +44,19 @@ Type Type::fromDynLocation(const Transl::DynLocation* dynLoc) { namespace { -Type vectorReturn(const IRInstruction* inst) { - return VectorEffects(inst).valType; +Type setElemReturn(const IRInstruction* inst) { + assert(inst->op() == SetElem || inst->op() == SetElemStk); + auto baseType = inst->src(vectorBaseIdx(inst))->type().strip(); + + // If the base is a Str, the result will always be a CountedStr (or + // an exception). If the baes might be a str, the result wil be + // CountedStr or Nullptr. Otherwise, the result is always Nullptr. + if (baseType.subtypeOf(Type::Str)) { + return Type::CountedStr; + } else if (baseType.maybe(Type::Str)) { + return Type::CountedStr | Type::Nullptr; + } + return Type::Nullptr; } Type builtinReturn(const IRInstruction* inst) { @@ -107,6 +118,9 @@ Type binArithResultType(Opcode op, Type t1, Type t2) { } Type outputType(const IRInstruction* inst, int dstId) { +#define IRT(name, ...) UNUSED static const Type name = Type::name; + IR_TYPES +#undef IRT #define D(type) return Type::type; #define DofS(n) return inst->src(n)->type(); @@ -116,9 +130,10 @@ Type outputType(const IRInstruction* inst, int dstId) { #define DMulti return Type::None; #define DStk(in) return stkReturn(inst, dstId, \ [&]() -> Type { in not_reached(); }); -#define DVector return vectorReturn(inst); +#define DSetElem return setElemReturn(inst); #define ND assert(0 && "outputType requires HasDest or NaryDest"); #define DBuiltin return builtinReturn(inst); +#define DSubtract(n, t) return inst->src(n)->type() - t; #define DArith return binArithResultType(inst->op(), \ inst->src(0)->type(), \ inst->src(1)->type()); @@ -139,9 +154,10 @@ Type outputType(const IRInstruction* inst, int dstId) { #undef DParam #undef DMulti #undef DStk -#undef DVector +#undef DSetElem #undef ND #undef DBuiltin +#undef DSubtract #undef DArith } @@ -280,9 +296,11 @@ void assertOperandTypes(const IRInstruction* inst) { #define ND #define DMulti #define DStk(...) -#define DVector +#define DSetElem #define D(...) #define DBuiltin +#define DSubtract(src, t)checkDst(src < inst->numSrcs(), \ + "invalid src num"); #define DUnbox(src) checkDst(src < inst->numSrcs(), \ "invalid src num"); #define DBox(src) checkDst(src < inst->numSrcs(), \ @@ -306,6 +324,7 @@ void assertOperandTypes(const IRInstruction* inst) { #undef O #undef NA +#undef SAny #undef S #undef C #undef CStr @@ -315,9 +334,12 @@ void assertOperandTypes(const IRInstruction* inst) { #undef ND #undef D +#undef DBuiltin +#undef DSubtract #undef DUnbox #undef DMulti #undef DStk +#undef DSetElem #undef DBox #undef DofS #undef DParam diff --git a/hphp/runtime/vm/translator/hopt/type.h b/hphp/runtime/vm/translator/hopt/type.h index a0e22c9be..6e64aee06 100644 --- a/hphp/runtime/vm/translator/hopt/type.h +++ b/hphp/runtime/vm/translator/hopt/type.h @@ -70,7 +70,8 @@ namespace JIT { IRT(TCA, 1ULL << 50) \ IRT(ActRec, 1ULL << 51) \ IRT(None, 1ULL << 52) \ - IRT(CacheHandle, 1ULL << 53) /* TargetCache::CacheHandle */ + IRT(CacheHandle, 1ULL << 53) /* TargetCache::CacheHandle */ \ + IRT(Nullptr, 1ULL << 54) // The definitions for these are in ir.cpp #define IRT_UNIONS \ @@ -82,6 +83,7 @@ namespace JIT { // would be nonsense types. #define IRT_SPECIAL \ IRT(Bottom, 0) \ + IRT(Top, 0xffffffffffffffffULL) \ IRT(Counted, kCountedStr|kCountedArr|kObj|kBoxedCell) \ IRT(PtrToCounted, kCounted << kPtrShift) \ IRT(Gen, kCell|kBoxedCell) \ diff --git a/hphp/runtime/vm/translator/hopt/vectortranslator.cpp b/hphp/runtime/vm/translator/hopt/vectortranslator.cpp index 93e51c8d3..0c5f97d3e 100644 --- a/hphp/runtime/vm/translator/hopt/vectortranslator.cpp +++ b/hphp/runtime/vm/translator/hopt/vectortranslator.cpp @@ -135,8 +135,7 @@ void VectorEffects::init(Opcode op, const Type origBase, assert_not_implemented(basePtr || baseType.subtypeOfAny(Type::Obj, Type::Arr)); - valType = origVal; - baseTypeChanged = baseValChanged = valTypeChanged = false; + baseTypeChanged = baseValChanged = false; // Canonicalize the op to SetProp/SetElem/UnsetElem/SetWithRefElem op = canonicalOp(op); @@ -147,11 +146,6 @@ void VectorEffects::init(Opcode op, const Type origBase, // since this shouldn't actually happen.) assert(key.equals(Type::None) || key.isKnownDataType() || key.equals(Type::Cell)); - if (op == SetWithRefElem) { - assert(origVal.isPtr()); - } else { - assert(origVal.equals(Type::None) || origVal.isKnownDataType()); - } if ((op == SetElem || op == SetProp || op == SetWithRefElem) && baseType.maybe(Type::Null | Type::Bool | Type::Str)) { @@ -196,36 +190,12 @@ void VectorEffects::init(Opcode op, const Type origBase, // warning, and evaulate to null. const Type badArrKey = Type::Arr | Type::Obj; - const bool maybeBadArrKey = key.maybe(badArrKey); - if ((op == SetElem && (!baseType.subtypeOf(okBase) || maybeBadArrKey)) || - (op == SetProp && !baseType.subtypeOf(Type::Obj))) { - // The set operation might fail, issue a warning, and evaluate to null. Any - // other invalid combinations of base/key will throw an exception/fatal. - - const bool definitelyFail = - // SetElem and the base is known to not be a good type - (op == SetElem && baseType.not(okBase)) || - // SetElem and the array key is known to be bad - (op == SetElem && key.subtypeOf(badArrKey)) || - // SetProp and the base is known to not be an object - (op == SetProp && baseType.not(Type::Obj)); - - valType = definitelyFail ? Type::InitNull : (valType | Type::InitNull); - } - - if (op == SetElem && baseType.maybe(Type::Str)) { - // If the base is a String, the result will be different from the value - // use valTypeChanged (even though the type may not have) - valTypeChanged = true; - } - // The final baseType should be a pointer/box iff the input was baseType = baseBoxed ? baseType.box() : baseType; baseType = basePtr ? baseType.ptr() : baseType; baseTypeChanged = baseTypeChanged || baseType != origBase; baseValChanged = baseValChanged || baseTypeChanged; - valTypeChanged = valTypeChanged || valType != origVal; } // vectorBaseIdx returns the src index for inst's base operand. @@ -292,17 +262,20 @@ HhbcTranslator::VectorTranslator::VectorTranslator( : m_ni(ni) , m_ht(ht) , m_tb(*m_ht.m_tb) + , m_irf(m_ht.m_irFactory) , m_mii(getMInstrInfo(ni.mInstrOp())) , m_needMIS(true) , m_misBase(nullptr) , m_base(nullptr) , m_result(nullptr) - , m_predictedResult(nullptr) + , m_strTestResult(nullptr) + , m_failedSetTrace(nullptr) { } template -SSATmp* HhbcTranslator::VectorTranslator::genStk(Opcode opc, Srcs... srcs) { +SSATmp* HhbcTranslator::VectorTranslator::genStk(Opcode opc, Trace* taken, + Srcs... srcs) { assert(opcodeHasFlags(opc, HasStackVersion)); assert(!opcodeHasFlags(opc, ModifiesStack)); std::vector srcVec({srcs...}); @@ -318,7 +291,7 @@ SSATmp* HhbcTranslator::VectorTranslator::genStk(Opcode opc, Srcs... srcs) { } } - return gen(opc, srcs...); + return gen(opc, taken, srcs...); } void HhbcTranslator::VectorTranslator::emit() { @@ -525,6 +498,7 @@ SSATmp* HhbcTranslator::VectorTranslator::getValAddr() { } else { assert(l.space == Location::Stack); assert(mapContains(m_stackInputs, 0)); + m_ht.spillStack(); return m_ht.ldStackAddr(m_stackInputs[0]); } } @@ -578,8 +552,8 @@ void HhbcTranslator::VectorTranslator::emitBaseLCR() { // Check for Uninit and warn/promote to InitNull as appropriate if (baseType.subtypeOf(Type::Uninit)) { if (mia & MIA_warn) { - m_ht.exceptionBarrier(); - gen(RaiseUninitLoc, LocalId(base.location.offset)); + gen(RaiseUninitLoc, m_ht.getCatchTrace(), + LocalId(base.location.offset)); } if (mia & MIA_define) { gen( @@ -620,6 +594,8 @@ void HhbcTranslator::VectorTranslator::emitBaseLCR() { m_base = m_ht.ldLocAddr(base.location.offset); } else { assert(base.location.space == Location::Stack); + // Make sure the stack is clean before getting a pointer to one of its + // elements. m_ht.spillStack(); assert(m_stackInputs.count(m_iInd)); m_base = m_ht.ldStackAddr(m_stackInputs[m_iInd]); @@ -715,8 +691,8 @@ void HhbcTranslator::VectorTranslator::emitBaseG() { static const OpFunc opFuncs[] = {baseG, baseGW, baseGD, baseGWD}; OpFunc opFunc = opFuncs[mia & MIA_base]; SSATmp* gblName = getBase(); - m_ht.exceptionBarrier(); m_base = gen(BaseG, + m_ht.getCatchTrace(), cns(reinterpret_cast(opFunc)), gblName, genMisPtr()); @@ -819,11 +795,12 @@ void HhbcTranslator::VectorTranslator::emitPropGeneric() { typedef TypedValue* (*OpFunc)(Class*, TypedValue*, TypedValue, MInstrState*); SSATmp* key = getKey(); BUILD_OPTAB(mia, m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); if (mia & Define) { - m_base = genStk(PropDX, cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); + m_base = genStk(PropDX, m_ht.getCatchTrace(), cns((TCA)opFunc), CTX(), + m_base, key, genMisPtr()); } else { - m_base = gen(PropX, cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); + m_base = gen(PropX, m_ht.getCatchTrace(), + cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); } } #undef HELPER_TABLE @@ -867,8 +844,7 @@ SSATmp* HhbcTranslator::VectorTranslator::checkInitProp( // init_null_variant. m_tb.hint(Block::Unlikely); if (doWarn && wantPropSpecializedWarnings()) { - // We did the exceptionBarrier for this up in emitPropSpecialized. - gen(RaiseUndefProp, baseAsObj, key); + gen(RaiseUndefProp, m_ht.getCatchTrace(), baseAsObj, key); } if (doDefine) { gen( @@ -896,20 +872,6 @@ void HhbcTranslator::VectorTranslator::emitPropSpecialized(const MInstrAttr mia, SSATmp* initNull = cns((const TypedValue*)&init_null_variant); - /* - * With the exception of single-dim CGetMs where hphpc knows the property - * can't be KindOfUninit, one of the conditional branches below may throw. - */ - if (doWarn && wantPropSpecializedWarnings()) { - if (m_ni.mInstrOp() == OpCGetM && m_mInd == 0) { - if (propInfo.hphpcType == KindOfInvalid) { - m_ht.exceptionBarrier(); - } - } else { - m_ht.exceptionBarrier(); - } - } - /* * Type-inference from hphpc only tells us that this is either an object of a * given class type or null. If it's not an object, it has to be a null type @@ -1056,12 +1018,12 @@ void HhbcTranslator::VectorTranslator::emitElem() { typedef TypedValue* (*OpFunc)(TypedValue*, TypedValue, MInstrState*); BUILD_OPTAB_HOT(getKeyTypeIS(key), mia); - m_ht.exceptionBarrier(); if (define || unset) { - m_base = genStk(define ? ElemDX : ElemUX, cns((TCA)opFunc), m_base, - key, genMisPtr()); + m_base = genStk(define ? ElemDX : ElemUX, m_ht.getCatchTrace(), + cns((TCA)opFunc), m_base, key, genMisPtr()); } else { - m_base = gen(ElemX, cns((TCA)opFunc), m_base, key, genMisPtr()); + m_base = gen(ElemX, m_ht.getCatchTrace(), + cns((TCA)opFunc), m_base, key, genMisPtr()); } } #undef HELPER_TABLE @@ -1190,8 +1152,8 @@ void HhbcTranslator::VectorTranslator::emitCGetProp() { typedef TypedValue (*OpFunc)(Class*, TypedValue*, TypedValue, MInstrState*); SSATmp* key = getKey(); BUILD_OPTAB_HOT(getKeyTypeS(key), m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); - m_result = gen(CGetProp, cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); + m_result = gen(CGetProp, m_ht.getCatchTrace(), + cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1232,8 +1194,7 @@ void HhbcTranslator::VectorTranslator::emitVGetProp() { SSATmp* key = getKey(); typedef RefData* (*OpFunc)(Class*, TypedValue*, TypedValue, MInstrState*); BUILD_OPTAB_HOT(getKeyTypeS(key), m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); - m_result = genStk(VGetProp, cns((TCA)opFunc), CTX(), + m_result = genStk(VGetProp, m_ht.getCatchTrace(), cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1265,9 +1226,8 @@ void HhbcTranslator::VectorTranslator::emitIssetEmptyProp(bool isEmpty) { SSATmp* key = getKey(); typedef uint64_t (*OpFunc)(Class*, TypedValue*, TypedValue); BUILD_OPTAB(isEmpty, m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); - m_result = gen(isEmpty ? EmptyProp : IssetProp, cns((TCA)opFunc), - CTX(), m_base, key); + m_result = gen(isEmpty ? EmptyProp : IssetProp, m_ht.getCatchTrace(), + cns((TCA)opFunc), CTX(), m_base, key); } #undef HELPER_TABLE @@ -1280,10 +1240,9 @@ void HhbcTranslator::VectorTranslator::emitEmptyProp() { } template -static inline TypedValue setPropImpl(Class* ctx, TypedValue* base, - TypedValue keyVal, Cell val) { - HPHP::SetProp(ctx, base, &keyVal, &val); - return val; +static inline void setPropImpl(Class* ctx, TypedValue* base, + TypedValue keyVal, Cell val) { + HPHP::SetProp(ctx, base, &keyVal, &val); } #define HELPER_TABLE(m) \ @@ -1292,8 +1251,8 @@ static inline TypedValue setPropImpl(Class* ctx, TypedValue* base, m(setPropCO, true) #define PROP(nm, ...) \ -TypedValue nm(Class* ctx, TypedValue* base, TypedValue key, Cell val) { \ - return setPropImpl<__VA_ARGS__>(ctx, base, key, val); \ +void nm(Class* ctx, TypedValue* base, TypedValue key, Cell val) { \ + setPropImpl<__VA_ARGS__>(ctx, base, key, val); \ } namespace VectorHelpers { HELPER_TABLE(PROP) @@ -1320,12 +1279,13 @@ void HhbcTranslator::VectorTranslator::emitSetProp() { } // Emit the appropriate helper call. - typedef TypedValue (*OpFunc)(Class*, TypedValue*, TypedValue, Cell); + typedef void (*OpFunc)(Class*, TypedValue*, TypedValue, Cell); SSATmp* key = getKey(); BUILD_OPTAB(m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); - m_result = genStk(SetProp, cns((TCA)opFunc), CTX(), m_base, key, value); - m_predictedResult = value; + m_failedSetTrace = m_ht.getCatchTrace(); + genStk(SetProp, m_failedSetTrace, cns((TCA)opFunc), CTX(), + m_base, key, value); + m_result = value; } #undef HELPER_TABLE @@ -1363,9 +1323,8 @@ void HhbcTranslator::VectorTranslator::emitSetOpProp() { Cell, MInstrState*, SetOpOp); BUILD_OPTAB(m_base->isA(Type::Obj)); m_tb.gen(StRaw, m_misBase, cns(RawMemSlot::MisCtx), CTX()); - m_ht.exceptionBarrier(); m_result = - genStk(SetOpProp, cns((TCA)opFunc), + genStk(SetOpProp, m_ht.getCatchTrace(), cns((TCA)opFunc), m_base, key, value, genMisPtr(), cns(op)); } #undef HELPER_TABLE @@ -1404,9 +1363,8 @@ void HhbcTranslator::VectorTranslator::emitIncDecProp() { MInstrState*, IncDecOp); BUILD_OPTAB(m_base->isA(Type::Obj)); m_tb.gen(StRaw, m_misBase, cns(RawMemSlot::MisCtx), CTX()); - m_ht.exceptionBarrier(); m_result = - genStk(IncDecProp, cns((TCA)opFunc), + genStk(IncDecProp, m_ht.getCatchTrace(), cns((TCA)opFunc), m_base, key, genMisPtr(), cns(op)); } #undef HELPER_TABLE @@ -1442,8 +1400,8 @@ void HhbcTranslator::VectorTranslator::emitBindProp() { typedef void (*OpFunc)(Class*, TypedValue*, TypedValue, RefData*, MInstrState*); BUILD_OPTAB(m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); - genStk(BindProp, cns((TCA)opFunc), CTX(), m_base, key, box, genMisPtr()); + genStk(BindProp, m_ht.getCatchTrace(), cns((TCA)opFunc), CTX(), + m_base, key, box, genMisPtr()); m_result = box; } #undef HELPER_TABLE @@ -1569,9 +1527,8 @@ void HhbcTranslator::VectorTranslator::emitCGetElem() { typedef TypedValue (*OpFunc)(TypedValue*, TypedValue, MInstrState*); BUILD_OPTAB_HOT(getKeyTypeIS(key)); - m_ht.exceptionBarrier(); - m_result = gen(CGetElem, cns((TCA)opFunc), - m_base, key, genMisPtr()); + m_result = gen(CGetElem, m_ht.getCatchTrace(), cns((TCA)opFunc), + m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1609,8 +1566,8 @@ void HhbcTranslator::VectorTranslator::emitVGetElem() { SSATmp* key = getKey(); typedef RefData* (*OpFunc)(TypedValue*, TypedValue, MInstrState*); BUILD_OPTAB(getKeyTypeIS(key)); - m_ht.exceptionBarrier(); - m_result = genStk(VGetElem, cns((TCA)opFunc), m_base, key, genMisPtr()); + m_result = genStk(VGetElem, m_ht.getCatchTrace(), cns((TCA)opFunc), + m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1649,9 +1606,8 @@ void HhbcTranslator::VectorTranslator::emitIssetEmptyElem(bool isEmpty) { typedef uint64_t (*OpFunc)(TypedValue*, TypedValue, MInstrState*); BUILD_OPTAB_HOT(getKeyTypeIS(key), isEmpty); - m_ht.exceptionBarrier(); - m_result = gen(isEmpty ? EmptyElem : IssetElem, - cns((TCA)opFunc), m_base, key, genMisPtr()); + m_result = gen(isEmpty ? EmptyElem : IssetElem, m_ht.getCatchTrace(), + cns((TCA)opFunc), m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1698,8 +1654,8 @@ void HhbcTranslator::VectorTranslator::emitArrayIsset() { typedef uint64_t (*OpFunc)(ArrayData*, TypedValue*); BUILD_OPTAB(keyType, checkForInt); assert(m_base->isA(Type::Arr)); - m_ht.exceptionBarrier(); - m_result = gen(ArrayIsset, cns((TCA)opFunc), m_base, key); + m_result = gen(ArrayIsset, m_ht.getCatchTrace(), + cns((TCA)opFunc), m_base, key); } #undef HELPER_TABLE @@ -1775,10 +1731,9 @@ void HhbcTranslator::VectorTranslator::emitArraySet(SSATmp* key, typedef ArrayData* (*OpFunc)(ArrayData*, TypedValue*, TypedValue, RefData*); BUILD_OPTAB_HOT(keyType, checkForInt, setRef); - // Don't exceptionBarrier below because the helper can't throw. It - // may reenter to call destructors so it has a sync point in - // nativecalls.cpp, but exceptions are swallowed at destructor - // boundaries right now: #2182869. + // No catch trace below because the helper can't throw. It may reenter to + // call destructors so it has a sync point in nativecalls.cpp, but exceptions + // are swallowed at destructor boundaries right now: #2182869. if (setRef) { assert(base.location.space == Location::Local || base.location.space == Location::Stack); @@ -1813,16 +1768,20 @@ namespace VectorHelpers { void setWithRefElemC(TypedValue* base, TypedValue keyVal, TypedValue* val, MInstrState* mis) { base = HPHP::ElemD(mis->tvScratch, mis->tvRef, base, &keyVal); - if (!(base == &mis->tvScratch && base->m_type == KindOfUninit)) { + if (base != &mis->tvScratch) { tvDup(val, base); + } else { + assert(base->m_type == KindOfUninit); } } void setWithRefNewElem(TypedValue* base, TypedValue* val, MInstrState* mis) { base = NewElem(mis->tvScratch, mis->tvRef, base); - if (!(base == &mis->tvScratch && base->m_type == KindOfUninit)) { + if (base != &mis->tvScratch) { tvDup(val, base); + } else { + assert(base->m_type == KindOfUninit); } } } @@ -1830,13 +1789,13 @@ void setWithRefNewElem(TypedValue* base, TypedValue* val, void HhbcTranslator::VectorTranslator::emitSetWithRefLElem() { SSATmp* key = getKey(); SSATmp* locAddr = getValAddr(); - m_ht.exceptionBarrier(); if (m_base->type().strip().subtypeOf(Type::Arr) && !locAddr->type().deref().maybeBoxed()) { emitSetElem(); - m_predictedResult = nullptr; + assert(m_strTestResult == nullptr); } else { - genStk(SetWithRefElem, cns((TCA)VectorHelpers::setWithRefElemC), + genStk(SetWithRefElem, m_ht.getCatchTrace(), + cns((TCA)VectorHelpers::setWithRefElemC), m_base, key, locAddr, genMisPtr()); } m_result = nullptr; @@ -1855,25 +1814,22 @@ void HhbcTranslator::VectorTranslator::emitSetWithRefRProp() { } void HhbcTranslator::VectorTranslator::emitSetWithRefNewElem() { - SSATmp* locAddr = getValAddr(); - m_ht.exceptionBarrier(); if (m_base->type().strip().subtypeOf(Type::Arr) && - !locAddr->type().deref().maybeBoxed()) { + getValue()->type().notBoxed()) { emitSetNewElem(); - m_predictedResult = nullptr; } else { - genStk(SetWithRefNewElem, cns((TCA)VectorHelpers::setWithRefNewElem), - m_base, locAddr, genMisPtr()); + genStk(SetWithRefNewElem, m_ht.getCatchTrace(), + cns((TCA)VectorHelpers::setWithRefNewElem), + m_base, getValAddr(), genMisPtr()); } m_result = nullptr; } template -static inline TypedValue setElemImpl(TypedValue* base, TypedValue keyVal, - Cell val) { +static inline StringData* setElemImpl(TypedValue* base, TypedValue keyVal, + Cell val) { TypedValue* key = keyPtr(keyVal); - HPHP::SetElem(base, key, &val); - return val; + return HPHP::SetElem(base, key, &val); } #define HELPER_TABLE(m) \ @@ -1884,7 +1840,7 @@ static inline TypedValue setElemImpl(TypedValue* base, TypedValue keyVal, #define ELEM(nm, hot, ...) \ hot \ -TypedValue nm(TypedValue* base, TypedValue key, Cell val) { \ +StringData* nm(TypedValue* base, TypedValue key, Cell val) { \ return setElemImpl<__VA_ARGS__>(base, key, val); \ } namespace VectorHelpers { @@ -1902,11 +1858,27 @@ void HhbcTranslator::VectorTranslator::emitSetElem() { } // Emit the appropriate helper call. - typedef TypedValue (*OpFunc)(TypedValue*, TypedValue, Cell); + typedef StringData* (*OpFunc)(TypedValue*, TypedValue, Cell); BUILD_OPTAB_HOT(getKeyTypeIS(key)); - m_ht.exceptionBarrier(); - m_result = genStk(SetElem, cns((TCA)opFunc), m_base, key, value); - m_predictedResult = value; + m_failedSetTrace = m_ht.getCatchTrace(); + SSATmp* result = genStk(SetElem, m_failedSetTrace, cns((TCA)opFunc), + m_base, key, value); + auto t = result->type(); + if (t.equals(Type::Nullptr)) { + // Base is not a string. Result is always value. + m_result = value; + } else if (t.equals(Type::CountedStr)) { + // Base is a string. Stack result is a new string so we're responsible for + // decreffing value. + m_result = result; + gen(DecRef, value); + } else { + assert(t.equals(Type::CountedStr | Type::Nullptr)); + // Base might be a string. Assume the result is value, then inform + // emitMPost that it needs to test the actual result. + m_result = value; + m_strTestResult = result; + } } #undef HELPER_TABLE @@ -1945,9 +1917,9 @@ void HhbcTranslator::VectorTranslator::emitSetOpElem() { # define SETOP_OP(op, bcOp) HELPER_TABLE(FILL_ROW, op) BUILD_OPTAB_ARG(SETOP_OPS, op); # undef SETOP_OP - m_ht.exceptionBarrier(); m_result = - genStk(SetOpElem, cns((TCA)opFunc), m_base, key, getValue(), genMisPtr()); + genStk(SetOpElem, m_ht.getCatchTrace(), cns((TCA)opFunc), + m_base, key, getValue(), genMisPtr()); } #undef HELPER_TABLE @@ -1980,8 +1952,8 @@ void HhbcTranslator::VectorTranslator::emitIncDecElem() { # define INCDEC_OP(op) HELPER_TABLE(FILL_ROW, op) BUILD_OPTAB_ARG(INCDEC_OPS, op); # undef INCDEC_OP - m_ht.exceptionBarrier(); - m_result = genStk(IncDecElem, cns((TCA)opFunc), m_base, key, genMisPtr()); + m_result = genStk(IncDecElem, m_ht.getCatchTrace(), cns((TCA)opFunc), + m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1998,8 +1970,7 @@ void bindElemC(TypedValue* base, TypedValue keyVal, RefData* val, void HhbcTranslator::VectorTranslator::emitBindElem() { SSATmp* key = getKey(); SSATmp* box = getValue(); - m_ht.exceptionBarrier(); - genStk(BindElem, cns((TCA)VectorHelpers::bindElemC), + genStk(BindElem, m_ht.getCatchTrace(), cns((TCA)VectorHelpers::bindElemC), m_base, key, box, genMisPtr()); m_result = box; } @@ -2043,8 +2014,7 @@ void HhbcTranslator::VectorTranslator::emitUnsetElem() { typedef void (*OpFunc)(TypedValue*, TypedValue); BUILD_OPTAB_HOT(getKeyTypeIS(key)); - m_ht.exceptionBarrier(); - genStk(UnsetElem, cns((TCA)opFunc), m_base, key); + genStk(UnsetElem, m_ht.getCatchTrace(), cns((TCA)opFunc), m_base, key); } #undef HELPER_TABLE @@ -2058,9 +2028,9 @@ void HhbcTranslator::VectorTranslator::emitVGetNewElem() { void HhbcTranslator::VectorTranslator::emitSetNewElem() { SSATmp* value = getValue(); - m_ht.exceptionBarrier(); - m_result = gen(SetNewElem, m_base, value); - m_predictedResult = value; + m_failedSetTrace = m_ht.getCatchTrace(); + gen(SetNewElem, m_failedSetTrace, m_base, value); + m_result = value; } void HhbcTranslator::VectorTranslator::emitSetOpNewElem() { @@ -2073,17 +2043,23 @@ void HhbcTranslator::VectorTranslator::emitIncDecNewElem() { void HhbcTranslator::VectorTranslator::emitBindNewElem() { SSATmp* box = getValue(); - m_ht.exceptionBarrier(); - genStk(BindNewElem, m_base, box, genMisPtr()); + genStk(BindNewElem, m_ht.getCatchTrace(), m_base, box, genMisPtr()); m_result = box; } void HhbcTranslator::VectorTranslator::emitMPost() { + SSATmp* catchSp = nullptr; + if (m_failedSetTrace) { + catchSp = m_failedSetTrace->back()->back()->src(0); + assert(catchSp->isA(Type::StkPtr)); + } + // Decref stack inputs. If we're translating a SetM or BindM, then input 0 is - // both our input and output so leave its refcount alone. The helpers for - // SetM can overwrite this value with InitNull if the operation fails, but - // they also decref the old value so it's still safe to leave its refcount - // alone. + // both our input and output so leave its refcount alone. If m_failedSetTrace + // is non-null, the final helper call may throw an InvalidSetMException. We + // need to add instructions to m_failedSetTrace to finish the vector + // instruction in case this happens, so any DecRefs emitted here are also + // added to m_failedSetTrace. unsigned nStack = (m_ni.mInstrOp() == OpSetM || m_ni.mInstrOp() == OpBindM) ? 1 : 0; for (unsigned i = nStack; i < m_ni.inputs.size(); ++i) { @@ -2094,6 +2070,10 @@ void HhbcTranslator::VectorTranslator::emitMPost() { auto input = getInput(i); if (input->isA(Type::Gen)) { gen(DecRef, input); + if (m_failedSetTrace) { + m_ht.genFor(m_failedSetTrace, DecRefStack, + StackOffset(m_stackInputs[i]), Type::Cell, catchSp); + } } break; } @@ -2116,7 +2096,7 @@ void HhbcTranslator::VectorTranslator::emitMPost() { // instead of the real result; its validity will be guarded later in this // function. if (m_result) { - m_ht.push(usePredictedResult() ? m_predictedResult : m_result); + m_ht.push(m_result); } else { assert(m_ni.mInstrOp() == OpUnsetM || m_ni.mInstrOp() == OpSetWithRefLM || @@ -2124,45 +2104,76 @@ void HhbcTranslator::VectorTranslator::emitMPost() { } // Clean up tvRef(2) - if (nLogicalRatchets() > 1) { - gen(DecRefMem, Type::Gen, m_misBase, cns(HHIR_MISOFF(tvRef2))); - } - if (nLogicalRatchets() > 0) { - gen(DecRefMem, Type::Gen, m_misBase, cns(HHIR_MISOFF(tvRef))); + static const size_t refOffs[] = { HHIR_MISOFF(tvRef), HHIR_MISOFF(tvRef2) }; + for (unsigned i = 0; i < std::min(nLogicalRatchets(), 2U); ++i) { + IRInstruction* inst = m_irf.gen(DecRefMem, Type::Gen, m_misBase, + cns(refOffs[i])); + m_tb.add(inst); + if (m_failedSetTrace) { + m_failedSetTrace->back()->push_back(inst->clone(&m_irf)); + } } - // Now that everything else is done, guard the predicted type of the result, - // if any. - if (usePredictedResult()) { - std::vector spillValues = m_ht.getSpillValues(); - assert(spillValues.back() == m_predictedResult); - spillValues.back() = m_result; - - gen(CheckType, - m_predictedResult->type(), - m_ht.getExitTrace(m_ht.nextSrcKey().offset(), spillValues), - m_result); - } + emitSideExits(catchSp, nStack); } -bool HhbcTranslator::VectorTranslator::usePredictedResult() { - // First check that we have a predicted result. - if (!m_predictedResult) return false; +void HhbcTranslator::VectorTranslator::emitSideExits(SSATmp* catchSp, + int nStack) { + const Offset nextOff = m_ht.nextSrcKey().offset(); + auto op = m_ni.mInstrOp(); + const bool isSetWithRef = op == OpSetWithRefLM || op == OpSetWithRefRM; - // Next, check that the predicted result is actually possible. This check - // will trigger if we can prove at compile time that the set operation will - // fail. - if (!m_predictedResult->isA(m_result->type())) return false; + if (m_failedSetTrace) { + assert(bool(m_result) ^ isSetWithRef); + // This catch trace currently ends with an EndCatch that will fall through + // if an InvalidSetMException was thrown. We need to emit code to clean up + // if that happens. If we're translating a SetWithRef* bytecode we don't + // have to do anything special to the stack since they have no stack + // output. Otherwise we need to pop our input value and push the value from + // the exception to the stack (done with a DecRefStack followed by a + // SpillStack). - // We can use the predicted result as long as we didn't do an operation on a - // possibly string base with a possibly string value. This is the only case - // where the result of a SetM might have the same type as its input without - // being the same value. - IRInstruction* inst = m_result->inst(); - SSATmp* base = inst->src(vectorBaseIdx(inst)); - SSATmp* value = inst->src(vectorValIdx(inst)); - return base->type().strip().not(Type::Str) || - value->type().strip().not(Type::Str); + std::vector args{ + catchSp, // sp from the previous SpillStack + cns(nStack), // cells popped since the last SpillStack + }; + + if (!isSetWithRef) { + m_ht.genFor(m_failedSetTrace, DecRefStack, StackOffset(0), + Type::Cell, catchSp); + args.push_back(m_ht.genFor(m_failedSetTrace, LdUnwinderValue, + Type::Cell)); + } + + SSATmp* sp = m_ht.genFor(m_failedSetTrace, SpillStack, + std::make_pair(args.size(), &args[0])); + m_ht.genFor(m_failedSetTrace, DeleteUnwinderException); + m_ht.genFor(m_failedSetTrace, SyncABIRegs, m_tb.getFp(), sp); + m_ht.genFor(m_failedSetTrace, ReqBindJmp, BCOffset(nextOff)); + } + + if (m_strTestResult) { + assert(!isSetWithRef); + // We expected SetElem's base to not be a Str but might be wrong. Make an + // exit trace to side exit to the next instruction, replacing our guess + // with the correct stack output. + + auto toSpill = m_ht.getSpillValues(); + assert(toSpill.size()); + assert(toSpill[0] == m_result); + SSATmp* str = m_irf.gen(AssertNonNull, m_strTestResult)->dst(); + toSpill[0] = str; + Trace* exit = m_ht.getExitTrace(nextOff, toSpill); + exit->front()->prepend(str->inst()); + exit->front()->prepend(m_irf.gen(DecRef, m_result)); + exit->front()->prepend( + m_irf.gen(IncStat, cns(Stats::TC_SetMStrGuess_Miss), + cns(1), cns(false))); + exit->front()->prepend(m_ht.makeMarker(m_ht.bcOff())); + + gen(CheckType, Type::Nullptr, exit, m_strTestResult); + gen(IncStat, cns(Stats::TC_SetMStrGuess_Hit), cns(1), cns(false)); + } } bool HhbcTranslator::VectorTranslator::needFirstRatchet() const { diff --git a/hphp/runtime/vm/translator/physreg.cpp b/hphp/runtime/vm/translator/physreg.cpp index 7637a640d..4fead77bc 100644 --- a/hphp/runtime/vm/translator/physreg.cpp +++ b/hphp/runtime/vm/translator/physreg.cpp @@ -38,12 +38,20 @@ PhysRegSaverParity::~PhysRegSaverParity() { // See above; stack parity. m_as. addq (m_adjust, reg::rsp); } - m_regs.forEachR([&] (PhysReg pr) { - m_as. pop (pr); + emitPops(m_as, m_regs); +} + +void PhysRegSaverParity::emitPops(X64Assembler& as, RegSet regs) { + regs.forEachR([&] (PhysReg pr) { + as. pop (pr); }); } int PhysRegSaverParity::rspAdjustment() const { + return m_adjust; +} + +int PhysRegSaverParity::rspTotalAdjustmentRegs() const { return m_regs.size() + m_adjust / sizeof(int64_t); } diff --git a/hphp/runtime/vm/translator/physreg.h b/hphp/runtime/vm/translator/physreg.h index 2c5f58965..870ebb47b 100644 --- a/hphp/runtime/vm/translator/physreg.h +++ b/hphp/runtime/vm/translator/physreg.h @@ -259,12 +259,15 @@ struct PhysRegSaverParity { PhysRegSaverParity(int parity, X64Assembler& as, RegSet regs); ~PhysRegSaverParity(); + static void emitPops(X64Assembler& as, RegSet regs); + PhysRegSaverParity(const PhysRegSaverParity&) = delete; PhysRegSaverParity(PhysRegSaverParity&&) = default; PhysRegSaverParity& operator=(const PhysRegSaverParity&) = delete; PhysRegSaverParity& operator=(PhysRegSaverParity&&) = default; int rspAdjustment() const; + int rspTotalAdjustmentRegs() const; void bytesPushed(int64_t bytes); private: diff --git a/hphp/runtime/vm/translator/targetcache.cpp b/hphp/runtime/vm/translator/targetcache.cpp index 7c701241a..fee28cea3 100644 --- a/hphp/runtime/vm/translator/targetcache.cpp +++ b/hphp/runtime/vm/translator/targetcache.cpp @@ -96,15 +96,16 @@ undefinedError(const char* msg, const char* name) { __thread void* tl_targetCaches = nullptr; __thread HphpArray* s_constants = nullptr; -static_assert(kConditionFlagsOff + sizeof(ssize_t) <= 64, - "kConditionFlagsOff too large"); -size_t s_frontier = kConditionFlagsOff + 64; +static const size_t kPreAllocatedBytes = 64; +size_t s_frontier = kPreAllocatedBytes; + +static_assert(sizeof(TargetCacheHeader) <= kPreAllocatedBytes, + "TargetCacheHeader doesn't fit in kPreAllocatedBytes"); size_t s_persistent_frontier = 0; size_t s_persistent_start = 0; static size_t s_next_bit; static size_t s_bits_to_go; static int s_tc_fd; -static const size_t kPreAllocatedBytes = kConditionFlagsOff + 64; // Mapping from names to targetcache locations. Protected by the translator // write lease. diff --git a/hphp/runtime/vm/translator/targetcache.h b/hphp/runtime/vm/translator/targetcache.h index 446693e93..00c72fece 100644 --- a/hphp/runtime/vm/translator/targetcache.h +++ b/hphp/runtime/vm/translator/targetcache.h @@ -19,6 +19,7 @@ #include "hphp/runtime/vm/func.h" #include "hphp/util/util.h" #include "hphp/runtime/vm/translator/types.h" +#include "hphp/runtime/vm/translator/unwind-x64.h" #include "hphp/util/asm-x64.h" #include @@ -47,7 +48,31 @@ extern size_t s_persistent_start; */ extern __thread HphpArray* s_constants; -static const int kConditionFlagsOff = 0; +/* + * The fields in TargetCacheHeader are pre-allocated at process + * startup and live at the beginning of the targetCache. + */ +struct TargetCacheHeader { + ssize_t conditionFlags; + + // Used to pass values between unwinder code and catch traces: + int64_t unwinderScratch; + TypedValue unwinderTv; + bool doSideExit; +}; + +inline TargetCacheHeader* header() { + return (TargetCacheHeader*)tl_targetCaches; +} + +constexpr int kConditionFlagsOff = + offsetof(TargetCacheHeader, conditionFlags); +constexpr int kUnwinderScratchOff = + offsetof(TargetCacheHeader, unwinderScratch); +constexpr int kUnwinderSideExitOff = + offsetof(TargetCacheHeader, doSideExit); +constexpr int kUnwinderTvOff = + offsetof(TargetCacheHeader, unwinderTv); /* * Some caches have different numbers of lines. This is our default. @@ -123,7 +148,7 @@ T& handleToRef(CacheHandle h) { } inline ssize_t* conditionFlagsPtr() { - return ((ssize_t*)handleToPtr(kConditionFlagsOff)); + return &header()->conditionFlags; } inline ssize_t loadConditionFlags() { diff --git a/hphp/runtime/vm/translator/translator-runtime.cpp b/hphp/runtime/vm/translator/translator-runtime.cpp index 092958ae8..d35fca2a4 100644 --- a/hphp/runtime/vm/translator/translator-runtime.cpp +++ b/hphp/runtime/vm/translator/translator-runtime.cpp @@ -36,9 +36,8 @@ ArrayData* addElemStringKeyHelper(ArrayData* ad, return array_setm_s0k1_v0(0, ad, key, &value); } -HOT_FUNC_VM TypedValue setNewElem(TypedValue* base, Cell val) { - SetNewElem(base, &val); - return val; +HOT_FUNC_VM void setNewElem(TypedValue* base, Cell val) { + SetNewElem(base, &val); } void bindNewElemIR(TypedValue* base, RefData* val, MInstrState* mis) { diff --git a/hphp/runtime/vm/translator/translator-runtime.h b/hphp/runtime/vm/translator/translator-runtime.h index e97098472..24d30b064 100644 --- a/hphp/runtime/vm/translator/translator-runtime.h +++ b/hphp/runtime/vm/translator/translator-runtime.h @@ -54,7 +54,7 @@ static_assert(sizeof(MInstrState) - sizeof(uintptr_t) // return address ArrayData* addElemIntKeyHelper(ArrayData* ad, int64_t key, TypedValue val); ArrayData* addElemStringKeyHelper(ArrayData* ad, StringData* key, TypedValue val); -TypedValue setNewElem(TypedValue* base, Cell val); +void setNewElem(TypedValue* base, Cell val); void bindNewElemIR(TypedValue* base, RefData* val, MInstrState* mis); RefData* box_value(TypedValue tv); ArrayData* convCellToArrHelper(TypedValue tv); diff --git a/hphp/runtime/vm/translator/translator-x64-internal.h b/hphp/runtime/vm/translator/translator-x64-internal.h index 51aa64fef..c773b2968 100644 --- a/hphp/runtime/vm/translator/translator-x64-internal.h +++ b/hphp/runtime/vm/translator/translator-x64-internal.h @@ -988,17 +988,17 @@ private: static inline void voidFunc() {} #define ID(argDbg) IE(debug, (argDbg), voidFunc()) -#define EMIT_CALL_PROLOGUE(a) do { \ +#define EMIT_CALL_PROLOGUE(a) do { \ SpaceRecorder sr("_HCallInclusive", a); \ - ArgManager _am(*this, a); \ + ArgManager _am(*this, a); \ prepareCallSaveRegs(); -#define EMIT_CALL_EPILOGUE(a, dest) \ - _am.emitArguments(); \ - { \ - SpaceRecorder sr("_HCallExclusive", a); \ - emitCall(a, (TCA)(dest), true); \ - } \ +#define EMIT_CALL_EPILOGUE(a, dest) \ + _am.emitArguments(); \ + { \ + SpaceRecorder sr("_HCallExclusive", a); \ + emitCall(a, (TCA)(dest), true); \ + } \ } while(0) #define EMIT_CALL(a, dest, ...) \ diff --git a/hphp/runtime/vm/translator/translator-x64.cpp b/hphp/runtime/vm/translator/translator-x64.cpp index 812e582c8..c418e2408 100644 --- a/hphp/runtime/vm/translator/translator-x64.cpp +++ b/hphp/runtime/vm/translator/translator-x64.cpp @@ -831,29 +831,6 @@ TranslatorX64::recordCallImpl(X64Assembler& a, recordSyncPoint(a, pcOff, stackOff); SKTRACE(2, sk, "record%sCall stackOff %d\n", reentrant ? "Reentrant" : "", int(stackOff)); - - /* - * Right now we assume call sites that need to record sync points - * may also throw exceptions. We record information about dirty - * callee-saved registers so we can spill their contents during - * unwinding. See unwind-x64.cpp. - */ - if (!m_pendingUnwindRegInfo.empty()) { - if (Trace::moduleLevel(Trace::tunwind) >= 2) { - sk.trace("recordCallImpl has dirty callee-saved regs\n"); - TRACE_MOD(Trace::tunwind, 2, - "CTCA: %p saving dirty callee regs:\n", - a.code.frontier); - for (int i = 0; i < UnwindRegInfo::kMaxCalleeSaved; ++i) { - if (m_pendingUnwindRegInfo.m_regs[i].dirty) { - TRACE_MOD(Trace::tunwind, 2, " %s\n", - m_pendingUnwindRegInfo.m_regs[i].pretty().c_str()); - } - } - } - m_unwindRegMap.insert(a.code.frontier, m_pendingUnwindRegInfo); - m_pendingUnwindRegInfo.clear(); - } } void @@ -864,24 +841,6 @@ TranslatorX64::recordEagerCall(X64Assembler& a, -i.stackOff * sizeof(TypedValue)); } -void TranslatorX64::prepareCallSaveRegs() { - emitCallSaveRegs(); // Clean caller-saved regs. - m_pendingUnwindRegInfo.clear(); - - RegSet rset = kGPCalleeSaved; - PhysReg reg; - while (rset.findFirst(reg)) { - rset.remove(reg); - if (!m_regMap.regIsDirty(reg)) continue; - const RegInfo* ri = m_regMap.getInfo(reg); - assert(ri->m_cont.m_kind == RegContent::Loc); - - // If the register is dirty, we'll record this so that we can - // restore it during stack unwinding if an exception is thrown. - m_pendingUnwindRegInfo.add(reg, ri->m_type, ri->m_cont.m_loc); - } -} - void TranslatorX64::emitIncRef(PhysReg base, DataType dtype) { emitIncRef(a, base, dtype); @@ -1235,7 +1194,7 @@ asm_label(a, release); { PhysRegSaver prs(a, kGPCallerSaved - RegSet(rdi)); callDestructor(a, rAsm, rax); - recordIndirectFixup(a.code.frontier, prs.rspAdjustment()); + recordIndirectFixup(a.code.frontier, prs.rspTotalAdjustmentRegs()); } a. ret (); @@ -11141,7 +11100,7 @@ TranslatorX64::TranslatorX64() m_irAstubsUsage(0), m_numHHIRTrans(0), m_regMap(kGPCallerSaved, kGPCalleeSaved, this), - m_unwindRegMap(128), + m_catchTraceMap(128), m_curTrace(0), m_curNI(0), m_curFile(nullptr), @@ -11481,6 +11440,16 @@ TranslatorX64::callBinaryStub(X64Assembler& a, const NormalizedInstruction& i, a. pop (rdi); } +void TranslatorX64::registerCatchTrace(CTCA ip, TCA trace) { + FTRACE(1, "registerCatchTrace: afterCall: {} trace: {}\n", ip, trace); + m_catchTraceMap.insert(ip, trace); +} + +TCA TranslatorX64::getCatchTrace(CTCA ip) const { + TCA* found = m_catchTraceMap.find(ip); + return found ? *found : nullptr; +} + namespace { struct DeferredFileInvalidate : public DeferredWorkItem { diff --git a/hphp/runtime/vm/translator/translator-x64.h b/hphp/runtime/vm/translator/translator-x64.h index e687d0253..c0f8c25a2 100644 --- a/hphp/runtime/vm/translator/translator-x64.h +++ b/hphp/runtime/vm/translator/translator-x64.h @@ -240,8 +240,8 @@ class TranslatorX64 : public Translator RegAlloc m_regMap; std::stack m_savedRegMaps; FixupMap m_fixupMap; - UnwindRegMap m_unwindRegMap; UnwindInfoHandle m_unwindRegistrar; + CatchTraceMap m_catchTraceMap; public: // Currently translating trace or instruction---only valid during @@ -262,7 +262,6 @@ private: m_tca(tca), m_fixup(fixup) { } }; vector m_pendingFixups; - UnwindRegInfo m_pendingUnwindRegInfo; void drawCFG(std::ofstream& out) const; static vector x64TranslRegs(); @@ -311,7 +310,7 @@ private: bool clearThis = true, uintptr_t varEnvInvName = 0); void emitCallSaveRegs(); - void prepareCallSaveRegs(); + void prepareCallSaveRegs() { not_reached(); } void emitCallStaticLocHelper(X64Assembler& as, const NormalizedInstruction& i, ScratchReg& output, @@ -641,9 +640,8 @@ PSEUDOINSTRS } } - const UnwindRegInfo* getUnwindInfo(CTCA ip) const { - return m_unwindRegMap.find(ip); - } + void registerCatchTrace(CTCA ip, TCA trace); + TCA getCatchTrace(CTCA ip) const; static void SEGVHandler(int signum, siginfo_t *info, void *ctx); diff --git a/hphp/runtime/vm/translator/translator.cpp b/hphp/runtime/vm/translator/translator.cpp index d780b4efa..2c6fa483f 100644 --- a/hphp/runtime/vm/translator/translator.cpp +++ b/hphp/runtime/vm/translator/translator.cpp @@ -41,6 +41,7 @@ #include "hphp/runtime/vm/translator/translator-inline.h" #include "hphp/runtime/vm/translator/translator-x64.h" #include "hphp/runtime/vm/translator/annotation.h" +#include "hphp/runtime/vm/translator/hopt/type.h" #include "hphp/runtime/vm/type_profile.h" #include "hphp/runtime/vm/runtime.h" @@ -48,6 +49,7 @@ namespace HPHP { namespace Transl { using namespace HPHP; +using HPHP::JIT::Type; TRACE_SET_MOD(trans) @@ -944,43 +946,50 @@ getDynLocType(const vector& inputs, op == OpDup ); - if (op == OpSetM || op == OpBindM) { + if (op == OpSetM) { /* - * these return null for "invalid" inputs - * if we cant prove everything's ok, we will - * have to insert a side exit + * SetM returns null for "invalid" inputs, or a string if the + * base was a string. VectorTranslator ensures that invalid + * inputs or a string output when we weren't expecting it will + * cause a side exit, so we can keep this fairly simple. */ - bool ok = inputs.size() <= 3; - if (ok) { - switch (inputs[1]->rtt.valueType()) { - case KindOfObject: - ok = mcodeMaybePropName(ni->immVecM[0]); - break; - case KindOfArray: - ok = mcodeMaybeArrayKey(ni->immVecM[0]); - break; - case KindOfNull: - case KindOfUninit: - break; - default: - ok = false; - } + + if (ni->immVecM.size() > 1) { + // We don't know the type of the base for the final + // operation so we can't assume anything about the output + // type. + return RuntimeType(KindOfAny); } - if (ok) { - for (int i = inputs.size(); --i >= 2; ) { - switch (inputs[i]->rtt.valueType()) { - case KindOfObject: - case KindOfArray: - ok = false; - break; - default: - continue; - } + + // For single-element vectors, we can determine the output + // type from the base. + Type baseType; + switch (ni->immVec.locationCode()) { + case LGL: case LGC: + case LNL: case LNC: + case LSL: case LSC: + baseType = Type::Gen; break; - } + + default: + baseType = Type::fromRuntimeType(inputs[1]->rtt); } - if (!ok) { - ni->outputPredicted = true; + + const bool setElem = mcodeMaybeArrayKey(ni->immVecM[0]); + const bool setNewElem = ni->immVecM[0] == MW; + const Type keyType = + setNewElem ? Type::None + : Type::fromRuntimeType(inputs[2]->rtt); + const Type valType = Type::fromRuntimeType(inputs[0]->rtt); + if (setElem && baseType.maybe(Type::Str)) { + if (baseType.isString()) { + // The base is a string so our output is a string. + return RuntimeType(KindOfString); + } else if (!valType.isString()) { + // The base might be a string and our value isn't known to + // be a string. The output type could be Str or valType. + return RuntimeType(KindOfAny); + } } } diff --git a/hphp/runtime/vm/translator/unwind-x64.cpp b/hphp/runtime/vm/translator/unwind-x64.cpp index a5985f8b0..9f422ea71 100644 --- a/hphp/runtime/vm/translator/unwind-x64.cpp +++ b/hphp/runtime/vm/translator/unwind-x64.cpp @@ -15,17 +15,20 @@ */ #include "hphp/runtime/vm/translator/unwind-x64.h" + #include -#include #include #include +#include #include -#include "hphp/runtime/vm/translator/translator-x64.h" -#include "hphp/runtime/vm/translator/runtime-type.h" #include "hphp/runtime/vm/translator/abi-x64.h" +#include "hphp/runtime/vm/translator/runtime-type.h" +#include "hphp/runtime/vm/translator/targetcache.h" +#include "hphp/runtime/vm/translator/translator-x64.h" #include "hphp/runtime/base/stats.h" #include "hphp/runtime/vm/runtime.h" +#include "hphp/runtime/vm/member_operations.h" // libgcc exports this for registering eh information for // dynamically-loaded objects. The pointer is to data in the format @@ -33,14 +36,10 @@ extern "C" void __register_frame(void*); extern "C" void __deregister_frame(void*); +TRACE_SET_MOD(tunwind); + namespace HPHP { namespace Transl { -////////////////////////////////////////////////////////////////////// - -const Trace::Module TRACEMOD = Trace::tunwind; - -////////////////////////////////////////////////////////////////////// - namespace { template @@ -56,8 +55,6 @@ void append_vec(std::vector& v, void sync_regstate(_Unwind_Context* context) { assert(tl_regState == REGSTATE_DIRTY); - TRACE(1, "unwind: doing fixup sync\n"); - uintptr_t frameRbp = _Unwind_GetGR(context, Debug::RBP); uintptr_t frameRip = _Unwind_GetGR(context, Debug::RIP); @@ -79,44 +76,31 @@ void sync_regstate(_Unwind_Context* context) { tl_regState = REGSTATE_CLEAN; } -void sync_callee_saved(_Unwind_Context* context) { - const CTCA frameIP = CTCA(_Unwind_GetGR(context, Debug::RIP)); - const UnwindRegInfo* const uInfo = tx64->getUnwindInfo(frameIP); - if (!uInfo) return; +bool install_catch_trace(_Unwind_Context* ctx, _Unwind_Exception* exn, + InvalidSetMException* ism) { + const CTCA rip = (CTCA)_Unwind_GetIP(ctx); + TCA catchTrace = tx64->getCatchTrace(rip); + assert(IMPLIES(ism, catchTrace)); + if (!catchTrace) return false; - ActRec* const frameAr = reinterpret_cast( - _Unwind_GetGR(context, Debug::RBP)); + FTRACE(1, "installing catch trace {} for call {} with ism {}, " + "returning _URC_INSTALL_CONTEXT\n", + catchTrace, rip, ism); - TRACE(1, "unwind: cleaning callee dirty regs\n"); - for (int i = 0; i < UnwindRegInfo::kMaxCalleeSaved; ++i) { - UnwindRegInfo::Data cri = uInfo->m_regs[i]; - if (!cri.dirty) break; - - TRACE(1, "unwind: unwind reg %s (val: %p)\n", cri.pretty().c_str(), - (void*)_Unwind_GetGR(context, cri.reg)); - - uintptr_t contents = _Unwind_GetGR(context, cri.reg); - TypedValue* cell; - if (cri.exStack) { - /* - * We don't keep information about how to restore the value of - * rVmSp (rbx) generally in tc frames, however the only time - * we'll need to sync dirty regs here is if we are the first tc - * frame below a C++ frame. In this case, rbx is properly - * restored for us by the unwinder, so we can use it here. - */ - static_assert(rVmSp == reg::rbx, - "unwind-x64.cpp expected rVmSp == rbx"); - TypedValue* frameSp = reinterpret_cast( - _Unwind_GetGR(context, Debug::RBX)); - cell = frameSp - (cri.locOffset + 1); - } else { - cell = frame_local(frameAr, cri.locOffset); - } - - cell->m_type = DataType(cri.dataType); - cell->m_data.num = contents; + // In theory the unwind api will let us set registers in the frame before + // executing our landing pad. In practice, trying to use their recommended + // scratch registers results in a SEGV inside _Unwind_SetGR, so we pass + // things to the handler using the target cache. This also simplifies the + // handler code because it doesn't have to worry about saving its arguments + // somewhere while executing the exit trace. + TargetCache::header()->unwinderScratch = (int64_t)exn; + TargetCache::header()->doSideExit = ism; + if (ism) { + TargetCache::header()->unwinderTv = ism->tv(); } + _Unwind_SetIP(ctx, (uint64_t)catchTrace); + + return true; } _Unwind_Reason_Code @@ -125,10 +109,34 @@ tc_unwind_personality(int version, uint64_t exceptionClass, _Unwind_Exception* exceptionObj, _Unwind_Context* context) { + using namespace abi; + // Exceptions thrown by g++-generated code will have the class "GNUCC++" + // packed into a 64-bit int. For now we shouldn't be seeing exceptions from + // any other runtimes but this may change in the future. + DEBUG_ONLY constexpr uint64_t kMagicClass = 0x474e5543432b2b00; + assert(exceptionClass == kMagicClass); assert(version == 1); - FTRACE(2, "unwind: tc_unwind_personality {}\n", - int(tl_regState)); + auto const& ti = typeInfoFromUnwindException(exceptionObj); + InvalidSetMException* ism = nullptr; + if (ti == typeid(InvalidSetMException)) { + ism = static_cast( + exceptionFromUnwindException(exceptionObj)); + } + + + if (Trace::moduleEnabled(TRACEMOD, 1)) { + DEBUG_ONLY auto const* unwindType = + (actions & _UA_SEARCH_PHASE) ? "search" : "cleanup"; + int status; + auto* exnType = __cxa_demangle(ti.name(), nullptr, nullptr, &status); + SCOPE_EXIT { free(exnType); }; + assert(status == 0); + FTRACE(1, "unwind {} exn {}: regState: {} ip: {} type: {}. ", + unwindType, exceptionObj, + tl_regState == REGSTATE_DIRTY ? "dirty" : "clean", + (TCA)_Unwind_GetIP(context), exnType); + } /* * We don't do anything during the search phase---before attempting @@ -137,22 +145,30 @@ tc_unwind_personality(int version, * tl_regState) and spilled any values they may have been holding in * callee-saved regs. */ - if (actions & _UA_SEARCH_PHASE) {} + if (actions & _UA_SEARCH_PHASE) { + if (ism) { + FTRACE(1, "thrown value: {} returning _URC_HANDLER_FOUND\n ", + ism->tv().pretty()); + return _URC_HANDLER_FOUND; + } + } /* - * During the cleanup phase, we can either use a landing pad to - * perform cleanup (with _Unwind_SetIP and _URC_INSTALL_CONTEXT), or - * we can do it here. We do it here currently so we can use unwind - * APIs to read the callee-saved register values instead of having - * to generate machine code that does it. + * During the cleanup phase, we can either use a landing pad to perform + * cleanup (with _Unwind_SetIP and _URC_INSTALL_CONTEXT), or we can do it + * here. We sync the VM registers here, then optionally use a landing pad, + * which is an exit traces from hhir with a few special instructions. */ else if (actions & _UA_CLEANUP_PHASE) { if (tl_regState == REGSTATE_DIRTY) { sync_regstate(context); } - sync_callee_saved(context); + if (install_catch_trace(context, exceptionObj, ism)) { + return _URC_INSTALL_CONTEXT; + } } + FTRACE(1, "returning _URC_CONTINUE_UNWIND\n"); return _URC_CONTINUE_UNWIND; } @@ -163,68 +179,11 @@ void deregister_unwind_region(std::vector* p) { } -////////////////////////////////////////////////////////////////////// - -UnwindRegInfo::UnwindRegInfo() { - clear(); -} - -std::string UnwindRegInfo::Data::pretty() const { - std::ostringstream out; - out << "(CRI r" - << reg << " dt:" << dataType << ' ' - << (exStack ? "stack" : "local") - << '@' << locOffset - << ")"; - return out.str(); -} - -void UnwindRegInfo::clear() { - memset(m_regs, 0, sizeof m_regs); -} - -bool UnwindRegInfo::empty() const { - return !m_regs[0].dirty; -} - -void UnwindRegInfo::add(RegNumber reg, - DataType type, - Location loc) { - assert(type >= -128 && type < 128 && - "UnwindRegInfo has only 8 bits for DataType"); - assert(loc.space == Location::Stack || loc.space == Location::Local); - assert(loc.offset >= std::numeric_limits::min() && - loc.offset <= std::numeric_limits::max() && - "UnwindRegInfo only has 16 bits for location offsets"); - - Data ent; - ent.dirty = true; - ent.dataType = type; - ent.exStack = loc.space == Location::Stack; - ent.locOffset = loc.offset; - - /* - * Important note: this is using asm-x64.h register numbers, - * although they are not the same as what dwarf/unwind uses. - * - * They happen to be the same for the specific (callee-saved) regs - * we care about right now, so this is ok. See kCalleeSaved. - */ - ent.reg = int(reg); - - for (int i = 0; i < kMaxCalleeSaved; ++i) { - if (!m_regs[i].dirty) { - m_regs[i] = ent; - return; - } - } - - assert(false && "Too many callee saved registers for UnwindRegInfo"); -} +/////////////////////////////////////////////////////////////////////////////// UnwindInfoHandle register_unwind_region(unsigned char* startAddr, size_t size) { - std::auto_ptr > bufferMem(new std::vector); + std::unique_ptr> bufferMem(new std::vector); std::vector& buffer = *bufferMem; { diff --git a/hphp/runtime/vm/translator/unwind-x64.h b/hphp/runtime/vm/translator/unwind-x64.h index 3ef3745d9..a955f1af6 100644 --- a/hphp/runtime/vm/translator/unwind-x64.h +++ b/hphp/runtime/vm/translator/unwind-x64.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "hphp/util/assertions.h" @@ -34,42 +35,24 @@ namespace HPHP { namespace Transl { ////////////////////////////////////////////////////////////////////// -/* - * Information about callee-saved registers that was dirty at a - * callsite. This should be registered during translation time so the - * unwinder can see it. - */ -struct UnwindRegInfo { - static const int kMaxCalleeSaved = 3; - - struct Data { - bool dirty : 1; - bool exStack : 1; // false means a local - signed dataType : 8; // DataType values - unsigned reg : 4; // Register id - int16_t locOffset; - - std::string pretty() const; - }; - static_assert(sizeof(Data) == sizeof(uint32_t), - "UnwindRegInfo::Data was too big"); - - explicit UnwindRegInfo(); - - void add(RegNumber reg, DataType type, Location loc); - void clear(); - bool empty() const; - - Data m_regs[kMaxCalleeSaved]; -}; - -static_assert(sizeof(UnwindRegInfo) <= sizeof(uint64_t) * 2, - "Unexpected size for UnwindRegInfo"); - -typedef TreadHashMap UnwindRegMap; +typedef TreadHashMap CatchTraceMap; ////////////////////////////////////////////////////////////////////// +inline const std::type_info& typeInfoFromUnwindException( + _Unwind_Exception* exceptionObj) +{ + constexpr size_t kTypeInfoOff = 112; + return **reinterpret_cast( + reinterpret_cast(exceptionObj + 1) - kTypeInfoOff); +} + +inline std::exception* exceptionFromUnwindException( + _Unwind_Exception* exceptionObj) +{ + return reinterpret_cast(exceptionObj + 1); +} + /* * Called whenever we create a new translation cache for the whole * region of code. diff --git a/hphp/util/trace.h b/hphp/util/trace.h index 469bd4dfd..b09927681 100644 --- a/hphp/util/trace.h +++ b/hphp/util/trace.h @@ -233,7 +233,9 @@ void dumpRingbuffer(); #define FTRACE(...) do { } while (0) #define TRACE_MOD(...) do { } while (0) #define FTRACE_MOD(...) do { } while (0) -#define TRACE_SET_MOD(...) /* nil */ +#define TRACE_SET_MOD(name) \ + DEBUG_ONLY static const HPHP::Trace::Module TRACEMOD = HPHP::Trace::name; + static const bool enabled = false; static inline void trace(const char*, ...) { }