From 8733cdc4f3c28a9e3e3cea40c92e31d96b47b264 Mon Sep 17 00:00:00 2001 From: bsimmers Date: Fri, 24 May 2013 11:53:07 -0700 Subject: [PATCH] Run hhir exit traces during unwinding; don't spillstack before most helpers Most of the SpillStacks we do are just to keep the in-memory VM stack clean in case a helper call throws an exception. Many of these exceptions are vanishingly rare, so let's stop making the fast path slower for them. This diff adds support for catch traces, which are just like normal exit traces with one major exception: they're never jumped to from translated code. If we're unwinding a TC frame and discover that the current rip has a catch trace registered, the unwinder will execute it before resuming unwinding. The unwinder parts were fairly straightfoward, then I ran into a bunch of issues with SetM. The SetElem instruction sometimes consumes a reference to one of its inputs, which is bad news for our optimizations. To make everything work again, I changed SetElem to throw an exception in cases where it would've previously decreffed the value input. This is a special exception that the new unwinding code recognizes. When one of these is caught, the catch trace executed for that TC frame will finish up the vector instruction and push the value provided by the exception on the stack. This allows us to optimize most traces under the assumption that SetM's output is the same as its input, letting the catch trace clean things up if the helper decides that's not the case and throws. There is one common case where SetM's output isn't the same as its input: setting an offset in a string base returns a new String. Luckily, we can detect most of these at compile time so SetElem returns a new StringData* when the base is known to be a string. If the base might be a string, SetElem will return nullptr if the base wasn't a string, and a StringData* if it was. We test the output in these cases and side exit if the return value of SetElem is non-null (I have yet to see this side exit happen outside of artificially created test cases). Once I got things working in the vector translator, I went through every other call to spillStack and exceptionBarrier, replacing them with catch traces as appropriate. --- hphp/doc/ir.specification | 93 +++-- hphp/runtime/base/stats.h | 4 + hphp/runtime/vm/bytecode.cpp | 19 +- hphp/runtime/vm/debug/debug.h | 4 +- hphp/runtime/vm/member_operations.h | 86 +++-- hphp/runtime/vm/translator/hopt/block.h | 28 +- hphp/runtime/vm/translator/hopt/check.cpp | 97 +++-- hphp/runtime/vm/translator/hopt/codegen.cpp | 121 +++++- hphp/runtime/vm/translator/hopt/codegen.h | 10 + hphp/runtime/vm/translator/hopt/dce.cpp | 6 +- hphp/runtime/vm/translator/hopt/extradata.h | 7 + .../vm/translator/hopt/hhbctranslator.cpp | 210 ++++++----- .../vm/translator/hopt/hhbctranslator.h | 36 +- hphp/runtime/vm/translator/hopt/ir.cpp | 43 ++- hphp/runtime/vm/translator/hopt/ir.h | 40 +- .../vm/translator/hopt/irinstruction.h | 2 +- .../vm/translator/hopt/irtranslator.cpp | 2 +- hphp/runtime/vm/translator/hopt/jumpopts.cpp | 4 +- hphp/runtime/vm/translator/hopt/mutation.cpp | 8 +- .../vm/translator/hopt/nativecalls.cpp | 6 +- .../vm/translator/hopt/predictionopts.cpp | 4 +- hphp/runtime/vm/translator/hopt/print.cpp | 5 +- .../runtime/vm/translator/hopt/simplifier.cpp | 56 +-- hphp/runtime/vm/translator/hopt/simplifier.h | 4 +- hphp/runtime/vm/translator/hopt/test/type.cpp | 11 + .../vm/translator/hopt/test/vector.cpp | 16 - hphp/runtime/vm/translator/hopt/trace.h | 9 + .../vm/translator/hopt/tracebuilder.cpp | 4 +- .../runtime/vm/translator/hopt/tracebuilder.h | 21 +- hphp/runtime/vm/translator/hopt/type.cpp | 32 +- hphp/runtime/vm/translator/hopt/type.h | 4 +- .../vm/translator/hopt/vectortranslator.cpp | 347 +++++++++--------- hphp/runtime/vm/translator/physreg.cpp | 12 +- hphp/runtime/vm/translator/physreg.h | 3 + hphp/runtime/vm/translator/targetcache.cpp | 9 +- hphp/runtime/vm/translator/targetcache.h | 29 +- .../vm/translator/translator-runtime.cpp | 5 +- .../vm/translator/translator-runtime.h | 2 +- .../vm/translator/translator-x64-internal.h | 16 +- hphp/runtime/vm/translator/translator-x64.cpp | 55 +-- hphp/runtime/vm/translator/translator-x64.h | 10 +- hphp/runtime/vm/translator/translator.cpp | 73 ++-- hphp/runtime/vm/translator/unwind-x64.cpp | 187 ++++------ hphp/runtime/vm/translator/unwind-x64.h | 49 +-- hphp/util/trace.h | 4 +- 45 files changed, 1083 insertions(+), 710 deletions(-) diff --git a/hphp/doc/ir.specification b/hphp/doc/ir.specification index f69d2d380..b60deb0ef 100644 --- a/hphp/doc/ir.specification +++ b/hphp/doc/ir.specification @@ -153,6 +153,7 @@ above PHP-facing types. PtrToT Exists for all T in Gen. Represents a TypedValue* None No value, KindOfInvalid Bottom No value, {}. Subtype of every other type + Top Supertype of every other type Cls Class* Func Func* VarEnv VarEnv* @@ -166,6 +167,7 @@ above PHP-facing types. StkPtr Pointer into VM execution stack FramePtr Pointer to a frame on the VM execution stack TCA Machine code address + Nullptr Represents a null pointer In the instruction descriptions below, we also use a "Const" prefix on several types loosly, which means the operand must be the requested @@ -327,7 +329,7 @@ Instruction set 1. Checks and Asserts -D:T = CheckType S0:Gen -> L +D:T = CheckType S0:{Gen|Nullptr} -> L Check that the type of the src S0 is T, and if so copy it to D. If S0 is not type T, branch to the label L. @@ -410,6 +412,10 @@ GuardRefs S0:FuncPtr S1:Int S2:Int S3:Int S4:Int S5:Int S6:Int If any of the checks fail, make a fallback jump. (Jump to a service request that will chain to a retranslation of this tracelet.) +D:CountedStr = AssertNonNull S0:{Nullptr|CountedStr} + + Returns S0, with Nullptr removed from its type. This instruction currently + supports a very limited range of types but can be expanded if needed. 2. Arithmetic @@ -810,12 +816,11 @@ D:PtrToGen = LdGblAddrDef S0:Str D:PtrToGen = LdClsPropAddr S0:Cls S1:Str S2:ConstCls [ -> L ] - Loads a pointer to a static class property. S0 points to the class, - S1 is the property name, and S2 is the class representing the - context of the code accessing the property. If class S0 does not - have a visible and accessible static property named S1, then this - instruction will either (1) throw a fatal error if the optional - label L is not present, or (2) jump to L if it is present. + Loads a pointer to a static class property. S0 points to the class, S1 is the + property name, and S2 is the class representing the context of the code + accessing the property. If class S0 does not have a visible and accessible + static property named S1, then this instruction will either (1) jump to L if + it is present and not a catch trace, or (2) throw a fatal error. D:PtrToGen = LdClsPropAddrCached S0:Cls S1:ConstStr S2:ConstStr S3:ConstCls [ -> L ] @@ -977,7 +982,7 @@ NativeImpl = S0:ConstFunc S1:FramePtr Execute a call to the native builtin specified by the func in S0. -D:T = CallBuiltin S0:FuncPtr S1...SN +D:T = CallBuiltin S0:FuncPtr S1:StkPtr S2...SN Call builtin function with N-1 arguments. Operands: @@ -1627,18 +1632,37 @@ D:PtrToCell = DefMIStateBase All of the remaining opcodes in this section are simple wrappers around helper functions (specified in S0) to perform the corresponding vector operation. If S1 is a ConstCls it represents the context class -for the operation. D:Vector indicates that the return value of the -opcode depends on the input types: +for the operation. -SetElem/SetOpElem/IncDecElem: -- If the base is not a subtype of Type::Arr or Type::Obj, the - operation will fail with a warning and the return value is - null. Otherwise the return value is the same as the value input. +SetElem, SetProp, and SetNewElem are used to implement part of the SetM hhbc +opcode, which almost always pushes its first stack input or a CountedStr as its +stack result. The combinations of input types that cause SetM to push anything +other than those two values are vanishingly rare in correct PHP programs, so +these three instructions have been optimized for the common cases. SetNewElem +and SetProp have no destination, allowing the compiler to predict that the +SetM's output will be the same as its input (and optimize accordingly). If that +turns out to not be the case at runtime, the instruction will throw an +InvalidSetMException. The exception will hold a Cell containing the value the +SetM should push on the stack instead of its input value. The runtime is +responsible for catching this exception, finishing execution of the SetM +instruction, pushing the value from the exception on the stack, and proceeding +as appropriate (most likely with a side exit to the next bytecode instruction, +since it has pushed an unexpected type onto the stack). -SetProp/SetOpProp/IncDecProp: -- If the base is not a subtype of Type::Obj, the operation will fail - with a warning and the return value is null. Otherwise the return - value is the same as the value input. +SetElem is similar to SetProp and SetNewElem but can also be used for setting +characters within strings. When given a string base and a valid offset, SetElem +returns a string representation of the newly inserted character. In all other +cases it returns nullptr or throws an InvalidSetMException. It will throw this +exception when it detects invalid input types, or when trying to set a string +offset that would grow the string beyond the maximum supported size. + +The input types that will cause the errors described above are listed here: + +SetNewElem will fail if the base is not a subtype of {Null|Str|Arr|Obj} and not + Bool. +SetElem has the same base constraint as SetNewElem. In addition, the key must + not be a subtype of {Arr|Obj}. +SetProp will fail if the base is not a subtype of {Obj|Null}. Any instructions that take a pointer to an MInstrState struct use the various fields of that struct for holding intermediate values. @@ -1677,9 +1701,8 @@ D:StkPtr = BindPropStk S0:ConstTCA S1:ConstCls S2:{Obj|PtrtoGen} S3:Gen Bind property with key S3 in base S2 to the reference in S4. S5 should point to an MInstrState struct. -D:Vector = SetProp S0:ConstTCA S1:ConstCls S2:{Obj|PtrToGen} S3:Gen S4:Cell -D:Vector, D:StkPtr = SetPropStk S0:ConstTCA S1:ConstCls S2:{Obj|PtrToGen} S3:Gen - S4:Cell +SetProp S0:ConstTCA S1:ConstCls S2:{Obj|PtrToGen} S3:Gen S4:Cell +D:StkPtr = SetPropStk S0:ConstTCA S1:ConstCls S2:{Obj|PtrToGen} S3:Gen S4:Cell Set property with key S3 in S2 to S4. @@ -1755,8 +1778,8 @@ ArraySet S0:ConstTCA S1:Arr S2:{Int|Str} S3:Cell S4:BoxedArr operation, it will be replaced with the new Array resulting from the set operation. -D:Vector = SetElem S0:ConstTCA S1:PtrToGen S2:Gen S3:Cell -D:Vector, D:StkPtr = SetElemStk S0:ConstTCA S1:PtrToGen S2:Gen S3:Cell +D:{CountedStr|Nullptr} = SetElem S0:ConstTCA S1:PtrToGen S2:Gen S3:Cell +D:{CountedStr|Nullptr} D:StkPtr = SetElemStk S0:ConstTCA S1:PtrToGen S2:Gen S3:Cell Set element with key S2 in S1 to S3. @@ -1784,8 +1807,8 @@ D:Cell, D:StkPtr = IncDecElemStk S0:ConstTCA S1:PtrToGen S2:Gen S3:PtrToCell Increment/decrement element with key S2 in base S1. S3 should point to an MInstrState struct. -D:Vector = SetNewElem S0:PtrToGen S1:Cell -D:Vector, D:StkPtr = SetNewElemStk S0:PtrToGen S1:Cell +SetNewElem S0:PtrToGen S1:Cell +D:StkPtr = SetNewElemStk S0:PtrToGen S1:Cell Append the value in S1 to S0. @@ -1812,3 +1835,23 @@ D:Bool = EmptyElem S0:ConstTCA S1:PtrToGen S2:Gen S3:PtrToCell Returns true iff the element at key S2 in S1 is set and not equal (as defined by the hhbc Eq instruction) to false. + +20. Exception/unwinding support + +BeginCatch + + Marks the beginning of a catch region. Exact behavior is implementation and + architecture specific. + +EndCatch S0:StkPtr + + Marks the end of a catch region and returns control to the unwinder if + appropriate. Exact behavior is implementation and architecture specific. + +D:Cell = LdUnwinderValue + + Loads the value contained by the current unwinder exception. + +DeleteUnwinderException + + Deletes the current unwinder exception. diff --git a/hphp/runtime/base/stats.h b/hphp/runtime/base/stats.h index 17eaf562b..96842cde7 100644 --- a/hphp/runtime/base/stats.h +++ b/hphp/runtime/base/stats.h @@ -134,6 +134,10 @@ namespace Stats { STAT(TC_TypePredMiss) \ STAT(TC_TypePredUnneeded) \ STAT(TC_TypePredOverridden) \ + STAT(TC_CatchTrace) \ + STAT(TC_CatchSideExit) \ + STAT(TC_SetMStrGuess_Hit) \ + STAT(TC_SetMStrGuess_Miss) \ /* Fixup */ \ STAT(Fixup_Find) \ STAT(Fixup_Probe) \ diff --git a/hphp/runtime/vm/bytecode.cpp b/hphp/runtime/vm/bytecode.cpp index 87b48df1f..a11fd9e00 100644 --- a/hphp/runtime/vm/bytecode.cpp +++ b/hphp/runtime/vm/bytecode.cpp @@ -3720,16 +3720,17 @@ inline void OPTBLD_INLINE VMExecutionContext::setHelperPost( } // NOTE: currently the only instructions using this that have return - // values on the stack also have more inputs than the K-vector, so + // values on the stack also have more inputs than the -vector, so // mdepth > 0. They also always return the original top value of // the stack. if (mdepth > 0) { assert(mdepth == 1 && - "We don't really support mdepth > 1 in setHelperPost"); + "We don't really support mdepth > 1 in setHelperPost"); - TypedValue* retSrc = m_stack.topTV(); if (ndiscard > 0) { + TypedValue* retSrc = m_stack.topTV(); TypedValue* dest = m_stack.indTV(ndiscard + mdepth - 1); + assert(dest != retSrc); memcpy(dest, retSrc, sizeof *dest); } } @@ -5229,9 +5230,15 @@ inline void OPTBLD_INLINE VMExecutionContext::iopSetM(PC& pc) { case MEL: case MEC: case MET: - case MEI: - SetElem(base, curMember, c1); + case MEI: { + StringData* result = SetElem(base, curMember, c1); + if (result) { + tvRefcountedDecRefCell(c1); + c1->m_type = KindOfString; + c1->m_data.pstr = result; + } break; + } case MPL: case MPC: case MPT: { @@ -5268,8 +5275,8 @@ inline void OPTBLD_INLINE VMExecutionContext::iopSetWithRefRM(PC& pc) { TypedValue* from = m_stack.top(); tvAsVariant(base) = withRefBind(tvAsVariant(from)); } + setHelperPost<0>(SETHELPERPOST_ARGS); m_stack.popTV(); - setHelperPost<1>(SETHELPERPOST_ARGS); } inline void OPTBLD_INLINE VMExecutionContext::iopSetOpL(PC& pc) { diff --git a/hphp/runtime/vm/debug/debug.h b/hphp/runtime/vm/debug/debug.h index b755ab677..1463a8a53 100644 --- a/hphp/runtime/vm/debug/debug.h +++ b/hphp/runtime/vm/debug/debug.h @@ -24,14 +24,12 @@ namespace HPHP { namespace Debug { -using namespace HPHP::Transl; - class DebugInfo { public: DebugInfo(); ~DebugInfo(); - void recordTracelet(TCRange range, + void recordTracelet(TCRange range, const Func* func, const Opcode *instr, bool exit, bool inPrologue); diff --git a/hphp/runtime/vm/member_operations.h b/hphp/runtime/vm/member_operations.h index 86b03e088..64e3e7ee8 100644 --- a/hphp/runtime/vm/member_operations.h +++ b/hphp/runtime/vm/member_operations.h @@ -27,6 +27,28 @@ namespace HPHP { +class InvalidSetMException : public std::runtime_error { + public: + InvalidSetMException() + : std::runtime_error("Empty InvalidSetMException") + , m_tv(HPHP::tv(KindOfNull, 0LL)) + {} + + explicit InvalidSetMException(const TypedValue& value) + : std::runtime_error(folly::format("InvalidSetMException containing {}", + value.pretty()).str()) + , m_tv(value) + {} + + ~InvalidSetMException() noexcept {} + + const TypedValue& tv() const { return m_tv; }; + private: + /* m_tv will contain a TypedValue with a reference destined for the + * VM eval stack. */ + const TypedValue m_tv; +}; + // When MoreWarnings is set to true, the VM will raise more warnings // on SetOpM, IncDecM and CGetG, intended to match Zend. const bool MoreWarnings = @@ -447,7 +469,7 @@ inline TypedValue* ElemU(TypedValue& tvScratch, TypedValue& tvRef, // $result = ($base[] = ...); inline TypedValue* NewElem(TypedValue& tvScratch, TypedValue& tvRef, - TypedValue* base) { + TypedValue* base) { TypedValue* result; DataType type; opPre(base, type); @@ -516,9 +538,12 @@ inline void SetElemEmptyish(TypedValue* base, TypedValue* key, template inline void SetElemNumberish(Cell* value) { raise_warning(Strings::CANNOT_USE_SCALAR_AS_ARRAY); + // XXX Flip? if (setResult) { tvRefcountedDecRefCell((TypedValue*)value); tvWriteNull((TypedValue*)value); + } else { + throw InvalidSetMException(tv(KindOfNull, 0LL)); } } @@ -605,17 +630,35 @@ inline void SetElemArray(TypedValue* base, TypedValue* key, if (setResult) { tvRefcountedDecRef(value); tvWriteNull(value); + } else { + throw InvalidSetMException(tv(KindOfNull, 0LL)); } } arrayRefShuffle(a, newData, base); } +template +inline int64_t castKeyToInt(TypedValue* key) { + TypedValue scratch; + initScratchKey(scratch, key); + if (key->m_type == KindOfInt64) { + return key->m_data.num; + } else { + return tvCastToInt64(key); + } +} + +template<> +inline int64_t castKeyToInt(TypedValue* key) { + return keyAsRaw(key); +} + // SetElem() leaves the result in 'value', rather than returning it as in // SetOpElem(), because doing so avoids a dup operation that SetOpElem() can't // get around. template -inline void SetElem(TypedValue* base, TypedValue* key, Cell* value) { +inline StringData* SetElem(TypedValue* base, TypedValue* key, Cell* value) { TypedValue scratch; DataType type; opPre(base, type); @@ -642,23 +685,20 @@ inline void SetElem(TypedValue* base, TypedValue* key, Cell* value) { } case KindOfStaticString: case KindOfString: { - initScratchKey(scratch, key); int baseLen = base->m_data.pstr->size(); if (baseLen == 0) { SetElemEmptyish(base, key, value); } else { // Convert key to string offset. - int64_t x; - if (LIKELY(key->m_type == KindOfInt64)) { - x = key->m_data.num; - } else { - x = tvCastToInt64(key); - } - if (x < 0 || x >= StringData::MaxSize) { - // Andrei: can't use PRId64 here because of order of inclusion - // issues + int64_t x = castKeyToInt(key); + if (UNLIKELY(x < 0 || x >= StringData::MaxSize)) { + // Can't use PRId64 here because of order of inclusion issues raise_warning("Illegal string offset: %lld", (long long)x); - break; + if (!setResult) { + throw InvalidSetMException(*value); + } else { + break; + } } // Compute how long the resulting string will be. Type needs // to agree with x. @@ -706,14 +746,10 @@ inline void SetElem(TypedValue* base, TypedValue* key, Cell* value) { base->m_data.pstr = sd; base->m_type = KindOfString; } - if (setResult) { - // Push y onto the stack. - tvRefcountedDecRef(value); - StringData* sd = NEW(StringData)(y, strlen(y), CopyString); - sd->incRefCount(); - value->m_data.pstr = sd; - value->m_type = KindOfString; - } + + StringData* sd = NEW(StringData)(y, strlen(y), CopyString); + sd->incRefCount(); + return sd; } break; } @@ -732,6 +768,8 @@ inline void SetElem(TypedValue* base, TypedValue* key, Cell* value) { } default: not_reached(); } + + return nullptr; } inline void SetNewElemEmptyish(TypedValue* base, Cell* value) { @@ -745,6 +783,8 @@ inline void SetNewElemNumberish(Cell* value) { if (setResult) { tvRefcountedDecRefCell((TypedValue*)value); tvWriteNull((TypedValue*)value); + } else { + throw InvalidSetMException(tv(KindOfNull, 0LL)); } } template @@ -1469,11 +1509,13 @@ inline bool IssetEmptyProp(Class* ctx, TypedValue* base, template inline void SetPropNull(Cell* val) { + raise_warning("Cannot access property on non-object"); if (setResult) { tvRefcountedDecRefCell(val); tvWriteNull(val); + } else { + throw InvalidSetMException(tv(KindOfNull, 0LL)); } - raise_warning("Cannot access property on non-object"); } inline void SetPropStdclass(TypedValue* base, TypedValue* key, Cell* val) { diff --git a/hphp/runtime/vm/translator/hopt/block.h b/hphp/runtime/vm/translator/hopt/block.h index 14daf6ed7..c6126123d 100644 --- a/hphp/runtime/vm/translator/hopt/block.h +++ b/hphp/runtime/vm/translator/hopt/block.h @@ -49,6 +49,13 @@ struct Block : boost::noncopyable { , m_hint(Neither) {} + const IRInstruction* beginCatch() const { + auto it = begin(); + ++it; + assert(it->op() == BeginCatch); + return &*it; + } + uint32_t id() const { return m_id; } Trace* trace() const { return m_trace; } void setTrace(Trace* t) { m_trace = t; } @@ -112,21 +119,30 @@ struct Block : boost::noncopyable { void setPostId(unsigned id) { m_postid = id; } /* - * Insert inst after this block's Marker, return an - * iterator to the newly inserted instruction. + * Insert inst after this block's optional DefLabel, optional + * BeginCatch, and Marker, return an iterator to the newly inserted + * instruction. * * Pre: the block contains a Marker after the optional DefLabel. */ iterator prepend(IRInstruction* inst) { - auto it = skipLabel(); + auto it = skipHeader(); assert(it->op() == Marker); return insert(++it, inst); } - // return iterator to first instruction after the (optional) label - iterator skipLabel() { + // return iterator to first instruction after the DefLabel (if + // present) and BeginCatch (if present). + iterator skipHeader() { auto it = begin(); - return it->op() == DefLabel ? ++it : it; + auto e = end(); + if (it != e && it->op() == DefLabel) ++it; + if (it != e && it->op() == BeginCatch) ++it; + return it; + } + + const_iterator skipHeader() const { + return const_cast(this)->skipHeader(); } // return iterator to last instruction diff --git a/hphp/runtime/vm/translator/hopt/check.cpp b/hphp/runtime/vm/translator/hopt/check.cpp index fd116466a..bdb36a9c4 100644 --- a/hphp/runtime/vm/translator/hopt/check.cpp +++ b/hphp/runtime/vm/translator/hopt/check.cpp @@ -62,48 +62,64 @@ DEBUG_ONLY static int numBlockParams(Block* b) { } /* - * Check one block for being well formed. It must: - * 1. Optionally start with DefLabel. DefLabel may not appear anywhere else. - * 2. Have either no other instructions, or else at least one Marker - * following the DefLabel before any other instructions. - * 3. If any instruction is isBlockEnd(), it must be last. - * 4. If the last instruction isTerminal(), block->next must be null. - * 5. If the block starts with a DefLabel with >=1 destinations, all the - * incoming must be from blocks listed in this block's trace. + * Check one block for being well formed. Invariants verified: + * 1. The block begins with an optional DefLabel, followed by an optional + * BeginCatch, followed by either a Marker or no more instructions. + * 2. DefLabel and BeginCatch may not appear anywhere in a block other than + * where specified in #1. + * 3. If the optional BeginCatch is present, the block must belong to an exit + * trace and must be the first block in its Trace's block list. + * 4. If any instruction is isBlockEnd(), it must be last. + * 5. If the last instruction isTerminal(), block->next must be null. + * 6. If the DefLabel produces a value, all of its incoming edges must be from + * blocks listed in the block list for this block's Trace. + * 7. Any path from this block to a Block that expects values must be + * from a Jmp_ instruciton. */ bool checkBlock(Block* b) { - if (b->skipLabel() != b->end()) { - assert(b->skipLabel()->op() == Marker); + auto it = b->begin(); + auto end = b->end(); + if (it == end) return true; + + // Invariant #1 + if (it->op() == DefLabel) ++it; + + // Invariant #1, #3 + if (it != end && it->op() == BeginCatch) { + ++it; + assert(!b->trace()->isMain()); + assert(b == b->trace()->front()); } - if (Block* next DEBUG_ONLY = b->next()) { - // cannot fall-through to join block expecting values - assert(numBlockParams(next) == 0); + + // Invariant #3 + if (it == end) return true; + assert(it->op() == Marker); + + // Invariants #2, #4 + if (++it == end) return true; + if (b->back()->isBlockEnd()) --end; + while (it != end && it->op() == Marker) ++it; + for (DEBUG_ONLY IRInstruction& inst : folly::makeRange(it, end)) { + assert(inst.op() != DefLabel); + assert(inst.op() != BeginCatch); + assert(!inst.isBlockEnd()); } - if (b->empty()) { - return true; + for (DEBUG_ONLY IRInstruction& inst : *b) { + assert(inst.block() == b); } - if (b->back()->isTerminal()) { - assert(!b->next()); - } - if (Block* taken DEBUG_ONLY = b->taken()) { + + // Invariant #5 + assert(IMPLIES(b->back()->isTerminal(), !b->next())); + + // Invariant #7 + if (b->taken()) { // only Jmp_ can branch to a join block expecting values. DEBUG_ONLY IRInstruction* branch = b->back(); DEBUG_ONLY auto numArgs = branch->op() == Jmp_ ? branch->numSrcs() : 0; - assert(numBlockParams(taken) == numArgs); - } - - { - auto i = b->skipLabel(), e = b->end(); - if (b->back()->isBlockEnd()) --e; - for (DEBUG_ONLY IRInstruction& inst : folly::makeRange(i, e)) { - assert(inst.op() != DefLabel); - assert(!inst.isBlockEnd()); - } - for (DEBUG_ONLY IRInstruction& inst : *b) { - assert(inst.block() == b); - } + assert(numBlockParams(b->taken()) == numArgs); } + // Invariant #6 if (b->front()->op() == DefLabel) { for (int i = 0; i < b->front()->numDsts(); ++i) { auto const traceBlocks = b->trace()->blocks(); @@ -117,6 +133,21 @@ bool checkBlock(Block* b) { return true; } +/* + * Check that every catch trace has at most one incoming branch and a single + * block. + */ +bool checkCatchTraces(Trace* trace, const IRFactory& irFactory) { + forEachTraceBlock(trace, [&](Block* b) { + auto trace = b->trace(); + if (trace->isCatch()) { + assert(trace->blocks().size() == 1); + assert(b->preds().size() <= 1); + } + }); + return true; +} + } const Edge* takenEdge(IRInstruction* inst) { @@ -167,6 +198,8 @@ bool checkCfg(Trace* trace, const IRFactory& factory) { } } + checkCatchTraces(trace, factory); + // visit dom tree in preorder, checking all tmps auto const children = findDomChildren(blocks); StateVector defined0(&factory, false); diff --git a/hphp/runtime/vm/translator/hopt/codegen.cpp b/hphp/runtime/vm/translator/hopt/codegen.cpp index c7141118b..d34d516a7 100644 --- a/hphp/runtime/vm/translator/hopt/codegen.cpp +++ b/hphp/runtime/vm/translator/hopt/codegen.cpp @@ -17,6 +17,7 @@ #include "hphp/runtime/vm/translator/hopt/codegen.h" #include +#include #include "folly/ScopeGuard.h" #include "folly/Format.h" @@ -318,6 +319,12 @@ Address CodeGenerator::cgInst(IRInstruction* inst) { Opcode opc = inst->op(); auto const start = m_as.code.frontier; m_rScratch = selectScratchReg(inst); + if (inst->taken() && inst->taken()->trace()->isCatch()) { + m_state.catchTrace = inst->taken()->trace(); + } else { + m_state.catchTrace = nullptr; + } + switch (opc) { #define O(name, dsts, srcs, flags) \ case name: FTRACE(7, "cg" #name "\n"); \ @@ -778,6 +785,58 @@ void CodeGenerator::emitReqBindJcc(ConditionCode cc, a. jmp (jccStub); } +void CodeGenerator::cgAssertNonNull(IRInstruction* inst) { + auto srcReg = m_regs[inst->src(0)].getReg(); + auto dstReg = m_regs[inst->dst()].getReg(); + if (RuntimeOption::EvalHHIRGenerateAsserts) { + Label nonNull; + m_as.testq (srcReg, srcReg); + m_as.jne8 (nonNull); + m_as.ud2(); + asm_label(m_as, nonNull); + } + emitMovRegReg(m_as, srcReg, dstReg); +} + +void CodeGenerator::cgLdUnwinderValue(IRInstruction* inst) { + cgLoad(rVmTl, TargetCache::kUnwinderTvOff, inst); +} + +void CodeGenerator::cgBeginCatch(IRInstruction* inst) { + auto const& info = m_state.catches[inst->block()]; + assert(info.afterCall); + + m_tx64->registerCatchTrace(info.afterCall, m_as.code.frontier); + + Stats::emitInc(m_as, Stats::TC_CatchTrace); + + // We want to restore state as though the call had completed + // successfully, so skip over any stack arguments and pop any + // saved registers. + if (info.rspOffset) { + m_as.subq(info.rspOffset, rsp); + } + PhysRegSaverParity::emitPops(m_as, info.savedRegs); +} + +void CodeGenerator::cgEndCatch(IRInstruction* inst) { + m_as.cmpb (0, rVmTl[TargetCache::kUnwinderSideExitOff]); + unlikelyIfBlock(CC_E, + [&](Asm& as) { // doSideExit == false, so call _Unwind_Resume + as.loadq(rVmTl[TargetCache::kUnwinderScratchOff], rdi); + as.call ((TCA)_Unwind_Resume); // pass control back to the unwinder + as.ud2(); + }); + + // doSideExit == true, so fall through to the side exit code + Stats::emitInc(m_as, Stats::TC_CatchSideExit); +} + +void CodeGenerator::cgDeleteUnwinderException(IRInstruction* inst) { + m_as.loadq(rVmTl[TargetCache::kUnwinderScratchOff], rdi); + m_as.call ((TCA)_Unwind_DeleteException); +} + void CodeGenerator::cgJcc(IRInstruction* inst) { emitCompare(inst->src(0), inst->src(1)); emitFwdJcc(opToConditionCode(inst->op()), inst->taken()); @@ -1073,6 +1132,14 @@ void CodeGenerator::cgCallHelper(Asm& a, recordSyncPoint(a, sync); } + if (m_state.catchTrace) { + auto& info = m_state.catches[m_state.catchTrace->front()]; + assert(!info.afterCall); + info.afterCall = a.code.frontier; + info.savedRegs = toSave; + info.rspOffset = regSaver.rspAdjustment(); + } + // copy the call result to the destination register(s) if (destType == DestType::TV) { // rax contains m_type and m_aux but we're expecting just the @@ -3525,8 +3592,8 @@ void CodeGenerator::cgCastStk(IRInstruction *inst) { void CodeGenerator::cgCallBuiltin(IRInstruction* inst) { SSATmp* f = inst->src(0); - auto args = inst->srcs().subpiece(1); - int32_t numArgs = args.size(); + auto args = inst->srcs().subpiece(2); + int32_t numArgs = args.size(); SSATmp* dst = inst->dst(); auto dstReg = m_regs[dst].getReg(0); auto dstType = m_regs[dst].getReg(1); @@ -4133,18 +4200,28 @@ void CodeGenerator::cgDefMIStateBase(IRInstruction* inst) { void CodeGenerator::cgCheckType(IRInstruction* inst) { auto const src = inst->src(0); + auto const t = inst->typeParam(); auto const rData = m_regs[src].getReg(0); auto const rType = m_regs[src].getReg(1); - emitTypeTest(inst->typeParam(), rType, rData, - [&](ConditionCode cc) { - emitFwdJcc(ccNegate(cc), inst->taken()); + auto doJcc = [&](ConditionCode cc) { + emitFwdJcc(ccNegate(cc), inst->taken()); + }; - auto const dstReg = m_regs[inst->dst()].getReg(); - if (dstReg != InvalidReg) { - emitMovRegReg(m_as, m_regs[src].getReg(0), dstReg); - } - }); + if (t.equals(Type::Nullptr)) { + if (!src->type().equals(Type::Nullptr | Type::CountedStr)) { + CG_PUNT(CheckType-Nullptr-UnsupportedType); + } + m_as.testq (rData, rData); + doJcc(CC_E); + } else { + emitTypeTest(inst->typeParam(), rType, rData, doJcc); + } + + auto const dstReg = m_regs[inst->dst()].getReg(); + if (dstReg != InvalidReg) { + emitMovRegReg(m_as, rData, dstReg); + } } void CodeGenerator::cgCheckTypeMem(IRInstruction* inst) { @@ -4502,11 +4579,15 @@ void CodeGenerator::cgLdClsPropAddrCached(IRInstruction* inst) { } void CodeGenerator::cgLdClsPropAddr(IRInstruction* inst) { - SSATmp* dst = inst->dst(); - SSATmp* cls = inst->src(0); - SSATmp* prop = inst->src(1); - SSATmp* cxt = inst->src(2); - Block* target = inst->taken(); + SSATmp* dst = inst->dst(); + SSATmp* cls = inst->src(0); + SSATmp* prop = inst->src(1); + SSATmp* ctx = inst->src(2); + Block* target = inst->taken(); + // If our label is a catch trace we pretend we don't have one, to + // avoid emitting a jmp to it or calling the wrong helper. + if (target && target->trace()->isCatch()) target = nullptr; + auto dstReg = m_regs[dst].getReg(); if (dstReg == InvalidReg && target) { // result is unused but this instruction was not eliminated @@ -4518,7 +4599,7 @@ void CodeGenerator::cgLdClsPropAddr(IRInstruction* inst) { : (TCA)SPropCache::lookupSProp, // raise on error dstReg, kSyncPoint, // could re-enter to initialize properties - ArgGroup(m_regs).ssa(cls).ssa(prop).ssa(cxt)); + ArgGroup(m_regs).ssa(cls).ssa(prop).ssa(ctx)); if (target) { m_as.testq(dstReg, dstReg); emitFwdJcc(m_as, CC_Z, target); @@ -4804,12 +4885,13 @@ void CodeGenerator::cgJmp_(IRInstruction* inst) { if (unsigned n = inst->numSrcs()) { // Parallel-copy sources to the label's destination registers. // TODO: t2040286: this only works if all destinations fit in registers. - SrcRange srcs = inst->srcs(); - DstRange dsts = target->front()->dsts(); + auto srcs = inst->srcs(); + auto dsts = target->front()->dsts(); ArgGroup args(m_regs); for (unsigned i = 0, j = 0; i < n; i++) { assert(srcs[i]->type().subtypeOf(dsts[i].type())); - SSATmp *dst = &dsts[i], *src = srcs[i]; + auto dst = &dsts[i]; + auto src = srcs[i]; // Currently, full XMM registers cannot be assigned to SSATmps // passed from to Jmp_ to DefLabel. If this changes, it'll require // teaching shuffleArgs() how to handle full XMM values. @@ -5290,6 +5372,7 @@ void CodeGenerator::cgBlock(Block* block, vector* bcMap) { IRInstruction* inst = &instr; if (inst->op() == Marker) { m_state.lastMarker = inst->extra(); + FTRACE(7, "lastMarker is now {}\n", inst->extra()->show()); if (m_tx64 && m_tx64->isTransDBEnabled() && bcMap) { bcMap->push_back((TransBCMapping){Offset(m_state.lastMarker->bcOff), m_as.code.frontier, diff --git a/hphp/runtime/vm/translator/hopt/codegen.h b/hphp/runtime/vm/translator/hopt/codegen.h index f2ddc179c..edb4d90ac 100644 --- a/hphp/runtime/vm/translator/hopt/codegen.h +++ b/hphp/runtime/vm/translator/hopt/codegen.h @@ -84,6 +84,8 @@ struct CodegenState { , liveRegs(liveRegs) , lifetime(lifetime) , asmInfo(asmInfo) + , catches(factory, CatchInfo()) + , catchTrace(nullptr) {} // Each block has a list of addresses to patch, and an address if @@ -113,6 +115,14 @@ struct CodegenState { // Output: start/end ranges of machine code addresses of each instruction. AsmInfo* asmInfo; + + // Used to pass information about the state of the world at native + // calls between cgCallHelper and cgBeginCatch. + StateVector catches; + + // If non-null, represents the catch trace for the current + // instruction, to be registered with the unwinder. + Trace* catchTrace; }; constexpr Reg64 rCgGP (reg::r11); diff --git a/hphp/runtime/vm/translator/hopt/dce.cpp b/hphp/runtime/vm/translator/hopt/dce.cpp index ed1ee8921..07d3c8293 100644 --- a/hphp/runtime/vm/translator/hopt/dce.cpp +++ b/hphp/runtime/vm/translator/hopt/dce.cpp @@ -122,6 +122,10 @@ void removeDeadInstructions(Trace* trace, const DceState& state) { auto cur = it; ++it; Block* block = *cur; block->remove_if([&] (const IRInstruction& inst) { + ONTRACE(7, + if (state[inst].isDead()) { + FTRACE(3, "Removing dead instruction {}\n", inst.toString()); + }); return state[inst].isDead(); }); // Marker and DefLabel instructions are marked live in reachable blocks @@ -204,7 +208,7 @@ BlockList removeUnreachable(Trace* trace, IRFactory* factory) { WorkList initInstructions(const BlockList& blocks, DceState& state) { - TRACE(5, "DCE:vvvvvvvvvvvvvvvvvvvv\n"); + TRACE(5, "DCE(initInstructions):vvvvvvvvvvvvvvvvvvvv\n"); // mark reachable, essential, instructions live and enqueue them WorkList wl; for (Block* block : blocks) { diff --git a/hphp/runtime/vm/translator/hopt/extradata.h b/hphp/runtime/vm/translator/hopt/extradata.h index aea66a343..af27d9334 100644 --- a/hphp/runtime/vm/translator/hopt/extradata.h +++ b/hphp/runtime/vm/translator/hopt/extradata.h @@ -112,6 +112,13 @@ struct MarkerData : IRExtraData { uint32_t bcOff; // the bytecode offset in unit int32_t stackOff; // stack off from start of trace const Func* func; // which func are we in + + std::string show() const { + return folly::format("--- bc {}, spOff {} ({})", + bcOff, + stackOff, + func->fullName()->data()).str(); + } }; struct LocalId : IRExtraData { diff --git a/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp b/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp index 0a2ede348..ae77ada6d 100644 --- a/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp +++ b/hphp/runtime/vm/translator/hopt/hhbctranslator.cpp @@ -268,18 +268,22 @@ bool HhbcTranslator::isInlining() const { return m_bcStateStack.size() > 1; } -void HhbcTranslator::emitMarker() { +IRInstruction* HhbcTranslator::makeMarker(Offset bcOff) { int32_t stackOff = m_tb->getSpOffset() + m_evalStack.numCells() - m_stackDeficit; FTRACE(2, "emitMarker: bc {} sp {} fn {}\n", - bcOff(), stackOff, getCurFunc()->fullName()->data()); + bcOff, stackOff, getCurFunc()->fullName()->data()); MarkerData marker; - marker.bcOff = bcOff(); + marker.bcOff = bcOff; marker.func = getCurFunc(); marker.stackOff = stackOff; - gen(Marker, marker); + return m_irFactory.gen(Marker, marker); +} + +void HhbcTranslator::emitMarker() { + m_tb->add(makeMarker(bcOff())); } void HhbcTranslator::profileFunctionEntry(const char* category) { @@ -518,7 +522,6 @@ void HhbcTranslator::emitCns(uint32_t id) { not_reached(); } } else { - spillStack(); // do this on main trace so we update stack tracking once. SSATmp* c1 = gen(LdCns, cnsType, cnsNameTmp); result = m_tb->cond( getCurFunc(), @@ -530,7 +533,7 @@ void HhbcTranslator::emitCns(uint32_t id) { }, [&] { // Taken: miss in TC, do lookup & init m_tb->hint(Block::Unlikely); - return gen(LookupCns, cnsType, cnsNameTmp); + return gen(LookupCns, getCatchTrace(), cnsType, cnsNameTmp); } ); } @@ -639,8 +642,9 @@ void HhbcTranslator::emitCGetL(int32_t id) { void HhbcTranslator::emitCGetL2(int32_t id) { Trace* exitTrace = getExitTrace(); + Trace* catchTrace = getCatchTrace(); SSATmp* oldTop = pop(Type::Gen); - pushIncRef(ldLocInnerWarn(id, exitTrace)); + pushIncRef(ldLocInnerWarn(id, exitTrace, catchTrace)); push(oldTop); } @@ -827,7 +831,6 @@ SSATmp* HhbcTranslator::emitIterInitCommon(int offset, Lambda genFunc) { if (!type.isArray() && type != Type::Obj) { PUNT(IterInit); } - spillStack(); SSATmp* res = genFunc(src); return emitJmpCondHelper(offset, true, res); } @@ -1024,7 +1027,10 @@ void HhbcTranslator::emitCreateCont(bool getArgs, } void HhbcTranslator::emitContEnter(int32_t returnBcOffset) { - spillStack(); + // The stack should always be clean here; this only appears in generated + // methods we control. + assert(m_evalStack.size() == 0); + assert(m_stackDeficit == 0); assert(getCurClass()); SSATmp* cont = gen(LdThis, m_tb->getFp()); @@ -1293,7 +1299,7 @@ SSATmp* HhbcTranslator::emitLdClsPropAddrOrExit(const StringData* propName, return emitLdClsPropAddrCached(propName, block); } - if (!block) exceptionBarrier(); + if (!block) block = getCatchTrace()->front(); SSATmp* clsTmp = popA(); SSATmp* prop = getStrName(propName); @@ -1466,13 +1472,14 @@ void HhbcTranslator::emitJmpNZ(Offset taken) { } void HhbcTranslator::emitCmp(Opcode opc) { + Trace* catchTrace = nullptr; if (cmpOpTypesMayReenter(opc, topC(0)->type(), topC(1)->type())) { - exceptionBarrier(); + catchTrace = getCatchTrace(); } // src2 opc src1 SSATmp* src1 = popC(); SSATmp* src2 = popC(); - push(gen(opc, src2, src1)); + push(gen(opc, catchTrace, src2, src1)); gen(DecRef, src2); gen(DecRef, src1); } @@ -1494,7 +1501,6 @@ void HhbcTranslator::emitClsCnsD(int32_t cnsNameStrId, int32_t clsNameStrId) { // if-then-else // todo: t2068502: refine the type? hhbc spec says null|bool|int|dbl|str // and, str should always be static-str. - exceptionBarrier(); // do on main trace so we update stack tracking once. Type cnsType = Type::Cell; SSATmp* c1 = gen(LdClsCns, cnsType, cnsNameTmp, clsNameTmp); SSATmp* result = m_tb->cond(getCurFunc(), @@ -1506,7 +1512,8 @@ void HhbcTranslator::emitClsCnsD(int32_t cnsNameStrId, int32_t clsNameStrId) { }, [&] { // Taken: miss in TC, do lookup & init m_tb->hint(Block::Unlikely); - return gen(LookupClsCns, cnsType, cnsNameTmp, clsNameTmp); + return gen(LookupClsCns, getCatchTrace(), + cnsType, cnsNameTmp, clsNameTmp); } ); push(result); @@ -1663,13 +1670,14 @@ void HhbcTranslator::emitFPushActRec(SSATmp* func, void HhbcTranslator::emitFPushCtorCommon(SSATmp* cls, SSATmp* obj, const Func* func, - int32_t numParams) { + int32_t numParams, + Trace* catchTrace) { push(obj); SSATmp* fn = nullptr; if (func) { fn = cns(func); } else { - fn = gen(LdClsCtor, cls); + fn = gen(LdClsCtor, catchTrace, cls); } SSATmp* obj2 = gen(IncRef, obj); int32_t numArgsAndCtorFlag = ActRec::encodeNumArgs(numParams, true); @@ -1677,10 +1685,10 @@ void HhbcTranslator::emitFPushCtorCommon(SSATmp* cls, } void HhbcTranslator::emitFPushCtor(int32_t numParams) { + Trace* catchTrace = getCatchTrace(); SSATmp* cls = popA(); - exceptionBarrier(); SSATmp* obj = gen(IncRef, gen(AllocObj, cls)); - emitFPushCtorCommon(cls, obj, nullptr, numParams); + emitFPushCtorCommon(cls, obj, nullptr, numParams, catchTrace); } static bool canInstantiateClass(const Class* cls) { @@ -1690,7 +1698,10 @@ static bool canInstantiateClass(const Class* cls) { void HhbcTranslator::emitFPushCtorD(int32_t numParams, int32_t classNameStrId) { const StringData* className = lookupStringId(classNameStrId); - exceptionBarrier(); + // The code generated for the catch trace depends on the environment at the + // call so we can't share them between instructions. + Trace* catchTrace1 = getCatchTrace(); + Trace* catchTrace2 = getCatchTrace(); const Class* cls = Unit::lookupUniqueClass(className); bool uniqueCls = classIsUnique(cls); @@ -1716,7 +1727,7 @@ void HhbcTranslator::emitFPushCtorD(int32_t numParams, int32_t classNameStrId) { if (persistentCls) { clss = cns(cls); } else { - clss = gen(LdClsCached, cns(className)); + clss = gen(LdClsCached, catchTrace1, cns(className)); } SSATmp* obj = nullptr; @@ -1726,7 +1737,7 @@ void HhbcTranslator::emitFPushCtorD(int32_t numParams, int32_t classNameStrId) { obj = gen(IncRef, gen(AllocObj, clss)); } - emitFPushCtorCommon(clss, obj, func, numParams); + emitFPushCtorCommon(clss, obj, func, numParams, catchTrace2); } /* @@ -1767,11 +1778,12 @@ void HhbcTranslator::emitFPushFuncD(int32_t numParams, int32_t funcId) { const bool immutable = func->isNameBindingImmutable(getCurUnit()); + Trace* catchTrace = nullptr; if (!immutable) { - exceptionBarrier(); // LdFuncCached can reenter + catchTrace = getCatchTrace(); // LdFuncCached can throw } SSATmp* ssaFunc = immutable ? cns(func) - : gen(LdFuncCached, cns(name)); + : gen(LdFuncCached, catchTrace, cns(name)); emitFPushActRec(ssaFunc, m_tb->genDefInitNull(), numParams, @@ -1795,8 +1807,7 @@ void HhbcTranslator::emitFPushFunc(int32_t numParams) { } void HhbcTranslator::emitFPushFunc(int32_t numParams, SSATmp* funcName) { - exceptionBarrier(); // LdFunc can reenter - emitFPushActRec(gen(LdFunc, funcName), + emitFPushActRec(gen(LdFunc, getCatchTrace(), funcName), m_tb->genDefInitNull(), numParams, nullptr); @@ -2019,9 +2030,11 @@ void HhbcTranslator::emitFCallBuiltin(uint32_t numArgs, // 1. some of the arguments may be passed by reference, for which // case we will pass a stack address. // 2. type conversions of the arguments (using tvCast* helpers) - // may throw an exception, so we need to have the VM stack - // in a clean state at that point. - exceptionBarrier(); + // may throw an exception, so we either need to have the VM stack + // in a clean state at that point or give each helper a catch + // trace. Since we have to spillstack anyway, the catch trace + // would be overkill. + spillStack(); // Convert types if needed. for (int i = 0; i < numNonDefault; i++) { @@ -2046,30 +2059,31 @@ void HhbcTranslator::emitFCallBuiltin(uint32_t numArgs, } // Pass arguments for CallBuiltin. - SSATmp* args[numArgs + 1]; + const int argsSize = numArgs + 2; + SSATmp* args[argsSize]; args[0] = cns(callee); + args[1] = m_tb->getSp(); for (int i = numArgs - 1; i >= 0; i--) { const Func::ParamInfo& pi = callee->params()[i]; switch (pi.builtinType()) { case KindOfBoolean: case KindOfInt64: - args[i + 1] = top(Type::fromDataType(pi.builtinType(), KindOfInvalid), + args[i + 2] = top(Type::fromDataType(pi.builtinType(), KindOfInvalid), numArgs - i - 1); break; case KindOfDouble: assert(false); default: - args[i + 1] = ldStackAddr(numArgs - i - 1); + args[i + 2] = ldStackAddr(numArgs - i - 1); break; } } // Generate call and set return type - SSATmp** decayedPtr = args; auto const ret = gen( CallBuiltin, Type::fromDataTypeWithRef(callee->returnType(), (callee->attrs() & ClassInfo::IsReference)), - std::make_pair(numArgs + 1, decayedPtr) + std::make_pair(argsSize, (SSATmp**)&args) ); // Decref and free args @@ -2263,12 +2277,12 @@ void HhbcTranslator::emitSwitch(const ImmVector& iv, switchVal, ssabase, ssatargets); } else if (type.subtypeOf(Type::Obj)) { // switchObjHelper can throw exceptions and reenter the VM + Trace* catchTrace = nullptr; if (type.subtypeOf(Type::Obj)) { - exceptionBarrier(); + catchTrace = getCatchTrace(); } bounded = false; - index = gen(LdSwitchObjIndex, - switchVal, ssabase, ssatargets); + index = gen(LdSwitchObjIndex, catchTrace, switchVal, ssabase, ssatargets); } else if (type.subtypeOf(Type::Arr)) { gen(DecRef, switchVal); gen(Jmp_, getExitTrace(defaultOff)); @@ -2315,8 +2329,9 @@ void HhbcTranslator::emitSSwitch(const ImmVector& iv) { } ); + Trace* catchTrace = nullptr; // The slow path can throw exceptions and reenter the VM. - if (!fastPath) exceptionBarrier(); + if (!fastPath) catchTrace = getCatchTrace(); auto const testVal = popC(); @@ -2334,9 +2349,10 @@ void HhbcTranslator::emitSSwitch(const ImmVector& iv) { data.defaultOff = bcOff() + iv.strvec()[iv.size() - 1].dest; SSATmp* dest = gen(fastPath ? LdSSwitchDestFast - : LdSSwitchDestSlow, - data, - testVal); + : LdSSwitchDestSlow, + catchTrace, + data, + testVal); gen(DecRef, testVal); auto const stack = spillStack(); gen(SyncABIRegs, m_tb->getFp(), stack); @@ -2458,9 +2474,8 @@ void HhbcTranslator::emitVerifyParamType(int32_t paramId) { return; } if (tc.isCallable()) { - exceptionBarrier(); locVal = gen(Unbox, getExitTrace(), locVal); - gen(VerifyParamCallable, locVal, cns(paramId)); + gen(VerifyParamCallable, getCatchTrace(), locVal, cns(paramId)); return; } @@ -2471,8 +2486,7 @@ void HhbcTranslator::emitVerifyParamType(int32_t paramId) { locVal = gen(LdRef, locVal->type().innerType(), getExitTrace(), locVal); } if (!tc.checkPrimitive(locType.toDataType())) { - exceptionBarrier(); - gen(VerifyParamFail, cns(paramId)); + gen(VerifyParamFail, getCatchTrace(), cns(paramId)); return; } return; @@ -2507,8 +2521,7 @@ void HhbcTranslator::emitVerifyParamType(int32_t paramId) { } else { // The hint was self or parent and there's no corresponding // class for the current func. This typehint will always fail. - exceptionBarrier(); - gen(VerifyParamFail, cns(paramId)); + gen(VerifyParamFail, getCatchTrace(), cns(paramId)); return; } } @@ -2528,23 +2541,22 @@ void HhbcTranslator::emitVerifyParamType(int32_t paramId) { SSATmp* isInstance = haveBit ? gen(InstanceOfBitmask, objClass, cns(clsName)) : gen(ExtendsClass, objClass, constraint); - exceptionBarrier(); m_tb->ifThen(getCurFunc(), [&](Block* taken) { gen(JmpZero, taken, isInstance); }, [&] { // taken: the param type does not match m_tb->hint(Block::Unlikely); - gen(VerifyParamFail, cns(paramId)); + gen(VerifyParamFail, getCatchTrace(), cns(paramId)); } ); } else { - exceptionBarrier(); gen(VerifyParamCls, - objClass, - constraint, - cns(paramId), - cns(uintptr_t(&tc))); + getCatchTrace(), + objClass, + constraint, + cns(paramId), + cns(uintptr_t(&tc))); } } @@ -2651,6 +2663,7 @@ void HhbcTranslator::emitCastBool() { } void HhbcTranslator::emitCastDouble() { + Trace* catchTrace = getCatchTrace(); SSATmp* src = popC(); Type fromType = src->type(); if (fromType.isDbl()) { @@ -2667,15 +2680,14 @@ void HhbcTranslator::emitCastDouble() { } else if (fromType.isString()) { push(gen(ConvStrToDbl, src)); } else if (fromType.isObj()) { - exceptionBarrier(); - push(gen(ConvObjToDbl, src)); + push(gen(ConvObjToDbl, catchTrace, src)); } else { - exceptionBarrier(); // may throw - push(gen(ConvCellToDbl, src)); + push(gen(ConvCellToDbl, catchTrace, src)); } } void HhbcTranslator::emitCastInt() { + Trace* catchTrace = getCatchTrace(); SSATmp* src = popC(); Type fromType = src->type(); if (fromType.isInt()) { @@ -2693,11 +2705,9 @@ void HhbcTranslator::emitCastInt() { push(gen(ConvStrToInt, src)); gen(DecRef, src); } else if (fromType.isObj()) { - exceptionBarrier(); - push(gen(ConvObjToInt, src)); + push(gen(ConvObjToInt, catchTrace, src)); } else { - exceptionBarrier(); - push(gen(ConvCellToInt, src)); + push(gen(ConvCellToInt, catchTrace, src)); } } @@ -2712,6 +2722,7 @@ void HhbcTranslator::emitCastObject() { } void HhbcTranslator::emitCastString() { + Trace* catchTrace = getCatchTrace(); SSATmp* src = popC(); Type fromType = src->type(); if (fromType.isString()) { @@ -2728,11 +2739,9 @@ void HhbcTranslator::emitCastString() { } else if (fromType.isInt()) { push(gen(ConvIntToStr, src)); } else if (fromType.isObj()) { - exceptionBarrier(); - push(gen(ConvObjToStr, src)); + push(gen(ConvObjToStr, catchTrace, src)); } else { - exceptionBarrier(); - push(gen(ConvCellToStr, src)); + push(gen(ConvCellToStr, catchTrace, src)); } } @@ -2778,8 +2787,7 @@ void HhbcTranslator::emitBindMem(SSATmp* ptr, SSATmp* src) { gen(StMem, ptr, cns(0), src); if (isRefCounted(src) && src->type().canRunDtor()) { Block* exitBlock = getExitTrace(nextSrcKey().offset())->front(); - Block::iterator markerInst = exitBlock->skipLabel(); - exitBlock->insert(++markerInst, m_irFactory.gen(DecRef, prevValue)); + exitBlock->prepend(m_irFactory.gen(DecRef, prevValue)); gen(DecRefNZOrBranch, exitBlock, prevValue); } else { gen(DecRef, prevValue); @@ -3076,7 +3084,8 @@ void HhbcTranslator::emitXor() { */ void HhbcTranslator::emitInterpOne(Type type, int numPopped, int numExtraPushed) { - exceptionBarrier(); + // We're calling into the interpreter so we want the stack synced to memory. + SSATmp* sp = spillStack(); // discard the top elements of the stack, which are consumed by this instr discard(numPopped); assert(numPopped == m_stackDeficit); @@ -3085,7 +3094,7 @@ void HhbcTranslator::emitInterpOne(Type type, int numPopped, InterpOne, type, m_tb->getFp(), - m_tb->getSp(), + sp, cns(bcOff()), cns(numPopped - numPushed) ); @@ -3093,18 +3102,20 @@ void HhbcTranslator::emitInterpOne(Type type, int numPopped, } void HhbcTranslator::emitInterpOneCF(int numPopped) { - exceptionBarrier(); + // We're calling into the interpreter so we want the stack synced to memory. + SSATmp* sp = spillStack(); // discard the top elements of the stack, which are consumed by this instr discard(numPopped); assert(numPopped == m_stackDeficit); - gen(InterpOneCF, m_tb->getFp(), m_tb->getSp(), cns(bcOff())); + gen(InterpOneCF, m_tb->getFp(), sp, cns(bcOff())); m_stackDeficit = 0; m_hasExit = true; } /* * Get SSATmps representing all the information on the virtual eval - * stack in preparation for a spill or exit trace. + * stack in preparation for a spill or exit trace. Top of stack will + * be at index 0. * * Doesn't actually remove these values from the eval stack. */ @@ -3196,15 +3207,45 @@ Trace* HhbcTranslator::getExitTraceImpl(Offset targetBcOff, return exit; } +/* + * Create a catch trace for the current state of the eval stack. This is a + * trace intended to be invoked by the unwinder while unwinding a frame + * containing a call to C++ from translated code. When attached to an + * instruction as its taken field, code will be generated and the trace will be + * registered with the unwinder automatically. + */ +Trace* HhbcTranslator::getCatchTrace() { + auto exit = m_tb->makeExitTrace(bcOff()); + assert(exit->blocks().size() == 1); + + genFor(exit, BeginCatch); + exit->front()->push_back(makeMarker(bcOff())); + auto sp = emitSpillStack(exit, m_tb->getSp(), getSpillValues()); + genFor(exit, EndCatch, sp); + + assert(exit->blocks().size() == 1); + return exit; +} + +SSATmp* HhbcTranslator::emitSpillStack(Trace* t, SSATmp* sp, + const std::vector& spillVals) { + std::vector ssaArgs{ sp, cns(int64_t(m_stackDeficit)) }; + ssaArgs.insert(ssaArgs.end(), spillVals.begin(), spillVals.end()); + + auto args = std::make_pair(ssaArgs.size(), &ssaArgs[0]); + if (t->isMain()) { + return gen(SpillStack, args); + } else { + return genFor(t, SpillStack, args); + } +} + SSATmp* HhbcTranslator::spillStack() { - auto ssaArgs = getSpillValues(); - ssaArgs.insert( - ssaArgs.begin(), - { m_tb->getSp(), cns(int64_t(m_stackDeficit)) } - ); + auto newSp = + emitSpillStack(m_tb->trace(), m_tb->getSp(), getSpillValues()); m_evalStack.clear(); m_stackDeficit = 0; - return gen(SpillStack, std::make_pair(ssaArgs.size(), &ssaArgs[0])); + return newSp; } void HhbcTranslator::exceptionBarrier() { @@ -3260,15 +3301,18 @@ SSATmp* HhbcTranslator::ldLocInner(uint32_t locId, Trace* exitTrace) { } /* - * This is a wrapper to ldLocInner that also emits the RaiseUninitLoc - * if the local is uninitialized + * This is a wrapper to ldLocInner that also emits the RaiseUninitLoc if the + * local is uninitialized. The catchTrace argument may be provided if the + * caller requires the catch trace to be generated at a point earlier than when + * it calls this function. */ -SSATmp* HhbcTranslator::ldLocInnerWarn(uint32_t id, Trace* target) { +SSATmp* HhbcTranslator::ldLocInnerWarn(uint32_t id, Trace* target, + Trace* catchTrace /* = nullptr */) { + if (!catchTrace) catchTrace = getCatchTrace(); auto const locVal = ldLocInner(id, target); if (locVal->type().subtypeOf(Type::Uninit)) { - exceptionBarrier(); - gen(RaiseUninitLoc, cns(getCurFunc()->localVarName(id))); + gen(RaiseUninitLoc, catchTrace, cns(getCurFunc()->localVarName(id))); return m_tb->genDefInitNull(); } diff --git a/hphp/runtime/vm/translator/hopt/hhbctranslator.h b/hphp/runtime/vm/translator/hopt/hhbctranslator.h index 1ff1472aa..a35766249 100644 --- a/hphp/runtime/vm/translator/hopt/hhbctranslator.h +++ b/hphp/runtime/vm/translator/hopt/hhbctranslator.h @@ -261,7 +261,8 @@ struct HhbcTranslator { void emitFPushCtorCommon(SSATmp* cls, SSATmp* obj, const Func* func, - int32_t numParams); + int32_t numParams, + Trace* catchTrace); void emitCreateCl(int32_t numParams, int32_t classNameStrId); void emitFCallArray(const Offset pcOffset, const Offset after); void emitFCall(uint32_t numParams, @@ -394,6 +395,7 @@ private: void emitMPre(); void emitFinalMOp(); void emitMPost(); + void emitSideExits(SSATmp* catchSp, int nStack); void emitMTrace(); // Bases @@ -455,13 +457,12 @@ private: * if appropriate. */ template - SSATmp* genStk(Opcode op, Srcs... srcs); + SSATmp* genStk(Opcode op, Trace* taken, Srcs... srcs); /* Various predicates about the current instruction */ bool isSimpleArrayOp(); bool isSimpleBase(); bool isSingleMember(); - bool usePredictedResult(); bool generateMVal() const; bool needFirstRatchet() const; @@ -482,6 +483,7 @@ private: const Transl::NormalizedInstruction& m_ni; HhbcTranslator& m_ht; TraceBuilder& m_tb; + IRFactory& m_irf; const MInstrInfo& m_mii; hphp_hash_map m_stackInputs; @@ -490,10 +492,29 @@ private: bool m_needMIS; + /* The base for any accesses to the current MInstrState. */ SSATmp* m_misBase; + + /* The value of the base for the next member operation. Starts as the base + * for the whole instruction and is updated as the translator makes + * progress. */ SSATmp* m_base; + + /* The result of the vector instruction. nullptr if the current instruction + * doesn't produce a result. */ SSATmp* m_result; - SSATmp* m_predictedResult; + + /* If set, contains a value of type CountedStr|Nullptr. If a runtime test + * determines that the value is not Nullptr, we incorrectly predicted the + * output type of the instruction and must side exit. */ + SSATmp* m_strTestResult; + + /* If set, contains the catch trace for the final set operation of this + * instruction. The operations that set this member may need to return an + * unexpected type, in which case they'll throw an InvalidSetMException. To + * handle this, emitMPost adds code to the catch trace to fish the correct + * value out of the exception and side exit. */ + Trace* m_failedSetTrace; }; private: // tracebuilder forwarding utilities @@ -571,6 +592,7 @@ private: void emitBinaryArith(Opcode); template SSATmp* emitIterInitCommon(int offset, Lambda genFunc); + IRInstruction* makeMarker(Offset bcOff); void emitMarker(); // Exit trace creation routines. @@ -581,6 +603,7 @@ private: std::vector& spillValues, const StringData* warning); Trace* getExitSlowTrace(); + Trace* getCatchTrace(); enum class ExitFlag { None, @@ -636,6 +659,8 @@ private: SSATmp* popF() { return pop(Type::Gen); } SSATmp* topC(uint32_t i = 0) { return top(Type::Cell, i); } std::vector getSpillValues() const; + SSATmp* emitSpillStack(Trace* t, SSATmp* sp, + const std::vector& spillVals); SSATmp* spillStack(); void exceptionBarrier(); SSATmp* ldStackAddr(int32_t offset); @@ -650,7 +675,8 @@ private: SSATmp* ldLoc(uint32_t id); SSATmp* ldLocAddr(uint32_t id); SSATmp* ldLocInner(uint32_t id, Trace* exitTrace); - SSATmp* ldLocInnerWarn(uint32_t id, Trace* target); + SSATmp* ldLocInnerWarn(uint32_t id, Trace* target, + Trace* catchTrace = nullptr); SSATmp* stLoc(uint32_t id, Trace* exitTrace, SSATmp* newVal); SSATmp* stLocNRC(uint32_t id, Trace* exitTrace, SSATmp* newVal); SSATmp* stLocImpl(uint32_t id, Trace*, SSATmp* newVal, bool doRefCount); diff --git a/hphp/runtime/vm/translator/hopt/ir.cpp b/hphp/runtime/vm/translator/hopt/ir.cpp index a261fe947..d5b0ef237 100644 --- a/hphp/runtime/vm/translator/hopt/ir.cpp +++ b/hphp/runtime/vm/translator/hopt/ir.cpp @@ -108,10 +108,11 @@ namespace { #define DParam HasDest #define DArith HasDest #define DMulti NaryDest -#define DVector HasDest +#define DSetElem HasDest #define DStk(x) ModifiesStack|(x) #define DPtrToParam HasDest #define DBuiltin HasDest +#define DSubtract(n,t) HasDest struct { const char* name; @@ -151,10 +152,11 @@ struct { #undef DParam #undef DArith #undef DMulti -#undef DVector +#undef DSetElem #undef DStk #undef DPtrToParam #undef DBuiltin +#undef DSubtract ////////////////////////////////////////////////////////////////////// @@ -408,7 +410,7 @@ bool IRInstruction::mayModifyRefs() const { // count. Therefore, its MayModifyRefs should be false. if (opc == DecRef) { auto type = src(0)->type(); - if (isControlFlowInstruction()) { + if (isControlFlow()) { // If the decref has a target label, then it exits if the destructor // has to be called, so it does not have any side effects on the main // trace. @@ -437,7 +439,7 @@ bool IRInstruction::isEssential() const { return true; } } - return isControlFlowInstruction() || + return isControlFlow() || opcodeHasFlags(opc, Essential) || mayReenterHelper(); } @@ -778,10 +780,9 @@ Opcode commuteQueryOp(Opcode opc) { // __toString function. bool cmpOpTypesMayReenter(Opcode op, Type t0, Type t1) { if (op == OpNSame || op == OpSame) return false; - assert(t0 != Type::Gen && t1 != Type::Gen); - return (t0 == Type::Cell || t1 == Type::Cell) || - ((t0 == Type::Obj || t1 == Type::Obj) && - (t0.isString() || t1.isString())); + assert(!t0.equals(Type::Gen) && !t1.equals(Type::Gen)); + return (t0.maybe(Type::Obj) && t1.maybe(Type::Str)) || + (t0.maybe(Type::Str) && t1.maybe(Type::Obj)); } bool isRefCounted(SSATmp* tmp) { @@ -810,19 +811,20 @@ void IRInstruction::convertToNop() { } void IRInstruction::convertToJmp() { - assert(isControlFlowInstruction()); - assert(block()->back() == this); + assert(isControlFlow()); + assert(IMPLIES(block(), block()->back() == this)); m_op = Jmp_; m_typeParam = Type::None; m_numSrcs = 0; m_numDsts = 0; m_srcs = nullptr; m_dst = nullptr; - block()->setNext(nullptr); + // Instructions in the simplifier don't have blocks yet. + if (block()) block()->setNext(nullptr); } void IRInstruction::convertToMov() { - assert(!isControlFlowInstruction()); + assert(!isControlFlow()); m_op = Mov; m_typeParam = Type::None; assert(m_numSrcs == 1); @@ -916,9 +918,12 @@ std::string IRInstruction::toString() const { return str.str(); } -int SSATmp::numNeededRegs() const { - auto t = type(); - if (t.subtypeOfAny(Type::None, Type::Null, Type::ActRec, Type::RetAddr)) { +namespace { +int typeNeededRegs(Type t) { + assert(!t.equals(Type::Bottom)); + + if (t.subtypeOfAny(Type::None, Type::Null, Type::ActRec, Type::RetAddr, + Type::Nullptr)) { // These don't need a register because their values are static or unused. // // RetAddr doesn't take any register because currently we only target x86, @@ -926,6 +931,9 @@ int SSATmp::numNeededRegs() const { // moved to a machine-specific section once we target other architectures. return 0; } + if (t.maybe(Type::Nullptr)) { + return typeNeededRegs(t - Type::Nullptr); + } if (t.subtypeOf(Type::Ctx) || t.isPtr()) { // Ctx and PtrTo* may be statically unknown but always need just 1 register. return 1; @@ -944,6 +952,11 @@ int SSATmp::numNeededRegs() const { assert(t.subtypeOf(Type::Gen)); return t.needsReg() ? 2 : 1; } +} + +int SSATmp::numNeededRegs() const { + return typeNeededRegs(type()); +} bool SSATmp::getValBool() const { assert(isConst()); diff --git a/hphp/runtime/vm/translator/hopt/ir.h b/hphp/runtime/vm/translator/hopt/ir.h index eb35f7901..531556916 100644 --- a/hphp/runtime/vm/translator/hopt/ir.h +++ b/hphp/runtime/vm/translator/hopt/ir.h @@ -98,13 +98,15 @@ class FailedIRGen : public std::exception { * DParam single dst has type of the instruction's type parameter * DArith single dst has a type based on arithmetic type rules * DMulti multiple dests. type and number depend on instruction - * DVector single dst depends on semantics of the vector instruction + * DSetElem single dst is a subset of CountedStr|Nullptr depending on + * sources * DStk(x) up to two dests. x should be another D* macro and indicates * the type of the first dest, if any. the second (or first, * depending on the presence of a primary destination), will be * of type Type::StkPtr. implies ModifiesStack. * DBuiltin single dst for CallBuiltin. This can return complex data * types such as (Type::Str | Type::Null) + * DSubtract(N,t) single dest has type of src N with t removed * * srcinfo: * @@ -153,7 +155,7 @@ class FailedIRGen : public std::exception { #define IR_OPCODES \ /* name dstinfo srcinfo flags */ \ -O(CheckType, DParam, S(Gen), C|E|CRc|PRc|P) \ +O(CheckType, DParam, S(Gen,Nullptr), C|E|CRc|PRc|P) \ O(CheckTypeMem, ND, S(PtrToGen), E) \ O(GuardLoc, ND, S(FramePtr), E) \ O(GuardStk, D(StkPtr), S(StkPtr), E) \ @@ -169,6 +171,10 @@ O(GuardRefs, ND, S(Func) \ S(Int), E) \ O(AssertLoc, ND, S(FramePtr), E) \ O(OverrideLoc, ND, S(FramePtr), E) \ +O(BeginCatch, ND, NA, E|Mem) \ +O(EndCatch, ND, S(StkPtr), E|Mem) \ +O(LdUnwinderValue, DParam, NA, PRc) \ +O(DeleteUnwinderException, ND, NA, N|E|Mem) \ O(OpAdd, DArith, S(Int,Dbl) S(Int,Dbl), C) \ O(OpSub, DArith, S(Int,Dbl) S(Int,Dbl), C) \ O(OpMul, DArith, S(Int,Dbl) S(Int,Dbl), C) \ @@ -269,10 +275,11 @@ O(JmpIndirect, ND, S(TCA), T|E) \ O(ExitWhenSurprised, ND, NA, E) \ O(ExitOnVarEnv, ND, S(FramePtr), E) \ O(ReleaseVVOrExit, ND, S(FramePtr), N|E) \ -O(RaiseError, ND, S(Str), E|N|Mem|Refs|T) \ +O(RaiseError, ND, S(Str), E|N|Mem|Refs|T|Er) \ O(RaiseWarning, ND, S(Str), E|N|Mem|Refs|Er) \ O(CheckInit, ND, S(Gen), NF) \ O(CheckInitMem, ND, S(PtrToGen) C(Int), NF) \ +O(AssertNonNull, DSubtract(0, Nullptr), S(Nullptr,CountedStr), NF) \ O(Unbox, DUnbox(0), S(Gen), NF) \ O(Box, DBox(0), S(Init), E|N|Mem|CRc|PRc) \ O(UnboxPtr, D(PtrToCell), S(PtrToGen), NF) \ @@ -505,7 +512,7 @@ O_STK(BindProp, ND, C(TCA) \ S(Cell) \ S(BoxedCell) \ S(PtrToCell),VProp|E|N|Mem|Refs|Er) \ -O_STK(SetProp, DVector, C(TCA) \ +O_STK(SetProp, ND, C(TCA) \ C(Cls) \ S(Obj,PtrToGen) \ S(Cell) \ @@ -570,7 +577,7 @@ O(ArraySetRef, ND, C(TCA) \ S(Int,Str) \ S(Cell) \ S(BoxedArr),E|N|PRc|CRc|Refs|Mem|K) \ -O_STK(SetElem, DVector, C(TCA) \ +O_STK(SetElem, DSetElem, C(TCA) \ S(PtrToGen) \ S(Cell) \ S(Cell), VElem|E|N|Mem|Refs|Er) \ @@ -591,7 +598,7 @@ O_STK(IncDecElem, D(Cell), C(TCA) \ S(PtrToGen) \ S(Cell) \ S(PtrToCell),VElem|E|N|Mem|Refs|Er) \ -O_STK(SetNewElem, DVector, S(PtrToGen) \ +O_STK(SetNewElem, ND, S(PtrToGen) \ S(Cell), VElem|E|N|Mem|Refs|Er) \ O_STK(SetWithRefNewElem, ND, C(TCA) \ S(PtrToGen) \ @@ -899,15 +906,32 @@ struct VectorEffects { VectorEffects(Opcode opc, const std::vector& srcs); Type baseType; - Type valType; bool baseTypeChanged; bool baseValChanged; - bool valTypeChanged; private: void init(Opcode op, const Type base, const Type key, const Type val); }; +struct CatchInfo { + /* afterCall is the address after the call instruction that this catch trace + * belongs to. It's the key used to look up catch traces by the + * unwinder, since it's the value of %rip during unwinding. */ + TCA afterCall; + + /* savedRegs contains the caller-saved registers that were pushed onto the + * C++ stack at the time of the call. The catch trace will pop these + * registers (in the same order as PhysRegSaver's destructor) before doing + * any real work to restore the register state from before the call. */ + RegSet savedRegs; + + /* rspOffset is the number of bytes pushed on the C++ stack after the + * registers in savedRegs were saved, typically from function calls with >6 + * arguments. The catch trace will adjust rsp by this amount before popping + * anything in savedRegs. */ + Offset rspOffset; +}; + typedef folly::Range TcaRange; /** diff --git a/hphp/runtime/vm/translator/hopt/irinstruction.h b/hphp/runtime/vm/translator/hopt/irinstruction.h index cbc7c438a..d244a4018 100644 --- a/hphp/runtime/vm/translator/hopt/irinstruction.h +++ b/hphp/runtime/vm/translator/hopt/irinstruction.h @@ -243,7 +243,7 @@ struct IRInstruction { else m_taken.setTo(b); } - bool isControlFlowInstruction() const { return bool(m_taken.to()); } + bool isControlFlow() const { return bool(m_taken.to()); } bool isBlockEnd() const { return m_taken.to() || isTerminal(); } bool isLoad() const; bool stores(uint32_t srcIdx) const; diff --git a/hphp/runtime/vm/translator/hopt/irtranslator.cpp b/hphp/runtime/vm/translator/hopt/irtranslator.cpp index 36ed4d606..78acba083 100644 --- a/hphp/runtime/vm/translator/hopt/irtranslator.cpp +++ b/hphp/runtime/vm/translator/hopt/irtranslator.cpp @@ -1977,8 +1977,8 @@ void TranslatorX64::hhirTraceCodeGen(vector* bcMap) { auto finishPass = [&](const char* msg, int level, const RegAllocInfo* regs = nullptr, const LifetimeInfo* lifetime = nullptr) { - assert(checkCfg(trace, *m_irFactory)); dumpTrace(level, trace, msg, regs, lifetime); + assert(checkCfg(trace, *m_irFactory)); }; finishPass(" after initial translation ", kIRLevel); diff --git a/hphp/runtime/vm/translator/hopt/jumpopts.cpp b/hphp/runtime/vm/translator/hopt/jumpopts.cpp index af9927465..22d9f7bdc 100644 --- a/hphp/runtime/vm/translator/hopt/jumpopts.cpp +++ b/hphp/runtime/vm/translator/hopt/jumpopts.cpp @@ -53,7 +53,7 @@ void elimUnconditionalJump(Trace* trace, IRFactory* irFactory) { IRInstruction& jmp = *lastInst; if (jmp.op() == Jmp_ && !isJoin[jmp.taken()->id()]) { Block* target = jmp.taken(); - lastBlock->splice(lastInst, target, target->skipLabel(), target->end()); + lastBlock->splice(lastInst, target, target->skipHeader(), target->end()); lastBlock->erase(lastInst); // delete the jmp } } @@ -92,7 +92,7 @@ Block* findMainExitBlock(Trace* trace, IRFactory* irFactory) { struct BlockMatcher { explicit BlockMatcher(Block* block) : m_block(block) - , m_it(block->skipLabel()) + , m_it(block->skipHeader()) {} bool match() { return true; } diff --git a/hphp/runtime/vm/translator/hopt/mutation.cpp b/hphp/runtime/vm/translator/hopt/mutation.cpp index 662aaf9b9..311c46e03 100644 --- a/hphp/runtime/vm/translator/hopt/mutation.cpp +++ b/hphp/runtime/vm/translator/hopt/mutation.cpp @@ -42,9 +42,9 @@ void cloneToBlock(const BlockList& rpoBlocks, } }; - auto targetIt = target->skipLabel(); + auto targetIt = target->skipHeader(); for (auto it = first; it != last; ++it) { - assert(!it->isControlFlowInstruction()); + assert(!it->isControlFlow()); FTRACE(5, "cloneToBlock({}): {}\n", target->id(), it->toString()); auto const newInst = irFactory->cloneInstruction(&*it); @@ -79,10 +79,10 @@ void moveToBlock(Block::iterator const first, auto const srcBlock = first->block(); - auto targetIt = target->skipLabel(); + auto targetIt = target->skipHeader(); for (auto it = first; it != last;) { auto const inst = &*it; - assert(!inst->isControlFlowInstruction()); + assert(!inst->isControlFlow()); FTRACE(5, "moveToBlock({}): {}\n", target->id(), diff --git a/hphp/runtime/vm/translator/hopt/nativecalls.cpp b/hphp/runtime/vm/translator/hopt/nativecalls.cpp index 65e302ffe..8d0844496 100644 --- a/hphp/runtime/vm/translator/hopt/nativecalls.cpp +++ b/hphp/runtime/vm/translator/hopt/nativecalls.cpp @@ -195,7 +195,7 @@ static CallMap s_callMap({ {{SSA, 1}, {SSA, 2}, {VecKeyS, 3}, {SSA, 4}}}, {BindProp, {FSSA, 0}, DNone, SSync, {{SSA, 1}, {SSA, 2}, {TV, 3}, {SSA, 4}, {SSA, 5}}}, - {SetProp, {FSSA, 0}, DTV, SSync, + {SetProp, {FSSA, 0}, DNone, SSync, {{SSA, 1}, {SSA, 2}, {TV, 3}, {TV, 4}}}, {UnsetProp, {FSSA, 0}, DNone, SSync, {{SSA, 1}, {SSA, 2}, {TV, 3}}}, @@ -227,7 +227,7 @@ static CallMap s_callMap({ {{SSA, 1}, {SSA, 2}, {TV, 3}}}, {ArraySetRef, {FSSA, 0}, DSSA, SSync, {{SSA, 1}, {SSA, 2}, {TV, 3}, {SSA, 4}}}, - {SetElem, {FSSA, 0}, DTV, SSync, + {SetElem, {FSSA, 0}, DSSA, SSync, {{SSA, 1}, {VecKeyIS, 2}, {TV, 3}}}, {UnsetElem, {FSSA, 0}, DNone, SSync, {{SSA, 1}, {VecKeyIS, 2}}}, @@ -235,7 +235,7 @@ static CallMap s_callMap({ {{SSA, 1}, {TV, 2}, {TV, 3}, {SSA, 4}}}, {IncDecElem, {FSSA, 0}, DTV, SSync, {{SSA, 1}, {TV, 2}, {SSA, 3}}}, - {SetNewElem, (TCA)setNewElem, DTV, SSync, {{SSA, 0}, {TV, 1}}}, + {SetNewElem, (TCA)setNewElem, DNone, SSync, {{SSA, 0}, {TV, 1}}}, {SetWithRefNewElem, {FSSA, 0}, DNone, SSync, {{SSA, 1}, {SSA, 2}, {SSA, 3}}}, {BindNewElem, (TCA)bindNewElemIR, DNone, SSync, diff --git a/hphp/runtime/vm/translator/hopt/predictionopts.cpp b/hphp/runtime/vm/translator/hopt/predictionopts.cpp index 557c12d86..a351e7006 100644 --- a/hphp/runtime/vm/translator/hopt/predictionopts.cpp +++ b/hphp/runtime/vm/translator/hopt/predictionopts.cpp @@ -123,7 +123,7 @@ void optimizePredictions(Trace* const trace, IRFactory* const irFactory) { // Clone the instructions to the exit before specializing. cloneToBlock(rpoSort, irFactory, sinkFirst, sinkLast, exit); - exit->insert(exit->skipLabel(), irFactory->cloneInstruction(lastMarker)); + exit->insert(exit->skipHeader(), irFactory->cloneInstruction(lastMarker)); /* * Specialize the LdMem left on the main trace after cloning the @@ -145,7 +145,7 @@ void optimizePredictions(Trace* const trace, IRFactory* const irFactory) { // Move the fallthrough case to specialized. moveToBlock(sinkFirst, boost::next(sinkLast), specialized); - specialized->insert(specialized->skipLabel(), + specialized->insert(specialized->skipHeader(), irFactory->cloneInstruction(lastMarker)); reflowTypes(specialized, rpoSort); diff --git a/hphp/runtime/vm/translator/hopt/print.cpp b/hphp/runtime/vm/translator/hopt/print.cpp index ee482fd12..ecea00d9c 100644 --- a/hphp/runtime/vm/translator/hopt/print.cpp +++ b/hphp/runtime/vm/translator/hopt/print.cpp @@ -132,10 +132,7 @@ void print(std::ostream& ostream, const IRInstruction* inst, if (inst->op() == Marker) { auto* marker = inst->extra(); ostream << color(ANSI_COLOR_BLUE) - << folly::format("--- bc {}, spOff {} ({})", - marker->bcOff, - marker->stackOff, - marker->func->fullName()->data()) + << marker->show() << color(ANSI_COLOR_END); return; } diff --git a/hphp/runtime/vm/translator/hopt/simplifier.cpp b/hphp/runtime/vm/translator/hopt/simplifier.cpp index 4314c7bfa..d72233876 100644 --- a/hphp/runtime/vm/translator/hopt/simplifier.cpp +++ b/hphp/runtime/vm/translator/hopt/simplifier.cpp @@ -264,7 +264,7 @@ SSATmp* Simplifier::simplify(IRInstruction* inst) { case OpNeq: case OpSame: case OpNSame: - return simplifyCmp(opc, src1, src2); + return simplifyCmp(opc, inst, src1, src2); case Concat: return simplifyConcat(src1, src2); case Mov: return simplifyMov(src1); @@ -323,6 +323,7 @@ SSATmp* Simplifier::simplify(IRInstruction* inst) { case DecRefNZ: return simplifyDecRef(inst); case IncRef: return simplifyIncRef(inst); case CheckType: return simplifyCheckType(inst); + case AssertNonNull:return simplifyAssertNonNull(inst); case LdCls: return simplifyLdCls(inst); case LdThis: return simplifyLdThis(inst); @@ -512,7 +513,7 @@ SSATmp* Simplifier::simplifyQueryJmp(IRInstruction* inst) { SSATmp* src2 = inst->src(1); Opcode opc = inst->op(); // reuse the logic in simplifyCmp. - SSATmp* newCmp = simplifyCmp(queryJmpToQueryOp(opc), src1, src2); + SSATmp* newCmp = simplifyCmp(queryJmpToQueryOp(opc), nullptr, src1, src2); if (!newCmp) return nullptr; SSATmp* newQueryJmp = makeInstruction( @@ -932,7 +933,11 @@ static typename std::common_type::type cmpOp(Opcode opName, T a, U b) { } } -SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { +SSATmp* Simplifier::simplifyCmp(Opcode opName, IRInstruction* inst, + SSATmp* src1, SSATmp* src2) { + auto newInst = [inst, this](Opcode op, SSATmp* src1, SSATmp* src2) { + return gen(op, inst ? inst->taken() : (Block*)nullptr, src1, src2); + }; // --------------------------------------------------------------------- // Perform some execution optimizations immediately // --------------------------------------------------------------------- @@ -985,9 +990,9 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { } // Type is neither a string nor an object - simplify to OpEq/OpNeq if (opName == OpSame) { - return gen(OpEq, src1, src2); + return newInst(OpEq, src1, src2); } - return gen(OpNeq, src1, src2); + return newInst(OpNeq, src1, src2); } // --------------------------------------------------------------------- @@ -1047,9 +1052,9 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { // E.g. `some-int > false` is equivalent to `some-int == true` if (opName != OpEq) { if (cmpOp(opName, false, b)) { - return gen(OpEq, src1, cns(false)); + return newInst(OpEq, src1, cns(false)); } else { - return gen(OpEq, src1, cns(true)); + return newInst(OpEq, src1, cns(true)); } } } @@ -1062,7 +1067,7 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { if (src1->type() == src2->type() || (src1->type().isString() && src2->type().isString())) { if (src1->isConst() && !src2->isConst()) { - return gen(commuteQueryOp(opName), src2, src1); + return newInst(commuteQueryOp(opName), src2, src1); } return nullptr; } @@ -1074,32 +1079,32 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { // nulls get canonicalized to the right if (src1->type().isNull()) { - return gen(commuteQueryOp(opName), src2, src1); + return newInst(commuteQueryOp(opName), src2, src1); } // case 1: null cmp string. Convert null to "" if (src1->type().isString() && src2->type().isNull()) { - return gen(opName, src1, cns(StringData::GetStaticString(""))); + return newInst(opName, src1, cns(StringData::GetStaticString(""))); } // case 2a: null cmp anything. Convert null to false if (src2->type().isNull()) { - return gen(opName, src1, cns(false)); + return newInst(opName, src1, cns(false)); } // bools get canonicalized to the right if (src1->type() == Type::Bool) { - return gen(commuteQueryOp(opName), src2, src1); + return newInst(commuteQueryOp(opName), src2, src1); } // case 2b: bool cmp anything. Convert anything to bool if (src2->type() == Type::Bool) { if (src1->isConst()) { if (src1->type() == Type::Int) { - return gen(opName, cns(bool(src1->getValInt())), src2); + return newInst(opName, cns(bool(src1->getValInt())), src2); } else if (src1->type().isString()) { auto str = src1->getValStr(); - return gen(opName, cns(str->toBoolean()), src2); + return newInst(opName, cns(str->toBoolean()), src2); } } @@ -1109,14 +1114,14 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { always_assert(opName == OpEq); if (src2->getValBool()) { - return gen(OpNeq, src1, cns(0)); + return newInst(OpNeq, src1, cns(0)); } else { - return gen(OpEq, src1, cns(0)); + return newInst(OpEq, src1, cns(0)); } } // Nothing fancy to do - perform juggling as normal. - return gen(opName, gen(ConvCellToBool, src1), src2); + return newInst(opName, gen(ConvCellToBool, src1), src2); } // From here on, we must be careful of how Type::Obj gets dealt with, @@ -1127,12 +1132,12 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { // strings get canonicalized to the left if (src2->type().isString()) { - return gen(commuteQueryOp(opName), src2, src1); + return newInst(commuteQueryOp(opName), src2, src1); } // ints get canonicalized to the right if (src1->type() == Type::Int) { - return gen(commuteQueryOp(opName), src2, src1); + return newInst(commuteQueryOp(opName), src2, src1); } // case 4: number/string/resource cmp. Convert to number (int OR double) @@ -1146,12 +1151,12 @@ SSATmp* Simplifier::simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2) { int64_t si; double sd; auto st = str->isNumericWithVal(si, sd, true /* allow errors */); if (st == KindOfDouble) { - return gen(opName, cns(sd), src2); + return newInst(opName, cns(sd), src2); } if (st == KindOfNull) { si = 0; } - return gen(opName, cns(si), src2); + return newInst(opName, cns(si), src2); } // case 5: array cmp array. No juggling to do @@ -1672,6 +1677,15 @@ SSATmp* Simplifier::simplifyDecRefStack(IRInstruction* inst) { return nullptr; } +SSATmp* Simplifier::simplifyAssertNonNull(IRInstruction* inst) { + auto t = inst->typeParam(); + assert(t.maybe(Type::Nullptr)); + if (t.subtypeOf(Type::Nullptr)) { + return inst->src(0); + } + return nullptr; +} + ////////////////////////////////////////////////////////////////////// }} diff --git a/hphp/runtime/vm/translator/hopt/simplifier.h b/hphp/runtime/vm/translator/hopt/simplifier.h index c7e9215cd..8feaea1c4 100644 --- a/hphp/runtime/vm/translator/hopt/simplifier.h +++ b/hphp/runtime/vm/translator/hopt/simplifier.h @@ -115,7 +115,8 @@ private: SSATmp* simplifyGetCtxFwdCall(IRInstruction* inst); SSATmp* simplifySpillStack(IRInstruction* inst); SSATmp* simplifyCall(IRInstruction* inst); - SSATmp* simplifyCmp(Opcode opName, SSATmp* src1, SSATmp* src2); + SSATmp* simplifyCmp(Opcode opName, IRInstruction* inst, + SSATmp* src1, SSATmp* src2); SSATmp* simplifyCondJmp(IRInstruction*); SSATmp* simplifyQueryJmp(IRInstruction*); SSATmp* simplifyExitOnVarEnv(IRInstruction*); @@ -127,6 +128,7 @@ private: SSATmp* simplifyDecRefLoc(IRInstruction*); SSATmp* simplifyLdLoc(IRInstruction*); SSATmp* simplifyStRef(IRInstruction*); + SSATmp* simplifyAssertNonNull(IRInstruction*); private: // tracebuilder forwarders template SSATmp* cns(Args&&...); diff --git a/hphp/runtime/vm/translator/hopt/test/type.cpp b/hphp/runtime/vm/translator/hopt/test/type.cpp index 3ce3f5663..d9d8d2f59 100644 --- a/hphp/runtime/vm/translator/hopt/test/type.cpp +++ b/hphp/runtime/vm/translator/hopt/test/type.cpp @@ -189,6 +189,7 @@ TEST(Type, CanRunDtor) { expectTrue(Type::Ctx); expectTrue(Type::Obj | Type::Func); expectTrue(Type::Init); + expectTrue(Type::Top); for (Type t : types) { EXPECT_FALSE(t.canRunDtor()) << t.toString() << ".canRunDtor == false"; @@ -202,4 +203,14 @@ TEST(Type, UnionOf) { EXPECT_EQ(Type::Gen, Type::unionOf(Type::Cell, Type::BoxedInt)); } +TEST(Type, Top) { + for (auto t : allTypes()) { + EXPECT_TRUE(t.subtypeOf(Type::Top)); + } + for (auto t : allTypes()) { + if (t.equals(Type::Top)) continue; + EXPECT_FALSE(Type::Top.subtypeOf(t)); + } +} + } } diff --git a/hphp/runtime/vm/translator/hopt/test/vector.cpp b/hphp/runtime/vm/translator/hopt/test/vector.cpp index a6ad6a1f3..cfacc025d 100644 --- a/hphp/runtime/vm/translator/hopt/test/vector.cpp +++ b/hphp/runtime/vm/translator/hopt/test/vector.cpp @@ -30,69 +30,53 @@ namespace HPHP { namespace JIT { TEST(VectorEffects, Basic) { VectorEffects elem(SetElem, Type::PtrToArr, Type::Int, Type::Str); EXPECT_TEQ(Type::PtrToArr, elem.baseType); - EXPECT_TEQ(Type::Str, elem.valType); EXPECT_FALSE(elem.baseTypeChanged); EXPECT_TRUE(elem.baseValChanged); - EXPECT_FALSE(elem.valTypeChanged); VectorEffects prop(SetProp, Type::Obj, Type::StaticStr, Type::Str); EXPECT_TEQ(Type::Obj, prop.baseType); - EXPECT_TEQ(Type::Str, prop.valType); EXPECT_FALSE(prop.baseTypeChanged); EXPECT_FALSE(prop.baseValChanged); - EXPECT_FALSE(prop.valTypeChanged); } TEST(VectorEffects, BadArrayKey) { VectorEffects ve(SetElem, Type::PtrToArr, Type::Arr, Type::Int); EXPECT_TEQ(Type::PtrToArr, ve.baseType); - EXPECT_TEQ(Type::InitNull, ve.valType); EXPECT_FALSE(ve.baseTypeChanged); EXPECT_TRUE(ve.baseValChanged); - EXPECT_TRUE(ve.valTypeChanged); } TEST(VectorEffects, NonObjProp) { VectorEffects ve(SetProp, Type::PtrToInt, Type::Str, Type::Dbl); EXPECT_TEQ(Type::PtrToInt, ve.baseType); - EXPECT_TEQ(Type::InitNull, ve.valType); EXPECT_FALSE(ve.baseTypeChanged); EXPECT_FALSE(ve.baseValChanged); - EXPECT_TRUE(ve.valTypeChanged); } TEST(VectorEffects, NonArrElem) { VectorEffects ve(SetElem, Type::PtrToDbl, Type::Int, Type::Obj); EXPECT_TEQ(Type::PtrToDbl, ve.baseType); - EXPECT_TEQ(Type::InitNull, ve.valType); EXPECT_FALSE(ve.baseTypeChanged); EXPECT_FALSE(ve.baseValChanged); - EXPECT_TRUE(ve.valTypeChanged); } TEST(VectorEffects, PromoteNull) { VectorEffects elem(SetElem, Type::PtrToNull, Type::Int, Type::Str); EXPECT_TEQ(Type::PtrToArr, elem.baseType); - EXPECT_TEQ(Type::Str, elem.valType); EXPECT_TRUE(elem.baseTypeChanged); EXPECT_TRUE(elem.baseValChanged); - EXPECT_FALSE(elem.valTypeChanged); VectorEffects prop(SetProp, Type::PtrToUninit, Type::StaticStr, Type::Str); EXPECT_TEQ(Type::PtrToObj, prop.baseType); - EXPECT_TEQ(Type::Str, prop.valType); EXPECT_TRUE(prop.baseTypeChanged); EXPECT_TRUE(prop.baseValChanged); - EXPECT_FALSE(prop.valTypeChanged); } TEST(VectorEffects, UnknownBase) { VectorEffects ve(SetElem, Type::PtrToCell, Type::Int, Type::Obj); EXPECT_TEQ(Type::PtrToCell, ve.baseType); - EXPECT_TEQ(Type::Obj|Type::InitNull, ve.valType); EXPECT_FALSE(ve.baseTypeChanged); EXPECT_TRUE(ve.baseValChanged); - EXPECT_TRUE(ve.valTypeChanged); } } } diff --git a/hphp/runtime/vm/translator/hopt/trace.h b/hphp/runtime/vm/translator/hopt/trace.h index 53f28313b..c53a8ca23 100644 --- a/hphp/runtime/vm/translator/hopt/trace.h +++ b/hphp/runtime/vm/translator/hopt/trace.h @@ -95,6 +95,15 @@ struct Trace : private boost::noncopyable { return exit; } bool isMain() const { return m_main == nullptr; } + bool isCatch() const { + auto it = front()->skipHeader(); + if (it == front()->end()) return false; + + assert(it->op() == Marker); + if (it == front()->begin()) return false; + + return (--it)->op() == BeginCatch; + } void setMain(Trace* t) { assert(m_main == nullptr); m_main = t; diff --git a/hphp/runtime/vm/translator/hopt/tracebuilder.cpp b/hphp/runtime/vm/translator/hopt/tracebuilder.cpp index 7ff9f8445..fb0076ec5 100644 --- a/hphp/runtime/vm/translator/hopt/tracebuilder.cpp +++ b/hphp/runtime/vm/translator/hopt/tracebuilder.cpp @@ -785,13 +785,13 @@ SSATmp* TraceBuilder::optimizeWork(IRInstruction* inst) { return nullptr; } -SSATmp* TraceBuilder::optimizeInst(IRInstruction* inst) { +SSATmp* TraceBuilder::optimizeInst(IRInstruction* inst, CloneFlag doClone) { if (SSATmp* tmp = optimizeWork(inst)) { return tmp; } // Couldn't CSE or simplify the instruction; clone it and append. if (inst->op() != Nop) { - inst = inst->clone(&m_irFactory); + if (doClone == CloneFlag::Yes) inst = inst->clone(&m_irFactory); appendInstruction(inst); // returns nullptr if instruction has no dest, returns the first // (possibly only) dest otherwise diff --git a/hphp/runtime/vm/translator/hopt/tracebuilder.h b/hphp/runtime/vm/translator/hopt/tracebuilder.h index 04ef089b7..2e52da733 100644 --- a/hphp/runtime/vm/translator/hopt/tracebuilder.h +++ b/hphp/runtime/vm/translator/hopt/tracebuilder.h @@ -125,7 +125,9 @@ struct TraceBuilder { template SSATmp* gen(Args&&... args) { return makeInstruction( - [this] (IRInstruction* inst) { return optimizeInst(inst); }, + [this] (IRInstruction* inst) { + return optimizeInst(inst, CloneFlag::Yes); + }, std::forward(args)... ); } @@ -134,17 +136,25 @@ struct TraceBuilder { * Create an IRInstruction, similar to gen(), except link it into * the Trace t instead of the current main trace. * - * Also does not run optimization passes. + * Does not run optimization passes. * * TODO(#2404447): run simplifier? */ template - SSATmp* genFor(Trace* t, Args... args) { - auto instr = m_irFactory.gen(args...); + SSATmp* genFor(Trace* t, Args&&... args) { + auto instr = m_irFactory.gen(std::forward(args)...); t->back()->push_back(instr); return instr->dst(); } + /* + * Add an already created instruction, running it through the normal + * optimization passes and updating tracked state. + */ + SSATmp* add(IRInstruction* inst) { + return optimizeInst(inst, CloneFlag::No); + } + ////////////////////////////////////////////////////////////////////// // constants @@ -300,7 +310,8 @@ private: SSATmp* preOptimize(IRInstruction* inst); SSATmp* optimizeWork(IRInstruction* inst); - SSATmp* optimizeInst(IRInstruction* inst); + enum class CloneFlag { Yes, No }; + SSATmp* optimizeInst(IRInstruction* inst, CloneFlag doclone); private: static void appendInstruction(IRInstruction* inst, Block* block); diff --git a/hphp/runtime/vm/translator/hopt/type.cpp b/hphp/runtime/vm/translator/hopt/type.cpp index 7089ba306..b438a9154 100644 --- a/hphp/runtime/vm/translator/hopt/type.cpp +++ b/hphp/runtime/vm/translator/hopt/type.cpp @@ -44,8 +44,19 @@ Type Type::fromDynLocation(const Transl::DynLocation* dynLoc) { namespace { -Type vectorReturn(const IRInstruction* inst) { - return VectorEffects(inst).valType; +Type setElemReturn(const IRInstruction* inst) { + assert(inst->op() == SetElem || inst->op() == SetElemStk); + auto baseType = inst->src(vectorBaseIdx(inst))->type().strip(); + + // If the base is a Str, the result will always be a CountedStr (or + // an exception). If the baes might be a str, the result wil be + // CountedStr or Nullptr. Otherwise, the result is always Nullptr. + if (baseType.subtypeOf(Type::Str)) { + return Type::CountedStr; + } else if (baseType.maybe(Type::Str)) { + return Type::CountedStr | Type::Nullptr; + } + return Type::Nullptr; } Type builtinReturn(const IRInstruction* inst) { @@ -107,6 +118,9 @@ Type binArithResultType(Opcode op, Type t1, Type t2) { } Type outputType(const IRInstruction* inst, int dstId) { +#define IRT(name, ...) UNUSED static const Type name = Type::name; + IR_TYPES +#undef IRT #define D(type) return Type::type; #define DofS(n) return inst->src(n)->type(); @@ -116,9 +130,10 @@ Type outputType(const IRInstruction* inst, int dstId) { #define DMulti return Type::None; #define DStk(in) return stkReturn(inst, dstId, \ [&]() -> Type { in not_reached(); }); -#define DVector return vectorReturn(inst); +#define DSetElem return setElemReturn(inst); #define ND assert(0 && "outputType requires HasDest or NaryDest"); #define DBuiltin return builtinReturn(inst); +#define DSubtract(n, t) return inst->src(n)->type() - t; #define DArith return binArithResultType(inst->op(), \ inst->src(0)->type(), \ inst->src(1)->type()); @@ -139,9 +154,10 @@ Type outputType(const IRInstruction* inst, int dstId) { #undef DParam #undef DMulti #undef DStk -#undef DVector +#undef DSetElem #undef ND #undef DBuiltin +#undef DSubtract #undef DArith } @@ -280,9 +296,11 @@ void assertOperandTypes(const IRInstruction* inst) { #define ND #define DMulti #define DStk(...) -#define DVector +#define DSetElem #define D(...) #define DBuiltin +#define DSubtract(src, t)checkDst(src < inst->numSrcs(), \ + "invalid src num"); #define DUnbox(src) checkDst(src < inst->numSrcs(), \ "invalid src num"); #define DBox(src) checkDst(src < inst->numSrcs(), \ @@ -306,6 +324,7 @@ void assertOperandTypes(const IRInstruction* inst) { #undef O #undef NA +#undef SAny #undef S #undef C #undef CStr @@ -315,9 +334,12 @@ void assertOperandTypes(const IRInstruction* inst) { #undef ND #undef D +#undef DBuiltin +#undef DSubtract #undef DUnbox #undef DMulti #undef DStk +#undef DSetElem #undef DBox #undef DofS #undef DParam diff --git a/hphp/runtime/vm/translator/hopt/type.h b/hphp/runtime/vm/translator/hopt/type.h index a0e22c9be..6e64aee06 100644 --- a/hphp/runtime/vm/translator/hopt/type.h +++ b/hphp/runtime/vm/translator/hopt/type.h @@ -70,7 +70,8 @@ namespace JIT { IRT(TCA, 1ULL << 50) \ IRT(ActRec, 1ULL << 51) \ IRT(None, 1ULL << 52) \ - IRT(CacheHandle, 1ULL << 53) /* TargetCache::CacheHandle */ + IRT(CacheHandle, 1ULL << 53) /* TargetCache::CacheHandle */ \ + IRT(Nullptr, 1ULL << 54) // The definitions for these are in ir.cpp #define IRT_UNIONS \ @@ -82,6 +83,7 @@ namespace JIT { // would be nonsense types. #define IRT_SPECIAL \ IRT(Bottom, 0) \ + IRT(Top, 0xffffffffffffffffULL) \ IRT(Counted, kCountedStr|kCountedArr|kObj|kBoxedCell) \ IRT(PtrToCounted, kCounted << kPtrShift) \ IRT(Gen, kCell|kBoxedCell) \ diff --git a/hphp/runtime/vm/translator/hopt/vectortranslator.cpp b/hphp/runtime/vm/translator/hopt/vectortranslator.cpp index 93e51c8d3..0c5f97d3e 100644 --- a/hphp/runtime/vm/translator/hopt/vectortranslator.cpp +++ b/hphp/runtime/vm/translator/hopt/vectortranslator.cpp @@ -135,8 +135,7 @@ void VectorEffects::init(Opcode op, const Type origBase, assert_not_implemented(basePtr || baseType.subtypeOfAny(Type::Obj, Type::Arr)); - valType = origVal; - baseTypeChanged = baseValChanged = valTypeChanged = false; + baseTypeChanged = baseValChanged = false; // Canonicalize the op to SetProp/SetElem/UnsetElem/SetWithRefElem op = canonicalOp(op); @@ -147,11 +146,6 @@ void VectorEffects::init(Opcode op, const Type origBase, // since this shouldn't actually happen.) assert(key.equals(Type::None) || key.isKnownDataType() || key.equals(Type::Cell)); - if (op == SetWithRefElem) { - assert(origVal.isPtr()); - } else { - assert(origVal.equals(Type::None) || origVal.isKnownDataType()); - } if ((op == SetElem || op == SetProp || op == SetWithRefElem) && baseType.maybe(Type::Null | Type::Bool | Type::Str)) { @@ -196,36 +190,12 @@ void VectorEffects::init(Opcode op, const Type origBase, // warning, and evaulate to null. const Type badArrKey = Type::Arr | Type::Obj; - const bool maybeBadArrKey = key.maybe(badArrKey); - if ((op == SetElem && (!baseType.subtypeOf(okBase) || maybeBadArrKey)) || - (op == SetProp && !baseType.subtypeOf(Type::Obj))) { - // The set operation might fail, issue a warning, and evaluate to null. Any - // other invalid combinations of base/key will throw an exception/fatal. - - const bool definitelyFail = - // SetElem and the base is known to not be a good type - (op == SetElem && baseType.not(okBase)) || - // SetElem and the array key is known to be bad - (op == SetElem && key.subtypeOf(badArrKey)) || - // SetProp and the base is known to not be an object - (op == SetProp && baseType.not(Type::Obj)); - - valType = definitelyFail ? Type::InitNull : (valType | Type::InitNull); - } - - if (op == SetElem && baseType.maybe(Type::Str)) { - // If the base is a String, the result will be different from the value - // use valTypeChanged (even though the type may not have) - valTypeChanged = true; - } - // The final baseType should be a pointer/box iff the input was baseType = baseBoxed ? baseType.box() : baseType; baseType = basePtr ? baseType.ptr() : baseType; baseTypeChanged = baseTypeChanged || baseType != origBase; baseValChanged = baseValChanged || baseTypeChanged; - valTypeChanged = valTypeChanged || valType != origVal; } // vectorBaseIdx returns the src index for inst's base operand. @@ -292,17 +262,20 @@ HhbcTranslator::VectorTranslator::VectorTranslator( : m_ni(ni) , m_ht(ht) , m_tb(*m_ht.m_tb) + , m_irf(m_ht.m_irFactory) , m_mii(getMInstrInfo(ni.mInstrOp())) , m_needMIS(true) , m_misBase(nullptr) , m_base(nullptr) , m_result(nullptr) - , m_predictedResult(nullptr) + , m_strTestResult(nullptr) + , m_failedSetTrace(nullptr) { } template -SSATmp* HhbcTranslator::VectorTranslator::genStk(Opcode opc, Srcs... srcs) { +SSATmp* HhbcTranslator::VectorTranslator::genStk(Opcode opc, Trace* taken, + Srcs... srcs) { assert(opcodeHasFlags(opc, HasStackVersion)); assert(!opcodeHasFlags(opc, ModifiesStack)); std::vector srcVec({srcs...}); @@ -318,7 +291,7 @@ SSATmp* HhbcTranslator::VectorTranslator::genStk(Opcode opc, Srcs... srcs) { } } - return gen(opc, srcs...); + return gen(opc, taken, srcs...); } void HhbcTranslator::VectorTranslator::emit() { @@ -525,6 +498,7 @@ SSATmp* HhbcTranslator::VectorTranslator::getValAddr() { } else { assert(l.space == Location::Stack); assert(mapContains(m_stackInputs, 0)); + m_ht.spillStack(); return m_ht.ldStackAddr(m_stackInputs[0]); } } @@ -578,8 +552,8 @@ void HhbcTranslator::VectorTranslator::emitBaseLCR() { // Check for Uninit and warn/promote to InitNull as appropriate if (baseType.subtypeOf(Type::Uninit)) { if (mia & MIA_warn) { - m_ht.exceptionBarrier(); - gen(RaiseUninitLoc, LocalId(base.location.offset)); + gen(RaiseUninitLoc, m_ht.getCatchTrace(), + LocalId(base.location.offset)); } if (mia & MIA_define) { gen( @@ -620,6 +594,8 @@ void HhbcTranslator::VectorTranslator::emitBaseLCR() { m_base = m_ht.ldLocAddr(base.location.offset); } else { assert(base.location.space == Location::Stack); + // Make sure the stack is clean before getting a pointer to one of its + // elements. m_ht.spillStack(); assert(m_stackInputs.count(m_iInd)); m_base = m_ht.ldStackAddr(m_stackInputs[m_iInd]); @@ -715,8 +691,8 @@ void HhbcTranslator::VectorTranslator::emitBaseG() { static const OpFunc opFuncs[] = {baseG, baseGW, baseGD, baseGWD}; OpFunc opFunc = opFuncs[mia & MIA_base]; SSATmp* gblName = getBase(); - m_ht.exceptionBarrier(); m_base = gen(BaseG, + m_ht.getCatchTrace(), cns(reinterpret_cast(opFunc)), gblName, genMisPtr()); @@ -819,11 +795,12 @@ void HhbcTranslator::VectorTranslator::emitPropGeneric() { typedef TypedValue* (*OpFunc)(Class*, TypedValue*, TypedValue, MInstrState*); SSATmp* key = getKey(); BUILD_OPTAB(mia, m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); if (mia & Define) { - m_base = genStk(PropDX, cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); + m_base = genStk(PropDX, m_ht.getCatchTrace(), cns((TCA)opFunc), CTX(), + m_base, key, genMisPtr()); } else { - m_base = gen(PropX, cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); + m_base = gen(PropX, m_ht.getCatchTrace(), + cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); } } #undef HELPER_TABLE @@ -867,8 +844,7 @@ SSATmp* HhbcTranslator::VectorTranslator::checkInitProp( // init_null_variant. m_tb.hint(Block::Unlikely); if (doWarn && wantPropSpecializedWarnings()) { - // We did the exceptionBarrier for this up in emitPropSpecialized. - gen(RaiseUndefProp, baseAsObj, key); + gen(RaiseUndefProp, m_ht.getCatchTrace(), baseAsObj, key); } if (doDefine) { gen( @@ -896,20 +872,6 @@ void HhbcTranslator::VectorTranslator::emitPropSpecialized(const MInstrAttr mia, SSATmp* initNull = cns((const TypedValue*)&init_null_variant); - /* - * With the exception of single-dim CGetMs where hphpc knows the property - * can't be KindOfUninit, one of the conditional branches below may throw. - */ - if (doWarn && wantPropSpecializedWarnings()) { - if (m_ni.mInstrOp() == OpCGetM && m_mInd == 0) { - if (propInfo.hphpcType == KindOfInvalid) { - m_ht.exceptionBarrier(); - } - } else { - m_ht.exceptionBarrier(); - } - } - /* * Type-inference from hphpc only tells us that this is either an object of a * given class type or null. If it's not an object, it has to be a null type @@ -1056,12 +1018,12 @@ void HhbcTranslator::VectorTranslator::emitElem() { typedef TypedValue* (*OpFunc)(TypedValue*, TypedValue, MInstrState*); BUILD_OPTAB_HOT(getKeyTypeIS(key), mia); - m_ht.exceptionBarrier(); if (define || unset) { - m_base = genStk(define ? ElemDX : ElemUX, cns((TCA)opFunc), m_base, - key, genMisPtr()); + m_base = genStk(define ? ElemDX : ElemUX, m_ht.getCatchTrace(), + cns((TCA)opFunc), m_base, key, genMisPtr()); } else { - m_base = gen(ElemX, cns((TCA)opFunc), m_base, key, genMisPtr()); + m_base = gen(ElemX, m_ht.getCatchTrace(), + cns((TCA)opFunc), m_base, key, genMisPtr()); } } #undef HELPER_TABLE @@ -1190,8 +1152,8 @@ void HhbcTranslator::VectorTranslator::emitCGetProp() { typedef TypedValue (*OpFunc)(Class*, TypedValue*, TypedValue, MInstrState*); SSATmp* key = getKey(); BUILD_OPTAB_HOT(getKeyTypeS(key), m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); - m_result = gen(CGetProp, cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); + m_result = gen(CGetProp, m_ht.getCatchTrace(), + cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1232,8 +1194,7 @@ void HhbcTranslator::VectorTranslator::emitVGetProp() { SSATmp* key = getKey(); typedef RefData* (*OpFunc)(Class*, TypedValue*, TypedValue, MInstrState*); BUILD_OPTAB_HOT(getKeyTypeS(key), m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); - m_result = genStk(VGetProp, cns((TCA)opFunc), CTX(), + m_result = genStk(VGetProp, m_ht.getCatchTrace(), cns((TCA)opFunc), CTX(), m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1265,9 +1226,8 @@ void HhbcTranslator::VectorTranslator::emitIssetEmptyProp(bool isEmpty) { SSATmp* key = getKey(); typedef uint64_t (*OpFunc)(Class*, TypedValue*, TypedValue); BUILD_OPTAB(isEmpty, m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); - m_result = gen(isEmpty ? EmptyProp : IssetProp, cns((TCA)opFunc), - CTX(), m_base, key); + m_result = gen(isEmpty ? EmptyProp : IssetProp, m_ht.getCatchTrace(), + cns((TCA)opFunc), CTX(), m_base, key); } #undef HELPER_TABLE @@ -1280,10 +1240,9 @@ void HhbcTranslator::VectorTranslator::emitEmptyProp() { } template -static inline TypedValue setPropImpl(Class* ctx, TypedValue* base, - TypedValue keyVal, Cell val) { - HPHP::SetProp(ctx, base, &keyVal, &val); - return val; +static inline void setPropImpl(Class* ctx, TypedValue* base, + TypedValue keyVal, Cell val) { + HPHP::SetProp(ctx, base, &keyVal, &val); } #define HELPER_TABLE(m) \ @@ -1292,8 +1251,8 @@ static inline TypedValue setPropImpl(Class* ctx, TypedValue* base, m(setPropCO, true) #define PROP(nm, ...) \ -TypedValue nm(Class* ctx, TypedValue* base, TypedValue key, Cell val) { \ - return setPropImpl<__VA_ARGS__>(ctx, base, key, val); \ +void nm(Class* ctx, TypedValue* base, TypedValue key, Cell val) { \ + setPropImpl<__VA_ARGS__>(ctx, base, key, val); \ } namespace VectorHelpers { HELPER_TABLE(PROP) @@ -1320,12 +1279,13 @@ void HhbcTranslator::VectorTranslator::emitSetProp() { } // Emit the appropriate helper call. - typedef TypedValue (*OpFunc)(Class*, TypedValue*, TypedValue, Cell); + typedef void (*OpFunc)(Class*, TypedValue*, TypedValue, Cell); SSATmp* key = getKey(); BUILD_OPTAB(m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); - m_result = genStk(SetProp, cns((TCA)opFunc), CTX(), m_base, key, value); - m_predictedResult = value; + m_failedSetTrace = m_ht.getCatchTrace(); + genStk(SetProp, m_failedSetTrace, cns((TCA)opFunc), CTX(), + m_base, key, value); + m_result = value; } #undef HELPER_TABLE @@ -1363,9 +1323,8 @@ void HhbcTranslator::VectorTranslator::emitSetOpProp() { Cell, MInstrState*, SetOpOp); BUILD_OPTAB(m_base->isA(Type::Obj)); m_tb.gen(StRaw, m_misBase, cns(RawMemSlot::MisCtx), CTX()); - m_ht.exceptionBarrier(); m_result = - genStk(SetOpProp, cns((TCA)opFunc), + genStk(SetOpProp, m_ht.getCatchTrace(), cns((TCA)opFunc), m_base, key, value, genMisPtr(), cns(op)); } #undef HELPER_TABLE @@ -1404,9 +1363,8 @@ void HhbcTranslator::VectorTranslator::emitIncDecProp() { MInstrState*, IncDecOp); BUILD_OPTAB(m_base->isA(Type::Obj)); m_tb.gen(StRaw, m_misBase, cns(RawMemSlot::MisCtx), CTX()); - m_ht.exceptionBarrier(); m_result = - genStk(IncDecProp, cns((TCA)opFunc), + genStk(IncDecProp, m_ht.getCatchTrace(), cns((TCA)opFunc), m_base, key, genMisPtr(), cns(op)); } #undef HELPER_TABLE @@ -1442,8 +1400,8 @@ void HhbcTranslator::VectorTranslator::emitBindProp() { typedef void (*OpFunc)(Class*, TypedValue*, TypedValue, RefData*, MInstrState*); BUILD_OPTAB(m_base->isA(Type::Obj)); - m_ht.exceptionBarrier(); - genStk(BindProp, cns((TCA)opFunc), CTX(), m_base, key, box, genMisPtr()); + genStk(BindProp, m_ht.getCatchTrace(), cns((TCA)opFunc), CTX(), + m_base, key, box, genMisPtr()); m_result = box; } #undef HELPER_TABLE @@ -1569,9 +1527,8 @@ void HhbcTranslator::VectorTranslator::emitCGetElem() { typedef TypedValue (*OpFunc)(TypedValue*, TypedValue, MInstrState*); BUILD_OPTAB_HOT(getKeyTypeIS(key)); - m_ht.exceptionBarrier(); - m_result = gen(CGetElem, cns((TCA)opFunc), - m_base, key, genMisPtr()); + m_result = gen(CGetElem, m_ht.getCatchTrace(), cns((TCA)opFunc), + m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1609,8 +1566,8 @@ void HhbcTranslator::VectorTranslator::emitVGetElem() { SSATmp* key = getKey(); typedef RefData* (*OpFunc)(TypedValue*, TypedValue, MInstrState*); BUILD_OPTAB(getKeyTypeIS(key)); - m_ht.exceptionBarrier(); - m_result = genStk(VGetElem, cns((TCA)opFunc), m_base, key, genMisPtr()); + m_result = genStk(VGetElem, m_ht.getCatchTrace(), cns((TCA)opFunc), + m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1649,9 +1606,8 @@ void HhbcTranslator::VectorTranslator::emitIssetEmptyElem(bool isEmpty) { typedef uint64_t (*OpFunc)(TypedValue*, TypedValue, MInstrState*); BUILD_OPTAB_HOT(getKeyTypeIS(key), isEmpty); - m_ht.exceptionBarrier(); - m_result = gen(isEmpty ? EmptyElem : IssetElem, - cns((TCA)opFunc), m_base, key, genMisPtr()); + m_result = gen(isEmpty ? EmptyElem : IssetElem, m_ht.getCatchTrace(), + cns((TCA)opFunc), m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1698,8 +1654,8 @@ void HhbcTranslator::VectorTranslator::emitArrayIsset() { typedef uint64_t (*OpFunc)(ArrayData*, TypedValue*); BUILD_OPTAB(keyType, checkForInt); assert(m_base->isA(Type::Arr)); - m_ht.exceptionBarrier(); - m_result = gen(ArrayIsset, cns((TCA)opFunc), m_base, key); + m_result = gen(ArrayIsset, m_ht.getCatchTrace(), + cns((TCA)opFunc), m_base, key); } #undef HELPER_TABLE @@ -1775,10 +1731,9 @@ void HhbcTranslator::VectorTranslator::emitArraySet(SSATmp* key, typedef ArrayData* (*OpFunc)(ArrayData*, TypedValue*, TypedValue, RefData*); BUILD_OPTAB_HOT(keyType, checkForInt, setRef); - // Don't exceptionBarrier below because the helper can't throw. It - // may reenter to call destructors so it has a sync point in - // nativecalls.cpp, but exceptions are swallowed at destructor - // boundaries right now: #2182869. + // No catch trace below because the helper can't throw. It may reenter to + // call destructors so it has a sync point in nativecalls.cpp, but exceptions + // are swallowed at destructor boundaries right now: #2182869. if (setRef) { assert(base.location.space == Location::Local || base.location.space == Location::Stack); @@ -1813,16 +1768,20 @@ namespace VectorHelpers { void setWithRefElemC(TypedValue* base, TypedValue keyVal, TypedValue* val, MInstrState* mis) { base = HPHP::ElemD(mis->tvScratch, mis->tvRef, base, &keyVal); - if (!(base == &mis->tvScratch && base->m_type == KindOfUninit)) { + if (base != &mis->tvScratch) { tvDup(val, base); + } else { + assert(base->m_type == KindOfUninit); } } void setWithRefNewElem(TypedValue* base, TypedValue* val, MInstrState* mis) { base = NewElem(mis->tvScratch, mis->tvRef, base); - if (!(base == &mis->tvScratch && base->m_type == KindOfUninit)) { + if (base != &mis->tvScratch) { tvDup(val, base); + } else { + assert(base->m_type == KindOfUninit); } } } @@ -1830,13 +1789,13 @@ void setWithRefNewElem(TypedValue* base, TypedValue* val, void HhbcTranslator::VectorTranslator::emitSetWithRefLElem() { SSATmp* key = getKey(); SSATmp* locAddr = getValAddr(); - m_ht.exceptionBarrier(); if (m_base->type().strip().subtypeOf(Type::Arr) && !locAddr->type().deref().maybeBoxed()) { emitSetElem(); - m_predictedResult = nullptr; + assert(m_strTestResult == nullptr); } else { - genStk(SetWithRefElem, cns((TCA)VectorHelpers::setWithRefElemC), + genStk(SetWithRefElem, m_ht.getCatchTrace(), + cns((TCA)VectorHelpers::setWithRefElemC), m_base, key, locAddr, genMisPtr()); } m_result = nullptr; @@ -1855,25 +1814,22 @@ void HhbcTranslator::VectorTranslator::emitSetWithRefRProp() { } void HhbcTranslator::VectorTranslator::emitSetWithRefNewElem() { - SSATmp* locAddr = getValAddr(); - m_ht.exceptionBarrier(); if (m_base->type().strip().subtypeOf(Type::Arr) && - !locAddr->type().deref().maybeBoxed()) { + getValue()->type().notBoxed()) { emitSetNewElem(); - m_predictedResult = nullptr; } else { - genStk(SetWithRefNewElem, cns((TCA)VectorHelpers::setWithRefNewElem), - m_base, locAddr, genMisPtr()); + genStk(SetWithRefNewElem, m_ht.getCatchTrace(), + cns((TCA)VectorHelpers::setWithRefNewElem), + m_base, getValAddr(), genMisPtr()); } m_result = nullptr; } template -static inline TypedValue setElemImpl(TypedValue* base, TypedValue keyVal, - Cell val) { +static inline StringData* setElemImpl(TypedValue* base, TypedValue keyVal, + Cell val) { TypedValue* key = keyPtr(keyVal); - HPHP::SetElem(base, key, &val); - return val; + return HPHP::SetElem(base, key, &val); } #define HELPER_TABLE(m) \ @@ -1884,7 +1840,7 @@ static inline TypedValue setElemImpl(TypedValue* base, TypedValue keyVal, #define ELEM(nm, hot, ...) \ hot \ -TypedValue nm(TypedValue* base, TypedValue key, Cell val) { \ +StringData* nm(TypedValue* base, TypedValue key, Cell val) { \ return setElemImpl<__VA_ARGS__>(base, key, val); \ } namespace VectorHelpers { @@ -1902,11 +1858,27 @@ void HhbcTranslator::VectorTranslator::emitSetElem() { } // Emit the appropriate helper call. - typedef TypedValue (*OpFunc)(TypedValue*, TypedValue, Cell); + typedef StringData* (*OpFunc)(TypedValue*, TypedValue, Cell); BUILD_OPTAB_HOT(getKeyTypeIS(key)); - m_ht.exceptionBarrier(); - m_result = genStk(SetElem, cns((TCA)opFunc), m_base, key, value); - m_predictedResult = value; + m_failedSetTrace = m_ht.getCatchTrace(); + SSATmp* result = genStk(SetElem, m_failedSetTrace, cns((TCA)opFunc), + m_base, key, value); + auto t = result->type(); + if (t.equals(Type::Nullptr)) { + // Base is not a string. Result is always value. + m_result = value; + } else if (t.equals(Type::CountedStr)) { + // Base is a string. Stack result is a new string so we're responsible for + // decreffing value. + m_result = result; + gen(DecRef, value); + } else { + assert(t.equals(Type::CountedStr | Type::Nullptr)); + // Base might be a string. Assume the result is value, then inform + // emitMPost that it needs to test the actual result. + m_result = value; + m_strTestResult = result; + } } #undef HELPER_TABLE @@ -1945,9 +1917,9 @@ void HhbcTranslator::VectorTranslator::emitSetOpElem() { # define SETOP_OP(op, bcOp) HELPER_TABLE(FILL_ROW, op) BUILD_OPTAB_ARG(SETOP_OPS, op); # undef SETOP_OP - m_ht.exceptionBarrier(); m_result = - genStk(SetOpElem, cns((TCA)opFunc), m_base, key, getValue(), genMisPtr()); + genStk(SetOpElem, m_ht.getCatchTrace(), cns((TCA)opFunc), + m_base, key, getValue(), genMisPtr()); } #undef HELPER_TABLE @@ -1980,8 +1952,8 @@ void HhbcTranslator::VectorTranslator::emitIncDecElem() { # define INCDEC_OP(op) HELPER_TABLE(FILL_ROW, op) BUILD_OPTAB_ARG(INCDEC_OPS, op); # undef INCDEC_OP - m_ht.exceptionBarrier(); - m_result = genStk(IncDecElem, cns((TCA)opFunc), m_base, key, genMisPtr()); + m_result = genStk(IncDecElem, m_ht.getCatchTrace(), cns((TCA)opFunc), + m_base, key, genMisPtr()); } #undef HELPER_TABLE @@ -1998,8 +1970,7 @@ void bindElemC(TypedValue* base, TypedValue keyVal, RefData* val, void HhbcTranslator::VectorTranslator::emitBindElem() { SSATmp* key = getKey(); SSATmp* box = getValue(); - m_ht.exceptionBarrier(); - genStk(BindElem, cns((TCA)VectorHelpers::bindElemC), + genStk(BindElem, m_ht.getCatchTrace(), cns((TCA)VectorHelpers::bindElemC), m_base, key, box, genMisPtr()); m_result = box; } @@ -2043,8 +2014,7 @@ void HhbcTranslator::VectorTranslator::emitUnsetElem() { typedef void (*OpFunc)(TypedValue*, TypedValue); BUILD_OPTAB_HOT(getKeyTypeIS(key)); - m_ht.exceptionBarrier(); - genStk(UnsetElem, cns((TCA)opFunc), m_base, key); + genStk(UnsetElem, m_ht.getCatchTrace(), cns((TCA)opFunc), m_base, key); } #undef HELPER_TABLE @@ -2058,9 +2028,9 @@ void HhbcTranslator::VectorTranslator::emitVGetNewElem() { void HhbcTranslator::VectorTranslator::emitSetNewElem() { SSATmp* value = getValue(); - m_ht.exceptionBarrier(); - m_result = gen(SetNewElem, m_base, value); - m_predictedResult = value; + m_failedSetTrace = m_ht.getCatchTrace(); + gen(SetNewElem, m_failedSetTrace, m_base, value); + m_result = value; } void HhbcTranslator::VectorTranslator::emitSetOpNewElem() { @@ -2073,17 +2043,23 @@ void HhbcTranslator::VectorTranslator::emitIncDecNewElem() { void HhbcTranslator::VectorTranslator::emitBindNewElem() { SSATmp* box = getValue(); - m_ht.exceptionBarrier(); - genStk(BindNewElem, m_base, box, genMisPtr()); + genStk(BindNewElem, m_ht.getCatchTrace(), m_base, box, genMisPtr()); m_result = box; } void HhbcTranslator::VectorTranslator::emitMPost() { + SSATmp* catchSp = nullptr; + if (m_failedSetTrace) { + catchSp = m_failedSetTrace->back()->back()->src(0); + assert(catchSp->isA(Type::StkPtr)); + } + // Decref stack inputs. If we're translating a SetM or BindM, then input 0 is - // both our input and output so leave its refcount alone. The helpers for - // SetM can overwrite this value with InitNull if the operation fails, but - // they also decref the old value so it's still safe to leave its refcount - // alone. + // both our input and output so leave its refcount alone. If m_failedSetTrace + // is non-null, the final helper call may throw an InvalidSetMException. We + // need to add instructions to m_failedSetTrace to finish the vector + // instruction in case this happens, so any DecRefs emitted here are also + // added to m_failedSetTrace. unsigned nStack = (m_ni.mInstrOp() == OpSetM || m_ni.mInstrOp() == OpBindM) ? 1 : 0; for (unsigned i = nStack; i < m_ni.inputs.size(); ++i) { @@ -2094,6 +2070,10 @@ void HhbcTranslator::VectorTranslator::emitMPost() { auto input = getInput(i); if (input->isA(Type::Gen)) { gen(DecRef, input); + if (m_failedSetTrace) { + m_ht.genFor(m_failedSetTrace, DecRefStack, + StackOffset(m_stackInputs[i]), Type::Cell, catchSp); + } } break; } @@ -2116,7 +2096,7 @@ void HhbcTranslator::VectorTranslator::emitMPost() { // instead of the real result; its validity will be guarded later in this // function. if (m_result) { - m_ht.push(usePredictedResult() ? m_predictedResult : m_result); + m_ht.push(m_result); } else { assert(m_ni.mInstrOp() == OpUnsetM || m_ni.mInstrOp() == OpSetWithRefLM || @@ -2124,45 +2104,76 @@ void HhbcTranslator::VectorTranslator::emitMPost() { } // Clean up tvRef(2) - if (nLogicalRatchets() > 1) { - gen(DecRefMem, Type::Gen, m_misBase, cns(HHIR_MISOFF(tvRef2))); - } - if (nLogicalRatchets() > 0) { - gen(DecRefMem, Type::Gen, m_misBase, cns(HHIR_MISOFF(tvRef))); + static const size_t refOffs[] = { HHIR_MISOFF(tvRef), HHIR_MISOFF(tvRef2) }; + for (unsigned i = 0; i < std::min(nLogicalRatchets(), 2U); ++i) { + IRInstruction* inst = m_irf.gen(DecRefMem, Type::Gen, m_misBase, + cns(refOffs[i])); + m_tb.add(inst); + if (m_failedSetTrace) { + m_failedSetTrace->back()->push_back(inst->clone(&m_irf)); + } } - // Now that everything else is done, guard the predicted type of the result, - // if any. - if (usePredictedResult()) { - std::vector spillValues = m_ht.getSpillValues(); - assert(spillValues.back() == m_predictedResult); - spillValues.back() = m_result; - - gen(CheckType, - m_predictedResult->type(), - m_ht.getExitTrace(m_ht.nextSrcKey().offset(), spillValues), - m_result); - } + emitSideExits(catchSp, nStack); } -bool HhbcTranslator::VectorTranslator::usePredictedResult() { - // First check that we have a predicted result. - if (!m_predictedResult) return false; +void HhbcTranslator::VectorTranslator::emitSideExits(SSATmp* catchSp, + int nStack) { + const Offset nextOff = m_ht.nextSrcKey().offset(); + auto op = m_ni.mInstrOp(); + const bool isSetWithRef = op == OpSetWithRefLM || op == OpSetWithRefRM; - // Next, check that the predicted result is actually possible. This check - // will trigger if we can prove at compile time that the set operation will - // fail. - if (!m_predictedResult->isA(m_result->type())) return false; + if (m_failedSetTrace) { + assert(bool(m_result) ^ isSetWithRef); + // This catch trace currently ends with an EndCatch that will fall through + // if an InvalidSetMException was thrown. We need to emit code to clean up + // if that happens. If we're translating a SetWithRef* bytecode we don't + // have to do anything special to the stack since they have no stack + // output. Otherwise we need to pop our input value and push the value from + // the exception to the stack (done with a DecRefStack followed by a + // SpillStack). - // We can use the predicted result as long as we didn't do an operation on a - // possibly string base with a possibly string value. This is the only case - // where the result of a SetM might have the same type as its input without - // being the same value. - IRInstruction* inst = m_result->inst(); - SSATmp* base = inst->src(vectorBaseIdx(inst)); - SSATmp* value = inst->src(vectorValIdx(inst)); - return base->type().strip().not(Type::Str) || - value->type().strip().not(Type::Str); + std::vector args{ + catchSp, // sp from the previous SpillStack + cns(nStack), // cells popped since the last SpillStack + }; + + if (!isSetWithRef) { + m_ht.genFor(m_failedSetTrace, DecRefStack, StackOffset(0), + Type::Cell, catchSp); + args.push_back(m_ht.genFor(m_failedSetTrace, LdUnwinderValue, + Type::Cell)); + } + + SSATmp* sp = m_ht.genFor(m_failedSetTrace, SpillStack, + std::make_pair(args.size(), &args[0])); + m_ht.genFor(m_failedSetTrace, DeleteUnwinderException); + m_ht.genFor(m_failedSetTrace, SyncABIRegs, m_tb.getFp(), sp); + m_ht.genFor(m_failedSetTrace, ReqBindJmp, BCOffset(nextOff)); + } + + if (m_strTestResult) { + assert(!isSetWithRef); + // We expected SetElem's base to not be a Str but might be wrong. Make an + // exit trace to side exit to the next instruction, replacing our guess + // with the correct stack output. + + auto toSpill = m_ht.getSpillValues(); + assert(toSpill.size()); + assert(toSpill[0] == m_result); + SSATmp* str = m_irf.gen(AssertNonNull, m_strTestResult)->dst(); + toSpill[0] = str; + Trace* exit = m_ht.getExitTrace(nextOff, toSpill); + exit->front()->prepend(str->inst()); + exit->front()->prepend(m_irf.gen(DecRef, m_result)); + exit->front()->prepend( + m_irf.gen(IncStat, cns(Stats::TC_SetMStrGuess_Miss), + cns(1), cns(false))); + exit->front()->prepend(m_ht.makeMarker(m_ht.bcOff())); + + gen(CheckType, Type::Nullptr, exit, m_strTestResult); + gen(IncStat, cns(Stats::TC_SetMStrGuess_Hit), cns(1), cns(false)); + } } bool HhbcTranslator::VectorTranslator::needFirstRatchet() const { diff --git a/hphp/runtime/vm/translator/physreg.cpp b/hphp/runtime/vm/translator/physreg.cpp index 7637a640d..4fead77bc 100644 --- a/hphp/runtime/vm/translator/physreg.cpp +++ b/hphp/runtime/vm/translator/physreg.cpp @@ -38,12 +38,20 @@ PhysRegSaverParity::~PhysRegSaverParity() { // See above; stack parity. m_as. addq (m_adjust, reg::rsp); } - m_regs.forEachR([&] (PhysReg pr) { - m_as. pop (pr); + emitPops(m_as, m_regs); +} + +void PhysRegSaverParity::emitPops(X64Assembler& as, RegSet regs) { + regs.forEachR([&] (PhysReg pr) { + as. pop (pr); }); } int PhysRegSaverParity::rspAdjustment() const { + return m_adjust; +} + +int PhysRegSaverParity::rspTotalAdjustmentRegs() const { return m_regs.size() + m_adjust / sizeof(int64_t); } diff --git a/hphp/runtime/vm/translator/physreg.h b/hphp/runtime/vm/translator/physreg.h index 2c5f58965..870ebb47b 100644 --- a/hphp/runtime/vm/translator/physreg.h +++ b/hphp/runtime/vm/translator/physreg.h @@ -259,12 +259,15 @@ struct PhysRegSaverParity { PhysRegSaverParity(int parity, X64Assembler& as, RegSet regs); ~PhysRegSaverParity(); + static void emitPops(X64Assembler& as, RegSet regs); + PhysRegSaverParity(const PhysRegSaverParity&) = delete; PhysRegSaverParity(PhysRegSaverParity&&) = default; PhysRegSaverParity& operator=(const PhysRegSaverParity&) = delete; PhysRegSaverParity& operator=(PhysRegSaverParity&&) = default; int rspAdjustment() const; + int rspTotalAdjustmentRegs() const; void bytesPushed(int64_t bytes); private: diff --git a/hphp/runtime/vm/translator/targetcache.cpp b/hphp/runtime/vm/translator/targetcache.cpp index 7c701241a..fee28cea3 100644 --- a/hphp/runtime/vm/translator/targetcache.cpp +++ b/hphp/runtime/vm/translator/targetcache.cpp @@ -96,15 +96,16 @@ undefinedError(const char* msg, const char* name) { __thread void* tl_targetCaches = nullptr; __thread HphpArray* s_constants = nullptr; -static_assert(kConditionFlagsOff + sizeof(ssize_t) <= 64, - "kConditionFlagsOff too large"); -size_t s_frontier = kConditionFlagsOff + 64; +static const size_t kPreAllocatedBytes = 64; +size_t s_frontier = kPreAllocatedBytes; + +static_assert(sizeof(TargetCacheHeader) <= kPreAllocatedBytes, + "TargetCacheHeader doesn't fit in kPreAllocatedBytes"); size_t s_persistent_frontier = 0; size_t s_persistent_start = 0; static size_t s_next_bit; static size_t s_bits_to_go; static int s_tc_fd; -static const size_t kPreAllocatedBytes = kConditionFlagsOff + 64; // Mapping from names to targetcache locations. Protected by the translator // write lease. diff --git a/hphp/runtime/vm/translator/targetcache.h b/hphp/runtime/vm/translator/targetcache.h index 446693e93..00c72fece 100644 --- a/hphp/runtime/vm/translator/targetcache.h +++ b/hphp/runtime/vm/translator/targetcache.h @@ -19,6 +19,7 @@ #include "hphp/runtime/vm/func.h" #include "hphp/util/util.h" #include "hphp/runtime/vm/translator/types.h" +#include "hphp/runtime/vm/translator/unwind-x64.h" #include "hphp/util/asm-x64.h" #include @@ -47,7 +48,31 @@ extern size_t s_persistent_start; */ extern __thread HphpArray* s_constants; -static const int kConditionFlagsOff = 0; +/* + * The fields in TargetCacheHeader are pre-allocated at process + * startup and live at the beginning of the targetCache. + */ +struct TargetCacheHeader { + ssize_t conditionFlags; + + // Used to pass values between unwinder code and catch traces: + int64_t unwinderScratch; + TypedValue unwinderTv; + bool doSideExit; +}; + +inline TargetCacheHeader* header() { + return (TargetCacheHeader*)tl_targetCaches; +} + +constexpr int kConditionFlagsOff = + offsetof(TargetCacheHeader, conditionFlags); +constexpr int kUnwinderScratchOff = + offsetof(TargetCacheHeader, unwinderScratch); +constexpr int kUnwinderSideExitOff = + offsetof(TargetCacheHeader, doSideExit); +constexpr int kUnwinderTvOff = + offsetof(TargetCacheHeader, unwinderTv); /* * Some caches have different numbers of lines. This is our default. @@ -123,7 +148,7 @@ T& handleToRef(CacheHandle h) { } inline ssize_t* conditionFlagsPtr() { - return ((ssize_t*)handleToPtr(kConditionFlagsOff)); + return &header()->conditionFlags; } inline ssize_t loadConditionFlags() { diff --git a/hphp/runtime/vm/translator/translator-runtime.cpp b/hphp/runtime/vm/translator/translator-runtime.cpp index 092958ae8..d35fca2a4 100644 --- a/hphp/runtime/vm/translator/translator-runtime.cpp +++ b/hphp/runtime/vm/translator/translator-runtime.cpp @@ -36,9 +36,8 @@ ArrayData* addElemStringKeyHelper(ArrayData* ad, return array_setm_s0k1_v0(0, ad, key, &value); } -HOT_FUNC_VM TypedValue setNewElem(TypedValue* base, Cell val) { - SetNewElem(base, &val); - return val; +HOT_FUNC_VM void setNewElem(TypedValue* base, Cell val) { + SetNewElem(base, &val); } void bindNewElemIR(TypedValue* base, RefData* val, MInstrState* mis) { diff --git a/hphp/runtime/vm/translator/translator-runtime.h b/hphp/runtime/vm/translator/translator-runtime.h index e97098472..24d30b064 100644 --- a/hphp/runtime/vm/translator/translator-runtime.h +++ b/hphp/runtime/vm/translator/translator-runtime.h @@ -54,7 +54,7 @@ static_assert(sizeof(MInstrState) - sizeof(uintptr_t) // return address ArrayData* addElemIntKeyHelper(ArrayData* ad, int64_t key, TypedValue val); ArrayData* addElemStringKeyHelper(ArrayData* ad, StringData* key, TypedValue val); -TypedValue setNewElem(TypedValue* base, Cell val); +void setNewElem(TypedValue* base, Cell val); void bindNewElemIR(TypedValue* base, RefData* val, MInstrState* mis); RefData* box_value(TypedValue tv); ArrayData* convCellToArrHelper(TypedValue tv); diff --git a/hphp/runtime/vm/translator/translator-x64-internal.h b/hphp/runtime/vm/translator/translator-x64-internal.h index 51aa64fef..c773b2968 100644 --- a/hphp/runtime/vm/translator/translator-x64-internal.h +++ b/hphp/runtime/vm/translator/translator-x64-internal.h @@ -988,17 +988,17 @@ private: static inline void voidFunc() {} #define ID(argDbg) IE(debug, (argDbg), voidFunc()) -#define EMIT_CALL_PROLOGUE(a) do { \ +#define EMIT_CALL_PROLOGUE(a) do { \ SpaceRecorder sr("_HCallInclusive", a); \ - ArgManager _am(*this, a); \ + ArgManager _am(*this, a); \ prepareCallSaveRegs(); -#define EMIT_CALL_EPILOGUE(a, dest) \ - _am.emitArguments(); \ - { \ - SpaceRecorder sr("_HCallExclusive", a); \ - emitCall(a, (TCA)(dest), true); \ - } \ +#define EMIT_CALL_EPILOGUE(a, dest) \ + _am.emitArguments(); \ + { \ + SpaceRecorder sr("_HCallExclusive", a); \ + emitCall(a, (TCA)(dest), true); \ + } \ } while(0) #define EMIT_CALL(a, dest, ...) \ diff --git a/hphp/runtime/vm/translator/translator-x64.cpp b/hphp/runtime/vm/translator/translator-x64.cpp index 812e582c8..c418e2408 100644 --- a/hphp/runtime/vm/translator/translator-x64.cpp +++ b/hphp/runtime/vm/translator/translator-x64.cpp @@ -831,29 +831,6 @@ TranslatorX64::recordCallImpl(X64Assembler& a, recordSyncPoint(a, pcOff, stackOff); SKTRACE(2, sk, "record%sCall stackOff %d\n", reentrant ? "Reentrant" : "", int(stackOff)); - - /* - * Right now we assume call sites that need to record sync points - * may also throw exceptions. We record information about dirty - * callee-saved registers so we can spill their contents during - * unwinding. See unwind-x64.cpp. - */ - if (!m_pendingUnwindRegInfo.empty()) { - if (Trace::moduleLevel(Trace::tunwind) >= 2) { - sk.trace("recordCallImpl has dirty callee-saved regs\n"); - TRACE_MOD(Trace::tunwind, 2, - "CTCA: %p saving dirty callee regs:\n", - a.code.frontier); - for (int i = 0; i < UnwindRegInfo::kMaxCalleeSaved; ++i) { - if (m_pendingUnwindRegInfo.m_regs[i].dirty) { - TRACE_MOD(Trace::tunwind, 2, " %s\n", - m_pendingUnwindRegInfo.m_regs[i].pretty().c_str()); - } - } - } - m_unwindRegMap.insert(a.code.frontier, m_pendingUnwindRegInfo); - m_pendingUnwindRegInfo.clear(); - } } void @@ -864,24 +841,6 @@ TranslatorX64::recordEagerCall(X64Assembler& a, -i.stackOff * sizeof(TypedValue)); } -void TranslatorX64::prepareCallSaveRegs() { - emitCallSaveRegs(); // Clean caller-saved regs. - m_pendingUnwindRegInfo.clear(); - - RegSet rset = kGPCalleeSaved; - PhysReg reg; - while (rset.findFirst(reg)) { - rset.remove(reg); - if (!m_regMap.regIsDirty(reg)) continue; - const RegInfo* ri = m_regMap.getInfo(reg); - assert(ri->m_cont.m_kind == RegContent::Loc); - - // If the register is dirty, we'll record this so that we can - // restore it during stack unwinding if an exception is thrown. - m_pendingUnwindRegInfo.add(reg, ri->m_type, ri->m_cont.m_loc); - } -} - void TranslatorX64::emitIncRef(PhysReg base, DataType dtype) { emitIncRef(a, base, dtype); @@ -1235,7 +1194,7 @@ asm_label(a, release); { PhysRegSaver prs(a, kGPCallerSaved - RegSet(rdi)); callDestructor(a, rAsm, rax); - recordIndirectFixup(a.code.frontier, prs.rspAdjustment()); + recordIndirectFixup(a.code.frontier, prs.rspTotalAdjustmentRegs()); } a. ret (); @@ -11141,7 +11100,7 @@ TranslatorX64::TranslatorX64() m_irAstubsUsage(0), m_numHHIRTrans(0), m_regMap(kGPCallerSaved, kGPCalleeSaved, this), - m_unwindRegMap(128), + m_catchTraceMap(128), m_curTrace(0), m_curNI(0), m_curFile(nullptr), @@ -11481,6 +11440,16 @@ TranslatorX64::callBinaryStub(X64Assembler& a, const NormalizedInstruction& i, a. pop (rdi); } +void TranslatorX64::registerCatchTrace(CTCA ip, TCA trace) { + FTRACE(1, "registerCatchTrace: afterCall: {} trace: {}\n", ip, trace); + m_catchTraceMap.insert(ip, trace); +} + +TCA TranslatorX64::getCatchTrace(CTCA ip) const { + TCA* found = m_catchTraceMap.find(ip); + return found ? *found : nullptr; +} + namespace { struct DeferredFileInvalidate : public DeferredWorkItem { diff --git a/hphp/runtime/vm/translator/translator-x64.h b/hphp/runtime/vm/translator/translator-x64.h index e687d0253..c0f8c25a2 100644 --- a/hphp/runtime/vm/translator/translator-x64.h +++ b/hphp/runtime/vm/translator/translator-x64.h @@ -240,8 +240,8 @@ class TranslatorX64 : public Translator RegAlloc m_regMap; std::stack m_savedRegMaps; FixupMap m_fixupMap; - UnwindRegMap m_unwindRegMap; UnwindInfoHandle m_unwindRegistrar; + CatchTraceMap m_catchTraceMap; public: // Currently translating trace or instruction---only valid during @@ -262,7 +262,6 @@ private: m_tca(tca), m_fixup(fixup) { } }; vector m_pendingFixups; - UnwindRegInfo m_pendingUnwindRegInfo; void drawCFG(std::ofstream& out) const; static vector x64TranslRegs(); @@ -311,7 +310,7 @@ private: bool clearThis = true, uintptr_t varEnvInvName = 0); void emitCallSaveRegs(); - void prepareCallSaveRegs(); + void prepareCallSaveRegs() { not_reached(); } void emitCallStaticLocHelper(X64Assembler& as, const NormalizedInstruction& i, ScratchReg& output, @@ -641,9 +640,8 @@ PSEUDOINSTRS } } - const UnwindRegInfo* getUnwindInfo(CTCA ip) const { - return m_unwindRegMap.find(ip); - } + void registerCatchTrace(CTCA ip, TCA trace); + TCA getCatchTrace(CTCA ip) const; static void SEGVHandler(int signum, siginfo_t *info, void *ctx); diff --git a/hphp/runtime/vm/translator/translator.cpp b/hphp/runtime/vm/translator/translator.cpp index d780b4efa..2c6fa483f 100644 --- a/hphp/runtime/vm/translator/translator.cpp +++ b/hphp/runtime/vm/translator/translator.cpp @@ -41,6 +41,7 @@ #include "hphp/runtime/vm/translator/translator-inline.h" #include "hphp/runtime/vm/translator/translator-x64.h" #include "hphp/runtime/vm/translator/annotation.h" +#include "hphp/runtime/vm/translator/hopt/type.h" #include "hphp/runtime/vm/type_profile.h" #include "hphp/runtime/vm/runtime.h" @@ -48,6 +49,7 @@ namespace HPHP { namespace Transl { using namespace HPHP; +using HPHP::JIT::Type; TRACE_SET_MOD(trans) @@ -944,43 +946,50 @@ getDynLocType(const vector& inputs, op == OpDup ); - if (op == OpSetM || op == OpBindM) { + if (op == OpSetM) { /* - * these return null for "invalid" inputs - * if we cant prove everything's ok, we will - * have to insert a side exit + * SetM returns null for "invalid" inputs, or a string if the + * base was a string. VectorTranslator ensures that invalid + * inputs or a string output when we weren't expecting it will + * cause a side exit, so we can keep this fairly simple. */ - bool ok = inputs.size() <= 3; - if (ok) { - switch (inputs[1]->rtt.valueType()) { - case KindOfObject: - ok = mcodeMaybePropName(ni->immVecM[0]); - break; - case KindOfArray: - ok = mcodeMaybeArrayKey(ni->immVecM[0]); - break; - case KindOfNull: - case KindOfUninit: - break; - default: - ok = false; - } + + if (ni->immVecM.size() > 1) { + // We don't know the type of the base for the final + // operation so we can't assume anything about the output + // type. + return RuntimeType(KindOfAny); } - if (ok) { - for (int i = inputs.size(); --i >= 2; ) { - switch (inputs[i]->rtt.valueType()) { - case KindOfObject: - case KindOfArray: - ok = false; - break; - default: - continue; - } + + // For single-element vectors, we can determine the output + // type from the base. + Type baseType; + switch (ni->immVec.locationCode()) { + case LGL: case LGC: + case LNL: case LNC: + case LSL: case LSC: + baseType = Type::Gen; break; - } + + default: + baseType = Type::fromRuntimeType(inputs[1]->rtt); } - if (!ok) { - ni->outputPredicted = true; + + const bool setElem = mcodeMaybeArrayKey(ni->immVecM[0]); + const bool setNewElem = ni->immVecM[0] == MW; + const Type keyType = + setNewElem ? Type::None + : Type::fromRuntimeType(inputs[2]->rtt); + const Type valType = Type::fromRuntimeType(inputs[0]->rtt); + if (setElem && baseType.maybe(Type::Str)) { + if (baseType.isString()) { + // The base is a string so our output is a string. + return RuntimeType(KindOfString); + } else if (!valType.isString()) { + // The base might be a string and our value isn't known to + // be a string. The output type could be Str or valType. + return RuntimeType(KindOfAny); + } } } diff --git a/hphp/runtime/vm/translator/unwind-x64.cpp b/hphp/runtime/vm/translator/unwind-x64.cpp index a5985f8b0..9f422ea71 100644 --- a/hphp/runtime/vm/translator/unwind-x64.cpp +++ b/hphp/runtime/vm/translator/unwind-x64.cpp @@ -15,17 +15,20 @@ */ #include "hphp/runtime/vm/translator/unwind-x64.h" + #include -#include #include #include +#include #include -#include "hphp/runtime/vm/translator/translator-x64.h" -#include "hphp/runtime/vm/translator/runtime-type.h" #include "hphp/runtime/vm/translator/abi-x64.h" +#include "hphp/runtime/vm/translator/runtime-type.h" +#include "hphp/runtime/vm/translator/targetcache.h" +#include "hphp/runtime/vm/translator/translator-x64.h" #include "hphp/runtime/base/stats.h" #include "hphp/runtime/vm/runtime.h" +#include "hphp/runtime/vm/member_operations.h" // libgcc exports this for registering eh information for // dynamically-loaded objects. The pointer is to data in the format @@ -33,14 +36,10 @@ extern "C" void __register_frame(void*); extern "C" void __deregister_frame(void*); +TRACE_SET_MOD(tunwind); + namespace HPHP { namespace Transl { -////////////////////////////////////////////////////////////////////// - -const Trace::Module TRACEMOD = Trace::tunwind; - -////////////////////////////////////////////////////////////////////// - namespace { template @@ -56,8 +55,6 @@ void append_vec(std::vector& v, void sync_regstate(_Unwind_Context* context) { assert(tl_regState == REGSTATE_DIRTY); - TRACE(1, "unwind: doing fixup sync\n"); - uintptr_t frameRbp = _Unwind_GetGR(context, Debug::RBP); uintptr_t frameRip = _Unwind_GetGR(context, Debug::RIP); @@ -79,44 +76,31 @@ void sync_regstate(_Unwind_Context* context) { tl_regState = REGSTATE_CLEAN; } -void sync_callee_saved(_Unwind_Context* context) { - const CTCA frameIP = CTCA(_Unwind_GetGR(context, Debug::RIP)); - const UnwindRegInfo* const uInfo = tx64->getUnwindInfo(frameIP); - if (!uInfo) return; +bool install_catch_trace(_Unwind_Context* ctx, _Unwind_Exception* exn, + InvalidSetMException* ism) { + const CTCA rip = (CTCA)_Unwind_GetIP(ctx); + TCA catchTrace = tx64->getCatchTrace(rip); + assert(IMPLIES(ism, catchTrace)); + if (!catchTrace) return false; - ActRec* const frameAr = reinterpret_cast( - _Unwind_GetGR(context, Debug::RBP)); + FTRACE(1, "installing catch trace {} for call {} with ism {}, " + "returning _URC_INSTALL_CONTEXT\n", + catchTrace, rip, ism); - TRACE(1, "unwind: cleaning callee dirty regs\n"); - for (int i = 0; i < UnwindRegInfo::kMaxCalleeSaved; ++i) { - UnwindRegInfo::Data cri = uInfo->m_regs[i]; - if (!cri.dirty) break; - - TRACE(1, "unwind: unwind reg %s (val: %p)\n", cri.pretty().c_str(), - (void*)_Unwind_GetGR(context, cri.reg)); - - uintptr_t contents = _Unwind_GetGR(context, cri.reg); - TypedValue* cell; - if (cri.exStack) { - /* - * We don't keep information about how to restore the value of - * rVmSp (rbx) generally in tc frames, however the only time - * we'll need to sync dirty regs here is if we are the first tc - * frame below a C++ frame. In this case, rbx is properly - * restored for us by the unwinder, so we can use it here. - */ - static_assert(rVmSp == reg::rbx, - "unwind-x64.cpp expected rVmSp == rbx"); - TypedValue* frameSp = reinterpret_cast( - _Unwind_GetGR(context, Debug::RBX)); - cell = frameSp - (cri.locOffset + 1); - } else { - cell = frame_local(frameAr, cri.locOffset); - } - - cell->m_type = DataType(cri.dataType); - cell->m_data.num = contents; + // In theory the unwind api will let us set registers in the frame before + // executing our landing pad. In practice, trying to use their recommended + // scratch registers results in a SEGV inside _Unwind_SetGR, so we pass + // things to the handler using the target cache. This also simplifies the + // handler code because it doesn't have to worry about saving its arguments + // somewhere while executing the exit trace. + TargetCache::header()->unwinderScratch = (int64_t)exn; + TargetCache::header()->doSideExit = ism; + if (ism) { + TargetCache::header()->unwinderTv = ism->tv(); } + _Unwind_SetIP(ctx, (uint64_t)catchTrace); + + return true; } _Unwind_Reason_Code @@ -125,10 +109,34 @@ tc_unwind_personality(int version, uint64_t exceptionClass, _Unwind_Exception* exceptionObj, _Unwind_Context* context) { + using namespace abi; + // Exceptions thrown by g++-generated code will have the class "GNUCC++" + // packed into a 64-bit int. For now we shouldn't be seeing exceptions from + // any other runtimes but this may change in the future. + DEBUG_ONLY constexpr uint64_t kMagicClass = 0x474e5543432b2b00; + assert(exceptionClass == kMagicClass); assert(version == 1); - FTRACE(2, "unwind: tc_unwind_personality {}\n", - int(tl_regState)); + auto const& ti = typeInfoFromUnwindException(exceptionObj); + InvalidSetMException* ism = nullptr; + if (ti == typeid(InvalidSetMException)) { + ism = static_cast( + exceptionFromUnwindException(exceptionObj)); + } + + + if (Trace::moduleEnabled(TRACEMOD, 1)) { + DEBUG_ONLY auto const* unwindType = + (actions & _UA_SEARCH_PHASE) ? "search" : "cleanup"; + int status; + auto* exnType = __cxa_demangle(ti.name(), nullptr, nullptr, &status); + SCOPE_EXIT { free(exnType); }; + assert(status == 0); + FTRACE(1, "unwind {} exn {}: regState: {} ip: {} type: {}. ", + unwindType, exceptionObj, + tl_regState == REGSTATE_DIRTY ? "dirty" : "clean", + (TCA)_Unwind_GetIP(context), exnType); + } /* * We don't do anything during the search phase---before attempting @@ -137,22 +145,30 @@ tc_unwind_personality(int version, * tl_regState) and spilled any values they may have been holding in * callee-saved regs. */ - if (actions & _UA_SEARCH_PHASE) {} + if (actions & _UA_SEARCH_PHASE) { + if (ism) { + FTRACE(1, "thrown value: {} returning _URC_HANDLER_FOUND\n ", + ism->tv().pretty()); + return _URC_HANDLER_FOUND; + } + } /* - * During the cleanup phase, we can either use a landing pad to - * perform cleanup (with _Unwind_SetIP and _URC_INSTALL_CONTEXT), or - * we can do it here. We do it here currently so we can use unwind - * APIs to read the callee-saved register values instead of having - * to generate machine code that does it. + * During the cleanup phase, we can either use a landing pad to perform + * cleanup (with _Unwind_SetIP and _URC_INSTALL_CONTEXT), or we can do it + * here. We sync the VM registers here, then optionally use a landing pad, + * which is an exit traces from hhir with a few special instructions. */ else if (actions & _UA_CLEANUP_PHASE) { if (tl_regState == REGSTATE_DIRTY) { sync_regstate(context); } - sync_callee_saved(context); + if (install_catch_trace(context, exceptionObj, ism)) { + return _URC_INSTALL_CONTEXT; + } } + FTRACE(1, "returning _URC_CONTINUE_UNWIND\n"); return _URC_CONTINUE_UNWIND; } @@ -163,68 +179,11 @@ void deregister_unwind_region(std::vector* p) { } -////////////////////////////////////////////////////////////////////// - -UnwindRegInfo::UnwindRegInfo() { - clear(); -} - -std::string UnwindRegInfo::Data::pretty() const { - std::ostringstream out; - out << "(CRI r" - << reg << " dt:" << dataType << ' ' - << (exStack ? "stack" : "local") - << '@' << locOffset - << ")"; - return out.str(); -} - -void UnwindRegInfo::clear() { - memset(m_regs, 0, sizeof m_regs); -} - -bool UnwindRegInfo::empty() const { - return !m_regs[0].dirty; -} - -void UnwindRegInfo::add(RegNumber reg, - DataType type, - Location loc) { - assert(type >= -128 && type < 128 && - "UnwindRegInfo has only 8 bits for DataType"); - assert(loc.space == Location::Stack || loc.space == Location::Local); - assert(loc.offset >= std::numeric_limits::min() && - loc.offset <= std::numeric_limits::max() && - "UnwindRegInfo only has 16 bits for location offsets"); - - Data ent; - ent.dirty = true; - ent.dataType = type; - ent.exStack = loc.space == Location::Stack; - ent.locOffset = loc.offset; - - /* - * Important note: this is using asm-x64.h register numbers, - * although they are not the same as what dwarf/unwind uses. - * - * They happen to be the same for the specific (callee-saved) regs - * we care about right now, so this is ok. See kCalleeSaved. - */ - ent.reg = int(reg); - - for (int i = 0; i < kMaxCalleeSaved; ++i) { - if (!m_regs[i].dirty) { - m_regs[i] = ent; - return; - } - } - - assert(false && "Too many callee saved registers for UnwindRegInfo"); -} +/////////////////////////////////////////////////////////////////////////////// UnwindInfoHandle register_unwind_region(unsigned char* startAddr, size_t size) { - std::auto_ptr > bufferMem(new std::vector); + std::unique_ptr> bufferMem(new std::vector); std::vector& buffer = *bufferMem; { diff --git a/hphp/runtime/vm/translator/unwind-x64.h b/hphp/runtime/vm/translator/unwind-x64.h index 3ef3745d9..a955f1af6 100644 --- a/hphp/runtime/vm/translator/unwind-x64.h +++ b/hphp/runtime/vm/translator/unwind-x64.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "hphp/util/assertions.h" @@ -34,42 +35,24 @@ namespace HPHP { namespace Transl { ////////////////////////////////////////////////////////////////////// -/* - * Information about callee-saved registers that was dirty at a - * callsite. This should be registered during translation time so the - * unwinder can see it. - */ -struct UnwindRegInfo { - static const int kMaxCalleeSaved = 3; - - struct Data { - bool dirty : 1; - bool exStack : 1; // false means a local - signed dataType : 8; // DataType values - unsigned reg : 4; // Register id - int16_t locOffset; - - std::string pretty() const; - }; - static_assert(sizeof(Data) == sizeof(uint32_t), - "UnwindRegInfo::Data was too big"); - - explicit UnwindRegInfo(); - - void add(RegNumber reg, DataType type, Location loc); - void clear(); - bool empty() const; - - Data m_regs[kMaxCalleeSaved]; -}; - -static_assert(sizeof(UnwindRegInfo) <= sizeof(uint64_t) * 2, - "Unexpected size for UnwindRegInfo"); - -typedef TreadHashMap UnwindRegMap; +typedef TreadHashMap CatchTraceMap; ////////////////////////////////////////////////////////////////////// +inline const std::type_info& typeInfoFromUnwindException( + _Unwind_Exception* exceptionObj) +{ + constexpr size_t kTypeInfoOff = 112; + return **reinterpret_cast( + reinterpret_cast(exceptionObj + 1) - kTypeInfoOff); +} + +inline std::exception* exceptionFromUnwindException( + _Unwind_Exception* exceptionObj) +{ + return reinterpret_cast(exceptionObj + 1); +} + /* * Called whenever we create a new translation cache for the whole * region of code. diff --git a/hphp/util/trace.h b/hphp/util/trace.h index 469bd4dfd..b09927681 100644 --- a/hphp/util/trace.h +++ b/hphp/util/trace.h @@ -233,7 +233,9 @@ void dumpRingbuffer(); #define FTRACE(...) do { } while (0) #define TRACE_MOD(...) do { } while (0) #define FTRACE_MOD(...) do { } while (0) -#define TRACE_SET_MOD(...) /* nil */ +#define TRACE_SET_MOD(name) \ + DEBUG_ONLY static const HPHP::Trace::Module TRACEMOD = HPHP::Trace::name; + static const bool enabled = false; static inline void trace(const char*, ...) { }