diff --git a/hphp/doc/ir.specification b/hphp/doc/ir.specification index 98debd10f..0e43313f0 100644 --- a/hphp/doc/ir.specification +++ b/hphp/doc/ir.specification @@ -385,6 +385,11 @@ D:StkPtr = AssertStk S0:StkPtr is similar to a GuardStk except that it does not imply a runtime check and cannot cause control flow. +D:StkPtr = AssertStkVal S0:StkPtr S1:Gen + + Returns a new StkPtr that represents the same stack as S0, but with the + knowledge that the slot at offset has the value S1. + D:StkPtr = CastStk S0:StkPtr Returns a new StkPtr that represents the same stack as S0, but with diff --git a/hphp/runtime/vm/jit/codegen.cpp b/hphp/runtime/vm/jit/codegen.cpp index 2e8b383cb..3b472d369 100644 --- a/hphp/runtime/vm/jit/codegen.cpp +++ b/hphp/runtime/vm/jit/codegen.cpp @@ -355,6 +355,7 @@ NOOP_OPCODE(DefSP) NOOP_OPCODE(AssertLoc) NOOP_OPCODE(OverrideLoc) NOOP_OPCODE(AssertStk) +NOOP_OPCODE(AssertStkVal) NOOP_OPCODE(Nop) NOOP_OPCODE(DefLabel) NOOP_OPCODE(ExceptionBarrier) diff --git a/hphp/runtime/vm/jit/extradata.h b/hphp/runtime/vm/jit/extradata.h index 742e299ed..ccb746347 100644 --- a/hphp/runtime/vm/jit/extradata.h +++ b/hphp/runtime/vm/jit/extradata.h @@ -338,6 +338,7 @@ X(GuardStk, StackOffset); X(CheckStk, StackOffset); X(CastStk, StackOffset); X(AssertStk, StackOffset); +X(AssertStkVal, StackOffset); X(ReDefSP, StackOffset); X(ReDefGeneratorSP, StackOffset); X(DefSP, StackOffset); diff --git a/hphp/runtime/vm/jit/hhbctranslator.cpp b/hphp/runtime/vm/jit/hhbctranslator.cpp index ea02e7753..b7b63d32e 100644 --- a/hphp/runtime/vm/jit/hhbctranslator.cpp +++ b/hphp/runtime/vm/jit/hhbctranslator.cpp @@ -38,7 +38,6 @@ using namespace HPHP::Transl; HhbcTranslator::HhbcTranslator(IRFactory& irFactory, Offset startOffset, - Offset nextTraceOffset, uint32_t initialSpOffsetFromFp, const Func* func) : m_irFactory(irFactory) @@ -48,7 +47,6 @@ HhbcTranslator::HhbcTranslator(IRFactory& irFactory, func)) , m_bcStateStack {BcState(startOffset, func)} , m_startBcOff(startOffset) - , m_nextTraceBcOff(nextTraceOffset) , m_lastBcOff(false) , m_hasExit(false) , m_stackDeficit(0) @@ -2380,8 +2378,19 @@ void HhbcTranslator::guardTypeLocal(uint32_t locId, Type type) { gen(GuardLoc, type, LocalId(locId), m_tb->fp()); } -void HhbcTranslator::checkTypeLocal(uint32_t locId, Type type) { - gen(CheckLoc, type, LocalId(locId), getExitTrace(), m_tb->fp()); +void HhbcTranslator::guardTypeLocation(const Location& loc, Type type) { + if (loc.isStack()) { + guardTypeStack(loc.offset, type); + } else if (loc.isLocal()) { + guardTypeLocal(loc.offset, type); + } else { + not_reached(); + } +} + +void HhbcTranslator::checkTypeLocal(uint32_t locId, Type type, + Offset dest /* = -1 */) { + gen(CheckLoc, type, LocalId(locId), getExitTrace(dest), m_tb->fp()); } void HhbcTranslator::assertTypeLocal(uint32_t locId, Type type) { @@ -2392,6 +2401,27 @@ void HhbcTranslator::overrideTypeLocal(uint32_t locId, Type type) { gen(OverrideLoc, type, LocalId(locId), m_tb->fp()); } +void HhbcTranslator::checkTypeLocation(const Location& loc, Type type, + Offset dest) { + if (loc.isStack()) { + checkTypeStack(loc.offset, type, dest); + } else if (loc.isLocal()) { + checkTypeLocal(loc.offset, type, dest); + } else { + not_reached(); + } +} + +void HhbcTranslator::assertTypeLocation(const Location& loc, Type type) { + if (loc.isStack()) { + assertTypeStack(loc.offset, type); + } else if (loc.isLocal()) { + assertTypeLocal(loc.offset, type); + } else { + not_reached(); + } +} + void HhbcTranslator::guardTypeStack(uint32_t stackIndex, Type type) { // Should not generate guards for class; instead assert their type if (type.subtypeOf(Type::Cls)) { @@ -2402,22 +2432,24 @@ void HhbcTranslator::guardTypeStack(uint32_t stackIndex, Type type) { gen(GuardStk, type, StackOffset(stackIndex), m_tb->sp()); } -void HhbcTranslator::checkTypeTopOfStack(Type type, Offset nextByteCode) { - IRTrace* exitTrace = getExitTrace(nextByteCode); - SSATmp* tmp = m_evalStack.top(); - if (!tmp) { - FTRACE(1, "checkTypeTopOfStack: no tmp: {}\n", type.toString()); - gen(CheckStk, type, exitTrace, StackOffset(0), m_tb->sp()); - push(pop(type)); +void HhbcTranslator::checkTypeStack(uint32_t idx, Type type, Offset dest) { + auto exitTrace = getExitTrace(dest); + if (idx >= m_evalStack.size()) { + FTRACE(1, "checkTypeStack({}): no tmp: {}\n", idx, type.toString()); + gen(CheckStk, type, exitTrace, StackOffset(idx), m_tb->sp()); } else { - FTRACE(1, "checkTypeTopOfStack: generating CheckType for {}\n", - type.toString()); - m_evalStack.pop(); - tmp = gen(CheckType, type, exitTrace, tmp); - push(tmp); + FTRACE(1, "checkTypeStack(){}: generating CheckType for {}\n", + idx, type.toString()); + SSATmp* tmp = m_evalStack.top(idx); + assert(tmp); + m_evalStack.replace(idx, gen(CheckType, type, exitTrace, tmp)); } } +void HhbcTranslator::checkTypeTopOfStack(Type type, Offset nextByteCode) { + checkTypeStack(0, type, nextByteCode); +} + void HhbcTranslator::assertTypeStack(uint32_t stackIndex, Type type) { SSATmp* tmp = m_evalStack.top(stackIndex); if (!tmp) { @@ -2434,6 +2466,73 @@ void HhbcTranslator::assertTypeStack(uint32_t stackIndex, Type type) { refineType(tmp, type); } +void HhbcTranslator::assertString(const Location& loc, const StringData* str) { + auto idx = loc.offset; + + if (loc.isStack()) { + if (idx >= m_evalStack.size()) { + gen(AssertStkVal, StackOffset(idx), cns(str)); + } else { + DEBUG_ONLY SSATmp* oldStr = m_evalStack.top(idx); + assert(oldStr->type().maybe(Type::Str)); + m_evalStack.replace(idx, cns(str)); + } + } else if (loc.isLocal()) { + assert(m_tb->getLocalType(loc.offset).maybe(Type::Str)); + m_tb->setLocalValue(idx, cns(str)); + } else { + not_reached(); + } +} + +/* + * Creates a RuntimeType struct from a program location. This needs access to + * more than just the location's type because RuntimeType includes known + * constant values. + */ +RuntimeType HhbcTranslator::rttFromLocation(const Location& loc) { + Type t; + SSATmp* val; + switch (loc.space) { + case Location::Stack: { + auto i = loc.offset; + assert(i >= 0); + if (i < m_evalStack.size()) { + val = m_evalStack.top(i); + t = val->type(); + } else { + auto stackVal = getStackValue(m_tb->sp(), i); + val = stackVal.value; + t = stackVal.knownType; + } + } break; + case Location::Local: { + auto l = loc.offset; + val = m_tb->getLocalValue(l); + t = val ? val->type() : m_tb->getLocalType(l); + } break; + case Location::Litstr: + return RuntimeType(curUnit()->lookupLitstrId(loc.offset)); + case Location::Litint: + return RuntimeType(loc.offset); + case Location::This: + return RuntimeType(KindOfObject, KindOfInvalid, curFunc()->cls()); + case Location::Invalid: + case Location::Iter: + not_reached(); + } + + assert(IMPLIES(val, val->type().equals(t))); + if (val && val->isConst()) { + // RuntimeType holds constant Bool, Int, Str, and Cls. + if (val->type().isBool()) return RuntimeType(val->getValBool()); + if (val->type().isInt()) return RuntimeType(val->getValInt()); + if (val->type().isString()) return RuntimeType(val->getValStr()); + if (val->type().isCls()) return RuntimeType(val->getValClass()); + } + return t.toRuntimeType(); +} + static uint64_t packBitVec(const vector& bits, unsigned i) { uint64_t retval = 0; assert(i % 64 == 0); @@ -3386,8 +3485,9 @@ SSATmp* HhbcTranslator::stLocImpl(uint32_t id, assert(!newVal->type().maybeBoxed()); auto const oldLoc = ldLoc(id); - assert((oldLoc->type().isBoxed() || oldLoc->type().notBoxed()) && - "We don't support maybeBoxed locals right now"); + if (!(oldLoc->type().isBoxed() || oldLoc->type().notBoxed())) { + PUNT(stLocImpl-maybeBoxedValue); + } if (oldLoc->type().notBoxed()) { gen(StLoc, LocalId(id), m_tb->fp(), newVal); @@ -3426,7 +3526,8 @@ SSATmp* HhbcTranslator::stLocNRC(uint32_t id, IRTrace* exit, SSATmp* newVal) { void HhbcTranslator::end() { if (m_hasExit) return; - auto const nextPc = m_nextTraceBcOff; + auto const nextSk = curSrcKey().advanced(curUnit()); + auto const nextPc = nextSk.offset(); if (nextPc >= curFunc()->past()) { // We have fallen off the end of the func's bytecodes. This happens // when the function's bytecodes end with an unconditional diff --git a/hphp/runtime/vm/jit/hhbctranslator.h b/hphp/runtime/vm/jit/hhbctranslator.h index 02134b1df..ffb6effa2 100644 --- a/hphp/runtime/vm/jit/hhbctranslator.h +++ b/hphp/runtime/vm/jit/hhbctranslator.h @@ -34,6 +34,9 @@ namespace HPHP { namespace Transl { struct PropInfo; } namespace JIT { +using Transl::Location; +using Transl::RuntimeType; + ////////////////////////////////////////////////////////////////////// struct EvalStack { @@ -102,7 +105,6 @@ private: struct HhbcTranslator { HhbcTranslator(IRFactory& irFactory, Offset startOffset, - Offset nextTraceOffset, uint32_t initialSpOffsetFromFp, const Func* func); @@ -118,6 +120,7 @@ struct HhbcTranslator { // Tracelet guards. void guardTypeStack(uint32_t stackIndex, Type type); void guardTypeLocal(uint32_t locId, Type type); + void guardTypeLocation(const Location& loc, Type type); void guardRefs(int64_t entryArDelta, const vector& mask, const vector& vals); @@ -125,9 +128,15 @@ struct HhbcTranslator { // Interface to irtranslator for predicted and inferred types. void assertTypeLocal(uint32_t localIndex, Type type); void assertTypeStack(uint32_t stackIndex, Type type); - void checkTypeLocal(uint32_t localIndex, Type type); + void checkTypeLocal(uint32_t localIndex, Type type, Offset dest = -1); + void checkTypeStack(uint32_t idx, Type type, Offset dest); void checkTypeTopOfStack(Type type, Offset nextByteCode); void overrideTypeLocal(uint32_t localIndex, Type type); + void assertTypeLocation(const Location& loc, Type type); + void checkTypeLocation(const Location& loc, Type type, Offset dest); + void assertString(const Location& loc, const StringData* sd); + + RuntimeType rttFromLocation(const Location& loc); // Inlining-related functions. void beginInlining(unsigned numArgs, @@ -795,10 +804,8 @@ private: std::vector m_bcStateStack; - // The first HHBC offset for this tracelet, and the offset for the - // next Traclet. + // The first HHBC offset for this tracelet const Offset m_startBcOff; - const Offset m_nextTraceBcOff; // True if we're on the last HHBC opcode that will be emitted for // this tracelet. diff --git a/hphp/runtime/vm/jit/ir.h b/hphp/runtime/vm/jit/ir.h index d2f1e4ac4..e1edf7752 100644 --- a/hphp/runtime/vm/jit/ir.h +++ b/hphp/runtime/vm/jit/ir.h @@ -163,6 +163,7 @@ O(CheckLoc, ND, S(FramePtr), E) \ O(CheckStk, D(StkPtr), S(StkPtr), E) \ O(CastStk, D(StkPtr), S(StkPtr), Mem|N|Er) \ O(AssertStk, D(StkPtr), S(StkPtr), E) \ +O(AssertStkVal, D(StkPtr), S(StkPtr) S(Gen), E) \ O(GuardRefs, ND, S(Func) \ S(Int) \ S(Int) \ diff --git a/hphp/runtime/vm/jit/irtranslator.cpp b/hphp/runtime/vm/jit/irtranslator.cpp index bc5830967..b013a7730 100644 --- a/hphp/runtime/vm/jit/irtranslator.cpp +++ b/hphp/runtime/vm/jit/irtranslator.cpp @@ -181,7 +181,6 @@ Translator::translateLtGtOp(const NormalizedInstruction& i) { const Opcode op = i.op(); assert(op == OpLt || op == OpLte || op == OpGt || op == OpGte); assert(i.inputs.size() == 2); - assert(i.outStack && !i.outLocal); assert(i.inputs[0]->outerType() != KindOfRef); assert(i.inputs[1]->outerType() != KindOfRef); @@ -259,9 +258,6 @@ Translator::translateAssignToLocalOp(const NormalizedInstruction& ni) { assert((op == OpBindL) == (ni.inputs[rhsIdx]->outerType() == KindOfRef)); - assert(!ni.outStack || ni.inputs[locIdx]->location != ni.outStack->location); - assert(ni.outLocal); - assert(ni.inputs[locIdx]->location == ni.outLocal->location); assert(ni.inputs[rhsIdx]->isStack()); if (op == OpSetL) { @@ -276,7 +272,6 @@ Translator::translateAssignToLocalOp(const NormalizedInstruction& ni) { void Translator::translatePopC(const NormalizedInstruction& i) { assert(i.inputs.size() == 1); - assert(!i.outStack && !i.outLocal); if (i.inputs[0]->rtt.isVagueValue()) { HHIR_EMIT(PopR); @@ -310,7 +305,6 @@ Translator::translateUnboxR(const NormalizedInstruction& i) { void Translator::translateNull(const NormalizedInstruction& i) { assert(i.inputs.size() == 0); - assert(!i.outLocal); HHIR_EMIT(Null); } @@ -323,7 +317,6 @@ Translator::translateNullUninit(const NormalizedInstruction& i) { void Translator::translateTrue(const NormalizedInstruction& i) { assert(i.inputs.size() == 0); - assert(!i.outLocal); HHIR_EMIT(True); } @@ -331,7 +324,6 @@ Translator::translateTrue(const NormalizedInstruction& i) { void Translator::translateFalse(const NormalizedInstruction& i) { assert(i.inputs.size() == 0); - assert(!i.outLocal); HHIR_EMIT(False); } @@ -339,7 +331,6 @@ Translator::translateFalse(const NormalizedInstruction& i) { void Translator::translateInt(const NormalizedInstruction& i) { assert(i.inputs.size() == 0); - assert(!i.outLocal); HHIR_EMIT(Int, i.imm[0].u_I64A); } @@ -352,7 +343,6 @@ Translator::translateDouble(const NormalizedInstruction& i) { void Translator::translateString(const NormalizedInstruction& i) { assert(i.inputs.size() == 0); - assert(!i.outLocal); HHIR_EMIT(String, (i.imm[0].u_SA)); } @@ -360,7 +350,6 @@ Translator::translateString(const NormalizedInstruction& i) { void Translator::translateArray(const NormalizedInstruction& i) { assert(i.inputs.size() == 0); - assert(!i.outLocal); HHIR_EMIT(Array, i.imm[0].u_AA); } @@ -383,7 +372,6 @@ Translator::translateAddElemC(const NormalizedInstruction& i) { void Translator::translateAddNewElemC(const NormalizedInstruction& i) { assert(i.inputs.size() == 2); - assert(i.outStack && !i.outLocal); assert(i.inputs[0]->outerType() != KindOfRef); assert(i.inputs[1]->outerType() != KindOfRef); assert(i.inputs[0]->isStack()); @@ -436,15 +424,12 @@ Translator::translateNot(const NormalizedInstruction& i) { void Translator::translateBitNot(const NormalizedInstruction& i) { - assert(i.outStack && !i.outLocal); - HHIR_EMIT(BitNot); } void Translator::translateCastInt(const NormalizedInstruction& i) { assert(i.inputs.size() == 1); - assert(i.outStack && !i.outLocal); HHIR_EMIT(CastInt); /* nop */ @@ -748,7 +733,6 @@ void Translator::translateFPassG(const NormalizedInstruction& ni) { void Translator::translateCheckTypeOp(const NormalizedInstruction& ni) { assert(ni.inputs.size() == 1); - assert(ni.outStack); const Opcode op = ni.op(); const int off = ni.inputs[0]->location.offset; @@ -804,7 +788,6 @@ void Translator::translateIncDecL(const NormalizedInstruction& i) { const vector& inputs = i.inputs; assert(inputs.size() == 1); - assert(i.outLocal); assert(inputs[0]->isLocal()); const IncDecOp oplet = IncDecOp(i.imm[1].u_OA); assert(oplet == PreInc || oplet == PostInc || oplet == PreDec || @@ -921,7 +904,7 @@ Translator::translateCheckThis(const NormalizedInstruction& i) { void Translator::translateInitThisLoc(const NormalizedInstruction& i) { - HHIR_EMIT(InitThisLoc, i.outLocal->location.offset); + HHIR_EMIT(InitThisLoc, i.imm[0].u_HA); } void @@ -1583,8 +1566,6 @@ void Translator::interpretInstr(const NormalizedInstruction& i) { } void Translator::translateInstr(const NormalizedInstruction& i) { - assert(!i.outStack || i.outStack->isStack()); - assert(!i.outLocal || i.outLocal->isLocal()); FTRACE(1, "translating: {}\n", opcodeToName(i.op())); m_hhbcTrans->setBcOff(i.source.offset(), @@ -1673,7 +1654,7 @@ static bool supportedInterpOne(const NormalizedInstruction* i) { } } -TranslatorX64::TranslateTraceletResult +TranslatorX64::TranslateResult TranslatorX64::irTranslateTracelet(Tracelet& t, const TCA start, const TCA stubStart, @@ -1747,11 +1728,11 @@ TranslatorX64::irTranslateTracelet(Tracelet& t, if (ni->breaksTracelet) break; } - hhirTraceEnd(); + traceEnd(); if (transResult != Retry) { try { transResult = Success; - hhirTraceCodeGen(bcMap); + traceCodeGen(bcMap); } catch (JIT::FailedCodeGen& fcg) { // Code-gen failed. Search for the bytecode instruction that caused the // problem, flag it to be interpreted, and retranslate the tracelet. @@ -1800,24 +1781,10 @@ TranslatorX64::irTranslateTracelet(Tracelet& t, FTRACE(1, "HHIR: FAILED with exception: {}\n", e.what()); assert(0); } - - if (transResult != Success) { - // The whole translation failed; give up on this BB. Since it is not - // linked into srcDB yet, it is guaranteed not to be reachable. - // Free IR resources for this trace, rollback the Translation cache - // frontiers, and discard any pending fixups. - hhirTraceFree(); - a.code.frontier = start; - astubs.code.frontier = stubStart; - m_pendingFixups.clear(); - // Reset additions to list of addresses which need to be patched - srcRec.clearInProgressTailJumps(); - } return transResult; } -void TranslatorX64::hhirTraceStart(Offset bcStartOffset, - Offset nextTraceletOffset) { +void Translator::traceStart(Offset bcStartOffset) { assert(!m_irFactory); Cell* fp = vmfp(); @@ -1831,10 +1798,10 @@ void TranslatorX64::hhirTraceStart(Offset bcStartOffset, m_irFactory.reset(new JIT::IRFactory()); m_hhbcTrans.reset(new JIT::HhbcTranslator( - *m_irFactory, bcStartOffset, nextTraceletOffset, fp - vmsp(), curFunc())); + *m_irFactory, bcStartOffset, fp - vmsp(), curFunc())); } -void TranslatorX64::hhirTraceEnd() { +void Translator::traceEnd() { m_hhbcTrans->end(); FTRACE(1, "{}{:-^40}{}\n", color(ANSI_COLOR_BLACK, ANSI_BGCOLOR_GREEN), @@ -1842,7 +1809,7 @@ void TranslatorX64::hhirTraceEnd() { color(ANSI_COLOR_END)); } -void TranslatorX64::hhirTraceCodeGen(vector* bcMap) { +void TranslatorX64::traceCodeGen(vector* bcMap) { using namespace JIT; HPHP::JIT::IRTrace* trace = m_hhbcTrans->trace(); @@ -1881,10 +1848,10 @@ void TranslatorX64::hhirTraceCodeGen(vector* bcMap) { } m_numHHIRTrans++; - hhirTraceFree(); + traceFree(); } -void TranslatorX64::hhirTraceFree() { +void Translator::traceFree() { FTRACE(1, "HHIR free: arena size: {}\n", m_irFactory->arena().size()); m_hhbcTrans.reset(); diff --git a/hphp/runtime/vm/jit/simplifier.cpp b/hphp/runtime/vm/jit/simplifier.cpp index 54e8bf3f9..a6b3ad96e 100644 --- a/hphp/runtime/vm/jit/simplifier.cpp +++ b/hphp/runtime/vm/jit/simplifier.cpp @@ -66,6 +66,12 @@ StackValueInfo getStackValue(SSATmp* sp, uint32_t index) { } return getStackValue(inst->src(0), index); + case AssertStkVal: + if (inst->extra()->offset == index) { + return StackValueInfo { inst->src(1) }; + } + return getStackValue(inst->src(0), index); + case CallArray: { if (index == 0) { // return value from call diff --git a/hphp/runtime/vm/jit/tracebuilder.cpp b/hphp/runtime/vm/jit/tracebuilder.cpp index c3a9ab83a..d73e2f8c4 100644 --- a/hphp/runtime/vm/jit/tracebuilder.cpp +++ b/hphp/runtime/vm/jit/tracebuilder.cpp @@ -211,6 +211,7 @@ void TraceBuilder::updateTrackedState(IRInstruction* inst) { break; case AssertStk: + case AssertStkVal: case CastStk: case CheckStk: case GuardStk: diff --git a/hphp/runtime/vm/jit/tracebuilder.h b/hphp/runtime/vm/jit/tracebuilder.h index 3ca84ee10..9e5e2b8a2 100644 --- a/hphp/runtime/vm/jit/tracebuilder.h +++ b/hphp/runtime/vm/jit/tracebuilder.h @@ -111,6 +111,10 @@ struct TraceBuilder { m_thisIsAvailable = true; } + Type getLocalType(unsigned id) const; + SSATmp* getLocalValue(unsigned id) const; + void setLocalValue(unsigned id, SSATmp* value); + /* * Run another pass of TraceBuilder-managed optimizations on this * trace. @@ -328,9 +332,6 @@ private: void killLocals(); void killLocalValue(uint32_t id); void setLocalType(uint32_t id, Type type); - Type getLocalType(unsigned id) const; - void setLocalValue(unsigned id, SSATmp* value); - SSATmp* getLocalValue(unsigned id) const; bool isValueAvailable(SSATmp*) const; bool anyLocalHasValue(SSATmp*) const; bool callerHasValueAvailable(SSATmp*) const; diff --git a/hphp/runtime/vm/jit/translator-x64.cpp b/hphp/runtime/vm/jit/translator-x64.cpp index 14edfb55a..d155cb5d1 100644 --- a/hphp/runtime/vm/jit/translator-x64.cpp +++ b/hphp/runtime/vm/jit/translator-x64.cpp @@ -817,18 +817,6 @@ TranslatorX64::createTranslation(const TranslArgs& args) { } } - /* - * First test if we have a region-selector that can handle this - * SrcKey. - */ - JIT::RegionContext rContext { curFunc(), sk.offset() }; - populateLiveContext(rContext); - if (auto UNUSED rd = JIT::selectRegion(rContext)) { - /* - * WIP. Unimplemented. - */ - } - // We put retranslate requests at the end of our slab to more frequently // allow conditional jump fall-throughs AHotSelector ahs(this, curFunc()->attrs() & AttrHot); @@ -883,7 +871,8 @@ TranslatorX64::translate(const TranslArgs& args) { } TCA start = a.code.frontier; - translateTracelet(args); + + translateWork(args); SKTRACE(1, args.m_sk, "translate moved head from %p to %p\n", getTopTranslation(args.m_sk), start); @@ -3273,14 +3262,14 @@ void dumpTranslationInfo(const Tracelet& t, TCA postGuards) { } void -TranslatorX64::translateTracelet(const TranslArgs& args) { +TranslatorX64::translateWork(const TranslArgs& args) { auto sk = args.m_sk; std::unique_ptr tp = analyze(sk); Tracelet& t = *tp; m_curTrace = &t; Nuller ctNuller(&m_curTrace); - SKTRACE(1, sk, "translateTracelet\n"); + SKTRACE(1, sk, "translateWork\n"); assert(m_srcDB.find(sk)); TCA start = a.code.frontier; @@ -3292,16 +3281,48 @@ TranslatorX64::translateTracelet(const TranslArgs& args) { TransKind transKind = TransInterp; if (!args.m_interp) { - TranslateTraceletResult result; - do { - hhirTraceStart(sk.offset(), t.nextSk().offset()); - SKTRACE(1, sk, "retrying irTranslateTracelet\n"); - result = irTranslateTracelet(t, start, stubStart, &bcMapping); - if (result == Retry) { - assert(a.code.frontier == start); - assert(astubs.code.frontier == stubStart); + // Attempt to create a region at this SrcKey + JIT::RegionContext rContext { curFunc(), args.m_sk.offset() }; + populateLiveContext(rContext); + auto region = JIT::selectRegion(rContext); + + TranslateResult result = Retry; + while (result == Retry) { + assert(a.code.frontier == start); + assert(astubs.code.frontier == stubStart); + + traceStart(sk.offset()); + + // Try translating a region if we have one, then fall back to using the + // Tracelet. + if (region) { + try { + result = translateRegion(*region, &bcMapping); + FTRACE(2, "translateRegion succeeded\n"); + } catch (const std::exception& e) { + FTRACE(1, "translateRegion failed with '{}'\n", e.what()); + traceEnd(); + result = Failure; + } } - } while (result == Retry); + if (!region || result == Failure) { + FTRACE(1, "trying irTranslateTracelet\n"); + result = irTranslateTracelet(*tp, start, stubStart, &bcMapping); + } + + if (result != Success) { + // The whole translation failed; give up on this SrcKey. Since it is not + // linked into srcDB yet, it is guaranteed not to be reachable. + // Free IR resources for this trace, rollback the Translation cache + // frontiers, and discard any pending fixups. + traceFree(); + a.code.frontier = start; + astubs.code.frontier = stubStart; + m_pendingFixups.clear(); + // Reset additions to list of addresses which need to be patched + srcRec.clearInProgressTailJumps(); + } + } if (result == Success) { m_irAUsage += (a.code.frontier - start); diff --git a/hphp/runtime/vm/jit/translator-x64.h b/hphp/runtime/vm/jit/translator-x64.h index a269fb336..a7cdadd83 100644 --- a/hphp/runtime/vm/jit/translator-x64.h +++ b/hphp/runtime/vm/jit/translator-x64.h @@ -192,11 +192,7 @@ class TranslatorX64 : public Translator // Data structures for HHIR-based translation uint64_t m_numHHIRTrans; - void hhirTraceStart(Offset bcStartOffset, Offset nextTraceOffset); - void hhirTraceCodeGen(vector* bcMap); - void hhirTraceEnd(); - void hhirTraceFree(); - + virtual void traceCodeGen(std::vector*); FixupMap m_fixupMap; UnwindInfoHandle m_unwindRegistrar; @@ -376,15 +372,10 @@ public: bool freeRequestStub(TCA stub); TCA getFreeStub(); bool checkTranslationLimit(SrcKey, const SrcRec&) const; - enum TranslateTraceletResult { - Failure, - Retry, - Success - }; - TranslateTraceletResult irTranslateTracelet(Tracelet& t, - const TCA start, - const TCA stubStart, - vector* bcMap); + TranslateResult irTranslateTracelet(Tracelet& t, + const TCA start, + const TCA stubStart, + vector* bcMap); void irAssertType(const Location& l, const RuntimeType& rtt); void checkType(Asm&, const Location& l, const RuntimeType& rtt, @@ -450,7 +441,7 @@ public: TCA createTranslation(const TranslArgs& args); TCA retranslate(const TranslArgs& args); TCA translate(const TranslArgs& args); - void translateTracelet(const TranslArgs& args); + void translateWork(const TranslArgs& args); TCA lookupTranslation(SrcKey sk) const; TCA retranslateOpt(TransID transId, bool align); diff --git a/hphp/runtime/vm/jit/translator.cpp b/hphp/runtime/vm/jit/translator.cpp index 34684060e..c96b03c6a 100644 --- a/hphp/runtime/vm/jit/translator.cpp +++ b/hphp/runtime/vm/jit/translator.cpp @@ -40,6 +40,7 @@ #include "hphp/runtime/vm/jit/annotation.h" #include "hphp/runtime/vm/jit/hhbctranslator.h" #include "hphp/runtime/vm/jit/irfactory.h" +#include "hphp/runtime/vm/jit/region_selection.h" #include "hphp/runtime/vm/jit/targetcache.h" #include "hphp/runtime/vm/jit/translator-inline.h" #include "hphp/runtime/vm/jit/translator-x64.h" @@ -88,7 +89,7 @@ struct TraceletContext { RuntimeType currentType(const Location& l) const; DynLocation* recordRead(const InputInfo& l, bool useHHIR, DataType staticType = KindOfInvalid); - void recordWrite(DynLocation* dl, NormalizedInstruction* source); + void recordWrite(DynLocation* dl); void recordDelete(const Location& l); void recordJmp(); void aliasTaint(); @@ -2397,11 +2398,9 @@ DynLocation* TraceletContext::recordRead(const InputInfo& ii, return dl; } -void TraceletContext::recordWrite(DynLocation* dl, - NormalizedInstruction* source) { +void TraceletContext::recordWrite(DynLocation* dl) { TRACE(2, "recordWrite: %s : %s\n", dl->location.pretty().c_str(), dl->rtt.pretty().c_str()); - dl->source = source; m_currentMap[dl->location] = dl; m_changeSet.insert(dl->location); m_deletedSet.erase(dl->location); @@ -3204,7 +3203,7 @@ void Translator::analyzeCallee(TraceletContext& tas, fcall->outputPredictionStatic = false; fcall->outStack = parent.newDynLocation(fcall->outStack->location, it->second->rtt); - tas.recordWrite(fcall->outStack, fcall); + tas.recordWrite(fcall->outStack); } } @@ -3432,7 +3431,7 @@ std::unique_ptr Translator::analyze(SrcKey sk, SKTRACE(2, sk, "inserting output t(%d->%d) #(%s, %d)\n", o->rtt.outerType(), o->rtt.innerType(), o->location.spaceName(), o->location.offset); - tas.recordWrite(o, ni); + tas.recordWrite(o); } } @@ -3629,6 +3628,233 @@ void Translator::populateImmediates(NormalizedInstruction& inst) { } } +/* + * Similar to applyInputMetaData, but designed to be used during ir + * generation. Reads and writes types of values using m_hhbcTrans. This will + * eventually replace applyInputMetaData. + */ +void Translator::readMetaData(Unit::MetaHandle& handle, + NormalizedInstruction& inst) { + if (!handle.findMeta(inst.unit(), inst.offset())) return; + + Unit::MetaInfo info; + if (!handle.nextArg(info)) return; + if (info.m_kind == Unit::MetaInfo::NopOut) { + inst.noOp = true; + return; + } + + /* + * We need to adjust the indexes in MetaInfo::m_arg if this instruction takes + * other stack arguments than those related to the MVector. (For example, + * the rhs of an assignment.) + */ + auto const& iInfo = instrInfo[inst.op()]; + if (iInfo.in & AllLocals) { + /* + * RetC/RetV dont care about their stack input, but it may have been + * annotated. Skip it (because RetC/RetV pretend they dont have a stack + * input). + */ + return; + } + if (iInfo.in == FuncdRef) { + /* + * FPassC* pretend to have no inputs + */ + return; + } + const int base = !(iInfo.in & MVector) ? 0 : + !(iInfo.in & Stack1) ? 0 : + !(iInfo.in & Stack2) ? 1 : + !(iInfo.in & Stack3) ? 2 : 3; + + do { + SKTRACE(3, inst.source, "considering MetaInfo of kind %d\n", info.m_kind); + + int arg = info.m_arg & Unit::MetaInfo::VectorArg ? + base + (info.m_arg & ~Unit::MetaInfo::VectorArg) : info.m_arg; + auto& input = *inst.inputs[arg]; + auto updateType = [&]{ + input.rtt = m_hhbcTrans->rttFromLocation(input.location); + }; + + switch (info.m_kind) { + case Unit::MetaInfo::NoSurprise: + inst.noSurprise = true; + break; + case Unit::MetaInfo::GuardedCls: + inst.guardedCls = true; + break; + case Unit::MetaInfo::ArrayCapacity: + inst.imm[0].u_IVA = info.m_data; + break; + case Unit::MetaInfo::DataTypePredicted: { + m_hhbcTrans->checkTypeLocation( + input.location, Type::fromDataType(DataType(info.m_data)), + inst.source.offset()); + updateType(); + break; + } + case Unit::MetaInfo::DataTypeInferred: { + m_hhbcTrans->assertTypeLocation( + input.location, Type::fromDataType(DataType(info.m_data))); + updateType(); + break; + } + case Unit::MetaInfo::String: { + m_hhbcTrans->assertString(input.location, + inst.unit()->lookupLitstrId(info.m_data)); + updateType(); + break; + } + case Unit::MetaInfo::Class: { + RuntimeType& rtt = inst.inputs[arg]->rtt; + if (rtt.valueType() != KindOfObject) { + continue; + } + + const StringData* metaName = inst.unit()->lookupLitstrId(info.m_data); + const StringData* rttName = + rtt.valueClass() ? rtt.valueClass()->name() : nullptr; + // The two classes might not be exactly the same, which is ok + // as long as metaCls is more derived than rttCls. + Class* metaCls = Unit::lookupUniqueClass(metaName); + Class* rttCls = rttName ? Unit::lookupUniqueClass(rttName) : nullptr; + if (metaCls && rttCls && metaCls != rttCls && + !metaCls->classof(rttCls)) { + // Runtime type is more derived + metaCls = 0; + } + if (metaCls && metaCls != rttCls) { + SKTRACE(1, inst.source, "replacing input %d with a MetaInfo-supplied " + "class of %s; old type = %s\n", + arg, metaName->data(), rtt.pretty().c_str()); + if (rtt.isRef()) { + rtt = RuntimeType(KindOfRef, KindOfObject, metaCls); + } else { + rtt = RuntimeType(KindOfObject, KindOfInvalid, metaCls); + } + } + break; + } + case Unit::MetaInfo::MVecPropClass: { + const StringData* metaName = inst.unit()->lookupLitstrId(info.m_data); + Class* metaCls = Unit::lookupUniqueClass(metaName); + if (metaCls) { + inst.immVecClasses[arg] = metaCls; + } + break; + } + case Unit::MetaInfo::NopOut: + // NopOut should always be the first and only annotation + // and was handled above. + not_reached(); + + case Unit::MetaInfo::GuardedThis: + case Unit::MetaInfo::NonRefCounted: + // fallthrough; these are handled in preInputApplyMetaData. + case Unit::MetaInfo::None: + break; + } + } while (handle.nextArg(info)); +} + +static Location toLocation(const RegionDesc::Location& loc) { + typedef RegionDesc::Location::Tag T; + switch (loc.tag()) { + case T::Stack: + return Location(Location::Stack, loc.stackOffset()); + + case T::Local: + return Location(Location::Local, loc.localId()); + } + not_reached(); +} + +Translator::TranslateResult +Translator::translateRegion(const RegionDesc& region, + vector* bcMap) { + FTRACE(1, "translateRegion starting with:\n{}\n", show(region)); + assert(!region.blocks.empty()); + + for (auto const& block : region.blocks) { + const SrcKey startSk = block->start(); + SrcKey sk = startSk; + auto predIt = block->typePreds().begin(); + auto const predEnd = block->typePreds().end(); + + for (unsigned i = 0; i < block->length(); ++i, sk.advance(block->unit())) { + // Emit prediction guards. If this is the first instruction in the + // region the guards will go to a retranslate request. Otherwise, they'll + // go to a side exit. + for (; predIt != predEnd && predIt->first == sk; ++predIt) { + auto const& pred = predIt->second; + if (block == region.blocks.front() && i == 0) { + m_hhbcTrans->guardTypeLocation(toLocation(pred.location), pred.type); + } else { + m_hhbcTrans->checkTypeLocation(toLocation(pred.location), pred.type, + sk.offset()); + } + } + + // Create and initialize the instruction. + NormalizedInstruction inst; + inst.source = sk; + inst.m_unit = block->unit(); + inst.preppedByRef = false; + inst.breaksTracelet = false; + inst.changesPC = opcodeChangesPC(inst.op()); + populateImmediates(inst); + + // Apply the first round of metadata from the repo and get a list of + // input locations. + InputInfos inputInfos; + Unit::MetaHandle metaHand; + preInputApplyMetaData(metaHand, &inst); + + // TranslatorX64 expected top of stack to be index -1, with indexes + // growing down from there. hhir defines top of stack to be index 1, with + // indexes growing up from there. To compensate we start with a stack + // offset of 1 and negate the index of any stack input after the call to + // getInputs. + int stackOff = 1; + getInputs(startSk, &inst, stackOff, inputInfos, [&](int i) { + return m_hhbcTrans->traceBuilder()->getLocalType(i); + }); + for (auto& info : inputInfos) { + if (info.loc.isStack()) info.loc.offset = -info.loc.offset; + } + + // Populate the NormalizedInstruction's input vector, using types from + // HhbcTranslator. + std::vector dynLocs; + dynLocs.reserve(inputInfos.size()); + auto newDynLoc = [&](const InputInfo& ii) { + dynLocs.emplace_back(ii.loc, m_hhbcTrans->rttFromLocation(ii.loc)); + FTRACE(2, "rttFromLocation: {} -> {}\n", + ii.loc.pretty(), dynLocs.back().rtt.pretty()); + return &dynLocs.back(); + }; + FTRACE(2, "populating inputs for {}\n", inst.toString()); + for (auto const& ii : inputInfos) { + inst.inputs.push_back(newDynLoc(ii)); + } + + // Apply the remaining metadata. This may change the types of some of + // inst's inputs. + readMetaData(metaHand, inst); + + Util::Nuller niNuller(&m_curNI); + m_curNI = &inst; + translateInstr(inst); + } + } + + traceCodeGen(bcMap); + return Success; +} + uint64_t* Translator::getTransCounterAddr() { if (!isTransDBEnabled()) return nullptr; diff --git a/hphp/runtime/vm/jit/translator.h b/hphp/runtime/vm/jit/translator.h index 32eaa0ee8..780573488 100644 --- a/hphp/runtime/vm/jit/translator.h +++ b/hphp/runtime/vm/jit/translator.h @@ -46,10 +46,12 @@ namespace HPHP { namespace JIT { class HhbcTranslator; class IRFactory; +class RegionDesc; } namespace Transl { using JIT::Type; +using JIT::RegionDesc; static const bool trustSigSegv = false; static const uint32_t transCountersPerChunk = 1024 * 1024 / 8; @@ -80,13 +82,12 @@ struct NormalizedInstruction; struct DynLocation { Location location; RuntimeType rtt; - NormalizedInstruction* source; - DynLocation(Location l, DataType t) : location(l), rtt(t), source(nullptr) {} + DynLocation(Location l, DataType t) : location(l), rtt(t) {} - DynLocation(Location l, RuntimeType t) : location(l), rtt(t), source(nullptr) {} + DynLocation(Location l, RuntimeType t) : location(l), rtt(t) {} - DynLocation() : location(), rtt(KindOfInvalid), source(nullptr) {} + DynLocation() : location(), rtt(KindOfInvalid) {} bool operator==(const DynLocation& r) const { return rtt == r.rtt && location == r.location; @@ -290,6 +291,7 @@ class NormalizedInstruction { NormalizedInstruction() : next(nullptr) , prev(nullptr) + , funcd(nullptr) , outStack(nullptr) , outLocal(nullptr) , outLocal2(nullptr) @@ -841,6 +843,8 @@ private: NormalizedInstruction* ni, TraceletContext& tas, InputInfos& ii); + void readMetaData(Unit::MetaHandle& handle, + NormalizedInstruction& inst); void getInputs(SrcKey startSk, NormalizedInstruction* ni, int& currentStackOffset, @@ -889,15 +893,22 @@ private: virtual void invalidateSrcKey(SrcKey sk) = 0; protected: + enum TranslateResult { + Failure, + Retry, + Success + }; void translateInstr(const NormalizedInstruction& i); + void traceStart(Offset bcStartOffset); + virtual void traceCodeGen(vector* bcMap) = 0; + void traceEnd(); + void traceFree(); + private: void interpretInstr(const NormalizedInstruction& i); void translateInstrWork(const NormalizedInstruction& i); void translateInstrDefault(const NormalizedInstruction& i); void passPredictedAndInferredTypes(const NormalizedInstruction& i); - - void translateReqLit(const NormalizedInstruction& i, - InclOpFlags flags); #define CASE(nm) void translate ## nm(const NormalizedInstruction& i); INSTRS PSEUDOINSTRS @@ -915,6 +926,8 @@ protected: void requestResetHighLevelTranslator(); void populateImmediates(NormalizedInstruction&); + TranslateResult translateRegion(const RegionDesc& region, + vector* bcMap); TCA m_resumeHelper; TCA m_resumeHelperRet; diff --git a/hphp/runtime/vm/jit/type.cpp b/hphp/runtime/vm/jit/type.cpp index 37517ea8f..6569fa678 100644 --- a/hphp/runtime/vm/jit/type.cpp +++ b/hphp/runtime/vm/jit/type.cpp @@ -32,6 +32,80 @@ TRACE_SET_MOD(hhir); ////////////////////////////////////////////////////////////////////// +DataType Type::toDataType() const { + assert(!isPtr()); + if (isBoxed()) { + return KindOfRef; + } + + // Order is important here: types must progress from more specific + // to less specific to return the most specific DataType. + if (subtypeOf(None)) return KindOfInvalid; + if (subtypeOf(Uninit)) return KindOfUninit; + if (subtypeOf(Null)) return KindOfNull; + if (subtypeOf(Bool)) return KindOfBoolean; + if (subtypeOf(Int)) return KindOfInt64; + if (subtypeOf(Dbl)) return KindOfDouble; + if (subtypeOf(StaticStr)) return KindOfStaticString; + if (subtypeOf(Str)) return KindOfString; + if (subtypeOf(Arr)) return KindOfArray; + if (subtypeOf(Obj)) return KindOfObject; + if (subtypeOf(Cls)) return KindOfClass; + if (subtypeOf(UncountedInit)) return KindOfUncountedInit; + if (subtypeOf(Uncounted)) return KindOfUncounted; + if (subtypeOf(Gen)) return KindOfAny; + not_reached(); +} + +RuntimeType Type::toRuntimeType() const { + assert(!isPtr()); + if (isBoxed()) { + return RuntimeType(KindOfRef, innerType().toDataType()); + } + return RuntimeType(toDataType()); +} + +Type Type::fromDataType(DataType outerType, + DataType innerType /* = KindOfInvalid */, + const Class* klass /* = nullptr */) { + assert(innerType != KindOfRef); + + switch (outerType) { + case KindOfInvalid : return None; + case KindOfUninit : return Uninit; + case KindOfNull : return InitNull; + case KindOfBoolean : return Bool; + case KindOfInt64 : return Int; + case KindOfDouble : return Dbl; + case KindOfStaticString : return StaticStr; + case KindOfString : return Str; + case KindOfArray : return Arr; + case KindOfObject : { + if (klass != nullptr) { + return Obj.specialize(klass); + } else { + return Obj; + } + } + case KindOfClass : return Cls; + case KindOfUncountedInit : return UncountedInit; + case KindOfUncounted : return Uncounted; + case KindOfAny : return Gen; + case KindOfRef: { + if (innerType == KindOfInvalid) { + return BoxedCell; + } else { + return fromDataType(innerType).box(); + } + } + default : not_reached(); + } +} + +Type Type::fromRuntimeType(const RuntimeType& rtt) { + return fromDataType(rtt.outerType(), rtt.innerType(), rtt.knownClass()); +} + Type Type::fromDynLocation(const Transl::DynLocation* dynLoc) { if (!dynLoc) { return Type::None; diff --git a/hphp/runtime/vm/jit/type.h b/hphp/runtime/vm/jit/type.h index 93c01c51a..7e2ec1fa7 100644 --- a/hphp/runtime/vm/jit/type.h +++ b/hphp/runtime/vm/jit/type.h @@ -25,6 +25,8 @@ struct DynLocation; } namespace JIT { +using Transl::RuntimeType; + #define IRT_BOXES(name, bit) \ IRT(name, (bit)) \ IRT(Boxed##name, (bit) << kBoxShift) \ @@ -351,6 +353,10 @@ public: return subtypeOf(Str); } + bool isCls() const { + return subtypeOf(Cls); + } + const Class* getClass() const { assert(isObj()); return m_class; @@ -436,68 +442,14 @@ public: != Type::Bottom; } - // translates a compiler Type to an HPHP::DataType - DataType toDataType() const { - assert(!isPtr()); - if (isBoxed()) { - return KindOfRef; - } - - // Order is important here: types must progress from more specific - // to less specific to return the most specific DataType. - if (subtypeOf(None)) return KindOfInvalid; - if (subtypeOf(Uninit)) return KindOfUninit; - if (subtypeOf(Null)) return KindOfNull; - if (subtypeOf(Bool)) return KindOfBoolean; - if (subtypeOf(Int)) return KindOfInt64; - if (subtypeOf(Dbl)) return KindOfDouble; - if (subtypeOf(StaticStr)) return KindOfStaticString; - if (subtypeOf(Str)) return KindOfString; - if (subtypeOf(Arr)) return KindOfArray; - if (subtypeOf(Obj)) return KindOfObject; - if (subtypeOf(Cls)) return KindOfClass; - if (subtypeOf(UncountedInit)) return KindOfUncountedInit; - if (subtypeOf(Uncounted)) return KindOfUncounted; - if (subtypeOf(Gen)) return KindOfAny; - not_reached(); - } + DataType toDataType() const; + RuntimeType toRuntimeType() const; static Type fromDataType(DataType outerType, DataType innerType = KindOfInvalid, - const Class* klass = nullptr) { - assert(innerType != KindOfRef); - - switch (outerType) { - case KindOfInvalid : return None; - case KindOfUninit : return Uninit; - case KindOfNull : return InitNull; - case KindOfBoolean : return Bool; - case KindOfInt64 : return Int; - case KindOfDouble : return Dbl; - case KindOfStaticString : return StaticStr; - case KindOfString : return Str; - case KindOfArray : return Arr; - case KindOfObject : { - if (klass != nullptr) { - return Obj.specialize(klass); - } else { - return Obj; - } - } - case KindOfClass : return Cls; - case KindOfUncountedInit : return UncountedInit; - case KindOfUncounted : return Uncounted; - case KindOfAny : return Gen; - case KindOfRef: { - if (innerType == KindOfInvalid) { - return BoxedCell; - } else { - return fromDataType(innerType).box(); - } - } - default : not_reached(); - } - } + const Class* klass = nullptr); + static Type fromRuntimeType(const Transl::RuntimeType& rtt); + static Type fromDynLocation(const Transl::DynLocation* dynLoc); // return true if this corresponds to a type that // is passed by value in C++ @@ -530,11 +482,6 @@ public: return isRef ? t.box() : t; } - static Type fromRuntimeType(const Transl::RuntimeType& rtt) { - return fromDataType(rtt.outerType(), rtt.innerType(), rtt.knownClass()); - } - - static Type fromDynLocation(const Transl::DynLocation* dynLoc); }; static_assert(sizeof(Type) <= 2 * sizeof(uint64_t),