wellton/hhvm
1
0
Fork 0
Código Issues Pull Requests Ações Pacotes Projetos Versões Wiki Atividade
Arquivos
cf87b7eec3b7aad20cbb0eab68e1edb8856023d9
hhvm/hphp/runtime/vm/jit/codegen.cpp
T
bsimmers cf87b7eec3 Basic region translator 
This diff adds a very, very basic region translator. Together with
-vEval.JitRegionSelector=method, it is capable of translating very simple
methods without using a Tracelet. It takes in a RegionDesc struct and creates
NormalizedInstructions on the stack to drive the translate* methods in
irtranslator.cpp. I started trying to get all tests passing with
JitRegionSelector=method (by replacing lots of asserts with punts) but decided
it's not worth it right now.

I'm planning on writing a Tracelet -> RegionDesc converter next and expanding
translateRegion to handle everything that throws at it.
2013-06-12 11:34:41 -07:00

5553 linhas
177 KiB
C++
Original Anotar Histórico

/*
+----------------------------------------------------------------------+
| HipHop for PHP |
+----------------------------------------------------------------------+
| Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com) |
+----------------------------------------------------------------------+
| This source file is subject to version 3.01 of the PHP license, |
| that is bundled with this package in the file LICENSE, and is |
| available through the world-wide-web at the following url: |
| http://www.php.net/license/3_01.txt |
| If you did not receive a copy of the PHP license and are unable to |
| obtain it through the world-wide-web, please send a note to |
| license@php.net so we can mail you a copy immediately. |
+----------------------------------------------------------------------+
*/
#include "hphp/runtime/vm/jit/codegen.h"
#include <cstring>
#include <unwind.h>
#include "folly/ScopeGuard.h"
#include "folly/Format.h"
#include "hphp/util/trace.h"
#include "hphp/util/util.h"
#include "hphp/runtime/base/array/hphp_array.h"
#include "hphp/runtime/base/comparisons.h"
#include "hphp/runtime/base/complex_types.h"
#include "hphp/runtime/base/runtime_option.h"
#include "hphp/runtime/base/string_data.h"
#include "hphp/runtime/base/types.h"
#include "hphp/runtime/ext/ext_closure.h"
#include "hphp/runtime/ext/ext_continuation.h"
#include "hphp/runtime/ext/ext_collections.h"
#include "hphp/runtime/vm/bytecode.h"
#include "hphp/runtime/vm/runtime.h"
#include "hphp/runtime/base/stats.h"
#include "hphp/runtime/vm/jit/targetcache.h"
#include "hphp/runtime/vm/jit/translator-inline.h"
#include "hphp/runtime/vm/jit/translator-x64.h"
#include "hphp/runtime/vm/jit/translator-x64-internal.h"
#include "hphp/runtime/vm/jit/translator.h"
#include "hphp/runtime/vm/jit/types.h"
#include "hphp/runtime/vm/jit/x64-util.h"
#include "hphp/runtime/vm/jit/ir.h"
#include "hphp/runtime/vm/jit/linearscan.h"
#include "hphp/runtime/vm/jit/nativecalls.h"
#include "hphp/runtime/vm/jit/print.h"
#include "hphp/runtime/vm/jit/layout.h"
using HPHP::Transl::TCA;
using namespace HPHP::Transl::TargetCache;
namespace HPHP {
namespace JIT {
namespace {
//////////////////////////////////////////////////////////////////////
using namespace Util;
using namespace Transl::reg;
TRACE_SET_MOD(hhir);
/*
* It's not normally ok to directly use tracelet abi registers in
* codegen, unless you're directly dealing with an instruction that
* does near-end-of-tracelet glue. (Or also we sometimes use them
* just for some static_assertions relating to calls to helpers from
* tx64 that hardcode these registers.)
*/
using Transl::rVmFp;
using Transl::rVmSp;
const size_t kTypeWordOffset = (offsetof(TypedValue, m_type) % 8);
const size_t kTypeShiftBits = kTypeWordOffset * CHAR_BIT;
// left shift an immediate DataType, for type, to the correct position
// within one of the registers used to pass a TypedValue by value.
uint64_t toDataTypeForCall(Type type) {
return uint64_t(type.toDataType()) << kTypeShiftBits;
}
int64_t spillSlotsToSize(int n) {
return n * sizeof(int64_t);
}
void cgPunt(const char* file, int line, const char* func, uint32_t bcOff) {
if (dumpIREnabled()) {
HPHP::Trace::trace("--------- CG_PUNT %s %d %s bcOff: %d \n",
file, line, func, bcOff);
}
throw FailedCodeGen(file, line, func, bcOff);
}
#define CG_PUNT(instr) cgPunt(__FILE__, __LINE__, #instr, m_curBcOff)
struct CycleInfo {
int node;
int length;
};
struct MoveInfo {
enum Kind { Move, Xchg };
MoveInfo(Kind kind, int reg1, int reg2):
m_kind(kind), m_reg1(reg1), m_reg2(reg2) {}
Kind m_kind;
PhysReg m_reg1, m_reg2;
};
template <int N>
static bool cycleHasXMMReg(const CycleInfo& cycle,
const int (&moves)[N]) {
int first = cycle.node;
int node = first;
do {
if (PhysReg(node).isXMM()) return true;
node = moves[node];
} while (node != first);
return false;
}
template <int N>
void doRegMoves(int (&moves)[N], int rTmp,
std::vector<MoveInfo>& howTo) {
assert(howTo.empty());
int outDegree[N];
CycleInfo cycles[N];
int numCycles = 0;
// Iterate over the nodes filling in outDegree[] and cycles[] as we go
{
int index[N];
for (int node = 0; node < N; ++node) {
// If a node's source is itself, its a nop
if (moves[node] == node) moves[node] = -1;
if (node == rTmp && moves[node] >= 0) {
// ERROR: rTmp cannot be referenced in moves[].
assert(false);
}
outDegree[node] = 0;
index[node] = -1;
}
int nextIndex = 0;
for (int startNode = 0; startNode < N; ++startNode) {
// If startNode has not been visited yet, begin walking
// a path from start node
if (index[startNode] < 0) {
int node = startNode;
pathloop:
index[node] = nextIndex++;
if (moves[node] >= 0) {
int nextNode = moves[node];
++outDegree[nextNode];
if (index[nextNode] < 0) {
// If there is an edge from v to nextNode and nextNode has not been
// visited, extend the current path to include nextNode and recurse
node = nextNode;
goto pathloop;
}
// There is an edge from v to nextNode but nextNode has already been
// visited, check if nextNode is on the current path
if (index[nextNode] >= index[startNode]) {
// nextNode is on the current path so we've found a cycle
int length = nextIndex - index[nextNode];
CycleInfo ci = { nextNode, length };
cycles[numCycles] = ci;
++numCycles;
}
}
}
}
}
// Handle all moves that aren't part of a cycle
{
int q[N];
int qBack = 0;
for (int node = 0; node < N; ++node) {
if (outDegree[node] == 0) {
q[qBack] = node;
++qBack;
}
}
for (int i = 0; i < qBack; ++i) {
int node = q[i];
if (moves[node] >= 0) {
int nextNode = moves[node];
howTo.push_back(MoveInfo(MoveInfo::Move, nextNode, node));
--outDegree[nextNode];
if (outDegree[nextNode] == 0) {
q[qBack] = nextNode;
++qBack;
}
}
}
}
// Deal with any cycles we encountered
for (int i = 0; i < numCycles; ++i) {
// can't use xchg if one of the registers is XMM
bool hasXMMReg = cycleHasXMMReg(cycles[i], moves);
if (cycles[i].length == 2 && !hasXMMReg) {
int v = cycles[i].node;
int w = moves[v];
howTo.push_back(MoveInfo(MoveInfo::Xchg, w, v));
} else if (cycles[i].length == 3 && !hasXMMReg) {
int v = cycles[i].node;
int w = moves[v];
howTo.push_back(MoveInfo(MoveInfo::Xchg, w, v));
int x = moves[w];
howTo.push_back(MoveInfo(MoveInfo::Xchg, x, w));
} else {
int v = cycles[i].node;
howTo.push_back(MoveInfo(MoveInfo::Move, v, rTmp));
int w = v;
int x = moves[w];
while (x != v) {
howTo.push_back(MoveInfo(MoveInfo::Move, x, w));
w = x;
x = moves[w];
}
howTo.push_back(MoveInfo(MoveInfo::Move, rTmp, w));
}
}
}
const char* getContextName(Class* ctx) {
return ctx ? ctx->name()->data() : ":anonymous:";
}
} // unnamed namespace
//////////////////////////////////////////////////////////////////////
ArgDesc::ArgDesc(SSATmp* tmp, const RegisterInfo& info, bool val)
: m_imm(-1), m_zeroExtend(false), m_done(false) {
if (tmp->type() == Type::None) {
assert(val);
m_kind = None;
return;
}
if (tmp->inst()->op() == DefConst) {
m_srcReg = InvalidReg;
if (val) {
m_imm = tmp->getValBits();
} else {
m_imm = toDataTypeForCall(tmp->type());
}
m_kind = Imm;
return;
}
if (tmp->type().isNull()) {
m_srcReg = InvalidReg;
if (val) {
m_imm = 0;
} else {
m_imm = toDataTypeForCall(tmp->type());
}
m_kind = Imm;
return;
}
if (val || tmp->numNeededRegs() > 1) {
auto reg = info.reg(val ? 0 : 1);
assert(reg != InvalidReg);
m_imm = 0;
// If val is false then we're passing tmp's type. TypeReg lets
// CodeGenerator know that the value might require some massaging
// to be in the right format for the call.
m_kind = val ? Reg : TypeReg;
// zero extend any boolean value that we pass to the helper in case
// the helper expects it (e.g., as TypedValue)
if (val && tmp->isA(Type::Bool)) m_zeroExtend = true;
m_srcReg = reg;
return;
}
m_srcReg = InvalidReg;
m_imm = toDataTypeForCall(tmp->type());
m_kind = Imm;
}
const Func* CodeGenerator::curFunc() const {
always_assert(m_state.lastMarker &&
"We shouldn't be looking for a func when we have no marker");
return m_state.lastMarker->func;
}
/*
* Select a scratch register to use in the given instruction, prefering the
* lower registers which don't require a REX prefix. The selected register
* must not be any of the instructions inputs or outputs, and neither a register
* that is alive across this instruction.
*/
PhysReg CodeGenerator::selectScratchReg(IRInstruction* inst) {
static const RegSet kLowGPRegs = RegSet()
| RegSet(reg::rax)
| RegSet(reg::rcx)
| RegSet(reg::rdx)
| RegSet(reg::rsi)
| RegSet(reg::rdi)
;
RegSet liveRegs = m_state.liveRegs[inst];
for (const auto& tmp : inst->srcs()) {
liveRegs |= m_regs[tmp].regs();
}
for (const auto& tmp : inst->dsts()) {
liveRegs |= m_regs[tmp].regs();
}
PhysReg selectedReg;
if ((kLowGPRegs - liveRegs).findFirst(selectedReg)) {
return selectedReg;
}
return rCgGP;
}
Address CodeGenerator::cgInst(IRInstruction* inst) {
Opcode opc = inst->op();
auto const start = m_as.code.frontier;
m_rScratch = selectScratchReg(inst);
if (inst->taken() && inst->taken()->trace()->isCatch()) {
m_state.catchTrace = inst->taken()->trace();
} else {
m_state.catchTrace = nullptr;
}
switch (opc) {
#define O(name, dsts, srcs, flags) \
case name: FTRACE(7, "cg" #name "\n"); \
cg ## name (inst); \
return m_as.code.frontier == start ? nullptr : start;
IR_OPCODES
#undef O
default:
assert(0);
return nullptr;
}
}
#define NOOP_OPCODE(opcode) \
void CodeGenerator::cg##opcode(IRInstruction*) {}
#define CALL_OPCODE(opcode) \
void CodeGenerator::cg##opcode(IRInstruction* i) { cgCallNative(i); }
#define CALL_STK_OPCODE(opcode) \
CALL_OPCODE(opcode) \
CALL_OPCODE(opcode ## Stk)
NOOP_OPCODE(DefConst)
NOOP_OPCODE(DefFP)
NOOP_OPCODE(DefSP)
NOOP_OPCODE(AssertLoc)
NOOP_OPCODE(OverrideLoc)
NOOP_OPCODE(AssertStk)
NOOP_OPCODE(AssertStkVal)
NOOP_OPCODE(Nop)
NOOP_OPCODE(DefLabel)
NOOP_OPCODE(ExceptionBarrier)
CALL_OPCODE(AddElemStrKey)
CALL_OPCODE(AddElemIntKey)
CALL_OPCODE(AddNewElem)
CALL_OPCODE(ArrayAdd)
CALL_OPCODE(Box)
CALL_OPCODE(ConvBoolToArr);
CALL_OPCODE(ConvDblToArr);
CALL_OPCODE(ConvIntToArr);
CALL_OPCODE(ConvObjToArr);
CALL_OPCODE(ConvStrToArr);
CALL_OPCODE(ConvCellToArr);
CALL_OPCODE(ConvArrToBool);
CALL_OPCODE(ConvStrToBool);
CALL_OPCODE(ConvCellToBool);
CALL_OPCODE(ConvArrToDbl);
CALL_OPCODE(ConvObjToDbl);
CALL_OPCODE(ConvStrToDbl);
CALL_OPCODE(ConvCellToDbl);
CALL_OPCODE(ConvArrToInt);
CALL_OPCODE(ConvDblToInt);
CALL_OPCODE(ConvObjToInt);
CALL_OPCODE(ConvStrToInt);
CALL_OPCODE(ConvCellToInt);
CALL_OPCODE(ConvCellToObj);
CALL_OPCODE(ConvDblToStr);
CALL_OPCODE(ConvIntToStr);
CALL_OPCODE(ConvObjToStr);
CALL_OPCODE(ConvCellToStr);
CALL_OPCODE(CreateCont)
CALL_OPCODE(FillContLocals)
CALL_OPCODE(NewArray)
CALL_OPCODE(NewTuple)
CALL_OPCODE(AllocObj)
CALL_OPCODE(LdClsCtor);
CALL_OPCODE(CreateCl)
CALL_OPCODE(PrintStr)
CALL_OPCODE(PrintInt)
CALL_OPCODE(PrintBool)
CALL_OPCODE(DbgAssertPtr)
CALL_OPCODE(LdSwitchDblIndex)
CALL_OPCODE(LdSwitchStrIndex)
CALL_OPCODE(LdSwitchObjIndex)
CALL_OPCODE(VerifyParamCallable)
CALL_OPCODE(VerifyParamFail)
CALL_OPCODE(RaiseUninitLoc)
CALL_OPCODE(WarnNonObjProp)
CALL_OPCODE(ThrowNonObjProp)
CALL_OPCODE(RaiseUndefProp)
CALL_OPCODE(RaiseError)
CALL_OPCODE(RaiseWarning)
CALL_OPCODE(IncStatGrouped)
CALL_OPCODE(StaticLocInit)
CALL_OPCODE(StaticLocInitCached)
CALL_OPCODE(OpMod)
CALL_OPCODE(ArrayIdx)
// Vector instruction helpers
CALL_OPCODE(BaseG)
CALL_OPCODE(PropX)
CALL_STK_OPCODE(PropDX)
CALL_OPCODE(CGetProp)
CALL_STK_OPCODE(VGetProp)
CALL_STK_OPCODE(BindProp)
CALL_STK_OPCODE(SetProp)
CALL_OPCODE(UnsetProp)
CALL_STK_OPCODE(SetOpProp)
CALL_STK_OPCODE(IncDecProp)
CALL_OPCODE(EmptyProp)
CALL_OPCODE(IssetProp)
CALL_OPCODE(ElemX)
CALL_STK_OPCODE(ElemDX)
CALL_STK_OPCODE(ElemUX)
CALL_OPCODE(ArrayGet)
CALL_OPCODE(VectorGet)
CALL_OPCODE(MapGet)
CALL_OPCODE(CGetElem)
CALL_STK_OPCODE(VGetElem)
CALL_STK_OPCODE(BindElem)
CALL_STK_OPCODE(SetWithRefElem)
CALL_STK_OPCODE(SetWithRefNewElem)
CALL_OPCODE(ArraySet)
CALL_OPCODE(VectorSet)
CALL_OPCODE(MapSet)
CALL_OPCODE(ArraySetRef)
CALL_STK_OPCODE(SetElem)
CALL_STK_OPCODE(UnsetElem)
CALL_STK_OPCODE(SetOpElem)
CALL_STK_OPCODE(IncDecElem)
CALL_STK_OPCODE(SetNewElem)
CALL_STK_OPCODE(BindNewElem)
CALL_OPCODE(ArrayIsset)
CALL_OPCODE(VectorIsset)
CALL_OPCODE(MapIsset)
CALL_OPCODE(IssetElem)
CALL_OPCODE(EmptyElem)
#undef NOOP_OPCODE
// Thread chain of patch locations using the 4 byte space in each jmp/jcc
static void prependPatchAddr(CodegenState& state,
Block* block,
TCA patchAddr) {
auto &patches = state.patches;
ssize_t diff = patches[block] ? (patchAddr - (TCA)patches[block]) : 0;
assert(deltaFits(diff, sz::dword));
*(int32_t*)(patchAddr) = (int32_t)diff;
patches[block] = patchAddr;
}
static void emitFwdJmp(Asm& a, Block* target, CodegenState& state) {
if (auto addr = state.addresses[target]) {
return a.jmpAuto(addr);
}
// TODO(#2101926): it'd be nice to get 1-byte forward jumps here
a.jmp(a.code.frontier);
TCA immPtr = a.code.frontier - 4;
prependPatchAddr(state, target, immPtr);
}
void CodeGenerator::emitFwdJcc(Asm& a, ConditionCode cc, Block* target) {
if (auto addr = m_state.addresses[target]) {
return a.jccAuto(cc, addr);
}
// TODO(#2101926): it'd be nice to get 1-byte forward jumps here
a.jcc(cc, a.code.frontier);
TCA immPtr = a.code.frontier - 4;
prependPatchAddr(m_state, target, immPtr);
}
void CodeGenerator::emitFwdJcc(ConditionCode cc, Block* target) {
emitFwdJcc(m_as, cc, target);
}
void emitLoadImm(CodeGenerator::Asm& as, int64_t val, PhysReg dstReg) {
as.emitImmReg(val, dstReg);
}
static void
emitMovRegReg(CodeGenerator::Asm& as, PhysReg srcReg, PhysReg dstReg) {
assert(srcReg != InvalidReg);
assert(dstReg != InvalidReg);
if (srcReg == dstReg) return;
if (srcReg.isGP()) {
if (dstReg.isGP()) { // GP => GP
as.movq(srcReg, dstReg);
} else { // GP => XMM
// This generates a movq x86 instruction, which zero extends
// the 64-bit value in srcReg into a 128-bit XMM register
as.mov_reg64_xmm(srcReg, dstReg);
}
} else {
if (dstReg.isGP()) { // XMM => GP
as.mov_xmm_reg64(srcReg, dstReg);
} else { // XMM => XMM
// This copies all 128 bits in XMM,
// thus avoiding partial register stalls
as.movdqa(srcReg, dstReg);
}
}
}
void CodeGenerator::emitLoadImm(CodeGenerator::Asm& as, int64_t val,
PhysReg dstReg) {
assert(dstReg != InvalidReg);
if (dstReg.isGP()) {
as.emitImmReg(val, dstReg);
} else {
assert(dstReg.isXMM());
if (val == 0) {
as.pxor_xmm_xmm(dstReg, dstReg);
} else {
// Can't move immediate directly into XMM register, so use m_rScratch
as.emitImmReg(val, m_rScratch);
emitMovRegReg(as, m_rScratch, dstReg);
}
}
}
static void emitLea(CodeGenerator::Asm& as, MemoryRef mr, PhysReg dst) {
if (dst == InvalidReg) return;
if (mr.r.disp == 0) {
emitMovRegReg(as, mr.r.base, dst);
} else {
as.lea(mr, dst);
}
}
template<class Mem>
static void emitLoadReg(CodeGenerator::Asm& as, Mem mem, PhysReg reg) {
assert(reg != InvalidReg);
if (reg.isGP()) {
as.loadq(mem, reg);
} else {
as.movsd(mem, reg);
}
}
template<class Mem>
static void emitStoreReg(CodeGenerator::Asm& as, PhysReg reg, Mem mem) {
assert(reg != InvalidReg);
if (reg.isGP()) {
as.storeq(reg, mem);
} else {
as.movsd(reg, mem);
}
}
static void shuffle2(CodeGenerator::Asm& a,
PhysReg s0, PhysReg s1, PhysReg d0, PhysReg d1) {
assert(s0 != s1);
if (d0 == s1 && d1 != InvalidReg) {
assert(d0 != d1);
if (d1 == s0) {
a. xchgq (s1, s0);
} else {
a. movq (s1, d1); // save s1 first; d1 != s0
a. movq (s0, d0);
}
} else {
if (d0 != InvalidReg) emitMovRegReg(a, s0, d0); // d0 != s1
if (d1 != InvalidReg) emitMovRegReg(a, s1, d1);
}
}
static void zeroExtendBool(X64Assembler& as, const RegisterInfo& info) {
auto reg = info.reg();
if (reg != InvalidReg) {
// zero-extend the bool from a byte to a quad
// note: movzbl actually extends the value to 64 bits.
as.movzbl(rbyte(reg), r32(reg));
}
}
static void zeroExtendIfBool(X64Assembler& as, const SSATmp* src,
const RegisterInfo& info) {
if (src->isA(Type::Bool)) {
zeroExtendBool(as, info);
}
}
static int64_t convIntToDouble(int64_t i) {
union {
double d;
int64_t i;
} u;
u.d = double(i);
return u.i;
}
/*
* Returns a XMM register containing the value of SSATmp tmp,
* which can be either a bool, an int, or a double.
* If the value is already in a XMM register, simply returns it.
* Otherwise, the value is moved into rCgXMM, which is returned.
* If instructions to convert to a double at runtime are needed,
* they're emitted in 'as'.
*/
PhysReg CodeGenerator::prepXMMReg(const SSATmp* tmp,
X64Assembler& as,
const RegAllocInfo& allocInfo,
RegXMM rCgXMM) {
assert(tmp->isA(Type::Bool) || tmp->isA(Type::Int) || tmp->isA(Type::Dbl));
PhysReg reg = allocInfo[tmp].reg();
// Case 1: tmp is already in a XMM register
if (reg.isXMM()) return reg;
// Case 2: tmp is in a GP register
if (reg != InvalidReg) {
// Case 2.a: Dbl stored in GP reg
if (tmp->isA(Type::Dbl)) {
emitMovRegReg(as, reg, rCgXMM);
return rCgXMM;
}
// Case 2.b: Bool or Int stored in GP reg
assert(tmp->isA(Type::Bool) || tmp->isA(Type::Int));
zeroExtendIfBool(as, tmp, allocInfo[tmp]);
as.pxor_xmm_xmm(rCgXMM, rCgXMM);
as.cvtsi2sd_reg64_xmm(reg, rCgXMM);
return rCgXMM;
}
// Case 3: tmp is a constant
assert(tmp->isConst());
int64_t val = tmp->getValRawInt();
if (!tmp->isA(Type::Dbl)) {
assert(tmp->isA(Type::Bool | Type::Int));
if (tmp->isA(Type::Bool)) val = val != 0; // see task #2401790
val = convIntToDouble(val);
}
emitLoadImm(as, val, m_rScratch);
emitMovRegReg(as, m_rScratch, rCgXMM);
return rCgXMM;
}
void CodeGenerator::doubleCmp(X64Assembler& a, RegXMM xmmReg0, RegXMM xmmReg1) {
a. ucomisd_xmm_xmm(xmmReg0, xmmReg1);
Label notPF;
a. jnp8(notPF);
// PF means the doubles were unordered. We treat this as !equal, so
// clear ZF.
a. or_imm32_reg64(1, m_rScratch);
asm_label(a, notPF);
}
static ConditionCode opToConditionCode(Opcode opc) {
using namespace HPHP::Transl;
switch (opc) {
case JmpGt: return CC_G;
case JmpGte: return CC_GE;
case JmpLt: return CC_L;
case JmpLte: return CC_LE;
case JmpEq: return CC_E;
case JmpNeq: return CC_NE;
case JmpSame: return CC_E;
case JmpNSame: return CC_NE;
case JmpInstanceOfBitmask: return CC_NZ;
case JmpNInstanceOfBitmask: return CC_Z;
case JmpIsType: return CC_NZ;
case JmpIsNType: return CC_Z;
case JmpZero: return CC_Z;
case JmpNZero: return CC_NZ;
case ReqBindJmpGt: return CC_G;
case ReqBindJmpGte: return CC_GE;
case ReqBindJmpLt: return CC_L;
case ReqBindJmpLte: return CC_LE;
case ReqBindJmpEq: return CC_E;
case ReqBindJmpNeq: return CC_NE;
case ReqBindJmpSame: return CC_E;
case ReqBindJmpNSame: return CC_NE;
case ReqBindJmpInstanceOfBitmask: return CC_NZ;
case ReqBindJmpNInstanceOfBitmask: return CC_Z;
case ReqBindJmpZero: return CC_Z;
case ReqBindJmpNZero: return CC_NZ;
default:
always_assert(0);
}
}
void CodeGenerator::emitCompare(SSATmp* src1, SSATmp* src2) {
auto const src1Type = src1->type();
auto const src2Type = src2->type();
// can't generate CMP instructions correctly for anything that isn't
// a bool or a numeric, and we can't mix bool/numerics because
// -1 == true in PHP, but not in HHIR binary representation
if (!((src1Type == Type::Int && src2Type == Type::Int) ||
((src1Type == Type::Int || src1Type == Type::Dbl) &&
(src2Type == Type::Int || src2Type == Type::Dbl)) ||
(src1Type == Type::Bool && src2Type == Type::Bool) ||
(src1Type == Type::Cls && src2Type == Type::Cls))) {
CG_PUNT(emitCompare);
}
if (src1Type == Type::Dbl || src2Type == Type::Dbl) {
PhysReg srcReg1 = prepXMMReg(src1, m_as, m_regs, rCgXMM0);
PhysReg srcReg2 = prepXMMReg(src2, m_as, m_regs, rCgXMM1);
assert(srcReg1 != rCgXMM1 && srcReg2 != rCgXMM0);
doubleCmp(m_as, srcReg1, srcReg2);
} else {
auto srcReg1 = m_regs[src1].reg();
auto srcReg2 = m_regs[src2].reg();
// Note: when both src1 and src2 are constants, we should transform the
// branch into an unconditional jump earlier in the IR.
if (src1->isConst()) {
// TODO: use compare with immediate or make sure simplifier
// canonicalizes this so that constant is src2
srcReg1 = m_rScratch;
m_as. mov_imm64_reg(src1->getValRawInt(), srcReg1);
}
if (src2->isConst()) {
if (src1Type.subtypeOf(Type::Bool)) {
m_as. cmpb (src2->getValRawInt(), Reg8(int(srcReg1)));
} else {
m_as. cmp_imm64_reg64(src2->getValRawInt(), srcReg1);
}
} else {
// Note the reverse syntax in the assembler.
// This cmp will compute srcReg1 - srcReg2
if (src1Type.subtypeOf(Type::Bool)) {
m_as. cmpb (Reg8(int(srcReg2)), Reg8(int(srcReg1)));
} else {
m_as. cmp_reg64_reg64(srcReg2, srcReg1);
}
}
}
}
void CodeGenerator::emitReqBindJcc(ConditionCode cc,
const ReqBindJccData* extra) {
auto& a = m_as;
assert(&m_as != &m_astubs &&
"ReqBindJcc only makes sense outside of astubs");
prepareForTestAndSmash(a, 0, kAlignJccAndJmp);
auto const patchAddr = a.code.frontier;
auto const jccStub = m_astubs.code.frontier;
{
auto& a = m_astubs;
// TODO(#2404398): move the setcc into the generic stub code so we
// don't need SRFlags::Persistent.
a. setcc (cc, rbyte(serviceReqArgRegs[4]));
m_tx64->emitServiceReq(
SRFlags::Persistent,
REQ_BIND_JMPCC_FIRST,
4ull,
patchAddr,
uint64_t(extra->taken),
uint64_t(extra->notTaken),
uint64_t(cc)
);
}
a. jcc (cc, jccStub);
a. jmp (jccStub);
}
void CodeGenerator::cgAssertNonNull(IRInstruction* inst) {
auto srcReg = m_regs[inst->src(0)].reg();
auto dstReg = m_regs[inst->dst()].reg();
if (RuntimeOption::EvalHHIRGenerateAsserts) {
Label nonNull;
m_as.testq (srcReg, srcReg);
m_as.jne8 (nonNull);
m_as.ud2();
asm_label(m_as, nonNull);
}
emitMovRegReg(m_as, srcReg, dstReg);
}
void CodeGenerator::cgLdUnwinderValue(IRInstruction* inst) {
cgLoad(rVmTl, TargetCache::kUnwinderTvOff, inst);
}
void CodeGenerator::cgBeginCatch(IRInstruction* inst) {
auto const& info = m_state.catches[inst->block()];
assert(info.afterCall);
m_tx64->registerCatchTrace(info.afterCall, m_as.code.frontier);
Stats::emitInc(m_as, Stats::TC_CatchTrace);
// We want to restore state as though the call had completed
// successfully, so skip over any stack arguments and pop any
// saved registers.
if (info.rspOffset) {
m_as.addq(info.rspOffset, rsp);
}
PhysRegSaverParity::emitPops(m_as, info.savedRegs);
}
void CodeGenerator::cgEndCatch(IRInstruction* inst) {
m_as.cmpb (0, rVmTl[TargetCache::kUnwinderSideExitOff]);
unlikelyIfBlock(CC_E,
[&](Asm& as) { // doSideExit == false, so call _Unwind_Resume
as.loadq(rVmTl[TargetCache::kUnwinderScratchOff], rdi);
as.call ((TCA)_Unwind_Resume); // pass control back to the unwinder
as.ud2();
});
// doSideExit == true, so fall through to the side exit code
Stats::emitInc(m_as, Stats::TC_CatchSideExit);
}
void CodeGenerator::cgDeleteUnwinderException(IRInstruction* inst) {
m_as.loadq(rVmTl[TargetCache::kUnwinderScratchOff], rdi);
m_as.call ((TCA)_Unwind_DeleteException);
}
void CodeGenerator::cgJcc(IRInstruction* inst) {
emitCompare(inst->src(0), inst->src(1));
emitFwdJcc(opToConditionCode(inst->op()), inst->taken());
}
void CodeGenerator::cgReqBindJcc(IRInstruction* inst) {
// TODO(#2404427): prepareForTestAndSmash?
emitCompare(inst->src(0), inst->src(1));
emitReqBindJcc(opToConditionCode(inst->op()),
inst->extra<ReqBindJccData>());
}
#define X(x) \
void CodeGenerator::cgReqBind##x(IRInstruction* i) { cgReqBindJcc(i); } \
void CodeGenerator::cg##x (IRInstruction* i) { cgJcc(i); }
X(JmpGt);
X(JmpGte);
X(JmpLt);
X(JmpLte);
X(JmpEq);
X(JmpNeq);
X(JmpSame);
X(JmpNSame);
#undef X
/**
* Once the arg sources and dests are all assigned; emit moves and exchanges to
* put all the args in desired registers. Any arguments that don't fit in
* registers will be put on the stack. In addition to moves and exchanges,
* shuffleArgs also handles adding lea-offsets for dest registers (dest = src +
* lea-offset) and zero extending bools (dest = zeroExtend(src)).
*/
typedef Transl::X64Assembler Asm;
static int64_t shuffleArgs(Asm& a, ArgGroup& args) {
// Compute the move/shuffle plan.
int moves[kNumRegs];
ArgDesc* argDescs[kNumRegs];
memset(moves, -1, sizeof moves);
memset(argDescs, 0, sizeof argDescs);
for (size_t i = 0; i < args.numRegArgs(); ++i) {
auto kind = args[i].kind();
if (!(kind == ArgDesc::Reg ||
kind == ArgDesc::Addr ||
kind == ArgDesc::TypeReg)) {
continue;
}
auto dstReg = args[i].dstReg();
auto srcReg = args[i].srcReg();
if (dstReg != srcReg) {
moves[int(dstReg)] = int(srcReg);
argDescs[int(dstReg)] = &args[i];
}
}
std::vector<MoveInfo> howTo;
doRegMoves(moves, int(rCgGP), howTo);
// Execute the plan
for (size_t i = 0; i < howTo.size(); ++i) {
if (howTo[i].m_kind == MoveInfo::Move) {
if (howTo[i].m_reg2 == rCgGP) {
emitMovRegReg(a, howTo[i].m_reg1, howTo[i].m_reg2);
} else {
ArgDesc* argDesc = argDescs[int(howTo[i].m_reg2)];
ArgDesc::Kind kind = argDesc->kind();
if (kind == ArgDesc::Reg || kind == ArgDesc::TypeReg) {
if (argDesc->isZeroExtend()) {
assert(howTo[i].m_reg1.isGP());
assert(howTo[i].m_reg2.isGP());
a. movzbl (rbyte(howTo[i].m_reg1), r32(howTo[i].m_reg2));
} else {
emitMovRegReg(a, howTo[i].m_reg1, howTo[i].m_reg2);
}
} else {
assert(kind == ArgDesc::Addr);
assert(howTo[i].m_reg1.isGP());
assert(howTo[i].m_reg2.isGP());
a. lea (howTo[i].m_reg1[argDesc->imm().q()],
howTo[i].m_reg2);
}
if (kind != ArgDesc::TypeReg) {
argDesc->markDone();
}
}
} else {
assert(howTo[i].m_reg1.isGP());
assert(howTo[i].m_reg2.isGP());
a. xchgq (howTo[i].m_reg1, howTo[i].m_reg2);
}
}
// Handle const-to-register moves, type shifting,
// load-effective address and zero extending for bools.
// Ignore args that have been handled by the
// move above.
for (size_t i = 0; i < args.numRegArgs(); ++i) {
if (!args[i].done()) {
ArgDesc::Kind kind = args[i].kind();
PhysReg dst = args[i].dstReg();
assert(dst.isGP());
if (kind == ArgDesc::Imm) {
a.emitImmReg(args[i].imm().q(), dst);
} else if (kind == ArgDesc::TypeReg) {
a. shlq (kTypeShiftBits, dst);
} else if (kind == ArgDesc::Addr) {
a. addq (args[i].imm(), dst);
} else if (args[i].isZeroExtend()) {
a. movzbl (rbyte(dst), r32(dst));
} else if (RuntimeOption::EvalHHIRGenerateAsserts &&
kind == ArgDesc::None) {
a.emitImmReg(0xbadbadbadbadbad, dst);
}
}
}
// Store any remaining arguments to the stack
for (int i = args.numStackArgs() - 1; i >= 0; --i) {
auto& arg = args.stk(i);
auto srcReg = arg.srcReg();
assert(arg.dstReg() == InvalidReg);
switch (arg.kind()) {
case ArgDesc::Reg:
if (arg.isZeroExtend()) {
a. movzbl(rbyte(srcReg), r32(rCgGP));
a. push(rCgGP);
} else {
if (srcReg.isXMM()) {
emitMovRegReg(a, srcReg, rCgGP);
a.push(rCgGP);
} else {
a.push(srcReg);
}
}
break;
case ArgDesc::TypeReg:
static_assert(kTypeWordOffset == 4 || kTypeWordOffset == 1,
"kTypeWordOffset value not supported");
assert(srcReg.isGP());
// x86 stacks grow down, so push higher offset items first
if (kTypeWordOffset == 4) {
a. pushl(r32(srcReg));
// 4 bytes of garbage:
a. pushl(eax);
} else {
// 4 bytes of garbage:
a. pushl(eax);
// get the type in the right place in rCgGP before pushing it
a. movb (rbyte(srcReg), rbyte(rCgGP));
a. shll (CHAR_BIT, r32(rCgGP));
a. pushl(r32(rCgGP));
}
break;
case ArgDesc::Imm:
a. emitImmReg(arg.imm(), rCgGP);
a. push(rCgGP);
break;
case ArgDesc::Addr:
not_implemented();
case ArgDesc::None:
a. push(rax);
if (RuntimeOption::EvalHHIRGenerateAsserts) {
a. storeq(0xbadbadbadbadbad, *rsp);
}
break;
}
}
return args.numStackArgs() * sizeof(int64_t);
}
void CodeGenerator::cgCallNative(Asm& a, IRInstruction* inst) {
using namespace NativeCalls;
Opcode opc = inst->op();
always_assert(CallMap::hasInfo(opc));
const CallInfo& info = CallMap::info(opc);
ArgGroup argGroup(m_regs);
for (auto const& arg : info.args) {
SSATmp* src = inst->src(arg.srcIdx);
switch (arg.type) {
case SSA:
argGroup.ssa(src);
break;
case TV:
argGroup.typedValue(src);
break;
case VecKeyS:
argGroup.vectorKeyS(src);
break;
case VecKeyIS:
argGroup.vectorKeyIS(src);
break;
case Immed:
always_assert(0 && "We can't generate a native call for this");
break;
}
}
TCA addr = nullptr;
switch (info.func.type) {
case FPtr:
addr = info.func.ptr;
break;
case FSSA:
addr = inst->src(info.func.srcIdx)->getValTCA();
break;
}
cgCallHelper(a,
addr,
info.dest != DestType::None ? inst->dst(0) : nullptr,
info.sync,
argGroup,
info.dest);
}
void CodeGenerator::cgCallHelper(Asm& a,
TCA addr,
SSATmp* dst,
SyncOptions sync,
ArgGroup& args,
DestType destType) {
PhysReg dstReg0 = InvalidReg;
PhysReg dstReg1 = InvalidReg;
if (dst) {
auto &info = m_regs[dst];
dstReg0 = info.reg(0);
dstReg1 = info.reg(1);
}
return cgCallHelper(a, Transl::CppCall(addr), dstReg0, dstReg1, sync, args,
destType);
}
void CodeGenerator::cgCallHelper(Asm& a,
TCA addr,
PhysReg dstReg,
SyncOptions sync,
ArgGroup& args,
DestType destType) {
cgCallHelper(a, Transl::CppCall(addr), dstReg, InvalidReg, sync, args,
destType);
}
void CodeGenerator::cgCallHelper(Asm& a,
const Transl::CppCall& call,
PhysReg dstReg,
SyncOptions sync,
ArgGroup& args,
DestType destType) {
cgCallHelper(a, call, dstReg, InvalidReg, sync, args, destType);
}
void CodeGenerator::cgCallHelper(Asm& a,
const Transl::CppCall& call,
PhysReg dstReg0,
PhysReg dstReg1,
SyncOptions sync,
ArgGroup& args,
DestType destType) {
cgCallHelper(a, call, dstReg0, dstReg1, sync, args,
m_state.liveRegs[m_curInst], destType);
}
void CodeGenerator::cgCallHelper(Asm& a,
const Transl::CppCall& call,
PhysReg dstReg0,
PhysReg dstReg1,
SyncOptions sync,
ArgGroup& args,
RegSet toSave,
DestType destType) {
assert(m_curInst->isNative());
// Save the caller-saved registers that are live across this
// instruction. The number of regs to save and the number of args
// being passed on the stack affect the parity of the PhysRegSaver,
// so we use the generic version here.
toSave = toSave & kCallerSaved;
assert((toSave & RegSet().add(dstReg0).add(dstReg1)).empty());
PhysRegSaverParity regSaver(1 + args.numStackArgs(), a, toSave);
// Assign registers to the arguments then prepare them for the call.
for (size_t i = 0; i < args.numRegArgs(); i++) {
args[i].setDstReg(argNumToRegName[i]);
}
regSaver.bytesPushed(shuffleArgs(a, args));
// do the call; may use a trampoline
m_tx64->emitCall(a, call);
if (sync != kNoSyncPoint) {
recordSyncPoint(a, sync);
}
if (m_state.catchTrace) {
auto& info = m_state.catches[m_state.catchTrace->front()];
assert(!info.afterCall);
info.afterCall = a.code.frontier;
info.savedRegs = toSave;
info.rspOffset = regSaver.rspAdjustment();
}
// copy the call result to the destination register(s)
if (destType == DestType::TV) {
// rax contains m_type and m_aux but we're expecting just the
// type in the lower bits, so shift the type result register.
auto rval = packed_tv ? reg::rdx : reg::rax;
auto rtyp = packed_tv ? reg::rax : reg::rdx;
if (kTypeShiftBits > 0) a.shrq(kTypeShiftBits, rtyp);
shuffle2(a, rval, rtyp, dstReg0, dstReg1);
} else if (destType == DestType::SSA) {
// copy the single-register result to dstReg0
assert(dstReg1 == InvalidReg);
if (dstReg0 != InvalidReg) emitMovRegReg(a, reg::rax, dstReg0);
} else {
// void return type, no registers have values
assert(dstReg0 == InvalidReg && dstReg1 == InvalidReg);
}
}
/*
* This doesn't really produce any code; it just keeps track of the current
* bytecode offset.
*/
void CodeGenerator::cgMarker(IRInstruction* inst) {
m_curBcOff = inst->extra<MarkerData>()->bcOff;
}
void CodeGenerator::cgMov(IRInstruction* inst) {
assert(!m_regs[inst->src(0)].hasReg(1));//TODO: t2082361: handle Gen & Cell
SSATmp* dst = inst->dst();
SSATmp* src = inst->src(0);
auto dstReg = m_regs[dst].reg();
if (!m_regs[src].hasReg(0)) {
assert(src->isConst());
if (src->type() == Type::Bool) {
emitLoadImm(m_as, (int64_t)src->getValBool(), dstReg);
} else {
emitLoadImm(m_as, src->getValRawInt(), dstReg);
}
} else {
auto srcReg = m_regs[src].reg();
emitMovRegReg(m_as, srcReg, dstReg);
}
}
template<class OpInstr, class Oper>
void CodeGenerator::cgUnaryIntOp(SSATmp* dst,
SSATmp* src,
OpInstr instr,
Oper oper) {
if (src->type() != Type::Int && src->type() != Type::Bool) {
assert(0); CG_PUNT(UnaryIntOp);
}
auto dstReg = m_regs[dst].reg();
auto srcReg = m_regs[src].reg();
assert(dstReg != InvalidReg);
auto& a = m_as;
// Integer operations require 64-bit representations
zeroExtendIfBool(a, src, m_regs[src]);
if (srcReg != InvalidReg) {
emitMovRegReg(a, srcReg, dstReg);
(a.*instr) (dstReg);
} else {
assert(src->isConst());
emitLoadImm(a, oper(src->getValRawInt()), dstReg);
}
}
void CodeGenerator::cgNegateWork(SSATmp* dst, SSATmp* src) {
cgUnaryIntOp(dst, src, &Asm::neg, [](int64_t i) { return -i; });
}
inline static Reg8 convertToReg8(PhysReg reg) { return rbyte(reg); }
inline static Reg64 convertToReg64(PhysReg reg) { return reg; }
template<class Oper, class RegType>
void CodeGenerator::cgBinaryIntOp(IRInstruction* inst,
void (Asm::*instrIR)(Immed, RegType),
void (Asm::*instrRR)(RegType, RegType),
void (Asm::*movInstr)(RegType, RegType),
Oper oper,
RegType (*convertReg)(PhysReg),
Commutativity commuteFlag) {
const SSATmp* dst = inst->dst();
const SSATmp* src1 = inst->src(0);
const SSATmp* src2 = inst->src(1);
if (!(src1->isA(Type::Bool) || src1->isA(Type::Int)) ||
!(src2->isA(Type::Bool) || src2->isA(Type::Int))) {
CG_PUNT(cgBinaryIntOp);
}
bool const commutative = commuteFlag == Commutative;
auto const dstReg = m_regs[dst].reg();
auto const src1Reg = m_regs[src1].reg();
auto const src2Reg = m_regs[src2].reg();
auto& a = m_as;
auto const dstOpReg = convertReg(dstReg);
auto const src1OpReg = convertReg(src1Reg);
auto const src2OpReg = convertReg(src2Reg);
auto const rOpScratch = convertReg(m_rScratch);
// Two registers.
if (src1Reg != InvalidReg && src2Reg != InvalidReg) {
if (dstReg == src1Reg) {
(a.*instrRR) (src2OpReg, dstOpReg);
} else if (dstReg == src2Reg) {
if (commutative) {
(a.*instrRR) (src1OpReg, dstOpReg);
} else {
(a.*movInstr)(src1OpReg, rOpScratch);
(a.*instrRR) (src2OpReg, rOpScratch);
(a.*movInstr)(rOpScratch, dstOpReg);
}
} else {
emitMovRegReg(a, src1Reg, dstReg);
(a.*instrRR) (src2OpReg, dstOpReg);
}
return;
}
// Two immediates.
if (src1Reg == InvalidReg && src2Reg == InvalidReg) {
assert(src1->isConst() && src2->isConst());
int64_t value = oper(src1->getValRawInt(), src2->getValRawInt());
emitLoadImm(a, value, dstReg);
return;
}
// One register, and one immediate.
if (commutative) {
auto immedSrc = (src2Reg == InvalidReg ? src2 : src1);
auto immed = immedSrc->getValRawInt();
auto srcReg = m_regs[(src2Reg == InvalidReg ? src1 : src2)].reg();
if (srcReg == dstReg) {
(a.*instrIR) (immed, dstOpReg);
} else {
emitLoadImm(a, immed, dstReg);
(a.*instrRR) (convertReg(srcReg), dstOpReg);
}
return;
}
// NonCommutative:
if (src1Reg == InvalidReg) {
if (dstReg == src2Reg) {
emitLoadImm(a, src1->getValRawInt(), m_rScratch);
(a.*instrRR) (src2OpReg, rOpScratch);
(a.*movInstr)(rOpScratch, dstOpReg);
} else {
emitLoadImm(a, src1->getValRawInt(), dstReg);
(a.*instrRR) (src2OpReg, dstOpReg);
}
return;
}
assert(src2Reg == InvalidReg);
emitMovRegReg(a, src1Reg, dstReg);
(a.*instrIR) (src2->getValRawInt(), dstOpReg);
}
template<class Oper, class RegType>
void CodeGenerator::cgBinaryOp(IRInstruction* inst,
void (Asm::*instrIR)(Immed, RegType),
void (Asm::*instrRR)(RegType, RegType),
void (Asm::*movInstr)(RegType, RegType),
void (Asm::*fpInstr)(RegXMM, RegXMM),
Oper oper,
RegType (*convertReg)(PhysReg),
Commutativity commuteFlag) {
const SSATmp* dst = inst->dst();
const SSATmp* src1 = inst->src(0);
const SSATmp* src2 = inst->src(1);
if (!(src1->isA(Type::Bool) || src1->isA(Type::Int) || src1->isA(Type::Dbl))
||
!(src2->isA(Type::Bool) || src2->isA(Type::Int) || src2->isA(Type::Dbl)) )
{
CG_PUNT(cgBinaryOp);
}
if (src1->isA(Type::Dbl) || src2->isA(Type::Dbl)) {
PhysReg dstReg = m_regs[dst].reg();
PhysReg resReg = dstReg.isXMM() && dstReg != m_regs[src2].reg() ?
dstReg : PhysReg(rCgXMM0);
assert(resReg.isXMM());
PhysReg srcReg1 = prepXMMReg(src1, m_as, m_regs, resReg);
PhysReg srcReg2 = prepXMMReg(src2, m_as, m_regs, rCgXMM1);
assert(srcReg1 != rCgXMM1 && srcReg2 != rCgXMM0);
emitMovRegReg(m_as, srcReg1, resReg);
(m_as.*fpInstr)(srcReg2, resReg);
emitMovRegReg(m_as, resReg, dstReg);
return;
}
cgBinaryIntOp(inst, instrIR, instrRR, movInstr,
oper, convertReg, commuteFlag);
}
bool CodeGenerator::emitIncDecHelper(SSATmp* dst, SSATmp* src1, SSATmp* src2,
void(Asm::*emitFunc)(Reg64)) {
if (m_regs[src1].reg() != InvalidReg &&
m_regs[dst].reg() != InvalidReg &&
src1->isA(Type::Int) &&
// src2 == 1:
src2->isConst() && src2->isA(Type::Int) && src2->getValInt() == 1) {
emitMovRegReg(m_as, m_regs[src1].reg(), m_regs[dst].reg());
(m_as.*emitFunc)(m_regs[dst].reg());
return true;
}
return false;
}
/*
* If src2 is 1, this generates dst = src1 + 1 using the "inc" x86 instruction.
* The return value is whether or not the instruction could be generated.
*/
bool CodeGenerator::emitInc(SSATmp* dst, SSATmp* src1, SSATmp* src2) {
return emitIncDecHelper(dst, src1, src2, &Asm::incq);
}
/*
* If src2 is 1, this generates dst = src1 - 1 using the "dec" x86 instruction.
* The return value is whether or not the instruction could be generated.
*/
bool CodeGenerator::emitDec(SSATmp* dst, SSATmp* src1, SSATmp* src2) {
return emitIncDecHelper(dst, src1, src2, &Asm::decq);
}
void CodeGenerator::cgOpAdd(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* src1 = inst->src(0);
SSATmp* src2 = inst->src(1);
// Special cases: x = y + 1
if (emitInc(dst, src1, src2) || emitInc(dst, src2, src1)) return;
cgBinaryOp(inst,
&Asm::addq,
&Asm::addq,
&Asm::movq,
&Asm::addsd_xmm_xmm,
std::plus<int64_t>(),
&convertToReg64,
Commutative);
}
void CodeGenerator::cgOpSub(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* src1 = inst->src(0);
SSATmp* src2 = inst->src(1);
if (emitDec(dst, src1, src2)) return;
if (src1->isConst() && src1->isA(Type::Int) && src1->getValInt() == 0 &&
src2->isA(Type::Int)) {
cgNegateWork(dst, src2);
return;
}
cgBinaryOp(inst,
&Asm::subq,
&Asm::subq,
&Asm::movq,
&Asm::subsd_xmm_xmm,
std::minus<int64_t>(),
&convertToReg64,
NonCommutative);
}
void CodeGenerator::cgOpDiv(IRInstruction* inst) {
not_implemented();
}
void CodeGenerator::cgOpBitAnd(IRInstruction* inst) {
cgBinaryIntOp(inst,
&Asm::andq,
&Asm::andq,
&Asm::movq,
[] (int64_t a, int64_t b) { return a & b; },
&convertToReg64,
Commutative);
}
void CodeGenerator::cgOpBitOr(IRInstruction* inst) {
cgBinaryIntOp(inst,
&Asm::orq,
&Asm::orq,
&Asm::movq,
[] (int64_t a, int64_t b) { return a | b; },
&convertToReg64,
Commutative);
}
void CodeGenerator::cgOpBitXor(IRInstruction* inst) {
cgBinaryIntOp(inst,
&Asm::xorq,
&Asm::xorq,
&Asm::movq,
[] (int64_t a, int64_t b) { return a ^ b; },
&convertToReg64,
Commutative);
}
void CodeGenerator::cgOpBitNot(IRInstruction* inst) {
cgUnaryIntOp(inst->dst(),
inst->src(0),
&Asm::not,
[](int64_t i) { return ~i; });
}
void CodeGenerator::cgOpLogicXor(IRInstruction* inst) {
cgBinaryIntOp(inst,
&Asm::xorb,
&Asm::xorb,
&Asm::movb,
[] (bool a, bool b) { return a ^ b; },
&convertToReg8,
Commutative);
}
void CodeGenerator::cgOpMul(IRInstruction* inst) {
cgBinaryOp(inst,
&Asm::imul,
&Asm::imul,
&Asm::movq,
&Asm::mulsd_xmm_xmm,
std::multiplies<int64_t>(),
&convertToReg64,
Commutative);
}
void CodeGenerator::cgOpNot(IRInstruction* inst) {
auto const src = inst->src(0);
auto const dstReg = m_regs[inst->dst()].reg();
auto& a = m_as;
if (src->isConst()) {
a. movb (!src->getValBool(), rbyte(dstReg));
} else {
if (dstReg != m_regs[src].reg()) {
a. movb (rbyte(m_regs[src].reg()), rbyte(dstReg));
}
a. xorb (1, rbyte(dstReg));
}
}
///////////////////////////////////////////////////////////////////////////////
// Comparison Operators
///////////////////////////////////////////////////////////////////////////////
#define DISPATCHER(name) \
HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, StringData* a2) \
{ return name(a1, a2); } \
HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, int64_t a2) \
{ return name(a1, a2); } \
HOT_FUNC_VM int64_t ccmp_ ## name (StringData* a1, ObjectData* a2) \
{ return name(a1, Object(a2)); } \
HOT_FUNC_VM int64_t ccmp_ ## name (ObjectData* a1, ObjectData* a2) \
{ return name(Object(a1), Object(a2)); } \
HOT_FUNC_VM int64_t ccmp_ ## name (ObjectData* a1, int64_t a2) \
{ return name(Object(a1), a2); } \
HOT_FUNC_VM int64_t ccmp_ ## name (ArrayData* a1, ArrayData* a2) \
{ return name(Array(a1), Array(a2)); }
DISPATCHER(same)
DISPATCHER(equal)
DISPATCHER(more)
DISPATCHER(less)
#undef DISPATCHER
template <typename A, typename B>
inline int64_t ccmp_nsame(A a, B b) { return !ccmp_same(a, b); }
template <typename A, typename B>
inline int64_t ccmp_nequal(A a, B b) { return !ccmp_equal(a, b); }
template <typename A, typename B>
inline int64_t ccmp_lte(A a, B b) { return !ccmp_more(a, b); }
template <typename A, typename B>
inline int64_t ccmp_gte(A a, B b) { return !ccmp_less(a, b); }
#define CG_OP_CMP(inst, setter, name) \
cgOpCmpHelper(inst, &Asm:: setter, ccmp_ ## name, ccmp_ ## name, \
ccmp_ ## name, ccmp_ ## name, ccmp_ ## name, ccmp_ ## name)
// SRON - string, resource, object, or number
static bool typeIsSRON(Type t) {
return t.isString()
|| t == Type::Obj // encompases object and resource
|| t == Type::Int
|| t == Type::Dbl
;
}
void CodeGenerator::cgOpCmpHelper(
IRInstruction* inst,
void (Asm::*setter)(Reg8),
int64_t (*str_cmp_str)(StringData*, StringData*),
int64_t (*str_cmp_int)(StringData*, int64_t),
int64_t (*str_cmp_obj)(StringData*, ObjectData*),
int64_t (*obj_cmp_obj)(ObjectData*, ObjectData*),
int64_t (*obj_cmp_int)(ObjectData*, int64_t),
int64_t (*arr_cmp_arr)( ArrayData*, ArrayData*)
) {
SSATmp* dst = inst->dst();
SSATmp* src1 = inst->src(0);
SSATmp* src2 = inst->src(1);
Type type1 = src1->type();
Type type2 = src2->type();
auto src1Reg = m_regs[src1].reg();
auto src2Reg = m_regs[src2].reg();
auto dstReg = m_regs[dst].reg();
auto setFromFlags = [&] {
(m_as.*setter)(rbyte(dstReg));
};
// It is possible that some pass has been done after simplification; if such
// a pass invalidates our invariants, then just punt.
// simplifyCmp has done const-const optimization
//
// If the types are the same and there is only one constant,
// simplifyCmp has moved it to the right.
if (src1->isConst()) {
CG_PUNT(cgOpCmpHelper_const);
}
/////////////////////////////////////////////////////////////////////////////
// case 1: null/string cmp string
// simplifyCmp has converted the null to ""
if (type1.isString() && type2.isString()) {
ArgGroup args(m_regs);
args.ssa(src1).ssa(src2);
cgCallHelper(m_as, (TCA)str_cmp_str, dst, kSyncPoint, args);
}
/////////////////////////////////////////////////////////////////////////////
// case 2: bool/null cmp anything
// simplifyCmp has converted all args to bool
else if (type1 == Type::Bool && type2 == Type::Bool) {
if (src2->isConst()) {
m_as. cmpb (src2->getValBool(), Reg8(int(src1Reg)));
} else {
m_as. cmpb (Reg8(int(src2Reg)), Reg8(int(src1Reg)));
}
setFromFlags();
}
/////////////////////////////////////////////////////////////////////////////
// case 3, 4, and 7: string/resource/object/number (sron) cmp sron
// These cases must be amalgamated because Type::Obj can refer to an object
// or to a resource.
// strings are canonicalized to the left, ints to the right
else if (typeIsSRON(type1) && typeIsSRON(type2)) {
// the common case: int cmp int
if (type1 == Type::Int && type2 == Type::Int) {
if (src2->isConst()) {
m_as.cmp_imm64_reg64(src2->getValInt(), src1Reg);
} else {
m_as.cmp_reg64_reg64(src2Reg, src1Reg);
}
setFromFlags();
}
else if (type1 == Type::Dbl || type2 == Type::Dbl) {
if ((type1 == Type::Dbl || type1 == Type::Int) &&
(type2 == Type::Dbl || type2 == Type::Int)) {
PhysReg srcReg1 = prepXMMReg(src1, m_as, m_regs, rCgXMM0);
PhysReg srcReg2 = prepXMMReg(src2, m_as, m_regs, rCgXMM1);
assert(srcReg1 != rCgXMM1 && srcReg2 != rCgXMM0);
doubleCmp(m_as, srcReg1, srcReg2);
setFromFlags();
} else {
CG_PUNT(cgOpCmpHelper_Dbl);
}
}
else if (type1.isString()) {
// string cmp string is dealt with in case 1
// string cmp double is punted above
if (type2 == Type::Int) {
ArgGroup args(m_regs);
args.ssa(src1).ssa(src2);
cgCallHelper(m_as, (TCA)str_cmp_int, dst, kSyncPoint, args);
} else if (type2 == Type::Obj) {
ArgGroup args(m_regs);
args.ssa(src1).ssa(src2);
cgCallHelper(m_as, (TCA)str_cmp_obj, dst, kSyncPoint, args);
} else {
CG_PUNT(cgOpCmpHelper_sx);
}
}
else if (type1 == Type::Obj) {
// string cmp object/resource is dealt with above
// object cmp double is punted above
if (type2 == Type::Obj) {
ArgGroup args(m_regs);
args.ssa(src1).ssa(src2);
cgCallHelper(m_as, (TCA)obj_cmp_obj, dst, kSyncPoint, args);
} else if (type2 == Type::Int) {
ArgGroup args(m_regs);
args.ssa(src1).ssa(src2);
cgCallHelper(m_as, (TCA)obj_cmp_int, dst, kSyncPoint, args);
} else {
CG_PUNT(cgOpCmpHelper_ox);
}
}
else NOT_REACHED();
}
/////////////////////////////////////////////////////////////////////////////
// case 5: array cmp array
else if (type1.isArray() && type2.isArray()) {
ArgGroup args(m_regs);
args.ssa(src1).ssa(src2);
cgCallHelper(m_as, (TCA)arr_cmp_arr, dst, kSyncPoint, args);
}
/////////////////////////////////////////////////////////////////////////////
// case 6: array cmp anything
// simplifyCmp has already dealt with this case.
/////////////////////////////////////////////////////////////////////////////
else {
// We have a type which is not a common type. It might be a cell or a box.
CG_PUNT(cgOpCmpHelper_unimplemented);
}
}
void CodeGenerator::cgOpEq(IRInstruction* inst) {
CG_OP_CMP(inst, sete, equal);
}
void CodeGenerator::cgOpNeq(IRInstruction* inst) {
CG_OP_CMP(inst, setne, nequal);
}
void CodeGenerator::cgOpSame(IRInstruction* inst) {
CG_OP_CMP(inst, sete, same);
}
void CodeGenerator::cgOpNSame(IRInstruction* inst) {
CG_OP_CMP(inst, setne, nsame);
}
void CodeGenerator::cgOpLt(IRInstruction* inst) {
CG_OP_CMP(inst, setl, less);
}
void CodeGenerator::cgOpGt(IRInstruction* inst) {
CG_OP_CMP(inst, setg, more);
}
void CodeGenerator::cgOpLte(IRInstruction* inst) {
CG_OP_CMP(inst, setle, lte);
}
void CodeGenerator::cgOpGte(IRInstruction* inst) {
CG_OP_CMP(inst, setge, gte);
}
///////////////////////////////////////////////////////////////////////////////
// Type check operators
///////////////////////////////////////////////////////////////////////////////
// Overloads to put the ObjectData* into a register so emitTypeTest
// can cmp to the Class* expected by the specialized Type
// Nothing to do, return the register that contain the ObjectData already
Reg64 getObjectDataEnregistered(Asm& as, PhysReg dataSrc, Reg64 scratch) {
return dataSrc;
}
// Enregister the meoryRef so it can be used with an offset by the
// cmp instruction
Reg64 getObjectDataEnregistered(Asm& as,
MemoryRef dataSrc,
Reg64 scratch) {
as.loadq(dataSrc, scratch);
return scratch;
}
template<class Loc1, class Loc2, class JmpFn>
void CodeGenerator::emitTypeTest(Type type, Loc1 typeSrc, Loc2 dataSrc,
JmpFn doJcc) {
assert(!type.subtypeOf(Type::Cls));
ConditionCode cc;
if (type.isString()) {
emitTestTVType(m_as, KindOfStringBit, typeSrc);
cc = CC_NZ;
} else if (type.equals(Type::UncountedInit)) {
emitTestTVType(m_as, KindOfUncountedInitBit, typeSrc);
cc = CC_NZ;
} else if (type.equals(Type::Uncounted)) {
emitCmpTVType(m_as, KindOfRefCountThreshold, typeSrc);
cc = CC_LE;
} else if (type.equals(Type::Cell)) {
emitCmpTVType(m_as, KindOfRef, typeSrc);
cc = CC_L;
} else if (type.equals(Type::Gen)) {
// nothing to check
return;
} else {
DataType dataType = type.toDataType();
assert(dataType == KindOfRef ||
(dataType >= KindOfUninit && dataType <= KindOfObject));
emitCmpTVType(m_as, dataType, typeSrc);
cc = CC_E;
}
doJcc(cc);
if (type.strictSubtypeOf(Type::Obj)) {
// emit the specific class test
assert(type.getClass()->attrs() & AttrFinal);
auto reg = getObjectDataEnregistered(m_as, dataSrc, m_rScratch);
m_as.cmpq(type.getClass(), reg[ObjectData::getVMClassOffset()]);
doJcc(cc);
}
}
template<class JmpFn>
void CodeGenerator::emitIsTypeTest(IRInstruction* inst, JmpFn doJcc) {
auto const src = inst->src(0);
// punt if specialized object for now
if (inst->typeParam().strictSubtypeOf(Type::Obj)) {
CG_PUNT(IsType-SpecializedUnsupported);
}
if (inst->typeParam().equals(Type::Obj)) {
auto const srcReg = m_regs[src].reg();
if (src->isA(Type::PtrToGen)) {
emitTestTVType(m_as, KindOfObject, srcReg[TVOFF(m_type)]);
TCA toPatch = m_as.code.frontier;
m_as. jne8(toPatch); // 1
// Get the ObjectData*
emitDeref(m_as, srcReg, m_rScratch);
m_as. cmpq(SystemLib::s_resourceClass,
m_rScratch[ObjectData::getVMClassOffset()]);
// 1:
m_as.patchJcc8(toPatch, m_as.code.frontier);
} else {
// Cases where src isn't an Obj should have been simplified away
if (!src->isA(Type::Obj)) {
CG_PUNT(IsType-KnownWrongType);
}
m_as. cmpq(SystemLib::s_resourceClass,
srcReg[ObjectData::getVMClassOffset()]);
}
// At this point, the flags say "equal" if is_object is false.
doJcc(CC_NE);
return;
}
if (src->isA(Type::PtrToGen)) {
PhysReg base = m_regs[src].reg();
emitTypeTest(inst->typeParam(), base[TVOFF(m_type)],
base[TVOFF(m_data)],
[&](ConditionCode cc) { doJcc(cc); });
return;
}
assert(src->isA(Type::Gen));
assert(!src->isConst());
PhysReg typeSrcReg = m_regs[src].reg(1); // type register
if (typeSrcReg == InvalidReg) {
CG_PUNT(IsType-KnownType);
}
PhysReg dataSrcReg = m_regs[src].reg(); // data register
emitTypeTest(inst->typeParam(), typeSrcReg, dataSrcReg,
[&](ConditionCode cc) { doJcc(cc); });
}
template<class Loc>
void CodeGenerator::emitTypeCheck(Type type,
Loc typeSrc,
Loc dataSrc,
Block* taken) {
emitTypeTest(type, typeSrc, dataSrc,
[&](ConditionCode cc) {
emitFwdJcc(ccNegate(cc), taken);
});
}
template<class Loc>
void CodeGenerator::emitTypeGuard(Type type, Loc typeSrc, Loc dataSrc) {
emitTypeTest(type, typeSrc, dataSrc,
[&](ConditionCode cc) {
auto const destSK = SrcKey(curFunc(), m_curTrace->bcOff());
auto const destSR = m_tx64->getSrcRec(destSK);
m_tx64->emitFallbackCondJmp(m_as, *destSR, ccNegate(cc));
});
}
void CodeGenerator::emitSetCc(IRInstruction* inst, ConditionCode cc) {
m_as.setcc(cc, rbyte(m_regs[inst->dst()].reg()));
}
void CodeGenerator::cgIsTypeMemCommon(IRInstruction* inst, bool negate) {
bool called = false; // check emitSetCc is called only once
emitIsTypeTest(inst,
[&](ConditionCode cc) {
assert(!called);
emitSetCc(inst, negate ? ccNegate(cc) : cc);
called = true;
});
}
void CodeGenerator::cgIsTypeCommon(IRInstruction* inst, bool negate) {
bool called = false; // check emitSetCc is called only once
emitIsTypeTest(inst,
[&](ConditionCode cc) {
assert(!called);
emitSetCc(inst, negate ? ccNegate(cc) : cc);
called = true;
});
}
void CodeGenerator::cgJmpIsTypeCommon(IRInstruction* inst, bool negate) {
emitIsTypeTest(inst,
[&](ConditionCode cc) {
emitFwdJcc(negate ? ccNegate(cc) : cc, inst->taken());
});
}
void CodeGenerator::cgIsType(IRInstruction* inst) {
cgIsTypeCommon(inst, false);
}
void CodeGenerator::cgIsNType(IRInstruction* inst) {
cgIsTypeCommon(inst, true);
}
// TODO(#2404341): remove JmpIs{N,}Type
void CodeGenerator::cgJmpIsType(IRInstruction* inst) {
cgJmpIsTypeCommon(inst, false);
}
void CodeGenerator::cgJmpIsNType(IRInstruction* inst) {
cgJmpIsTypeCommon(inst, true);
}
void CodeGenerator::cgIsTypeMem(IRInstruction* inst) {
cgIsTypeMemCommon(inst, false);
}
void CodeGenerator::cgIsNTypeMem(IRInstruction* inst) {
cgIsTypeMemCommon(inst, true);
}
///////////////////////////////////////////////////////////////////////////////
HOT_FUNC_VM static bool instanceOfHelper(const Class* objClass,
const Class* testClass) {
return testClass && objClass->classof(testClass);
}
void CodeGenerator::cgInstanceOf(IRInstruction* inst) {
cgCallHelper(m_as,
TCA(instanceOfHelper),
inst->dst(),
kNoSyncPoint,
ArgGroup(m_regs)
.ssa(inst->src(0))
.ssa(inst->src(1)));
}
/*
* Check instanceof using instance bitmasks.
*
* Note it's not necessary to check whether the test class is defined:
* if it doesn't exist than the candidate can't be an instance of it
* and will fail this check.
*/
void CodeGenerator::emitInstanceBitmaskCheck(IRInstruction* inst) {
auto const rObjClass = m_regs[inst->src(0)].reg(0);
auto const testClassName = inst->src(1)->getValStr();
auto& a = m_as;
int offset;
uint8_t mask;
if (!Class::getInstanceBitMask(testClassName, offset, mask)) {
always_assert(!"cgInstanceOfBitmask had no bitmask");
}
a. testb (int8_t(mask), rObjClass[offset]);
}
void CodeGenerator::cgInstanceOfBitmask(IRInstruction* inst) {
auto& a = m_as;
emitInstanceBitmaskCheck(inst);
a. setnz (rbyte(m_regs[inst->dst()].reg()));
}
void CodeGenerator::cgNInstanceOfBitmask(IRInstruction* inst) {
auto& a = m_as;
emitInstanceBitmaskCheck(inst);
a. setz (rbyte(m_regs[inst->dst()].reg()));
}
void CodeGenerator::cgJmpInstanceOfBitmask(IRInstruction* inst) {
emitInstanceBitmaskCheck(inst);
emitFwdJcc(CC_NZ, inst->taken());
}
void CodeGenerator::cgJmpNInstanceOfBitmask(IRInstruction* inst) {
emitInstanceBitmaskCheck(inst);
emitFwdJcc(CC_Z, inst->taken());
}
void CodeGenerator::cgReqBindJmpInstanceOfBitmask(IRInstruction* inst) {
emitInstanceBitmaskCheck(inst);
emitReqBindJcc(opToConditionCode(inst->op()),
inst->extra<ReqBindJccData>());
}
void CodeGenerator::cgReqBindJmpNInstanceOfBitmask(IRInstruction* inst) {
emitInstanceBitmaskCheck(inst);
emitReqBindJcc(opToConditionCode(inst->op()),
inst->extra<ReqBindJccData>());
}
/*
* Check instanceof using the superclass vector on the end of the
* Class entry.
*/
void CodeGenerator::cgExtendsClass(IRInstruction* inst) {
auto const rObjClass = m_regs[inst->src(0)].reg();
auto const testClass = inst->src(1)->getValClass();
auto rTestClass = m_regs[inst->src(1)].reg();
auto const rdst = rbyte(m_regs[inst->dst()].reg());
auto& a = m_as;
Label out;
Label notExact;
Label falseLabel;
if (rTestClass == InvalidReg) { // TODO(#2031606)
rTestClass = m_rScratch; // careful below about asm-x64 smashing this
emitLoadImm(a, (int64_t)testClass, rTestClass);
}
// Test if it is the exact same class. TODO(#2044801): we should be
// doing this control flow at the IR level.
if (!(testClass->attrs() & AttrAbstract)) {
a. cmpq (rTestClass, rObjClass);
a. jne8 (notExact);
a. movb (1, rdst);
a. jmp8 (out);
}
auto const vecOffset = Class::classVecOff() +
sizeof(Class*) * (testClass->classVecLen() - 1);
// Check the length of the class vectors---if the candidate's is at
// least as long as the potential base (testClass) it might be a
// subclass.
asm_label(a, notExact);
a. cmpl (testClass->classVecLen(),
rObjClass[Class::classVecLenOff()]);
a. jb8 (falseLabel);
// If it's a subclass, rTestClass must be at the appropriate index.
a. cmpq (rTestClass, rObjClass[vecOffset]);
a. sete (rdst);
a. jmp8 (out);
asm_label(a, falseLabel);
a. xorl (r32(rdst), r32(rdst));
asm_label(a, out);
}
void CodeGenerator::cgConvDblToBool(IRInstruction* inst) {
SSATmp* dst = inst->dst();
auto dstReg = m_regs[dst].reg();
assert(dstReg != InvalidReg);
SSATmp* src = inst->src(0);
auto srcReg = m_regs[src].reg();
if (srcReg == InvalidReg) {
assert(src->isConst());
double constVal = src->getValDbl();
if (constVal == 0.0) {
m_as.xor_reg64_reg64(dstReg, dstReg);
} else {
m_as.mov_imm64_reg(1, dstReg);
}
} else {
emitMovRegReg(m_as, srcReg, dstReg);
m_as.shlq(1, dstReg); // 0.0 stays zero and -0.0 is now 0.0
m_as.setne(rbyte(dstReg)); // lower byte becomes 1 if dstReg != 0
m_as.movzbl(rbyte(dstReg), r32(dstReg));
}
}
void CodeGenerator::cgConvIntToBool(IRInstruction* inst) {
SSATmp* dst = inst->dst();
auto dstReg = m_regs[dst].reg();
assert(dstReg != InvalidReg);
SSATmp* src = inst->src(0);
auto srcReg = m_regs[src].reg();
if (srcReg == InvalidReg) {
assert(src->isConst());
int64_t constVal = src->getValInt();
if (constVal == 0) {
m_as.xor_reg64_reg64(dstReg, dstReg);
} else {
m_as.mov_imm64_reg(1, dstReg);
}
} else {
m_as.test_reg64_reg64(srcReg, srcReg);
m_as.setne(rbyte(dstReg));
m_as.movzbl(rbyte(dstReg), r32(dstReg));
}
}
void CodeGenerator::emitConvBoolOrIntToDbl(IRInstruction* inst) {
SSATmp* src = inst->src(0);
SSATmp* dst = inst->dst();
PhysReg dstReg = m_regs[dst].reg();
assert(src->isA(Type::Bool) || src->isA(Type::Int));
assert(dstReg != InvalidReg);
if (src->isConst()) {
int64_t constVal = src->getValRawInt();
if (src->isA(Type::Bool)) constVal = constVal != 0; // see task #2401790
constVal = convIntToDouble(constVal);
emitLoadImm(m_as, constVal, dstReg);
} else {
// cvtsi2sd doesn't modify the high bits of its target, which can
// cause false dependencies to prevent register renaming from kicking
// in. Break the dependency chain by zeroing out the XMM reg.
PhysReg srcReg = m_regs[src].reg();
PhysReg xmmReg = dstReg.isXMM() ? dstReg : PhysReg(rCgXMM0);
m_as.pxor_xmm_xmm(xmmReg, xmmReg);
m_as.cvtsi2sd_reg64_xmm(srcReg, xmmReg);
zeroExtendIfBool(m_as, src, m_regs[src]);
emitMovRegReg(m_as, xmmReg, dstReg);
}
}
void CodeGenerator::cgConvBoolToDbl(IRInstruction* inst) {
emitConvBoolOrIntToDbl(inst);
}
void CodeGenerator::cgConvIntToDbl(IRInstruction* inst) {
emitConvBoolOrIntToDbl(inst);
}
void CodeGenerator::cgConvBoolToInt(IRInstruction* inst) {
SSATmp* dst = inst->dst();
auto dstReg = m_regs[dst].reg();
assert(dstReg != InvalidReg);
SSATmp* src = inst->src(0);
auto srcReg = m_regs[src].reg();
assert(src->isConst() == (srcReg == InvalidReg));
if (srcReg == InvalidReg) {
int64_t constVal = src->getValRawInt();
if (constVal == 0) {
m_as.xor_reg64_reg64(dstReg, dstReg);
} else {
m_as.mov_imm64_reg(1, dstReg);
}
} else {
m_as.movzbl(rbyte(srcReg), r32(dstReg));
}
}
void CodeGenerator::cgConvBoolToStr(IRInstruction* inst) {
SSATmp* dst = inst->dst();
auto dstReg = m_regs[dst].reg();
assert(dstReg != InvalidReg);
SSATmp* src = inst->src(0);
auto srcReg = m_regs[src].reg();
assert(src->isConst() == (srcReg == InvalidReg));
if (srcReg == InvalidReg) {
auto constVal = src->getValBool();
if (!constVal) {
m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString(""), dstReg);
} else {
m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString("1"), dstReg);
}
} else {
m_as.testb(Reg8(int(srcReg)), Reg8(int(srcReg)));
m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString(""), dstReg);
m_as.mov_imm64_reg((uint64_t)StringData::GetStaticString("1"), m_rScratch);
m_as.cmov_reg64_reg64(CC_NZ, m_rScratch, dstReg);
}
}
void CodeGenerator::cgUnboxPtr(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* src = inst->src(0);
auto srcReg = m_regs[src].reg();
auto dstReg = m_regs[dst].reg();
assert(srcReg != InvalidReg);
assert(dstReg != InvalidReg);
emitMovRegReg(m_as, srcReg, dstReg);
emitDerefIfVariant(m_as, PhysReg(dstReg));
}
void CodeGenerator::cgUnbox(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* src = inst->src(0);
auto dstValReg = m_regs[dst].reg(0);
auto dstTypeReg = m_regs[dst].reg(1);
auto srcValReg = m_regs[src].reg(0);
auto srcTypeReg = m_regs[src].reg(1);
assert(dstValReg != dstTypeReg);
assert(src->type().equals(Type::Gen));
assert(dst->type().notBoxed());
emitCmpTVType(m_as, HPHP::KindOfRef, srcTypeReg);
ifThenElse(CC_E, [&] {
// srcTypeReg == KindOfRef; srcValReg is RefData*
const size_t ref_tv_off = RefData::tvOffset();
if (dstValReg != srcValReg) {
emitLoadReg(m_as, srcValReg[ref_tv_off + TVOFF(m_data)], dstValReg);
emitLoadTVType(m_as, srcValReg[ref_tv_off + TVOFF(m_type)],
r32(dstTypeReg));
} else {
emitLoadTVType(m_as, srcValReg[ref_tv_off + TVOFF(m_type)],
r32(dstTypeReg));
m_as.loadq(srcValReg[ref_tv_off + TVOFF(m_data)], dstValReg);
}
}, [&] {
// srcTypeReg != KindOfRef; copy src -> dst
shuffle2(m_as, srcValReg, srcTypeReg, dstValReg, dstTypeReg);
});
}
void CodeGenerator::cgLdFuncCachedCommon(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* methodName = inst->src(0);
const StringData* name = methodName->getValStr();
CacheHandle ch = TargetCache::allocFixedFunction(name);
size_t funcCacheOff = ch + offsetof(FixedFuncCache, m_func);
auto dstReg = m_regs[dst].reg();
if (dstReg == InvalidReg) {
// happens if LdFixedFunc and FCall not in same trace
m_as. cmpq(0, rVmTl[funcCacheOff]);
} else {
m_as. loadq (rVmTl[funcCacheOff], dstReg);
m_as. testq (dstReg, dstReg);
}
}
void CodeGenerator::cgLdFuncCached(IRInstruction* inst) {
cgLdFuncCachedCommon(inst);
// jz off to the helper call in astubs
unlikelyIfBlock(CC_Z, [&] (Asm& a) {
// this helper tries the autoload map, and fatals on failure
cgCallNative(a, inst);
});
}
void CodeGenerator::cgLdFuncCachedSafe(IRInstruction* inst) {
cgLdFuncCachedCommon(inst);
if (Block* taken = inst->taken()) {
emitFwdJcc(m_as, CC_Z, taken);
}
}
void CodeGenerator::cgLdFunc(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* methodName = inst->src(0);
TargetCache::CacheHandle ch = TargetCache::FuncCache::alloc();
// raises an error if function not found
cgCallHelper(m_as, (TCA)FuncCache::lookup, m_regs[dst].reg(), kSyncPoint,
ArgGroup(m_regs).imm(ch).ssa(methodName));
}
static void emitLdObjClass(CodeGenerator::Asm& a,
PhysReg objReg,
PhysReg dstReg) {
a.loadq (objReg[ObjectData::getVMClassOffset()], dstReg);
}
void CodeGenerator::cgLdObjClass(IRInstruction* inst) {
auto dstReg = m_regs[inst->dst()].reg();
auto objReg = m_regs[inst->src(0)].reg();
emitLdObjClass(m_as, objReg, dstReg);
}
void CodeGenerator::cgLdObjMethod(IRInstruction *inst) {
auto cls = inst->src(0);
auto clsReg = m_regs[cls].reg();
auto name = inst->src(1);
auto actRec = inst->src(2);
auto actRecReg = m_regs[actRec].reg();
CacheHandle handle = Transl::TargetCache::MethodCache::alloc();
// lookup in the targetcache
assert(MethodCache::kNumLines == 1);
if (debug) {
MethodCache::Pair p;
static_assert(sizeof(p.m_value) == 8,
"MethodCache::Pair::m_value assumed to be 8 bytes");
static_assert(sizeof(p.m_key) == 8,
"MethodCache::Pair::m_key assumed to be 8 bytes");
}
// preload handle->m_value
m_as.loadq(rVmTl[handle + offsetof(MethodCache::Pair, m_value)], m_rScratch);
m_as.cmpq (rVmTl[handle + offsetof(MethodCache::Pair, m_key)], clsReg);
ifThenElse(CC_E, // if handle->key == cls
[&] { // then actReg->m_func = handle->value
m_as.storeq(m_rScratch, actRecReg[AROFF(m_func)]);
},
[&] { // else call slow path helper
cgCallHelper(m_as, (TCA)methodCacheSlowPath, InvalidReg,
kSyncPoint,
ArgGroup(m_regs).addr(rVmTl, handle)
.ssa(actRec)
.ssa(name)
.ssa(cls));
});
}
void CodeGenerator::cgStRetVal(IRInstruction* inst) {
auto const rFp = m_regs[inst->src(0)].reg();
auto* const val = inst->src(1);
cgStore(rFp, AROFF(m_r), val);
}
void CodeGenerator::cgRetAdjustStack(IRInstruction* inst) {
auto const rFp = m_regs[inst->src(0)].reg();
auto const dstSp = m_regs[inst->dst()].reg();
auto& a = m_as;
a. lea (rFp[AROFF(m_r)], dstSp);
}
void CodeGenerator::cgLdRetAddr(IRInstruction* inst) {
auto fpReg = m_regs[inst->src(0)].reg(0);
assert(fpReg != InvalidReg);
m_as.push(fpReg[AROFF(m_savedRip)]);
}
void checkFrame(ActRec* fp, Cell* sp, bool checkLocals) {
const Func* func = fp->m_func;
if (fp->hasVarEnv()) {
assert(fp->getVarEnv()->getCfp() == fp);
}
// TODO: validate this pointer from actrec
int numLocals = func->numLocals();
assert(sp <= (Cell*)fp - func->numSlotsInFrame()
|| func->isGenerator());
if (checkLocals) {
int numParams = func->numParams();
for (int i=0; i < numLocals; i++) {
if (i >= numParams && func->isGenerator() && i < func->numNamedLocals()) {
continue;
}
assert(checkTv(frame_local(fp, i)));
}
}
// We unfortunately can't do the same kind of check for the stack
// without knowing about FPI regions, because it may contain
// ActRecs.
}
void traceRet(ActRec* fp, Cell* sp, void* rip) {
if (rip == TranslatorX64::Get()->getCallToExit()) {
return;
}
checkFrame(fp, sp, /*checkLocals*/ false);
assert(sp <= (Cell*)fp || fp->m_func->isGenerator());
// check return value if stack not empty
if (sp < (Cell*)fp) assertTv(sp);
}
void CodeGenerator::emitTraceRet(CodeGenerator::Asm& a) {
// call to a trace function
a. movq (rVmFp, rdi);
a. movq (rVmSp, rsi);
a. loadq (*rsp, rdx); // return ip from native stack
// do the call; may use a trampoline
m_tx64->emitCall(a, TCA(traceRet));
}
void CodeGenerator::cgRetCtrl(IRInstruction* inst) {
SSATmp* sp = inst->src(0);
SSATmp* fp = inst->src(1);
// Make sure rVmFp and rVmSp are set appropriately
emitMovRegReg(m_as, m_regs[sp].reg(), rVmSp);
emitMovRegReg(m_as, m_regs[fp].reg(), rVmFp);
// Return control to caller
if (RuntimeOption::EvalHHIRGenerateAsserts) {
emitTraceRet(m_as);
}
m_as.ret();
if (RuntimeOption::EvalHHIRGenerateAsserts) {
m_as.ud2();
}
}
void CodeGenerator::emitReqBindAddr(const Func* func,
TCA& dest,
Offset offset) {
dest = m_tx64->emitServiceReq(SRFlags::None,
REQ_BIND_ADDR,
2ull,
&dest,
offset);
}
void CodeGenerator::cgJmpSwitchDest(IRInstruction* inst) {
JmpSwitchData* data = inst->extra<JmpSwitchDest>();
SSATmp* index = inst->src(0);
auto indexReg = m_regs[index].reg();
if (!index->isConst()) {
if (data->bounded) {
if (data->base) {
m_as. subq(data->base, indexReg);
}
m_as. cmpq(data->cases - 2, indexReg);
prepareForSmash(m_as, kJmpccLen);
TCA def = m_tx64->emitServiceReq(REQ_BIND_JMPCC_SECOND, 3,
m_as.code.frontier, data->defaultOff, CC_AE);
m_as. jae(def);
}
TCA* table = m_tx64->allocData<TCA>(sizeof(TCA), data->cases);
TCA afterLea = m_as.code.frontier + kLeaRipLen;
ptrdiff_t diff = (TCA)table - afterLea;
assert(deltaFits(diff, sz::dword));
m_as. lea(rip[diff], m_rScratch);
assert(m_as.code.frontier == afterLea);
m_as. jmp(m_rScratch[indexReg*8]);
for (int i = 0; i < data->cases; i++) {
emitReqBindAddr(data->func, table[i], data->targets[i]);
}
} else {
int64_t indexVal = index->getValInt();
if (data->bounded) {
indexVal -= data->base;
if (indexVal >= data->cases - 2 || indexVal < 0) {
m_tx64->emitBindJmp(m_as, SrcKey(data->func, data->defaultOff));
return;
}
}
m_tx64->emitBindJmp(m_as, SrcKey(data->func, data->targets[indexVal]));
}
}
typedef FixedStringMap<TCA,true> SSwitchMap;
static TCA sswitchHelperFast(const StringData* val,
const SSwitchMap* table,
TCA* def) {
TCA* dest = table->find(val);
return dest ? *dest : *def;
}
void CodeGenerator::cgLdSSwitchDestFast(IRInstruction* inst) {
auto data = inst->extra<LdSSwitchDestFast>();
auto table = m_tx64->allocData<SSwitchMap>(64);
table->init(data->numCases);
for (int64_t i = 0; i < data->numCases; ++i) {
table->add(data->cases[i].str, nullptr);
TCA* addr = table->find(data->cases[i].str);
emitReqBindAddr(data->func, *addr, data->cases[i].dest);
}
TCA* def = m_tx64->allocData<TCA>(sizeof(TCA), 1);
emitReqBindAddr(data->func, *def, data->defaultOff);
cgCallHelper(m_as,
TCA(sswitchHelperFast),
inst->dst(),
kNoSyncPoint,
ArgGroup(m_regs)
.ssa(inst->src(0))
.immPtr(table)
.immPtr(def));
}
static TCA sswitchHelperSlow(TypedValue typedVal,
const StringData** strs,
int numStrs,
TCA* jmptab) {
TypedValue* cell = tvToCell(&typedVal);
for (int i = 0; i < numStrs; ++i) {
if (tvAsCVarRef(cell).equal(strs[i])) return jmptab[i];
}
return jmptab[numStrs]; // default case
}
void CodeGenerator::cgLdSSwitchDestSlow(IRInstruction* inst) {
auto data = inst->extra<LdSSwitchDestSlow>();
auto strtab = m_tx64->allocData<const StringData*>(
sizeof(const StringData*), data->numCases);
auto jmptab = m_tx64->allocData<TCA>(sizeof(TCA), data->numCases + 1);
for (int i = 0; i < data->numCases; ++i) {
strtab[i] = data->cases[i].str;
emitReqBindAddr(data->func, jmptab[i], data->cases[i].dest);
}
emitReqBindAddr(data->func, jmptab[data->numCases], data->defaultOff);
cgCallHelper(m_as,
TCA(sswitchHelperSlow),
inst->dst(),
kSyncPoint,
ArgGroup(m_regs)
.typedValue(inst->src(0))
.immPtr(strtab)
.imm(data->numCases)
.immPtr(jmptab));
}
/*
* It'd be nice not to have the cgMov here (and just copy propagate
* the source or something), but for now we're keeping it allocated to
* rVmFp so inlined calls to C++ helpers that use the rbp chain to
* find the caller's ActRec will work correctly.
*
* This instruction primarily exists to assist in optimizing away
* unused activation records, so it's usually not going to happen
* anyway.
*/
void CodeGenerator::cgDefInlineFP(IRInstruction* inst) {
auto const fp = m_regs[inst->src(0)].reg();
auto const fakeRet = m_tx64->getRetFromInlinedFrame();
auto const retBCOff = inst->extra<DefInlineFP>()->retBCOff;
m_as. storeq (fakeRet, fp[AROFF(m_savedRip)]);
m_as. storel (retBCOff, fp[AROFF(m_soff)]);
cgMov(inst);
}
void CodeGenerator::cgInlineReturn(IRInstruction* inst) {
auto fpReg = m_regs[inst->src(0)].reg();
assert(fpReg == rVmFp);
m_as. loadq (fpReg[AROFF(m_savedRbp)], rVmFp);
}
void CodeGenerator::cgReDefSP(IRInstruction* inst) {
// TODO(#2288359): this instruction won't be necessary (for
// non-generator frames) when we don't track rVmSp independently
// from rVmFp. In generator frames we'll have to track offsets from
// a DefGeneratorSP or something similar.
auto fp = m_regs[inst->src(0)].reg();
auto dst = m_regs[inst->dst()].reg();
auto off = -inst->extra<ReDefSP>()->offset * sizeof(Cell);
emitLea(m_as, fp[off], dst);
}
void CodeGenerator::cgStashGeneratorSP(IRInstruction* inst) {
cgMov(inst);
}
void CodeGenerator::cgReDefGeneratorSP(IRInstruction* inst) {
cgMov(inst);
}
void CodeGenerator::cgFreeActRec(IRInstruction* inst) {
m_as.loadq(m_regs[inst->src(0)].reg()[AROFF(m_savedRbp)],
m_regs[inst->dst()].reg());
}
void CodeGenerator::cgSpill(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* src = inst->src(0);
assert(dst->numNeededRegs() == src->numNeededRegs());
for (int locIndex = 0; locIndex < m_regs[src].numAllocatedRegs();
++locIndex) {
// We do not need to mask booleans, since the IR will reload the spill
auto srcReg = m_regs[src].reg(locIndex);
auto sinfo = m_regs[dst].spillInfo(locIndex);
if (m_regs[src].isFullXMM()) {
m_as.movdqa(srcReg, reg::rsp[sinfo.offset()]);
} else {
int offset = sinfo.offset();
if (locIndex == 0 || packed_tv) {
emitStoreReg(m_as, srcReg, reg::rsp[offset]);
} else {
// Note that type field is shifted in memory
assert(srcReg.isGP());
offset += TVOFF(m_type) - (TVOFF(m_data) + sizeof(Value));
emitStoreTVType(m_as, srcReg, reg::rsp[offset]);
}
}
}
}
void CodeGenerator::cgReload(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* src = inst->src(0);
assert(dst->numNeededRegs() == src->numNeededRegs());
for (int locIndex = 0; locIndex < m_regs[dst].numAllocatedRegs();
++locIndex) {
auto dstReg = m_regs[dst].reg(locIndex);
auto sinfo = m_regs[src].spillInfo(locIndex);
if (m_regs[dst].isFullXMM()) {
assert(dstReg.isXMM());
m_as.movdqa(reg::rsp[sinfo.offset()], dstReg);
} else {
int offset = sinfo.offset();
if (locIndex == 0 || packed_tv) {
emitLoadReg(m_as, reg::rsp[offset], dstReg);
} else {
// Note that type field is shifted in memory
offset += TVOFF(m_type) - (TVOFF(m_data) + sizeof(Value));
assert(dstReg.isGP());
emitLoadTVType(m_as, reg::rsp[offset], dstReg);
}
}
}
}
void CodeGenerator::cgStPropWork(IRInstruction* inst, bool genTypeStore) {
SSATmp* obj = inst->src(0);
SSATmp* prop = inst->src(1);
SSATmp* src = inst->src(2);
cgStore(m_regs[obj].reg(), prop->getValInt(), src, genTypeStore);
}
void CodeGenerator::cgStProp(IRInstruction* inst) {
cgStPropWork(inst, true);
}
void CodeGenerator::cgStPropNT(IRInstruction* inst) {
cgStPropWork(inst, false);
}
void CodeGenerator::cgStMemWork(IRInstruction* inst, bool genStoreType) {
SSATmp* addr = inst->src(0);
SSATmp* offset = inst->src(1);
SSATmp* src = inst->src(2);
cgStore(m_regs[addr].reg(), offset->getValInt(), src, genStoreType);
}
void CodeGenerator::cgStMem(IRInstruction* inst) {
cgStMemWork(inst, true);
}
void CodeGenerator::cgStMemNT(IRInstruction* inst) {
cgStMemWork(inst, false);
}
void CodeGenerator::cgStRefWork(IRInstruction* inst, bool genStoreType) {
auto destReg = m_regs[inst->dst()].reg();
auto addrReg = m_regs[inst->src(0)].reg();
SSATmp* src = inst->src(1);
always_assert(!m_regs[src].isFullXMM());
cgStore(addrReg, RefData::tvOffset(), src, genStoreType);
if (destReg != InvalidReg) emitMovRegReg(m_as, addrReg, destReg);
}
void CodeGenerator::cgStRef(IRInstruction* inst) {
cgStRefWork(inst, true);
}
void CodeGenerator::cgStRefNT(IRInstruction* inst) {
cgStRefWork(inst, false);
}
static int64_t localOffset(int64_t index) {
return -cellsToBytes(index + 1);
}
static int64_t localOffset(SSATmp* index) {
return localOffset(index->getValInt());
}
int CodeGenerator::iterOffset(SSATmp* tmp) {
const Func* func = curFunc();
int64_t index = tmp->getValInt();
return -cellsToBytes(((index + 1) * kNumIterCells + func->numLocals()));
}
int CodeGenerator::iterOffset(uint32_t id) {
const Func* func = curFunc();
return -cellsToBytes(((id + 1) * kNumIterCells + func->numLocals()));
}
void CodeGenerator::cgStLoc(IRInstruction* inst) {
cgStore(m_regs[inst->src(0)].reg(),
localOffset(inst->extra<StLoc>()->locId),
inst->src(1),
true /* store type */);
}
void CodeGenerator::cgStLocNT(IRInstruction* inst) {
cgStore(m_regs[inst->src(0)].reg(),
localOffset(inst->extra<StLocNT>()->locId),
inst->src(1),
false /* store type */);
}
void CodeGenerator::cgSyncABIRegs(IRInstruction* inst) {
emitMovRegReg(m_as, m_regs[inst->src(0)].reg(), rVmFp);
emitMovRegReg(m_as, m_regs[inst->src(1)].reg(), rVmSp);
}
void CodeGenerator::cgReqBindJmp(IRInstruction* inst) {
m_tx64->emitBindJmp(
m_as,
SrcKey(curFunc(), inst->extra<ReqBindJmp>()->offset)
);
}
static void emitExitNoIRStats(Asm& a,
TranslatorX64* tx64,
const Func* func,
SrcKey dest) {
if (RuntimeOption::EnableInstructionCounts ||
HPHP::Trace::moduleEnabled(HPHP::Trace::stats, 3)) {
Stats::emitInc(a,
Stats::opcodeToIRPreStatCounter(
Op(*func->unit()->at(dest.offset()))),
-1,
Transl::CC_None,
true);
}
}
void CodeGenerator::cgReqBindJmpNoIR(IRInstruction* inst) {
auto const dest = SrcKey(curFunc(),
inst->extra<ReqBindJmpNoIR>()->offset);
emitExitNoIRStats(m_as, m_tx64, curFunc(), dest);
m_tx64->emitBindJmp(m_as, dest, REQ_BIND_JMP_NO_IR);
}
void CodeGenerator::cgReqRetranslateNoIR(IRInstruction* inst) {
auto const dest = SrcKey(curFunc(),
inst->extra<ReqRetranslateNoIR>()->offset);
emitExitNoIRStats(m_as, m_tx64, curFunc(), dest);
m_tx64->emitReqRetransNoIR(m_as, dest);
}
void CodeGenerator::cgReqRetranslate(IRInstruction* inst) {
auto const destSK = SrcKey(curFunc(), m_curTrace->bcOff());
auto const destSR = m_tx64->getSrcRec(destSK);
m_tx64->emitFallbackUncondJmp(m_as, *destSR);
}
static void emitAssertFlagsNonNegative(CodeGenerator::Asm& as) {
ifThen(as, CC_NGE, [&] { as.ud2(); });
}
static void emitAssertRefCount(CodeGenerator::Asm& as, PhysReg base) {
as.cmpl(HPHP::RefCountStaticValue, base[FAST_REFCOUNT_OFFSET]);
ifThen(as, CC_NBE, [&] { as.ud2(); });
}
static void emitIncRef(CodeGenerator::Asm& as, PhysReg base) {
if (RuntimeOption::EvalHHIRGenerateAsserts) {
emitAssertRefCount(as, base);
}
// emit incref
as.incl(base[FAST_REFCOUNT_OFFSET]);
if (RuntimeOption::EvalHHIRGenerateAsserts) {
// Assert that the ref count is greater than zero
emitAssertFlagsNonNegative(as);
}
}
void CodeGenerator::cgIncRefWork(Type type, SSATmp* src) {
assert(type.maybeCounted());
auto increfMaybeStatic = [&] {
auto base = m_regs[src].reg(0);
if (!type.needsStaticBitCheck()) {
emitIncRef(m_as, base);
} else {
m_as.cmpl(RefCountStaticValue, base[FAST_REFCOUNT_OFFSET]);
ifThen(m_as, CC_NE, [&] { emitIncRef(m_as, base); });
}
};
if (type.isKnownDataType()) {
assert(IS_REFCOUNTED_TYPE(type.toDataType()));
increfMaybeStatic();
} else {
emitCmpTVType(m_as, KindOfRefCountThreshold, m_regs[src].reg(1));
ifThen(m_as, CC_NLE, [&] { increfMaybeStatic(); });
}
}
void CodeGenerator::cgIncRef(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* src = inst->src(0);
Type type = src->type();
cgIncRefWork(type, src);
shuffle2(m_as, m_regs[src].reg(0), m_regs[src].reg(1),
m_regs[dst].reg(0), m_regs[dst].reg(1));
}
void CodeGenerator::cgDecRefStack(IRInstruction* inst) {
cgDecRefMem(inst->typeParam(),
m_regs[inst->src(0)].reg(),
cellsToBytes(inst->extra<DecRefStack>()->offset),
nullptr);
}
void CodeGenerator::cgDecRefThis(IRInstruction* inst) {
SSATmp* fp = inst->src(0);
Block* exit = inst->taken();
auto fpReg = m_regs[fp].reg();
auto scratchReg = m_rScratch;
// Load AR->m_this into m_rScratch
m_as.loadq(fpReg[AROFF(m_this)], scratchReg);
auto decrefIfAvailable = [&] {
// Check if this is available and we're not in a static context instead
m_as.testb(1, rbyte(scratchReg));
ifThen(m_as, CC_Z, [&] {
cgDecRefStaticType(
Type::Obj,
scratchReg,
exit,
true /* genZeroCheck */
);
});
};
if (curFunc()->isPseudoMain()) {
// In pseudo-mains, emit check for presence of m_this
m_as.testq(scratchReg, scratchReg);
ifThen(m_as, CC_NZ, [&] { decrefIfAvailable(); });
} else {
decrefIfAvailable();
}
}
void CodeGenerator::cgDecRefLoc(IRInstruction* inst) {
cgDecRefMem(inst->typeParam(),
m_regs[inst->src(0)].reg(),
localOffset(inst->extra<DecRefLoc>()->locId),
inst->taken());
}
void CodeGenerator::cgGenericRetDecRefs(IRInstruction* inst) {
auto const rFp = m_regs[inst->src(0)].reg();
auto const numLocals = inst->src(1)->getValInt();
auto const rDest = m_regs[inst->dst()].reg();
auto& a = m_as;
assert(rFp == rVmFp &&
"free locals helper assumes the frame pointer is rVmFp");
assert(rDest == rVmSp &&
"free locals helper adjusts rVmSp, which must be our dst reg");
if (numLocals == 0) return;
// The helpers called below use a special ABI, in which r15 is not saved.
// So save r15 on the stack if it's live.
bool saveR15 = m_state.liveRegs[inst].contains(r15);
int stackAdjust = 8;
if (saveR15) {
a.push(r15);
stackAdjust = 16;
}
auto const target = numLocals > kNumFreeLocalsHelpers
? m_tx64->m_freeManyLocalsHelper
: m_tx64->m_freeLocalsHelpers[numLocals - 1];
a.subq(stackAdjust, rsp); // For parity; callee does retq $0x8.
a.lea(rFp[-numLocals * sizeof(TypedValue)], rDest);
a.call(target);
recordSyncPoint(a);
if (saveR15) {
a.addq(8, rsp);
a.pop(r15);
}
}
static void
tv_release_generic(TypedValue* tv) {
assert(Transl::tx64->stateIsDirty());
assert(tv->m_type >= KindOfString && tv->m_type <= KindOfRef);
g_destructors[typeToDestrIndex(tv->m_type)](tv->m_data.pref);
}
static void
tv_release_typed(RefData* pv, DataType dt) {
assert(Transl::tx64->stateIsDirty());
assert(dt >= KindOfString && dt <= KindOfRef);
g_destructors[typeToDestrIndex(dt)](pv);
}
Address CodeGenerator::getDtorGeneric() {
return (Address)tv_release_generic;
}
Address CodeGenerator::getDtorTyped() {
return (Address)tv_release_typed;
}
//
// This method generates code that checks the static bit and jumps if the bit
// is set. If regIsCount is true, reg contains the _count field. Otherwise,
// it's assumed to contain m_data field.
//
// Return value: the address to be patched with the address to jump to in case
// the static bit is set. If the check is unnecessary, this method retuns NULL.
Address CodeGenerator::cgCheckStaticBit(Type type,
PhysReg reg,
bool regIsCount) {
if (!type.needsStaticBitCheck()) return NULL;
if (regIsCount) {
// reg has the _count value
m_as.cmp_imm32_reg32(RefCountStaticValue, reg);
} else {
// reg has the data pointer
m_as.cmp_imm32_disp_reg32(RefCountStaticValue, FAST_REFCOUNT_OFFSET, reg);
}
Address addrToPatch = m_as.code.frontier;
m_as.jcc8(CC_E, addrToPatch);
return addrToPatch;
}
//
// Using the given dataReg, this method generates code that checks the static
// bit out of dataReg, and emits a DecRef if needed.
// NOTE: the flags are left with the result of the DecRef's subtraction,
// which can then be tested immediately after this.
//
// Return value: the address to be patched if a RefCountedStaticValue check is
// emitted; NULL otherwise.
//
Address CodeGenerator::cgCheckStaticBitAndDecRef(Type type,
PhysReg dataReg,
Block* exit) {
assert(type.maybeCounted());
Address patchStaticCheck = nullptr;
const auto scratchReg = m_rScratch;
bool canUseScratch = dataReg != scratchReg;
// TODO: run experiments to check whether the 'if' code sequence
// is any better than the 'else' branch below; otherwise, always
// use the 'else' code sequence
if (type.needsStaticBitCheck() && canUseScratch) {
// If we need to check for static value, then load the _count into a
// register to avoid doing two loads. The generated sequence is:
//
// scratchReg = [dataReg + offset(_count)]
// if scratchReg == RefCountStaticValue then skip DecRef
// scratchReg = scratchReg - 1
// ( if exit != NULL, emit:
// jz exit
// )
// [dataReg + offset(_count)] = scratchReg
if (RuntimeOption::EvalHHIRGenerateAsserts) {
emitAssertRefCount(m_as, dataReg);
}
// Load _count in scratchReg
m_as.loadl(dataReg[FAST_REFCOUNT_OFFSET], r32(scratchReg));
// Check for RefCountStaticValue
patchStaticCheck = cgCheckStaticBit(type, scratchReg,
true /* reg has _count */);
// Decrement count and store it back in memory.
// If there's an exit, emit jump to it when _count would get down to 0
m_as.decq(scratchReg);
if (exit) {
emitFwdJcc(CC_E, exit);
}
if (RuntimeOption::EvalHHIRGenerateAsserts) {
// Assert that the ref count is greater than zero
emitAssertFlagsNonNegative(m_as);
}
m_as.store_reg32_disp_reg64(scratchReg, FAST_REFCOUNT_OFFSET, dataReg);
} else {
// Can't use scratch reg, so emit code that operates directly in
// memory. Compared to the sequence above, this will result in one
// extra load, but it has the advantage of producing a instruction
// sequence.
//
// ( if needStaticBitCheck, emit :
// cmp [dataReg + offset(_count)], RefCountStaticValue
// je LabelAfterDecRef
// )
// ( if exit != NULL, emit:
// cmp [dataReg + offset(_count)], 1
// jz exit
// )
// sub [dataReg + offset(_count)], 1
// If necessary, check for RefCountStaticValue
patchStaticCheck = cgCheckStaticBit(type, dataReg,
false /* passing dataReg */);
// If there's an exit, emit jump to it if _count would get down to 0
if (exit) {
m_as.cmp_imm32_disp_reg32(1, FAST_REFCOUNT_OFFSET, dataReg);
emitFwdJcc(CC_E, exit);
}
if (RuntimeOption::EvalHHIRGenerateAsserts) {
emitAssertRefCount(m_as, dataReg);
}
// Decrement _count
m_as.decl(dataReg[FAST_REFCOUNT_OFFSET]);
if (RuntimeOption::EvalHHIRGenerateAsserts) {
// Assert that the ref count is not less than zero
emitAssertFlagsNonNegative(m_as);
}
}
return patchStaticCheck;
}
//
// Returns the address to be patched with the address to jump to in case
// the type is not ref-counted.
//
Address CodeGenerator::cgCheckRefCountedType(PhysReg typeReg) {
emitCmpTVType(m_as, KindOfRefCountThreshold, typeReg);
Address addrToPatch = m_as.code.frontier;
m_as.jcc8(CC_LE, addrToPatch);
return addrToPatch;
}
Address CodeGenerator::cgCheckRefCountedType(PhysReg baseReg, int64_t offset) {
emitCmpTVType(m_as, KindOfRefCountThreshold, baseReg[offset + TVOFF(m_type)]);
Address addrToPatch = m_as.code.frontier;
m_as.jcc8(CC_LE, addrToPatch);
return addrToPatch;
}
//
// Generates dec-ref of a typed value with statically known type.
//
void CodeGenerator::cgDecRefStaticType(Type type,
PhysReg dataReg,
Block* exit,
bool genZeroCheck) {
assert(type != Type::Cell && type != Type::Gen);
assert(type.isKnownDataType());
if (type.notCounted()) return;
// Check for RefCountStaticValue if needed, do the actual DecRef,
// and leave flags set based on the subtract result, which is
// tested below
Address patchStaticCheck;
if (genZeroCheck) {
patchStaticCheck = cgCheckStaticBitAndDecRef(type, dataReg, exit);
} else {
// Set exit as NULL so that the code doesn't jump to error checking.
patchStaticCheck = cgCheckStaticBitAndDecRef(type, dataReg, nullptr);
}
// If not exiting on count down to zero, emit the zero-check and
// release call
if (genZeroCheck && exit == nullptr) {
// Emit jump to m_astubs (to call release) if count got down to zero
unlikelyIfBlock(CC_Z, [&] (Asm& a) {
// Emit the call to release in m_astubs
cgCallHelper(a, m_tx64->getDtorCall(type.toDataType()),
InvalidReg, InvalidReg, kSyncPoint,
ArgGroup(m_regs).reg(dataReg));
});
}
if (patchStaticCheck) {
m_as.patchJcc8(patchStaticCheck, m_as.code.frontier);
}
}
//
// Generates dec-ref of a typed value with dynamic (statically unknown) type,
// when the type is stored in typeReg.
//
void CodeGenerator::cgDecRefDynamicType(PhysReg typeReg,
PhysReg dataReg,
Block* exit,
bool genZeroCheck) {
// Emit check for ref-counted type
Address patchTypeCheck = cgCheckRefCountedType(typeReg);
// Emit check for RefCountStaticValue and the actual DecRef
Address patchStaticCheck;
if (genZeroCheck) {
patchStaticCheck = cgCheckStaticBitAndDecRef(Type::Cell, dataReg, exit);
} else {
patchStaticCheck = cgCheckStaticBitAndDecRef(Type::Cell, dataReg, nullptr);
}
// If not exiting on count down to zero, emit the zero-check and release call
if (genZeroCheck && exit == nullptr) {
// Emit jump to m_astubs (to call release) if count got down to zero
unlikelyIfBlock(CC_Z, [&] (Asm& a) {
// Emit call to release in m_astubs
cgCallHelper(a, getDtorTyped(), InvalidReg, kSyncPoint,
ArgGroup(m_regs).reg(dataReg).reg(typeReg));
});
}
// Patch checks to jump around the DecRef
if (patchTypeCheck) m_as.patchJcc8(patchTypeCheck, m_as.code.frontier);
if (patchStaticCheck) m_as.patchJcc8(patchStaticCheck, m_as.code.frontier);
}
//
// Generates dec-ref of a typed value with dynamic (statically
// unknown) type, when all we have is the baseReg and offset of
// the typed value. This method assumes that baseReg is not the
// scratch register.
//
void CodeGenerator::cgDecRefDynamicTypeMem(PhysReg baseReg,
int64_t offset,
Block* exit) {
auto scratchReg = m_rScratch;
assert(baseReg != scratchReg);
// Emit check for ref-counted type
Address patchTypeCheck = cgCheckRefCountedType(baseReg, offset);
if (exit == nullptr && RuntimeOption::EvalHHIRGenericDtorHelper) {
{
// This PhysRegSaverStub saves rdi redundantly if
// !liveRegs[m_curInst].contains(rdi), but its
// necessary to maintain stack alignment. We can do better
// by making the helpers adjust the stack for us in the cold
// path, which calls the destructor.
PhysRegSaverStub regSaver(m_as, RegSet(rdi));
/*
* rVmSp is ok here because this is part of the special
* ABI to m_irPopRHelper. We're not using a secret dependency
* on the frame or stack---we're only going to use that ABI if
* we happen to have that register allocated for baseReg.
*/
if (offset == 0 && baseReg == rVmSp) {
// Decref'ing top of vm stack, very likely a popR
m_tx64->emitCall(m_as, m_tx64->m_irPopRHelper);
} else {
if (baseReg == rsp) {
// Because we just pushed %rdi, %rsp is 8 bytes below where
// offset is expecting it to be.
offset += sizeof(int64_t);
}
m_as.lea(baseReg[offset], rdi);
m_tx64->emitCall(m_as, m_tx64->m_dtorGenericStub);
}
recordSyncPoint(m_as);
}
if (patchTypeCheck) {
m_as.patchJcc8(patchTypeCheck, m_as.code.frontier);
}
return;
}
// Load m_data into the scratch reg
m_as.loadq(baseReg[offset + TVOFF(m_data)], scratchReg);
// Emit check for RefCountStaticValue and the actual DecRef
Address patchStaticCheck = cgCheckStaticBitAndDecRef(Type::Cell, scratchReg,
exit);
// If not exiting on count down to zero, emit the zero-check and release call
if (exit == nullptr) {
// Emit jump to m_astubs (to call release) if count got down to zero
unlikelyIfBlock(CC_Z, [&] (Asm& a) {
// Emit call to release in m_astubs
a.lea(baseReg[offset], scratchReg);
cgCallHelper(a, getDtorGeneric(), InvalidReg, kSyncPoint,
ArgGroup(m_regs).reg(scratchReg));
});
}
// Patch checks to jump around the DecRef
if (patchTypeCheck) m_as.patchJcc8(patchTypeCheck, m_as.code.frontier);
if (patchStaticCheck) m_as.patchJcc8(patchStaticCheck, m_as.code.frontier);
}
//
// Generates the dec-ref of a typed value in memory address [baseReg + offset].
// This handles cases where type is either static or dynamic.
//
void CodeGenerator::cgDecRefMem(Type type,
PhysReg baseReg,
int64_t offset,
Block* exit) {
auto scratchReg = m_rScratch;
assert(baseReg != scratchReg);
if (type.needsReg()) {
// The type is dynamic, but we don't have two registers available
// to load the type and the data.
cgDecRefDynamicTypeMem(baseReg, offset, exit);
} else if (type.maybeCounted()) {
m_as.loadq(baseReg[offset + TVOFF(m_data)], scratchReg);
cgDecRefStaticType(type, scratchReg, exit, true);
}
}
void CodeGenerator::cgDecRefMem(IRInstruction* inst) {
assert(inst->src(0)->type().isPtr());
cgDecRefMem(inst->typeParam(),
m_regs[inst->src(0)].reg(),
inst->src(1)->getValInt(),
inst->taken());
}
void CodeGenerator::cgDecRefWork(IRInstruction* inst, bool genZeroCheck) {
SSATmp* src = inst->src(0);
if (!isRefCounted(src)) return;
Block* exit = inst->taken();
Type type = src->type();
if (type.isKnownDataType()) {
cgDecRefStaticType(type, m_regs[src].reg(), exit, genZeroCheck);
} else {
cgDecRefDynamicType(m_regs[src].reg(1),
m_regs[src].reg(0),
exit,
genZeroCheck);
}
}
void CodeGenerator::cgDecRef(IRInstruction *inst) {
// DecRef may bring the count to zero, and run the destructor.
// Generate code for this.
assert(!inst->taken());
cgDecRefWork(inst, true);
}
void CodeGenerator::cgDecRefNZ(IRInstruction* inst) {
// DecRefNZ cannot bring the count to zero.
// Therefore, we don't generate zero-checking code.
assert(!inst->taken());
cgDecRefWork(inst, false);
}
void CodeGenerator::cgDecRefNZOrBranch(IRInstruction* inst) {
assert(inst->taken());
cgDecRefWork(inst, true);
}
void CodeGenerator::cgCufIterSpillFrame(IRInstruction* inst) {
auto const sp = inst->src(0);
auto const fp = inst->src(1);
auto const nArgs = inst->extra<CufIterSpillFrame>()->args;
auto const iterId = inst->extra<CufIterSpillFrame>()->iterId;
auto const itOff = iterOffset(iterId);
const int64_t spOffset = -kNumActRecCells * sizeof(Cell);
auto spReg = m_regs[sp].reg();
auto fpReg = m_regs[fp].reg();
m_as.loadq (fpReg[itOff + CufIter::funcOff()], m_rScratch);
m_as.storeq (m_rScratch, spReg[spOffset + int(AROFF(m_func))]);
m_as.loadq (fpReg[itOff + CufIter::ctxOff()], m_rScratch);
m_as.storeq (m_rScratch, spReg[spOffset + int(AROFF(m_this))]);
m_as.shrq (1, m_rScratch);
ifThen(m_as, CC_NBE, [this] {
m_as.shlq(1, m_rScratch);
emitIncRef(m_as, m_rScratch);
});
m_as.loadq (fpReg[itOff + CufIter::nameOff()], m_rScratch);
m_as.testq (m_rScratch, m_rScratch);
ifThen(m_as, CC_NZ, [this] {
m_as.cmpl(RefCountStaticValue, m_rScratch[FAST_REFCOUNT_OFFSET]);
ifThen(m_as, CC_NE, [&] { emitIncRef(m_as, m_rScratch); });
m_as.orq (ActRec::kInvNameBit, m_rScratch);
});
m_as.storeq (m_rScratch, spReg[spOffset + int(AROFF(m_invName))]);
m_as.storeq (fpReg, spReg[spOffset + int(AROFF(m_savedRbp))]);
m_as.storel (nArgs, spReg[spOffset + int(AROFF(m_numArgsAndCtorFlag))]);
emitAdjustSp(spReg,
m_regs[inst->dst()].reg(),
spOffset);
}
void CodeGenerator::cgSpillFrame(IRInstruction* inst) {
auto const sp = inst->src(0);
auto const fp = inst->src(1);
auto const func = inst->src(2);
auto const objOrCls = inst->src(3);
auto const magicName = inst->extra<SpillFrame>()->invName;
auto const nArgs = inst->extra<SpillFrame>()->numArgs;
const int64_t spOffset = -kNumActRecCells * sizeof(Cell);
DEBUG_ONLY bool setThis = true;
auto spReg = m_regs[sp].reg();
// actRec->m_this
if (objOrCls->isA(Type::Cls)) {
// store class
if (objOrCls->isConst()) {
m_as.store_imm64_disp_reg64(uintptr_t(objOrCls->getValClass()) | 1,
spOffset + int(AROFF(m_this)),
spReg);
} else {
Reg64 clsPtrReg = m_regs[objOrCls].reg();
m_as.movq (clsPtrReg, m_rScratch);
m_as.orq (1, m_rScratch);
m_as.storeq(m_rScratch, spReg[spOffset + int(AROFF(m_this))]);
}
} else if (objOrCls->isA(Type::Obj)) {
// store this pointer
m_as.store_reg64_disp_reg64(m_regs[objOrCls].reg(),
spOffset + int(AROFF(m_this)),
spReg);
} else if (objOrCls->isA(Type::Ctx)) {
// Stores either a this pointer or a Cctx -- statically unknown.
Reg64 objOrClsPtrReg = m_regs[objOrCls].reg();
m_as.storeq(objOrClsPtrReg, spReg[spOffset + int(AROFF(m_this))]);
} else {
assert(objOrCls->isA(Type::InitNull));
// no obj or class; this happens in FPushFunc
int offset_m_this = spOffset + int(AROFF(m_this));
// When func is either Type::FuncCls or Type::FuncCtx,
// m_this/m_cls will be initialized below
if (!func->isConst() && (func->isA(Type::FuncCtx))) {
// m_this is unioned with m_cls and will be initialized below
setThis = false;
} else {
m_as.store_imm64_disp_reg64(0, offset_m_this, spReg);
}
}
// actRec->m_invName
// ActRec::m_invName is encoded as a pointer with bit kInvNameBit
// set to distinguish it from m_varEnv and m_extrArgs
uintptr_t invName = !magicName
? 0
: reinterpret_cast<uintptr_t>(magicName) | ActRec::kInvNameBit;
m_as.store_imm64_disp_reg64(invName,
spOffset + int(AROFF(m_invName)),
spReg);
// actRec->m_func and possibly actRec->m_cls
// Note m_cls is unioned with m_this and may overwrite previous value
if (func->type().isNull()) {
assert(func->isConst());
} else if (func->isConst()) {
const Func* f = func->getValFunc();
m_as. mov_imm64_reg((uint64_t)f, m_rScratch);
m_as.store_reg64_disp_reg64(m_rScratch,
spOffset + int(AROFF(m_func)),
spReg);
if (func->isA(Type::FuncCtx)) {
// Fill in m_cls if provided with both func* and class*
CG_PUNT(cgAllocActRec);
}
} else {
int offset_m_func = spOffset + int(AROFF(m_func));
m_as.store_reg64_disp_reg64(m_regs[func].reg(0),
offset_m_func,
spReg);
if (func->isA(Type::FuncCtx)) {
int offset_m_cls = spOffset + int(AROFF(m_cls));
m_as.store_reg64_disp_reg64(m_regs[func].reg(1),
offset_m_cls,
spReg);
setThis = true; /* m_this and m_cls are in a union */
}
}
assert(setThis);
// actRec->m_savedRbp
m_as.store_reg64_disp_reg64(m_regs[fp].reg(),
spOffset + int(AROFF(m_savedRbp)),
spReg);
// actRec->m_numArgsAndCtorFlag
m_as.store_imm32_disp_reg(nArgs,
spOffset + int(AROFF(m_numArgsAndCtorFlag)),
spReg);
emitAdjustSp(spReg,
m_regs[inst->dst()].reg(),
spOffset);
}
const Func* loadClassCtor(Class* cls) {
const Func* f = cls->getCtor();
if (UNLIKELY(!(f->attrs() & AttrPublic))) {
VMRegAnchor _;
UNUSED MethodLookup::LookupResult res =
g_vmContext->lookupCtorMethod(f, cls, true /*raise*/);
assert(res == MethodLookup::MethodFoundWithThis);
}
return f;
}
Instance* createClHelper(Class* cls, int numArgs, ActRec* ar, TypedValue* sp) {
Instance* newObj = newInstance(cls);
newObj->incRefCount();
return static_cast<c_Closure*>(newObj)->init(numArgs, ar, sp);
}
void CodeGenerator::cgAllocObjFast(IRInstruction* inst) {
const Class* cls = inst->src(0)->getValClass();
auto dstReg = m_regs[inst->dst()].reg();
// First, make sure our property init vectors are all set up
bool props = cls->pinitVec().size() > 0;
bool sprops = cls->numStaticProperties() > 0;
assert((props || sprops) == cls->needInitialization());
if (cls->needInitialization()) {
if (props) {
cls->initPropHandle();
m_as.testq(-1, rVmTl[cls->propHandle()]);
unlikelyIfBlock(CC_Z, [&] (Asm& a) {
cgCallHelper(a,
(TCA)getMethodPtr(&Class::initProps),
InvalidReg,
kSyncPoint,
ArgGroup(m_regs).imm((uint64_t)cls));
});
}
if (sprops) {
cls->initSPropHandle();
m_as.testq(-1, rVmTl[cls->sPropHandle()]);
unlikelyIfBlock(CC_Z, [&] (Asm& a) {
cgCallHelper(a,
(TCA)getMethodPtr(&Class::initSProps),
InvalidReg,
kSyncPoint,
ArgGroup(m_regs).imm((uint64_t)cls));
});
}
}
// Next, allocate the object
if (cls->instanceCtor()) {
cgCallHelper(m_as,
(TCA)cls->instanceCtor(),
dstReg,
kSyncPoint,
ArgGroup(m_regs).imm((uint64_t)cls));
} else {
size_t size = Instance::sizeForNProps(cls->numDeclProperties());
int allocator = object_alloc_size_to_index(size);
assert(allocator != -1);
cgCallHelper(m_as,
(TCA)getMethodPtr(&Instance::newInstanceRaw),
dstReg,
kSyncPoint,
ArgGroup(m_regs).imm((uint64_t)cls).imm(allocator));
}
// Set the attributes, if any
int odAttrs = cls->getODAttrs();
if (odAttrs) {
// o_attribute is 16 bits but the fact that we're or-ing a mask makes it ok
assert(!(odAttrs & 0xffff0000));
m_as.orq(odAttrs, dstReg[ObjectData::attributeOff()]);
}
// Initialize the properties
size_t nProps = cls->numDeclProperties();
if (nProps > 0) {
m_as.push(dstReg);
m_as.subq(8, reg::rsp);
if (cls->pinitVec().size() == 0) {
// Fast case: copy from a known address in the Class
ArgGroup args = ArgGroup(m_regs)
.addr(dstReg, sizeof(ObjectData) + cls->builtinPropSize())
.imm(int64_t(&cls->declPropInit()[0]))
.imm(cellsToBytes(nProps));
cgCallHelper(m_as,
(TCA)memcpy,
InvalidReg,
kNoSyncPoint,
args);
} else {
// Slower case: we have to load the src address from the targetcache
auto rPropData = m_rScratch;
// Load the Class's propInitVec from the targetcache
m_as.loadq(rVmTl[cls->propHandle()], rPropData);
// propData holds the PropInitVec. We want &(*propData)[0]
m_as.loadq(rPropData[Class::PropInitVec::dataOff()], rPropData);
if (!cls->hasDeepInitProps()) {
ArgGroup args = ArgGroup(m_regs)
.addr(dstReg, sizeof(ObjectData) + cls->builtinPropSize())
.reg(rPropData)
.imm(cellsToBytes(nProps));
cgCallHelper(m_as,
(TCA)memcpy,
InvalidReg,
kNoSyncPoint,
args);
} else {
ArgGroup args = ArgGroup(m_regs)
.addr(dstReg, sizeof(ObjectData) + cls->builtinPropSize())
.reg(rPropData)
.imm(nProps);
cgCallHelper(m_as,
(TCA)deepInitHelper,
InvalidReg,
kNoSyncPoint,
args);
}
}
m_as.addq(8, reg::rsp);
m_as.pop(dstReg);
}
if (cls->callsCustomInstanceInit()) {
// callCustomInstanceInit returns the instance in rax
cgCallHelper(m_as,
(TCA)getMethodPtr(&Instance::callCustomInstanceInit),
dstReg,
kSyncPoint,
ArgGroup(m_regs).reg(dstReg));
}
}
void CodeGenerator::cgInlineCreateCont(IRInstruction* inst) {
auto const& data = *inst->extra<InlineCreateCont>();
auto const helper = data.origFunc->isMethod()
? &VMExecutionContext::createContinuationHelper<true>
: &VMExecutionContext::createContinuationHelper<false>;
cgCallHelper(
m_as,
reinterpret_cast<TCA>(helper),
inst->dst(),
kSyncPoint,
ArgGroup(m_regs)
.immPtr(data.origFunc)
.immPtr(data.genFunc)
.ssa(inst->src(0))
// Deliberately ignoring frameStaticClass parameter, because
// it's unused if we have a $this pointer, and we don't inline
// functions with a null $this.
);
if (data.origFunc->isMethod()) {
// We can't support a null $this.
assert(inst->src(0)->isA(Type::Obj));
}
}
void CodeGenerator::cgCallArray(IRInstruction* inst) {
Offset pc = inst->extra<CallArray>()->pc;
Offset after = inst->extra<CallArray>()->after;
ArgGroup args(m_regs);
args.imm(pc).imm(after);
// fCallArrayHelper makes the actual call by smashing its return address.
cgCallHelper(m_as, (TCA)TranslatorX64::fCallArrayHelper,
nullptr, kSyncPoint, args);
}
void CodeGenerator::cgCall(IRInstruction* inst) {
SSATmp* actRec = inst->src(0);
SSATmp* returnBcOffset = inst->src(1);
SSATmp* func = inst->src(2);
SrcRange args = inst->srcs().subpiece(3);
int32_t numArgs = args.size();
auto spReg = m_regs[actRec].reg();
// put all outgoing arguments onto the VM stack
int64_t adjustment = (-(int64_t)numArgs) * sizeof(Cell);
for (int32_t i = 0; i < numArgs; i++) {
// Type::None here means that the simplifier proved that the value
// matches the value already in memory, thus the store is redundant.
if (args[i]->type() != Type::None) {
cgStore(spReg, -(i + 1) * sizeof(Cell), args[i]);
}
}
// store the return bytecode offset into the outgoing actrec
uint64_t returnBc = returnBcOffset->getValInt();
m_as.store_imm32_disp_reg(returnBc, AROFF(m_soff), spReg);
if (adjustment != 0) {
m_as.add_imm32_reg64(adjustment, spReg);
}
assert(m_state.lastMarker);
SrcKey srcKey = SrcKey(m_state.lastMarker->func, m_state.lastMarker->bcOff);
bool isImmutable = (func->isConst() && !func->type().isNull());
const Func* funcd = isImmutable ? func->getValFunc() : nullptr;
assert(&m_as == &m_tx64->getAsm());
int32_t adjust = m_tx64->emitBindCall(srcKey, funcd, numArgs);
if (adjust) {
m_as.addq (adjust, rVmSp);
}
}
void CodeGenerator::cgCastStk(IRInstruction *inst) {
Type type = inst->typeParam();
SSATmp* sp = inst->src(0);
uint32_t offset = inst->extra<CastStk>()->offset;
PhysReg spReg = m_regs[sp].reg();
ArgGroup args(m_regs);
args.addr(spReg, cellsToBytes(offset));
TCA tvCastHelper;
if (type.subtypeOf(Type::Bool)) {
tvCastHelper = (TCA)tvCastToBooleanInPlace;
} else if (type.subtypeOf(Type::Int)) {
// if casting to integer, pass 10 as the base for the conversion
args.imm(10);
tvCastHelper = (TCA)tvCastToInt64InPlace;
} else if (type.subtypeOf(Type::Dbl)) {
tvCastHelper = (TCA)tvCastToDoubleInPlace;
} else if (type.subtypeOf(Type::Arr)) {
tvCastHelper = (TCA)tvCastToArrayInPlace;
} else if (type.subtypeOf(Type::Str)) {
tvCastHelper = (TCA)tvCastToStringInPlace;
} else if (type.subtypeOf(Type::Obj)) {
tvCastHelper = (TCA)tvCastToObjectInPlace;
} else {
not_reached();
}
cgCallHelper(m_as, tvCastHelper, nullptr,
kSyncPoint, args, DestType::None);
}
void CodeGenerator::cgCallBuiltin(IRInstruction* inst) {
SSATmp* f = inst->src(0);
auto args = inst->srcs().subpiece(2);
int32_t numArgs = args.size();
SSATmp* dst = inst->dst();
auto dstReg = m_regs[dst].reg(0);
auto dstType = m_regs[dst].reg(1);
Type returnType = inst->typeParam();
const Func* func = f->getValFunc();
DataType funcReturnType = func->returnType();
int returnOffset = HHIR_MISOFF(tvBuiltinReturn);
if (TranslatorX64::eagerRecord(func)) {
const uchar* pc = curUnit()->entry() + m_state.lastMarker->bcOff;
// we have spilled all args to stack, so spDiff is 0
m_tx64->emitEagerSyncPoint(m_as, pc, 0);
}
// RSP points to the MInstrState we need to use.
// workaround the fact that rsp moves when we spill registers around call
PhysReg misReg = m_rScratch;
emitMovRegReg(m_as, reg::rsp, misReg);
ArgGroup callArgs(m_regs);
if (isCppByRef(funcReturnType)) {
// first arg is pointer to storage for that return value
if (isSmartPtrRef(funcReturnType)) {
returnOffset += TVOFF(m_data);
}
// misReg is pointing to an MInstrState struct on the C stack. Pass
// the address of tvBuiltinReturn to the native function as the location
// it can construct the return Array, String, Object, or Variant.
callArgs.addr(misReg, returnOffset); // &misReg[returnOffset]
}
// non-pointer args are plain values passed by value. String, Array,
// Object, and Variant are passed by const&, ie a pointer to stack memory
// holding the value, so expect PtrToT types for these.
// Pointers to smartptr types (String, Array, Object) need adjusting to
// point to &ptr->m_data.
for (int i = 0; i < numArgs; i++) {
const Func::ParamInfo& pi = func->params()[i];
if (TVOFF(m_data) && isSmartPtrRef(pi.builtinType())) {
assert(args[i]->type().isPtr() && m_regs[args[i]].reg() != InvalidReg);
callArgs.addr(m_regs[args[i]].reg(), TVOFF(m_data));
} else {
callArgs.ssa(args[i]);
}
}
// if the return value is returned by reference, we don't need the
// return value from this call since we know where the value is.
cgCallHelper(m_as, Transl::CppCall((TCA)func->nativeFuncPtr()),
isCppByRef(funcReturnType) ? InvalidReg : dstReg,
kSyncPoint, callArgs);
// load return value from builtin
// for primitive return types (int, bool), the return value
// is already in dstReg (the builtin call returns in rax). For return
// by reference (String, Object, Array, Variant), the builtin writes the
// return value into MInstrState::tvBuiltinReturn TV, from where it
// has to be tested and copied.
if (dstReg == InvalidReg || returnType.isSimpleType()) {
return;
}
// after the call, RSP is back pointing to MInstrState and rSratch
// has been clobberred.
misReg = rsp;
if (returnType.isReferenceType()) {
assert(isCppByRef(funcReturnType) && isSmartPtrRef(funcReturnType));
// return type is String, Array, or Object; fold nullptr to KindOfNull
m_as. loadq (misReg[returnOffset], dstReg);
emitLoadImm(m_as, returnType.toDataType(), dstType);
emitLoadImm(m_as, KindOfNull, m_rScratch);
m_as. testq (dstReg, dstReg);
m_as. cmov_reg64_reg64 (CC_Z, m_rScratch, dstType);
return;
}
if (returnType.subtypeOf(Type::Cell)
|| returnType.subtypeOf(Type::BoxedCell)) {
// return type is Variant; fold KindOfUninit to KindOfNull
assert(isCppByRef(funcReturnType) && !isSmartPtrRef(funcReturnType));
assert(misReg != dstType);
emitLoadTVType(m_as, misReg[returnOffset + TVOFF(m_type)], dstType);
m_as. loadq (misReg[returnOffset + TVOFF(m_data)], dstReg);
emitLoadImm(m_as, KindOfNull, m_rScratch);
static_assert(KindOfUninit == 0, "KindOfUninit must be 0 for test");
m_as. testb (rbyte(dstType), rbyte(dstType));
m_as. cmov_reg64_reg64 (CC_Z, m_rScratch, dstType);
return;
}
not_reached();
}
void CodeGenerator::cgSpillStack(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* sp = inst->src(0);
auto const spDeficit = inst->src(1)->getValInt();
auto const spillVals = inst->srcs().subpiece(2);
auto const numSpillSrcs = spillVals.size();
auto const dstReg = m_regs[dst].reg();
auto const spReg = m_regs[sp].reg();
auto const spillCells = spillValueCells(inst);
int64_t adjustment = (spDeficit - spillCells) * sizeof(Cell);
for (uint32_t i = 0; i < numSpillSrcs; ++i) {
const int64_t offset = i * sizeof(Cell) + adjustment;
if (spillVals[i]->type() == Type::None) {
// The simplifier detected that we're storing the same value
// already in there.
continue;
}
auto* val = spillVals[i];
auto* inst = val->inst();
while (inst->isPassthrough()) {
inst = inst->getPassthroughValue()->inst();
}
// If our value came from a LdStack on the same sp and offset,
// we don't need to spill it.
if (inst->op() == LdStack && inst->src(0) == sp &&
inst->extra<LdStack>()->offset * sizeof(Cell) == offset) {
FTRACE(6, "{}: Not spilling spill value {} from {}\n",
__func__, i, inst->toString());
} else {
cgStore(spReg, offset, val);
}
}
emitAdjustSp(spReg, dstReg, adjustment);
}
void CodeGenerator::emitAdjustSp(PhysReg spReg,
PhysReg dstReg,
int64_t adjustment /* bytes */) {
if (adjustment != 0) {
if (dstReg != spReg) {
m_as. lea (spReg[adjustment], dstReg);
} else {
m_as. addq (adjustment, dstReg);
}
} else {
emitMovRegReg(m_as, spReg, dstReg);
}
}
void CodeGenerator::cgNativeImpl(IRInstruction* inst) {
SSATmp* func = inst->src(0);
SSATmp* fp = inst->src(1);
assert(func->isConst());
assert(func->type() == Type::Func);
const Func* fn = func->getValFunc();
BuiltinFunction builtinFuncPtr = func->getValFunc()->builtinFuncPtr();
emitMovRegReg(m_as, m_regs[fp].reg(), argNumToRegName[0]);
if (TranslatorX64::eagerRecord(fn)) {
m_tx64->emitEagerSyncPoint(m_as, fn->getEntry(), 0);
}
m_as.call((TCA)builtinFuncPtr);
recordSyncPoint(m_as);
}
void CodeGenerator::cgLdThis(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* src = inst->src(0);
Block* label = inst->taken();
// mov dst, [fp + 0x20]
auto dstReg = m_regs[dst].reg();
// the destination of LdThis could be dead but the instruction
// itself still useful because of the checks that it does (if it has
// a label). So we need to make sure there is a dstReg for this
// instruction.
if (dstReg != InvalidReg) {
// instruction's result is not dead
m_as.loadq(m_regs[src].reg()[AROFF(m_this)], dstReg);
}
if (label == NULL) return; // no need to perform its checks
if (dstReg != InvalidReg) {
// test 0x01, dst
m_as.testb(1, rbyte(dstReg));
} else {
m_as.testb(1, m_regs[src].reg()[AROFF(m_this)]);
}
// jnz label
emitFwdJcc(CC_NZ, label);
}
static void emitLdClsCctx(CodeGenerator::Asm& a,
PhysReg srcReg,
PhysReg dstReg) {
emitMovRegReg(a, srcReg, dstReg);
a. decq(dstReg);
}
void CodeGenerator::cgLdClsCtx(IRInstruction* inst) {
PhysReg srcReg = m_regs[inst->src(0)].reg();
PhysReg dstReg = m_regs[inst->dst()].reg();
// Context could be either a this object or a class ptr
m_as. testb(1, rbyte(srcReg));
ifThenElse(CC_NZ,
[&] { emitLdClsCctx(m_as, srcReg, dstReg); }, // ctx is a class
[&] { emitLdObjClass(m_as, srcReg, dstReg); } // ctx is this ptr
);
}
void CodeGenerator::cgLdClsCctx(IRInstruction* inst) {
PhysReg srcReg = m_regs[inst->src(0)].reg();
PhysReg dstReg = m_regs[inst->dst()].reg();
emitLdClsCctx(m_as, srcReg, dstReg);
}
void CodeGenerator::cgLdCtx(IRInstruction* inst) {
PhysReg dstReg = m_regs[inst->dst()].reg();
PhysReg srcReg = m_regs[inst->src(0)].reg();
if (dstReg != InvalidReg) {
m_as.loadq(srcReg[AROFF(m_this)], dstReg);
}
}
void CodeGenerator::cgLdCctx(IRInstruction* inst) {
return cgLdCtx(inst);
}
void CodeGenerator::cgLdConst(IRInstruction* inst) {
auto const dstReg = m_regs[inst->dst()].reg();
auto const val = inst->extra<LdConst>()->as<uintptr_t>();
if (dstReg == InvalidReg) return;
emitLoadImm(m_as, val, dstReg);
}
void CodeGenerator::cgLdARFuncPtr(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* baseAddr = inst->src(0);
SSATmp* offset = inst->src(1);
auto dstReg = m_regs[dst].reg();
auto baseReg = m_regs[baseAddr].reg();
assert(offset->isConst());
m_as.load_reg64_disp_reg64(baseReg,
offset->getValInt() + AROFF(m_func),
dstReg);
}
static int getNativeTypeSize(Type type) {
if (type.subtypeOf(Type::Int | Type::Func)) return sz::qword;
if (type.subtypeOf(Type::Bool)) return sz::byte;
not_implemented();
}
void CodeGenerator::cgLdRaw(IRInstruction* inst) {
SSATmp* dest = inst->dst();
SSATmp* addr = inst->src(0);
SSATmp* offset = inst->src(1);
assert(!(dest->isConst()));
Reg64 addrReg = m_regs[addr].reg();
PhysReg destReg = m_regs[dest].reg();
if (addr->isConst()) {
addrReg = m_rScratch;
emitLoadImm(m_as, addr->getValRawInt(), addrReg);
}
if (offset->isConst()) {
assert(offset->type() == Type::Int);
int64_t kind = offset->getValInt();
RawMemSlot& slot = RawMemSlot::Get(RawMemSlot::Kind(kind));
int ldSize = slot.size();
int64_t off = slot.offset();
if (ldSize == sz::qword) {
m_as.loadq (addrReg[off], destReg);
} else if (ldSize == sz::dword) {
m_as.loadl (addrReg[off], r32(destReg));
} else {
assert(ldSize == sz::byte);
m_as.loadzbl (addrReg[off], r32(destReg));
}
} else {
int ldSize = getNativeTypeSize(dest->type());
Reg64 offsetReg = r64(m_regs[offset].reg());
if (ldSize == sz::qword) {
m_as.loadq (addrReg[offsetReg], destReg);
} else {
// Not yet supported by our assembler
assert(ldSize == sz::byte);
not_implemented();
}
}
}
void CodeGenerator::cgStRaw(IRInstruction* inst) {
auto baseReg = m_regs[inst->src(0)].reg();
int64_t kind = inst->src(1)->getValInt();
SSATmp* value = inst->src(2);
RawMemSlot& slot = RawMemSlot::Get(RawMemSlot::Kind(kind));
assert(value->type().equals(slot.type()));
int stSize = slot.size();
int64_t off = slot.offset();
auto dest = baseReg[off];
if (value->isConst()) {
if (stSize == sz::qword) {
m_as.storeq(value->getValRawInt(), dest);
} else if (stSize == sz::dword) {
m_as.storel(value->getValRawInt(), dest);
} else {
assert(stSize == sz::byte);
m_as.storeb(value->getValBool(), dest);
}
} else {
if (stSize == sz::qword) {
m_as.storeq(r64(m_regs[value].reg()), dest);
} else if (stSize == sz::dword) {
m_as.storel(r32(m_regs[value].reg()), dest);
} else {
assert(stSize == sz::byte);
m_as.storeb(rbyte(m_regs[value].reg()), dest);
}
}
}
void CodeGenerator::cgLdStaticLocCached(IRInstruction* inst) {
auto ch = inst->src(0)->getValRawInt();
auto outReg = m_regs[inst->dst()].reg();
m_as.loadq (rVmTl[ch], outReg);
m_as.testq (outReg, outReg);
emitFwdJcc(m_as, CC_Z, inst->taken());
}
// If label is set and type is not Gen, this method generates a check
// that bails to the label if the loaded typed value doesn't match type.
void CodeGenerator::cgLoadTypedValue(PhysReg base,
int64_t off,
IRInstruction* inst) {
Block* label = inst->taken();
Type type = inst->typeParam();
SSATmp* dst = inst->dst();
assert(type == dst->type());
assert(type.needsReg());
auto valueDstReg = m_regs[dst].reg(0);
auto typeDstReg = m_regs[dst].reg(1);
if (valueDstReg.isXMM()) {
// Whole typed value is stored in single XMM reg valueDstReg
assert(RuntimeOption::EvalHHIRAllocXMMRegs);
assert(typeDstReg == InvalidReg);
m_as.movdqa(base[off + TVOFF(m_data)], valueDstReg);
return;
}
if (valueDstReg == InvalidReg && typeDstReg == InvalidReg &&
(label == nullptr || type == Type::Gen)) {
// a dead load
return;
}
bool useScratchReg = (base == typeDstReg && valueDstReg != InvalidReg);
if (useScratchReg) {
// Save base to m_rScratch, because base will be overwritten.
m_as.mov_reg64_reg64(base, m_rScratch);
}
// Load type if it's not dead
if (typeDstReg != InvalidReg) {
emitLoadTVType(m_as, base[off + TVOFF(m_type)], typeDstReg);
if (label) {
emitTypeCheck(inst->typeParam(), typeDstReg,
valueDstReg, inst->taken());
}
} else if (label) {
emitTypeCheck(inst->typeParam(),
base[off + TVOFF(m_type)],
base[off + TVOFF(m_data)],
inst->taken());
}
// Load value if it's not dead
if (valueDstReg == InvalidReg) return;
if (useScratchReg) {
m_as.loadq(m_rScratch[off + TVOFF(m_data)], valueDstReg);
} else {
m_as.loadq(base[off + TVOFF(m_data)], valueDstReg);
}
}
void CodeGenerator::cgStoreTypedValue(PhysReg base,
int64_t off,
SSATmp* src) {
assert(src->type().needsReg());
auto srcReg0 = m_regs[src].reg(0);
auto srcReg1 = m_regs[src].reg(1);
if (srcReg0.isXMM()) {
// Whole typed value is stored in single XMM reg srcReg0
assert(RuntimeOption::EvalHHIRAllocXMMRegs);
assert(srcReg1 == InvalidReg);
m_as.movdqa(srcReg0, base[off + TVOFF(m_data)]);
return;
}
m_as.storeq(srcReg0, base[off + TVOFF(m_data)]);
emitStoreTVType(m_as, srcReg1, base[off + TVOFF(m_type)]);
}
void CodeGenerator::cgStore(PhysReg base,
int64_t off,
SSATmp* src,
bool genStoreType) {
Type type = src->type();
if (type.needsReg()) {
cgStoreTypedValue(base, off, src);
return;
}
// store the type
if (genStoreType) {
emitStoreTVType(m_as, type.toDataType(), base[off + TVOFF(m_type)]);
}
if (type.isNull()) {
// no need to store a value for null or uninit
return;
}
if (src->isConst()) {
int64_t val = 0;
if (type.subtypeOf(Type::Bool | Type::Int | Type::Dbl |
Type::Arr | Type::StaticStr | Type::Cls)) {
val = src->getValBits();
} else {
not_reached();
}
m_as.storeq(val, base[off + TVOFF(m_data)]);
} else {
zeroExtendIfBool(m_as, src, m_regs[src]);
emitStoreReg(m_as, m_regs[src].reg(), base[off + TVOFF(m_data)]);
}
}
void CodeGenerator::cgLoad(PhysReg base,
int64_t off,
IRInstruction* inst) {
Type type = inst->typeParam();
if (type.needsReg()) {
return cgLoadTypedValue(base, off, inst);
}
Block* label = inst->taken();
if (label != NULL) {
emitTypeCheck(inst->typeParam(),
base[off + TVOFF(m_type)],
base[off + TVOFF(m_data)],
inst->taken());
}
if (type.isNull()) return; // these are constants
auto dstReg = m_regs[inst->dst()].reg();
// if dstReg == InvalidReg then the value of this load is dead
if (dstReg == InvalidReg) return;
if (type == Type::Bool) {
m_as.load_reg64_disp_reg32(base, off + TVOFF(m_data), dstReg);
} else {
emitLoadReg(m_as, base[off + TVOFF(m_data)], dstReg);
}
}
void CodeGenerator::cgLdProp(IRInstruction* inst) {
cgLoad(m_regs[inst->src(0)].reg(), inst->src(1)->getValInt(), inst);
}
void CodeGenerator::cgLdMem(IRInstruction * inst) {
cgLoad(m_regs[inst->src(0)].reg(), inst->src(1)->getValInt(), inst);
}
void CodeGenerator::cgLdRef(IRInstruction* inst) {
cgLoad(m_regs[inst->src(0)].reg(), RefData::tvOffset(), inst);
}
void CodeGenerator::recordSyncPoint(Asm& as,
SyncOptions sync /* = kSyncPoint */) {
assert(m_state.lastMarker);
Offset stackOff = m_state.lastMarker->stackOff;
switch (sync) {
case kSyncPointAdjustOne:
stackOff -= 1;
break;
case kSyncPoint:
break;
case kNoSyncPoint:
assert(0);
}
Offset pcOff = m_state.lastMarker->bcOff - m_state.lastMarker->func->base();
FTRACE(5, "IR recordSyncPoint: {} {} {}\n", as.code.frontier, pcOff,
stackOff);
m_tx64->recordSyncPoint(as, pcOff, stackOff);
}
void CodeGenerator::cgLdAddr(IRInstruction* inst) {
auto base = m_regs[inst->src(0)].reg();
int64_t offset = inst->src(1)->getValInt();
m_as.lea (base[offset], m_regs[inst->dst()].reg());
}
void CodeGenerator::cgLdLoc(IRInstruction* inst) {
cgLoad(m_regs[inst->src(0)].reg(),
localOffset(inst->extra<LdLoc>()->locId),
inst);
}
void CodeGenerator::cgLdLocAddr(IRInstruction* inst) {
auto const fpReg = m_regs[inst->src(0)].reg();
auto const offset = localOffset(inst->extra<LdLocAddr>()->locId);
if (m_regs[inst->dst()].hasReg()) {
m_as.lea(fpReg[offset], m_regs[inst->dst()].reg());
}
}
void CodeGenerator::cgLdStackAddr(IRInstruction* inst) {
auto const base = m_regs[inst->src(0)].reg();
auto const offset = cellsToBytes(inst->extra<LdStackAddr>()->offset);
m_as.lea (base[offset], m_regs[inst->dst()].reg());
}
void CodeGenerator::cgLdStack(IRInstruction* inst) {
assert(inst->taken() == nullptr);
cgLoad(m_regs[inst->src(0)].reg(),
cellsToBytes(inst->extra<LdStack>()->offset),
inst);
}
void CodeGenerator::cgGuardStk(IRInstruction* inst) {
auto const rSP = m_regs[inst->src(0)].reg();
auto const baseOff = cellsToBytes(inst->extra<GuardStk>()->offset);
emitTypeGuard(inst->typeParam(),
rSP[baseOff + TVOFF(m_type)],
rSP[baseOff + TVOFF(m_data)]);
}
void CodeGenerator::cgCheckStk(IRInstruction* inst) {
auto const rbase = m_regs[inst->src(0)].reg();
auto const baseOff = cellsToBytes(inst->extra<CheckStk>()->offset);
emitTypeCheck(inst->typeParam(), rbase[baseOff + TVOFF(m_type)],
rbase[baseOff + TVOFF(m_data)], inst->taken());
}
void CodeGenerator::cgGuardLoc(IRInstruction* inst) {
auto const rFP = m_regs[inst->src(0)].reg();
auto const baseOff = localOffset(inst->extra<GuardLoc>()->locId);
emitTypeGuard(inst->typeParam(),
rFP[baseOff + TVOFF(m_type)],
rFP[baseOff + TVOFF(m_data)]);
}
void CodeGenerator::cgCheckLoc(IRInstruction* inst) {
auto const rbase = m_regs[inst->src(0)].reg();
auto const baseOff = localOffset(inst->extra<CheckLoc>()->locId);
emitTypeCheck(inst->typeParam(), rbase[baseOff + TVOFF(m_type)],
rbase[baseOff + TVOFF(m_data)], inst->taken());
}
template<class Loc>
void CodeGenerator::emitSideExitGuard(Type type,
Loc typeSrc,
Loc dataSrc,
Offset taken) {
emitTypeTest(type, typeSrc, dataSrc,
[&](ConditionCode cc) {
auto const sk = SrcKey(curFunc(), taken);
m_tx64->emitBindJcc(m_as, ccNegate(cc), sk, REQ_BIND_SIDE_EXIT);
});
}
void CodeGenerator::cgSideExitGuardLoc(IRInstruction* inst) {
auto const fp = m_regs[inst->src(0)].reg();
auto const extra = inst->extra<SideExitGuardLoc>();
emitSideExitGuard(inst->typeParam(),
fp[localOffset(extra->checkedSlot) + TVOFF(m_type)],
fp[localOffset(extra->checkedSlot) + TVOFF(m_data)],
extra->taken);
}
void CodeGenerator::cgSideExitGuardStk(IRInstruction* inst) {
auto const sp = m_regs[inst->src(0)].reg();
auto const extra = inst->extra<SideExitGuardStk>();
emitSideExitGuard(inst->typeParam(),
sp[cellsToBytes(extra->checkedSlot) + TVOFF(m_type)],
sp[cellsToBytes(extra->checkedSlot) + TVOFF(m_data)],
extra->taken);
}
void CodeGenerator::cgDefMIStateBase(IRInstruction* inst) {
assert(inst->dst()->type() == Type::PtrToCell);
assert(m_regs[inst->dst()].reg() == rsp);
}
void CodeGenerator::cgCheckType(IRInstruction* inst) {
auto const src = inst->src(0);
auto const t = inst->typeParam();
auto const rData = m_regs[src].reg(0);
auto const rType = m_regs[src].reg(1);
auto doJcc = [&](ConditionCode cc) {
emitFwdJcc(ccNegate(cc), inst->taken());
};
if (t.equals(Type::Nullptr)) {
if (!src->type().equals(Type::Nullptr | Type::CountedStr)) {
CG_PUNT(CheckType-Nullptr-UnsupportedType);
}
m_as.testq (rData, rData);
doJcc(CC_E);
} else {
emitTypeTest(inst->typeParam(), rType, rData, doJcc);
}
auto const dstReg = m_regs[inst->dst()].reg();
if (dstReg != InvalidReg) {
emitMovRegReg(m_as, rData, dstReg);
}
}
void CodeGenerator::cgCheckTypeMem(IRInstruction* inst) {
auto const reg = m_regs[inst->src(0)].reg();
emitTypeCheck(inst->typeParam(), reg[TVOFF(m_type)],
reg[TVOFF(m_data)], inst->taken());
}
void CodeGenerator::cgGuardRefs(IRInstruction* inst) {
assert(inst->numSrcs() == 5);
SSATmp* funcPtrTmp = inst->src(0);
SSATmp* nParamsTmp = inst->src(1);
SSATmp* firstBitNumTmp = inst->src(2);
SSATmp* mask64Tmp = inst->src(3);
SSATmp* vals64Tmp = inst->src(4);
// Get values in place
assert(funcPtrTmp->type() == Type::Func);
auto funcPtrReg = m_regs[funcPtrTmp].reg();
assert(funcPtrReg != InvalidReg);
assert(nParamsTmp->type() == Type::Int);
auto nParamsReg = m_regs[nParamsTmp].reg();
assert(nParamsReg != InvalidReg || nParamsTmp->isConst());
assert(firstBitNumTmp->isConst() && firstBitNumTmp->type() == Type::Int);
uint32_t firstBitNum = (uint32_t)(firstBitNumTmp->getValInt());
assert(mask64Tmp->type() == Type::Int);
assert(mask64Tmp->isConst());
auto mask64Reg = m_regs[mask64Tmp].reg();
assert(mask64Reg != InvalidReg || mask64Tmp->inst()->op() != LdConst);
uint64_t mask64 = mask64Tmp->getValInt();
assert(mask64);
assert(vals64Tmp->type() == Type::Int);
assert(vals64Tmp->isConst());
auto vals64Reg = m_regs[vals64Tmp].reg();
assert(vals64Reg != InvalidReg || vals64Tmp->inst()->op() != LdConst);
uint64_t vals64 = vals64Tmp->getValInt();
assert((vals64 & mask64) == vals64);
auto const destSK = SrcKey(curFunc(), m_curTrace->bcOff());
auto const destSR = m_tx64->getSrcRec(destSK);
auto thenBody = [&] {
auto bitsOff = sizeof(uint64_t) * (firstBitNum / 64);
auto cond = CC_NE;
auto bitsPtrReg = m_rScratch;
if (firstBitNum == 0) {
bitsOff = Func::refBitValOff();
bitsPtrReg = funcPtrReg;
} else {
m_as.loadq(funcPtrReg[Func::sharedOff()], bitsPtrReg);
bitsOff -= sizeof(uint64_t);
}
if (vals64 == 0 || (mask64 & (mask64 - 1)) == 0) {
// If vals64 is zero, or we're testing a single
// bit, we can get away with a single test,
// rather than mask-and-compare
if (mask64Reg != InvalidReg) {
m_as.testq (mask64Reg, bitsPtrReg[bitsOff]);
} else {
if (mask64 < 256) {
m_as.testb((int8_t)mask64, bitsPtrReg[bitsOff]);
} else {
m_as.testl((int32_t)mask64, bitsPtrReg[bitsOff]);
}
}
if (vals64) cond = CC_E;
} else {
auto bitsValReg = m_rScratch;
m_as. loadq (bitsPtrReg[bitsOff], bitsValReg);
if (debug) bitsPtrReg = InvalidReg;
// bitsValReg <- bitsValReg & mask64
if (mask64Reg != InvalidReg) {
m_as. andq (mask64Reg, bitsValReg);
} else if (mask64 < 256) {
m_as. andb ((int8_t)mask64, rbyte(bitsValReg));
} else {
m_as. andl ((int32_t)mask64, r32(bitsValReg));
}
// If bitsValReg != vals64, then goto Exit
if (vals64Reg != InvalidReg) {
m_as. cmpq (vals64Reg, bitsValReg);
} else if (mask64 < 256) {
assert(vals64 < 256);
m_as. cmpb ((int8_t)vals64, rbyte(bitsValReg));
} else {
m_as. cmpl ((int32_t)vals64, r32(bitsValReg));
}
}
m_tx64->emitFallbackCondJmp(m_as, *destSR, cond);
};
if (firstBitNum == 0) {
assert(nParamsReg == InvalidReg);
// This is the first 64 bits. No need to check
// nParams.
thenBody();
} else {
assert(nParamsReg != InvalidReg);
// Check number of args...
m_as. cmpq (firstBitNum, nParamsReg);
if (vals64 != 0 && vals64 != mask64) {
// If we're beyond nParams, then either all params
// are refs, or all params are non-refs, so if vals64
// isn't 0 and isnt mask64, there's no possibility of
// a match
m_tx64->emitFallbackCondJmp(m_as, *destSR, CC_LE);
thenBody();
} else {
ifThenElse(CC_NLE, thenBody, /* else */ [&] {
// If not special builtin...
m_as.testl(AttrVariadicByRef, funcPtrReg[Func::attrsOff()]);
m_tx64->emitFallbackCondJmp(m_as, *destSR, vals64 ? CC_Z : CC_NZ);
});
}
}
}
void CodeGenerator::cgLdPropAddr(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* obj = inst->src(0);
SSATmp* prop = inst->src(1);
assert(prop->isConst() && prop->type() == Type::Int);
auto dstReg = m_regs[dst].reg();
auto objReg = m_regs[obj].reg();
assert(objReg != InvalidReg);
assert(dstReg != InvalidReg);
int64_t offset = prop->getValInt();
m_as.lea_reg64_disp_reg64(objReg, offset, dstReg);
}
void CodeGenerator::cgLdClsMethod(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* cls = inst->src(0);
SSATmp* mSlot = inst->src(1);
assert(cls->type() == Type::Cls);
assert(mSlot->isConst() && mSlot->type() == Type::Int);
uint64_t mSlotInt64 = mSlot->getValRawInt();
// We're going to multiply mSlotVal by sizeof(Func*) and use
// it as a 32-bit offset (methOff) below.
if (mSlotInt64 > (std::numeric_limits<uint32_t>::max() / sizeof(Func*))) {
CG_PUNT(cgLdClsMethod_large_offset);
}
int32_t mSlotVal = (uint32_t) mSlotInt64;
Reg64 dstReg = m_regs[dst].reg();
assert(dstReg != InvalidReg);
Reg64 clsReg = m_regs[cls].reg();
if (clsReg == InvalidReg) {
CG_PUNT(LdClsMethod);
}
Offset vecOff = Class::getMethodsOffset() + Class::MethodMap::vecOff();
int32_t methOff = mSlotVal * sizeof(Func*);
m_as.loadq(clsReg[vecOff], dstReg);
m_as.loadq(dstReg[methOff], dstReg);
}
void CodeGenerator::cgLdClsMethodCache(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* className = inst->src(0);
SSATmp* methodName = inst->src(1);
SSATmp* baseClass = inst->src(2);
Block* label = inst->taken();
// Stats::emitInc(a, Stats::TgtCache_StaticMethodHit);
const StringData* cls = className->getValStr();
const StringData* method = methodName->getValStr();
auto const ne = baseClass->getValNamedEntity();
TargetCache::CacheHandle ch =
TargetCache::StaticMethodCache::alloc(cls,
method,
getContextName(curClass()));
auto funcDestReg = m_regs[dst].reg(0);
auto classDestReg = m_regs[dst].reg(1);
auto offsetof_func = offsetof(TargetCache::StaticMethodCache, m_func);
auto offsetof_cls = offsetof(TargetCache::StaticMethodCache, m_cls);
assert(funcDestReg != InvalidReg && classDestReg != InvalidReg);
// Attempt to retrieve the func* and class* from cache
m_as.loadq(rVmTl[ch + offsetof_func], funcDestReg);
m_as.loadq(rVmTl[ch + offsetof_cls], classDestReg);
m_as.testq(funcDestReg, funcDestReg);
// May have retrieved a NULL from the cache
// handle case where method is not entered in the cache
unlikelyIfBlock(CC_E, [&] (Asm& a) {
if (false) { // typecheck
const UNUSED Func* f = StaticMethodCache::lookupIR(ch, ne, cls, method);
}
// can raise an error if class is undefined
cgCallHelper(a,
(TCA)StaticMethodCache::lookupIR,
funcDestReg,
kSyncPoint,
ArgGroup(m_regs).imm(ch) // Handle ch
.immPtr(ne) // NamedEntity* np.second
.immPtr(cls) // className
.immPtr(method) // methodName
);
// recordInstrCall is done in cgCallHelper
a.testq(funcDestReg, funcDestReg);
a.loadq(rVmTl[ch + offsetof_cls], classDestReg);
// if StaticMethodCache::lookupIR() returned NULL, jmp to label
emitFwdJcc(a, CC_Z, label);
});
}
/**
* Helper to emit getting the value for ActRec's m_this/m_cls slot
* from a This pointer depending on whether the callee method is
* static or not.
*/
void CodeGenerator::emitGetCtxFwdCallWithThis(PhysReg ctxReg,
bool staticCallee) {
if (staticCallee) {
// Load (this->m_cls | 0x1) into ctxReg.
m_as.loadq(ctxReg[ObjectData::getVMClassOffset()], ctxReg);
m_as.orq(1, ctxReg);
} else {
// Just incref $this.
emitIncRef(m_as, ctxReg);
}
}
/**
* This method is similar to emitGetCtxFwdCallWithThis above, but
* whether or not the callee is a static method is unknown at JIT
* time, and that is determined dynamically by looking up into the
* StaticMethodFCache.
*/
void CodeGenerator::emitGetCtxFwdCallWithThisDyn(PhysReg destCtxReg,
PhysReg thisReg,
CacheHandle& ch) {
Label NonStaticCall, End;
// thisReg is holding $this. Should we pass it to the callee?
m_as.cmpl(1, rVmTl[ch + offsetof(StaticMethodFCache, m_static)]);
m_as.jcc8(CC_NE, NonStaticCall);
// If calling a static method...
{
// Load (this->m_cls | 0x1) into destCtxReg
m_as.loadq(thisReg[ObjectData::getVMClassOffset()], destCtxReg);
m_as.orq(1, destCtxReg);
m_as.jmp8(End);
}
// Else: calling non-static method
{
asm_label(m_as, NonStaticCall);
emitMovRegReg(m_as, thisReg, destCtxReg);
emitIncRef(m_as, destCtxReg);
}
asm_label(m_as, End);
}
void CodeGenerator::cgGetCtxFwdCall(IRInstruction* inst) {
PhysReg destCtxReg = m_regs[inst->dst()].reg(0);
SSATmp* srcCtxTmp = inst->src(0);
const Func* callee = inst->src(1)->getValFunc();
bool withThis = srcCtxTmp->isA(Type::Obj);
// Eagerly move src into the dest reg
emitMovRegReg(m_as, m_regs[srcCtxTmp].reg(0), destCtxReg);
Label End;
// If we don't know whether we have a This, we need to check dynamically
if (!withThis) {
m_as.testb(1, rbyte(destCtxReg));
m_as.jcc8(CC_NZ, End);
}
// If we have a This pointer in destCtxReg, then select either This
// or its Class based on whether callee is static or not
emitGetCtxFwdCallWithThis(destCtxReg, (callee->attrs() & AttrStatic));
asm_label(m_as, End);
}
void CodeGenerator::cgLdClsMethodFCache(IRInstruction* inst) {
PhysReg funcDestReg = m_regs[inst->dst()].reg(0);
PhysReg destCtxReg = m_regs[inst->dst()].reg(1);
const Class* cls = inst->src(0)->getValClass();
const StringData* methName = inst->src(1)->getValStr();
SSATmp* srcCtxTmp = inst->src(2);
PhysReg srcCtxReg = m_regs[srcCtxTmp].reg(0);
Block* exitLabel = inst->taken();
const StringData* clsName = cls->name();
CacheHandle ch = StaticMethodFCache::alloc(clsName, methName,
getContextName(curClass()));
assert(funcDestReg != InvalidReg && destCtxReg != InvalidReg);
emitMovRegReg(m_as, srcCtxReg, destCtxReg);
m_as.loadq(rVmTl[ch], funcDestReg);
m_as.testq(funcDestReg, funcDestReg);
Label End;
// Handle case where method is not entered in the cache
unlikelyIfBlock(CC_E, [&] (Asm& a) {
const Func* (*lookup)(CacheHandle, const Class*, const StringData*) =
StaticMethodFCache::lookupIR;
// preserve destCtxReg across the call since it wouldn't be otherwise
RegSet toSave = m_state.liveRegs[inst] | RegSet(destCtxReg);
cgCallHelper(a, Transl::CppCall((TCA)lookup),
funcDestReg, InvalidReg,
kSyncPoint,
ArgGroup(m_regs).imm(ch)
.immPtr(cls)
.immPtr(methName),
toSave);
// If entry found in target cache, jump back to m_as.
// Otherwise, bail to exit label
a.testq(funcDestReg, funcDestReg);
emitFwdJcc(a, CC_Z, exitLabel);
});
auto t = srcCtxTmp->type();
assert(!t.equals(Type::Cls));
if (t.equals(Type::Cctx)) {
return; // done: destCtxReg already has srcCtxReg
} else if (t == Type::Obj) {
// unconditionally run code produced by emitGetCtxFwdCallWithThisDyn below
// break
} else if (t == Type::Ctx) {
// dynamically check if we have a This pointer and
// call emitGetCtxFwdCallWithThisDyn below
m_as.testb(1, rbyte(destCtxReg));
m_as.jcc8(CC_NZ, End);
} else {
not_reached();
}
// If we have a 'this' pointer ...
emitGetCtxFwdCallWithThisDyn(destCtxReg, destCtxReg, ch);
asm_label(m_as, End);
}
void CodeGenerator::cgLdClsPropAddrCached(IRInstruction* inst) {
using namespace Transl::TargetCache;
SSATmp* dst = inst->dst();
SSATmp* cls = inst->src(0);
SSATmp* propName = inst->src(1);
SSATmp* clsName = inst->src(2);
SSATmp* cxt = inst->src(3);
Block* target = inst->taken();
const StringData* propNameString = propName->getValStr();
const StringData* clsNameString = clsName->getValStr();
string sds(Util::toLower(clsNameString->data()) + ":" +
string(propNameString->data(), propNameString->size()));
StackStringData sd(sds.c_str(), sds.size(), AttachLiteral);
CacheHandle ch = SPropCache::alloc(&sd);
auto dstReg = m_regs[dst].reg();
// Cls is live in the slow path call to lookupIR, so we have to be
// careful not to clobber it before the branch to slow path. So
// use the scratch register as a temporary destination if cls is
// assigned the same register as the dst register.
auto tmpReg = dstReg;
if (dstReg == InvalidReg || dstReg == m_regs[cls].reg()) {
tmpReg = PhysReg(m_rScratch);
}
// Could be optimized to cmp against zero when !label && dstReg == InvalidReg
m_as.loadq(rVmTl[ch], tmpReg);
m_as.testq(tmpReg, tmpReg);
unlikelyIfBlock(CC_E, [&] (Asm& a) {
cgCallHelper(a,
target ? (TCA)SPropCache::lookupIR<false>
: (TCA)SPropCache::lookupIR<true>, // raise on error
tmpReg,
kSyncPoint, // could re-enter to initialize properties
ArgGroup(m_regs).imm(ch).ssa(cls).ssa(propName).ssa(cxt));
if (target) {
a.testq(tmpReg, tmpReg);
emitFwdJcc(a, CC_Z, target);
}
});
if (dstReg != InvalidReg) {
emitMovRegReg(m_as, tmpReg, dstReg);
}
}
void CodeGenerator::cgLdClsPropAddr(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* cls = inst->src(0);
SSATmp* prop = inst->src(1);
SSATmp* ctx = inst->src(2);
Block* target = inst->taken();
// If our label is a catch trace we pretend we don't have one, to
// avoid emitting a jmp to it or calling the wrong helper.
if (target && target->trace()->isCatch()) target = nullptr;
auto dstReg = m_regs[dst].reg();
if (dstReg == InvalidReg && target) {
// result is unused but this instruction was not eliminated
// because its essential
dstReg = m_rScratch;
}
cgCallHelper(m_as,
target ? (TCA)SPropCache::lookupSProp<false>
: (TCA)SPropCache::lookupSProp<true>, // raise on error
dstReg,
kSyncPoint, // could re-enter to initialize properties
ArgGroup(m_regs).ssa(cls).ssa(prop).ssa(ctx));
if (target) {
m_as.testq(dstReg, dstReg);
emitFwdJcc(m_as, CC_Z, target);
}
}
TargetCache::CacheHandle CodeGenerator::cgLdClsCachedCommon(
IRInstruction* inst) {
SSATmp* dst = inst->dst();
const StringData* className = inst->src(0)->getValStr();
auto ch = TargetCache::allocKnownClass(className);
auto dstReg = m_regs[dst].reg();
if (dstReg == InvalidReg) {
m_as. cmpq (0, rVmTl[ch]);
} else {
m_as. loadq (rVmTl[ch], dstReg);
m_as. testq (dstReg, dstReg);
}
return ch;
}
void CodeGenerator::cgLdClsCached(IRInstruction* inst) {
auto ch = cgLdClsCachedCommon(inst);
unlikelyIfBlock(CC_E, [&] (Asm& a) {
// Passing only two arguments to lookupKnownClass, since the
// third is ignored in the checkOnly==false case.
cgCallHelper(a,
(TCA)TargetCache::lookupKnownClass<false>,
inst->dst(),
kSyncPoint,
ArgGroup(m_regs).addr(rVmTl, intptr_t(ch)).ssas(inst, 0));
});
}
void CodeGenerator::cgLdClsCachedSafe(IRInstruction* inst) {
cgLdClsCachedCommon(inst);
if (Block* taken = inst->taken()) {
emitFwdJcc(CC_Z, taken);
}
}
void CodeGenerator::cgLdCls(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* className = inst->src(0);
CacheHandle ch = ClassCache::alloc();
cgCallHelper(m_as, (TCA)ClassCache::lookup, dst, kSyncPoint,
ArgGroup(m_regs).imm(ch).ssa(className));
}
static StringData* fullConstName(const StringData* cls,
const StringData* cnsName) {
return StringData::GetStaticString(
Util::toLower(cls->data()) + "::" + cnsName->data()
);
}
void CodeGenerator::cgLdClsCns(IRInstruction* inst) {
auto const extra = inst->extra<LdClsCns>();
auto const fullName = fullConstName(extra->clsName, extra->cnsName);
auto const ch = TargetCache::allocClassConstant(fullName);
cgLoad(rVmTl, ch, inst);
}
void CodeGenerator::cgLookupClsCns(IRInstruction* inst) {
auto const extra = inst->extra<LookupClsCns>();
auto const fullName = fullConstName(extra->clsName, extra->cnsName);
auto const ch = TargetCache::allocClassConstant(fullName);
cgCallHelper(
m_as,
TCA(TargetCache::lookupClassConstantTv),
inst->dst(),
kSyncPoint,
ArgGroup(m_regs)
.addr(rVmTl, ch)
.immPtr(Unit::GetNamedEntity(extra->clsName))
.immPtr(extra->clsName)
.immPtr(extra->cnsName),
DestType::TV
);
}
void CodeGenerator::cgLdCns(IRInstruction* inst) {
const StringData* cnsName = inst->src(0)->getValStr();
TargetCache::CacheHandle ch = StringData::DefCnsHandle(cnsName, false);
// Has an unlikely branch to a LookupCns
cgLoad(rVmTl, ch, inst);
}
static TypedValue lookupCnsHelper(const TypedValue* tv, StringData* nm) {
assert(tv->m_type == KindOfUninit);
TypedValue *cns = nullptr;
TypedValue c1;
if (UNLIKELY(tv->m_data.pref != nullptr)) {
ClassInfo::ConstantInfo* ci =
(ClassInfo::ConstantInfo*)(void*)tv->m_data.pref;
cns = const_cast<Variant&>(ci->getDeferredValue()).asTypedValue();
tvReadCell(cns, &c1);
} else {
if (UNLIKELY(TargetCache::s_constants != nullptr)) {
cns = TargetCache::s_constants->HphpArray::nvGet(nm);
}
if (!cns) {
cns = Unit::loadCns(const_cast<StringData*>(nm));
}
if (UNLIKELY(!cns)) {
raise_notice(Strings::UNDEFINED_CONSTANT, nm->data(), nm->data());
c1.m_data.pstr = const_cast<StringData*>(nm);
c1.m_type = KindOfStaticString;
} else {
c1.m_type = cns->m_type;
c1.m_data = cns->m_data;
}
}
return c1;
}
void CodeGenerator::cgLookupCns(IRInstruction* inst) {
SSATmp* cnsNameTmp = inst->src(0);
assert(inst->typeParam() == Type::Cell);
assert(cnsNameTmp->isConst() && cnsNameTmp->type() == Type::StaticStr);
const StringData* cnsName = cnsNameTmp->getValStr();
TargetCache::CacheHandle ch = StringData::DefCnsHandle(cnsName, false);
ArgGroup args(m_regs);
args.addr(rVmTl, ch)
.immPtr(cnsName);
cgCallHelper(m_as, TCA(lookupCnsHelper),
inst->dst(), kSyncPoint, args, DestType::TV);
}
HOT_FUNC_VM
static inline int64_t ak_exist_string_helper(StringData* key, ArrayData* arr) {
int64_t n;
if (key->isStrictlyInteger(n)) {
return arr->exists(n);
}
return arr->exists(StrNR(key));
}
HOT_FUNC_VM
static int64_t ak_exist_string(StringData* key, ArrayData* arr) {
int64_t res = ak_exist_string_helper(key, arr);
return res;
}
HOT_FUNC_VM
static int64_t ak_exist_int(int64_t key, ArrayData* arr) {
bool res = arr->exists(key);
return res;
}
HOT_FUNC_VM
static int64_t ak_exist_string_obj(StringData* key, ObjectData* obj) {
if (obj->isCollection()) {
return collectionOffsetContains(obj, key);
}
CArrRef arr = obj->o_toArray();
int64_t res = ak_exist_string_helper(key, arr.get());
return res;
}
HOT_FUNC_VM
static int64_t ak_exist_int_obj(int64_t key, ObjectData* obj) {
if (obj->isCollection()) {
return collectionOffsetContains(obj, key);
}
CArrRef arr = obj->o_toArray();
bool res = arr.get()->exists(key);
return res;
}
void CodeGenerator::cgAKExists(IRInstruction* inst) {
SSATmp* arr = inst->src(0);
SSATmp* key = inst->src(1);
if (key->type().isNull()) {
if (arr->isA(Type::Arr)) {
cgCallHelper(m_as,
(TCA)ak_exist_string,
inst->dst(),
kNoSyncPoint,
ArgGroup(m_regs).immPtr(empty_string.get()).ssa(arr));
} else {
m_as.mov_imm64_reg(0, m_regs[inst->dst()].reg());
}
return;
}
TCA helper_func =
arr->isA(Type::Obj)
? (key->isA(Type::Int) ? (TCA)ak_exist_int_obj : (TCA)ak_exist_string_obj)
: (key->isA(Type::Int) ? (TCA)ak_exist_int : (TCA)ak_exist_string);
cgCallHelper(m_as,
helper_func,
inst->dst(),
kNoSyncPoint,
ArgGroup(m_regs).ssa(key).ssa(arr));
}
HOT_FUNC_VM static TypedValue* ldGblAddrHelper(StringData* name) {
return g_vmContext->m_globalVarEnv->lookup(name);
}
HOT_FUNC_VM static TypedValue* ldGblAddrDefHelper(StringData* name) {
TypedValue* r = g_vmContext->m_globalVarEnv->lookupAdd(name);
decRefStr(name);
return r;
}
void CodeGenerator::cgLdGblAddr(IRInstruction* inst) {
auto dstReg = m_regs[inst->dst()].reg();
cgCallHelper(m_as, (TCA)ldGblAddrHelper, dstReg, kNoSyncPoint,
ArgGroup(m_regs).ssa(inst->src(0)));
m_as.testq(dstReg, dstReg);
emitFwdJcc(CC_Z, inst->taken());
}
void CodeGenerator::cgLdGblAddrDef(IRInstruction* inst) {
cgCallHelper(m_as, (TCA)ldGblAddrDefHelper, inst->dst(), kNoSyncPoint,
ArgGroup(m_regs).ssa(inst->src(0)));
}
void CodeGenerator::emitTestZero(SSATmp* src) {
auto& a = m_as;
auto reg = m_regs[src].reg();
/*
* If src is const, normally a earlier optimization pass should have
* converted the thing testing this condition into something
* unconditional. So rather than supporting constants efficiently
* here, we just materialize the value into a register.
*/
if (reg == InvalidReg) {
reg = m_rScratch;
a. movq (src->getValBits(), reg);
}
if (src->isA(Type::Bool)) {
a. testb (rbyte(reg), rbyte(reg));
} else {
a. testq (reg, reg);
}
}
void CodeGenerator::cgJmpZero(IRInstruction* inst) {
emitTestZero(inst->src(0));
emitFwdJcc(CC_Z, inst->taken());
}
void CodeGenerator::cgJmpNZero(IRInstruction* inst) {
emitTestZero(inst->src(0));
emitFwdJcc(CC_NZ, inst->taken());
}
void CodeGenerator::cgReqBindJmpZero(IRInstruction* inst) {
// TODO(#2404427): prepareForTestAndSmash?
emitTestZero(inst->src(0));
emitReqBindJcc(CC_Z, inst->extra<ReqBindJmpZero>());
}
void CodeGenerator::cgReqBindJmpNZero(IRInstruction* inst) {
// TODO(#2404427): prepareForTestAndSmash?
emitTestZero(inst->src(0));
emitReqBindJcc(CC_NZ, inst->extra<ReqBindJmpNZero>());
}
void CodeGenerator::cgJmp_(IRInstruction* inst) {
Block* target = inst->taken();
if (unsigned n = inst->numSrcs()) {
// Parallel-copy sources to the label's destination registers.
// TODO: t2040286: this only works if all destinations fit in registers.
auto srcs = inst->srcs();
auto dsts = target->front()->dsts();
ArgGroup args(m_regs);
for (unsigned i = 0, j = 0; i < n; i++) {
assert(srcs[i]->type().subtypeOf(dsts[i].type()));
auto dst = &dsts[i];
auto src = srcs[i];
// Currently, full XMM registers cannot be assigned to SSATmps
// passed from to Jmp_ to DefLabel. If this changes, it'll require
// teaching shuffleArgs() how to handle full XMM values.
assert(!m_regs[src].isFullXMM() && !m_regs[dst].isFullXMM());
if (m_regs[dst].reg(0) == InvalidReg) continue; // dst is unused.
// first dst register
args.ssa(src);
args[j++].setDstReg(m_regs[dst].reg(0));
// second dst register, if any
if (dst->numNeededRegs() == 2) {
if (src->numNeededRegs() < 2) {
// src has known data type, but dst doesn't - pass immediate type
assert(src->type().isKnownDataType());
args.imm(src->type().toDataType());
} else {
// pass src's second register
assert(m_regs[src].reg(1) != InvalidReg);
args.reg(m_regs[src].reg(1));
}
args[j++].setDstReg(m_regs[dst].reg(1));
}
}
assert(args.numStackArgs() == 0 &&
"Jmp_ doesn't support passing arguments on the stack yet.");
shuffleArgs(m_as, args);
}
if (!m_state.noTerminalJmp_) {
emitFwdJmp(m_as, target, m_state);
}
}
void CodeGenerator::cgJmpIndirect(IRInstruction* inst) {
m_as.jmp(m_regs[inst->src(0)].reg());
}
void CodeGenerator::cgCheckInit(IRInstruction* inst) {
Block* label = inst->taken();
assert(label);
SSATmp* src = inst->src(0);
if (src->type().isInit()) return;
auto typeReg = m_regs[src].reg(1);
assert(typeReg != InvalidReg);
static_assert(KindOfUninit == 0, "cgCheckInit assumes KindOfUninit == 0");
m_as.testb (rbyte(typeReg), rbyte(typeReg));
emitFwdJcc(CC_Z, label);
}
void CodeGenerator::cgCheckInitMem(IRInstruction* inst) {
Block* label = inst->taken();
assert(label);
SSATmp* base = inst->src(0);
int64_t offset = inst->src(1)->getValInt();
Type t = base->type().deref();
if (t.isInit()) return;
auto basereg = m_regs[base].reg();
emitCmpTVType(m_as, KindOfUninit, basereg[offset + TVOFF(m_type)]);
emitFwdJcc(CC_Z, label);
}
void CodeGenerator::cgExitWhenSurprised(IRInstruction* inst) {
Block* label = inst->taken();
m_tx64->emitTestSurpriseFlags(m_as);
emitFwdJcc(CC_NZ, label);
}
void CodeGenerator::cgExitOnVarEnv(IRInstruction* inst) {
SSATmp* fp = inst->src(0);
Block* label = inst->taken();
assert(!(fp->isConst()));
auto fpReg = m_regs[fp].reg();
m_as. cmpq (0, fpReg[AROFF(m_varEnv)]);
emitFwdJcc(CC_NE, label);
}
void CodeGenerator::cgReleaseVVOrExit(IRInstruction* inst) {
auto* const label = inst->taken();
auto const rFp = m_regs[inst->src(0)].reg();
m_as. cmpq (0, rFp[AROFF(m_varEnv)]);
unlikelyIfBlock(CC_NZ, [&] (Asm& a) {
a. testl (ActRec::kExtraArgsBit, rFp[AROFF(m_varEnv)]);
emitFwdJcc(a, CC_Z, label);
cgCallHelper(
a,
TCA(static_cast<void (*)(ActRec*)>(ExtraArgs::deallocate)),
nullptr,
kSyncPoint,
ArgGroup(m_regs).reg(rFp),
DestType::None
);
});
}
void CodeGenerator::cgBoxPtr(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* addr = inst->src(0);
auto base = m_regs[addr].reg();
auto dstReg = m_regs[dst].reg();
emitMovRegReg(m_as, base, dstReg);
emitTypeTest(Type::BoxedCell, base[TVOFF(m_type)],
base[TVOFF(m_data)],
[&](ConditionCode cc) {
ifThen(m_as, ccNegate(cc), [&] {
cgCallHelper(m_as, (TCA)tvBox, dstReg, kNoSyncPoint,
ArgGroup(m_regs).ssa(addr));
});
});
}
void CodeGenerator::cgDefCns(IRInstruction* inst) {
UNUSED SSATmp* dst = inst->dst();
UNUSED SSATmp* cnsName = inst->src(0);
UNUSED SSATmp* val = inst->src(1);
using namespace TargetCache;
CG_PUNT(DefCns);
}
// TODO: Kill this #2031980
static StringData* concat_value(TypedValue tv1, TypedValue tv2) {
return concat_tv(tv1.m_type, tv1.m_data.num, tv2.m_type, tv2.m_data.num);
}
void CodeGenerator::cgConcat(IRInstruction* inst) {
SSATmp* dst = inst->dst();
SSATmp* tl = inst->src(0);
SSATmp* tr = inst->src(1);
Type lType = tl->type();
Type rType = tr->type();
// We have specialized helpers for concatenating two strings, a
// string and an int, and an int and a string.
void* fptr = nullptr;
if (lType.isString() && rType.isString()) {
fptr = (void*)concat_ss;
} else if (lType.isString() && rType == Type::Int) {
fptr = (void*)concat_si;
} else if (lType == Type::Int && rType.isString()) {
fptr = (void*)concat_is;
}
if (fptr) {
cgCallHelper(m_as, (TCA)fptr, dst, kNoSyncPoint,
ArgGroup(m_regs).ssa(tl).ssa(tr));
} else {
if (lType.subtypeOf(Type::Obj) || lType.needsReg() ||
rType.subtypeOf(Type::Obj) || rType.needsReg()) {
CG_PUNT(cgConcat);
}
cgCallHelper(m_as, (TCA)concat_value, dst, kNoSyncPoint,
ArgGroup(m_regs).typedValue(tl).typedValue(tr));
}
}
void CodeGenerator::cgInterpOne(IRInstruction* inst) {
SSATmp* fp = inst->src(0);
SSATmp* sp = inst->src(1);
SSATmp* pcOffTmp = inst->src(2);
SSATmp* spAdjustmentTmp = inst->src(3);
int64_t pcOff = pcOffTmp->getValInt();
auto opc = *(curFunc()->unit()->at(pcOff));
void* interpOneHelper = interpOneEntryPoints[opc];
auto dstReg = InvalidReg;
cgCallHelper(m_as, (TCA)interpOneHelper, dstReg, kSyncPoint,
ArgGroup(m_regs).ssa(fp).ssa(sp).imm(pcOff));
auto newSpReg = m_regs[inst->dst()].reg();
assert(newSpReg == m_regs[sp].reg());
int64_t spAdjustBytes = cellsToBytes(spAdjustmentTmp->getValInt());
if (spAdjustBytes != 0) {
m_as.addq(spAdjustBytes, newSpReg);
}
}
void CodeGenerator::cgInterpOneCF(IRInstruction* inst) {
SSATmp* fp = inst->src(0);
SSATmp* sp = inst->src(1);
int64_t pcOff = inst->src(2)->getValInt();
auto opc = *(curFunc()->unit()->at(pcOff));
void* interpOneHelper = interpOneEntryPoints[opc];
auto dstReg = InvalidReg;
cgCallHelper(m_as, (TCA)interpOneHelper, dstReg, kSyncPoint,
ArgGroup(m_regs).ssa(fp).ssa(sp).imm(pcOff));
// The interpOne method returns a pointer to the current ExecutionContext
// in rax. Use it read the 'm_fp' and 'm_stack.m_top' fields into the
// rVmFp and rVmSp registers.
m_as.loadq(rax[offsetof(VMExecutionContext, m_fp)], rVmFp);
m_as.loadq(rax[offsetof(VMExecutionContext, m_stack) +
Stack::topOfStackOffset()], rVmSp);
m_tx64->emitServiceReq(SRFlags::EmitInA, REQ_RESUME, 0ull);
}
void CodeGenerator::cgContEnter(IRInstruction* inst) {
auto contAR = inst->src(0);
auto addr = inst->src(1);
auto returnOff = inst->src(2);
auto curFp = m_regs[inst->src(3)].reg();
auto contARReg = m_regs[contAR].reg();
m_as. storel (returnOff->getValInt(), contARReg[AROFF(m_soff)]);
m_as. storeq (curFp, contARReg[AROFF(m_savedRbp)]);
m_as. movq (contARReg, rStashedAR);
m_as. call (m_regs[addr].reg());
}
void CodeGenerator::emitContVarEnvHelperCall(SSATmp* fp, TCA helper) {
auto scratch = m_rScratch;
m_as. loadq (m_regs[fp].reg()[AROFF(m_varEnv)], scratch);
m_as. testq (scratch, scratch);
unlikelyIfBlock(CC_NZ, [&] (Asm& a) {
cgCallHelper(a, helper, InvalidReg, kNoSyncPoint,
ArgGroup(m_regs).ssa(fp));
});
}
void CodeGenerator::cgUnlinkContVarEnv(IRInstruction* inst) {
emitContVarEnvHelperCall(
inst->src(0),
(TCA)VMExecutionContext::packContVarEnvLinkage);
}
void CodeGenerator::cgLinkContVarEnv(IRInstruction* inst) {
emitContVarEnvHelperCall(
inst->src(0),
(TCA)VMExecutionContext::unpackContVarEnvLinkage);
}
void CodeGenerator::cgContPreNext(IRInstruction* inst) {
auto contReg = m_regs[inst->src(0)].reg();
const Offset doneOffset = c_Continuation::doneOffset();
static_assert((doneOffset + 1) == c_Continuation::runningOffset(),
"done should immediately precede running");
// Check done and running at the same time
m_as.test_imm16_disp_reg16(0x0101, doneOffset, contReg);
emitFwdJcc(CC_NZ, inst->taken());
// ++m_index
m_as.add_imm64_disp_reg64(0x1, CONTOFF(m_index), contReg);
// running = true
m_as.store_imm8_disp_reg(0x1, c_Continuation::runningOffset(), contReg);
}
void CodeGenerator::cgContStartedCheck(IRInstruction* inst) {
m_as.cmp_imm64_disp_reg64(0, CONTOFF(m_index),
m_regs[inst->src(0)].reg());
emitFwdJcc(CC_L, inst->taken());
}
void CodeGenerator::cgIterInit(IRInstruction* inst) {
cgIterInitCommon(inst);
}
void CodeGenerator::cgIterInitK(IRInstruction* inst) {
cgIterInitCommon(inst);
}
void CodeGenerator::cgWIterInit(IRInstruction* inst) {
cgIterInitCommon(inst);
}
void CodeGenerator::cgWIterInitK(IRInstruction* inst) {
cgIterInitCommon(inst);
}
void CodeGenerator::cgIterInitCommon(IRInstruction* inst) {
bool isInitK = inst->op() == IterInitK || inst->op() == WIterInitK;
bool isWInit = inst->op() == WIterInit || inst->op() == WIterInitK;
PhysReg fpReg = m_regs[inst->src(1)].reg();
int64_t iterOffset = this->iterOffset(inst->src(2));
int64_t valLocalOffset = localOffset(inst->src(3));
SSATmp* src = inst->src(0);
ArgGroup args(m_regs);
args.addr(fpReg, iterOffset).ssa(src);
if (src->isArray()) {
args.addr(fpReg, valLocalOffset);
if (isInitK) {
args.addr(fpReg, localOffset(inst->src(4)));
} else if (isWInit) {
args.imm(0);
}
TCA helperAddr = isWInit ? (TCA)new_iter_array_key<true> :
isInitK ? (TCA)new_iter_array_key<false> : (TCA)new_iter_array;
cgCallHelper(m_as, helperAddr, inst->dst(), kSyncPoint, args);
} else {
assert(src->type() == Type::Obj);
args.imm(uintptr_t(curClass())).addr(fpReg, valLocalOffset);
if (isInitK) {
args.addr(fpReg, localOffset(inst->src(4)));
} else {
args.imm(0);
}
// new_iter_object decrefs its src object if it propagates an
// exception out, so we use kSyncPointAdjustOne, which adjusts the
// stack pointer by 1 stack element on an unwind, skipping over
// the src object.
cgCallHelper(m_as, (TCA)new_iter_object, inst->dst(),
kSyncPointAdjustOne, args);
}
}
void CodeGenerator::cgIterNext(IRInstruction* inst) {
cgIterNextCommon(inst);
}
void CodeGenerator::cgIterNextK(IRInstruction* inst) {
cgIterNextCommon(inst);
}
void CodeGenerator::cgWIterNext(IRInstruction* inst) {
cgIterNextCommon(inst);
}
void CodeGenerator::cgWIterNextK(IRInstruction* inst) {
cgIterNextCommon(inst);
}
void CodeGenerator::cgIterNextCommon(IRInstruction* inst) {
bool isNextK = inst->op() == IterNextK || inst->op() == WIterNextK;
bool isWNext = inst->op() == WIterNext || inst->op() == WIterNextK;
PhysReg fpReg = m_regs[inst->src(0)].reg();
ArgGroup args(m_regs);
args.addr(fpReg, iterOffset(inst->src(1)))
.addr(fpReg, localOffset(inst->src(2)));
if (isNextK) {
args.addr(fpReg, localOffset(inst->src(3)));
} else if (isWNext) {
args.imm(0);
}
TCA helperAddr = isWNext ? (TCA)iter_next_key<true> :
isNextK ? (TCA)iter_next_key<false> : (TCA)iter_next;
cgCallHelper(m_as, helperAddr, inst->dst(), kSyncPoint, args);
}
void iterFreeHelper(Iter* iter) {
iter->free();
}
void citerFreeHelper(Iter* iter) {
iter->cfree();
}
void CodeGenerator::cgIterFree(IRInstruction* inst) {
PhysReg fpReg = m_regs[inst->src(0)].reg();
int64_t offset = iterOffset(inst->extra<IterFree>()->iterId);
cgCallHelper(m_as, (TCA)iterFreeHelper, InvalidReg, kSyncPoint,
ArgGroup(m_regs).addr(fpReg, offset));
}
void CodeGenerator::cgDecodeCufIter(IRInstruction* inst) {
PhysReg fpReg = m_regs[inst->src(1)].reg();
int64_t offset = iterOffset(inst->extra<DecodeCufIter>()->iterId);
cgCallHelper(m_as, (TCA)decodeCufIterHelper, inst->dst(), kSyncPoint,
ArgGroup(m_regs).addr(fpReg, offset).typedValue(inst->src(0)));
}
void CodeGenerator::cgCIterFree(IRInstruction* inst) {
PhysReg fpReg = m_regs[inst->src(0)].reg();
int64_t offset = iterOffset(inst->extra<CIterFree>()->iterId);
cgCallHelper(m_as, (TCA)citerFreeHelper, InvalidReg, kSyncPoint,
ArgGroup(m_regs).addr(fpReg, offset));
}
void CodeGenerator::cgIncStat(IRInstruction *inst) {
Stats::emitInc(m_as,
Stats::StatCounter(inst->src(0)->getValInt()),
inst->src(1)->getValInt(),
Transl::CC_None,
inst->src(2)->getValBool());
}
void CodeGenerator::cgIncTransCounter(IRInstruction* inst) {
m_tx64->emitTransCounterInc(m_as);
}
void CodeGenerator::cgDbgAssertRefCount(IRInstruction* inst) {
emitAssertRefCount(m_as, m_regs[inst->src(0)].reg());
}
void traceCallback(ActRec* fp, Cell* sp, int64_t pcOff, void* rip) {
if (HPHP::Trace::moduleEnabled(HPHP::Trace::hhirTracelets)) {
FTRACE(0, "{} {} {}\n", fp->m_func->fullName()->data(), pcOff, rip);
}
checkFrame(fp, sp, /*checkLocals*/true);
}
void CodeGenerator::cgDbgAssertType(IRInstruction* inst) {
emitTypeTest(inst->typeParam(),
m_regs[inst->src(0)].reg(1),
m_regs[inst->src(0)].reg(0),
[&](ConditionCode cc) {
ifThen(m_as, ccNegate(cc), [&] { m_as.ud2(); });
});
}
void CodeGenerator::cgVerifyParamCls(IRInstruction* inst) {
SSATmp* objClass = inst->src(0);
assert(!objClass->isConst());
auto objClassReg = m_regs[objClass].reg();
SSATmp* constraint = inst->src(1);
if (constraint->isConst()) {
m_as. cmpq(constraint->getValClass(), objClassReg);
} else {
m_as. cmpq(m_regs[constraint].reg(), objClassReg);
}
// The native call for this instruction is the slow path that does
// proper subtype checking. The comparison above is just to
// short-circuit the overhead when the Classes are an exact match.
ifThen(m_as, CC_NE, [&]{ cgCallNative(inst); });
}
static void emitTraceCall(CodeGenerator::Asm& as,
int64_t pcOff,
Transl::TranslatorX64* tx64) {
// call to a trace function
as.mov_imm64_reg((int64_t)as.code.frontier, reg::rcx);
as.mov_reg64_reg64(rVmFp, reg::rdi);
as.mov_reg64_reg64(rVmSp, reg::rsi);
as.mov_imm64_reg(pcOff, reg::rdx);
// do the call; may use a trampoline
tx64->emitCall(as, (TCA)traceCallback);
}
void CodeGenerator::print() const {
JIT::print(std::cout, m_curTrace, &m_state.regs, m_state.lifetime,
m_state.asmInfo);
}
static void patchJumps(Asm& as, CodegenState& state, Block* block) {
void* list = state.patches[block];
Address labelAddr = as.code.frontier;
while (list) {
int32_t* toPatch = (int32_t*)list;
int32_t diffToNext = *toPatch;
ssize_t diff = labelAddr - ((Address)list + sizeof(int32_t));
*toPatch = safe_cast<int32_t>(diff); // patch the jump address
if (diffToNext == 0) break;
void* next = (TCA)list - diffToNext;
list = next;
}
}
void CodeGenerator::cgBlock(Block* block, vector<TransBCMapping>* bcMap) {
FTRACE(6, "cgBlock: {}\n", block->id());
for (IRInstruction& instr : *block) {
IRInstruction* inst = &instr;
if (inst->op() == Marker) {
m_state.lastMarker = inst->extra<Marker>();
FTRACE(7, "lastMarker is now {}\n", inst->extra<Marker>()->show());
if (m_tx64 && m_tx64->isTransDBEnabled() && bcMap) {
bcMap->push_back((TransBCMapping){Offset(m_state.lastMarker->bcOff),
m_as.code.frontier,
m_astubs.code.frontier});
}
}
m_curInst = inst;
auto nuller = folly::makeGuard([&]{ m_curInst = nullptr; });
auto* addr = cgInst(inst);
if (m_state.asmInfo && addr) {
m_state.asmInfo->instRanges[inst] = TcaRange(addr, m_as.code.frontier);
m_state.asmInfo->asmRanges[block] =
TcaRange(m_state.asmInfo->asmRanges[block].start(), m_as.code.frontier);
}
}
}
/*
* Compute and save registers that are live *across* each inst, not including
* registers whose lifetimes end at inst, nor registers defined by inst.
*/
LiveRegs computeLiveRegs(const IRFactory* factory, const RegAllocInfo& regs,
Block* start_block) {
StateVector<Block, RegSet> liveMap(factory, RegSet());
LiveRegs live_regs(factory, RegSet());
postorderWalk(
[&](Block* block) {
RegSet& live = liveMap[block];
if (Block* taken = block->taken()) live = liveMap[taken];
if (Block* next = block->next()) live |= liveMap[next];
for (auto it = block->end(); it != block->begin(); ) {
IRInstruction& inst = *--it;
for (const SSATmp& dst : inst.dsts()) {
live -= regs[dst].regs();
}
live_regs[inst] = live;
for (SSATmp* src : inst.srcs()) {
live |= regs[src].regs();
}
}
},
factory->numBlocks(),
start_block
);
return live_regs;
}
void genCodeForTrace(IRTrace* trace,
CodeGenerator::Asm& as,
CodeGenerator::Asm& astubs,
IRFactory* irFactory,
vector<TransBCMapping>* bcMap,
Transl::TranslatorX64* tx64,
const RegAllocInfo& regs,
const LifetimeInfo* lifetime,
AsmInfo* asmInfo) {
assert(trace->isMain());
LiveRegs live_regs = computeLiveRegs(irFactory, regs, trace->front());
CodegenState state(irFactory, regs, live_regs, lifetime, asmInfo);
// Returns: whether a block has already been emitted.
DEBUG_ONLY auto isEmitted = [&](Block* block) {
return state.addresses[block];
};
/*
* Emit the given block on the supplied assembler. The `nextBlock'
* is the nextBlock that will be emitted on this assembler. If is
* not the fallthrough block, emit a patchable jump to the
* fallthrough block.
*/
auto emitBlock = [&](Asm& a, Block* block, Block* nextBlock) {
assert(!isEmitted(block));
FTRACE(6, "cgBlock {} on {}\n", block->id(),
&a == &astubs ? "astubs" : "a");
auto const aStart = a.code.frontier;
auto const astubsStart = astubs.code.frontier;
patchJumps(a, state, block);
state.addresses[block] = aStart;
// If the block ends with a Jmp_ and the next block is going to be
// its target, we don't need to actually emit it.
IRInstruction* last = block->back();
state.noTerminalJmp_ = last->op() == Jmp_ && nextBlock == last->taken();
CodeGenerator cg(trace, a, astubs, tx64, state);
if (state.asmInfo) {
state.asmInfo->asmRanges[block] = TcaRange(aStart, a.code.frontier);
}
cg.cgBlock(block, bcMap);
state.lastMarker = nullptr;
if (auto next = block->next()) {
if (next != nextBlock) {
// If there's a fallthrough block and it's not the next thing
// going into this assembler, then emit a jump to it.
emitFwdJmp(a, next, state);
}
}
if (state.asmInfo) {
state.asmInfo->asmRanges[block] = TcaRange(aStart, a.code.frontier);
if (&a != &astubs) {
state.asmInfo->astubRanges[block] = TcaRange(astubsStart,
astubs.code.frontier);
}
}
};
if (RuntimeOption::EvalHHIRGenerateAsserts && trace->isMain()) {
emitTraceCall(as, trace->bcOff(), tx64);
}
auto const linfo = layoutBlocks(trace, *irFactory);
for (auto it = linfo.blocks.begin(); it != linfo.astubsIt; ++it) {
Block* nextBlock = boost::next(it) != linfo.astubsIt
? *boost::next(it) : nullptr;
emitBlock(as, *it, nextBlock);
}
for (auto it = linfo.astubsIt; it != linfo.blocks.end(); ++it) {
Block* nextBlock = boost::next(it) != linfo.blocks.end()
? *boost::next(it) : nullptr;
emitBlock(astubs, *it, nextBlock);
}
if (debug) {
for (Block* UNUSED block : linfo.blocks) {
assert(isEmitted(block));
}
}
}
ALWAYS_INLINE
TypedValue& getDefaultIfNullCell(TypedValue* tv, TypedValue& def) {
if (UNLIKELY(nullptr == tv)) {
// refcount is already correct since def was never decrefed
return def;
}
tvRefcountedDecRef(&def);
TypedValue* ret = tvToCell(tv);
tvRefcountedIncRef(ret);
return *ret;
}
HOT_FUNC_VM
TypedValue arrayIdxS(ArrayData* a, StringData* key, TypedValue def) {
return getDefaultIfNullCell(a->nvGet(key), def);
}
HOT_FUNC_VM
TypedValue arrayIdxSi(ArrayData* a, StringData* key, TypedValue def) {
int64_t i;
return UNLIKELY(key->isStrictlyInteger(i)) ?
getDefaultIfNullCell(a->nvGet(i), def) :
getDefaultIfNullCell(a->nvGet(key), def);
}
HOT_FUNC_VM
TypedValue arrayIdxI(ArrayData* a, int64_t key, TypedValue def) {
return getDefaultIfNullCell(a->nvGet(key), def);
}
}}
Referência em uma Nova Issue Ver Git Blame Copiar Link Permanente