From 22058efe41b507f77b3f0b98ae8818d34d9eceae Mon Sep 17 00:00:00 2001 From: smith Date: Fri, 29 Mar 2013 06:07:19 -0700 Subject: [PATCH] Allow different RefData and TypedValue layouts Fixing various bugs all over the VM that make assumptions about RefData and TypedValue layout. Here are the assumptions fixed by this diff: offsetof(RefData, m_tv) == 0. Both JIT's assumed this in many subtle ways, by punning RefData* as TypedValue* without adding an offset. This assumption also causes RefData._count to overlap TypedValue.m_aux, which constraints TypedValue layout. offsetof(TypedValue, m_data) == 0. gen_ext_hhvm.php assumes you can cast TypedValue* to Value*; the JITs often weren't using offsetof(TypedValue, m_data) in their addressing calculations. HHIR assumed return-by-value TV's have m_data/m_type in rax/rdx, which can change when TV layout changes. offsetof(TypedValue, m_type) > 8 is an assumption baked into the pass-by-value register assignment logic in HHIR's codegen.cpp; if the type is in the low word, register assignment is swapped. sizeof(TypedValue::m_type) == 4. We used dword-sized operations in both JIT's when accessing m_type. Now, we use helper functions that are sensitive to sizeof(DataType) Configuration: DEBUG=: (opt) same layouts as trunk for RefData & TypedValue DEBUG=1: (dbg) new RefData layout (m_tv doesn't overlap RefData::_count) PACKED_TV=1, DEBUG=*: new RefData and TypedValue layout. --- hphp/runtime/base/array/hphp_array.cpp | 43 +++-- hphp/runtime/base/array/hphp_array.h | 16 +- hphp/runtime/base/hphp_value.h | 63 +++++-- hphp/runtime/base/memory/memory_manager.cpp | 9 +- hphp/runtime/base/memory/smart_allocator.cpp | 3 + hphp/runtime/base/memory/smart_allocator.h | 3 + hphp/runtime/base/ref_data.cpp | 1 + hphp/runtime/base/ref_data.h | 63 +++++-- hphp/runtime/base/shared/shared_variant.cpp | 5 +- hphp/runtime/base/shared/shared_variant.h | 87 +++++----- hphp/runtime/base/type_array.h | 2 - hphp/runtime/base/type_object.h | 6 - hphp/runtime/base/type_string.h | 2 - hphp/runtime/base/types.h | 52 +++--- hphp/runtime/vm/blob_helper.h | 15 +- hphp/runtime/vm/bytecode.cpp | 15 +- hphp/runtime/vm/translator/hopt/codegen.cpp | 157 +++++++----------- hphp/runtime/vm/translator/hopt/codegen.h | 13 +- .../vm/translator/translator-x64-internal.h | 51 +++++- .../vm/translator/translator-x64-vector.cpp | 67 ++++---- hphp/runtime/vm/translator/translator-x64.cpp | 122 ++++++++------ hphp/runtime/vm/translator/translator-x64.h | 2 +- hphp/util/atomic.h | 3 +- hphp/util/base.h | 8 + hphp/util/trace.h | 2 + 25 files changed, 454 insertions(+), 356 deletions(-) diff --git a/hphp/runtime/base/array/hphp_array.cpp b/hphp/runtime/base/array/hphp_array.cpp index d262f9044..29c0b6d77 100644 --- a/hphp/runtime/base/array/hphp_array.cpp +++ b/hphp/runtime/base/array/hphp_array.cpp @@ -1752,7 +1752,7 @@ void setmDecRef(StringData* sd) { decRefStr(sd); } template static inline ArrayData* -array_setm(TypedValue* cell, ArrayData* ad, Key key, TypedValue* value) { +array_setm(RefData* ref, ArrayData* ad, Key key, TypedValue* value) { ArrayData* retval; bool copy = ad->getCount() > 1; // nvSet will decRef any old value that may have been overwritten @@ -1765,12 +1765,9 @@ array_setm(TypedValue* cell, ArrayData* ad, Key key, TypedValue* value) { // methods that didn't bump up the refcount so that we didn't // have to decrement it here if (DecRefValue) tvRefcountedDecRef(value); - if (cell == nullptr) { - return arrayRefShuffle(ad, retval, cell); - } else { - arrayRefShuffle(ad, retval, cell); - return nullptr; - } + if (!ref) return arrayRefShuffle(ad, retval, nullptr); + arrayRefShuffle(ad, retval, ref->tv()); + return nullptr; } /** @@ -1781,14 +1778,14 @@ array_setm(TypedValue* cell, ArrayData* ad, Key key, TypedValue* value) { * array_setm_ik1_v0 -- * Don't count the array's reference to the polymorphic value. */ -ArrayData* array_setm_ik1_v(TypedValue* cell, ArrayData* ad, int64_t key, +ArrayData* array_setm_ik1_v(RefData* ref, ArrayData* ad, int64_t key, TypedValue* value) { - return array_setm(cell, ad, key, value); + return array_setm(ref, ad, key, value); } -ArrayData* array_setm_ik1_v0(TypedValue* cell, ArrayData* ad, int64_t key, +ArrayData* array_setm_ik1_v0(RefData* ref, ArrayData* ad, int64_t key, TypedValue* value) { - return array_setm(cell, ad, key, value); + return array_setm(ref, ad, key, value); } /** @@ -1809,34 +1806,34 @@ ArrayData* array_setm_ik1_v0(TypedValue* cell, ArrayData* ad, int64_t key, * Dont decRef the key, and skip the check for * whether the key is really an integer. */ -ArrayData* array_setm_sk1_v(TypedValue* cell, ArrayData* ad, StringData* key, +ArrayData* array_setm_sk1_v(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value) { - return array_setm(cell, ad, key, value); + return array_setm(ref, ad, key, value); } -ArrayData* array_setm_sk1_v0(TypedValue* cell, ArrayData* ad, StringData* key, +ArrayData* array_setm_sk1_v0(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value) { - return array_setm(cell, ad, key, value); + return array_setm(ref, ad, key, value); } -ArrayData* array_setm_s0k1_v(TypedValue* cell, ArrayData* ad, StringData* key, +ArrayData* array_setm_s0k1_v(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value) { - return array_setm(cell, ad, key, value); + return array_setm(ref, ad, key, value); } -ArrayData* array_setm_s0k1_v0(TypedValue* cell, ArrayData* ad, StringData* key, +ArrayData* array_setm_s0k1_v0(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value) { - return array_setm(cell, ad, key, value); + return array_setm(ref, ad, key, value); } -ArrayData* array_setm_s0k1nc_v(TypedValue* cell, ArrayData* ad, StringData* key, +ArrayData* array_setm_s0k1nc_v(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value) { - return array_setm(cell, ad, key, value); + return array_setm(ref, ad, key, value); } -ArrayData* array_setm_s0k1nc_v0(TypedValue* cell, ArrayData* ad, +ArrayData* array_setm_s0k1nc_v0(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value) { - return array_setm(cell, ad, key, value); + return array_setm(ref, ad, key, value); } /** diff --git a/hphp/runtime/base/array/hphp_array.h b/hphp/runtime/base/array/hphp_array.h index 907c54e7b..d687f6363 100644 --- a/hphp/runtime/base/array/hphp_array.h +++ b/hphp/runtime/base/array/hphp_array.h @@ -522,21 +522,21 @@ enum ArrayGetFlags { CheckInts = 2 }; -ArrayData* array_setm_ik1_v(TypedValue* cell, ArrayData* ad, int64_t key, +ArrayData* array_setm_ik1_v(RefData* ref, ArrayData* ad, int64_t key, TypedValue* value); -ArrayData* array_setm_ik1_v0(TypedValue* cell, ArrayData* ad, int64_t key, +ArrayData* array_setm_ik1_v0(RefData* ref, ArrayData* ad, int64_t key, TypedValue* value); -ArrayData* array_setm_sk1_v(TypedValue* cell, ArrayData* ad, StringData* key, +ArrayData* array_setm_sk1_v(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value); -ArrayData* array_setm_sk1_v0(TypedValue* cell, ArrayData* ad, StringData* key, +ArrayData* array_setm_sk1_v0(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value); -ArrayData* array_setm_s0k1_v(TypedValue* cell, ArrayData* ad, StringData* key, +ArrayData* array_setm_s0k1_v(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value); -ArrayData* array_setm_s0k1_v0(TypedValue* cell, ArrayData* ad, StringData* key, +ArrayData* array_setm_s0k1_v0(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value); -ArrayData* array_setm_s0k1nc_v(TypedValue* cell, ArrayData* ad, StringData* key, +ArrayData* array_setm_s0k1nc_v(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value); -ArrayData* array_setm_s0k1nc_v0(TypedValue* cell, ArrayData* ad, +ArrayData* array_setm_s0k1nc_v0(RefData* ref, ArrayData* ad, StringData* key, TypedValue* value); ArrayData* array_setm_wk1_v0(ArrayData* ad, TypedValue* value); ArrayData* array_getm_i(void* hphpArray, int64_t key, TypedValue* out); diff --git a/hphp/runtime/base/hphp_value.h b/hphp/runtime/base/hphp_value.h index 9e83c4cd7..b07584a45 100644 --- a/hphp/runtime/base/hphp_value.h +++ b/hphp/runtime/base/hphp_value.h @@ -39,7 +39,7 @@ struct TypedValue; * when the type is known beforehand. */ union Value { - int64_t num; // KindOfInt64, KindOfBool + int64_t num; // KindOfInt64, KindOfBool (must be zero-extended) double dbl; // KindOfDouble StringData *pstr; // KindOfString, KindOfStaticString ArrayData *parr; // KindOfArray @@ -51,30 +51,59 @@ union Value { enum VarNrFlag { NR_FLAG = 1<<29 }; +union AuxUnion { + int32_t u_hash; // key type and hash for HphpArray and [Stable]Map + VarNrFlag u_varNrFlag; // magic number for asserts in VarNR + bool u_deepInit; // used by Class::initPropsImpl for deep init + int32_t u_cacheHandle; // used by unit.cpp to squirrel away cache handles +}; + +/* + * 7pack format: + * experimental "Packed" format for TypedValues. By grouping 7 tags + * and 7 values separately, we can fit 7 TypedValues in 63 bytes (64 with + * a throw-away alignment byte (t0): + * + * 0 1 2 7 8 16 56 + * [t0][t1][t2]..[t7][value1][value2]..[value7] + * + * With this layout, a single TypedValue requires 16 bytes, and still has + * room for a 32-bit padding field, which we still use in a few places: + * + * 0 1 2 3 4 8 + * [t0][m_type][t2][t3][m_pad][m_data] + */ + /* * A TypedValue is a descriminated PHP Value. m_tag describes the contents * of m_data. m_aux is described above, and must only be read or written * in specialized contexts. */ +#ifdef PACKED_TV +// This TypedValue layout is a subset of the full 7pack format. Client +// code should not mess with the _t0 or _tags padding fields. struct TypedValue { - /** - * The order of the data members is significant. The m_aux field must - * be exactly FAST_REFCOUNT_OFFSET bytes from the beginning of the object. - */ - Value m_data; -private: - friend struct TypedValueAux; union { - int32_t u_hash; // key type and hash for HphpArray and [Stable]Map - VarNrFlag u_varNrFlag; // magic number for asserts in VarNR - bool u_deepInit; // used by Class::initPropsImpl for deep init - int32_t u_cacheHandle; // used by unit.cpp to squirrel away cache handles - } m_aux; -public: + uint8_t _tags[8]; + struct { + uint8_t _t0; + DataType m_type; + AuxUnion m_aux; + }; + }; + Value m_data; + + std::string pretty() const; +}; +#else +struct TypedValue { + Value m_data; + AuxUnion m_aux; DataType m_type; std::string pretty() const; // debug formatting. see trace.h }; +#endif /* * This TypedValue subclass exposes a 32-bit "aux" field somewhere inside it. @@ -93,6 +122,12 @@ struct TypedValueAux : TypedValue { const bool& deepInit() const { return m_aux.u_deepInit; } VarNrFlag& varNrFlag() { return m_aux.u_varNrFlag; } const VarNrFlag& varNrFlag() const { return m_aux.u_varNrFlag; } + +private: + static void assertions() { + static_assert(sizeof(TypedValueAux) <= 16, + "don't add big things to AuxUnion"); + } }; /////////////////////////////////////////////////////////////////////////////// diff --git a/hphp/runtime/base/memory/memory_manager.cpp b/hphp/runtime/base/memory/memory_manager.cpp index dcd03b54b..745751e09 100644 --- a/hphp/runtime/base/memory/memory_manager.cpp +++ b/hphp/runtime/base/memory/memory_manager.cpp @@ -31,10 +31,12 @@ #include #include #include +#include namespace HPHP { /////////////////////////////////////////////////////////////////////////////// +TRACE_SET_MOD(smartalloc); #ifdef USE_JEMALLOC bool MemoryManager::s_statsEnabled = false; size_t MemoryManager::s_cactiveLimitCeiling = 0; @@ -460,8 +462,11 @@ void* SmartAllocatorImpl::alloc(size_t nbytes) { assert(nbytes == size_t(m_itemSize)); MM().getStats().usage += nbytes; void* ptr = m_free.maybePop(); - if (LIKELY(ptr != nullptr)) return ptr; - return MM().slabAlloc(nbytes); + if (UNLIKELY(!ptr)) { + ptr = MM().slabAlloc(nbytes); + } + TRACE(1, "alloc %zu -> %p\n", nbytes, ptr); + return ptr; } /////////////////////////////////////////////////////////////////////////////// diff --git a/hphp/runtime/base/memory/smart_allocator.cpp b/hphp/runtime/base/memory/smart_allocator.cpp index 66e86a1ad..ef9fd13e3 100644 --- a/hphp/runtime/base/memory/smart_allocator.cpp +++ b/hphp/runtime/base/memory/smart_allocator.cpp @@ -20,6 +20,7 @@ #include #include #include +#include /* * Enabling these will prevent us from allocating out of the free list @@ -32,6 +33,8 @@ namespace HPHP { +TRACE_SET_MOD(smartalloc); + /////////////////////////////////////////////////////////////////////////////// // initializer diff --git a/hphp/runtime/base/memory/smart_allocator.h b/hphp/runtime/base/memory/smart_allocator.h index 77a3795fb..c6313cc28 100644 --- a/hphp/runtime/base/memory/smart_allocator.h +++ b/hphp/runtime/base/memory/smart_allocator.h @@ -27,6 +27,7 @@ #include #include #include +#include namespace HPHP { @@ -181,6 +182,7 @@ public: void* alloc() { return alloc(m_itemSize); } void* alloc(size_t size); void dealloc(void *obj) { + TRACE(1, "dealloc %p\n", obj); assert(memset(obj, kSmartFreeFill, m_itemSize)); m_free.push(obj); MemoryManager::TheMemoryManager()->getStats().usage -= m_itemSize; @@ -195,6 +197,7 @@ public: // keep these frequently used fields together. private: + TRACE_SET_MOD(smartalloc); GarbageList m_free; const int m_itemSize; const Name m_name; diff --git a/hphp/runtime/base/ref_data.cpp b/hphp/runtime/base/ref_data.cpp index 35cec531c..19315bce4 100644 --- a/hphp/runtime/base/ref_data.cpp +++ b/hphp/runtime/base/ref_data.cpp @@ -21,6 +21,7 @@ namespace HPHP { IMPLEMENT_SMART_ALLOCATION(RefData); RefData::~RefData() { + assert(m_magic == kMagic); tvAsVariant(&m_tv).~Variant(); } diff --git a/hphp/runtime/base/ref_data.h b/hphp/runtime/base/ref_data.h index 9fd3aff89..7695631a0 100644 --- a/hphp/runtime/base/ref_data.h +++ b/hphp/runtime/base/ref_data.h @@ -30,16 +30,19 @@ namespace HPHP { * but the value held here must not be KindOfRef. */ class RefData { + enum Magic : uint64_t { kMagic = 0xfacefaceb00cb00c }; public: - enum NullInit { - nullinit, - }; - RefData() {} + enum NullInit { nullinit }; + RefData() { assert(m_magic = kMagic); } RefData(NullInit) { + assert(m_magic = kMagic); _count = 1; m_tv.m_type = KindOfNull; } - RefData(DataType t, int64_t datum) { init(t, datum); } + RefData(DataType t, int64_t datum) { + assert(m_magic = kMagic); + init(t, datum); + } ~RefData(); // Don't extend Countable but use these methods to directly @@ -50,16 +53,28 @@ public: DECLARE_SMART_ALLOCATION(RefData); void dump() const; - const TypedValue* tv() const { return &m_tv; } - TypedValue* tv() { return &m_tv; } - const Variant* var() const { return (const Variant*)&m_tv; } - Variant* var() { return (Variant*)&m_tv; } + const TypedValue* tv() const { + assert(m_magic == kMagic); + return &m_tv; + } + TypedValue* tv() { + assert(m_magic == kMagic); + return &m_tv; + } + const Variant* var() const { return (const Variant*)tv(); } + Variant* var() { return reinterpret_cast(tv()); } static constexpr size_t tvOffset() { return offsetof(RefData, m_tv); } + void assertValid() const { + assert(m_magic == kMagic); + } + // TODO: t2221110: get rid of this hack. static RefData* refDataFromVariantIfYouDare(const Variant* var) { - return reinterpret_cast(uintptr_t(var) - tvOffset()); + RefData* ref = reinterpret_cast(uintptr_t(var) - tvOffset()); + ref->assertValid(); + return ref; } private: @@ -73,13 +88,35 @@ private: m_tv.m_type = KindOfNull; } } + + static void compileTimeAsserts() { + static_assert(offsetof(RefData, _count) == FAST_REFCOUNT_OFFSET, ""); + } + +#if defined(DEBUG) || defined(PACKED_TV) +// don't overlap TypedValue with _count. sizeof(*this) = 32. +private: Magic m_magic; +public: mutable int32_t _count; +private: TypedValue m_tv; + static void layoutAsserts() { + static_assert(offsetof(RefData, m_tv) > + offsetof(RefData, _count) + sizeof(int32_t), ""); + }; +#else +// overlap TypedValue with count private: union { - // overlap TypedValue with an explicit struct to expose the _count - // field to the macro-expanded refcount methods above. - struct { void* ptr; mutable int32_t _count; int32_t type; }; + struct { + void* _ptr; + mutable int32_t _count; + }; TypedValue m_tv; }; + static void layoutAsserts() { + static_assert(offsetof(RefData, _count) == TypedValueAux::auxOffset, ""); + static_assert(sizeof(RefData::_count) == TypedValueAux::auxSize, ""); + } +#endif }; ALWAYS_INLINE inline void decRefRef(RefData* ref) { diff --git a/hphp/runtime/base/shared/shared_variant.cpp b/hphp/runtime/base/shared/shared_variant.cpp index d24e1287b..e45fb78ac 100644 --- a/hphp/runtime/base/shared/shared_variant.cpp +++ b/hphp/runtime/base/shared/shared_variant.cpp @@ -26,11 +26,10 @@ namespace HPHP { SharedVariant::SharedVariant(CVarRef source, bool serialized, bool inner /* = false */, bool unserializeObj /* = false */) - : m_count (1), m_shouldCache(false), m_flags(0){ + : m_shouldCache(false), m_flags(0) { assert(!serialized || source.isString()); - + m_count = 1; m_type = source.getType(); - switch (m_type) { case KindOfBoolean: { diff --git a/hphp/runtime/base/shared/shared_variant.h b/hphp/runtime/base/shared/shared_variant.h index 40ecd9d06..cf46713cc 100644 --- a/hphp/runtime/base/shared/shared_variant.h +++ b/hphp/runtime/base/shared/shared_variant.h @@ -163,54 +163,49 @@ private: void operator delete(void* ptr, int num) { free(ptr); } }; - /* This macro is to help making the object layout binary compatible with - * Variant for primitive types. We want to have compile time assertion to - * guard it but still want to have anonymous struct. For non-refcounted - * types, m_shouldCache and m_flags are guaranteed to be 0, and other parts - * of runtime code will not touch the count.*/ -#ifdef WORDS_BIGENDIAN - #define SharedVarData \ - union {\ - int64_t num;\ - double dbl;\ - StringData *str;\ - ImmutableMap* map;\ - VectorData* vec;\ - ImmutableObj* obj;\ - } m_data;\ - int m_count;\ - bool m_shouldCache;\ - uint8_t m_flags;\ - uint16_t m_type + /* + * Keep the object layout binary compatible with Variant for primitive types. + * We want to have compile time assertion to guard it but still want to have + * anonymous struct. For non-refcounted types, m_shouldCache and m_flags are + * guaranteed to be 0, and other parts of runtime will not touch the count. + */ -#else - #define SharedVarData \ - union {\ - int64_t num;\ - double dbl;\ - StringData *str;\ - ImmutableMap* map;\ - VectorData* vec;\ - ImmutableObj* obj;\ - } m_data;\ - int m_count;\ - uint16_t m_type;\ - bool m_shouldCache;\ - uint8_t m_flags - -#endif - - struct SharedVar { - SharedVarData; + union SharedData { + int64_t num; + double dbl; + StringData *str; + ImmutableMap* map; + VectorData* vec; + ImmutableObj* obj; }; union { - struct { - SharedVarData; - }; TypedValue m_tv; + struct { +#if PACKED_TV + uint8_t _typePad; + DataType m_type; + bool m_shouldCache; + uint8_t m_flags; + uint32_t m_count; + SharedData m_data; +#else + #ifdef WORDS_BIGENDIAN + SharedData m_data; + uint32_t m_count; + bool m_shouldCache; + uint8_t m_flags; + uint16_t m_type; + #else + SharedData m_data; + uint32_t m_count; + uint16_t m_type; + bool m_shouldCache; + uint8_t m_flags; + #endif +#endif + }; }; -#undef SharedVarData const static uint8_t SerializedArray = (1<<0); const static uint8_t IsVector = (1<<1); @@ -218,12 +213,14 @@ private: const static uint8_t ObjAttempted = (1<<3); static void compileTimeAssertions() { - static_assert(offsetof(SharedVar, m_data) == offsetof(TypedValue, m_data), + static_assert(offsetof(SharedVariant, m_data) == offsetof(TypedValue, m_data), "Offset of m_data must be equal in SharedVar and TypedValue"); - static_assert(offsetof(SharedVar, m_count) == TypedValueAux::auxOffset, + static_assert(offsetof(SharedVariant, m_count) == TypedValueAux::auxOffset, "Offset of m_count must equal offset of TV.m_aux"); - static_assert(offsetof(SharedVar, m_type) == offsetof(TypedValue, m_type), + static_assert(offsetof(SharedVariant, m_type) == offsetof(TypedValue, m_type), "Offset of m_type must be equal in SharedVar and TypedValue"); + static_assert(sizeof(SharedVariant) == sizeof(TypedValue), + "Be careful with field layout"); } bool getSerializedArray() const { return (bool)(m_flags & SerializedArray);} diff --git a/hphp/runtime/base/type_array.h b/hphp/runtime/base/type_array.h index f547e13d5..aae046a8d 100644 --- a/hphp/runtime/base/type_array.h +++ b/hphp/runtime/base/type_array.h @@ -571,8 +571,6 @@ class Array : protected ArrayBase { } static void compileTimeAssertions() { - static_assert(offsetof(Array, m_px) == offsetof(TypedValue, m_data), - "Offset of m_px in Array must be offset of m_data in TV"); BOOST_STATIC_ASSERT((offsetof(Array, m_px) == kExpectedMPxOffset)); } }; diff --git a/hphp/runtime/base/type_object.h b/hphp/runtime/base/type_object.h index eae83de70..b36977a6d 100644 --- a/hphp/runtime/base/type_object.h +++ b/hphp/runtime/base/type_object.h @@ -201,12 +201,6 @@ public: m_px = nullptr; return ret; } - - private: - static void compileTimeAssertions() { - static_assert(offsetof(Object, m_px) == offsetof(TypedValue, m_data), - "Offset of m_px in Object must be offset of m_data in TV"); - } }; /////////////////////////////////////////////////////////////////////////////// diff --git a/hphp/runtime/base/type_string.h b/hphp/runtime/base/type_string.h index 0192949a8..a52c62469 100644 --- a/hphp/runtime/base/type_string.h +++ b/hphp/runtime/base/type_string.h @@ -471,8 +471,6 @@ public: } static void compileTimeAssertions() { - static_assert(offsetof(String, m_px) == offsetof(TypedValue, m_data), - "Offset of m_px in String must be offset of m_data in TV"); BOOST_STATIC_ASSERT((offsetof(String, m_px) == kExpectedMPxOffset)); } }; diff --git a/hphp/runtime/base/types.h b/hphp/runtime/base/types.h index da0a356fb..4e8dbff4b 100644 --- a/hphp/runtime/base/types.h +++ b/hphp/runtime/base/types.h @@ -27,6 +27,7 @@ #include #include +#include namespace HPHP { /////////////////////////////////////////////////////////////////////////////// @@ -125,7 +126,8 @@ class VariableUnserializer; * */ -enum DataType { +typedef std::conditional::type DataTypeInt; +enum DataType: DataTypeInt { KindOfClass = -13, MinDataType = -13, @@ -173,31 +175,31 @@ enum DataType { KindOfUncountedInitBit = 8, }; -BOOST_STATIC_ASSERT(KindOfString & KindOfStringBit); -BOOST_STATIC_ASSERT(KindOfStaticString & KindOfStringBit); -BOOST_STATIC_ASSERT(!(KindOfUninit & KindOfStringBit)); -BOOST_STATIC_ASSERT(!(KindOfNull & KindOfStringBit)); -BOOST_STATIC_ASSERT(!(KindOfBoolean & KindOfStringBit)); -BOOST_STATIC_ASSERT(!(KindOfInt64 & KindOfStringBit)); -BOOST_STATIC_ASSERT(!(KindOfDouble & KindOfStringBit)); -BOOST_STATIC_ASSERT(!(KindOfArray & KindOfStringBit)); -BOOST_STATIC_ASSERT(!(KindOfObject & KindOfStringBit)); -BOOST_STATIC_ASSERT(!(KindOfRef & KindOfStringBit)); -BOOST_STATIC_ASSERT(!(KindOfIndirect & KindOfStringBit)); -BOOST_STATIC_ASSERT(!(KindOfClass & KindOfStringBit)); +static_assert(KindOfString & KindOfStringBit, ""); +static_assert(KindOfStaticString & KindOfStringBit, ""); +static_assert(!(KindOfUninit & KindOfStringBit), ""); +static_assert(!(KindOfNull & KindOfStringBit), ""); +static_assert(!(KindOfBoolean & KindOfStringBit), ""); +static_assert(!(KindOfInt64 & KindOfStringBit), ""); +static_assert(!(KindOfDouble & KindOfStringBit), ""); +static_assert(!(KindOfArray & KindOfStringBit), ""); +static_assert(!(KindOfObject & KindOfStringBit), ""); +static_assert(!(KindOfRef & KindOfStringBit), ""); +static_assert(!(KindOfIndirect & KindOfStringBit), ""); +static_assert(!(KindOfClass & KindOfStringBit), ""); -BOOST_STATIC_ASSERT(KindOfNull & KindOfUncountedInitBit); -BOOST_STATIC_ASSERT(KindOfBoolean & KindOfUncountedInitBit); -BOOST_STATIC_ASSERT(KindOfInt64 & KindOfUncountedInitBit); -BOOST_STATIC_ASSERT(KindOfDouble & KindOfUncountedInitBit); -BOOST_STATIC_ASSERT(KindOfStaticString & KindOfUncountedInitBit); -BOOST_STATIC_ASSERT(!(KindOfUninit & KindOfUncountedInitBit)); -BOOST_STATIC_ASSERT(!(KindOfString & KindOfUncountedInitBit)); -BOOST_STATIC_ASSERT(!(KindOfArray & KindOfUncountedInitBit)); -BOOST_STATIC_ASSERT(!(KindOfObject & KindOfUncountedInitBit)); -BOOST_STATIC_ASSERT(!(KindOfRef & KindOfUncountedInitBit)); -BOOST_STATIC_ASSERT(!(KindOfIndirect & KindOfUncountedInitBit)); -BOOST_STATIC_ASSERT(!(KindOfClass & KindOfUncountedInitBit)); +static_assert(KindOfNull & KindOfUncountedInitBit, ""); +static_assert(KindOfBoolean & KindOfUncountedInitBit, ""); +static_assert(KindOfInt64 & KindOfUncountedInitBit, ""); +static_assert(KindOfDouble & KindOfUncountedInitBit, ""); +static_assert(KindOfStaticString & KindOfUncountedInitBit, ""); +static_assert(!(KindOfUninit & KindOfUncountedInitBit), ""); +static_assert(!(KindOfString & KindOfUncountedInitBit), ""); +static_assert(!(KindOfArray & KindOfUncountedInitBit), ""); +static_assert(!(KindOfObject & KindOfUncountedInitBit), ""); +static_assert(!(KindOfRef & KindOfUncountedInitBit), ""); +static_assert(!(KindOfIndirect & KindOfUncountedInitBit), ""); +static_assert(!(KindOfClass & KindOfUncountedInitBit), ""); // assume KindOfUninit == 0 in ClsCns static_assert(KindOfUninit == 0, diff --git a/hphp/runtime/vm/blob_helper.h b/hphp/runtime/vm/blob_helper.h index 7f925e84a..81a7a3a75 100644 --- a/hphp/runtime/vm/blob_helper.h +++ b/hphp/runtime/vm/blob_helper.h @@ -121,13 +121,19 @@ struct BlobEncoder { const_cast(t).serde(*this); } + void encode(DataType t) { + // always encode DataType as int8 even if it's a bigger size. + assert(DataType(int8_t(t)) == t); + encode(int8_t(t)); + } + void encode(const StringData* sd) { if (!sd) return encode(uint32_t(-1)); uint32_t sz = sd->size(); encode(sz); const size_t start = m_blob.size(); - m_blob.resize(m_blob.size() + sd->size()); + m_blob.resize(start + sz); std::copy(sd->data(), sd->data() + sz, &m_blob[start]); } @@ -220,6 +226,13 @@ struct BlobDecoder { t.serde(*this); } + void decode(DataType& t) { + // always decode DataType as int8 even if it's a bigger size. + int8_t t2; + decode(t2); + t = DataType(t2); + } + void decode(const StringData*& sd) { String s(decodeString()); sd = s.get() ? StringData::GetStaticString(s) : 0; diff --git a/hphp/runtime/vm/bytecode.cpp b/hphp/runtime/vm/bytecode.cpp index 5c29060a3..c6c45d063 100644 --- a/hphp/runtime/vm/bytecode.cpp +++ b/hphp/runtime/vm/bytecode.cpp @@ -684,26 +684,31 @@ void Stack::toStringElm(std::ostream& os, TypedValue* tv, const ActRec* fp) } case KindOfStaticString: case KindOfString: { - assert(tv->m_data.pstr->getCount() > 0); int len = tv->m_data.pstr->size(); bool truncated = false; if (len > 128) { len = 128; truncated = true; } - os << tv->m_data.pstr << ":\"" + os << tv->m_data.pstr + << "c(" << tv->m_data.pstr->getCount() << ")" + << ":\"" << Util::escapeStringForCPP(tv->m_data.pstr->data(), len) << "\"" << (truncated ? "..." : ""); break; } case KindOfArray: { assert(tv->m_data.parr->getCount() > 0); - os << tv->m_data.parr << ":Array"; - break; + os << tv->m_data.parr + << "c(" << tv->m_data.parr->getCount() << ")" + << ":Array"; + break; } case KindOfObject: { assert(tv->m_data.pobj->getCount() > 0); - os << tv->m_data.pobj << ":Object(" + os << tv->m_data.pobj + << "c(" << tv->m_data.pobj->getCount() << ")" + << ":Object(" << tvAsVariant(tv).asObjRef().get()->o_getClassName().get()->data() << ")"; break; diff --git a/hphp/runtime/vm/translator/hopt/codegen.cpp b/hphp/runtime/vm/translator/hopt/codegen.cpp index 1f2ec4f91..da01b4b7d 100644 --- a/hphp/runtime/vm/translator/hopt/codegen.cpp +++ b/hphp/runtime/vm/translator/hopt/codegen.cpp @@ -43,33 +43,9 @@ #include "runtime/vm/translator/hopt/linearscan.h" #include "runtime/vm/translator/hopt/nativecalls.h" -using HPHP::DataType; -using HPHP::TypedValue; using HPHP::VM::Transl::TCA; using namespace HPHP::VM::Transl::TargetCache; -// emitDispDeref -- -// emitDeref -- -// -// Helpers for common cell operations. -// -// Dereference the cell whose address lives in src into dest. -/*static */void -emitDispDeref(X64Assembler &a, PhysReg src, int disp, PhysReg dest) { - a. load_reg64_disp_reg64(src, disp + TVOFF(m_data), dest); -} - -/*static*/ void -emitDeref(X64Assembler &a, PhysReg src, PhysReg dest) { - emitDispDeref(a, src, 0, dest); -} - -/* static */ void -emitDerefIfVariant(X64Assembler &a, PhysReg reg) { - emitCmpTVType(a, HPHP::KindOfRef, reg[TVOFF(m_type)]); - a.cload_reg64_disp_reg64(CC_Z, reg, 0, reg); -} - namespace HPHP { namespace VM { namespace JIT { @@ -86,9 +62,12 @@ static const HPHP::Trace::Module TRACEMOD = HPHP::Trace::hhir; using Transl::rVmSp; using Transl::rVmFp; -const int64_t kTypeShiftBits = sizeof(int32_t) * CHAR_BIT; +const size_t kTypeShiftBits = (offsetof(TypedValue, m_type) % 8) * 8; + +// left shift an immediate DataType, for type, to the correct position +// within one of the registers used to pass a TypedValue by value. uint64_t toDataTypeForCall(Type type) { - return (uint64_t)type.toDataType() << (sizeof(int32_t) * CHAR_BIT); + return uint64_t(type.toDataType()) << kTypeShiftBits; } int64_t spillSlotsToSize(int n) { @@ -803,10 +782,12 @@ void CodeGenerator::cgCallHelper(Asm& a, // copy the call result to the destination register(s) if (destType == DestType::TV) { - // rdx contains m_type and m_aux; we need to put just the type in the - // lower bits, so right-shift. rax contains m_data. - if (kTypeShiftBits > 0) a.shrq(kTypeShiftBits, reg::rdx); - shuffle2(a, reg::rax, reg::rdx, dstReg0, dstReg1); + // rax contains m_type and m_aux but we're expecting just the + // type in the lower bits, so shift the type result register. + auto rval = packed_tv ? reg::rdx : reg::rax; + auto rtyp = packed_tv ? reg::rax : reg::rdx; + if (kTypeShiftBits > 0) a.shrq(kTypeShiftBits, rtyp); + shuffle2(a, rval, rtyp, dstReg0, dstReg1); } else if (destType == DestType::SSA) { // copy the single-register result to dstReg0 assert(dstReg1 == InvalidReg); @@ -1347,10 +1328,8 @@ void CodeGenerator::cgOpGte(IRInstruction* inst) { template ConditionCode CodeGenerator::emitTypeTest(Type type, OpndType src, bool negate) { - ConditionCode cc; - assert(!type.subtypeOf(Type::Cls)); - + ConditionCode cc; if (type.isString()) { emitTestTVType(m_as, KindOfStringBit, src); cc = CC_NZ; @@ -1401,7 +1380,7 @@ ConditionCode CodeGenerator::emitIsTypeTest(IRInstruction* inst, bool negate) { assert(src->isA(Type::Gen)); assert(!src->isConst()); PhysReg srcReg = src->getReg(1); // type register - return emitTypeTest(inst, r32(srcReg), negate); + return emitTypeTest(inst, srcReg, negate); } template @@ -1857,39 +1836,27 @@ void CodeGenerator::cgUnbox(IRInstruction* inst) { auto srcValReg = src->getReg(0); auto srcTypeReg = src->getReg(1); + assert(dstValReg != dstTypeReg); assert(src->getType().equals(Type::Gen)); assert(dst->getType().notBoxed()); - - // cmpq KindOfRef, srcTypeReg - // mov srcTypeReg --> dstTypeReg - // - // -- srcTypeReg is now dead - // -- if srcValReg == dstTypeReg, then use scratch for dstTypeReg; otherwise, - // -- the next load will kill srcValReg - // - // cload.z [srcValReg + m_type] --> dstTypeReg - // mov srcValReg --> dstValReg (potentially move up 1 instruction) - // cload.z [srcValReg + m_data] --> dstValReg - - PhysReg tmpDstTypeReg = srcValReg == dstTypeReg ? PhysReg(rScratch) - : dstTypeReg; - - bool useCMov = sizeof(DataType) == 4; - emitMovRegReg(m_as, srcTypeReg, tmpDstTypeReg); - emitMovRegReg(m_as, srcValReg, dstValReg); emitCmpTVType(m_as, HPHP::KindOfRef, srcTypeReg); - // XXX: hardcodes RefData <-> TypedValue pun. - if (useCMov) { - m_as. cload_reg64_disp_reg32(CC_Z, srcValReg, TVOFF(m_type), tmpDstTypeReg); - m_as. cload_reg64_disp_reg64(CC_Z, srcValReg, TVOFF(m_data), dstValReg); - } else { - ifThen(m_as, CC_E, [&]() { - emitLoadTVType(m_as, srcValReg[TVOFF(m_type)], tmpDstTypeReg); - m_as.loadq(srcValReg[TVOFF(m_data)], dstValReg); - }); - } - emitMovRegReg(m_as, tmpDstTypeReg, dstTypeReg); + ifThenElse(CC_E, [&] { + // srcTypeReg == KindOfRef; srcValReg is RefData* + const size_t ref_tv_off = RefData::tvOffset(); + if (dstValReg != srcValReg) { + m_as.loadq(srcValReg[ref_tv_off + TVOFF(m_data)], dstValReg); + emitLoadTVType(m_as, srcValReg[ref_tv_off + TVOFF(m_type)], + r32(dstTypeReg)); + } else { + emitLoadTVType(m_as, srcValReg[ref_tv_off + TVOFF(m_type)], + r32(dstTypeReg)); + m_as.loadq(srcValReg[ref_tv_off + TVOFF(m_data)], dstValReg); + } + }, [&] { + // srcTypeReg != KindOfRef; copy src -> dst + shuffle2(m_as, srcValReg, srcTypeReg, dstValReg, dstTypeReg); + }); } void CodeGenerator::cgLdFixedFunc(IRInstruction* inst) { @@ -2049,16 +2016,17 @@ void traceRet(ActRec* fp, Cell* sp, void* rip) { if (rip == TranslatorX64::Get()->getCallToExit()) { return; } - checkFrame(fp, sp, false); - assertTv(sp); // check return value + checkFrame(fp, sp, /*checkLocals*/ false); + assert(sp <= (Cell*)fp); + // check return value if stack not empty + if (sp < (Cell*)fp) assertTv(sp); } void CodeGenerator::emitTraceRet(CodeGenerator::Asm& a) { // call to a trace function - // ld return ip from native stack into rdx - a. loadq (*rsp, rdx); a. movq (rVmFp, rdi); a. movq (rVmSp, rsi); + a. loadq (*rsp, rdx); // return ip from native stack // do the call; may use a trampoline m_tx64->emitCall(a, TCA(traceRet)); } @@ -2244,7 +2212,6 @@ void CodeGenerator::cgSpill(IRInstruction* inst) { auto sinfo = dst->getSpillInfo(locIndex); assert(sinfo.type() == SpillInfo::Memory); m_as. storeq(srcReg, reg::rsp[sizeof(uint64_t) * sinfo.mem()]); - } } @@ -2292,7 +2259,7 @@ void CodeGenerator::cgStRefWork(IRInstruction* inst, bool genStoreType) { auto destReg = inst->getDst()->getReg(); auto addrReg = inst->getSrc(0)->getReg(); SSATmp* src = inst->getSrc(1); - cgStore(addrReg, 0, src, genStoreType); + cgStore(addrReg, RefData::tvOffset(), src, genStoreType); if (destReg != InvalidReg) emitMovRegReg(m_as, addrReg, destReg); } @@ -2814,21 +2781,16 @@ Address CodeGenerator::cgCheckStaticBitAndDecRef(Type type, // the type is not ref-counted. // Address CodeGenerator::cgCheckRefCountedType(PhysReg typeReg) { - - m_as.cmp_imm32_reg32(KindOfRefCountThreshold, typeReg); - - Address addrToPatch = m_as.code.frontier; + emitCmpTVType(m_as, KindOfRefCountThreshold, typeReg); + Address addrToPatch = m_as.code.frontier; m_as.jcc8(CC_LE, addrToPatch); - return addrToPatch; } -Address CodeGenerator::cgCheckRefCountedType(PhysReg baseReg, - int64_t offset) { +Address CodeGenerator::cgCheckRefCountedType(PhysReg baseReg, int64_t offset) { emitCmpTVType(m_as, KindOfRefCountThreshold, baseReg[offset + TVOFF(m_type)]); - Address addrToPatch = m_as.code.frontier; + Address addrToPatch = m_as.code.frontier; m_as.jcc8(CC_LE, addrToPatch); - return addrToPatch; } @@ -2949,7 +2911,7 @@ void CodeGenerator::cgDecRefDynamicTypeMem(PhysReg baseReg, return; } // Load m_data into the scratch reg - m_as.load_reg64_disp_reg64(baseReg, offset + TVOFF(m_data), scratchReg); + m_as.loadq(baseReg[offset + TVOFF(m_data)], scratchReg); // Emit check for RefCountStaticValue and the actual DecRef Address patchStaticCheck = cgCheckStaticBitAndDecRef(Type::Cell, scratchReg, @@ -2960,7 +2922,7 @@ void CodeGenerator::cgDecRefDynamicTypeMem(PhysReg baseReg, // Emit jump to m_astubs (to call release) if count got down to zero unlikelyIfBlock(CC_Z, [&] { // Emit call to release in m_astubs - m_astubs.lea_reg64_disp_reg64(baseReg, offset, scratchReg); + m_astubs.lea(baseReg[offset], scratchReg); cgCallHelper(m_astubs, getDtorGeneric(), InvalidReg, kSyncPoint, ArgGroup().reg(scratchReg)); }); @@ -2987,7 +2949,7 @@ void CodeGenerator::cgDecRefMem(Type type, // to load the type and the data. cgDecRefDynamicTypeMem(baseReg, offset, exit); } else if (type.maybeCounted()) { - m_as.load_reg64_disp_reg64(baseReg, offset, scratchReg); + m_as.loadq(baseReg[offset + TVOFF(m_data)], scratchReg); cgDecRefStaticType(type, scratchReg, exit, true); } } @@ -3270,7 +3232,7 @@ void CodeGenerator::cgCastStk(IRInstruction *inst) { not_reached(); } cgCallHelper(m_as, tvCastHelper, nullptr, - kSyncPoint, args); + kSyncPoint, args, DestType::None); } void CodeGenerator::cgCallBuiltin(IRInstruction* inst) { @@ -3626,7 +3588,7 @@ void CodeGenerator::cgLoadTypedValue(PhysReg base, emitLoadTVType(m_as, base[off + TVOFF(m_type)], typeDstReg); if (label) { // Check type needed - emitGuardType(r32(typeDstReg), inst); + emitGuardType(typeDstReg, inst); } } else if (label) { // Check type needed @@ -3637,9 +3599,9 @@ void CodeGenerator::cgLoadTypedValue(PhysReg base, if (valueDstReg == InvalidReg) return; if (useScratchReg) { - m_as.load_reg64_disp_reg64(reg::rScratch, off + TVOFF(m_data), valueDstReg); + m_as.loadq(reg::rScratch[off + TVOFF(m_data)], valueDstReg); } else { - m_as.load_reg64_disp_reg64(base, off + TVOFF(m_data), valueDstReg); + m_as.loadq(base[off + TVOFF(m_data)], valueDstReg); } } @@ -3647,9 +3609,7 @@ void CodeGenerator::cgStoreTypedValue(PhysReg base, int64_t off, SSATmp* src) { assert(src->getType().needsReg()); - m_as.store_reg64_disp_reg64(src->getReg(0), - off + TVOFF(m_data), - base); + m_as.storeq(src->getReg(0), base[off + TVOFF(m_data)]); emitStoreTVType(m_as, src->getReg(1), base[off + TVOFF(m_type)]); } @@ -3678,12 +3638,10 @@ void CodeGenerator::cgStore(PhysReg base, } else { not_reached(); } - m_as.store_imm64_disp_reg64(val, off + TVOFF(m_data), base); + m_as.storeq(val, base[off + TVOFF(m_data)]); } else { zeroExtendIfBool(m_as, src); - m_as.store_reg64_disp_reg64(src->getReg(), - off + TVOFF(m_data), - base); + m_as.storeq(src->getReg(), base[off + TVOFF(m_data)]); } } @@ -3736,7 +3694,7 @@ void CodeGenerator::cgLdMem(IRInstruction * inst) { } void CodeGenerator::cgLdRef(IRInstruction* inst) { - cgLoad(inst->getSrc(0)->getReg(), 0, inst); + cgLoad(inst->getSrc(0)->getReg(), RefData::tvOffset(), inst); } void CodeGenerator::recordSyncPoint(Asm& as, @@ -3805,7 +3763,7 @@ void CodeGenerator::cgGuardType(IRInstruction* inst) { assert(srcTypeReg != InvalidReg); ConditionCode cc; - cc = emitTypeTest(type, r32(srcTypeReg), true); + cc = emitTypeTest(type, srcTypeReg, true); emitFwdJcc(cc, inst->getTaken()); auto dstReg = inst->getDst()->getReg(); @@ -3877,12 +3835,12 @@ void CodeGenerator::cgGuardRefs(IRInstruction* inst) { } else if (vals64 != 0) { ifThenElse(CC_NL, thenBody, /* else */ [&] { // If not special builtin... - m_as.test_imm32_disp_reg32(AttrVariadicByRef, Func::attrsOff(), funcPtrReg); + m_as.testl(AttrVariadicByRef, funcPtrReg[Func::attrsOff()]); emitFwdJcc(CC_Z, exitLabel); }); } else { ifThenElse(CC_NL, thenBody, /* else */ [&] { - m_as.test_imm32_disp_reg32(AttrVariadicByRef, Func::attrsOff(), funcPtrReg); + m_as.testl(AttrVariadicByRef, funcPtrReg[Func::attrsOff()]); emitFwdJcc(CC_NZ, exitLabel); }); } @@ -4441,7 +4399,7 @@ void CodeGenerator::cgCheckInit(IRInstruction* inst) { assert(typeReg != InvalidReg); static_assert(KindOfUninit == 0, "cgCheckInit assumes KindOfUninit == 0"); - m_as.testl (r32(typeReg), r32(typeReg)); + m_as.testb (rbyte(typeReg), rbyte(typeReg)); emitFwdJcc(CC_Z, label); } @@ -4452,8 +4410,7 @@ void CodeGenerator::cgCheckInitMem(IRInstruction* inst) { int64_t offset = inst->getSrc(1)->getValInt(); Type t = base->getType().deref(); if (t.isInit()) return; - - m_as.cmpl (KindOfUninit, base->getReg()[offset + TVOFF(m_type)]); + emitCmpTVType(m_as, KindOfUninit, base->getReg()[offset + TVOFF(m_type)]); emitFwdJcc(CC_Z, label); } @@ -4808,7 +4765,7 @@ void traceCallback(ActRec* fp, Cell* sp, int64_t pcOff, void* rip) { << " " << rip << std::endl; #endif - checkFrame(fp, sp, true); + checkFrame(fp, sp, /*checkLocals*/true); } void CodeGenerator::emitTraceCall(CodeGenerator::Asm& as, int64_t pcOff, diff --git a/hphp/runtime/vm/translator/hopt/codegen.h b/hphp/runtime/vm/translator/hopt/codegen.h index c43502ef2..ba26e7c05 100644 --- a/hphp/runtime/vm/translator/hopt/codegen.h +++ b/hphp/runtime/vm/translator/hopt/codegen.h @@ -91,15 +91,6 @@ struct CodegenState { AsmInfo* asmInfo; }; -// Generate an if-then block into a. thenBlock is executed if cc is true. -template -void ifThen(Transl::X64Assembler& a, ConditionCode cc, Then thenBlock) { - Label done; - a.jcc8(ccNegate(cc), done); - thenBlock(); - asm_label(a, done); -} - struct CodeGenerator { typedef Transl::X64Assembler Asm; @@ -432,7 +423,7 @@ struct ArgGroup { * Pass tmp as a TypedValue passed by value. */ ArgGroup& typedValue(SSATmp* tmp) { - return ssa(tmp).type(tmp); + return packed_tv ? type(tmp).ssa(tmp) : ssa(tmp).type(tmp); } ArgGroup& none() { @@ -451,7 +442,7 @@ struct ArgGroup { private: ArgGroup& vectorKeyImpl(SSATmp* key, bool allowInt) { if (key->isString() || (allowInt && key->isA(Type::Int))) { - return ssa(key).none(); + return packed_tv ? none().ssa(key) : ssa(key).none(); } return typedValue(key); } diff --git a/hphp/runtime/vm/translator/translator-x64-internal.h b/hphp/runtime/vm/translator/translator-x64-internal.h index d4a487cce..fdbb5fab6 100644 --- a/hphp/runtime/vm/translator/translator-x64-internal.h +++ b/hphp/runtime/vm/translator/translator-x64-internal.h @@ -33,6 +33,16 @@ namespace Transl { static const Trace::Module TRACEMOD = Trace::tx64; static const DataType BitwiseKindOfString = KindOfString; + +// Generate an if-then block into a. thenBlock is executed if cc is true. +template +void ifThen(Transl::X64Assembler& a, ConditionCode cc, Then thenBlock) { + Label done; + a.jcc8(ccNegate(cc), done); + thenBlock(); + asm_label(a, done); +} + // RAII aids to machine code. // In shared stubs, we've already made the stack odd by calling @@ -571,7 +581,7 @@ asm_label(a, likely); // a ref-counted cell. // // It's ok to do reconcilable register operations in the body. -template struct CondBlock { X64Assembler& m_a; @@ -658,6 +668,8 @@ emitStoreImm(X64Assembler& a, uint64_t imm, PhysReg r, int off, a. store_reg64_disp_reg64(immReg, off, r); } else if (size == sz::dword) { a. store_imm32_disp_reg(imm, off, r); + } else if (size == sz::byte) { + a. store_imm8_disp_reg(imm, off, r); } else { not_implemented(); } @@ -713,6 +725,7 @@ static inline Reg8 toByte(const Reg64& x) { return rbyte(x); } static inline Reg8 toByte(PhysReg x) { return rbyte(x); } static inline Reg32 toReg32(const Reg64& x) { return r32(x); } +static inline Reg32 toReg32(const Reg8& x) { return r32(x); } static inline Reg32 toReg32(PhysReg x) { return r32(x); } // For other operand types, let whatever conversions (or compile @@ -788,6 +801,7 @@ emitStoreTVType(X64Assembler& a, OpndType tvOp, DestType dest) { // dest. static inline void emitDispDeref(X64Assembler &a, Reg64 src, int disp, Reg64 dest) { + // src is a RefData, dest will be m_data field of inner gizmoom. a. loadq (src[disp + TVOFF(m_data)], dest); } @@ -796,10 +810,27 @@ emitDeref(X64Assembler &a, PhysReg src, PhysReg dest) { emitDispDeref(a, src, 0, dest); } +static inline void +emitDerefRef(X64Assembler &a, Reg64 src, int disp, Reg64 dest) { + emitDispDeref(a, src, disp + RefData::tvOffset(), dest); +} + +static inline void +emitDerefRef(X64Assembler &a, Reg64 src, Reg64 dest) { + emitDerefRef(a, src, 0, dest); +} + static inline void emitDerefIfVariant(X64Assembler &a, PhysReg reg) { emitCmpTVType(a, KindOfRef, reg[TVOFF(m_type)]); - a.cload_reg64_disp_reg64(CC_Z, reg, TVOFF(m_data), reg); + if (RefData::tvOffset() == 0) { + a. cload_reg64_disp_reg64(CC_E, reg, TVOFF(m_data), reg); + } else { + ifThen(a, CC_E, [&] { + a. loadq(reg[TVOFF(m_data)], reg); + a. addq(RefData::tvOffset(), reg); + }); + } } // NB: leaves count field unmodified. Does not store to m_data if type @@ -812,16 +843,19 @@ emitStoreTypedValue(X64Assembler& a, DataType type, PhysReg val, } if (!IS_NULL_TYPE(type)) { assert(val != reg::noreg); - a. store_reg64_disp_reg64(val, disp + TVOFF(m_data), dest); + a. storeq(val, dest[disp + TVOFF(m_data)]); } } +static inline void +emitStoreToRefData(X64Assembler& a, DataType type, PhysReg val, + int disp, PhysReg dest) { + emitStoreTypedValue(a, type, val, disp + RefData::tvOffset(), dest); +} + static inline void emitStoreInvalid(X64Assembler& a, int disp, PhysReg dest) { - static_assert(TypedValueAux::auxSize == sizeof(int32_t), - "emitStoreInvalid assumes m_aux is dword sized."); a. storeq (0xfacefacefaceface, dest[disp + TVOFF(m_data)]); - a. storel ((signed int)0xfaceface, dest[disp + TypedValueAux::auxOffset]); emitStoreTVType(a, KindOfInvalid, dest[disp + TVOFF(m_type)]); } @@ -891,6 +925,8 @@ public: void addLoc(const Location &loc); + void addLocRef(const Location &loc); + void addDeref(const Location &loc); void addReg(PhysReg reg); @@ -918,7 +954,7 @@ public: private: struct ArgContent { enum ArgKind { - ArgImm, ArgLoc, ArgDeref, ArgReg, ArgRegPlus, ArgLocAddr + ArgImm, ArgLoc, ArgLocRef, ArgDeref, ArgReg, ArgRegPlus, ArgLocAddr } m_kind; PhysReg m_reg; const Location *m_loc; @@ -948,6 +984,7 @@ private: // arguments #define IMM(i) _am.addImm(i) #define V(loc) _am.addLoc(loc) +#define VREF(loc) _am.addLocRef(loc) #define DEREF(loc) _am.addDeref(loc) #define R(r) _am.addReg(r) #define RPLUS(r,off) _am.addRegPlus(r, off) diff --git a/hphp/runtime/vm/translator/translator-x64-vector.cpp b/hphp/runtime/vm/translator/translator-x64-vector.cpp index 53d2c21b1..61e5ddc5b 100644 --- a/hphp/runtime/vm/translator/translator-x64-vector.cpp +++ b/hphp/runtime/vm/translator/translator-x64-vector.cpp @@ -130,7 +130,8 @@ static const PhysReg unsafe_rsp = rsp; PhysReg reg; \ int disp; \ locToRegDisp(val.location, ®, &disp); \ - a. load_reg64_disp_reg64(reg, disp + TVOFF(m_data), r(rVal)); \ + a. loadq(reg[disp + TVOFF(m_data)], r(rVal)); \ + if (auto offset = RefData::tvOffset()) a.addq(offset, r(rVal)); \ } #define VAL(useRVal) \ IE((useRVal), \ @@ -410,7 +411,8 @@ void TranslatorX64::emitBaseLCR(const Tracelet& t, m_regMap.cleanSmashLoc(base.location); if (base.isVariant()) { // Get inner value. - a. load_reg64_disp_reg64(pr, disp + TVOFF(m_data), r(rBase)); + a. loadq(pr[disp + TVOFF(m_data)], r(rBase)); + if (auto offset = RefData::tvOffset()) a.addq (offset, r(rBase)); } else { a. lea_reg64_disp_reg64(pr, disp, r(rBase)); } @@ -1520,7 +1522,7 @@ void TranslatorX64::emitIssetEmptyProp(const Tracelet& t, BUILD_OPTABH(getKeyTypeS(memb), memb.isVariant(), useEmpty, m_vecState->isObj()); EMIT_RCALL(a, ni, opFunc, CTX(), R(rBase), SML(memb)); - a. and_imm32_reg64(1, rax); // Mask garbage bits. + a. andq(1, rax); // Mask garbage bits. ScratchReg rIssetEmpty(m_regMap, rax); assert(!ni.outLocal); if (useTvResult(t, ni, mii)) { @@ -1668,7 +1670,7 @@ void TranslatorX64::emitSetProp(const Tracelet& t, LazyScratchReg tmp(m_regMap); if (val.isVariant() && !IS_NULL_TYPE(val.rtt.valueType())) { tmp.alloc(); - emitDeref(a, rhsReg, r(tmp)); + emitDerefRef(a, rhsReg, r(tmp)); rhsReg = r(tmp); } @@ -1989,12 +1991,13 @@ void TranslatorX64::emitIncDecProp(const Tracelet& t, #undef HELPER_TABLE template -static inline void bindElemImpl(TypedValue* base, TypedValue* key, RefData* val, +static inline void bindElemImpl(TypedValue* base, TypedValue* key, TypedValue* val, MInstrState* mis) { key = unbox(key); base = ElemD(mis->tvScratch, mis->tvRef, base, key); + assert(val->m_type == KindOfRef); if (!(base == &mis->tvScratch && base->m_type == KindOfUninit)) { - tvBind(val->tv(), base); + tvBind(val, base); } } @@ -2008,8 +2011,8 @@ static inline void bindElemImpl(TypedValue* base, TypedValue* key, RefData* val, m(bindElemS, StrKey, false) #define ELEM(nm, ...) \ -static void nm(TypedValue* base, TypedValue* key, RefData* val, \ - MInstrState* mis) { \ +static void nm(TypedValue* base, TypedValue* key, TypedValue* val, \ + MInstrState* mis) { \ bindElemImpl<__VA_ARGS__>(base, key, val, mis); \ } HELPER_TABLE(ELEM) @@ -2028,7 +2031,7 @@ void TranslatorX64::emitBindElem(const Tracelet& t, cleanOutLocal(ni); assert(!forceMValIncDec(t, ni, mii)); assert(val.isVariant()); - typedef void (*OpFunc)(TypedValue*, TypedValue*, RefData*, MInstrState*); + typedef void (*OpFunc)(TypedValue*, TypedValue*, TypedValue*, MInstrState*); BUILD_OPTAB(getKeyTypeIS(key), key.isVariant()); EMIT_RCALL(a, ni, opFunc, R(rBase), ISML(key), A(val.location), R(mis_rsp)); invalidateOutLocal(ni); @@ -2037,12 +2040,13 @@ void TranslatorX64::emitBindElem(const Tracelet& t, template static inline void bindPropImpl(Class* ctx, TypedValue* base, TypedValue* key, - RefData* val, MInstrState* mis) { + TypedValue* val, MInstrState* mis) { key = unbox(key); base = Prop(mis->tvScratch, mis->tvRef, ctx, base, key); + assert(val->m_type == KindOfRef); if (!(base == &mis->tvScratch && base->m_type == KindOfUninit)) { - tvBind(val->tv(), base); + tvBind(val, base); } } @@ -2059,8 +2063,8 @@ static inline void bindPropImpl(Class* ctx, TypedValue* base, TypedValue* key, #define PROP(nm, ...) \ static inline void nm(Class* ctx, TypedValue* base, TypedValue* key, \ - RefData* val, \ - MInstrState* mis) { \ + TypedValue* val, \ + MInstrState* mis) { \ bindPropImpl<__VA_ARGS__>(ctx, base, key, val, mis); \ } HELPER_TABLE(PROP) @@ -2083,7 +2087,7 @@ void TranslatorX64::emitBindProp(const Tracelet& t, PREP_CTX(argNumToRegName[0]); // Emit the appropriate helper call. assert(!forceMValIncDec(t, ni, mii)); - typedef void (*OpFunc)(Class*, TypedValue*, TypedValue*, RefData*, + typedef void (*OpFunc)(Class*, TypedValue*, TypedValue*, TypedValue*, MInstrState*); BUILD_OPTAB(getKeyTypeS(key), key.isVariant(), m_vecState->isObj()); EMIT_RCALL(a, ni, opFunc, @@ -2373,11 +2377,12 @@ void TranslatorX64::emitIncDecNewElem(const Tracelet& t, } } -static inline void bindNewElem(TypedValue* base, RefData* val, +static inline void bindNewElem(TypedValue* base, TypedValue* val, MInstrState* mis) { base = NewElem(mis->tvScratch, mis->tvRef, base); + assert(val->m_type == KindOfRef); if (!(base == &mis->tvScratch && base->m_type == KindOfUninit)) { - tvBind(val->tv(), base); + tvBind(val, base); } } @@ -2390,10 +2395,8 @@ void TranslatorX64::emitBindNewElem(const Tracelet& t, assert(generateMVal(t, ni, mii)); const DynLocation& val = *ni.inputs[0]; m_regMap.cleanSmashLoc(val.location); - // Emit the appropriate helper call. - void (*bindNewElemOp)(TypedValue*, RefData*, MInstrState*) = bindNewElem; assert(val.isVariant()); - EMIT_RCALL(a, ni, bindNewElemOp, R(rBase), A(val.location), R(mis_rsp)); + EMIT_RCALL(a, ni, bindNewElem, R(rBase), A(val.location), R(mis_rsp)); } void TranslatorX64::emitNotSuppNewElem(const Tracelet& t, @@ -2533,7 +2536,7 @@ void TranslatorX64::emitMPre(const Tracelet& t, RegInfo::CLEAN); if (val.isVariant()) { tmp.alloc(); - emitDeref(a, rVal, r(tmp)); + emitDerefRef(a, rVal, r(tmp)); rVal = r(tmp); } // Copy val to tvVal for later assignment and decref. @@ -2850,7 +2853,7 @@ TranslatorX64::translateCGetM(const Tracelet& t, LazyScratchReg baseScratch(m_regMap); if (base.isVariant()) { baseScratch.alloc(); - emitDeref(a, baseReg, r(baseScratch)); + emitDerefRef(a, baseReg, r(baseScratch)); baseReg = r(baseScratch); } Stats::emitInc(a, Stats::Tx64_CGetMArray); @@ -2908,7 +2911,7 @@ void TranslatorX64::translateIssetMFast(const Tracelet& t, LazyScratchReg scratch(m_regMap); if (base.isVariant()) { scratch.alloc(); - emitDeref(a, arrReg, r(scratch)); + emitDerefRef(a, arrReg, r(scratch)); arrReg = r(scratch); SKTRACE(1, ni.source, "loaded variant\n"); } @@ -3077,18 +3080,18 @@ TranslatorX64::translateSetMArray(const Tracelet& t, bool useBoxedForm = arr.isVariant(); void* fptr; if (false) { // helper type-checks - TypedValue* cell = nullptr; + RefData* ref = nullptr; ArrayData* arr = nullptr; TypedValue* rhs = nullptr; StringData* strKey = nullptr; UNUSED ArrayData* ret; - ret = array_setm_ik1_v(cell, arr, 12, rhs); - ret = array_setm_sk1_v(cell, arr, strKey, rhs); - ret = array_setm_sk1_v0(cell, arr, strKey, rhs); - ret = array_setm_s0k1_v(cell, arr, strKey, rhs); - ret = array_setm_s0k1_v0(cell, arr, strKey, rhs); - ret = array_setm_s0k1nc_v(cell, arr, strKey, rhs); - ret = array_setm_s0k1nc_v0(cell, arr, strKey, rhs); + ret = array_setm_ik1_v(ref, arr, 12, rhs); + ret = array_setm_sk1_v(ref, arr, strKey, rhs); + ret = array_setm_sk1_v0(ref, arr, strKey, rhs); + ret = array_setm_s0k1_v(ref, arr, strKey, rhs); + ret = array_setm_s0k1_v0(ref, arr, strKey, rhs); + ret = array_setm_s0k1nc_v(ref, arr, strKey, rhs); + ret = array_setm_s0k1nc_v0(ref, arr, strKey, rhs); } /* @@ -3126,7 +3129,7 @@ TranslatorX64::translateSetMArray(const Tracelet& t, PhysReg rhsReg = getReg(valLoc); if (valIsVariant) { tmp.alloc(); - emitDeref(a, rhsReg, r(tmp)); + emitDerefRef(a, rhsReg, r(tmp)); rhsReg = r(tmp); } emitIncRef(rhsReg, KindOfArray); @@ -3135,7 +3138,7 @@ TranslatorX64::translateSetMArray(const Tracelet& t, useBoxedForm ? V(arrLoc) : IMM(0), useBoxedForm ? DEREF(arrLoc) : V(arrLoc), V(keyLoc), - valIsVariant ? V(valLoc) : A(valLoc)); + valIsVariant ? VREF(valLoc) : A(valLoc)); recordReentrantCall(i); // If we did not used boxed form, we need to tell the register allocator diff --git a/hphp/runtime/vm/translator/translator-x64.cpp b/hphp/runtime/vm/translator/translator-x64.cpp index 813f7034f..0f829e910 100644 --- a/hphp/runtime/vm/translator/translator-x64.cpp +++ b/hphp/runtime/vm/translator/translator-x64.cpp @@ -131,9 +131,6 @@ enum TransPerfCounter { static __thread int64_t s_perfCounters[tpc_num_counters]; #define INC_TPC(n) ++s_perfCounters[tpc_ ## n]; -#define KindOfString \ -#error You probably do not mean to use KindOfString in this file. - #define NULLCASE() \ case KindOfUninit: case KindOfNull @@ -580,6 +577,7 @@ void ArgManager::computeUsed(std::map &used, m_args[i].m_kind == ArgContent::ArgRegPlus) { reg = m_args[i].m_reg; } else if (m_args[i].m_kind == ArgContent::ArgLoc || + m_args[i].m_kind == ArgContent::ArgLocRef || m_args[i].m_kind == ArgContent::ArgDeref) { reg = m_tx64.getReg(*m_args[i].m_loc); } else { @@ -727,16 +725,22 @@ void ArgManager::shuffleRegisters(std::map &used, void ArgManager::emitValues(std::vector &actual) { for (size_t i = 0; i < m_args.size(); i++) { - switch(m_args[i].m_kind) { + auto kind = m_args[i].m_kind; + auto argReg = argNumToRegName[i]; + switch (kind) { case ArgContent::ArgLoc: + case ArgContent::ArgLocRef: case ArgContent::ArgDeref: case ArgContent::ArgReg: TRACE(6, "ArgManager: copying arg %zd from r%d to r%d\n", - i, int(actual[i]), int(argNumToRegName[i])); - emitMovRegReg(m_a, actual[i], argNumToRegName[i]); + i, int(actual[i]), int(argReg)); + emitMovRegReg(m_a, actual[i], argReg); // Emit dereference if needed - if (m_args[i].m_kind == ArgContent::ArgDeref) { - emitDeref(m_a, argNumToRegName[i], argNumToRegName[i]); + if (kind == ArgContent::ArgDeref) { + emitDerefRef(m_a, argReg, argReg); + } else if (kind == ArgContent::ArgLocRef && RefData::tvOffset()) { + // argReg holds a RefData*; adjust it to be TypedValue* to the value. + m_a.addq(RefData::tvOffset(), argReg); } break; @@ -744,12 +748,12 @@ void ArgManager::emitValues(std::vector &actual) { // If it was used previously by an input value, shuffleRegisters // should have moved it to the proper register from argNumToRegName. case ArgContent::ArgImm: - emitImmReg(m_a, m_args[i].m_imm, argNumToRegName[i]); + emitImmReg(m_a, m_args[i].m_imm, argReg); break; case ArgContent::ArgRegPlus: if (m_args[i].m_imm) { - m_a. add_imm32_reg64(m_args[i].m_imm, argNumToRegName[i]); + m_a. add_imm32_reg64(m_args[i].m_imm, argReg); } break; @@ -758,7 +762,7 @@ void ArgManager::emitValues(std::vector &actual) { PhysReg base; int disp; locToRegDisp(*m_args[i].m_loc, &base, &disp); - emitLea(m_a, base, disp, argNumToRegName[i]); + emitLea(m_a, base, disp, argReg); } break; @@ -1159,7 +1163,7 @@ void TranslatorX64::emitDecRefGenericReg(PhysReg rData, PhysReg rType) { }; Op op = m_curNI->op(); - a. cmpl (KindOfRefCountThreshold, r32(rType)); + emitCmpTVType(a, KindOfRefCountThreshold, rType); if (op == OpSetM || op == OpContSend || op == OpSetG) { // Semi-likely cases semiLikelyIfBlock(CC_A, a, std::bind(body, std::ref(a))); @@ -2391,7 +2395,8 @@ TranslatorX64::emitPrologue(Func* func, int nPassed) { } else { PhysReg base; int disp, k; - if (numLocals < func->numLocals()) { + static_assert(KindOfUninit == 0, ""); + if (numParams < func->numLocals()) { a.xorl (eax, eax); } for (k = numLocals; k < func->numLocals(); ++k) { @@ -3881,7 +3886,7 @@ TranslatorX64::emitUnboxTopOfStack(const NormalizedInstruction& i) { // for the output location m_regMap.allocOutputRegs(i); PhysReg rDest = getReg(i.outStack->location); - emitDeref(a, rSrc, rDest); + emitDerefRef(a, rSrc, rDest); emitIncRef(rDest, outType); // decRef the var on the evaluation stack emitDecRef(i, rSrc, KindOfRef); @@ -4031,12 +4036,12 @@ TranslatorX64::binaryArithLocal(const NormalizedInstruction &i, // The local is a var, so we have to read its value into outReg // on operate on that. We will need to write the result back // to the local after the operation. - emitDeref(a, localReg, r(scr)); + emitDerefRef(a, localReg, r(scr)); emitBody(r(scr)); // We operated on outReg, so we need to write the result back to the // local emitMovRegReg(a, r(scr), outReg); - a. store_reg64_disp_reg64(r(scr), 0, localReg); + a. storeq (r(scr), localReg[RefData::tvOffset() + TVOFF(m_data)]); } } @@ -4646,7 +4651,7 @@ TranslatorX64::translateUnaryBooleanOp(const Tracelet& t, PhysReg reg = getReg(inLoc); PhysReg outReg = getReg(i.outStack->location); if (boxedForm) { - emitDeref(a, reg, outReg); + emitDerefRef(a, reg, outReg); } else { emitMovRegReg(a, reg, outReg); } @@ -4660,7 +4665,7 @@ TranslatorX64::translateUnaryBooleanOp(const Tracelet& t, PhysReg outReg = getReg(i.outStack->location); ScratchReg scratch(m_regMap); if (boxedForm) { - emitDeref(a, reg, r(scratch)); + emitDerefRef(a, reg, r(scratch)); emitConvertToBool(a, r(scratch), outReg, instrNeg); } else { emitConvertToBool(a, reg, outReg, instrNeg); @@ -4891,7 +4896,7 @@ TranslatorX64::translateCGetL(const Tracelet& t, } else { ScratchReg rTmp(m_regMap); PhysReg localReg = getReg(inputs[0]->location); - a.store_reg64_disp_reg64(localReg, stackDisp + TVOFF(m_data), stackBase); + a. storeq (localReg, stackBase[stackDisp + TVOFF(m_data)]); emitLoadTVType(a, locBase[locDisp + TVOFF(m_type)], r(rTmp)); emitStoreTVType(a, r(rTmp), stackBase[stackDisp + TVOFF(m_type)]); } @@ -4932,7 +4937,7 @@ TranslatorX64::translateCGetL(const Tracelet& t, emitMovRegReg(a, localReg, dest); } if (inputs[0]->isVariant()) { - emitDeref(a, dest, dest); + emitDerefRef(a, dest, dest); } assert(outType != KindOfStaticString); emitIncRef(dest, outType); @@ -4983,7 +4988,7 @@ TranslatorX64::translateCGetL2(const Tracelet& t, const PhysReg cellOut = getReg(ni.outStack->location); assert(cellOut != stackIn); if (ni.inputs[locIdx]->isVariant()) { - emitDeref(a, localIn, cellOut); + emitDerefRef(a, localIn, cellOut); } else if (!undefinedLocal) { emitMovRegReg(a, localIn, cellOut); } @@ -5127,20 +5132,20 @@ TranslatorX64::translateAssignToLocalOp(const Tracelet& t, (!locTypeRelaxed && IS_REFCOUNTED_TYPE(decRefType)); if (useOldType) { oldLocalReg.alloc(); - emitDeref(a, localReg, r(oldLocalReg)); + emitDerefRef(a, localReg, r(oldLocalReg)); } if (rhsTypeRelaxed) { PhysReg base; int disp; ScratchReg rTmp(m_regMap); locToRegDisp(ni.inputs[rhsIdx]->location, &base, &disp); - size_t typeOff = TVOFF(m_type); - size_t dataOff = TVOFF(m_data); + size_t typeOff = RefData::tvOffset() + TVOFF(m_type); + size_t dataOff = RefData::tvOffset() + TVOFF(m_data); emitLoadTVType(a, base[disp + TVOFF(m_type)], r(rTmp)); a. storeq (rhsReg, localReg[dataOff]); emitStoreTVType(a, r(rTmp), localReg[typeOff]); } else { - emitStoreTypedValue(a, rhsType, rhsReg, 0, localReg); + emitStoreToRefData(a, rhsType, rhsReg, 0, localReg); } } else if (rhsTypeRelaxed) { PhysReg rhsBase; @@ -5520,14 +5525,14 @@ TranslatorX64::translateAddElemC(const Tracelet& t, // not (for cases where the key is a local). assert(key.rtt.isInt() || key.rtt.isString()); if (false) { // type-check - TypedValue* cell = nullptr; + RefData* ref = nullptr; TypedValue* rhs = nullptr; StringData* strkey = nullptr; ArrayData* arr = nullptr; ArrayData* ret; - ret = array_setm_ik1_v0(cell, arr, 12, rhs); + ret = array_setm_ik1_v0(ref, arr, 12, rhs); printf("%p", ret); // use ret - ret = array_setm_sk1_v0(cell, arr, strkey, rhs); + ret = array_setm_sk1_v0(ref, arr, strkey, rhs); printf("%p", ret); // use ret } // Otherwise, we pass the rhs by address @@ -6648,6 +6653,13 @@ void TranslatorX64::emitReturnVal( tvWriteUninit(&tv); tv.m_data.num = 0; // to keep the compiler happy + auto moveRetValIfNeeded = [&] { + if (thisBase != dstBase || + thisOffset != (dstOffset + TVOFF(m_data))) { + a. loadq(thisBase[thisOffset], scratch); + a. storeq(scratch, dstBase[dstOffset + TVOFF(m_data)]); + } + }; /* * We suppressed the write of the (literal) return value * to the stack. Figure out what it was. @@ -6680,15 +6692,13 @@ void TranslatorX64::emitReturnVal( tv.m_data.parr = curUnit()->lookupArrayId(prev->imm[0].u_AA); break; case OpThis: { - if (thisBase != dstBase || thisOffset != dstOffset) { - a. load_reg64_disp_reg64(thisBase, thisOffset, scratch); - a. store_reg64_disp_reg64(scratch, dstOffset, dstBase); - } + moveRetValIfNeeded(); emitStoreTVType(a, KindOfObject, dstBase[dstOffset + TVOFF(m_type)]); return; } case OpBareThis: { assert(curFunc()->cls()); + moveRetValIfNeeded(); a. mov_imm32_reg32(KindOfNull, scratch); a. testb(1, thisBase[thisOffset]); { @@ -6696,10 +6706,6 @@ void TranslatorX64::emitReturnVal( a. mov_imm32_reg32(KindOfObject, scratch); } emitStoreTVType(a, scratch, dstBase[dstOffset + TVOFF(m_type)]); - if (thisBase != dstBase || thisOffset != dstOffset) { - a. load_reg64_disp_reg64(thisBase, thisOffset, scratch); - a. store_reg64_disp_reg64(scratch, dstOffset, dstBase); - } return; } default: @@ -6709,7 +6715,7 @@ void TranslatorX64::emitReturnVal( emitStoreTVType(a, tv.m_type, r64(dstBase)[dstOffset + TVOFF(m_type)]); if (tv.m_type != KindOfNull) { emitStoreImm(a, tv.m_data.num, - dstBase, dstOffset, sz::qword); + dstBase, dstOffset + TVOFF(m_data), sz::qword); } } @@ -7218,7 +7224,7 @@ TranslatorX64::emitObjToClass(const NormalizedInstruction& i) { PhysReg src = getReg(in); ScratchReg tmp(m_regMap); if (i.inputs[kEmitClsLocalIdx]->rtt.isVariant()) { - emitDeref(a, src, r(tmp)); + emitDerefRef(a, src, r(tmp)); src = r(tmp); } assert(i.outStack->valueType() == KindOfClass); @@ -7424,15 +7430,15 @@ void TranslatorX64::translateCreateCont(const Tracelet& t, // Deal with a potential $this local in the generator body if (fillThis) { assert(thisId != kInvalidId); - a.load_reg64_disp_reg64(rax, CONTOFF(m_obj), r(rScratch)); - a.test_reg64_reg64(r(rScratch), r(rScratch)); + a. load_reg64_disp_reg64(rax, CONTOFF(m_obj), r(rScratch)); + a. test_reg64_reg64(r(rScratch), r(rScratch)); { JccBlock ifObj(a); const int thisOff = cellsToBytes(genLocals - thisId - 1); // We don't have to check for a static refcount since we // know it's an Object - a.add_imm32_disp_reg32(1, FAST_REFCOUNT_OFFSET, r(rScratch)); - a.store_reg64_disp_reg64(r(rScratch), thisOff + TVOFF(m_data), r(rDest)); + a. addl(1, r(rScratch)[FAST_REFCOUNT_OFFSET]); + a. storeq(r(rScratch), r(rDest)[thisOff + TVOFF(m_data)]); emitStoreTVType(a, KindOfObject, r(rDest)[thisOff + TVOFF(m_type)]); } } @@ -9447,7 +9453,7 @@ TranslatorX64::translateThis(const Tracelet &t, curFunc()->isClosureBody()); m_regMap.allocOutputRegs(i); PhysReg out = getReg(i.outStack->location); - a. load_reg64_disp_reg64(rVmFp, AROFF(m_this), out); + a. loadq(rVmFp[AROFF(m_this)], out); if (!i.guardedThis) { emitThisCheck(i, out); @@ -9497,7 +9503,7 @@ TranslatorX64::translateBareThis(const Tracelet &t, emitIncRef(out, KindOfObject); if (i.outStack->rtt.isVagueValue()) { emitStoreTVType(a, KindOfObject, base[offset + TVOFF(m_type)]); - a. store_reg64_disp_reg64(out, TVOFF(m_data) + offset, base); + a. storeq(out, base[TVOFF(m_data) + offset]); } else { assert(i.outStack->isObject()); m_regMap.bindScratch(outScratch, i.outStack->location, KindOfObject, @@ -9526,20 +9532,20 @@ TranslatorX64::translateInitThisLoc(const Tracelet& t, assert(base == rVmFp); ScratchReg thiz(m_regMap); - a.load_reg64_disp_reg64(rVmFp, AROFF(m_this), r(thiz)); + a. load_reg64_disp_reg64(rVmFp, AROFF(m_this), r(thiz)); if (curFunc()->cls() == nullptr) { // If we're in a pseudomain, m_this could be NULL - a.test_reg64_reg64(r(thiz), r(thiz)); - a.jz(astubs.code.frontier); // jz if_null + a. testq (r(thiz), r(thiz)); + a. jz (astubs.code.frontier); // jz if_null } // Ok, it's not NULL but it might be a Class which should be treated // equivalently - a.testb(1, rbyte(thiz)); - a.jnz(astubs.code.frontier); // jnz if_null + a. testb(1, rbyte(thiz)); + a. jnz(astubs.code.frontier); // jnz if_null // We have a valid $this! emitStoreTVType(a, KindOfObject, base[offset + TVOFF(m_type)]); - a.store_reg64_disp_reg64(r(thiz), offset + TVOFF(m_data), base); + a. storeq(r(thiz), base[offset + TVOFF(m_data)]); emitIncRef(r(thiz), KindOfObject); // if_null: @@ -10146,11 +10152,11 @@ void TranslatorX64::translateFCallBuiltin(const Tracelet& t, // For bool return value, get the %al byte case KindOfBoolean: a. movzbl (al, eax); // sign extend byte->qword - emitStoreTypedValue(a, func->returnType(), rax, disp, base, true); + emitStoreTypedValue(a, returnType, rax, disp, base, true); break; case KindOfNull: /* void return type */ case KindOfInt64: - emitStoreTypedValue(a, func->returnType(), rax, disp, base, true); + emitStoreTypedValue(a, returnType, rax, disp, base, true); break; STRINGCASE(): case KindOfArray: @@ -10370,7 +10376,7 @@ TranslatorX64::translateVerifyParamType(const Tracelet& t, PhysReg src = getReg(in); ScratchReg inCls(m_regMap); if (i.inputs[0]->rtt.isVariant()) { - emitDeref(a, src, r(inCls)); + emitDerefRef(a, src, r(inCls)); src = r(inCls); } a. load_reg64_disp_reg64(src, ObjectData::getVMClassOffset(), r(inCls)); @@ -10530,7 +10536,7 @@ TranslatorX64::translateInstanceOfD(const Tracelet& t, PhysReg baseReg = srcReg; if (input0->rtt.isVariant()) { assert(input0IsLoc); - emitDeref(a, srcReg, r(inCls)); + emitDerefRef(a, srcReg, r(inCls)); baseReg = r(inCls); } a. load_reg64_disp_reg64(baseReg, ObjectData::getVMClassOffset(), @@ -11088,7 +11094,7 @@ TranslatorX64::emitVariantGuards(const Tracelet& t, } if (isRef && !m_useHHIR) { m_regMap.allocInputReg(i, in); - emitOneGuard(t, *base, getReg(input->location), 0, + emitOneGuard(t, *base, getReg(input->location), RefData::tvOffset(), input->rtt.innerType(), sideExit); } } @@ -12569,6 +12575,12 @@ void ArgManager::addLoc(const Location &loc) { m_args.push_back(ArgContent(ArgContent::ArgLoc, loc)); } +void ArgManager::addLocRef(const Location &loc) { + TRACE(6, "ArgManager: push arg ref %zd loc:(%s, %" PRId64 ")\n", + m_args.size(), loc.spaceName(), loc.offset); + m_args.push_back(ArgContent(ArgContent::ArgLocRef, loc)); +} + void ArgManager::addLocAddr(const Location &loc) { TRACE(6, "ArgManager: push arg %zd addr:(%s, %" PRId64 ")\n", m_args.size(), loc.spaceName(), loc.offset); diff --git a/hphp/runtime/vm/translator/translator-x64.h b/hphp/runtime/vm/translator/translator-x64.h index fb921af92..0773a128a 100644 --- a/hphp/runtime/vm/translator/translator-x64.h +++ b/hphp/runtime/vm/translator/translator-x64.h @@ -116,7 +116,7 @@ class TranslatorX64 : public Translator friend class HPHP::VM::JIT::CodeGenerator; friend class HPHP::VM::JIT::HhbcTranslator; // packBitVec() friend TCA funcBodyHelper(ActRec* fp); - template friend class CondBlock; + template friend class CondBlock; template friend class JccBlock; template friend class IfElseBlock; friend class UnlikelyIfBlock; diff --git a/hphp/util/atomic.h b/hphp/util/atomic.h index 9badea93a..da173ab25 100644 --- a/hphp/util/atomic.h +++ b/hphp/util/atomic.h @@ -83,7 +83,8 @@ static inline T atomic_inc(T &count) { return __sync_fetch_and_add(&count, 1) + 1; } -static inline int atomic_dec(int &count) { +template +static inline T atomic_dec(T &count) { assert_address_is_atomically_accessible(&count); return __sync_fetch_and_add(&count, -1) - 1; } diff --git a/hphp/util/base.h b/hphp/util/base.h index 8259d070b..d7d906b1d 100644 --- a/hphp/util/base.h +++ b/hphp/util/base.h @@ -144,6 +144,14 @@ static const bool use_jemalloc = #endif ; +static const bool packed_tv = +#ifdef PACKED_TV + true +#else + false +#endif + ; + /** * Guard bug-for-bug hphpi compatibility code with this predicate. */ diff --git a/hphp/util/trace.h b/hphp/util/trace.h index 2b5f5276e..dcd1bce5d 100644 --- a/hphp/util/trace.h +++ b/hphp/util/trace.h @@ -92,6 +92,8 @@ namespace Trace { /* Jit bisection interval */ \ TM(txOpBisectLow) \ TM(txOpBisectHigh) \ + /* smart alloc */ \ + TM(smartalloc) \ /* Temporary catetories, to save compilation time */ \ TM(tmp0) TM(tmp1) TM(tmp2) TM(tmp3) \ TM(tmp4) TM(tmp5) TM(tmp6) TM(tmp7) \