Allow different RefData and TypedValue layouts

Fixing various bugs all over the VM that make assumptions about RefData
and TypedValue layout.  Here are the assumptions fixed by this diff:

offsetof(RefData, m_tv) == 0.  Both JIT's assumed this in many subtle
ways, by punning RefData* as TypedValue* without adding an offset.
This assumption also causes RefData._count to overlap TypedValue.m_aux,
which constraints TypedValue layout.

offsetof(TypedValue, m_data) == 0.  gen_ext_hhvm.php assumes you
can cast TypedValue* to Value*; the JITs often weren't using
offsetof(TypedValue, m_data) in their addressing calculations.  HHIR
assumed return-by-value TV's have m_data/m_type in rax/rdx, which
can change when TV layout changes.

offsetof(TypedValue, m_type) > 8 is an assumption baked into the
pass-by-value register assignment logic in HHIR's codegen.cpp; if
the type is in the low word, register assignment is swapped.

sizeof(TypedValue::m_type) == 4.  We used dword-sized operations
in both JIT's when accessing m_type.  Now, we use helper functions
that are sensitive to sizeof(DataType)

Configuration:
DEBUG=: (opt)  same layouts as trunk for RefData & TypedValue
DEBUG=1: (dbg) new RefData layout (m_tv doesn't overlap RefData::_count)
PACKED_TV=1, DEBUG=*:  new RefData and TypedValue layout.

hphp/runtime/base/array/hphp_array.cpp

@@ -1752,7 +1752,7 @@ void setmDecRef(StringData* sd) { decRefStr(sd); }
 template<typename Key, bool DecRefValue, bool CheckInt, bool DecRefKey>
 static inline
 ArrayData*
-array_setm(TypedValue* cell, ArrayData* ad, Key key, TypedValue* value) {
+array_setm(RefData* ref, ArrayData* ad, Key key, TypedValue* value) {
   ArrayData* retval;
   bool copy = ad->getCount() > 1;
   // nvSet will decRef any old value that may have been overwritten
@@ -1765,12 +1765,9 @@ array_setm(TypedValue* cell, ArrayData* ad, Key key, TypedValue* value) {
   // methods that didn't bump up the refcount so that we didn't
   // have to decrement it here
   if (DecRefValue) tvRefcountedDecRef(value);
-  if (cell == nullptr) {
-    return arrayRefShuffle<false>(ad, retval, cell);
-  } else {
-    arrayRefShuffle<true>(ad, retval, cell);
-    return nullptr;
-  }
+  if (!ref) return arrayRefShuffle<false>(ad, retval, nullptr);
+  arrayRefShuffle<true>(ad, retval, ref->tv());
+  return nullptr;
 }
 
 /**
@@ -1781,14 +1778,14 @@ array_setm(TypedValue* cell, ArrayData* ad, Key key, TypedValue* value) {
  *    array_setm_ik1_v0 --
  *       Don't count the array's reference to the polymorphic value.
  */
-ArrayData* array_setm_ik1_v(TypedValue* cell, ArrayData* ad, int64_t key,
+ArrayData* array_setm_ik1_v(RefData* ref, ArrayData* ad, int64_t key,
                             TypedValue* value) {
-  return array_setm<int64_t, false, false, false>(cell, ad, key, value);
+  return array_setm<int64_t, false, false, false>(ref, ad, key, value);
 }
 
-ArrayData* array_setm_ik1_v0(TypedValue* cell, ArrayData* ad, int64_t key,
+ArrayData* array_setm_ik1_v0(RefData* ref, ArrayData* ad, int64_t key,
                              TypedValue* value) {
-  return array_setm<int64_t, true, false, false>(cell, ad, key, value);
+  return array_setm<int64_t, true, false, false>(ref, ad, key, value);
 }
 
 /**
@@ -1809,34 +1806,34 @@ ArrayData* array_setm_ik1_v0(TypedValue* cell, ArrayData* ad, int64_t key,
  *       Dont decRef the key, and skip the check for
  *       whether the key is really an integer.
  */
-ArrayData* array_setm_sk1_v(TypedValue* cell, ArrayData* ad, StringData* key,
+ArrayData* array_setm_sk1_v(RefData* ref, ArrayData* ad, StringData* key,
                             TypedValue* value) {
-  return array_setm<StringData*, false, true, true>(cell, ad, key, value);
+  return array_setm<StringData*, false, true, true>(ref, ad, key, value);
 }
 
-ArrayData* array_setm_sk1_v0(TypedValue* cell, ArrayData* ad, StringData* key,
+ArrayData* array_setm_sk1_v0(RefData* ref, ArrayData* ad, StringData* key,
                              TypedValue* value) {
-  return array_setm<StringData*, true, true, true>(cell, ad, key, value);
+  return array_setm<StringData*, true, true, true>(ref, ad, key, value);
 }
 
-ArrayData* array_setm_s0k1_v(TypedValue* cell, ArrayData* ad, StringData* key,
+ArrayData* array_setm_s0k1_v(RefData* ref, ArrayData* ad, StringData* key,
                              TypedValue* value) {
-  return array_setm<StringData*, false, true, false>(cell, ad, key, value);
+  return array_setm<StringData*, false, true, false>(ref, ad, key, value);
 }
 
-ArrayData* array_setm_s0k1_v0(TypedValue* cell, ArrayData* ad, StringData* key,
+ArrayData* array_setm_s0k1_v0(RefData* ref, ArrayData* ad, StringData* key,
                               TypedValue* value) {
-  return array_setm<StringData*, true, true, false>(cell, ad, key, value);
+  return array_setm<StringData*, true, true, false>(ref, ad, key, value);
 }
 
-ArrayData* array_setm_s0k1nc_v(TypedValue* cell, ArrayData* ad, StringData* key,
+ArrayData* array_setm_s0k1nc_v(RefData* ref, ArrayData* ad, StringData* key,
                                TypedValue* value) {
-  return array_setm<StringData*, false, false, false>(cell, ad, key, value);
+  return array_setm<StringData*, false, false, false>(ref, ad, key, value);
 }
 
-ArrayData* array_setm_s0k1nc_v0(TypedValue* cell, ArrayData* ad,
+ArrayData* array_setm_s0k1nc_v0(RefData* ref, ArrayData* ad,
                                 StringData* key, TypedValue* value) {
-  return array_setm<StringData*, true, false, false>(cell, ad, key, value);
+  return array_setm<StringData*, true, false, false>(ref, ad, key, value);
 }
 
 /**

hphp/runtime/base/array/hphp_array.h

@@ -522,21 +522,21 @@ enum ArrayGetFlags {
   CheckInts = 2
 };
 
-ArrayData* array_setm_ik1_v(TypedValue* cell, ArrayData* ad, int64_t key,
+ArrayData* array_setm_ik1_v(RefData* ref, ArrayData* ad, int64_t key,
                             TypedValue* value);
-ArrayData* array_setm_ik1_v0(TypedValue* cell, ArrayData* ad, int64_t key,
+ArrayData* array_setm_ik1_v0(RefData* ref, ArrayData* ad, int64_t key,
                              TypedValue* value);
-ArrayData* array_setm_sk1_v(TypedValue* cell, ArrayData* ad, StringData* key,
+ArrayData* array_setm_sk1_v(RefData* ref, ArrayData* ad, StringData* key,
                             TypedValue* value);
-ArrayData* array_setm_sk1_v0(TypedValue* cell, ArrayData* ad, StringData* key,
+ArrayData* array_setm_sk1_v0(RefData* ref, ArrayData* ad, StringData* key,
                              TypedValue* value);
-ArrayData* array_setm_s0k1_v(TypedValue* cell, ArrayData* ad, StringData* key,
+ArrayData* array_setm_s0k1_v(RefData* ref, ArrayData* ad, StringData* key,
                              TypedValue* value);
-ArrayData* array_setm_s0k1_v0(TypedValue* cell, ArrayData* ad, StringData* key,
+ArrayData* array_setm_s0k1_v0(RefData* ref, ArrayData* ad, StringData* key,
                               TypedValue* value);
-ArrayData* array_setm_s0k1nc_v(TypedValue* cell, ArrayData* ad, StringData* key,
+ArrayData* array_setm_s0k1nc_v(RefData* ref, ArrayData* ad, StringData* key,
                                TypedValue* value);
-ArrayData* array_setm_s0k1nc_v0(TypedValue* cell, ArrayData* ad,
+ArrayData* array_setm_s0k1nc_v0(RefData* ref, ArrayData* ad,
                                 StringData* key, TypedValue* value);
 ArrayData* array_setm_wk1_v0(ArrayData* ad, TypedValue* value);
 ArrayData* array_getm_i(void* hphpArray, int64_t key, TypedValue* out);

hphp/runtime/base/hphp_value.h

@@ -39,7 +39,7 @@ struct TypedValue;
  * when the type is known beforehand.
  */
 union Value {
-  int64_t     num;  // KindOfInt64, KindOfBool
+  int64_t     num;  // KindOfInt64, KindOfBool (must be zero-extended)
   double      dbl;  // KindOfDouble
   StringData *pstr; // KindOfString, KindOfStaticString
   ArrayData  *parr; // KindOfArray
@@ -51,30 +51,59 @@ union Value {
 
 enum VarNrFlag { NR_FLAG = 1<<29 };
 
+union AuxUnion {
+  int32_t u_hash;        // key type and hash for HphpArray and [Stable]Map
+  VarNrFlag u_varNrFlag; // magic number for asserts in VarNR
+  bool u_deepInit;       // used by Class::initPropsImpl for deep init
+  int32_t u_cacheHandle; // used by unit.cpp to squirrel away cache handles
+};
+
+/*
+ * 7pack format:
+ * experimental "Packed" format for TypedValues.  By grouping 7 tags
+ * and 7 values separately, we can fit 7 TypedValues in 63 bytes (64 with
+ * a throw-away alignment byte (t0):
+ *
+ *   0   1   2     7   8       16        56
+ *   [t0][t1][t2]..[t7][value1][value2]..[value7]
+ *
+ * With this layout, a single TypedValue requires 16 bytes, and still has
+ * room for a 32-bit padding field, which we still use in a few places:
+ *
+ *   0   1       2   3   4      8
+ *   [t0][m_type][t2][t3][m_pad][m_data]
+ */
+
 /*
  * A TypedValue is a descriminated PHP Value.  m_tag describes the contents
  * of m_data.  m_aux is described above, and must only be read or written
  * in specialized contexts.
  */
+#ifdef PACKED_TV
+// This TypedValue layout is a subset of the full 7pack format.  Client
+// code should not mess with the _t0 or _tags padding fields.
 struct TypedValue {
-  /**
-   * The order of the data members is significant. The m_aux field must
-   * be exactly FAST_REFCOUNT_OFFSET bytes from the beginning of the object.
-   */
-  Value m_data;
-private:
-  friend struct TypedValueAux;
   union {
-    int32_t u_hash;        // key type and hash for HphpArray and [Stable]Map
-    VarNrFlag u_varNrFlag; // magic number for asserts in VarNR
-    bool u_deepInit;       // used by Class::initPropsImpl for deep init
-    int32_t u_cacheHandle; // used by unit.cpp to squirrel away cache handles
-  } m_aux;
-public:
+    uint8_t _tags[8];
+    struct {
+      uint8_t _t0;
+      DataType m_type;
+      AuxUnion m_aux;
+    };
+  };
+  Value m_data;
+
+  std::string pretty() const;
+};
+#else
+struct TypedValue {
+  Value m_data;
+  AuxUnion m_aux;
   DataType m_type;
 
   std::string pretty() const; // debug formatting. see trace.h
 };
+#endif
 
 /*
  * This TypedValue subclass exposes a 32-bit "aux" field somewhere inside it.
@@ -93,6 +122,12 @@ struct TypedValueAux : TypedValue {
   const bool& deepInit() const { return m_aux.u_deepInit; }
   VarNrFlag& varNrFlag() { return m_aux.u_varNrFlag; }
   const VarNrFlag& varNrFlag() const { return m_aux.u_varNrFlag; }
+
+private:
+  static void assertions() {
+    static_assert(sizeof(TypedValueAux) <= 16,
+                  "don't add big things to AuxUnion");
+  }
 };
 
 ///////////////////////////////////////////////////////////////////////////////

hphp/runtime/base/memory/memory_manager.cpp

@@ -31,10 +31,12 @@
 #include <runtime/base/server/http_server.h>
 #include <util/alloc.h>
 #include <util/process.h>
+#include <util/trace.h>
 
 namespace HPHP {
 ///////////////////////////////////////////////////////////////////////////////
 
+TRACE_SET_MOD(smartalloc);
 #ifdef USE_JEMALLOC
 bool MemoryManager::s_statsEnabled = false;
 size_t MemoryManager::s_cactiveLimitCeiling = 0;
@@ -460,8 +462,11 @@ void* SmartAllocatorImpl::alloc(size_t nbytes) {
   assert(nbytes == size_t(m_itemSize));
   MM().getStats().usage += nbytes;
   void* ptr = m_free.maybePop();
-  if (LIKELY(ptr != nullptr)) return ptr;
-  return MM().slabAlloc(nbytes);
+  if (UNLIKELY(!ptr)) {
+    ptr = MM().slabAlloc(nbytes);
+  }
+  TRACE(1, "alloc %zu -> %p\n", nbytes, ptr);
+  return ptr;
 }
 
 ///////////////////////////////////////////////////////////////////////////////

hphp/runtime/base/memory/smart_allocator.cpp

@@ -20,6 +20,7 @@
 #include <runtime/base/server/server_stats.h>
 #include <runtime/base/runtime_option.h>
 #include <util/logger.h>
+#include <util/trace.h>
 
 /*
  * Enabling these will prevent us from allocating out of the free list
@@ -32,6 +33,8 @@
 
 namespace HPHP {
 
+TRACE_SET_MOD(smartalloc);
+
 ///////////////////////////////////////////////////////////////////////////////
 // initializer
 

hphp/runtime/base/memory/smart_allocator.h

@@ -27,6 +27,7 @@
 #include <runtime/base/types.h>
 #include <runtime/base/util/countable.h>
 #include <runtime/base/memory/memory_usage_stats.h>
+#include <util/trace.h>
 
 namespace HPHP {
 
@@ -181,6 +182,7 @@ public:
   void* alloc() { return alloc(m_itemSize); }
   void* alloc(size_t size);
   void dealloc(void *obj) {
+    TRACE(1, "dealloc %p\n", obj);
     assert(memset(obj, kSmartFreeFill, m_itemSize));
     m_free.push(obj);
     MemoryManager::TheMemoryManager()->getStats().usage -= m_itemSize;
@@ -195,6 +197,7 @@ public:
 
   // keep these frequently used fields together.
 private:
+  TRACE_SET_MOD(smartalloc);
   GarbageList m_free;
   const int m_itemSize;
   const Name m_name;

hphp/runtime/base/ref_data.cpp

@@ -21,6 +21,7 @@ namespace HPHP {
 IMPLEMENT_SMART_ALLOCATION(RefData);
 
 RefData::~RefData() {
+  assert(m_magic == kMagic);
   tvAsVariant(&m_tv).~Variant();
 }
 

hphp/runtime/base/ref_data.h

@@ -30,16 +30,19 @@ namespace HPHP {
  * but the value held here must not be KindOfRef.
  */
 class RefData {
+  enum Magic : uint64_t { kMagic = 0xfacefaceb00cb00c };
 public:
-  enum NullInit {
-    nullinit,
-  };
-  RefData() {}
+  enum NullInit { nullinit };
+  RefData() { assert(m_magic = kMagic); }
   RefData(NullInit) {
+    assert(m_magic = kMagic);
     _count = 1;
     m_tv.m_type = KindOfNull;
   }
-  RefData(DataType t, int64_t datum) { init(t, datum); }
+  RefData(DataType t, int64_t datum) {
+    assert(m_magic = kMagic);
+    init(t, datum);
+  }
   ~RefData();
 
   // Don't extend Countable but use these methods to directly
@@ -50,16 +53,28 @@ public:
   DECLARE_SMART_ALLOCATION(RefData);
   void dump() const;
 
-  const TypedValue* tv() const { return &m_tv; }
-  TypedValue* tv() { return &m_tv; }
-  const Variant* var() const { return (const Variant*)&m_tv; }
-  Variant* var() { return (Variant*)&m_tv; }
+  const TypedValue* tv() const {
+    assert(m_magic == kMagic);
+    return &m_tv;
+  }
+  TypedValue* tv() {
+    assert(m_magic == kMagic);
+    return &m_tv;
+  }
+  const Variant* var() const { return (const Variant*)tv(); }
+  Variant* var() { return reinterpret_cast<Variant*>(tv()); }
 
   static constexpr size_t tvOffset() { return offsetof(RefData, m_tv); }
 
+  void assertValid() const {
+    assert(m_magic == kMagic);
+  }
+
   // TODO: t2221110: get rid of this hack.
   static RefData* refDataFromVariantIfYouDare(const Variant* var) {
-    return reinterpret_cast<RefData*>(uintptr_t(var) - tvOffset());
+    RefData* ref = reinterpret_cast<RefData*>(uintptr_t(var) - tvOffset());
+    ref->assertValid();
+    return ref;
   }
 
 private:
@@ -73,13 +88,35 @@ private:
       m_tv.m_type = KindOfNull;
     }
   }
+
+  static void compileTimeAsserts() {
+    static_assert(offsetof(RefData, _count) == FAST_REFCOUNT_OFFSET, "");
+  }
+
+#if defined(DEBUG) || defined(PACKED_TV)
+// don't overlap TypedValue with _count.  sizeof(*this) = 32.
+private: Magic m_magic;
+public:  mutable int32_t _count;
+private: TypedValue m_tv;
+  static void layoutAsserts() {
+    static_assert(offsetof(RefData, m_tv) >
+                  offsetof(RefData, _count) + sizeof(int32_t), "");
+  };
+#else
+// overlap TypedValue with count
 private:
   union {
-    // overlap TypedValue with an explicit struct to expose the _count
-    // field to the macro-expanded refcount methods above.
-    struct { void* ptr; mutable int32_t _count; int32_t type; };
+    struct {
+      void* _ptr;
+      mutable int32_t _count;
+    };
     TypedValue m_tv;
   };
+  static void layoutAsserts() {
+    static_assert(offsetof(RefData, _count) == TypedValueAux::auxOffset, "");
+    static_assert(sizeof(RefData::_count) == TypedValueAux::auxSize, "");
+  }
+#endif
 };
 
 ALWAYS_INLINE inline void decRefRef(RefData* ref) {

hphp/runtime/base/shared/shared_variant.cpp

@@ -26,11 +26,10 @@ namespace HPHP {
 SharedVariant::SharedVariant(CVarRef source, bool serialized,
                              bool inner /* = false */,
                              bool unserializeObj /* = false */)
-  : m_count (1), m_shouldCache(false), m_flags(0){
+  : m_shouldCache(false), m_flags(0) {
   assert(!serialized || source.isString());
-
+  m_count = 1;
   m_type = source.getType();
-
   switch (m_type) {
   case KindOfBoolean:
     {

hphp/runtime/base/shared/shared_variant.h

@@ -163,54 +163,49 @@ private:
     void operator delete(void* ptr, int num) { free(ptr); }
   };
 
-  /* This macro is to help making the object layout binary compatible with
-   * Variant for primitive types. We want to have compile time assertion to
-   * guard it but still want to have anonymous struct. For non-refcounted
-   * types, m_shouldCache and m_flags are guaranteed to be 0, and other parts
-   * of runtime code will not touch the count.*/
-#ifdef WORDS_BIGENDIAN
- #define SharedVarData \
-  union {\
-    int64_t num;\
-    double dbl;\
-    StringData *str;\
-    ImmutableMap* map;\
-    VectorData* vec;\
-    ImmutableObj* obj;\
-  } m_data;\
-  int m_count;\
-  bool m_shouldCache;\
-  uint8_t m_flags;\
-  uint16_t m_type
+  /*
+   * Keep the object layout binary compatible with Variant for primitive types.
+   * We want to have compile time assertion to guard it but still want to have
+   * anonymous struct. For non-refcounted types, m_shouldCache and m_flags are
+   * guaranteed to be 0, and other parts of runtime will not touch the count.
+   */
 
-#else
- #define SharedVarData \
-  union {\
-    int64_t num;\
-    double dbl;\
-    StringData *str;\
-    ImmutableMap* map;\
-    VectorData* vec;\
-    ImmutableObj* obj;\
-  } m_data;\
-  int m_count;\
-  uint16_t m_type;\
-  bool m_shouldCache;\
-  uint8_t m_flags
-
-#endif
-
-  struct SharedVar {
-    SharedVarData;
+  union SharedData {
+    int64_t num;
+    double dbl;
+    StringData *str;
+    ImmutableMap* map;
+    VectorData* vec;
+    ImmutableObj* obj;
   };
 
   union {
-    struct {
-      SharedVarData;
-    };
     TypedValue m_tv;
+    struct {
+#if PACKED_TV
+      uint8_t _typePad;
+      DataType m_type;
+      bool m_shouldCache;
+      uint8_t m_flags;
+      uint32_t m_count;
+      SharedData m_data;
+#else
+ #ifdef WORDS_BIGENDIAN
+      SharedData m_data;
+      uint32_t m_count;
+      bool m_shouldCache;
+      uint8_t m_flags;
+      uint16_t m_type;
+ #else
+      SharedData m_data;
+      uint32_t m_count;
+      uint16_t m_type;
+      bool m_shouldCache;
+      uint8_t m_flags;
+ #endif
+#endif
+    };
   };
-#undef SharedVarData
 
   const static uint8_t SerializedArray = (1<<0);
   const static uint8_t IsVector = (1<<1);
@@ -218,12 +213,14 @@ private:
   const static uint8_t ObjAttempted = (1<<3);
 
   static void compileTimeAssertions() {
-    static_assert(offsetof(SharedVar, m_data) == offsetof(TypedValue, m_data),
+    static_assert(offsetof(SharedVariant, m_data) == offsetof(TypedValue, m_data),
                   "Offset of m_data must be equal in SharedVar and TypedValue");
-    static_assert(offsetof(SharedVar, m_count) == TypedValueAux::auxOffset,
+    static_assert(offsetof(SharedVariant, m_count) == TypedValueAux::auxOffset,
                   "Offset of m_count must equal offset of TV.m_aux");
-    static_assert(offsetof(SharedVar, m_type) == offsetof(TypedValue, m_type),
+    static_assert(offsetof(SharedVariant, m_type) == offsetof(TypedValue, m_type),
                   "Offset of m_type must be equal in SharedVar and TypedValue");
+    static_assert(sizeof(SharedVariant) == sizeof(TypedValue),
+                  "Be careful with field layout");
   }
 
   bool getSerializedArray() const { return (bool)(m_flags & SerializedArray);}

hphp/runtime/base/type_array.h

@@ -571,8 +571,6 @@ class Array : protected ArrayBase {
   }
 
   static void compileTimeAssertions() {
-    static_assert(offsetof(Array, m_px) == offsetof(TypedValue, m_data),
-                  "Offset of m_px in Array must be offset of m_data in TV");
     BOOST_STATIC_ASSERT((offsetof(Array, m_px) == kExpectedMPxOffset));
   }
 };

hphp/runtime/base/type_object.h

@@ -201,12 +201,6 @@ public:
     m_px = nullptr;
     return ret;
   }
-
- private:
-  static void compileTimeAssertions() {
-    static_assert(offsetof(Object, m_px) == offsetof(TypedValue, m_data),
-                  "Offset of m_px in Object must be offset of m_data in TV");
-  }
 };
 
 ///////////////////////////////////////////////////////////////////////////////

hphp/runtime/base/type_string.h

@@ -471,8 +471,6 @@ public:
   }
 
   static void compileTimeAssertions() {
-    static_assert(offsetof(String, m_px) == offsetof(TypedValue, m_data),
-                  "Offset of m_px in String must be offset of m_data in TV");
     BOOST_STATIC_ASSERT((offsetof(String, m_px) == kExpectedMPxOffset));
   }
 };

hphp/runtime/base/types.h

@@ -27,6 +27,7 @@
 
 #include <boost/static_assert.hpp>
 #include <boost/intrusive_ptr.hpp>
+#include <type_traits>
 
 namespace HPHP {
 ///////////////////////////////////////////////////////////////////////////////
@@ -125,7 +126,8 @@ class VariableUnserializer;
  *
  */
 
-enum DataType {
+typedef std::conditional<packed_tv, int8_t, int32_t>::type DataTypeInt;
+enum DataType: DataTypeInt {
   KindOfClass            = -13,
   MinDataType            = -13,
 
@@ -173,31 +175,31 @@ enum DataType {
   KindOfUncountedInitBit = 8,
 };
 
-BOOST_STATIC_ASSERT(KindOfString       & KindOfStringBit);
-BOOST_STATIC_ASSERT(KindOfStaticString & KindOfStringBit);
-BOOST_STATIC_ASSERT(!(KindOfUninit     & KindOfStringBit));
-BOOST_STATIC_ASSERT(!(KindOfNull       & KindOfStringBit));
-BOOST_STATIC_ASSERT(!(KindOfBoolean    & KindOfStringBit));
-BOOST_STATIC_ASSERT(!(KindOfInt64      & KindOfStringBit));
-BOOST_STATIC_ASSERT(!(KindOfDouble     & KindOfStringBit));
-BOOST_STATIC_ASSERT(!(KindOfArray      & KindOfStringBit));
-BOOST_STATIC_ASSERT(!(KindOfObject     & KindOfStringBit));
-BOOST_STATIC_ASSERT(!(KindOfRef        & KindOfStringBit));
-BOOST_STATIC_ASSERT(!(KindOfIndirect   & KindOfStringBit));
-BOOST_STATIC_ASSERT(!(KindOfClass      & KindOfStringBit));
+static_assert(KindOfString       & KindOfStringBit, "");
+static_assert(KindOfStaticString & KindOfStringBit, "");
+static_assert(!(KindOfUninit     & KindOfStringBit), "");
+static_assert(!(KindOfNull       & KindOfStringBit), "");
+static_assert(!(KindOfBoolean    & KindOfStringBit), "");
+static_assert(!(KindOfInt64      & KindOfStringBit), "");
+static_assert(!(KindOfDouble     & KindOfStringBit), "");
+static_assert(!(KindOfArray      & KindOfStringBit), "");
+static_assert(!(KindOfObject     & KindOfStringBit), "");
+static_assert(!(KindOfRef        & KindOfStringBit), "");
+static_assert(!(KindOfIndirect   & KindOfStringBit), "");
+static_assert(!(KindOfClass      & KindOfStringBit), "");
 
-BOOST_STATIC_ASSERT(KindOfNull         & KindOfUncountedInitBit);
-BOOST_STATIC_ASSERT(KindOfBoolean      & KindOfUncountedInitBit);
-BOOST_STATIC_ASSERT(KindOfInt64        & KindOfUncountedInitBit);
-BOOST_STATIC_ASSERT(KindOfDouble       & KindOfUncountedInitBit);
-BOOST_STATIC_ASSERT(KindOfStaticString & KindOfUncountedInitBit);
-BOOST_STATIC_ASSERT(!(KindOfUninit     & KindOfUncountedInitBit));
-BOOST_STATIC_ASSERT(!(KindOfString     & KindOfUncountedInitBit));
-BOOST_STATIC_ASSERT(!(KindOfArray      & KindOfUncountedInitBit));
-BOOST_STATIC_ASSERT(!(KindOfObject     & KindOfUncountedInitBit));
-BOOST_STATIC_ASSERT(!(KindOfRef        & KindOfUncountedInitBit));
-BOOST_STATIC_ASSERT(!(KindOfIndirect   & KindOfUncountedInitBit));
-BOOST_STATIC_ASSERT(!(KindOfClass      & KindOfUncountedInitBit));
+static_assert(KindOfNull         & KindOfUncountedInitBit, "");
+static_assert(KindOfBoolean      & KindOfUncountedInitBit, "");
+static_assert(KindOfInt64        & KindOfUncountedInitBit, "");
+static_assert(KindOfDouble       & KindOfUncountedInitBit, "");
+static_assert(KindOfStaticString & KindOfUncountedInitBit, "");
+static_assert(!(KindOfUninit     & KindOfUncountedInitBit), "");
+static_assert(!(KindOfString     & KindOfUncountedInitBit), "");
+static_assert(!(KindOfArray      & KindOfUncountedInitBit), "");
+static_assert(!(KindOfObject     & KindOfUncountedInitBit), "");
+static_assert(!(KindOfRef        & KindOfUncountedInitBit), "");
+static_assert(!(KindOfIndirect   & KindOfUncountedInitBit), "");
+static_assert(!(KindOfClass      & KindOfUncountedInitBit), "");
 
 // assume KindOfUninit == 0 in ClsCns
 static_assert(KindOfUninit == 0,

hphp/runtime/vm/blob_helper.h

@@ -121,13 +121,19 @@ struct BlobEncoder {
     const_cast<T&>(t).serde(*this);
   }
 
+  void encode(DataType t) {
+    // always encode DataType as int8 even if it's a bigger size.
+    assert(DataType(int8_t(t)) == t);
+    encode(int8_t(t));
+  }
+
   void encode(const StringData* sd) {
     if (!sd) return encode(uint32_t(-1));
     uint32_t sz = sd->size();
     encode(sz);
 
     const size_t start = m_blob.size();
-    m_blob.resize(m_blob.size() + sd->size());
+    m_blob.resize(start + sz);
     std::copy(sd->data(), sd->data() + sz, &m_blob[start]);
   }
 
@@ -220,6 +226,13 @@ struct BlobDecoder {
     t.serde(*this);
   }
 
+  void decode(DataType& t) {
+    // always decode DataType as int8 even if it's a bigger size.
+    int8_t t2;
+    decode(t2);
+    t = DataType(t2);
+  }
+
   void decode(const StringData*& sd) {
     String s(decodeString());
     sd = s.get() ? StringData::GetStaticString(s) : 0;

hphp/runtime/vm/bytecode.cpp

@@ -684,26 +684,31 @@ void Stack::toStringElm(std::ostream& os, TypedValue* tv, const ActRec* fp)
   }
   case KindOfStaticString:
   case KindOfString: {
-    assert(tv->m_data.pstr->getCount() > 0);
     int len = tv->m_data.pstr->size();
     bool truncated = false;
     if (len > 128) {
       len = 128;
       truncated = true;
     }
-    os << tv->m_data.pstr << ":\""
+    os << tv->m_data.pstr
+       << "c(" << tv->m_data.pstr->getCount() << ")"
+       << ":\""
        << Util::escapeStringForCPP(tv->m_data.pstr->data(), len)
        << "\"" << (truncated ? "..." : "");
     break;
   }
   case KindOfArray: {
     assert(tv->m_data.parr->getCount() > 0);
-    os << tv->m_data.parr << ":Array";
-    break;
+    os << tv->m_data.parr
+       << "c(" << tv->m_data.parr->getCount() << ")"
+       << ":Array";
+     break;
   }
   case KindOfObject: {
     assert(tv->m_data.pobj->getCount() > 0);
-    os << tv->m_data.pobj << ":Object("
+    os << tv->m_data.pobj
+       << "c(" << tv->m_data.pobj->getCount() << ")"
+       << ":Object("
        << tvAsVariant(tv).asObjRef().get()->o_getClassName().get()->data()
        << ")";
     break;

hphp/runtime/vm/translator/hopt/codegen.cpp

@@ -43,33 +43,9 @@
 #include "runtime/vm/translator/hopt/linearscan.h"
 #include "runtime/vm/translator/hopt/nativecalls.h"
 
-using HPHP::DataType;
-using HPHP::TypedValue;
 using HPHP::VM::Transl::TCA;
 using namespace HPHP::VM::Transl::TargetCache;
 
-// emitDispDeref --
-// emitDeref --
-//
-//   Helpers for common cell operations.
-//
-//   Dereference the cell whose address lives in src into dest.
-/*static */void
-emitDispDeref(X64Assembler &a, PhysReg src, int disp, PhysReg dest) {
-  a.    load_reg64_disp_reg64(src, disp + TVOFF(m_data), dest);
-}
-
-/*static*/ void
-emitDeref(X64Assembler &a, PhysReg src, PhysReg dest) {
-  emitDispDeref(a, src, 0, dest);
-}
-
-/* static */ void
-emitDerefIfVariant(X64Assembler &a, PhysReg reg) {
-  emitCmpTVType(a, HPHP::KindOfRef, reg[TVOFF(m_type)]);
-  a.cload_reg64_disp_reg64(CC_Z, reg, 0, reg);
-}
-
 namespace HPHP {
 namespace VM {
 namespace JIT {
@@ -86,9 +62,12 @@ static const HPHP::Trace::Module TRACEMOD = HPHP::Trace::hhir;
 using Transl::rVmSp;
 using Transl::rVmFp;
 
-const int64_t kTypeShiftBits = sizeof(int32_t) * CHAR_BIT;
+const size_t kTypeShiftBits = (offsetof(TypedValue, m_type) % 8) * 8;
+
+// left shift an immediate DataType, for type, to the correct position
+// within one of the registers used to pass a TypedValue by value.
 uint64_t toDataTypeForCall(Type type) {
-  return (uint64_t)type.toDataType() << (sizeof(int32_t) * CHAR_BIT);
+  return uint64_t(type.toDataType()) << kTypeShiftBits;
 }
 
 int64_t spillSlotsToSize(int n) {
@@ -803,10 +782,12 @@ void CodeGenerator::cgCallHelper(Asm& a,
 
   // copy the call result to the destination register(s)
   if (destType == DestType::TV) {
-    // rdx contains m_type and m_aux; we need to put just the type in the
-    // lower bits, so right-shift.  rax contains m_data.
-    if (kTypeShiftBits > 0) a.shrq(kTypeShiftBits, reg::rdx);
-    shuffle2(a, reg::rax, reg::rdx, dstReg0, dstReg1);
+    // rax contains m_type and m_aux but we're expecting just the
+    // type in the lower bits, so shift the type result register.
+    auto rval = packed_tv ? reg::rdx : reg::rax;
+    auto rtyp = packed_tv ? reg::rax : reg::rdx;
+    if (kTypeShiftBits > 0) a.shrq(kTypeShiftBits, rtyp);
+    shuffle2(a, rval, rtyp, dstReg0, dstReg1);
   } else if (destType == DestType::SSA) {
     // copy the single-register result to dstReg0
     assert(dstReg1 == InvalidReg);
@@ -1347,10 +1328,8 @@ void CodeGenerator::cgOpGte(IRInstruction* inst) {
 template<class OpndType>
 ConditionCode CodeGenerator::emitTypeTest(Type type, OpndType src,
                                           bool negate) {
-  ConditionCode cc;
-
   assert(!type.subtypeOf(Type::Cls));
-
+  ConditionCode cc;
   if (type.isString()) {
     emitTestTVType(m_as, KindOfStringBit, src);
     cc = CC_NZ;
@@ -1401,7 +1380,7 @@ ConditionCode CodeGenerator::emitIsTypeTest(IRInstruction* inst, bool negate) {
   assert(src->isA(Type::Gen));
   assert(!src->isConst());
   PhysReg srcReg = src->getReg(1); // type register
-  return emitTypeTest(inst, r32(srcReg), negate);
+  return emitTypeTest(inst, srcReg, negate);
 }
 
 template<class OpndType>
@@ -1857,39 +1836,27 @@ void CodeGenerator::cgUnbox(IRInstruction* inst) {
   auto srcValReg  = src->getReg(0);
   auto srcTypeReg = src->getReg(1);
 
+  assert(dstValReg != dstTypeReg);
   assert(src->getType().equals(Type::Gen));
   assert(dst->getType().notBoxed());
 
-
-  // cmpq   KindOfRef, srcTypeReg
-  // mov    srcTypeReg --> dstTypeReg
-  //
-  // -- srcTypeReg is now dead
-  // -- if srcValReg == dstTypeReg, then use scratch for dstTypeReg; otherwise,
-  // -- the next load will kill srcValReg
-  //
-  // cload.z [srcValReg + m_type] --> dstTypeReg
-  // mov    srcValReg --> dstValReg (potentially move up 1 instruction)
-  // cload.z [srcValReg + m_data] --> dstValReg
-
-  PhysReg tmpDstTypeReg = srcValReg == dstTypeReg ? PhysReg(rScratch)
-                                                  : dstTypeReg;
-
-  bool useCMov = sizeof(DataType) == 4;
-  emitMovRegReg(m_as, srcTypeReg, tmpDstTypeReg);
-  emitMovRegReg(m_as, srcValReg, dstValReg);
   emitCmpTVType(m_as, HPHP::KindOfRef, srcTypeReg);
-  // XXX: hardcodes RefData <-> TypedValue pun.
-  if (useCMov) {
-    m_as.  cload_reg64_disp_reg32(CC_Z, srcValReg, TVOFF(m_type), tmpDstTypeReg);
-    m_as.  cload_reg64_disp_reg64(CC_Z, srcValReg, TVOFF(m_data), dstValReg);
-  } else {
-    ifThen(m_as, CC_E, [&]() {
-      emitLoadTVType(m_as, srcValReg[TVOFF(m_type)], tmpDstTypeReg);
-      m_as.loadq(srcValReg[TVOFF(m_data)], dstValReg);
-    });
-  }
-  emitMovRegReg(m_as, tmpDstTypeReg, dstTypeReg);
+  ifThenElse(CC_E, [&] {
+    // srcTypeReg == KindOfRef; srcValReg is RefData*
+    const size_t ref_tv_off = RefData::tvOffset();
+    if (dstValReg != srcValReg) {
+      m_as.loadq(srcValReg[ref_tv_off + TVOFF(m_data)], dstValReg);
+      emitLoadTVType(m_as, srcValReg[ref_tv_off + TVOFF(m_type)],
+                     r32(dstTypeReg));
+    } else {
+      emitLoadTVType(m_as, srcValReg[ref_tv_off + TVOFF(m_type)],
+                     r32(dstTypeReg));
+      m_as.loadq(srcValReg[ref_tv_off + TVOFF(m_data)], dstValReg);
+    }
+  }, [&] {
+    // srcTypeReg != KindOfRef; copy src -> dst
+    shuffle2(m_as, srcValReg, srcTypeReg, dstValReg, dstTypeReg);
+  });
 }
 
 void CodeGenerator::cgLdFixedFunc(IRInstruction* inst) {
@@ -2049,16 +2016,17 @@ void traceRet(ActRec* fp, Cell* sp, void* rip) {
   if (rip == TranslatorX64::Get()->getCallToExit()) {
     return;
   }
-  checkFrame(fp, sp, false);
-  assertTv(sp); // check return value
+  checkFrame(fp, sp, /*checkLocals*/ false);
+  assert(sp <= (Cell*)fp);
+  // check return value if stack not empty
+  if (sp < (Cell*)fp) assertTv(sp);
 }
 
 void CodeGenerator::emitTraceRet(CodeGenerator::Asm& a) {
   // call to a trace function
-  // ld return ip from native stack into rdx
-  a.    loadq (*rsp, rdx);
   a.    movq  (rVmFp, rdi);
   a.    movq  (rVmSp, rsi);
+  a.    loadq (*rsp, rdx); // return ip from native stack
   // do the call; may use a trampoline
   m_tx64->emitCall(a, TCA(traceRet));
 }
@@ -2244,7 +2212,6 @@ void CodeGenerator::cgSpill(IRInstruction* inst) {
     auto sinfo = dst->getSpillInfo(locIndex);
     assert(sinfo.type() == SpillInfo::Memory);
     m_as.    storeq(srcReg, reg::rsp[sizeof(uint64_t) * sinfo.mem()]);
-
   }
 }
 
@@ -2292,7 +2259,7 @@ void CodeGenerator::cgStRefWork(IRInstruction* inst, bool genStoreType) {
   auto destReg = inst->getDst()->getReg();
   auto addrReg = inst->getSrc(0)->getReg();
   SSATmp* src  = inst->getSrc(1);
-  cgStore(addrReg, 0, src, genStoreType);
+  cgStore(addrReg, RefData::tvOffset(), src, genStoreType);
   if (destReg != InvalidReg)  emitMovRegReg(m_as, addrReg, destReg);
 }
 
@@ -2814,21 +2781,16 @@ Address CodeGenerator::cgCheckStaticBitAndDecRef(Type type,
 // the type is not ref-counted.
 //
 Address CodeGenerator::cgCheckRefCountedType(PhysReg typeReg) {
-
-  m_as.cmp_imm32_reg32(KindOfRefCountThreshold, typeReg);
-
-  Address addrToPatch =  m_as.code.frontier;
+  emitCmpTVType(m_as, KindOfRefCountThreshold, typeReg);
+  Address addrToPatch = m_as.code.frontier;
   m_as.jcc8(CC_LE, addrToPatch);
-
   return addrToPatch;
 }
 
-Address CodeGenerator::cgCheckRefCountedType(PhysReg baseReg,
-                                             int64_t offset) {
+Address CodeGenerator::cgCheckRefCountedType(PhysReg baseReg, int64_t offset) {
   emitCmpTVType(m_as, KindOfRefCountThreshold, baseReg[offset + TVOFF(m_type)]);
-  Address addrToPatch =  m_as.code.frontier;
+  Address addrToPatch = m_as.code.frontier;
   m_as.jcc8(CC_LE, addrToPatch);
-
   return addrToPatch;
 }
 
@@ -2949,7 +2911,7 @@ void CodeGenerator::cgDecRefDynamicTypeMem(PhysReg baseReg,
     return;
   }
   // Load m_data into the scratch reg
-  m_as.load_reg64_disp_reg64(baseReg, offset + TVOFF(m_data), scratchReg);
+  m_as.loadq(baseReg[offset + TVOFF(m_data)], scratchReg);
 
   // Emit check for RefCountStaticValue and the actual DecRef
   Address patchStaticCheck = cgCheckStaticBitAndDecRef(Type::Cell, scratchReg,
@@ -2960,7 +2922,7 @@ void CodeGenerator::cgDecRefDynamicTypeMem(PhysReg baseReg,
     // Emit jump to m_astubs (to call release) if count got down to zero
     unlikelyIfBlock(CC_Z, [&] {
       // Emit call to release in m_astubs
-      m_astubs.lea_reg64_disp_reg64(baseReg, offset, scratchReg);
+      m_astubs.lea(baseReg[offset], scratchReg);
       cgCallHelper(m_astubs, getDtorGeneric(), InvalidReg, kSyncPoint,
                    ArgGroup().reg(scratchReg));
     });
@@ -2987,7 +2949,7 @@ void CodeGenerator::cgDecRefMem(Type type,
     // to load the type and the data.
     cgDecRefDynamicTypeMem(baseReg, offset, exit);
   } else if (type.maybeCounted()) {
-    m_as.load_reg64_disp_reg64(baseReg, offset, scratchReg);
+    m_as.loadq(baseReg[offset + TVOFF(m_data)], scratchReg);
     cgDecRefStaticType(type, scratchReg, exit, true);
   }
 }
@@ -3270,7 +3232,7 @@ void CodeGenerator::cgCastStk(IRInstruction *inst) {
     not_reached();
   }
   cgCallHelper(m_as, tvCastHelper, nullptr,
-               kSyncPoint, args);
+               kSyncPoint, args, DestType::None);
 }
 
 void CodeGenerator::cgCallBuiltin(IRInstruction* inst) {
@@ -3626,7 +3588,7 @@ void CodeGenerator::cgLoadTypedValue(PhysReg base,
     emitLoadTVType(m_as, base[off + TVOFF(m_type)], typeDstReg);
     if (label) {
       // Check type needed
-      emitGuardType(r32(typeDstReg), inst);
+      emitGuardType(typeDstReg, inst);
     }
   } else if (label) {
     // Check type needed
@@ -3637,9 +3599,9 @@ void CodeGenerator::cgLoadTypedValue(PhysReg base,
   if (valueDstReg == InvalidReg) return;
 
   if (useScratchReg) {
-    m_as.load_reg64_disp_reg64(reg::rScratch, off + TVOFF(m_data), valueDstReg);
+    m_as.loadq(reg::rScratch[off + TVOFF(m_data)], valueDstReg);
   } else {
-    m_as.load_reg64_disp_reg64(base, off + TVOFF(m_data), valueDstReg);
+    m_as.loadq(base[off + TVOFF(m_data)], valueDstReg);
   }
 }
 
@@ -3647,9 +3609,7 @@ void CodeGenerator::cgStoreTypedValue(PhysReg base,
                                       int64_t off,
                                       SSATmp* src) {
   assert(src->getType().needsReg());
-  m_as.store_reg64_disp_reg64(src->getReg(0),
-                              off + TVOFF(m_data),
-                              base);
+  m_as.storeq(src->getReg(0),        base[off + TVOFF(m_data)]);
   emitStoreTVType(m_as, src->getReg(1), base[off + TVOFF(m_type)]);
 }
 
@@ -3678,12 +3638,10 @@ void CodeGenerator::cgStore(PhysReg base,
     } else {
       not_reached();
     }
-    m_as.store_imm64_disp_reg64(val, off + TVOFF(m_data), base);
+    m_as.storeq(val, base[off + TVOFF(m_data)]);
   } else {
     zeroExtendIfBool(m_as, src);
-    m_as.store_reg64_disp_reg64(src->getReg(),
-                                off + TVOFF(m_data),
-                                base);
+    m_as.storeq(src->getReg(), base[off + TVOFF(m_data)]);
   }
 }
 
@@ -3736,7 +3694,7 @@ void CodeGenerator::cgLdMem(IRInstruction * inst) {
 }
 
 void CodeGenerator::cgLdRef(IRInstruction* inst) {
-  cgLoad(inst->getSrc(0)->getReg(), 0, inst);
+  cgLoad(inst->getSrc(0)->getReg(), RefData::tvOffset(), inst);
 }
 
 void CodeGenerator::recordSyncPoint(Asm& as,
@@ -3805,7 +3763,7 @@ void CodeGenerator::cgGuardType(IRInstruction* inst) {
   assert(srcTypeReg != InvalidReg);
 
   ConditionCode cc;
-  cc = emitTypeTest(type, r32(srcTypeReg), true);
+  cc = emitTypeTest(type, srcTypeReg, true);
   emitFwdJcc(cc, inst->getTaken());
 
   auto dstReg = inst->getDst()->getReg();
@@ -3877,12 +3835,12 @@ void CodeGenerator::cgGuardRefs(IRInstruction* inst) {
   } else if (vals64 != 0) {
     ifThenElse(CC_NL, thenBody, /* else */ [&] {
       //   If not special builtin...
-      m_as.test_imm32_disp_reg32(AttrVariadicByRef, Func::attrsOff(), funcPtrReg);
+      m_as.testl(AttrVariadicByRef, funcPtrReg[Func::attrsOff()]);
       emitFwdJcc(CC_Z, exitLabel);
     });
   } else {
     ifThenElse(CC_NL, thenBody, /* else */ [&] {
-      m_as.test_imm32_disp_reg32(AttrVariadicByRef, Func::attrsOff(), funcPtrReg);
+      m_as.testl(AttrVariadicByRef, funcPtrReg[Func::attrsOff()]);
       emitFwdJcc(CC_NZ, exitLabel);
     });
   }
@@ -4441,7 +4399,7 @@ void CodeGenerator::cgCheckInit(IRInstruction* inst) {
   assert(typeReg != InvalidReg);
 
   static_assert(KindOfUninit == 0, "cgCheckInit assumes KindOfUninit == 0");
-  m_as.testl (r32(typeReg), r32(typeReg));
+  m_as.testb (rbyte(typeReg), rbyte(typeReg));
   emitFwdJcc(CC_Z, label);
 }
 
@@ -4452,8 +4410,7 @@ void CodeGenerator::cgCheckInitMem(IRInstruction* inst) {
   int64_t offset = inst->getSrc(1)->getValInt();
   Type t = base->getType().deref();
   if (t.isInit()) return;
-
-  m_as.cmpl (KindOfUninit, base->getReg()[offset + TVOFF(m_type)]);
+  emitCmpTVType(m_as, KindOfUninit, base->getReg()[offset + TVOFF(m_type)]);
   emitFwdJcc(CC_Z, label);
 }
 
@@ -4808,7 +4765,7 @@ void traceCallback(ActRec* fp, Cell* sp, int64_t pcOff, void* rip) {
             << " " << rip
             << std::endl;
 #endif
-  checkFrame(fp, sp, true);
+  checkFrame(fp, sp, /*checkLocals*/true);
 }
 
 void CodeGenerator::emitTraceCall(CodeGenerator::Asm& as, int64_t pcOff,

hphp/runtime/vm/translator/hopt/codegen.h

@@ -91,15 +91,6 @@ struct CodegenState {
   AsmInfo* asmInfo;
 };
 
-// Generate an if-then block into a.  thenBlock is executed if cc is true.
-template <class Then>
-void ifThen(Transl::X64Assembler& a, ConditionCode cc, Then thenBlock) {
-  Label done;
-  a.jcc8(ccNegate(cc), done);
-  thenBlock();
-  asm_label(a, done);
-}
-
 struct CodeGenerator {
   typedef Transl::X64Assembler Asm;
 
@@ -432,7 +423,7 @@ struct ArgGroup {
    * Pass tmp as a TypedValue passed by value.
    */
   ArgGroup& typedValue(SSATmp* tmp) {
-    return ssa(tmp).type(tmp);
+    return packed_tv ? type(tmp).ssa(tmp) : ssa(tmp).type(tmp);
   }
 
   ArgGroup& none() {
@@ -451,7 +442,7 @@ struct ArgGroup {
 private:
   ArgGroup& vectorKeyImpl(SSATmp* key, bool allowInt) {
     if (key->isString() || (allowInt && key->isA(Type::Int))) {
-      return ssa(key).none();
+      return packed_tv ? none().ssa(key) : ssa(key).none();
     }
     return typedValue(key);
   }

hphp/runtime/vm/translator/translator-x64-internal.h

@@ -33,6 +33,16 @@ namespace Transl {
 
 static const Trace::Module TRACEMOD = Trace::tx64;
 static const DataType BitwiseKindOfString = KindOfString;
+
+// Generate an if-then block into a.  thenBlock is executed if cc is true.
+template <class Then>
+void ifThen(Transl::X64Assembler& a, ConditionCode cc, Then thenBlock) {
+  Label done;
+  a.jcc8(ccNegate(cc), done);
+  thenBlock();
+  asm_label(a, done);
+}
+
 // RAII aids to machine code.
 
 // In shared stubs, we've already made the stack odd by calling
@@ -571,7 +581,7 @@ asm_label(a, likely);
 // a ref-counted cell.
 //
 // It's ok to do reconcilable register operations in the body.
-template<int FieldOffset, int FieldValue, ConditionCode Jcc,
+template<unsigned FieldOffset, unsigned FieldValue, ConditionCode Jcc,
          typename FieldType>
 struct CondBlock {
   X64Assembler& m_a;
@@ -658,6 +668,8 @@ emitStoreImm(X64Assembler& a, uint64_t imm, PhysReg r, int off,
     a.   store_reg64_disp_reg64(immReg, off, r);
   } else if (size == sz::dword) {
     a.   store_imm32_disp_reg(imm, off, r);
+  } else if (size == sz::byte) {
+    a.   store_imm8_disp_reg(imm, off, r);
   } else {
     not_implemented();
   }
@@ -713,6 +725,7 @@ static inline Reg8 toByte(const Reg64& x)   { return rbyte(x); }
 static inline Reg8 toByte(PhysReg x)        { return rbyte(x); }
 
 static inline Reg32 toReg32(const Reg64& x) { return r32(x); }
+static inline Reg32 toReg32(const Reg8& x)  { return r32(x); }
 static inline Reg32 toReg32(PhysReg x)      { return r32(x); }
 
 // For other operand types, let whatever conversions (or compile
@@ -788,6 +801,7 @@ emitStoreTVType(X64Assembler& a, OpndType tvOp, DestType dest) {
 //   dest.
 static inline void
 emitDispDeref(X64Assembler &a, Reg64 src, int disp, Reg64 dest) {
+  // src is a RefData, dest will be m_data field of inner gizmoom.
   a.    loadq (src[disp + TVOFF(m_data)], dest);
 }
 
@@ -796,10 +810,27 @@ emitDeref(X64Assembler &a, PhysReg src, PhysReg dest) {
   emitDispDeref(a, src, 0, dest);
 }
 
+static inline void
+emitDerefRef(X64Assembler &a, Reg64 src, int disp, Reg64 dest) {
+  emitDispDeref(a, src, disp + RefData::tvOffset(), dest);
+}
+
+static inline void
+emitDerefRef(X64Assembler &a, Reg64 src, Reg64 dest) {
+  emitDerefRef(a, src, 0, dest);
+}
+
 static inline void
 emitDerefIfVariant(X64Assembler &a, PhysReg reg) {
   emitCmpTVType(a, KindOfRef, reg[TVOFF(m_type)]);
-  a.cload_reg64_disp_reg64(CC_Z, reg, TVOFF(m_data), reg);
+  if (RefData::tvOffset() == 0) {
+    a.    cload_reg64_disp_reg64(CC_E, reg, TVOFF(m_data), reg);
+  } else {
+    ifThen(a, CC_E, [&] {
+      a.  loadq(reg[TVOFF(m_data)], reg);
+      a.  addq(RefData::tvOffset(), reg);
+    });
+  }
 }
 
 // NB: leaves count field unmodified. Does not store to m_data if type
@@ -812,16 +843,19 @@ emitStoreTypedValue(X64Assembler& a, DataType type, PhysReg val,
   }
   if (!IS_NULL_TYPE(type)) {
     assert(val != reg::noreg);
-    a.  store_reg64_disp_reg64(val, disp + TVOFF(m_data), dest);
+    a.  storeq(val, dest[disp + TVOFF(m_data)]);
   }
 }
 
+static inline void
+emitStoreToRefData(X64Assembler& a, DataType type, PhysReg val,
+                   int disp, PhysReg dest) {
+  emitStoreTypedValue(a, type, val, disp + RefData::tvOffset(), dest);
+}
+
 static inline void
 emitStoreInvalid(X64Assembler& a, int disp, PhysReg dest) {
-  static_assert(TypedValueAux::auxSize == sizeof(int32_t),
-                "emitStoreInvalid assumes m_aux is dword sized.");
   a.    storeq (0xfacefacefaceface,     dest[disp + TVOFF(m_data)]);
-  a.    storel ((signed int)0xfaceface, dest[disp + TypedValueAux::auxOffset]);
   emitStoreTVType(a, KindOfInvalid,     dest[disp + TVOFF(m_type)]);
 }
 
@@ -891,6 +925,8 @@ public:
 
   void addLoc(const Location &loc);
 
+  void addLocRef(const Location &loc);
+
   void addDeref(const Location &loc);
 
   void addReg(PhysReg reg);
@@ -918,7 +954,7 @@ public:
 private:
   struct ArgContent {
     enum ArgKind {
-      ArgImm, ArgLoc, ArgDeref, ArgReg, ArgRegPlus, ArgLocAddr
+      ArgImm, ArgLoc, ArgLocRef, ArgDeref, ArgReg, ArgRegPlus, ArgLocAddr
     } m_kind;
     PhysReg m_reg;
     const Location *m_loc;
@@ -948,6 +984,7 @@ private:
 // arguments
 #define IMM(i)       _am.addImm(i)
 #define V(loc)       _am.addLoc(loc)
+#define VREF(loc)    _am.addLocRef(loc)
 #define DEREF(loc)   _am.addDeref(loc)
 #define R(r)         _am.addReg(r)
 #define RPLUS(r,off) _am.addRegPlus(r, off)

hphp/runtime/vm/translator/translator-x64-vector.cpp

@@ -130,7 +130,8 @@ static const PhysReg unsafe_rsp = rsp;
     PhysReg reg;                                                    \
     int disp;                                                       \
     locToRegDisp(val.location, &reg, &disp);                        \
-    a.  load_reg64_disp_reg64(reg, disp + TVOFF(m_data), r(rVal));  \
+    a.  loadq(reg[disp + TVOFF(m_data)], r(rVal));                  \
+    if (auto offset = RefData::tvOffset()) a.addq(offset, r(rVal)); \
   }
 #define VAL(useRVal)  \
   IE((useRVal),       \
@@ -410,7 +411,8 @@ void TranslatorX64::emitBaseLCR(const Tracelet& t,
     m_regMap.cleanSmashLoc(base.location);
     if (base.isVariant()) {
       // Get inner value.
-      a.  load_reg64_disp_reg64(pr, disp + TVOFF(m_data), r(rBase));
+      a.  loadq(pr[disp + TVOFF(m_data)], r(rBase));
+      if (auto offset = RefData::tvOffset()) a.addq (offset, r(rBase));
     } else {
       a.  lea_reg64_disp_reg64(pr, disp, r(rBase));
     }
@@ -1520,7 +1522,7 @@ void TranslatorX64::emitIssetEmptyProp(const Tracelet& t,
   BUILD_OPTABH(getKeyTypeS(memb), memb.isVariant(), useEmpty,
               m_vecState->isObj());
   EMIT_RCALL(a, ni, opFunc, CTX(), R(rBase), SML(memb));
-  a.   and_imm32_reg64(1, rax); // Mask garbage bits.
+  a.   andq(1, rax); // Mask garbage bits.
   ScratchReg rIssetEmpty(m_regMap, rax);
   assert(!ni.outLocal);
   if (useTvResult(t, ni, mii)) {
@@ -1668,7 +1670,7 @@ void TranslatorX64::emitSetProp(const Tracelet& t,
     LazyScratchReg tmp(m_regMap);
     if (val.isVariant() && !IS_NULL_TYPE(val.rtt.valueType())) {
       tmp.alloc();
-      emitDeref(a, rhsReg, r(tmp));
+      emitDerefRef(a, rhsReg, r(tmp));
       rhsReg = r(tmp);
     }
 
@@ -1989,12 +1991,13 @@ void TranslatorX64::emitIncDecProp(const Tracelet& t,
 #undef HELPER_TABLE
 
 template <KeyType keyType, bool unboxKey>
-static inline void bindElemImpl(TypedValue* base, TypedValue* key, RefData* val,
+static inline void bindElemImpl(TypedValue* base, TypedValue* key, TypedValue* val,
                                 MInstrState* mis) {
   key = unbox<keyType, unboxKey>(key);
   base = ElemD<false, true, keyType>(mis->tvScratch, mis->tvRef, base, key);
+  assert(val->m_type == KindOfRef);
   if (!(base == &mis->tvScratch && base->m_type == KindOfUninit)) {
-    tvBind(val->tv(), base);
+    tvBind(val, base);
   }
 }
 
@@ -2008,8 +2011,8 @@ static inline void bindElemImpl(TypedValue* base, TypedValue* key, RefData* val,
   m(bindElemS,   StrKey,  false)
 
 #define ELEM(nm, ...)                                                   \
-static void nm(TypedValue* base, TypedValue* key, RefData* val,         \
-               MInstrState* mis) {                       \
+static void nm(TypedValue* base, TypedValue* key, TypedValue* val,      \
+               MInstrState* mis) {                                      \
   bindElemImpl<__VA_ARGS__>(base, key, val, mis);                       \
 }
 HELPER_TABLE(ELEM)
@@ -2028,7 +2031,7 @@ void TranslatorX64::emitBindElem(const Tracelet& t,
   cleanOutLocal(ni);
   assert(!forceMValIncDec(t, ni, mii));
   assert(val.isVariant());
-  typedef void (*OpFunc)(TypedValue*, TypedValue*, RefData*, MInstrState*);
+  typedef void (*OpFunc)(TypedValue*, TypedValue*, TypedValue*, MInstrState*);
   BUILD_OPTAB(getKeyTypeIS(key), key.isVariant());
   EMIT_RCALL(a, ni, opFunc, R(rBase), ISML(key), A(val.location), R(mis_rsp));
   invalidateOutLocal(ni);
@@ -2037,12 +2040,13 @@ void TranslatorX64::emitBindElem(const Tracelet& t,
 
 template <KeyType keyType, bool unboxKey, bool isObj>
 static inline void bindPropImpl(Class* ctx, TypedValue* base, TypedValue* key,
-                                RefData* val, MInstrState* mis) {
+                                TypedValue* val, MInstrState* mis) {
   key = unbox<keyType, unboxKey>(key);
   base = Prop<false, true, false, isObj, keyType>(mis->tvScratch, mis->tvRef,
                                                   ctx, base, key);
+  assert(val->m_type == KindOfRef);
   if (!(base == &mis->tvScratch && base->m_type == KindOfUninit)) {
-    tvBind(val->tv(), base);
+    tvBind(val, base);
   }
 }
 
@@ -2059,8 +2063,8 @@ static inline void bindPropImpl(Class* ctx, TypedValue* base, TypedValue* key,
 
 #define PROP(nm, ...)                                                   \
 static inline void nm(Class* ctx, TypedValue* base, TypedValue* key,    \
-                                RefData* val,                           \
-                                MInstrState* mis) {      \
+                                TypedValue* val,                        \
+                                MInstrState* mis) {                     \
   bindPropImpl<__VA_ARGS__>(ctx, base, key, val, mis);                  \
 }
 HELPER_TABLE(PROP)
@@ -2083,7 +2087,7 @@ void TranslatorX64::emitBindProp(const Tracelet& t,
   PREP_CTX(argNumToRegName[0]);
   // Emit the appropriate helper call.
   assert(!forceMValIncDec(t, ni, mii));
-  typedef void (*OpFunc)(Class*, TypedValue*, TypedValue*, RefData*,
+  typedef void (*OpFunc)(Class*, TypedValue*, TypedValue*, TypedValue*,
                          MInstrState*);
   BUILD_OPTAB(getKeyTypeS(key), key.isVariant(), m_vecState->isObj());
   EMIT_RCALL(a, ni, opFunc,
@@ -2373,11 +2377,12 @@ void TranslatorX64::emitIncDecNewElem(const Tracelet& t,
   }
 }
 
-static inline void bindNewElem(TypedValue* base, RefData* val,
+static inline void bindNewElem(TypedValue* base, TypedValue* val,
                                MInstrState* mis) {
   base = NewElem(mis->tvScratch, mis->tvRef, base);
+  assert(val->m_type == KindOfRef);
   if (!(base == &mis->tvScratch && base->m_type == KindOfUninit)) {
-    tvBind(val->tv(), base);
+    tvBind(val, base);
   }
 }
 
@@ -2390,10 +2395,8 @@ void TranslatorX64::emitBindNewElem(const Tracelet& t,
   assert(generateMVal(t, ni, mii));
   const DynLocation& val = *ni.inputs[0];
   m_regMap.cleanSmashLoc(val.location);
-  // Emit the appropriate helper call.
-  void (*bindNewElemOp)(TypedValue*, RefData*, MInstrState*) = bindNewElem;
   assert(val.isVariant());
-  EMIT_RCALL(a, ni, bindNewElemOp, R(rBase), A(val.location), R(mis_rsp));
+  EMIT_RCALL(a, ni, bindNewElem, R(rBase), A(val.location), R(mis_rsp));
 }
 
 void TranslatorX64::emitNotSuppNewElem(const Tracelet& t,
@@ -2533,7 +2536,7 @@ void TranslatorX64::emitMPre(const Tracelet& t,
                                        RegInfo::CLEAN);
     if (val.isVariant()) {
       tmp.alloc();
-      emitDeref(a, rVal, r(tmp));
+      emitDerefRef(a, rVal, r(tmp));
       rVal = r(tmp);
     }
     // Copy val to tvVal for later assignment and decref.
@@ -2850,7 +2853,7 @@ TranslatorX64::translateCGetM(const Tracelet& t,
     LazyScratchReg baseScratch(m_regMap);
     if (base.isVariant()) {
       baseScratch.alloc();
-      emitDeref(a, baseReg, r(baseScratch));
+      emitDerefRef(a, baseReg, r(baseScratch));
       baseReg = r(baseScratch);
     }
     Stats::emitInc(a, Stats::Tx64_CGetMArray);
@@ -2908,7 +2911,7 @@ void TranslatorX64::translateIssetMFast(const Tracelet& t,
   LazyScratchReg scratch(m_regMap);
   if (base.isVariant()) {
     scratch.alloc();
-    emitDeref(a, arrReg, r(scratch));
+    emitDerefRef(a, arrReg, r(scratch));
     arrReg = r(scratch);
     SKTRACE(1, ni.source, "loaded variant\n");
   }
@@ -3077,18 +3080,18 @@ TranslatorX64::translateSetMArray(const Tracelet& t,
   bool useBoxedForm = arr.isVariant();
   void* fptr;
   if (false) { // helper type-checks
-    TypedValue* cell = nullptr;
+    RefData* ref = nullptr;
     ArrayData* arr = nullptr;
     TypedValue* rhs = nullptr;
     StringData* strKey = nullptr;
     UNUSED ArrayData* ret;
-    ret = array_setm_ik1_v(cell, arr, 12, rhs);
-    ret = array_setm_sk1_v(cell, arr, strKey, rhs);
-    ret = array_setm_sk1_v0(cell, arr, strKey, rhs);
-    ret = array_setm_s0k1_v(cell, arr, strKey, rhs);
-    ret = array_setm_s0k1_v0(cell, arr, strKey, rhs);
-    ret = array_setm_s0k1nc_v(cell, arr, strKey, rhs);
-    ret = array_setm_s0k1nc_v0(cell, arr, strKey, rhs);
+    ret = array_setm_ik1_v(ref, arr, 12, rhs);
+    ret = array_setm_sk1_v(ref, arr, strKey, rhs);
+    ret = array_setm_sk1_v0(ref, arr, strKey, rhs);
+    ret = array_setm_s0k1_v(ref, arr, strKey, rhs);
+    ret = array_setm_s0k1_v0(ref, arr, strKey, rhs);
+    ret = array_setm_s0k1nc_v(ref, arr, strKey, rhs);
+    ret = array_setm_s0k1nc_v0(ref, arr, strKey, rhs);
   }
 
   /*
@@ -3126,7 +3129,7 @@ TranslatorX64::translateSetMArray(const Tracelet& t,
     PhysReg rhsReg = getReg(valLoc);
     if (valIsVariant) {
       tmp.alloc();
-      emitDeref(a, rhsReg, r(tmp));
+      emitDerefRef(a, rhsReg, r(tmp));
       rhsReg = r(tmp);
     }
     emitIncRef(rhsReg, KindOfArray);
@@ -3135,7 +3138,7 @@ TranslatorX64::translateSetMArray(const Tracelet& t,
             useBoxedForm ? V(arrLoc) : IMM(0),
             useBoxedForm ? DEREF(arrLoc) : V(arrLoc),
             V(keyLoc),
-            valIsVariant ? V(valLoc) : A(valLoc));
+            valIsVariant ? VREF(valLoc) : A(valLoc));
 
   recordReentrantCall(i);
   // If we did not used boxed form, we need to tell the register allocator

hphp/runtime/vm/translator/translator-x64.cpp

@@ -131,9 +131,6 @@ enum TransPerfCounter {
 static __thread int64_t s_perfCounters[tpc_num_counters];
 #define INC_TPC(n) ++s_perfCounters[tpc_ ## n];
 
-#define KindOfString \
-#error You probably do not mean to use KindOfString in this file.
-
 #define NULLCASE() \
   case KindOfUninit: case KindOfNull
 
@@ -580,6 +577,7 @@ void ArgManager::computeUsed(std::map<PhysReg, size_t> &used,
         m_args[i].m_kind == ArgContent::ArgRegPlus) {
       reg = m_args[i].m_reg;
     } else if (m_args[i].m_kind == ArgContent::ArgLoc ||
+               m_args[i].m_kind == ArgContent::ArgLocRef ||
                m_args[i].m_kind == ArgContent::ArgDeref) {
       reg = m_tx64.getReg(*m_args[i].m_loc);
     } else {
@@ -727,16 +725,22 @@ void ArgManager::shuffleRegisters(std::map<PhysReg, size_t> &used,
 
 void ArgManager::emitValues(std::vector<PhysReg> &actual) {
   for (size_t i = 0; i < m_args.size(); i++) {
-    switch(m_args[i].m_kind) {
+    auto kind = m_args[i].m_kind;
+    auto argReg = argNumToRegName[i];
+    switch (kind) {
     case ArgContent::ArgLoc:
+    case ArgContent::ArgLocRef:
     case ArgContent::ArgDeref:
     case ArgContent::ArgReg:
       TRACE(6, "ArgManager: copying arg %zd from r%d to r%d\n",
-            i, int(actual[i]), int(argNumToRegName[i]));
-      emitMovRegReg(m_a, actual[i], argNumToRegName[i]);
+            i, int(actual[i]), int(argReg));
+      emitMovRegReg(m_a, actual[i], argReg);
       // Emit dereference if needed
-      if (m_args[i].m_kind == ArgContent::ArgDeref) {
-        emitDeref(m_a, argNumToRegName[i], argNumToRegName[i]);
+      if (kind == ArgContent::ArgDeref) {
+        emitDerefRef(m_a, argReg, argReg);
+      } else if (kind == ArgContent::ArgLocRef && RefData::tvOffset()) {
+        // argReg holds a RefData*; adjust it to be TypedValue* to the value.
+        m_a.addq(RefData::tvOffset(), argReg);
       }
       break;
 
@@ -744,12 +748,12 @@ void ArgManager::emitValues(std::vector<PhysReg> &actual) {
     // If it was used previously by an input value, shuffleRegisters
     // should have moved it to the proper register from argNumToRegName.
     case ArgContent::ArgImm:
-      emitImmReg(m_a, m_args[i].m_imm, argNumToRegName[i]);
+      emitImmReg(m_a, m_args[i].m_imm, argReg);
       break;
 
     case ArgContent::ArgRegPlus:
       if (m_args[i].m_imm) {
-        m_a.  add_imm32_reg64(m_args[i].m_imm, argNumToRegName[i]);
+        m_a.  add_imm32_reg64(m_args[i].m_imm, argReg);
       }
       break;
 
@@ -758,7 +762,7 @@ void ArgManager::emitValues(std::vector<PhysReg> &actual) {
         PhysReg base;
         int disp;
         locToRegDisp(*m_args[i].m_loc, &base, &disp);
-        emitLea(m_a, base, disp, argNumToRegName[i]);
+        emitLea(m_a, base, disp, argReg);
       }
       break;
 
@@ -1159,7 +1163,7 @@ void TranslatorX64::emitDecRefGenericReg(PhysReg rData, PhysReg rType) {
   };
 
   Op op = m_curNI->op();
-  a.    cmpl   (KindOfRefCountThreshold, r32(rType));
+  emitCmpTVType(a, KindOfRefCountThreshold, rType);
   if (op == OpSetM || op == OpContSend || op == OpSetG) {
     // Semi-likely cases
     semiLikelyIfBlock(CC_A, a, std::bind(body, std::ref(a)));
@@ -2391,7 +2395,8 @@ TranslatorX64::emitPrologue(Func* func, int nPassed) {
     } else {
       PhysReg base;
       int disp, k;
-      if (numLocals < func->numLocals()) {
+      static_assert(KindOfUninit == 0, "");
+      if (numParams < func->numLocals()) {
         a.xorl (eax, eax);
       }
       for (k = numLocals; k < func->numLocals(); ++k) {
@@ -3881,7 +3886,7 @@ TranslatorX64::emitUnboxTopOfStack(const NormalizedInstruction& i) {
   // for the output location
   m_regMap.allocOutputRegs(i);
   PhysReg rDest = getReg(i.outStack->location);
-  emitDeref(a, rSrc, rDest);
+  emitDerefRef(a, rSrc, rDest);
   emitIncRef(rDest, outType);
   // decRef the var on the evaluation stack
   emitDecRef(i, rSrc, KindOfRef);
@@ -4031,12 +4036,12 @@ TranslatorX64::binaryArithLocal(const NormalizedInstruction &i,
     // The local is a var, so we have to read its value into outReg
     // on operate on that. We will need to write the result back
     // to the local after the operation.
-    emitDeref(a, localReg, r(scr));
+    emitDerefRef(a, localReg, r(scr));
     emitBody(r(scr));
     // We operated on outReg, so we need to write the result back to the
     // local
     emitMovRegReg(a, r(scr), outReg);
-    a.    store_reg64_disp_reg64(r(scr), 0, localReg);
+    a.    storeq (r(scr), localReg[RefData::tvOffset() + TVOFF(m_data)]);
   }
 }
 
@@ -4646,7 +4651,7 @@ TranslatorX64::translateUnaryBooleanOp(const Tracelet& t,
       PhysReg reg = getReg(inLoc);
       PhysReg outReg = getReg(i.outStack->location);
       if (boxedForm) {
-        emitDeref(a, reg, outReg);
+        emitDerefRef(a, reg, outReg);
       } else {
         emitMovRegReg(a, reg, outReg);
       }
@@ -4660,7 +4665,7 @@ TranslatorX64::translateUnaryBooleanOp(const Tracelet& t,
       PhysReg outReg = getReg(i.outStack->location);
       ScratchReg scratch(m_regMap);
       if (boxedForm) {
-        emitDeref(a, reg, r(scratch));
+        emitDerefRef(a, reg, r(scratch));
         emitConvertToBool(a, r(scratch), outReg, instrNeg);
       } else {
         emitConvertToBool(a, reg, outReg, instrNeg);
@@ -4891,7 +4896,7 @@ TranslatorX64::translateCGetL(const Tracelet& t,
     } else {
       ScratchReg rTmp(m_regMap);
       PhysReg localReg = getReg(inputs[0]->location);
-      a.store_reg64_disp_reg64(localReg, stackDisp + TVOFF(m_data), stackBase);
+      a.    storeq (localReg, stackBase[stackDisp + TVOFF(m_data)]);
       emitLoadTVType(a, locBase[locDisp + TVOFF(m_type)], r(rTmp));
       emitStoreTVType(a, r(rTmp), stackBase[stackDisp + TVOFF(m_type)]);
     }
@@ -4932,7 +4937,7 @@ TranslatorX64::translateCGetL(const Tracelet& t,
     emitMovRegReg(a, localReg, dest);
   }
   if (inputs[0]->isVariant()) {
-    emitDeref(a, dest, dest);
+    emitDerefRef(a, dest, dest);
   }
   assert(outType != KindOfStaticString);
   emitIncRef(dest, outType);
@@ -4983,7 +4988,7 @@ TranslatorX64::translateCGetL2(const Tracelet& t,
   const PhysReg cellOut = getReg(ni.outStack->location);
   assert(cellOut != stackIn);
   if (ni.inputs[locIdx]->isVariant()) {
-    emitDeref(a, localIn, cellOut);
+    emitDerefRef(a, localIn, cellOut);
   } else if (!undefinedLocal) {
     emitMovRegReg(a, localIn, cellOut);
   }
@@ -5127,20 +5132,20 @@ TranslatorX64::translateAssignToLocalOp(const Tracelet& t,
       (!locTypeRelaxed && IS_REFCOUNTED_TYPE(decRefType));
     if (useOldType) {
       oldLocalReg.alloc();
-      emitDeref(a, localReg, r(oldLocalReg));
+      emitDerefRef(a, localReg, r(oldLocalReg));
     }
     if (rhsTypeRelaxed) {
       PhysReg base;
       int disp;
       ScratchReg rTmp(m_regMap);
       locToRegDisp(ni.inputs[rhsIdx]->location, &base, &disp);
-      size_t typeOff = TVOFF(m_type);
-      size_t dataOff = TVOFF(m_data);
+      size_t typeOff = RefData::tvOffset() + TVOFF(m_type);
+      size_t dataOff = RefData::tvOffset() + TVOFF(m_data);
       emitLoadTVType(a, base[disp + TVOFF(m_type)], r(rTmp));
       a.    storeq (rhsReg, localReg[dataOff]);
       emitStoreTVType(a, r(rTmp), localReg[typeOff]);
     } else {
-      emitStoreTypedValue(a, rhsType, rhsReg, 0, localReg);
+      emitStoreToRefData(a, rhsType, rhsReg, 0, localReg);
     }
   } else if (rhsTypeRelaxed) {
     PhysReg rhsBase;
@@ -5520,14 +5525,14 @@ TranslatorX64::translateAddElemC(const Tracelet& t,
   // not (for cases where the key is a local).
   assert(key.rtt.isInt() || key.rtt.isString());
   if (false) { // type-check
-    TypedValue* cell = nullptr;
+    RefData* ref = nullptr;
     TypedValue* rhs = nullptr;
     StringData* strkey = nullptr;
     ArrayData* arr = nullptr;
     ArrayData* ret;
-    ret = array_setm_ik1_v0(cell, arr, 12, rhs);
+    ret = array_setm_ik1_v0(ref, arr, 12, rhs);
     printf("%p", ret); // use ret
-    ret = array_setm_sk1_v0(cell, arr, strkey, rhs);
+    ret = array_setm_sk1_v0(ref, arr, strkey, rhs);
     printf("%p", ret); // use ret
   }
   // Otherwise, we pass the rhs by address
@@ -6648,6 +6653,13 @@ void TranslatorX64::emitReturnVal(
   tvWriteUninit(&tv);
   tv.m_data.num = 0; // to keep the compiler happy
 
+  auto moveRetValIfNeeded = [&] {
+    if (thisBase != dstBase ||
+        thisOffset != (dstOffset + TVOFF(m_data))) {
+      a.  loadq(thisBase[thisOffset], scratch);
+      a.  storeq(scratch, dstBase[dstOffset + TVOFF(m_data)]);
+    }
+  };
   /*
    * We suppressed the write of the (literal) return value
    * to the stack. Figure out what it was.
@@ -6680,15 +6692,13 @@ void TranslatorX64::emitReturnVal(
       tv.m_data.parr = curUnit()->lookupArrayId(prev->imm[0].u_AA);
       break;
     case OpThis: {
-      if (thisBase != dstBase || thisOffset != dstOffset) {
-        a.  load_reg64_disp_reg64(thisBase, thisOffset, scratch);
-        a.  store_reg64_disp_reg64(scratch, dstOffset, dstBase);
-      }
+      moveRetValIfNeeded();
       emitStoreTVType(a, KindOfObject, dstBase[dstOffset + TVOFF(m_type)]);
       return;
     }
     case OpBareThis: {
       assert(curFunc()->cls());
+      moveRetValIfNeeded();
       a.    mov_imm32_reg32(KindOfNull, scratch);
       a.    testb(1, thisBase[thisOffset]);
       {
@@ -6696,10 +6706,6 @@ void TranslatorX64::emitReturnVal(
         a.  mov_imm32_reg32(KindOfObject, scratch);
       }
       emitStoreTVType(a, scratch, dstBase[dstOffset + TVOFF(m_type)]);
-      if (thisBase != dstBase || thisOffset != dstOffset) {
-        a.  load_reg64_disp_reg64(thisBase, thisOffset, scratch);
-        a.  store_reg64_disp_reg64(scratch, dstOffset, dstBase);
-      }
       return;
     }
     default:
@@ -6709,7 +6715,7 @@ void TranslatorX64::emitReturnVal(
   emitStoreTVType(a, tv.m_type, r64(dstBase)[dstOffset + TVOFF(m_type)]);
   if (tv.m_type != KindOfNull) {
     emitStoreImm(a, tv.m_data.num,
-                 dstBase, dstOffset, sz::qword);
+                 dstBase, dstOffset + TVOFF(m_data), sz::qword);
   }
 
 }
@@ -7218,7 +7224,7 @@ TranslatorX64::emitObjToClass(const NormalizedInstruction& i) {
   PhysReg src = getReg(in);
   ScratchReg tmp(m_regMap);
   if (i.inputs[kEmitClsLocalIdx]->rtt.isVariant()) {
-    emitDeref(a, src, r(tmp));
+    emitDerefRef(a, src, r(tmp));
     src = r(tmp);
   }
   assert(i.outStack->valueType() == KindOfClass);
@@ -7424,15 +7430,15 @@ void TranslatorX64::translateCreateCont(const Tracelet& t,
     // Deal with a potential $this local in the generator body
     if (fillThis) {
       assert(thisId != kInvalidId);
-      a.load_reg64_disp_reg64(rax, CONTOFF(m_obj), r(rScratch));
-      a.test_reg64_reg64(r(rScratch), r(rScratch));
+      a.    load_reg64_disp_reg64(rax, CONTOFF(m_obj), r(rScratch));
+      a.    test_reg64_reg64(r(rScratch), r(rScratch));
       {
         JccBlock<CC_Z> ifObj(a);
         const int thisOff = cellsToBytes(genLocals - thisId - 1);
         // We don't have to check for a static refcount since we
         // know it's an Object
-        a.add_imm32_disp_reg32(1, FAST_REFCOUNT_OFFSET, r(rScratch));
-        a.store_reg64_disp_reg64(r(rScratch), thisOff + TVOFF(m_data), r(rDest));
+        a.  addl(1, r(rScratch)[FAST_REFCOUNT_OFFSET]);
+        a.  storeq(r(rScratch), r(rDest)[thisOff + TVOFF(m_data)]);
         emitStoreTVType(a, KindOfObject, r(rDest)[thisOff + TVOFF(m_type)]);
       }
     }
@@ -9447,7 +9453,7 @@ TranslatorX64::translateThis(const Tracelet &t,
          curFunc()->isClosureBody());
   m_regMap.allocOutputRegs(i);
   PhysReg out = getReg(i.outStack->location);
-  a.   load_reg64_disp_reg64(rVmFp, AROFF(m_this), out);
+  a.   loadq(rVmFp[AROFF(m_this)], out);
 
   if (!i.guardedThis) {
     emitThisCheck(i, out);
@@ -9497,7 +9503,7 @@ TranslatorX64::translateBareThis(const Tracelet &t,
   emitIncRef(out, KindOfObject);
   if (i.outStack->rtt.isVagueValue()) {
     emitStoreTVType(a, KindOfObject, base[offset + TVOFF(m_type)]);
-    a. store_reg64_disp_reg64(out, TVOFF(m_data) + offset, base);
+    a. storeq(out, base[TVOFF(m_data) + offset]);
   } else {
     assert(i.outStack->isObject());
     m_regMap.bindScratch(outScratch, i.outStack->location, KindOfObject,
@@ -9526,20 +9532,20 @@ TranslatorX64::translateInitThisLoc(const Tracelet& t,
   assert(base == rVmFp);
 
   ScratchReg thiz(m_regMap);
-  a.load_reg64_disp_reg64(rVmFp, AROFF(m_this), r(thiz));
+  a.    load_reg64_disp_reg64(rVmFp, AROFF(m_this), r(thiz));
   if (curFunc()->cls() == nullptr) {
     // If we're in a pseudomain, m_this could be NULL
-    a.test_reg64_reg64(r(thiz), r(thiz));
-    a.jz(astubs.code.frontier); // jz if_null
+    a.  testq (r(thiz), r(thiz));
+    a.  jz (astubs.code.frontier); // jz if_null
   }
   // Ok, it's not NULL but it might be a Class which should be treated
   // equivalently
-  a.testb(1, rbyte(thiz));
-  a.jnz(astubs.code.frontier); // jnz if_null
+  a.    testb(1, rbyte(thiz));
+  a.    jnz(astubs.code.frontier); // jnz if_null
 
   // We have a valid $this!
   emitStoreTVType(a, KindOfObject, base[offset + TVOFF(m_type)]);
-  a.store_reg64_disp_reg64(r(thiz), offset + TVOFF(m_data), base);
+  a.    storeq(r(thiz), base[offset + TVOFF(m_data)]);
   emitIncRef(r(thiz), KindOfObject);
 
   // if_null:
@@ -10146,11 +10152,11 @@ void TranslatorX64::translateFCallBuiltin(const Tracelet& t,
     // For bool return value, get the %al byte
     case KindOfBoolean:
       a.  movzbl (al, eax);  // sign extend byte->qword
-      emitStoreTypedValue(a, func->returnType(), rax, disp, base, true);
+      emitStoreTypedValue(a, returnType, rax, disp, base, true);
       break;
     case KindOfNull:  /* void return type */
     case KindOfInt64:
-      emitStoreTypedValue(a, func->returnType(), rax, disp, base, true);
+      emitStoreTypedValue(a, returnType, rax, disp, base, true);
       break;
     STRINGCASE():
     case KindOfArray:
@@ -10370,7 +10376,7 @@ TranslatorX64::translateVerifyParamType(const Tracelet& t,
   PhysReg src = getReg(in);
   ScratchReg inCls(m_regMap);
   if (i.inputs[0]->rtt.isVariant()) {
-    emitDeref(a, src, r(inCls));
+    emitDerefRef(a, src, r(inCls));
     src = r(inCls);
   }
   a.  load_reg64_disp_reg64(src, ObjectData::getVMClassOffset(), r(inCls));
@@ -10530,7 +10536,7 @@ TranslatorX64::translateInstanceOfD(const Tracelet& t,
     PhysReg baseReg = srcReg;
     if (input0->rtt.isVariant()) {
       assert(input0IsLoc);
-      emitDeref(a, srcReg, r(inCls));
+      emitDerefRef(a, srcReg, r(inCls));
       baseReg = r(inCls);
     }
     a.  load_reg64_disp_reg64(baseReg, ObjectData::getVMClassOffset(),
@@ -11088,7 +11094,7 @@ TranslatorX64::emitVariantGuards(const Tracelet& t,
     }
     if (isRef && !m_useHHIR) {
       m_regMap.allocInputReg(i, in);
-      emitOneGuard(t, *base, getReg(input->location), 0,
+      emitOneGuard(t, *base, getReg(input->location), RefData::tvOffset(),
                    input->rtt.innerType(), sideExit);
     }
   }
@@ -12569,6 +12575,12 @@ void ArgManager::addLoc(const Location &loc) {
   m_args.push_back(ArgContent(ArgContent::ArgLoc, loc));
 }
 
+void ArgManager::addLocRef(const Location &loc) {
+  TRACE(6, "ArgManager: push arg ref %zd loc:(%s, %" PRId64 ")\n",
+        m_args.size(), loc.spaceName(), loc.offset);
+  m_args.push_back(ArgContent(ArgContent::ArgLocRef, loc));
+}
+
 void ArgManager::addLocAddr(const Location &loc) {
   TRACE(6, "ArgManager: push arg %zd addr:(%s, %" PRId64 ")\n",
         m_args.size(), loc.spaceName(), loc.offset);

hphp/runtime/vm/translator/translator-x64.h

@@ -116,7 +116,7 @@ class TranslatorX64 : public Translator
   friend class HPHP::VM::JIT::CodeGenerator;
   friend class HPHP::VM::JIT::HhbcTranslator; // packBitVec()
   friend TCA funcBodyHelper(ActRec* fp);
-  template<int, int, ConditionCode, class> friend class CondBlock;
+  template<unsigned, unsigned, ConditionCode, class> friend class CondBlock;
   template<ConditionCode, typename smasher> friend class JccBlock;
   template<ConditionCode> friend class IfElseBlock;
   friend class UnlikelyIfBlock;

hphp/util/atomic.h

@@ -83,7 +83,8 @@ static inline T atomic_inc(T &count) {
   return __sync_fetch_and_add(&count, 1) + 1;
 }
 
-static inline int atomic_dec(int &count) {
+template<typename T>
+static inline T atomic_dec(T &count) {
   assert_address_is_atomically_accessible(&count);
   return __sync_fetch_and_add(&count, -1) - 1;
 }

hphp/util/base.h

@@ -144,6 +144,14 @@ static const bool use_jemalloc =
 #endif
   ;
 
+static const bool packed_tv =
+#ifdef PACKED_TV
+  true
+#else
+  false
+#endif
+  ;
+
 /**
  * Guard bug-for-bug hphpi compatibility code with this predicate.
  */

hphp/util/trace.h

@@ -92,6 +92,8 @@ namespace Trace {
       /* Jit bisection interval */ \
       TM(txOpBisectLow) \
       TM(txOpBisectHigh) \
+      /* smart alloc */ \
+      TM(smartalloc) \
       /* Temporary catetories, to save compilation time */ \
       TM(tmp0)  TM(tmp1)  TM(tmp2)  TM(tmp3)               \
       TM(tmp4)  TM(tmp5)  TM(tmp6)  TM(tmp7)               \