HphpArray Vector specializations

Introduces a new m_kind for vector-shaped HphpArray instances.
A vector has integer indexes in the range [0..size), and no
tombstones.  Internally it does not store the integer keys or
hashtable, although this version still allocates space for it.

hphp/runtime/base/array_data-defs.h

@@ -0,0 +1,206 @@
+/*
+   +----------------------------------------------------------------------+
+   | HipHop for PHP                                                       |
+   +----------------------------------------------------------------------+
+   | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com)     |
+   +----------------------------------------------------------------------+
+   | This source file is subject to version 3.01 of the PHP license,      |
+   | that is bundled with this package in the file LICENSE, and is        |
+   | available through the world-wide-web at the following url:           |
+   | http://www.php.net/license/3_01.txt                                  |
+   | If you did not receive a copy of the PHP license and are unable to   |
+   | obtain it through the world-wide-web, please send a note to          |
+   | license@php.net so we can mail you a copy immediately.               |
+   +----------------------------------------------------------------------+
+*/
+
+#ifndef incl_HPHP_ARRAY_INLINE_H_
+#define incl_HPHP_ARRAY_INLINE_H_
+
+#include "hphp/runtime/base/array_data.h"
+#include "hphp/runtime/base/complex_types.h"
+
+namespace HPHP {
+///////////////////////////////////////////////////////////////////////////////
+
+namespace {
+inline bool isIntKey(const Cell* cell) {
+  return IS_INT_KEY_TYPE(cell->m_type);
+}
+
+inline int64_t getIntKey(const Cell* cell) {
+  assert(cell->m_type == KindOfInt64);
+  return cell->m_data.num;
+}
+
+inline StringData* getStringKey(const Cell* cell) {
+  assert(IS_STRING_TYPE(cell->m_type));
+  return cell->m_data.pstr;
+}
+}
+
+inline bool ArrayData::exists(CStrRef k) const {
+  assert(IsValidKey(k));
+  return exists(k.get());
+}
+
+inline bool ArrayData::exists(CVarRef k) const {
+  assert(IsValidKey(k));
+  auto const cell = k.asCell();
+  return isIntKey(cell) ? exists(getIntKey(cell))
+                        : exists(getStringKey(cell));
+}
+
+inline CVarRef ArrayData::get(CStrRef k, bool error) const {
+  assert(IsValidKey(k));
+  return get(k.get(), error);
+}
+
+inline CVarRef ArrayData::get(int64_t k, bool error) const {
+  auto tv = nvGet(k);
+  return tv ? tvAsCVarRef(tv) : getNotFound(k, error);
+}
+
+inline CVarRef ArrayData::get(const StringData* k, bool error) const {
+  auto tv = nvGet(k);
+  return tv ? tvAsCVarRef(tv) : getNotFound(k, error);
+}
+
+inline ArrayData* ArrayData::lval(CStrRef k, Variant *&ret, bool copy,
+                                  bool checkExist) {
+  assert(IsValidKey(k));
+  return lval(k.get(), ret, copy, checkExist);
+}
+
+inline ArrayData* ArrayData::lval(CVarRef k, Variant *&ret, bool copy,
+                                  bool checkExist) {
+  assert(IsValidKey(k));
+  auto const cell = k.asCell();
+  return isIntKey(cell) ? lval(getIntKey(cell), ret, copy, checkExist)
+                        : lval(getStringKey(cell), ret, copy, checkExist);
+}
+
+inline
+ArrayData *ArrayData::createLvalPtr(CStrRef k, Variant *&ret, bool copy) {
+  assert(IsValidKey(k));
+  return createLvalPtr(k.get(), ret, copy);
+}
+
+inline ArrayData *ArrayData::getLvalPtr(CStrRef k, Variant *&ret, bool copy) {
+  assert(IsValidKey(k));
+  return getLvalPtr(k.get(), ret, copy);
+}
+
+inline ArrayData* ArrayData::set(CStrRef k, CVarRef v, bool copy) {
+  assert(IsValidKey(k));
+  return set(k.get(), v, copy);
+}
+
+inline ArrayData* ArrayData::set(CVarRef k, CVarRef v, bool copy) {
+  assert(IsValidKey(k));
+  auto const cell = k.asCell();
+  return isIntKey(cell) ? set(getIntKey(cell), v, copy)
+                        : set(getStringKey(cell), v, copy);
+}
+
+inline ArrayData* ArrayData::setRef(CStrRef k, CVarRef v, bool copy) {
+  assert(IsValidKey(k));
+  return setRef(k.get(), v, copy);
+}
+
+inline ArrayData* ArrayData::setRef(CVarRef k, CVarRef v, bool copy) {
+  assert(IsValidKey(k));
+  auto const cell = k.asCell();
+  return isIntKey(cell) ? setRef(getIntKey(cell), v, copy)
+                        : setRef(getStringKey(cell), v, copy);
+}
+
+inline ArrayData* ArrayData::add(CStrRef k, CVarRef v, bool copy) {
+  assert(IsValidKey(k));
+  return add(k.get(), v, copy);
+}
+
+inline ArrayData* ArrayData::add(CVarRef k, CVarRef v, bool copy) {
+  assert(IsValidKey(k));
+  auto const cell = k.asCell();
+  return isIntKey(cell) ? add(getIntKey(cell), v, copy)
+                        : add(getStringKey(cell), v, copy);
+}
+
+inline ArrayData* ArrayData::addLval(CStrRef k, Variant *&ret, bool copy) {
+  assert(IsValidKey(k));
+  return addLval(k.get(), ret, copy);
+}
+
+inline ArrayData* ArrayData::addLval(CVarRef k, Variant *&ret, bool copy) {
+  assert(IsValidKey(k));
+  auto const cell = k.asCell();
+  return isIntKey(cell) ? addLval(getIntKey(cell), ret, copy)
+                        : addLval(getStringKey(cell), ret, copy);
+}
+
+inline ArrayData* ArrayData::remove(CStrRef k, bool copy) {
+  assert(IsValidKey(k));
+  return remove(k.get(), copy);
+}
+
+inline ArrayData* ArrayData::remove(CVarRef k, bool copy) {
+  assert(IsValidKey(k));
+  auto const cell = k.asCell();
+  return isIntKey(cell) ? remove(getIntKey(cell), copy)
+                        : remove(getStringKey(cell), copy);
+}
+
+inline Variant ArrayData::getValue(ssize_t pos) const {
+  return getValueRef(pos);
+}
+
+inline TypedValue* ArrayData::nvGetValueRef(ssize_t pos) {
+  return const_cast<TypedValue*>(getValueRef(pos).asTypedValue());
+}
+
+inline Variant ArrayData::getKey(ssize_t pos) const {
+  TypedValue tv;
+  nvGetKey(&tv, pos);
+  return std::move(tvAsVariant(&tv));
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
+inline unsigned ArrayData::index() const {
+  assert(unsigned(m_kind) < unsigned(ArrayKind::kNumKinds));
+  return unsigned(m_kind);
+}
+
+inline void ArrayData::release() {
+  return g_array_funcs.release[index()](this);
+}
+
+inline ArrayData* ArrayData::append(CVarRef v, bool copy) {
+  return g_array_funcs.append[index()](this, v, copy);
+}
+
+inline TypedValue* ArrayData::nvGet(int64_t ikey) const {
+  return g_array_funcs.nvGetInt[index()](this, ikey);
+}
+
+inline TypedValue* ArrayData::nvGet(const StringData* skey) const {
+  return g_array_funcs.nvGetStr[index()](this, skey);
+}
+
+inline void ArrayData::nvGetKey(TypedValue* out, ssize_t pos) const {
+  g_array_funcs.nvGetKey[index()](this, out, pos);
+}
+
+inline ArrayData* ArrayData::set(int64_t k, CVarRef v, bool copy) {
+  return g_array_funcs.setInt[index()](this, k, v, copy);
+}
+
+inline ArrayData* ArrayData::set(StringData* k, CVarRef v, bool copy) {
+  return g_array_funcs.setStr[index()](this, k, v, copy);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+}
+
+#endif // incl_HPHP_ARRAY_INLINE_H_

hphp/runtime/base/array_data.cpp

@@ -30,6 +30,7 @@
 #include "hphp/runtime/base/shared_map.h"
 #include "hphp/runtime/base/policy_array.h"
 #include "hphp/runtime/base/comparisons.h"
+#include "hphp/runtime/vm/name_value_table_wrapper.h"
 
 namespace HPHP {
 ///////////////////////////////////////////////////////////////////////////////
@@ -61,6 +62,49 @@ ArrayData *ArrayData::GetScalarArray(ArrayData *arr,
   return acc->second;
 }
 
+///////////////////////////////////////////////////////////////////////////////
+
+// order: kVector, kHphpArray, kSharedMap, kNameValueTableWrapper, kPolicyArray
+extern const ArrayFunctions g_array_funcs = {
+  // release
+  { &HphpArray::ReleaseVec, &HphpArray::Release,
+    &SharedMap::Release,
+    &NameValueTableWrapper::Release,
+    &PolicyArray::Release },
+  // append
+  { &HphpArray::AppendVec, &HphpArray::Append,
+    &SharedMap::Append,
+    &NameValueTableWrapper::Append,
+    &PolicyArray::Append },
+  // nvGetInt
+  { &HphpArray::NvGetIntVec, &HphpArray::NvGetInt,
+    &SharedMap::NvGetInt,
+    &NameValueTableWrapper::NvGetInt,
+    &PolicyArray::NvGetInt },
+  // nvGetStr
+  { &HphpArray::NvGetStrVec, &HphpArray::NvGetStr,
+    &SharedMap::NvGetStr,
+    &NameValueTableWrapper::NvGetStr,
+    &PolicyArray::NvGetStr },
+  // nvGetKey
+  { &HphpArray::NvGetKeyVec, &HphpArray::NvGetKey,
+    &SharedMap::NvGetKey,
+    &NameValueTableWrapper::NvGetKey,
+    &PolicyArray::NvGetKey },
+  // setInt
+  { &HphpArray::SetIntVec, &HphpArray::SetInt,
+    &SharedMap::SetInt,
+    &NameValueTableWrapper::SetInt,
+    &PolicyArray::SetInt },
+  // setStr
+  { &HphpArray::SetStrVec, &HphpArray::SetStr,
+    &SharedMap::SetStr,
+    &NameValueTableWrapper::SetStr,
+    &PolicyArray::SetStr },
+};
+
+///////////////////////////////////////////////////////////////////////////////
+
 // In general, arrays can contain int-valued-strings, even though
 // plain array access converts them to integers.  non-int-string
 // assersions should go upstream of the ArrayData api.
@@ -112,27 +156,6 @@ ArrayData *ArrayData::nonSmartCopy() const {
   throw FatalErrorException("nonSmartCopy not implemented.");
 }
 
-HOT_FUNC
-void ArrayData::release() {
-  if (isHphpArray()) {
-    HphpArray* that = static_cast<HphpArray*>(this);
-    that->release();
-    return;
-  }
-  if (isSharedMap()) {
-    SharedMap* that = static_cast<SharedMap*>(this);
-    that->release();
-    return;
-  }
-  if (isPolicyArray()) {
-    auto that = static_cast<PolicyArray*>(this);
-    that->release();
-    return;
-  }
-  assert(m_kind == ArrayKind::kNameValueTableWrapper);
-  // NameValueTableWrapper: nop.
-}
-
 ///////////////////////////////////////////////////////////////////////////////
 // reads
 
@@ -262,7 +285,7 @@ void ArrayData::newFullPos(FullPos &fp) {
 }
 
 void ArrayData::freeFullPos(FullPos &fp) {
-  assert(strongIterators() != 0 && fp.getContainer() == (ArrayData*)this);
+  assert(strongIterators() && fp.getContainer() == this);
   // search for fp in our list, then remove it. Usually its the first one.
   FullPos* p = strongIterators();
   if (p == &fp) {
@@ -286,7 +309,7 @@ void ArrayData::freeStrongIterators() {
   for (FullPosRange r(strongIterators()); !r.empty(); r.popFront()) {
     r.front()->setContainer(nullptr);
   }
-  setStrongIterators(0);
+  setStrongIterators(nullptr);
 }
 
 void ArrayData::moveStrongIterators(ArrayData* dest, ArrayData* src) {

hphp/runtime/base/array_data.h

@@ -40,10 +40,12 @@ class ArrayData : public Countable {
   // enum of possible array types, so we can guard nonvirtual
   // fast paths in runtime code.
   enum class ArrayKind : uint8_t {
+    kVector,
     kHphpArray,
     kSharedMap,
     kNameValueTableWrapper,
     kPolicyArray,
+    kNumKinds // insert new values before kNumKinds.
   };
 
 public:
@@ -162,7 +164,11 @@ public:
    * Specific derived class type querying operators.
    */
   bool isPolicyArray() const { return m_kind == ArrayKind::kPolicyArray; }
-  bool isHphpArray() const { return m_kind == ArrayKind::kHphpArray; }
+  bool isVector() const { return m_kind == ArrayKind::kVector; }
+  bool isHphpArray() const {
+    return m_kind <= ArrayKind::kHphpArray;
+    static_assert(ArrayKind::kVector < ArrayKind::kHphpArray, "");
+  }
   bool isSharedMap() const { return m_kind == ArrayKind::kSharedMap; }
   bool isNameValueTableWrapper() const {
     return m_kind == ArrayKind::kNameValueTableWrapper;
@@ -208,11 +214,14 @@ public:
 
   /**
    * Interface for VM helpers.  ArrayData implements generic versions
-   * using the other ArrayData api; subclasses may do better.
+   * using the other ArrayData api; subclasses may customize methods either
+   * by overriding a virtual method or providing a custom static method,
+   * depending on how the method is dispatched.
+   * todo: t2608483 eliminate the remaining virtual methods.
    */
-  virtual TypedValue* nvGet(int64_t k) const = 0;
-  virtual TypedValue* nvGet(const StringData* k) const = 0;
-  virtual void nvGetKey(TypedValue* out, ssize_t pos) const = 0;
+  TypedValue* nvGet(int64_t k) const;
+  TypedValue* nvGet(const StringData* k) const;
+  void nvGetKey(TypedValue* out, ssize_t pos) const;
 
   // nonvirtual wrappers that call virtual getValueRef()
   TypedValue* nvGetValueRef(ssize_t pos);
@@ -250,8 +259,8 @@ public:
    * then set the value. Return NULL if escalation is not needed, or an
    * escalated array data.
    */
-  virtual ArrayData *set(int64_t k, CVarRef v, bool copy) = 0;
-  virtual ArrayData *set(StringData* k, CVarRef v, bool copy) = 0;
+  ArrayData *set(int64_t k, CVarRef v, bool copy);
+  ArrayData *set(StringData* k, CVarRef v, bool copy);
 
   virtual ArrayData *setRef(int64_t k, CVarRef v, bool copy) = 0;
   virtual ArrayData *setRef(StringData* k, CVarRef v, bool copy) = 0;
@@ -381,8 +390,8 @@ public:
    * then append the value. Return NULL if escalation is not needed, or an
    * escalated array data.
    */
-  virtual ArrayData *append(CVarRef v, bool copy) = 0;
-  virtual ArrayData *appendRef(CVarRef v, bool copy) = 0;
+  ArrayData* append(CVarRef v, bool copy);
+  virtual ArrayData* appendRef(CVarRef v, bool copy) = 0;
 
   /**
    * Similar to append(v, copy), with reference in v preserved.
@@ -454,6 +463,9 @@ public:
     static_assert(offsetof(ArrayData, _count) == FAST_REFCOUNT_OFFSET,
                   "Offset of _count in ArrayData must be FAST_REFCOUNT_OFFSET");
   }
+  // safely convert m_kind to an array index for dispatching
+  unsigned index() const;
+
  protected:
   void freeStrongIterators();
   static void moveStrongIterators(ArrayData* dest, ArrayData* src);
@@ -475,6 +487,12 @@ public:
   static bool IsValidKey(CVarRef k);
   static bool IsValidKey(const StringData* k) { return k; }
 
+  // allocation helpers either call smart_malloc api or regular
+  // malloc, depending on m_allocMode
+  void* modeAlloc(size_t nbytes) const;
+  void* modeRealloc(void* ptr, size_t nbytes) const;
+  void modeFree(void* ptr) const;
+
  protected:
   // Layout starts with 64 bits for vtable, then 32 bits for m_count
   // from Countable base, then...
@@ -483,7 +501,7 @@ private:
   FullPos* m_strongIterators; // head of linked list
 protected:
   int32_t m_pos;
-  const ArrayKind m_kind;
+  ArrayKind m_kind;
   const AllocationMode m_allocMode;
 
  public: // for the JIT
@@ -495,6 +513,26 @@ protected:
   void getChildren(std::vector<TypedValue *> &out);
 };
 
+/*
+ * ArrayFunctions is a hand-built virtual dispatch table.  Each field represents
+ * one virtual method with an array of function pointers, one per ArrayKind.
+ * There is one global instance of this table.  Arranging it this way allows
+ * dispatch to be done with a single indexed load, using m_kind as the index.
+ */
+struct ArrayFunctions {
+  // NK stands for number of array kinds; here just for shorthand.
+  static auto const NK = size_t(ArrayData::ArrayKind::kNumKinds);
+  void (*release[NK])(ArrayData*);
+  ArrayData* (*append[NK])(ArrayData*, CVarRef v, bool copy);
+  TypedValue* (*nvGetInt[NK])(const ArrayData*, int64_t k);
+  TypedValue* (*nvGetStr[NK])(const ArrayData*, const StringData* k);
+  void (*nvGetKey[NK])(const ArrayData*, TypedValue* out, ssize_t pos);
+  ArrayData* (*setInt[NK])(ArrayData*, int64_t k, CVarRef v, bool copy);
+  ArrayData* (*setStr[NK])(ArrayData*, StringData* k, CVarRef v, bool copy);
+};
+
+extern const ArrayFunctions g_array_funcs;
+
 ALWAYS_INLINE inline
 void decRefArr(ArrayData* arr) {
   if (arr->decRefCount() == 0) arr->release();

hphp/runtime/base/array_iterator.cpp

@@ -22,6 +22,9 @@
 #include "hphp/runtime/base/object_data.h"
 #include "hphp/runtime/ext/ext_collections.h"
 
+// inline methods of HphpArray.
+#include "hphp/runtime/base/hphp_array-defs.h"
+
 namespace HPHP {
 ///////////////////////////////////////////////////////////////////////////////
 // Static strings.
@@ -909,27 +912,6 @@ int64_t new_iter_array_cold(Iter* dest, ArrayData* arr, TypedValue* valOut,
   return 0LL;
 }
 
-template <bool withRef>
-inline ALWAYS_INLINE
-void getHphpArrayElm(HphpArray::Elm* elm, TypedValue* valOut,
-                     TypedValue* keyOut) {
-  if (withRef) {
-    tvAsVariant(valOut) = withRefBind(tvAsVariant(&elm->data));
-    if (LIKELY(keyOut != nullptr)) {
-      DataType t = keyOut->m_type;
-      uint64_t d = keyOut->m_data.num;
-      HphpArray::getElmKey(elm, keyOut);
-      tvRefcountedDecRefHelper(t, d);
-    }
-  } else {
-    TypedValue* cur = tvToCell(&elm->data);
-    cellDup(*cur, *valOut);
-    if (keyOut) {
-      HphpArray::getElmKey(elm, keyOut);
-    }
-  }
-}
-
 HOT_FUNC
 int64_t new_iter_array(Iter* dest, ArrayData* ad, TypedValue* valOut) {
   TRACE(2, "%s: I %p, ad %p\n", __func__, dest, ad);
@@ -948,8 +930,7 @@ int64_t new_iter_array(Iter* dest, ArrayData* ad, TypedValue* valOut) {
       // we do not need to adjust the refcount.
       (void) new (&dest->arr()) ArrayIter(arr, ArrayIter::noIncNonNull);
       dest->arr().setIterType(ArrayIter::TypeArray);
-      HphpArray::Elm* elm = arr->getElm(dest->arr().m_pos);
-      getHphpArrayElm<false>(elm, valOut, nullptr);
+      arr->getArrayElm<false>(dest->arr().m_pos, valOut, nullptr);
       return 1LL;
     }
     // We did not transfer ownership of the array to an iterator, so we need
@@ -991,8 +972,7 @@ int64_t new_iter_array_key(Iter* dest, ArrayData* ad,
       // we do not need to adjust the refcount.
       (void) new (&dest->arr()) ArrayIter(arr, ArrayIter::noIncNonNull);
       dest->arr().setIterType(ArrayIter::TypeArray);
-      HphpArray::Elm* elm = arr->getElm(dest->arr().m_pos);
-      getHphpArrayElm<withRef>(elm, valOut, keyOut);
+      arr->getArrayElm<withRef>(dest->arr().m_pos, valOut, keyOut);
       return 1LL;
     }
     // We did not transfer ownership of the array to an iterator, so we need
@@ -1236,7 +1216,6 @@ int64_t iter_next(Iter* iter, TypedValue* valOut) {
     }
     const HphpArray* arr = (HphpArray*)ad;
     ssize_t pos = arrIter->getPos();
-    HphpArray::Elm* elm;
     do {
       if (size_t(++pos) >= size_t(arr->iterLimit())) {
         if (UNLIKELY(arr->getCount() == 1)) {
@@ -1248,14 +1227,13 @@ int64_t iter_next(Iter* iter, TypedValue* valOut) {
         }
         return 0;
       }
-      elm = arr->getElm(pos);
-    } while (HphpArray::isTombstone(elm->data.m_type));
+    } while (arr->isTombstone(pos));
     if (UNLIKELY(tvWillBeReleased(valOut))) {
       goto cold;
     }
     tvDecRefOnly(valOut);
     arrIter->setPos(pos);
-    getHphpArrayElm<false>(elm, valOut, nullptr);
+    arr->getArrayElm<false>(pos, valOut, nullptr);
     return 1;
   }
 cold:
@@ -1283,7 +1261,6 @@ int64_t iter_next_key(Iter* iter, TypedValue* valOut, TypedValue* keyOut) {
     }
     const HphpArray* arr = (HphpArray*)ad;
     ssize_t pos = arrIter->getPos();
-    HphpArray::Elm* elm;
     do {
       ++pos;
       if (size_t(pos) >= size_t(arr->iterLimit())) {
@@ -1296,8 +1273,7 @@ int64_t iter_next_key(Iter* iter, TypedValue* valOut, TypedValue* keyOut) {
         }
         return 0;
       }
-      elm = arr->getElm(pos);
-    } while (HphpArray::isTombstone(elm->data.m_type));
+    } while (arr->isTombstone(pos));
     if (!withRef) {
       if (UNLIKELY(tvWillBeReleased(valOut))) {
         goto cold;
@@ -1309,7 +1285,7 @@ int64_t iter_next_key(Iter* iter, TypedValue* valOut, TypedValue* keyOut) {
       tvDecRefOnly(keyOut);
     }
     arrIter->setPos(pos);
-    getHphpArrayElm<withRef>(elm, valOut, keyOut);
+    arr->getArrayElm<withRef>(pos, valOut, keyOut);
     return 1;
   }
   cold:

hphp/runtime/base/complex_types.h

@@ -27,7 +27,7 @@
 #include "hphp/runtime/base/ref_data.h"
 #include "hphp/runtime/base/tv_helpers.h"
 #include "hphp/runtime/base/type_variant.h"
-#include "hphp/runtime/base/array_data-inl.h"
+#include "hphp/runtime/base/array_data-defs.h"
 #include "hphp/runtime/base/array_init.h"
 
 #undef incl_HPHP_INSIDE_HPHP_COMPLEX_TYPES_H_

hphp/runtime/base/hphp_array-defs.h

@@ -0,0 +1,145 @@
+/*
+   +----------------------------------------------------------------------+
+   | HipHop for PHP                                                       |
+   +----------------------------------------------------------------------+
+   | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com)     |
+   +----------------------------------------------------------------------+
+   | This source file is subject to version 3.01 of the PHP license,      |
+   | that is bundled with this package in the file LICENSE, and is        |
+   | available through the world-wide-web at the following url:           |
+   | http://www.php.net/license/3_01.txt                                  |
+   | If you did not receive a copy of the PHP license and are unable to   |
+   | obtain it through the world-wide-web, please send a note to          |
+   | license@php.net so we can mail you a copy immediately.               |
+   +----------------------------------------------------------------------+
+*/
+
+#ifndef incl_HPHP_HPHP_ARRAY_INL_H_
+#define incl_HPHP_HPHP_ARRAY_INL_H_
+
+#include "hphp/runtime/base/hphp_array.h"
+#include "hphp/util/stacktrace_profiler.h"
+
+namespace HPHP {
+///////////////////////////////////////////////////////////////////////////////
+
+inline void* ArrayData::modeAlloc(size_t nbytes) const {
+  return m_allocMode == AllocationMode::smart ? smart_malloc(nbytes) :
+         Util::safe_malloc(nbytes);
+}
+
+inline void* ArrayData::modeRealloc(void* ptr, size_t nbytes) const {
+  return m_allocMode == AllocationMode::smart ? smart_realloc(ptr, nbytes) :
+         Util::safe_realloc(ptr, nbytes);
+}
+
+inline void ArrayData::modeFree(void* ptr) const {
+  return LIKELY(m_allocMode == AllocationMode::smart) ? smart_free(ptr) :
+         free(ptr);
+}
+
+inline void HphpArray::initHash(size_t tableSize) {
+  assert(HphpArray::ElmIndEmpty == -1);
+  memset(m_hash, 0xffU, tableSize * sizeof(*m_hash));
+  m_hLoad = 0;
+}
+
+inline ALWAYS_INLINE
+HphpArray::ElmInd* HphpArray::findForNewInsert(size_t h0) const {
+  assert(!isVector());
+  size_t tableMask = m_tableMask;
+  size_t probeIndex = h0 & tableMask;
+  ElmInd* ei = &m_hash[probeIndex];
+  return !validElmInd(*ei) ? ei :
+         findForNewInsertLoop(tableMask, h0);
+}
+
+inline bool HphpArray::isTombstone(ssize_t pos) const {
+  assert(size_t(pos) <= m_used);
+  return isTombstone(m_data[pos].data.m_type);
+}
+
+inline void HphpArray::getElmKey(const Elm& e, TypedValue* out) {
+  if (e.hasIntKey()) {
+    out->m_data.num = e.ikey;
+    out->m_type = KindOfInt64;
+    return;
+  }
+  auto str = e.key;
+  out->m_data.pstr = str;
+  out->m_type = KindOfString;
+  str->incRefCount();
+}
+
+template <bool withRef> inline ALWAYS_INLINE
+void HphpArray::getArrayElm(ssize_t pos, TypedValue* valOut,
+                            TypedValue* keyOut) const {
+  assert(size_t(pos) < m_used);
+  auto& elm = m_data[pos];
+  if (withRef) {
+    tvAsVariant(valOut) = withRefBind(tvAsVariant(&elm.data));
+    if (LIKELY(keyOut != nullptr)) {
+      DataType t = keyOut->m_type;
+      uint64_t d = keyOut->m_data.num;
+      if (isVector()) {
+        keyOut->m_data.num = pos;
+        keyOut->m_type = KindOfInt64;
+      } else {
+        HphpArray::getElmKey(elm, keyOut);
+      }
+      tvRefcountedDecRefHelper(t, d);
+    }
+  } else {
+    TypedValue* cur = tvToCell(&elm.data);
+    cellDup(*cur, *valOut);
+    if (keyOut) {
+      if (isVector()) {
+        keyOut->m_data.num = pos;
+        keyOut->m_type = KindOfInt64;
+      } else {
+        HphpArray::getElmKey(elm, keyOut);
+      }
+    }
+  }
+}
+
+inline HphpArray* HphpArray::asVector(ArrayData* ad) {
+  assert(ad->kind() == ArrayKind::kVector);
+  assert(dynamic_cast<HphpArray*>(ad));
+  auto a = static_cast<HphpArray*>(ad);
+  assert(a->checkInvariants());
+  return a;
+}
+
+inline const HphpArray* HphpArray::asVector(const ArrayData* ad) {
+  assert(ad->kind() == ArrayKind::kVector);
+  assert(dynamic_cast<const HphpArray*>(ad));
+  auto a = static_cast<const HphpArray*>(ad);
+  assert(a->checkInvariants());
+  return a;
+}
+
+inline HphpArray* HphpArray::asHphpArray(ArrayData* ad) {
+  assert(ad->kind() == ArrayKind::kHphpArray);
+  assert(dynamic_cast<HphpArray*>(ad));
+  auto a = static_cast<HphpArray*>(ad);
+  assert(a->checkInvariants());
+  return a;
+}
+
+inline const HphpArray* HphpArray::asHphpArray(const ArrayData* ad) {
+  assert(ad->kind() == ArrayKind::kHphpArray);
+  assert(dynamic_cast<const HphpArray*>(ad));
+  auto a = static_cast<const HphpArray*>(ad);
+  assert(a->checkInvariants());
+  return a;
+}
+
+inline HphpArray* HphpArray::copyImpl() const {
+  return isVector() ? copyVec() : copyGeneric();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+}
+
+#endif // incl_HPHP_HPHP_ARRAY_INL_H_

hphp/runtime/base/hphp_array.h

@@ -45,8 +45,20 @@ public:
   }
 
 private:
-  // for copy-on-write escalation
-  explicit HphpArray(AllocationMode);
+  enum class CopyVector {};
+  HphpArray(const HphpArray& other, AllocationMode, CopyVector);
+
+  enum class CopyGeneric {};
+  HphpArray(const HphpArray& other, AllocationMode, CopyGeneric);
+
+  // convert in-place from kVector to kHphpArray: fill in keys & hashtable
+  HphpArray* vectorToGeneric();
+
+  // Safe downcast helpers
+  static HphpArray* asVector(ArrayData* ad);
+  static const HphpArray* asVector(const ArrayData* ad);
+  static HphpArray* asHphpArray(ArrayData* ad);
+  static const HphpArray* asHphpArray(const ArrayData* ad);
 
 public:
   // Create an empty array with enough capacity for nSize elements.
@@ -57,6 +69,8 @@ public:
   HphpArray(uint size, const TypedValue* vals); // make tuple
 
   virtual ~HphpArray();
+  void destroyVec();
+  void destroy();
 
   // unlike ArrayData::size(), this functions doesn't delegate
   // to the virtual vsize() functions, so its more efficient to
@@ -90,6 +104,7 @@ public:
   using ArrayData::add;
   using ArrayData::addLval;
   using ArrayData::remove;
+  using ArrayData::nvGet;
 
   // implements ArrayData
   ssize_t vsize() const;
@@ -117,8 +132,10 @@ public:
   ArrayData* getLvalPtr(StringData* k, Variant*& ret, bool copy);
 
   // implements ArrayData
-  ArrayData* set(int64_t k, CVarRef v, bool copy);
-  ArrayData* set(StringData* k, CVarRef v, bool copy);
+  static ArrayData* SetIntVec(ArrayData*, int64_t k, CVarRef v, bool copy);
+  static ArrayData* SetStrVec(ArrayData*, StringData* k, CVarRef v, bool copy);
+  static ArrayData* SetInt(ArrayData*, int64_t k, CVarRef v, bool copy);
+  static ArrayData* SetStr(ArrayData*, StringData* k, CVarRef v, bool copy);
 
   // implements ArrayData
   ArrayData* setRef(int64_t k, CVarRef v, bool copy);
@@ -140,8 +157,11 @@ public:
 
   ArrayData* nonSmartCopy() const;
   HphpArray* copyImpl() const;
+  HphpArray* copyVec() const;
+  HphpArray* copyGeneric() const;
 
-  ArrayData* append(CVarRef v, bool copy);
+  static ArrayData* AppendVec(ArrayData*, CVarRef v, bool copy);
+  static ArrayData* Append(ArrayData*, CVarRef v, bool copy);
   ArrayData* appendRef(CVarRef v, bool copy);
   ArrayData* appendWithRef(CVarRef v, bool copy);
   ArrayData* plus(const ArrayData* elems, bool copy);
@@ -165,8 +185,10 @@ public:
 
   // nvGet returns a pointer to the value if the specified key is in the
   // array, NULL otherwise.
-  TypedValue* nvGet(int64_t ki) const;
-  TypedValue* nvGet(const StringData* k) const;
+  static TypedValue* NvGetIntVec(const ArrayData*, int64_t ki);
+  static TypedValue* NvGetInt(const ArrayData*, int64_t ki);
+  static TypedValue* NvGetStrVec(const ArrayData*, const StringData* k);
+  static TypedValue* NvGetStr(const ArrayData*, const StringData* k);
 
   void nvBind(int64_t ki, const TypedValue* v) {
     updateRef(ki, tvAsCVarRef(v));
@@ -175,10 +197,11 @@ public:
     updateRef(k, tvAsCVarRef(v));
   }
   void nvAppend(const TypedValue* v) {
-    nextInsert(tvAsCVarRef(v));
+    nextInsertVec(tvAsCVarRef(v));
   }
   ArrayData* nvNew(TypedValue*& v, bool copy);
-  void nvGetKey(TypedValue* out, ssize_t pos) const;
+  static void NvGetKeyVec(const ArrayData*, TypedValue* out, ssize_t pos);
+  static void NvGetKey(const ArrayData*, TypedValue* out, ssize_t pos);
   bool nvInsert(StringData* k, TypedValue *v);
 
   /**
@@ -275,26 +298,16 @@ public:
   static const uint32_t SmallMask = SmallHashSize - 1;
   static const uint32_t SmallSize = SmallHashSize - SmallHashSize / LoadScale;
 
-  struct InlineSlots {
-    Elm slots[SmallSize];
-    ElmInd hash[SmallHashSize];
-  };
-
   uint32_t iterLimit() const { return m_used; }
-  Elm* getElm(ssize_t pos) const {
-    assert(size_t(pos) < m_used);
-    return &m_data[pos];
-  }
-  static void getElmKey(Elm* e, TypedValue* out) {
-    if (e->hasIntKey()) {
-      out->m_data.num = e->ikey;
-      out->m_type = KindOfInt64;
-      return;
-    }
-    out->m_data.pstr = e->key;
-    out->m_type = KindOfString;
-    e->key->incRefCount();
-  }
+
+  // Fetch a value and optional key (if keyPos != nullptr), given an
+  // iterator pos.  If withref is true, copy the value with "withRef"
+  // semantics, and decref the previous key before copying the key.
+  // Otherwise get the value cell (unboxing), and initialize keyOut.
+  template <bool withRef>
+  void getArrayElm(ssize_t pos, TypedValue* out, TypedValue* keyOut) const;
+  bool isTombstone(ssize_t pos) const;
+
 private:
   // Small: Array elements and the hash table are allocated inline.
   //
@@ -333,14 +346,18 @@ private:
   // m_hash --> |                    | 2^K hash table entries.
   //            +--------------------+
 
-  uint32_t m_used;       // number of used elements (values or tombstones)
+  uint32_t m_used;       // Number of used elements (values or tombstones)
+  uint32_t m_cap;        // Number of Elms we can use before having to grow.
   uint32_t m_tableMask;  // Bitmask used when indexing into the hash table.
   uint32_t m_hLoad;      // Hash table load (# of non-empty slots).
   int64_t  m_nextKI;     // Next integer key to use for append.
   Elm*     m_data;       // Contains elements and hash table.
   ElmInd*  m_hash;       // Hash table.
   union {
-    InlineSlots m_inline_data;
+    struct {
+      Elm slots[SmallSize];
+      ElmInd hash[SmallHashSize];
+    } m_inline_data;
     ElmInd m_inline_hash[sizeof(m_inline_data) / sizeof(ElmInd)];
   };
 
@@ -355,6 +372,11 @@ private:
   }
   ssize_t prevElm(Elm* elms, ssize_t ei) const;
 
+  // Assert a bunch of invariants about this array then return true.
+  // usage:  assert(checkInvariants());
+  bool checkInvariants() const;
+
+  static void getElmKey(const Elm& e, TypedValue* out);
   ssize_t find(int64_t ki) const;
   ssize_t find(const StringData* s, strhash_t prehash) const;
   ElmInd* findForInsert(int64_t ki) const;
@@ -367,18 +389,11 @@ private:
    * the relevant key is not already present in the array. Otherwise this can
    * put the array into a bad state; use with caution.
    */
+  ElmInd* findForNewInsert(size_t h0) const;
   ElmInd* findForNewInsertLoop(size_t tableMask, size_t h0) const;
 
-  inline ALWAYS_INLINE
-  ElmInd* findForNewInsert(size_t h0) const {
-    size_t tableMask = m_tableMask;
-    size_t probeIndex = h0 & tableMask;
-    ElmInd* ei = &m_hash[probeIndex];
-    return !validElmInd(*ei) ? ei :
-           findForNewInsertLoop(tableMask, h0);
-  }
-
   bool nextInsert(CVarRef data);
+  HphpArray* nextInsertVec(CVarRef data);
   ArrayData* nextInsertRef(CVarRef data);
   ArrayData* nextInsertWithRef(CVarRef data);
   ArrayData* addLvalImpl(int64_t ki, Variant** pDest);
@@ -402,6 +417,7 @@ private:
   Elm* newElmGrow(size_t h0);
   Elm* allocElm(ElmInd* ei);
   Elm* allocElmFast(ElmInd* ei);
+  TypedValue& allocNextElm(uint32_t i);
   void initElmInt(Elm* e, int64_t ki, CVarRef data, bool byRef=false);
   void initElmStr(Elm* e, strhash_t h, StringData* key, CVarRef data,
                   bool byRef=false);
@@ -409,16 +425,8 @@ private:
                       bool byRef=false);
   void newElmStr(ElmInd* ei, strhash_t h, StringData* key, CVarRef data,
                       bool byRef=false);
-  void allocData(size_t maxElms, size_t tableSize);
-  void reallocData(size_t maxElms, size_t tableSize, uint oldMask);
-
-  /**
-   * init(size) allocates space for size elements but initializes as
-   * an empty array. The "WithoutHash" version does not initialize the
-   * hash table and returns the hash table size.
-   */
-  uint32_t initWithoutHash(uint size);
-  void init(uint size);
+  ElmInd* allocData(size_t maxElms, size_t tableSize);
+  ElmInd* reallocData(size_t maxElms, size_t tableSize);
 
   /**
    * grow() increases the hash table size and the number of slots for
@@ -426,6 +434,7 @@ private:
    * does not compact the elements.
    */
   void grow() ATTRIBUTE_COLD;
+  void growVec() ATTRIBUTE_COLD;
 
   /**
    * compact() does not change the hash table size or the number of slots
@@ -448,6 +457,8 @@ private:
 
   // Memory allocator methods.
   DECLARE_SMART_ALLOCATION(HphpArray);
+  static void ReleaseVec(ArrayData*);
+  static void Release(ArrayData*);
 
 private:
   enum EmptyMode { StaticEmptyArray };
@@ -456,10 +467,8 @@ private:
   // isHphpArray implementation; HphpArray should be effectively final.
   static HphpArray s_theEmptyArray;
 
-  static void initHash(HphpArray::ElmInd* hash, size_t tableSize) {
-    assert(HphpArray::ElmIndEmpty == -1);
-    memset(hash, 0xffU, tableSize * sizeof(HphpArray::ElmInd));
-  }
+  void initHash(size_t tableSize);
+  void initNonEmpty(const HphpArray& other);
 
 public:
   static bool validElmInd(ssize_t /*HphpArray::ElmInd*/ ei) {
@@ -479,9 +488,6 @@ public:
       computeMaxElms(tableMask) * sizeof(HphpArray::Elm);
   }
 
-private:
-  HphpArray* clone(AllocationMode am) const;
-  void cloneNonEmpty(HphpArray* target) const;
 };
 
 //=============================================================================
@@ -496,6 +502,11 @@ inline HphpArray* ArrayData::Make(uint size, const TypedValue* data) {
   return NEW(HphpArray)(size, data);
 }
 
+// HphpArray has more than one kind, so reuse ArrayData's virtual dispatch.
+inline void HphpArray::release() {
+  ArrayData::release();
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 }
 

hphp/runtime/base/hphp_array_sort.cpp

@@ -23,6 +23,9 @@
 #include "hphp/runtime/base/execution_context.h"
 #include "hphp/runtime/vm/jit/translator-inline.h"
 
+// inline methods of HphpArray
+#include "hphp/runtime/base/hphp_array-defs.h"
+
 namespace HPHP {
 ///////////////////////////////////////////////////////////////////////////////
 
@@ -61,6 +64,10 @@ template <typename AccessorT>
 HphpArray::SortFlavor
 HphpArray::preSort(const AccessorT& acc, bool checkTypes) {
   assert(m_size > 0);
+  if (isVector()) {
+    // todo t2607563: this is pessimistic.
+    vectorToGeneric();
+  }
   if (!checkTypes && m_size == m_used) {
     // No need to loop over the elements, we're done
     return GenericSort;
@@ -117,8 +124,7 @@ done:
 void HphpArray::postSort(bool resetKeys) {
   assert(m_size > 0);
   size_t tableSize = computeTableSize(m_tableMask);
-  initHash(m_hash, tableSize);
-  m_hLoad = 0;
+  initHash(tableSize);
   if (resetKeys) {
     for (uint32_t pos = 0; pos < m_used; ++pos) {
       Elm* e = &m_data[pos];

hphp/runtime/base/policy_array.cpp

@@ -24,8 +24,8 @@
 
 TRACE_SET_MOD(runtime);
 #define MYLOG if (true) {} else LOG(INFO)
-#define APILOG MYLOG << "{" << this << ":m_size=" << this->m_size     \
-  << ";cap=" << this->capacity() << ";m_pos=" << this->m_pos << "}->" \
+#define APILOG(a) MYLOG << "{" << (a) << ":m_size=" << (a)->m_size    \
+  << ";cap=" << (a)->capacity() << ";m_pos=" << (a)->m_pos << "}->" \
   << __FUNCTION__
 
 namespace HPHP {
@@ -192,7 +192,7 @@ PolicyArray::PolicyArray(uint capacity)
   m_pos = invalid_index;
   // Log at the end of the ctor so as to show the properly initialized
   // members.
-  APILOG << "(" << capacity << ");";
+  APILOG(this) << "(" << capacity << ");";
 }
 
 PolicyArray::PolicyArray(const PolicyArray& rhs, uint capacity,
@@ -203,22 +203,38 @@ PolicyArray::PolicyArray(const PolicyArray& rhs, uint capacity,
   m_pos = rhs.m_pos;
   // Log at the end of the ctor so as to show the properly initialized
   // members.
-  APILOG << "(" << &rhs << ", " << capacity << ", " << uint(am) << ");";
+  APILOG(this) << "(" << &rhs << ", " << capacity << ", " << uint(am) << ");";
 }
 
 PolicyArray::~PolicyArray() {
-  APILOG << "()";
+  APILOG(this) << "()";
   destroy(m_size, m_allocMode);
 }
 
+void PolicyArray::Release(ArrayData* ad) {
+  asPolicyArray(ad)->release();
+}
+
+inline PolicyArray* PolicyArray::asPolicyArray(ArrayData* ad) {
+  assert(ad->kind() == ArrayKind::kPolicyArray);
+  assert(dynamic_cast<PolicyArray*>(ad));
+  return static_cast<PolicyArray*>(ad);
+}
+
+inline const PolicyArray* PolicyArray::asPolicyArray(const ArrayData* ad) {
+  assert(ad->kind() == ArrayKind::kPolicyArray);
+  assert(dynamic_cast<const PolicyArray*>(ad));
+  return static_cast<const PolicyArray*>(ad);
+}
+
 const Variant& PolicyArray::getValueRef(ssize_t pos) const {
-  APILOG << "(" << pos << ")";
+  APILOG(this) << "(" << pos << ")";
   assert(size_t(pos) < m_size);
   return val(toPos(pos));
 }
 
 bool PolicyArray::isVectorData() const {
-  APILOG << "()";
+  APILOG(this) << "()";
   for (ssize_t i = 0; i < m_size; ++i) {
     if (Store::find(i, m_size) != toPos(i)) return false;
   }
@@ -229,29 +245,32 @@ static_assert(ArrayData::invalid_index == size_t(-1), "ehm");
 
 template <class K>
 TypedValue* PolicyArray::nvGetImpl(K k) const {
-  APILOG << "(" << keystr(k) << ")";
+  APILOG(this) << "(" << keystr(k) << ")";
   auto const pos = find(k, m_size);
   return LIKELY(pos != PosType::invalid)
     ? reinterpret_cast<TypedValue*>(&lval(pos))
     : nullptr;
 }
 
-void PolicyArray::nvGetKey(TypedValue* out, ssize_t pos) const {
-  APILOG << "(" << out << ", " << pos << ")";
-  assert(size_t(pos) < m_size);
-  new(out) Variant(key(toPos(pos)));
+TypedValue* PolicyArray::NvGetInt(const ArrayData* ad, int64_t k) {
+  return asPolicyArray(ad)->nvGetImpl(k);
 }
 
-// template <class K>
-// ssize_t ArrayShell::getIndexImpl(K k) const {
-//   APILOG << "(" << keystr(k) << ")";
-//   return toInt<int64_t>(find(k, m_size));
-// }
+TypedValue* PolicyArray::NvGetStr(const ArrayData* ad, const StringData* k) {
+  return asPolicyArray(ad)->nvGetImpl(k);
+}
+
+void PolicyArray::NvGetKey(const ArrayData* ad, TypedValue* out, ssize_t pos) {
+  auto a = asPolicyArray(ad);
+  APILOG(a) << "(" << out << ", " << pos << ")";
+  assert(size_t(pos) < a->m_size);
+  new(out) Variant(a->key(toPos(pos)));
+}
 
 template <class K>
 ArrayData *PolicyArray::lvalImpl(K k, Variant*& ret,
                                   bool copy, bool checkExist) {
-  APILOG << "(" << keystr(k) << ", " << ret << ", "
+  APILOG(this) << "(" << keystr(k) << ", " << ret << ", "
          << copy << ", " << checkExist << ")";
 
   if (copy) {
@@ -315,12 +334,12 @@ ArrayData *PolicyArray::lvalNew(Variant *&ret, bool copy) {
 }
 
 ArrayData *PolicyArray::createLvalPtr(StringData* k, Variant *&ret, bool copy) {
-  APILOG << "(" << keystr(k) << ", " << ret << ", " << copy << ")";
+  APILOG(this) << "(" << keystr(k) << ", " << ret << ", " << copy << ")";
   return addLval(k, ret, copy);
 }
 
 ArrayData *PolicyArray::getLvalPtr(StringData* k, Variant *&ret, bool copy) {
-  APILOG << "(" << keystr(k) << ", " << ret << ", " << copy << ")";
+  APILOG(this) << "(" << keystr(k) << ", " << ret << ", " << copy << ")";
   if (copy) {
     return PolicyArray::copy()->getLvalPtr(k, ret, false);
   }
@@ -333,7 +352,7 @@ ArrayData *PolicyArray::getLvalPtr(StringData* k, Variant *&ret, bool copy) {
 
 template <class K>
 PolicyArray* PolicyArray::setImpl(K k, const Variant& v, bool copy) {
-  APILOG << "(" << keystr(k) << ", " << valstr(v) << ", " << copy
+  APILOG(this) << "(" << keystr(k) << ", " << valstr(v) << ", " << copy
          << ")";
   PolicyArray* result = this;
   if (copy) result = PolicyArray::copy();
@@ -345,9 +364,19 @@ PolicyArray* PolicyArray::setImpl(K k, const Variant& v, bool copy) {
   return result;
 }
 
+ArrayData*
+PolicyArray::SetInt(ArrayData* ad, int64_t k, CVarRef v, bool copy) {
+  return asPolicyArray(ad)->setImpl(k, v, copy);
+}
+
+ArrayData*
+PolicyArray::SetStr(ArrayData* ad, StringData* k, CVarRef v, bool copy) {
+  return asPolicyArray(ad)->setImpl(k, v, copy);
+}
+
 template <class K>
 ArrayData *PolicyArray::setRefImpl(K k, CVarRef v, bool copy) {
-  APILOG << "(" << keystr(k) << ", " << valstr(v) << ", " << copy << ")";
+  APILOG(this) << "(" << keystr(k) << ", " << valstr(v) << ", " << copy << ")";
 
   if (copy) {
     return PolicyArray::copy()->setRef(k, v, false);
@@ -372,7 +401,7 @@ ArrayData *PolicyArray::setRefImpl(K k, CVarRef v, bool copy) {
 
 template <class K>
 ArrayData *PolicyArray::addImpl(K k, const Variant& v, bool copy) {
-  APILOG << "(" << keystr(k) << ", " << valstr(v) << ", " << copy << ");";
+  APILOG(this) << "(" << keystr(k) << ", " << valstr(v) << ", " << copy << ");";
   if (copy) {
     auto result = PolicyArray::copy(m_size * 2 + 1);
     result->add(k, v, false);
@@ -389,7 +418,7 @@ ArrayData *PolicyArray::addImpl(K k, const Variant& v, bool copy) {
 
 template <class K>
 PolicyArray *PolicyArray::addLvalImpl(K k, Variant*& ret, bool copy) {
-  APILOG << "(" << k << ", " << ret << ", " << copy << ")";
+  APILOG(this) << "(" << k << ", " << ret << ", " << copy << ")";
   if (copy) {
     return PolicyArray::copy()->addLval(k, ret, false);
   }
@@ -404,7 +433,7 @@ PolicyArray *PolicyArray::addLvalImpl(K k, Variant*& ret, bool copy) {
 
 template <class K>
 ArrayData *PolicyArray::removeImpl(K k, bool copy) {
-  APILOG << "(" << keystr(k) << ", " << copy << ")";
+  APILOG(this) << "(" << keystr(k) << ", " << copy << ")";
 
   if (copy) {
     return PolicyArray::copy()->remove(k, false);
@@ -441,35 +470,35 @@ ArrayData *PolicyArray::removeImpl(K k, bool copy) {
 }
 
 ssize_t PolicyArray::iter_begin() const {
-  APILOG << "()";
+  APILOG(this) << "()";
   return m_size ? toInt<int64_t>(firstIndex(m_size)) : invalid_index;
 }
 
 ssize_t PolicyArray::iter_end() const {
-  APILOG << "()";
+  APILOG(this) << "()";
   return ssize_t(lastIndex(m_size));
 }
 
 ssize_t PolicyArray::iter_advance(ssize_t prev) const {
-  APILOG << "(" << prev << ")";
+  APILOG(this) << "(" << prev << ")";
   auto const result = toInt<int64_t>(nextIndex(toPos(prev), m_size));
   MYLOG << "returning " << result;
   return result;
 }
 
 ssize_t PolicyArray::iter_rewind(ssize_t prev) const {
-  APILOG << "(" << prev << ")";
+  APILOG(this) << "(" << prev << ")";
   return toInt<int64_t>(prevIndex(toPos(prev), m_size));
 }
 
 bool PolicyArray::validFullPos(const FullPos& fp) const {
-  APILOG << "(" << fp.m_pos << ";" << fp.getResetFlag() << ")";
+  APILOG(this) << "(" << fp.m_pos << ";" << fp.getResetFlag() << ")";
   assert(fp.getContainer() == this);
   return fp.m_pos != invalid_index;
 }
 
 bool PolicyArray::advanceFullPos(FullPos &fp) {
-  APILOG << "(" << fp.m_pos << ";" << fp.getResetFlag() << ")";
+  APILOG(this) << "(" << fp.m_pos << ";" << fp.getResetFlag() << ")";
   assert(fp.getContainer() == this);
   if (fp.getResetFlag()) {
     fp.setResetFlag(false);
@@ -500,12 +529,12 @@ HphpArray* PolicyArray::toHphpArray() const {
 }
 
 ArrayData* PolicyArray::escalateForSort() {
-  APILOG << "()";
+  APILOG(this) << "()";
   return toHphpArray();
 }
 
 PolicyArray *PolicyArray::copy() const {
-  APILOG << "()";
+  APILOG(this) << "()";
   auto result = NEW(PolicyArray)(
     *this,
     capacity() + (m_size == capacity()),
@@ -515,12 +544,12 @@ PolicyArray *PolicyArray::copy() const {
 }
 
 PolicyArray* PolicyArray::copy(uint capacity) {
-  APILOG << "(" << capacity << ")";
+  APILOG(this) << "(" << capacity << ")";
   return NEW(PolicyArray)(*this, capacity, m_allocMode);
 }
 
 PolicyArray *PolicyArray::copyWithStrongIterators() const {
-  APILOG << "()";
+  APILOG(this) << "()";
   auto result = PolicyArray::copy();
   moveStrongIterators(result, const_cast<PolicyArray*>(this));
   assert(result->getCount() == 0);
@@ -528,23 +557,22 @@ PolicyArray *PolicyArray::copyWithStrongIterators() const {
 }
 
 ArrayData *PolicyArray::nonSmartCopy() const {
-  APILOG << "()";
+  APILOG(this) << "()";
   //return NEW(PolicyArray)(*this, capacity(), true);
   return toHphpArray()->nonSmartCopy();
 }
 
-PolicyArray *PolicyArray::append(const Variant& v, bool copy) {
-  APILOG << "(" << valstr(v) << ", " << copy << ")";
-  if (copy) {
-    return PolicyArray::copy()->append(v, false);
-  }
-  grow(m_size, m_size + 1, m_size * 2 + 1, m_allocMode);
-  appendNoGrow(nextKeyBump(), v);
-  return this;
+ArrayData *PolicyArray::Append(ArrayData* ad, const Variant& v, bool copy) {
+  auto a = static_cast<PolicyArray*>(ad);
+  APILOG(a) << "(" << valstr(v) << ", " << copy << ")";
+  if (copy) a = a->PolicyArray::copy();
+  a->grow(a->m_size, a->m_size + 1, a->m_size * 2 + 1, a->m_allocMode);
+  a->appendNoGrow(a->nextKeyBump(), v);
+  return a;
 }
 
 PolicyArray *PolicyArray::appendRef(const Variant& v, bool copy) {
-  APILOG << "(" << valstr(v) << ", " << copy << ")";
+  APILOG(this) << "(" << valstr(v) << ", " << copy << ")";
   if (copy) {
     return PolicyArray::copy()->appendRef(v, false);
   }
@@ -562,7 +590,7 @@ PolicyArray *PolicyArray::appendRef(const Variant& v, bool copy) {
    * Similar to append(v, copy), with reference in v preserved.
    */
 ArrayData *PolicyArray::appendWithRef(CVarRef v, bool copy) {
-  APILOG << "(" << valstr(v) << ", " << copy << ")";
+  APILOG(this) << "(" << valstr(v) << ", " << copy << ")";
   if (copy) {
     return PolicyArray::copy()->appendWithRef(v, false);
   }
@@ -611,7 +639,7 @@ void PolicyArray::nextInsertWithRef(const Variant& v) {
 }
 
 ArrayData *PolicyArray::plus(const ArrayData *elems, bool copy) {
-  APILOG << "(" << elems << ", " << copy << ")";
+  APILOG(this) << "(" << elems << ", " << copy << ")";
   if (copy) {
     return PolicyArray::copy()->plus(elems, false);
   }
@@ -632,7 +660,7 @@ ArrayData *PolicyArray::plus(const ArrayData *elems, bool copy) {
 }
 
 ArrayData *PolicyArray::merge(const ArrayData *elems, bool copy) {
-  APILOG << "(" << elems << ", " << copy << ")";
+  APILOG(this) << "(" << elems << ", " << copy << ")";
   if (copy) {
     return PolicyArray::copy()->merge(elems, false);
   }
@@ -660,7 +688,7 @@ ArrayData *PolicyArray::merge(const ArrayData *elems, bool copy) {
    * Stack function: pop the last item and return it.
    */
 ArrayData* PolicyArray::pop(Variant &value) {
-  APILOG << "(" << &value << ")";
+  APILOG(this) << "(" << &value << ")";
   if (getCount() > 1) {
     return PolicyArray::copy()->pop(value);
   }
@@ -687,7 +715,7 @@ ArrayData* PolicyArray::pop(Variant &value) {
 }
 
 ArrayData *PolicyArray::dequeue(Variant &value) {
-  APILOG << "(" << &value << ")";
+  APILOG(this) << "(" << &value << ")";
   if (getCount() > 1) {
     return PolicyArray::copy()->dequeue(value);
   }
@@ -714,7 +742,7 @@ ArrayData *PolicyArray::dequeue(Variant &value) {
 }
 
 ArrayData* PolicyArray::prepend(CVarRef v, bool copy) {
-  APILOG << "(" << valstr(v) << ", " << copy << ")";
+  APILOG(this) << "(" << valstr(v) << ", " << copy << ")";
   if (copy) {
     return PolicyArray::copy()->prepend(v, false);
   }
@@ -734,7 +762,7 @@ ArrayData* PolicyArray::prepend(CVarRef v, bool copy) {
 }
 
 void PolicyArray::renumber() {
-  APILOG << "()";
+  APILOG(this) << "()";
   if (!m_size) {
     return;
   }
@@ -790,7 +818,7 @@ void PolicyArray::renumber() {
 }
 
 void PolicyArray::onSetEvalScalar() {
-  APILOG << "()";
+  APILOG(this) << "()";
   //FOR_EACH_RANGE (pos, 0, m_size) {
   for (auto pos = firstIndex(m_size); pos != PosType::invalid;
        pos = nextIndex(pos, m_size)) {
@@ -811,7 +839,7 @@ void PolicyArray::onSetEvalScalar() {
 }
 
 ArrayData *PolicyArray::escalate() const {
-  APILOG << "()";
+  APILOG(this) << "()";
   return ArrayData::escalate();
 }
 

hphp/runtime/base/policy_array.h

@@ -318,9 +318,14 @@ class PolicyArray : public ArrayData, private SimpleArrayStore {
   // Copy to a brand new HphpArray
   HphpArray* toHphpArray() const;
 
+  // Safe downcast helpers
+  static PolicyArray* asPolicyArray(ArrayData* ad);
+  static const PolicyArray* asPolicyArray(const ArrayData* ad);
+
 public:
   // Memory allocator methods.
   DECLARE_SMART_ALLOCATION(PolicyArray);
+  static void Release(ArrayData*);
 
   explicit PolicyArray(uint size);
 
@@ -364,13 +369,9 @@ public:
    * Interface for VM helpers.  ArrayData implements generic versions
    * using the other ArrayData api; subclasses may do better.
    */
-  virtual TypedValue* nvGet(int64_t k) const FOLLY_OVERRIDE {
-    return nvGetImpl(k);
-  }
-  virtual TypedValue* nvGet(const StringData* k) const FOLLY_OVERRIDE {
-    return nvGetImpl(k);
-  }
-  virtual void nvGetKey(TypedValue* out, ssize_t pos) const FOLLY_OVERRIDE;
+  static TypedValue* NvGetInt(const ArrayData*, int64_t k);
+  static TypedValue* NvGetStr(const ArrayData*, const StringData* k);
+  static void NvGetKey(const ArrayData*, TypedValue* out, ssize_t pos);
 
 private:
   template <class K>
@@ -425,12 +426,8 @@ private:
   PolicyArray* setImpl(K k, CVarRef v, bool copy);
 
 public:
-  virtual ArrayData* set(int64_t k, CVarRef v, bool copy) FOLLY_OVERRIDE {
-    return setImpl(k, v, copy);
-  }
-  virtual ArrayData* set(StringData* k, CVarRef v, bool copy) FOLLY_OVERRIDE {
-    return setImpl(k, v, copy);
-  }
+  static ArrayData* SetInt(ArrayData*, int64_t k, CVarRef v, bool copy);
+  static ArrayData* SetStr(ArrayData*, StringData* k, CVarRef v, bool copy);
 
 private:
   template <class K>
@@ -558,7 +555,7 @@ public:
    * then append the value. Return NULL if escalation is not needed, or an
    * escalated array data.
    */
-  virtual PolicyArray *append(CVarRef v, bool copy) FOLLY_OVERRIDE;
+  static ArrayData* Append(ArrayData*, CVarRef v, bool copy);
   virtual PolicyArray *appendRef(CVarRef v, bool copy) FOLLY_OVERRIDE;
 
   /**

hphp/runtime/base/shared_map.cpp

@@ -56,6 +56,23 @@ SharedMap::~SharedMap() {
   }
 }
 
+HOT_FUNC
+void SharedMap::Release(ArrayData* ad) {
+  asSharedMap(ad)->release();
+}
+
+inline SharedMap* SharedMap::asSharedMap(ArrayData* ad) {
+  assert(ad->kind() == ArrayKind::kSharedMap);
+  assert(dynamic_cast<SharedMap*>(ad));
+  return static_cast<SharedMap*>(ad);
+}
+
+inline const SharedMap* SharedMap::asSharedMap(const ArrayData* ad) {
+  assert(ad->kind() == ArrayKind::kSharedMap);
+  assert(dynamic_cast<const SharedMap*>(ad));
+  return static_cast<const SharedMap*>(ad);
+}
+
 ssize_t SharedMap::getIndex(const StringData* k) const {
   if (isVector()) return -1;
   return m_map->indexOf(k);
@@ -110,13 +127,15 @@ ArrayData *SharedMap::lvalNew(Variant *&ret, bool copy) {
   return releaseIfCopied(escalated, escalated->lvalNew(ret, false));
 }
 
-ArrayData *SharedMap::set(int64_t k, CVarRef v, bool copy) {
-  ArrayData *escalated = SharedMap::escalate();
+ArrayData*
+SharedMap::SetInt(ArrayData* ad, int64_t k, CVarRef v, bool copy) {
+  ArrayData *escalated = asSharedMap(ad)->SharedMap::escalate();
   return releaseIfCopied(escalated, escalated->set(k, v, false));
 }
 
-ArrayData *SharedMap::set(StringData* k, CVarRef v, bool copy) {
-  ArrayData *escalated = SharedMap::escalate();
+ArrayData*
+SharedMap::SetStr(ArrayData* ad, StringData* k, CVarRef v, bool copy) {
+  ArrayData *escalated = asSharedMap(ad)->SharedMap::escalate();
   return releaseIfCopied(escalated, escalated->set(k, v, false));
 }
 
@@ -144,8 +163,9 @@ ArrayData *SharedMap::copy() const {
   return SharedMap::escalate();
 }
 
-ArrayData *SharedMap::append(CVarRef v, bool copy) {
-  ArrayData *escalated = SharedMap::escalate();
+ArrayData *SharedMap::Append(ArrayData* ad, CVarRef v, bool copy) {
+  auto a = asSharedMap(ad);
+  ArrayData *escalated = a->SharedMap::escalate();
   return releaseIfCopied(escalated, escalated->append(v, false));
 }
 
@@ -180,24 +200,27 @@ ArrayData *SharedMap::escalate() const {
   return ret;
 }
 
-TypedValue* SharedMap::nvGet(int64_t k) const {
-  int index = getIndex(k);
+TypedValue* SharedMap::NvGetInt(const ArrayData* ad, int64_t k) {
+  auto a = asSharedMap(ad);
+  auto index = a->getIndex(k);
   if (index == -1) return nullptr;
-  return (TypedValue*)&getValueRef(index);
+  return (TypedValue*)&a->getValueRef(index);
 }
 
-TypedValue* SharedMap::nvGet(const StringData* key) const {
-  int index = getIndex(key);
+TypedValue* SharedMap::NvGetStr(const ArrayData* ad, const StringData* key) {
+  auto a = asSharedMap(ad);
+  auto index = a->getIndex(key);
   if (index == -1) return nullptr;
-  return (TypedValue*)&getValueRef(index);
+  return (TypedValue*)&a->getValueRef(index);
 }
 
-void SharedMap::nvGetKey(TypedValue* out, ssize_t pos) const {
-  if (isVector()) {
+void SharedMap::NvGetKey(const ArrayData* ad, TypedValue* out, ssize_t pos) {
+  auto a = asSharedMap(ad);
+  if (a->isVector()) {
     out->m_data.num = pos;
     out->m_type = KindOfInt64;
   } else {
-    Variant k = m_map->getKey(pos);
+    Variant k = a->m_map->getKey(pos);
     TypedValue* tv = k.asTypedValue();
     // copy w/out clobbering out->_count.
     out->m_type = tv->m_type;

hphp/runtime/base/shared_map.h

@@ -73,8 +73,8 @@ public:
                           bool checkExist = false);
   ArrayData *lvalNew(Variant *&ret, bool copy);
 
-  ArrayData *set(int64_t k, CVarRef v, bool copy);
-  ArrayData *set(StringData* k, CVarRef v, bool copy);
+  static ArrayData *SetInt(ArrayData*, int64_t k, CVarRef v, bool copy);
+  static ArrayData *SetStr(ArrayData*, StringData* k, CVarRef v, bool copy);
   ArrayData *setRef(int64_t k, CVarRef v, bool copy);
   ArrayData *setRef(StringData* k, CVarRef v, bool copy);
 
@@ -85,7 +85,7 @@ public:
   /**
    * Copy (escalate) the SharedMap without triggering local cache.
    */
-  ArrayData *append(CVarRef v, bool copy);
+  static ArrayData *Append(ArrayData* a, CVarRef v, bool copy);
   ArrayData *appendRef(CVarRef v, bool copy);
   ArrayData *appendWithRef(CVarRef v, bool copy);
   ArrayData *plus(const ArrayData *elems, bool copy);
@@ -96,9 +96,9 @@ public:
   /**
    * Non-Variant virtual methods that override ArrayData
    */
-  TypedValue* nvGet(int64_t k) const;
-  TypedValue* nvGet(const StringData* k) const;
-  void nvGetKey(TypedValue* out, ssize_t pos) const;
+  static TypedValue* NvGetInt(const ArrayData*, int64_t k);
+  static TypedValue* NvGetStr(const ArrayData*, const StringData* k);
+  static void NvGetKey(const ArrayData*, TypedValue* out, ssize_t pos);
 
   bool isVectorData() const;
 
@@ -114,6 +114,7 @@ public:
    * Memory allocator methods.
    */
   DECLARE_SMART_ALLOCATION(SharedMap);
+  static void Release(ArrayData*);
 
   virtual ArrayData *escalate() const;
   virtual ArrayData* escalateForSort();
@@ -123,6 +124,8 @@ public:
 private:
   ssize_t getIndex(int64_t k) const;
   ssize_t getIndex(const StringData* k) const;
+  static SharedMap* asSharedMap(ArrayData* ad);
+  static const SharedMap* asSharedMap(const ArrayData* ad);
 
 private:
   bool m_isVector;

hphp/runtime/vm/name_value_table_wrapper.cpp

@@ -23,6 +23,19 @@ namespace HPHP {
 
 //////////////////////////////////////////////////////////////////////
 
+inline NameValueTableWrapper* NameValueTableWrapper::asNVTW(ArrayData* ad) {
+  assert(ad->kind() == ArrayKind::kNameValueTableWrapper);
+  assert(dynamic_cast<NameValueTableWrapper*>(ad));
+  return static_cast<NameValueTableWrapper*>(ad);
+}
+
+inline const NameValueTableWrapper*
+NameValueTableWrapper::asNVTW(const ArrayData* ad) {
+  assert(ad->kind() == ArrayKind::kNameValueTableWrapper);
+  assert(dynamic_cast<const NameValueTableWrapper*>(ad));
+  return static_cast<const NameValueTableWrapper*>(ad);
+}
+
 ssize_t NameValueTableWrapper::vsize() const {
   // We need to iterate to find out the actual size, since
   // KindOfIndirect elements in the array may have been set to
@@ -36,8 +49,10 @@ ssize_t NameValueTableWrapper::vsize() const {
   return count;
 }
 
-void NameValueTableWrapper::nvGetKey(TypedValue* out, ssize_t pos) const {
-  NameValueTable::Iterator iter(m_tab, pos);
+void NameValueTableWrapper::NvGetKey(const ArrayData* ad, TypedValue* out,
+                                     ssize_t pos) {
+  auto a = asNVTW(ad);
+  NameValueTable::Iterator iter(a->m_tab, pos);
   if (iter.valid()) {
     auto k = iter.curKey();
     out->m_data.pstr = const_cast<StringData*>(k);
@@ -67,12 +82,13 @@ bool NameValueTableWrapper::exists(const StringData* k) const {
   return m_tab->lookup(k);
 }
 
-TypedValue* NameValueTableWrapper::nvGet(const StringData* k) const {
-  return m_tab->lookup(k);
+TypedValue*
+NameValueTableWrapper::NvGetStr(const ArrayData* ad, const StringData* k) {
+  return asNVTW(ad)->m_tab->lookup(k);
 }
 
-TypedValue* NameValueTableWrapper::nvGet(int64_t k) const {
-  return m_tab->lookup(String(k).get());
+TypedValue* NameValueTableWrapper::NvGetInt(const ArrayData* ad, int64_t k) {
+  return asNVTW(ad)->m_tab->lookup(String(k).get());
 }
 
 ArrayData* NameValueTableWrapper::lval(int64_t k, Variant*& ret, bool copy,
@@ -97,14 +113,16 @@ ArrayData* NameValueTableWrapper::lvalNew(Variant*& ret, bool copy) {
   return this;
 }
 
-ArrayData* NameValueTableWrapper::set(int64_t k, CVarRef v, bool copy) {
-  return set(String(k), v, copy);
+ArrayData* NameValueTableWrapper::SetInt(ArrayData* ad, int64_t k,
+                                         CVarRef v, bool copy) {
+  return SetStr(ad, String(k).get(), v, copy);
 }
 
-ArrayData* NameValueTableWrapper::set(StringData* k, CVarRef v,
-                                      bool copy) {
-  tvAsVariant(m_tab->lookupAdd(k)).assignVal(v);
-  return this;
+ArrayData* NameValueTableWrapper::SetStr(ArrayData* ad, StringData* k,
+                                         CVarRef v, bool copy) {
+  auto a = asNVTW(ad);
+  tvAsVariant(a->m_tab->lookupAdd(k)).assignVal(v);
+  return a;
 }
 
 ArrayData* NameValueTableWrapper::setRef(int64_t k, CVarRef v, bool copy) {
@@ -131,7 +149,7 @@ ArrayData* NameValueTableWrapper::remove(const StringData* k, bool copy) {
  * is currently $GLOBALS.
  */
 
-ArrayData* NameValueTableWrapper::append(CVarRef v, bool copy) {
+ArrayData* NameValueTableWrapper::Append(ArrayData*, CVarRef v, bool copy) {
   throw NotImplementedException("append on $GLOBALS");
 }
 

hphp/runtime/vm/name_value_table_wrapper.h

@@ -59,10 +59,10 @@ struct NameValueTableWrapper : public ArrayData {
   { }
 
 public: // ArrayData implementation
+  static void Release(ArrayData*) {}
 
-  // these using directives ensure the full set of overloaded functions
-  // are visible in this class, to avoid triggering implicit conversions
-  // from a CVarRef key to int64.
+  // These using directives ensure the full set of overloaded functions
+  // are visible in this class, to avoid triggering implicit conversions.
   using ArrayData::exists;
   using ArrayData::lval;
   using ArrayData::lvalNew;
@@ -71,21 +71,22 @@ public: // ArrayData implementation
   using ArrayData::add;
   using ArrayData::addLval;
   using ArrayData::remove;
+  using ArrayData::nvGet;
 
   Variant& getRef(CStrRef k) {
     return tvAsVariant(nvGet(k.get()));
   }
 
   virtual ssize_t vsize() const;
-  virtual void nvGetKey(TypedValue* out, ssize_t pos) const;
+  static void NvGetKey(const ArrayData* ad, TypedValue* out, ssize_t pos);
   virtual CVarRef getValueRef(ssize_t pos) const;
   virtual bool noCopyOnWrite() const;
 
   virtual bool exists(int64_t k) const;
   virtual bool exists(const StringData* k) const;
 
-  virtual TypedValue* nvGet(int64_t k) const;
-  virtual TypedValue* nvGet(const StringData* k) const;
+  static TypedValue* NvGetInt(const ArrayData*, int64_t k);
+  static TypedValue* NvGetStr(const ArrayData*, const StringData* k);
 
   virtual ArrayData* lval(int64_t k, Variant*& ret, bool copy,
                           bool checkExist = false);
@@ -93,8 +94,8 @@ public: // ArrayData implementation
                           bool copy, bool checkExist = false);
   virtual ArrayData* lvalNew(Variant*& ret, bool copy);
 
-  virtual ArrayData* set(int64_t k, CVarRef v, bool copy);
-  virtual ArrayData* set(StringData* k, CVarRef v, bool copy);
+  static ArrayData* SetInt(ArrayData*, int64_t k, CVarRef v, bool copy);
+  static ArrayData* SetStr(ArrayData*, StringData* k, CVarRef v, bool copy);
   virtual ArrayData* setRef(int64_t k, CVarRef v, bool copy);
   virtual ArrayData* setRef(StringData* k, CVarRef v, bool copy);
   virtual ArrayData* remove(int64_t k, bool copy);
@@ -102,7 +103,7 @@ public: // ArrayData implementation
 
   virtual ArrayData* copy() const { return 0; }
 
-  virtual ArrayData* append(CVarRef v, bool copy);
+  static ArrayData* Append(ArrayData*, CVarRef v, bool copy);
   virtual ArrayData* appendRef(CVarRef v, bool copy);
   virtual ArrayData* appendWithRef(CVarRef v, bool copy);
 
@@ -128,6 +129,10 @@ public: // ArrayData implementation
   virtual void usort(CVarRef cmp_function);
   virtual void uasort(CVarRef cmp_function);
 
+private:
+  static NameValueTableWrapper* asNVTW(ArrayData* ad);
+  static const NameValueTableWrapper* asNVTW(const ArrayData* ad);
+
 private:
   NameValueTable* const m_tab;
 };