/*
   +----------------------------------------------------------------------+
   | HipHop for PHP                                                       |
   +----------------------------------------------------------------------+
   | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com)     |
   +----------------------------------------------------------------------+
   | This source file is subject to version 3.01 of the PHP license,      |
   | that is bundled with this package in the file LICENSE, and is        |
   | available through the world-wide-web at the following url:           |
   | http://www.php.net/license/3_01.txt                                  |
   | If you did not receive a copy of the PHP license and are unable to   |
   | obtain it through the world-wide-web, please send a note to          |
   | license@php.net so we can mail you a copy immediately.               |
   +----------------------------------------------------------------------+
*/

#define INLINE_VARIANT_HELPER 1

#include "hphp/runtime/base/hphp-array.h"

#include "hphp/runtime/base/array-init.h"
#include "hphp/runtime/base/array-iterator.h"
#include "hphp/runtime/base/complex-types.h"
#include "hphp/runtime/base/runtime-option.h"
#include "hphp/runtime/base/runtime-error.h"
#include "hphp/runtime/base/variable-serializer.h"
#include "hphp/runtime/base/apc-local-array.h"
#include "hphp/util/hash.h"
#include "hphp/util/lock.h"
#include "hphp/util/alloc.h"
#include "hphp/util/trace.h"
#include "hphp/util/util.h"
#include "hphp/runtime/base/execution-context.h"
#include "hphp/runtime/vm/member-operations.h"
#include "hphp/runtime/base/stats.h"

// inline methods of HphpArray
#include "hphp/runtime/base/hphp-array-defs.h"

namespace HPHP {

TRACE_SET_MOD(runtime);

//////////////////////////////////////////////////////////////////////

/*
 * Allocation of HphpArray buffers works like this: the smallest buffer
 * size is allocated inline in HphpArray.  Larger buffer sizes are smart
 * allocated or malloc-allocated depending on whether the array itself
 * was smart-allocated or not.  (nonSmartCopy() is used to create static
 * arrays).  HphpArray::m_allocMode tracks the state as it progresses:
 *
 *   kInline -> kSmart, or
 *           -> kMalloc
 *
 * Hashtables never shrink, so the allocMode Never goes backwards.
 * If an array is pre-sized, we might skip directly to kSmart or kMalloc.
 * If an array is created via nonSmartCopy(), we skip kSmart.
 * Since kMalloc is only used for static arrays, and static arrays are
 * never swept, we don't need any sweep method.
 *
 * For kInline, we use space in HphpArray defined as InlineSlots, which
 * has enough room for slots and the hashtable.  The next few larger array
 * sizes use the inline space for just the hashtable, with slots allocated
 * separately.  Even larger tables allocate the hashtable and slots
 * contiguously.
 */

//=============================================================================
// Static members.

std::aligned_storage<
  sizeof(HphpArray) +
    sizeof(HphpArray::Elm) * HphpArray::SmallSize,
    // No need for space for the hash because the empty array is
    // kPackedKind.
  alignof(HphpArray)
>::type s_theEmptyArray;

struct HphpArray::EmptyArrayInitializer {
  EmptyArrayInitializer() ATTRIBUTE_COLD {
    void* vpEmpty = &s_theEmptyArray;

    auto const ad = static_cast<HphpArray*>(vpEmpty);
    ad->m_kind            = kPackedKind;
    ad->m_allocMode       = AllocationMode::smart;
    ad->m_size            = 0;
    ad->m_pos             = ArrayData::invalid_index;
    ad->m_count           = 0;
    ad->m_strongIterators = nullptr;
    ad->m_used            = 0;
    ad->m_tableMask       = SmallHashSize - 1;

    ad->m_cap  = SmallSize;
    ad->m_hash = nullptr;

    ad->setStatic();

    assert(ad->checkInvariants());
  }
};
HphpArray::EmptyArrayInitializer HphpArray::s_arrayInitializer;

//=============================================================================
// Helpers.

namespace {

ALWAYS_INLINE
uint32_t computeMaskFromNumElms(uint32_t n) {
  assert(n <= 0x7fffffffU);
  auto lgSize = HphpArray::MinLgTableSize;
  auto maxElms = HphpArray::SmallSize;
  assert(lgSize >= 2);

  // Note: it's tempting to convert this loop into something involving
  // x64 bsr and a shift.  Naive attempts currently actually add more
  // branches, because we need to initially check whether `n' is less
  // than SmallSize, and after finding the next power of two we need a
  // branch to see if it was big enough for the desired load factor.
  // This is probably still worth revisiting (e.g., MakeReserve could
  // have a precondition that n is at least SmallSize).
  while (maxElms < n) {
    ++lgSize;
    maxElms <<= 1;
  }
  assert(lgSize <= 32);

  // return 2^lgSize - 1
  return ((size_t(1U)) << lgSize) - 1;
  static_assert(HphpArray::MinLgTableSize >= 2,
                "lower limit for 0.75 load factor");
}

ALWAYS_INLINE
std::pair<uint32_t,uint32_t> computeCapAndMask(uint32_t minimumMaxElms) {
  auto const mask = computeMaskFromNumElms(minimumMaxElms);
  auto const cap  = HphpArray::computeMaxElms(mask);
  return std::make_pair(cap, mask);
}

ALWAYS_INLINE
uint32_t computeAllocBytes(uint32_t cap, uint32_t mask) {
  auto const tabSize    = mask + 1;
  auto const tabBytes   = tabSize * sizeof(int32_t);
  auto const dataBytes  = cap * sizeof(HphpArray::Elm);
  return sizeof(HphpArray) + tabBytes + dataBytes;
}

ALWAYS_INLINE
HphpArray* smartAllocArray(uint32_t cap, uint32_t mask) {
  /*
   * Note: we're currently still allocating the memory for the hash
   * for a packed array even if we aren't going to use it yet.
   */
  auto const allocBytes = computeAllocBytes(cap, mask);
  return static_cast<HphpArray*>(MM().objMalloc(allocBytes));
}

ALWAYS_INLINE
HphpArray* mallocArray(uint32_t cap, uint32_t mask) {
  auto const allocBytes = computeAllocBytes(cap, mask);
  return static_cast<HphpArray*>(std::malloc(allocBytes));
}

}

//=============================================================================
// Construction

HOT_FUNC_VM NEVER_INLINE
HphpArray* HphpArray::MakeReserve(uint32_t capacity) {
  auto const cmret = computeCapAndMask(capacity);
  auto const cap   = cmret.first;
  auto const mask  = cmret.second;
  auto const ad    = smartAllocArray(cap, mask);

  ad->m_kindModeAndSize = static_cast<uint32_t>(AllocationMode::smart) << 8 |
                            kPackedKind; // 0
  ad->m_posAndCount     = uint64_t{1} << 32 |
                           static_cast<uint32_t>(ArrayData::invalid_index);
  ad->m_strongIterators = nullptr;
  ad->m_capAndUsed      = cap;
  ad->m_tableMask       = mask;

  assert(ad->m_kind == kPackedKind);
  assert(ad->m_allocMode == AllocationMode::smart);
  assert(ad->m_size == 0);
  assert(ad->m_pos == ArrayData::invalid_index);
  assert(ad->m_count == 1);
  assert(ad->m_cap == cap);
  assert(ad->m_used == 0);
  assert(ad->checkInvariants());
  return ad;
}

HOT_FUNC_VM
HphpArray* HphpArray::MakePacked(uint32_t size, const TypedValue* values) {
  assert(size > 0);

  auto const cmret = computeCapAndMask(size);
  auto const cap   = cmret.first;
  auto const mask  = cmret.second;
  auto const ad    = smartAllocArray(cap, mask);

  auto const shiftedSize = uint64_t{size} << 32;
  ad->m_kindModeAndSize  = shiftedSize |
                            static_cast<uint32_t>(AllocationMode::smart) << 8 |
                            kPackedKind;
  ad->m_posAndCount      = uint64_t{1} << 32;
  ad->m_strongIterators  = nullptr;
  ad->m_capAndUsed       = shiftedSize | cap;
  ad->m_tableMask        = mask;

  auto const data = reinterpret_cast<Elm*>(ad + 1);

  // Append values by moving -- Caller assumes we update refcount.
  // Values are in reverse order since they come from the stack, which
  // grows down.
  for (uint32_t i = 0; i < size; i++) {
    const auto& tv = values[size - i - 1];
    data[i].data.m_data = tv.m_data;
    data[i].data.m_type = tv.m_type;
  }

  assert(ad->m_kind == kPackedKind);
  assert(ad->m_allocMode == AllocationMode::smart);
  assert(ad->m_size == size);
  assert(ad->m_pos == 0);
  assert(ad->m_count == 1);
  assert(ad->m_cap == cap);
  assert(ad->m_used == size);
  assert(ad->checkInvariants());
  return ad;
}

// for internal use by nonSmartCopy() and copyPacked()
ALWAYS_INLINE
HphpArray* HphpArray::CopyPacked(const HphpArray& other, AllocationMode mode) {
  assert(other.isPacked());

  auto const cap  = other.m_cap;
  auto const mask = other.m_tableMask;
  auto const ad = mode == AllocationMode::smart
    ? smartAllocArray(cap, mask)
    : mallocArray(cap, mask);

  ad->m_kindModeAndSize = uint64_t{other.m_size} << 32 |
                           static_cast<uint32_t>(mode) << 8 |
                           kPackedKind;
  ad->m_posAndCount     = static_cast<uint32_t>(other.m_pos);
  ad->m_strongIterators = nullptr;
  ad->m_capAndUsed      = uint64_t{other.m_used} << 32 | cap;
  ad->m_tableMask       = mask;

  auto const targetElms = reinterpret_cast<Elm*>(ad + 1);

  // Copy the elements and bump up refcounts as needed.
  auto const elms = other.data();
  for (uint32_t i = 0, limit = ad->m_used; i < limit; ++i) {
    tvDupFlattenVars(&elms[i].data, &targetElms[i].data, &other);
  }

  assert(ad->m_kind == kPackedKind);
  assert(ad->m_allocMode == mode);
  assert(ad->m_size == other.m_size);
  assert(ad->m_pos == other.m_pos);
  assert(ad->m_count == 0);
  assert(ad->m_used == other.m_used);
  assert(ad->m_cap == cap);
  assert(ad->m_tableMask == mask);
  assert(ad->checkInvariants());
  return ad;
}

// For internal use by nonSmartCopy() and copyMixed()
ALWAYS_INLINE
HphpArray* HphpArray::CopyMixed(const HphpArray& other, AllocationMode mode) {
  assert(other.m_kind == kMixedKind);

  auto const cap  = other.m_cap;
  auto const mask = other.m_tableMask;
  auto const ad = mode == AllocationMode::smart
    ? smartAllocArray(cap, mask)
    : mallocArray(cap, mask);

  ad->m_kindModeAndSize = uint64_t{other.m_size} << 32 |
                           static_cast<uint32_t>(mode) << 8 |
                           kMixedKind;
  ad->m_posAndCount     = static_cast<uint32_t>(other.m_pos);
  ad->m_strongIterators = nullptr;
  ad->m_capAndUsed      = uint64_t{other.m_used} << 32 | cap;
  ad->m_maskAndLoad     = uint64_t{other.m_hLoad} << 32 | mask;
  ad->m_nextKI          = other.m_nextKI;

  auto const data      = reinterpret_cast<Elm*>(ad + 1);
  auto const hash      = reinterpret_cast<int32_t*>(data + cap);

  ad->m_hash = static_cast<int32_t*>(
    memcpy(hash, other.m_hash, (mask + 1) * sizeof *hash)
  );

  // Copy the elements and bump up refcounts as needed.
  auto const elms = other.data();
  for (uint32_t i = 0, limit = ad->m_used; i < limit; ++i) {
    auto const& e = elms[i];
    auto& te = data[i];
    if (!isTombstone(e.data.m_type)) {
      te.key = e.key;
      te.data.hash() = e.data.hash();
      if (te.hasStrKey()) te.key->incRefCount();
      tvDupFlattenVars(&e.data, &te.data, &other);
      assert(te.hash() == e.hash()); // ensure not clobbered.
    } else {
      // Tombstone.
      te.data.m_type = KindOfInvalid;
    }
  }

  // If the element density dropped below 50% due to indirect elements
  // being converted into tombstones, we should do a compaction
  if (ad->m_size < ad->m_used / 2) {
    ad->compact(false);
  }

  assert(ad->m_kind == other.m_kind);
  assert(ad->m_allocMode == mode);
  assert(ad->m_size == other.m_size);
  assert(ad->m_pos == other.m_pos);
  assert(ad->m_count == 0);
  assert(ad->m_used == other.m_used);
  assert(ad->m_cap == cap);
  assert(ad->m_tableMask == mask);
  assert(ad->m_hLoad == other.m_hLoad);
  assert(ad->checkInvariants());
  return ad;
}

NEVER_INLINE ArrayData* HphpArray::NonSmartCopy(const ArrayData* in) {
  auto a = asHphpArray(in);
  assert(a->checkInvariants());
  return a->isPacked()
    ? CopyPacked(*a, AllocationMode::nonSmart)
    : CopyMixed(*a, AllocationMode::nonSmart);
}

NEVER_INLINE HphpArray* HphpArray::copyPacked() const {
  assert(checkInvariants());
  return CopyPacked(*this, AllocationMode::smart);
}

NEVER_INLINE HphpArray* HphpArray::copyMixed() const {
  assert(checkInvariants());
  return CopyMixed(*this, AllocationMode::smart);
}

ALWAYS_INLINE
HphpArray* HphpArray::copyPackedAndResizeIfNeeded() const {
  if (LIKELY(!isFullPacked())) return copyPacked();
  return copyPackedAndResizeIfNeededSlow();
}

NEVER_INLINE
HphpArray* HphpArray::copyPackedAndResizeIfNeededSlow() const {
  assert(isFullPacked());
  // Note: this path will have to handle splitting strong iterators
  // later when we combine copyPacked & GrowPacked into one operation.
  // For now I'm just making use of copyPacked to do it for me before
  // GrowPacked happens.
  auto const copy = copyPacked();
  auto const ret  = GrowPacked(copy);
  assert(ret != copy);
  assert(copy->getCount() == 0);
  ReleasePacked(copy);
  return ret;
}

ALWAYS_INLINE
HphpArray* HphpArray::copyMixedAndResizeIfNeeded() const {
  if (LIKELY(!isFull())) return copyMixed();
  return copyMixedAndResizeIfNeededSlow();
}

NEVER_INLINE
HphpArray* HphpArray::copyMixedAndResizeIfNeededSlow() const {
  assert(isFull());

  // Note: this path will have to handle splitting strong iterators
  // later when we combine copyPacked & GrowPacked into one operation.
  // For now I'm just making use of copyPacked to do it for me before
  // GrowPacked happens.
  auto const copy = copyMixed();
  auto const ret  = Grow(copy);
  assert(ret != copy);
  assert(copy->getCount() == 0);
  Release(copy);
  return ret;
}

//=============================================================================
// Destruction

HOT_FUNC_VM NEVER_INLINE
void HphpArray::ReleasePacked(ArrayData* in) {
  auto const ad = asPacked(in);

  if (!ad->isZombie()) {
    auto const data = ad->data();
    auto const stop = data + ad->m_used;

    for (auto ptr = data; ptr != stop; ++ptr) {
      tvRefcountedDecRef(ptr->data);
    }

    auto fullPos = ad->m_strongIterators;
    while (UNLIKELY(fullPos != nullptr)) {
      fullPos->setContainer(nullptr);
      fullPos = fullPos->getNext();
    }
  }

  auto const cap  = ad->m_cap;
  auto const mask = ad->m_tableMask;
  MM().objFree(ad, computeAllocBytes(cap, mask));
}

HOT_FUNC_VM NEVER_INLINE
void HphpArray::Release(ArrayData* in) {
  auto const ad = asMixed(in);

  if (!ad->isZombie()) {
    auto const data = ad->data();
    auto const stop = data + ad->m_used;

    for (auto ptr = data; ptr != stop; ++ptr) {
      if (isTombstone(ptr->data.m_type)) continue;
      if (ptr->hasStrKey()) decRefStr(ptr->key);
      tvRefcountedDecRef(&ptr->data);
    }

    auto fullPos = ad->m_strongIterators;
    while (UNLIKELY(fullPos != nullptr)) {
      fullPos->setContainer(nullptr);
      fullPos = fullPos->getNext();
    }
  }

  auto const cap  = ad->m_cap;
  auto const mask = ad->m_tableMask;
  MM().objFree(ad, computeAllocBytes(cap, mask));
}

//=============================================================================
// State transitions

NEVER_INLINE HOT_FUNC_VM
HphpArray* HphpArray::packedToMixed() {
  assert(isPacked());

  auto const size      = m_size;
  auto const tableMask = m_tableMask;
  auto pdata           = data();
  auto hash            = reinterpret_cast<int32_t*>(pdata + m_cap);

  m_kind   = kMixedKind;
  m_hLoad  = size;
  m_nextKI = size;
  m_hash   = hash;

  uint32_t i = 0;
  for (; i < size; ++i) {
    pdata->setIntKey(i);
    *hash = i;
    ++pdata;
    ++hash;
  }
  for (; i <= tableMask; ++i) {
    *hash = Empty;
    ++hash;
  }

  assert(checkInvariants());
  return this;
}

//=============================================================================

/*
 * Invariants:
 *
 *  All arrays are either in a mode, or in zombie state.  The zombie
 *  state happens if an array is "moved from"---the only legal
 *  operation on a zombie array is to decref and release it.
 *
 * All arrays (zombie or not):
 *
 *   m_tableMask is 2^k - 1 (required for quadratic probe)
 *   m_tableMask == nextPower2(m_cap) - 1;
 *   m_cap == computeMaxElms(m_tableMask);
 *
 * Zombie state:
 *
 *   m_used == UINT32_MAX
 *   no FullPos's are pointing to this array
 *
 * Non-zombie:
 *
 *   m_size <= m_used; m_used <= m_cap
 *   last element cannot be a tombstone
 *   m_pos and all external iterators can't be on a tombstone
 *
 * kMixedKind:
 *   m_nextKI >= highest actual int key
 *   Elm.data.m_type maybe KindOfInvalid (tombstone)
 *   hash[] maybe Tombstone
 *   m_hLoad >= m_size, == number of non-Empty hash entries
 *
 * kPackedKind:
 *   m_size == m_used
 *   m_nextKI = uninitialized
 *   m_hLoad = uninitialized
 *   m_hash = uninitialized
 *   Elm.key uninitialized
 *   Elm.hash uninitialized
 *   no KindOfInvalid tombstones
 */
bool HphpArray::checkInvariants() const {
  static_assert(sizeof(Elm) == 24, "");
  static_assert(sizeof(ArrayData) == 3 * sizeof(uint64_t), "");
  static_assert(
    sizeof(HphpArray) == sizeof(ArrayData) + 4 * sizeof(uint64_t),
    "Performance is sensitive to sizeof(HphpArray)."
    " Make sure you changed it with good reason and then update this assert."
  );

  // All arrays:
  assert(m_tableMask > 0 && ((m_tableMask+1) & m_tableMask) == 0);
  assert(m_tableMask == Util::nextPower2(m_cap) - 1);
  assert(m_cap == computeMaxElms(m_tableMask));

  if (isZombie()) return true;

  // Non-zombie:
  assert(m_size <= m_used);
  assert(m_used <= m_cap);
  if (m_pos != invalid_index) {
    assert(size_t(m_pos) < m_used);
    assert(!isTombstone(data()[m_pos].data.m_type));
  }
  if (m_used > 0) {
    // can't have a tombstone at the end; m_used should have been trimmed.
    assert(!isTombstone(data()[m_used - 1].data.m_type));
  }

  // Type specific:
  switch (m_kind) {
  case kPackedKind:
    assert(m_size == m_used);
    break;
  case kMixedKind: {
    assert(m_hash);
    assert(m_hLoad >= m_size);
    size_t load = 0;
    return true;
    // The following loop is for debugging arrays only; it slows
    // things down too much for general use
    for (size_t i = 0; i <= m_tableMask; i++) {
      load += m_hash[i] != Empty;
    }
    assert(m_hLoad == load);
    break;
  }
  default:
    assert(false);
    break;
  }

  if (this == GetStaticEmptyArray()) {
    assert(m_size == 0);
    assert(m_used == 0);
    assert(isPacked());
    assert(m_pos == invalid_index);
  }
  return true;
}

//=============================================================================
// Iteration.

inline ssize_t HphpArray::prevElm(Elm* elms, ssize_t ei) const {
  assert(ei <= ssize_t(m_used));
  while (ei > 0) {
    --ei;
    if (!isTombstone(elms[ei].data.m_type)) {
      return ei;
    }
  }
  return invalid_index;
}

ssize_t HphpArray::IterBegin(const ArrayData* ad) {
  auto a = asHphpArray(ad);
  return a->nextElm(a->data(), invalid_index);
}

ssize_t HphpArray::IterEnd(const ArrayData* ad) {
  auto a = asHphpArray(ad);
  return a->prevElm(a->data(), a->m_used);
}

ssize_t HphpArray::IterAdvance(const ArrayData* ad, ssize_t pos) {
  auto a = asHphpArray(ad);
  // Since m_used is always less than 2^32 and invalid_index == -1,
  // we can save a check by doing an unsigned comparison instead
  // of a signed comparison.
  if (size_t(++pos) < a->m_used && !isTombstone(a->data()[pos].data.m_type)) {
    return pos;
  }
  return a->iter_advance_helper(pos);
  static_assert(invalid_index == -1, "");
}

// caller has already incremented pos but encountered a tombstone
ssize_t HphpArray::iter_advance_helper(ssize_t next_pos) const {
  Elm* elms = data();
  // Since m_used is always less than 2^32 and invalid_index == -1,
  // we can save a check by doing an unsigned comparison instead of
  // a signed comparison.
  for (auto limit = m_used; size_t(next_pos) < limit; ++next_pos) {
    if (!isTombstone(elms[next_pos].data.m_type)) {
      return next_pos;
    }
  }
  return invalid_index;
}

ssize_t HphpArray::IterRewind(const ArrayData* ad, ssize_t pos) {
  if (pos == invalid_index) return invalid_index;
  auto a = asHphpArray(ad);
  return a->prevElm(a->data(), pos);
}

CVarRef HphpArray::GetValueRef(const ArrayData* ad, ssize_t pos) {
  auto a = asHphpArray(ad);
  assert(a->checkInvariants());
  assert(pos != invalid_index);
  auto& e = a->data()[pos];
  assert(!isTombstone(e.data.m_type));
  return tvAsCVarRef(&e.data);
}

bool HphpArray::IsVectorDataPacked(const ArrayData*) {
  return true;
}

bool HphpArray::IsVectorData(const ArrayData* ad) {
  auto a = asMixed(ad);
  if (a->m_size == 0) {
    // any 0-length array is "vector-like" for the sake of this
    // function, even if m_kind != kVector.
    return true;
  }
  auto const elms = a->data();
  int64_t i = 0;
  for (uint32_t pos = 0, limit = a->m_used; pos < limit; ++pos) {
    auto const& e = elms[pos];
    if (isTombstone(e.data.m_type)) {
      continue;
    }
    if (e.hasStrKey() || e.ikey != i) {
      return false;
    }
    ++i;
  }
  return true;
}

//=============================================================================
// Lookup.

static bool hitStringKey(const HphpArray::Elm& e, const StringData* s,
                         int32_t hash) {
  // hitStringKey() should only be called on an Elm that is referenced by a
  // hash table entry. HphpArray guarantees that when it adds a hash table
  // entry that it always sets it to refer to a valid element. Likewise when
  // it removes an element it always removes the corresponding hash entry.
  // Therefore the assertion below must hold.
  assert(!HphpArray::isTombstone(e.data.m_type));
  return hash == e.hash() && (s == e.key || s->same(e.key));
}

static bool hitIntKey(const HphpArray::Elm& e, int64_t ki) {
  // hitIntKey() should only be called on an Elm that is referenced by a
  // hash table entry. HphpArray guarantees that when it adds a hash table
  // entry that it always sets it to refer to a valid element. Likewise when
  // it removes an element it always removes the corresponding hash entry.
  // Therefore the assertion below must hold.
  assert(!HphpArray::isTombstone(e.data.m_type));
  return e.ikey == ki && e.hasIntKey();
}

// Quadratic probe is:
//
//   h(k, i) = (k + c1*i + c2*(i^2)) % tableSize
//
// Use 1/2 for c1 and c2.  In combination with a table size that is a power of
// 2, this guarantees a probe sequence of length tableSize that probes all
// table elements exactly once.

template <class Hit> ALWAYS_INLINE
ssize_t HphpArray::findImpl(size_t h0, Hit hit) const {
  // tableMask, probeIndex, and pos are explicitly 64-bit, because performance
  // regressed when they were 32-bit types via auto.  Test carefully.
  size_t tableMask = m_tableMask;
  size_t probeIndex = h0 & tableMask;
  auto* elms = data();
  auto* hashtable = m_hash;
  ssize_t pos = hashtable[probeIndex];
  if ((validPos(pos) && hit(elms[pos])) || pos == Empty) {
    return pos;
  }
  for (size_t i = 1;; ++i) {
    assert(i <= tableMask);
    probeIndex = (probeIndex + i) & tableMask;
    assert(probeIndex == ((h0 + (i + i*i) / 2) & tableMask));
    pos = hashtable[probeIndex];
    if ((validPos(pos) && hit(elms[pos])) || pos == Empty) {
      return pos;
    }
  }
}

NEVER_INLINE
ssize_t HphpArray::find(int64_t ki) const {
  // all vector methods should work w/out touching the hashtable
  assert(!isPacked());
  return findImpl(ki, [ki] (const Elm& e) {
    return hitIntKey(e, ki);
  });
}

NEVER_INLINE
ssize_t HphpArray::find(const StringData* s, strhash_t prehash) const {
  // all vector methods should work w/out touching the hashtable
  assert(!isPacked());
  auto h = prehash | STRHASH_MSB;
  return findImpl(prehash, [s, h] (const Elm& e) {
    return hitStringKey(e, s, h);
  });
}

NEVER_INLINE
int32_t* warnUnbalanced(size_t n, int32_t* ei) {
  if (n > size_t(RuntimeOption::MaxArrayChain)) {
    raise_error("Array is too unbalanced (%lu)", n);
  }
  return ei;
}

template <class Hit> ALWAYS_INLINE
int32_t* HphpArray::findForInsertImpl(size_t h0, Hit hit) const {
  // tableMask, probeIndex, and pos are explicitly 64-bit, because performance
  // regressed when they were 32-bit types via auto.  Test carefully.
  assert(m_hLoad <= computeMaxElms(m_tableMask));
  size_t tableMask = m_tableMask;
  auto* elms = data();
  auto* hashtable = m_hash;
  int32_t* ret = nullptr;
  size_t probeIndex = h0 & tableMask;
  auto* ei = &hashtable[probeIndex];
  ssize_t pos = *ei;
  if ((validPos(pos) && hit(elms[pos])) || pos == Empty) {
    return ei;
  }
  if (!validPos(pos)) ret = ei;
  for (size_t i = 1;; ++i) {
    assert(i <= tableMask);
    probeIndex = (probeIndex + i) & tableMask;
    assert(probeIndex == ((h0 + (i + i*i) / 2) & tableMask));
    ei = &hashtable[probeIndex];
    pos = *ei;
    if (validPos(pos)) {
      if (hit(elms[pos])) {
        return ei;
      }
    } else {
      if (!ret) ret = ei;
      if (pos == Empty) {
        return LIKELY(i <= 100) ? ret : warnUnbalanced(i, ret);
      }
    }
  }
}

int32_t* HphpArray::findForInsert(int64_t ki) const {
  // all vector methods should work w/out touching the hashtable
  assert(!isPacked());
  return findForInsertImpl(ki, [ki] (const Elm& e) {
    return hitIntKey(e, ki);
  });
}

int32_t*
HphpArray::findForInsert(const StringData* s, strhash_t prehash) const {
  // all vector methods should work w/out touching the hashtable
  assert(!isPacked());
  auto h = prehash | STRHASH_MSB;
  return findForInsertImpl(prehash, [s, h] (const Elm& e) {
    return hitStringKey(e, s, h);
  });
}

HphpArray::InsertPos HphpArray::insert(int64_t k) {
  assert(!isFull());
  auto ei = findForInsert(k);
  if (validPos(*ei)) {
    return InsertPos(true, data()[*ei].data);
  }
  if (k >= m_nextKI && m_nextKI >= 0) m_nextKI = k + 1;
  auto& e = allocElm(ei);
  e.setIntKey(k);
  return InsertPos(false, e.data);
}

HphpArray::InsertPos HphpArray::insert(StringData* k) {
  assert(!isFull());
  strhash_t h = k->hash();
  auto ei = findForInsert(k, h);
  if (validPos(*ei)) {
    return InsertPos(true, data()[*ei].data);
  }
  auto& e = allocElm(ei);
  e.setStrKey(k, h);
  return InsertPos(false, e.data);
}

template <class Hit, class Remove> ALWAYS_INLINE
ssize_t HphpArray::findForRemoveImpl(size_t h0, Hit hit, Remove remove) const {
  assert(m_hLoad <= computeMaxElms(m_tableMask));
  size_t mask = m_tableMask;
  auto* elms = data();
  auto* hashtable = m_hash;
  for (size_t i = 1, probe = h0;; ++i) {
    auto* ei = &hashtable[probe & mask];
    ssize_t pos = *ei;
    if (validPos(pos)) {
      if (hit(elms[pos])) {
        remove(elms[pos]);
        *ei = Tombstone;
        return pos;
      }
    } else {
      if (pos == Empty) {
        // not found, terminate search
        return pos;
      }
    }
    probe += i;
    assert(probe == (h0 + (i + i*i) / 2));
    assert(i <= mask);
  }
}

NEVER_INLINE
ssize_t HphpArray::findForRemove(int64_t ki, bool updateNext) {
  // all vector methods should work w/out touching the hashtable
  assert(!isPacked());
  return findForRemoveImpl(ki,
      [&] (const Elm& e) {
        return hitIntKey(e, ki);
      },
      [this, ki, updateNext] (Elm& e) {
        assert(ki == e.ikey);
        // Match PHP 5.3.1 semantics
        // Hacky: don't removed the unsigned cast, else g++ can optimize away
        // the check for == 0x7fff..., since there is no signed int k
        // for which k-1 == 0x7fff...
        if ((uint64_t)ki == (uint64_t)m_nextKI-1
              && (ki == 0x7fffffffffffffffLL || updateNext)) {
          --m_nextKI;
        }
      }
  );
}

NEVER_INLINE ssize_t
HphpArray::findForRemove(const StringData* s, strhash_t prehash) {
  // all vector methods should work w/out touching the hashtable
  assert(!isPacked());
  auto h = prehash | STRHASH_MSB;
  return findForRemoveImpl(prehash,
      [&] (const Elm& e) {
        return hitStringKey(e, s, h);
      },
      [] (Elm& e) {
        decRefStr(e.key);
        e.setIntKey(0);
      }
    );
}

NEVER_INLINE int32_t*
HphpArray::findForNewInsertLoop(int32_t* table, size_t h0, size_t mask) {
  /* Quadratic probe. */
  for (size_t i = 1, probe = h0;; ++i) {
    assert(i <= mask);
    probe += i;
    assert(probe == h0 + ((i + i * i) / 2));
    auto ei = &table[probe & mask];
    ssize_t pos = *ei;
    if (!validPos(pos)) return ei;
  }
}

bool HphpArray::ExistsIntPacked(const ArrayData* ad, int64_t k) {
  auto a = asPacked(ad);
  return size_t(k) < a->m_size;
}

bool HphpArray::ExistsInt(const ArrayData* ad, int64_t k) {
  auto a = asMixed(ad);
  return validPos(a->find(k));
}

bool HphpArray::ExistsStrPacked(const ArrayData* ad, const StringData* k) {
  assert(asPacked(ad));
  return false;
}

bool HphpArray::ExistsStr(const ArrayData* ad, const StringData* k) {
  auto a = asMixed(ad);
  return validPos(a->find(k, k->hash()));
}

//=============================================================================
// Append/insert/update.

ALWAYS_INLINE bool HphpArray::isFull() const {
  assert(!isPacked());
  assert(m_used <= m_cap);
  assert(m_hLoad <= m_cap);
  return m_used == m_cap || m_hLoad == m_cap;
}

ALWAYS_INLINE bool HphpArray::isFullPacked() const {
  assert(isPacked());
  assert(m_size <= m_cap);
  return m_size == m_cap;
}

ALWAYS_INLINE HphpArray::Elm& HphpArray::allocElm(int32_t* ei) {
  assert(!validPos(*ei) && !isFull());
  assert(m_size != 0 || m_used == 0);
  ++m_size;
  m_hLoad += (*ei == Empty);
  size_t i = m_used;
  (*ei) = i;
  m_used = i + 1;
  if (m_pos == invalid_index) m_pos = i;
  return data()[i];
}

ALWAYS_INLINE TypedValue& HphpArray::allocNextElm(uint32_t i) {
  assert(isPacked() && i == m_size);
  assert(!isFullPacked());
  auto next = i + 1;
  if (m_pos == invalid_index) m_pos = i;
  m_used = m_size = next;
  return data()[i].data;
}

ALWAYS_INLINE
HphpArray* HphpArray::initVal(TypedValue& tv, CVarRef v) {
  tvAsUninitializedVariant(&tv).constructValHelper(v);
  return this;
}

ALWAYS_INLINE
HphpArray* HphpArray::initRef(TypedValue& tv, CVarRef v) {
  tvAsUninitializedVariant(&tv).constructRefHelper(v);
  return this;
}

ALWAYS_INLINE
HphpArray* HphpArray::getLval(TypedValue& tv, Variant*& ret) {
  ret = &tvAsVariant(&tv);
  return this;
}

ALWAYS_INLINE
HphpArray* HphpArray::initLval(TypedValue& tv, Variant*& ret) {
  tvWriteNull(&tv);
  ret = &tvAsVariant(&tv);
  return this;
}

ALWAYS_INLINE
HphpArray* HphpArray::initWithRef(TypedValue& tv, CVarRef v) {
  tvWriteNull(&tv);
  tvAsVariant(&tv).setWithRef(v);
  return this;
}

ALWAYS_INLINE
HphpArray* HphpArray::setVal(TypedValue& tv, CVarRef v) {
  tvAsVariant(&tv).assignValHelper(v);
  return this;
}

ALWAYS_INLINE
HphpArray* HphpArray::setRef(TypedValue& tv, CVarRef v) {
  tvAsVariant(&tv).assignRefHelper(v);
  return this;
}

/*
 * This is a streamlined copy of Variant.constructValHelper()
 * with no incref+decref because we're moving v to this array.
 */
ALWAYS_INLINE
HphpArray* HphpArray::moveVal(TypedValue& tv, TypedValue v) {
  tv.m_type = typeInitNull(v.m_type);
  tv.m_data.num = v.m_data.num;
  return this;
}

ALWAYS_INLINE HphpArray* HphpArray::resizeIfNeeded() {
  if (UNLIKELY(isFull())) return resize();
  return this;
}

ALWAYS_INLINE HphpArray* HphpArray::resizePackedIfNeeded() {
  if (UNLIKELY(isFullPacked())) return GrowPacked(this);
  return this;
}

NEVER_INLINE HphpArray* HphpArray::resize() {
  uint32_t maxElms = computeMaxElms(m_tableMask);
  assert(m_used <= maxElms);
  assert(m_hLoad <= maxElms);
  // At a minimum, compaction is required.  If the load factor would be >0.5
  // even after compaction, grow instead, in order to avoid the possibility
  // of repeated compaction if the load factor were to hover at nearly 0.75.
  if (m_size > maxElms / 2) {
    return Grow(this);
  }
  compact(false);
  return this;
}

HphpArray* HphpArray::Grow(HphpArray* old) {
  assert(!old->isPacked());
  assert(old->m_allocMode == AllocationMode::smart);
  assert(old->m_tableMask <= 0x7fffffffU);

  DEBUG_ONLY auto oldPos = old->m_pos;

  auto const oldMask    = old->m_tableMask;
  auto const mask       = oldMask * 2 + 1;
  auto const cap        = computeMaxElms(mask);
  auto const ad         = smartAllocArray(cap, mask);

  auto const oldSize        = old->m_size;
  auto const oldPosUnsigned = static_cast<uint32_t>(old->m_pos);
  auto const oldUsed        = old->m_used;
  auto const oldStrongIters = old->m_strongIterators;

  ad->m_kindModeAndSize = uint64_t{oldSize} << 32 |
                            static_cast<uint16_t>(AllocationMode::smart) << 8 |
                            kMixedKind;
  ad->m_posAndCount     = oldPosUnsigned;
  ad->m_strongIterators = oldStrongIters; // could be nullptr
  ad->m_capAndUsed      = uint64_t{oldUsed} << 32 | cap;
  ad->m_maskAndLoad     = uint64_t{oldSize} << 32 | mask;
  ad->m_nextKI          = old->m_nextKI;
  ad->m_hash            = reinterpret_cast<int32_t*>(ad->data() + cap);

  // Migrate all strong iterators to the new array.
  auto si = oldStrongIters;
  while (UNLIKELY(si != nullptr)) {
    si->setContainer(ad);
    si = si->getNext();
  }

  // Copy the old element array, and initialize the hashtable to all
  // empty.
  assert(HphpArray::Empty == -1);
  memcpy(ad->data(), old->data(), oldUsed * sizeof(Elm));
  memset(ad->m_hash, 0xffU, (mask + 1) * sizeof *ad->m_hash);

  // TODO(#2942020): findForNewInsert will reload m_hash and
  // m_tableMask each time through the loop.  A naive attempt to keep
  // them in temporaries just ends up spilling/filling things from the
  // stack.  We can probably do better ...
  auto iter = ad->data();
  auto const stop = iter + oldUsed;
  for (uint32_t i = 0; iter != stop; ++iter, ++i) {
    auto& e = *iter;
    if (isTombstone(e.data.m_type)) continue;
    *ad->findForNewInsert(e.hasIntKey() ? e.ikey : e.hash()) = i;
  }

  old->m_used = -uint32_t{1};

  assert(old->isZombie());
  assert(ad->m_allocMode == AllocationMode::smart);
  assert(ad->m_kind == kMixedKind);
  assert(ad->m_size == oldSize);
  assert(ad->m_count == 0);
  assert(ad->m_pos == oldPos);
  assert(ad->m_used == oldUsed);
  assert(ad->m_tableMask == mask);
  assert(ad->m_hLoad == oldSize);
  assert(ad->checkInvariants());
  return ad;
}

NEVER_INLINE
HphpArray* HphpArray::GrowPacked(HphpArray* old) {
  assert(old->isPacked());
  assert(old->m_allocMode == AllocationMode::smart);
  assert(old->m_cap == old->m_used);

  DEBUG_ONLY auto const oldSize = old->m_size;
  DEBUG_ONLY auto const oldPos  = old->m_pos;

  auto const oldCap     = old->m_cap;
  auto const oldMask    = old->m_tableMask;
  auto const cap        = oldCap * 2;
  auto const mask       = oldMask * 2 + 1;
  auto const ad         = smartAllocArray(cap, mask);

  auto const oldUsed            = old->m_used;
  auto const oldKindModeAndSize = old->m_kindModeAndSize;
  auto const oldPosUnsigned     = uint64_t{static_cast<uint32_t>(old->m_pos)};
  auto const oldStrongIters     = old->m_strongIterators;

  ad->m_kindModeAndSize = oldKindModeAndSize;
  ad->m_posAndCount     = oldPosUnsigned;
  ad->m_strongIterators = oldStrongIters; // could be nullptr
  ad->m_capAndUsed      = uint64_t{oldUsed} << 32 | cap;
  ad->m_tableMask       = mask;

  // Migrate all strong iterators to the new array.
  auto si = oldStrongIters;
  while (UNLIKELY(si != nullptr)) {
    si->setContainer(ad);
    si = si->getNext();
  }

  // Steal the old array payload.
  old->m_used = -uint32_t{1};
  memcpy(ad->data(), old->data(), oldUsed * sizeof(Elm));

  // TODO(#2926276): it would be good to refactor callers to expect
  // our refcount to start at 1.

  assert(old->isZombie());
  assert(ad->m_kind == kPackedKind);
  assert(ad->m_allocMode == AllocationMode::smart);
  assert(ad->m_pos == oldPos);
  assert(ad->m_count == 0);
  assert(ad->m_used == oldUsed);
  assert(ad->m_cap == cap);
  assert(ad->m_size == oldSize);
  assert(ad->checkInvariants());
  return ad;
}

namespace {
  struct ElmKey {
    ElmKey() {}
    ElmKey(int32_t hash, StringData* key) {
      this->hash = hash;
      this->key = key;
    }
    int32_t hash;
    union {
      StringData* key;
      int64_t ikey;
    };
  };
}

void HphpArray::compact(bool renumber /* = false */) {
  assert(!isPacked());
  ElmKey mPos;
  if (m_pos != invalid_index) {
    // Cache key for element associated with m_pos in order to update m_pos
    // below.
    assert(size_t(m_pos) < m_used);
    auto& e = data()[m_pos];
    mPos.hash = e.hasIntKey() ? 0 : e.hash();
    mPos.key = e.key;
  } else {
    // Silence compiler warnings.
    mPos.hash = 0;
    mPos.key = nullptr;
  }
  TinyVector<ElmKey, 3> siKeys;
  for (FullPosRange r(strongIterators()); !r.empty(); r.popFront()) {
    auto ei = r.front()->m_pos;
    if (ei != invalid_index) {
      auto& e = data()[ei];
      siKeys.push_back(ElmKey(e.hash(), e.key));
    }
  }
  if (renumber) {
    m_nextKI = 0;
  }
  auto elms = data();
  size_t tableSize = computeTableSize(m_tableMask);
  initHash(tableSize);
  for (uint32_t frPos = 0, toPos = 0; toPos < m_size; ++toPos, ++frPos) {
    while (isTombstone(elms[frPos].data.m_type)) {
      assert(frPos + 1 < m_used);
      ++frPos;
    }
    auto& toE = elms[toPos];
    if (toPos != frPos) {
      toE = elms[frPos];
    }
    if (renumber && !toE.hasStrKey()) {
      toE.ikey = m_nextKI++;
    }
    auto ie = findForNewInsert(toE.hasIntKey() ? toE.ikey : toE.hash());
    *ie = toPos;
  }
  m_used = m_size;
  m_hLoad = m_size;
  if (m_pos != invalid_index) {
    // Update m_pos, now that compaction is complete.
    if (mPos.hash) {
      m_pos = ssize_t(find(mPos.key, mPos.hash));
    } else {
      m_pos = ssize_t(find(mPos.ikey));
    }
  }
  // Update strong iterators, now that compaction is complete.
  int key = 0;
  for (FullPosRange r(strongIterators()); !r.empty(); r.popFront()) {
    FullPos* fp = r.front();
    if (fp->m_pos != invalid_index) {
      auto& k = siKeys[key];
      key++;
      if (k.hash) { // string key
        fp->m_pos = ssize_t(find(k.key, k.hash));
      } else { // int key
        fp->m_pos = ssize_t(find(k.ikey));
      }
    }
  }
}

bool HphpArray::nextInsert(CVarRef data) {
  assert(m_nextKI >= 0);
  assert(!isPacked());
  assert(!isFull());

  int64_t ki = m_nextKI;
  // The check above enforces an invariant that allows us to always
  // know that m_nextKI is not present in the array, so it is safe
  // to use findForNewInsert()
  auto ei = findForNewInsert(ki);
  assert(!validPos(*ei));
  // Allocate and initialize a new element.
  auto& e = allocElm(ei);
  e.setIntKey(ki);
  m_nextKI = ki + 1; // Update next free element.
  initVal(e.data, data);
  return true;
}

ArrayData* HphpArray::nextInsertRef(CVarRef data) {
  assert(!isPacked());
  assert(!isFull());
  assert(m_nextKI >= 0);

  int64_t ki = m_nextKI;
  // The check above enforces an invariant that allows us to always
  // know that m_nextKI is not present in the array, so it is safe
  // to use findForNewInsert()
  auto ei = findForNewInsert(ki);
  auto& e = allocElm(ei);
  e.setIntKey(ki);
  m_nextKI = ki + 1; // Update next free element.
  return initRef(e.data, data);
}

ArrayData* HphpArray::nextInsertWithRef(CVarRef data) {
  assert(!isFull());

  int64_t ki = m_nextKI;
  auto ei = findForNewInsert(ki);
  assert(!validPos(*ei));

  // Allocate a new element.
  auto& e = allocElm(ei);
  e.setIntKey(ki);
  m_nextKI = ki + 1; // Update next free element.
  return initWithRef(e.data, data);
}

template <class K>
ArrayData* HphpArray::addLvalImpl(K k, Variant*& ret) {
  assert(!isPacked());
  assert(!isFull());
  auto p = insert(k);
  if (!p.found) tvWriteNull(&p.tv);
  return getLval(p.tv, ret);
}

inline ArrayData* HphpArray::addVal(int64_t ki, CVarRef data) {
  assert(!exists(ki));
  assert(!isPacked());
  assert(!isFull());
  auto ei = findForNewInsert(ki);
  auto& e = allocElm(ei);
  e.setIntKey(ki);
  if (ki >= m_nextKI && m_nextKI >= 0) m_nextKI = ki + 1;
  return initVal(e.data, data);
}

inline ArrayData* HphpArray::addVal(StringData* key, CVarRef data) {
  assert(!exists(key));
  assert(!isPacked());
  assert(!isFull());
  strhash_t h = key->hash();
  auto ei = findForNewInsert(h);
  auto& e = allocElm(ei);
  e.setStrKey(key, h);
  return initVal(e.data, data);
}

template <class K> INLINE_SINGLE_CALLER
ArrayData* HphpArray::update(K k, CVarRef data) {
  assert(!isPacked());
  assert(!isFull());
  auto p = insert(k);
  if (p.found) {
    return setVal(p.tv, data);
  }
  return initVal(p.tv, data);
}

template <class K>
ArrayData* HphpArray::updateRef(K k, CVarRef data) {
  assert(!isPacked());
  assert(!isFull());
  auto p = insert(k);
  if (p.found) {
    return setRef(p.tv, data);
  }
  return initRef(p.tv, data);
}

ArrayData* HphpArray::LvalIntPacked(ArrayData* ad, int64_t k, Variant*& ret,
                                    bool copy) {
  auto a = asPacked(ad);

  if (size_t(k) < a->m_size) {
    if (copy) a = a->copyPacked();
    return a->getLval(a->data()[k].data, ret);
  }

  if (copy) {
    a = a->copyPackedAndResizeIfNeeded();
  } else {
    a = a->resizePackedIfNeeded();
  }

  if (size_t(k) == a->m_size) {
    auto& tv = a->allocNextElm(k);
    tvWriteNull(&tv);
    ret = &tvAsVariant(&tv);
    return a;
  }

  // todo t2606310: we know key is new.  use add/findForNewInsert
  a = a->packedToMixed(); // in place
  return a->addLvalImpl(k, ret);
}

ArrayData* HphpArray::LvalInt(ArrayData* ad, int64_t k, Variant*& ret,
                              bool copy) {
  auto a = asMixed(ad);
  if (copy) {
    a = a->copyMixedAndResizeIfNeeded();
  } else {
    a = a->resizeIfNeeded();
  }
  return a->addLvalImpl(k, ret);
}

ArrayData*
HphpArray::LvalStrPacked(ArrayData* ad, StringData* key, Variant*& ret,
                         bool copy) {
  auto a = asPacked(ad);
  a = copy ? a->copyPackedAndResizeIfNeeded()
           : a->resizePackedIfNeeded();
  a = a->packedToMixed();
  return a->addLvalImpl(key, ret);
}

ArrayData* HphpArray::LvalStr(ArrayData* ad, StringData* key, Variant*& ret,
                              bool copy) {
  auto a = asMixed(ad);
  a = copy ? a->copyMixedAndResizeIfNeeded()
           : a->resizeIfNeeded();
  return a->addLvalImpl(key, ret);
}

ArrayData* HphpArray::LvalNewPacked(ArrayData* ad, Variant*& ret, bool copy) {
  auto a = asPacked(ad);
  a = copy ? a->copyPackedAndResizeIfNeeded()
           : a->resizePackedIfNeeded();
  auto& tv = a->allocNextElm(a->m_size);
  tvWriteUninit(&tv);  // TODO(#2942090)
  ret = &tvAsVariant(&tv);
  return a;
}

ArrayData* HphpArray::LvalNew(ArrayData* ad, Variant*& ret, bool copy) {
  auto a = asMixed(ad);
  if (UNLIKELY(a->m_nextKI < 0)) {
    raise_warning("Cannot add element to the array as the next element is "
                  "already occupied");
    ret = &Variant::lvalBlackHole();
    return a;
  }

  a = copy ? a->copyMixedAndResizeIfNeeded()
           : a->resizeIfNeeded();

  if (UNLIKELY(!a->nextInsert(uninit_null()))) {
    ret = &Variant::lvalBlackHole();
    return a;
  }

  ret = &tvAsVariant(&a->data()[a->m_used - 1].data);
  return a;
}

ArrayData*
HphpArray::SetIntPacked(ArrayData* ad, int64_t k, CVarRef v, bool copy) {
  auto a = asPacked(ad);

  if (size_t(k) < a->m_size) {
    if (copy) a = a->copyPacked();
    return a->setVal(a->data()[k].data, v);
  }

  a = copy ? a->copyPackedAndResizeIfNeeded()
           : a->resizePackedIfNeeded();

  if (size_t(k) == a->m_size) {
    auto& tv = a->allocNextElm(k);
    return a->initVal(tv, v);
  }

  // Must escalate to mixed, but call addVal() since key doesn't
  // exist.
  a = a->packedToMixed();
  return a->addVal(k, v);
}

ArrayData* HphpArray::SetInt(ArrayData* ad, int64_t k, CVarRef v, bool copy) {
  auto a = asMixed(ad);
  a = copy ? a->copyMixedAndResizeIfNeeded()
           : a->resizeIfNeeded();
  return a->update(k, v);
}

ArrayData*
HphpArray::SetStrPacked(ArrayData* ad, StringData* k, CVarRef v, bool copy) {
  auto a = asPacked(ad);
  if (copy) a = a->copyPacked();
  // must convert to mixed, but call addVal() since key doesn't exist.
  a = a->resizePackedIfNeeded();
  a = a->packedToMixed();
  return a->addVal(k, v);
}

ArrayData*
HphpArray::SetStr(ArrayData* ad, StringData* k, CVarRef v, bool copy) {
  auto a = asMixed(ad);
  a = copy ? a->copyMixedAndResizeIfNeeded()
           : a->resizeIfNeeded();
  return a->update(k, v);
}

ArrayData*
HphpArray::SetRefIntPacked(ArrayData* ad, int64_t k, CVarRef v, bool copy) {
  auto a = asPacked(ad);

  if (size_t(k) < a->m_size) {
    if (copy) a = a->copyPacked();
    return a->setRef(a->data()[k].data, v);
  }

  a = copy ? a->copyPackedAndResizeIfNeeded()
           : a->resizePackedIfNeeded();

  if (size_t(k) == a->m_size) {
    auto& tv = a->allocNextElm(k);
    return a->initRef(tv, v);
  }

  // todo t2606310: key can't exist.  use add/findForNewInsert
  a = a->packedToMixed();
  return a->updateRef(k, v);
}

ArrayData*
HphpArray::SetRefInt(ArrayData* ad, int64_t k, CVarRef v, bool copy) {
  auto a = asMixed(ad);
  a = copy ? a->copyMixedAndResizeIfNeeded()
           : a->resizeIfNeeded();
  return a->updateRef(k, v);
}

ArrayData*
HphpArray::SetRefStrPacked(ArrayData* ad, StringData* k, CVarRef v, bool copy) {
  auto a = asPacked(ad);
  if (copy) a = a->copyPacked();
  // todo t2606310: key can't exist.  use add/findForNewInsert
  a = a->resizePackedIfNeeded();
  a = a->packedToMixed();
  return a->updateRef(k, v);
}

ArrayData*
HphpArray::SetRefStr(ArrayData* ad, StringData* k, CVarRef v, bool copy) {
  auto a = asMixed(ad);
  a = copy ? a->copyMixedAndResizeIfNeeded()
           : a->resizeIfNeeded();
  return a->updateRef(k, v);
}

ArrayData*
HphpArray::AddIntPacked(ArrayData* ad, int64_t k, CVarRef v, bool copy) {
  assert(!ad->exists(k));
  auto a = asPacked(ad);

  a = copy ? a->copyPackedAndResizeIfNeeded()
           : a->resizePackedIfNeeded();

  if (size_t(k) == a->m_size) {
    auto& tv = a->allocNextElm(k);
    return a->initVal(tv, v);
  }

  a = a->packedToMixed();
  return a->addVal(k, v);
}

ArrayData*
HphpArray::AddInt(ArrayData* ad, int64_t k, CVarRef v, bool copy) {
  assert(!ad->exists(k));
  auto a = asMixed(ad);
  a = copy ? a->copyMixedAndResizeIfNeeded()
           : a->resizeIfNeeded();
  return a->addVal(k, v);
}

ArrayData*
HphpArray::AddStr(ArrayData* ad, StringData* k, CVarRef v, bool copy) {
  assert(!ad->exists(k));
  auto a = asMixed(ad);
  a = copy ? a->copyMixedAndResizeIfNeeded()
           : a->resizeIfNeeded();
  return a->addVal(k, v);
}

//=============================================================================
// Delete.

NEVER_INLINE
void HphpArray::adjustFullPos(ssize_t pos) {
  ssize_t eIPrev = Tombstone;
  for (FullPosRange r(strongIterators()); !r.empty(); r.popFront()) {
    FullPos* fp = r.front();
    if (fp->m_pos == pos) {
      if (eIPrev == Tombstone) {
        // eIPrev will actually be used, so properly initialize it with the
        // previous element before pos, or invalid_index if pos is the first
        // element.
        eIPrev = prevElm(data(), pos);
      }
      if (eIPrev == Empty) {
        fp->setResetFlag(true);
      }
      fp->m_pos = eIPrev;
    }
  }
}

void HphpArray::erase(ssize_t pos) {
  assert(validPos(pos));

  // move strong iterators to the previous element
  if (strongIterators()) adjustFullPos(pos);

  // If the internal pointer points to this element, advance it.
  Elm* elms = data();
  if (m_pos == pos) {
    m_pos = nextElm(elms, pos);
  }

  auto& e = elms[pos];
  // Mark the value as a tombstone.
  TypedValue* tv = &e.data;
  DataType oldType = tv->m_type;
  uint64_t oldDatum = tv->m_data.num;
  tv->m_type = KindOfInvalid;
  --m_size;
  // If this element was last, adjust m_used.
  if (size_t(pos + 1) == m_used) {
    do {
      --m_used;
    } while (m_used > 0 && isTombstone(elms[m_used - 1].data.m_type));
  }
  // Mark the hash entry as "deleted".
  assert(m_used <= m_cap);
  assert(m_hLoad <= m_cap);

  // Finally, decref the old value
  tvRefcountedDecRefHelper(oldType, oldDatum);

  if (m_size < m_used / 2) {
    // Compact in order to keep elms from being overly sparse.
    compact(false);
  }
}

ArrayData* HphpArray::RemoveIntPacked(ArrayData* ad, int64_t k, bool copy) {
  auto a = asPacked(ad);
  if (copy) a = a->copyPacked();
  // todo t2606310: what is probability of (k == size-1)
  if (size_t(k) < a->m_size) {
    a->packedToMixed();
    auto pos = a->findForRemove(k, false);
    if (validPos(pos)) a->erase(pos);
  }
  return a; // key didn't exist, so we're still vector
}

ArrayData* HphpArray::RemoveInt(ArrayData* ad, int64_t k, bool copy) {
  auto a = asMixed(ad);
  if (copy) a = a->copyMixed();
  auto pos = a->findForRemove(k, false);
  if (validPos(pos)) a->erase(pos);
  return a;
}

ArrayData*
HphpArray::RemoveStrPacked(ArrayData* ad, const StringData* key, bool copy) {
  auto a = asPacked(ad);
  if (copy) a = a->copyPacked();
  return a;
}

ArrayData*
HphpArray::RemoveStr(ArrayData* ad, const StringData* key, bool copy) {
  auto a = asMixed(ad);
  if (copy) a = a->copyMixed();
  auto pos = a->findForRemove(key, key->hash());
  if (validPos(pos)) a->erase(pos);
  return a;
}

ArrayData* HphpArray::CopyPacked(const ArrayData* ad) {
  return asPacked(ad)->copyPacked();
}

ArrayData* HphpArray::Copy(const ArrayData* ad) {
  return asMixed(ad)->copyMixed();
}

ArrayData* HphpArray::CopyWithStrongIterators(const ArrayData* ad) {
  auto a = asHphpArray(ad);
  auto copied = a->copyImpl();
  moveStrongIterators(copied, const_cast<HphpArray*>(a));
  return copied;
}

//=============================================================================
// non-variant interface

TypedValue* HphpArray::NvGetIntPacked(const ArrayData* ad, int64_t ki) {
  auto a = asPacked(ad);
  return LIKELY(size_t(ki) < a->m_size) ? &a->data()[ki].data : nullptr;
}

TypedValue* HphpArray::NvGetInt(const ArrayData* ad, int64_t ki) {
  auto a = asMixed(ad);
  auto i = a->find(ki);
  return LIKELY(validPos(i)) ? &a->data()[i].data : nullptr;
}

TypedValue*
HphpArray::NvGetStrPacked(const ArrayData* ad, const StringData* k) {
  assert(asPacked(ad));
  return nullptr;
}

TypedValue* HphpArray::NvGetStr(const ArrayData* ad, const StringData* k) {
  auto a = asMixed(ad);
  auto i = a->find(k, k->hash());
  if (LIKELY(validPos(i))) {
    return &a->data()[i].data;
  }
  return nullptr;
}

// nvGetKey does not touch out->_count, so can be used
// for inner or outer cells.
void HphpArray::NvGetKeyPacked(const ArrayData* ad, TypedValue* out,
                               ssize_t pos) {
  DEBUG_ONLY auto a = asPacked(ad);
  assert(pos != invalid_index);
  assert(!isTombstone(a->data()[pos].data.m_type));
  out->m_data.num = pos;
  out->m_type = KindOfInt64;
}

void HphpArray::NvGetKey(const ArrayData* ad, TypedValue* out, ssize_t pos) {
  auto a = asMixed(ad);
  assert(pos != invalid_index);
  assert(!isTombstone(a->data()[pos].data.m_type));
  getElmKey(a->data()[pos], out);
}

ArrayData* HphpArray::AppendPacked(ArrayData* ad, CVarRef v, bool copy) {
  auto a = asPacked(ad);
  a = copy ? a->copyPackedAndResizeIfNeeded()
           : a->resizePackedIfNeeded();
  auto& tv = a->allocNextElm(a->m_size);
  return a->initVal(tv, v);
}

ArrayData* HphpArray::Append(ArrayData* ad, CVarRef v, bool copy) {
  auto a = asMixed(ad);
  if (UNLIKELY(a->m_nextKI < 0)) {
    raise_warning("Cannot add element to the array as the next element is "
                  "already occupied");
    return a;
  }
  a = copy ? a->copyMixedAndResizeIfNeeded()
           : a->resizeIfNeeded();
  a->nextInsert(v);
  return a;
}

/*
 * Cold path helper for AddNewElemC delegates to the ArrayData::append
 * virtual method.
 */
static NEVER_INLINE
ArrayData* genericAddNewElemC(ArrayData* a, TypedValue value) {
  ArrayData* r = a->append(tvAsCVarRef(&value), a->getCount() != 1);
  if (UNLIKELY(r != a)) {
    r->incRefCount();
    decRefArr(a);
  }
  tvRefcountedDecRef(value);
  return r;
}

/*
 * The pass-by-value and move semantics of this helper are slightly different
 * than other array helpers, but tuned for the opcode.  See doc comment in
 * hphp_array.h.
 */
ArrayData* HphpArray::AddNewElemC(ArrayData* ad, TypedValue value) {
  assert(value.m_type != KindOfRef);
  HphpArray* a;
  int64_t k;
  if (LIKELY(ad->isPacked()) &&
      ((a = asPacked(ad)), LIKELY(a->m_pos >= 0)) &&
      LIKELY(a->getCount() <= 1) &&
      ((k = a->m_size), LIKELY(size_t(k) < a->m_cap))) {
    assert(a->checkInvariants());
    auto& tv = a->allocNextElm(k);
    return a->moveVal(tv, value);
  }
  return genericAddNewElemC(ad, value);
}

ArrayData* HphpArray::AppendRefPacked(ArrayData* ad, CVarRef v, bool copy) {
  auto a = asPacked(ad);
  a = copy ? a->copyPackedAndResizeIfNeeded()
           : a->resizePackedIfNeeded();
  auto& tv = a->allocNextElm(a->m_size);
  return a->initRef(tv, v);
}

ArrayData* HphpArray::AppendRef(ArrayData* ad, CVarRef v, bool copy) {
  auto a = asMixed(ad);
  a = copy ? a->copyMixedAndResizeIfNeeded()
           : a->resizeIfNeeded();

  // Note: preserving behavior, but I think this can leak the copy if
  // the user error handler throws.
  if (UNLIKELY(a->m_nextKI < 0)) {
    raise_warning("Cannot add element to the array as the next element is "
                  "already occupied");
    return a;
  }

  // TODO: maybe inline manually (only caller).
  return a->nextInsertRef(v);
}

ArrayData *HphpArray::AppendWithRefPacked(ArrayData* ad, CVarRef v, bool copy) {
  auto a = asPacked(ad);
  a = copy ? a->copyPackedAndResizeIfNeeded()
           : a->resizePackedIfNeeded();
  auto& tv = a->allocNextElm(a->m_size);
  return a->initWithRef(tv, v);
}

ArrayData *HphpArray::AppendWithRef(ArrayData* ad, CVarRef v, bool copy) {
  auto a = asMixed(ad);
  a = copy ? a->copyMixedAndResizeIfNeeded()
           : a->resizeIfNeeded();
  return a->nextInsertWithRef(v);
}

/*
 * Copy an array to a new array of mixed kind, with a particular
 * pre-reserved size.  The input array may be either packed or mixed.
 */
NEVER_INLINE
HphpArray* HphpArray::CopyReserve(const HphpArray* src,
                                  size_t expectedSize) {
  auto const cmret = computeCapAndMask(expectedSize);
  auto const cap   = cmret.first;
  auto const mask  = cmret.second;
  auto const ad    = smartAllocArray(cap, mask);

  auto const oldPacked      = src->isPacked();
  auto const oldSize        = src->m_size;
  auto const oldPosUnsigned = static_cast<uint32_t>(src->m_pos);
  auto const oldNextKI      = oldPacked ? oldSize : src->m_nextKI;
  auto const oldUsed        = src->m_used;

  ad->m_kindModeAndSize = uint64_t{oldSize} << 32 |
                            static_cast<uint32_t>(AllocationMode::smart) << 8 |
                            kMixedKind;
  ad->m_posAndCount     = uint64_t{1} << 32 | oldPosUnsigned;
  ad->m_strongIterators = nullptr;
  ad->m_maskAndLoad     = uint64_t{oldSize} << 32 | mask;
  ad->m_nextKI          = oldNextKI;

  auto const data = ad->data();
  auto const hash = reinterpret_cast<int32_t*>(data + cap);
  ad->m_hash      = hash;

  assert(Empty == -1);
  memset(hash, 0xffu, (mask + 1) * sizeof *hash);

  auto dstElm = data;
  auto srcElm = src->data();
  auto const srcStop = src->data() + oldUsed;
  uint32_t i = 0;
  if (oldPacked) {
    for (; srcElm != srcStop; ++srcElm, ++i) {
      tvDupFlattenVars(&srcElm->data, &dstElm->data, src);
      dstElm->setIntKey(i);
      *ad->findForNewInsert(i) = i;
      ++dstElm;
    }
    assert(ad->m_pos == oldPosUnsigned);
  } else {
    // We're not copying the tombstones over to the new array, so the
    // positions of the elements in the new array may be shifted. Cache
    // the key for element associated with src->m_pos so that we can
    // properly initialize ad->m_pos below.
    ElmKey mPos;
    if (src->m_pos != invalid_index) {
      assert(size_t(src->m_pos) < src->m_used);
      auto& e = srcElm[src->m_pos];
      mPos.hash = e.hasIntKey() ? 0 : e.hash();
      mPos.key = e.key;
    } else {
      // Silence compiler warnings.
      mPos.hash = 0;
      mPos.key = nullptr;
    }
    // Copy the elements
    for (; srcElm != srcStop; ++srcElm) {
      if (isTombstone(srcElm->data.m_type)) continue;
      tvDupFlattenVars(&srcElm->data, &dstElm->data, src);
      auto const hasIntKey = srcElm->hasIntKey();
      auto const hash = hasIntKey ? srcElm->ikey : srcElm->hash();
      if (hasIntKey) {
        dstElm->setIntKey(srcElm->ikey);
      } else {
        srcElm->key->incRefCount();
        dstElm->setStrKey(srcElm->key, hash);
      }
      *ad->findForNewInsert(hash) = i;
      ++dstElm;
      ++i;
    }
    // Now that we have finished copying the elements, update ad->m_pos
    if (src->m_pos != invalid_index) {
      ad->m_pos = mPos.hash
        ? ssize_t(ad->find(mPos.key, mPos.hash))
        : ssize_t(ad->find(mPos.ikey));
    }
  }

  // Set new used value (we've removed any tombstones).
  assert(i == dstElm - data);
  ad->m_capAndUsed = static_cast<uint64_t>(i) << 32 | cap;

  assert(ad->m_kind == kMixedKind);
  assert(ad->m_allocMode == AllocationMode::smart);
  assert(ad->m_size == oldSize);
  assert(ad->m_count == 1);
  assert(ad->m_cap == cap);
  assert(ad->m_used <= oldUsed);
  assert(ad->m_used == dstElm - data);
  assert(ad->m_hLoad == oldSize);
  assert(ad->m_tableMask == mask);
  assert(ad->m_nextKI == oldNextKI);
  assert(ad->checkInvariants());
  return ad;
}

ArrayData* HphpArray::Plus(ArrayData* ad, const ArrayData* elems) {
  auto const ret = CopyReserve(asHphpArray(ad), ad->size() + elems->size());

  for (ArrayIter it(elems); !it.end(); it.next()) {
    Variant key = it.first();
    CVarRef value = it.secondRef();
    auto tv = key.asTypedValue();
    auto p = tv->m_type == KindOfInt64
      ? ret->insert(tv->m_data.num)
      : ret->insert(tv->m_data.pstr);
    if (!p.found) {
      ret->initWithRef(p.tv, value);
    }
  }

  return ret;
}

ArrayData* HphpArray::Merge(ArrayData* ad, const ArrayData* elems) {
  auto const ret = CopyReserve(asHphpArray(ad), ad->size() + elems->size());

  for (ArrayIter it(elems); !it.end(); it.next()) {
    Variant key = it.first();
    CVarRef value = it.secondRef();
    if (key.asTypedValue()->m_type == KindOfInt64) {
      ret->nextInsertWithRef(value);
    } else {
      Variant* p;
      StringData* sd = key.getStringData();
      ret->addLvalImpl(sd, p);
      p->setWithRef(value);
    }
  }

  // Note: currently caller is responsible for calling renumber after
  // this.  Should refactor so we handle it (we already know things
  // about the array).

  return ret;
}

ArrayData* HphpArray::PopPacked(ArrayData* ad, Variant& value) {
  auto a = asPacked(ad);
  if (a->getCount() > 1) a = a->copyPacked();
  if (a->m_size > 0) {
    auto i = a->m_size - 1;
    auto& tv = a->data()[i].data;
    value = tvAsCVarRef(&tv);
    if (a->strongIterators()) a->adjustFullPos(i);
    auto oldType = tv.m_type;
    auto oldDatum = tv.m_data.num;
    a->m_size = a->m_used = i;
    a->m_pos = a->m_size > 0 ? 0 : invalid_index; // reset internal iterator
    tvRefcountedDecRefHelper(oldType, oldDatum);
    return a;
  }
  value = uninit_null();
  a->m_pos = invalid_index; // reset internal iterator
  return a;
}

ArrayData* HphpArray::Pop(ArrayData* ad, Variant& value) {
  auto a = asMixed(ad);
  if (a->getCount() > 1) a = a->copyMixed();
  auto elms = a->data();
  ssize_t pos = IterEnd(a);
  if (validPos(pos)) {
    auto& e = elms[pos];
    assert(!isTombstone(e.data.m_type));
    value = tvAsCVarRef(&e.data);
    auto pos2 = e.hasStrKey() ? a->findForRemove(e.key, e.hash()) :
                a->findForRemove(e.ikey, true);
    assert(pos2 == pos);
    a->erase(pos2);
  } else {
    value = uninit_null();
  }
  // To conform to PHP behavior, the pop operation resets the array's
  // internal iterator.
  a->m_pos = a->nextElm(elms, invalid_index);
  return a;
}

ArrayData* HphpArray::DequeuePacked(ArrayData* ad, Variant& value) {
  auto a = asPacked(ad);
  if (a->getCount() > 1) a = a->copyPacked();
  // To conform to PHP behavior, we invalidate all strong iterators when an
  // element is removed from the beginning of the array.
  a->freeStrongIterators();
  auto elms = a->data();
  if (a->m_size > 0) {
    auto n = a->m_size - 1;
    auto& tv = elms[0].data;
    value = std::move(tvAsVariant(&tv)); // no incref+decref
    memmove(&elms[0], &elms[1], n * sizeof(elms[0]));
    a->m_size = a->m_used = n;
    a->m_pos = n > 0 ? 0 : invalid_index;
  } else {
    value = uninit_null();
    a->m_pos = invalid_index;
  }
  return a;
}

ArrayData* HphpArray::Dequeue(ArrayData* ad, Variant& value) {
  auto a = asMixed(ad);
  if (a->getCount() > 1) a = a->copyMixed();
  // To conform to PHP behavior, we invalidate all strong iterators when an
  // element is removed from the beginning of the array.
  a->freeStrongIterators();
  auto elms = a->data();
  ssize_t pos = a->nextElm(elms, invalid_index);
  if (validPos(pos)) {
    auto& e = elms[pos];
    value = tvAsCVarRef(&e.data);
    auto pos2 = e.hasStrKey() ? a->findForRemove(e.key, e.hash()) :
                a->findForRemove(e.ikey, false);
    assert(pos2 == pos);
    a->erase(pos2);
    a->compact(true);
  } else {
    value = uninit_null();
  }
  // To conform to PHP behavior, the dequeue operation resets the array's
  // internal iterator
  a->m_pos = a->nextElm(elms, invalid_index);
  return a;
}

ArrayData* HphpArray::PrependPacked(ArrayData* ad, CVarRef v, bool copy) {
  auto a = asPacked(ad);
  if (a->getCount() > 1) a = a->copyPackedAndResizeIfNeeded();
  // To conform to PHP behavior, we invalidate all strong iterators when an
  // element is added to the beginning of the array.
  a->freeStrongIterators();
  size_t n = a->m_size;
  if (n > 0) {
    if (n == a->m_cap) a = GrowPacked(a);
    auto elms = a->data();
    memmove(&elms[1], &elms[0], n * sizeof(elms[0]));
  }
  a->m_size = a->m_used = n + 1;
  a->m_pos = 0;
  a->initVal(a->data()[0].data, v);
  return a;
}

ArrayData* HphpArray::Prepend(ArrayData* ad, CVarRef v, bool copy) {
  auto a = asMixed(ad);
  if (a->getCount() > 1) a = a->copyMixedAndResizeIfNeeded();

  // To conform to PHP behavior, we invalidate all strong iterators when an
  // element is added to the beginning of the array.
  a->freeStrongIterators();

  auto elms = a->data();
  if (a->m_used == 0 || !isTombstone(elms[0].data.m_type)) {
    // Make sure there is room to insert an element.
    a = a->resizeIfNeeded();
    // Recompute elms, in case resizeIfNeeded() had side effects.
    elms = a->data();
    // Move the existing elements to make element 0 available.
    memmove(&elms[1], &elms[0], a->m_used * sizeof(*elms));
    ++a->m_used;
  }

  // Prepend.
  ++a->m_size;
  auto& e = elms[0];
  e.setIntKey(0);
  a->initVal(e.data, v);

  // Renumber.
  a->compact(true);
  // To conform to PHP behavior, the prepend operation resets the array's
  // internal iterator
  a->m_pos = a->nextElm(elms, invalid_index);
  return a;
}

void HphpArray::RenumberPacked(ArrayData* ad) {
  assert(asPacked(ad)); // for the checkInvariants() call
  // renumber has no effect on Vector and doesn't move internal pos
}

void HphpArray::Renumber(ArrayData* ad) {
  asMixed(ad)->compact(true);
}

void HphpArray::OnSetEvalScalarPacked(ArrayData* ad) {
  auto a = asPacked(ad);
  Elm* elms = a->data();
  for (uint32_t i = 0, limit = a->m_size; i < limit; ++i) {
    tvAsVariant(&elms[i].data).setEvalScalar();
  }
}

void HphpArray::OnSetEvalScalar(ArrayData* ad) {
  auto a = asMixed(ad);
  auto elms = a->data();
  for (uint32_t i = 0, limit = a->m_used; i < limit; ++i) {
    auto& e = elms[i];
    if (!isTombstone(e.data.m_type)) {
      auto key = e.key;
      if (e.hasStrKey() && !key->isStatic()) {
        e.key = makeStaticString(key);
        decRefStr(key);
      }
      tvAsVariant(&e.data).setEvalScalar();
    }
  }
}

bool HphpArray::ValidFullPos(const ArrayData* ad, const FullPos &fp) {
  assert(fp.getContainer() == asHphpArray(ad));
  if (fp.getResetFlag()) return false;
  return fp.m_pos != invalid_index;
}

bool HphpArray::AdvanceFullPos(ArrayData* ad, FullPos& fp) {
  auto a = asHphpArray(ad);
  Elm* elms = a->data();
  if (fp.getResetFlag()) {
    fp.setResetFlag(false);
    fp.m_pos = invalid_index;
  } else if (fp.m_pos == invalid_index) {
    return false;
  }
  fp.m_pos = a->nextElm(elms, fp.m_pos);
  if (fp.m_pos == invalid_index) {
    return false;
  }
  // To conform to PHP behavior, we need to set the internal
  // cursor to point to the next element.
  a->m_pos = a->nextElm(elms, fp.m_pos);
  return true;
}

//////////////////////////////////////////////////////////////////////

}