diff --git a/hphp/compiler/analysis/emitter.cpp b/hphp/compiler/analysis/emitter.cpp index a1f0ceadc..5607176e9 100644 --- a/hphp/compiler/analysis/emitter.cpp +++ b/hphp/compiler/analysis/emitter.cpp @@ -7284,8 +7284,7 @@ void EmitterVisitor::initScalar(TypedValue& tvVal, ExpressionPtr val) { case Expression::KindOfUnaryOpExpression: { UnaryOpExpressionPtr u(static_pointer_cast(val)); if (u->getOp() == T_ARRAY) { - auto a = ArrayData::Make(0); - a->incRefCount(); + auto a = HphpArray::MakeReserve(0); m_staticArrays.push_back(a); visit(u->getExpression()); diff --git a/hphp/doc/bytecode.specification b/hphp/doc/bytecode.specification index aebde9419..4c4991557 100644 --- a/hphp/doc/bytecode.specification +++ b/hphp/doc/bytecode.specification @@ -852,9 +852,10 @@ NewArray [] -> [C:Arr] NewTuple [C..C] -> [C] - New array. Creates a new array from the top %1 cells on the stack, pops those - cells, then pushes the new array onto the stack. Elements are implicitly - numbered from 0 to num elems - 1; $1 is at index 0, $2 is at 1, and so on. + New array. Creates a new array from the top %1 cells on the stack, + pops those cells, then pushes the new array onto the stack. Elements + are implicitly numbered from 0 to num elems - 1; $1 is at index 0, + $2 is at 1, and so on. %1 must be greater than zero. AddElemC [C C C] -> [C:Arr] diff --git a/hphp/runtime/base/array-data.h b/hphp/runtime/base/array-data.h index caa279c4d..05f9511a8 100644 --- a/hphp/runtime/base/array-data.h +++ b/hphp/runtime/base/array-data.h @@ -54,6 +54,12 @@ class ArrayData { static const ssize_t invalid_index = -1; protected: + /* + * NOTE: HphpArray no longer calls these constructors. If you + * change them, change the HphpArray::Make functions as + * appropriate. + */ + explicit ArrayData(ArrayKind kind) : m_kind(kind) , m_allocMode(AllocationMode::smart) @@ -89,7 +95,6 @@ class ArrayData { , m_count(0) , m_strongIterators(nullptr) {} - void setRefCount(RefCount n) { m_count = n; } void destroy() { // If there are any strong iterators pointing to this array, they need @@ -101,6 +106,7 @@ class ArrayData { public: IMPLEMENT_COUNTABLE_METHODS + void setRefCount(RefCount n) { m_count = n; } /** * Create a new ArrayData with specified array element(s). @@ -111,20 +117,6 @@ public: static ArrayData *CreateRef(CVarRef value); static ArrayData *CreateRef(CVarRef name, CVarRef value); - /* - * Create a new array with estimated capacity. The returned array - * is not yet incref'd. - */ - static HphpArray* Make(uint capacity); - - /* - * Create an array or tuple-style array. - * - * N.B. unlike the above, the returned HphpArray is already incref'd. - */ - static HphpArray* MakeReserve(uint capacity); - static HphpArray* MakeTuple(uint size, const TypedValue*); - /** * Type conversion functions. All other types are handled inside Array class. */ @@ -521,28 +513,34 @@ public: static bool IsValidKey(CVarRef k); static bool IsValidKey(const StringData* k) { return k; } - // allocation helpers either call smart_malloc api or regular - // malloc, depending on m_allocMode - void* modeAlloc(size_t nbytes) const; - void* modeRealloc(void* ptr, size_t nbytes) const; - void modeFree(void* ptr) const; - - protected: - ArrayKind m_kind; - const AllocationMode m_allocMode; - UNUSED uint16_t m_forSubClasses; // unused space that subclasses may use - uint32_t m_size; - int32_t m_pos; - mutable RefCount m_count; - private: +protected: + // The following fields are blocked into unions with qwords so we + // can combine the stores when initializing arrays. (gcc won't do + // this on its own.) + union { + struct { + ArrayKind m_kind; + AllocationMode m_allocMode; + UNUSED uint16_t m_forSubClasses; // unused space that subclasses may use + uint32_t m_size; + }; + uint64_t m_kindModeAndSize; + }; + union { + struct { + int32_t m_pos; + mutable RefCount m_count; + }; + uint64_t m_posAndCount; // be careful, m_pos is signed + }; FullPos* m_strongIterators; // head of linked list - public: // for the JIT +public: // for the JIT static uint32_t getKindOff() { return (uintptr_t)&((ArrayData*)0)->m_kind; } - public: // for heap profiler +public: // for heap profiler void getChildren(std::vector &out); }; diff --git a/hphp/runtime/base/array-init.cpp b/hphp/runtime/base/array-init.cpp index abaa24c09..7343a6aa4 100644 --- a/hphp/runtime/base/array-init.cpp +++ b/hphp/runtime/base/array-init.cpp @@ -30,13 +30,15 @@ ArrayInit::ArrayInit(ssize_t n) { // Force compilation of PolicyArray m_data = NEW(PolicyArray)(n); } else { - m_data = ArrayData::Make(n); + m_data = HphpArray::MakeReserve(n); + m_data->setRefCount(0); } } HOT_FUNC ArrayInit::ArrayInit(ssize_t n, MapInit) - : m_data(ArrayData::Make(n)) { + : m_data(HphpArray::MakeReserve(n)) { + m_data->setRefCount(0); } /////////////////////////////////////////////////////////////////////////////// diff --git a/hphp/runtime/base/array-init.h b/hphp/runtime/base/array-init.h index 5231888b8..9a14d4ec4 100644 --- a/hphp/runtime/base/array-init.h +++ b/hphp/runtime/base/array-init.h @@ -305,7 +305,9 @@ private: */ class PackedArrayInit { public: - explicit PackedArrayInit(size_t n) : m_vec(ArrayData::Make(n)) {} + explicit PackedArrayInit(size_t n) : m_vec(HphpArray::MakeReserve(n)) { + m_vec->setRefCount(0); + } PackedArrayInit(PackedArrayInit&& other) : m_vec(other.m_vec) { other.m_vec = nullptr; diff --git a/hphp/runtime/base/execution-context.h b/hphp/runtime/base/execution-context.h index 7089ec58c..c5e88549f 100644 --- a/hphp/runtime/base/execution-context.h +++ b/hphp/runtime/base/execution-context.h @@ -540,8 +540,7 @@ public: return it->second; } - auto array = ArrayData::Make(f->numStaticLocals()); - array->incRefCount(); + auto array = HphpArray::MakeReserve(f->numStaticLocals()); return m_funcStaticCtx[f] = array; } diff --git a/hphp/runtime/base/hphp-array-defs.h b/hphp/runtime/base/hphp-array-defs.h index c13b82a29..76c43efa9 100644 --- a/hphp/runtime/base/hphp-array-defs.h +++ b/hphp/runtime/base/hphp-array-defs.h @@ -23,21 +23,6 @@ namespace HPHP { /////////////////////////////////////////////////////////////////////////////// -inline void* ArrayData::modeAlloc(size_t nbytes) const { - return m_allocMode == AllocationMode::smart ? smart_malloc(nbytes) : - Util::safe_malloc(nbytes); -} - -inline void* ArrayData::modeRealloc(void* ptr, size_t nbytes) const { - return m_allocMode == AllocationMode::smart ? smart_realloc(ptr, nbytes) : - Util::safe_realloc(ptr, nbytes); -} - -inline void ArrayData::modeFree(void* ptr) const { - return LIKELY(m_allocMode == AllocationMode::smart) ? smart_free(ptr) : - free(ptr); -} - inline void HphpArray::initHash(size_t tableSize) { assert(HphpArray::Empty == -1); memset(m_hash, 0xffU, tableSize * sizeof(*m_hash)); diff --git a/hphp/runtime/base/hphp-array.cpp b/hphp/runtime/base/hphp-array.cpp index a98143fd1..a35d40d89 100644 --- a/hphp/runtime/base/hphp-array.cpp +++ b/hphp/runtime/base/hphp-array.cpp @@ -17,6 +17,7 @@ #define INLINE_VARIANT_HELPER 1 #include "hphp/runtime/base/hphp-array.h" + #include "hphp/runtime/base/array-init.h" #include "hphp/runtime/base/array-iterator.h" #include "hphp/runtime/base/complex-types.h" @@ -38,19 +39,9 @@ namespace HPHP { -static_assert( - sizeof(HphpArray) == 152, - "Performance is sensitive to sizeof(HphpArray)." - " Make sure you changed it with good reason and then update this assert." -); - -static_assert( - sizeof(ArrayData) == 24, - "ArrayData is expected to take 3 q-words" -); - TRACE_SET_MOD(runtime); -/////////////////////////////////////////////////////////////////////////////// + +////////////////////////////////////////////////////////////////////// /* * Allocation of HphpArray buffers works like this: the smallest buffer @@ -74,19 +65,39 @@ TRACE_SET_MOD(runtime); * separately. Even larger tables allocate the hashtable and slots * contiguously. */ -void* HphpArray::SmaAllocatorInitSetup = SmartAllocatorInitSetup(); //============================================================================= // Static members. std::aligned_storage< - sizeof(HphpArray), + sizeof(HphpArray) + + sizeof(HphpArray::Elm) * HphpArray::SmallSize, + // No need for space for the hash because the empty array is + // kPackedKind. alignof(HphpArray) >::type s_theEmptyArray; struct HphpArray::EmptyArrayInitializer { - EmptyArrayInitializer() { - new (HphpArray::GetStaticEmptyArray()) HphpArray(StaticEmptyArray); + EmptyArrayInitializer() ATTRIBUTE_COLD { + void* vpEmpty = &s_theEmptyArray; + + auto const ad = static_cast(vpEmpty); + ad->m_kind = kPackedKind; + ad->m_allocMode = AllocationMode::smart; + ad->m_size = 0; + ad->m_pos = ArrayData::invalid_index; + ad->m_count = 0; + ad->m_strongIterators = nullptr; + ad->m_used = 0; + ad->m_tableMask = SmallHashSize - 1; + + ad->m_cap = SmallSize; + ad->m_data = reinterpret_cast(ad + 1); + ad->m_hash = nullptr; + + ad->setStatic(); + + assert(ad->checkInvariants()); } }; HphpArray::EmptyArrayInitializer HphpArray::s_arrayInitializer; @@ -94,101 +105,219 @@ HphpArray::EmptyArrayInitializer HphpArray::s_arrayInitializer; //============================================================================= // Helpers. -static inline uint32_t computeMaskFromNumElms(const uint32_t n) { +namespace { + +ALWAYS_INLINE +uint32_t computeMaskFromNumElms(uint32_t n) { assert(n <= 0x7fffffffU); auto lgSize = HphpArray::MinLgTableSize; auto maxElms = HphpArray::SmallSize; assert(lgSize >= 2); + + // Note: it's tempting to convert this loop into something involving + // x64 bsr and a shift. Naive attempts currently actually add more + // branches, because we need to initially check whether `n' is less + // than SmallSize, and after finding the next power of two we need a + // branch to see if it was big enough for the desired load factor. + // This is probably still worth revisiting (e.g., MakeReserve could + // have a precondition that n is at least SmallSize). while (maxElms < n) { ++lgSize; maxElms <<= 1; } assert(lgSize <= 32); + // return 2^lgSize - 1 return ((size_t(1U)) << lgSize) - 1; static_assert(HphpArray::MinLgTableSize >= 2, "lower limit for 0.75 load factor"); } -//============================================================================= -// Construction/destruction. - -HphpArray::HphpArray(uint capacity) - : ArrayData(kPackedKind, AllocationMode::smart, 0) - , m_used(0) { - assert(m_size == 0); - const auto mask = computeMaskFromNumElms(capacity); - m_tableMask = mask; - allocData(computeMaxElms(mask), computeTableSize(mask)); - assert(checkInvariants()); +ALWAYS_INLINE +std::pair computeCapAndMask(uint32_t minimumMaxElms) { + auto const mask = computeMaskFromNumElms(minimumMaxElms); + auto const cap = HphpArray::computeMaxElms(mask); + return std::make_pair(cap, mask); } -HphpArray::HphpArray(uint size, const TypedValue* values) - : ArrayData(kPackedKind, AllocationMode::smart, size) - , m_used(size) { - const auto mask = computeMaskFromNumElms(size); - m_tableMask = mask; - allocData(computeMaxElms(mask), computeTableSize(mask)); - // append values by moving -- Caller assumes we update refcount. Values - // are in reverse order since they come from the stack, which grows down. - Elm* data = m_data; +ALWAYS_INLINE +uint32_t computeAllocBytesPromoted(uint32_t cap, uint32_t mask) { + auto const tabSize = mask + 1; + auto const tabBytes = tabSize * sizeof(int32_t); + auto const dataBytes = cap * sizeof(HphpArray::Elm); + return tabBytes + dataBytes; +} + +ALWAYS_INLINE +uint32_t computeAllocBytesFlat(uint32_t cap, uint32_t mask) { + return sizeof(HphpArray) + computeAllocBytesPromoted(cap, mask); +} + +ALWAYS_INLINE +HphpArray* smartAllocFlat(uint32_t cap, uint32_t mask) { + /* + * Note: we're currently still allocating the memory for the hash + * for a packed array even if we aren't going to use it yet. + */ + auto const allocBytes = computeAllocBytesFlat(cap, mask); + return static_cast(MM().objMalloc(allocBytes)); +} + +ALWAYS_INLINE +HphpArray* mallocFlat(uint32_t cap, uint32_t mask) { + auto const allocBytes = computeAllocBytesFlat(cap, mask); + return static_cast(std::malloc(allocBytes)); +} + +} + +//============================================================================= +// Construction + +HOT_FUNC_VM +HphpArray* HphpArray::MakeReserve(uint32_t capacity) { + auto const cmret = computeCapAndMask(capacity); + auto const cap = cmret.first; + auto const mask = cmret.second; + auto const ad = smartAllocFlat(cap, mask); + + ad->m_kindModeAndSize = static_cast(AllocationMode::smart) << 8 | + kPackedKind; // 0 + ad->m_posAndCount = uint64_t{1} << 32 | + static_cast(ArrayData::invalid_index); + ad->m_strongIterators = nullptr; + ad->m_data = reinterpret_cast(ad + 1); + ad->m_capAndUsed = cap; + ad->m_tableMask = mask; + + assert(ad->m_kind == kPackedKind); + assert(ad->m_allocMode == AllocationMode::smart); + assert(ad->m_size == 0); + assert(ad->m_pos == ArrayData::invalid_index); + assert(ad->m_count == 1); + assert(ad->m_cap == cap); + assert(ad->m_used == 0); + assert(ad->isFlat()); + assert(ad->checkInvariants()); + return ad; +} + +HOT_FUNC_VM +HphpArray* HphpArray::MakeTuple(uint32_t size, const TypedValue* values) { + assert(size > 0); + + auto const cmret = computeCapAndMask(size); + auto const cap = cmret.first; + auto const mask = cmret.second; + auto const ad = smartAllocFlat(cap, mask); + + auto const shiftedSize = uint64_t{size} << 32; + ad->m_kindModeAndSize = shiftedSize | + static_cast(AllocationMode::smart) << 8 | + kPackedKind; + ad->m_posAndCount = uint64_t{1} << 32; + ad->m_strongIterators = nullptr; + ad->m_capAndUsed = shiftedSize | cap; + ad->m_tableMask = mask; + + auto const data = reinterpret_cast(ad + 1); + ad->m_data = data; + + // Append values by moving -- Caller assumes we update refcount. + // Values are in reverse order since they come from the stack, which + // grows down. for (uint32_t i = 0; i < size; i++) { const auto& tv = values[size - i - 1]; data[i].data.m_data = tv.m_data; data[i].data.m_type = tv.m_type; } - assert(size == 0 || m_pos == 0); - assert(checkInvariants()); -} -HphpArray::HphpArray(EmptyMode) - : ArrayData(kPackedKind, AllocationMode::smart, 0) - , m_used(0) - , m_tableMask(SmallHashSize - 1) { - allocData(SmallSize, SmallHashSize); - setStatic(); - assert(checkInvariants()); + assert(ad->m_kind == kPackedKind); + assert(ad->m_allocMode == AllocationMode::smart); + assert(ad->m_size == size); + assert(ad->m_pos == 0); + assert(ad->m_count == 1); + assert(ad->m_cap == cap); + assert(ad->m_used == size); + assert(ad->isFlat()); + assert(ad->checkInvariants()); + return ad; } // for internal use by nonSmartCopy() and copyPacked() ALWAYS_INLINE -HphpArray::HphpArray(const HphpArray& other, AllocationMode mode, ClonePacked) - : ArrayData(other.m_kind, mode, other.m_size) - , m_used(other.m_used) - , m_tableMask(other.m_tableMask) { +HphpArray* HphpArray::CopyPacked(const HphpArray& other, AllocationMode mode) { assert(other.isPacked()); - m_pos = other.m_pos; - allocData(other.m_cap, computeTableSize(m_tableMask)); + + auto const cap = other.m_cap; + auto const mask = other.m_tableMask; + auto const ad = mode == AllocationMode::smart + ? smartAllocFlat(cap, mask) + : mallocFlat(cap, mask); + + ad->m_kindModeAndSize = uint64_t{other.m_size} << 32 | + static_cast(mode) << 8 | + kPackedKind; + ad->m_posAndCount = static_cast(other.m_pos); + ad->m_strongIterators = nullptr; + ad->m_capAndUsed = uint64_t{other.m_used} << 32 | cap; + ad->m_tableMask = mask; + + auto const targetElms = reinterpret_cast(ad + 1); + ad->m_data = targetElms; + // Copy the elements and bump up refcounts as needed. - Elm* elms = other.m_data; - Elm* targetElms = m_data; - for (uint32_t i = 0, limit = m_used; i < limit; ++i) { + auto const elms = other.m_data; + for (uint32_t i = 0, limit = ad->m_used; i < limit; ++i) { tvDupFlattenVars(&elms[i].data, &targetElms[i].data, &other); } - assert(checkInvariants()); + + assert(ad->m_kind == kPackedKind); + assert(ad->m_allocMode == mode); + assert(ad->m_size == other.m_size); + assert(ad->m_pos == other.m_pos); + assert(ad->m_count == 0); + assert(ad->m_used == other.m_used); + assert(ad->m_cap == cap); + assert(ad->m_tableMask == mask); + assert(ad->isFlat()); + assert(ad->checkInvariants()); + return ad; } // For internal use by nonSmartCopy() and copyMixed() ALWAYS_INLINE -HphpArray::HphpArray(const HphpArray& other, AllocationMode mode, CloneMixed) - : ArrayData(other.m_kind, mode, other.m_size) - , m_used(other.m_used) - , m_tableMask(other.m_tableMask) - , m_hLoad(other.m_hLoad) - , m_nextKI(other.m_nextKI) { - assert(!other.isPacked()); - m_pos = other.m_pos; - auto maxElms = other.m_cap; - auto tableSize = computeTableSize(m_tableMask); - m_hash = allocData(maxElms, tableSize); - // Copy the hash. - memcpy(m_hash, other.m_hash, tableSize * sizeof(*m_hash)); +HphpArray* HphpArray::CopyMixed(const HphpArray& other, AllocationMode mode) { + assert(other.m_kind == kMixedKind); + + auto const cap = other.m_cap; + auto const mask = other.m_tableMask; + auto const ad = mode == AllocationMode::smart + ? smartAllocFlat(cap, mask) + : mallocFlat(cap, mask); + + ad->m_kindModeAndSize = uint64_t{other.m_size} << 32 | + static_cast(mode) << 8 | + kMixedKind; + ad->m_posAndCount = static_cast(other.m_pos); + ad->m_strongIterators = nullptr; + ad->m_capAndUsed = uint64_t{other.m_used} << 32 | cap; + ad->m_maskAndLoad = uint64_t{other.m_hLoad} << 32 | mask; + ad->m_nextKI = other.m_nextKI; + + auto const data = reinterpret_cast(ad + 1); + auto const hash = reinterpret_cast(data + cap); + + ad->m_data = data; + ad->m_hash = static_cast( + memcpy(hash, other.m_hash, (mask + 1) * sizeof *hash) + ); + // Copy the elements and bump up refcounts as needed. - auto elms = other.m_data; - auto targetElms = m_data; - for (uint32_t i = 0, limit = m_used; i < limit; ++i) { - const auto& e = elms[i]; - auto& te = targetElms[i]; + auto const elms = other.m_data; + for (uint32_t i = 0, limit = ad->m_used; i < limit; ++i) { + auto const& e = elms[i]; + auto& te = data[i]; if (!isTombstone(e.data.m_type)) { te.key = e.key; te.data.hash() = e.data.hash(); @@ -200,76 +329,146 @@ HphpArray::HphpArray(const HphpArray& other, AllocationMode mode, CloneMixed) te.data.m_type = KindOfInvalid; } } + // If the element density dropped below 50% due to indirect elements // being converted into tombstones, we should do a compaction - if (m_size < m_used / 2) { - compact(false); + if (ad->m_size < ad->m_used / 2) { + ad->compact(false); } + + assert(ad->m_kind == other.m_kind); + assert(ad->m_allocMode == mode); + assert(ad->m_size == other.m_size); + assert(ad->m_pos == other.m_pos); + assert(ad->m_count == 0); + assert(ad->m_used == other.m_used); + assert(ad->m_cap == cap); + assert(ad->m_tableMask == mask); + assert(ad->m_hLoad == other.m_hLoad); + assert(ad->isFlat()); + assert(ad->checkInvariants()); + return ad; +} + +NEVER_INLINE ArrayData* HphpArray::NonSmartCopy(const ArrayData* in) { + auto a = asHphpArray(in); + assert(a->checkInvariants()); + return a->isPacked() + ? CopyPacked(*a, AllocationMode::nonSmart) + : CopyMixed(*a, AllocationMode::nonSmart); +} + +NEVER_INLINE HphpArray* HphpArray::copyPacked() const { assert(checkInvariants()); + return CopyPacked(*this, AllocationMode::smart); } -inline void HphpArray::destroyPacked() { - auto const elms = m_data; - for (uint32_t i = 0, n = m_used; i < n; ++i) { - tvRefcountedDecRef(&elms[i].data); - } - if (elms != m_inline_data.slots) modeFree(elms); +NEVER_INLINE HphpArray* HphpArray::copyMixed() const { + assert(checkInvariants()); + return CopyMixed(*this, AllocationMode::smart); } -inline void HphpArray::destroy() { - auto const elms = m_data; - for (uint32_t i = 0, n = m_used; i < n; ++i) { - auto& e = elms[i]; - if (isTombstone(e.data.m_type)) continue; - if (e.hasStrKey()) decRefStr(e.key); - tvRefcountedDecRef(&e.data); +//============================================================================= +// Destruction + +HOT_FUNC_VM +void HphpArray::ReleasePacked(ArrayData* in) { + auto const ad = asPacked(in); + auto const data = ad->m_data; + + auto const stop = data + ad->m_used; + for (auto ptr = data; ptr != stop; ++ptr) { + tvRefcountedDecRef(ptr->data); } - if (elms != m_inline_data.slots) modeFree(elms); + + ad->ArrayData::destroy(); + + auto const cap = ad->m_cap; + auto const mask = ad->m_tableMask; + + // Calling isFlat() here instead of manually inlining it does a + // re-load from m_data (the compiler doesn't know whether + // ArrayData::destroy modified it). + if (UNLIKELY(data != static_cast(ad + 1))) { + MM().objFree(data, computeAllocBytesPromoted(cap, mask)); + auto& payload = ad->promotedPayload(); + MM().objFree( + ad, + computeAllocBytesFlat(payload.oldCap, payload.oldMask) + ); + return; + } + + MM().objFree(ad, computeAllocBytesFlat(cap, mask)); } HOT_FUNC_VM -void HphpArray::ReleasePacked(ArrayData* ad) { - auto a = asPacked(ad); - a->destroyPacked(); - a->ArrayData::destroy(); - HphpArray::AllocatorType::getNoCheck()->dealloc(a); +void HphpArray::Release(ArrayData* in) { + auto const ad = asMixed(in); + auto const data = ad->m_data; + + auto const stop = data + ad->m_used; + for (auto ptr = data; ptr != stop; ++ptr) { + if (isTombstone(ptr->data.m_type)) continue; + if (ptr->hasStrKey()) decRefStr(ptr->key); + tvRefcountedDecRef(&ptr->data); + } + + ad->ArrayData::destroy(); + + auto const cap = ad->m_cap; + auto const mask = ad->m_tableMask; + + // Calling isFlat() here instead of manually inlining it does a + // re-load from m_data. (The compiler doesn't know whether + // ArrayData::destroy modified it.) + if (UNLIKELY(data != static_cast(ad + 1))) { + MM().objFree(data, computeAllocBytesPromoted(cap, mask)); + + auto& payload = ad->promotedPayload(); + MM().objFree( + ad, + computeAllocBytesFlat(payload.oldCap, payload.oldMask) + ); + + return; + } + + MM().objFree(ad, computeAllocBytesFlat(cap, mask)); } -HOT_FUNC_VM -void HphpArray::Release(ArrayData* ad) { - auto a = asMixed(ad); - a->destroy(); - a->ArrayData::destroy(); - HphpArray::AllocatorType::getNoCheck()->dealloc(a); -} +//============================================================================= +// State transitions NEVER_INLINE HOT_FUNC_VM HphpArray* HphpArray::packedToMixed() { assert(isPacked()); - if (m_data == m_inline_data.slots) { - m_hash = m_inline_data.hash; - } else { - auto dataSize = m_cap * sizeof(*m_data); - auto hashSize = computeTableSize(m_tableMask) * sizeof(*m_hash); - m_hash = hashSize <= sizeof(m_inline_hash) ? m_inline_hash : - (int32_t*)(uintptr_t(m_data) + dataSize); - } + + auto const data = m_data; + auto const hash = reinterpret_cast(data + m_cap); + m_kind = kMixedKind; + m_hash = hash; + + auto const size = m_size; + m_hLoad = size; + m_nextKI = size; + uint32_t i = 0; - auto size = m_size; for (; i < size; ++i) { - m_data[i].setIntKey(i); - m_hash[i] = i; + data[i].setIntKey(i); + hash[i] = i; } for (; i <= m_tableMask; ++i) { - m_hash[i] = Empty; + hash[i] = Empty; } - m_hLoad = size; - m_nextKI = size; + assert(checkInvariants()); return this; } +//============================================================================= + // Invariants: // // m_size <= m_used; m_used <= m_cap @@ -295,7 +494,13 @@ HphpArray* HphpArray::packedToMixed() { // no KindOfInvalid tombstones // bool HphpArray::checkInvariants() const { - static_assert(sizeof(*m_data) == 24, ""); + static_assert(sizeof *m_data == 24, ""); + static_assert(sizeof(ArrayData) == 3 * sizeof(uint64_t), ""); + static_assert( + sizeof(HphpArray) == sizeof(ArrayData) + 5 * sizeof(uint64_t), + "Performance is sensitive to sizeof(HphpArray)." + " Make sure you changed it with good reason and then update this assert." + ); assert(m_size <= m_used); assert(m_used <= m_cap); @@ -699,39 +904,6 @@ HphpArray* HphpArray::moveVal(TypedValue& tv, TypedValue v) { return this; } -int32_t* HphpArray::allocData(size_t maxElms, size_t tableSize) { - m_cap = maxElms; - if (maxElms <= SmallSize) { - m_data = m_inline_data.slots; - return m_inline_data.hash; - } - size_t hashSize = tableSize * sizeof(*m_hash); - size_t dataSize = maxElms * sizeof(*m_data); - size_t allocSize = hashSize <= sizeof(m_inline_hash) ? dataSize : - dataSize + hashSize; - m_data = (Elm*) modeAlloc(allocSize); - return hashSize <= sizeof(m_inline_hash) ? m_inline_hash : - (int32_t*)(uintptr_t(m_data) + dataSize); -} - -int32_t* HphpArray::reallocData(size_t maxElms, size_t tableSize) { - assert(m_data && m_cap > 0 && maxElms > SmallSize); - size_t hashSize = tableSize * sizeof(*m_hash); - size_t dataSize = maxElms * sizeof(*m_data); - size_t allocSize = hashSize <= sizeof(m_inline_hash) ? dataSize : - dataSize + hashSize; - size_t oldDataSize = m_cap * sizeof(*m_data); // slots only. - if (m_data == m_inline_data.slots) { - m_data = (Elm*) modeAlloc(allocSize); - memcpy(m_data, m_inline_data.slots, oldDataSize); - } else { - m_data = (Elm*) modeRealloc(m_data, allocSize); - } - m_cap = maxElms; - return hashSize <= sizeof(m_inline_hash) ? m_inline_hash : - (int32_t*)(uintptr_t(m_data) + dataSize); -} - ALWAYS_INLINE void HphpArray::resizeIfNeeded() { if (isFull()) resize(); } @@ -753,30 +925,80 @@ NEVER_INLINE void HphpArray::resize() { void HphpArray::grow() { assert(!isPacked()); assert(m_tableMask <= 0x7fffffffU); - m_tableMask = 2 * m_tableMask + 1; - auto tableSize = computeTableSize(m_tableMask); - auto maxElms = computeMaxElms(m_tableMask); - m_hash = reallocData(maxElms, tableSize); - // All the elements have been copied and their offsets from the base are - // still the same, so we just need to build the new hash table. - initHash(tableSize); - auto elms = m_data; + + auto const oldMask = m_tableMask; + auto const mask = oldMask * 2 + 1; + auto const cap = computeMaxElms(mask); + auto const allocBytes = computeAllocBytesPromoted(cap, mask); + auto data = static_cast(MM().objMalloc(allocBytes)); + auto const oldData = m_data; + + m_data = data; + m_tableMask = mask; + m_hash = reinterpret_cast(data + cap); + + // Load oldCap down here instead of earlier to keep its lifetime + // small. (This reduces the amount of spills vs doing it up near + // oldMask.) + auto const oldCap = m_cap; + m_cap = cap; + + // Preserve data without using a register. + data = static_cast( + memcpy(data, oldData, oldCap * sizeof(Elm)) + ); + + // If we call isFlat() here it will reload from m_data---use our + // oldData. + if (oldData == static_cast(this + 1)) { + promotedPayload().oldCap = oldCap; + promotedPayload().oldMask = oldMask; + } else { + MM().objFree( + oldData, + computeAllocBytesPromoted(oldCap, oldMask) + ); + } + + initHash(mask + 1); + for (uint32_t i = 0, limit = m_used; i < limit; ++i) { - auto& e = elms[i]; + auto& e = data[i]; if (isTombstone(e.data.m_type)) continue; auto* ei = findForNewInsert(e.hasIntKey() ? e.ikey : e.hash()); *ei = i; } + m_hLoad = m_size; } NEVER_INLINE void HphpArray::growPacked() { assert(isPacked()); - auto maxElms = m_cap * 2; - auto mask = m_tableMask * 2 + 1; + + auto const oldCap = m_cap; + auto const oldMask = m_tableMask; + auto const cap = oldCap * 2; + auto const mask = oldMask * 2 + 1; + auto const allocBytes = computeAllocBytesPromoted(cap, mask); + auto const ptr = MM().objMalloc(allocBytes); + auto const oldData = m_data; + + m_data = static_cast(ptr); m_tableMask = mask; - reallocData(maxElms, computeTableSize(mask)); + m_cap = cap; + + memcpy(ptr, oldData, oldCap * sizeof(Elm)); + + if (oldData == static_cast(this + 1)) { + promotedPayload().oldCap = oldCap; + promotedPayload().oldMask = oldMask; + } else { + MM().objFree( + oldData, + computeAllocBytesPromoted(oldCap, oldMask) + ); + } } void HphpArray::compact(bool renumber /* = false */) { @@ -1708,44 +1930,6 @@ bool HphpArray::AdvanceFullPos(ArrayData* ad, FullPos& fp) { return true; } -//============================================================================= +////////////////////////////////////////////////////////////////////// -NEVER_INLINE ArrayData* HphpArray::NonSmartCopy(const ArrayData* ad) { - auto a = asHphpArray(ad); - return a->isPacked() ? - new HphpArray(*a, AllocationMode::nonSmart, ClonePacked()) : - new HphpArray(*a, AllocationMode::nonSmart, CloneMixed()); -} - -NEVER_INLINE HphpArray* HphpArray::copyPacked() const { - assert(checkInvariants()); - return new (HphpArray::AllocatorType::getNoCheck()->alloc(sizeof(*this))) - HphpArray(*this, AllocationMode::smart, ClonePacked()); -} - -NEVER_INLINE HphpArray* HphpArray::copyMixed() const { - assert(checkInvariants()); - return new (HphpArray::AllocatorType::getNoCheck()->alloc(sizeof(*this))) - HphpArray(*this, AllocationMode::smart, CloneMixed()); -} - -//============================================================================= - -HOT_FUNC_VM -HphpArray* ArrayData::MakeTuple(uint size, const TypedValue* data) { - auto a = HphpArray::Make(size, data); - a->setRefCount(1); - TRACE(2, "MakeTuple: size %d\n", size); - return a; -} - -HOT_FUNC_VM -HphpArray* ArrayData::MakeReserve(uint capacity) { - auto a = HphpArray::Make(capacity); - a->setRefCount(1); - TRACE(2, "MakeReserve: capacity %d\n", capacity); - return a; -} - -/////////////////////////////////////////////////////////////////////////////// } diff --git a/hphp/runtime/base/hphp-array.h b/hphp/runtime/base/hphp-array.h index 6c15e787d..6d7be613a 100644 --- a/hphp/runtime/base/hphp-array.h +++ b/hphp/runtime/base/hphp-array.h @@ -72,15 +72,34 @@ public: } }; - template static HphpArray* Make(Args&&... args) { - return NEW(HphpArray)(std::forward(args)...); - } + /* + * Allocate a new, empty, request-local HphpArray in packed mode, + * with enough space reserved for `capacity' members. + * + * The returned array is already incref'd. + */ + static HphpArray* MakeReserve(uint32_t capacity); + /* + * Allocate a "tuple"-style HphpArray. This is an array in packed + * mode, containing `size' values, in the reverse order of the + * `values' array. + * + * This function takes ownership of the TypedValues in `values'. + * + * The returned array is already incref'd. + * + * Pre: size > 0 + */ + static HphpArray* MakeTuple(uint32_t size, const TypedValue* values); + + /* + * Return a pointer to the singleton static empty array. This is + * used for initial empty arrays (COW will cause it to escalate to a + * request-local array if it is modified). + */ static HphpArray* GetStaticEmptyArray(); - void destroyPacked(); - void destroy(); - // This behaves the same as iter_begin except that it assumes // this array is not empty and its not virtual. ssize_t getIterBegin() const { @@ -91,7 +110,7 @@ public: return nextElm(m_data, 0); } - // these using directives ensure the full set of overloaded functions + // These using directives ensure the full set of overloaded functions // are visible in this class, to avoid triggering implicit conversions // from a CVarRef key to int64. using ArrayData::exists; @@ -102,6 +121,7 @@ public: using ArrayData::add; using ArrayData::remove; using ArrayData::nvGet; + using ArrayData::release; // implements ArrayData static CVarRef GetValueRef(const ArrayData*, ssize_t pos); @@ -291,8 +311,12 @@ private: enum EmptyMode { StaticEmptyArray }; enum SortFlavor { IntegerSort, StringSort, GenericSort }; + struct PromotedPayload { + uint32_t oldCap; + uint32_t oldMask; + }; + private: - DECLARE_SMART_ALLOCATION(HphpArray); static EmptyArrayInitializer s_arrayInitializer; private: @@ -304,20 +328,26 @@ private: static HphpArray* asHphpArray(ArrayData* ad); static const HphpArray* asHphpArray(const ArrayData* ad); + static HphpArray* Make(EmptyMode); static void getElmKey(const Elm& e, TypedValue* out); -private: // Private construction/destruction, use ::Make publically - explicit HphpArray(EmptyMode); - explicit HphpArray(uint nSize); - HphpArray(uint size, const TypedValue* vals); // make tuple - HphpArray(const HphpArray& other, AllocationMode, ClonePacked); - HphpArray(const HphpArray& other, AllocationMode, CloneMixed); +private: + static HphpArray* CopyPacked(const HphpArray& other, AllocationMode); + static HphpArray* CopyMixed(const HphpArray& other, AllocationMode); + + HphpArray() = delete; + HphpArray(const HphpArray&) = delete; + HphpArray& operator=(const HphpArray&) = delete; ~HphpArray() = delete; private: void initHash(size_t tableSize); void initNonEmpty(const HphpArray& other); + PromotedPayload& promotedPayload() { + return *reinterpret_cast(this + 1); + } + template SortFlavor preSort(const AccessorT& acc, bool checkTypes); void postSort(bool resetKeys); @@ -399,10 +429,7 @@ private: HphpArray* initWithRef(TypedValue& tv, CVarRef v); HphpArray* moveVal(TypedValue& tv, TypedValue v); - int32_t* allocData(size_t maxElms, size_t tableSize); - int32_t* reallocData(size_t maxElms, size_t tableSize); - - /** + /* * grow() increases the hash table size and the number of slots for * elements by a factor of 2. grow() rebuilds the hash table, but it * does not compact the elements. @@ -429,6 +456,10 @@ private: void resize(); void resizeIfNeeded(); + bool isFlat() const { + return m_data == static_cast(this + 1); + } + private: // Small: Array elements and the hash table are allocated inline. // @@ -467,24 +498,31 @@ private: // m_hash --> | | 2^K hash table entries. // +--------------------+ - uint32_t m_used; // Number of used elements (values or tombstones) - uint32_t m_cap; // Number of Elms we can use before having to grow. - uint32_t m_tableMask; // Bitmask used when indexing into the hash table. - uint32_t m_hLoad; // Hash table load (# of non-empty slots). - int64_t m_nextKI; // Next integer key to use for append. - Elm* m_data; // Contains elements and hash table. - int32_t* m_hash; // Hash table. + // Some of these are packed into qword-sized unions so we can + // combine stores during initialization. (gcc won't do it on its + // own.) union { struct { - Elm slots[SmallSize]; - int32_t hash[SmallHashSize]; - } m_inline_data; - int32_t m_inline_hash[sizeof(m_inline_data) / sizeof(int32_t)]; + uint32_t m_cap; // Number of Elms we can use before having to grow. + uint32_t m_used; // Number of used elements (values or tombstones) + }; + uint64_t m_capAndUsed; }; + union { + struct { + uint32_t m_tableMask; // Bitmask used when indexing into the hash table. + uint32_t m_hLoad; // Hash table load (# of non-empty slots). + }; + uint64_t m_maskAndLoad; + }; + int64_t m_nextKI; // Next integer key to use for append. + Elm* m_data; // Contains elements and hash table. + int32_t* m_hash; // Hash table. }; extern std::aligned_storage< - sizeof(HphpArray), + sizeof(HphpArray) + + sizeof(HphpArray::Elm) * HphpArray::SmallSize, alignof(HphpArray) >::type s_theEmptyArray; @@ -495,15 +533,6 @@ inline HphpArray* HphpArray::GetStaticEmptyArray() { return static_cast(vp); } -inline HphpArray* ArrayData::Make(uint capacity) { - return HphpArray::Make(capacity); -} - -// HphpArray has more than one kind, so reuse ArrayData's dispatch. -inline void HphpArray::release() { - ArrayData::release(); -} - /////////////////////////////////////////////////////////////////////////////// } diff --git a/hphp/runtime/base/memory-manager-inl.h b/hphp/runtime/base/memory-manager-inl.h index afedc6df5..9547ebfd8 100644 --- a/hphp/runtime/base/memory-manager-inl.h +++ b/hphp/runtime/base/memory-manager-inl.h @@ -35,18 +35,6 @@ inline uint32_t MemoryManager::smartSizeClass(uint32_t reqBytes) { return ret; } -// allocate nbytes from the current slab, aligned to 16-bytes -inline void* MemoryManager::slabAlloc(size_t nbytes) { - const size_t kAlignMask = 15; - assert((nbytes & 7) == 0); - char* ptr = (char*)(uintptr_t(m_front + kAlignMask) & ~kAlignMask); - if (ptr + nbytes <= m_limit) { - m_front = ptr + nbytes; - return ptr; - } - return newSlab(nbytes); -} - inline void* MemoryManager::smartMallocSize(uint32_t bytes) { assert(bytes > 0); assert(bytes <= kMaxSmartSize); diff --git a/hphp/runtime/base/memory-manager.cpp b/hphp/runtime/base/memory-manager.cpp index 6f369dcd6..dc17f41ac 100644 --- a/hphp/runtime/base/memory-manager.cpp +++ b/hphp/runtime/base/memory-manager.cpp @@ -402,6 +402,18 @@ NEVER_INLINE char* MemoryManager::newSlab(size_t nbytes) { return slab; } +// allocate nbytes from the current slab, aligned to 16-bytes +void* MemoryManager::slabAlloc(size_t nbytes) { + const size_t kAlignMask = 15; + assert((nbytes & 7) == 0); + char* ptr = (char*)(uintptr_t(m_front + kAlignMask) & ~kAlignMask); + if (ptr + nbytes <= m_limit) { + m_front = ptr + nbytes; + return ptr; + } + return newSlab(nbytes); +} + NEVER_INLINE void* MemoryManager::smartMallocSlab(size_t padbytes) { SmallNode* n = (SmallNode*) slabAlloc(padbytes); diff --git a/hphp/runtime/base/memory-manager.h b/hphp/runtime/base/memory-manager.h index 96f3d8233..6656fad06 100644 --- a/hphp/runtime/base/memory-manager.h +++ b/hphp/runtime/base/memory-manager.h @@ -355,6 +355,13 @@ public: return smartMallocSizeBig(size); } + ALWAYS_INLINE void objFree(void* vp, size_t size) { + if (LIKELY(size <= kMaxSmartSize)) { + return smartFreeSize(vp, size); + } + return smartFreeSizeBig(vp, size); + } + /* * Allocate/deallocate smart-allocated memory that is too big for * the small size classes. diff --git a/hphp/runtime/base/memory-profile.cpp b/hphp/runtime/base/memory-profile.cpp index fb04a4125..2fe57c1ec 100644 --- a/hphp/runtime/base/memory-profile.cpp +++ b/hphp/runtime/base/memory-profile.cpp @@ -134,9 +134,7 @@ size_t MemoryProfile::getSizeOfArray(ArrayData *arr) { if (maxElms > HphpArray::SmallSize) { size_t hashSize = tableSize * sizeof(int32_t); size_t dataSize = maxElms * sizeof(HphpArray::Elm); - size += (hashSize <= sizeof(ha->m_inline_hash)) - ? dataSize - : dataSize + hashSize; + size += dataSize + hashSize; } } return size; diff --git a/hphp/runtime/base/object-data.cpp b/hphp/runtime/base/object-data.cpp index a761c16a0..ffa0c4d90 100644 --- a/hphp/runtime/base/object-data.cpp +++ b/hphp/runtime/base/object-data.cpp @@ -406,7 +406,8 @@ Array ObjectData::o_toIterArray(CStrRef context, bool getRef /* = false */) { size_t size = m_cls->declPropNumAccessible() + (o_properties.get() ? o_properties.get()->size() : 0); - auto retval = ArrayData::Make(size); + auto retval = HphpArray::MakeReserve(size); + Array returnArray { Array::attach(retval) }; Class* ctx = nullptr; if (!context.empty()) { ctx = Unit::lookupClass(context.get()); @@ -477,7 +478,7 @@ Array ObjectData::o_toIterArray(CStrRef context, } } - return Array(retval); + return returnArray; } static bool decode_invoke(CStrRef s, ObjectData* obj, bool fatal, @@ -892,7 +893,7 @@ Object ObjectData::FromArray(ArrayData* properties) { void ObjectData::initDynProps(int numDynamic /* = 0 */) { // Create o_properties with room for numDynamic - o_properties.asArray() = ArrayData::Make(numDynamic); + o_properties.asArray() = Array::attach(HphpArray::MakeReserve(numDynamic)); } Slot ObjectData::declPropInd(TypedValue* prop) const { diff --git a/hphp/runtime/base/policy-array.cpp b/hphp/runtime/base/policy-array.cpp index fed116cdd..e632dcca4 100644 --- a/hphp/runtime/base/policy-array.cpp +++ b/hphp/runtime/base/policy-array.cpp @@ -512,7 +512,8 @@ bool PolicyArray::AdvanceFullPos(ArrayData* ad, FullPos &fp) { } HphpArray* PolicyArray::toHphpArray() const { - auto result = ArrayData::Make(m_size); + auto result = HphpArray::MakeReserve(m_size); + result->setRefCount(0); FOR_EACH_RANGE (i, 0, m_size) { if (hasStrKey(toPos(i))) { result->add(key(toPos(i)).getStringData(), val(toPos(i)), false); diff --git a/hphp/runtime/base/shared-variant.cpp b/hphp/runtime/base/shared-variant.cpp index 0bbce4e2a..f5c39b54e 100644 --- a/hphp/runtime/base/shared-variant.cpp +++ b/hphp/runtime/base/shared-variant.cpp @@ -143,6 +143,7 @@ StringCase: assert(m_type != KindOfResource); } +// Defined here for inlining into MakeSVSlowPath below. ALWAYS_INLINE void StringData::enlist() { assert(isShared()); auto& head = MemoryManager::TheMemoryManager()->m_strings; @@ -209,6 +210,7 @@ StringData* StringData::Make(SharedVariant* shared) { pdst[len] = 0; auto const mcret = memcpy(pdst, psrc, len); auto const ret = reinterpret_cast(mcret) - 1; + // Recalculating ret from mcret avoids a spill. // Note: this return value thing is doing a dead lea into %rsi in // the caller for some reason. diff --git a/hphp/runtime/base/smart-ptr.h b/hphp/runtime/base/smart-ptr.h index c903f495c..fd7eac174 100644 --- a/hphp/runtime/base/smart-ptr.h +++ b/hphp/runtime/base/smart-ptr.h @@ -42,6 +42,9 @@ public: if (m_px) m_px->incRefCount(); } + enum class NoIncRef {}; + explicit SmartPtr(T* px, NoIncRef) : m_px(px) {} + enum class NonNull { Tag }; explicit SmartPtr(T* px, NonNull) : m_px(px) { assert(px); diff --git a/hphp/runtime/base/string-data.cpp b/hphp/runtime/base/string-data.cpp index a9f6292f4..0f04ea521 100644 --- a/hphp/runtime/base/string-data.cpp +++ b/hphp/runtime/base/string-data.cpp @@ -95,6 +95,7 @@ StringData* StringData::MakeStatic(StringSlice sl) { data[sl.len] = 0; auto const mcret = memcpy(data, sl.ptr, sl.len); auto const ret = reinterpret_cast(mcret) - 1; + // Recalculating ret from mcret avoids a spill. assert(ret->m_hash == 0); assert(ret->m_count == 0); @@ -158,6 +159,7 @@ StringData* StringData::Make(StringSlice sl, CopyStringMode) { data[sl.len] = 0; auto const mcret = memcpy(data, sl.ptr, sl.len); auto const ret = reinterpret_cast(mcret) - 1; + // Recalculating ret from mcret avoids a spill. assert(ret == sd); assert(ret->m_len == sl.len); @@ -192,6 +194,7 @@ StringData* StringData::MakeMalloced(const char* data, int len) { sd->m_data[len] = 0; auto const mcret = memcpy(sd->m_data, data, len); auto const ret = reinterpret_cast(mcret) - 1; + // Recalculating ret from mcret avoids a spill. assert(ret == sd); assert(ret->m_hash == 0); @@ -315,12 +318,16 @@ StringData* StringData::reserve(int cap) { cap += cap >> 2; if (cap > MaxCap) cap = MaxCap; + auto const sd = Make(cap); auto const src = slice(); auto const dst = sd->mutableData(); sd->setSize(src.len); + auto const mcret = memcpy(dst, src.ptr, src.len); auto const ret = static_cast(mcret) - 1; + // Recalculating ret from mcret avoids a spill. + assert(ret == sd); assert(ret->checkSane()); return ret; @@ -334,8 +341,11 @@ StringData* StringData::escalate(uint32_t cap) { auto const src = slice(); auto const dst = sd->mutableData(); sd->setSize(src.len); + auto const mcret = memcpy(dst, src.ptr, src.len); auto const ret = static_cast(mcret) - 1; + // Recalculating ret from mcret avoids a spill. + assert(ret == sd); assert(ret->checkSane()); return ret; diff --git a/hphp/runtime/base/string-data.h b/hphp/runtime/base/string-data.h index e0934dbe9..ab50b20e9 100644 --- a/hphp/runtime/base/string-data.h +++ b/hphp/runtime/base/string-data.h @@ -50,10 +50,10 @@ enum CopyStringMode { CopyString }; /* * Runtime representation of PHP strings. * - * StringData's have several different modes, not all of which we want - * to keep forever. The main mode is Flat, which means StringData - * is a header in a contiguous allocation with the character array for - * the string. + * StringData's have two different modes, not all of which we want to + * keep forever. The main mode is Flat, which means StringData is a + * header in a contiguous allocation with the character array for the + * string. The other is for SharedVariant-backed StringDatas. * * StringDatas can also be allocated in multiple ways. Normally, they * are created through one of the Make overloads, which drops them in diff --git a/hphp/runtime/base/type-array.h b/hphp/runtime/base/type-array.h index f300664d8..6a76fb567 100644 --- a/hphp/runtime/base/type-array.h +++ b/hphp/runtime/base/type-array.h @@ -46,9 +46,13 @@ class ArrayIter; * escalation. */ class Array : protected SmartPtr { - typedef SmartPtr ArrayBase; + typedef SmartPtr ArrayBase; - public: + explicit Array(ArrayData* ad, ArrayBase::NoIncRef) + : ArrayBase(ad, ArrayBase::NoIncRef{}) + {} + +public: /** * Create an empty array or an array with one element. Note these are * different than those copying constructors that also take one value. @@ -57,10 +61,16 @@ class Array : protected SmartPtr { static Array Create(CVarRef value) { return ArrayData::Create(value);} static Array Create(CVarRef key, CVarRef value); - public: +public: Array() {} ~Array(); + static Array attach(ArrayData* ad) { + Array a(ad, SmartPtr::NoIncRef{}); + a.m_px = ad; + return a; + } + ArrayData* get() const { return m_px; } void reset() { ArrayBase::reset(); } diff --git a/hphp/runtime/ext/ext_class.cpp b/hphp/runtime/ext/ext_class.cpp index d2f09bc41..577104675 100644 --- a/hphp/runtime/ext/ext_class.cpp +++ b/hphp/runtime/ext/ext_class.cpp @@ -99,20 +99,22 @@ Array f_get_class_methods(CVarRef class_or_object) { if (!cls) return Array(); VMRegAnchor _; - auto retVal = ArrayData::Make(cls->numMethods()); + auto retVal = HphpArray::MakeReserve(cls->numMethods()); + auto arrayHolder = Array::attach(retVal); cls->getMethodNames(arGetContextClassFromBuiltin(g_vmContext->getFP()), retVal); - return Array(retVal).keys(); + return arrayHolder.keys(); } Array vm_get_class_constants(CStrRef className) { Class* cls = Unit::loadClass(className.get()); + if (cls == NULL) { - return ArrayData::Make(0); + return Array::attach(HphpArray::MakeReserve(0)); } size_t numConstants = cls->numConstants(); - auto retVal = ArrayData::Make(numConstants); + auto retVal = HphpArray::MakeReserve(numConstants); const Class::Const* consts = cls->constants(); for (size_t i = 0; i < numConstants; i++) { // Note: hphpc doesn't include inherited constants in @@ -128,7 +130,7 @@ Array vm_get_class_constants(CStrRef className) { } } - return retVal; + return Array::attach(retVal); } Array f_get_class_constants(CStrRef class_name) { @@ -160,7 +162,7 @@ Array vm_get_class_vars(CStrRef className) { CallerFrame cf; Class* ctx = arGetContextClass(cf()); - auto ret = ArrayData::Make(numDeclProps + numSProps); + auto ret = HphpArray::MakeReserve(numDeclProps + numSProps); for (size_t i = 0; i < numDeclProps; ++i) { StringData* name = const_cast(propInfo[i].m_name); @@ -181,7 +183,7 @@ Array vm_get_class_vars(CStrRef className) { } } - return ret; + return Array::attach(ret); } Array f_get_class_vars(CStrRef class_name) { diff --git a/hphp/runtime/ext/ext_closure.cpp b/hphp/runtime/ext/ext_closure.cpp index f63a3fd9f..54e31ffb5 100644 --- a/hphp/runtime/ext/ext_closure.cpp +++ b/hphp/runtime/ext/ext_closure.cpp @@ -75,7 +75,10 @@ c_Closure* c_Closure::Clone(ObjectData* obj) { HphpArray* c_Closure::getStaticLocals() { if (m_VMStatics.get() == NULL) { - m_VMStatics = ArrayData::Make(1); + m_VMStatics = SmartPtr( + HphpArray::MakeReserve(1), + SmartPtr::NoIncRef{} + ); } return m_VMStatics.get(); } diff --git a/hphp/runtime/ext/ext_function.cpp b/hphp/runtime/ext/ext_function.cpp index e42046fb1..5a55b64f7 100644 --- a/hphp/runtime/ext/ext_function.cpp +++ b/hphp/runtime/ext/ext_function.cpp @@ -332,7 +332,7 @@ Array hhvm_get_frame_args(const ActRec* ar) { } int numParams = ar->m_func->numParams(); int numArgs = ar->numArgs(); - auto retval = ArrayData::Make(numArgs); + auto retval = HphpArray::MakeReserve(numArgs); TypedValue* local = (TypedValue*)(uintptr_t(ar) - sizeof(TypedValue)); for (int i = 0; i < numArgs; ++i) { @@ -347,7 +347,7 @@ Array hhvm_get_frame_args(const ActRec* ar) { } } - return Array(retval); + return Array::attach(retval); } Variant f_func_get_args() { diff --git a/hphp/runtime/ext_zend_compat/php-src/Zend/zend_alloc.h b/hphp/runtime/ext_zend_compat/php-src/Zend/zend_alloc.h index 7446018bd..42ecc3b12 100644 --- a/hphp/runtime/ext_zend_compat/php-src/Zend/zend_alloc.h +++ b/hphp/runtime/ext_zend_compat/php-src/Zend/zend_alloc.h @@ -148,8 +148,10 @@ END_EXTERN_C() efree_rel(z) /* fast cache for HashTables */ -#define ALLOC_HASHTABLE(ht) \ - (ht) = HashTable::Make(0); +#define ALLOC_HASHTABLE(ht) \ + (ht) = [&]{ auto ret = HPHP::HphpArray::MakeReserve(0); \ + ret->setRefCount(0); \ + return ret; }() #define FREE_HASHTABLE(ht) diff --git a/hphp/runtime/ext_zend_compat/php-src/Zend/zend_hash.h b/hphp/runtime/ext_zend_compat/php-src/Zend/zend_hash.h index 466865d8c..9b9c969b5 100644 --- a/hphp/runtime/ext_zend_compat/php-src/Zend/zend_hash.h +++ b/hphp/runtime/ext_zend_compat/php-src/Zend/zend_hash.h @@ -26,6 +26,7 @@ #include "zend.h" #include "hphp/runtime/base/string-data.h" +#include "hphp/runtime/base/hphp-array.h" #define HASH_KEY_IS_STRING 1 #define HASH_KEY_IS_LONG 2 diff --git a/hphp/runtime/vm/bytecode.cpp b/hphp/runtime/vm/bytecode.cpp index 080ca41b4..7a5c174ce 100644 --- a/hphp/runtime/vm/bytecode.cpp +++ b/hphp/runtime/vm/bytecode.cpp @@ -1338,7 +1338,6 @@ void VMExecutionContext::shuffleMagicArgs(ActRec* ar) { // We need to make an array containing all the arguments passed by the // caller and put it where the second argument is ArrayData* argArray = pack_args_into_array(ar, nargs); - argArray->incRefCount(); // Remove the arguments from the stack for (int i = 0; i < nargs; ++i) { m_stack.popC(); @@ -3419,7 +3418,7 @@ OPTBLD_INLINE void VMExecutionContext::iopArray(PC& pc) { OPTBLD_INLINE void VMExecutionContext::iopNewArray(PC& pc) { NEXT(); - auto arr = ArrayData::MakeReserve(HphpArray::SmallSize); + auto arr = HphpArray::MakeReserve(HphpArray::SmallSize); m_stack.pushArrayNoRc(arr); } diff --git a/hphp/runtime/vm/class.cpp b/hphp/runtime/vm/class.cpp index e65e76c93..f1e544a3c 100644 --- a/hphp/runtime/vm/class.cpp +++ b/hphp/runtime/vm/class.cpp @@ -456,8 +456,8 @@ Class::PropInitVec* Class::initPropsImpl() const { { Array args; - HphpArray* propArr = ArrayData::Make(nProps); - Variant arg0(propArr); + HphpArray* propArr = HphpArray::MakeReserve(nProps); + Variant arg0(Array::attach(propArr)); args.appendRef(arg0); assert(propArr->getCount() == 1); // Don't want to trigger COW @@ -652,8 +652,8 @@ TypedValue* Class::initSPropsImpl() const { // They'll put their initialized values into an array, and we'll read any // values we need out of the array later. if (hasNonscalarInit) { - HphpArray* propData = ArrayData::Make(m_staticProperties.size()); - Variant arg0(propData); + HphpArray* propData = HphpArray::MakeReserve(m_staticProperties.size()); + Variant arg0(Array::attach(propData)); // The 86sinit functions will initialize some subset of the static props. // Set all of them to a sentinel object so we can distinguish these. @@ -792,8 +792,7 @@ TypedValue Class::getStaticPropInitVal(const SProp& prop) { HphpArray* Class::initClsCnsData() const { Slot nConstants = m_constants.size(); - HphpArray* constants = ArrayData::Make(nConstants); - constants->incRefCount(); + HphpArray* constants = HphpArray::MakeReserve(nConstants); if (m_parent.get() != nullptr) { if (g_vmContext->getClsCnsData(m_parent.get()) == nullptr) { diff --git a/hphp/runtime/vm/event-hook.cpp b/hphp/runtime/vm/event-hook.cpp index a14b192dd..977497107 100644 --- a/hphp/runtime/vm/event-hook.cpp +++ b/hphp/runtime/vm/event-hook.cpp @@ -109,7 +109,7 @@ void EventHook::RunUserProfiler(const ActRec* ar, int mode) { static Array get_frame_args_with_ref(const ActRec* ar) { int numParams = ar->m_func->numParams(); int numArgs = ar->numArgs(); - HphpArray* retval = ArrayData::Make(numArgs); + HphpArray* retval = HphpArray::MakeReserve(numArgs); TypedValue* local = (TypedValue*)(uintptr_t(ar) - sizeof(TypedValue)); for (int i = 0; i < numArgs; ++i) { @@ -124,7 +124,7 @@ static Array get_frame_args_with_ref(const ActRec* ar) { } } - return Array(retval); + return Array::attach(retval); } bool EventHook::RunInterceptHandler(ActRec* ar) { diff --git a/hphp/runtime/vm/jit/native-calls.cpp b/hphp/runtime/vm/jit/native-calls.cpp index 8c5ec47bc..4874e06cd 100644 --- a/hphp/runtime/vm/jit/native-calls.cpp +++ b/hphp/runtime/vm/jit/native-calls.cpp @@ -173,8 +173,8 @@ static CallMap s_callMap { {{SSA, 0}, {TV, 1}}}, {ArrayAdd, array_add, DSSA, SNone, {{SSA, 0}, {SSA, 1}}}, {Box, box_value, DSSA, SNone, {{TV, 0}}}, - {NewArray, ArrayData::MakeReserve, DSSA, SNone, {{SSA, 0}}}, - {NewTuple, ArrayData::MakeTuple, DSSA, SNone, + {NewArray, HphpArray::MakeReserve, DSSA, SNone, {{SSA, 0}}}, + {NewTuple, HphpArray::MakeTuple, DSSA, SNone, {{SSA, 0}, {SSA, 1}}}, {AllocObj, newInstance, DSSA, SSync, {{SSA, 0}}}, diff --git a/hphp/runtime/vm/jit/translator-x64.cpp b/hphp/runtime/vm/jit/translator-x64.cpp index 89452ee58..5735b122c 100644 --- a/hphp/runtime/vm/jit/translator-x64.cpp +++ b/hphp/runtime/vm/jit/translator-x64.cpp @@ -622,8 +622,7 @@ TranslatorX64::shuffleArgsForMagicCall(ActRec* ar) { int nargs = ar->numArgs(); // We need to make an array containing all the arguments passed by the // caller and put it where the second argument is - HphpArray* argArray = ArrayData::Make(nargs); - argArray->incRefCount(); + HphpArray* argArray = HphpArray::MakeReserve(nargs); for (int i = 0; i < nargs; ++i) { TypedValue* tv = (TypedValue*)(uintptr_t(ar) - (i+1) * sizeof(TypedValue)); diff --git a/hphp/runtime/vm/runtime.cpp b/hphp/runtime/vm/runtime.cpp index 8a736ba04..904f107c1 100644 --- a/hphp/runtime/vm/runtime.cpp +++ b/hphp/runtime/vm/runtime.cpp @@ -308,9 +308,9 @@ Unit* compile_string(const char* s, size_t sz, const char* fname) { return g_hphp_compiler_parse(s, sz, md5, fname); } -// Returned array has refcount zero! Caller must refcount. +// Returned array has refcount one! Caller must not incref. HphpArray* pack_args_into_array(ActRec* ar, int nargs) { - HphpArray* argArray = ArrayData::Make(nargs); + HphpArray* argArray = HphpArray::MakeReserve(nargs); for (int i = 0; i < nargs; ++i) { TypedValue* tv = (TypedValue*)(ar) - (i+1); argArray->HphpArray::appendWithRef(tvAsCVarRef(tv), false); @@ -320,7 +320,7 @@ HphpArray* pack_args_into_array(ActRec* ar, int nargs) { return argArray; } // This is a magic call, so we need to shuffle the args - HphpArray* magicArgs = ArrayData::Make(2); + HphpArray* magicArgs = HphpArray::MakeReserve(2); magicArgs->append(ar->getInvName(), false); magicArgs->append(argArray, false); return magicArgs; diff --git a/hphp/runtime/vm/unit.cpp b/hphp/runtime/vm/unit.cpp index 914371b89..dbb2c5e3a 100644 --- a/hphp/runtime/vm/unit.cpp +++ b/hphp/runtime/vm/unit.cpp @@ -989,8 +989,7 @@ bool Unit::defCns(const StringData* cnsName, const TypedValue* value, * static string. Not worth presizing or otherwise * optimizing for. */ - TargetCache::s_constants = ArrayData::Make(1); - TargetCache::s_constants->incRefCount(); + TargetCache::s_constants = HphpArray::MakeReserve(1); } if (TargetCache::s_constants->nvInsert( const_cast(cnsName), const_cast(value))) {