From 1efb5915a0241351856879a3e0d76c4dd27a345a Mon Sep 17 00:00:00 2001 From: Sara Golemon Date: Wed, 15 May 2013 13:36:36 -0700 Subject: [PATCH] Update folly --- hphp/third_party/folly/folly/Arena-inl.h | 7 + hphp/third_party/folly/folly/Arena.h | 13 +- .../folly/folly/AtomicHashArray-inl.h | 54 ++-- .../third_party/folly/folly/AtomicHashArray.h | 19 +- .../folly/folly/AtomicHashMap-inl.h | 86 +++---- hphp/third_party/folly/folly/AtomicHashMap.h | 12 +- hphp/third_party/folly/folly/Bits.h | 51 ++-- hphp/third_party/folly/folly/Conv.h | 56 ++++- hphp/third_party/folly/folly/Exception.h | 67 +++-- hphp/third_party/folly/folly/FBString.h | 7 +- hphp/third_party/folly/folly/FBVector.h | 4 +- hphp/third_party/folly/folly/File.cpp | 46 +++- hphp/third_party/folly/folly/File.h | 21 ++ hphp/third_party/folly/folly/Format-inl.h | 8 +- hphp/third_party/folly/folly/FormatArg.h | 7 +- hphp/third_party/folly/folly/Malloc.h | 2 +- hphp/third_party/folly/folly/Memory.h | 57 +++++ .../third_party/folly/folly/MemoryMapping.cpp | 2 +- hphp/third_party/folly/folly/Optional.h | 87 ++++--- hphp/third_party/folly/folly/Portability.h | 63 +++-- hphp/third_party/folly/folly/Range.cpp | 2 +- hphp/third_party/folly/folly/Range.h | 89 ++++--- hphp/third_party/folly/folly/ScopeGuard.h | 2 +- hphp/third_party/folly/folly/String-inl.h | 136 ++++++++++ hphp/third_party/folly/folly/String.cpp | 3 +- hphp/third_party/folly/folly/String.h | 79 ++++++ hphp/third_party/folly/folly/Subprocess.cpp | 237 ++++++++++++++---- hphp/third_party/folly/folly/Subprocess.h | 61 ++++- hphp/third_party/folly/folly/ThreadLocal.h | 41 ++- hphp/third_party/folly/folly/Traits.h | 117 ++++++++- hphp/third_party/folly/folly/detail/Stats.h | 31 +++ .../folly/folly/detail/ThreadLocalDetail.h | 104 ++++---- hphp/third_party/folly/folly/dynamic-inl.h | 1 + hphp/third_party/folly/folly/dynamic.cpp | 4 + hphp/third_party/folly/folly/dynamic.h | 5 + .../folly/experimental/EliasFanoCoding.h | 3 +- .../folly/folly/experimental/Gen-inl.h | 179 ++++++++++++- .../folly/folly/experimental/Gen.h | 17 +- .../folly/folly/experimental/TestUtil.cpp | 104 ++++---- .../folly/folly/experimental/TestUtil.h | 46 +++- .../exception_tracer/ExceptionTracer.cpp | 1 + .../exception_tracer/ExceptionTracer.h | 3 +- .../exception_tracer/ExceptionTracerLib.cpp | 11 +- .../exception_tracer/ExceptionTracerTest.cpp | 1 + .../folly/experimental/symbolizer/Elf.cpp | 5 +- hphp/third_party/folly/folly/io/Cursor.h | 70 ++++++ .../folly/stats/BucketedTimeSeries-defs.h | 28 ++- .../folly/folly/stats/BucketedTimeSeries.h | 60 +++-- 48 files changed, 1633 insertions(+), 476 deletions(-) diff --git a/hphp/third_party/folly/folly/Arena-inl.h b/hphp/third_party/folly/folly/Arena-inl.h index 719a35ec9..d97ea6973 100644 --- a/hphp/third_party/folly/folly/Arena-inl.h +++ b/hphp/third_party/folly/folly/Arena-inl.h @@ -45,6 +45,13 @@ template void* Arena::allocateSlow(size_t size) { std::pair p; char* start; + + + size_t allocSize = std::max(size, minBlockSize()) + sizeof(Block); + if(sizeLimit_ && allocSize > sizeLimit_ - totalAllocatedSize_) { + throw std::bad_alloc(); + } + if (size > minBlockSize()) { // Allocate a large block for this chunk only, put it at the back of the // list so it doesn't get used for small allocations; don't change ptr_ diff --git a/hphp/third_party/folly/folly/Arena.h b/hphp/third_party/folly/folly/Arena.h index bb43f45e7..a8d8d7f80 100644 --- a/hphp/third_party/folly/folly/Arena.h +++ b/hphp/third_party/folly/folly/Arena.h @@ -60,12 +60,14 @@ template class Arena { public: explicit Arena(const Alloc& alloc, - size_t minBlockSize = kDefaultMinBlockSize) + size_t minBlockSize = kDefaultMinBlockSize, + size_t sizeLimit = 0) : allocAndSize_(alloc, minBlockSize) , ptr_(nullptr) , end_(nullptr) , totalAllocatedSize_(0) - , bytesUsed_(0) { + , bytesUsed_(0) + , sizeLimit_(sizeLimit) { } ~Arena(); @@ -192,6 +194,7 @@ class Arena { char* end_; size_t totalAllocatedSize_; size_t bytesUsed_; + size_t sizeLimit_; }; /** @@ -231,8 +234,10 @@ struct ArenaAllocatorTraits { */ class SysArena : public Arena { public: - explicit SysArena(size_t minBlockSize = kDefaultMinBlockSize) - : Arena(SysAlloc(), minBlockSize) { + explicit SysArena( + size_t minBlockSize = kDefaultMinBlockSize, + size_t sizeLimit = 0) + : Arena(SysAlloc(), minBlockSize, sizeLimit) { } }; diff --git a/hphp/third_party/folly/folly/AtomicHashArray-inl.h b/hphp/third_party/folly/folly/AtomicHashArray-inl.h index afb30e93d..236c7c223 100644 --- a/hphp/third_party/folly/folly/AtomicHashArray-inl.h +++ b/hphp/third_party/folly/folly/AtomicHashArray-inl.h @@ -24,8 +24,8 @@ namespace folly { // AtomicHashArray private constructor -- -template -AtomicHashArray:: +template +AtomicHashArray:: AtomicHashArray(size_t capacity, KeyT emptyKey, KeyT lockedKey, KeyT erasedKey, double maxLoadFactor, size_t cacheSize) : capacity_(capacity), maxEntries_(size_t(maxLoadFactor * capacity_ + 0.5)), @@ -41,9 +41,9 @@ AtomicHashArray(size_t capacity, KeyT emptyKey, KeyT lockedKey, * of key and returns true, or if key does not exist returns false and * ret.index is set to capacity_. */ -template -typename AtomicHashArray::SimpleRetT -AtomicHashArray:: +template +typename AtomicHashArray::SimpleRetT +AtomicHashArray:: findInternal(const KeyT key_in) { DCHECK_NE(key_in, kEmptyKey_); DCHECK_NE(key_in, kLockedKey_); @@ -52,7 +52,7 @@ findInternal(const KeyT key_in) { ; idx = probeNext(idx, numProbes)) { const KeyT key = acquireLoadKey(cells_[idx]); - if (LIKELY(key == key_in)) { + if (LIKELY(EqualFcn()(key, key_in))) { return SimpleRetT(idx, true); } if (UNLIKELY(key == kEmptyKey_)) { @@ -77,10 +77,10 @@ findInternal(const KeyT key_in) { * this will be the previously inserted value, and if the map is full it is * default. */ -template +template template -typename AtomicHashArray::SimpleRetT -AtomicHashArray:: +typename AtomicHashArray::SimpleRetT +AtomicHashArray:: insertInternal(KeyT key_in, T&& value) { const short NO_NEW_INSERTS = 1; const short NO_PENDING_INSERTS = 2; @@ -140,6 +140,8 @@ insertInternal(KeyT key_in, T&& value) { --numPendingEntries_; throw; } + // Direct comparison rather than EqualFcn ok here + // (we just inserted it) DCHECK(relaxedLoadKey(*cell) == key_in); --numPendingEntries_; ++numEntries_; // This is a thread cached atomic increment :) @@ -159,7 +161,7 @@ insertInternal(KeyT key_in, T&& value) { } const KeyT thisKey = acquireLoadKey(*cell); - if (thisKey == key_in) { + if (EqualFcn()(thisKey, key_in)) { // Found an existing entry for our key, but we don't overwrite the // previous value. return SimpleRetT(idx, false); @@ -191,8 +193,8 @@ insertInternal(KeyT key_in, T&& value) { * erased key will never be reused. If there's an associated value, we won't * touch it either. */ -template -size_t AtomicHashArray:: +template +size_t AtomicHashArray:: erase(KeyT key_in) { CHECK_NE(key_in, kEmptyKey_); CHECK_NE(key_in, kLockedKey_); @@ -208,10 +210,10 @@ erase(KeyT key_in) { // is similar to how it's handled in find(). return 0; } - if (key_in == currentKey) { + if (EqualFcn()(currentKey, key_in)) { // Found an existing entry for our key, attempt to mark it erased. // Some other thread may have erased our key, but this is ok. - KeyT expect = key_in; + KeyT expect = currentKey; if (cellKeyPtr(*cell)->compare_exchange_strong(expect, kErasedKey_)) { numErases_.fetch_add(1, std::memory_order_relaxed); @@ -234,13 +236,13 @@ erase(KeyT key_in) { } } -template -const typename AtomicHashArray::Config -AtomicHashArray::defaultConfig; +template +const typename AtomicHashArray::Config +AtomicHashArray::defaultConfig; -template -typename AtomicHashArray::SmartPtr -AtomicHashArray:: +template +typename AtomicHashArray::SmartPtr +AtomicHashArray:: create(size_t maxSize, const Config& c) { CHECK_LE(c.maxLoadFactor, 1.0); CHECK_GT(c.maxLoadFactor, 0.0); @@ -272,8 +274,8 @@ create(size_t maxSize, const Config& c) { return map; } -template -void AtomicHashArray:: +template +void AtomicHashArray:: destroy(AtomicHashArray* p) { assert(p); FOR_EACH_RANGE(i, 0, p->capacity_) { @@ -286,8 +288,8 @@ destroy(AtomicHashArray* p) { } // clear -- clears all keys and values in the map and resets all counters -template -void AtomicHashArray:: +template +void AtomicHashArray:: clear() { FOR_EACH_RANGE(i, 0, capacity_) { if (cells_[i].first != kEmptyKey_) { @@ -305,9 +307,9 @@ clear() { // Iterator implementation -template +template template -struct AtomicHashArray::aha_iterator +struct AtomicHashArray::aha_iterator : boost::iterator_facade, IterVal, boost::forward_traversal_tag> diff --git a/hphp/third_party/folly/folly/AtomicHashArray.h b/hphp/third_party/folly/folly/AtomicHashArray.h index 56bee6a26..e6b6c7213 100644 --- a/hphp/third_party/folly/folly/AtomicHashArray.h +++ b/hphp/third_party/folly/folly/AtomicHashArray.h @@ -42,13 +42,16 @@ namespace folly { -template > +template , class EqualFcn = std::equal_to> class AtomicHashMap; -template > +template , class EqualFcn = std::equal_to> class AtomicHashArray : boost::noncopyable { static_assert((std::is_convertible::value || - std::is_convertible::value), + std::is_convertible::value || + std::is_convertible::value), "You are trying to use AtomicHashArray with disallowed key " "types. You must use atomically compare-and-swappable integer " "keys, or a different container class."); @@ -117,13 +120,15 @@ class AtomicHashArray : boost::noncopyable { KeyT lockedKey; KeyT erasedKey; double maxLoadFactor; + double growthFactor; int entryCountThreadCacheSize; size_t capacity; // if positive, overrides maxLoadFactor - constexpr Config() : emptyKey(static_cast(-1ul)), - lockedKey(static_cast(-2ul)), - erasedKey(static_cast(-3ul)), + constexpr Config() : emptyKey((KeyT)-1), + lockedKey((KeyT)-2), + erasedKey((KeyT)-3), maxLoadFactor(0.8), + growthFactor(-1), entryCountThreadCacheSize(1000), capacity(0) {} }; @@ -210,7 +215,7 @@ class AtomicHashArray : boost::noncopyable { /* Private data and helper functions... */ private: - friend class AtomicHashMap; + friend class AtomicHashMap; struct SimpleRetT { size_t idx; bool success; SimpleRetT(size_t i, bool s) : idx(i), success(s) {} diff --git a/hphp/third_party/folly/folly/AtomicHashMap-inl.h b/hphp/third_party/folly/folly/AtomicHashMap-inl.h index e109084dd..1548b7f06 100644 --- a/hphp/third_party/folly/folly/AtomicHashMap-inl.h +++ b/hphp/third_party/folly/folly/AtomicHashMap-inl.h @@ -22,16 +22,17 @@ namespace folly { -template -const typename AtomicHashMap::Config -AtomicHashMap::defaultConfig; +template +const typename AtomicHashMap::Config +AtomicHashMap::defaultConfig; // AtomicHashMap constructor -- Atomic wrapper that allows growth // This class has a lot of overhead (184 Bytes) so only use for big maps -template -AtomicHashMap:: +template +AtomicHashMap:: AtomicHashMap(size_t size, const Config& config) - : kGrowthFrac_(1.0 - config.maxLoadFactor) { + : kGrowthFrac_(config.growthFactor < 0 ? + 1.0 - config.maxLoadFactor : config.growthFactor) { CHECK(config.maxLoadFactor > 0.0 && config.maxLoadFactor < 1.0); subMaps_[0].store(SubMap::create(size, config).release(), std::memory_order_relaxed); @@ -43,9 +44,9 @@ AtomicHashMap(size_t size, const Config& config) } // insert -- -template -std::pair::iterator,bool> -AtomicHashMap:: +template +std::pair::iterator,bool> +AtomicHashMap:: insert(key_type k, const mapped_type& v) { SimpleRetT ret = insertInternal(k,v); SubMap* subMap = subMaps_[ret.i].load(std::memory_order_relaxed); @@ -53,9 +54,9 @@ insert(key_type k, const mapped_type& v) { ret.success); } -template -std::pair::iterator,bool> -AtomicHashMap:: +template +std::pair::iterator,bool> +AtomicHashMap:: insert(key_type k, mapped_type&& v) { SimpleRetT ret = insertInternal(k, std::move(v)); SubMap* subMap = subMaps_[ret.i].load(std::memory_order_relaxed); @@ -64,15 +65,14 @@ insert(key_type k, mapped_type&& v) { } // insertInternal -- Allocates new sub maps as existing ones fill up. -template +template template -typename AtomicHashMap::SimpleRetT -AtomicHashMap:: +typename AtomicHashMap::SimpleRetT +AtomicHashMap:: insertInternal(key_type key, T&& value) { beginInsertInternal: int nextMapIdx = // this maintains our state numMapsAllocated_.load(std::memory_order_acquire); - uint32_t idx = 0; typename SubMap::SimpleRetT ret; FOR_EACH_RANGE(i, 0, nextMapIdx) { // insert in each map successively. If one succeeds, we're done! @@ -142,9 +142,9 @@ insertInternal(key_type key, T&& value) { } // find -- -template -typename AtomicHashMap::iterator -AtomicHashMap:: +template +typename AtomicHashMap::iterator +AtomicHashMap:: find(KeyT k) { SimpleRetT ret = findInternal(k); if (ret.i >= numMapsAllocated_.load(std::memory_order_acquire)) { @@ -154,17 +154,17 @@ find(KeyT k) { return iterator(this, ret.i, subMap->makeIter(ret.j)); } -template -typename AtomicHashMap::const_iterator -AtomicHashMap:: +template +typename AtomicHashMap::const_iterator +AtomicHashMap:: find(KeyT k) const { return const_cast(this)->find(k); } // findInternal -- -template -typename AtomicHashMap::SimpleRetT -AtomicHashMap:: +template +typename AtomicHashMap::SimpleRetT +AtomicHashMap:: findInternal(const KeyT k) const { SubMap* const primaryMap = subMaps_[0].load(std::memory_order_relaxed); typename SubMap::SimpleRetT ret = primaryMap->findInternal(k); @@ -185,9 +185,9 @@ findInternal(const KeyT k) const { } // findAtInternal -- see encodeIndex() for details. -template -typename AtomicHashMap::SimpleRetT -AtomicHashMap:: +template +typename AtomicHashMap::SimpleRetT +AtomicHashMap:: findAtInternal(uint32_t idx) const { uint32_t subMapIdx, subMapOffset; if (idx & kSecondaryMapBit_) { @@ -205,9 +205,9 @@ findAtInternal(uint32_t idx) const { } // erase -- -template -typename AtomicHashMap::size_type -AtomicHashMap:: +template +typename AtomicHashMap::size_type +AtomicHashMap:: erase(const KeyT k) { int const numMaps = numMapsAllocated_.load(std::memory_order_acquire); FOR_EACH_RANGE(i, 0, numMaps) { @@ -221,8 +221,8 @@ erase(const KeyT k) { } // capacity -- summation of capacities of all submaps -template -size_t AtomicHashMap:: +template +size_t AtomicHashMap:: capacity() const { size_t totalCap(0); int const numMaps = numMapsAllocated_.load(std::memory_order_acquire); @@ -234,8 +234,8 @@ capacity() const { // spaceRemaining -- // number of new insertions until current submaps are all at max load -template -size_t AtomicHashMap:: +template +size_t AtomicHashMap:: spaceRemaining() const { size_t spaceRem(0); int const numMaps = numMapsAllocated_.load(std::memory_order_acquire); @@ -251,8 +251,8 @@ spaceRemaining() const { // clear -- Wipes all keys and values from primary map and destroys // all secondary maps. Not thread safe. -template -void AtomicHashMap:: +template +void AtomicHashMap:: clear() { subMaps_[0].load(std::memory_order_relaxed)->clear(); int const numMaps = numMapsAllocated_ @@ -267,8 +267,8 @@ clear() { } // size -- -template -size_t AtomicHashMap:: +template +size_t AtomicHashMap:: size() const { size_t totalSize(0); int const numMaps = numMapsAllocated_.load(std::memory_order_acquire); @@ -296,8 +296,8 @@ size() const { // 31 1 // 27-30 which subMap // 0-26 subMap offset (index_ret input) -template -inline uint32_t AtomicHashMap:: +template +inline uint32_t AtomicHashMap:: encodeIndex(uint32_t subMap, uint32_t offset) { DCHECK_EQ(offset & kSecondaryMapBit_, 0); // offset can't be too big if (subMap == 0) return offset; @@ -313,9 +313,9 @@ encodeIndex(uint32_t subMap, uint32_t offset) { // Iterator implementation -template +template template -struct AtomicHashMap::ahm_iterator +struct AtomicHashMap::ahm_iterator : boost::iterator_facade, IterVal, boost::forward_traversal_tag> diff --git a/hphp/third_party/folly/folly/AtomicHashMap.h b/hphp/third_party/folly/folly/AtomicHashMap.h index 5de4f5035..2738f3997 100644 --- a/hphp/third_party/folly/folly/AtomicHashMap.h +++ b/hphp/third_party/folly/folly/AtomicHashMap.h @@ -143,10 +143,6 @@ namespace folly { * - We don't take the Hash function object as an instance in the * constructor. * - * - We don't take a Compare template parameter (since our keys must - * be integers, and the underlying hash array here uses atomic - * compare-and-swap instructions, we only allow normal integer - * comparisons). */ // Thrown when insertion fails due to running out of space for @@ -157,16 +153,16 @@ struct AtomicHashMapFullError : std::runtime_error { {} }; -template +template class AtomicHashMap : boost::noncopyable { - typedef AtomicHashArray SubMap; + typedef AtomicHashArray SubMap; public: typedef KeyT key_type; typedef ValueT mapped_type; typedef std::pair value_type; typedef HashFcn hasher; - typedef std::equal_to key_equal; + typedef EqualFcn key_equal; typedef value_type* pointer; typedef value_type& reference; typedef const value_type& const_reference; @@ -204,7 +200,7 @@ class AtomicHashMap : boost::noncopyable { } } - key_equal key_eq() const { return key_eq(); } + key_equal key_eq() const { return key_equal(); } hasher hash_function() const { return hasher(); } // TODO: emplace() support would be nice. diff --git a/hphp/third_party/folly/folly/Bits.h b/hphp/third_party/folly/folly/Bits.h index b206968bd..83c61708e 100644 --- a/hphp/third_party/folly/folly/Bits.h +++ b/hphp/third_party/folly/folly/Bits.h @@ -65,14 +65,17 @@ #error GCC required #endif +#include "folly/folly-config.h" #include "folly/detail/BitsDetail.h" #include "folly/detail/BitIteratorDetail.h" #include "folly/Likely.h" -#include +#if FOLLY_HAVE_BYTESWAP_H +# include +#endif + #include #include -#include #include #include #include @@ -230,23 +233,41 @@ struct EndianIntBase { static T swap(T x); }; +/** + * If we have the bswap_16 macro from byteswap.h, use it; otherwise, provide our + * own definition. + */ +#ifdef bswap_16 +# define our_bswap16 bswap_16 +#else + +template +inline constexpr typename std::enable_if< + sizeof(Int16) == 2, + Int16>::type +our_bswap16(Int16 x) { + return ((x >> 8) & 0xff) | ((x & 0xff) << 8); +} +#endif + #define FB_GEN(t, fn) \ template<> inline t EndianIntBase::swap(t x) { return fn(x); } // fn(x) expands to (x) if the second argument is empty, which is exactly -// what we want for [u]int8_t +// what we want for [u]int8_t. Also, gcc 4.7 on Intel doesn't have +// __builtin_bswap16 for some reason, so we have to provide our own. FB_GEN( int8_t,) FB_GEN(uint8_t,) -FB_GEN( int64_t, bswap_64) -FB_GEN(uint64_t, bswap_64) -FB_GEN( int32_t, bswap_32) -FB_GEN(uint32_t, bswap_32) -FB_GEN( int16_t, bswap_16) -FB_GEN(uint16_t, bswap_16) +FB_GEN( int64_t, __builtin_bswap64) +FB_GEN(uint64_t, __builtin_bswap64) +FB_GEN( int32_t, __builtin_bswap32) +FB_GEN(uint32_t, __builtin_bswap32) +FB_GEN( int16_t, our_bswap16) +FB_GEN(uint16_t, our_bswap16) #undef FB_GEN -#if __BYTE_ORDER == __LITTLE_ENDIAN +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ template struct EndianInt : public detail::EndianIntBase { @@ -255,7 +276,7 @@ struct EndianInt : public detail::EndianIntBase { static T little(T x) { return x; } }; -#elif __BYTE_ORDER == __BIG_ENDIAN +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ template struct EndianInt : public detail::EndianIntBase { @@ -266,7 +287,7 @@ struct EndianInt : public detail::EndianIntBase { #else # error Your machine uses a weird endianness! -#endif /* __BYTE_ORDER */ +#endif /* __BYTE_ORDER__ */ } // namespace detail @@ -296,13 +317,13 @@ class Endian { }; static constexpr Order order = -#if __BYTE_ORDER == __LITTLE_ENDIAN +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ Order::LITTLE; -#elif __BYTE_ORDER == __BIG_ENDIAN +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ Order::BIG; #else # error Your machine uses a weird endianness! -#endif /* __BYTE_ORDER */ +#endif /* __BYTE_ORDER__ */ template static T swap(T x) { return detail::EndianInt::swap(x); diff --git a/hphp/third_party/folly/folly/Conv.h b/hphp/third_party/folly/folly/Conv.h index 4ccacf226..4d14a559e 100644 --- a/hphp/third_party/folly/folly/Conv.h +++ b/hphp/third_party/folly/folly/Conv.h @@ -64,13 +64,17 @@ typename std::enable_if< to(const Src & value) { /* static */ if (std::numeric_limits::max() < std::numeric_limits::max()) { - FOLLY_RANGE_CHECK(value <= std::numeric_limits::max(), - "Overflow"); + FOLLY_RANGE_CHECK( + (!greater_than::max()>(value)), + "Overflow" + ); } /* static */ if (std::is_signed::value && (!std::is_signed::value || sizeof(Src) > sizeof(Tgt))) { - FOLLY_RANGE_CHECK(value >= std::numeric_limits::min(), - "Negative overflow"); + FOLLY_RANGE_CHECK( + (!less_than::min()>(value)), + "Negative overflow" + ); } return static_cast(value); } @@ -472,6 +476,38 @@ typename std::enable_if::value>::type toAppend(Tgt* result) { } +/** + * Variadic base case: do nothing. + */ +template +typename std::enable_if::value>::type +toAppendDelim(const Delimiter& delim, Tgt* result) { +} + +/** + * 1 element: same as toAppend. + */ +template +typename std::enable_if::value>::type +toAppendDelim(const Delimiter& delim, const T& v, Tgt* tgt) { + toAppend(v, tgt); +} + +/** + * Append to string with a delimiter in between elements. + */ +template +typename std::enable_if= 2 + && detail::IsSomeString< + typename std::remove_pointer< + typename std::tuple_element< + sizeof...(Ts) - 1, std::tuple + >::type>::type>::value>::type +toAppendDelim(const Delimiter& delim, const T& v, const Ts&... vs) { + toAppend(v, delim, detail::getLastElement(vs...)); + toAppendDelim(delim, vs...); +} + /** * to(v1, v2, ...) uses toAppend() (see below) as back-end * for all types. @@ -484,6 +520,18 @@ to(const Ts&... vs) { return result; } +/** + * toDelim(delim, v1, v2, ...) uses toAppendDelim() as + * back-end for all types. + */ +template +typename std::enable_if::value, Tgt>::type +toDelim(const Delim& delim, const Ts&... vs) { + Tgt result; + toAppendDelim(delim, vs..., &result); + return result; +} + /******************************************************************************* * Conversions from string types to integral types. ******************************************************************************/ diff --git a/hphp/third_party/folly/folly/Exception.h b/hphp/third_party/folly/folly/Exception.h index c2e6eea78..fad0b691b 100644 --- a/hphp/third_party/folly/folly/Exception.h +++ b/hphp/third_party/folly/folly/Exception.h @@ -19,60 +19,93 @@ #include +#include #include #include #include "folly/Conv.h" +#include "folly/FBString.h" #include "folly/Likely.h" +#include "folly/Portability.h" namespace folly { +// Various helpers to throw appropriate std::system_error exceptions from C +// library errors (returned in errno, as positive return values (many POSIX +// functions), or as negative return values (Linux syscalls)) +// +// The *Explicit functions take an explicit value for errno. + // Helper to throw std::system_error -void throwSystemError(int err, const char* msg) __attribute__((noreturn)); -inline void throwSystemError(int err, const char* msg) { +void throwSystemErrorExplicit(int err, const char*) FOLLY_NORETURN; +inline void throwSystemErrorExplicit(int err, const char* msg) { throw std::system_error(err, std::system_category(), msg); } -// Helper to throw std::system_error from errno -void throwSystemError(const char* msg) __attribute__((noreturn)); -inline void throwSystemError(const char* msg) { - throwSystemError(errno, msg); +template +void throwSystemErrorExplicit(int, Args&&... args) FOLLY_NORETURN; +template +void throwSystemErrorExplicit(int err, Args&&... args) { + throwSystemErrorExplicit( + err, to(std::forward(args)...).c_str()); } // Helper to throw std::system_error from errno and components of a string template -void throwSystemError(Args... args) __attribute__((noreturn)); +void throwSystemError(Args&&... args) FOLLY_NORETURN; template -inline void throwSystemError(Args... args) { - throwSystemError(errno, folly::to(args...)); +void throwSystemError(Args&&... args) { + throwSystemErrorExplicit(errno, std::forward(args)...); } // Check a Posix return code (0 on success, error number on error), throw // on error. -inline void checkPosixError(int err, const char* msg) { +template +void checkPosixError(int err, Args&&... args) { if (UNLIKELY(err != 0)) { - throwSystemError(err, msg); + throwSystemErrorExplicit(err, std::forward(args)...); } } // Check a Linux kernel-style return code (>= 0 on success, negative error // number on error), throw on error. -inline void checkKernelError(ssize_t ret, const char* msg) { +template +void checkKernelError(ssize_t ret, Args&&... args) { if (UNLIKELY(ret < 0)) { - throwSystemError(-ret, msg); + throwSystemErrorExplicit(-ret, std::forward(args)...); } } // Check a traditional Unix return code (-1 and sets errno on error), throw // on error. -inline void checkUnixError(ssize_t ret, const char* msg) { +template +void checkUnixError(ssize_t ret, Args&&... args) { if (UNLIKELY(ret == -1)) { - throwSystemError(msg); + throwSystemError(std::forward(args)...); } } -inline void checkUnixError(ssize_t ret, int savedErrno, const char* msg) { + +template +void checkUnixErrorExplicit(ssize_t ret, int savedErrno, Args&&... args) { if (UNLIKELY(ret == -1)) { - throwSystemError(savedErrno, msg); + throwSystemErrorExplicit(savedErrno, std::forward(args)...); + } +} + +// Check the return code from a fopen-style function (returns a non-nullptr +// FILE* on success, nullptr on error, sets errno). Works with fopen, fdopen, +// freopen, tmpfile, etc. +template +void checkFopenError(FILE* fp, Args&&... args) { + if (UNLIKELY(!fp)) { + throwSystemError(std::forward(args)...); + } +} + +template +void checkFopenErrorExplicit(FILE* fp, int savedErrno, Args&&... args) { + if (UNLIKELY(!fp)) { + throwSystemErrorExplicit(savedErrno, std::forward(args)...); } } diff --git a/hphp/third_party/folly/folly/FBString.h b/hphp/third_party/folly/folly/FBString.h index e0c562508..ea1541485 100644 --- a/hphp/third_party/folly/folly/FBString.h +++ b/hphp/third_party/folly/folly/FBString.h @@ -1070,8 +1070,11 @@ public: // Move assignment basic_fbstring& operator=(basic_fbstring&& goner) { - // Self move assignment is illegal, see 17.6.4.9 for the explanation - assert(&goner != this); + if (FBSTRING_UNLIKELY(&goner == this)) { + // Compatibility with std::basic_string<>, + // 21.4.2 [string.cons] / 23 requires self-move-assignment support. + return *this; + } // No need of this anymore this->~basic_fbstring(); // Move the goner into this diff --git a/hphp/third_party/folly/folly/FBVector.h b/hphp/third_party/folly/folly/FBVector.h index fbae7e2c8..9e593c608 100644 --- a/hphp/third_party/folly/folly/FBVector.h +++ b/hphp/third_party/folly/folly/FBVector.h @@ -1095,7 +1095,7 @@ public: if (impl_.b_) M_deallocate(impl_.b_, impl_.z_ - impl_.b_); impl_.z_ = newB + newCap; - impl_.e_ += newB - impl_.b_; // speed hax + impl_.e_ = newB + (impl_.e_ - impl_.b_); impl_.b_ = newB; } @@ -1128,7 +1128,7 @@ public: if (impl_.b_) M_deallocate(impl_.b_, impl_.z_ - impl_.b_); impl_.z_ = newB + newCap; - impl_.e_ += newB - impl_.b_; // speed hax + impl_.e_ = newB + (impl_.e_ - impl_.b_); impl_.b_ = newB; } } diff --git a/hphp/third_party/folly/folly/File.cpp b/hphp/third_party/folly/folly/File.cpp index 1262dd1c3..aaf109f07 100644 --- a/hphp/third_party/folly/folly/File.cpp +++ b/hphp/third_party/folly/folly/File.cpp @@ -15,7 +15,13 @@ */ #include "folly/File.h" + +#include +#include +#include + #include "folly/Format.h" +#include "folly/Exception.h" #include "folly/ScopeGuard.h" #include @@ -37,11 +43,9 @@ File::File(int fd, bool ownsFd) File::File(const char* name, int flags, mode_t mode) : fd_(::open(name, flags, mode)) , ownsFd_(false) { - - if (fd_ < 0) { - throw std::system_error(errno, std::system_category(), - folly::format("open(\"{}\", {:#o}, 0{:#o}) failed", - name, flags, mode).str()); + if (fd_ == -1) { + throwSystemError(folly::format("open(\"{}\", {:#o}, 0{:#o}) failed", + name, flags, mode).fbstr()); } ownsFd_ = true; } @@ -66,15 +70,11 @@ File::~File() { /* static */ File File::temporary() { // make a temp file with tmpfile(), dup the fd, then return it in a File. FILE* tmpFile = tmpfile(); - if (!tmpFile) { - throw std::system_error(errno, std::system_category(), "tmpfile() failed"); - } + checkFopenError(tmpFile, "tmpfile() failed"); SCOPE_EXIT { fclose(tmpFile); }; int fd = dup(fileno(tmpFile)); - if (fd < 0) { - throw std::system_error(errno, std::system_category(), "dup() failed"); - } + checkUnixError(fd, "dup() failed"); return File(fd, true); } @@ -96,7 +96,7 @@ void swap(File& a, File& b) { void File::close() { if (!closeNoThrow()) { - throw std::system_error(errno, std::system_category(), "close() failed"); + throwSystemError("close() failed"); } } @@ -106,4 +106,26 @@ bool File::closeNoThrow() { return r == 0; } +void File::lock() { doLock(LOCK_EX); } +bool File::try_lock() { return doTryLock(LOCK_EX); } +void File::lock_shared() { doLock(LOCK_SH); } +bool File::try_lock_shared() { return doTryLock(LOCK_SH); } + +void File::doLock(int op) { + checkUnixError(flock(fd_, op), "flock() failed (lock)"); +} + +bool File::doTryLock(int op) { + int r = flock(fd_, op | LOCK_NB); + // flock returns EWOULDBLOCK if already locked + if (r == -1 && errno == EWOULDBLOCK) return false; + checkUnixError(r, "flock() failed (try_lock)"); + return true; +} + +void File::unlock() { + checkUnixError(flock(fd_, LOCK_UN), "flock() failed (unlock)"); +} +void File::unlock_shared() { unlock(); } + } // namespace folly diff --git a/hphp/third_party/folly/folly/File.h b/hphp/third_party/folly/folly/File.h index e990c91bf..71fcc0336 100644 --- a/hphp/third_party/folly/folly/File.h +++ b/hphp/third_party/folly/folly/File.h @@ -20,6 +20,7 @@ #include #include #include +#include namespace folly { @@ -92,7 +93,26 @@ class File { File(File&&); File& operator=(File&&); + // FLOCK (INTERPROCESS) LOCKS + // + // NOTE THAT THESE LOCKS ARE flock() LOCKS. That is, they may only be used + // for inter-process synchronization -- an attempt to acquire a second lock + // on the same file descriptor from the same process may succeed. Attempting + // to acquire a second lock on a different file descriptor for the same file + // should fail, but some systems might implement flock() using fcntl() locks, + // in which case it will succeed. + void lock(); + bool try_lock(); + void unlock(); + + void lock_shared(); + bool try_lock_shared(); + void unlock_shared(); + private: + void doLock(int op); + bool doTryLock(int op); + // unique File(const File&) = delete; File& operator=(const File&) = delete; @@ -103,6 +123,7 @@ class File { void swap(File& a, File& b); + } // namespace folly #endif /* FOLLY_FILE_H_ */ diff --git a/hphp/third_party/folly/folly/Format-inl.h b/hphp/third_party/folly/folly/Format-inl.h index 831d3c821..70672d476 100644 --- a/hphp/third_party/folly/folly/Format-inl.h +++ b/hphp/third_party/folly/folly/Format-inl.h @@ -18,6 +18,8 @@ #error This file may only be included from Format.h. #endif +#include "folly/Traits.h" + namespace folly { namespace detail { @@ -45,7 +47,7 @@ size_t uintToHex(char* buffer, size_t bufLen, Uint v, const char (&repr)[256][2]) { // 'v >>= 7, v >>= 1' is no more than a work around to get rid of shift size // warning when Uint = uint8_t (it's false as v >= 256 implies sizeof(v) > 1). - for (; v >= 256; v >>= 7, v >>= 1) { + for (; !less_than(v); v >>= 7, v >>= 1) { auto b = v & 0xff; bufLen -= 2; buffer[bufLen] = repr[b][0]; @@ -89,7 +91,7 @@ size_t uintToOctal(char* buffer, size_t bufLen, Uint v) { auto& repr = formatOctal; // 'v >>= 7, v >>= 2' is no more than a work around to get rid of shift size // warning when Uint = uint8_t (it's false as v >= 512 implies sizeof(v) > 1). - for (; v >= 512; v >>= 7, v >>= 2) { + for (; !less_than(v); v >>= 7, v >>= 2) { auto b = v & 0x1ff; bufLen -= 3; buffer[bufLen] = repr[b][0]; @@ -562,8 +564,6 @@ class FormatValue { arg.precision = 6; } - bool done = false; - // 2+: for null terminator and optional sign shenanigans. char buf[2 + std::max({ (2 + DoubleToStringConverter::kMaxFixedDigitsBeforePoint + diff --git a/hphp/third_party/folly/folly/FormatArg.h b/hphp/third_party/folly/folly/FormatArg.h index c72c03005..124d621d7 100644 --- a/hphp/third_party/folly/folly/FormatArg.h +++ b/hphp/third_party/folly/folly/FormatArg.h @@ -18,9 +18,10 @@ #define FOLLY_FORMATARG_H_ #include -#include "folly/Range.h" -#include "folly/Likely.h" #include "folly/Conv.h" +#include "folly/Likely.h" +#include "folly/Portability.h" +#include "folly/Range.h" namespace folly { @@ -71,7 +72,7 @@ struct FormatArg { } template - void error(Args&&... args) const __attribute__((noreturn)); + void error(Args&&... args) const FOLLY_NORETURN; /** * Full argument string, as passed in to the constructor. */ diff --git a/hphp/third_party/folly/folly/Malloc.h b/hphp/third_party/folly/folly/Malloc.h index cd5e56e72..ab165229f 100644 --- a/hphp/third_party/folly/folly/Malloc.h +++ b/hphp/third_party/folly/folly/Malloc.h @@ -43,7 +43,7 @@ namespace folly { #pragma GCC system_header #define FOLLY_HAVE_MALLOC_H 1 #else -#include "folly-config.h" +#include "folly/Portability.h" #endif // for malloc_usable_size diff --git a/hphp/third_party/folly/folly/Memory.h b/hphp/third_party/folly/folly/Memory.h index 09f10f36b..0b6c0a11c 100644 --- a/hphp/third_party/folly/folly/Memory.h +++ b/hphp/third_party/folly/folly/Memory.h @@ -78,9 +78,28 @@ template class StlAllocator { typedef void value_type; typedef void* pointer; typedef const void* const_pointer; + + StlAllocator() : alloc_(nullptr) { } + explicit StlAllocator(Alloc* alloc) : alloc_(alloc) { } + + Alloc* alloc() const { + return alloc_; + } + template struct rebind { typedef StlAllocator other; }; + + bool operator!=(const StlAllocator& other) const { + return alloc_ != other.alloc_; + } + + bool operator==(const StlAllocator& other) const { + return alloc_ == other.alloc_; + } + + private: + Alloc* alloc_; }; template @@ -150,6 +169,18 @@ class StlAllocator { Alloc* alloc_; }; +/** + * Helper function to obtain rebound allocators + * + * @author: Marcelo Juchem + */ +template +typename Allocator::template rebind::other rebind_allocator( + Allocator const &allocator +) { + return typename Allocator::template rebind::other(allocator); +} + /* * Helper classes/functions for creating a unique_ptr using a custom allocator * @@ -210,6 +241,18 @@ public: && !has_destroy::value; }; +template +struct as_stl_allocator { + typedef typename std::conditional< + is_simple_allocator::value, + folly::StlAllocator< + typename std::remove_reference::type, + typename std::remove_reference::type + >, + typename std::remove_reference::type + >::type type; +}; + template typename std::enable_if< is_simple_allocator::value, @@ -232,6 +275,13 @@ typename std::enable_if< return std::move(allocator); } +/** + * AllocatorUniquePtr: a unique_ptr that supports both STL-style + * allocators and SimpleAllocator + * + * @author: Marcelo Juchem + */ + template struct AllocatorUniquePtr { typedef std::unique_ptr type; }; +/** + * Functions to allocate a unique_ptr / shared_ptr, supporting both + * STL-style allocators and SimpleAllocator, analog to std::allocate_shared + * + * @author: Marcelo Juchem + */ + template typename AllocatorUniquePtr::type allocate_unique( Allocator&& allocator, Args&&... args diff --git a/hphp/third_party/folly/folly/MemoryMapping.cpp b/hphp/third_party/folly/folly/MemoryMapping.cpp index 447a192b4..3e0b93a20 100644 --- a/hphp/third_party/folly/folly/MemoryMapping.cpp +++ b/hphp/third_party/folly/folly/MemoryMapping.cpp @@ -22,7 +22,7 @@ #include #include -#define FLAGS_mlock_chunk_size (1<<20) +#define FLAGS_mlock_chunk_size (1 << 20) namespace folly { diff --git a/hphp/third_party/folly/folly/Optional.h b/hphp/third_party/folly/folly/Optional.h index 871357cea..4295e2715 100644 --- a/hphp/third_party/folly/folly/Optional.h +++ b/hphp/third_party/folly/folly/Optional.h @@ -61,6 +61,7 @@ #include + namespace folly { namespace detail { struct NoneHelper {}; } @@ -81,10 +82,7 @@ const None none = nullptr; #endif // __GNUC__ template -class Optional : boost::totally_ordered, - boost::totally_ordered, Value>> { - typedef void (Optional::*bool_type)() const; - void truthy() const {}; +class Optional { public: static_assert(!std::is_reference::value, "Optional may not be used with reference types"); @@ -110,7 +108,7 @@ class Optional : boost::totally_ordered, } } - /* implicit */ Optional(const None& empty) + /* implicit */ Optional(const None&) : hasValue_(false) { } @@ -179,32 +177,6 @@ class Optional : boost::totally_ordered, return *this; } - bool operator<(const Optional& other) const { - if (hasValue() != other.hasValue()) { - return hasValue() < other.hasValue(); - } - if (hasValue()) { - return value() < other.value(); - } - return false; // both empty - } - - bool operator<(const Value& other) const { - return !hasValue() || value() < other; - } - - bool operator==(const Optional& other) const { - if (hasValue()) { - return other.hasValue() && value() == other.value(); - } else { - return !other.hasValue(); - } - } - - bool operator==(const Value& other) const { - return hasValue() && value() == other; - } - template void emplace(Args&&... args) { clear(); @@ -230,9 +202,8 @@ class Optional : boost::totally_ordered, bool hasValue() const { return hasValue_; } - /* safe bool idiom */ - operator bool_type() const { - return hasValue() ? &Optional::truthy : nullptr; + explicit operator bool() const { + return hasValue(); } const Value& operator*() const { return value(); } @@ -286,6 +257,54 @@ Opt make_optional(T&& v) { return Opt(std::forward(v)); } +template +bool operator< (const Optional& a, const Optional& b) { + if (a.hasValue() != b.hasValue()) { return a.hasValue() < b.hasValue(); } + if (a.hasValue()) { return a.value() < b.value(); } + return false; +} + +template +bool operator==(const Optional& a, const Optional& b) { + if (a.hasValue() != b.hasValue()) { return false; } + if (a.hasValue()) { return a.value() == b.value(); } + return true; +} + +template +bool operator<=(const Optional& a, const Optional& b) { + return !(b < a); +} + +template +bool operator!=(const Optional& a, const Optional& b) { + return !(b == a); +} + +template +bool operator>=(const Optional& a, const Optional& b) { + return !(a < b); +} + +template +bool operator> (const Optional& a, const Optional& b) { + return b < a; +} + +// To supress comparability of Optional with T, despite implicit conversion. +template bool operator< (const Optional&, const V& other) = delete; +template bool operator<=(const Optional&, const V& other) = delete; +template bool operator==(const Optional&, const V& other) = delete; +template bool operator!=(const Optional&, const V& other) = delete; +template bool operator>=(const Optional&, const V& other) = delete; +template bool operator> (const Optional&, const V& other) = delete; +template bool operator< (const V& other, const Optional&) = delete; +template bool operator<=(const V& other, const Optional&) = delete; +template bool operator==(const V& other, const Optional&) = delete; +template bool operator!=(const V& other, const Optional&) = delete; +template bool operator>=(const V& other, const Optional&) = delete; +template bool operator> (const V& other, const Optional&) = delete; + } // namespace folly #endif//FOLLY_OPTIONAL_H_ diff --git a/hphp/third_party/folly/folly/Portability.h b/hphp/third_party/folly/folly/Portability.h index 14b7c5b08..f4064a222 100644 --- a/hphp/third_party/folly/folly/Portability.h +++ b/hphp/third_party/folly/folly/Portability.h @@ -19,6 +19,11 @@ #include "folly-config.h" +#ifdef FOLLY_HAVE_FEATURES_H +#include +#endif + + #ifdef FOLLY_HAVE_SCHED_H #include #ifndef FOLLY_HAVE_PTHREAD_YIELD @@ -26,32 +31,6 @@ #endif #endif -// Define macro wrappers for C++11's "final" and "override" keywords, which -// are supported in gcc 4.7 but not gcc 4.6. -// -// TODO(tudorb/agallagher): Autotoolize this. -#undef FOLLY_FINAL -#undef FOLLY_OVERRIDE - -#if defined(__clang__) -# define FOLLY_FINAL final -# define FOLLY_OVERRIDE override -#elif defined(__GNUC__) -# include -# if __GNUC_PREREQ(4,7) -# define FOLLY_FINAL final -# define FOLLY_OVERRIDE override -# endif -#endif - -#ifndef FOLLY_FINAL -# define FOLLY_FINAL -#endif - -#ifndef FOLLY_OVERRIDE -# define FOLLY_OVERRIDE -#endif - // MaxAlign: max_align_t isn't supported by gcc #ifdef __GNUC__ @@ -60,4 +39,36 @@ struct MaxAlign { char c; } __attribute__((aligned)); # error Cannot define MaxAlign on this platform #endif + +// noreturn +#if defined(__clang__) || defined(__GNUC__) +# define FOLLY_NORETURN __attribute__((noreturn)) +#else +# define FOLLY_NORETURN +#endif + + +/* Define macro wrappers for C++11's "final" and "override" keywords, which + * are supported in gcc 4.7 but not gcc 4.6. */ +#if !defined(FOLLY_FINAL) && !defined(FOLLY_OVERRIDE) +# if defined(__clang__) || \ + (defined(__GNUC__) && defined(__GNUC_MINOR__) && \ + ((__GNUC__ << 16) + __GNUC_MINOR__) >= ((4 << 16) + 7)) +# define FOLLY_FINAL final +# define FOLLY_OVERRIDE override +# else +# define FOLLY_FINAL /**/ +# define FOLLY_OVERRIDE /**/ +# endif +#endif + + +// Define to 1 if you have the `preadv' and `pwritev' functions, respectively +#if !defined(FOLLY_HAVE_PREADV) && !defined(FOLLY_HAVE_PWRITEV) +# if defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 10) +# define FOLLY_HAVE_PREADV 1 +# define FOLLY_HAVE_PWRITEV 1 +# endif +#endif + #endif // FOLLY_PORTABILITY_H_ diff --git a/hphp/third_party/folly/folly/Range.cpp b/hphp/third_party/folly/folly/Range.cpp index 2796cd470..b4359bb1e 100644 --- a/hphp/third_party/folly/folly/Range.cpp +++ b/hphp/third_party/folly/folly/Range.cpp @@ -20,7 +20,7 @@ #include "folly/Range.h" #include // __v16qi -#include "folly/Likely.h" +#include namespace folly { diff --git a/hphp/third_party/folly/folly/Range.h b/hphp/third_party/folly/folly/Range.h index 57db55c83..592aead5c 100644 --- a/hphp/third_party/folly/folly/Range.h +++ b/hphp/third_party/folly/folly/Range.h @@ -20,21 +20,20 @@ #ifndef FOLLY_RANGE_H_ #define FOLLY_RANGE_H_ -#include "folly/folly-config.h" +#include "folly/Portability.h" #include "folly/FBString.h" #include #include #include -#include +#include #include #include #include #include -#include -#include #include #include "folly/CpuId.h" #include "folly/Traits.h" +#include "folly/Likely.h" namespace folly { @@ -77,9 +76,9 @@ namespace detail { * For random-access iterators, the value before is simply i[-1]. */ template -typename boost::enable_if_c< - boost::is_same::iterator_category, - std::random_access_iterator_tag>::value, +typename std::enable_if< + std::is_same::iterator_category, + std::random_access_iterator_tag>::value, typename std::iterator_traits::reference>::type value_before(Iter i) { return i[-1]; @@ -89,9 +88,9 @@ value_before(Iter i) { * For all other iterators, we need to use the decrement operator. */ template -typename boost::enable_if_c< - !boost::is_same::iterator_category, - std::random_access_iterator_tag>::value, +typename std::enable_if< + !std::is_same::iterator_category, + std::random_access_iterator_tag>::value, typename std::iterator_traits::reference>::type value_before(Iter i) { return *--i; @@ -116,7 +115,7 @@ public: typedef std::size_t size_type; typedef Iter iterator; typedef Iter const_iterator; - typedef typename boost::remove_reference< + typedef typename std::remove_reference< typename std::iterator_traits::reference>::type value_type; typedef typename std::iterator_traits::reference reference; @@ -145,7 +144,9 @@ public: : b_(str.data()), e_(b_ + str.size()) {} // Works only for Range Range(const std::string& str, std::string::size_type startFrom) { - CHECK_LE(startFrom, str.size()); + if (UNLIKELY(startFrom > str.size())) { + throw std::out_of_range("index out of range"); + } b_ = str.data() + startFrom; e_ = str.data() + str.size(); } @@ -153,32 +154,52 @@ public: Range(const std::string& str, std::string::size_type startFrom, std::string::size_type size) { - CHECK_LE(startFrom + size, str.size()); + if (UNLIKELY(startFrom > str.size())) { + throw std::out_of_range("index out of range"); + } b_ = str.data() + startFrom; - e_ = b_ + size; + if (str.size() - startFrom < size) { + e_ = str.data() + str.size(); + } else { + e_ = b_ + size; + } } Range(const Range& str, size_t startFrom, size_t size) { - CHECK_LE(startFrom + size, str.size()); + if (UNLIKELY(startFrom > str.size())) { + throw std::out_of_range("index out of range"); + } b_ = str.b_ + startFrom; - e_ = b_ + size; + if (str.size() - startFrom < size) { + e_ = str.e_; + } else { + e_ = b_ + size; + } } // Works only for Range /* implicit */ Range(const fbstring& str) : b_(str.data()), e_(b_ + str.size()) { } // Works only for Range Range(const fbstring& str, fbstring::size_type startFrom) { - CHECK_LE(startFrom, str.size()); + if (UNLIKELY(startFrom > str.size())) { + throw std::out_of_range("index out of range"); + } b_ = str.data() + startFrom; e_ = str.data() + str.size(); } // Works only for Range Range(const fbstring& str, fbstring::size_type startFrom, fbstring::size_type size) { - CHECK_LE(startFrom + size, str.size()); + if (UNLIKELY(startFrom > str.size())) { + throw std::out_of_range("index out of range"); + } b_ = str.data() + startFrom; - e_ = b_ + size; + if (str.size() - startFrom < size) { + e_ = str.data() + str.size(); + } else { + e_ = b_ + size; + } } // Allow implicit conversion from Range (aka StringPiece) to @@ -301,12 +322,16 @@ public: } void advance(size_type n) { - CHECK_LE(n, size()); + if (UNLIKELY(n > size())) { + throw std::out_of_range("index out of range"); + } b_ += n; } void subtract(size_type n) { - CHECK_LE(n, size()); + if (UNLIKELY(n > size())) { + throw std::out_of_range("index out of range"); + } e_ -= n; } @@ -322,7 +347,9 @@ public: Range subpiece(size_type first, size_type length = std::string::npos) const { - CHECK_LE(first, size()); + if (UNLIKELY(first > size())) { + throw std::out_of_range("index out of range"); + } return Range(b_ + first, std::min(length, size() - first)); } @@ -452,11 +479,11 @@ template struct ComparableAsStringPiece { enum { value = - (boost::is_convertible::value - && boost::is_same::value) + (std::is_convertible::value + && std::is_same::value) || - (boost::is_convertible::value - && boost::is_same::value) + (std::is_convertible::value + && std::is_same::value) }; }; @@ -467,7 +494,7 @@ struct ComparableAsStringPiece { */ template typename -boost::enable_if_c::value, bool>::type +std::enable_if::value, bool>::type operator==(const T& lhs, const U& rhs) { return StringPiece(lhs) == StringPiece(rhs); } @@ -477,7 +504,7 @@ operator==(const T& lhs, const U& rhs) { */ template typename -boost::enable_if_c::value, bool>::type +std::enable_if::value, bool>::type operator<(const T& lhs, const U& rhs) { return StringPiece(lhs) < StringPiece(rhs); } @@ -487,7 +514,7 @@ operator<(const T& lhs, const U& rhs) { */ template typename -boost::enable_if_c::value, bool>::type +std::enable_if::value, bool>::type operator>(const T& lhs, const U& rhs) { return StringPiece(lhs) > StringPiece(rhs); } @@ -497,7 +524,7 @@ operator>(const T& lhs, const U& rhs) { */ template typename -boost::enable_if_c::value, bool>::type +std::enable_if::value, bool>::type operator<=(const T& lhs, const U& rhs) { return StringPiece(lhs) <= StringPiece(rhs); } @@ -507,7 +534,7 @@ operator<=(const T& lhs, const U& rhs) { */ template typename -boost::enable_if_c::value, bool>::type +std::enable_if::value, bool>::type operator>=(const T& lhs, const U& rhs) { return StringPiece(lhs) >= StringPiece(rhs); } diff --git a/hphp/third_party/folly/folly/ScopeGuard.h b/hphp/third_party/folly/folly/ScopeGuard.h index f6f2e2c3a..a3050a847 100644 --- a/hphp/third_party/folly/folly/ScopeGuard.h +++ b/hphp/third_party/folly/folly/ScopeGuard.h @@ -104,7 +104,7 @@ class ScopeGuardImpl : public ScopeGuardImplBase { } } -private: + private: void* operator new(size_t) = delete; void execute() noexcept { function_(); } diff --git a/hphp/third_party/folly/folly/String-inl.h b/hphp/third_party/folly/folly/String-inl.h index b9f7c2370..c608398ac 100644 --- a/hphp/third_party/folly/folly/String-inl.h +++ b/hphp/third_party/folly/folly/String-inl.h @@ -149,6 +149,95 @@ void cUnescape(StringPiece str, String& out, bool strict) { out.append(&*last, p - last); } +namespace detail { +// Map from character code to escape mode: +// 0 = pass through +// 1 = unused +// 2 = pass through in PATH mode +// 3 = space, replace with '+' in QUERY mode +// 4 = percent-encode +extern const unsigned char uriEscapeTable[]; +} // namespace detail + +template +void uriEscape(StringPiece str, String& out, UriEscapeMode mode) { + static const char hexValues[] = "0123456789abcdef"; + char esc[3]; + esc[0] = '%'; + // Preallocate assuming that 25% of the input string will be escaped + out.reserve(out.size() + str.size() + 3 * (str.size() / 4)); + auto p = str.begin(); + auto last = p; // last regular character + // We advance over runs of passthrough characters and copy them in one go; + // this is faster than calling push_back repeatedly. + unsigned char minEncode = static_cast(mode); + while (p != str.end()) { + char c = *p; + unsigned char v = static_cast(c); + unsigned char discriminator = detail::uriEscapeTable[v]; + if (LIKELY(discriminator <= minEncode)) { + ++p; + } else if (mode == UriEscapeMode::QUERY && discriminator == 3) { + out.append(&*last, p - last); + out.push_back('+'); + ++p; + last = p; + } else { + out.append(&*last, p - last); + esc[1] = hexValues[v >> 4]; + esc[2] = hexValues[v & 0x0f]; + out.append(esc, 3); + ++p; + last = p; + } + } + out.append(&*last, p - last); +} + +template +void uriUnescape(StringPiece str, String& out, UriEscapeMode mode) { + out.reserve(out.size() + str.size()); + auto p = str.begin(); + auto last = p; + // We advance over runs of passthrough characters and copy them in one go; + // this is faster than calling push_back repeatedly. + while (p != str.end()) { + char c = *p; + unsigned char v = static_cast(v); + switch (c) { + case '%': + { + if (UNLIKELY(std::distance(p, str.end()) < 3)) { + throw std::invalid_argument("incomplete percent encode sequence"); + } + auto h1 = detail::hexTable[static_cast(p[1])]; + auto h2 = detail::hexTable[static_cast(p[2])]; + if (UNLIKELY(h1 == 16 || h2 == 16)) { + throw std::invalid_argument("invalid percent encode sequence"); + } + out.append(&*last, p - last); + out.push_back((h1 << 4) | h2); + p += 3; + last = p; + break; + } + case '+': + if (mode == UriEscapeMode::QUERY) { + out.append(&*last, p - last); + out.push_back(' '); + ++p; + last = p; + break; + } + // else fallthrough + default: + ++p; + break; + } + } + out.append(&*last, p - last); +} + namespace detail { /* @@ -258,6 +347,39 @@ template StringPiece prepareDelim(const String& s) { } inline char prepareDelim(char c) { return c; } +template +bool splitFixed(const Delim& delimiter, + StringPiece input, + StringPiece& out) { + if (exact && UNLIKELY(std::string::npos != input.find(delimiter))) { + return false; + } + out = input; + return true; +} + +template +bool splitFixed(const Delim& delimiter, + StringPiece input, + StringPiece& outHead, + StringPieces&... outTail) { + size_t cut = input.find(delimiter); + if (UNLIKELY(cut == std::string::npos)) { + return false; + } + StringPiece head(input.begin(), input.begin() + cut); + StringPiece tail(input.begin() + cut + detail::delimSize(delimiter), + input.end()); + if (LIKELY(splitFixed(delimiter, tail, outTail...))) { + outHead = head; + return true; + } + return false; +} + } ////////////////////////////////////////////////////////////////////// @@ -299,6 +421,20 @@ void splitTo(const Delim& delimiter, ignoreEmpty); } +template +bool split(const Delim& delimiter, + StringPiece input, + StringPiece& outHead, + StringPieces&... outTail) { + return detail::splitFixed( + detail::prepareDelim(delimiter), + input, + outHead, + outTail...); +} + namespace detail { template diff --git a/hphp/third_party/folly/folly/String.cpp b/hphp/third_party/folly/folly/String.cpp index cfcb35821..4f1303eea 100644 --- a/hphp/third_party/folly/folly/String.cpp +++ b/hphp/third_party/folly/folly/String.cpp @@ -242,8 +242,7 @@ fbstring errnoStr(int err) { fbstring result; // http://www.kernel.org/doc/man-pages/online/pages/man3/strerror.3.html -#if (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600 || \ - !FOLLY_HAVE_FEATURES_H) && !_GNU_SOURCE +#if (_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && !_GNU_SOURCE // Using XSI-compatible strerror_r int r = strerror_r(err, buf, sizeof(buf)); diff --git a/hphp/third_party/folly/folly/String.h b/hphp/third_party/folly/folly/String.h index c843eed8b..93a4c6583 100644 --- a/hphp/third_party/folly/folly/String.h +++ b/hphp/third_party/folly/folly/String.h @@ -112,6 +112,56 @@ String cUnescape(StringPiece str, bool strict = true) { return out; } +/** + * URI-escape a string. Appends the result to the output string. + * + * Alphanumeric characters and other characters marked as "unreserved" in RFC + * 3986 ( -_.~ ) are left unchanged. In PATH mode, the forward slash (/) is + * also left unchanged. In QUERY mode, spaces are replaced by '+'. All other + * characters are percent-encoded. + */ +enum class UriEscapeMode : unsigned char { + // The values are meaningful, see generate_escape_tables.py + ALL = 0, + QUERY = 1, + PATH = 2 +}; +template +void uriEscape(StringPiece str, + String& out, + UriEscapeMode mode = UriEscapeMode::ALL); + +/** + * Similar to uriEscape above, but returns the escaped string. + */ +template +String uriEscape(StringPiece str, UriEscapeMode mode = UriEscapeMode::ALL) { + String out; + uriEscape(str, out, mode); + return out; +} + +/** + * URI-unescape a string. Appends the result to the output string. + * + * In QUERY mode, '+' are replaced by space. %XX sequences are decoded if + * XX is a valid hex sequence, otherwise we throw invalid_argument. + */ +template +void uriUnescape(StringPiece str, + String& out, + UriEscapeMode mode = UriEscapeMode::ALL); + +/** + * Similar to uriUnescape above, but returns the unescaped string. + */ +template +String uriUnescape(StringPiece str, UriEscapeMode mode = UriEscapeMode::ALL) { + String out; + uriUnescape(str, out, mode); + return out; +} + /** * stringPrintf is much like printf but deposits its result into a * string. Two signatures are supported: the first simply returns the @@ -344,6 +394,35 @@ void splitTo(const Delim& delimiter, OutputIterator out, bool ignoreEmpty = false); +/* + * Split a string into a fixed number of pieces by delimiter. Returns 'true' if + * the fields were all successfully populated. + * + * Example: + * + * folly::StringPiece name, key, value; + * if (folly::split('\t', line, name, key, value)) + * ... + * + * The 'exact' template paremeter specifies how the function behaves when too + * many fields are present in the input string. When 'exact' is set to its + * default value of 'true', a call to split will fail if the number of fields in + * the input string does not exactly match the number of output parameters + * passed. If 'exact' is overridden to 'false', all remaining fields will be + * stored, unsplit, in the last field, as shown below: + * + * folly::StringPiece x, y. + * if (folly::split(':', "a:b:c", x, y)) + * assert(x == "a" && y == "b:c"); + */ +template +bool split(const Delim& delimiter, + StringPiece input, + StringPiece& outHead, + StringPieces&... outTail); + /* * Join list of tokens. * diff --git a/hphp/third_party/folly/folly/Subprocess.cpp b/hphp/third_party/folly/folly/Subprocess.cpp index e2d5f7bd8..11026c29d 100644 --- a/hphp/third_party/folly/folly/Subprocess.cpp +++ b/hphp/third_party/folly/folly/Subprocess.cpp @@ -16,6 +16,7 @@ #include "folly/Subprocess.h" +#include #include #include #include @@ -32,12 +33,16 @@ #include "folly/Conv.h" #include "folly/Exception.h" +#include "folly/FileUtil.h" #include "folly/ScopeGuard.h" #include "folly/String.h" #include "folly/io/Cursor.h" extern char** environ; +constexpr int kExecFailure = 127; +constexpr int kChildFailure = 126; + namespace folly { ProcessReturnCode::State ProcessReturnCode::state() const { @@ -90,6 +95,16 @@ CalledProcessError::CalledProcessError(ProcessReturnCode rc) what_(returnCode_.str()) { } +SubprocessSpawnError::SubprocessSpawnError(const char* executable, + int errCode, + int errnoValue) + : errnoValue_(errnoValue), + what_(to(errCode == kExecFailure ? + "failed to execute " : + "error preparing to execute ", + executable, ": ", errnoStr(errnoValue))) { +} + namespace { // Copy pointers to the given strings in a format suitable for posix_spawn @@ -166,12 +181,29 @@ Subprocess::Subprocess( Subprocess::~Subprocess() { CHECK_NE(returnCode_.state(), ProcessReturnCode::RUNNING) << "Subprocess destroyed without reaping child"; + closeAll(); } namespace { void closeChecked(int fd) { checkUnixError(::close(fd), "close"); } + +struct ChildErrorInfo { + int errCode; + int errnoValue; +}; + +void childError(int errFd, int errCode, int errnoValue) FOLLY_NORETURN; +void childError(int errFd, int errCode, int errnoValue) { + ChildErrorInfo info = {errCode, errnoValue}; + // Write the error information over the pipe to our parent process. + // We can't really do anything else if this write call fails. + writeNoInt(errFd, &info, sizeof(info)); + // exit + _exit(errCode); +} + } // namespace void Subprocess::closeAll() { @@ -204,12 +236,79 @@ void Subprocess::spawn( // Make a copy, we'll mutate options Options options(optionsIn); + // On error, close all of the pipes_ + auto pipesGuard = makeGuard([&] { + for (auto& p : this->pipes_) { + CHECK_ERR(::close(p.parentFd)); + } + }); + + // Create a pipe to use to receive error information from the child, + // in case it fails before calling exec() + int errFds[2]; + int r = ::pipe(errFds); + checkUnixError(r, "pipe"); + SCOPE_EXIT { + CHECK_ERR(::close(errFds[0])); + if (errFds[1] >= 0) { + CHECK_ERR(::close(errFds[1])); + } + }; + // Ask the child to close the read end of the error pipe. + options.fdActions_[errFds[0]] = CLOSE; + // Set the close-on-exec flag on the write side of the pipe. + // This way the pipe will be closed automatically in the child if execve() + // succeeds. If the exec fails the child can write error information to the + // pipe. + r = fcntl(errFds[1], F_SETFD, FD_CLOEXEC); + checkUnixError(r, "set FD_CLOEXEC"); + + // Perform the actual work of setting up pipes then forking and + // executing the child. + spawnInternal(std::move(argv), executable, options, env, errFds[1]); + + // After spawnInternal() returns the child is alive. We have to be very + // careful about throwing after this point. We are inside the constructor, + // so if we throw the Subprocess object will have never existed, and the + // destructor will never be called. + // + // We should only throw if we got an error via the errFd, and we know the + // child has exited and can be immediately waited for. In all other cases, + // we have no way of cleaning up the child. + + // Close writable side of the errFd pipe in the parent process + CHECK_ERR(::close(errFds[1])); + errFds[1] = -1; + + // Read from the errFd pipe, to tell if the child ran into any errors before + // calling exec() + readChildErrorPipe(errFds[0], executable); + + // We have fully succeeded now, so release the guard on pipes_ + pipesGuard.dismiss(); +} + +void Subprocess::spawnInternal( + std::unique_ptr argv, + const char* executable, + Options& options, + const std::vector* env, + int errFd) { // Parent work, pre-fork: create pipes std::vector childFds; + // Close all of the childFds as we leave this scope + SCOPE_EXIT { + // These are only pipes, closing them shouldn't fail + for (int cfd : childFds) { + CHECK_ERR(::close(cfd)); + } + }; + + int r; for (auto& p : options.fdActions_) { if (p.second == PIPE_IN || p.second == PIPE_OUT) { int fds[2]; - int r = ::pipe(fds); + r = ::pipe(fds); checkUnixError(r, "pipe"); PipeInfo pinfo; pinfo.direction = p.second; @@ -261,72 +360,62 @@ void Subprocess::spawn( // // The parent also unblocks all signals as soon as vfork() returns. sigset_t allBlocked; - int r = ::sigfillset(&allBlocked); + r = ::sigfillset(&allBlocked); checkUnixError(r, "sigfillset"); sigset_t oldSignals; + r = pthread_sigmask(SIG_SETMASK, &allBlocked, &oldSignals); checkPosixError(r, "pthread_sigmask"); + SCOPE_EXIT { + // Restore signal mask + r = pthread_sigmask(SIG_SETMASK, &oldSignals, nullptr); + CHECK_EQ(r, 0) << "pthread_sigmask: " << errnoStr(r); // shouldn't fail + }; pid_t pid = vfork(); if (pid == 0) { - // While all signals are blocked, we must reset their - // dispositions to default. - for (int sig = 1; sig < NSIG; ++sig) { - ::signal(sig, SIG_DFL); + int errnoValue = prepareChild(options, &oldSignals); + if (errnoValue != 0) { + childError(errFd, kChildFailure, errnoValue); } - // Unblock signals; restore signal mask. - int r = pthread_sigmask(SIG_SETMASK, &oldSignals, nullptr); - if (r != 0) abort(); - runChild(executable, argVec, envVec, options); - // This should never return, but there's nothing else we can do here. - abort(); + errnoValue = runChild(executable, argVec, envVec, options); + // If we get here, exec() failed. + childError(errFd, kExecFailure, errnoValue); } - // In parent. We want to restore the signal mask even if vfork fails, - // so we'll save errno here, restore the signal mask, and only then - // throw. - int savedErrno = errno; + // In parent. Make sure vfork() succeeded. + checkUnixError(pid, errno, "vfork"); - // Restore signal mask; do this even if vfork fails! - // We only check for errors from pthread_sigmask after we recorded state - // that the child is alive, so we know to reap it. - r = pthread_sigmask(SIG_SETMASK, &oldSignals, nullptr); - checkUnixError(pid, savedErrno, "vfork"); - - // Child is alive + // Child is alive. We have to be very careful about throwing after this + // point. We are inside the constructor, so if we throw the Subprocess + // object will have never existed, and the destructor will never be called. + // + // We should only throw if we got an error via the errFd, and we know the + // child has exited and can be immediately waited for. In all other cases, + // we have no way of cleaning up the child. pid_ = pid; returnCode_ = ProcessReturnCode(RV_RUNNING); +} - // Parent work, post-fork: close child's ends of pipes - for (int f : childFds) { - closeChecked(f); +int Subprocess::prepareChild(const Options& options, + const sigset_t* sigmask) const { + // While all signals are blocked, we must reset their + // dispositions to default. + for (int sig = 1; sig < NSIG; ++sig) { + ::signal(sig, SIG_DFL); + } + // Unblock signals; restore signal mask. + int r = pthread_sigmask(SIG_SETMASK, sigmask, nullptr); + if (r != 0) { + return r; // pthread_sigmask() returns an errno value } - checkPosixError(r, "pthread_sigmask"); -} - -namespace { - -// Checked version of close() to use in the child: abort() on error -void childClose(int fd) { - int r = ::close(fd); - if (r == -1) abort(); -} - -// Checked version of dup2() to use in the child: abort() on error -void childDup2(int oldfd, int newfd) { - int r = ::dup2(oldfd, newfd); - if (r == -1) abort(); -} - -} // namespace - -void Subprocess::runChild(const char* executable, - char** argv, char** env, - const Options& options) const { // Close parent's ends of all pipes for (auto& p : pipes_) { - childClose(p.parentFd); + r = ::close(p.parentFd); + if (r == -1) { + return errno; + } } // Close all fds that we're supposed to close. @@ -336,7 +425,10 @@ void Subprocess::runChild(const char* executable, if (p.second == CLOSE) { ::close(p.first); } else { - childDup2(p.second, p.first); + r = ::dup2(p.second, p.first); + if (r == -1) { + return errno; + } } } @@ -351,6 +443,20 @@ void Subprocess::runChild(const char* executable, } } + // Opt to receive signal on parent death, if requested + if (options.parentDeathSignal_ != 0) { + r = prctl(PR_SET_PDEATHSIG, options.parentDeathSignal_, 0, 0, 0); + if (r == -1) { + return errno; + } + } + + return 0; +} + +int Subprocess::runChild(const char* executable, + char** argv, char** env, + const Options& options) const { // Now, finally, exec. int r; if (options.usePath_) { @@ -358,9 +464,36 @@ void Subprocess::runChild(const char* executable, } else { ::execve(executable, argv, env); } + return errno; +} - // If we're here, something's wrong. - abort(); +void Subprocess::readChildErrorPipe(int pfd, const char* executable) { + ChildErrorInfo info; + auto rc = readNoInt(pfd, &info, sizeof(info)); + if (rc == 0) { + // No data means the child executed successfully, and the pipe + // was closed due to the close-on-exec flag being set. + return; + } else if (rc != sizeof(ChildErrorInfo)) { + // An error occurred trying to read from the pipe, or we got a partial read. + // Neither of these cases should really occur in practice. + // + // We can't get any error data from the child in this case, and we don't + // know if it is successfully running or not. All we can do is to return + // normally, as if the child executed successfully. If something bad + // happened the caller should at least get a non-normal exit status from + // the child. + LOG(ERROR) << "unexpected error trying to read from child error pipe " << + "rc=" << rc << ", errno=" << errno; + return; + } + + // We got error data from the child. The child should exit immediately in + // this case, so wait on it to clean up. + wait(); + + // Throw to signal the error + throw SubprocessSpawnError(executable, info.errCode, info.errnoValue); } ProcessReturnCode Subprocess::poll() { diff --git a/hphp/third_party/folly/folly/Subprocess.h b/hphp/third_party/folly/folly/Subprocess.h index 47ec42601..58c9c9c82 100644 --- a/hphp/third_party/folly/folly/Subprocess.h +++ b/hphp/third_party/folly/folly/Subprocess.h @@ -144,10 +144,15 @@ class ProcessReturnCode { int rawStatus_; }; +/** + * Base exception thrown by the Subprocess methods. + */ +class SubprocessError : public std::exception {}; + /** * Exception thrown by *Checked methods of Subprocess. */ -class CalledProcessError : public std::exception { +class CalledProcessError : public SubprocessError { public: explicit CalledProcessError(ProcessReturnCode rc); ~CalledProcessError() throw() { } @@ -158,6 +163,21 @@ class CalledProcessError : public std::exception { std::string what_; }; +/** + * Exception thrown if the subprocess cannot be started. + */ +class SubprocessSpawnError : public SubprocessError { + public: + SubprocessSpawnError(const char* executable, int errCode, int errnoValue); + ~SubprocessSpawnError() throw() {} + const char* what() const throw() FOLLY_OVERRIDE { return what_.c_str(); } + int errnoValue() const { return errnoValue_; } + + private: + int errnoValue_; + std::string what_; +}; + /** * Subprocess. */ @@ -179,7 +199,11 @@ class Subprocess : private boost::noncopyable { class Options : private boost::orable { friend class Subprocess; public: - Options() : closeOtherFds_(false), usePath_(false) { } + Options() + : closeOtherFds_(false), + usePath_(false), + parentDeathSignal_(0) { + } /** * Change action for file descriptor fd. @@ -233,6 +257,14 @@ class Subprocess : private boost::noncopyable { */ Options& usePath() { usePath_ = true; return *this; } + /** + * Child will receive a signal when the parent exits. + */ + Options& parentDeathSignal(int sig) { + parentDeathSignal_ = sig; + return *this; + } + /** * Helpful way to combine Options. */ @@ -243,6 +275,7 @@ class Subprocess : private boost::noncopyable { FdMap fdActions_; bool closeOtherFds_; bool usePath_; + int parentDeathSignal_; }; static Options pipeStdin() { return Options().stdin(PIPE); } @@ -444,16 +477,34 @@ class Subprocess : private boost::noncopyable { static const int RV_RUNNING = ProcessReturnCode::RV_RUNNING; static const int RV_NOT_STARTED = ProcessReturnCode::RV_NOT_STARTED; + // spawn() sets up a pipe to read errors from the child, + // then calls spawnInternal() to do the bulk of the work. Once + // spawnInternal() returns it reads the error pipe to see if the child + // encountered any errors. void spawn( std::unique_ptr argv, const char* executable, const Options& options, const std::vector* env); + void spawnInternal( + std::unique_ptr argv, + const char* executable, + Options& options, + const std::vector* env, + int errFd); - // Action to run in child. + // Actions to run in child. // Note that this runs after vfork(), so tread lightly. - void runChild(const char* executable, char** argv, char** env, - const Options& options) const; + // Returns 0 on success, or an errno value on failure. + int prepareChild(const Options& options, const sigset_t* sigmask) const; + int runChild(const char* executable, char** argv, char** env, + const Options& options) const; + + /** + * Read from the error pipe, and throw SubprocessSpawnError if the child + * failed before calling exec(). + */ + void readChildErrorPipe(int pfd, const char* executable); /** * Close all file descriptors. diff --git a/hphp/third_party/folly/folly/ThreadLocal.h b/hphp/third_party/folly/folly/ThreadLocal.h index c11819354..b4c031e75 100644 --- a/hphp/third_party/folly/folly/ThreadLocal.h +++ b/hphp/third_party/folly/folly/ThreadLocal.h @@ -42,22 +42,6 @@ #include "folly/Likely.h" #include -// Use noexcept on gcc 4.6 or higher -#undef FOLLY_NOEXCEPT -#ifdef __GNUC__ -# ifdef HAVE_FEATURES_H -# include -# if __GNUC_PREREQ(4,6) -# define FOLLY_NOEXCEPT noexcept -# define FOLLY_ASSERT(x) x -# endif -# endif -#endif - -#ifndef FOLLY_NOEXCEPT -# define FOLLY_NOEXCEPT -# define FOLLY_ASSERT(x) /**/ -#endif namespace folly { enum class TLPDestructionMode { @@ -79,11 +63,12 @@ class ThreadLocal { T* get() const { T* ptr = tlp_.get(); - if (UNLIKELY(ptr == nullptr)) { - ptr = new T(); - tlp_.reset(ptr); + if (LIKELY(ptr != nullptr)) { + return ptr; } - return ptr; + + // separated new item creation out to speed up the fast path. + return makeTlp(); } T* operator->() const { @@ -112,6 +97,12 @@ class ThreadLocal { ThreadLocal(const ThreadLocal&) = delete; ThreadLocal& operator=(const ThreadLocal&) = delete; + T* makeTlp() const { + T* ptr = new T(); + tlp_.reset(ptr); + return ptr; + } + mutable ThreadLocalPtr tlp_; }; @@ -276,7 +267,7 @@ class ThreadLocalPtr { Accessor(const Accessor&) = delete; Accessor& operator=(const Accessor&) = delete; - Accessor(Accessor&& other) FOLLY_NOEXCEPT + Accessor(Accessor&& other) noexcept : meta_(other.meta_), lock_(other.lock_), id_(other.id_) { @@ -284,7 +275,7 @@ class ThreadLocalPtr { other.lock_ = nullptr; } - Accessor& operator=(Accessor&& other) FOLLY_NOEXCEPT { + Accessor& operator=(Accessor&& other) noexcept { // Each Tag has its own unique meta, and accessors with different Tags // have different types. So either *this is empty, or this and other // have the same tag. But if they have the same tag, they have the same @@ -324,8 +315,8 @@ class ThreadLocalPtr { // accessor allows a client to iterate through all thread local child // elements of this ThreadLocal instance. Holds a global lock for each Accessor accessAllThreads() const { - FOLLY_ASSERT(static_assert(!std::is_same::value, - "Must use a unique Tag to use the accessAllThreads feature")); + static_assert(!std::is_same::value, + "Must use a unique Tag to use the accessAllThreads feature"); return Accessor(id_); } @@ -343,8 +334,6 @@ class ThreadLocalPtr { int id_; // every instantiation has a unique id }; -#undef FOLLY_NOEXCEPT - } // namespace folly #endif /* FOLLY_THREADLOCAL_H_ */ diff --git a/hphp/third_party/folly/folly/Traits.h b/hphp/third_party/folly/folly/Traits.h index bc0a0bf68..db8bd33a3 100644 --- a/hphp/third_party/folly/folly/Traits.h +++ b/hphp/third_party/folly/folly/Traits.h @@ -20,6 +20,7 @@ #define FOLLY_BASE_TRAITS_H_ #include +#include #include #include @@ -277,10 +278,42 @@ struct IsOneOf { enum { value = std::is_same::value || IsOneOf::value }; }; -/* - * Complementary type traits to check for a negative/non-positive value. +/** + * A traits class to check for incomplete types. * - * `if(x < 0)` yields an error in clang for unsigned types when -Werror is used + * Example: + * + * struct FullyDeclared {}; // complete type + * struct ForwardDeclared; // incomplete type + * + * is_complete::value // evaluates to true + * is_complete::value // evaluates to true + * is_complete::value // evaluates to false + * + * struct ForwardDeclared {}; // declared, at last + * + * is_complete::value // now it evaluates to true + * + * @author: Marcelo Juchem + */ +template +class is_complete { + template struct sfinae {}; + template + constexpr static bool test(sfinae*) { return true; } + template constexpr static bool test(...) { return false; } +public: + constexpr static bool value = test(nullptr); +}; + +/* + * Complementary type traits for integral comparisons. + * + * For instance, `if(x < 0)` yields an error in clang for unsigned types + * when -Werror is used due to -Wtautological-compare + * + * + * @author: Marcelo Juchem */ namespace detail { @@ -295,6 +328,68 @@ struct is_negative_impl { constexpr static bool check(T x) { return false; } }; +template +bool less_than_impl( + typename std::enable_if< + (rhs <= std::numeric_limits::max() + && rhs >= std::numeric_limits::min()), + LHS + >::type const lhs +) { + return lhs < rhs; +} + +template +bool less_than_impl( + typename std::enable_if< + (rhs > std::numeric_limits::max()), + LHS + >::type const +) { + return true; +} + +template +bool less_than_impl( + typename std::enable_if< + (rhs < std::numeric_limits::min()), + LHS + >::type const +) { + return false; +} + +template +bool greater_than_impl( + typename std::enable_if< + (lhs <= std::numeric_limits::max() + && lhs >= std::numeric_limits::min()), + RHS + >::type const rhs +) { + return lhs < rhs; +} + +template +bool greater_than_impl( + typename std::enable_if< + (lhs > std::numeric_limits::max()), + RHS + >::type const +) { + return false; +} + +template +bool greater_than_impl( + typename std::enable_if< + (lhs < std::numeric_limits::min()), + RHS + >::type const +) { + return true; +} + } // namespace detail { // same as `x < 0` @@ -307,6 +402,20 @@ constexpr bool is_negative(T x) { template constexpr bool is_non_positive(T x) { return !x || folly::is_negative(x); } +template +bool less_than(LHS const lhs) { + return detail::less_than_impl< + RHS, rhs, typename std::remove_reference::type + >(lhs); +} + +template +bool greater_than(RHS const rhs) { + return detail::greater_than_impl< + LHS, lhs, typename std::remove_reference::type + >(rhs); +} + } // namespace folly FOLLY_ASSUME_FBVECTOR_COMPATIBLE_3(std::basic_string); @@ -382,6 +491,8 @@ FOLLY_ASSUME_FBVECTOR_COMPATIBLE_1(boost::shared_ptr); * cout << "Does class Bar have a member double test(const string&, long)? " * << boolalpha << has_test_traits::value; * } + * + * @author: Marcelo Juchem */ #define FOLLY_CREATE_HAS_MEMBER_FN_TRAITS(classname, func_name) \ template class classname; \ diff --git a/hphp/third_party/folly/folly/detail/Stats.h b/hphp/third_party/folly/folly/detail/Stats.h index 3e5ef06ff..4729eeed4 100644 --- a/hphp/third_party/folly/folly/detail/Stats.h +++ b/hphp/third_party/folly/folly/detail/Stats.h @@ -17,6 +17,7 @@ #ifndef FOLLY_DETAIL_STATS_H_ #define FOLLY_DETAIL_STATS_H_ +#include #include #include @@ -49,6 +50,36 @@ avgHelper(ValueType sum, uint64_t count) { return static_cast(sumf / countf); } +/* + * Helper function to compute the rate per Interval, + * given the specified count recorded over the elapsed time period. + */ +template +ReturnType rateHelper(ReturnType count, TimeType elapsed) { + if (elapsed == TimeType(0)) { + return 0; + } + + // Use std::chrono::duration_cast to convert between the native + // duration and the desired interval. However, convert the rates, + // rather than just converting the elapsed duration. Converting the + // elapsed time first may collapse it down to 0 if the elapsed interval + // is less than the desired interval, which will incorrectly result in + // an infinite rate. + typedef std::chrono::duration< + ReturnType, std::ratio> NativeRate; + typedef std::chrono::duration< + ReturnType, std::ratio> DesiredRate; + + NativeRate native(count / elapsed.count()); + DesiredRate desired = std::chrono::duration_cast(native); + return desired.count(); +} + template struct Bucket { diff --git a/hphp/third_party/folly/folly/detail/ThreadLocalDetail.h b/hphp/third_party/folly/folly/detail/ThreadLocalDetail.h index 0cc9193b0..537e6b6b7 100644 --- a/hphp/third_party/folly/folly/detail/ThreadLocalDetail.h +++ b/hphp/third_party/folly/folly/detail/ThreadLocalDetail.h @@ -145,6 +145,7 @@ struct StaticMeta { // Leak it on exit, there's only one per process and we don't have to // worry about synchronization with exiting threads. static bool constructed = (inst = new StaticMeta()); + (void)constructed; // suppress unused warning return *inst; } @@ -259,59 +260,66 @@ struct StaticMeta { } } - static ElementWrapper& get(int id) { + /** + * Reserve enough space in the threadEntry_.elements for the item + * @id to fit in. + */ + static void reserve(int id) { size_t prevSize = threadEntry_.elementsCapacity; - if (prevSize <= id) { - size_t newSize = static_cast((id + 5) * 1.7); - auto & meta = instance(); - ElementWrapper* ptr = NULL; - // Rely on jemalloc to zero the memory if possible -- maybe it knows - // it's already zeroed and saves us some work. - if (!usingJEMalloc() || - prevSize < jemallocMinInPlaceExpandable || - (rallocm( - static_cast(static_cast(&threadEntry_.elements)), - NULL, newSize * sizeof(ElementWrapper), 0, - ALLOCM_NO_MOVE | ALLOCM_ZERO) != ALLOCM_SUCCESS)) { - // Sigh, must realloc, but we can't call realloc here, as elements is - // still linked in meta, so another thread might access invalid memory - // after realloc succeeds. We'll copy by hand and update threadEntry_ - // under the lock. - // - // Note that we're using calloc instead of malloc in order to zero - // the entire region. rallocm (ALLOCM_ZERO) will only zero newly - // allocated memory, so if a previous allocation allocated more than - // we requested, it's our responsibility to guarantee that the tail - // is zeroed. calloc() is simpler than malloc() followed by memset(), - // and potentially faster when dealing with a lot of memory, as - // it can get already-zeroed pages from the kernel. - if ((ptr = static_cast( - calloc(newSize, sizeof(ElementWrapper)))) != NULL) { - memcpy(ptr, threadEntry_.elements, - sizeof(ElementWrapper) * prevSize); - } else { - throw std::bad_alloc(); - } + size_t newSize = static_cast((id + 5) * 1.7); + auto& meta = instance(); + ElementWrapper* ptr = nullptr; + // Rely on jemalloc to zero the memory if possible -- maybe it knows + // it's already zeroed and saves us some work. + if (!usingJEMalloc() || + prevSize < jemallocMinInPlaceExpandable || + (rallocm( + static_cast(static_cast(&threadEntry_.elements)), + NULL, newSize * sizeof(ElementWrapper), 0, + ALLOCM_NO_MOVE | ALLOCM_ZERO) != ALLOCM_SUCCESS)) { + // Sigh, must realloc, but we can't call realloc here, as elements is + // still linked in meta, so another thread might access invalid memory + // after realloc succeeds. We'll copy by hand and update threadEntry_ + // under the lock. + // + // Note that we're using calloc instead of malloc in order to zero + // the entire region. rallocm (ALLOCM_ZERO) will only zero newly + // allocated memory, so if a previous allocation allocated more than + // we requested, it's our responsibility to guarantee that the tail + // is zeroed. calloc() is simpler than malloc() followed by memset(), + // and potentially faster when dealing with a lot of memory, as + // it can get already-zeroed pages from the kernel. + if ((ptr = static_cast( + calloc(newSize, sizeof(ElementWrapper)))) != nullptr) { + memcpy(ptr, threadEntry_.elements, sizeof(ElementWrapper) * prevSize); + } else { + throw std::bad_alloc(); } + } - // Success, update the entry - { - boost::lock_guard g(meta.lock_); - if (prevSize == 0) { - meta.push_back(&threadEntry_); - } - if (ptr) { - using std::swap; - swap(ptr, threadEntry_.elements); - } - threadEntry_.elementsCapacity = newSize; - } - - free(ptr); - + // Success, update the entry + { + boost::lock_guard g(meta.lock_); if (prevSize == 0) { - pthread_setspecific(meta.pthreadKey_, &meta); + meta.push_back(&threadEntry_); } + if (ptr) { + using std::swap; + swap(ptr, threadEntry_.elements); + } + threadEntry_.elementsCapacity = newSize; + } + + free(ptr); + + if (prevSize == 0) { + pthread_setspecific(meta.pthreadKey_, &meta); + } + } + + static ElementWrapper& get(int id) { + if (UNLIKELY(threadEntry_.elementsCapacity <= id)) { + reserve(id); } return threadEntry_.elements[id]; } diff --git a/hphp/third_party/folly/folly/dynamic-inl.h b/hphp/third_party/folly/folly/dynamic-inl.h index ae38798b2..bae776a24 100644 --- a/hphp/third_party/folly/folly/dynamic-inl.h +++ b/hphp/third_party/folly/folly/dynamic-inl.h @@ -325,6 +325,7 @@ inline dynamic::const_iterator dynamic::end() const { template struct dynamic::IterableProxy { typedef It const_iterator; + typedef typename It::value_type value_type; /* implicit */ IterableProxy(const dynamic::ObjectImpl* o) : o_(o) { } diff --git a/hphp/third_party/folly/folly/dynamic.cpp b/hphp/third_party/folly/folly/dynamic.cpp index 25ab1032b..0278dad9b 100644 --- a/hphp/third_party/folly/folly/dynamic.cpp +++ b/hphp/third_party/folly/folly/dynamic.cpp @@ -34,6 +34,10 @@ DEF_TYPE(dynamic::ObjectImpl, "object", dynamic::OBJECT); #undef DEF_TYPE +const char* dynamic::typeName() const { + return typeName(type_); +} + ////////////////////////////////////////////////////////////////////// } diff --git a/hphp/third_party/folly/folly/dynamic.h b/hphp/third_party/folly/folly/dynamic.h index 88d23d9f5..6ab5ca1bf 100644 --- a/hphp/third_party/folly/folly/dynamic.h +++ b/hphp/third_party/folly/folly/dynamic.h @@ -253,6 +253,11 @@ public: */ Type type() const; + /* + * Returns the type of this dynamic as a printable string. + */ + const char* typeName() const; + /* * Extract a value while trying to convert to the specified type. * Throws exceptions if we cannot convert from the real type to the diff --git a/hphp/third_party/folly/folly/experimental/EliasFanoCoding.h b/hphp/third_party/folly/folly/experimental/EliasFanoCoding.h index 06d54978a..d6232b2df 100644 --- a/hphp/third_party/folly/folly/experimental/EliasFanoCoding.h +++ b/hphp/third_party/folly/folly/experimental/EliasFanoCoding.h @@ -33,7 +33,6 @@ #endif #include -#include #include #include #include @@ -44,7 +43,7 @@ #include "folly/Likely.h" #include "folly/Range.h" -#if __BYTE_ORDER != __LITTLE_ENDIAN +#if __BYTE_ORDER__ != __ORDER_LITTLE_ENDIAN__ #error EliasFanoCoding.h requires little endianness #endif diff --git a/hphp/third_party/folly/folly/experimental/Gen-inl.h b/hphp/third_party/folly/folly/experimental/Gen-inl.h index dd7425006..6a90d2665 100644 --- a/hphp/third_party/folly/folly/experimental/Gen-inl.h +++ b/hphp/third_party/folly/folly/experimental/Gen-inl.h @@ -156,8 +156,10 @@ class GenImpl : public FBounded { /** * apply() - Send all values produced by this generator to given - * handler until it returns false. Returns true if the false iff the handler - * returned false. + * handler until the handler returns false. Returns true until the handler + * returns false. GOTCHA: It should return true even if it completes (without + * the handler returning false), as 'Chain' uses the return value of apply + * to determine if it should process the second object in its chain. */ template bool apply(Handler&& handler) const; @@ -773,6 +775,84 @@ class Take : public Operator { } }; +/** + * Sample - For taking a random sample of N elements from a sequence + * (without replacement). + */ +template +class Sample : public Operator> { + size_t count_; + Random rng_; + public: + explicit Sample(size_t count, Random rng) + : count_(count), rng_(std::move(rng)) {} + + template::type> + class Generator : + public GenImpl> { + static_assert(!Source::infinite, "Cannot sample infinite source!"); + // It's too easy to bite ourselves if random generator is only 16-bit + static_assert(Random::max() >= std::numeric_limits::max() - 1, + "Random number generator must support big values"); + Source source_; + size_t count_; + mutable Rand rng_; + public: + explicit Generator(Source source, size_t count, Random rng) + : source_(std::move(source)) , count_(count), rng_(std::move(rng)) {} + + template + bool apply(Handler&& handler) const { + if (count_ == 0) { return false; } + std::vector v; + v.reserve(count_); + // use reservoir sampling to give each source value an equal chance + // of appearing in our output. + size_t n = 1; + source_.foreach([&](Value value) -> void { + if (v.size() < count_) { + v.push_back(std::forward(value)); + } else { + // alternatively, we could create a std::uniform_int_distribution + // instead of using modulus, but benchmarks show this has + // substantial overhead. + size_t index = rng_() % n; + if (index < v.size()) { + v[index] = std::forward(value); + } + } + ++n; + }); + + // output is unsorted! + for (auto& val: v) { + if (!handler(std::move(val))) { + return false; + } + } + return true; + } + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), count_, rng_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), count_, rng_); + } +}; + /** * Skip - For skipping N items from the beginning of a source generator. * @@ -901,11 +981,11 @@ class Order : public Operator> { } public: Generator(Source source, - const Selector& selector, - const Comparer& comparer) + Selector selector, + Comparer comparer) : source_(std::move(source)), - selector_(selector), - comparer_(comparer) {} + selector_(std::move(selector)), + comparer_(std::move(comparer)) {} VectorType operator|(const Collect&) const { return asVector(); @@ -956,6 +1036,86 @@ class Order : public Operator> { } }; +/** + * Distinct - For filtering duplicates out of a sequence. A selector may be + * provided to generate a key to uniquify for each value. + * + * This type is usually used through the 'distinct' helper function, like: + * + * auto closest = from(results) + * | distinctBy([](Item& i) { + * return i.target; + * }) + * | take(10); + */ +template +class Distinct : public Operator> { + Selector selector_; + public: + Distinct() {} + + explicit Distinct(Selector selector) + : selector_(std::move(selector)) + {} + + template + class Generator : public GenImpl> { + Source source_; + Selector selector_; + + typedef typename std::decay::type StorageType; + + // selector_ cannot be passed an rvalue or it would end up passing the husk + // of a value to the downstream operators. + typedef const StorageType& ParamType; + + typedef typename std::result_of::type KeyType; + typedef typename std::decay::type KeyStorageType; + + public: + Generator(Source source, + Selector selector) + : source_(std::move(source)), + selector_(std::move(selector)) {} + + template + void foreach(Body&& body) const { + std::unordered_set keysSeen; + source_.foreach([&](Value value) { + if (keysSeen.insert(selector_(ParamType(value))).second) { + body(std::forward(value)); + } + }); + } + + template + bool apply(Handler&& handler) const { + std::unordered_set keysSeen; + return source_.apply([&](Value value) -> bool { + if (keysSeen.insert(selector_(ParamType(value))).second) { + return handler(std::forward(value)); + } + return true; + }); + } + }; + + template> + Gen compose(GenImpl&& source) const { + return Gen(std::move(source.self()), selector_); + } + + template> + Gen compose(const GenImpl& source) const { + return Gen(source.self(), selector_); + } +}; + /** * Composed - For building up a pipeline of operations to perform, absent any * particular source generator. Useful for building up custom pipelines. @@ -1614,6 +1774,8 @@ static const detail::Min max; static const detail::Order order; +static const detail::Distinct distinct; + static const detail::Map move; static const detail::Concat concat; @@ -1624,6 +1786,11 @@ inline detail::Take take(size_t count) { return detail::Take(count); } +template +inline detail::Sample sample(size_t count, Random rng = Random()) { + return detail::Sample(count, std::move(rng)); +} + inline detail::Skip skip(size_t count) { return detail::Skip(count); } diff --git a/hphp/third_party/folly/folly/experimental/Gen.h b/hphp/third_party/folly/folly/experimental/Gen.h index da64fde93..130cc7732 100644 --- a/hphp/third_party/folly/folly/experimental/Gen.h +++ b/hphp/third_party/folly/folly/experimental/Gen.h @@ -21,6 +21,9 @@ #include #include #include +#include +#include +#include #include "folly/Range.h" #include "folly/Optional.h" @@ -208,11 +211,17 @@ class Until; class Take; +template +class Sample; + class Skip; template class Order; +template +class Distinct; + template class Composed; @@ -294,7 +303,7 @@ From from(std::initializer_list source) { template> + Container>> From from(Container&& source) { return From(std::move(source)); } @@ -380,6 +389,12 @@ Order orderByDescending(Selector selector = Identity()) { return Order(std::move(selector)); } +template> +Distinct distinctBy(Selector selector = Identity()) { + return Distinct(std::move(selector)); +} + template>> Get get() { diff --git a/hphp/third_party/folly/folly/experimental/TestUtil.cpp b/hphp/third_party/folly/folly/experimental/TestUtil.cpp index c7384fca6..d68235fb1 100644 --- a/hphp/third_party/folly/folly/experimental/TestUtil.cpp +++ b/hphp/third_party/folly/folly/experimental/TestUtil.cpp @@ -16,62 +16,56 @@ #include "folly/experimental/TestUtil.h" -#include -#include -#include -#include +#include +#include +#include -#include "folly/Format.h" +#include "folly/Conv.h" +#include "folly/Exception.h" namespace folly { namespace test { -TemporaryFile::TemporaryFile(const char* prefix, Scope scope, +namespace { + +fs::path generateUniquePath(fs::path path, StringPiece namePrefix) { + if (path.empty()) { + path = fs::temp_directory_path(); + } + if (namePrefix.empty()) { + path /= fs::unique_path(); + } else { + path /= fs::unique_path( + to(namePrefix, ".%%%%-%%%%-%%%%-%%%%")); + } + return path; +} + +} // namespce + +TemporaryFile::TemporaryFile(StringPiece namePrefix, + fs::path dir, + Scope scope, bool closeOnDestruction) : scope_(scope), - closeOnDestruction_(closeOnDestruction) { - static const char* suffix = ".XXXXXX"; // per mkstemp(3) - if (!prefix || prefix[0] == '\0') { - prefix = "temp"; - } - const char* dir = nullptr; - if (!strchr(prefix, '/')) { - // Not a full path, try getenv("TMPDIR") or "/tmp" - dir = getenv("TMPDIR"); - if (!dir) { - dir = "/tmp"; - } - // The "!" is a placeholder to ensure that &(path[0]) is null-terminated. - // This is the only standard-compliant way to get at a null-terminated - // non-const char string inside a std::string: put the null-terminator - // yourself. - path_ = format("{}/{}{}!", dir, prefix, suffix).str(); - } else { - path_ = format("{}{}!", prefix, suffix).str(); - } - - // Replace the '!' with a null terminator, we'll get rid of it later - path_[path_.size() - 1] = '\0'; - - fd_ = mkstemp(&(path_[0])); - if (fd_ == -1) { - throw std::system_error(errno, std::system_category(), - format("mkstemp failed: {}", path_).str().c_str()); - } - - DCHECK_EQ(path_[path_.size() - 1], '\0'); - path_.erase(path_.size() - 1); + closeOnDestruction_(closeOnDestruction), + fd_(-1), + path_(generateUniquePath(std::move(dir), namePrefix)) { + fd_ = open(path_.c_str(), O_RDWR | O_CREAT | O_EXCL, 0644); + checkUnixError(fd_, "open failed"); if (scope_ == Scope::UNLINK_IMMEDIATELY) { - if (unlink(path_.c_str()) == -1) { - throw std::system_error(errno, std::system_category(), - format("unlink failed: {}", path_).str().c_str()); + boost::system::error_code ec; + fs::remove(path_, ec); + if (ec) { + LOG(WARNING) << "unlink on construction failed: " << ec; + } else { + path_.clear(); } - path_.clear(); // path no longer available or meaningful } } -const std::string& TemporaryFile::path() const { +const fs::path& TemporaryFile::path() const { CHECK(scope_ != Scope::UNLINK_IMMEDIATELY); DCHECK(!path_.empty()); return path_; @@ -87,8 +81,28 @@ TemporaryFile::~TemporaryFile() { // If we previously failed to unlink() (UNLINK_IMMEDIATELY), we'll // try again here. if (scope_ != Scope::PERMANENT && !path_.empty()) { - if (unlink(path_.c_str()) == -1) { - PLOG(ERROR) << "unlink(" << path_ << ") failed"; + boost::system::error_code ec; + fs::remove(path_, ec); + if (ec) { + LOG(WARNING) << "unlink on destruction failed: " << ec; + } + } +} + +TemporaryDirectory::TemporaryDirectory(StringPiece namePrefix, + fs::path dir, + Scope scope) + : scope_(scope), + path_(generateUniquePath(std::move(dir), namePrefix)) { + fs::create_directory(path_); +} + +TemporaryDirectory::~TemporaryDirectory() { + if (scope_ == Scope::DELETE_ON_DESTRUCTION) { + boost::system::error_code ec; + fs::remove_all(path_, ec); + if (ec) { + LOG(WARNING) << "recursive delete on destruction failed: " << ec; } } } diff --git a/hphp/third_party/folly/folly/experimental/TestUtil.h b/hphp/third_party/folly/folly/experimental/TestUtil.h index c04061076..06e2f0487 100644 --- a/hphp/third_party/folly/folly/experimental/TestUtil.h +++ b/hphp/third_party/folly/folly/experimental/TestUtil.h @@ -18,6 +18,8 @@ #define FOLLY_TESTUTIL_H_ #include +#include "folly/Range.h" +#include "folly/experimental/io/FsUtil.h" namespace folly { namespace test { @@ -27,8 +29,7 @@ namespace test { * * By default, the file is created in a system-specific location (the value * of the TMPDIR environment variable, or /tmp), but you can override that - * by making "prefix" be a path (containing a '/'; use a prefix starting with - * './' to create a file in the current directory). + * with a different (non-empty) directory passed to the constructor. * * By default, the file is closed and deleted when the TemporaryFile object * is destroyed, but both these behaviors can be overridden with arguments @@ -41,19 +42,50 @@ class TemporaryFile { UNLINK_IMMEDIATELY, UNLINK_ON_DESTRUCTION }; - explicit TemporaryFile(const char* prefix=nullptr, - Scope scope=Scope::UNLINK_ON_DESTRUCTION, - bool closeOnDestruction=true); + explicit TemporaryFile(StringPiece namePrefix = StringPiece(), + fs::path dir = fs::path(), + Scope scope = Scope::UNLINK_ON_DESTRUCTION, + bool closeOnDestruction = true); ~TemporaryFile(); int fd() const { return fd_; } - const std::string& path() const; + const fs::path& path() const; private: Scope scope_; bool closeOnDestruction_; int fd_; - std::string path_; + fs::path path_; +}; + +/** + * Temporary directory. + * + * By default, the temporary directory is created in a system-specific + * location (the value of the TMPDIR environment variable, or /tmp), but you + * can override that with a non-empty directory passed to the constructor. + * + * By default, the directory is recursively deleted when the TemporaryDirectory + * object is destroyed, but that can be overridden with an argument + * to the constructor. + */ + +class TemporaryDirectory { + public: + enum class Scope { + PERMANENT, + DELETE_ON_DESTRUCTION + }; + explicit TemporaryDirectory(StringPiece namePrefix = StringPiece(), + fs::path dir = fs::path(), + Scope scope = Scope::DELETE_ON_DESTRUCTION); + ~TemporaryDirectory(); + + const fs::path& path() const { return path_; } + + private: + Scope scope_; + fs::path path_; }; } // namespace test diff --git a/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracer.cpp b/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracer.cpp index d79eab853..a2bd9ef02 100644 --- a/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracer.cpp +++ b/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracer.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include "folly/experimental/exception_tracer/ExceptionAbi.h" diff --git a/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracer.h b/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracer.h index 84800bf39..809d81af8 100644 --- a/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracer.h +++ b/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracer.h @@ -20,7 +20,8 @@ #ifndef FOLLY_EXPERIMENTAL_EXCEPTION_TRACER_EXCEPTIONTRACER_H_ #define FOLLY_EXPERIMENTAL_EXCEPTION_TRACER_EXCEPTIONTRACER_H_ -#include +#include +#include #include #include diff --git a/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracerLib.cpp b/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracerLib.cpp index 2a7c954b8..eab33fd3e 100644 --- a/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracerLib.cpp +++ b/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracerLib.cpp @@ -20,6 +20,7 @@ #include +#include "folly/Portability.h" #include "folly/experimental/exception_tracer/StackTrace.h" #include "folly/experimental/exception_tracer/ExceptionAbi.h" #include "folly/experimental/exception_tracer/ExceptionTracer.h" @@ -28,9 +29,9 @@ namespace __cxxabiv1 { extern "C" { void __cxa_throw(void* thrownException, std::type_info* type, - void (*destructor)(void)) __attribute__((noreturn)); + void (*destructor)(void)) FOLLY_NORETURN; void* __cxa_begin_catch(void* excObj); -void __cxa_rethrow(void) __attribute__((noreturn)); +void __cxa_rethrow(void) FOLLY_NORETURN; void __cxa_end_catch(void); } @@ -45,10 +46,10 @@ pthread_once_t initialized = PTHREAD_ONCE_INIT; extern "C" { typedef void (*CxaThrowType)(void*, std::type_info*, void (*)(void)) - __attribute__((noreturn)); + FOLLY_NORETURN; typedef void* (*CxaBeginCatchType)(void*); typedef void (*CxaRethrowType)(void) - __attribute__((noreturn)); + FOLLY_NORETURN; typedef void (*CxaEndCatchType)(void); CxaThrowType orig_cxa_throw; @@ -58,7 +59,7 @@ CxaEndCatchType orig_cxa_end_catch; } // extern "C" typedef void (*RethrowExceptionType)(std::exception_ptr) - __attribute__((noreturn)); + FOLLY_NORETURN; RethrowExceptionType orig_rethrow_exception; void initialize() { diff --git a/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracerTest.cpp b/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracerTest.cpp index d18364bdf..5bf2dff46 100644 --- a/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracerTest.cpp +++ b/hphp/third_party/folly/folly/experimental/exception_tracer/ExceptionTracerTest.cpp @@ -15,6 +15,7 @@ */ +#include #include #include "folly/experimental/exception_tracer/ExceptionTracer.h" diff --git a/hphp/third_party/folly/folly/experimental/symbolizer/Elf.cpp b/hphp/third_party/folly/folly/experimental/symbolizer/Elf.cpp index f28ed07b0..33e20ddef 100644 --- a/hphp/third_party/folly/folly/experimental/symbolizer/Elf.cpp +++ b/hphp/third_party/folly/folly/experimental/symbolizer/Elf.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #include @@ -130,9 +129,9 @@ void ElfFile::init() { #undef EXPECTED_CLASS // Validate ELF data encoding (LSB/MSB) -#if __BYTE_ORDER == __LITTLE_ENDIAN +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ # define EXPECTED_ENCODING ELFDATA2LSB -#elif __BYTE_ORDER == __BIG_ENDIAN +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ # define EXPECTED_ENCODING ELFDATA2MSB #else # error Unsupported byte order diff --git a/hphp/third_party/folly/folly/io/Cursor.h b/hphp/third_party/folly/folly/io/Cursor.h index 73a5493a7..790b9cca4 100644 --- a/hphp/third_party/folly/folly/io/Cursor.h +++ b/hphp/third_party/folly/folly/io/Cursor.h @@ -84,6 +84,76 @@ class CursorBase { return Endian::little(read()); } + /** + * Read a fixed-length string. + * + * The std::string-based APIs should probably be avoided unless you + * ultimately want the data to live in an std::string. You're better off + * using the pull() APIs to copy into a raw buffer otherwise. + */ + std::string readFixedString(size_t len) { + std::string str; + + str.reserve(len); + for (;;) { + // Fast path: it all fits in one buffer. + size_t available = length(); + if (LIKELY(available >= len)) { + str.append(reinterpret_cast(data()), len); + offset_ += len; + return str; + } + + str.append(reinterpret_cast(data()), available); + if (UNLIKELY(!tryAdvanceBuffer())) { + throw std::out_of_range("string underflow"); + } + len -= available; + } + } + + /** + * Read a string consisting of bytes until the given terminator character is + * seen. Raises an std::length_error if maxLength bytes have been processed + * before the terminator is seen. + * + * See comments in readFixedString() about when it's appropriate to use this + * vs. using pull(). + */ + std::string readTerminatedString( + char termChar = '\0', + size_t maxLength = std::numeric_limits::max()) { + std::string str; + + for (;;) { + const uint8_t* buf = data(); + size_t buflen = length(); + + size_t i = 0; + while (i < buflen && buf[i] != termChar) { + ++i; + + // Do this check after incrementing 'i', as even though we start at the + // 0 byte, it still represents a single character + if (str.length() + i >= maxLength) { + throw std::length_error("string overflow"); + } + } + + str.append(reinterpret_cast(buf), i); + if (i < buflen) { + skip(i + 1); + return str; + } + + skip(i); + + if (UNLIKELY(!tryAdvanceBuffer())) { + throw std::out_of_range("string underflow"); + } + } + } + explicit CursorBase(BufType* buf) : crtBuf_(buf) , offset_(0) diff --git a/hphp/third_party/folly/folly/stats/BucketedTimeSeries-defs.h b/hphp/third_party/folly/folly/stats/BucketedTimeSeries-defs.h index a4d7c4387..e5892e62c 100644 --- a/hphp/third_party/folly/folly/stats/BucketedTimeSeries-defs.h +++ b/hphp/third_party/folly/folly/stats/BucketedTimeSeries-defs.h @@ -161,13 +161,12 @@ void BucketedTimeSeries::clear() { template -TT BucketedTimeSeries::elapsed() const { +TT BucketedTimeSeries::getEarliestTime() const { if (empty()) { return TimeType(0); } - if (isAllTime()) { - return latestTime_ - firstTime_ + TimeType(1); + return firstTime_; } size_t currentBucket; @@ -183,7 +182,28 @@ TT BucketedTimeSeries::elapsed() const { // We're never tracking data before firstTime_ earliestTime = std::max(earliestTime, firstTime_); - return latestTime_ - earliestTime + TimeType(1); + return earliestTime; +} + +template +TT BucketedTimeSeries::elapsed() const { + if (empty()) { + return TimeType(0); + } + + // Add 1 since [latestTime_, earliestTime] is an inclusive interval. + return latestTime_ - getEarliestTime() + TimeType(1); +} + +template +TT BucketedTimeSeries::elapsed(TimeType start, TimeType end) const { + if (empty()) { + return TimeType(0); + } + start = std::max(start, getEarliestTime()); + end = std::min(end, latestTime_ + TimeType(1)); + end = std::max(start, end); + return end - start; } template diff --git a/hphp/third_party/folly/folly/stats/BucketedTimeSeries.h b/hphp/third_party/folly/folly/stats/BucketedTimeSeries.h index ef852b53d..22d8421dc 100644 --- a/hphp/third_party/folly/folly/stats/BucketedTimeSeries.h +++ b/hphp/third_party/folly/folly/stats/BucketedTimeSeries.h @@ -104,11 +104,29 @@ class BucketedTimeSeries { /* * Get the latest time that has ever been passed to update() or addValue(). + * + * If no data has ever been added to this timeseries, 0 will be returned. */ TimeType getLatestTime() const { return latestTime_; } + /* + * Get the time of the earliest data point stored in this timeseries. + * + * If no data has ever been added to this timeseries, 0 will be returned. + * + * If isAllTime() is true, this is simply the time when the first data point + * was recorded. + * + * For non-all-time data, the timestamp reflects the first data point still + * remembered. As new data points are added, old data will be expired. + * getEarliestTime() returns the timestamp of the oldest bucket still present + * in the timeseries. This will never be older than (getLatestTime() - + * duration()). + */ + TimeType getEarliestTime() const; + /* * Return the number of buckets. */ @@ -170,6 +188,16 @@ class BucketedTimeSeries { */ TimeType elapsed() const; + /* + * Get the amount of time tracked by this timeseries, between the specified + * start and end times. + * + * If the timeseries contains data for the entire time range specified, this + * simply returns (end - start). However, if start is earlier than + * getEarliestTime(), this returns (end - getEarliestTime()). + */ + TimeType elapsed(TimeType start, TimeType end) const; + /* * Return the sum of all the data points currently tracked by this * BucketedTimeSeries. @@ -243,8 +271,8 @@ class BucketedTimeSeries { * * Note that data outside of the timeseries duration will no longer be * available for use in the estimation. Specifying a start time earlier than - * (getLatestTime() - elapsed()) will not have much effect, since only data - * points after that point in time will be counted. + * getEarliestTime() will not have much effect, since only data points after + * that point in time will be counted. * * Note that the value returned is an estimate, and may not be precise. */ @@ -277,7 +305,8 @@ class BucketedTimeSeries { template ReturnType rate(TimeType start, TimeType end) const { ValueType intervalSum = sum(start, end); - return rateHelper(intervalSum, end - start); + TimeType interval = elapsed(start, end); + return rateHelper(intervalSum, interval); } /* @@ -290,7 +319,8 @@ class BucketedTimeSeries { template ReturnType countRate(TimeType start, TimeType end) const { uint64_t intervalCount = count(start, end); - return rateHelper(intervalCount, end - start); + TimeType interval = elapsed(start, end); + return rateHelper(intervalCount, interval); } /* @@ -342,26 +372,8 @@ class BucketedTimeSeries { private: template ReturnType rateHelper(ReturnType numerator, TimeType elapsed) const { - if (elapsed == TimeType(0)) { - return 0; - } - - // Use std::chrono::duration_cast to convert between the native - // duration and the desired interval. However, convert the rates, - // rather than just converting the elapsed duration. Converting the - // elapsed time first may collapse it down to 0 if the elapsed interval - // is less than the desired interval, which will incorrectly result in - // an infinite rate. - typedef std::chrono::duration< - ReturnType, std::ratio> NativeRate; - typedef std::chrono::duration< - ReturnType, std::ratio> DesiredRate; - - NativeRate native(numerator / elapsed.count()); - DesiredRate desired = std::chrono::duration_cast(native); - return desired.count(); + return detail::rateHelper(numerator, + elapsed); } ValueType rangeAdjust(TimeType bucketStart, TimeType nextBucketStart,