From 7449c3afeaa66a521aba8ba554615e97820659ce Mon Sep 17 00:00:00 2001 From: Jordan DeLong Date: Sat, 20 Apr 2013 12:12:57 -0700 Subject: [PATCH] Remove util/lfu_table.h This is unused. --- hphp/test/test_util.cpp | 142 --------- hphp/util/lfu_table.h | 637 ---------------------------------------- 2 files changed, 779 deletions(-) delete mode 100644 hphp/util/lfu_table.h diff --git a/hphp/test/test_util.cpp b/hphp/test/test_util.cpp index 19237e13c..c79cd7d4d 100644 --- a/hphp/test/test_util.cpp +++ b/hphp/test/test_util.cpp @@ -16,7 +16,6 @@ #include #include -#include #include #include #include @@ -37,7 +36,6 @@ TestUtil::TestUtil() { bool TestUtil::RunTests(const std::string &which) { bool ret = true; - //RUN_TEST(TestLFUTable); RUN_TEST(TestSharedString); RUN_TEST(TestCanonicalize); RUN_TEST(TestHDF); @@ -66,146 +64,6 @@ struct IEq { bool operator()(int i, int j) const { return i == j; } }; -typedef LFUTable TestMap; - -bool TestUtil::TestLFUTable() { - { - TestMap table(1, 5, 2); - table.insert(1,1); - int r = 0; - VERIFY(table.lookup(1, r) && r == 1); - VERIFY(table.lookup(1, r) && r == 1); - VERIFY(table.lookup(1, r) && r == 1); - } - { - TestMap table(100, 5, 2); - for (int i = 0; i < 10; i++) { - table.insert(i,i); - } - VERIFY(table.check()); - int r = 0; - VERIFY(!table.lookup(0, r)); - VERIFY(!table.lookup(1, r)); - VERIFY(!table.lookup(2, r)); - VERIFY(!table.lookup(3, r)); - VERIFY(!table.lookup(4, r)); - VERIFY(table.lookup(5, r) && r == 5); - VERIFY(table.lookup(6, r) && r == 6); - VERIFY(table.lookup(7, r) && r == 7); - VERIFY(table.lookup(8, r) && r == 8); - VERIFY(table.lookup(9, r) && r == 9); - VERIFY(table.check()); - } - { - TestMap table(1, 5, 2); - for (int i = 0; i < 5; i++) { - table.insert(i,i); - } - sleep(1); - for (int i = 5; i < 10; i++) { - table.insert(i,i); - } - VERIFY(table.check()); - int r = 0; - VERIFY(!table.lookup(0, r)); - VERIFY(!table.lookup(1, r)); - VERIFY(!table.lookup(2, r)); - VERIFY(!table.lookup(3, r)); - VERIFY(!table.lookup(4, r)); - VERIFY(table.lookup(5, r) && r == 5); - VERIFY(table.lookup(6, r) && r == 6); - VERIFY(table.lookup(7, r) && r == 7); - VERIFY(table.lookup(8, r) && r == 8); - VERIFY(table.lookup(9, r) && r == 9); - VERIFY(table.check()); - } - { - TestMap table(1, 5, 2); - for (int i = 0; i < 5; i++) { - table.insert(i,i); - } - int r = 0; - VERIFY(table.lookup(0, r) && r == 0); - VERIFY(table.lookup(0, r) && r == 0); - VERIFY(table.lookup(3, r) && r == 3); - sleep(1); - for (int i = 5; i < 8; i++) { - table.insert(i,i); - } - VERIFY(table.lookup(0, r) && r == 0); - VERIFY(!table.lookup(1, r)); - VERIFY(!table.lookup(2, r)); - VERIFY(table.lookup(3, r) && r == 3); - VERIFY(!table.lookup(4, r)); - VERIFY(table.lookup(5, r) && r == 5); - VERIFY(table.lookup(6, r) && r == 6); - VERIFY(table.lookup(7, r) && r == 7); - int ct = 0; - for (TestMap::const_iterator it = table.begin(); - it != table.end(); ++it) { - ct++; - VERIFY(it.first() == it.second()); - } - VERIFY(ct == 5); - } - { - TestMap table(1, 5, 2); - for (int i = 0; i < 5; i++) { - table[i] = i; - } - int r = 0; - VERIFY(table[0] == 0); - VERIFY(table[0] == 0); - VERIFY(table[3] == 3); - sleep(1); - for (int i = 5; i < 8; i++) { - table[i] = i; - } - VERIFY(table.check()); - VERIFY(table.lookup(0, r) && r == 0); - VERIFY(!table.lookup(1, r)); - VERIFY(!table.lookup(2, r)); - VERIFY(table.lookup(3, r) && r == 3); - VERIFY(!table.lookup(4, r)); - VERIFY(table.lookup(5, r) && r == 5); - VERIFY(table.lookup(6, r) && r == 6); - VERIFY(table.lookup(7, r) && r == 7); - VERIFY(table.check()); - int ct = 0; - for (TestMap::const_iterator it = table.begin(); - it != table.end(); ++it) { - ct++; - VERIFY(it.first() == it.second()); - } - VERIFY(ct == 5); - } - { - class Reader : public TestMap::AtomicReader { - public: - void read(int const &k, int const &v) { - - } - }; - class Updater : public TestMap::AtomicUpdater { - public: - bool update(int const &k, int &v, bool newly) { - v = k; - return false; - } - }; - Reader r; - Updater u; - TestMap table(1, 5, 2); - for (int i = 0; i < 5; i++) { - table.atomicUpdate(i, u, true); - } - for (int i = 0; i < 5; i++) { - VERIFY(table.atomicRead(i, r)); - } - } - return Count(true); -} - bool TestUtil::TestSharedString() { { SharedString foo = "foo"; diff --git a/hphp/util/lfu_table.h b/hphp/util/lfu_table.h deleted file mode 100644 index d861277d8..000000000 --- a/hphp/util/lfu_table.h +++ /dev/null @@ -1,637 +0,0 @@ -/* - +----------------------------------------------------------------------+ - | HipHop for PHP | - +----------------------------------------------------------------------+ - | Copyright (c) 2010- Facebook, Inc. (http://www.facebook.com) | - +----------------------------------------------------------------------+ - | This source file is subject to version 3.01 of the PHP license, | - | that is bundled with this package in the file LICENSE, and is | - | available through the world-wide-web at the following url: | - | http://www.php.net/license/3_01.txt | - | If you did not receive a copy of the PHP license and are unable to | - | obtain it through the world-wide-web, please send a note to | - | license@php.net so we can mail you a copy immediately. | - +----------------------------------------------------------------------+ -*/ - -#ifndef incl_HPHP_UTIL_LFU_TABLE_H_ -#define incl_HPHP_UTIL_LFU_TABLE_H_ - -#include -#include - -namespace HPHP { -/////////////////////////////////////////////////////////////////////////////// - -template -class LFUNullDestructor { -public: - void operator()(const K &k, V &v) {} -}; - -/** - * A lookup table with a maximum size. Once it reaches maximum size, - * inserts will evict the least frequently used item. - * - * It consists of 3 parts: a queue, a heap, and a map. - * The map is used for lookup by key. The queue is a staging area for - * values that are too new to have an accurate frequency, and the heap - * serves as a priority queue for looking up the element that has - * the lowest frequency. - * - * Frequencies are updated only when the hit count hits a multiple of the - * period option. When a frequency is updated, the heap is also updated - * to maintain the heap property. Frequency should be locked by the queue - * lock since it must not change during heap operations. - * - * There are two locks that make it thread safe. There is read/write lock - * for the map, and a lock for the queue/heap. The acquisition order - * is map then queue/heap. - * - * An element is in the map if it is in the queue xor the heap. - * If max capacity is 0, then all queue operations are disabled. - * Elements can be added to the map but not the queue. - */ -template > -class LFUTable { - class Node { - public: - Node(const K &k) : key(k), prev(nullptr), next(nullptr), hits(0), heapIndex(0), - immortal(false), m_freq(0), m_timestamp(time(nullptr)) {} - Node(const K &k, const V &v) - : key(k), val(v), prev(nullptr), next(nullptr), hits(0), heapIndex(0), - immortal(false), m_freq(0), m_timestamp(time(nullptr)) {} - ~Node() { - D d; - d(key, val); - } - const K &key; - V val; - Node *prev; - Node *next; - std::atomic hits; - uint heapIndex; - bool immortal; - - - // Frequency updates must be accompanied by heap updates since otherwise - // it may break the heap property. It should probably be under lock too. - bool updateFrequency() { - double lifetime = time(nullptr) - m_timestamp; - double oldFreq = m_freq; - if (lifetime > 0) { - m_freq = hits / lifetime; - } - return m_freq > oldFreq; - } - double frequency() const { - return m_freq; - } - time_t timestamp() { - return m_timestamp; - } - private: - double m_freq; - time_t m_timestamp; - }; - // The map - class Map : public hphp_hash_map {}; - //typedef std::map Map; - -public: - class LFUTableConstIterator; - class LFUTableIterator { - public: - LFUTableIterator(typename LFUTable::Map::iterator &it) : m_it(it) {} - LFUTableIterator(const LFUTableIterator &it) : m_it(it.m_it) {} - const K &first() const { - return m_it->first; - } - V &second() const { - return m_it->second->val; - } - LFUTableIterator &operator++() { - ++m_it; - return *this; - } - bool operator==(const LFUTableIterator &it) const { - return m_it == it.m_it; - } - bool operator!=(const LFUTableIterator &it) const { - return m_it != it.m_it; - } - private: - friend class LFUTable::LFUTableConstIterator; - friend class LFUTable; - typename LFUTable::Map::iterator m_it; - }; - class LFUTableConstIterator { - public: - LFUTableConstIterator(typename LFUTable::Map::const_iterator &it) - : m_it(it) {} - LFUTableConstIterator(const LFUTableConstIterator &it) : m_it(it.m_it) { - } - LFUTableConstIterator(const LFUTableIterator &it) : m_it(it.m_it) { - } - const K &first() const { - return m_it->first; - } - const V &second() const { - return m_it->second->val; - } - LFUTableConstIterator &operator++() { - ++m_it; - return *this; - } - bool operator==(const LFUTableConstIterator &it) const { - return m_it == it.m_it; - } - bool operator!=(const LFUTableConstIterator &it) const { - return m_it != it.m_it; - } - private: - friend class LFUTable; - typename LFUTable::Map::const_iterator m_it; - }; - /** - * AtomicReader is used to perform a lookup under the internal - * lock of the table. It takes the read lock. - */ - class AtomicReader { - public: - virtual ~AtomicReader() {} - virtual void read(const K &key, const V &val) = 0; - }; - /** - * AtomicUpdater is used to perform a update/insert under the internal - * lock of the table. It takes the write lock. - * The update method is given a reference to the value in the table and - * a bool that specifies if the element was newly added. - * It the method returns false, the element is deleted. - */ - class AtomicUpdater { - public: - virtual ~AtomicUpdater() {} - virtual bool update(const K &key, V &val, bool newlyCreated) = 0; - }; - -public: - LFUTable(time_t maturity, size_t maxCap, int updatePeriod) - : m_head(nullptr), m_tail(nullptr), m_immortalCount(0), - m_maturityThreshold(maturity), m_maximumCapacity(maxCap), - m_updatePeriod(updatePeriod) { - //Lfu table is currently buggy and at the moment not worth fixing - assert(false); - } - ~LFUTable() { - clear(); - } - - typedef LFUTableConstIterator const_iterator; - typedef LFUTableIterator iterator; - - const_iterator begin() const { - return const_iterator(m_map.begin()); - } - const_iterator end() const { - return const_iterator(m_map.end()); - } - iterator begin() { - typename Map::iterator it = m_map.begin(); - return iterator(it); - } - iterator end() { - typename Map::iterator it = m_map.end(); - return iterator(it); - } - - void insert(const K &k, const V &v) { - WriteLock lock(m_mapLock); - if (_contains(k)) { - _erase(k); - } - _makeRoom(); - // Add to the map - typename Map::iterator ins = m_map.insert(std::pair(k,nullptr)) - .first; - Node *n = new Node(ins->first, v); - ins->second = n; - // Add to queue - insertQueue(n); - } - - bool atomicRead(const K &k, AtomicReader &reader) { - ReadLock lock(m_mapLock); - Node *n = _getNode(k, false); - if (n) { - reader.read(n->key, n->val); - return true; - } - return false; - } - - void atomicForeach(AtomicReader &reader) { - ReadLock lock(m_mapLock); - for (typename Map::const_iterator it = m_map.begin(); - it != m_map.end(); ++it) { - reader.read(it->first, it->second->val); - } - } - - void atomicUpdate(const K &k, AtomicUpdater &updater, bool createNew, - bool immortal = false) { - WriteLock lock(m_mapLock); - bool erase = false; - Node *n = _getNode(k, false); - bool created = false; - if (!n && createNew) { - n = _createNode(k, immortal); - created = true; - } - if (n) { - erase = updater.update(n->key, n->val, created); - } - if (erase) { - _erase(k); - } - } - - void erase(const K &k) { - WriteLock lock(m_mapLock); - _erase(k); - } - void erase(const iterator &it) { - WriteLock lock(m_mapLock); - _erase(it); - } - - bool lookup(const K &k, V &result) { - ReadLock lock(m_mapLock); - Node *n = _getNode(k, false); - if (n) { - result = n->val; - return true; - } - return false; - } - iterator find(const K &k) { - ReadLock lock(m_mapLock); - typename Map::iterator it = m_map.find(k); - if (it != m_map.end()) { - Node *n = it->second; - _bumpNode(n); - } - return iterator(it); - } - V &operator[](const K &k) { - WriteLock lock(m_mapLock); - return _getNode(k, true)->val; - } - - size_t size() const { - return m_map.size(); - } - size_t maximumCapacity() const { - return m_maximumCapacity; - } - size_t immortalCount() const { - return m_immortalCount; - } - - void clear() { - WriteLock lock(m_mapLock); - typename Map::iterator it = m_map.begin(); - while (it != m_map.end()) { - typename Map::iterator cit = it++; - Node *n = cit->second; - if (n->immortal) { - // Trade immortal for mortal - ++m_maximumCapacity; - --m_immortalCount; - } else { - removeQueue(cit->second); - } - m_map.erase(cit); - delete n; - } - assert(m_head == nullptr); - assert(m_tail == nullptr); - assert(m_heap.size() == 0); - } - - bool check() { - WriteLock lock(m_mapLock); - Lock qlock(m_queueLock); - - bool fail = false; - Node *prev = nullptr; - Node *n = m_head; - while (n) { - if (m_map.find(n->key) == m_map.end()) { - fail = true; - Logger::Error("Value in queue not in map"); - assert(!fail); - } - if (n->prev != prev) { - fail = true; - Logger::Error("Queue list corrupted"); - assert(!fail); - } - prev = n; - n = n->next; - if (!n && prev != m_tail) { - fail = true; - Logger::Error("Queue tail incorrect"); - assert(!fail); - } - } - uint hsize = m_heap.size(); - for (uint i = 0; i < hsize; i++) { - Node *n = m_heap[i]; - if (m_map.find(n->key) == m_map.end()) { - fail = true; - Logger::Error("Value in queue not in heap"); - assert(!fail); - } - size_t child = (i+1) * 2; - if (child <= hsize && n->frequency() > m_heap[child-1]->frequency()) { - fail = true; - Logger::Error("Heap property violated"); - assert(!fail); - } - child++; - if (child <= hsize && n->frequency() > m_heap[child-1]->frequency()) { - fail = true; - Logger::Error("Heap property violated"); - assert(!fail); - } - } - return !fail; - } - -private: - -private: - ////////////////////////////////////////////////////////////////////////////// - // These methods are to be used when the map lock is already acquired. - bool _contains(const K &k) { - return m_map.find(k) != m_map.end(); - } - void _erase(const K &k) { - typename Map::iterator it = m_map.find(k); - if (it != m_map.end()) { - _erase(it); - } - } - void _erase(const iterator &it) { - Node *n = it.m_it->second; - m_map.erase(it.m_it); - if (n->immortal) { - // We trade an immortal for a mortal value - --m_immortalCount; - ++m_maximumCapacity; - } else { - removeQueue(n); - } - delete n; - } - - void _bumpNode(Node *n) { - if (!m_maximumCapacity) return; - if (n->hits.fetch_add(1, std::memory_order_relaxed) % m_updatePeriod == 0) { - Lock lock(m_queueLock); - // hits could have increased between incrementing and taking the lock, - // but it does not matter. - // It is also possible (if unlikely) that two threads could be trying to - // do this update, but that doesn't matter either. - bool increase = n->updateFrequency(); - _updateQueue(n, increase); - } - } - void _makeRoom() { - if (!m_maximumCapacity) return; - while (size() - m_immortalCount >= m_maximumCapacity) { - Node *m = popQueue(); - assert(m); - m_map.erase(m->key); - delete m; - } - } - Node *_getNode(const K &k, bool force, bool immortal = false) { - typename Map::iterator it = m_map.find(k); - if (it != m_map.end()) { - _bumpNode(it->second); - return it->second; - } - if (force) { - return _createNode(k, immortal); - } else { - return nullptr; - } - } - - Node *_createNode(const K &k, bool immortal = false) { - if (immortal) { - ++m_immortalCount; - } else { - _makeRoom(); - } - // Add to the map - typename Map::iterator ins = m_map.insert(std::pair(k,nullptr)) - .first; - Node *n = new Node(ins->first); - ins->second = n; - n->immortal = immortal; - if (!immortal) { - // Add to queue - insertQueue(n); - } - return n; - } - - void _dumpStatus() { - std::cout << "in map:\n"; - for (typename Map::const_iterator it = m_map.begin(); it != m_map.end(); - ++it) { - std::cout << it->first << " : " << it->second->frequency() << "\n"; - } - if (!m_maximumCapacity) return; - std::cout << "in queue:\n"; - Node *n = m_head; - while (n) { - std::cout << n->key << " : " << n->frequency() << "\n"; - n = n->next; - } - std::cout << "in heap\n"; - for (uint i = 0; i < m_heap.size(); i++) { - std::cout << m_heap[i]->key << " : " << m_heap[i]->frequency() << "\n"; - } - } - -private: - ////////////////////////////////////////////////////////////////////////////// - // Queue interface methods - - // Add item to queue - void insertQueue(Node *item) { - if (!m_maximumCapacity) return; - Lock lock(m_queueLock); - // Add to head of list - item->heapIndex = 0; - item->prev = nullptr; - item->next = m_head; - if (m_head) m_head->prev = item; - m_head = item; - if (!m_tail) m_tail = item; - _shiftMature(); - } - - void removeQueue(Node *item) { - if (!m_maximumCapacity) return; - Lock lock(m_queueLock); - if (item == m_head) { - m_head = item->next; - } - if (item == m_tail) { - m_tail = item->prev; - } - if (item->prev) { - item->prev->next = item->next; - } - if (item->next) { - item->next->prev = item->prev; - } - item->prev = item->next = nullptr; - if (item->heapIndex != 0) { - int pos = item->heapIndex; - int last = m_heap.size(); - item->heapIndex = 0; - m_heap[pos-1] = m_heap[last-1]; - m_heap[pos-1]->heapIndex = pos; - m_heap.pop_back(); - _heapifyDown(pos); - } - } - // Pop the minimum - Node *popQueue() { - if (!m_maximumCapacity) return nullptr; - Lock lock(m_queueLock); - _shiftMature(); - if (m_heap.size() == 0) { - Node *res = m_tail; - if (!res) return nullptr; - if (res->prev) { - res->prev->next = nullptr; - } - m_tail = res->prev; - res->prev = nullptr; - if (res == m_head) m_head = nullptr; - return res; - } - Node *res = m_heap[0]; - res->heapIndex = 0; - m_heap[0] = m_heap[m_heap.size() - 1]; - m_heap[0]->heapIndex = 1; - m_heap.pop_back(); - _heapifyDown(1); - return res; - } - - ////////////////////////////////////////////////////////////////////////////// - // Queue helper methods - - void _shiftMature() { - // Move mature items to heap - time_t now = time(nullptr); - while (m_tail && ((now - m_tail->timestamp()) >= m_maturityThreshold)) { - Node *last = m_tail; - if (last->prev) { - last->prev->next = nullptr; - } - m_tail = last->prev; - if (m_head == last) m_head = nullptr; - last->prev = nullptr; - last->updateFrequency(); - _heapInsert(last); - } - } - - void _updateQueue(const Node *item, bool increase) { - if (!m_maximumCapacity) return; - if (item->heapIndex != 0) { - if (increase) { - _heapifyDown(item->heapIndex); - } else { - _heapifyUp(item->heapIndex); - } - } - } - - void _heapInsert(Node *item) { - m_heap.push_back(item); - item->heapIndex = m_heap.size(); - int pos = m_heap.size(); - _heapifyUp(pos); - } - - void _heapifyUp(int pos) { - while (pos != 1) { - Node *&child = m_heap[pos - 1]; - Node *&parent = m_heap[pos/2 - 1]; - if (parent->frequency() > child->frequency()) { - std::swap(child, parent); - child->heapIndex = pos; - parent->heapIndex = pos/2; - pos = pos/2; - } else { - break; - } - } - } - - void _heapifyDown(int pos) { - int size = m_heap.size(); - for (;;) { - int child; - if (pos * 2 > size) break; - if (pos * 2 + 1 > size || (m_heap[pos*2 - 1]->frequency() < - m_heap[pos*2]->frequency())) { - child = pos * 2; - } else { - child = pos * 2 + 1; - } - if (m_heap[pos - 1]->frequency() > m_heap[child - 1]->frequency()) { - std::swap(m_heap[pos - 1], m_heap[child - 1]); - m_heap[pos - 1]->heapIndex = pos; - m_heap[child - 1]->heapIndex = child; - pos = child; - } else { - break; - } - } - } - -private: - // The queue. A doubly linked list since I need to remove elements at will. - Node *m_head; - Node *m_tail; - // The heap - std::vector m_heap; - Map m_map; - size_t m_immortalCount; - - // Locks in acquisition order - ReadWriteMutex m_mapLock; - Mutex m_queueLock; - - // Options - time_t m_maturityThreshold; - size_t m_maximumCapacity; - int m_updatePeriod; -}; - -/////////////////////////////////////////////////////////////////////////////// -} - -#endif // incl_HPHP_UTIL_LFU_TABLE_H_