move warmup thread functionality out of LibEventServer

This moves the warmup thread logic from LibEventServer to a new
WarmupRequestHandler class.  This will allow the logic to work with any
server implementation, since it isn't really specific to LibEventServer.
(Eventually it would perhaps be nice to decouple the worker thread
behavior from the server I/O implementation entirely.)

hphp/runtime/base/server/http_server.cpp

@@ -21,6 +21,7 @@
 #include "hphp/runtime/base/server/http_request_handler.h"
 #include "hphp/runtime/base/server/admin_request_handler.h"
 #include "hphp/runtime/base/server/server_stats.h"
+#include "hphp/runtime/base/server/warmup_request_handler.h"
 #include "hphp/runtime/base/server/xbox_server.h"
 #include "hphp/runtime/base/runtime_option.h"
 #include "hphp/runtime/base/server/static_content_cache.h"
@@ -58,30 +59,20 @@ HttpServer::HttpServer(void *sslCTX /* = NULL */)
   // enabling mutex profiling, but it's not turned on
   LockProfiler::s_pfunc_profile = server_stats_log_mutex;
 
-  bool const useWarmupThrottle =
-    RuntimeOption::ServerWarmupThrottleRequestCount > 0 &&
-    RuntimeOption::ServerThreadCount > kNumProcessors;
-
-  auto maybeEnableThrottle = [&] (LibEventServer* s) {
-    if (!useWarmupThrottle) return;
-
-    s->enableWarmupThrottle(RuntimeOption::ServerThreadCount - kNumProcessors,
-                            RuntimeOption::ServerWarmupThrottleRequestCount);
-    Logger::Info("Starting with %d threads for the first %d requests\n",
-                 kNumProcessors,
-                 RuntimeOption::ServerWarmupThrottleRequestCount);
-  };
-
-  int const startingThreadCount = !useWarmupThrottle
-    ? RuntimeOption::ServerThreadCount
-    : kNumProcessors;
+  int startingThreadCount = RuntimeOption::ServerThreadCount;
+  uint32_t additionalThreads = 0;
+  RequestHandlerFactory handlerFactory;
+  if (RuntimeOption::ServerWarmupThrottleRequestCount > 0 &&
+      RuntimeOption::ServerThreadCount > kNumProcessors) {
+    startingThreadCount = kNumProcessors;
+    additionalThreads = RuntimeOption::ServerThreadCount - startingThreadCount;
+  }
 
   if (RuntimeOption::ServerPortFd != -1 || RuntimeOption::SSLPortFd != -1) {
     auto const server = boost::make_shared<LibEventServerWithFd>(
         RuntimeOption::ServerIP, RuntimeOption::ServerPort,
         startingThreadCount,
         RuntimeOption::RequestTimeoutSeconds);
-    maybeEnableThrottle(server.get());
     server->setServerSocketFd(RuntimeOption::ServerPortFd);
     server->setSSLSocketFd(RuntimeOption::SSLPortFd);
     m_pageServer = server;
@@ -90,20 +81,28 @@ HttpServer::HttpServer(void *sslCTX /* = NULL */)
         RuntimeOption::ServerIP, RuntimeOption::ServerPort,
         startingThreadCount,
         RuntimeOption::RequestTimeoutSeconds);
-    maybeEnableThrottle(server.get());
     m_pageServer = server;
   } else {
     auto const server = boost::make_shared<LibEventServerWithTakeover>(
         RuntimeOption::ServerIP, RuntimeOption::ServerPort,
         startingThreadCount,
         RuntimeOption::RequestTimeoutSeconds);
-    maybeEnableThrottle(server.get());
     server->setTransferFilename(RuntimeOption::TakeoverFilename);
     m_pageServer = server;
   }
-  m_pageServer->setRequestHandlerFactory<HttpRequestHandler>();
   m_pageServer->addTakeoverListener(this);
 
+  if (additionalThreads) {
+    auto handlerFactory = boost::make_shared<WarmupRequestHandlerFactory>(
+        m_pageServer, additionalThreads,
+        RuntimeOption::ServerWarmupThrottleRequestCount);
+    m_pageServer->setRequestHandlerFactory([handlerFactory] {
+      return handlerFactory->createHandler();
+    });
+  } else {
+    m_pageServer->setRequestHandlerFactory<HttpRequestHandler>();
+  }
+
   if (RuntimeOption::EnableSSL && m_sslCTX) {
     assert(SSLInit::IsInited());
     m_pageServer->enableSSL(m_sslCTX, RuntimeOption::SSLPort);

hphp/runtime/base/server/libevent_server.cpp

@@ -88,8 +88,6 @@ void LibEventWorker::doJob(LibEventJobPtr job) {
   assert(m_opaque);
   LibEventServer *server = (LibEventServer*)m_opaque;
 
-  server->bumpReqCount();
-
   LibEventTransport transport(server, request, m_id);
 #ifdef _EVENT_USE_OPENSSL
   if (evhttp_is_connection_ssl(job->request->evcon)) {
@@ -153,9 +151,6 @@ void LibEventWorker::onThreadExit() {
 LibEventServer::LibEventServer(const std::string &address, int port,
                                int thread, int timeoutSeconds)
   : Server(address, port, thread),
-    m_warmup_thread_slack(0),
-    m_req_number(0),
-    m_warmup_req_threshold(0),
     m_accept_sock(-1),
     m_accept_sock_ssl(-1),
     m_timeoutThreadData(timeoutSeconds),
@@ -207,24 +202,6 @@ int LibEventServer::getLibEventConnectionCount() {
   return evhttp_get_connection_count(m_server);
 }
 
-void LibEventServer::enableWarmupThrottle(int threadSlack, int reqCount) {
-  m_warmup_thread_slack = threadSlack;
-  const_cast<int&>(m_warmup_req_threshold) = reqCount;
-}
-
-void LibEventServer::bumpReqCount() {
-  auto const oldReqNum = m_req_number.fetch_add(1, std::memory_order_relaxed);
-  if (oldReqNum == m_warmup_req_threshold) {
-    if (auto const num = m_warmup_thread_slack.load()) {
-      Logger::Info("Finished warmup; adding %d new worker threads\n", num);
-      m_dispatcher.addWorkers(num);
-
-      // Set to zero so we can't do it if the req counter wraps.
-      m_warmup_thread_slack.store(0);
-    }
-  }
-}
-
 void LibEventServer::start() {
   if (getStatus() == RUNNING) return;
 

hphp/runtime/base/server/libevent_server.h

@@ -131,14 +131,6 @@ public:
                  int timeoutSeconds);
   ~LibEventServer();
 
-  /*
-   * Function to enable reducing the thread count during initial
-   * warmup requests.
-   *
-   * See RuntimeOption::ServerWarmupThrottleRequestCount.
-   */
-  void enableWarmupThrottle(int threadSlack, int reqCount);
-
   // implementing Server
   virtual void start();
   virtual void waitForEnd();
@@ -146,6 +138,9 @@ public:
   virtual int getActiveWorker() {
     return m_dispatcher.getActiveWorker();
   }
+  virtual void addWorkers(int numWorkers) {
+    m_dispatcher.addWorkers(numWorkers);
+  }
   virtual int getQueuedJobs() {
     return m_dispatcher.getQueuedJobs();
   }
@@ -180,17 +175,6 @@ protected:
   virtual int getAcceptSocket();
   virtual int getAcceptSocketSSL();
 
-private:
-  // Number of threads to start when warmup request counter passes the
-  // throttled request threshold.
-  friend class LibEventWorker;
-  void bumpReqCount();
-
-private:
-  std::atomic<int> m_warmup_thread_slack;
-  std::atomic<int32_t> m_req_number;
-  int const m_warmup_req_threshold;
-
 protected:
   int m_accept_sock;
   int m_accept_sock_ssl;

hphp/runtime/base/server/server.h

@@ -164,6 +164,11 @@ public:
   virtual void addTakeoverListener(TakeoverListener* lisener) {}
   virtual void removeTakeoverListener(TakeoverListener* lisener) {}
 
+  /**
+   * Add additional worker threads
+   */
+  virtual void addWorkers(int numWorkers) = 0;
+
   /**
    * Informational.
    */

hphp/runtime/base/server/warmup_request_handler.cpp

@@ -0,0 +1,63 @@
+/*
+   +----------------------------------------------------------------------+
+   | HipHop for PHP                                                       |
+   +----------------------------------------------------------------------+
+   | Copyright (c) 2010- Facebook, Inc. (http://www.facebook.com)         |
+   +----------------------------------------------------------------------+
+   | This source file is subject to version 3.01 of the PHP license,      |
+   | that is bundled with this package in the file LICENSE, and is        |
+   | available through the world-wide-web at the following url:           |
+   | http://www.php.net/license/3_01.txt                                  |
+   | If you did not receive a copy of the PHP license and are unable to   |
+   | obtain it through the world-wide-web, please send a note to          |
+   | license@php.net so we can mail you a copy immediately.               |
+   +----------------------------------------------------------------------+
+*/
+
+#include "hphp/runtime/base/server/warmup_request_handler.h"
+
+#include "folly/Memory.h"
+
+using folly::make_unique;
+
+namespace HPHP {
+///////////////////////////////////////////////////////////////////////////////
+
+void WarmupRequestHandler::handleRequest(Transport *transport) {
+  // There is one WarmupRequestHandler per-thread, but we want to track request
+  // count across all threads.  Therefore we let WarmupRequestHandlerFactory
+  // track the global request count.
+  m_factory->bumpReqCount();
+  m_reqHandler.handleRequest(transport);
+}
+
+std::unique_ptr<RequestHandler> WarmupRequestHandlerFactory::createHandler() {
+  return make_unique<WarmupRequestHandler>(shared_from_this());
+}
+
+void WarmupRequestHandlerFactory::bumpReqCount() {
+  // Bump the request count.  When we hit m_warmupReqThreshold,
+  // add additional threads to the server.
+  auto const oldReqNum = m_reqNumber.fetch_add(1, std::memory_order_relaxed);
+  if (oldReqNum != m_warmupReqThreshold) {
+    return;
+  }
+
+  auto const num = m_additionalThreads.load();
+  if (!num) {
+    return;
+  }
+  ServerPtr server = m_server.lock();
+  if (!server) {
+    return;
+  }
+
+  Logger::Info("Finished warmup; adding %d new worker threads", num);
+  server->addWorkers(num);
+
+  // Set to zero so we can't do it if the req counter wraps.
+  m_additionalThreads.store(0);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+}

hphp/runtime/base/server/warmup_request_handler.h

@@ -0,0 +1,72 @@
+/*
+   +----------------------------------------------------------------------+
+   | HipHop for PHP                                                       |
+   +----------------------------------------------------------------------+
+   | Copyright (c) 2010- Facebook, Inc. (http://www.facebook.com)         |
+   +----------------------------------------------------------------------+
+   | This source file is subject to version 3.01 of the PHP license,      |
+   | that is bundled with this package in the file LICENSE, and is        |
+   | available through the world-wide-web at the following url:           |
+   | http://www.php.net/license/3_01.txt                                  |
+   | If you did not receive a copy of the PHP license and are unable to   |
+   | obtain it through the world-wide-web, please send a note to          |
+   | license@php.net so we can mail you a copy immediately.               |
+   +----------------------------------------------------------------------+
+*/
+
+#ifndef incl_HPHP_WARMUP_REQUEST_HANDLER_H_
+#define incl_HPHP_WARMUP_REQUEST_HANDLER_H_
+
+#include "hphp/runtime/base/server/server.h"
+#include "hphp/runtime/base/server/http_request_handler.h"
+
+namespace HPHP {
+///////////////////////////////////////////////////////////////////////////////
+
+DECLARE_BOOST_TYPES(WarmupRequestHandlerFactory);
+
+/**
+ * WarmupRequestHandler is a small shim on top of HttpRequestHandler.
+ * It counts the number of requests, and adds additional worker threads to the
+ * server after a specified threshold.
+ */
+class WarmupRequestHandler : public RequestHandler {
+public:
+  explicit WarmupRequestHandler(const WarmupRequestHandlerFactoryPtr& factory)
+    : m_factory(factory) {}
+
+  virtual void handleRequest(Transport *transport);
+
+private:
+  WarmupRequestHandlerFactoryPtr m_factory;
+  HttpRequestHandler m_reqHandler;
+};
+
+class WarmupRequestHandlerFactory :
+  public boost::enable_shared_from_this<WarmupRequestHandlerFactory> {
+public:
+  WarmupRequestHandlerFactory(ServerPtr server,
+                              uint32_t additionalThreads,
+                              uint32_t reqCount)
+    : m_additionalThreads(additionalThreads),
+      m_reqNumber(0),
+      m_warmupReqThreshold(reqCount),
+      m_server(server) {}
+
+  std::unique_ptr<RequestHandler> createHandler();
+
+  void bumpReqCount();
+
+private:
+  std::atomic<uint32_t> m_additionalThreads;
+  std::atomic<uint32_t> m_reqNumber;
+  uint32_t const m_warmupReqThreshold;
+  // The server has a shared pointer to us, so use a weak pointer to the
+  // server to avoid a circular reference.
+  ServerWeakPtr m_server;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+}
+
+#endif // incl_HPHP_WARMUP_REQUEST_HANDLER_H_