Embed systemlib.php into hhvm

Also, since its always present, remove the special case
code for adding systemlib to a RepoAuthoritative repo,
and clean up all the magic variables for finding systemlib.

If HHVM_SYSTEMLIB is set, and the file exists, it will be
used as systemlib, otherwise, the embedded one will be used.

CMake/FindLibElf.cmake

@@ -51,5 +51,14 @@ FIND_PACKAGE_HANDLE_STANDARD_ARGS(LibElf DEFAULT_MSG
     LIBELF_LIBRARIES
     LIBELF_INCLUDE_DIRS)
 
+SET(CMAKE_REQUIRED_LIBRARIES elf)
+INCLUDE(CheckCXXSourceCompiles)
+CHECK_CXX_SOURCE_COMPILES("#include <libelf.h>
+int main() {
+  Elf *e = (Elf*)0;
+  size_t sz;
+  elf_getshdrstrndx(e, &sz);
+  return 0;
+}" ELF_GETSHDRSTRNDX)
 
-mark_as_advanced(LIBELF_INCLUDE_DIRS LIBELF_LIBRARIES)
+mark_as_advanced(LIBELF_INCLUDE_DIRS LIBELF_LIBRARIES ELF_GETSHDRSTRNDX)

CMake/HPHPFindLibs.cmake

@@ -266,6 +266,9 @@ include_directories(${LIBDWARF_INCLUDE_DIRS})
 
 find_package(LibElf REQUIRED)
 include_directories(${LIBELF_INCLUDE_DIRS})
+if (ELF_GETSHDRSTRNDX)
+        add_definitions("-DHAVE_ELF_GETSHDRSTRNDX")
+endif()
 
 CONTAINS_STRING("${CCLIENT_INCLUDE_PATH}/utf8.h" U8T_DECOMPOSE RECENT_CCLIENT)
 if (NOT RECENT_CCLIENT)

hphp/compiler/analysis/emitter.cpp

@@ -7314,53 +7314,6 @@ static void batchCommit(std::vector<UnitEmitter*>& ues) {
   ues.clear();
 }
 
-static void emitSystemLib() {
-  if (!Option::WholeProgram) return;
-
-  string slib = systemlib_path();
-  if (slib.empty()) return;
-
-  FILE* sl = fopen(slib.c_str(), "r");
-  if (!sl) return;
-  fseek(sl, 0, SEEK_END);
-  long len = ftell(sl);
-  fseek(sl, 0, SEEK_SET);
-
-  char *code = (char*)malloc(len + 1);
-  Option::WholeProgram = false;
-  SystemLib::s_inited = false;
-
-  SCOPE_EXIT {
-    SystemLib::s_inited = true;
-    Option::WholeProgram = true;
-    free(code);
-  };
-
-  if (fread(code, len, 1, sl) != 1) {
-    throw Exception("Failed to read systemlib");
-  }
-
-  code[len] = 0;
-  AnalysisResultPtr ar(new AnalysisResult());
-  Scanner scanner(code, len, RuntimeOption::ScannerType, "/:systemlib.php");
-  Parser parser(scanner, "/:systemlib.php", ar, len);
-  parser.parse();
-  FileScopePtr fsp = parser.getFileScope();
-  fsp->setOuterScope(ar);
-
-  ar->loadBuiltins();
-  ar->setPhase(AnalysisResult::AnalyzeAll);
-  fsp->analyzeProgram(ar);
-
-  int md5len;
-  char* md5str = string_md5(code, len, false, md5len);
-  MD5 md5(md5str);
-  free(md5str);
-
-  UnitEmitter* ue = emitHHBCUnitEmitter(ar, fsp, md5);
-  Repo::get().commitUnit(ue, UnitOriginFile);
-}
-
 /**
  * This is the entry point for offline bytecode generation.
  */
@@ -7415,8 +7368,6 @@ void emitAllHHBC(AnalysisResultPtr ar) {
         break;
       }
     }
-
-    emitSystemLib();
   } else {
     dispatcher.waitEmpty();
   }

hphp/compiler/builtin_symbols.cpp

@@ -311,24 +311,6 @@ void BuiltinSymbols::ImportExtClasses(AnalysisResultPtr ar) {
   }
 }
 
-void BuiltinSymbols::Parse(AnalysisResultPtr ar,
-                           const std::string& phpBaseName,
-                           const std::string& phpFileName) {
-  const char *baseName = s_strings.add(phpBaseName.c_str());
-  const char *fileName = s_strings.add(phpFileName.c_str());
-  try {
-    Scanner scanner(fileName, Option::ScannerType);
-    Compiler::Parser parser(scanner, baseName, ar);
-    if (!parser.parse()) {
-      Logger::Error("Unable to parse file %s: %s", fileName,
-                    parser.getMessage().c_str());
-      assert(false);
-    }
-  } catch (FileOpenException &e) {
-    Logger::Error("%s", e.getMessage().c_str());
-  }
-}
-
 bool BuiltinSymbols::Load(AnalysisResultPtr ar, bool extOnly /* = false */) {
   if (Loaded) return true;
   Loaded = true;
@@ -346,17 +328,17 @@ bool BuiltinSymbols::Load(AnalysisResultPtr ar, bool extOnly /* = false */) {
   if (!extOnly) {
     ar = AnalysisResultPtr(new AnalysisResult());
     ar->loadBuiltinFunctions();
-    string slib = systemlib_path();
-    if (slib.empty()) {
-      for (const char **cls = SystemClasses; *cls; cls++) {
-        string phpBaseName = "/system/classes/";
-        phpBaseName += *cls;
-        phpBaseName += ".php";
-        Parse(ar, phpBaseName, Option::GetSystemRoot() + phpBaseName);
-      }
-    } else {
-      Parse(ar, slib, slib);
+    string slib = get_systemlib();
+
+    Scanner scanner(slib.c_str(), slib.size(),
+                    Option::ScannerType, "systemlib.php");
+    Compiler::Parser parser(scanner, "systemlib.php", ar);
+    if (!parser.parse()) {
+      Logger::Error("Unable to parse systemlib.php: %s",
+                    parser.getMessage().c_str());
+      assert(false);
     }
+
     ar->analyzeProgram(true);
     ar->inferTypes();
     const StringToFileScopePtrMap &files = ar->getAllFiles();

hphp/compiler/builtin_symbols.h

@@ -73,9 +73,6 @@ private:
   static const char *SystemClasses[];
 
   static AnalysisResultPtr LoadGlobalSymbols(const char *fileName);
-  static void Parse(AnalysisResultPtr ar,
-                    const std::string& phpBaseName,
-                    const std::string& phpFileName);
 
   static StringToTypePtrMap s_superGlobals;
 

hphp/hhvm/CMakeLists.txt

@@ -5,6 +5,10 @@ list(APPEND CXX_SOURCES ${files})
 add_executable(hhvm ${CXX_SOURCES})
 target_link_libraries(hhvm hphp_analysis hphp_runtime_static ext_hhvm_static)
 
-install(TARGETS hhvm DESTINATION bin)
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/../../bin/systemlib.php DESTINATION bin)
+add_custom_command(TARGET hhvm POST_BUILD
+                   COMMAND "objcopy"
+                   ARGS "--add-section" "systemlib=${HPHP_HOME}/bin/systemlib.php"
+                        "${CMAKE_CURRENT_SOURCE_DIR}/hhvm"
+                   COMMENT "Embedding systemlib.php in hhvm")
 
+install(TARGETS hhvm DESTINATION bin)

hphp/hhvm/main.cpp

@@ -16,9 +16,12 @@
 
 #include "runtime/base/program_functions.h"
 #include "hhvm/process_init.h"
-#include "runtime/vm/embedded_repo.h"
 #include "compiler/compiler.h"
 
+#include "util/embedded_data.h"
+#include "util/embedded_vfs.h"
+#include "util/util.h"
+
 int main(int argc, char** argv) {
   if (!argc) return 0;
   if (argc > 1 && !strcmp(argv[1], "--hphp")) {
@@ -26,10 +29,16 @@ int main(int argc, char** argv) {
     return HPHP::compiler_main(argc - 1, argv + 1);
   }
   HPHP::register_process_init();
-  std::string repo = HPHP::get_embedded_repo();
-  if (repo.empty()) {
+  HPHP::Util::embedded_data data;
+  if (!HPHP::Util::get_embedded_data("repo", &data)) {
     return HPHP::execute_program(argc, argv);
   }
+  std::string repo;
+  HPHP::Util::string_printf(repo, "%s:%u:%u",
+                            data.m_filename.c_str(),
+                            (unsigned)data.m_start, (unsigned)data.m_len);
+  HPHP::sqlite3_embedded_initialize(nullptr, true);
+
   std::vector<char*> args;
   args.push_back(argv[0]);
   args.push_back("-vRepo.Authoritative=true");

hphp/hhvm/process_init.cpp

@@ -123,37 +123,17 @@ void ProcessInit() {
                                                 hhbc_ext_funcs_count);
   SystemLib::s_nativeFuncUnit = nativeFuncUnit;
 
-  // Search for systemlib.php in the following places:
-  // 1) ${HHVM_LIB_PATH}/systemlib.php
-  // 2) <dirname(realpath(hhvm))>/systemlib.php (requires proc filesystem)
-  // 3) <HHVM_LIB_PATH_DEFAULT>/systemlib.php
-  //
-  // HHVM_LIB_PATH allows a manual override at runtime. If systemlib.php
-  // exists next to the hhvm binary, that is likely to be the next best
-  // version to use. The realpath()-based lookup will succeed as long as the
-  // proc filesystem exists (e.g. on Linux and some FreeBSD configurations)
-  // and no hard links are in use for the executable. Failing all of those
-  // options, the HHVM_LIB_PATH_DEFAULT-based lookup will always succeed,
-  // assuming that the application was built and installed correctly.
   String currentDir = g_vmContext->getCwd();
-  HPHP::Eval::PhpFile* file = nullptr;
 
-  string slib = RuntimeOption::RepoAuthoritative ?
-    string("/:systemlib.php") : systemlib_path();
+  string slib = get_systemlib();
 
-  if (!slib.empty()) {
-    file = g_vmContext->lookupPhpFile(String(slib).get(),
-                                      currentDir.data(), nullptr);
-  }
-  if (!file) {
+  if (slib.empty()) {
     // Die a horrible death.
     Logger::Error("Unable to find/load systemlib.php");
     _exit(1);
   }
 
-  SystemLib::s_phpFile = file;
-  file->incRef();
-  SystemLib::s_unit = file->unit();
+  SystemLib::s_unit = compile_string(slib.c_str(), slib.size());
 
   // Restore most settings before merging anything,
   // because of optimizations that depend on them

hphp/runtime/base/program_functions.cpp

@@ -34,6 +34,7 @@
 #include "runtime/base/memory/memory_manager.h"
 #include "util/process.h"
 #include "util/capability.h"
+#include "util/embedded_data.h"
 #include "util/timer.h"
 #include "util/stack_trace.h"
 #include "util/light_process.h"
@@ -1116,30 +1117,29 @@ String canonicalize_path(CStrRef p, const char* root, int rootLen) {
   return path;
 }
 
-static bool check_path(const char *path) {
-  if (!path) return false;
-
-  string file = string(path) + "/systemlib.php";
-  struct stat result;
-  if (stat(file.c_str(), &result)) {
-    return false;
+// Search for systemlib.php in the following places:
+// 1) ${HHVM_SYSTEMLIB}
+// 2) section "systemlib" in the current executable
+// and return its contents
+string get_systemlib() {
+  if (char *file = getenv("HHVM_SYSTEMLIB")) {
+    std::ifstream ifs(file);
+    if (ifs.good()) {
+      return std::string(std::istreambuf_iterator<char>(ifs),
+                         std::istreambuf_iterator<char>());
+    }
   }
-  return S_ISREG(result.st_mode);
-}
 
-string systemlib_path() {
-  char *file;
-  char buf[PATH_MAX + 1];
-  if (!check_path(file = getenv("HHVM_LIB_PATH")) &&
-      !(realpath("/proc/self/exe", buf) &&
-        check_path(file = dirname(buf)))
-#ifdef HHVM_LIB_PATH_DEFAULT
-      && !(check_path(file = HHVM_LIB_PATH_DEFAULT))
-#endif
-     ) {
-    return "";
-  }
-  return string(file) + "/systemlib.php";
+  Util::embedded_data desc;
+  if (!Util::get_embedded_data("systemlib", &desc)) return "";
+
+  std::ifstream ifs(desc.m_filename);
+  if (!ifs.good()) return "";
+  ifs.seekg(desc.m_start, std::ios::beg);
+  std::unique_ptr<char[]> data(new char[desc.m_len]);
+  ifs.read(data.get(), desc.m_len);
+  string result(data.get(), desc.m_len);
+  return result;
 }
 
 ///////////////////////////////////////////////////////////////////////////////

hphp/runtime/base/program_functions.h

@@ -81,7 +81,7 @@ void hphp_thread_exit();
 void hphp_session_exit();
 void hphp_process_exit() ATTRIBUTE_COLD;
 bool hphp_is_warmup_enabled();
-std::string systemlib_path();
+std::string get_systemlib();
 
 // Generated by hphp/util/generate_buildinfo.sh.
 extern const char* const kCompilerId;

hphp/runtime/vm/translator/test/main.cpp

@@ -16,7 +16,20 @@
 #include <gtest/gtest.h>
 #include "hhvm/process_init.h"
 
+#include <string>
+
 int main(int argc, char** argv) {
+  char buf[PATH_MAX];
+  if (realpath("/proc/self/exe", buf)) {
+    for (int i = 0; i < 4; i++) {
+      char* p = strrchr(buf, '/');
+      assert(p);
+      *p = 0;
+    }
+    std::string slib = buf;
+    slib += "/ext_hhvm/systemlib.php";
+    setenv("HHVM_SYSTEMLIB", slib.c_str(), true);
+  }
   testing::InitGoogleTest(&argc, argv);
   HPHP::init_for_unit_test();
   return RUN_ALL_TESTS();

hphp/system/lib/systemlib.cpp

@@ -33,7 +33,6 @@ namespace HPHP {
   }
 
 bool SystemLib::s_inited = false;
-HPHP::Eval::PhpFile* SystemLib::s_phpFile = nullptr;
 HPHP::VM::Unit* SystemLib::s_unit = nullptr;
 HPHP::VM::Unit* SystemLib::s_nativeFuncUnit = nullptr;
 HPHP::VM::Unit* SystemLib::s_nativeClassUnit = nullptr;

hphp/system/lib/systemlib.h

@@ -14,11 +14,11 @@
    +----------------------------------------------------------------------+
 */
 
-#include <runtime/base/types.h>
-
 #ifndef incl_HPHP_SYSTEMLIB_H_
 #define incl_HPHP_SYSTEMLIB_H_
 
+#include <runtime/base/types.h>
+
 namespace HPHP {
 ///////////////////////////////////////////////////////////////////////////////
 
@@ -71,7 +71,6 @@ namespace Eval {
 class SystemLib {
  public:
   static bool s_inited;
-  static HPHP::Eval::PhpFile* s_phpFile;
   static HPHP::VM::Unit* s_unit;
   static HPHP::VM::Unit* s_nativeFuncUnit;
   static HPHP::VM::Unit* s_nativeClassUnit;

hphp/test/quick/Setprofile.php.expectf

@@ -200,7 +200,7 @@ array(3) {
         [0]=>
         array(6) {
           ["file"]=>
-          string(%d) "%s"
+          string(%d) "%S"
           ["line"]=>
           int(42)
           ["function"]=>

hphp/tools/generated_files.sh

@@ -13,7 +13,13 @@ check_err()
 VERBOSE=1
 
 HHVM=$HPHP_HOME/hphp/hhvm/hhvm
-[ ! -x "$HHVM" ] && HHVM=`which hhvm`
+if [ -x "$HHVM" ]; then
+  export HHVM_SYSTEMLIB=$HPHP_HOME/bin/systemlib.php
+else
+  HHVM=`which hhvm`
+  export HHVM_SYSTEMLIB=`dirname $HHVM`/systemlib.php
+fi
+
 [ ! -x "$HHVM" ] && check_err 1 "$HHVM is not executable"
 
 HPHP_TOOLS=$HPHP_HOME/hphp/tools/

hphp/util/embedded_data.cpp

@@ -14,6 +14,8 @@
    +----------------------------------------------------------------------+
 */
 
+#include "util/embedded_data.h"
+
 #include <libelf.h>
 #include <gelf.h>
 #include <stdio.h>
@@ -22,44 +24,46 @@
 #include <fcntl.h>
 #include <string.h>
 
-#include <runtime/vm/embedded_repo.h>
-#include <util/embedded_vfs.h>
+namespace HPHP { namespace Util {
 
-namespace HPHP {
-
-std::string get_embedded_repo() {
+bool get_embedded_data(const char *section, embedded_data* desc) {
   GElf_Shdr shdr;
   size_t shstrndx;
   char *name;
   Elf_Scn *scn;
 
-  if (elf_version(EV_CURRENT) == EV_NONE) return "";
+  if (elf_version(EV_CURRENT) == EV_NONE) return false;
 
   int fd = open("/proc/self/exe", O_RDONLY, 0);
-  if (fd < 0) return "";
+  if (fd < 0) return false;
   Elf* e = elf_begin(fd, ELF_C_READ, nullptr);
 
   if (!e ||
       elf_kind(e) != ELF_K_ELF ||
-      !elf_getshstrndx(e, &shstrndx)) {
-    return "";
+#ifdef HAVE_ELF_GETSHDRSTRNDX
+      elf_getshdrstrndx(e, &shstrndx)
+#else
+      !elf_getshstrndx(e, &shstrndx)
+#endif
+      ) {
+    return false;
   }
   scn = nullptr;
   while ((scn = elf_nextscn(e, scn)) != nullptr) {
     if (gelf_getshdr(scn, &shdr) != &shdr ||
         !(name = elf_strptr(e, shstrndx , shdr.sh_name))) {
-      return "";
+      return false;
     }
-    if (!strcmp("repo", name)) {
+    if (!strcmp(section, name)) {
       GElf_Shdr ghdr;
-      if (gelf_getshdr(scn, &ghdr) != &ghdr) return "";
-      char buf[512];
-      sprintf(buf, "/proc/self/exe:%lu:%lu", ghdr.sh_offset, ghdr.sh_size);
-      sqlite3_embedded_initialize(nullptr, true);
-      return buf;
+      if (gelf_getshdr(scn, &ghdr) != &ghdr) return false;
+      desc->m_filename = "/proc/self/exe";
+      desc->m_start = ghdr.sh_offset;
+      desc->m_len = ghdr.sh_size;
+      return true;
     }
   }
-  return "";
+  return false;
 }
 
-}
+} }

hphp/util/embedded_data.h

@@ -19,10 +19,16 @@
 
 #include <string>
 
-namespace HPHP {
+namespace HPHP { namespace Util {
 
-std::string get_embedded_repo();
+struct embedded_data {
+  std::string   m_filename;
+  uint32_t      m_start;
+  uint32_t      m_len;
+};
 
-}
+bool get_embedded_data(const char *section, embedded_data* desc);
+
+} }
 
 #endif