Basic heap tracing framework + debugger command

Adds an extension command to the debugger which provides
heap tracing functionality, and adds the framework needed to get
heap traces. The heaptrace command can dump the current heap to the
debugger session or it can save a GraphViz specification to a file.

I'm hoping to add a backend which can put the graph in GML format,
so it can be explored interactively. I also hope at some point to
see this integrated into FBIDE if we can do that.

hphp/runtime/base/array/array_data.cpp

@@ -581,5 +581,19 @@ void ArrayData::dump(std::ostream &out) {
   }
 }
 
+void ArrayData::getChildren(std::vector<TypedValue *> &out) {
+  if (isSharedMap()) {
+    SharedMap *sm = static_cast<SharedMap *>(this);
+    sm->getChildren(out);
+    return;
+  }
+  for (ssize_t pos = iter_begin();
+      pos != ArrayData::invalid_index;
+      pos = iter_advance(pos)) {
+    TypedValue *tv = nvGetValueRef(pos);
+    out.push_back(tv);
+  }
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 }

hphp/runtime/base/array/array_data.h

@@ -494,6 +494,9 @@ protected:
   static uint32_t getKindOff() {
     return (uintptr_t)&((ArrayData*)0)->m_kind;
   }
+
+ public:
+  void getChildren(std::vector<TypedValue *> &out);
 };
 
 ALWAYS_INLINE inline

hphp/runtime/base/array/policy_array.cpp

@@ -941,5 +941,4 @@ ArrayData *PolicyArray::escalate() const {
   return ArrayData::escalate();
 }
 
-
 } // namespace HPHP

hphp/runtime/base/memory/tracer.h

@@ -0,0 +1,120 @@
+/*
+   +----------------------------------------------------------------------+
+   | HipHop for PHP                                                       |
+   +----------------------------------------------------------------------+
+   | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com)     |
+   +----------------------------------------------------------------------+
+   | This source file is subject to version 3.01 of the PHP license,      |
+   | that is bundled with this package in the file LICENSE, and is        |
+   | available through the world-wide-web at the following url:           |
+   | http://www.php.net/license/3_01.txt                                  |
+   | If you did not receive a copy of the PHP license and are unable to   |
+   | obtain it through the world-wide-web, please send a note to          |
+   | license@php.net so we can mail you a copy immediately.               |
+   +----------------------------------------------------------------------+
+*/
+
+#ifndef incl_HPHP_TRACER_H
+#define incl_HPHP_TRACER_H
+
+#include <functional>
+#include <queue>
+#include <set>
+#include <vector>
+#include "hphp/runtime/base/complex_types.h"
+#include "hphp/util/trace.h"
+
+namespace HPHP {
+
+template<typename Accum>
+struct Tracer {
+  typedef std::function<void(TypedValue *,             Accum&)> NodeFunc;
+  typedef std::function<void(TypedValue *,TypedValue *,Accum&)> EdgeFunc;
+
+  static void trace(TypedValue *self,
+                    NodeFunc atNode,
+                    EdgeFunc atEdge,
+                    Accum& accumulator) {
+    std::set<TypedValue *> visited;
+    traceImpl(self, visited, atNode, atEdge, accumulator);
+  }
+
+  static void traceAll(std::vector<TypedValue *> &ts,
+                       NodeFunc atNode,
+                       EdgeFunc atEdge,
+                       Accum& accumulator) {
+    std::set<TypedValue *> visited;
+    for (TypedValue *t : ts) {
+      traceImpl(t, visited, atNode, atEdge, accumulator);
+    }
+  }
+
+private:
+  static void traceImpl(TypedValue *self,
+                        std::set<TypedValue *> &visited,
+                        NodeFunc atNode,
+                        EdgeFunc atEdge,
+                        Accum& accumulator) {
+    std::queue<TypedValue *> tvs;
+    const auto traceSingle = [&](TypedValue *parent, TypedValue *child) {
+      atEdge(parent, child, accumulator);
+      tvs.push(child);
+    };
+    const auto traceMultiple = [&](TypedValue *parent) {
+      std::vector<TypedValue *> children;
+      switch(parent->m_type) {
+        case KindOfArray:
+          parent->m_data.parr->getChildren(children);
+          break;
+        case KindOfObject:
+          parent->m_data.pobj->getChildren(children);
+          break;
+        default:
+          not_reached();
+      }
+      for (TypedValue *child : children) {
+        traceSingle(parent, child);
+      }
+    };
+
+    tvs.push(self);
+    while (!tvs.empty()) {
+      TypedValue *top = tvs.front();
+      tvs.pop();
+      if (visited.find(top) != visited.end()) {
+        continue;
+      }
+      visited.insert(top);
+      atNode(top, accumulator);
+      switch (top->m_type) {
+        case KindOfUninit:
+        case KindOfNull:
+        case KindOfBoolean:
+        case KindOfInt64:
+        case KindOfDouble:
+        case KindOfStaticString:
+        case KindOfString:
+          break;
+        case KindOfArray:
+        case KindOfObject:
+          traceMultiple(top);
+          break;
+        case KindOfRef:
+          traceSingle(top, top->m_data.pref->tv());
+          break;
+        case KindOfIndirect:
+          traceSingle(top, top->m_data.pind);
+          break;
+        default:
+          not_reached();
+      }
+    }
+  }
+};
+
+}
+
+#undef TRACE_SINGLE_CHILD
+#undef TRACE_MULTIPLE_CHILDREN
+
+#endif

hphp/runtime/base/object_data.h

@@ -279,6 +279,14 @@ class ObjectData : public CountableNF {
     }
     delete this;
   }
+
+ public:
+  void getChildren(std::vector<TypedValue *> &out) {
+    ArrayData *props = o_properties.get();
+    if (props) {
+      props->getChildren(out);
+    }
+  }
 } __attribute__((aligned(16)));
 
 template<> inline SmartPtr<ObjectData>::~SmartPtr() {}

hphp/runtime/base/shared/shared_map.cpp

@@ -251,5 +251,18 @@ ArrayData* SharedMap::loadElems(bool mapInit /* = false */) const {
   return elems;
 }
 
+void SharedMap::getChildren(std::vector<TypedValue *> &out) {
+  if (m_localCache) {
+    TypedValue *localCacheEnd = m_localCache + size();
+    for (TypedValue *tv = m_localCache;
+         tv < localCacheEnd;
+         ++tv) {
+      if (tv->m_type != KindOfUninit) {
+        out.push_back(tv);
+      }
+    }
+  }
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 }

hphp/runtime/base/shared/shared_map.h

@@ -136,6 +136,9 @@ private:
   };
   mutable TypedValue* m_localCache;
   bool isVector() const { return m_isVector; }
+
+public:
+  void getChildren(std::vector<TypedValue *> &out);
 };
 
 ///////////////////////////////////////////////////////////////////////////////

hphp/runtime/base/util/thrift_buffer.h

@@ -225,6 +225,34 @@ public:
       write(data[i]);
     }
   }
+  template<typename T1, typename T2>
+  void read(std::pair<T1, T2> &data) {
+    read(data.first);
+    read(data.second);
+  }
+  template<typename T1, typename T2>
+  void write(const std::pair<T1, T2> &data) {
+    write(data.first);
+    write(data.second);
+  }
+  template<typename T1, typename T2>
+  void read(std::map<T1, T2> &data) {
+    int32_t size;
+    read(size);
+    for (int i = 0; i < size; i++) {
+      std::pair<T1, T2> entry;
+      read(entry);
+      data.insert(entry);
+    }
+  }
+  template<typename T1, typename T2>
+  void write(const std::map<T1, T2> &data) {
+    int32_t size = data.size();
+    write(size);
+    for (const std::pair<T1, T2> &entry : data) {
+      write(entry);
+    }
+  }
   template<typename T>
   void read(boost::shared_ptr<T> &data) {
     bool has;

hphp/runtime/debugger/cmd/all.h

@@ -51,6 +51,7 @@
 #include "hphp/runtime/debugger/cmd/cmd_macro.h"
 #include "hphp/runtime/debugger/cmd/cmd_config.h"
 #include "hphp/runtime/debugger/cmd/cmd_complete.h"
+#include "hphp/runtime/debugger/cmd/cmd_heaptrace.h"
 #include "hphp/runtime/debugger/cmd/cmd_internal_testing.h"
 //tag: new_cmd.php inserts new command here, do NOT remove/modify this line
 

hphp/runtime/debugger/cmd/cmd_extended.cpp

@@ -177,12 +177,11 @@ bool CmdExtended::onServer(DebuggerProxy &proxy) {
 ///////////////////////////////////////////////////////////////////////////////
 
 const ExtendedCommandMap &CmdExtended::GetExtendedCommandMap() {
-  static ExtendedCommandMap s_command_map;
-  if (s_command_map.empty()) {
-    // add one line for each command
-    s_command_map["ample"]   = "CmdExample";
-    s_command_map["tension"] = "CmdExtension";
-  }
+  static ExtendedCommandMap s_command_map = {
+    { "ample"    , "CmdExample" },
+    { "tension"  , "CmdExtension" },
+    { "heaptrace", "CmdHeaptrace" }
+  };
   return s_command_map;
 }
 
@@ -196,7 +195,7 @@ DebuggerCommandPtr CmdExtended::CreateExtendedCommand(const std::string &cls) {
     // add one line for each command
     ELSE_IF_CMD(Example);
     ELSE_IF_CMD(Extension);
-
+    ELSE_IF_CMD(Heaptrace);
   }
 
   if (ret) {

hphp/runtime/debugger/cmd/cmd_heaptrace.cpp

@@ -0,0 +1,196 @@
+/*
+   +----------------------------------------------------------------------+
+   | HipHop for PHP                                                       |
+   +----------------------------------------------------------------------+
+   | Copyright (c) 2010 Facebook, Inc. (http://www.facebook.com)          |
+   +----------------------------------------------------------------------+
+   | This source file is subject to version 3.01 of the PHP license,      |
+   | that is bundled with this package in the file LICENSE, and is        |
+   | available through the world-wide-web at the following url:           |
+   | http://www.php.net/license/3_01.txt                                  |
+   | If you did not receive a copy of the PHP license and are unable to   |
+   | obtain it through the world-wide-web, please send a note to          |
+   | license@php.net so we can mail you a copy immediately.               |
+   +----------------------------------------------------------------------+
+*/
+
+#include "hphp/runtime/debugger/cmd/cmd_heaptrace.h"
+
+#include "hphp/runtime/vm/unit.h"
+
+namespace HPHP { namespace Eval {
+///////////////////////////////////////////////////////////////////////////////
+
+void CmdHeaptrace::sendImpl(DebuggerThriftBuffer &thrift) {
+  DebuggerCommand::sendImpl(thrift);
+  thrift.write(m_accum.typesMap);
+  thrift.write(m_accum.adjacencyList);
+}
+
+void CmdHeaptrace::recvImpl(DebuggerThriftBuffer &thrift) {
+  DebuggerCommand::recvImpl(thrift);
+  thrift.read(m_accum.typesMap);
+  thrift.read(m_accum.adjacencyList);
+}
+
+void CmdHeaptrace::list(DebuggerClient &client) {
+}
+
+void CmdHeaptrace::help(DebuggerClient &client) {
+  client.helpTitle("Heaptrace Command");
+  client.helpCmds(
+    "[h]eaptrace",              "dumps all currently reachable values",
+    "[h]eaptrace {filename}",   "dumps heap to GraphViz graph file",
+    nullptr
+  );
+  client.helpBody(
+    "This will print the locations and types of all reachable values "
+    "in the heap."
+  );
+}
+
+static const char *typeName(int8_t type) {
+  switch (type) {
+    case KindOfUninit:
+      return "uninit";
+    case KindOfNull:
+      return "null";
+    case KindOfBoolean:
+      return "boolean";
+    case KindOfInt64:
+      return "integer";
+    case KindOfDouble:
+      return "double";
+    case KindOfStaticString:
+      return "string (static)";
+    case KindOfString:
+      return "string";
+    case KindOfArray:
+      return "array";
+    case KindOfObject:
+      return "object";
+    case KindOfRef:
+      return "reference";
+    case KindOfIndirect:
+      return "indirect";
+    default:
+      return "unknown";
+  }
+}
+
+void CmdHeaptrace::printHeap(DebuggerClient &client) {
+  for (const auto &pair : m_accum.typesMap) {
+    client.print(folly::stringPrintf("Found TV at %p with type %s (%u)",
+                                     (void *)pair.first,
+                                     typeName(pair.second),
+                                     pair.second));
+    std::vector<int64_t> &adjList = m_accum.adjacencyList[pair.first];
+    if (!adjList.empty()) {
+      std::string children = "  -> found children: ";
+      bool first = true;
+      for (const int64_t &adjacent : adjList) {
+        if (!first) {
+          children += ", ";
+        }
+        children += folly::stringPrintf("%p", (void *)adjacent);
+        first = false;
+      }
+      client.print(children);
+    }
+  }
+}
+
+void CmdHeaptrace::printGraphToFile(DebuggerClient &client, String filename) {
+  const char *name = filename->data();
+  FILE *graphFile = fopen(name, "w");
+  if (!graphFile) {
+    client.print("Could not open file!");
+    return;
+  }
+
+  fprintf(graphFile, "digraph {\n  node [shape=box];\n");
+  for (const auto &pair : m_accum.typesMap) {
+    void *ptr = (void *)pair.first;
+    std::string n = folly::stringPrintf(
+      "  node%p [label=\"TV at %p\\ntype %s\"];\n",
+      ptr,
+      ptr,
+      typeName(pair.second)
+    );
+    fprintf(graphFile, "%s", n.c_str());
+    std::vector<int64_t> &adjList = m_accum.adjacencyList[pair.first];
+    for (const int64_t adjacent : adjList) {
+      std::string e = folly::stringPrintf(
+        "  node%p -> node%p;\n",
+        ptr,
+        (void *)adjacent
+      );
+      fprintf(graphFile, "%s", e.c_str());
+    }
+  }
+  fprintf(graphFile, "}\n");
+  fclose(graphFile);
+
+  client.print(folly::stringPrintf("Wrote heap graph to %s.", name));
+}
+
+void CmdHeaptrace::onClientImpl(DebuggerClient &client) {
+  if (DebuggerCommand::displayedHelp(client)) return;
+
+  String file;
+  if (client.argCount() == 2) {
+    file = client.argValue(2);
+  } else if (client.argCount() != 1) {
+    help(client);
+    return;
+  }
+
+  CmdHeaptracePtr cmd = client.xend<CmdHeaptrace>(this);
+
+  if (file.empty()) {
+    cmd->printHeap(client);
+  } else {
+    cmd->printGraphToFile(client, file);
+  }
+
+}
+
+bool CmdHeaptrace::onServer(DebuggerProxy &proxy) {
+
+  // globals
+  std::vector<TypedValue *> roots;
+  CArrRef arr = g_vmContext->m_globalVarEnv->getDefinedVariables();
+  arr->getChildren(roots);
+
+  // static properties
+  for (AllClasses ac; !ac.empty();) {
+    Class *c = ac.popFront();
+    c->getChildren(roots);
+  }
+
+  // locals
+  int numFrames = proxy.getRealStackDepth();
+  std::vector<Array> locs;
+  for (int i = 0; i < numFrames; ++i) {
+    locs.push_back(g_vmContext->getLocalDefinedVariables(i));
+  }
+  for (CArrRef locArr : locs) {
+    locArr->getChildren(roots);
+  }
+
+  Tracer<Accum>::traceAll(
+    roots,
+    [](TypedValue *node, Accum &accum) {
+      accum.typesMap[(int64_t)node] = (int8_t)node->m_type;
+    },
+    [](TypedValue *parent, TypedValue *child, Accum &accum) {
+      accum.adjacencyList[(int64_t)parent].push_back((int64_t)child);
+    },
+    m_accum
+  );
+
+  return proxy.sendToClient(this);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+}}

hphp/runtime/debugger/cmd/cmd_heaptrace.h

@@ -0,0 +1,55 @@
+/*
+   +----------------------------------------------------------------------+
+   | HipHop for PHP                                                       |
+   +----------------------------------------------------------------------+
+   | Copyright (c) 2010 Facebook, Inc. (http://www.facebook.com)          |
+   +----------------------------------------------------------------------+
+   | This source file is subject to version 3.01 of the PHP license,      |
+   | that is bundled with this package in the file LICENSE, and is        |
+   | available through the world-wide-web at the following url:           |
+   | http://www.php.net/license/3_01.txt                                  |
+   | If you did not receive a copy of the PHP license and are unable to   |
+   | obtain it through the world-wide-web, please send a note to          |
+   | license@php.net so we can mail you a copy immediately.               |
+   +----------------------------------------------------------------------+
+*/
+
+#ifndef incl_HPHP_EVAL_DEBUGGER_CMD_HEAPTRACE_H_
+#define incl_HPHP_EVAL_DEBUGGER_CMD_HEAPTRACE_H_
+
+#include <map>
+#include <vector>
+
+#include "hphp/runtime/debugger/cmd/cmd_extended.h"
+
+#include "hphp/runtime/base/memory/tracer.h"
+
+namespace HPHP { namespace Eval {
+///////////////////////////////////////////////////////////////////////////////
+
+DECLARE_BOOST_TYPES(CmdHeaptrace);
+class CmdHeaptrace : public CmdExtended {
+public:
+  virtual void list(DebuggerClient &client);
+  virtual void help(DebuggerClient &client);
+  virtual bool onServer(DebuggerProxy &proxy);
+
+protected:
+  virtual void onClientImpl(DebuggerClient &client);
+  virtual void sendImpl(DebuggerThriftBuffer &thrift);
+  virtual void recvImpl(DebuggerThriftBuffer &thrift);
+
+private:
+  void printHeap(DebuggerClient &client);
+  void printGraphToFile(DebuggerClient &client, String filename);
+
+  struct Accum {
+    std::map<int64_t, int8_t> typesMap;
+    std::map<int64_t, std::vector<int64_t>> adjacencyList;
+  } m_accum;
+};
+
+///////////////////////////////////////////////////////////////////////////////
+}}
+
+#endif // __HPHP_EVAL_DEBUGGER_CMD_HEAPTRACE_H__

hphp/runtime/vm/bytecode.cpp

@@ -64,6 +64,7 @@
 #include "hphp/runtime/vm/type_profile.h"
 #include "hphp/runtime/base/server/source_root_info.h"
 #include "hphp/runtime/base/util/extended_logger.h"
+#include "hphp/runtime/base/memory/tracer.h"
 
 #include "hphp/system/systemlib.h"
 #include "hphp/runtime/ext/ext_collections.h"

hphp/runtime/vm/class.cpp

@@ -2640,4 +2640,11 @@ void Class::setSPropData(TypedValue* sPropData) const {
   handleToRef<TypedValue*>(m_propSDataCache) = sPropData;
 }
 
+void Class::getChildren(std::vector<TypedValue *> &out) {
+  for (Slot i = 0; i < m_staticProperties.size(); ++i) {
+    if (m_staticProperties[i].m_class != this) continue;
+    out.push_back(&m_staticProperties[i].m_val);
+  }
+}
+
  } // HPHP::VM

hphp/runtime/vm/class.h

@@ -1027,6 +1027,9 @@ private:
 
   MethodToTraitListMap m_importMethToTraitMap;
 
+public:
+  void getChildren(std::vector<TypedValue *> &out);
+
 public: // used in Unit
   Class* m_nextClass;
 private:

hphp/util/trace.h

@@ -104,6 +104,8 @@ namespace Trace {
       TM(txOpBisectHigh) \
       /* smart alloc */ \
       TM(smartalloc) \
+      /* Heap tracing */ \
+      TM(heap) \
       /* Temporary catetories, to save compilation time */ \
       TM(tmp0)  TM(tmp1)  TM(tmp2)  TM(tmp3)               \
       TM(tmp4)  TM(tmp5)  TM(tmp6)  TM(tmp7)               \