src: make heap snapshot & embedder graph accessible for tests

Add methods that allow inspection of heap snapshots and a JS
version of our own embedder graph.

These can be used in tests and might also prove useful for
ad-hoc debugging. Usage requires `--expose-internals` and
prints a warning similar to our other modules whose primary
purpose is test support.

PR-URL: #21741
Reviewed-By: James M Snell <[email protected]>
Reviewed-By: Joyee Cheung <[email protected]>
Reviewed-By: Refael Ackermann <[email protected]>

lib/internal/test/heap.js

@@ -0,0 +1,87 @@
+'use strict';
+
+process.emitWarning(
+  'These APIs are exposed only for testing and are not ' +
+  'tracked by any versioning system or deprecation process.',
+  'internal/test/heap');
+
+const { internalBinding } = require('internal/bootstrap/loaders');
+const { createHeapDump, buildEmbedderGraph } = internalBinding('heap_utils');
+const assert = require('assert');
+
+// This is not suitable for production code. It creates a full V8 heap dump,
+// parses it as JSON, and then creates complex objects from it, leading
+// to significantly increased memory usage.
+function createJSHeapDump() {
+  const dump = createHeapDump();
+  const meta = dump.snapshot.meta;
+
+  const nodes =
+    readHeapInfo(dump.nodes, meta.node_fields, meta.node_types, dump.strings);
+  const edges =
+    readHeapInfo(dump.edges, meta.edge_fields, meta.edge_types, dump.strings);
+
+  for (const node of nodes) {
+    node.incomingEdges = [];
+    node.outgoingEdges = [];
+  }
+
+  let fromNodeIndex = 0;
+  let edgeIndex = 0;
+  for (const { type, name_or_index, to_node } of edges) {
+    while (edgeIndex === nodes[fromNodeIndex].edge_count) {
+      edgeIndex = 0;
+      fromNodeIndex++;
+    }
+    const toNode = nodes[to_node / meta.node_fields.length];
+    const fromNode = nodes[fromNodeIndex];
+    const edge = {
+      type,
+      toNode,
+      fromNode,
+      name: typeof name_or_index === 'string' ? name_or_index : null
+    };
+    toNode.incomingEdges.push(edge);
+    fromNode.outgoingEdges.push(edge);
+    edgeIndex++;
+  }
+
+  for (const node of nodes)
+    assert.strictEqual(node.edge_count, node.outgoingEdges.length);
+
+  return nodes;
+}
+
+function readHeapInfo(raw, fields, types, strings) {
+  const items = [];
+
+  for (var i = 0; i < raw.length; i += fields.length) {
+    const item = {};
+    for (var j = 0; j < fields.length; j++) {
+      const name = fields[j];
+      let type = types[j];
+      if (Array.isArray(type)) {
+        item[name] = type[raw[i + j]];
+      } else if (name === 'name_or_index') {  // type === 'string_or_number'
+        if (item.type === 'element' || item.type === 'hidden')
+          type = 'number';
+        else
+          type = 'string';
+      }
+
+      if (type === 'string') {
+        item[name] = strings[raw[i + j]];
+      } else if (type === 'number' || type === 'node') {
+        item[name] = raw[i + j];
+      }
+    }
+    items.push(item);
+  }
+
+  return items;
+}
+
+module.exports = {
+  createJSHeapDump,
+  buildEmbedderGraph
+};

node.gyp

@@ -146,6 +146,7 @@
       'lib/internal/repl/await.js',
       'lib/internal/socket_list.js',
       'lib/internal/test/binding.js',
+      'lib/internal/test/heap.js',
       'lib/internal/test/unicode.js',
       'lib/internal/timers.js',
       'lib/internal/tls.js',
@@ -328,6 +329,7 @@
         'src/exceptions.cc',
         'src/fs_event_wrap.cc',
         'src/handle_wrap.cc',
+        'src/heap_utils.cc',
         'src/js_stream.cc',
         'src/module_wrap.cc',
         'src/node.cc',

src/heap_utils.cc

@@ -0,0 +1,232 @@
+#include "node_internals.h"
+#include "env.h"
+
+using v8::Array;
+using v8::Boolean;
+using v8::Context;
+using v8::EmbedderGraph;
+using v8::EscapableHandleScope;
+using v8::FunctionCallbackInfo;
+using v8::HandleScope;
+using v8::HeapSnapshot;
+using v8::Isolate;
+using v8::JSON;
+using v8::Local;
+using v8::MaybeLocal;
+using v8::Number;
+using v8::Object;
+using v8::String;
+using v8::Value;
+
+namespace node {
+namespace heap {
+
+class JSGraphJSNode : public EmbedderGraph::Node {
+ public:
+  const char* Name() override { return "<JS Node>"; }
+  size_t SizeInBytes() override { return 0; }
+  bool IsEmbedderNode() override { return false; }
+  Local<Value> JSValue() { return StrongPersistentToLocal(persistent_); }
+
+  int IdentityHash() {
+    Local<Value> v = JSValue();
+    if (v->IsObject()) return v.As<Object>()->GetIdentityHash();
+    if (v->IsName()) return v.As<v8::Name>()->GetIdentityHash();
+    if (v->IsInt32()) return v.As<v8::Int32>()->Value();
+    return 0;
+  }
+
+  JSGraphJSNode(Isolate* isolate, Local<Value> val)
+      : persistent_(isolate, val) {
+    CHECK(!val.IsEmpty());
+  }
+
+  struct Hash {
+    inline size_t operator()(JSGraphJSNode* n) const {
+      return n->IdentityHash();
+    }
+  };
+
+  struct Equal {
+    inline bool operator()(JSGraphJSNode* a, JSGraphJSNode* b) const {
+      return a->JSValue()->SameValue(b->JSValue());
+    }
+  };
+
+ private:
+  Persistent<Value> persistent_;
+};
+
+class JSGraph : public EmbedderGraph {
+ public:
+  explicit JSGraph(Isolate* isolate) : isolate_(isolate) {}
+
+  Node* V8Node(const Local<Value>& value) override {
+    std::unique_ptr<JSGraphJSNode> n { new JSGraphJSNode(isolate_, value) };
+    auto it = engine_nodes_.find(n.get());
+    if (it != engine_nodes_.end())
+      return *it;
+    engine_nodes_.insert(n.get());
+    return AddNode(std::unique_ptr<Node>(n.release()));
+  }
+
+  Node* AddNode(std::unique_ptr<Node> node) override {
+    Node* n = node.get();
+    nodes_.emplace(std::move(node));
+    return n;
+  }
+
+  void AddEdge(Node* from, Node* to) override {
+    edges_[from].insert(to);
+  }
+
+  MaybeLocal<Array> CreateObject() const {
+    EscapableHandleScope handle_scope(isolate_);
+    Local<Context> context = isolate_->GetCurrentContext();
+
+    std::unordered_map<Node*, Local<Object>> info_objects;
+    Local<Array> nodes = Array::New(isolate_, nodes_.size());
+    Local<String> edges_string = FIXED_ONE_BYTE_STRING(isolate_, "edges");
+    Local<String> is_root_string = FIXED_ONE_BYTE_STRING(isolate_, "isRoot");
+    Local<String> name_string = FIXED_ONE_BYTE_STRING(isolate_, "name");
+    Local<String> size_string = FIXED_ONE_BYTE_STRING(isolate_, "size");
+    Local<String> value_string = FIXED_ONE_BYTE_STRING(isolate_, "value");
+    Local<String> wraps_string = FIXED_ONE_BYTE_STRING(isolate_, "wraps");
+
+    for (const std::unique_ptr<Node>& n : nodes_)
+      info_objects[n.get()] = Object::New(isolate_);
+
+    {
+      HandleScope handle_scope(isolate_);
+      size_t i = 0;
+      for (const std::unique_ptr<Node>& n : nodes_) {
+        Local<Object> obj = info_objects[n.get()];
+        Local<Value> value;
+        if (!String::NewFromUtf8(isolate_, n->Name(),
+                                 v8::NewStringType::kNormal).ToLocal(&value) ||
+            obj->Set(context, name_string, value).IsNothing() ||
+            obj->Set(context, is_root_string,
+                     Boolean::New(isolate_, n->IsRootNode())).IsNothing() ||
+            obj->Set(context, size_string,
+                     Number::New(isolate_, n->SizeInBytes())).IsNothing() ||
+            obj->Set(context, edges_string,
+                     Array::New(isolate_)).IsNothing()) {
+          return MaybeLocal<Array>();
+        }
+        if (nodes->Set(context, i++, obj).IsNothing())
+          return MaybeLocal<Array>();
+        if (!n->IsEmbedderNode()) {
+          value = static_cast<JSGraphJSNode*>(n.get())->JSValue();
+          if (obj->Set(context, value_string, value).IsNothing())
+            return MaybeLocal<Array>();
+        }
+      }
+    }
+
+    for (const std::unique_ptr<Node>& n : nodes_) {
+      Node* wraps = n->WrapperNode();
+      if (wraps == nullptr) continue;
+      Local<Object> from = info_objects[n.get()];
+      Local<Object> to = info_objects[wraps];
+      if (from->Set(context, wraps_string, to).IsNothing())
+        return MaybeLocal<Array>();
+    }
+
+    for (const auto& edge_info : edges_) {
+      Node* source = edge_info.first;
+      Local<Value> edges;
+      if (!info_objects[source]->Get(context, edges_string).ToLocal(&edges) ||
+          !edges->IsArray()) {
+        return MaybeLocal<Array>();
+      }
+
+      size_t i = 0;
+      for (Node* target : edge_info.second) {
+        if (edges.As<Array>()->Set(context,
+                                   i++,
+                                   info_objects[target]).IsNothing()) {
+          return MaybeLocal<Array>();
+        }
+      }
+    }
+
+    return handle_scope.Escape(nodes);
+  }
+
+ private:
+  Isolate* isolate_;
+  std::unordered_set<std::unique_ptr<Node>> nodes_;
+  std::unordered_set<JSGraphJSNode*, JSGraphJSNode::Hash, JSGraphJSNode::Equal>
+      engine_nodes_;
+  std::unordered_map<Node*, std::unordered_set<Node*>> edges_;
+};
+
+void BuildEmbedderGraph(const FunctionCallbackInfo<Value>& args) {
+  Environment* env = Environment::GetCurrent(args);
+  JSGraph graph(env->isolate());
+  Environment::BuildEmbedderGraph(env->isolate(), &graph, env);
+  Local<Array> ret;
+  if (graph.CreateObject().ToLocal(&ret))
+    args.GetReturnValue().Set(ret);
+}
+
+
+class BufferOutputStream : public v8::OutputStream {
+ public:
+  BufferOutputStream() : buffer_(new JSString()) {}
+
+  void EndOfStream() override {}
+  int GetChunkSize() override { return 1024 * 1024; }
+  WriteResult WriteAsciiChunk(char* data, int size) override {
+    buffer_->Append(data, size);
+    return kContinue;
+  }
+
+  Local<String> ToString(Isolate* isolate) {
+    return String::NewExternalOneByte(isolate,
+                                      buffer_.release()).ToLocalChecked();
+  }
+
+ private:
+  class JSString : public String::ExternalOneByteStringResource {
+   public:
+    void Append(char* data, size_t count) {
+      store_.append(data, count);
+    }
+
+    const char* data() const override { return store_.data(); }
+    size_t length() const override { return store_.size(); }
+
+   private:
+    std::string store_;
+  };
+
+  std::unique_ptr<JSString> buffer_;
+};
+
+void CreateHeapDump(const FunctionCallbackInfo<Value>& args) {
+  Isolate* isolate = args.GetIsolate();
+  const HeapSnapshot* snapshot = isolate->GetHeapProfiler()->TakeHeapSnapshot();
+  BufferOutputStream out;
+  snapshot->Serialize(&out, HeapSnapshot::kJSON);
+  const_cast<HeapSnapshot*>(snapshot)->Delete();
+  Local<Value> ret;
+  if (JSON::Parse(isolate->GetCurrentContext(),
+                  out.ToString(isolate)).ToLocal(&ret)) {
+    args.GetReturnValue().Set(ret);
+  }
+}
+
+void Initialize(Local<Object> target,
+                Local<Value> unused,
+                Local<Context> context) {
+  Environment* env = Environment::GetCurrent(context);
+
+  env->SetMethodNoSideEffect(target, "buildEmbedderGraph", BuildEmbedderGraph);
+  env->SetMethodNoSideEffect(target, "createHeapDump", CreateHeapDump);
+}
+
+}  // namespace heap
+}  // namespace node
+
+NODE_MODULE_CONTEXT_AWARE_INTERNAL(heap_utils, node::heap::Initialize)

src/node_internals.h

@@ -111,6 +111,7 @@ struct sockaddr;
     V(domain)                                                                 \
     V(fs)                                                                     \
     V(fs_event_wrap)                                                          \
+    V(heap_utils)                                                             \
     V(http2)                                                                  \
     V(http_parser)                                                            \
     V(inspector)                                                              \