feat: truncate payloads according to intake API limits (#8)

elastic · Nov 8, 2018 · 7d15249 · 7d15249
1 parent 9e01f66
commit 7d15249
Show file tree

Hide file tree

Showing 8 changed files with 488 additions and 72 deletions.
diff --git a/README.md b/README.md
@@ -105,9 +105,17 @@ Streaming configuration:
 
 Data sanitizing configuration:
 
-- `truncateStringsAt` - Maximum size in bytes for strings stored as
+- `truncateKeywordsAt` - Maximum size in bytes for strings stored as
   Elasticsearch keywords. Strings larger than this will be trucated
   (default: `1024` bytes)
+- `truncateErrorMessagesAt` - The maximum size in bytes for error
+  messages. Messages above this length will be truncated. Set to `-1` to
+  disable truncation. This applies to the following properties:
+  `error.exception.message` and `error.log.message` (default: `2048`
+  bytes)
+- `truncateSourceLinesAt` - The maximum size in bytes for souce code
+  lines in stack traces. Lines above this length will be truncated
+  (default: `1000` bytes)
 
 ### Event: `close`
 

diff --git a/index.js b/index.js
@@ -9,8 +9,8 @@ const pump = require('pump')
 const eos = require('end-of-stream')
 const streamToBuffer = require('fast-stream-to-buffer')
 const StreamChopper = require('stream-chopper')
-const truncate = require('unicode-byte-truncate')
 const ndjson = require('./lib/ndjson')
+const truncate = require('./lib/truncate')
 const pkg = require('./package')
 
 module.exports = Client
@@ -43,7 +43,7 @@ util.inherits(Client, Writable)
 function Client (opts) {
   if (!(this instanceof Client)) return new Client(opts)
 
-  opts = normalizeOptions(opts)
+  this._opts = opts = normalizeOptions(opts)
 
   Writable.call(this, opts)
 
@@ -96,6 +96,13 @@ Client.prototype._write = function (obj, enc, cb) {
       this._chopper.chop(cb)
     }
   } else {
+    if ('transaction' in obj) {
+      truncate.transaction(obj.transaction, this._opts)
+    } else if ('span' in obj) {
+      truncate.span(obj.span, this._opts)
+    } else if ('error' in obj) {
+      truncate.error(obj.error, this._opts)
+    }
     this._received++
     this._chopper.write(ndjson.serialize(obj), cb)
   }
@@ -221,7 +228,9 @@ function onStream (opts, client, onerror) {
     })
 
     // All requests to the APM Server must start with a metadata object
-    stream.write(ndjson.serialize({metadata: metadata(opts)}))
+    const metadata = getMetadata(opts)
+    truncate.metadata(metadata, opts)
+    stream.write(ndjson.serialize({metadata}))
   }
 }
 
@@ -255,7 +264,9 @@ function normalizeOptions (opts) {
   if (!normalized.serverTimeout && normalized.serverTimeout !== 0) normalized.serverTimeout = 15000
   if (!normalized.serverUrl) normalized.serverUrl = 'http://localhost:8200'
   if (!normalized.hostname) normalized.hostname = hostname
-  if (!normalized.truncateStringsAt) normalized.truncateStringsAt = 1024
+  if (!normalized.truncateKeywordsAt) normalized.truncateKeywordsAt = 1024
+  if (!normalized.truncateErrorMessagesAt) normalized.truncateErrorMessagesAt = 2048
+  if (!normalized.truncateSourceLinesAt) normalized.truncateSourceLinesAt = 1000
   normalized.keepAlive = normalized.keepAlive !== false
 
   // process
@@ -287,7 +298,7 @@ function getHeaders (opts) {
   return Object.assign(headers, opts.headers)
 }
 
-function metadata (opts) {
+function getMetadata (opts) {
   var payload = {
     service: {
       name: opts.serviceName,
@@ -306,7 +317,7 @@ function metadata (opts) {
     process: {
       pid: process.pid,
       ppid: process.ppid,
-      title: truncate(String(process.title), opts.truncateStringsAt),
+      title: process.title,
       argv: process.argv
     },
     system: {

diff --git a/lib/truncate.js b/lib/truncate.js
@@ -0,0 +1,130 @@
+'use strict'
+
+var truncate = require('unicode-byte-truncate')
+
+exports.metadata = truncMetadata
+exports.transaction = truncTransaction
+exports.span = truncSpan
+exports.error = truncError
+
+function truncMetadata (metadata, opts) {
+  metadata.process.title = truncate(String(metadata.process.title), opts.truncateKeywordsAt)
+}
+
+function truncTransaction (trans, opts) {
+  trans.name = truncate(String(trans.name), opts.truncateKeywordsAt)
+  trans.type = truncate(String(trans.type), opts.truncateKeywordsAt)
+  trans.result = truncate(String(trans.result), opts.truncateKeywordsAt)
+
+  // Unless sampled, context will be null
+  if (trans.sampled) truncContext(trans.context, opts.truncateKeywordsAt)
+}
+
+function truncSpan (span, opts) {
+  span.name = truncate(String(span.name), opts.truncateKeywordsAt)
+  span.type = truncate(String(span.type), opts.truncateKeywordsAt)
+  if (span.stacktrace) span.stacktrace = truncFrames(span.stacktrace, opts.truncateSourceLinesAt)
+}
+
+function truncError (error, opts) {
+  if (error.log) {
+    if (error.log.level) {
+      error.log.level = truncate(String(error.log.level), opts.truncateKeywordsAt)
+    }
+    if (error.log.logger_name) {
+      error.log.logger_name = truncate(String(error.log.logger_name), opts.truncateKeywordsAt)
+    }
+    if (error.log.message && opts.truncateErrorMessagesAt >= 0) {
+      error.log.message = truncate(String(error.log.message), opts.truncateErrorMessagesAt)
+    }
+    if (error.log.param_message) {
+      error.log.param_message = truncate(String(error.log.param_message), opts.truncateKeywordsAt)
+    }
+    if (error.log.stacktrace) {
+      error.log.stacktrace = truncFrames(error.log.stacktrace, opts.truncateSourceLinesAt)
+    }
+  }
+
+  if (error.exception) {
+    if (error.exception.message && opts.truncateErrorMessagesAt >= 0) {
+      error.exception.message = truncate(String(error.exception.message), opts.truncateErrorMessagesAt)
+    }
+    if (error.exception.type) {
+      error.exception.type = truncate(String(error.exception.type), opts.truncateKeywordsAt)
+    }
+    if (error.exception.code) {
+      error.exception.code = truncate(String(error.exception.code), opts.truncateKeywordsAt)
+    }
+    if (error.exception.module) {
+      error.exception.module = truncate(String(error.exception.module), opts.truncateKeywordsAt)
+    }
+    if (error.exception.stacktrace) {
+      error.exception.stacktrace = truncFrames(error.exception.stacktrace, opts.truncateSourceLinesAt)
+    }
+  }
+
+  truncContext(error.context, opts.truncateKeywordsAt)
+}
+
+function truncContext (context, max) {
+  if (!context) return
+
+  if (context.request) {
+    if (context.request.method) {
+      context.request.method = truncate(String(context.request.method), max)
+    }
+    if (context.request.url) {
+      if (context.request.url.protocol) {
+        context.request.url.protocol = truncate(String(context.request.url.protocol), max)
+      }
+      if (context.request.url.hostname) {
+        context.request.url.hostname = truncate(String(context.request.url.hostname), max)
+      }
+      if (context.request.url.port) {
+        context.request.url.port = truncate(String(context.request.url.port), max)
+      }
+      if (context.request.url.pathname) {
+        context.request.url.pathname = truncate(String(context.request.url.pathname), max)
+      }
+      if (context.request.url.search) {
+        context.request.url.search = truncate(String(context.request.url.search), max)
+      }
+      if (context.request.url.hash) {
+        context.request.url.hash = truncate(String(context.request.url.hash), max)
+      }
+      if (context.request.url.raw) {
+        context.request.url.raw = truncate(String(context.request.url.raw), max)
+      }
+      if (context.request.url.full) {
+        context.request.url.full = truncate(String(context.request.url.full), max)
+      }
+    }
+  }
+  if (context.user) {
+    if (context.user.id) {
+      context.user.id = truncate(String(context.user.id), max)
+    }
+    if (context.user.email) {
+      context.user.email = truncate(String(context.user.email), max)
+    }
+    if (context.user.username) {
+      context.user.username = truncate(String(context.user.username), max)
+    }
+  }
+}
+
+function truncFrames (frames, max) {
+  frames.forEach(function (frame, i) {
+    if (frame.pre_context) frame.pre_context = truncEach(frame.pre_context, max)
+    if (frame.context_line) frame.context_line = truncate(String(frame.context_line), max)
+    if (frame.post_context) frame.post_context = truncEach(frame.post_context, max)
+  })
+
+  return frames
+}
+
+function truncEach (arr, len) {
+  return arr.map(function (str) {
+    return truncate(String(str), len)
+  })
+}
diff --git a/package.json b/package.json
@@ -8,7 +8,7 @@
   },
   "scripts": {
     "coverage": "nyc report --reporter=text-lcov > coverage.lcov && codecov",
-    "test": "standard && nyc node test/test.js"
+    "test": "standard && nyc tape test/*.js"
   },
   "engines": {
     "node": "6 || 8 || 10"

diff --git a/test/unref-client.js → test/lib/unref-client.js b/test/unref-client.js → test/lib/unref-client.js
@@ -1,6 +1,6 @@
 'use strict'
 
-const Client = require('../')
+const Client = require('../../')
 
 const client = new Client({
   serverUrl: process.argv[2],

diff --git a/test/utils.js → test/lib/utils.js b/test/utils.js → test/lib/utils.js
@@ -6,13 +6,14 @@ const zlib = require('zlib')
 const semver = require('semver')
 const pem = require('https-pem')
 const ndjson = require('ndjson')
-const pkg = require('../package')
-const Client = require('../')
+const pkg = require('../../package')
+const Client = require('../../')
 
 exports.APMServer = APMServer
 exports.processReq = processReq
 exports.assertReq = assertReq
 exports.assertMetadata = assertMetadata
+exports.assertEvent = assertEvent
 exports.validOpts = validOpts
 
 function APMServer (opts, onreq) {
@@ -79,11 +80,20 @@ function assertMetadata (t, obj) {
   const _process = metadata.process
   t.ok(_process.pid > 0)
   t.ok(_process.ppid > 0)
-  t.ok(/(\/node|^node)$/.test(_process.title), `process.title should match /(\\/node|^node)$/ (was: ${_process.title})`)
+
+  if (_process.title.length === 1) {
+    // because of truncation test
+    t.equal(_process.title, process.title[0])
+  } else {
+    const regex = /(\/node|^node)$/
+    t.ok(regex.test(_process.title), `process.title should match ${regex} (was: ${_process.title})`)
+  }
+
   t.ok(Array.isArray(_process.argv), 'process.title should be an array')
   t.ok(_process.argv.length >= 2, 'process.title should contain at least two elements')
   t.ok(/\/node$/.test(_process.argv[0]), `process.argv[0] should match /\\/node$/ (was: ${_process.argv[0]})`)
-  t.ok(/\/test\/(test|unref-client)\.js$/.test(_process.argv[1]), `process.argv[1] should match /\\/test\\/(test|unref-client)\\.js$/ (was: ${_process.argv[1]})"`)
+  const regex = /(\/test\/(test|truncate|lib\/unref-client)\.js|node_modules\/\.bin\/tape)$/
+  t.ok(regex.test(_process.argv[1]), `process.argv[1] should match ${regex} (was: ${_process.argv[1]})"`)
   const system = metadata.system
   t.ok(typeof system.hostname, 'string')
   t.ok(system.hostname.length > 0)
@@ -94,6 +104,30 @@ function assertMetadata (t, obj) {
 }
 assertMetadata.asserts = 22
 
+function assertEvent (expect) {
+  return function (t, obj) {
+    const key = Object.keys(expect)[0]
+    const val = expect[key]
+    switch (key) {
+      case 'transaction':
+        if (!('name' in val)) val.name = 'undefined'
+        if (!('type' in val)) val.type = 'undefined'
+        if (!('result' in val)) val.result = 'undefined'
+        break
+      case 'span':
+        if (!('name' in val)) val.name = 'undefined'
+        if (!('type' in val)) val.type = 'undefined'
+        break
+      case 'error':
+        break
+      default:
+        t.fail('unexpected event type: ' + key)
+    }
+    t.deepEqual(obj, expect)
+  }
+}
+assertEvent.asserts = 1
+
 function validOpts (opts) {
   return Object.assign({
     agentName: 'my-agent-name',