feat(metrics): add metricsLimit option (#1273)

Fixes #1249

Author: Qard

docs/configuration.asciidoc

@@ -719,6 +719,17 @@ or should include a time suffix.
 To disable metrics reporting,
 set the interval to `0`.
 
+[[metrics-limit]]
+==== `metricsLimit`
+
+* *Type:* Number
+* *Default:* `1000`
+* *Env:* `ELASTIC_APM_METRICS_LIMIT`
+
+Specify the maximum number of metrics to track at any given time.
+When a new metric is inserted which would exceed the limit,
+the oldest metric will be dropped to give it space.
+
 [[global-labels]]
 ==== `globalLabels`
 

lib/config.js

@@ -68,6 +68,7 @@ var DEFAULTS = {
   kubernetesPodUID: undefined,
   logLevel: 'info',
   metricsInterval: '30s',
+  metricsLimit: 1000,
   centralConfig: true,
   serverTimeout: '30s',
   sourceLinesErrorAppFrames: 5,
@@ -111,6 +112,7 @@ var ENV_TABLE = {
   kubernetesPodUID: ['ELASTIC_APM_KUBERNETES_POD_UID', 'KUBERNETES_POD_UID'],
   logLevel: 'ELASTIC_APM_LOG_LEVEL',
   metricsInterval: 'ELASTIC_APM_METRICS_INTERVAL',
+  metricsLimit: 'ELASTIC_APM_METRICS_LIMIT',
   payloadLogFile: 'ELASTIC_APM_PAYLOAD_LOG_FILE',
   centralConfig: 'ELASTIC_APM_CENTRAL_CONFIG',
   secretToken: 'ELASTIC_APM_SECRET_TOKEN',
@@ -153,6 +155,7 @@ var BOOL_OPTS = [
 ]
 
 var NUM_OPTS = [
+  'metricsLimit',
   'sourceLinesErrorAppFrames',
   'sourceLinesErrorLibraryFrames',
   'sourceLinesSpanAppFrames',

lib/metrics/registry.js

@@ -3,6 +3,7 @@
 const os = require('os')
 
 const { SelfReportingMetricsRegistry } = require('measured-reporting')
+const DimensionAwareMetricsRegistry = require('measured-reporting/lib/registries/DimensionAwareMetricsRegistry')
 
 const MetricsReporter = require('./reporter')
 const createRuntimeMetrics = require('./runtime')
@@ -11,7 +12,7 @@ const createSystemMetrics = process.platform === 'linux'
   : require('./platforms/generic')
 
 class MetricsRegistry extends SelfReportingMetricsRegistry {
-  constructor (agent, { reporterOptions, registryOptions } = {}) {
+  constructor (agent, { reporterOptions, registryOptions = {} } = {}) {
     const defaultReporterOptions = {
       defaultDimensions: {
         hostname: agent._conf.hostname || os.hostname(),
@@ -22,6 +23,10 @@ class MetricsRegistry extends SelfReportingMetricsRegistry {
     const options = Object.assign({}, defaultReporterOptions, reporterOptions)
     const reporter = new MetricsReporter(agent, options)
 
+    registryOptions.registry = new DimensionAwareMetricsRegistry({
+      metricLimit: agent._conf.metricsLimit
+    })
+
     super(reporter, registryOptions)
     this._agent = agent
 

lib/metrics/reporter.js

@@ -28,6 +28,9 @@ class MetricsReporter extends Reporter {
       const seen = new ObjectIdentityMap()
 
       for (const metric of metrics) {
+        // Due to limitations in measured-reporting, metrics dropped
+        // due to `metricsLimit` leave empty slots in the list.
+        if (!metric) continue
         const data = seen.ensure(metric.dimensions, () => {
           const metricData = unflattenBreakdown(metric.dimensions)
           const merged = Object.assign({ samples: {} }, baseDimensions, metricData)

package.json

@@ -84,7 +84,7 @@
     "fast-safe-stringify": "^2.0.6",
     "http-headers": "^3.0.2",
     "is-native": "^1.0.1",
-    "measured-reporting": "^1.49.0",
+    "measured-reporting": "^1.50.0",
     "monitor-event-loop-delay": "^1.0.0",
     "object-filter-sequence": "^1.0.0",
     "object-identity-map": "^1.0.1",

test/config.js

@@ -52,6 +52,7 @@ var optionFixtures = [
   ['kubernetesPodUID', 'KUBERNETES_POD_UID'],
   ['logLevel', 'LOG_LEVEL', 'info'],
   ['metricsInterval', 'METRICS_INTERVAL', 30],
+  ['metricsLimit', 'METRICS_LIMIT', 1000],
   ['secretToken', 'SECRET_TOKEN'],
   ['serverTimeout', 'SERVER_TIMEOUT', 30],
   ['serverUrl', 'SERVER_URL'],

test/metrics/index.js

@@ -145,6 +145,26 @@ test('reports expected metrics', function (t) {
   metrics.start()
 })
 
+test('applies metrics limit', function (t) {
+  agent = mockAgent({
+    metricsInterval: 10,
+    metricsLimit: 2,
+    hostname: 'foo',
+    environment: 'bar'
+  }, (metricset = {}) => {
+    t.equal(Object.keys(metricset.samples).length, 2, 'has expected number of metrics')
+    t.end()
+  })
+
+  metrics = new Metrics(agent)
+  metrics.start()
+
+  // Ensure there are at least two counters
+  metrics.getOrCreateCounter('first').inc()
+  metrics.getOrCreateCounter('second').inc()
+  metrics.getOrCreateCounter('third').inc()
+})
+
 function spinCPUFor (durationMs) {
   const start = Date.now()
   while (Date.now() - start < durationMs) {}