feat(debugger): add snapshot time budget

Enforce a per-snapshot time budget. By default this budget is 10ms, but
can be modified by the experimental config, either as an environment
variable:

    DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS=20

Or programatically:

    tracer.init({
      dynamicInstrumentation: {
        captureTimeoutMs: 20
      }
    })

When the budget is exceeded, remaining values that are not already
resolved, are marked with `notCapturedReason: 'timeout'`.

Author: watson

integration-tests/debugger/snapshot-time-budget.spec.js

@@ -0,0 +1,44 @@
+'use strict'
+
+const assert = require('node:assert')
+const { setup } = require('./utils')
+
+describe('Dynamic Instrumentation', function () {
+  // Force a very small time budget in ms to trigger partial snapshots
+  const t = setup({
+    dependencies: ['fastify'],
+    env: { DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS: '1' }
+  })
+
+  describe('input messages', function () {
+    describe('with snapshot under tight time budget', function () {
+      beforeEach(t.triggerBreakpoint)
+
+      it('should include partial snapshot marked with notCapturedReason: timeout', function (done) {
+        t.agent.on('debugger-input', ({ payload: [{ debugger: { snapshot: { captures } } }] }) => {
+          const { locals } = captures.lines[t.breakpoint.line]
+          assert.strictEqual(
+            containsTimeBudget(locals),
+            true,
+            'expected at least one field/element to be marked with notCapturedReason: "timeout"'
+          )
+          done()
+        })
+
+        t.agent.addRemoteConfig(t.generateRemoteConfig({
+          captureSnapshot: true,
+          capture: { maxReferenceDepth: 5 }
+        }))
+      })
+    })
+  })
+})
+
+function containsTimeBudget (node) {
+  if (node == null || typeof node !== 'object') return false
+  if (node.notCapturedReason === 'timeout') return true
+  for (const value of Object.values(node)) {
+    if (containsTimeBudget(value)) return true
+  }
+  return false
+}

integration-tests/debugger/target-app/snapshot-time-budget.js

@@ -0,0 +1,96 @@
+'use strict'
+
+require('dd-trace/init')
+
+const Fastify = require('fastify')
+
+const fastify = Fastify({ logger: { level: 'error' } })
+
+fastify.get('/:name', function handler (request) {
+  // The size of `obj` generated is carefully tuned to never result in a snapshot larger than the 1MB size limit, while
+  // still being large enough to trigger the time budget limit. However, the size of `fastify` and `request` is not
+  // stable across Node.js and Fastify versions, so the generated object might need to be adjusted.
+  const obj = generateObject(5, 6) // eslint-disable-line no-unused-vars
+
+  return { hello: request.params.name } // BREAKPOINT: /foo
+})
+
+fastify.listen({ port: process.env.APP_PORT || 0 }, (err) => {
+  if (err) {
+    fastify.log.error(err)
+    process.exit(1)
+  }
+  process.send?.({ port: fastify.server.address().port })
+})
+
+const leafValues = [null, undefined, true, 1, '']
+const complexTypes = ['object', 'array', 'map', 'set']
+
+/**
+ * Generate a complex nested object that requires a lot of async CDP calls to traverse
+ */
+function generateObject (depth, breath) {
+  const obj = {}
+  for (let i = 0; i < breath; i++) {
+    const key = `p${i}`
+    if (depth === 0) {
+      obj[key] = leafValues[i % leafValues.length]
+    } else {
+      const type = complexTypes[i % complexTypes.length]
+      obj[key] = generateType(type, depth - 1, breath)
+    }
+  }
+  return obj
+}
+
+function generateArray (depth, breath) {
+  const arr = []
+  for (let i = 0; i < breath; i++) {
+    if (depth === 0) {
+      arr.push(leafValues[i % leafValues.length])
+    } else {
+      const type = complexTypes[i % complexTypes.length]
+      arr.push(generateType(type, depth - 1, breath))
+    }
+  }
+  return arr
+}
+
+function generateMap (depth, breath) {
+  const map = new Map()
+  for (let i = 0; i < breath; i++) {
+    if (depth === 0) {
+      map.set(i, leafValues[i % leafValues.length])
+    } else {
+      const type = complexTypes[i % complexTypes.length]
+      map.set(i, generateType(type, depth - 1, breath))
+    }
+  }
+  return map
+}
+
+function generateSet (depth, breath) {
+  const set = new Set()
+  for (let i = 0; i < breath; i++) {
+    if (depth === 0) {
+      set.add(leafValues[i % leafValues.length])
+    } else {
+      const type = complexTypes[i % complexTypes.length]
+      set.add(generateType(type, depth - 1, breath))
+    }
+  }
+  return set
+}
+
+function generateType (type, depth, breath) {
+  switch (type) {
+    case 'object':
+      return generateObject(depth, breath)
+    case 'array':
+      return generateArray(depth, breath)
+    case 'map':
+      return generateMap(depth, breath)
+    case 'set':
+      return generateSet(depth, breath)
+  }
+}

packages/dd-trace/src/config.js

@@ -445,6 +445,7 @@ class Config {
       DD_DBM_PROPAGATION_MODE,
       DD_DOGSTATSD_HOST,
       DD_DOGSTATSD_PORT,
+      DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS,
       DD_DYNAMIC_INSTRUMENTATION_ENABLED,
       DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE,
       DD_DYNAMIC_INSTRUMENTATION_REDACTED_IDENTIFIERS,
@@ -683,6 +684,7 @@ class Config {
     this.#setString(target, 'dogstatsd.hostname', DD_DOGSTATSD_HOST)
     this.#setString(target, 'dogstatsd.port', DD_DOGSTATSD_PORT)
     this.#setBoolean(target, 'dsmEnabled', DD_DATA_STREAMS_ENABLED)
+    target['dynamicInstrumentation.captureTimeoutMs'] = maybeFloat(DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS)
     this.#setBoolean(target, 'dynamicInstrumentation.enabled', DD_DYNAMIC_INSTRUMENTATION_ENABLED)
     this.#setString(target, 'dynamicInstrumentation.probeFile', DD_DYNAMIC_INSTRUMENTATION_PROBE_FILE)
     this.#setArray(target, 'dynamicInstrumentation.redactedIdentifiers',
@@ -983,6 +985,8 @@ class Config {
       this.#setString(opts, 'dogstatsd.port', options.dogstatsd.port)
     }
     this.#setBoolean(opts, 'dsmEnabled', options.dsmEnabled)
+    opts['dynamicInstrumentation.captureTimeoutMs'] = maybeFloat(options.dynamicInstrumentation?.captureTimeoutMs)
+    this.#optsUnprocessed['dynamicInstrumentation.captureTimeoutMs'] = options.dynamicInstrumentation?.captureTimeoutMs
     this.#setBoolean(opts, 'dynamicInstrumentation.enabled', options.dynamicInstrumentation?.enabled)
     this.#setString(opts, 'dynamicInstrumentation.probeFile', options.dynamicInstrumentation?.probeFile)
     this.#setArray(

packages/dd-trace/src/config_defaults.js

@@ -63,6 +63,7 @@ module.exports = {
   'dogstatsd.hostname': '127.0.0.1',
   'dogstatsd.port': '8125',
   dsmEnabled: false,
+  'dynamicInstrumentation.captureTimeoutMs': 10,
   'dynamicInstrumentation.enabled': false,
   'dynamicInstrumentation.probeFile': undefined,
   'dynamicInstrumentation.redactedIdentifiers': [],

packages/dd-trace/src/debugger/devtools_client/index.js

@@ -7,7 +7,7 @@ const { getLocalStateForCallFrame } = require('./snapshot')
 const send = require('./send')
 const { getStackFromCallFrames } = require('./state')
 const { ackEmitting } = require('./status')
-const { parentThreadId } = require('./config')
+const config = require('./config')
 const { MAX_SNAPSHOTS_PER_SECOND_GLOBALLY } = require('./defaults')
 const log = require('./log')
 const { version } = require('../../../../../package.json')
@@ -33,8 +33,8 @@ const getDDTagsExpression = `(() => {
 
 // There doesn't seem to be an official standard for the content of these fields, so we're just populating them with
 // something that should be useful to a Node.js developer.
-const threadId = parentThreadId === 0 ? `pid:${process.pid}` : `pid:${process.pid};tid:${parentThreadId}`
-const threadName = parentThreadId === 0 ? 'MainThread' : `WorkerThread:${parentThreadId}`
+const threadId = config.parentThreadId === 0 ? `pid:${process.pid}` : `pid:${process.pid};tid:${config.parentThreadId}`
+const threadName = config.parentThreadId === 0 ? 'MainThread' : `WorkerThread:${config.parentThreadId}`
 
 const SUPPORT_ARRAY_BUFFER_RESIZE = NODE_MAJOR >= 20
 const oneSecondNs = 1_000_000_000n
@@ -162,6 +162,7 @@ session.on('Debugger.paused', async ({ params }) => {
   // TODO: Create unique states for each affected probe based on that probes unique `capture` settings (DEBUG-2863)
   const processLocalState = numberOfProbesWithSnapshots !== 0 && await getLocalStateForCallFrame(
     params.callFrames[0],
+    start + BigInt(config.dynamicInstrumentation.captureTimeoutMs ?? 10) * 1_000_000n,
     { maxReferenceDepth, maxCollectionSize, maxFieldCount, maxLength }
   )
 

packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js

@@ -1,13 +1,13 @@
 'use strict'
 
-const { collectionSizeSym, fieldCountSym } = require('./symbols')
+const { collectionSizeSym, fieldCountSym, timeBudgetSym } = require('./symbols')
 const session = require('../session')
 
 const LEAF_SUBTYPES = new Set(['date', 'regexp'])
 const ITERABLE_SUBTYPES = new Set(['map', 'set', 'weakmap', 'weakset'])
 
 module.exports = {
-  getRuntimeObject: getObject
+  getRuntimeObject: getObject // TODO: Called once per stack frame, but doesn't retain the `deadlineReached` flag.
 }
 
 async function getObject (objectId, opts, depth = 0, collection = false) {
@@ -47,6 +47,10 @@ async function traverseGetPropertiesResult (props, opts, depth) {
 
   for (const prop of props) {
     if (prop.value === undefined) continue
+    if (overBudget(opts)) {
+      prop.value[timeBudgetSym] = true
+      continue
+    }
     const { value: { type, objectId, subtype } } = prop
     if (type === 'object') {
       if (objectId === undefined) continue // if `subtype` is "null"
@@ -189,3 +193,9 @@ function removeNonEnumerableProperties (props) {
     }
   }
 }
+
+function overBudget (opts) {
+  if (opts.deadlineReached) return true
+  opts.deadlineReached = process.hrtime.bigint() >= opts.deadlineNs
+  return opts.deadlineReached
+}

packages/dd-trace/src/debugger/devtools_client/snapshot/index.js

@@ -19,6 +19,7 @@ function returnError () {
 
 async function getLocalStateForCallFrame (
   callFrame,
+  deadlineNs,
   {
     maxReferenceDepth = DEFAULT_MAX_REFERENCE_DEPTH,
     maxCollectionSize = DEFAULT_MAX_COLLECTION_SIZE,
@@ -34,7 +35,7 @@ async function getLocalStateForCallFrame (
       if (scope.type === 'global') return // The global scope is too noisy
       rawState.push(...await getRuntimeObject(
         scope.object.objectId,
-        { maxReferenceDepth, maxCollectionSize, maxFieldCount }
+        { maxReferenceDepth, maxCollectionSize, maxFieldCount, deadlineNs }
       ))
     }))
   } catch (err) {