Include system_configuration, driver_connection_params, and auth_type in all telemetry logs

samikshya-db · claude · samikshya-db · commit a37fdf083ff2 · 2026-01-30T09:21:50.000Z
- Cache driver config in MetricsAggregator when connection event is processed
- Include cached driver config in all statement and error metrics
- Export system_configuration, driver_connection_params, and auth_type for every log
- Each telemetry log is now self-contained with full context

This ensures every telemetry event (connection, statement, error) includes
the driver configuration context, making logs independently analyzable.

Co-Authored-By: Claude Sonnet 4.5 &lt;noreply@anthropic.com&gt;
diff --git a/lib/telemetry/DatabricksTelemetryExporter.ts b/lib/telemetry/DatabricksTelemetryExporter.ts
@@ -288,9 +288,8 @@ export default class DatabricksTelemetryExporter {
       },
     };
 
-    // Add metric-specific fields based on proto definition
-    if (metric.metricType === 'connection' && metric.driverConfig) {
-      // Map driverConfig to system_configuration (snake_case as per proto)
+    // Include system_configuration, driver_connection_params, and auth_type for ALL metrics (if available)
+    if (metric.driverConfig) {
       log.entry.sql_driver_log.system_configuration = {
         driver_version: metric.driverConfig.driverVersion,
         driver_name: metric.driverConfig.driverName,
@@ -304,10 +303,7 @@ export default class DatabricksTelemetryExporter {
         char_set_encoding: metric.driverConfig.charSetEncoding,
         process_name: metric.driverConfig.processName,
       };
-      // Include connection open latency
-      if (metric.latencyMs !== undefined) {
-        log.entry.sql_driver_log.operation_latency_ms = metric.latencyMs;
-      }
+
       // Include driver connection params (only if we have fields to include)
       if (
         metric.driverConfig.httpPath ||
@@ -326,8 +322,17 @@ export default class DatabricksTelemetryExporter {
           }),
         };
       }
+
       // Include auth type at top level
       log.entry.sql_driver_log.auth_type = metric.driverConfig.authType;
+    }
+
+    // Add metric-specific fields based on proto definition
+    if (metric.metricType === 'connection') {
+      // Include connection open latency
+      if (metric.latencyMs !== undefined) {
+        log.entry.sql_driver_log.operation_latency_ms = metric.latencyMs;
+      }
     } else if (metric.metricType === 'statement') {
       log.entry.sql_driver_log.operation_latency_ms = metric.latencyMs;
 
diff --git a/lib/telemetry/MetricsAggregator.ts b/lib/telemetry/MetricsAggregator.ts
@@ -16,7 +16,13 @@
 
 import IClientContext from '../contracts/IClientContext';
 import { LogLevel } from '../contracts/IDBSQLLogger';
-import { TelemetryEvent, TelemetryEventType, TelemetryMetric, DEFAULT_TELEMETRY_CONFIG } from './types';
+import {
+  TelemetryEvent,
+  TelemetryEventType,
+  TelemetryMetric,
+  DriverConfiguration,
+  DEFAULT_TELEMETRY_CONFIG,
+} from './types';
 import DatabricksTelemetryExporter from './DatabricksTelemetryExporter';
 import ExceptionClassifier from './ExceptionClassifier';
 
@@ -64,6 +70,8 @@ export default class MetricsAggregator {
 
   private flushIntervalMs: number;
 
+  private driverConfig?: DriverConfiguration;
+
   constructor(private context: IClientContext, private exporter: DatabricksTelemetryExporter) {
     try {
       const config = context.getConfig();
@@ -118,6 +126,11 @@ export default class MetricsAggregator {
    * Process connection event (emit immediately)
    */
   private processConnectionEvent(event: TelemetryEvent): void {
+    // Cache driver config for use in all subsequent metrics
+    if (event.driverConfig) {
+      this.driverConfig = event.driverConfig;
+    }
+
     const metric: TelemetryMetric = {
       metricType: 'connection',
       timestamp: event.timestamp,
@@ -153,13 +166,14 @@ export default class MetricsAggregator {
         details.errors.push(event);
         this.completeStatement(event.statementId);
       } else {
-        // Standalone error - emit immediately
+        // Standalone error - emit immediately (include cached driver config for context)
         const metric: TelemetryMetric = {
           metricType: 'error',
           timestamp: event.timestamp,
           sessionId: event.sessionId,
           statementId: event.statementId,
           workspaceId: event.workspaceId,
+          driverConfig: this.driverConfig,
           errorName: event.errorName,
           errorMessage: event.errorMessage,
         };
@@ -245,13 +259,14 @@ export default class MetricsAggregator {
         return;
       }
 
-      // Create statement metric
+      // Create statement metric (include cached driver config for context)
       const metric: TelemetryMetric = {
         metricType: 'statement',
         timestamp: details.startTime,
         sessionId: details.sessionId,
         statementId: details.statementId,
         workspaceId: details.workspaceId,
+        driverConfig: this.driverConfig,
         operationType: details.operationType,
         latencyMs: details.executionLatencyMs,
         resultFormat: details.resultFormat,
@@ -263,14 +278,15 @@ export default class MetricsAggregator {
 
       this.addPendingMetric(metric);
 
-      // Add buffered error metrics
+      // Add buffered error metrics (include cached driver config for context)
       for (const errorEvent of details.errors) {
         const errorMetric: TelemetryMetric = {
           metricType: 'error',
           timestamp: errorEvent.timestamp,
           sessionId: details.sessionId,
           statementId: details.statementId,
           workspaceId: details.workspaceId,
+          driverConfig: this.driverConfig,
           errorName: errorEvent.errorName,
           errorMessage: errorEvent.errorMessage,
         };
diff --git a/lib/telemetry/types.ts b/lib/telemetry/types.ts
@@ -151,7 +151,7 @@ export interface TelemetryMetric {
   /** Workspace ID */
   workspaceId?: string;
 
-  /** Driver configuration (for connection metrics) */
+  /** Driver configuration (included in all metrics for context) */
   driverConfig?: DriverConfiguration;
 
   /** Execution latency in milliseconds */
diff --git a/tests/e2e/telemetry-local.test.ts b/tests/e2e/telemetry-local.test.ts
@@ -0,0 +1,109 @@
+/**
+ * LOCAL TELEMETRY TEST - NOT FOR COMMIT
+ *
+ * This test verifies telemetry requests are properly sent.
+ * Run locally with valid credentials to check telemetry payload structure.
+ *
+ * Set environment variables:
+ * - DATABRICKS_SERVER_HOSTNAME
+ * - DATABRICKS_HTTP_PATH
+ * - DATABRICKS_TOKEN
+ */
+
+import { DBSQLClient, LogLevel } from '../../lib';
+import IDBSQLLogger from '../../lib/contracts/IDBSQLLogger';
+
+// Custom logger to capture telemetry debug logs
+class DebugLogger implements IDBSQLLogger {
+  async log(level: LogLevel, message: string): Promise<void> {
+    const timestamp = new Date().toISOString();
+    const levelStr = LogLevel[level].padEnd(5);
+
+    // Highlight telemetry-related logs
+    if (message.includes('telemetry') || message.includes('Telemetry')) {
+      console.log(`\x1b[36m[${timestamp}] [${levelStr}] ${message}\x1b[0m`);
+    } else {
+      console.log(`[${timestamp}] [${levelStr}] ${message}`);
+    }
+  }
+}
+
+describe('Telemetry E2E Test (Local Only)', () => {
+  it('should send telemetry for SELECT 1 query', async function () {
+    this.timeout(30000);
+
+    // Check for required environment variables
+    const host = process.env.DATABRICKS_SERVER_HOSTNAME;
+    const path = process.env.DATABRICKS_HTTP_PATH;
+    const token = process.env.DATABRICKS_TOKEN;
+
+    if (!host || !path || !token) {
+      console.log('\n❌ Skipping test: Missing environment variables');
+      console.log('Set the following variables to run this test:');
+      console.log('  - DATABRICKS_SERVER_HOSTNAME');
+      console.log('  - DATABRICKS_HTTP_PATH');
+      console.log('  - DATABRICKS_TOKEN\n');
+      this.skip();
+      return;
+    }
+
+    console.log('\n' + '='.repeat(60));
+    console.log('TELEMETRY E2E TEST');
+    console.log('='.repeat(60));
+
+    const client = new DBSQLClient({
+      logger: new DebugLogger(),
+    });
+
+    console.log('\n📡 Connecting with telemetry enabled...\n');
+
+    const connection = await client.connect({
+      host,
+      path,
+      token,
+      telemetryEnabled: true,
+      telemetryBatchSize: 1, // Flush immediately for testing
+    });
+
+    console.log('\n' + '='.repeat(60));
+    console.log('EXECUTING SELECT 1');
+    console.log('='.repeat(60) + '\n');
+
+    const session = await connection.openSession();
+    const queryOperation = await session.executeStatement('SELECT 1', {
+      runAsync: false,
+    });
+
+    const result = await queryOperation.fetchAll();
+    console.log('\n✅ Query Result:', JSON.stringify(result, null, 2));
+
+    await queryOperation.close();
+    console.log('\n📝 Statement closed - waiting for telemetry flush...\n');
+
+    // Wait for telemetry to flush
+    await new Promise<void>((resolve) => {
+      setTimeout(resolve, 3000);
+    });
+
+    console.log('\n' + '='.repeat(60));
+    console.log('CLEANING UP');
+    console.log('='.repeat(60) + '\n');
+
+    await session.close();
+    await connection.close();
+
+    // Wait for final flush
+    await new Promise<void>((resolve) => {
+      setTimeout(resolve, 2000);
+    });
+
+    console.log('\n' + '='.repeat(60));
+    console.log('TEST COMPLETE');
+    console.log('='.repeat(60));
+    console.log('\nCheck the logs above for telemetry-related messages (shown in cyan)');
+    console.log('Look for:');
+    console.log('  - "Exporting N telemetry metrics"');
+    console.log('  - "Successfully exported N telemetry metrics"');
+    console.log('  - "Feature flag enabled: true"\n');
+  });
+});
