feat(csharp/src/Drivers/Databricks): Add Logging Capabilities to Databricks driver

csharp/src/Drivers/Databricks/Log/DatabricksLogger.cs

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.IO;
+
+namespace Apache.Arrow.Adbc.Drivers.Databricks.Log;
+
+public class DatabricksLogger
+{
+    public static readonly string DATABRICKS_LOG_FILE = "databricks_adbc.log";
+    public static readonly DatabricksLogger LOGGER = new DatabricksLogger();
+
+    private static readonly object _logLock = new object();
+    private static readonly string _logFilePath = Path.Combine(Path.GetTempPath(), DATABRICKS_LOG_FILE);
+
+    public void trace(string message)
+    {
+        WriteLog("TRACE", message);
+    }
+
+    public void debug(string message)
+    {
+        WriteLog("DEBUG", message);
+    }
+
+    public void info(string message)
+    {
+        WriteLog("INFO", message);
+    }
+
+    public void warn(string message)
+    {
+        WriteLog("WARN", message);
+    }
+
+    public void error(string message)
+    {
+        WriteLog("ERROR", message);
+    }
+
+    public void fatal(string message)
+    {
+        WriteLog("FATAL", message);
+    }
+
+    private static void WriteLog(string level, string message)
+    {
+        var logMessage = $"{DateTime.Now:yyyy-MM-dd HH:mm:ss.fff} [{level}] {message}";
+
+        lock (_logLock)
+        {
+            try
+            {
+                File.AppendAllText(_logFilePath, logMessage + Environment.NewLine);
+            }
+            catch (Exception)
+            {
+            }
+        }
+    }
+}

csharp/src/Drivers/Databricks/Telemetry/DatabricksActivityListener.cs

@@ -0,0 +1,70 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using Apache.Arrow.Adbc.Tracing;
+using Apache.Arrow.Adbc.Drivers.Apache;
+using Apache.Arrow.Adbc.Drivers.Databricks.Telemetry.Model;
+using Apache.Arrow.Adbc.Drivers.Databricks.Telemetry.Enums;
+using static Apache.Arrow.Adbc.Drivers.Databricks.Log.DatabricksLogger;
+
+namespace Apache.Arrow.Adbc.Drivers.Databricks.Telemetry
+{
+    internal class DatabricksActivityListener : IDisposable
+    {
+        private readonly ActivityListener _activityListener;
+        private TelemetryHelper? _telemetryHelper;
+
+        public DatabricksActivityListener(TelemetryHelper? telemetryHelper, string sourceName, Guid guid)
+        {
+            this._telemetryHelper = telemetryHelper;
+            this._activityListener = new ActivityListener
+            {
+                ShouldListenTo = (activitySource) => activitySource.Tags?.Any(kvp => kvp.Key == "guid" && kvp.Value?.Equals(guid) == true) == true,
+                Sample = (ref ActivityCreationOptions<ActivityContext> _) => ActivitySamplingResult.AllData,
+                ActivityStopped = OnActivityStopped,
+            };
+            ActivitySource.AddActivityListener(_activityListener);
+        }
+
+        private void OnActivityStopped(Activity activity)
+        {       
+            if(_telemetryHelper == null)
+            {
+                return;
+            }
+
+            if(activity.OperationName?.EndsWith("ExecuteStatementAsync") == true)
+            {   
+                LOGGER.info("OnActivityStopped: " + activity.OperationName);
+                var sqlExecutionEvent = new SqlExecutionEvent();
+                var operationDetail = new OperationDetail();
+                operationDetail.OperationType = Util.StringToOperationType("EXECUTE_STATEMENT_ASYNC");
+                sqlExecutionEvent.OperationDetail = operationDetail;
+                _telemetryHelper.AddSqlExecutionEvent(sqlExecutionEvent, Convert.ToInt64(activity.Duration.TotalMilliseconds));
+            }
+        }
+
+        public void Dispose()
+        {   
+            this._activityListener.Dispose();
+        }
+    }
+}

csharp/src/Drivers/Databricks/Telemetry/DatabricksConnectionConfig.cs

@@ -0,0 +1,24 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+namespace Apache.Arrow.Adbc.Drivers.Databricks.Telemetry;
+
+internal class DatabricksConnectionConfig
+{
+    public static readonly int MAX_BATCH_SIZE = 200;
+    public static readonly int FLUSH_INTERVAL_MILLIS = 300000; // 5 minutes
+}

csharp/src/Drivers/Databricks/Telemetry/README.md

@@ -0,0 +1,7 @@
+# Databricks Telemetry
+
+Uses `DatabricksActivityListener.cs` to listen and record events to `TelemetryHelper.cs` using ActivityListener.
+
+`TelemetryHelper.cs` stores telemetry logs and telemetry parameters.
+
+`TelemtryClient.cs` sends logs to `/telemetry` or `/telemetry-unauth` depending on authentication level.

csharp/src/Drivers/Databricks/Telemetry/TelemetryClient.cs

@@ -0,0 +1,124 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net.Http;
+using System.Net.Http.Headers;
+using System.Text.Json;
+using System.Threading.Tasks;
+using Apache.Arrow.Adbc.Drivers.Databricks.Telemetry.Model;
+using static Apache.Arrow.Adbc.Drivers.Databricks.Log.DatabricksLogger;
+
+namespace Apache.Arrow.Adbc.Drivers.Databricks.Telemetry
+{
+    internal class TelemetryClient
+    {
+        private readonly HttpClient _httpClient;
+        private readonly string? _telemetryUrl;
+        private readonly string? _accessToken;
+
+        public TelemetryClient(HttpClient httpClient, string? hostUrl, string? accessToken)
+        {
+            _httpClient = httpClient;
+            _accessToken = accessToken;
+            _telemetryUrl = !string.IsNullOrEmpty(hostUrl) ? accessToken != null ? $"https://{hostUrl}/telemetry-ext" : $"https://{hostUrl}/telemetry-unauth" : null;
+        }
+
+        /// <summary>
+        /// Sends a batch of telemetry events asynchronously
+        /// </summary>
+        /// <param name="telemetryBatch">List of telemetry events to send</param>
+        /// <returns>Task representing the async operation</returns>
+        public async Task<bool> SendTelemetryBatchAsync(List<TelemetryFrontendLog> telemetryBatch)
+        {
+            if (string.IsNullOrEmpty(_telemetryUrl) || telemetryBatch.Count == 0)
+            {
+                return false;
+            }
+
+            try
+            {
+                var request = new HttpRequestMessage(HttpMethod.Post, _telemetryUrl);
+
+                // Serialize the batch to JSON
+                var telemetryRequest = new TelemetryRequest();
+                telemetryRequest.UploadTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+                telemetryRequest.ProtoLogs = telemetryBatch.Select(x => JsonSerializer.Serialize(x)).ToList();
+                request.Content = new StringContent(JsonSerializer.Serialize(telemetryRequest));
+
+                // Set headers
+                request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+                request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
+                if(_accessToken != null)
+                {
+                    request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _accessToken);
+                }
+
+                var response = await _httpClient.SendAsync(request);
+                return response.IsSuccessStatusCode;
+            }
+            catch (Exception ex)
+            {
+                LOGGER.error($"Failed to send telemetry: {ex.Message}");
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Sends a single telemetry event asynchronously
+        /// </summary>
+        /// <param name="telemetryEvent">Single telemetry event to send</param>
+        /// <returns>Task representing the async operation</returns>
+        public async Task<bool> SendTelemetryAsync(TelemetryFrontendLog telemetryEvent)
+        {
+            if (string.IsNullOrEmpty(_telemetryUrl))
+            {
+                return false;
+            }
+
+            try
+            {
+                var request = new HttpRequestMessage(HttpMethod.Post, _telemetryUrl);
+
+                // Serialize the event to JSON
+                var telemetryRequest = new TelemetryRequest();
+                telemetryRequest.UploadTime = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds();
+                telemetryRequest.ProtoLogs = new List<string> { JsonSerializer.Serialize(telemetryEvent) };
+                request.Content = new StringContent(JsonSerializer.Serialize(telemetryRequest));
+
+                // Set headers
+                request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+                request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
+                if(_accessToken != null)
+                {
+                    request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _accessToken);
+                }
+
+                var response = await _httpClient.SendAsync(request);
+                LOGGER.info($"Telemetry response: {response.StatusCode}");
+                return response.IsSuccessStatusCode;
+            }
+            catch (Exception ex)
+            {
+                LOGGER.error($"Failed to send telemetry: {ex.Message}");
+                return false;
+            }
+        }
+    }
+}