Add remote symbolication support with build-id and PC offset

Author: jbachorik

README.md

@@ -402,6 +402,25 @@ Improved thread-local storage initialization to prevent race conditions:
 
 These architectural improvements focus on eliminating race conditions, improving performance in high-throughput scenarios, and providing better debugging capabilities for the native profiling engine.
 
+### Remote Symbolication Support (2025)
+
+Added support for remote symbolication to enable offloading symbol resolution from the agent to backend services:
+
+- **Build-ID extraction**: Automatically extracts GNU build-id from ELF binaries on Linux
+- **Raw addressing information**: Stores build-id and PC offset instead of resolved symbol names
+- **Remote symbolication mode**: Enable with `remotesym=true` profiler argument
+- **JFR integration**: Remote frames serialized with build-id and offset for backend resolution
+- **Zero encoding overhead**: Uses dedicated frame type (FRAME_NATIVE_REMOTE) for efficient serialization
+
+**Benefits**:
+- Reduces agent overhead by eliminating local symbol resolution
+- Enables centralized symbol resolution with better caching
+- Supports scenarios where debug symbols are not available locally
+
+**Key files**: `elfBuildId.h`, `elfBuildId.cpp`, `profiler.cpp`, `flightRecorder.cpp`
+
+For detailed documentation, see [doc/REMOTE_SYMBOLICATION.md](doc/REMOTE_SYMBOLICATION.md).
+
 ## Contributing
 1. Fork the repository
 2. Create a feature branch

ddprof-lib/src/main/cpp/arguments.cpp

@@ -88,7 +88,10 @@ static const Multiplier UNIVERSAL[] = {
 //                          samples
 //     generations        - track surviving generations
 //     lightweight[=BOOL] - enable lightweight profiling - events without
-//     stacktraces (default: true) jfr                - dump events in Java
+//     stacktraces (default: true)
+//     remotesymbolication[=BOOL] - enable remote symbolication for native frames
+//                          (stores build-id and PC offset instead of symbol names)
+//     jfr                - dump events in Java
 //     Flight Recorder format interval=N         - sampling interval in ns
 //     (default: 10'000'000, i.e. 10 ms) jstackdepth=N      - maximum Java stack
 //     depth (default: 2048) safemode=BITS      - disable stack recovery
@@ -317,17 +320,30 @@ Error Arguments::parse(const char *args) {
           _lightweight = false;
         }
       }
-            CASE("wallsampler")
-                if (value != NULL) {
-                    switch (value[0]) {
-                        case 'j':
-                            _wallclock_sampler = JVMTI;
-                            break;
-                        case 'a':
-                        default:
-                            _wallclock_sampler = ASGCT;
-                    }
-                }
+
+      CASE("remotesym")
+      if (value != NULL) {
+        switch (value[0]) {
+        case 'y': // yes
+        case 't': // true
+          _remote_symbolication = true;
+          break;
+        default:
+          _remote_symbolication = false;
+        }
+      }
+
+      CASE("wallsampler")
+      if (value != NULL) {
+          switch (value[0]) {
+              case 'j':
+                  _wallclock_sampler = JVMTI;
+                  break;
+              case 'a':
+              default:
+                  _wallclock_sampler = ASGCT;
+          }
+      }
 
       DEFAULT()
       if (_unknown_arg == NULL)

ddprof-lib/src/main/cpp/arguments.h

@@ -147,6 +147,7 @@ class Arguments {
   int _jfr_options;
   std::vector<std::string> _context_attributes;
   bool _lightweight;
+  bool _remote_symbolication;  // Enable remote symbolication for native frames
 
   Arguments(bool persistent = false)
       : _buf(NULL),
@@ -177,7 +178,8 @@ class Arguments {
         _jfr_options(0),
         _context_attributes({}),
         _wallclock_sampler(ASGCT),
-        _lightweight(false) {}
+        _lightweight(false),
+        _remote_symbolication(false) {}
 
   ~Arguments();
 

ddprof-lib/src/main/cpp/codeCache.cpp

@@ -37,6 +37,11 @@ CodeCache::CodeCache(const char *name, short lib_index,
   _plt_size = 0;
   _debug_symbols = false;
 
+  // Initialize build-id fields
+  _build_id = nullptr;
+  _build_id_len = 0;
+  _load_bias = 0;
+
   memset(_imports, 0, sizeof(_imports));
   _imports_patchable = imports_patchable;
 
@@ -54,10 +59,27 @@ CodeCache::CodeCache(const CodeCache &other) {
   _min_address = other._min_address;
   _max_address = other._max_address;
   _text_base = other._text_base;
+  _image_base = other._image_base;
 
-  _imports_patchable = other._imports_patchable;
   _plt_offset = other._plt_offset;
   _plt_size = other._plt_size;
+  _debug_symbols = other._debug_symbols;
+
+  // Copy build-id information
+  _build_id_len = other._build_id_len;
+  if (other._build_id != nullptr && other._build_id_len > 0) {
+    size_t hex_str_len = strlen(other._build_id);
+    _build_id = static_cast<char*>(malloc(hex_str_len + 1));
+    if (_build_id != nullptr) {
+      strcpy(_build_id, other._build_id);
+    }
+  } else {
+    _build_id = nullptr;
+  }
+  _load_bias = other._load_bias;
+
+  memset(_imports, 0, sizeof(_imports));
+  _imports_patchable = other._imports_patchable;
 
   _dwarf_table_length = other._dwarf_table_length;
   _dwarf_table = new FrameDesc[_dwarf_table_length];
@@ -77,17 +99,34 @@ CodeCache &CodeCache::operator=(const CodeCache &other) {
     delete _name;
     delete _dwarf_table;
     delete _blobs;
+    free(_build_id);  // Free existing build-id
 
     _name = NativeFunc::create(other._name, -1);
     _lib_index = other._lib_index;
     _min_address = other._min_address;
     _max_address = other._max_address;
     _text_base = other._text_base;
-
-    _imports_patchable = other._imports_patchable;
+    _image_base = other._image_base;
 
     _plt_offset = other._plt_offset;
     _plt_size = other._plt_size;
+    _debug_symbols = other._debug_symbols;
+
+    // Copy build-id information
+    _build_id_len = other._build_id_len;
+    if (other._build_id != nullptr && other._build_id_len > 0) {
+      size_t hex_str_len = strlen(other._build_id);
+      _build_id = static_cast<char*>(malloc(hex_str_len + 1));
+      if (_build_id != nullptr) {
+        strcpy(_build_id, other._build_id);
+      }
+    } else {
+      _build_id = nullptr;
+    }
+    _load_bias = other._load_bias;
+
+    memset(_imports, 0, sizeof(_imports));
+    _imports_patchable = other._imports_patchable;
 
     _dwarf_table_length = other._dwarf_table_length;
     _dwarf_table = new FrameDesc[_dwarf_table_length];
@@ -110,6 +149,7 @@ CodeCache::~CodeCache() {
   NativeFunc::destroy(_name);
   delete[] _blobs;
   delete _dwarf_table;
+  free(_build_id);  // Free build-id memory
 }
 
 void CodeCache::expand() {
@@ -387,3 +427,23 @@ FrameDesc CodeCache::findFrameDesc(const void *pc) {
     return FrameDesc::default_frame;
   }
 }
+
+void CodeCache::setBuildId(const char* build_id, size_t build_id_len) {
+  // Free existing build-id if any
+  free(_build_id);
+  _build_id = nullptr;
+  _build_id_len = 0;
+
+  if (build_id != nullptr && build_id_len > 0) {
+    // build_id is a hex string, allocate based on actual string length
+    size_t hex_str_len = strlen(build_id);
+    _build_id = static_cast<char*>(malloc(hex_str_len + 1));
+
+    if (_build_id != nullptr) {
+      // Copy the hex string
+      strcpy(_build_id, build_id);
+      // Store the original byte length (not hex string length)
+      _build_id_len = build_id_len;
+    }
+  }
+}

ddprof-lib/src/main/cpp/codeCache.h

@@ -116,6 +116,11 @@ class CodeCache {
   unsigned int _plt_offset;
   unsigned int _plt_size;
 
+  // Build-ID and load bias for remote symbolication
+  char *_build_id;           // GNU build-id (hex string, null if not available)
+  size_t _build_id_len;      // Build-id length in bytes (raw, not hex string length)
+  uintptr_t _load_bias;      // Load bias (image_base - file_base address)
+
   void **_imports[NUM_IMPORTS][NUM_IMPORT_TYPES];
   bool _imports_patchable;
   bool _debug_symbols;
@@ -169,6 +174,19 @@ class CodeCache {
 
   void setDebugSymbols(bool debug_symbols) { _debug_symbols = debug_symbols; }
 
+  // Build-ID and remote symbolication support
+  const char* buildId() const { return _build_id; }
+  size_t buildIdLen() const { return _build_id_len; }
+  bool hasBuildId() const { return _build_id != nullptr; }
+  uintptr_t loadBias() const { return _load_bias; }
+  short libIndex() const { return _lib_index; }
+
+  // Sets the build-id (hex string) and stores the original byte length
+  // build_id: null-terminated hex string (e.g., "abc123..." for 40-char string)
+  // build_id_len: original byte length before hex conversion (e.g., 20 bytes)
+  void setBuildId(const char* build_id, size_t build_id_len);
+  void setLoadBias(uintptr_t load_bias) { _load_bias = load_bias; }
+
   void add(const void *start, int length, const char *name,
            bool update_bounds = false);
   void updateBounds(const void *start, const void *end);

ddprof-lib/src/main/cpp/elfBuildId.cpp

@@ -0,0 +1,149 @@
+/*
+ * Copyright 2025, Datadog, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifdef __linux__
+
+#include "elfBuildId.h"
+#include <elf.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+// GNU build-id note constants
+#define NT_GNU_BUILD_ID 3
+#define GNU_BUILD_ID_NAME "GNU"
+
+char* ElfBuildIdExtractor::extractBuildId(const char* file_path, size_t* build_id_len) {
+    if (!file_path || !build_id_len) {
+        return nullptr;
+    }
+
+    int fd = open(file_path, O_RDONLY);
+    if (fd < 0) {
+        return nullptr;
+    }
+
+    struct stat st;
+    if (fstat(fd, &st) < 0) {
+        close(fd);
+        return nullptr;
+    }
+
+    void* elf_base = mmap(nullptr, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+    close(fd);
+
+    if (elf_base == MAP_FAILED) {
+        return nullptr;
+    }
+
+    char* result = extractBuildIdFromMemory(elf_base, st.st_size, build_id_len);
+
+    munmap(elf_base, st.st_size);
+    return result;
+}
+
+char* ElfBuildIdExtractor::extractBuildIdFromMemory(const void* elf_base, size_t elf_size, size_t* build_id_len) {
+    if (!elf_base || !build_id_len || elf_size < sizeof(Elf64_Ehdr)) {
+        return nullptr;
+    }
+
+    const Elf64_Ehdr* ehdr = static_cast<const Elf64_Ehdr*>(elf_base);
+
+    // Verify ELF magic
+    if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) {
+        return nullptr;
+    }
+
+    // Only handle 64-bit ELF for now
+    if (ehdr->e_ident[EI_CLASS] != ELFCLASS64) {
+        return nullptr;
+    }
+
+    // Check if we have program headers
+    if (ehdr->e_phoff == 0 || ehdr->e_phnum == 0) {
+        return nullptr;
+    }
+
+    const char* base = static_cast<const char*>(elf_base);
+    const Elf64_Phdr* phdr = reinterpret_cast<const Elf64_Phdr*>(base + ehdr->e_phoff);
+
+    // Search for PT_NOTE segments
+    for (int i = 0; i < ehdr->e_phnum; i++) {
+        if (phdr[i].p_type == PT_NOTE && phdr[i].p_filesz > 0) {
+            // Ensure note segment is within file bounds
+            if (phdr[i].p_offset + phdr[i].p_filesz > elf_size) {
+                continue;
+            }
+
+            const void* note_data = base + phdr[i].p_offset;
+            const uint8_t* build_id_bytes = findBuildIdInNotes(note_data, phdr[i].p_filesz, build_id_len);
+
+            if (build_id_bytes) {
+                return buildIdToHex(build_id_bytes, *build_id_len);
+            }
+        }
+    }
+
+    return nullptr;
+}
+
+const uint8_t* ElfBuildIdExtractor::findBuildIdInNotes(const void* note_data, size_t note_size, size_t* build_id_len) {
+    const char* data = static_cast<const char*>(note_data);
+    size_t offset = 0;
+
+    while (offset + sizeof(Elf64_Nhdr) < note_size) {
+        const Elf64_Nhdr* nhdr = reinterpret_cast<const Elf64_Nhdr*>(data + offset);
+
+        // Calculate aligned sizes
+        size_t name_size_aligned = (nhdr->n_namesz + 3) & ~3;
+        size_t desc_size_aligned = (nhdr->n_descsz + 3) & ~3;
+
+        // Check bounds
+        if (offset + sizeof(Elf64_Nhdr) + name_size_aligned + desc_size_aligned > note_size) {
+            break;
+        }
+
+        // Check if this is a GNU build-id note
+        if (nhdr->n_type == NT_GNU_BUILD_ID && nhdr->n_namesz > 0 && nhdr->n_descsz > 0) {
+            const char* name = data + offset + sizeof(Elf64_Nhdr);
+
+            // Verify GNU build-id name (including null terminator)
+            if (nhdr->n_namesz == 4 && strncmp(name, GNU_BUILD_ID_NAME, 3) == 0 && name[3] == '\0') {
+                const uint8_t* desc = reinterpret_cast<const uint8_t*>(data + offset + sizeof(Elf64_Nhdr) + name_size_aligned);
+                *build_id_len = nhdr->n_descsz;
+                return desc;
+            }
+        }
+
+        offset += sizeof(Elf64_Nhdr) + name_size_aligned + desc_size_aligned;
+    }
+
+    return nullptr;
+}
+
+char* ElfBuildIdExtractor::buildIdToHex(const uint8_t* build_id_bytes, size_t byte_len) {
+    if (!build_id_bytes || byte_len == 0) {
+        return nullptr;
+    }
+
+    // Allocate string for hex representation (2 chars per byte + null terminator)
+    char* hex_str = static_cast<char*>(malloc(byte_len * 2 + 1));
+    if (!hex_str) {
+        return nullptr;
+    }
+
+    for (size_t i = 0; i < byte_len; i++) {
+        snprintf(hex_str + i * 2, 3, "%02x", build_id_bytes[i]);
+    }
+
+    hex_str[byte_len * 2] = '\0';
+    return hex_str;
+}
+
+#endif // __linux__