diff --git a/.clang-format b/.clang-format index 2943a03c8..d8b84514f 100644 --- a/.clang-format +++ b/.clang-format @@ -5,13 +5,10 @@ IndentWidth: 4 ColumnLimit: 100 BreakBeforeBraces: Attach AllowShortFunctionsOnASingleLine: None -AllowShortIfStatementsOnASingleLine: false +AllowShortIfStatementsOnASingleLine: Never AllowShortLoopsOnASingleLine: false -AllowShortBlocksOnASingleLine: false +AllowShortBlocksOnASingleLine: Never AllowShortCaseLabelsOnASingleLine: false AllowShortNamespacesOnASingleLine: false -AllowShortStructsOnASingleLine: false AllowShortEnumsOnASingleLine: false -AllowShortLambdasOnASingleLine: false -AllowShortCompoundStatementsOnASingleLine: false -AllowShortAlwaysBreakType: None +AllowShortLambdasOnASingleLine: None diff --git a/.github/scripts/generate-test-summary.sh b/.github/scripts/generate-test-summary.sh index 8404e3d87..a6cbcfcc5 100755 --- a/.github/scripts/generate-test-summary.sh +++ b/.github/scripts/generate-test-summary.sh @@ -239,34 +239,29 @@ log "Generating markdown summary..." fi echo "" - # Status matrix table + # Status matrix table (JDK versions as rows, platforms as columns) if ((${#sorted_platforms[@]} > 0 && ${#sorted_java[@]} > 0)); then echo "### Status Overview" echo "" - # Header row - printf "| Platform |" - for java in "${sorted_java[@]}"; do - # Shorten java version for header (e.g., "17-graal" -> "17-gr") - short_java="${java}" - if [[ ${#java} -gt 6 ]]; then - short_java="${java:0:6}" - fi - printf " %s |" "$short_java" + # Header row - platforms as columns + printf "| JDK |" + for platform in "${sorted_platforms[@]}"; do + printf " %s |" "$platform" done echo "" # Separator row - printf "%s" "|----------|" - for _ in "${sorted_java[@]}"; do + printf "%s" "|-----|" + for _ in "${sorted_platforms[@]}"; do printf "%s" "--------|" done echo "" - # Data rows - for platform in "${sorted_platforms[@]}"; do - printf "| %s |" "$platform" - for java in "${sorted_java[@]}"; do + # Data rows - JDK versions as rows + for java in "${sorted_java[@]}"; do + printf "| %s |" "$java" + for platform in "${sorted_platforms[@]}"; do key="${platform}|${java}" status="${job_status[$key]:-}" url="${job_url[$key]:-}" @@ -320,21 +315,16 @@ log "Generating markdown summary..." done fi - # Summary statistics - echo "### Summary" - echo "" - echo "| Metric | Value |" - echo "|--------|-------|" - echo "| Total jobs | $total_jobs |" - echo "| Passed | $passed_jobs |" - echo "| Failed | $failed_count |" + # Summary statistics (single line) + printf "**Summary:** Total: %d | Passed: %d | Failed: %d" "$total_jobs" "$passed_jobs" "$failed_count" if ((skipped_jobs > 0)); then - echo "| Skipped | $skipped_jobs |" + printf " | Skipped: %d" "$skipped_jobs" fi if ((cancelled_jobs > 0)); then - echo "| Cancelled | $cancelled_jobs |" + printf " | Cancelled: %d" "$cancelled_jobs" fi echo "" + echo "" echo "---" echo "*Updated: $(date -u '+%Y-%m-%d %H:%M:%S UTC')*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7d7f8ed1..47dd5696c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,7 +163,7 @@ jobs: summarize-tests: needs: [test-matrix] - if: always() && github.event_name == 'pull_request' + if: always() && !cancelled() && github.event_name == 'pull_request' runs-on: ubuntu-latest permissions: contents: read diff --git a/.gitignore b/.gitignore index 98ebc2deb..765412a2e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ **/build/ **/build_*/ **/build-*/ +!build-logic/ /nbproject/ /out/ /.idea/ @@ -14,6 +15,7 @@ .project .settings .gradle +.kotlin .tmp *.iml /ddprof-stresstest/jmh-result.* @@ -34,3 +36,9 @@ datadog/maven/resources .history .claude/settings.local.json /jmh-* + +# Temporary documentation and work state +doc/temp/ + +# CLAUDE.md is auto-generated from AGENTS.md bootstrap instructions +CLAUDE.md diff --git a/CLAUDE.md b/AGENTS.md similarity index 54% rename from CLAUDE.md rename to AGENTS.md index 11dc420af..8a7e8e91f 100644 --- a/CLAUDE.md +++ b/AGENTS.md @@ -1,6 +1,22 @@ + + +# AGENTS.md + +This file provides guidance to AI coding assistants when working with code in this repository. ## Project Overview @@ -9,11 +25,11 @@ This is the Datadog Java Profiler Library, a specialized profiler derived from a **Key Technologies:** - Java 8+ (main API and library loading) - C++17 (native profiling engine) -- Gradle (build system with custom native compilation) +- Gradle (build system with custom native compilation tasks) - JNI (Java Native Interface for C++ integration) -- CMake (for C++ unit tests via Google Test) +- Google Test (for C++ unit tests, compiled via custom Gradle tasks) -## Project Operating Guide for Claude (Main Session) +## Project Operating Guide (Main Session) You are the **Main Orchestrator** for this repository. @@ -33,16 +49,16 @@ You are the **Main Orchestrator** for this repository. - Assume macOS/Linux unless I explicitly say Windows; if Windows, use PowerShell equivalents. - If a step fails, print the failing command and a one-line hint, then stop. -### Implementation Hints for You +### Implementation Hints - For builds, always use: `--console=plain -i` (or `-d` if I ask for debug). - Use `mkdir -p build/logs build/reports/claude` before writing. - Timestamp format: `$(date +%Y%m%d-%H%M%S)`. - After the build finishes, call the sub-agent like: - “Use `gradle-log-analyst` to parse LOG_PATH; write the two reports; reply with only a 3–6 line status and the two relative file paths.” + "Use `gradle-log-analyst` to parse LOG_PATH; write the two reports; reply with only a 3–6 line status and the two relative file paths." ### Shortcuts I Expect - `./gradlew ` to do everything in one step. -- If I just say “build assembleDebugJar”, interpret that as the shortcut above. +- If I just say "build assembleDebugJar", interpret that as the shortcut above. ## Build Commands Never use 'gradle' or 'gradlew' directly. Instead, use the '/build-and-summarize' command. @@ -79,14 +95,78 @@ Never use 'gradle' or 'gradlew' directly. Instead, use the '/build-and-summarize ./gradlew testAsan ./gradlew testTsan -# Run C++ unit tests only -./gradlew gtestDebug -./gradlew gtestRelease +# Run C++ unit tests only (via GtestPlugin) +./gradlew :ddprof-lib:gtestDebug # All debug tests +./gradlew :ddprof-lib:gtestRelease # All release tests +./gradlew :ddprof-lib:gtest # All tests, all configs + +# Run individual C++ test +./gradlew :ddprof-lib:gtestDebug_test_callTraceStorage # Cross-JDK testing JAVA_TEST_HOME=/path/to/test/jdk ./gradlew testDebug ``` +#### Google Test Plugin + +The project uses a custom `GtestPlugin` (in `build-logic/`) for C++ unit testing with Google Test. The plugin automatically: +- Discovers `.cpp` test files in `src/test/cpp/` +- Creates compilation, linking, and execution tasks for each test +- Filters configurations by current platform/architecture +- Integrates with NativeCompileTask and NativeLinkExecutableTask + +**Key features:** +- Platform-aware: Only creates tasks for matching OS/arch +- Assertion control: Removes `-DNDEBUG` to enable assertions in tests +- Symbol preservation: Keeps debug symbols in release test builds +- Task aggregation: Per-config (`gtestDebug`) and master (`gtest`) tasks +- Shared configurations: Uses BuildConfiguration from NativeBuildPlugin + +**Configuration example (ddprof-lib/build.gradle.kts):** +```kotlin +plugins { + id("com.datadoghq.native-build") + id("com.datadoghq.gtest") +} + +gtest { + testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) + mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) + includes.from( + "src/main/cpp", + "${javaHome}/include", + "${javaHome}/include/${platformInclude}" + ) + // Optional + enableAssertions.set(true) // Remove -DNDEBUG (default: true) + keepSymbols.set(true) // Keep symbols in release (default: true) + failFast.set(false) // Stop on first failure (default: false) +} +``` + +**See:** `build-logic/README.md` for full documentation + +#### Debug Symbol Extraction + +Release builds automatically extract debug symbols via `NativeLinkTask`, reducing production binary size (~69% smaller) while maintaining separate debug files for offline debugging. + +**Key features:** +- Platform-aware: Uses `objcopy`/`strip` on Linux, `dsymutil`/`strip` on macOS +- Automatic workflow: Extract symbols → Add GNU debuglink (Linux) → Strip library → Copy artifacts +- Size optimization: Stripped ~1.2MB production library from ~6.1MB with embedded debug info +- Debug preservation: Separate `.debug` files (Linux) or `.dSYM` bundles (macOS) + +**Tool requirements:** +- Linux: `binutils` package (objcopy, strip) +- macOS: Xcode Command Line Tools (dsymutil, strip) + +**Skip extraction:** +```bash +./gradlew buildRelease -Pskip-debug-extraction=true +``` + +**See:** `build-logic/README.md` for full documentation + ### Build Options ```bash # Skip native compilation @@ -103,6 +183,11 @@ JAVA_TEST_HOME=/path/to/test/jdk ./gradlew testDebug # Skip debug symbol extraction ./gradlew buildRelease -Pskip-debug-extraction=true + +# Force specific compiler (auto-detects clang++ or g++ by default) +./gradlew build -Pnative.forceCompiler=clang++ +./gradlew build -Pnative.forceCompiler=g++ +./gradlew build -Pnative.forceCompiler=/usr/bin/g++-13 ``` ### Code Quality @@ -145,7 +230,7 @@ The project supports multiple build configurations per platform: ### Platform Support - **Linux**: x64, arm64 (primary platforms) - **macOS**: arm64, x64 -- **Architecture detection**: Automatic via `common.gradle` +- **Architecture detection**: Automatic via `PlatformUtils` in build-logic - **musl libc detection**: Automatic detection and handling ### Debug Information Handling @@ -273,10 +358,141 @@ The profiler uses a sophisticated double-buffered storage system for call traces - **malloc-shim**: Linux memory allocation interceptor ### Native Compilation Pipeline -- **Platform Detection**: Automatic OS and architecture detection via `common.gradle` +- **Platform Detection**: Automatic OS and architecture detection via `PlatformUtils` in build-logic - **Configuration Matrix**: Multiple build configs (release/debug/asan/tsan) per platform - **Symbol Processing**: Automatic debug symbol extraction for release builds - **Library Packaging**: Final JAR contains all platform-specific native libraries +- **Compiler Detection**: Auto-detects clang++ (preferred) or g++ (fallback); override with `-Pnative.forceCompiler` + +### Native Build Plugin (build-logic) +The project includes a Kotlin-based native build plugin (`build-logic/`) for type-safe C++ compilation: +- **Composite Build**: Independent Gradle project for build logic versioning +- **Type-Safe DSL**: Kotlin-based configuration with compile-time checking +- **Auto Task Generation**: Creates compile, link, and assemble tasks per configuration +- **Debug Symbol Extraction**: Automatic split debug info for release builds (69% size reduction) +- **Source Sets**: Per-directory compiler flags for legacy/third-party code +- **Symbol Visibility**: Linux version scripts and macOS exported symbols lists + +**See:** `build-logic/README.md` for full documentation + +### Custom Native Build Plugin (build-logic) +The project uses a custom Kotlin-based native build plugin in `build-logic/` instead of Gradle's `cpp-library` and `cpp-application` plugins. This is intentional: + +**Why not cpp-library/cpp-application plugins:** +- Gradle's native plugins parse compiler version strings which breaks with newer gcc/clang versions +- JNI header detection has issues with non-standard JAVA_HOME layouts +- Plugin maintainers are unresponsive to fixes +- The plugins use undocumented internals that change between Gradle versions + +**Plugin components (`com.datadoghq.native-build`):** +- `NativeCompileTask` - Parallel C++ compilation with source sets support +- `NativeLinkTask` - Links shared libraries (.so/.dylib) with symbol visibility +- `PlatformUtils` - Platform detection and compiler location + +**Plugin components (`com.datadoghq.gtest`):** +- `NativeLinkExecutableTask` - Links executables (for gtest) +- `GtestPlugin` - Google Test integration and task generation + +**Key principle:** Direct compiler invocation without version parsing. The tasks simply find `clang++` or `g++` on PATH and invoke them with the configured flags. + +#### Configuring Build Tasks + +All build tasks support industry-standard configuration options. Configuration is done using Kotlin DSL: + +**Basic compilation:** +```kotlin +tasks.register("compileLib", NativeCompileTask::class) { + compiler.set("clang++") + compilerArgs.set(listOf("-O3", "-std=c++17", "-fPIC")) + sources.from(fileTree("src/main/cpp") { include("**/*.cpp") }) + includes.from("src/main/cpp", "${System.getenv("JAVA_HOME")}/include") + objectFileDir.set(file("build/obj")) +} +``` + +**Advanced configuration with source sets:** +```kotlin +tasks.register("compileLib", NativeCompileTask::class) { + compiler.set("clang++") + compilerArgs.set(listOf("-Wall", "-O3")) // Base flags for all files + + // Multiple source directories with per-directory flags + sourceSets { + create("main") { + sources.from(fileTree("src/main/cpp")) + compilerArgs.add("-fPIC") + } + create("legacy") { + sources.from(fileTree("src/legacy")) + compilerArgs.addAll("-Wno-deprecated", "-std=c++11") + excludes.add("**/broken/*.cpp") + } + } + + // Logging + logLevel.set(LogLevel.VERBOSE) + + objectFileDir.set(file("build/obj")) +} +``` + +**Linking shared libraries with symbol management:** +```kotlin +tasks.register("linkLib", NativeLinkTask::class) { + linker.set("clang++") + linkerArgs.set(listOf("-O3")) + objectFiles.from(fileTree("build/obj") { include("*.o") }) + outputFile.set(file("build/lib/libjavaProfiler.so")) + + // Symbol visibility control + exportSymbols.set(listOf("Java_*", "JNI_OnLoad", "JNI_OnUnload")) + hideSymbols.set(listOf("*_internal*")) + + // Libraries + lib("pthread", "dl", "m") + libPath("/usr/local/lib") + + logLevel.set(LogLevel.VERBOSE) +} +``` + +**Executable linking (for gtest):** +```kotlin +tasks.register("linkTest", NativeLinkExecutableTask::class) { + linker.set("clang++") + objectFiles.from(fileTree("build/obj/gtest") { include("*.o") }) + outputFile.set(file("build/bin/callTrace_test")) + + // Library management + lib("gtest", "gtest_main", "pthread") + libPath("/usr/local/lib") + runtimePath("/opt/lib", "/usr/local/lib") + + logLevel.set(LogLevel.VERBOSE) +} +``` + +**Task properties:** + +*NativeCompileTask:* +- `compiler`, `compilerArgs` - Compiler and flags +- `sources`, `includes` - Source files and include paths +- `sourceSets` - Per-directory compiler flag overrides +- `objectFileDir` - Output directory for object files +- `logLevel` - QUIET, NORMAL, VERBOSE, DEBUG + +*NativeLinkTask:* +- `linker`, `linkerArgs` - Linker and flags +- `objectFiles`, `outputFile` - Input objects and output library +- `exportSymbols`, `hideSymbols` - Symbol visibility control +- `lib()`, `libPath()` - Library convenience methods +- `logLevel`, `showCommandLine` - Logging options + +*NativeLinkExecutableTask:* +- `linker`, `linkerArgs` - Linker and flags +- `objectFiles`, `outputFile` - Input objects and output executable +- `lib()`, `libPath()`, `runtimePath()` - Library and rpath convenience methods +- `logLevel`, `showCommandLine` - Logging options ### Artifact Structure Final artifacts maintain a specific structure for deployment: @@ -285,6 +501,46 @@ META-INF/native-libs/{os}-{arch}/libjavaProfiler.{so|dylib} ``` With separate debug symbol packages for production debugging support. +## Build System Maintenance + +> **Detailed guide**: [doc/build/BuildSystemGuide.md](doc/build/BuildSystemGuide.md) + +### Quick Reference + +**Convention plugins** (in `build-logic/conventions/`): +- `com.datadoghq.native-build` - Multi-config C++ compilation +- `com.datadoghq.gtest` - Google Test integration +- `com.datadoghq.profiler-test` - Multi-config Java test generation +- `com.datadoghq.simple-native-lib` - Simple single-library builds + +**Key principle**: Build configurations (release/debug/asan/tsan/fuzzer) are **discovered dynamically**, not hardcoded. Add new configs in `ConfigurationPresets.kt` only. + +**Key files**: +- `ConfigurationPresets.kt` - Defines all build configurations and their flags +- `PlatformUtils.kt` - Platform detection and compiler finding +- `NativeBuildPlugin.kt` - Creates compile/link tasks per configuration + +### Common Tasks + +| Task | Description | +|------|-------------| +| Add compiler flag to all configs | Edit `commonLinuxCompilerArgs()` in `ConfigurationPresets.kt` | +| Add new build configuration | Add `register("name")` block in `ConfigurationPresets.kt` | +| Create new convention plugin | Create class, register in `build.gradle.kts`, see [guide](doc/BUILD-SYSTEM-GUIDE.md#creating-a-new-convention-plugin) | + +### Gradle Properties + +See `gradle.properties.template` for all options. Key ones: +- `skip-tests`, `skip-native`, `skip-gtest` - Skip build phases +- `native.forceCompiler` - Override compiler detection +- `ddprof_version` - Override version + +### Troubleshooting + +- **Plugin changes not taking effect**: Run `./gradlew --stop` +- **"Task not found"**: Tasks are created dynamically; check `PlatformUtils` detection +- **"Configuration not found"**: Access configurations in `afterEvaluate` + ## Legacy and Compatibility - Java 8 compatibility maintained throughout @@ -295,7 +551,7 @@ With separate debug symbol packages for production debugging support. - For OpenJ9 specific issues consul the openj9 github project - don't use assemble task. Use assembleDebug or assembleRelease instead - gtest tests are located in ddprof-lib/src/test/cpp -- Module ddprof-lib/gtest is only containing the gtest build setup +- GtestPlugin in build-logic handles gtest build setup - Java unit tests are in ddprof-test module - Always run /build-and-summarize spotlessApply before commiting the changes @@ -344,6 +600,10 @@ With separate debug symbol packages for production debugging support. - This ensures the full build log is captured to a file and only a summary is shown in the main session. +## Documentation Rules +- All documentation files in `doc/` must use **PascalCase** naming (e.g., `BuildSystemGuide.md`, `CallTraceStorage.md`) +- The `doc/README.md` index file is the only lowercase exception + ## Ground rules - Never replace the code you work on with stubs - Never 'fix' the tests by testing constants against constants diff --git a/README.md b/README.md index 40f359fb6..e94163bb1 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,9 @@ If you need a full-fledged Java profiler head back to [async-profiler](https://g ### Prerequisites 1. JDK 8 or later (required for building) 2. Gradle (included in wrapper) -3. C++ compiler (gcc/g++ or clang) +3. C++ compiler (clang++ preferred, g++ supported) + - Build system auto-detects clang++ or g++ + - Override with: `./gradlew build -Pnative.forceCompiler=g++` 4. Make (included in XCode on Macos) 5. Google Test (for unit testing) - On Ubuntu/Debian: `sudo apt install libgtest-dev` @@ -289,6 +291,27 @@ ddprof-lib/build/ - Alpine: `apk add binutils` - macOS: Included with Xcode command line tools +### Compiler Selection +The build system automatically detects the best available C++ compiler (prefers clang++, falls back to g++). + +```bash +# Auto-detection (default) +./gradlew build + +# Force specific compiler +./gradlew build -Pnative.forceCompiler=clang++ +./gradlew build -Pnative.forceCompiler=g++ +./gradlew build -Pnative.forceCompiler=/usr/bin/g++-13 + +# Test with specific compiler +./gradlew testDebug -Pnative.forceCompiler=g++ +``` + +This is useful for: +- **Reproducibility**: Ensure builds use the same compiler across machines +- **clang-only systems**: macOS with Xcode but no gcc (sanitizer builds work) +- **Testing**: Verify code compiles with both gcc and clang + ## Development ### Code Quality diff --git a/build-logic/QUICKSTART.md b/build-logic/QUICKSTART.md new file mode 100644 index 000000000..6cceaf1b2 --- /dev/null +++ b/build-logic/QUICKSTART.md @@ -0,0 +1,1196 @@ +# Native Build Plugins - Quick Start Guide + +This guide provides practical examples, workflows, and tips for using the Datadog native build plugin suite. For architectural details and reference documentation, see [README.md](README.md). + +## Table of Contents + +- [Getting Started](#getting-started) +- [Common Workflows](#common-workflows) +- [How-To Guides](#how-to-guides) +- [Tips and Tricks](#tips-and-tricks) +- [Troubleshooting](#troubleshooting) + +--- + +## Getting Started + +### Minimal Setup + +**build.gradle.kts:** +```kotlin +plugins { + id("com.datadoghq.native-build") +} + +nativeBuild { + version.set(project.version.toString()) +} +``` + +That's it! The plugin will: +- Auto-detect your compiler (clang++ or g++) +- Create standard configurations (release, debug, asan, tsan, fuzzer) +- Generate compile, link, and assemble tasks +- Use `src/main/cpp` as default source directory + +### Quick Build + +```bash +# Build release configuration +./gradlew assembleRelease + +# Build all active configurations +./gradlew assembleAll + +# Build specific configuration +./gradlew assembleDebug +``` + +### Adding Tests + +**build.gradle.kts:** +```kotlin +plugins { + id("com.datadoghq.native-build") + id("com.datadoghq.gtest") +} + +gtest { + testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) + mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) + + val javaHome = com.datadoghq.native.util.PlatformUtils.javaHome() + includes.from("src/main/cpp", "$javaHome/include") +} +``` + +```bash +# Run all tests +./gradlew gtest + +# Run tests for specific configuration +./gradlew gtestDebug +./gradlew gtestRelease +``` + +--- + +## Common Workflows + +### 1. Development: Fast Iteration with Debug Build + +```bash +# One-time: ensure you have debug config +./gradlew tasks --group=build | grep Debug + +# Edit code, then compile and link +./gradlew assembleDebug + +# Run debug tests +./gradlew gtestDebug +``` + +**Why debug config?** +- No optimization (`-O0`) = faster compilation +- Full debug symbols embedded +- Assertions enabled (no `-DNDEBUG`) +- No symbol stripping + +### 2. Testing: Memory Safety with ASan + +```bash +# Check if ASan is available +./gradlew tasks | grep -i asan + +# Build with AddressSanitizer +./gradlew assembleAsan + +# Run tests with ASan instrumentation +./gradlew gtestAsan +``` + +**ASan detects:** +- Heap buffer overflow/underflow +- Stack buffer overflow +- Use-after-free +- Use-after-return +- Memory leaks +- Double-free + +### 3. Testing: Thread Safety with TSan + +```bash +# Build with ThreadSanitizer +./gradlew assembleTsan + +# Run tests with TSan instrumentation +./gradlew gtestTsan +``` + +**TSan detects:** +- Data races +- Deadlocks +- Thread leaks +- Signal-unsafe functions in signal handlers + +### 4. Release: Production Build with Debug Symbols + +```bash +# Build release configuration +./gradlew assembleRelease + +# Output structure: +# build/lib/main/release/{platform}/{arch}/ +# ├── libjavaProfiler.so # Stripped library (~1.2MB) +# └── debug/ +# └── libjavaProfiler.so.debug # Debug symbols (~6MB) +``` + +**Key features:** +- Fully optimized (`-O3 -DNDEBUG`) +- Debug symbols extracted to separate file +- 69% size reduction in production binary +- Symbols linked via `.gnu_debuglink` + +### 5. Static Analysis: Clang scan-build + +The `scanbuild` plugin integrates Clang's static analyzer to detect bugs without running code. + +**build.gradle.kts:** +```kotlin +plugins { + id("com.datadoghq.scanbuild") +} + +scanBuild { + makefileDir.set(layout.projectDirectory.dir("src/test/make")) + outputDir.set(layout.buildDirectory.dir("reports/scan-build")) + analyzer.set("/usr/bin/clang++") + parallelJobs.set(4) + makeTargets.set(listOf("all", "test")) // Optional: specify make targets +} +``` + +```bash +# Run static analysis +./gradlew scanBuild + +# View HTML report +open build/reports/scan-build/*/index.html + +# Or on Linux +xdg-open build/reports/scan-build/*/index.html +``` + +**What scan-build detects:** +- Null pointer dereferences +- Memory leaks +- Use of uninitialized values +- Dead stores (unused assignments) +- Division by zero +- API misuse +- Logic errors +- Buffer overflows + +**Note:** scan-build is only available on Linux by default. The plugin will skip on macOS unless you have scan-build installed via Homebrew. + +--- + +## How-To Guides + +### Override Compiler + +```bash +# Use specific compiler +./gradlew build -Pnative.forceCompiler=clang++ +./gradlew build -Pnative.forceCompiler=g++-13 +./gradlew build -Pnative.forceCompiler=/usr/local/bin/clang++ + +# The plugin validates the compiler exists and works +``` + +### Customize Source Directories + +```kotlin +nativeBuild { + version.set("1.2.3") + cppSourceDirs.set(listOf( + "src/main/cpp", + "src/vendor/library" + )) + includeDirectories.set(listOf( + "src/main/cpp", + "src/vendor/library/include", + "/usr/local/include" + )) +} +``` + +### Add Custom Configurations + +```kotlin +nativeBuild { + version.set(project.version.toString()) + + // Override standard configs + buildConfigurations { + // Create custom configuration + register("production") { + platform.set(com.datadoghq.native.model.Platform.LINUX) + architecture.set(com.datadoghq.native.model.Architecture.X64) + active.set(true) + + compilerArgs.set(listOf( + "-O3", + "-DNDEBUG", + "-march=native", // Optimize for current CPU + "-flto", // Link-time optimization + "-fPIC" + )) + + linkerArgs.set(listOf( + "-Wl,--gc-sections", + "-flto" + )) + } + } +} +``` + +**Generated tasks:** +- `compileProduction` +- `linkProduction` +- `assembleProduction` + +### Add Common Flags to All Configurations + +```kotlin +nativeBuild { + version.set(project.version.toString()) + + // Apply to all configurations + commonCompilerArgs( + "-Wall", + "-Wextra", + "-Werror" + ) + + commonLinkerArgs( + "-Wl,--as-needed" + ) +} +``` + +### Control Google Test Behavior + +```kotlin +gtest { + testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) + mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) + + // Custom Google Test location (macOS) + googleTestHome.set(file("/usr/local/opt/googletest")) + + // Always enable assertions (remove -DNDEBUG) + enableAssertions.set(true) + + // Keep debug symbols in release test builds + keepSymbols.set(true) + + // Stop on first test failure + failFast.set(true) + + // Always re-run tests (ignore up-to-date checks) + alwaysRun.set(true) + + // Skip building native test support libraries + buildNativeLibs.set(false) + + includes.from( + "src/main/cpp", + "third-party/include" + ) +} +``` + +### Skip Builds Selectively + +```bash +# Skip all tests +./gradlew build -Pskip-tests + +# Skip only gtest (keep Java tests) +./gradlew build -Pskip-gtest + +# Skip all native compilation +./gradlew build -Pskip-native +``` + +### Cross-Platform Configuration + +```kotlin +nativeBuild { + buildConfigurations { + // Linux x64 release + register("linuxRelease") { + platform.set(Platform.LINUX) + architecture.set(Architecture.X64) + active.set(PlatformUtils.currentPlatform == Platform.LINUX) + // ... compiler/linker args + } + + // macOS ARM release + register("macosRelease") { + platform.set(Platform.MACOS) + architecture.set(Architecture.ARM64) + active.set(PlatformUtils.currentPlatform == Platform.MACOS) + // ... compiler/linker args + } + } +} +``` + +### Integrate with Java Resource Packaging + +```kotlin +// Copy native libraries to Java resources +val copyReleaseLibs by tasks.registering(Copy::class) { + from("build/lib/main/release") + into(layout.buildDirectory.dir("resources/main/native")) + + dependsOn(tasks.named("assembleRelease")) +} + +tasks.named("processResources") { + dependsOn(copyReleaseLibs) +} + +// Include in JAR +tasks.named("jar") { + from(layout.buildDirectory.dir("resources/main/native")) { + into("native") + } +} +``` + +### Configure Static Analysis with scan-build + +The scan-build plugin requires a Makefile-based build for analysis. This is typically separate from the Gradle native build. + +**Basic configuration:** +```kotlin +plugins { + id("com.datadoghq.scanbuild") +} + +scanBuild { + // Directory containing Makefile (required) + makefileDir.set(layout.projectDirectory.dir("src/test/make")) + + // Where to output HTML reports + outputDir.set(layout.buildDirectory.dir("reports/scan-build")) + + // Clang analyzer to use + analyzer.set("/usr/bin/clang++") + + // Parallel make jobs + parallelJobs.set(4) + + // Make targets to build (default: ["all"]) + makeTargets.set(listOf("all")) +} +``` + +**Advanced configuration:** +```kotlin +scanBuild { + makefileDir.set(layout.projectDirectory.dir("src/test/make")) + outputDir.set(layout.buildDirectory.dir("reports/scan-build")) + + // Use specific clang version + analyzer.set("/usr/bin/clang++-15") + + // Increase parallelism for faster analysis + parallelJobs.set(Runtime.getRuntime().availableProcessors()) + + // Analyze multiple targets + makeTargets.set(listOf("library", "tests")) +} +``` + +**Example Makefile structure:** + +Create `src/test/make/Makefile`: +```makefile +# Compiler (will be intercepted by scan-build) +CXX = clang++ +CXXFLAGS = -std=c++17 -Wall -Wextra -I../../main/cpp + +# Source files +SOURCES = $(wildcard ../../main/cpp/*.cpp) +OBJECTS = $(SOURCES:.cpp=.o) + +# Targets +all: library + +library: $(OBJECTS) + $(CXX) -shared -o libjavaProfiler.so $(OBJECTS) + +%.o: %.cpp + $(CXX) $(CXXFLAGS) -c $< -o $@ + +clean: + rm -f $(OBJECTS) libjavaProfiler.so + +.PHONY: all clean +``` + +**Integration with CI:** +```kotlin +// Make scanBuild part of verification +tasks.named("check") { + dependsOn("scanBuild") +} +``` + +**Platform-specific configuration:** +```kotlin +import com.datadoghq.native.util.PlatformUtils +import com.datadoghq.native.model.Platform + +if (PlatformUtils.currentPlatform == Platform.LINUX) { + scanBuild { + makefileDir.set(layout.projectDirectory.dir("src/test/make")) + outputDir.set(layout.buildDirectory.dir("reports/scan-build")) + } +} +``` + +--- + +## Tips and Tricks + +### Performance Tips + +#### 1. Parallel Compilation +Gradle automatically parallelizes compilation at the file level. Each `.cpp` file compiles independently. + +```bash +# Use more parallel workers +./gradlew build --parallel --max-workers=8 +``` + +#### 2. Incremental Builds +The plugin tracks: +- Source file changes +- Header file changes (via `-MMD` dependency tracking) +- Compiler flag changes + +Only modified files recompile: +```bash +# First build +./gradlew assembleDebug # Compiles all files + +# Edit one file +vim src/main/cpp/profiler.cpp + +# Second build +./gradlew assembleDebug # Only recompiles profiler.cpp +``` + +#### 3. Build Faster During Development + +```bash +# Use debug config (no optimization) +./gradlew assembleDebug + +# Skip tests +./gradlew assembleDebug -Pskip-tests + +# Use clang++ (generally faster than g++) +./gradlew assembleDebug -Pnative.forceCompiler=clang++ +``` + +### Debugging Tips + +#### 1. Verbose Compiler Output + +```kotlin +tasks.withType { + doFirst { + println("Compiler: ${compiler.get()}") + println("Args: ${compilerArgs.get()}") + println("Sources: ${sources.files}") + } +} +``` + +#### 2. Inspect Generated Build Files + +```bash +# Debug symbols location +ls -lh build/lib/main/release/*/*/debug/ + +# Object files +ls -lh build/obj/main/release/ + +# Task dependency tree +./gradlew assembleRelease --dry-run +``` + +#### 3. Check Active Configurations + +```bash +# View all build tasks +./gradlew tasks --group=build + +# The plugin logs active configurations: +./gradlew assembleAll | grep "Active configurations" +``` + +#### 4. Validate Debug Symbol Extraction + +**Linux:** +```bash +# Check if symbols are stripped +nm build/lib/main/release/linux/x64/libjavaProfiler.so | wc -l + +# Verify debug link +readelf -p .gnu_debuglink build/lib/main/release/linux/x64/libjavaProfiler.so + +# Check debug file +file build/lib/main/release/linux/x64/debug/libjavaProfiler.so.debug +``` + +**macOS:** +```bash +# Check stripped library +nm -gU build/lib/main/release/macos/arm64/libjavaProfiler.dylib + +# Verify dSYM bundle +dwarfdump --uuid build/lib/main/release/macos/arm64/libjavaProfiler.dylib.dSYM +``` + +### Testing Tips + +#### 1. Run Specific Test + +```bash +# Run one test from specific config +./gradlew gtestDebug_test_callTraceStorage +``` + +#### 2. Test with Multiple Configurations + +```bash +# Run tests with sanitizers in parallel +./gradlew gtestDebug gtestAsan gtestTsan --parallel +``` + +#### 3. Investigate Test Failures + +```bash +# Enable detailed test output +./gradlew gtestDebug --info + +# Run specific test binary directly +./build/bin/gtest/debug_test_callTraceStorage/test_callTraceStorage +``` + +#### 4. Test Environment Variables + +ASan and TSan tests automatically set environment variables: +```kotlin +// In BuildConfiguration: +testEnvironment.put("ASAN_OPTIONS", "...") +testEnvironment.put("TSAN_OPTIONS", "...") + +// Add custom variables: +buildConfigurations { + named("debug") { + testEnvironment.put("LOG_LEVEL", "debug") + testEnvironment.put("TEST_DATA_DIR", "$projectDir/testdata") + } +} +``` + +### Static Analysis Tips + +#### 1. Reading scan-build Reports + +```bash +# Run analysis +./gradlew scanBuild + +# Open report in browser +open build/reports/scan-build/*/index.html + +# Reports are organized by bug type: +# - Dead store: Unused assignments +# - Memory leak: Leaked allocations +# - Null dereference: Potential null pointer access +# - Uninitialized value: Use of uninitialized variables +``` + +#### 2. Focus on High-Priority Issues + +scan-build categorizes bugs by severity. Start with: +1. **Logic errors** - Wrong behavior +2. **Memory errors** - Leaks, use-after-free +3. **Null pointer issues** - Crashes +4. **Dead code** - Optimization opportunities + +#### 3. Incremental Analysis + +```bash +# Analyze after significant changes +./gradlew scanBuild + +# Compare with previous run +diff -u old-report/index.html build/reports/scan-build/*/index.html +``` + +#### 4. Customize Analyzer Options + +```kotlin +scanBuild { + makefileDir.set(layout.projectDirectory.dir("src/test/make")) + outputDir.set(layout.buildDirectory.dir("reports/scan-build")) + + // Use latest clang for better analysis + analyzer.set("/usr/bin/clang++-15") + + // Faster analysis with more parallelism + parallelJobs.set(8) +} +``` + +#### 5. Integration with Code Review + +```bash +# Run in CI before merging +./gradlew scanBuild + +# Fail build on new issues (requires custom script) +if grep -q "bugs found" build/reports/scan-build/*/index.html; then + echo "Static analysis found issues" + exit 1 +fi +``` + +#### 6. Suppressing False Positives + +If scan-build reports false positives, add assertions in code: +```cpp +void processData(Data* data) { + // Tell analyzer this can't be null + assert(data != nullptr); + + // Now scan-build knows data is valid + data->process(); +} +``` + +#### 7. Combine with Other Tools + +```bash +# Static analysis + runtime sanitizers = comprehensive coverage +./gradlew scanBuild # Find logic errors +./gradlew gtestAsan # Find memory errors +./gradlew gtestTsan # Find race conditions +``` + +### CI/CD Tips + +#### 1. Minimal CI Build +```bash +# Quick validation build +./gradlew assembleRelease -Pskip-tests +``` + +#### 2. Full CI Build +```bash +# Build all configs and run all tests +./gradlew assembleAll gtest +``` + +#### 3. CI with Sanitizers +```bash +# Test memory safety in CI +./gradlew gtestAsan gtestTsan + +# These are conditionally skipped if libasan/libtsan not available +``` + +#### 4. CI with Static Analysis +```bash +# Run static analysis in CI (Linux only) +./gradlew scanBuild + +# Archive reports as CI artifacts +# GitHub Actions example: +# - name: Upload scan-build reports +# uses: actions/upload-artifact@v3 +# with: +# name: scan-build-reports +# path: build/reports/scan-build/ + +# GitLab CI example: +# artifacts: +# paths: +# - build/reports/scan-build/ +# when: always +``` + +#### 5. Comprehensive CI Pipeline +```bash +# Full verification pipeline +./gradlew clean \ + assembleAll \ + gtest \ + gtestAsan \ + gtestTsan \ + scanBuild + +# This covers: +# - Compilation for all configs +# - Unit tests +# - Memory safety (ASan) +# - Thread safety (TSan) +# - Static analysis (scan-build) +``` + +#### 6. Release Artifact Packaging +```bash +# Build release with extracted debug symbols +./gradlew assembleRelease + +# Package production library (stripped) +tar czf library.tar.gz \ + build/lib/main/release/linux/x64/libjavaProfiler.so + +# Package debug symbols separately +tar czf library-debug.tar.gz \ + build/lib/main/release/linux/x64/debug/ +``` + +### Platform-Specific Tips + +#### macOS: Homebrew Google Test +```bash +# Install Google Test +brew install googletest + +# Plugin auto-detects at: +# /opt/homebrew/opt/googletest (Apple Silicon) +# /usr/local/opt/googletest (Intel) +``` + +#### Linux: System Google Test +```bash +# Ubuntu/Debian +sudo apt-get install libgtest-dev libgmock-dev + +# Fedora/RHEL +sudo dnf install gtest-devel gmock-devel +``` + +#### musl libc Detection +The plugin automatically detects musl libc and adds `-D__musl__`: +```bash +# Check musl detection +./gradlew assembleRelease | grep __musl__ + +# Force musl detection (advanced) +./gradlew -Pnative.musl=true assembleRelease +``` + +#### Linux: scan-build Installation + +scan-build is typically available on Linux but needs separate installation: + +```bash +# Ubuntu/Debian - includes clang static analyzer +sudo apt-get install clang-tools + +# Fedora/RHEL +sudo dnf install clang-tools-extra + +# Arch Linux +sudo pacman -S clang + +# Verify installation +which scan-build +scan-build --version +``` + +**Common scan-build locations:** +- `/usr/bin/scan-build` (most distros) +- `/usr/lib/llvm-*/bin/scan-build` (Ubuntu with specific LLVM versions) + +If you have multiple clang versions: +```bash +# List available scan-build versions +ls /usr/lib/llvm-*/bin/scan-build + +# Use specific version in plugin +scanBuild { + analyzer.set("/usr/lib/llvm-15/bin/clang++") +} +``` + +--- + +## Troubleshooting + +### Compiler Not Found + +**Problem:** +``` +Could not find any suitable C++ compiler +``` + +**Solutions:** +1. Install a compiler: + - macOS: `xcode-select --install` + - Linux: `sudo apt-get install build-essential` or `sudo dnf install gcc-c++` + +2. Force specific compiler: + ```bash + ./gradlew build -Pnative.forceCompiler=/path/to/compiler + ``` + +### Google Test Not Found + +**Problem:** +``` +WARNING: Google Test not found - skipping native tests +``` + +**Solutions:** +1. Install Google Test (see Platform-Specific Tips above) + +2. Set custom location (macOS): + ```kotlin + gtest { + googleTestHome.set(file("/custom/path/googletest")) + } + ``` + +3. Skip tests if not needed: + ```bash + ./gradlew build -Pskip-gtest + ``` + +### Sanitizer Libraries Not Available + +**Problem:** +ASan/TSan configurations are inactive. + +**Check availability:** +```bash +# Check for libasan +find /usr/lib /usr/local/lib -name "libasan.so*" 2>/dev/null + +# Check for libtsan +find /usr/lib /usr/local/lib -name "libtsan.so*" 2>/dev/null +``` + +**Solutions:** +1. Install sanitizer libraries: + - Ubuntu/Debian: `sudo apt-get install libasan6 libtsan0` + - Fedora/RHEL: `sudo dnf install libasan libtsan` + +2. Use clang (includes sanitizers): + ```bash + ./gradlew build -Pnative.forceCompiler=clang++ + ``` + +### Compilation Errors + +**Problem:** +``` +error: 'std::optional' is not a member of 'std' +``` + +**Solution:** +Ensure C++17 standard: +```kotlin +nativeBuild { + commonCompilerArgs("-std=c++17") +} +``` + +### Linking Errors + +**Problem:** +``` +undefined reference to `pthread_create' +``` + +**Solution:** +Add missing linker flags: +```kotlin +buildConfigurations { + named("debug") { + linkerArgs.add("-lpthread") + } +} +``` + +### Test Failures with ASan/TSan + +**Problem:** +Tests pass in debug but fail with ASan/TSan. + +**Analysis:** +This indicates real bugs! ASan/TSan found: +- Memory leaks +- Race conditions +- Use-after-free +- Buffer overflows + +**Solutions:** +1. Review the sanitizer output carefully +2. Fix the underlying bug (don't suppress unless false positive) +3. Add suppressions only for false positives: + ```bash + # ASan suppressions + echo "leak:third_party_library" >> gradle/sanitizers/asan.supp + + # TSan suppressions + echo "race:false_positive_function" >> gradle/sanitizers/tsan.supp + ``` + +### Clean Build Issues + +**Problem:** +Build fails after clean. + +**Solution:** +```bash +# Full clean including native artifacts +./gradlew clean + +# Rebuild +./gradlew assembleAll +``` + +### Task Not Found + +**Problem:** +``` +Task 'assembleAsan' not found +``` + +**Cause:** +Configuration is inactive (sanitizer not available). + +**Check:** +```bash +./gradlew tasks --group=build | grep -i asan +``` + +If not listed, the configuration is skipped on your platform. + +### scan-build Not Found + +**Problem:** +``` +scan-build not found in PATH - scanBuild task will fail if executed +``` + +**Solutions:** +1. Install scan-build on Linux: + ```bash + # Ubuntu/Debian + sudo apt-get install clang-tools + + # Fedora/RHEL + sudo dnf install clang-tools-extra + + # Arch Linux + sudo pacman -S clang + ``` + +2. Install on macOS (optional, not typical): + ```bash + brew install llvm + # Add to PATH: + export PATH="/opt/homebrew/opt/llvm/bin:$PATH" + ``` + +3. Verify installation: + ```bash + which scan-build + scan-build --help + ``` + +### scan-build Fails on macOS + +**Problem:** +Plugin skips scan-build task on macOS. + +**Explanation:** +By design, the plugin only runs scan-build on Linux. This matches typical CI environments. + +**Solution:** +If you need scan-build on macOS: +1. Install via Homebrew (see above) +2. Modify plugin check (advanced): + ```kotlin + // Override platform check + tasks.register("scanBuildMac", Exec::class) { + workingDir = file("src/test/make") + commandLine( + "scan-build", + "-o", "build/reports/scan-build", + "make", "all" + ) + } + ``` + +### scan-build Reports No Issues + +**Problem:** +scan-build runs but reports 0 bugs, even when issues exist. + +**Possible causes:** +1. **Makefile not using compiler correctly:** + ```makefile + # Wrong - hardcoded command + build: + /usr/bin/g++ -o output source.cpp + + # Correct - use $(CXX) variable + build: + $(CXX) -o output source.cpp + ``` + +2. **Precompiled objects:** + ```bash + # Clean before analysis + cd src/test/make + make clean + ./gradlew scanBuild + ``` + +3. **Compiler wrappers not intercepted:** + ```bash + # Verify scan-build is wrapping compiler + ./gradlew scanBuild --info | grep "scan-build" + ``` + +### scan-build Makefile Errors + +**Problem:** +``` +make: *** No rule to make target 'all'. Stop. +``` + +**Solution:** +Verify Makefile exists and has required targets: +```bash +# Check Makefile location +ls -la src/test/make/Makefile + +# Test make directly +cd src/test/make +make all + +# If it works, then try scan-build +./gradlew scanBuild +``` + +### scan-build Reports Inaccessible + +**Problem:** +Can't find or open HTML reports. + +**Solution:** +```bash +# Find the latest report directory +find build/reports/scan-build -name "index.html" + +# Open with browser +open $(find build/reports/scan-build -name "index.html" | head -1) + +# Or view summary in terminal +grep -A 5 "bugs found" build/reports/scan-build/*/index.html +``` + +--- + +## Advanced Topics + +### Custom Task Integration + +Hook into native build lifecycle: + +```kotlin +// Run custom validation after compilation +tasks.named("compileRelease") { + doLast { + println("Compiled ${outputs.files.files.size} object files") + } +} + +// Custom post-link processing +tasks.named("linkRelease") { + doLast { + val library = outputs.files.singleFile + println("Built library: ${library.absolutePath}") + println("Size: ${library.length() / 1024}KB") + } +} +``` + +### Multi-Project Builds + +Share native configurations across projects: + +**root/buildSrc/src/main/kotlin/NativeConventions.kt:** +```kotlin +fun Project.configureNative() { + apply(plugin = "com.datadoghq.native-build") + + configure { + version.set(rootProject.version.toString()) + commonCompilerArgs("-Wall", "-Wextra") + } +} +``` + +**subproject/build.gradle.kts:** +```kotlin +configureNative() + +nativeBuild { + cppSourceDirs.set(listOf("src/cpp")) +} +``` + +### Conditional Platform Builds + +```kotlin +import com.datadoghq.native.util.PlatformUtils +import com.datadoghq.native.model.Platform + +if (PlatformUtils.currentPlatform == Platform.LINUX) { + tasks.register("packageDeb") { + dependsOn("assembleRelease") + doLast { + // Create .deb package + } + } +} +``` + +--- + +## Further Reading + +- [README.md](README.md) - Full reference documentation +- [Plugin source code](conventions/src/main/kotlin/) - Implementation details +- [Gradle documentation](https://docs.gradle.org/) - Gradle build system +- [Google Test documentation](https://google.github.io/googletest/) - Unit testing framework diff --git a/build-logic/README.md b/build-logic/README.md new file mode 100644 index 000000000..16900afc5 --- /dev/null +++ b/build-logic/README.md @@ -0,0 +1,359 @@ +# Native Build Plugins + +This directory contains a Gradle composite build that provides plugins for building C++ libraries and tests: + +- **`com.datadoghq.native-build`** - Core C++ compilation and linking +- **`com.datadoghq.gtest`** - Google Test integration for C++ unit tests +- **`com.datadoghq.scanbuild`** - Clang static analyzer integration + +> **📚 New to these plugins?** Check out [QUICKSTART.md](QUICKSTART.md) for practical examples, common workflows, tips and tricks, and troubleshooting guidance. + +## Architecture + +The plugin uses Kotlin DSL for type-safe build configuration and follows modern Gradle conventions: + +- **Composite Build**: Independent Gradle project for build logic versioning +- **Type-Safe DSL**: Kotlin-based configuration with compile-time checking +- **Property API**: Lazy evaluation using Gradle's Property types +- **Automatic Task Generation**: Creates compile, link, and assemble tasks per configuration + +## Plugin Usage + +```kotlin +plugins { + id("com.datadoghq.native-build") +} + +nativeBuild { + version.set(project.version.toString()) + cppSourceDirs.set(listOf("src/main/cpp")) + includeDirectories.set(listOf("src/main/cpp")) +} +``` + +The plugin automatically creates standard configurations (release, debug, asan, tsan, fuzzer) and generates tasks: +- `compile{Config}` - Compiles C++ sources +- `link{Config}` - Links shared library +- `assemble{Config}` - Assembles configuration +- `assembleAll` - Builds all active configurations + +## Standard Configurations + +### Release +- **Optimization**: `-O3 -DNDEBUG` +- **Debug symbols**: Extracted to separate files (69% size reduction) +- **Strip**: Yes (production binaries) +- **Output**: Stripped library + .dSYM bundle (macOS) or .debug file (Linux) + +### Debug +- **Optimization**: `-O0 -g` +- **Debug symbols**: Embedded +- **Strip**: No +- **Output**: Full debug library + +### ASan (AddressSanitizer) +- Conditionally active if libasan is available +- Memory error detection + +### TSan (ThreadSanitizer) +- Conditionally active if libtsan is available +- Thread safety validation + +### Fuzzer +- Fuzzing instrumentation +- Requires libFuzzer + +## Compiler Detection + +The plugin automatically detects and selects the best available C++ compiler: + +### Auto-Detection (Default) +```bash +./gradlew build +# Logs: "Auto-detected compiler: clang++" +# or: "Auto-detected compiler: g++" +``` + +**Detection order:** +1. `clang++` (preferred - better optimization and diagnostics) +2. `g++` (fallback) +3. `c++` (last resort) + +If no compiler is found, the build fails with a clear error message. + +### Force Specific Compiler +Use the `-Pnative.forceCompiler` property to override auto-detection: + +```bash +# Force clang++ +./gradlew build -Pnative.forceCompiler=clang++ + +# Force g++ +./gradlew build -Pnative.forceCompiler=g++ + +# Force specific version (full path) +./gradlew build -Pnative.forceCompiler=/usr/bin/g++-13 +./gradlew build -Pnative.forceCompiler=/opt/homebrew/bin/clang++ +``` + +**Validation:** The specified compiler is validated by running ` --version`. If validation fails, the build errors immediately with an actionable message. + +### Sanitizer Library Detection +ASan and TSan library detection uses the detected/forced compiler instead of hardcoding `gcc`. This enables sanitizer builds on clang-only systems (e.g., macOS with Xcode but no gcc installed). + +## Debug Symbol Extraction + +Release builds automatically extract debug symbols for optimal production deployment: + +### macOS +``` +dsymutil library.dylib -o library.dylib.dSYM +strip -S library.dylib +``` +- **Stripped library**: ~404KB (production) +- **Debug bundle**: ~3.7MB (.dSYM) + +### Linux +``` +objcopy --only-keep-debug library.so library.so.debug +objcopy --strip-debug library.so +objcopy --add-gnu-debuglink=library.so.debug library.so +``` +- **Stripped library**: ~1.2MB (production) +- **Debug file**: ~6MB (.debug) + +## Advanced Features + +### Source Sets + +Source sets allow different parts of the codebase to have different compilation flags. This is useful for: +- Legacy code requiring older C++ standards +- Third-party code with specific compiler warnings +- Platform-specific optimizations + +**Example:** +```kotlin +tasks.register("compileLib", NativeCompileTask::class) { + compiler.set("clang++") + compilerArgs.set(listOf("-std=c++17", "-O3")) // Base flags for all files + includes.from("src/main/cpp") + + // Define source sets with per-set compiler flags + sourceSets { + create("main") { + sources.from(fileTree("src/main/cpp")) + compilerArgs.add("-fPIC") // Additional flags for main code + } + create("legacy") { + sources.from(fileTree("src/legacy")) + compilerArgs.addAll("-Wno-deprecated", "-std=c++11") // Different standard + excludes.add("**/broken/*.cpp") // Exclude specific files + } + } + + objectFileDir.set(file("build/obj")) +} +``` + +**Key features:** +- **Include/exclude patterns**: Ant-style patterns (e.g., `**/*.cpp`, `**/test_*.cpp`) +- **Merged compiler args**: Base args + source-set-specific args +- **Conveniences**: `from()`, `include()`, `exclude()`, `compileWith()` methods + +### Symbol Visibility Control + +Symbol visibility controls which symbols are exported from shared libraries. This is essential for: +- Hiding internal implementation details +- Reducing symbol table size +- Preventing symbol conflicts +- Creating clean JNI interfaces + +**Example:** +```kotlin +tasks.register("linkLib", NativeLinkTask::class) { + linker.set("clang++") + objectFiles.from(fileTree("build/obj")) + outputFile.set(file("build/lib/libjavaProfiler.dylib")) + + // Export only JNI symbols + exportSymbols.set(listOf( + "Java_*", // All JNI methods + "JNI_OnLoad", // JNI initialization + "JNI_OnUnload" // JNI cleanup + )) + + // Hide specific internal symbols (overrides exports) + hideSymbols.set(listOf( + "*_internal*", // Internal functions + "*_test*" // Test utilities + )) +} +``` + +**Platform-specific implementation:** +- **Linux**: Generates version script (`.ver` file) with wildcard pattern support (e.g., `Java_*` matches all JNI methods) +- **macOS**: Generates exported symbols list (`.exp` file) - **Note:** Wildcards are not supported on macOS. Patterns like `Java_*` are treated as literal symbol names. For JNI exports, you must either list individual symbols or use `-fvisibility` compiler flags instead. + +**Generated files** (in `temporaryDir`): +- Linux: `library.ver` → `-Wl,--version-script=library.ver` +- macOS: `library.exp` → `-Wl,-exported_symbols_list,library.exp` + +**Symbol visibility best practices:** +1. Start with `-fvisibility=hidden` compiler flag +2. Mark public API with `__attribute__((visibility("default")))` in source +3. OR use `exportSymbols` linker flag for pattern-based export +4. Verify with: `nm -gU library.dylib` (macOS) or `nm -D library.so` (Linux) + +## Task Dependencies + +``` +compileConfig → linkConfig → assembleConfig + ↓ + extractDebugSymbols (release only) + ↓ + stripSymbols (release only) + ↓ + copyConfigLibs → assembleConfigJar +``` + +## Design Benefits + +The Kotlin-based build system provides: +- ✅ Compile-time type checking via Kotlin DSL +- ✅ Gradle idiomatic design (Property API, composite builds) +- ✅ Automatic debug symbol extraction (69% size reduction) +- ✅ Clean builds work from scratch +- ✅ Centralized configuration definitions + +--- + +# Google Test Plugin + +The `com.datadoghq.gtest` plugin provides Google Test integration for C++ unit testing. + +## Plugin Usage + +```kotlin +plugins { + id("com.datadoghq.native-build") // Required - provides configurations + id("com.datadoghq.gtest") +} + +gtest { + testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) + mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) + + includes.from( + "src/main/cpp", + "${javaHome}/include", + "${javaHome}/include/${platformInclude}" + ) +} +``` + +## Generated Tasks + +For each test file in `testSourceDir`, the plugin creates: + +| Task Pattern | Description | +|--------------|-------------| +| `compileGtest{Config}_{TestName}` | Compile main sources + test file | +| `linkGtest{Config}_{TestName}` | Link test executable with gtest libraries | +| `gtest{Config}_{TestName}` | Execute the test | + +Aggregation tasks: +- `gtest` - Run all tests across all configurations +- `gtest{Config}` - Run all tests for a specific configuration (e.g., `gtestDebug`) + +## Configuration Options + +| Property | Type | Default | Description | +|----------|------|---------|-------------| +| `testSourceDir` | `DirectoryProperty` | Required | Directory containing test `.cpp` files | +| `mainSourceDir` | `DirectoryProperty` | Required | Directory containing main source files | +| `includes` | `ConfigurableFileCollection` | Empty | Include directories for compilation | +| `googleTestHome` | `DirectoryProperty` | Auto-detected | Google Test installation directory (macOS) | +| `enableAssertions` | `Property` | `true` | Remove `-DNDEBUG` to enable assertions | +| `keepSymbols` | `Property` | `true` | Keep debug symbols in release test builds | +| `failFast` | `Property` | `false` | Stop on first test failure | +| `alwaysRun` | `Property` | `true` | Ignore up-to-date checks for tests | +| `buildNativeLibs` | `Property` | `true` | Build native test support libraries (Linux) | + +## Platform Detection + +The plugin automatically detects Google Test installation: + +- **macOS**: `/opt/homebrew/opt/googletest` (Homebrew default) +- **Linux**: System includes (`/usr/include/gtest`) + +Override with `googleTestHome`: +```kotlin +gtest { + googleTestHome.set(file("/custom/path/to/googletest")) +} +``` + +## Integration with NativeBuildPlugin + +GtestPlugin consumes configurations from NativeBuildPlugin: + +1. **Shared configurations**: Uses the same release/debug/asan/tsan/fuzzer configs +2. **Compiler detection**: Uses `PlatformUtils.findCompiler()` with `-Pnative.forceCompiler` support +3. **Consistent flags**: Inherits compiler/linker args from build configurations + +## Example Output + +``` +$ ./gradlew gtestDebug + +> Task :ddprof-lib:compileGtestDebug_test_callTraceStorage +Compiling 45 C++ source files with clang++... + +> Task :ddprof-lib:linkGtestDebug_test_callTraceStorage +Linking executable: test_callTraceStorage + +> Task :ddprof-lib:gtestDebug_test_callTraceStorage +[==========] Running 5 tests from 1 test suite. +... +[ PASSED ] 5 tests. + +BUILD SUCCESSFUL +``` + +## Skip Options + +```bash +# Skip all tests +./gradlew build -Pskip-tests + +# Skip only gtest (keep Java tests) +./gradlew build -Pskip-gtest + +# Skip native compilation entirely +./gradlew build -Pskip-native +``` + +--- + +## Files + +- `settings.gradle` - Composite build configuration +- `conventions/build.gradle.kts` - Plugin module +- `conventions/src/main/kotlin/` - Plugin implementation + - `NativeBuildPlugin.kt` - Native build plugin + - `NativeBuildExtension.kt` - Native build DSL extension + - `gtest/GtestPlugin.kt` - Google Test plugin + - `gtest/GtestExtension.kt` - Google Test DSL extension + - `scanbuild/ScanBuildPlugin.kt` - Static analysis plugin + - `scanbuild/ScanBuildExtension.kt` - Static analysis DSL extension + - `model/` - Type-safe configuration models + - `tasks/` - Compile and link tasks + - `config/` - Configuration presets + - `util/` - Platform utilities + +--- + +## Documentation + +- **[QUICKSTART.md](QUICKSTART.md)** - Quick start guide with practical examples, workflows, tips and troubleshooting +- **[README.md](README.md)** (this file) - Architecture details, API reference, and design documentation diff --git a/build-logic/conventions/build.gradle.kts b/build-logic/conventions/build.gradle.kts new file mode 100644 index 000000000..633469242 --- /dev/null +++ b/build-logic/conventions/build.gradle.kts @@ -0,0 +1,58 @@ +plugins { + `kotlin-dsl` +} + +repositories { + gradlePluginPortal() + mavenCentral() +} + +dependencies { + implementation("org.jetbrains.kotlin:kotlin-stdlib") + implementation("com.diffplug.spotless:spotless-plugin-gradle:6.11.0") +} + +gradlePlugin { + plugins { + create("nativeBuild") { + id = "com.datadoghq.native-build" + implementationClass = "com.datadoghq.native.NativeBuildPlugin" + } + create("nativeRoot") { + id = "com.datadoghq.native-root" + implementationClass = "com.datadoghq.native.RootProjectPlugin" + } + create("gtest") { + id = "com.datadoghq.gtest" + implementationClass = "com.datadoghq.native.gtest.GtestPlugin" + } + create("scanbuild") { + id = "com.datadoghq.scanbuild" + implementationClass = "com.datadoghq.native.scanbuild.ScanBuildPlugin" + } + create("profilerTest") { + id = "com.datadoghq.profiler-test" + implementationClass = "com.datadoghq.profiler.ProfilerTestPlugin" + } + create("spotlessConvention") { + id = "com.datadoghq.spotless-convention" + implementationClass = "com.datadoghq.profiler.SpotlessConventionPlugin" + } + create("javaConventions") { + id = "com.datadoghq.java-conventions" + implementationClass = "com.datadoghq.profiler.JavaConventionsPlugin" + } + create("fuzzTargets") { + id = "com.datadoghq.fuzz-targets" + implementationClass = "com.datadoghq.native.fuzz.FuzzTargetsPlugin" + } + create("simpleNativeLib") { + id = "com.datadoghq.simple-native-lib" + implementationClass = "com.datadoghq.native.SimpleNativeLibPlugin" + } + create("versionedSources") { + id = "com.datadoghq.versioned-sources" + implementationClass = "com.datadoghq.java.versionedsources.VersionedSourcesPlugin" + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourceSet.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourceSet.kt new file mode 100644 index 000000000..a4e075d1c --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourceSet.kt @@ -0,0 +1,54 @@ + +package com.datadoghq.java.versionedsources + +import org.gradle.api.Named +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.provider.Property +import javax.inject.Inject + +/** + * Represents a versioned source set for Java version-specific code. + * + * Each instance defines a Java version-specific source directory that contains + * classes compiled with a specific Java release target. These classes are selected + * at runtime via Class.forName() based on the running JVM version. + * + * @property name The source set name (e.g., "java9", "java11") + */ +abstract class VersionedSourceSet @Inject constructor( + private val name: String +) : Named { + + /** + * The Java release version (e.g., 9, 11, 17, 21). + * Used with javac --release flag for compilation. + */ + abstract val release: Property + + /** + * Source directory for version-specific classes. + * Default: src/main/{name} (e.g., src/main/java9) + */ + abstract val sourceDir: DirectoryProperty + + /** + * Minimum toolchain version required to compile this source set. + * Must be >= release version. Defaults to the release version. + * Use when a higher JDK is needed (e.g., compile Java 9 code with JDK 11). + */ + abstract val minToolchainVersion: Property + + override fun getName(): String = name + + /** + * Returns the capitalized name for task naming. + * Example: "Java9" for name "java9" + */ + fun capitalizedName(): String = name.replaceFirstChar { it.titlecase() } + + /** + * The compile task name for this versioned source set. + * Example: "compileJava9Java" for name "java9" + */ + val compileTaskName: String get() = "compile${capitalizedName()}Java" +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesExtension.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesExtension.kt new file mode 100644 index 000000000..9f55a3462 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesExtension.kt @@ -0,0 +1,109 @@ + +package com.datadoghq.java.versionedsources + +import org.gradle.api.Action +import org.gradle.api.JavaVersion +import org.gradle.api.NamedDomainObjectContainer +import org.gradle.api.Project +import org.gradle.api.model.ObjectFactory +import org.gradle.api.tasks.SourceSetContainer +import org.gradle.jvm.tasks.Jar +import javax.inject.Inject + +/** + * Extension for configuring versioned source sets. + * + * This plugin helps manage Java version-specific source directories where code + * requires different Java versions to compile (e.g., VarHandle in Java 9+). + * Classes are compiled separately and merged into the JAR root for runtime + * selection via Class.forName(). + * + * Usage: + * ```kotlin + * versionedSources { + * versions { + * register("java9") { + * release.set(9) + * minToolchainVersion.set(11) + * } + * register("java11") { + * release.set(11) + * } + * } + * } + * ``` + */ +abstract class VersionedSourcesExtension @Inject constructor( + private val project: Project, + objects: ObjectFactory +) { + /** + * Container for versioned source set configurations. + */ + val versions: NamedDomainObjectContainer = + objects.domainObjectContainer(VersionedSourceSet::class.java) + + /** + * Configure versioned source sets using a DSL block. + */ + fun versions(action: Action>) { + action.execute(versions) + } + + /** + * Get all versioned source sets that can be compiled with the current JDK. + */ + fun getCompilableVersions(): List { + val available = JavaVersion.current().majorVersion.toInt() + return versions.filter { version -> + val required = version.minToolchainVersion.orNull ?: version.release.get() + required <= available + } + } + + /** + * Configures a JAR task to include all versioned classes at the JAR root. + * + * This adds the classes compiled from version-specific source sets to the JAR root, + * alongside the main source set classes. This is useful when the application uses + * runtime class loading (Class.forName) to select the appropriate implementation + * based on Java version. + * + * Usage: + * ```kotlin + * tasks.register("myJar", Jar::class) { + * from(sourceSets.main.get().output) + * versionedSources.configureJar(this) + * } + * ``` + */ + fun configureJar(jar: Jar) { + val sourceSets = project.extensions.getByType(SourceSetContainer::class.java) + + versions.forEach { version -> + val sourceSet = sourceSets.findByName(version.name) ?: return@forEach + jar.from(sourceSet.output.classesDirs) + jar.dependsOn(version.compileTaskName) + } + } + + /** + * Configures a source JAR task to include all versioned source files. + * + * Usage: + * ```kotlin + * val sourcesJar by tasks.registering(Jar::class) { + * from(sourceSets.main.get().allJava) + * versionedSources.configureSourceJar(this) + * } + * ``` + */ + fun configureSourceJar(jar: Jar) { + val sourceSets = project.extensions.getByType(SourceSetContainer::class.java) + + versions.forEach { version -> + val sourceSet = sourceSets.findByName(version.name) ?: return@forEach + jar.from(sourceSet.allJava) + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesPlugin.kt new file mode 100644 index 000000000..57c94418d --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesPlugin.kt @@ -0,0 +1,131 @@ + +package com.datadoghq.java.versionedsources + +import org.gradle.api.JavaVersion +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.plugins.JavaPlugin +import org.gradle.api.tasks.SourceSetContainer +import org.gradle.api.tasks.compile.JavaCompile +import org.gradle.jvm.toolchain.JavaLanguageVersion +import org.gradle.jvm.toolchain.JavaToolchainService + +/** + * Gradle plugin for managing versioned source sets. + * + * This plugin simplifies compiling Java version-specific code by: + * 1. Creating versioned source sets automatically + * 2. Configuring compilation with correct --release flags and toolchains + * 3. Setting up classpath dependencies (version N depends on main) + * 4. Providing utilities for JAR task configuration + * + * Classes are added to the JAR root (not META-INF/versions/) for runtime + * selection via Class.forName() based on Java version. + * + * Usage: + * ```kotlin + * plugins { + * java + * id("com.datadoghq.versioned-sources") + * } + * + * versionedSources { + * versions { + * register("java9") { + * release.set(9) + * minToolchainVersion.set(11) + * } + * } + * } + * ``` + */ +class VersionedSourcesPlugin : Plugin { + + override fun apply(project: Project) { + // Ensure Java plugin is applied + project.pluginManager.apply(JavaPlugin::class.java) + + // Create extension + val extension = project.extensions.create( + "versionedSources", + VersionedSourcesExtension::class.java, + project, + project.objects + ) + + // Configure after project evaluation to access all registered versions + project.afterEvaluate { + configureVersionedSourceSets(project, extension) + configureCompilationTasks(project, extension) + } + } + + private fun configureVersionedSourceSets( + project: Project, + extension: VersionedSourcesExtension + ) { + val sourceSets = project.extensions.getByType(SourceSetContainer::class.java) + + extension.versions.forEach { version -> + // Create source set if it doesn't exist + val sourceSet = sourceSets.maybeCreate(version.name) + + // Configure source directory + val srcDir = if (version.sourceDir.isPresent) { + version.sourceDir.get().asFile.path + } else { + "src/main/${version.name}" + } + + sourceSet.java.srcDirs(srcDir) + + project.logger.info( + "VersionedSources: Created source set '${version.name}' with source dir: $srcDir" + ) + } + } + + private fun configureCompilationTasks( + project: Project, + extension: VersionedSourcesExtension + ) { + val sourceSets = project.extensions.getByType(SourceSetContainer::class.java) + val mainSourceSet = sourceSets.getByName("main") + val javaToolchains = project.extensions.getByType(JavaToolchainService::class.java) + + extension.versions.forEach { version -> + val compileTask = project.tasks.findByName(version.compileTaskName) as? JavaCompile + ?: return@forEach + + val releaseVersion = version.release.get() + val minToolchain = version.minToolchainVersion.orNull ?: releaseVersion + + // Determine actual toolchain version to use (current JDK if >= minToolchain) + val currentJdk = JavaVersion.current().majorVersion.toInt() + val toolchainVersion = if (currentJdk >= minToolchain) currentJdk else minToolchain + + compileTask.apply { + // Set toolchain for compilation + javaCompiler.set( + javaToolchains.compilerFor { + languageVersion.set(JavaLanguageVersion.of(toolchainVersion)) + } + ) + + // Set release flag for target version + options.release.set(releaseVersion) + + // Set classpath to include main source set output + compile dependencies + classpath = mainSourceSet.output + project.configurations.getByName("compileClasspath") + + // Depend on main compilation + dependsOn(project.tasks.named("compileJava")) + } + + project.logger.info( + "VersionedSources: Configured ${version.compileTaskName}: " + + "release=$releaseVersion, toolchain=$toolchainVersion" + ) + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildExtension.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildExtension.kt new file mode 100644 index 000000000..d1a1872c9 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildExtension.kt @@ -0,0 +1,136 @@ + +package com.datadoghq.native + +import com.datadoghq.native.model.Architecture +import com.datadoghq.native.model.BuildConfiguration +import com.datadoghq.native.model.Platform +import com.datadoghq.native.util.PlatformUtils +import org.gradle.api.Action +import org.gradle.api.NamedDomainObjectContainer +import org.gradle.api.Project +import org.gradle.api.file.Directory +import org.gradle.api.model.ObjectFactory +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property +import org.gradle.api.provider.Provider +import javax.inject.Inject + +abstract class NativeBuildExtension @Inject constructor( + private val project: Project, + private val objects: ObjectFactory +) { + /** + * Container for build configurations (release, debug, asan, tsan, etc.) + */ + val buildConfigurations: NamedDomainObjectContainer = + objects.domainObjectContainer(BuildConfiguration::class.java) + + /** + * Project version to embed in binaries + */ + abstract val version: Property + + /** + * Source directories for C++ code + */ + abstract val cppSourceDirs: ListProperty + + /** + * Include directories for compilation + */ + abstract val includeDirectories: ListProperty + + init { + version.convention(project.version.toString()) + cppSourceDirs.convention(listOf("src/main/cpp")) + includeDirectories.convention(emptyList()) + } + + /** + * Configure build configurations using a DSL block + */ + fun buildConfigurations(action: Action>) { + action.execute(buildConfigurations) + } + + /** + * Get all configurations that are active for the current platform and architecture + */ + fun getActiveConfigurations(platform: Platform, architecture: Architecture): List { + return buildConfigurations.filter { it.isActiveFor(platform, architecture) } + } + + /** + * Get configuration names for the current platform/architecture + */ + fun getActiveConfigurationNames(): List { + val platform = PlatformUtils.currentPlatform + val arch = PlatformUtils.currentArchitecture + return getActiveConfigurations(platform, arch).map { it.name } + } + + /** + * Convenience method to define common compiler args for all configurations + */ + fun commonCompilerArgs(vararg args: String) { + buildConfigurations.configureEach { + compilerArgs.addAll(*args) + } + } + + /** + * Convenience method to define common linker args for all configurations + */ + fun commonLinkerArgs(vararg args: String) { + buildConfigurations.configureEach { + linkerArgs.addAll(*args) + } + } + + // ==================== Path Utilities ==================== + // These methods provide consistent path calculations for native library locations + + /** + * Platform identifier for library paths (e.g., "LINUX-x64", "MACOS-arm64-musl") + */ + fun platformIdentifier(): String { + val muslSuffix = if (PlatformUtils.isMusl()) "-musl" else "" + return "${PlatformUtils.currentPlatform}-${PlatformUtils.currentArchitecture}$muslSuffix" + } + + /** + * Directory where native libraries are built for a given configuration. + * Structure: build/lib/main/{config}/{platform}/{arch}/ + */ + fun librarySourceDir(config: String): Provider { + return project.layout.buildDirectory.dir( + "lib/main/$config/${PlatformUtils.currentPlatform}/${PlatformUtils.currentArchitecture}" + ) + } + + /** + * Directory for packaging native libraries into JAR. + * Structure: build/native/{config}/META-INF/native-libs/{platform-arch}/ + */ + fun libraryTargetDir(config: String): Provider { + return project.layout.buildDirectory.dir( + "native/$config/META-INF/native-libs/${platformIdentifier()}" + ) + } + + /** + * Base directory for native library packaging (without platform subdirs). + * Structure: build/native/{config}/ + */ + fun libraryTargetBase(config: String): Provider { + return project.layout.buildDirectory.dir("native/$config") + } + + /** + * Returns the platform-appropriate shared library filename. + * Examples: "libjavaProfiler.so" (Linux), "libjavaProfiler.dylib" (macOS) + */ + fun sharedLibraryName(baseName: String): String { + return "lib$baseName.${PlatformUtils.sharedLibExtension()}" + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildPlugin.kt new file mode 100644 index 000000000..33628c1f3 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildPlugin.kt @@ -0,0 +1,171 @@ + +package com.datadoghq.native + +import com.datadoghq.native.config.ConfigurationPresets +import com.datadoghq.native.model.Architecture +import com.datadoghq.native.model.BuildConfiguration +import com.datadoghq.native.model.Platform +import com.datadoghq.native.tasks.NativeCompileTask +import com.datadoghq.native.tasks.NativeLinkTask +import com.datadoghq.native.util.PlatformUtils +import org.gradle.api.Plugin +import org.gradle.api.Project + +/** + * Gradle plugin that provides native C++ build configuration management. + * + * This plugin: + * 1. Creates a `nativeBuild` extension for configuring build configurations + * 2. Automatically generates compile, link, and assemble tasks for each configuration + * 3. Provides standard configuration presets (release, debug, asan, tsan, fuzzer) + * 4. Handles platform-specific compiler and linker flags + * + * Usage example: + * ``` + * plugins { + * id("com.datadoghq.native-build") + * } + * + * nativeBuild { + * buildConfigurations { + * register("release") { + * platform.set(Platform.LINUX) + * architecture.set(Architecture.X64) + * compilerArgs.set(listOf("-O3", "-DNDEBUG")) + * } + * } + * } + * ``` + */ +class NativeBuildPlugin : Plugin { + override fun apply(project: Project) { + // Create the extension + val extension = project.extensions.create( + "nativeBuild", + NativeBuildExtension::class.java, + project, + project.objects + ) + + // Setup standard configurations after project evaluation + project.afterEvaluate { + setupStandardConfigurations(project, extension) + createTasks(project, extension) + } + } + + private fun setupStandardConfigurations(project: Project, extension: NativeBuildExtension) { + ConfigurationPresets.setupStandardConfigurations(extension, project) + } + + private fun createTasks(project: Project, extension: NativeBuildExtension) { + val currentPlatform = PlatformUtils.currentPlatform + val currentArch = PlatformUtils.currentArchitecture + + // Get active configurations for current platform + val activeConfigs = extension.getActiveConfigurations(currentPlatform, currentArch) + + project.logger.lifecycle("Active configurations: ${activeConfigs.map { it.name }.joinToString(", ")}") + + // Create tasks for each active configuration + activeConfigs.forEach { config -> + createConfigurationTasks(project, extension, config) + } + + // Create aggregation tasks + createAggregationTasks(project, activeConfigs) + } + + private fun createConfigurationTasks( + project: Project, + extension: NativeBuildExtension, + config: BuildConfiguration + ) { + val configName = config.capitalizedName() + val platform = config.platform.get() + val arch = config.architecture.get() + + // Define paths + val objDir = project.file("build/obj/main/${config.name}") + val libDir = project.file("build/lib/main/${config.name}/$platform/$arch") + val libName = "libjavaProfiler.${PlatformUtils.sharedLibExtension()}" + val outputLib = project.file("$libDir/$libName") + + // Create compile task + val compileTask = project.tasks.register("compile$configName", NativeCompileTask::class.java) { + group = "build" + description = "Compiles C++ sources for ${config.name} configuration" + + // Find compiler + val compilerPath = findCompiler(project) + compiler.set(compilerPath) + compilerArgs.set(config.compilerArgs.get()) + + // Set sources - default to src/main/cpp + val srcDirs = extension.cppSourceDirs.get() + sources.from(srcDirs.map { dir -> + project.fileTree(dir) { + include("**/*.cpp", "**/*.cc", "**/*.c") + } + }) + + // Set includes - default + JNI + val includeList = extension.includeDirectories.get().toMutableList() + includeList.addAll(PlatformUtils.jniIncludePaths()) + includes.from(includeList) + + objectFileDir.set(objDir) + } + + // Create link task + val linkTask = project.tasks.register("link$configName", NativeLinkTask::class.java) { + group = "build" + description = "Links ${config.name} shared library" + dependsOn(compileTask) + + val compilerPath = findCompiler(project) + linker.set(compilerPath) + linkerArgs.set(config.linkerArgs.get()) + objectFiles.from(project.fileTree(objDir) { + include("*.o") + }) + outputFile.set(outputLib) + + // Enable debug symbol extraction for release builds + if (config.name == "release") { + extractDebugSymbols.set(true) + stripSymbols.set(true) + debugSymbolsDir.set(project.file("$libDir/debug")) + } + } + + // Create assemble task + project.tasks.register("assemble$configName") { + group = "build" + description = "Assembles ${config.name} configuration" + dependsOn(linkTask) + } + + project.logger.debug("Created tasks for configuration: ${config.name}") + } + + private fun findCompiler(project: Project): String = PlatformUtils.findCompiler(project) + + private fun createAggregationTasks( + project: Project, + activeConfigs: List + ) { + // Create assembleAll task that depends on all assemble tasks + project.tasks.register("assembleAll") { + group = "build" + description = "Assembles all active build configurations" + // Depend on all individual assemble tasks + activeConfigs.forEach { config -> + val configName = config.capitalizedName() + dependsOn("assemble$configName") + } + } + + project.logger.lifecycle("Created assembleAll task") + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/RootProjectPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/RootProjectPlugin.kt new file mode 100644 index 000000000..0808db99a --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/RootProjectPlugin.kt @@ -0,0 +1,53 @@ + +package com.datadoghq.native + +import com.datadoghq.native.config.ConfigurationPresets +import com.datadoghq.native.util.PlatformUtils +import org.gradle.api.GradleException +import org.gradle.api.Plugin +import org.gradle.api.Project + +/** + * Gradle plugin that provides native build configuration access to the root project. + * + * This plugin creates a `nativeBuild` extension on the root project that exposes + * standard build configurations (release, debug, asan, tsan, fuzzer) to all subprojects. + * This allows subprojects to query and access build configurations in a type-safe manner. + * + * Usage example: + * ``` + * // In root build.gradle.kts + * plugins { + * id("com.datadoghq.native-root") + * } + * + * // In subproject build.gradle.kts + * val nativeBuildExt = rootProject.extensions.getByType(NativeBuildExtension::class.java) + * val activeConfigs = nativeBuildExt.getActiveConfigurations(currentPlatform, currentArch) + * ``` + */ +class RootProjectPlugin : Plugin { + override fun apply(project: Project) { + // Only apply to root project + if (project != project.rootProject) { + throw GradleException("RootProjectPlugin must be applied to root project only") + } + + // Create nativeBuild extension on root + val extension = project.extensions.create( + "nativeBuild", + NativeBuildExtension::class.java, + project, + project.objects + ) + + // Setup standard configurations after project evaluation + project.afterEvaluate { + setupStandardConfigurationsIfNeeded(project, extension) + } + } + + private fun setupStandardConfigurationsIfNeeded(project: Project, extension: NativeBuildExtension) { + ConfigurationPresets.setupStandardConfigurations(extension, project) + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/SimpleNativeLibPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/SimpleNativeLibPlugin.kt new file mode 100644 index 000000000..f39087c02 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/SimpleNativeLibPlugin.kt @@ -0,0 +1,226 @@ + +package com.datadoghq.native + +import com.datadoghq.native.model.Platform +import com.datadoghq.native.tasks.NativeCompileTask +import com.datadoghq.native.tasks.NativeLinkTask +import com.datadoghq.native.util.PlatformUtils +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property +import javax.inject.Inject + +/** + * Simplified plugin for single-output native library projects. + * + * Creates standard compile and link tasks with sensible defaults: + * - compileLib: Compiles C/C++ sources + * - linkLib: Links shared library + * - Wires linkLib into assemble lifecycle + * + * Usage: + * ```kotlin + * plugins { + * id("com.datadoghq.simple-native-lib") + * } + * + * simpleNativeLib { + * libraryName.set("mylib") + * sourceDir.set(file("src/main/cpp")) + * compilerArgs.set(listOf("-O3", "-fPIC")) + * } + * ``` + */ +class SimpleNativeLibPlugin : Plugin { + override fun apply(project: Project) { + val extension = project.extensions.create( + "simpleNativeLib", + SimpleNativeLibExtension::class.java, + project + ) + + project.afterEvaluate { + if (!extension.enabled.get()) { + return@afterEvaluate + } + + val compiler = extension.compiler.getOrElse(PlatformUtils.findCompiler(project)) + val linker = extension.linker.getOrElse(compiler) + val libraryName = extension.libraryName.get() + val objectDir = extension.objectDir.get().asFile + val outputLib = extension.outputDir.get().asFile.resolve( + "lib$libraryName.${PlatformUtils.sharedLibExtension()}" + ) + + // Compile task + val compileTask = project.tasks.register("compileLib", NativeCompileTask::class.java) { + onlyIf { extension.enabled.get() && !project.hasProperty("skip-native") } + group = "build" + description = "Compile the $libraryName library" + + this.compiler.set(compiler) + this.compilerArgs.set(extension.compilerArgs.get()) + sources.from(extension.sourceDir.map { dir -> + project.fileTree(dir) { + include("**/*.cpp", "**/*.cc", "**/*.c") + } + }) + if (extension.includeJni.get()) { + includes.from(PlatformUtils.jniIncludePaths()) + } + includes.from(extension.includeDirs.get()) + objectFileDir.set(objectDir) + } + + // Link task + val linkTask = project.tasks.register("linkLib", NativeLinkTask::class.java) { + onlyIf { extension.enabled.get() && !project.hasProperty("skip-native") } + dependsOn(compileTask) + group = "build" + description = "Link the $libraryName shared library" + + this.linker.set(linker) + this.linkerArgs.set(extension.linkerArgs.get()) + objectFiles.from(project.fileTree(objectDir) { include("*.o") }) + outputFile.set(outputLib) + } + + // Wire into assemble + project.tasks.named("assemble") { + dependsOn(linkTask) + } + + // Create consumable configurations if requested + if (extension.createConfigurations.get()) { + createConsumableConfigurations(project, extension, compileTask, linkTask) + } + } + } + + private fun createConsumableConfigurations( + project: Project, + extension: SimpleNativeLibExtension, + compileTask: org.gradle.api.tasks.TaskProvider, + linkTask: org.gradle.api.tasks.TaskProvider + ) { + // Runtime library configuration + project.configurations.create("nativeLib") { + isCanBeConsumed = true + isCanBeResolved = false + description = "Native shared library for runtime loading" + outgoing.artifact(linkTask.flatMap { it.outputFile }) { + type = "native-lib" + } + } + + // Object files configuration + project.configurations.create("nativeObjects") { + isCanBeConsumed = true + isCanBeResolved = false + description = "Object files for static linking" + outgoing.artifact(compileTask.flatMap { it.objectFileDir }) { + type = "native-objects" + } + } + + // Library directory configuration + project.configurations.create("nativeLibDir") { + isCanBeConsumed = true + isCanBeResolved = false + description = "Directory containing native library" + outgoing.artifact(extension.outputDir) { + type = "native-lib-dir" + } + } + + // Headers configuration + project.configurations.create("nativeHeaders") { + isCanBeConsumed = true + isCanBeResolved = false + description = "Header files for compilation" + outgoing.artifact(extension.sourceDir) { + type = "native-headers" + } + } + } +} + +/** + * Extension for simple native library configuration. + */ +abstract class SimpleNativeLibExtension @Inject constructor(project: Project) { + /** + * Whether this native build is enabled (e.g., platform-specific builds). + */ + val enabled: Property = project.objects.property(Boolean::class.java) + + /** + * Name of the library (without lib prefix or extension). + */ + val libraryName: Property = project.objects.property(String::class.java) + + /** + * Source directory containing C/C++ files. + */ + val sourceDir: DirectoryProperty = project.objects.directoryProperty() + + /** + * Output directory for compiled library. + */ + val outputDir: DirectoryProperty = project.objects.directoryProperty() + + /** + * Object file directory. + */ + val objectDir: DirectoryProperty = project.objects.directoryProperty() + + /** + * Compiler to use (auto-detected if not set). + */ + val compiler: Property = project.objects.property(String::class.java) + + /** + * Linker to use (defaults to compiler if not set). + */ + val linker: Property = project.objects.property(String::class.java) + + /** + * Compiler arguments. + */ + val compilerArgs: ListProperty = project.objects.listProperty(String::class.java) + + /** + * Linker arguments. + */ + val linkerArgs: ListProperty = project.objects.listProperty(String::class.java) + + /** + * Additional include directories. + */ + val includeDirs: ListProperty = project.objects.listProperty(String::class.java) + + /** + * Whether to include JNI headers. + */ + val includeJni: Property = project.objects.property(Boolean::class.java) + + /** + * Whether to create consumable configurations (nativeLib, nativeObjects, etc.). + */ + val createConfigurations: Property = project.objects.property(Boolean::class.java) + + init { + enabled.convention(true) + libraryName.convention("native") + sourceDir.convention(project.layout.projectDirectory.dir("src/main/cpp")) + outputDir.convention(project.layout.buildDirectory.dir("lib")) + objectDir.convention(project.layout.buildDirectory.dir("obj")) + compilerArgs.convention(listOf("-fPIC", "-O3")) + linkerArgs.convention(emptyList()) + includeDirs.convention(emptyList()) + includeJni.convention(false) + createConfigurations.convention(false) + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/config/ConfigurationPresets.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/config/ConfigurationPresets.kt new file mode 100644 index 000000000..13233eb22 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/config/ConfigurationPresets.kt @@ -0,0 +1,323 @@ + +package com.datadoghq.native.config + +import com.datadoghq.native.model.Architecture +import com.datadoghq.native.model.BuildConfiguration +import com.datadoghq.native.model.Platform +import com.datadoghq.native.util.PlatformUtils +import java.io.File + +/** + * Provides factory methods for creating standard build configurations + * (release, debug, asan, tsan, fuzzer) with appropriate compiler and linker flags. + */ +object ConfigurationPresets { + + /** + * Sets up standard build configurations (release, debug, asan, tsan, fuzzer) on the extension. + * This is the shared implementation used by both NativeBuildPlugin and RootProjectPlugin. + * + * @param extension The NativeBuildExtension to configure + * @param project The Gradle project (used for rootDir, logger, and compiler detection) + */ + fun setupStandardConfigurations( + extension: com.datadoghq.native.NativeBuildExtension, + project: org.gradle.api.Project + ) { + if (extension.buildConfigurations.isNotEmpty()) { + return // Don't override explicitly defined configurations + } + + val currentPlatform = PlatformUtils.currentPlatform + val currentArch = PlatformUtils.currentArchitecture + val version = extension.version.get() + val rootDir = project.rootDir + val compiler = PlatformUtils.findCompiler(project) + + project.logger.lifecycle("Setting up standard build configurations for $currentPlatform-$currentArch") + project.logger.lifecycle("Using compiler: $compiler") + + extension.buildConfigurations.apply { + register("release") { + configureRelease(this, currentPlatform, currentArch, version) + } + register("debug") { + configureDebug(this, currentPlatform, currentArch, version) + } + register("asan") { + configureAsan(this, currentPlatform, currentArch, version, rootDir, compiler) + } + register("tsan") { + configureTsan(this, currentPlatform, currentArch, version, rootDir, compiler) + } + register("fuzzer") { + configureFuzzer(this, currentPlatform, currentArch, version, rootDir) + } + } + + val activeConfigs = extension.getActiveConfigurations(currentPlatform, currentArch) + project.logger.lifecycle("Active configurations: ${activeConfigs.map { it.name }.joinToString(", ")}") + } + + private fun commonLinuxCompilerArgs(version: String): List { + val args = mutableListOf( + "-fPIC", + "-fno-omit-frame-pointer", + "-momit-leaf-frame-pointer", + "-fvisibility=hidden", + "-fdata-sections", + "-ffunction-sections", + "-std=c++17", + "-DPROFILER_VERSION=\"$version\"", + "-DCOUNTERS" + ) + // Define __musl__ when building on musl libc (it doesn't define this by default) + if (PlatformUtils.isMusl()) { + args.add("-D__musl__") + } + return args + } + + private fun commonLinuxLinkerArgs(): List = listOf( + "-ldl", + "-Wl,-z,defs", + "--verbose", + "-lpthread", + "-lm", + "-lrt", + "-Wl,--build-id" + ) + + private fun commonMacosCompilerArgs(version: String): List = + commonLinuxCompilerArgs(version) + listOf("-D_XOPEN_SOURCE", "-D_DARWIN_C_SOURCE") + + fun configureRelease( + config: BuildConfiguration, + platform: Platform, + architecture: Architecture, + version: String + ) { + config.platform.set(platform) + config.architecture.set(architecture) + config.active.set(true) + + when (platform) { + Platform.LINUX -> { + config.compilerArgs.set( + listOf("-O3", "-DNDEBUG", "-g") + commonLinuxCompilerArgs(version) + ) + config.linkerArgs.set( + commonLinuxLinkerArgs() + listOf( + "-Wl,-z,nodelete", + "-static-libstdc++", + "-static-libgcc", + "-Wl,--exclude-libs,ALL", + "-Wl,--gc-sections" + ) + ) + } + Platform.MACOS -> { + config.compilerArgs.set( + commonMacosCompilerArgs(version) + listOf("-O3", "-DNDEBUG", "-g") + ) + config.linkerArgs.set(emptyList()) + } + } + } + + fun configureDebug( + config: BuildConfiguration, + platform: Platform, + architecture: Architecture, + version: String + ) { + config.platform.set(platform) + config.architecture.set(architecture) + config.active.set(true) + + when (platform) { + Platform.LINUX -> { + config.compilerArgs.set( + listOf("-O0", "-g", "-DDEBUG") + commonLinuxCompilerArgs(version) + ) + config.linkerArgs.set(commonLinuxLinkerArgs()) + } + Platform.MACOS -> { + config.compilerArgs.set( + commonMacosCompilerArgs(version) + listOf("-O0", "-g", "-DDEBUG") + ) + config.linkerArgs.set(emptyList()) + } + } + } + + fun configureAsan( + config: BuildConfiguration, + platform: Platform, + architecture: Architecture, + version: String, + rootDir: File, + compiler: String = "gcc" + ) { + config.platform.set(platform) + config.architecture.set(architecture) + config.active.set(PlatformUtils.hasAsan(compiler)) + + val asanCompilerArgs = listOf( + "-g", + "-DDEBUG", + "-fPIC", + "-fsanitize=address", + "-fsanitize=undefined", + "-fno-sanitize-recover=all", + "-fsanitize=float-divide-by-zero", + "-fstack-protector-all", + "-fsanitize=leak", + "-fsanitize=pointer-overflow", + "-fsanitize=return", + "-fsanitize=bounds", + "-fsanitize=alignment", + "-fsanitize=object-size", + "-fno-omit-frame-pointer", + "-fno-optimize-sibling-calls" + ) + + when (platform) { + Platform.LINUX -> { + config.compilerArgs.set(asanCompilerArgs + commonLinuxCompilerArgs(version)) + + val libasan = PlatformUtils.locateLibasan(compiler) + val asanLinkerArgs = if (libasan != null) { + listOf( + "-L${File(libasan).parent}", + "-lasan", + "-lubsan", + "-fsanitize=address", + "-fsanitize=undefined", + "-fno-omit-frame-pointer" + ) + } else { + emptyList() + } + + config.linkerArgs.set(commonLinuxLinkerArgs() + asanLinkerArgs) + + if (libasan != null) { + config.testEnvironment.apply { + put("LD_PRELOAD", libasan) + put("ASAN_OPTIONS", "allocator_may_return_null=1:unwind_abort_on_malloc=1:use_sigaltstack=0:detect_stack_use_after_return=0:handle_segv=1:halt_on_error=0:abort_on_error=0:print_stacktrace=1:symbolize=1:suppressions=$rootDir/gradle/sanitizers/asan.supp") + put("UBSAN_OPTIONS", "halt_on_error=0:abort_on_error=0:print_stacktrace=1:suppressions=$rootDir/gradle/sanitizers/ubsan.supp") + put("LSAN_OPTIONS", "detect_leaks=0") + } + } + } + Platform.MACOS -> { + // ASAN not typically configured for macOS in this project + config.active.set(false) + } + } + } + + fun configureTsan( + config: BuildConfiguration, + platform: Platform, + architecture: Architecture, + version: String, + rootDir: File, + compiler: String = "gcc" + ) { + config.platform.set(platform) + config.architecture.set(architecture) + config.active.set(PlatformUtils.hasTsan(compiler)) + + val tsanCompilerArgs = listOf( + "-g", + "-DDEBUG", + "-fPIC", + "-fsanitize=thread", + "-fno-omit-frame-pointer", + "-fno-optimize-sibling-calls" + ) + + when (platform) { + Platform.LINUX -> { + config.compilerArgs.set(tsanCompilerArgs + commonLinuxCompilerArgs(version)) + + val libtsan = PlatformUtils.locateLibtsan(compiler) + val tsanLinkerArgs = if (libtsan != null) { + listOf( + "-L${File(libtsan).parent}", + "-ltsan", + "-fsanitize=thread", + "-fno-omit-frame-pointer" + ) + } else { + emptyList() + } + + config.linkerArgs.set(commonLinuxLinkerArgs() + tsanLinkerArgs) + + if (libtsan != null) { + config.testEnvironment.apply { + put("LD_PRELOAD", libtsan) + put("TSAN_OPTIONS", "suppressions=$rootDir/gradle/sanitizers/tsan.supp") + } + } + } + Platform.MACOS -> { + // TSAN not typically configured for macOS in this project + config.active.set(false) + } + } + } + + fun configureFuzzer( + config: BuildConfiguration, + platform: Platform, + architecture: Architecture, + version: String, + rootDir: File + ) { + config.platform.set(platform) + config.architecture.set(architecture) + config.active.set(PlatformUtils.hasFuzzer()) + + val fuzzerCompilerArgs = listOf( + "-g", + "-DDEBUG", + "-fPIC", + "-fsanitize=address", + "-fsanitize=undefined", + "-fno-sanitize-recover=all", + "-fno-omit-frame-pointer", + "-fno-optimize-sibling-calls" + ) + + val fuzzerLinkerArgs = listOf( + "-fsanitize=address", + "-fsanitize=undefined", + "-fno-omit-frame-pointer" + ) + + when (platform) { + Platform.LINUX -> { + config.compilerArgs.set(fuzzerCompilerArgs + commonLinuxCompilerArgs(version)) + config.linkerArgs.set(commonLinuxLinkerArgs() + fuzzerLinkerArgs) + + config.testEnvironment.apply { + put("ASAN_OPTIONS", "allocator_may_return_null=1:detect_stack_use_after_return=0:handle_segv=0:abort_on_error=1:symbolize=1:suppressions=$rootDir/gradle/sanitizers/asan.supp") + put("UBSAN_OPTIONS", "halt_on_error=1:abort_on_error=1:print_stacktrace=1:suppressions=$rootDir/gradle/sanitizers/ubsan.supp") + } + } + Platform.MACOS -> { + config.compilerArgs.set(fuzzerCompilerArgs + commonMacosCompilerArgs(version)) + config.linkerArgs.set(fuzzerLinkerArgs) + + config.testEnvironment.apply { + put("ASAN_OPTIONS", "allocator_may_return_null=1:detect_stack_use_after_return=0:abort_on_error=1:symbolize=1") + put("UBSAN_OPTIONS", "halt_on_error=1:abort_on_error=1:print_stacktrace=1") + } + } + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/fuzz/FuzzTargetsPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/fuzz/FuzzTargetsPlugin.kt new file mode 100644 index 000000000..b6c5a0b56 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/fuzz/FuzzTargetsPlugin.kt @@ -0,0 +1,313 @@ + +package com.datadoghq.native.fuzz + +import com.datadoghq.native.model.Platform +import com.datadoghq.native.tasks.NativeCompileTask +import com.datadoghq.native.tasks.NativeLinkExecutableTask +import com.datadoghq.native.util.PlatformUtils +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.Exec +import java.io.File +import javax.inject.Inject + +/** + * Plugin for libFuzzer-based fuzz testing. + * + * Automatically discovers fuzz targets (*.cpp files) in a source directory and generates: + * - compileFuzz_{name} - Compile task + * - linkFuzz_{name} - Link task + * - fuzz_{name} - Execute task + * - fuzz - Aggregate task running all fuzz targets + * - listFuzzTargets - Help task + * + * Usage: + * ```kotlin + * plugins { + * id("com.datadoghq.fuzz-targets") + * } + * + * fuzzTargets { + * fuzzSourceDir.set(file("src/test/fuzz")) + * corpusDir.set(file("src/test/fuzz/corpus")) + * profilerSourceDir.set(file("src/main/cpp")) + * duration.set(60) + * } + * ``` + */ +class FuzzTargetsPlugin : Plugin { + override fun apply(project: Project) { + val extension = project.extensions.create( + "fuzzTargets", + FuzzTargetsExtension::class.java, + project + ) + + project.afterEvaluate { + configureFuzzTargets(project, extension) + } + } + + private fun configureFuzzTargets(project: Project, extension: FuzzTargetsExtension) { + val hasFuzzer = PlatformUtils.hasFuzzer() + + // Master fuzz task + val fuzzAll = project.tasks.register("fuzz") { + onlyIf { + hasFuzzer && + !project.hasProperty("skip-tests") && + !project.hasProperty("skip-native") && + !project.hasProperty("skip-fuzz") + } + group = "verification" + description = "Run all fuzz targets" + + doFirst { + if (!hasFuzzer) { + project.logger.warn("WARNING: libFuzzer not available - skipping fuzz tests (requires clang with -fsanitize=fuzzer)") + } + } + } + + if (!hasFuzzer) { + createListFuzzTargetsTask(project, extension) + return + } + + val compiler = PlatformUtils.findFuzzerCompiler(project) + val homebrewLLVM = PlatformUtils.findHomebrewLLVM() + val clangResourceDir = PlatformUtils.findClangResourceDir(homebrewLLVM) + + // Build include paths + val includeFiles = buildIncludePaths(project, extension, homebrewLLVM) + + // Build compiler/linker args + val compilerArgs = buildFuzzCompilerArgs() + val linkerArgs = buildFuzzLinkerArgs(homebrewLLVM, clangResourceDir, project.logger) + + val fuzzSourceDir = extension.fuzzSourceDir.get().asFile + val corpusBaseDir = extension.corpusDir.get().asFile + val crashDir = extension.crashDir.get().asFile + val duration = extension.duration.get() + + // Discover and create tasks for each fuzz target + if (fuzzSourceDir.exists()) { + fuzzSourceDir.listFiles()?.filter { file -> file.name.endsWith(".cpp") }?.forEach { fuzzFile -> + val fullName = fuzzFile.nameWithoutExtension + val fuzzName = if (fullName.startsWith("fuzz_")) fullName.substring(5) else fullName + + val objDir = project.file("${project.layout.buildDirectory.get()}/obj/fuzz/$fuzzName") + val binDir = project.file("${project.layout.buildDirectory.get()}/bin/fuzz/$fuzzName") + val binary = project.file("$binDir/$fuzzName") + val targetCorpusDir = File(corpusBaseDir, fuzzName) + + // Compile task + val compileTask = project.tasks.register("compileFuzz_$fuzzName", NativeCompileTask::class.java) { + onlyIf { hasFuzzer && !project.hasProperty("skip-tests") && !project.hasProperty("skip-native") && !project.hasProperty("skip-fuzz") } + group = "build" + description = "Compile the fuzz target $fuzzName" + + this.compiler.set(compiler) + this.compilerArgs.set(compilerArgs) + sources.from( + extension.profilerSourceDir.map { dir -> + project.fileTree(dir) { include("**/*.cpp") } + }, + fuzzFile + ) + includes.from(includeFiles) + objectFileDir.set(objDir) + } + + // Link task + val linkTask = project.tasks.register("linkFuzz_$fuzzName", NativeLinkExecutableTask::class.java) { + onlyIf { hasFuzzer && !project.hasProperty("skip-tests") && !project.hasProperty("skip-native") && !project.hasProperty("skip-fuzz") } + dependsOn(compileTask) + group = "build" + description = "Link the fuzz target $fuzzName" + + linker.set(compiler) + this.linkerArgs.set(linkerArgs) + objectFiles.from(project.fileTree(objDir) { include("*.o") }) + outputFile.set(binary) + } + + // Execute task + val executeTask = project.tasks.register("fuzz_$fuzzName", Exec::class.java) { + onlyIf { hasFuzzer && !project.hasProperty("skip-tests") && !project.hasProperty("skip-native") && !project.hasProperty("skip-fuzz") } + dependsOn(linkTask) + group = "verification" + description = "Run the fuzz target $fuzzName for $duration seconds" + + doFirst { + crashDir.mkdirs() + targetCorpusDir.mkdirs() + } + + executable = binary.absolutePath + args( + targetCorpusDir.absolutePath, + "-max_total_time=$duration", + "-artifact_prefix=${crashDir.absolutePath}/$fuzzName-", + "-print_final_stats=1" + ) + + inputs.files(binary) + outputs.upToDateWhen { false } + } + + fuzzAll.configure { dependsOn(executeTask) } + } + } + + createListFuzzTargetsTask(project, extension) + } + + private fun buildIncludePaths(project: Project, extension: FuzzTargetsExtension, homebrewLLVM: String?): ConfigurableFileCollection { + val javaHome = PlatformUtils.javaHome() + val platformInclude = when (PlatformUtils.currentPlatform) { + Platform.LINUX -> "linux" + Platform.MACOS -> "darwin" + } + + val includes = project.files() + includes.from( + extension.profilerSourceDir, + "$javaHome/include", + "$javaHome/include/$platformInclude" + ) + + // Add additional include directories + extension.additionalIncludes.get().forEach { dir -> + includes.from(dir) + } + + // Add Homebrew LLVM includes on macOS + if (PlatformUtils.currentPlatform == Platform.MACOS && homebrewLLVM != null) { + includes.from("$homebrewLLVM/include") + } + + return includes + } + + private fun buildFuzzCompilerArgs(): List { + val args = mutableListOf( + "-O1", + "-g", + "-fno-omit-frame-pointer", + "-fsanitize=fuzzer,address,undefined", + "-fvisibility=hidden", + "-std=c++17", + "-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION" + ) + if (PlatformUtils.currentPlatform == Platform.LINUX && PlatformUtils.isMusl()) { + args.add("-D__musl__") + } + return args + } + + private fun buildFuzzLinkerArgs(homebrewLLVM: String?, clangResourceDir: String?, logger: org.gradle.api.logging.Logger): List { + val args = mutableListOf() + + // libFuzzer linking strategy + if (PlatformUtils.currentPlatform == Platform.MACOS && clangResourceDir != null) { + val fuzzerLib = "$clangResourceDir/lib/darwin/libclang_rt.fuzzer_osx.a" + if (File(fuzzerLib).exists()) { + logger.info("Using Homebrew libFuzzer: $fuzzerLib") + args.add(fuzzerLib) + args.add("-L$homebrewLLVM/lib/c++") + args.add("-lc++") + args.add("-Wl,-rpath,$homebrewLLVM/lib/c++") + } else { + logger.warn("Homebrew libFuzzer not found, falling back to -fsanitize=fuzzer") + args.add("-fsanitize=fuzzer,address,undefined") + } + } else { + args.add("-fsanitize=fuzzer,address,undefined") + } + + args.addAll(listOf("-ldl", "-lpthread", "-lm")) + if (PlatformUtils.currentPlatform == Platform.LINUX) { + args.add("-lrt") + } + + return args + } + + private fun createListFuzzTargetsTask(project: Project, extension: FuzzTargetsExtension) { + project.tasks.register("listFuzzTargets") { + group = "help" + description = "List all available fuzz targets" + doLast { + val fuzzSrcDir = extension.fuzzSourceDir.get().asFile + if (fuzzSrcDir.exists()) { + println("Available fuzz targets:") + fuzzSrcDir.listFiles()?.filter { file -> file.name.endsWith(".cpp") }?.forEach { fuzzFile -> + val fullName = fuzzFile.nameWithoutExtension + val fuzzName = if (fullName.startsWith("fuzz_")) fullName.substring(5) else fullName + println(" - fuzz_$fuzzName") + } + println() + println("Run individual targets with: ./gradlew :${project.path}:fuzz_") + println("Run all targets with: ./gradlew :${project.path}:fuzz") + println("Configure duration with: -Pfuzz-duration= (default: ${extension.duration.get()})") + } else { + println("No fuzz targets found. Create .cpp files in ${fuzzSrcDir.path}") + } + } + } + } +} + +/** + * Extension for configuring fuzz targets. + */ +abstract class FuzzTargetsExtension @Inject constructor(project: Project) { + /** + * Directory containing fuzz target source files (*.cpp). + */ + val fuzzSourceDir: DirectoryProperty = project.objects.directoryProperty() + + /** + * Directory for seed corpus files. + */ + val corpusDir: DirectoryProperty = project.objects.directoryProperty() + + /** + * Directory for crash artifacts. + */ + val crashDir: DirectoryProperty = project.objects.directoryProperty() + + /** + * Directory containing the profiler C++ sources to compile with fuzz targets. + */ + val profilerSourceDir: DirectoryProperty = project.objects.directoryProperty() + + /** + * Additional include directories. + */ + val additionalIncludes: ListProperty = project.objects.listProperty(String::class.java) + + /** + * Fuzz duration in seconds. + */ + val duration: Property = project.objects.property(Int::class.java) + + init { + // Set reasonable defaults - these should be overridden by the user + crashDir.convention(project.layout.buildDirectory.dir("fuzz-crashes")) + additionalIncludes.convention(emptyList()) + + // Duration from property or default + val propDuration = if (project.hasProperty("fuzz-duration")) { + project.property("fuzz-duration").toString().toIntOrNull() ?: 60 + } else { + 60 + } + duration.convention(propDuration) + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestExtension.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestExtension.kt new file mode 100644 index 000000000..0767b576b --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestExtension.kt @@ -0,0 +1,110 @@ + +package com.datadoghq.native.gtest + +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.model.ObjectFactory +import org.gradle.api.provider.Property +import javax.inject.Inject + +/** + * Extension for configuring Google Test integration in C++ projects. + * + * Provides a declarative DSL for setting up Google Test compilation, linking, and execution + * across multiple build configurations (debug, release, asan, tsan). + * + * Usage example: + * ```kotlin + * gtest { + * testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) + * mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) + * includes.from("src/main/cpp", "${javaHome}/include") + * } + * ``` + */ +abstract class GtestExtension @Inject constructor(objects: ObjectFactory) { + + // === Source Directories === + + /** + * Directory containing test source files (.cpp). + * Required - must be set explicitly. + */ + abstract val testSourceDir: DirectoryProperty + + /** + * Directory containing main source files to compile with tests. + * Required - must be set explicitly. + */ + abstract val mainSourceDir: DirectoryProperty + + /** + * Optional Google Test installation directory. + * Used for include and library paths on macOS. + * Default: /opt/homebrew/opt/googletest on macOS + */ + abstract val googleTestHome: DirectoryProperty + + // === Compiler/Linker Configuration === + + /** + * Include directories for compilation. + * Should include main source, JNI headers, and any dependencies. + */ + abstract val includes: ConfigurableFileCollection + + // === Test Behavior === + + /** + * Enable assertions by removing -DNDEBUG from compiler args. + * Default: true + */ + abstract val enableAssertions: Property + + /** + * Keep symbols in release builds (skip minimizing linker flags). + * Default: true + */ + abstract val keepSymbols: Property + + /** + * Stop on first test failure (fail-fast). + * Default: false (collect all failures) + */ + abstract val failFast: Property + + /** + * Always re-run tests (ignore up-to-date checks). + * Default: true + */ + abstract val alwaysRun: Property + + // === Build Native Libs Task === + + /** + * Enable building native test support libraries (Linux only). + * Default: true + */ + abstract val buildNativeLibs: Property + + /** + * Directory containing native test library sources. + * Default: src/test/resources/native-libs + */ + abstract val nativeLibsSourceDir: DirectoryProperty + + /** + * Output directory for built native test libraries. + * Default: build/test/resources/native-libs + */ + abstract val nativeLibsOutputDir: DirectoryProperty + + init { + // Set default conventions + enableAssertions.convention(true) + keepSymbols.convention(true) + failFast.convention(false) + alwaysRun.convention(true) + buildNativeLibs.convention(true) + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestPlugin.kt new file mode 100644 index 000000000..a58ff29c2 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestPlugin.kt @@ -0,0 +1,227 @@ + +package com.datadoghq.native.gtest + +import com.datadoghq.native.NativeBuildExtension +import com.datadoghq.native.model.BuildConfiguration +import com.datadoghq.native.model.Platform +import com.datadoghq.native.util.PlatformUtils +import org.gradle.api.Plugin +import org.gradle.api.Project +import java.io.File + +/** + * Gradle plugin for Google Test integration in C++ projects. + * + * This plugin automatically creates compilation, linking, and execution tasks for Google Test + * tests across multiple build configurations. It handles platform-specific differences (macOS, Linux) + * and integrates with the NativeBuildPlugin's BuildConfiguration model. + * + * For each test file in the test source directory, the plugin creates: + * - compileGtest{Config}_{TestName} - Compile all main sources + test file + * - linkGtest{Config}_{TestName} - Link test executable with gtest libraries + * - gtest{Config}_{TestName} - Execute the test + * + * Aggregation tasks: + * - gtest{Config} - Run all tests for a specific configuration (e.g., gtestDebug) + * - gtest - Run all tests across all configurations + * + * Usage: + * ```kotlin + * plugins { + * id("com.datadoghq.native-build") + * id("com.datadoghq.gtest") + * } + * + * gtest { + * testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) + * mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) + * includes.from("src/main/cpp", "${javaHome}/include") + * } + * ``` + */ +class GtestPlugin : Plugin { + + override fun apply(project: Project) { + // Create extension + val extension = project.extensions.create("gtest", GtestExtension::class.java) + + // Register tasks after project evaluation + project.afterEvaluate { + // Check if testSourceDir is set + if (!extension.testSourceDir.isPresent) { + project.logger.warn("WARNING: gtest.testSourceDir not configured - skipping Google Test tasks") + return@afterEvaluate + } + + // Get configurations from NativeBuildExtension + val nativeBuildExtension = project.extensions.findByType(NativeBuildExtension::class.java) + if (nativeBuildExtension == null) { + project.logger.warn("WARNING: NativeBuildExtension not found - apply com.datadoghq.native-build plugin first") + return@afterEvaluate + } + + val activeConfigs = nativeBuildExtension.getActiveConfigurations( + PlatformUtils.currentPlatform, + PlatformUtils.currentArchitecture + ) + + if (activeConfigs.isEmpty()) { + project.logger.warn("WARNING: No active build configurations - skipping Google Test tasks") + return@afterEvaluate + } + + // Check if gtest is available + val hasGtest = checkGtestAvailable() + if (!hasGtest) { + project.logger.warn("WARNING: Google Test not found - skipping native tests") + } + + // Create buildNativeLibs task (Linux only) + if (extension.buildNativeLibs.get()) { + createBuildNativeLibsTask(project, extension, hasGtest) + } + + // Create master aggregation task + val gtestAll = createMasterAggregationTask(project, hasGtest) + + // Create tasks for each active configuration + activeConfigs.forEach { config -> + createConfigTasks(project, extension, config, hasGtest, gtestAll) + } + } + } + + private fun checkGtestAvailable(): Boolean { + // Check common gtest locations + val locations = when (PlatformUtils.currentPlatform) { + Platform.MACOS -> listOf( + "/opt/homebrew/opt/googletest", + "/usr/local/opt/googletest" + ) + Platform.LINUX -> listOf( + "/usr/include/gtest", + "/usr/local/include/gtest" + ) + } + return locations.any { File(it).exists() } + } + + private fun createBuildNativeLibsTask(project: Project, extension: GtestExtension, hasGtest: Boolean) { + project.tasks.register("buildNativeLibs") { + group = "build" + description = "Build the native libs for the Google Tests" + + onlyIf { + hasGtest && + !project.hasProperty("skip-native") && + !project.hasProperty("skip-gtest") && + PlatformUtils.currentPlatform == Platform.LINUX && + extension.nativeLibsSourceDir.isPresent && + extension.nativeLibsOutputDir.isPresent + } + + val srcDir = if (extension.nativeLibsSourceDir.isPresent) { + extension.nativeLibsSourceDir.get().asFile + } else { + project.file("src/test/resources/native-libs") + } + val targetDir = if (extension.nativeLibsOutputDir.isPresent) { + extension.nativeLibsOutputDir.get().asFile + } else { + project.file("build/test/resources/native-libs") + } + + doLast { + if (!srcDir.exists()) { + project.logger.info("Native libs source directory does not exist: $srcDir") + return@doLast + } + + srcDir.listFiles()?.filter { it.isDirectory }?.forEach { dir -> + val libName = dir.name + val libDir = File("$targetDir/$libName") + val libSrcDir = File("$srcDir/$libName") + + project.exec { + commandLine("sh", "-c", """ + echo "Processing library: $libName @ $libSrcDir" + mkdir -p $libDir + cd $libSrcDir + make TARGET_DIR=$libDir + """.trimIndent()) + } + } + } + + inputs.files(project.fileTree(srcDir)) + outputs.dir(targetDir) + } + } + + private fun createMasterAggregationTask(project: Project, hasGtest: Boolean): org.gradle.api.tasks.TaskProvider<*> { + return project.tasks.register("gtest") { + group = "verification" + description = "Run all Google Tests for all build configurations of the library" + + onlyIf { + hasGtest && + !project.hasProperty("skip-tests") && + !project.hasProperty("skip-native") && + !project.hasProperty("skip-gtest") + } + } + } + + private fun createConfigTasks( + project: Project, + extension: GtestExtension, + config: BuildConfiguration, + hasGtest: Boolean, + gtestAll: org.gradle.api.tasks.TaskProvider<*> + ) { + // Find compiler and build include paths + val compiler = findCompiler(project) + val includeFiles = extension.includes.plus(project.files(getGtestIncludes(extension))) + + // Create per-config aggregation task + val gtestConfigTask = project.tasks.register("gtest${config.capitalizedName()}") { + group = "verification" + description = "Run all Google Tests for the ${config.name} build of the library" + } + + // Discover and create tasks for each test file using builder + val testDir = extension.testSourceDir.get().asFile + if (!testDir.exists()) { + project.logger.info("Test source directory does not exist: $testDir") + return + } + + testDir.listFiles()?.filter { it.name.endsWith(".cpp") }?.forEach { testFile -> + val executeTask = GtestTaskBuilder(project, extension, config) + .forTest(testFile) + .withCompiler(compiler) + .withIncludes(includeFiles) + .onlyIfGtest(hasGtest) + .build() + + gtestConfigTask.configure { dependsOn(executeTask) } + gtestAll.configure { dependsOn(executeTask) } + } + } + + private fun findCompiler(project: Project): String = PlatformUtils.findCompiler(project) + + private fun getGtestIncludes(extension: GtestExtension): List { + return when (PlatformUtils.currentPlatform) { + Platform.MACOS -> { + val gtestPath = if (extension.googleTestHome.isPresent) { + extension.googleTestHome.get().asFile.absolutePath + } else { + "/opt/homebrew/opt/googletest" + } + listOf(File("$gtestPath/include")) + } + Platform.LINUX -> emptyList() // System includes + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestTaskBuilder.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestTaskBuilder.kt new file mode 100644 index 000000000..3e6d99ece --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestTaskBuilder.kt @@ -0,0 +1,206 @@ + +package com.datadoghq.native.gtest + +import com.datadoghq.native.model.BuildConfiguration +import com.datadoghq.native.model.Platform +import com.datadoghq.native.tasks.NativeCompileTask +import com.datadoghq.native.tasks.NativeLinkExecutableTask +import com.datadoghq.native.util.PlatformUtils +import org.gradle.api.Project +import org.gradle.api.file.FileCollection +import org.gradle.api.tasks.Exec +import org.gradle.api.tasks.TaskProvider +import java.io.File + +/** + * Builder for creating Google Test compile, link, and execute tasks. + * + * Groups related configuration and provides a fluent API for task creation. + * + * Usage: + * ```kotlin + * GtestTaskBuilder(project, extension, config) + * .forTest(testFile) + * .withCompiler(compiler) + * .withIncludes(includeFiles) + * .onlyIf { hasGtest } + * .build() + * ``` + */ +class GtestTaskBuilder( + private val project: Project, + private val extension: GtestExtension, + private val config: BuildConfiguration +) { + private lateinit var testFile: File + private lateinit var testName: String + private lateinit var compiler: String + private lateinit var includeFiles: FileCollection + private var hasGtest: Boolean = true + + private val configName: String get() = config.capitalizedName() + + /** + * Set the test file to build tasks for. + */ + fun forTest(file: File): GtestTaskBuilder { + testFile = file + testName = file.nameWithoutExtension + return this + } + + /** + * Set the compiler to use. + */ + fun withCompiler(comp: String): GtestTaskBuilder { + compiler = comp + return this + } + + /** + * Set include directories. + */ + fun withIncludes(includes: FileCollection): GtestTaskBuilder { + includeFiles = includes + return this + } + + /** + * Set whether gtest is available. + */ + fun onlyIfGtest(available: Boolean): GtestTaskBuilder { + hasGtest = available + return this + } + + /** + * Build all tasks (compile, link, execute) and return the execute task provider. + */ + fun build(): TaskProvider { + val compileTask = buildCompileTask() + val linkTask = buildLinkTask(compileTask) + return buildExecuteTask(linkTask) + } + + private fun buildCompileTask(): TaskProvider { + val compilerArgs = adjustCompilerArgs() + val objDir = project.file("${project.layout.buildDirectory.get()}/obj/gtest/${config.name}/$testName") + + return project.tasks.register("compileGtest${configName}_$testName", NativeCompileTask::class.java) { + onlyIf { hasGtest && !skipConditions() } + group = "build" + description = "Compile the Google Test $testName for the ${config.name} build" + + this.compiler.set(this@GtestTaskBuilder.compiler) + this.compilerArgs.set(compilerArgs) + + sources.from( + project.fileTree(extension.mainSourceDir.get()) { include("**/*.cpp") }, + testFile + ) + includes.from(includeFiles) + objectFileDir.set(objDir) + } + } + + private fun buildLinkTask(compileTask: TaskProvider): TaskProvider { + val linkerArgs = config.linkerArgs.get() + val objDir = project.file("${project.layout.buildDirectory.get()}/obj/gtest/${config.name}/$testName") + val binary = project.file("${project.layout.buildDirectory.get()}/bin/gtest/${config.name}_$testName/$testName") + + return project.tasks.register("linkGtest${configName}_$testName", NativeLinkExecutableTask::class.java) { + onlyIf { hasGtest && !skipConditions() } + dependsOn(compileTask) + group = "build" + description = "Link the Google Test $testName for the ${config.name} build" + + linker.set(compiler) + this.linkerArgs.set(linkerArgs) + objectFiles.from(project.fileTree(objDir) { include("*.o") }) + outputFile.set(binary) + + // Add gtest library paths + when (PlatformUtils.currentPlatform) { + Platform.MACOS -> { + val gtestPath = gtestHomePath() + libPath("$gtestPath/lib") + } + Platform.LINUX -> { /* System paths */ } + } + + // Add gtest libraries + lib("gtest", "gtest_main", "gmock", "gmock_main", "dl", "pthread", "m") + if (PlatformUtils.currentPlatform == Platform.LINUX) { + lib("rt") + } + } + } + + private fun buildExecuteTask(linkTask: TaskProvider): TaskProvider { + val binary = project.file("${project.layout.buildDirectory.get()}/bin/gtest/${config.name}_$testName/$testName") + + return project.tasks.register("gtest${configName}_$testName", Exec::class.java) { + onlyIf { hasGtest && !skipConditions() } + dependsOn(linkTask) + + // Add dependency on buildNativeLibs if it exists (Linux only) + if (PlatformUtils.currentPlatform == Platform.LINUX && extension.buildNativeLibs.get()) { + project.tasks.findByName("buildNativeLibs")?.let { dependsOn(it) } + } + + group = "verification" + description = "Run the Google Test $testName for the ${config.name} build" + + executable = binary.absolutePath + + // Set test environment variables from configuration + config.testEnvironment.get().forEach { (key, value) -> + environment(key, value) + } + + inputs.files(binary) + + if (extension.alwaysRun.get()) { + outputs.upToDateWhen { false } + } + + isIgnoreExitValue = !extension.failFast.get() + } + } + + private fun skipConditions(): Boolean { + return project.hasProperty("skip-tests") || + project.hasProperty("skip-native") || + project.hasProperty("skip-gtest") + } + + private fun gtestHomePath(): String { + return if (extension.googleTestHome.isPresent) { + extension.googleTestHome.get().asFile.absolutePath + } else { + "/opt/homebrew/opt/googletest" + } + } + + private fun adjustCompilerArgs(): List { + val args = config.compilerArgs.get().toMutableList() + + // Remove -std= so we can re-add it consistently + args.removeAll { it.startsWith("-std=") } + + // Remove -DNDEBUG if assertions are enabled + if (extension.enableAssertions.get()) { + args.removeAll { it == "-DNDEBUG" } + } + + // Re-add C++17 standard + args.add("-std=c++17") + + // Add musl define if needed + if (PlatformUtils.currentPlatform == Platform.LINUX && PlatformUtils.isMusl()) { + args.add("-D__musl__") + } + + return args + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Architecture.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Architecture.kt new file mode 100644 index 000000000..1bab5b103 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Architecture.kt @@ -0,0 +1,29 @@ + +package com.datadoghq.native.model + +enum class Architecture { + X64, + ARM64, + X86, + ARM; + + override fun toString(): String = when (this) { + X64 -> "x64" + ARM64 -> "arm64" + X86 -> "x86" + ARM -> "arm" + } + + companion object { + fun current(): Architecture { + val osArch = System.getProperty("os.arch").lowercase() + return when { + osArch.contains("amd64") || osArch.contains("x86_64") -> X64 + osArch.contains("aarch64") || osArch.contains("arm64") -> ARM64 + osArch.contains("x86") || osArch.contains("i386") -> X86 + osArch.contains("arm") -> ARM + else -> throw IllegalStateException("Unsupported architecture: $osArch") + } + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/BuildConfiguration.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/BuildConfiguration.kt new file mode 100644 index 000000000..aa7a5c3c7 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/BuildConfiguration.kt @@ -0,0 +1,56 @@ + +package com.datadoghq.native.model + +import org.gradle.api.Named +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.MapProperty +import org.gradle.api.provider.Property +import javax.inject.Inject + +abstract class BuildConfiguration @Inject constructor( + private val configName: String +) : Named { + abstract val platform: Property + abstract val architecture: Property + abstract val compilerArgs: ListProperty + abstract val linkerArgs: ListProperty + abstract val testEnvironment: MapProperty + abstract val active: Property + + override fun getName(): String = configName + + init { + // Default to active unless overridden + active.convention(true) + testEnvironment.convention(emptyMap()) + } + + fun isActiveFor(targetPlatform: Platform, targetArch: Architecture): Boolean { + return active.get() && + platform.get() == targetPlatform && + architecture.get() == targetArch + } + + /** + * Returns a unique identifier for this configuration combining name, platform, and architecture. + * Example: "releaseLinuxX64" + */ + fun identifier(): String { + val platformStr = platform.get().toString() + val archStr = architecture.get().toString() + return "$configName${platformStr.replaceFirstChar { it.titlecase() }}${archStr.replaceFirstChar { it.titlecase() }}" + } + + /** + * Returns the capitalized name for task generation. + * Example: "Release" for name "release" + */ + fun capitalizedName(): String = configName.replaceFirstChar { it.titlecase() } + + // Task name helpers for consistent naming across plugins + val compileTaskName: String get() = "compile${capitalizedName()}" + val linkTaskName: String get() = "link${capitalizedName()}" + val assembleTaskName: String get() = "assemble${capitalizedName()}" + val testTaskName: String get() = "test${capitalizedName()}" + val gtestTaskName: String get() = "gtest${capitalizedName()}" +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/ErrorHandlingMode.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/ErrorHandlingMode.kt new file mode 100644 index 000000000..04a0cad05 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/ErrorHandlingMode.kt @@ -0,0 +1,13 @@ + +package com.datadoghq.native.model + +/** + * Error handling strategy for compilation. + */ +enum class ErrorHandlingMode { + /** Stop on first error (default) */ + FAIL_FAST, + + /** Compile all files, collect all errors, report at end */ + COLLECT_ALL +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/LogLevel.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/LogLevel.kt new file mode 100644 index 000000000..3797cbc7d --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/LogLevel.kt @@ -0,0 +1,19 @@ + +package com.datadoghq.native.model + +/** + * Logging verbosity level for native build tasks. + */ +enum class LogLevel { + /** Only errors */ + QUIET, + + /** Standard lifecycle messages (default) */ + NORMAL, + + /** Detailed progress information */ + VERBOSE, + + /** Full command lines and output */ + DEBUG +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Platform.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Platform.kt new file mode 100644 index 000000000..861e9637b --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Platform.kt @@ -0,0 +1,20 @@ + +package com.datadoghq.native.model + +enum class Platform { + LINUX, + MACOS; + + override fun toString(): String = name.lowercase() + + companion object { + fun current(): Platform { + val osName = System.getProperty("os.name").lowercase() + return when { + osName.contains("mac") || osName.contains("darwin") -> MACOS + osName.contains("linux") -> LINUX + else -> throw IllegalStateException("Unsupported OS: $osName") + } + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/SourceSet.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/SourceSet.kt new file mode 100644 index 000000000..4325265ef --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/SourceSet.kt @@ -0,0 +1,93 @@ + +package com.datadoghq.native.model + +import org.gradle.api.Named +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.provider.ListProperty +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.Internal +import org.gradle.api.tasks.Optional +import javax.inject.Inject + +/** + * Represents a named set of source files with optional per-set compiler flags. + * Allows different parts of the codebase to have different compilation settings. + * + * Example usage: + * sourceSets { main { sources.from(fileTree("src/main/cpp")) } } + */ +abstract class SourceSet @Inject constructor( + private val name: String +) : Named { + + /** + * Source files for this source set. + */ + @get:InputFiles + abstract val sources: ConfigurableFileCollection + + /** + * Include patterns for filtering source files (Ant-style). + * Default: all C++ source files (.cpp, .c, .cc) + */ + @get:Input + @get:Optional + abstract val includes: ListProperty + + /** + * Exclude patterns for filtering source files (Ant-style). + */ + @get:Input + @get:Optional + abstract val excludes: ListProperty + + /** + * Additional compiler arguments specific to this source set. + * These are added to the base compiler arguments. + */ + @get:Input + @get:Optional + abstract val compilerArgs: ListProperty + + init { + includes.convention(listOf("**/*.cpp", "**/*.c", "**/*.cc")) + excludes.convention(emptyList()) + compilerArgs.convention(emptyList()) + } + + @Internal + override fun getName(): String = name + + /** + * Convenience method to set source directory. + */ + fun from(vararg sources: Any) { + this.sources.from(*sources) + } + + /** + * Convenience method to add include patterns. + */ + fun include(vararg patterns: String) { + includes.addAll(*patterns) + } + + /** + * Convenience method to add exclude patterns. + */ + fun exclude(vararg patterns: String) { + excludes.addAll(*patterns) + } + + /** + * Convenience method to add compiler args. + */ + fun compileWith(vararg args: String) { + compilerArgs.addAll(*args) + } + + override fun toString(): String { + return "SourceSet[name=$name, sources=${sources.files.size} files]" + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildExtension.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildExtension.kt new file mode 100644 index 000000000..a7808165f --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildExtension.kt @@ -0,0 +1,51 @@ + +package com.datadoghq.native.scanbuild + +import org.gradle.api.Project +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property +import javax.inject.Inject + +/** + * Extension for configuring the scan-build static analysis task. + */ +abstract class ScanBuildExtension @Inject constructor(project: Project) { + /** + * Directory containing the Makefile for scan-build. + * Default: src/test/make + */ + abstract val makefileDir: DirectoryProperty + + /** + * Output directory for scan-build reports. + * Default: build/reports/scan-build + */ + abstract val outputDir: DirectoryProperty + + /** + * Path to the clang analyzer to use. + * Default: /usr/bin/clang++ + */ + abstract val analyzer: Property + + /** + * Number of parallel jobs for make. + * Default: 4 + */ + abstract val parallelJobs: Property + + /** + * Make targets to build. + * Default: ["all"] + */ + abstract val makeTargets: ListProperty + + init { + makefileDir.convention(project.layout.projectDirectory.dir("src/test/make")) + outputDir.convention(project.layout.buildDirectory.dir("reports/scan-build")) + analyzer.convention("/usr/bin/clang++") + parallelJobs.convention(4) + makeTargets.convention(listOf("all")) + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildPlugin.kt new file mode 100644 index 000000000..aff275b52 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildPlugin.kt @@ -0,0 +1,98 @@ + +package com.datadoghq.native.scanbuild + +import com.datadoghq.native.model.Platform +import com.datadoghq.native.util.PlatformUtils +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.Exec + +/** + * Gradle plugin that provides clang static analysis via scan-build. + * + * This plugin creates a `scanBuild` task that runs the clang static analyzer + * on the C++ codebase using a Makefile-based build. + * + * Usage: + * ```kotlin + * plugins { + * id("com.datadoghq.scanbuild") + * } + * + * scanBuild { + * makefileDir.set(layout.projectDirectory.dir("src/test/make")) + * outputDir.set(layout.buildDirectory.dir("reports/scan-build")) + * analyzer.set("/usr/bin/clang++") + * parallelJobs.set(4) + * } + * ``` + */ +class ScanBuildPlugin : Plugin { + override fun apply(project: Project) { + // Create the extension + val extension = project.extensions.create( + "scanBuild", + ScanBuildExtension::class.java, + project + ) + + // Create the task after project evaluation + project.afterEvaluate { + createScanBuildTask(project, extension) + } + } + + private fun createScanBuildTask(project: Project, extension: ScanBuildExtension) { + // Only create the task on Linux (scan-build is typically Linux-only in CI) + if (PlatformUtils.currentPlatform != Platform.LINUX) { + project.logger.info("Skipping scanBuild task - only available on Linux") + return + } + + // Check if scan-build is available + if (!isScanBuildAvailable()) { + project.logger.warn("scan-build not found in PATH - scanBuild task will fail if executed") + } + + val makefileDir = extension.makefileDir.get().asFile + val outputDir = extension.outputDir.get().asFile + val analyzer = extension.analyzer.get() + val parallelJobs = extension.parallelJobs.get() + val makeTargets = extension.makeTargets.get() + + val scanBuildTask = project.tasks.register("scanBuild", Exec::class.java) + scanBuildTask.configure { + group = "verification" + description = "Run clang static analyzer via scan-build" + + workingDir(makefileDir) + + // Build command line as a single list to avoid vararg ambiguity + val command = mutableListOf( + "scan-build", + "-o", outputDir.absolutePath, + "--force-analyze-debug-code", + "--use-analyzer", analyzer, + "make", "-j$parallelJobs" + ) + command.addAll(makeTargets) + commandLine(command) + + // Ensure output directory exists + doFirst { + outputDir.mkdirs() + } + } + } + + private fun isScanBuildAvailable(): Boolean { + return try { + val process = ProcessBuilder("which", "scan-build") + .redirectErrorStream(true) + .start() + process.waitFor() == 0 + } catch (e: Exception) { + false + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeCompileTask.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeCompileTask.kt new file mode 100644 index 000000000..4a9ec0e75 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeCompileTask.kt @@ -0,0 +1,473 @@ + +package com.datadoghq.native.tasks + +import com.datadoghq.native.model.ErrorHandlingMode +import com.datadoghq.native.model.LogLevel +import com.datadoghq.native.model.SourceSet +import org.gradle.api.DefaultTask +import org.gradle.api.NamedDomainObjectContainer +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.model.ObjectFactory +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.* +import org.gradle.process.ExecOperations +import java.io.ByteArrayOutputStream +import java.io.File +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.atomic.AtomicInteger +import javax.inject.Inject + +/** + * Kotlin-based C++ compilation task that directly invokes gcc/clang. + * + * Supports both simple mode (single sources collection) and source sets mode + * (multiple source collections with per-set compiler flags). + */ +abstract class NativeCompileTask @Inject constructor( + private val execOperations: ExecOperations, + private val objects: ObjectFactory +) : DefaultTask() { + + /** + * The C++ compiler executable (e.g., 'g++', 'clang++', or full path). + */ + @get:Input + abstract val compiler: Property + + /** + * Compiler arguments (flags) to pass to the compiler. + */ + @get:Input + abstract val compilerArgs: ListProperty + + /** + * The C++ source files to compile. + */ + @get:InputFiles + @get:SkipWhenEmpty + abstract val sources: ConfigurableFileCollection + + /** + * Include directories for header file lookup. + */ + @get:InputFiles + @get:Optional + abstract val includes: ConfigurableFileCollection + + /** + * Output directory for object files. + */ + @get:OutputDirectory + abstract val objectFileDir: DirectoryProperty + + /** + * Number of parallel compilation jobs. + */ + @get:Input + @get:Optional + abstract val parallelJobs: Property + + /** + * Show detailed compilation output. + */ + @get:Input + @get:Optional + abstract val verbose: Property + + /** + * Source sets for per-directory compiler flags. + * When used, the simple 'sources' property is ignored. + */ + @get:Nested + @get:Optional + val sourceSets: NamedDomainObjectContainer = objects.domainObjectContainer(SourceSet::class.java) + + // === Logging and Verbosity === + + /** + * Logging verbosity level. + * Default: NORMAL + */ + @get:Input + @get:Optional + abstract val logLevel: Property + + /** + * Progress reporting interval (log every N files during compilation). + * Default: 10 + */ + @get:Input + @get:Optional + abstract val progressReportInterval: Property + + /** + * Show full command line for each file compilation. + * Default: false (only shown at DEBUG level) + */ + @get:Input + @get:Optional + abstract val showCommandLines: Property + + /** + * Enable ANSI color codes in output. + * Default: true + */ + @get:Input + @get:Optional + abstract val colorOutput: Property + + // === Error Handling === + + /** + * Error handling mode. + * FAIL_FAST: Stop on first compilation error (default) + * COLLECT_ALL: Compile all files, collect errors, report at end + */ + @get:Input + @get:Optional + abstract val errorHandling: Property + + /** + * Maximum number of errors to show when using COLLECT_ALL mode. + * Default: 10 + */ + @get:Input + @get:Optional + abstract val maxErrorsToShow: Property + + /** + * Treat compiler warnings as errors (-Werror). + * Default: false + */ + @get:Input + @get:Optional + abstract val treatWarningsAsErrors: Property + + // === Convenience Properties === + + /** + * Compiler defines (-D flags). + * Use define() method to add: define("DEBUG", "VERSION=\"1.0\"") + */ + @get:Input + @get:Optional + abstract val defines: ListProperty + + /** + * Compiler undefines (-U flags). + * Use undefine() method to add: undefine("NDEBUG") + */ + @get:Input + @get:Optional + abstract val undefines: ListProperty + + /** + * C++ standard version (e.g., "c++17", "c++20"). + * Use standard() method to set: standard("c++20") + */ + @get:Input + @get:Optional + abstract val standardVersion: Property + + init { + parallelJobs.convention(Runtime.getRuntime().availableProcessors()) + verbose.convention(false) + logLevel.convention(LogLevel.NORMAL) + progressReportInterval.convention(10) + showCommandLines.convention(false) + colorOutput.convention(true) + errorHandling.convention(ErrorHandlingMode.FAIL_FAST) + maxErrorsToShow.convention(10) + treatWarningsAsErrors.convention(false) + defines.convention(emptyList()) + undefines.convention(emptyList()) + group = "build" + description = "Compiles C++ source files" + } + + /** + * Configure source sets using a DSL block. + */ + fun sourceSets(action: org.gradle.api.Action>) { + action.execute(sourceSets) + } + + // === Convenience Methods === + + /** + * Add compiler defines (-D flags). + * Example: define("DEBUG", "VERSION=\"1.0\"") + */ + fun define(vararg defs: String) { + defines.addAll(*defs) + } + + /** + * Add compiler undefines (-U flags). + * Example: undefine("NDEBUG", "DEBUG") + */ + fun undefine(vararg undefs: String) { + undefines.addAll(*undefs) + } + + /** + * Set C++ standard version. + * Example: standard("c++20") generates -std=c++20 + */ + fun standard(version: String) { + standardVersion.set(version) + } + + // === Logging Helpers === + + private fun logNormal(message: String) { + if (logLevel.get() >= LogLevel.NORMAL) { + logger.lifecycle(message) + } + } + + private fun logVerbose(message: String) { + if (logLevel.get() >= LogLevel.VERBOSE) { + logger.lifecycle(message) + } + } + + private fun logDebug(message: String) { + if (logLevel.get() == LogLevel.DEBUG) { + logger.lifecycle(message) + } + } + + private fun shouldShowCommandLine(): Boolean { + return showCommandLines.get() || logLevel.get() == LogLevel.DEBUG + } + + @TaskAction + fun compile() { + val objDir = objectFileDir.get().asFile + objDir.mkdirs() + + // Build base compiler arguments with convenience properties + val baseArgs = compilerArgs.get().toMutableList() + + // Add C++ standard if specified + if (standardVersion.isPresent) { + baseArgs.add("-std=${standardVersion.get()}") + } + + // Add defines (-D) + defines.get().forEach { define -> + baseArgs.add("-D$define") + } + + // Add undefines (-U) + undefines.get().forEach { undefine -> + baseArgs.add("-U$undefine") + } + + // Add -Werror if warnings should be treated as errors + if (treatWarningsAsErrors.get()) { + baseArgs.add("-Werror") + } + + // Build include arguments + val includeArgs = mutableListOf() + includes.files.forEach { dir -> + if (dir.exists()) { + includeArgs.add("-I") + includeArgs.add(dir.absolutePath) + } + } + + val errors = ConcurrentLinkedQueue() + val compiled = AtomicInteger(0) + + // Choose compilation mode: source sets or simple sources + if (sourceSets.isEmpty()) { + // Simple mode: compile all sources with base args + compileSimpleMode(objDir, baseArgs, includeArgs, compiled, errors) + } else { + // Source sets mode: compile each set with merged args + compileSourceSetsMode(objDir, baseArgs, includeArgs, compiled, errors) + } + + // Report errors if any + if (errors.isNotEmpty()) { + val maxErrors = maxErrorsToShow.get() + val errorMsg = buildString { + appendLine("Compilation failed with ${errors.size} error(s):") + errors.take(maxErrors).forEach { error -> + appendLine(" - $error") + } + if (errors.size > maxErrors) { + appendLine(" ... and ${errors.size - maxErrors} more error(s)") + } + } + throw RuntimeException(errorMsg) + } + + logNormal("Successfully compiled ${compiled.get()} file${if (compiled.get() == 1) "" else "s"}") + } + + private fun compileSimpleMode( + objDir: File, + baseArgs: List, + includeArgs: List, + compiled: AtomicInteger, + errors: ConcurrentLinkedQueue + ) { + val sourceFiles = sources.files.toList() + if (sourceFiles.isEmpty()) { + logNormal("No source files to compile") + return + } + + val total = sourceFiles.size + logNormal("Compiling $total C++ source file${if (total == 1) "" else "s"} with ${compiler.get()}...") + + // Compile files in parallel (or sequentially for FAIL_FAST) + if (errorHandling.get() == ErrorHandlingMode.FAIL_FAST) { + // Use sequential stream for FAIL_FAST to ensure immediate termination on error + sourceFiles.stream().forEach { sourceFile -> + try { + compileFile(sourceFile, objDir, baseArgs, includeArgs, compiled, total, errors) + } catch (e: Exception) { + errors.add("Exception compiling ${sourceFile.name}: ${e.message}") + throw e // Re-throw to stop compilation immediately in FAIL_FAST mode + } + } + } else { + // Use parallel stream for COLLECT_ALL mode + sourceFiles.parallelStream().forEach { sourceFile -> + try { + compileFile(sourceFile, objDir, baseArgs, includeArgs, compiled, total, errors) + } catch (e: Exception) { + errors.add("Exception compiling ${sourceFile.name}: ${e.message}") + } + } + } + } + + private fun compileSourceSetsMode( + objDir: File, + baseArgs: List, + includeArgs: List, + compiled: AtomicInteger, + errors: ConcurrentLinkedQueue + ) { + // Collect all files from all source sets + val allFiles = mutableListOf>>() + + sourceSets.forEach { sourceSet -> + val setFiles = sourceSet.sources.asFileTree + .matching { + sourceSet.includes.get().forEach { pattern -> include(pattern) } + sourceSet.excludes.get().forEach { pattern -> exclude(pattern) } + } + .files + .toList() + + // Merge base args with source-set-specific args + val mergedArgs = baseArgs + sourceSet.compilerArgs.get() + + setFiles.forEach { file -> + allFiles.add(file to mergedArgs) + } + } + + if (allFiles.isEmpty()) { + logNormal("No source files to compile in source sets") + return + } + + val total = allFiles.size + logNormal("Compiling $total C++ source file${if (total == 1) "" else "s"} from ${sourceSets.size} source set${if (sourceSets.size == 1) "" else "s"} with ${compiler.get()}...") + + // Compile files in parallel (or sequentially for FAIL_FAST) with their specific args + if (errorHandling.get() == ErrorHandlingMode.FAIL_FAST) { + // Use sequential stream for FAIL_FAST to ensure immediate termination on error + allFiles.stream().forEach { (sourceFile, specificArgs) -> + try { + compileFile(sourceFile, objDir, specificArgs, includeArgs, compiled, total, errors) + } catch (e: Exception) { + errors.add("Exception compiling ${sourceFile.name}: ${e.message}") + throw e // Re-throw to stop compilation immediately in FAIL_FAST mode + } + } + } else { + // Use parallel stream for COLLECT_ALL mode + allFiles.parallelStream().forEach { (sourceFile, specificArgs) -> + try { + compileFile(sourceFile, objDir, specificArgs, includeArgs, compiled, total, errors) + } catch (e: Exception) { + errors.add("Exception compiling ${sourceFile.name}: ${e.message}") + } + } + } + } + + private fun compileFile( + sourceFile: File, + objDir: File, + baseArgs: List, + includeArgs: List, + compiled: AtomicInteger, + total: Int, + errors: ConcurrentLinkedQueue + ) { + // Determine object file name + val baseName = sourceFile.nameWithoutExtension + val objectFile = File(objDir, "$baseName.o") + + // Build full command line + val cmdLine = mutableListOf().apply { + add(compiler.get()) + addAll(baseArgs) + addAll(includeArgs) + add("-c") + add(sourceFile.absolutePath) + add("-o") + add(objectFile.absolutePath) + } + + if (shouldShowCommandLine()) { + logDebug(" ${cmdLine.joinToString(" ")}") + } + + // Execute compilation + val stdout = ByteArrayOutputStream() + val stderr = ByteArrayOutputStream() + + val result = execOperations.exec { + commandLine(cmdLine) + standardOutput = stdout + errorOutput = stderr + isIgnoreExitValue = true + } + + if (result.exitValue != 0) { + val allOutput = (stdout.toString() + stderr.toString()).trim() + val errorMsg = buildString { + append("Failed to compile ${sourceFile.name}: exit code ${result.exitValue}") + if (allOutput.isNotEmpty()) { + appendLine() + append(allOutput) + } + } + errors.add(errorMsg) + + // FAIL_FAST: throw immediately on first error + if (errorHandling.get() == ErrorHandlingMode.FAIL_FAST) { + throw RuntimeException(errorMsg) + } + } else { + val count = compiled.incrementAndGet() + val interval = progressReportInterval.get() + if (logLevel.get() >= LogLevel.VERBOSE && (count % interval == 0 || count == total)) { + logVerbose(" Compiled $count/$total files...") + } + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkExecutableTask.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkExecutableTask.kt new file mode 100644 index 000000000..d2d8d5e08 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkExecutableTask.kt @@ -0,0 +1,196 @@ + +package com.datadoghq.native.tasks + +import com.datadoghq.native.model.LogLevel +import com.datadoghq.native.util.PlatformUtils +import org.gradle.api.DefaultTask +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.* +import org.gradle.process.ExecOperations +import java.io.ByteArrayOutputStream +import javax.inject.Inject + +/** + * Kotlin-based executable linking task that directly invokes the linker. + * + * Used for linking test executables (gtest) and other standalone binaries. + */ +abstract class NativeLinkExecutableTask @Inject constructor( + private val execOperations: ExecOperations +) : DefaultTask() { + + /** + * The linker executable (usually same as compiler: 'g++', 'clang++'). + */ + @get:Input + abstract val linker: Property + + /** + * Linker arguments (flags). + */ + @get:Input + abstract val linkerArgs: ListProperty + + /** + * The object files to link. + */ + @get:InputFiles + @get:SkipWhenEmpty + abstract val objectFiles: ConfigurableFileCollection + + /** + * The output executable file. + */ + @get:OutputFile + abstract val outputFile: RegularFileProperty + + /** + * Library search paths (-L). + */ + @get:Input + @get:Optional + abstract val libraryPaths: ListProperty + + /** + * Libraries to link against (-l). + */ + @get:Input + @get:Optional + abstract val libraries: ListProperty + + /** + * Runtime library search paths (-Wl,-rpath). + */ + @get:Input + @get:Optional + abstract val runtimePaths: ListProperty + + /** + * Logging verbosity level. + */ + @get:Input + @get:Optional + abstract val logLevel: Property + + /** + * Show full command line. + */ + @get:Input + @get:Optional + abstract val showCommandLine: Property + + init { + libraryPaths.convention(emptyList()) + libraries.convention(emptyList()) + runtimePaths.convention(emptyList()) + logLevel.convention(LogLevel.NORMAL) + showCommandLine.convention(false) + group = "build" + description = "Links object files into an executable" + } + + /** + * Add libraries to link against. + */ + fun lib(vararg libs: String) { + libraries.addAll(*libs) + } + + /** + * Add library search paths. + */ + fun libPath(vararg paths: String) { + libraryPaths.addAll(*paths) + } + + /** + * Add runtime library search paths. + */ + fun runtimePath(vararg paths: String) { + runtimePaths.addAll(*paths) + } + + private fun logNormal(message: String) { + if (logLevel.get() >= LogLevel.NORMAL) { + logger.lifecycle(message) + } + } + + private fun logDebug(message: String) { + if (logLevel.get() == LogLevel.DEBUG) { + logger.lifecycle(message) + } + } + + @TaskAction + fun link() { + val outFile = outputFile.get().asFile + outFile.parentFile.mkdirs() + + val objectPaths = objectFiles.files.map { it.absolutePath } + + // Build command line + val cmdLine = mutableListOf().apply { + add(linker.get()) + addAll(objectPaths) + addAll(linkerArgs.get()) + + // Add library search paths (-L) + libraryPaths.get().forEach { path -> + add("-L$path") + } + + // Add libraries (-l) + libraries.get().forEach { lib -> + add("-l$lib") + } + + // Add runtime search paths (-rpath) + runtimePaths.get().forEach { path -> + add("-Wl,-rpath,$path") + } + + // Add output file + add("-o") + add(outFile.absolutePath) + } + + logNormal("Linking executable: ${outFile.name}") + + if (showCommandLine.get() || logLevel.get() == LogLevel.DEBUG) { + logDebug(" ${cmdLine.joinToString(" ")}") + } + + // Execute linking + val stdout = ByteArrayOutputStream() + val stderr = ByteArrayOutputStream() + + val result = execOperations.exec { + commandLine(cmdLine) + standardOutput = stdout + errorOutput = stderr + isIgnoreExitValue = true + } + + if (result.exitValue != 0) { + val allOutput = (stdout.toString() + stderr.toString()).trim() + val errorMsg = buildString { + append("Failed to link executable: exit code ${result.exitValue}") + if (allOutput.isNotEmpty()) { + appendLine() + append(allOutput) + } + } + throw RuntimeException(errorMsg) + } + + // Make executable + outFile.setExecutable(true) + + val sizeKB = outFile.length() / 1024 + logNormal("Successfully linked ${outFile.name} (${sizeKB}KB)") + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkTask.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkTask.kt new file mode 100644 index 000000000..59185c678 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkTask.kt @@ -0,0 +1,543 @@ + +package com.datadoghq.native.tasks + +import com.datadoghq.native.model.LogLevel +import com.datadoghq.native.model.Platform +import com.datadoghq.native.util.PlatformUtils +import org.gradle.api.DefaultTask +import org.gradle.api.file.ConfigurableFileCollection +import org.gradle.api.file.DirectoryProperty +import org.gradle.api.file.RegularFileProperty +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.* +import org.gradle.process.ExecOperations +import java.io.ByteArrayOutputStream +import javax.inject.Inject + +/** + * Kotlin-based shared library linking task that directly invokes the linker. + * + * Simplified from the Groovy SimpleLinkShared to focus on core functionality: + * - Linking object files into shared libraries + * - Library path and library flag management + * - Platform-specific flag handling (soname vs install_name) + * - Symbol stripping (optional) + */ +abstract class NativeLinkTask @Inject constructor( + private val execOperations: ExecOperations +) : DefaultTask() { + + /** + * The linker executable (usually same as compiler: 'g++', 'clang++'). + */ + @get:Input + abstract val linker: Property + + /** + * Linker arguments (flags and libraries). + */ + @get:Input + abstract val linkerArgs: ListProperty + + /** + * The object files to link. + */ + @get:InputFiles + @get:SkipWhenEmpty + abstract val objectFiles: ConfigurableFileCollection + + /** + * The output shared library file. + */ + @get:OutputFile + abstract val outputFile: RegularFileProperty + + /** + * Library search paths (-L). + */ + @get:Input + @get:Optional + abstract val libraryPaths: ListProperty + + /** + * Libraries to link against (-l). + */ + @get:Input + @get:Optional + abstract val libraries: ListProperty + + /** + * SO name for Linux (-Wl,-soname). + */ + @get:Input + @get:Optional + abstract val soname: Property + + /** + * Install name for macOS (-Wl,-install_name). + */ + @get:Input + @get:Optional + abstract val installName: Property + + /** + * Strip symbols after linking. + */ + @get:Input + @get:Optional + abstract val stripSymbols: Property + + /** + * Extract debug symbols to separate file before stripping. + */ + @get:Input + @get:Optional + abstract val extractDebugSymbols: Property + + /** + * Output directory for extracted debug symbols. + */ + @get:OutputDirectory + @get:Optional + abstract val debugSymbolsDir: DirectoryProperty + + /** + * Show detailed linking output. + */ + @get:Input + @get:Optional + abstract val verbose: Property + + // === Logging and Verbosity === + + /** + * Logging verbosity level. + * Default: NORMAL + */ + @get:Input + @get:Optional + abstract val logLevel: Property + + /** + * Show full command line for the link operation. + * Default: false (only shown at DEBUG level) + */ + @get:Input + @get:Optional + abstract val showCommandLine: Property + + /** + * Show linker map (symbol resolution details). + * Default: false + */ + @get:Input + @get:Optional + abstract val showLinkerMap: Property + + /** + * Linker map output file (when showLinkerMap is true). + * Default: null (stdout/stderr) + */ + @get:OutputFile + @get:Optional + abstract val linkerMapFile: RegularFileProperty + + /** + * Enable ANSI color codes in output. + * Default: true + */ + @get:Input + @get:Optional + abstract val colorOutput: Property + + // === Symbol Management === + + /** + * Symbol patterns to export (make visible). + * For example: ["Java_*", "JNI_OnLoad", "JNI_OnUnload"] + */ + @get:Input + @get:Optional + abstract val exportSymbols: ListProperty + + /** + * Symbol patterns to hide (make not visible). + * Applied after exportSymbols. + */ + @get:Input + @get:Optional + abstract val hideSymbols: ListProperty + + // === Library Path Management === + + /** + * Runtime library search paths (-Wl,-rpath). + * Use runtimePath() method to add. + */ + @get:Input + @get:Optional + abstract val runtimePaths: ListProperty + + // === Verification === + + /** + * Check for undefined symbols after linking. + * Default: false + */ + @get:Input + @get:Optional + abstract val checkUndefinedSymbols: Property + + /** + * Verify the shared library after linking (ldd/otool -L). + * Default: false + */ + @get:Input + @get:Optional + abstract val verifySharedLib: Property + + init { + libraryPaths.convention(emptyList()) + libraries.convention(emptyList()) + runtimePaths.convention(emptyList()) + stripSymbols.convention(false) + extractDebugSymbols.convention(false) + verbose.convention(false) + logLevel.convention(LogLevel.NORMAL) + showCommandLine.convention(false) + showLinkerMap.convention(false) + colorOutput.convention(true) + exportSymbols.convention(emptyList()) + hideSymbols.convention(emptyList()) + checkUndefinedSymbols.convention(false) + verifySharedLib.convention(false) + group = "build" + description = "Links object files into a shared library" + } + + fun lib(vararg libs: String) { + libraries.addAll(*libs) + } + + fun libPath(vararg paths: String) { + libraryPaths.addAll(*paths) + } + + fun runtimePath(vararg paths: String) { + runtimePaths.addAll(*paths) + } + + // === Logging Helpers === + + private fun logNormal(message: String) { + if (logLevel.get() >= LogLevel.NORMAL) { + logger.lifecycle(message) + } + } + + private fun logVerbose(message: String) { + if (logLevel.get() >= LogLevel.VERBOSE) { + logger.lifecycle(message) + } + } + + private fun logDebug(message: String) { + if (logLevel.get() == LogLevel.DEBUG) { + logger.lifecycle(message) + } + } + + private fun shouldShowCommandLine(): Boolean { + return showCommandLine.get() || logLevel.get() == LogLevel.DEBUG + } + + @TaskAction + fun link() { + val outFile = outputFile.get().asFile + outFile.parentFile.mkdirs() + + val objectPaths = objectFiles.files.map { it.absolutePath } + + // Determine shared library flag based on platform + val sharedFlag = when (PlatformUtils.currentPlatform) { + Platform.MACOS -> "-dynamiclib" + Platform.LINUX -> "-shared" + } + + // Build command line + val cmdLine = mutableListOf().apply { + add(linker.get()) + add(sharedFlag) + addAll(objectPaths) + addAll(linkerArgs.get()) + + // Add library search paths (-L) + libraryPaths.get().forEach { path -> + add("-L$path") + } + + // Add libraries (-l) + libraries.get().forEach { lib -> + add("-l$lib") + } + + // Add runtime search paths (-rpath) + runtimePaths.get().forEach { path -> + add("-Wl,-rpath,$path") + } + + // Add soname/install_name based on platform + when (PlatformUtils.currentPlatform) { + Platform.LINUX -> { + if (soname.isPresent) { + add("-Wl,-soname,${soname.get()}") + } + } + Platform.MACOS -> { + if (installName.isPresent) { + add("-Wl,-install_name,${installName.get()}") + } + } + } + + // Add symbol visibility control if specified + if (exportSymbols.get().isNotEmpty() || hideSymbols.get().isNotEmpty()) { + addAll(generateSymbolVisibilityFlags(outFile)) + } + + // Add output file + add("-o") + add(outFile.absolutePath) + } + + logNormal("Linking shared library: ${outFile.name}") + + if (shouldShowCommandLine()) { + logDebug(" ${cmdLine.joinToString(" ")}") + } + + // Execute linking + val stdout = ByteArrayOutputStream() + val stderr = ByteArrayOutputStream() + + val result = execOperations.exec { + commandLine(cmdLine) + standardOutput = stdout + errorOutput = stderr + isIgnoreExitValue = true + } + + if (result.exitValue != 0) { + val allOutput = (stdout.toString() + stderr.toString()).trim() + val errorMsg = buildString { + append("Failed to link shared library: exit code ${result.exitValue}") + if (allOutput.isNotEmpty()) { + appendLine() + append(allOutput) + } + } + throw RuntimeException(errorMsg) + } + + // Extract debug symbols before stripping if requested + if (extractDebugSymbols.get()) { + extractDebugInfo(outFile) + } + + // Strip symbols if requested + if (stripSymbols.get()) { + stripLibrary(outFile) + } + + val sizeKB = outFile.length() / 1024 + logNormal("Successfully linked ${outFile.name} (${sizeKB}KB)") + } + + /** + * Generate platform-specific symbol visibility flags. + * Returns linker flags to control symbol export/hiding. + */ + private fun generateSymbolVisibilityFlags(outFile: java.io.File): List { + return when (PlatformUtils.currentPlatform) { + Platform.LINUX -> { + generateLinuxVersionScript(outFile) + } + Platform.MACOS -> { + generateMacOSExportList(outFile) + } + } + } + + /** + * Generate Linux version script for symbol visibility control. + */ + private fun generateLinuxVersionScript(outFile: java.io.File): List { + val versionScript = java.io.File(temporaryDir, "${outFile.nameWithoutExtension}.ver") + + val scriptContent = buildString { + appendLine("{") + appendLine(" global:") + + // Export specified symbols + exportSymbols.get().forEach { pattern -> + appendLine(" $pattern;") + } + + // Consolidate all hidden symbols in a single local section + appendLine(" local:") + + // Explicitly hide specified symbols (override exports) + hideSymbols.get().forEach { pattern -> + appendLine(" $pattern;") + } + + // Hide everything else unless it was explicitly exported + if (exportSymbols.get().isNotEmpty() || hideSymbols.get().isNotEmpty()) { + appendLine(" *;") + } + + appendLine("};") + } + + versionScript.writeText(scriptContent) + logVerbose("Generated version script: ${versionScript.name}") + + return listOf("-Wl,--version-script=${versionScript.absolutePath}") + } + + /** + * Generate macOS exported symbols list for symbol visibility control. + */ + private fun generateMacOSExportList(outFile: java.io.File): List { + val exportList = java.io.File(temporaryDir, "${outFile.nameWithoutExtension}.exp") + + // Warn if wildcards are used - macOS doesn't support them + exportSymbols.get().forEach { pattern -> + if (pattern.contains('*') || pattern.contains('?')) { + logger.warn("Symbol pattern '$pattern' contains wildcards which are not supported on macOS. " + + "Pattern will be treated as a literal symbol name. " + + "Consider using -fvisibility compiler flags instead, or list symbols explicitly.") + } + } + + val listContent = buildString { + // Export specified symbols (macOS needs leading underscore for C symbols) + exportSymbols.get().forEach { pattern -> + // Convert glob patterns to exact names or keep as-is + // macOS export list doesn't support wildcards like Linux version scripts + // For wildcards, we'd need to use -exported_symbols_list with all matching symbols + // For now, treat patterns as literal symbol names + val symbol = if (pattern.startsWith("_")) pattern else "_$pattern" + appendLine(symbol) + } + } + + exportList.writeText(listContent) + logVerbose("Generated export list: ${exportList.name}") + + val flags = mutableListOf() + + // Add export list + if (exportSymbols.get().isNotEmpty()) { + flags.add("-Wl,-exported_symbols_list,${exportList.absolutePath}") + } + + // For hiding, use -unexported_symbols_list if needed + if (hideSymbols.get().isNotEmpty()) { + val hideList = java.io.File(temporaryDir, "${outFile.nameWithoutExtension}.hide") + val hideContent = buildString { + hideSymbols.get().forEach { pattern -> + val symbol = if (pattern.startsWith("_")) pattern else "_$pattern" + appendLine(symbol) + } + } + hideList.writeText(hideContent) + flags.add("-Wl,-unexported_symbols_list,${hideList.absolutePath}") + } + + return flags + } + + private fun extractDebugInfo(libFile: java.io.File) { + val debugDir = if (debugSymbolsDir.isPresent) { + debugSymbolsDir.get().asFile + } else { + libFile.parentFile + } + debugDir.mkdirs() + + when (PlatformUtils.currentPlatform) { + Platform.LINUX -> { + extractDebugInfoLinux(libFile, debugDir) + } + Platform.MACOS -> { + extractDebugInfoMacOS(libFile, debugDir) + } + } + } + + private fun extractDebugInfoLinux(libFile: java.io.File, debugDir: java.io.File) { + val debugFile = java.io.File(debugDir, "${libFile.name}.debug") + + logNormal("Extracting debug symbols to ${debugFile.name}...") + + // Extract debug symbols + val extractResult = execOperations.exec { + commandLine("objcopy", "--only-keep-debug", libFile.absolutePath, debugFile.absolutePath) + isIgnoreExitValue = true + } + + if (extractResult.exitValue != 0) { + logger.warn("Failed to extract debug symbols (exit code ${extractResult.exitValue})") + return + } + + // Add GNU debuglink to stripped library + val debuglinkResult = execOperations.exec { + commandLine("objcopy", "--add-gnu-debuglink=${debugFile.absolutePath}", libFile.absolutePath) + isIgnoreExitValue = true + } + + if (debuglinkResult.exitValue != 0) { + logger.warn("Failed to add debuglink (exit code ${debuglinkResult.exitValue})") + } else { + logNormal("Created debug file: ${debugFile.name} (${debugFile.length() / 1024}KB)") + } + } + + private fun extractDebugInfoMacOS(libFile: java.io.File, debugDir: java.io.File) { + val dsymBundle = java.io.File(debugDir, "${libFile.name}.dSYM") + + logNormal("Creating dSYM bundle...") + + val result = execOperations.exec { + commandLine("dsymutil", libFile.absolutePath, "-o", dsymBundle.absolutePath) + isIgnoreExitValue = true + } + + if (result.exitValue != 0) { + logger.warn("Failed to create dSYM bundle (exit code ${result.exitValue})") + } else { + logNormal("Created dSYM bundle: ${dsymBundle.name}") + } + } + + private fun stripLibrary(libFile: java.io.File) { + logNormal("Stripping symbols from ${libFile.name}...") + + val stripCmd = when (PlatformUtils.currentPlatform) { + Platform.LINUX -> listOf("strip", "--strip-debug", libFile.absolutePath) + Platform.MACOS -> listOf("strip", "-S", libFile.absolutePath) + } + + val result = execOperations.exec { + commandLine(stripCmd) + isIgnoreExitValue = true + } + + if (result.exitValue != 0) { + logger.warn("Failed to strip symbols (exit code ${result.exitValue}), continuing...") + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/util/PlatformUtils.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/util/PlatformUtils.kt new file mode 100644 index 000000000..0d529db45 --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/util/PlatformUtils.kt @@ -0,0 +1,292 @@ + +package com.datadoghq.native.util + +import com.datadoghq.native.model.Architecture +import com.datadoghq.native.model.Platform +import org.gradle.api.GradleException +import org.gradle.api.Project +import java.io.File +import java.util.concurrent.TimeUnit + +object PlatformUtils { + val currentPlatform: Platform by lazy { Platform.current() } + val currentArchitecture: Architecture by lazy { Architecture.current() } + + fun isMusl(): Boolean { + if (currentPlatform != Platform.LINUX) { + return false + } + + // Check if running on musl libc by scanning /lib/ld-musl-*.so.1 + return File("/lib").listFiles()?.any { + it.name.startsWith("ld-musl-") && it.name.endsWith(".so.1") + } ?: false + } + + fun javaHome(): String { + return System.getenv("JAVA_HOME") + ?: System.getProperty("java.home") + ?: throw IllegalStateException("Neither JAVA_HOME environment variable nor java.home system property is set") + } + + /** + * Resolve JAVA_HOME for test execution, preferring JAVA_TEST_HOME if set. + * This allows running tests with a different JDK than the build JDK. + */ + fun testJavaHome(): String { + return System.getenv("JAVA_TEST_HOME") + ?: System.getenv("JAVA_HOME") + ?: System.getProperty("java.home") + ?: throw IllegalStateException("Neither JAVA_TEST_HOME, JAVA_HOME, nor java.home is set") + } + + /** + * Get the java executable path for test execution. + */ + fun testJavaExecutable(): String { + return "${testJavaHome()}/bin/java" + } + + fun jniIncludePaths(): List { + val javaHome = javaHome() + val platform = when (currentPlatform) { + Platform.LINUX -> "linux" + Platform.MACOS -> "darwin" + } + return listOf( + "$javaHome/include", + "$javaHome/include/$platform" + ) + } + + /** + * Check if a compiler is available and functional + */ + fun isCompilerAvailable(compiler: String): Boolean { + return try { + val process = ProcessBuilder(compiler, "--version") + .redirectErrorStream(true) + .start() + process.waitFor(5, TimeUnit.SECONDS) + process.exitValue() == 0 + } catch (e: Exception) { + false + } + } + + /** + * Locate a library using compiler's -print-file-name. + * Uses the specified compiler, falling back to gcc if not available. + */ + fun locateLibrary(libName: String, compiler: String = "gcc"): String? { + if (currentPlatform != Platform.LINUX) { + return null + } + + return try { + // Try the specified compiler first, fall back to gcc + val compilerToUse = if (isCompilerAvailable(compiler)) { + compiler + } else if (compiler != "gcc" && isCompilerAvailable("gcc")) { + "gcc" + } else { + return null + } + + val process = ProcessBuilder(compilerToUse, "-print-file-name=$libName.so") + .redirectErrorStream(true) + .start() + + val output = process.inputStream.bufferedReader().readText().trim() + process.waitFor() + + if (process.exitValue() == 0 && !output.endsWith("$libName.so")) { + output + } else { + null + } + } catch (e: Exception) { + null + } + } + + fun locateLibasan(compiler: String = "gcc"): String? = locateLibrary("libasan", compiler) + + fun locateLibtsan(compiler: String = "gcc"): String? = locateLibrary("libtsan", compiler) + + fun checkFuzzerSupport(): Boolean { + return try { + val testFile = createTempFile("fuzzer_check", ".cpp") + try { + testFile.writeText("extern \"C\" int LLVMFuzzerTestOneInput(const char*, long) { return 0; }") + + val process = ProcessBuilder( + "clang++", + "-fsanitize=fuzzer", + "-c", + testFile.absolutePath, + "-o", + "/dev/null" + ).redirectErrorStream(true).start() + + process.waitFor() + process.exitValue() == 0 + } finally { + testFile.delete() + } + } catch (e: Exception) { + false + } + } + + fun hasAsan(compiler: String = "gcc"): Boolean { + return !isMusl() && locateLibasan(compiler) != null + } + + fun hasTsan(compiler: String = "gcc"): Boolean { + return !isMusl() && locateLibtsan(compiler) != null + } + + fun hasFuzzer(): Boolean { + return !isMusl() && checkFuzzerSupport() + } + + fun sharedLibExtension(): String = when (currentPlatform) { + Platform.LINUX -> "so" + Platform.MACOS -> "dylib" + } + + /** + * Find Homebrew LLVM installation on macOS. + * Returns the LLVM installation path or null if not found. + */ + fun findHomebrewLLVM(): String? { + if (currentPlatform != Platform.MACOS) { + return null + } + + val possiblePaths = listOf( + "/opt/homebrew/opt/llvm", // Apple Silicon + "/usr/local/opt/llvm" // Intel Mac + ) + + for (path in possiblePaths) { + if (File(path).exists() && File("$path/bin/clang++").exists()) { + return path + } + } + + // Try using brew command + return try { + val process = ProcessBuilder("brew", "--prefix", "llvm") + .redirectErrorStream(true) + .start() + process.waitFor(5, TimeUnit.SECONDS) + if (process.exitValue() == 0) { + val brewPath = process.inputStream.bufferedReader().readText().trim() + if (File("$brewPath/bin/clang++").exists()) { + brewPath + } else { + null + } + } else { + null + } + } catch (e: Exception) { + null + } + } + + /** + * Find the clang resource directory within an LLVM installation. + * This is needed for locating libFuzzer on macOS with Homebrew LLVM. + */ + fun findClangResourceDir(llvmPath: String?): String? { + if (llvmPath == null) { + return null + } + + val clangLibDir = File("$llvmPath/lib/clang") + if (!clangLibDir.exists()) { + return null + } + + // Find the version directory (e.g., 18.1.8 or 19) + val versions = clangLibDir.listFiles() + ?.filter { it.isDirectory } + ?.sortedByDescending { it.name } + + return if (versions != null && versions.isNotEmpty()) { + "$llvmPath/lib/clang/${versions[0].name}" + } else { + null + } + } + + /** + * Find a compiler suitable for fuzzing. + * On macOS, prefers Homebrew LLVM's clang++ for libFuzzer support. + */ + fun findFuzzerCompiler(project: Project): String { + if (currentPlatform == Platform.MACOS) { + val homebrewLLVM = findHomebrewLLVM() + if (homebrewLLVM != null) { + return "$homebrewLLVM/bin/clang++" + } + } + return findCompiler(project) + } + + /** + * Detect the installed clang-format version. + * Returns null if clang-format is not available. + */ + fun clangFormatVersion(): String? { + return try { + val process = ProcessBuilder("clang-format", "--version").start() + process.waitFor(5, TimeUnit.SECONDS) + if (process.exitValue() == 0) { + val output = process.inputStream.bufferedReader().readText().trim() + val match = Regex("""clang-format version (\d+\.\d+\.\d+)""").find(output) + match?.groupValues?.get(1) + } else { + null + } + } catch (e: Exception) { + null + } + } + + /** + * Find a C++ compiler, respecting -Pnative.forceCompiler property. + * Auto-detects clang++ or g++ if not specified. + */ + fun findCompiler(project: Project): String { + // Check for forced compiler override + val forcedCompiler = project.findProperty("native.forceCompiler") as? String + if (forcedCompiler != null) { + if (isCompilerAvailable(forcedCompiler)) { + project.logger.info("Using forced compiler: $forcedCompiler") + return forcedCompiler + } + throw GradleException( + "Forced compiler '$forcedCompiler' is not available. " + + "Verify the path or remove -Pnative.forceCompiler to auto-detect." + ) + } + + // Auto-detect: prefer clang++, then g++, then c++ + val compilers = listOf("clang++", "g++", "c++") + for (compiler in compilers) { + if (isCompilerAvailable(compiler)) { + project.logger.info("Auto-detected compiler: $compiler") + return compiler + } + } + + throw GradleException( + "No C++ compiler found. Please install clang++ or g++, " + + "or specify one with -Pnative.forceCompiler=/path/to/compiler" + ) + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/JavaConventionsPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/JavaConventionsPlugin.kt new file mode 100644 index 000000000..85983bcee --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/JavaConventionsPlugin.kt @@ -0,0 +1,40 @@ + +package com.datadoghq.profiler + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.compile.JavaCompile + +/** + * Convention plugin for Java compilation settings. + * + * Applies standard Java compilation options across all subprojects: + * - Java 8 release target for broad JVM compatibility + * + * Requires JDK 9+ for building (uses --release flag). + * The compiled bytecode targets Java 8 runtime. + * + * Usage: + * ```kotlin + * plugins { + * id("com.datadoghq.java-conventions") + * } + * ``` + */ +class JavaConventionsPlugin : Plugin { + override fun apply(project: Project) { + val javaVersion = System.getProperty("java.specification.version")?.toDoubleOrNull() ?: 0.0 + + project.tasks.withType(JavaCompile::class.java).configureEach { + if (javaVersion >= 9) { + // JDK 9+ supports --release flag which handles source, target, and boot classpath + options.compilerArgs.addAll(listOf("--release", "8")) + } else { + // Fallback for JDK 8 (not recommended for building) + sourceCompatibility = "8" + targetCompatibility = "8" + project.logger.warn("Building with JDK 8 is not recommended. Use JDK 11+ with --release 8 for better compatibility.") + } + } + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt new file mode 100644 index 000000000..aeb7c9d4d --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt @@ -0,0 +1,418 @@ + +package com.datadoghq.profiler + +import com.datadoghq.native.NativeBuildExtension +import com.datadoghq.native.util.PlatformUtils +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.artifacts.Configuration +import org.gradle.api.provider.ListProperty +import org.gradle.api.provider.Property +import org.gradle.api.tasks.JavaExec +import org.gradle.api.tasks.SourceSetContainer +import org.gradle.api.tasks.testing.Test +import org.gradle.api.tasks.testing.logging.TestLogEvent +import javax.inject.Inject + +/** + * Convention plugin for profiler test configuration. + * + * Provides: + * - Standard JVM arguments for profiler testing (attach self, error files, etc.) + * - Java executable selection (JAVA_TEST_HOME or JAVA_HOME) + * - Common environment variables (CI, rate limiting) + * - JUnit Platform configuration + * - Automatic multi-config test task generation from NativeBuildExtension + * + * Usage: + * ```kotlin + * plugins { + * id("com.datadoghq.profiler-test") + * } + * + * profilerTest { + * // Required: the project providing the profiler library + * profilerLibProject.set(":ddprof-lib") + * + * // Optional: override native library directory + * nativeLibDir.set(layout.buildDirectory.dir("libs/native")) + * + * // Optional: add extra JVM args + * extraJvmArgs.add("-Xms256m") + * + * // Optional: specify which configs get application tasks (default: release, debug) + * applicationConfigs.set(listOf("release", "debug")) + * + * // Optional: main class for application tasks + * applicationMainClass.set("com.datadoghq.profiler.unwinding.UnwindingValidator") + * } + * ``` + */ +class ProfilerTestPlugin : Plugin { + override fun apply(project: Project) { + val extension = project.extensions.create( + "profilerTest", + ProfilerTestExtension::class.java, + project + ) + + // Create base configurations eagerly so they can be extended by build scripts + // without needing afterEvaluate + project.configurations.maybeCreate("testCommon").apply { + isCanBeConsumed = true + isCanBeResolved = true + } + project.configurations.maybeCreate("mainCommon").apply { + isCanBeConsumed = true + isCanBeResolved = true + } + + // Configure all Test tasks with standard settings + project.tasks.withType(Test::class.java).configureEach { + configureTestTask(this, extension, project) + } + + // Configure all JavaExec tasks with standard settings + project.tasks.withType(JavaExec::class.java).configureEach { + configureJavaExecTask(this, extension, project) + } + + // After evaluation, generate multi-config tasks if profilerLibProject is set + project.afterEvaluate { + if (extension.profilerLibProject.isPresent) { + generateMultiConfigTasks(project, extension) + } + } + } + + private fun configureTestTask(task: Test, extension: ProfilerTestExtension, project: Project) { + task.onlyIf { !project.hasProperty("skip-tests") } + + // Use JUnit Platform + task.useJUnitPlatform() + + // Configure Java executable - use centralized utility for JAVA_TEST_HOME/JAVA_HOME resolution + task.setExecutable(PlatformUtils.testJavaExecutable()) + + // Standard environment variables + task.environment("DDPROF_TEST_DISABLE_RATE_LIMIT", "1") + task.environment("CI", project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false) + + // Test logging + task.testLogging.showStandardStreams = true + task.testLogging.events(TestLogEvent.FAILED, TestLogEvent.SKIPPED) + + // JVM arguments - combine standard + extra + task.doFirst { + val allArgs = mutableListOf() + allArgs.addAll(extension.standardJvmArgs.get()) + + // Add native library path if configured + if (extension.nativeLibDir.isPresent) { + allArgs.add("-Djava.library.path=${extension.nativeLibDir.get().asFile.absolutePath}") + } + + allArgs.addAll(extension.extraJvmArgs.get()) + task.jvmArgs(allArgs) + } + } + + private fun configureJavaExecTask(task: JavaExec, extension: ProfilerTestExtension, project: Project) { + // Configure Java executable - use centralized utility for JAVA_TEST_HOME/JAVA_HOME resolution + task.setExecutable(PlatformUtils.testJavaExecutable()) + + // JVM arguments for JavaExec tasks + task.doFirst { + val allArgs = mutableListOf() + allArgs.addAll(extension.standardJvmArgs.get()) + allArgs.addAll(extension.extraJvmArgs.get()) + task.jvmArgs(allArgs) + } + } + + private fun generateMultiConfigTasks(project: Project, extension: ProfilerTestExtension) { + val nativeBuildExt = project.rootProject.extensions.findByType(NativeBuildExtension::class.java) + ?: return // No native build extension, nothing to generate + + val currentPlatform = PlatformUtils.currentPlatform + val currentArchitecture = PlatformUtils.currentArchitecture + val activeConfigurations = nativeBuildExt.getActiveConfigurations(currentPlatform, currentArchitecture) + + if (activeConfigurations.isEmpty()) { + return + } + + val profilerLibProjectPath = extension.profilerLibProject.get() + val tracerProjectPath = extension.tracerProject.getOrElse(":ddprof-test-tracer") + // Default to all active configs if not explicitly specified + val explicitAppConfigs = extension.applicationConfigs.get() + val applicationConfigs = if (explicitAppConfigs.isEmpty()) { + activeConfigurations.map { it.name } + } else { + explicitAppConfigs + } + val appMainClass = extension.applicationMainClass.getOrElse("") + + val sourceSets = project.extensions.getByType(SourceSetContainer::class.java) + + // Get the base configurations (created eagerly in apply()) + val testCommon = project.configurations.getByName("testCommon") + val mainCommon = project.configurations.getByName("mainCommon") + + // Add common dependencies to base configurations + addCommonTestDependencies(project, testCommon, tracerProjectPath) + addCommonMainDependencies(project, mainCommon, tracerProjectPath) + + val configNames = mutableListOf() + + // Generate tasks for each active configuration + activeConfigurations.forEach { config -> + val configName = config.name + val isActive = config.active.get() + val testEnv = config.testEnvironment.get() + + configNames.add(configName) + + // Create test configuration + val testCfg = project.configurations.maybeCreate("test${configName.capitalize()}Implementation").apply { + isCanBeConsumed = true + isCanBeResolved = true + extendsFrom(testCommon) + } + testCfg.dependencies.add( + project.dependencies.project(mapOf("path" to profilerLibProjectPath, "configuration" to configName)) + ) + + // Create test task using configuration closure + project.tasks.register("test$configName", Test::class.java) { + val testTask = this + testTask.onlyIf { isActive } + testTask.dependsOn(project.tasks.named("compileTestJava")) + testTask.description = "Runs unit tests with the $configName library variant" + testTask.group = "verification" + + // Filter classpath to include only necessary dependencies + testTask.classpath = sourceSets.getByName("test").runtimeClasspath.filter { file -> + !file.name.contains("ddprof-") || file.name.contains("test-tracer") + } + testCfg + + // Apply test environment from config + if (testEnv.isNotEmpty()) { + testEnv.forEach { (key, value) -> + testTask.environment(key, value) + } + } + + // Sanitizer-specific conditions + when (configName) { + "asan" -> testTask.onlyIf { PlatformUtils.locateLibasan() != null } + "tsan" -> testTask.onlyIf { PlatformUtils.locateLibtsan() != null } + else -> { /* no additional conditions */ } + } + } + + // Create application tasks for specified configs + if (configName in applicationConfigs && appMainClass.isNotEmpty()) { + // Create main configuration + val mainCfg = project.configurations.maybeCreate("${configName}Implementation").apply { + isCanBeConsumed = true + isCanBeResolved = true + extendsFrom(mainCommon) + } + mainCfg.dependencies.add( + project.dependencies.project(mapOf("path" to profilerLibProjectPath, "configuration" to configName)) + ) + + // Create run task + project.tasks.register("runUnwindingValidator${configName.capitalize()}", JavaExec::class.java) { + val runTask = this + runTask.onlyIf { isActive } + runTask.dependsOn(project.tasks.named("compileJava")) + runTask.description = "Run the unwinding validator application ($configName config)" + runTask.group = "application" + runTask.mainClass.set(appMainClass) + runTask.classpath = sourceSets.getByName("main").runtimeClasspath + mainCfg + + if (testEnv.isNotEmpty()) { + testEnv.forEach { (key, value) -> + runTask.environment(key, value) + } + } + + // Handle validatorArgs property + if (project.hasProperty("validatorArgs")) { + runTask.setArgs((project.property("validatorArgs") as String).split(" ")) + } + } + + // Create report task + project.tasks.register("unwindingReport${configName.capitalize()}", JavaExec::class.java) { + val reportTask = this + reportTask.onlyIf { isActive } + reportTask.dependsOn(project.tasks.named("compileJava")) + reportTask.description = "Generate unwinding report for CI ($configName config)" + reportTask.group = "verification" + reportTask.mainClass.set(appMainClass) + reportTask.classpath = sourceSets.getByName("main").runtimeClasspath + mainCfg + reportTask.args = listOf( + "--output-format=markdown", + "--output-file=build/reports/unwinding-summary.md" + ) + + if (testEnv.isNotEmpty()) { + testEnv.forEach { (key, value) -> + reportTask.environment(key, value) + } + } + reportTask.environment("CI", project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false) + + reportTask.doFirst { + project.file("${project.layout.buildDirectory.get()}/reports").mkdirs() + } + } + } + } + + // Create convenience delegate tasks + if (applicationConfigs.isNotEmpty()) { + project.tasks.register("runUnwindingValidator", DefaultTask::class.java) { + val delegateTask = this + delegateTask.description = "Run unwinding validator (delegates to release if available, otherwise debug)" + delegateTask.group = "application" + delegateTask.dependsOn( + project.provider { + when { + project.tasks.findByName("runUnwindingValidatorRelease") != null -> listOf("runUnwindingValidatorRelease") + project.tasks.findByName("runUnwindingValidatorDebug") != null -> listOf("runUnwindingValidatorDebug") + else -> throw GradleException("No suitable build configuration available for unwinding validator") + } + } + ) + } + + project.tasks.register("unwindingReport", DefaultTask::class.java) { + val delegateTask = this + delegateTask.description = "Generate unwinding report (delegates to release if available, otherwise debug)" + delegateTask.group = "verification" + delegateTask.dependsOn( + project.provider { + when { + project.tasks.findByName("unwindingReportRelease") != null -> listOf("unwindingReportRelease") + project.tasks.findByName("unwindingReportDebug") != null -> listOf("unwindingReportDebug") + else -> throw GradleException("No suitable build configuration available for unwinding report") + } + } + ) + } + } + + // Wire up assemble/gtest dependencies + project.gradle.projectsEvaluated { + configNames.forEach { cfgName -> + val testTask = project.tasks.findByName("test$cfgName") + val profilerLibProject = project.rootProject.findProject(profilerLibProjectPath) + + if (profilerLibProject != null) { + val assembleTask = profilerLibProject.tasks.findByName("assemble${cfgName.capitalize()}") + if (testTask != null && assembleTask != null) { + assembleTask.dependsOn(testTask) + } + + val gtestTask = profilerLibProject.tasks.findByName("gtest${cfgName.capitalize()}") + if (testTask != null && gtestTask != null) { + testTask.dependsOn(gtestTask) + } + } + } + } + } + + private fun addCommonTestDependencies(project: Project, configuration: Configuration, tracerProjectPath: String) { + // Add tracer project dependency - other deps added via version catalog in build file + configuration.dependencies.add( + project.dependencies.project(mapOf("path" to tracerProjectPath)) + ) + } + + private fun addCommonMainDependencies(project: Project, configuration: Configuration, tracerProjectPath: String) { + // Add tracer project dependency - other deps added via version catalog in build file + configuration.dependencies.add( + project.dependencies.project(mapOf("path" to tracerProjectPath)) + ) + } +} + +/** + * Extension for profiler test configuration. + */ +abstract class ProfilerTestExtension @Inject constructor( + project: Project, + objects: org.gradle.api.model.ObjectFactory +) { + + /** + * Standard JVM arguments applied to all Test and JavaExec tasks. + * These are the common profiler testing requirements. + */ + abstract val standardJvmArgs: ListProperty + + /** + * Additional JVM arguments to add beyond the standard set. + */ + abstract val extraJvmArgs: ListProperty + + /** + * Directory containing native test libraries. + * When set, adds -Djava.library.path to Test tasks. + */ + val nativeLibDir: org.gradle.api.file.DirectoryProperty = objects.directoryProperty() + + /** + * Whether to skip tests when -Pskip-tests is set. + * Default: true + */ + abstract val respectSkipTests: Property + + /** + * The project path providing the profiler library (e.g., ":ddprof-lib"). + * When set, enables automatic multi-config task generation. + */ + abstract val profilerLibProject: Property + + /** + * The project path providing the test tracer (default: ":ddprof-test-tracer"). + */ + abstract val tracerProject: Property + + /** + * Configurations that should have application tasks (runUnwindingValidator*, unwindingReport*). + * Default: empty list means all active configurations get application tasks. + * Set explicitly to restrict which configs get app tasks. + */ + abstract val applicationConfigs: ListProperty + + /** + * Main class for application tasks. + * Default: "com.datadoghq.profiler.unwinding.UnwindingValidator" + */ + abstract val applicationMainClass: Property + + init { + // Standard JVM arguments for profiler testing + standardJvmArgs.convention(listOf( + "-Djdk.attach.allowAttachSelf", // Allow profiler to attach to self + "-Djol.tryWithSudo=true", // JOL memory layout analysis + "-XX:ErrorFile=build/hs_err_pid%p.log", // HotSpot error file location + "-XX:+ResizeTLAB", // Allow TLAB resizing for allocation profiling + "-Xmx512m" // Default heap size for tests + )) + + extraJvmArgs.convention(emptyList()) + respectSkipTests.convention(true) + tracerProject.convention(":ddprof-test-tracer") + applicationConfigs.convention(emptyList()) // Empty = all active configs get app tasks + applicationMainClass.convention("com.datadoghq.profiler.unwinding.UnwindingValidator") + } +} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/SpotlessConventionPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/SpotlessConventionPlugin.kt new file mode 100644 index 000000000..4c1236a8f --- /dev/null +++ b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/SpotlessConventionPlugin.kt @@ -0,0 +1,105 @@ + +package com.datadoghq.profiler + +import com.diffplug.gradle.spotless.SpotlessExtension +import org.gradle.api.Plugin +import org.gradle.api.Project + +/** + * Convention plugin that configures Spotless code formatting for all project types. + * Supports Java, Groovy, Kotlin, Scala, C++, and miscellaneous file formatting. + */ +class SpotlessConventionPlugin : Plugin { + override fun apply(project: Project) { + project.pluginManager.apply("com.diffplug.spotless") + + val configPath = project.rootProject.file("gradle").absolutePath + val spotless = project.extensions.getByType(SpotlessExtension::class.java) + + // Java formatting (Google Java Format 1.7 - last version supporting Java 8) + if (project.plugins.hasPlugin("java")) { + spotless.java { + toggleOffOn() + target("src/**/*.java") + targetExclude("src/test/resources/**") + googleJavaFormat("1.7") + } + } + + // Groovy Gradle files + spotless.groovyGradle { + toggleOffOn() + target("*.gradle", "gradle/**/*.gradle") + greclipse().configFile("$configPath/enforcement/spotless-groovy.properties") + } + + // Kotlin Gradle files + spotless.kotlinGradle { + toggleOffOn() + target("*.gradle.kts") + // ktlint 0.41.0 is compatible with older Kotlin versions in this build + ktlint("0.41.0").userData( + mapOf( + "indent_size" to "2", + "continuation_indent_size" to "2" + ) + ) + } + + // Groovy source files + if (project.plugins.hasPlugin("groovy")) { + val skipJavaExclude = project.findProperty("groovySkipJavaExclude") as? Boolean ?: false + spotless.groovy { + toggleOffOn() + if (!skipJavaExclude) { + excludeJava() + } + greclipse().configFile("$configPath/enforcement/spotless-groovy.properties") + } + } + + // Scala source files + if (project.plugins.hasPlugin("scala")) { + spotless.scala { + toggleOffOn() + scalafmt("2.7.5").configFile("$configPath/enforcement/spotless-scalafmt.conf") + } + } + + // TODO: Enable C++ formatting in a follow-up PR + // This requires reformatting all C/C++ source files which would pollute this PR. + // + // project.pluginManager.withPlugin("com.datadoghq.native-build") { + // val clangVersion = PlatformUtils.clangFormatVersion() + // if (clangVersion != null) { + // spotless.cpp { + // target("src/main/cpp/**") + // clangFormat(clangVersion).style("file") + // } + // } + // } + + // Miscellaneous files (markdown, shell scripts, etc.) + spotless.format("misc") { + toggleOffOn() + target( + ".gitignore", + "*.md", + ".github/**/*.md", + "src/**/*.md", + "application/**/*.md", + "*.sh", + "tooling/*.sh", + ".circleci/*.sh" + ) + indentWithSpaces() + trimTrailingWhitespace() + endWithNewline() + } + + // Wire spotlessCheck into the check task + project.tasks.named("check") { + dependsOn("spotlessCheck") + } + } +} diff --git a/build-logic/settings.gradle b/build-logic/settings.gradle new file mode 100644 index 000000000..b4be215bb --- /dev/null +++ b/build-logic/settings.gradle @@ -0,0 +1,3 @@ +rootProject.name = 'build-logic' + +include 'conventions' diff --git a/build.gradle b/build.gradle.kts similarity index 59% rename from build.gradle rename to build.gradle.kts index 80c62eed1..2ed1cf320 100644 --- a/build.gradle +++ b/build.gradle.kts @@ -1,3 +1,5 @@ +import java.net.URI + buildscript { dependencies { classpath("com.dipien:semantic-version-gradle-plugin:2.0.0") @@ -10,18 +12,17 @@ buildscript { } plugins { - id 'io.github.gradle-nexus.publish-plugin' version '2.0.0' - id "com.diffplug.spotless" version "6.11.0" + id("io.github.gradle-nexus.publish-plugin") version "2.0.0" + id("com.datadoghq.native-root") } version = "1.38.0" -apply plugin: "com.dipien.semantic-version" -version = project.findProperty("ddprof_version") ?: version +apply(plugin = "com.dipien.semantic-version") +version = findProperty("ddprof_version") as? String ?: version allprojects { repositories { - mavenCentral() mavenCentral() gradlePluginPortal() } @@ -34,63 +35,59 @@ repositories { maven { content { - includeGroup "com.datadoghq" + includeGroup("com.datadoghq") } mavenContent { snapshotsOnly() } // see https://central.sonatype.org/publish/publish-portal-snapshots/#consuming-via-gradle - url 'https://central.sonatype.com/repository/maven-snapshots/' + url = URI("https://central.sonatype.com/repository/maven-snapshots/") } } allprojects { - group = 'com.datadoghq' + group = "com.datadoghq" - apply from: rootProject.file('common.gradle') - apply from: rootProject.file('gradle/configurations.gradle') - apply from: "$rootDir/gradle/spotless.gradle" + // Apply spotless formatting convention to all projects + apply(plugin = "com.datadoghq.spotless-convention") } subprojects { version = rootProject.version } -apply from: rootProject.file('common.gradle') -apply from: rootProject.file('gradle/configurations.gradle') - -def isCI = project.hasProperty("CI") || Boolean.parseBoolean(System.getenv("CI")) +val isCI = hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false nexusPublishing { repositories { - def forceLocal = project.hasProperty('forceLocal') + val forceLocal = hasProperty("forceLocal") if (forceLocal && !isCI) { - local { + create("local") { // For testing use with https://hub.docker.com/r/sonatype/nexus // docker run --rm -d -p 8081:8081 --name nexus sonatype/nexus // Doesn't work for testing releases though... (due to staging) - nexusUrl = uri("http://localhost:8081/nexus/content/repositories/releases/") - snapshotRepositoryUrl = uri("http://localhost:8081/nexus/content/repositories/snapshots/") - username = "admin" - password = "admin123" + nexusUrl.set(URI("http://localhost:8081/nexus/content/repositories/releases/")) + snapshotRepositoryUrl.set(URI("http://localhost:8081/nexus/content/repositories/snapshots/")) + username.set("admin") + password.set("admin123") } } else { // see https://github.com/gradle-nexus/publish-plugin#publishing-to-maven-central-via-sonatype-central // For official documentation: // staging repo publishing https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/#configuration // snapshot publishing https://central.sonatype.org/publish/publish-portal-snapshots/#publishing-via-other-methods - sonatype { + create("sonatype") { // see https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/#configuration // see https://github.com/gradle-nexus/publish-plugin#publishing-to-maven-central-via-sonatype-central // Also for official doc // staging repo publishing https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/#configuration // snapshot publishing https://central.sonatype.org/publish/publish-portal-snapshots/#publishing-via-other-methods - nexusUrl.set(uri("https://ossrh-staging-api.central.sonatype.com/service/local/")) - snapshotRepositoryUrl.set(uri("https://central.sonatype.com/repository/maven-snapshots/")) + nexusUrl.set(URI("https://ossrh-staging-api.central.sonatype.com/service/local/")) + snapshotRepositoryUrl.set(URI("https://central.sonatype.com/repository/maven-snapshots/")) - username = System.getenv("SONATYPE_USERNAME") - password = System.getenv("SONATYPE_PASSWORD") + username.set(System.getenv("SONATYPE_USERNAME")) + password.set(System.getenv("SONATYPE_PASSWORD")) } } } diff --git a/common.gradle b/common.gradle deleted file mode 100644 index 2275f66f5..000000000 --- a/common.gradle +++ /dev/null @@ -1,148 +0,0 @@ -import java.nio.charset.Charset -import java.nio.file.Files -import java.nio.file.Paths - -buildscript { - repositories { - mavenLocal() - mavenCentral() - gradlePluginPortal() - } -} - -def static os() { - return org.gradle.internal.os.OperatingSystem.current() -} - -def static osIdentifier() { - if (os().isMacOsX()) { - return 'macos' - } else if (os().isLinux()) { - return 'linux' - } - throw new RuntimeException("Unknown OS: ${os().toString()}") -} - -def static arch() { - return System.getProperty('os.arch') -} - -@SuppressWarnings('GroovyFallthrough') -def static archIdentifier() { - String reportedArch = System.getProperty('os.arch') - switch (reportedArch) { - case 'x86_64': - case 'amd64': - case 'k8': return 'x64' - case 'x86': - case 'i386': - case 'i486': - case 'i586': - case 'i686': return 'x86' - case 'ARM': - case 'aarch32': return 'arm' - case 'arm64': - case 'aarch64': return 'arm64' - default: throw new RuntimeException("Unknown arch type: ${reportedArch}") - } -} - -def static javaHome() { - def jhome = System.getenv('JAVA_HOME') - if (jhome == null) { - jhome = System.getProperty('java.home') - } - return jhome -} - -def static containsArray(byte[] container, int offset, byte[] contained) { - for (int i = 0; i < contained.length; i++) { - int leftPos = offset + i - if (leftPos >= container.length) { - return false - } - if (container[leftPos] != contained[i]) { - return false - } - } - return true -} - -/** - * There is information about the linking in the ELF file. Since properly parsing ELF is not - * trivial this code will attempt a brute-force approach and will scan the first 4096 bytes - * of the 'java' program image for anything prefixed with `/ld-` - in practice this will contain - * `/ld-musl` for musl systems and probably something else for non-musl systems (eg. `/ld-linux-...`). - * However, if such string is missing should indicate that the system is not a musl one. - */ -def static isMusl() { - def magic = new byte[]{ - 127, 69, 76, 70 - } // *ELF - def prefix = '/ld-'.getBytes(Charset.defaultCharset()) // '/ld-*' - def musl = 'musl'.getBytes(Charset.defaultCharset()) // 'musl' - - java.nio.file.Path binary = Paths.get(javaHome(), "bin", "java") - byte[] buffer = new byte[4096] - - try (InputStream is = Files.newInputStream(binary)) { - int read = is.read(buffer, 0, 4) - if (read != 4 || !containsArray(buffer, 0, magic)) { - return false - } - read = is.read(buffer) - if (read <= 0) { - return false - } - int prefixPos = 0 - for (int i = 0; i < read; i++) { - if (buffer[i] == prefix[prefixPos]) { - if (++prefixPos == prefix.length) { - return containsArray(buffer, i + 1, musl) - } - } else { - prefixPos = 0 - } - } - } - return false -} - -ext.hasGtest = false - -// Define potential GTest locations for MacOS and Linux -def gtestLocations = [ - macos: ['/opt/homebrew/opt/googletest', '/usr/local/opt/googletest'], - linux: ['/usr/include/gtest', '/usr/local/include/gtest'] -] - -// Function to check if any of the specified paths exist -def checkGtestPaths(paths) { - for (path in paths) { - if (file(path).exists()) { - return true - } - } - return false -} - -// Determine OS and check for GTest -if (os().isMacOsX()) { - ext.hasGtest = checkGtestPaths(gtestLocations.macos) -} else if (os().isLinux()) { - ext.hasGtest = checkGtestPaths(gtestLocations.linux) -} - -// Log a message for debugging -if (!ext.hasGtest) { - println "GTest not found. Please install GTest or configure paths." -} - -ext { - isMusl = this.&isMusl - os = this.&os - osIdentifier = this.&osIdentifier - arch = this.&arch - archIdentifier = this.&archIdentifier - javaHome = this.&javaHome -} \ No newline at end of file diff --git a/ddprof-lib/benchmarks/build.gradle b/ddprof-lib/benchmarks/build.gradle deleted file mode 100644 index c6bd1db5c..000000000 --- a/ddprof-lib/benchmarks/build.gradle +++ /dev/null @@ -1,60 +0,0 @@ -plugins { - id 'cpp-application' -} - -// this feels weird but it is the only way invoking `./gradlew :ddprof-lib:*` tasks will work -if (rootDir.toString().endsWith("ddprof-lib/gradle")) { - apply from: rootProject.file('../../common.gradle') -} - -application { - baseName = "unwind_failures_benchmark" - source.from file('src') - privateHeaders.from file('src') - - targetMachines = [machines.macOS, machines.linux.x86_64] -} - -// Include the main library headers -tasks.withType(CppCompile).configureEach { - includes file('../src/main/cpp').toString() -} - -// Add a task to run the benchmark -tasks.register('runBenchmark', Exec) { - dependsOn 'assemble' - workingDir = buildDir - - doFirst { - // Find the executable by looking for it in the build directory - def executableName = "unwind_failures_benchmark" - def executable = null - - // Search for the executable in the build directory - buildDir.eachFileRecurse { file -> - if (file.isFile() && file.name == executableName && file.canExecute()) { - executable = file - return true // Stop searching once found - } - } - - if (executable == null) { - throw new GradleException("Executable '${executableName}' not found in ${buildDir.absolutePath}. Make sure the build was successful.") - } - - // Build command line with the executable path and any additional arguments - def cmd = [executable.absolutePath] - - // Add any additional arguments passed to the Gradle task - if (project.hasProperty('args')) { - cmd.addAll(project.args.split(' ')) - } - - println "Running benchmark using executable at: ${executable.absolutePath}" - commandLine = cmd - } - - doLast { - println "Benchmark completed." - } -} diff --git a/ddprof-lib/benchmarks/build.gradle.kts b/ddprof-lib/benchmarks/build.gradle.kts new file mode 100644 index 000000000..d3f0d2f3a --- /dev/null +++ b/ddprof-lib/benchmarks/build.gradle.kts @@ -0,0 +1,83 @@ +/* + * Benchmark for testing unwinding failures. + * Uses NativeCompileTask/NativeLinkExecutableTask from build-logic. + */ + +import com.datadoghq.native.model.Platform +import com.datadoghq.native.tasks.NativeCompileTask +import com.datadoghq.native.tasks.NativeLinkExecutableTask +import com.datadoghq.native.util.PlatformUtils + +plugins { + base + // Note: Does NOT use native-build plugin - tasks are created manually below + // because this project has a single benchmark executable, not the standard + // multi-config library structure +} + +val benchmarkName = "unwind_failures_benchmark" + +// Determine if we should build for this platform +val shouldBuild = PlatformUtils.currentPlatform == Platform.MACOS || + PlatformUtils.currentPlatform == Platform.LINUX + +if (shouldBuild) { + val compiler = PlatformUtils.findCompiler(project) + + // Compile task + val compileTask = tasks.register("compileBenchmark") { + onlyIf { shouldBuild && !project.hasProperty("skip-native") } + group = "build" + description = "Compile the unwinding failures benchmark" + + this.compiler.set(compiler) + compilerArgs.set(listOf("-O2", "-g", "-std=c++17")) + sources.from(file("src/unwindFailuresBenchmark.cpp")) + includes.from(project(":ddprof-lib").file("src/main/cpp")) + objectFileDir.set(file("${layout.buildDirectory.get()}/obj/benchmark")) + } + + // Link task + val binary = file("${layout.buildDirectory.get()}/bin/$benchmarkName") + val linkTask = tasks.register("linkBenchmark") { + onlyIf { shouldBuild && !project.hasProperty("skip-native") } + dependsOn(compileTask) + group = "build" + description = "Link the unwinding failures benchmark" + + linker.set(compiler) + val args = mutableListOf("-ldl", "-lpthread") + if (PlatformUtils.currentPlatform == Platform.LINUX) { + args.add("-lrt") + } + linkerArgs.set(args) + objectFiles.from(fileTree("${layout.buildDirectory.get()}/obj/benchmark") { include("*.o") }) + outputFile.set(binary) + } + + // Wire linkBenchmark into the standard assemble lifecycle + tasks.named("assemble") { + dependsOn(linkTask) + } + + // Add a task to run the benchmark + tasks.register("runBenchmark") { + dependsOn(linkTask) + group = "verification" + description = "Run the unwinding failures benchmark" + + executable = binary.absolutePath + + // Add any additional arguments passed to the Gradle task + doFirst { + if (project.hasProperty("args")) { + args(project.property("args").toString().split(" ")) + } + println("Running benchmark: ${binary.absolutePath}") + } + + doLast { + println("Benchmark completed.") + } + } +} diff --git a/ddprof-lib/build.gradle b/ddprof-lib/build.gradle deleted file mode 100644 index fc303fd53..000000000 --- a/ddprof-lib/build.gradle +++ /dev/null @@ -1,629 +0,0 @@ -plugins { - id 'cpp-library' - id 'java' - id 'maven-publish' - id 'signing' - - id 'com.github.ben-manes.versions' version '0.27.0' - id 'de.undercouch.download' version '4.1.1' -} - -// Helper function to check if objcopy is available -def checkObjcopyAvailable() { - try { - def process = ['objcopy', '--version'].execute() - process.waitFor() - return process.exitValue() == 0 - } catch (Exception e) { - return false - } -} - -// Helper function to check if dsymutil is available (for macOS) -def checkDsymutilAvailable() { - try { - def process = ['dsymutil', '--version'].execute() - process.waitFor() - return process.exitValue() == 0 - } catch (Exception e) { - return false - } -} - -// Helper function to check if debug extraction should be skipped -def shouldSkipDebugExtraction() { - // Skip if explicitly disabled - if (project.hasProperty('skip-debug-extraction')) { - return true - } - - // Skip if required tools are not available - if (os().isLinux() && !checkObjcopyAvailable()) { - return true - } - - if (os().isMacOsX() && !checkDsymutilAvailable()) { - return true - } - - return false -} - -// Helper function to get debug file path for a given config -def getDebugFilePath(config) { - def extension = os().isLinux() ? 'so' : 'dylib' - return file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/debug/libjavaProfiler.${extension}.debug") -} - -// Helper function to get stripped file path for a given config -def getStrippedFilePath(config) { - def extension = os().isLinux() ? 'so' : 'dylib' - return file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/stripped/libjavaProfiler.${extension}") -} - -// Helper function to create error message for missing tools -def getMissingToolErrorMessage(toolName, installInstructions) { - return """ - |${toolName} is not available but is required for split debug information. - | - |To fix this issue: - |${installInstructions} - | - |If you want to build without split debug info, set -Pskip-debug-extraction=true - """.stripMargin() -} - -// Helper function to create debug extraction task -def createDebugExtractionTask(config, linkTask) { - return tasks.register('extractDebugLibRelease', Exec) { - onlyIf { - !shouldSkipDebugExtraction() - } - dependsOn linkTask - description = 'Extract debug symbols from release library' - workingDir project.buildDir - - // Declare outputs so Gradle knows what files this task creates - outputs.file getDebugFilePath(config) - - doFirst { - def sourceFile = linkTask.get().linkedFile.get().asFile - def debugFile = getDebugFilePath(config) - - // Ensure debug directory exists - debugFile.parentFile.mkdirs() - - // Set the command line based on platform - if (os().isLinux()) { - commandLine = ['objcopy', '--only-keep-debug', sourceFile.absolutePath, debugFile.absolutePath] - } else { - // For macOS, we'll use dsymutil instead - commandLine = ['dsymutil', sourceFile.absolutePath, '-o', debugFile.absolutePath.replace('.debug', '.dSYM')] - } - } - } -} - -// Helper function to create debug link task (Linux only) -def createDebugLinkTask(config, linkTask, extractDebugTask) { - return tasks.register('addDebugLinkLibRelease', Exec) { - onlyIf { - os().isLinux() && !shouldSkipDebugExtraction() - } - dependsOn extractDebugTask - description = 'Add debug link to the original library' - - inputs.files linkTask, extractDebugTask - outputs.file { linkTask.get().linkedFile.get().asFile } - - doFirst { - def sourceFile = linkTask.get().linkedFile.get().asFile - def debugFile = getDebugFilePath(config) - - commandLine = ['objcopy', '--add-gnu-debuglink=' + debugFile.absolutePath, sourceFile.absolutePath] - } - } -} - -// Helper function to create debug file copy task -def createDebugCopyTask(config, extractDebugTask) { - return tasks.register('copyReleaseDebugFiles', Copy) { - onlyIf { - !shouldSkipDebugExtraction() - } - dependsOn extractDebugTask - from file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/debug") - into file(libraryTargetPath(config.name)) - include '**/*.debug' - include '**/*.dSYM/**' - } -} - -// Main function to setup debug extraction for release builds -def setupDebugExtraction(config, linkTask) { - if (config.name == 'release' && config.active && !project.hasProperty('skip-native')) { - // Create all debug-related tasks - def extractDebugTask = createDebugExtractionTask(config, linkTask) - def addDebugLinkTask = createDebugLinkTask(config, linkTask, extractDebugTask) - - // Create the strip task and configure it properly - def stripTask = tasks.register('stripLibRelease', StripSymbols) { - // No onlyIf needed here - setupDebugExtraction already handles the main conditions - dependsOn addDebugLinkTask - } - - // Configure the strip task after registration - stripTask.configure { - targetPlatform = linkTask.get().targetPlatform - toolChain = linkTask.get().toolChain - binaryFile = linkTask.get().linkedFile.get().asFile - outputFile = getStrippedFilePath(config) - } - - def copyDebugTask = createDebugCopyTask(config, extractDebugTask) - - // Wire up the copy task to use stripped binaries - def copyTask = tasks.findByName("copyReleaseLibs") - if (copyTask != null) { - copyTask.dependsOn stripTask - copyTask.inputs.files stripTask.get().outputs.files - - // Create an extra folder for the debug symbols - copyTask.dependsOn copyDebugTask - } - } -} - -def libraryName = "ddprof" - -description = "Datadog Java Profiler Library" - -def component_version = project.hasProperty("ddprof_version") ? project.ddprof_version : project.version - -// this feels weird but it is the only way invoking `./gradlew :ddprof-lib:*` tasks will work -if (rootDir.toString().endsWith("ddprof-lib")) { - apply from: rootProject.file('../common.gradle') -} - -dependencies { - if (os().isLinux()) { - // the malloc shim works only on linux - project(':malloc-shim') - } - project(':ddprof-lib:gtest') -} - -// Add a task to run all benchmarks -tasks.register('runBenchmarks') { - dependsOn ':ddprof-lib:benchmarks:runBenchmark' - group = 'verification' - description = 'Run all benchmarks' -} - -test { - onlyIf { - !project.hasProperty('skip-tests') - } - useJUnitPlatform() -} - -def libraryTargetBase(type) { - return "${projectDir}/build/native/${type}" -} - -def osarchext() { - if (osIdentifier() == 'linux' && archIdentifier() != 'x64') { - // when built on aarch64 the location the library is built in is 'x86-64' ¯\_(ツ)_/¯ - return "x86-64" - } else if (osIdentifier() == 'macos') { - return archIdentifier() == 'x64' ? 'x86-64' : 'arm64' - } else { - return archIdentifier() - } -} - -def libraryTargetPath(type) { - return "${libraryTargetBase(type)}/META-INF/native-libs/${osIdentifier()}-${archIdentifier()}${isMusl() ? '-musl' : ''}" -} - -def librarySourcePath(type, qualifier = "") { - return "${projectDir}/build/lib/main/${type}/${osIdentifier()}/${archIdentifier()}/${qualifier}/libjavaProfiler.${osIdentifier() == 'macos' ? 'dylib' : 'so'}" -} - -ext { - libraryTargetBase = this.&libraryTargetBase - libraryTargetPath = this.&libraryTargetPath - librarySourcePath = this.&librarySourcePath -} - -sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 - -// Configure Java 9+ source set for multi-release JAR -sourceSets { - java9 { - java { - srcDirs = ['src/main/java9'] - } - } -} - -def current = JavaVersion.current().majorVersion.toInteger() -def requested = current >= 11 ? current : 11 - -// Configure Java 9 compilation with Java 11 toolchain -tasks.named('compileJava9Java') { - javaCompiler = javaToolchains.compilerFor { - languageVersion = JavaLanguageVersion.of(requested) - } - options.release = 9 - - // Add main source set output to classpath - classpath = sourceSets.main.output + configurations.compileClasspath - dependsOn tasks.named('compileJava') -} - -def isGitlabCI = System.getenv("GITLAB_CI") != null -def buildTempDir = "${projectDir}/build/tmp" - -// Allow specifying the external location for the native libraries -// The libraries should be properly sorted into subfolders corresponding to the `libraryTargetPath` value for each -// os/arch/libc combination -tasks.register('copyExternalLibs', Copy) { - if (project.hasProperty("with-libs")) { - from(project.getProperty("with-libs")) { - include "**/*.so" - include "**/*.dylib" - include "**/*.debug" - include "**/*.dSYM/**" - } - into "${projectDir}/build/classes/java/main/META-INF/native-libs" - } -} - -tasks.register('assembleAll') {} - -// use the build config names to create configurations, copy lib and asemble jar tasks -buildConfigNames().each { name -> - configurations.create(name) { - canBeConsumed = true - canBeResolved = false - extendsFrom configurations.implementation } - - def copyTask = tasks.register("copy${name.capitalize()}Libs", Copy) { - from file(librarySourcePath(name, name == 'release' ? 'stripped' : '')).parent // the release build is stripped - into file(libraryTargetPath(name)) - - if (name == 'release') { - def stripTask = tasks.findByName('stripLibRelease') - if (stripTask != null) { - dependsOn stripTask - } - } - } - def assembleJarTask = tasks.register("assemble${name.capitalize()}Jar", Jar) { - group = 'build' - description = "Assemble the ${name} build of the library" - dependsOn copyExternalLibs - dependsOn tasks.named('compileJava9Java') - if (!project.hasProperty('skip-native')) { - dependsOn copyTask - } - - if (name == 'debug') { - manifest { - attributes 'Premain-Class': 'com.datadoghq.profiler.Main' - } - } - - from sourceSets.main.output.classesDirs - from sourceSets.java9.output.classesDirs - from files(libraryTargetBase(name)) { - include "**/*" - } - archiveBaseName = libraryName - archiveClassifier = name == 'release' ? '' : name // the release qualifier is empty - archiveVersion = component_version - } - // We need this second level indirection such that we can make the assembling dependent on the tests - // The catch is that the test tasks depend on the assembled jar so we need a wrapper assemble task instead - def assembleTask = tasks.register("assemble${name.capitalize()}", Task) { - dependsOn assembleJarTask - } - - tasks.assembleAll.dependsOn assembleTask -} -configurations { - // the 'all' configuration is used to aggregate all the build configurations - assembled { - canBeConsumed = true - canBeResolved = false - extendsFrom implementation - } -} - -// We need this trickery to reuse the toolchain and system config from tasks created by the cpp-library plugin -// Basically, we are listening when the default 'comile' and 'link' (eg. 'compileReleaseCpp') is added and then -// we are adding our own tasks for each build configuration, inheriting the part of the configuration which was -// added by the cpp-library plugin -tasks.whenTaskAdded { task -> - if (task instanceof CppCompile) { - if (!task.name.startsWith('compileLib') && task.name.contains('Release')) { - buildConfigurations.each { config -> - if (config.os == osIdentifier() && config.arch == archIdentifier()) { - def cppTask = tasks.register("compileLib${config.name.capitalize()}", CppCompile) { - onlyIf { - config.active - } - group = 'build' - description = "Compile the ${config.name} build of the library" - objectFileDir = file("$buildDir/obj/main/${config.name}") - compilerArgs.addAll(config.compilerArgs) - if (os().isLinux() && isMusl()) { - compilerArgs.add('-D__musl__') - } - toolChain = task.toolChain - targetPlatform = task.targetPlatform - includes task.includes - includes project(':ddprof-lib').file('src/main/cpp').toString() - includes "${javaHome()}/include" - includes project(':malloc-shim').file('src/main/public').toString() - if (os().isMacOsX()) { - includes "${javaHome()}/include/darwin" - } else if (os().isLinux()) { - includes "${javaHome()}/include/linux" - } - systemIncludes.from task.systemIncludes - source task.source - inputs.files source - outputs.dir objectFileDir - } - def linkTask = tasks.findByName("linkLib${config.name.capitalize()}".toString()) - if (linkTask != null) { - linkTask.dependsOn cppTask - } - } - } - } - } else if (task instanceof LinkSharedLibrary) { - if (!task.name.startsWith('linkLib') && task.name.contains('Release')) { - buildConfigurations.each { config -> - if (config.os == osIdentifier() && config.arch == archIdentifier()) { - def linkTask = tasks.register("linkLib${config.name.capitalize()}", LinkSharedLibrary) { - onlyIf { - config.active - } - group = 'build' - description = "Link the ${config.name} build of the library" - source = fileTree("$buildDir/obj/main/${config.name}") - linkedFile = file("$buildDir/lib/main/${config.name}/${osIdentifier()}/${archIdentifier()}/libjavaProfiler.${os().isLinux() ? 'so' : 'dylib'}") - def compileTask = tasks.findByName("compileLib${config.name.capitalize()}".toString()) - if (compileTask != null) { - dependsOn compileTask - } - linkerArgs.addAll(config.linkerArgs) - toolChain = task.toolChain - targetPlatform = task.targetPlatform - libs = task.libs - inputs.files source - outputs.file linkedFile - } - if (config.name == 'release') { - setupDebugExtraction(config, linkTask) - } - } - } - } - } -} - -// configure the compiler here -tasks.withType(CppCompile).configureEach { - if (name.startsWith('compileRelease') || name.startsWith('compileDebug')) { - onlyIf { - // disable the built-in compiler task for release; we are using the custom compiler task - false - } - } else { - onlyIf { - !project.hasProperty('skip-native') - } - } -} - -// configure linker -tasks.withType(LinkSharedLibrary).configureEach { - if (name.startsWith('linkRelease') || name.startsWith('linkDebug')) { - onlyIf { - // disable the built-in linker task for release; we are using the custom linker task - false - } - } else { - onlyIf { - !project.hasProperty('skip-native') - } - } -} - -library { - baseName = "javaProfiler" - source.from file('src/main/cpp') - privateHeaders.from file('src/main/cpp') - - // aarch64 support is still incubating - // for the time being an aarch64 linux machine will match 'machines.linux.x86_64' - targetMachines = [machines.macOS, machines.linux.x86_64] - linkage = [Linkage.SHARED] -} - -tasks.withType(StripSymbols).configureEach { - onlyIf { - name == ("stripLibRelease") && !project.hasProperty('skip-native') - } -} - -jar { - dependsOn copyExternalLibs - dependsOn tasks.named('compileJava9Java') -} - -tasks.register('sourcesJar', Jar) { - from sourceSets.main.allJava - from sourceSets.java9.allJava - archiveBaseName = libraryName - archiveClassifier = "sources" - archiveVersion = component_version -} - -tasks.withType(Javadoc).configureEach { - // Allow javadoc to access internal sun.nio.ch package used by BufferWriter8 - options.addStringOption('-add-exports', 'java.base/sun.nio.ch=ALL-UNNAMED') -} - -tasks.register('javadocJar', Jar) { - dependsOn javadoc - archiveBaseName = libraryName - archiveClassifier = 'javadoc' - archiveVersion = component_version - from javadoc.destinationDir -} - - - -tasks.register('scanBuild', Exec) { - workingDir "${projectDir}/src/test/make" - commandLine 'scan-build' - args "-o", "${projectDir}/build/reports/scan-build", - "--force-analyze-debug-code", - "--use-analyzer", "/usr/bin/clang++", - "make", "-j4", "all" -} - -tasks.withType(Test) { - onlyIf { - !project.hasProperty('skip-tests') - } - def javaHome = System.getenv("JAVA_TEST_HOME") - if (javaHome == null) { - javaHome = System.getenv("JAVA_HOME") - } - executable = file("${javaHome}/bin/java") - javaLauncher.set(javaToolchains.launcherFor { - languageVersion = JavaLanguageVersion.of(11) - }) -} - -// relink the tasks when all are created -gradle.projectsEvaluated { - buildConfigNames().each { - def compileTask = tasks.findByName("compileLib${it.capitalize()}") - def linkTask = tasks.findByName("linkLib${it.capitalize()}") - if (linkTask != null) { - if (it != 'release') { - def copyTask = tasks.findByName("copy${it.capitalize()}Libs") - if (copyTask != null) { - copyTask.dependsOn linkTask - } - } - } - def javadocTask = tasks.findByName("javadoc") - def copyReleaseLibs = tasks.findByName("copyReleaseLibs") - if (javadocTask != null && copyReleaseLibs != null) { - javadocTask.dependsOn copyReleaseLibs - } - } -} - -artifacts { - // create artifacts for all configures build config names - buildConfigNames().each { - def task = tasks.named("assemble${it.capitalize()}Jar") - artifacts.add('assembled', task) - artifacts.add(it, task) - } -} - -publishing { - publications { - assembled(MavenPublication) { publication -> - buildConfigNames().each { - publication.artifact tasks.named("assemble${it.capitalize()}Jar") - } - publication.artifact sourcesJar - publication.artifact javadocJar - - publication.groupId = 'com.datadoghq' - publication.artifactId = 'ddprof' - } - } -} - -tasks.withType(GenerateMavenPom).configureEach { - doFirst { - MavenPom pom = it.pom - pom.name = project.name - pom.description = "${project.description} (${component_version})" - pom.packaging = "jar" - pom.url = "https://github.com/datadog/java-profiler" - pom.licenses { - license { - name = "The Apache Software License, Version 2.0" - url = "http://www.apache.org/licenses/LICENSE-2.0.txt" - distribution = "repo" - } - } - pom.scm { - connection = "scm:https://datadog@github.com/datadog/java-profiler" - developerConnection = "scm:git@github.com:datadog/java-profiler" - url = "https://github.com/datadog/java-profiler" - } - pom.developers { - developer { - id = "datadog" - name = "Datadog" - } - } - } -} - -signing { - useInMemoryPgpKeys(System.getenv("GPG_PRIVATE_KEY"), System.getenv("GPG_PASSWORD")) - sign publishing.publications.assembled -} - -tasks.withType(Sign).configureEach { - // Only sign in Gitlab CI - onlyIf { isGitlabCI || (System.getenv("GPG_PRIVATE_KEY") != null && System.getenv("GPG_PASSWORD") != null) } -} - -/** - * State assertions below... - */ - -gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph -> - if (taskGraph.hasTask(publish) || taskGraph.hasTask("publishToSonatype")) { - assert project.findProperty("removeJarVersionNumbers") != true - if (taskGraph.hasTask("publishToSonatype")) { - assert System.getenv("SONATYPE_USERNAME") != null - assert System.getenv("SONATYPE_PASSWORD") != null - if (isCI) { - assert System.getenv("GPG_PRIVATE_KEY") != null - assert System.getenv("GPG_PASSWORD") != null - } - } - } -} - -afterEvaluate { - assert description: "Project $project.path is published, must have a description" -} - -// we are publishing very customized artifacts - we are attaching the native library to the resulting JAR artifact -tasks.withType(AbstractPublishToMaven).configureEach { - if (it.name.contains('AssembledPublication')) { - it.dependsOn assembleReleaseJar - } - rootProject.subprojects { - mustRunAfter tasks.matching { it instanceof VerificationTask } - } -} diff --git a/ddprof-lib/build.gradle.kts b/ddprof-lib/build.gradle.kts new file mode 100644 index 000000000..b5c04397e --- /dev/null +++ b/ddprof-lib/build.gradle.kts @@ -0,0 +1,181 @@ +import com.datadoghq.native.model.Platform +import com.datadoghq.native.util.PlatformUtils + +plugins { + java + `maven-publish` + signing + id("com.github.ben-manes.versions") version "0.27.0" + id("de.undercouch.download") version "4.1.1" + id("com.datadoghq.native-build") + id("com.datadoghq.gtest") + id("com.datadoghq.scanbuild") + id("com.datadoghq.versioned-sources") +} + +val libraryName = "ddprof" +description = "Datadog Java Profiler Library" + +val componentVersion = findProperty("ddprof_version") as? String ?: version.toString() + +// Configure native build with the new plugin +nativeBuild { + version.set(componentVersion) + cppSourceDirs.set(listOf("src/main/cpp")) + includeDirectories.set( + listOf( + "src/main/cpp", + "${project(":malloc-shim").file("src/main/public")}" + ) + ) +} + +// Configure Google Test +gtest { + testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) + mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) + + // Include paths for compilation + val javaHome = PlatformUtils.javaHome() + val platformInclude = when (PlatformUtils.currentPlatform) { + Platform.LINUX -> "linux" + Platform.MACOS -> "darwin" + } + + includes.from( + "src/main/cpp", + "$javaHome/include", + "$javaHome/include/$platformInclude", + project(":malloc-shim").file("src/main/public") + ) +} + +// Java configuration - using sourceCompatibility (not --release 8) +// because BufferWriter8 needs access to internal sun.nio.ch package +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} + +// Configure versioned sources for runtime version-specific implementations +versionedSources { + versions { + register("java9") { + release.set(9) + minToolchainVersion.set(11) // Compile Java 9 code with JDK 11+ + } + } +} + +// Test configuration +tasks.test { + onlyIf { + !project.hasProperty("skip-tests") + } + useJUnitPlatform() +} + +// Copy external libs task +val copyExternalLibs by tasks.registering(Copy::class) { + if (project.hasProperty("with-libs")) { + from(project.property("with-libs") as String) { + include("**/*.so", "**/*.dylib", "**/*.debug", "**/*.dSYM/**") + } + into("$projectDir/build/classes/java/main/META-INF/native-libs") + } +} + +// Create JAR tasks for each build configuration using nativeBuild extension utilities +// Uses afterEvaluate to discover configurations dynamically from NativeBuildExtension +afterEvaluate { + nativeBuild.buildConfigurations.names.forEach { name -> + val capitalizedName = name.replaceFirstChar { it.uppercase() } + + val copyTask = tasks.register("copy${capitalizedName}Libs", Copy::class) { + from(nativeBuild.librarySourceDir(name)) { + // Exclude debug symbols from production JAR + exclude("debug/**", "*.debug", "*.dSYM/**") + } + into(nativeBuild.libraryTargetDir(name)) + + // Ensure library is built before copying (link task created by NativeBuildPlugin) + val linkTaskName = "link$capitalizedName" + if (tasks.names.contains(linkTaskName)) { + dependsOn(linkTaskName) + } + } + + val assembleJarTask = tasks.register("assemble${capitalizedName}Jar", Jar::class) { + group = "build" + description = "Assemble the $name build of the library" + dependsOn(copyExternalLibs) + + if (!project.hasProperty("skip-native")) { + dependsOn(copyTask) + } + + if (name == "debug") { + manifest { + attributes("Premain-Class" to "com.datadoghq.profiler.Main") + } + } + + from(sourceSets.main.get().output.classesDirs) + versionedSources.configureJar(this) + from(nativeBuild.libraryTargetBase(name)) { + include("**/*") + // Exclude debug symbols from production JAR + exclude("**/debug/**", "**/*.debug", "**/*.dSYM/**") + } + archiveBaseName.set(libraryName) + archiveClassifier.set(if (name == "release") "" else name) + archiveVersion.set(componentVersion) + } + + // Create consumable configuration for inter-project dependencies + // This allows other projects to depend on specific build configurations + configurations.create(name) { + isCanBeConsumed = true + isCanBeResolved = false + outgoing.artifact(assembleJarTask) + } + } +} + +// Add runBenchmarks task +tasks.register("runBenchmarks") { + dependsOn(":ddprof-lib:benchmarks:runBenchmark") + group = "verification" + description = "Run all benchmarks" +} + +// Standard JAR task +tasks.jar { + dependsOn(copyExternalLibs) +} + +// Source JAR +val sourcesJar by tasks.registering(Jar::class) { + from(sourceSets.main.get().allJava) + versionedSources.configureSourceJar(this) + archiveBaseName.set(libraryName) + archiveClassifier.set("sources") + archiveVersion.set(componentVersion) +} + +// Javadoc configuration +tasks.withType { + // Allow javadoc to access internal sun.nio.ch package used by BufferWriter8 + (options as StandardJavadocDocletOptions).addStringOption("-add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") +} + +// Javadoc JAR +val javadocJar by tasks.registering(Jar::class) { + dependsOn(tasks.javadoc) + archiveBaseName.set(libraryName) + archiveClassifier.set("javadoc") + archiveVersion.set(componentVersion) + from(tasks.javadoc.get().destinationDir) +} + +// Publishing configuration will be added later diff --git a/ddprof-lib/fuzz/build.gradle b/ddprof-lib/fuzz/build.gradle deleted file mode 100644 index ed706cb0e..000000000 --- a/ddprof-lib/fuzz/build.gradle +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * Gradle build file for libFuzzer-based fuzz testing. - * This module compiles and runs fuzz targets against the profiler's C++ code - * to discover bugs through automated input generation. - */ - -plugins { - id 'cpp-application' -} - -// Access to common utilities and build configurations -if (rootDir.toString().endsWith("ddprof-lib/fuzz")) { - apply from: rootProject.file('../../common.gradle') - apply from: rootProject.file('../../gradle/configurations.gradle') -} - -// disable the default compile and link tasks not to interfere with our custom ones -tasks.withType(CppCompile).configureEach { task -> - if (task.name.startsWith('compileRelease') || task.name.startsWith('compileDebug')) { - task.onlyIf { - false - } - } -} - -tasks.withType(LinkExecutable).configureEach { task -> - if (task.name.startsWith('linkRelease') || task.name.startsWith('linkDebug')) { - task.onlyIf { - false - } - } -} - -tasks.withType(ExtractSymbols).configureEach { task -> - task.onlyIf { - false - } -} - -tasks.withType(StripSymbols).configureEach { task -> - task.onlyIf { - false - } -} - -// Default fuzz duration in seconds (can be overridden with -Pfuzz-duration=N) -def fuzzDuration = project.hasProperty('fuzz-duration') ? project.getProperty('fuzz-duration').toInteger() : 60 - -// Directory for crash artifacts -def crashDir = file("${buildDir}/fuzz-crashes") - -// Directory for seed corpus -def corpusDir = project(':ddprof-lib').file('src/test/fuzz/corpus') - -// Helper to detect Homebrew LLVM on macOS -def findHomebrewLLVM() { - if (!os().isMacOsX()) { - return null - } - - def possiblePaths = ["/opt/homebrew/opt/llvm", // Apple Silicon - "/usr/local/opt/llvm" // Intel Mac - ] - - for (path in possiblePaths) { - def llvmDir = file(path) - if (llvmDir.exists() && file("${path}/bin/clang++").exists()) { - logger.info("Found Homebrew LLVM at: ${path}") - return path - } - } - - // Try using brew command - try { - def process = ["brew", "--prefix", "llvm"].execute() - process.waitFor() - if (process.exitValue() == 0) { - def brewPath = process.in.text.trim() - if (file("${brewPath}/bin/clang++").exists()) { - logger.info("Found Homebrew LLVM via brew command at: ${brewPath}") - return brewPath - } - } - } catch (Exception e) { - // brew not available or failed - } - - return null -} - -def homebrewLLVM = findHomebrewLLVM() - -// Find the clang version directory within Homebrew LLVM -def findClangResourceDir(String llvmPath) { - if (llvmPath == null) { - return null - } - - def clangLibDir = file("${llvmPath}/lib/clang") - if (!clangLibDir.exists()) { - return null - } - - // Find the version directory (e.g., 18.1.8 or 19) - def versions = clangLibDir.listFiles()?.findAll { it.isDirectory() }?.sort { a, b -> - b.name <=> a.name // Sort descending to get latest version - } - - if (versions && versions.size() > 0) { - def resourceDir = "${llvmPath}/lib/clang/${versions[0].name}" - logger.info("Using clang resource directory: ${resourceDir}") - return resourceDir - } - - return null -} - -def clangResourceDir = findClangResourceDir(homebrewLLVM) - -def fuzzAll = tasks.register("fuzz") { - onlyIf { - hasFuzzer() && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native') && !project.hasProperty('skip-fuzz') - } - group = 'verification' - description = "Run all fuzz targets" - - if (!hasFuzzer()) { - logger.warn("WARNING: libFuzzer not available - skipping fuzz tests (requires clang with -fsanitize=fuzzer)") - } -} - -// We need this trickery to reuse the toolchain and system config from tasks created by the cpp-application plugin -tasks.whenTaskAdded { task -> - if (task instanceof CppCompile) { - if (!task.name.startsWith('compileFuzz') && task.name.contains('Release')) { - // Only create fuzz tasks for the 'fuzzer' configuration - buildConfigurations.findAll { it.name == 'fuzzer' }.each { config -> - if (config.os == osIdentifier() && config.arch == archIdentifier()) { - def fuzzSrcDir = project(':ddprof-lib').file("src/test/fuzz/") - if (fuzzSrcDir.exists()) { - fuzzSrcDir.eachFile { fuzzFile -> - if (fuzzFile.name.endsWith('.cpp')) { - def fullName = fuzzFile.name.substring(0, fuzzFile.name.lastIndexOf('.')) - // Strip "fuzz_" prefix from filename to get the target name - def fuzzName = fullName.startsWith('fuzz_') ? fullName.substring(5) : fullName - def fuzzCompileTask = tasks.register("compileFuzz_${fuzzName}", CppCompile) { - onlyIf { - config.active && hasFuzzer() && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native') && !project.hasProperty('skip-fuzz') - } - group = 'build' - description = "Compile the fuzz target ${fuzzName}" - objectFileDir = file("$buildDir/obj/fuzz/${fuzzName}") - // Use fuzzer config's compiler args, but we need to use clang - compilerArgs.addAll(config.compilerArgs.findAll { - // drop -std flag to add our own - it != '-std=c++17' && it != '-DNDEBUG' - }) - if (os().isLinux() && isMusl()) { - compilerArgs.add('-D__musl__') - } - compilerArgs.add('-std=c++17') - // Add fuzzer-specific compile flags (but not -fsanitize=fuzzer for compilation) - compilerArgs.add('-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION') - - toolChain = task.toolChain - targetPlatform = task.targetPlatform - includes task.includes - includes project(':ddprof-lib').file('src/main/cpp').toString() - includes "${javaHome()}/include" - includes project(':malloc-shim').file('src/main/public').toString() - if (os().isMacOsX()) { - includes "${javaHome()}/include/darwin" - } else if (os().isLinux()) { - includes "${javaHome()}/include/linux" - } - includes task.systemIncludes - // Compile main profiler sources (needed for fuzzing the actual code) - source project(':ddprof-lib').fileTree('src/main/cpp') { - include '**/*' - } - // Compile the fuzz target itself - source fuzzFile - - inputs.files source - outputs.dir objectFileDir - } - def linkTask = tasks.findByName("linkFuzz_${fuzzName}".toString()) - if (linkTask != null) { - linkTask.dependsOn fuzzCompileTask - } - } - } - } - } - } - } - } else if (task instanceof LinkExecutable) { - if (!task.name.startsWith('linkFuzz') && task.name.contains('Release')) { - buildConfigurations.findAll { it.name == 'fuzzer' }.each { config -> - if (config.os == osIdentifier() && config.arch == archIdentifier()) { - def fuzzSrcDir = project(':ddprof-lib').file("src/test/fuzz/") - if (fuzzSrcDir.exists()) { - fuzzSrcDir.eachFile { fuzzFile -> - if (fuzzFile.name.endsWith('.cpp')) { - def fullName = fuzzFile.name.substring(0, fuzzFile.name.lastIndexOf('.')) - // Strip "fuzz_" prefix from filename to get the target name - def fuzzName = fullName.startsWith('fuzz_') ? fullName.substring(5) : fullName - def binary = file("$buildDir/bin/fuzz/${fuzzName}/${fuzzName}") - def fuzzLinkTask = tasks.register("linkFuzz_${fuzzName}", LinkExecutable) { - onlyIf { - config.active && hasFuzzer() && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native') && !project.hasProperty('skip-fuzz') - } - group = 'build' - description = "Link the fuzz target ${fuzzName}" - source = fileTree("$buildDir/obj/fuzz/${fuzzName}") - linkedFile = binary - // Add linker args from config - linkerArgs.addAll(config.linkerArgs) - - // libFuzzer linking strategy: - // On macOS with Homebrew LLVM, explicitly link the library file - // because system clang looks in the wrong location - if (os().isMacOsX() && clangResourceDir != null) { - def fuzzerLib = "${clangResourceDir}/lib/darwin/libclang_rt.fuzzer_osx.a" - if (file(fuzzerLib).exists()) { - logger.info("Using Homebrew libFuzzer: ${fuzzerLib}") - // Explicitly link the fuzzer library file - linkerArgs.add(fuzzerLib) - // Also link Homebrew's libc++ to match the fuzzer library's ABI - linkerArgs.add("-L${homebrewLLVM}/lib/c++") - linkerArgs.add("-lc++") - linkerArgs.add("-Wl,-rpath,${homebrewLLVM}/lib/c++") - } else { - logger.warn("Homebrew libFuzzer not found, falling back to -fsanitize=fuzzer") - linkerArgs.add("-fsanitize=fuzzer") - } - } else { - // Standard libFuzzer linkage for Linux or when Homebrew not available - linkerArgs.add("-fsanitize=fuzzer") - } - - linkerArgs.addAll("-ldl", "-lpthread", "-lm") - if (os().isLinux()) { - linkerArgs.add("-lrt") - } - toolChain = task.toolChain - targetPlatform = task.targetPlatform - libs = task.libs - inputs.files source - outputs.file linkedFile - } - - // Create corpus directory for this fuzz target - def targetCorpusDir = file("${corpusDir}/${fuzzName}") - - def fuzzExecuteTask = tasks.register("fuzz_${fuzzName}", Exec) { - onlyIf { - config.active && hasFuzzer() && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native') && !project.hasProperty('skip-fuzz') - } - group = 'verification' - description = "Run the fuzz target ${fuzzName} for ${fuzzDuration} seconds" - dependsOn fuzzLinkTask - - doFirst { - // Ensure crash directory exists - crashDir.mkdirs() - // Ensure corpus directory exists (even if empty) - targetCorpusDir.mkdirs() - } - - executable binary - // libFuzzer arguments: - // - corpus directory (positional) - // - max_total_time: stop after N seconds - // - artifact_prefix: where to save crash files - // - print_final_stats: show coverage stats at end - args targetCorpusDir.absolutePath, - "-max_total_time=${fuzzDuration}", - "-artifact_prefix=${crashDir.absolutePath}/${fuzzName}-", - "-print_final_stats=1" - - config.testEnv.each { key, value -> - environment key, value - } - - inputs.files binary - // Fuzz tasks should run every time - outputs.upToDateWhen { false } - } - - def compileTask = tasks.findByName("compileFuzz_${fuzzName}") - if (compileTask != null) { - fuzzLinkTask.get().dependsOn compileTask - } - fuzzAll.get().dependsOn fuzzExecuteTask.get() - } - } - } - } - } - } - } -} - -// Task to list available fuzz targets -tasks.register("listFuzzTargets") { - group = 'help' - description = "List all available fuzz targets" - doLast { - def fuzzSrcDir = project(':ddprof-lib').file("src/test/fuzz/") - if (fuzzSrcDir.exists()) { - println "Available fuzz targets:" - fuzzSrcDir.eachFile { fuzzFile -> - if (fuzzFile.name.endsWith('.cpp')) { - def fullName = fuzzFile.name.substring(0, fuzzFile.name.lastIndexOf('.')) - // Strip "fuzz_" prefix from filename to get the target name - def fuzzName = fullName.startsWith('fuzz_') ? fullName.substring(5) : fullName - println " - fuzz_${fuzzName}" - } - } - println "" - println "Run individual targets with: ./gradlew :ddprof-lib:fuzz:fuzz_" - println "Run all targets with: ./gradlew :ddprof-lib:fuzz:fuzz" - println "Configure duration with: -Pfuzz-duration= (default: 60)" - } else { - println "No fuzz targets found. Create .cpp files in ddprof-lib/src/test/fuzz/" - } - } -} diff --git a/ddprof-lib/fuzz/build.gradle.kts b/ddprof-lib/fuzz/build.gradle.kts new file mode 100644 index 000000000..c47eb472d --- /dev/null +++ b/ddprof-lib/fuzz/build.gradle.kts @@ -0,0 +1,28 @@ +/* + * Gradle build file for libFuzzer-based fuzz testing. + * This module compiles and runs fuzz targets against the profiler's C++ code + * to discover bugs through automated input generation. + */ + +plugins { + base + id("com.datadoghq.fuzz-targets") +} + +fuzzTargets { + // Source directory containing fuzz target files (fuzz_*.cpp) + fuzzSourceDir.set(project(":ddprof-lib").file("src/test/fuzz")) + + // Seed corpus directory for each target (subdirectories named by target) + corpusDir.set(project(":ddprof-lib").file("src/test/fuzz/corpus")) + + // Main profiler sources to compile with fuzz targets + profilerSourceDir.set(project(":ddprof-lib").file("src/main/cpp")) + + // Additional include directories + additionalIncludes.set( + listOf( + project(":malloc-shim").file("src/main/public").absolutePath + ) + ) +} diff --git a/ddprof-lib/gtest/build.gradle b/ddprof-lib/gtest/build.gradle deleted file mode 100644 index d7ee20272..000000000 --- a/ddprof-lib/gtest/build.gradle +++ /dev/null @@ -1,210 +0,0 @@ -plugins { - id 'cpp-application' -} - -// this feels weird but it is the only way invoking `./gradlew :ddprof-lib:*` tasks will work -if (rootDir.toString().endsWith("ddprof-lib/gradle")) { - apply from: rootProject.file('../../common.gradle') -} - - -// disable the default compile and link tasks not to interfere with our custom ones -tasks.withType(CppCompile).configureEach { task -> - if (task.name.startsWith('compileRelease') || task.name.startsWith('compileDebug')) { - task.onlyIf { - false - } - } -} - -tasks.withType(LinkExecutable).configureEach { task -> - if (task.name.startsWith('linkRelease') || task.name.startsWith('linkDebug')) { - task.onlyIf { - false - } - } -} - -tasks.withType(ExtractSymbols).configureEach { task -> - task.onlyIf { - false - } -} - -tasks.withType(StripSymbols).configureEach { task -> - task.onlyIf { - false - } -} - -def buildNativeLibsTask = tasks.register("buildNativeLibs") { - group = 'build' - description = "Build the native libs for the Google Tests" - - onlyIf { - hasGtest && !project.hasProperty('skip-native') && !project.hasProperty('skip-gtest') && os().isLinux() - } - - def srcDir = project(':ddprof-lib').file('src/test/resources/native-libs') - def targetDir = project(':ddprof-lib').file('build/test/resources/native-libs/') - - // Move the exec calls to the execution phase - doLast { - srcDir.eachDir { dir -> - def libName = dir.name - def libDir = file("${targetDir}/${libName}") - def libSrcDir = file("${srcDir}/${libName}") - - exec { - commandLine "sh", "-c", """ - echo "Processing library: ${libName} @ ${libSrcDir}" - mkdir -p ${libDir} - cd ${libSrcDir} - make TARGET_DIR=${libDir} - """ - } - } - } - - inputs.files project(':ddprof-lib').files('src/test/resources/native-libs/**/*') - outputs.dir "${targetDir}" -} - -def gtestAll = tasks.register("gtest") { - onlyIf { - hasGtest && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native') && !project.hasProperty('skip-gtest') - } - group = 'verification' - description = "Run all Google Tests for all build configurations of the library" - - if (!hasGtest) { - logger.warn("WARNING: Google Test not found - skipping native tests") - } -} - -// we need this trickery to reuse the toolchain and system config from tasks created by the cpp-application plugin -tasks.whenTaskAdded { task -> - if (task instanceof CppCompile) { - if (!task.name.startsWith('compileGtest') && task.name.contains('Release')) { - buildConfigurations.each { config -> - if (config.os == osIdentifier() && config.arch == archIdentifier()) { - project(':ddprof-lib').file("src/test/cpp/").eachFile { - def testFile = it - def testName = it.name.substring(0, it.name.lastIndexOf('.')) - def gtestCompileTask = tasks.register("compileGtest${config.name.capitalize()}_${testName}", CppCompile) { - onlyIf { - config.active && hasGtest && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native') && !project.hasProperty('skip-gtest') - } - group = 'build' - description = "Compile the Google Test ${testName} for the ${config.name} build of the library" - objectFileDir = file("$buildDir/obj/gtest/${config.name}/${testName}") - compilerArgs.addAll(config.compilerArgs.findAll { - // need to drop the -std and -DNDEBUG flags because we need a different standard and assertions enabled - it != '-std=c++17' && it != '-DNDEBUG' - }) - if (os().isLinux() && isMusl()) { - compilerArgs.add('-D__musl__') - } - compilerArgs.add('-std=c++17') - toolChain = task.toolChain - targetPlatform = task.targetPlatform - includes task.includes - includes project(':ddprof-lib').file('src/main/cpp').toString() - includes "${javaHome()}/include" - includes project(':malloc-shim').file('src/main/public').toString() - if (os().isMacOsX()) { - includes "${javaHome()}/include/darwin" - includes "/opt/homebrew/opt/googletest/include/" - } else if (os().isLinux()) { - includes "${javaHome()}/include/linux" - } - includes task.systemIncludes - source project(':ddprof-lib').fileTree('src/main/cpp') { - include '**/*' - } - source testFile - - inputs.files source - outputs.dir objectFileDir - } - def linkTask = tasks.named("linkGtest${config.name.capitalize()}_${testName}") - if (linkTask != null) { - linkTask.get().dependsOn gtestCompileTask - } - } - } - } - } - } else if (task instanceof LinkExecutable) { - if (!task.name.startsWith('linkGtest') && task.name.contains('Release')) { - buildConfigurations.each { config -> - if (config.os == osIdentifier() && config.arch == archIdentifier()) { - def gtestTask = tasks.register("gtest${config.name.capitalize()}") { - group = 'verification' - description = "Run all Google Tests for the ${config.name} build of the library" - } - project(':ddprof-lib').file("src/test/cpp/").eachFile { - - def testFile = it - def testName = it.name.substring(0, it.name.lastIndexOf('.')) - def binary = file("$buildDir/bin/gtest/${config.name}_${testName}/${testName}") - def gtestLinkTask = tasks.register("linkGtest${config.name.capitalize()}_${testName}", LinkExecutable) { - onlyIf { - config.active && hasGtest && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native') - && !project.hasProperty('skip-gtest') - } - group = 'build' - description = "Link the Google Test for the ${config.name} build of the library" - source = fileTree("$buildDir/obj/gtest/${config.name}/${testName}") - linkedFile = binary - if (config.name != 'release') { - // linking the gtests using the minimizing release flags is making gtest unhappy - linkerArgs.addAll(config.linkerArgs) - } - linkerArgs.addAll("-lgtest", "-lgtest_main", "-lgmock", "-lgmock_main", "-ldl", "-lpthread", "-lm") - if (os().isMacOsX()) { - linkerArgs.addAll("-L/opt/homebrew/opt/googletest/lib") - } else { - linkerArgs.add("-lrt") - } - toolChain = task.toolChain - targetPlatform = task.targetPlatform - libs = task.libs - inputs.files source - outputs.file linkedFile - } - def gtestExecuteTask = tasks.register("gtest${config.name.capitalize()}_${testName}", Exec) { - onlyIf { - config.active && hasGtest && !project.hasProperty('skip-tests') && !project.hasProperty('skip-native') - && !project.hasProperty('skip-gtest') - } - group = 'verification' - description = "Run the Google Test ${testName} for the ${config.name} build of the library" - dependsOn gtestLinkTask - executable binary - - config.testEnv.each { key, value -> - environment key, value - } - - inputs.files binary - // Test tasks should run every time the test command is run - outputs.upToDateWhen { false } - } - - def compileTask = tasks.findByName("compileGtest${config.name.capitalize()}_${testName}") - if (compileTask != null) { - gtestLinkTask.dependsOn compileTask - } - gtestTask.get().dependsOn gtestExecuteTask.get() - if (os().isLinux()) { - // custom binaries for tests are built only on linux - gtestExecuteTask.get().dependsOn(buildNativeLibs) - } - gtestAll.get().dependsOn gtestExecuteTask.get() - } - } - } - } - } -} \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/os_linux.cpp b/ddprof-lib/src/main/cpp/os_linux.cpp index 496e0d27f..4232485f2 100644 --- a/ddprof-lib/src/main/cpp/os_linux.cpp +++ b/ddprof-lib/src/main/cpp/os_linux.cpp @@ -522,7 +522,7 @@ static bool readProcessCmdline(int pid, ProcessInfo* info) { size_t len = 0; ssize_t r; - while (r = read(fd, info->cmdline + len, max_read - len)) { + while ((r = read(fd, info->cmdline + len, max_read - len))) { if (r > 0) { len += (size_t)r; if (len == max_read) break; @@ -564,7 +564,7 @@ static bool readProcessStats(int pid, ProcessInfo* info) { size_t len = 0; ssize_t r; - while (r = read(fd, buffer + len, sizeof(buffer) - 1 - len)) { + while ((r = read(fd, buffer + len, sizeof(buffer) - 1 - len))) { if (r > 0) { len += (size_t)r; if (len == sizeof(buffer) - 1) break; diff --git a/ddprof-stresstest/build.gradle b/ddprof-stresstest/build.gradle deleted file mode 100644 index 14880b615..000000000 --- a/ddprof-stresstest/build.gradle +++ /dev/null @@ -1,71 +0,0 @@ -plugins { - id 'java' - id 'me.champeau.jmh' version '0.7.1' -} - -repositories { - mavenCentral() -} - -dependencies { - implementation project(path: ":ddprof-lib", configuration: 'debug') - implementation project(path: ":ddprof-test-tracer") - implementation 'org.openjdk.jmh:jmh-core:1.36' - implementation 'org.openjdk.jmh:jmh-generator-annprocess:1.36' -} - -sourceSets { - jmh { - java { - runtimeClasspath += project(':ddprof-lib').sourceSets.main.output - } - } -} - -jmh { - def javaTestHome = System.getenv("JAVA_TEST_HOME") - def javaHome = javaTestHome != null ? javaTestHome : System.getenv("JAVA_HOME") - - // Set the JVM executable - this is used by the JMH plugin to fork benchmark processes - jvm = "${javaHome}/bin/java" - - // Explicitly set fork to use external JVM (not Gradle's JVM) - fork = 3 - - iterations = 3 - timeOnIteration = '3s' - warmup = '1s' - warmupIterations = 3 -} - -// Configure all JMH-related JavaExec tasks to use the correct JDK -tasks.withType(JavaExec).matching { it.name.startsWith('jmh') }.configureEach { - def javaTestHome = System.getenv("JAVA_TEST_HOME") - def javaHome = javaTestHome != null ? javaTestHome : System.getenv("JAVA_HOME") - - executable = "${javaHome}/bin/java" -} - -jmhJar { - manifest { - attributes( - 'Main-Class': 'com.datadoghq.profiler.stresstest.Main' - ) - } - archiveFileName = "stresstests.jar" -} - -task runStressTests(type: Exec) { - dependsOn jmhJar - def javaHome = System.getenv("JAVA_TEST_HOME") - if (javaHome == null) { - javaHome = System.getenv("JAVA_HOME") - } - group = 'Execution' - description = 'Run JMH stresstests' - commandLine "${javaHome}/bin/java", '-jar', 'build/libs/stresstests.jar', '-prof', 'com.datadoghq.profiler.stresstest.WhiteboxProfiler', 'counters.*' -} - -tasks.withType(JavaCompile).configureEach { - options.compilerArgs.addAll(['--release', '8']) -} \ No newline at end of file diff --git a/ddprof-stresstest/build.gradle.kts b/ddprof-stresstest/build.gradle.kts new file mode 100644 index 000000000..42222b5bb --- /dev/null +++ b/ddprof-stresstest/build.gradle.kts @@ -0,0 +1,63 @@ +import com.datadoghq.native.util.PlatformUtils + +plugins { + java + id("me.champeau.jmh") version "0.7.1" + id("com.datadoghq.java-conventions") +} + +dependencies { + implementation(project(mapOf("path" to ":ddprof-lib", "configuration" to "debug"))) + implementation(project(":ddprof-test-tracer")) + implementation(libs.bundles.jmh) +} + +sourceSets { + named("jmh") { + java { + runtimeClasspath += project(":ddprof-lib").sourceSets["main"].output + } + } +} + +jmh { + // Set the JVM executable - this is used by the JMH plugin to fork benchmark processes + jvm.set(PlatformUtils.testJavaExecutable()) + + // Explicitly set fork to use external JVM (not Gradle's JVM) + fork.set(3) + + iterations.set(3) + timeOnIteration.set("3s") + warmup.set("1s") + warmupIterations.set(3) +} + +// Configure all JMH-related JavaExec tasks to use the correct JDK +tasks.withType().matching { it.name.startsWith("jmh") }.configureEach { + executable = PlatformUtils.testJavaExecutable() +} + +tasks.named("jmhJar") { + manifest { + attributes( + "Main-Class" to "com.datadoghq.profiler.stresstest.Main" + ) + } + archiveFileName.set("stresstests.jar") +} + +tasks.register("runStressTests") { + dependsOn(tasks.named("jmhJar")) + + group = "application" + description = "Run JMH stresstests" + commandLine( + PlatformUtils.testJavaExecutable(), + "-jar", + "build/libs/stresstests.jar", + "-prof", + "com.datadoghq.profiler.stresstest.WhiteboxProfiler", + "counters.*" + ) +} diff --git a/ddprof-test-native/build.gradle.kts b/ddprof-test-native/build.gradle.kts new file mode 100644 index 000000000..51e9841ab --- /dev/null +++ b/ddprof-test-native/build.gradle.kts @@ -0,0 +1,41 @@ +/* + * Native test helper libraries (JNI helpers for Java tests). + */ + +import com.datadoghq.native.model.Platform +import com.datadoghq.native.util.PlatformUtils + +plugins { + id("com.datadoghq.simple-native-lib") +} + +description = "Native test helper libraries (JNI helpers for Java tests)" + +simpleNativeLib { + libraryName.set("ddproftest") + + // Use C compiler (not C++) for .c files + compiler.set(if (PlatformUtils.currentPlatform == Platform.MACOS) "clang" else "gcc") + linker.set(if (PlatformUtils.currentPlatform == Platform.MACOS) "clang" else "gcc") + + includeJni.set(true) + + // Note: No optimization (-O0) to prevent inlining of static functions like do_primes() + // which need to be visible in stack traces for profiler testing + compilerArgs.set( + when (PlatformUtils.currentPlatform) { + Platform.LINUX -> listOf("-fPIC") + Platform.MACOS -> emptyList() + } + ) + + linkerArgs.set( + when (PlatformUtils.currentPlatform) { + Platform.LINUX -> listOf("-shared", "-Wl,--build-id") + Platform.MACOS -> listOf("-dynamiclib") + } + ) + + // Create consumable configurations for other projects to depend on + createConfigurations.set(true) +} diff --git a/ddprof-test/src/test/cpp/nativethread.c b/ddprof-test-native/src/main/cpp/nativethread.c similarity index 100% rename from ddprof-test/src/test/cpp/nativethread.c rename to ddprof-test-native/src/main/cpp/nativethread.c diff --git a/ddprof-test/src/test/cpp/remotesym.c b/ddprof-test-native/src/main/cpp/remotesym.c similarity index 100% rename from ddprof-test/src/test/cpp/remotesym.c rename to ddprof-test-native/src/main/cpp/remotesym.c diff --git a/ddprof-test-tracer/build.gradle b/ddprof-test-tracer/build.gradle deleted file mode 100644 index 09901ac41..000000000 --- a/ddprof-test-tracer/build.gradle +++ /dev/null @@ -1,15 +0,0 @@ -plugins { - id 'java' -} - -repositories { - mavenCentral() -} - -dependencies { - implementation project(path: ":ddprof-lib", configuration: 'release') -} - -tasks.withType(JavaCompile).configureEach { - options.compilerArgs.addAll(['--release', '8']) -} \ No newline at end of file diff --git a/ddprof-test-tracer/build.gradle.kts b/ddprof-test-tracer/build.gradle.kts new file mode 100644 index 000000000..ed13dcd45 --- /dev/null +++ b/ddprof-test-tracer/build.gradle.kts @@ -0,0 +1,8 @@ +plugins { + java + id("com.datadoghq.java-conventions") +} + +dependencies { + implementation(project(mapOf("path" to ":ddprof-lib", "configuration" to "release"))) +} diff --git a/ddprof-test/build.gradle b/ddprof-test/build.gradle deleted file mode 100644 index e19250e6d..000000000 --- a/ddprof-test/build.gradle +++ /dev/null @@ -1,343 +0,0 @@ -plugins { - id 'java' - id 'java-library' - id 'application' -} - -repositories { - mavenCentral() -} - -// 1. Define paths and properties -def nativeSrcDir = file('src/test/cpp') -def jniHeadersDir = layout.buildDirectory.dir("generated/jni-headers").get().asFile -def outputLibDir = layout.buildDirectory.dir("libs/native").get().asFile -// Define the name of your JNI library (e.g., "ddproftest" becomes libddproftest.so/ddproftest.dll/libddproftest.dylib) -def libraryName = "ddproftest" - -// Determine OS-specific file extensions and library names -def osName = org.gradle.internal.os.OperatingSystem.current() -def libFileExtension = (os().isMacOsX() ? "dylib" : "so") -def libraryFileName = "lib${libraryName}.${libFileExtension}" - -// 2. Generate JNI headers using javac -tasks.named('compileJava') { - // Tell javac to generate the JNI headers into the specified directory - options.compilerArgs += ['-h', jniHeadersDir] -} - -// 3. Define a task to compile the native code -tasks.register('buildNativeJniLibrary', Exec) { - description 'Compiles the JNI C/C++ sources into a shared library' - group 'build' - - // Ensure Java compilation (and thus header generation) happens first - dependsOn tasks.named('compileJava') - - // Clean up previous build artifacts - doFirst { - outputLibDir.mkdirs() - } - - // Assume GCC/Clang on Linux/macOS - commandLine 'gcc' - args "-I${System.getenv('JAVA_HOME')}/include" // Standard JNI includes - if (os().isMacOsX()) { - args "-I${System.getenv('JAVA_HOME')}/include/darwin" // macOS-specific includes - args "-dynamiclib" // Build a dynamic library on macOS - } else if (os().isLinux()) { - args "-I${System.getenv('JAVA_HOME')}/include/linux" // Linux-specific includes - args "-fPIC" - args "-shared" // Build a shared library on Linux - args "-Wl,--build-id" // Embed GNU build-id for remote symbolication testing - } - args nativeSrcDir.listFiles()*.getAbsolutePath() // Source files - args "-o", "${outputLibDir.absolutePath}/${libraryFileName}" // Output file path -} - -apply from: rootProject.file('common.gradle') - - -def addCommonTestDependencies(Configuration configuration) { - configuration.dependencies.add(project.dependencies.create('org.junit.jupiter:junit-jupiter-api:5.9.2')) - configuration.dependencies.add(project.dependencies.create('org.junit.jupiter:junit-jupiter-engine:5.9.2')) - configuration.dependencies.add(project.dependencies.create('org.junit.jupiter:junit-jupiter-params:5.9.2')) - configuration.dependencies.add(project.dependencies.create('org.slf4j:slf4j-simple:1.7.32')) - configuration.dependencies.add(project.dependencies.create('org.openjdk.jmc:flightrecorder:8.1.0')) - configuration.dependencies.add(project.dependencies.create('org.openjdk.jol:jol-core:0.16')) - configuration.dependencies.add(project.dependencies.create('org.junit-pioneer:junit-pioneer:1.9.1')) - configuration.dependencies.add(project.dependencies.create('org.lz4:lz4-java:1.8.0')) - configuration.dependencies.add(project.dependencies.create('org.xerial.snappy:snappy-java:1.1.10.1')) - configuration.dependencies.add(project.dependencies.create('com.github.luben:zstd-jni:1.5.5-4')) - configuration.dependencies.add(project.dependencies.create('org.ow2.asm:asm:9.6')) - configuration.dependencies.add(project.dependencies.project(path: ":ddprof-test-tracer")) -} - -def addCommonMainDependencies(Configuration configuration) { - // Main dependencies for the unwinding validator application - configuration.dependencies.add(project.dependencies.create('org.slf4j:slf4j-simple:1.7.32')) - configuration.dependencies.add(project.dependencies.create('org.openjdk.jmc:flightrecorder:8.1.0')) - configuration.dependencies.add(project.dependencies.create('org.openjdk.jol:jol-core:0.16')) - configuration.dependencies.add(project.dependencies.create('org.lz4:lz4-java:1.8.0')) - configuration.dependencies.add(project.dependencies.create('org.xerial.snappy:snappy-java:1.1.10.1')) - configuration.dependencies.add(project.dependencies.create('com.github.luben:zstd-jni:1.5.5-4')) - configuration.dependencies.add(project.dependencies.project(path: ":ddprof-test-tracer")) -} - -configurations.create('testCommon') { - canBeConsumed = true - canBeResolved = true -} - -// Configuration for main source set dependencies -configurations.create('mainCommon') { - canBeConsumed = true - canBeResolved = true -} - -// Application configuration -application { - mainClass = 'com.datadoghq.profiler.unwinding.UnwindingValidator' -} - -buildConfigurations.each { config -> - def name = config.name - if (config.os != osIdentifier() || config.arch != archIdentifier()) { - return - } - logger.debug("Creating configuration for ${name}") - - // Test configuration - def cfg = configurations.create("test${name.capitalize()}Implementation") { - canBeConsumed = true - canBeResolved = true - extendsFrom configurations.testCommon - } - addCommonTestDependencies(cfg) - cfg.dependencies.add(project.dependencies.project(path: ":ddprof-lib", configuration: name)) - - def task = tasks.register("test${name}", Test) { - onlyIf { - config.active - } - dependsOn compileTestJava - description = "Runs the unit tests with the ${name} library variant" - group = 'verification' - // Filter classpath to include only necessary dependencies for this test task - classpath = sourceSets.test.runtimeClasspath.filter { file -> - !file.name.contains('ddprof-') || file.name.contains('test-tracer') - } + cfg - - if (!config.testEnv.empty) { - config.testEnv.each { key, value -> - environment key, value - } - logger.debug("Setting environment variables for ${name}: ${config.testEnv}") - } - if (config.name == "asan") { - onlyIf { - locateLibasan() != null - } - } else if (config.name == "tsan") { - onlyIf { - locateLibtsan() != null - } - } - } - - // Main/application configuration for unwinding validator (release and debug configs) - if (name == "release" || name == "debug") { - def mainCfg = configurations.create("${name}Implementation") { - canBeConsumed = true - canBeResolved = true - extendsFrom configurations.mainCommon - } - addCommonMainDependencies(mainCfg) - mainCfg.dependencies.add(project.dependencies.project(path: ":ddprof-lib", configuration: name)) - - // Manual execution task - tasks.register("runUnwindingValidator${name.capitalize()}", JavaExec) { - onlyIf { - config.active - } - dependsOn compileJava - description = "Run the unwinding validator application (release or debug config)" - group = 'application' - mainClass = 'com.datadoghq.profiler.unwinding.UnwindingValidator' - classpath = sourceSets.main.runtimeClasspath + mainCfg - - if (!config.testEnv.empty) { - config.testEnv.each { key, value -> - environment key, value - } - } - - def javaHome = System.getenv("JAVA_TEST_HOME") - if (javaHome == null) { - javaHome = System.getenv("JAVA_HOME") - } - executable = new File("${javaHome}", 'bin/java') - - jvmArgs '-Djdk.attach.allowAttachSelf', '-Djol.tryWithSudo=true', - '-XX:ErrorFile=build/hs_err_pid%p.log', '-XX:+ResizeTLAB', - '-Xmx512m' - } - - // Configure arguments for runUnwindingValidator task - tasks.named("runUnwindingValidator${name.capitalize()}") { - if (project.hasProperty('validatorArgs')) { - setArgs(project.property('validatorArgs').split(' ').toList()) - } - } - - // CI reporting task - tasks.register("unwindingReport${name.capitalize()}", JavaExec) { - onlyIf { - config.active - } - dependsOn compileJava - description = "Generate unwinding report for CI (release or debug config)" - group = 'verification' - mainClass = 'com.datadoghq.profiler.unwinding.UnwindingValidator' - classpath = sourceSets.main.runtimeClasspath + mainCfg - args = [ - '--output-format=markdown', - '--output-file=build/reports/unwinding-summary.md' - ] - - if (!config.testEnv.empty) { - config.testEnv.each { key, value -> - environment key, value - } - } - environment("CI", project.hasProperty("CI") || Boolean.parseBoolean(System.getenv("CI"))) - - def javaHome = System.getenv("JAVA_TEST_HOME") - if (javaHome == null) { - javaHome = System.getenv("JAVA_HOME") - } - executable = new File("${javaHome}", 'bin/java') - - jvmArgs '-Djdk.attach.allowAttachSelf', '-Djol.tryWithSudo=true', - '-XX:ErrorFile=build/hs_err_pid%p.log', '-XX:+ResizeTLAB', - '-Xmx512m' - - doFirst { - file("${buildDir}/reports").mkdirs() - } - } - } -} - -// Create convenience tasks that delegate to the appropriate config -task runUnwindingValidator { - description = "Run the unwinding validator application (delegates to release if available, otherwise debug)" - group = 'application' - dependsOn { - if (tasks.findByName('runUnwindingValidatorRelease')) { - return 'runUnwindingValidatorRelease' - } else if (tasks.findByName('runUnwindingValidatorDebug')) { - return 'runUnwindingValidatorDebug' - } else { - throw new GradleException("No suitable build configuration available for unwinding validator") - } - } - - doLast { - // Delegate to the appropriate task - actual work is done by dependency - } -} - -task unwindingReport { - description = "Generate unwinding report for CI (delegates to release if available, otherwise debug)" - group = 'verification' - dependsOn { - if (tasks.findByName('unwindingReportRelease')) { - return 'unwindingReportRelease' - } else if (tasks.findByName('unwindingReportDebug')) { - return 'unwindingReportDebug' - } else { - throw new GradleException("No suitable build configuration available for unwinding report") - } - } - - doLast { - // Delegate to the appropriate task - actual work is done by dependency - } -} - -tasks.withType(Test).configureEach { - dependsOn tasks.named('buildNativeJniLibrary') - - // this is a shared configuration for all test tasks - onlyIf { - !project.hasProperty('skip-tests') - } - - def config = it.name.replace("test", "") - - def keepRecordings = project.hasProperty("keepJFRs") || Boolean.parseBoolean(System.getenv("KEEP_JFRS")) - environment("CI", project.hasProperty("CI") || Boolean.parseBoolean(System.getenv("CI"))) - environment("DDPROF_TEST_DISABLE_RATE_LIMIT", "1") // Disable PID controller rate limiting in tests - - // Base JVM arguments - def jvmArgsList = [ - "-Dddprof_test.keep_jfrs=${keepRecordings}", - '-Djdk.attach.allowAttachSelf', - '-Djol.tryWithSudo=true', - "-Dddprof_test.config=${config}", - "-Dddprof_test.ci=${project.hasProperty('CI')}", - "-Dddprof.disable_unsafe=true", - '-XX:ErrorFile=build/hs_err_pid%p.log', - '-XX:+ResizeTLAB', - '-Xmx512m', - '-XX:OnError=/tmp/do_stuff.sh', - "-Djava.library.path=${outputLibDir.absolutePath}" - ] - - jvmArgs jvmArgsList - - def javaHome = System.getenv("JAVA_TEST_HOME") - if (javaHome == null) { - javaHome = System.getenv("JAVA_HOME") - } - useJUnitPlatform() - executable = new File("${javaHome}", 'bin/java') - - testLogging { - showStandardStreams = true - } -} - -test { - onlyIf { - false - } -} - -tasks.withType(JavaCompile).configureEach { - options.compilerArgs.addAll(['--release', '8']) - - if (name == "compileTestJava") { - sourceCompatibility = '8' - targetCompatibility = '8' - } - classpath += configurations.testReleaseImplementation -} - -// Make the assemble* tasks depend on the test* tasks -gradle.projectsEvaluated { - buildConfigNames().each { - def testTask = tasks.findByName("test${it}") - def assembleTask = project(':ddprof-lib').tasks.findByName("assemble${it.capitalize()}") - if (testTask && assembleTask) { - assembleTask.dependsOn testTask - } - - // Hook C++ gtest tasks to run as part of the corresponding Java test tasks - def gtestTask = project(':ddprof-lib:gtest').tasks.findByName("gtest${it.capitalize()}") - if (testTask && gtestTask) { - testTask.dependsOn gtestTask - } - } -} diff --git a/ddprof-test/build.gradle.kts b/ddprof-test/build.gradle.kts new file mode 100644 index 000000000..dd3d6e247 --- /dev/null +++ b/ddprof-test/build.gradle.kts @@ -0,0 +1,84 @@ +import com.datadoghq.profiler.ProfilerTestExtension + +plugins { + java + `java-library` + application + id("com.datadoghq.profiler-test") + id("com.datadoghq.java-conventions") +} + +// Reference to native test helpers library directory +val testNativeLibDir = project(":ddprof-test-native").layout.buildDirectory.dir("lib") + +// Configure profiler test plugin - this generates all multi-config tasks automatically +configure { + // Native library path for JNI test helpers + nativeLibDir.set(testNativeLibDir) + + // Enable multi-config task generation + profilerLibProject.set(":ddprof-lib") + + // Extra JVM args specific to this project's tests + extraJvmArgs.addAll( + "-Dddprof.disable_unsafe=true", + "-XX:OnError=/tmp/do_stuff.sh" + ) +} + +// Generate JNI headers using javac +val jniHeadersDir = layout.buildDirectory.dir("generated/jni-headers") +tasks.named("compileJava") { + options.compilerArgs.addAll(listOf("-h", jniHeadersDir.get().asFile.absolutePath)) +} + +// Application configuration (for the run task) +application { + mainClass.set("com.datadoghq.profiler.unwinding.UnwindingValidator") +} + +// Add common dependencies to test and main configurations +// The plugin creates testCommon and mainCommon configurations eagerly +dependencies { + // Test dependencies + "testCommon"(libs.bundles.testing) + "testCommon"(libs.bundles.profiler.runtime) + "testCommon"(libs.asm) + + // Main/application dependencies + "mainCommon"(libs.slf4j.simple) + "mainCommon"(libs.bundles.profiler.runtime) +} + +// Additional test task configuration beyond what the plugin provides +tasks.withType().configureEach { + // Ensure native test library is built before running tests + dependsOn(":ddprof-test-native:linkLib") + + // Extract config name from task name for test-specific JVM args + val configName = name.replace("test", "") + val keepRecordings = project.hasProperty("keepJFRs") || System.getenv("KEEP_JFRS")?.toBoolean() ?: false + + jvmArgs( + "-Dddprof_test.keep_jfrs=$keepRecordings", + "-Dddprof_test.config=$configName", + "-Dddprof_test.ci=${project.hasProperty("CI")}" + ) +} + +// Disable the default 'test' task - we use config-specific tasks instead +tasks.named("test") { + onlyIf { false } +} + +// Java compilation settings handled by java-conventions plugin (--release 8) + +// Ensure compileTestJava has access to the test dependencies for compilation +// (must be set after project evaluation when the configuration is created) +gradle.projectsEvaluated { + configurations.findByName("testReleaseImplementation")?.let { testReleaseCfg -> + tasks.withType().configureEach { + classpath += testReleaseCfg + } + } +} diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 000000000..8f4661d1f --- /dev/null +++ b/doc/README.md @@ -0,0 +1,36 @@ +# Java Profiler Documentation + +## Directory Structure + +| Directory | Purpose | +|-----------|---------| +| `architecture/` | Profiler internal architecture and design documents | +| `build/` | Build system documentation (Gradle, native compilation) | +| `reference/` | Reference documentation for profiler features | +| `temp/` | Work logs, session state, and analysis reports | + +## Naming Convention + +All documentation files use **PascalCase** naming (e.g., `BuildSystemGuide.md`). + +## Quick Navigation + +### Architecture +- [CallTraceStorage](architecture/CallTraceStorage.md) - Triple-buffer architecture for call traces +- [TLSContext](architecture/TLSContext.md) - Thread-local context for distributed tracing +- [TLSPriming](architecture/TLSPriming.md) - Signal-safe TLS initialization + +### Build System +- [BuildSystemGuide](build/BuildSystemGuide.md) - Comprehensive build system documentation +- [GradleTasks](build/GradleTasks.md) - Available Gradle tasks reference +- [NativeBuildPlugin](build/NativeBuildPlugin.md) - Native C++ compilation plugin + +### Reference +- [ProfilerMemoryRequirements](reference/ProfilerMemoryRequirements.md) - Memory usage and limits +- [EventTypeSystem](reference/EventTypeSystem.md) - Profiler event types +- [RemoteSymbolication](reference/RemoteSymbolication.md) - Symbol resolution +- [RemoteSymbolicationFrameTypes](reference/RemoteSymbolicationFrameTypes.md) - Frame type design for symbolication +- [TestFlakinessAnalysis](reference/TestFlakinessAnalysis.md) - Test flakiness investigation results + +### Work State (temp/) +Session logs and analysis reports for ongoing work. Not considered permanent documentation. diff --git a/doc/architecture/TlsPriming.md b/doc/architecture/TLSPriming.md similarity index 100% rename from doc/architecture/TlsPriming.md rename to doc/architecture/TLSPriming.md diff --git a/doc/build/BuildSystemGuide.md b/doc/build/BuildSystemGuide.md new file mode 100644 index 000000000..ffbc373d8 --- /dev/null +++ b/doc/build/BuildSystemGuide.md @@ -0,0 +1,466 @@ +# Build System Maintenance Guide + +This document provides detailed guidance for maintaining and extending the Gradle build system. + +## Convention Plugins Overview + +All custom build logic lives in `build-logic/conventions/`. The plugins are: + +| Plugin ID | Class | Purpose | +|-----------|-------|---------| +| `com.datadoghq.native-build` | `NativeBuildPlugin` | Multi-config C++ compilation | +| `com.datadoghq.native-root` | `RootProjectPlugin` | Root-level config discovery | +| `com.datadoghq.gtest` | `GtestPlugin` | Google Test integration | +| `com.datadoghq.simple-native-lib` | `SimpleNativeLibPlugin` | Simple single-library builds | +| `com.datadoghq.profiler-test` | `ProfilerTestPlugin` | Multi-config Java test generation | +| `com.datadoghq.java-conventions` | `JavaConventionsPlugin` | Java 8 compilation settings | +| `com.datadoghq.versioned-sources` | `VersionedSourcesPlugin` | Java version-specific code | +| `com.datadoghq.fuzz-targets` | `FuzzTargetsPlugin` | libFuzzer integration | +| `com.datadoghq.scanbuild` | `ScanBuildPlugin` | Clang static analysis | +| `com.datadoghq.spotless-convention` | `SpotlessConventionPlugin` | Code formatting | + +## Key Files and Their Purposes + +``` +build-logic/conventions/src/main/kotlin/com/datadoghq/ +├── native/ +│ ├── NativeBuildPlugin.kt # Creates compile/link tasks per config +│ ├── NativeBuildExtension.kt # DSL: nativeBuild { ... } +│ ├── RootProjectPlugin.kt # Provides configs to all subprojects +│ ├── SimpleNativeLibPlugin.kt # Simplified single-lib builds +│ ├── config/ +│ │ └── ConfigurationPresets.kt # Defines release/debug/asan/tsan/fuzzer +│ ├── model/ +│ │ ├── BuildConfiguration.kt # Config model (platform, arch, flags) +│ │ ├── Platform.kt # LINUX, MACOS enum +│ │ └── Architecture.kt # X64, ARM64, etc. +│ ├── tasks/ +│ │ ├── NativeCompileTask.kt # C++ compilation task +│ │ ├── NativeLinkTask.kt # Shared library linking +│ │ └── NativeLinkExecutableTask.kt # Executable linking +│ ├── util/ +│ │ └── PlatformUtils.kt # Platform detection, compiler finding +│ ├── gtest/ +│ │ ├── GtestPlugin.kt # Google Test task generation +│ │ ├── GtestExtension.kt # DSL: gtest { ... } +│ │ └── GtestTaskBuilder.kt # Per-test task creation +│ └── fuzz/ +│ └── FuzzTargetsPlugin.kt # Fuzz testing support +├── profiler/ +│ ├── ProfilerTestPlugin.kt # Multi-config test generation +│ ├── JavaConventionsPlugin.kt # Java 8 --release flag +│ └── SpotlessConventionPlugin.kt # Code formatting +└── java/versionedsources/ + └── VersionedSourcesPlugin.kt # Java 9+ version-specific code +``` + +## How Build Configurations Work + +Build configurations (release, debug, asan, tsan, fuzzer) are defined in `ConfigurationPresets.kt`: + +1. **Registration**: Configs are registered in `NativeBuildExtension.buildConfigurations` +2. **Activation**: Each config checks if it's available on current platform + - release/debug: Always active + - asan: Active if `libasan.so` found via `gcc -print-file-name=libasan.so` + - tsan: Active if `libtsan.so` found + - fuzzer: Active if clang supports `-fsanitize=fuzzer` +3. **Discovery**: Subprojects query active configs via `nativeBuild.buildConfigurations.names` + +### Configuration Flow Diagram + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ settings.gradle.kts │ +│ includeBuild("build-logic") ← loads convention plugins │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ build.gradle.kts (root) │ +│ plugins { id("com.datadoghq.native-root") } │ +│ ↓ │ +│ RootProjectPlugin creates nativeBuild extension │ +│ ↓ │ +│ afterEvaluate: ConfigurationPresets.setupStandardConfigurations│ +│ → Registers: release, debug, asan, tsan, fuzzer │ +│ → Checks availability via PlatformUtils │ +└─────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────┐ +│ ddprof-lib/build.gradle.kts │ +│ plugins { id("com.datadoghq.native-build") } │ +│ ↓ │ +│ afterEvaluate: queries nativeBuild.buildConfigurations.names │ +│ → Creates tasks: compileRelease, linkRelease, assembleRelease │ +│ → Creates tasks: compileDebug, linkDebug, assembleDebug │ +│ → (only for active configs on current platform) │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Adding a New Build Configuration + +To add a new configuration (e.g., "coverage"): + +### Step 1: Add to ConfigurationPresets.kt + +```kotlin +// In setupStandardConfigurations() +extension.buildConfigurations.apply { + // ... existing configs ... + register("coverage") { + configureCoverage(this, currentPlatform, currentArch, version, rootDir) + } +} + +// Add configuration function +fun configureCoverage( + config: BuildConfiguration, + platform: Platform, + architecture: Architecture, + version: String, + rootDir: File +) { + config.platform.set(platform) + config.architecture.set(architecture) + config.active.set(PlatformUtils.hasGcov()) // Add detection in PlatformUtils + + when (platform) { + Platform.LINUX -> { + config.compilerArgs.set( + listOf("-O0", "-g", "--coverage", "-fprofile-arcs", "-ftest-coverage") + + commonLinuxCompilerArgs(version) + ) + config.linkerArgs.set(commonLinuxLinkerArgs() + listOf("--coverage")) + } + Platform.MACOS -> { + config.compilerArgs.set( + listOf("-O0", "-g", "--coverage") + commonMacosCompilerArgs(version) + ) + config.linkerArgs.set(listOf("--coverage")) + } + } +} +``` + +### Step 2: Add Detection to PlatformUtils.kt (if needed) + +```kotlin +fun hasGcov(): Boolean { + return isCompilerAvailable("gcov") +} +``` + +### Step 3: No Changes Needed in Build Scripts + +Configurations are discovered dynamically - all subprojects automatically pick up the new config. + +## Modifying Compiler/Linker Flags + +Flags are centralized in `ConfigurationPresets.kt`: + +| Function | Purpose | +|----------|---------| +| `commonLinuxCompilerArgs()` | Shared C++ flags for all Linux configs | +| `commonLinuxLinkerArgs()` | Shared linker flags for Linux | +| `commonMacosCompilerArgs()` | Shared C++ flags for macOS | +| `configureRelease()` | Release-specific flags | +| `configureDebug()` | Debug-specific flags | +| `configureAsan()` | AddressSanitizer flags | +| `configureTsan()` | ThreadSanitizer flags | +| `configureFuzzer()` | libFuzzer flags | + +### Example: Adding a Flag to All Configs + +```kotlin +private fun commonLinuxCompilerArgs(version: String): List { + return mutableListOf( + "-fPIC", + "-fno-omit-frame-pointer", + "-momit-leaf-frame-pointer", + "-fvisibility=hidden", + "-fdata-sections", + "-ffunction-sections", + "-std=c++17", + // Add new flag here: + "-fstack-protector-strong", + "-DPROFILER_VERSION=\"$version\"", + "-DCOUNTERS" + ) +} +``` + +### Example: Adding a Flag to One Config + +```kotlin +fun configureRelease(...) { + when (platform) { + Platform.LINUX -> { + config.compilerArgs.set( + listOf("-O3", "-DNDEBUG", "-g", "-flto") + // Added -flto + commonLinuxCompilerArgs(version) + ) + } + } +} +``` + +## Creating a New Convention Plugin + +### Step 1: Create the Plugin Class + +Create in `build-logic/conventions/src/main/kotlin/com/datadoghq/example/`: + +```kotlin +package com.datadoghq.example + +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.provider.Property +import javax.inject.Inject + +class MyPlugin : Plugin { + override fun apply(project: Project) { + // Create extension for DSL configuration + val extension = project.extensions.create( + "myPlugin", + MyExtension::class.java, + project + ) + + // Create configurations eagerly (so build scripts can reference them) + val myConfig = project.configurations.create("myConfiguration") + + // Configure tasks in afterEvaluate (when extension values are set) + project.afterEvaluate { + if (extension.enabled.get()) { + createTasks(project, extension) + } + } + } + + private fun createTasks(project: Project, extension: MyExtension) { + project.tasks.register("myTask") { + group = "my-group" + description = "Does something useful" + doLast { + println("Setting: ${extension.someSetting.get()}") + } + } + } +} + +abstract class MyExtension @Inject constructor(project: Project) { + val enabled: Property = project.objects.property(Boolean::class.java) + val someSetting: Property = project.objects.property(String::class.java) + + init { + enabled.convention(true) + someSetting.convention("default-value") + } +} +``` + +### Step 2: Register in build.gradle.kts + +Add to `build-logic/conventions/build.gradle.kts`: + +```kotlin +gradlePlugin { + plugins { + // ... existing plugins ... + create("myPlugin") { + id = "com.datadoghq.my-plugin" + implementationClass = "com.datadoghq.example.MyPlugin" + } + } +} +``` + +### Step 3: Use in Subproject + +```kotlin +plugins { + id("com.datadoghq.my-plugin") +} + +myPlugin { + enabled.set(true) + someSetting.set("custom-value") +} +``` + +## Best Practices for Plugin Development + +### Prefer Lazy Configuration + +```kotlin +// GOOD - lazy task registration +project.tasks.register("myTask", MyTask::class.java) { + inputFile.set(extension.someFile) // Evaluated when task executes +} + +// AVOID - eager task creation +project.tasks.create("myTask", MyTask::class.java) { + inputFile.set(extension.someFile) // Evaluated immediately +} +``` + +### Create Configurations Eagerly, Populate Lazily + +```kotlin +override fun apply(project: Project) { + // Create configurations immediately (so build scripts can use them) + val myConfig = project.configurations.create("myConfiguration").apply { + isCanBeConsumed = true + isCanBeResolved = true + } + + // Populate dependencies in afterEvaluate + project.afterEvaluate { + myConfig.dependencies.add( + project.dependencies.create("com.example:lib:1.0") + ) + } +} +``` + +### Use Lazy Task References + +```kotlin +// GOOD - lazy reference, doesn't force task creation +val linkTask = tasks.named("linkRelease") +myTask.dependsOn(linkTask) + +// AVOID - forces immediate task creation/lookup +val linkTask = tasks.getByName("linkRelease") +``` + +### Check Task Existence Safely + +```kotlin +// GOOD - safe check +if (tasks.names.contains("linkRelease")) { + dependsOn("linkRelease") +} + +// AVOID - throws exception if not found +dependsOn(tasks.getByName("linkRelease")) +``` + +## Testing Build Changes + +After modifying build-logic: + +```bash +# Kill Gradle daemon to pick up plugin changes +./gradlew --stop + +# Verify configuration loads without errors +./.claude/commands/build-and-summarize tasks --group=build -q + +# Verify Java compilation +./.claude/commands/build-and-summarize :ddprof-lib:compileJava :ddprof-test:compileJava -Pskip-native + +# Verify native compilation (on Linux) +./.claude/commands/build-and-summarize assembleRelease + +# Verify tests can be configured +./.claude/commands/build-and-summarize :ddprof-test:testRelease -Pskip-native + +# Verify all tasks are generated correctly +./.claude/commands/build-and-summarize :ddprof-lib:tasks --all +``` + +## Gradle Properties Reference + +All configurable properties (see `gradle.properties.template` for full documentation): + +| Property | Default | Description | +|----------|---------|-------------| +| `skip-tests` | false | Skip Java test execution | +| `skip-native` | false | Skip C++ compilation | +| `skip-gtest` | false | Skip Google Test execution | +| `skip-fuzz` | false | Skip fuzz testing | +| `native.forceCompiler` | auto | Force specific C++ compiler path | +| `ddprof_version` | from build.gradle.kts | Override project version | +| `with-libs` | - | Path to pre-built native libraries | +| `keepJFRs` | false | Keep JFR recordings after tests | +| `validatorArgs` | - | Arguments for UnwindingValidator | +| `CI` | from env | CI environment flag | +| `forceLocal` | false | Use local Nexus for testing | + +## Troubleshooting + +### "Configuration not found" + +Configurations are created in `afterEvaluate`. Ensure you're accessing them in `afterEvaluate` or later: + +```kotlin +// WRONG - runs during configuration phase +val cfg = configurations.getByName("testReleaseImplementation") // May not exist yet + +// RIGHT - runs after all afterEvaluate blocks +project.afterEvaluate { + val cfg = configurations.getByName("testReleaseImplementation") +} + +// ALSO RIGHT - use gradle.projectsEvaluated for cross-project +gradle.projectsEvaluated { + val cfg = configurations.getByName("testReleaseImplementation") +} +``` + +### "Task not found" + +Tasks are created dynamically based on active configs. Verify: + +1. The config is active: Check `PlatformUtils` detection methods +2. The plugin is applied: Check `plugins { }` block +3. You're in the right phase: Use `afterEvaluate` or `gradle.projectsEvaluated` + +### Plugin Changes Not Taking Effect + +The Gradle daemon caches compiled plugins: + +```bash +./gradlew --stop # Kill daemon +./gradlew tasks --no-daemon # Run without daemon +``` + +### Circular Dependencies + +Avoid cross-project `afterEvaluate` dependencies. Use `gradle.projectsEvaluated` instead: + +```kotlin +// WRONG - may cause ordering issues +project.afterEvaluate { + val otherProject = rootProject.findProject(":other") + otherProject?.afterEvaluate { ... } // Nested afterEvaluate +} + +// RIGHT - runs after ALL projects are evaluated +gradle.projectsEvaluated { + val otherProject = rootProject.findProject(":other") + // Safe to access other project's tasks/configs +} +``` + +### Debug Gradle Configuration + +```bash +# Show task dependencies +./gradlew :ddprof-lib:assembleRelease --dry-run + +# Show dependency resolution +./gradlew :ddprof-test:dependencies --configuration testReleaseImplementation + +# Enable debug logging +./gradlew build -d 2>&1 | head -500 +``` + +## See Also + +- [GRADLE-TASKS.md](GRADLE-TASKS.md) - Task reference and workflows +- [build-logic/README.md](../build-logic/README.md) - Plugin implementation details +- [gradle.properties.template](../gradle.properties.template) - All configurable properties diff --git a/doc/build/GradleTasks.md b/doc/build/GradleTasks.md new file mode 100644 index 000000000..866ad6117 --- /dev/null +++ b/doc/build/GradleTasks.md @@ -0,0 +1,205 @@ +# Gradle Tasks Reference + +This document describes the custom Gradle tasks available in the java-profiler project. + +## Quick Reference + +| Task | Description | +|------|-------------| +| `./gradlew assembleRelease` | Build release native library and JAR | +| `./gradlew testRelease` | Run Java tests with release library | +| `./gradlew gtestDebug` | Run C++ unit tests (debug config) | +| `./gradlew spotlessApply` | Auto-format all source files | + +## Build Tasks + +### Native Library Compilation + +Tasks are generated for each active build configuration (release, debug, asan, tsan, fuzzer). +Configuration availability depends on platform and compiler capabilities: +- **release/debug**: Always available +- **asan**: Available on Linux when libasan is found +- **tsan**: Available on Linux when libtsan is found +- **fuzzer**: Available when clang with libFuzzer support is detected + +``` +compile{Config} # Compile C++ sources +link{Config} # Link shared library +assemble{Config} # Full build for configuration +assemble{Config}Jar # Build JAR with native library +assembleAll # Build all active configurations +``` + +**Examples:** +```bash +./gradlew assembleRelease # Build release library +./gradlew assembleDebug # Build debug library +./gradlew assembleAll # Build all active configs +``` + +### JAR Tasks + +``` +jar # Standard JAR (requires external libs) +assembleReleaseJar # JAR with release native library +assembleDebugJar # JAR with debug native library +sourcesJar # Sources JAR for publishing +javadocJar # Javadoc JAR for publishing +``` + +## Test Tasks + +### Java Tests (ddprof-test) + +``` +test{Config} # Run Java tests with specified config +testRelease # Java tests with release library +testDebug # Java tests with debug library +testAsan # Java tests with ASAN library (Linux) +testTsan # Java tests with TSAN library (Linux) +``` + +**Examples:** +```bash +./gradlew :ddprof-test:testRelease +./gradlew :ddprof-test:testDebug --tests "*.ProfilerTest" +``` + +### C++ Unit Tests (Google Test) + +``` +gtest{Config}_{TestName} # Run specific test (e.g., gtestDebug_SymbolAnalyzerTest) +gtest{Config} # Run all tests for config +gtest # Run all tests for all configs +``` + +**Examples:** +```bash +./gradlew :ddprof-lib:gtestDebug # All debug tests +./gradlew :ddprof-lib:gtest # All tests, all configs +``` + +### Stress Tests (JMH Benchmarks) + +``` +jmh # Run JMH benchmarks +jmhJar # Build executable JMH JAR +runStressTests # Run stress tests with profiler +``` + +## Application Tasks + +### Unwinding Validator + +``` +runUnwindingValidator{Config} # Run validator with config +runUnwindingValidator # Run validator (release or debug) +unwindingReport{Config} # Generate markdown report +unwindingReport # Generate report (release or debug) +``` + +**Examples:** +```bash +./gradlew :ddprof-test:runUnwindingValidatorRelease +./gradlew :ddprof-test:runUnwindingValidatorRelease -PvalidatorArgs="--verbose" +``` + +## Code Quality Tasks + +### Formatting (Spotless) + +``` +spotlessCheck # Check formatting violations +spotlessApply # Auto-fix formatting issues +``` + +### Static Analysis + +``` +scanbuild{Config} # Run Clang Static Analyzer +``` + +## Fuzz Testing + +``` +fuzz_{TargetName} # Run specific fuzz target +compileFuzz_{TargetName} # Compile fuzz target +linkFuzz_{TargetName} # Link fuzz target +``` + +**Example:** +```bash +./gradlew :ddprof-lib:fuzz:fuzz_symbolAnalyzer +``` + +## Task Dependency Graph + +``` + ┌─────────────────┐ + │ assembleAll │ + └────────┬────────┘ + │ + ┌───────────────────┼───────────────────┐ + ▼ ▼ ▼ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ assembleRelease │ │ assembleDebug │ │ assembleAsan │ ... +└────────┬────────┘ └────────┬────────┘ └────────┬────────┘ + │ │ │ + ▼ ▼ ▼ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ linkRelease │ │ linkDebug │ │ linkAsan │ +└────────┬────────┘ └────────┬────────┘ └────────┬────────┘ + │ │ │ + ▼ ▼ ▼ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ compileRelease │ │ compileDebug │ │ compileAsan │ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + + + ┌─────────────────┐ + │ gtest │ + └────────┬────────┘ + │ + ┌───────────────────┼───────────────────┐ + ▼ ▼ ▼ +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ gtestRelease │ │ gtestDebug │ │ gtestAsan │ +└────────┬────────┘ └────────┬────────┘ └────────┬────────┘ + │ │ │ + (per-test tasks) (per-test tasks) (per-test tasks) +``` + +## Common Workflows + +### Full Build and Test +```bash +./gradlew assembleRelease :ddprof-test:testRelease :ddprof-lib:gtestRelease +``` + +### Quick Development Cycle +```bash +./gradlew assembleDebug :ddprof-test:testDebug --tests "*.MyTest" +``` + +### Pre-commit Checks +```bash +./gradlew spotlessCheck :ddprof-lib:gtestDebug :ddprof-test:testDebug +``` + +### CI Pipeline +```bash +./gradlew -PCI assembleAll :ddprof-test:testRelease :ddprof-lib:gtest spotlessCheck +``` + +## Gradle Properties + +See `gradle.properties.template` for all available properties: + +| Property | Description | +|----------|-------------| +| `skip-tests` | Skip Java test execution | +| `skip-native` | Skip native C++ compilation | +| `skip-gtest` | Skip Google Test execution | +| `native.forceCompiler` | Force specific C++ compiler | +| `ddprof_version` | Override project version | +| `CI` | Enable CI-specific behavior | diff --git a/doc/build/NativeBuildPlugin.md b/doc/build/NativeBuildPlugin.md new file mode 100644 index 000000000..965d4f230 --- /dev/null +++ b/doc/build/NativeBuildPlugin.md @@ -0,0 +1,410 @@ +# Native Build Plugin Architecture + +This document describes the architecture of the Kotlin-based native build plugin (`com.datadoghq.native-build`) used for C++ compilation in the Datadog Java Profiler project. + +## Overview + +The native build plugin replaces Gradle's built-in `cpp-library` and `cpp-application` plugins with a custom, type-safe solution that directly invokes compilers without version string parsing. This design avoids known issues with Gradle's native plugins while providing a clean DSL for configuration. + +## Why Custom Build Tasks? + +Gradle's native plugins have several problems: + +1. **Version Parsing Failures**: The plugins parse compiler version strings which breaks with newer gcc/clang versions +2. **JNI Header Detection Issues**: Problems with non-standard JAVA_HOME layouts +3. **Unresponsive Maintainers**: Plugin maintainers are unresponsive to fixes +4. **Undocumented Internals**: The plugins use internals that change between Gradle versions + +**Solution**: Direct compiler invocation without version parsing. The tasks simply find `clang++` or `g++` on PATH and invoke them with configured flags. + +## Component Architecture + +``` +build-logic/ +└── conventions/ + └── src/main/kotlin/com/datadoghq/native/ + ├── NativeBuildPlugin.kt # Main native build plugin + ├── NativeBuildExtension.kt # DSL extension for configuration + ├── config/ + │ └── ConfigurationPresets.kt # Standard build configurations + ├── gtest/ + │ ├── GtestPlugin.kt # Google Test integration plugin + │ └── GtestExtension.kt # DSL extension for gtest config + ├── model/ + │ ├── Architecture.kt # x64, arm64 enum + │ ├── Platform.kt # linux, macos enum + │ ├── BuildConfiguration.kt # Configuration model + │ ├── LogLevel.kt # QUIET, NORMAL, VERBOSE, DEBUG + │ ├── ErrorHandlingMode.kt # FAIL_FAST, COLLECT_ALL + │ └── SourceSet.kt # Per-directory compiler flags + ├── tasks/ + │ ├── NativeCompileTask.kt # C++ compilation task + │ ├── NativeLinkTask.kt # Library linking task + │ └── NativeLinkExecutableTask.kt # Executable linking task + └── util/ + └── PlatformUtils.kt # Platform detection utilities +``` + +## Plugin Lifecycle + +### 1. Plugin Application + +When `com.datadoghq.native-build` is applied to a project: + +```kotlin +plugins { + id("com.datadoghq.native-build") +} +``` + +The plugin: +1. Creates the `nativeBuild` extension for DSL configuration +2. Registers an `afterEvaluate` hook for task generation + +### 2. Configuration Phase + +During project evaluation, users configure the build: + +```kotlin +nativeBuild { + version.set(project.version.toString()) + cppSourceDirs.set(listOf("src/main/cpp")) + includeDirectories.set(listOf("src/main/cpp")) +} +``` + +### 3. Task Generation (afterEvaluate) + +After project evaluation, the plugin: + +1. **Detects Current Platform**: Uses `PlatformUtils.currentPlatform` and `PlatformUtils.currentArchitecture` + +2. **Detects Compiler**: Runs the compiler detection algorithm (see below) + +3. **Creates Standard Configurations**: If no configurations are explicitly defined, creates release, debug, asan, tsan, and fuzzer configurations + +4. **Filters Active Configurations**: Only configurations matching the current platform/architecture are processed + +5. **Generates Tasks**: For each active configuration, creates: + - `compile{Config}` - Compiles C++ sources + - `link{Config}` - Links shared library + - `assemble{Config}` - Aggregates the above + +6. **Creates Aggregation Tasks**: `assembleAll` depends on all individual assemble tasks + +## Compiler Detection + +The compiler detection algorithm prioritizes explicit overrides, then auto-detection: + +``` +┌─────────────────────────────────────────┐ +│ Check -Pnative.forceCompiler property │ +└─────────────────┬───────────────────────┘ + │ + ┌─────────▼─────────┐ + │ Property defined? │ + └─────────┬─────────┘ + Yes │ No + ┌─────────▼─────────┐ ┌─────────────────────┐ + │ Validate compiler │ │ Try clang++ │ + │ with --version │ │ (preferred) │ + └─────────┬─────────┘ └──────────┬──────────┘ + │ │ + ┌─────────▼─────────┐ ┌──────────▼──────────┐ + │ Available? │ │ Available? │ + └─────────┬─────────┘ └──────────┬──────────┘ + Yes │ No Yes │ No + ▼ │ ▼ │ + Return │ Return │ + ▼ ▼ + GradleException Try g++ → c++ + │ + ┌─────▼─────┐ + │ None found│ + └─────┬─────┘ + ▼ + GradleException +``` + +**Usage:** +```bash +# Auto-detect (default) +./gradlew build + +# Force specific compiler +./gradlew build -Pnative.forceCompiler=clang++ +./gradlew build -Pnative.forceCompiler=/usr/bin/g++-13 +``` + +## Build Configurations + +### Standard Configurations + +| Config | Active When | Optimization | Debug | Sanitizers | +|---------|--------------------------------------|--------------|-------|------------| +| release | Always | `-O3` | `-g` | None | +| debug | Always | `-O0` | `-g` | None | +| asan | `libasan` found + not musl | None | `-g` | ASan, UBSan, LSan | +| tsan | `libtsan` found + not musl | None | `-g` | TSan | +| fuzzer | clang++ with libFuzzer + not musl | None | `-g` | ASan, UBSan | + +### Configuration Model + +Each `BuildConfiguration` contains: + +```kotlin +abstract class BuildConfiguration { + val platform: Property // LINUX or MACOS + val architecture: Property // X64 or ARM64 + val compilerArgs: ListProperty // Compiler flags + val linkerArgs: ListProperty // Linker flags + val testEnvironment: MapProperty // Test env vars + val active: Property // Whether to build +} +``` + +### Platform-Specific Flags + +**Common Linux Flags:** +``` +-fPIC -fno-omit-frame-pointer -momit-leaf-frame-pointer +-fvisibility=hidden -fdata-sections -ffunction-sections -std=c++17 +``` + +**Common macOS Additions:** +``` +-D_XOPEN_SOURCE -D_DARWIN_C_SOURCE +``` + +**Release Linker Flags (Linux):** +``` +-Wl,-z,nodelete -static-libstdc++ -static-libgcc +-Wl,--exclude-libs,ALL -Wl,--gc-sections +``` + +## Task Architecture + +### NativeCompileTask + +Compiles C++ source files in parallel: + +``` +┌──────────────────────────────────────────────────────┐ +│ NativeCompileTask │ +├──────────────────────────────────────────────────────┤ +│ Inputs: │ +│ - compiler: String (e.g., "clang++") │ +│ - compilerArgs: List │ +│ - sources: FileCollection │ +│ - includes: FileCollection │ +│ - sourceSets: NamedDomainObjectContainer│ +│ │ +│ Outputs: │ +│ - objectFileDir: Directory │ +│ │ +│ Features: │ +│ - Parallel compilation (configurable jobs) │ +│ - Per-source-set compiler flags │ +│ - FAIL_FAST or COLLECT_ALL error modes │ +│ - Configurable logging verbosity │ +│ - Convenience methods: define(), standard() │ +└──────────────────────────────────────────────────────┘ +``` + +**Source Sets Support:** + +Source sets allow different parts of the codebase to have different compilation flags: + +```kotlin +tasks.register("compile", NativeCompileTask::class) { + compilerArgs.set(listOf("-std=c++17", "-O3")) // Base flags + + sourceSets { + create("main") { + sources.from(fileTree("src/main/cpp")) + compilerArgs.add("-fPIC") + } + create("legacy") { + sources.from(fileTree("src/legacy")) + compilerArgs.addAll("-Wno-deprecated", "-std=c++11") + excludes.add("**/broken/*.cpp") + } + } +} +``` + +### NativeLinkTask + +Links object files into shared libraries: + +``` +┌──────────────────────────────────────────────────────┐ +│ NativeLinkTask │ +├──────────────────────────────────────────────────────┤ +│ Inputs: │ +│ - linker: String │ +│ - linkerArgs: List │ +│ - objectFiles: FileCollection │ +│ - exportSymbols: List │ +│ - hideSymbols: List │ +│ │ +│ Outputs: │ +│ - outputFile: RegularFile │ +│ - debugSymbolsDir: Directory (optional) │ +│ │ +│ Features: │ +│ - Symbol visibility control (version scripts) │ +│ - Debug symbol extraction (release builds) │ +│ - Platform-specific linking │ +│ - macOS wildcard warning │ +└──────────────────────────────────────────────────────┘ +``` + +**Symbol Visibility:** + +The task generates platform-specific symbol export files: + +- **Linux**: Version script (`.ver`) with wildcard support (`Java_*`) +- **macOS**: Exported symbols list (`.exp`) - **no wildcard support** + +```kotlin +tasks.register("link", NativeLinkTask::class) { + exportSymbols.set(listOf("Java_*", "JNI_OnLoad", "JNI_OnUnload")) + hideSymbols.set(listOf("*_internal*")) +} +``` + +**Note:** On macOS, the task warns when wildcards are used since they're not supported. + +## Task Dependencies + +``` +compile{Config} + │ + ▼ + link{Config} + │ + ├──────────────────┐ + │ │ + ▼ ▼ +extractDebugLib stripLib{Config} + (release only) (release only) + │ │ + └────────┬─────────┘ + │ + ▼ + assemble{Config} + │ + ▼ + assembleAll +``` + +## Debug Symbol Extraction + +Release builds automatically extract debug symbols for optimal deployment: + +### Linux Workflow +```bash +objcopy --only-keep-debug library.so library.so.debug +objcopy --add-gnu-debuglink=library.so.debug library.so +strip --strip-debug library.so +``` + +### macOS Workflow +```bash +dsymutil library.dylib -o library.dylib.dSYM +strip -S library.dylib +``` + +### Size Reduction +- Original with debug: ~6.1 MB +- Stripped library: ~1.2 MB (80% reduction) +- Debug symbols: ~6.1 MB (separate file) + +## Platform Utilities + +`PlatformUtils` provides platform detection and tool location: + +| Function | Description | +|----------|-------------| +| `currentPlatform` | Detects LINUX or MACOS | +| `currentArchitecture` | Detects X64 or ARM64 | +| `isMusl()` | Detects musl libc (Alpine Linux) | +| `javaHome()` | Finds JAVA_HOME | +| `jniIncludePaths()` | Returns JNI header paths | +| `isCompilerAvailable(compiler)` | Tests compiler with `--version` | +| `locateLibasan(compiler)` | Finds ASan library path | +| `locateLibtsan(compiler)` | Finds TSan library path | +| `hasFuzzer()` | Tests libFuzzer support | +| `sharedLibExtension()` | Returns "so" or "dylib" | + +## Plugin Components + +The `build-logic` directory contains all native build plugins: + +| Component | Plugin ID | Purpose | +|-----------|-----------|---------| +| `NativeBuildPlugin` | `com.datadoghq.native-build` | C++ compilation and linking | +| `GtestPlugin` | `com.datadoghq.gtest` | Google Test integration | +| `NativeCompileTask` | - | Parallel C++ compilation task | +| `NativeLinkTask` | - | Shared library linking task | +| `NativeLinkExecutableTask` | - | Executable linking task (for gtest) | +| `PlatformUtils` | - | Platform detection and compiler location | + +## GtestPlugin Integration + +The `GtestPlugin` consumes configurations from `NativeBuildPlugin`: + +```kotlin +plugins { + id("com.datadoghq.native-build") + id("com.datadoghq.gtest") +} + +gtest { + testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) + mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) + includes.from("src/main/cpp", "$javaHome/include") +} +``` + +For each test file, GtestPlugin creates: +- `compileGtest{Config}_{TestName}` - Compile sources with test +- `linkGtest{Config}_{TestName}` - Link test executable +- `gtest{Config}_{TestName}` - Execute the test + +See `build-logic/README.md` for full GtestPlugin documentation. + +## Error Handling Modes + +### FAIL_FAST (Default) +- Stops compilation on first error +- Uses sequential stream processing +- Provides immediate feedback + +### COLLECT_ALL +- Compiles all files regardless of errors +- Uses parallel stream processing +- Reports all errors at end +- Configurable max errors to show + +## Logging Levels + +| Level | Description | +|-------|-------------| +| QUIET | Minimal output | +| NORMAL | Standard progress (default) | +| VERBOSE | Progress per N files | +| DEBUG | Full command lines | + +## Future Considerations + +1. **Windows Support**: Add MSVC/MinGW compiler support if needed +2. **Fuzzer Compiler Detection**: Currently hardcodes clang++ +3. **Per-Configuration Compiler**: Allow different compilers per configuration +4. **Incremental Compilation**: Track source dependencies for partial rebuilds + +## Related Documentation + +- `build-logic/README.md` - Native build and GtestPlugin usage documentation +- `CLAUDE.md` - Build commands reference diff --git a/doc/EventTypeSystem.md b/doc/reference/EventTypeSystem.md similarity index 100% rename from doc/EventTypeSystem.md rename to doc/reference/EventTypeSystem.md diff --git a/doc/ProfilerMemoryRequirements.md b/doc/reference/ProfilerMemoryRequirements.md similarity index 100% rename from doc/ProfilerMemoryRequirements.md rename to doc/reference/ProfilerMemoryRequirements.md diff --git a/doc/RemoteSymbolication.md b/doc/reference/RemoteSymbolication.md similarity index 100% rename from doc/RemoteSymbolication.md rename to doc/reference/RemoteSymbolication.md diff --git a/doc/ModifierAllocation.md b/doc/reference/RemoteSymbolicationFrameTypes.md similarity index 100% rename from doc/ModifierAllocation.md rename to doc/reference/RemoteSymbolicationFrameTypes.md diff --git a/doc/test-flakiness-analysis.md b/doc/reference/TestFlakinessAnalysis.md similarity index 100% rename from doc/test-flakiness-analysis.md rename to doc/reference/TestFlakinessAnalysis.md diff --git a/gradle.properties.template b/gradle.properties.template new file mode 100644 index 000000000..b56022b3d --- /dev/null +++ b/gradle.properties.template @@ -0,0 +1,75 @@ +# Java Profiler - Gradle Properties Template +# Copy this file to gradle.properties and customize as needed. +# +# Note: Property values can be overridden via command line with -P flag +# Example: ./gradlew build -Pskip-tests + +# ============================================================================= +# VERSION MANAGEMENT +# ============================================================================= + +# Override the project version (useful for custom builds) +# ddprof_version=1.38.0-custom + +# ============================================================================= +# BUILD CONTROL +# ============================================================================= + +# Skip test execution (Java tests) +# skip-tests=true + +# Skip native C++ compilation +# skip-native=true + +# Skip Google Test (C++ unit tests) +# skip-gtest=true + +# Skip fuzz testing +# skip-fuzz=true + +# ============================================================================= +# COMPILER CONFIGURATION +# ============================================================================= + +# Force a specific C++ compiler (auto-detects clang++ or g++ if not set) +# native.forceCompiler=/usr/bin/clang++ + +# ============================================================================= +# EXTERNAL DEPENDENCIES +# ============================================================================= + +# Path to pre-built native libraries (skips native compilation if set) +# with-libs=/path/to/external/libs + +# ============================================================================= +# TESTING +# ============================================================================= + +# Keep JFR recordings after test runs (also respects KEEP_JFRS env var) +# keepJFRs=true + +# Arguments to pass to UnwindingValidator +# validatorArgs=--verbose --output-format=json + +# ============================================================================= +# CI / PUBLISHING +# ============================================================================= + +# CI environment flag (auto-detected from CI env var) +# CI=true + +# Use local Nexus for testing (requires running docker container) +# forceLocal=true + +# ============================================================================= +# MEMORY / PERFORMANCE +# ============================================================================= + +# Gradle JVM arguments (default values shown) +# org.gradle.jvmargs=-Xmx2g -XX:+HeapDumpOnOutOfMemoryError + +# Enable Gradle build cache +# org.gradle.caching=true + +# Enable parallel project execution +# org.gradle.parallel=true diff --git a/gradle/configurations.gradle b/gradle/configurations.gradle deleted file mode 100644 index b539e395e..000000000 --- a/gradle/configurations.gradle +++ /dev/null @@ -1,348 +0,0 @@ -apply from: rootProject.file('common.gradle') - -ext.buildConfigurations = [] - -@SuppressWarnings('GrMethodMayBeStatic') -def buildConfigNames() { - buildConfigurations.findAll { - it.os == osIdentifier() && it.arch == archIdentifier() - }.collect { it.name }.toSet() -} - -@SuppressWarnings('GrMethodMayBeStatic') -def buildConfigs() { - return buildConfigurations.findAll { - it.os == osIdentifier() && it.arch == archIdentifier() - } -} - -def locate(def file) { - if (os().isLinux()) { - try { - def locateCommand = "locate ${file}" - def process = locateCommand.execute() - process.waitFor() - - if (process.exitValue() == 0) { - return process.in.text.trim() - } - } catch (Exception e) { - logger.warn("Exception when locating ${file}: ${e.message}") - } - } - return null -} - -def locateLibrary(String libName) { - if (os().isLinux()) { - try { - def locateCommand = "gcc -print-file-name=${libName}.so" - def process = locateCommand.execute() - process.waitFor() - - if (process.exitValue() == 0) { - def output = process.in.text.trim() - if (output.endsWith("${libName}.so")) { - return output - } - } - } catch (Exception e) { - logger.warn("Exception when locating ${libName} library: ${e.message}") - } - } - return null -} - -def locateLibasan() { - return locateLibrary('libasan') -} - -def locateLibtsan() { - return locateLibrary('libtsan') -} - -def checkFuzzerSupport() { - // libFuzzer requires clang - check if clang supports -fsanitize=fuzzer - try { - def testFile = File.createTempFile("fuzzer_check", ".cpp") - testFile.text = "extern \"C\" int LLVMFuzzerTestOneInput(const char*, long) { return 0; }" - testFile.deleteOnExit() - - def process = ["clang++", "-fsanitize=fuzzer", "-c", testFile.absolutePath, "-o", "/dev/null"].execute() - process.waitFor() - testFile.delete() - - return process.exitValue() == 0 - } catch (Exception e) { - logger.debug("Exception when checking fuzzer support: ${e.message}") - return false - } -} - -ext.libasan = locateLibasan() -ext.libtsan = locateLibtsan() -ext.hasFuzzerSupport = checkFuzzerSupport() - -if (ext.libasan == null) { - logger.debug("Asan library not found") -} -if (ext.libtsan == null) { - logger.debug("Tsan library not found") -} -if (!ext.hasFuzzerSupport) { - logger.debug("libFuzzer not available (requires clang with -fsanitize=fuzzer support)") -} - -def hasAsan() { - return isMusl() ? false : ext.libasan != null -} - -def hasTsan() { - return isMusl() ? false : ext.libtsan != null -} - -def hasFuzzer() { - return isMusl() ? false : ext.hasFuzzerSupport -} - -def isActive(def name) { - def methodName = "has${name.capitalize()}" - if (this.metaClass.respondsTo(this, methodName)) { - this."${methodName}"() - } else { - // no activation method defined; default to 'true' - true - } -} - -ext.addBuildConfiguration = { String name, String os, String arch, List compilerArgs, List linkerArgs, Map testEnv = [:] -> - buildConfigurations << [active: isActive(name), name: name, os: os, arch: arch, compilerArgs: compilerArgs, linkerArgs: linkerArgs, testEnv: testEnv] -} - -ext { - hasAsan = this.&hasAsan - hasTsan = this.&hasTsan - hasFuzzer = this.&hasFuzzer - buildConfigNames = this.&buildConfigNames - buildConfigs = this.&buildConfigs - locateLibasan = this.&locateLibasan - locateLibtsan = this.&locateLibtsan -} - -// make sure the provided version is correctly propagated to the build -version = project.findProperty("ddprof_version") ?: version - -// ======= Define build configurations below ======== - -def commonLinuxCompilerArgs = [ - "-fPIC", - "-fno-omit-frame-pointer", - "-momit-leaf-frame-pointer", - "-fvisibility=hidden", - "-fdata-sections", - "-ffunction-sections", - "-std=c++17", - "-DPROFILER_VERSION=\"${version}\"", - "-DCOUNTERS" -] - -def commonLinuxLinkerArgs = ["-ldl", "-Wl,-z,defs", "--verbose", "-lpthread", "-lm", "-lrt", "-v", "-Wl,--build-id"] - -def commonMacosCompilerArgs = commonLinuxCompilerArgs + ["-D_XOPEN_SOURCE", "-D_DARWIN_C_SOURCE"] - -def asanEnv = hasAsan() ? - ['LD_PRELOAD': libasan, - // warning: stack use after return can cause slowness on arm64 - "ASAN_OPTIONS" : "allocator_may_return_null=1:unwind_abort_on_malloc=1:use_sigaltstack=0:detect_stack_use_after_return=0:handle_segv=1:halt_on_error=0:abort_on_error=0:print_stacktrace=1:symbolize=1:suppressions=${rootDir}/gradle/sanitizers/asan.supp", - "UBSAN_OPTIONS" : "halt_on_error=0:abort_on_error=0:print_stacktrace=1:suppressions=${rootDir}/gradle/sanitizers/ubsan.supp", - // lsan still does not run for all tests - manually trigger on some tests - "LSAN_OPTIONS" : "detect_leaks=0" - ] : [:] - -def asanCompilerArgs = hasAsan() ? [ - '-g', - // Generate debug information - '-DDEBUG', - // Enable debug mode - '-fPIC', - // Position-independent code - '-fsanitize=address', - // AddressSanitizer for memory errors - '-fsanitize=undefined', - // UndefinedBehaviorSanitizer for undefined behavior - '-fno-sanitize-recover=all', - // Disable recovery to ensure program halts on error - '-fsanitize=float-divide-by-zero', - // Catch floating point division by zero - '-fstack-protector-all', - // Protect against stack buffer overflows - '-fsanitize=leak', - // LeakSanitizer to detect memory leaks - '-fsanitize=pointer-overflow', - // Catch pointer overflows - '-fsanitize=return', - // Check for uninitialized return values - '-fsanitize=bounds', - // BoundsSanitizer for array bounds - '-fsanitize=alignment', - // Catch misaligned pointer access - '-fsanitize=object-size', - // Catch accesses that are likely out of bounds - '-fno-omit-frame-pointer', - // Keep frame pointer for better debugging - '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces -] : [] - -def asanLinkerArgs = hasAsan() ? [ - "-L${file(libasan).parent}", - "-lasan", - "-lubsan", - "-fsanitize=address", - "-fsanitize=undefined", - "-fno-omit-frame-pointer" // Often recommended with sanitizers -] : [] - -def tsanCompilerArgs = hasTsan() ? [ - '-g', - // Generate debug information - '-DDEBUG', - // Enable debug mode - '-fPIC', - // Position-independent code - '-fsanitize=thread', - // ThreadSanitizer for - '-fno-omit-frame-pointer', - // Keep frame pointer for better debugging - '-fno-optimize-sibling-calls' // Disable tail call optimization for better stack traces -] : [] - -def tsanLinkerArgs = hasTsan() ? [ - "-L${file(libtsan).parent}", - "-ltsan", - "-fsanitize=thread", - "-fno-omit-frame-pointer" // Often recommended with sanitizers -] : [] - -def tsanEnv = hasTsan() ? - ['LD_PRELOAD': libtsan, - "TSAN_OPTIONS" : "suppressions=${rootDir}/gradle/sanitizers/tsan.supp" - ] : [:] - -// Linux -addBuildConfiguration 'release', 'linux', 'x64', - ['-O3', '-DNDEBUG', '-g'] + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + [ - "-Wl,-z,nodelete", - "-static-libstdc++", - "-static-libgcc", - "-Wl,--exclude-libs,ALL", - "-Wl,--gc-sections" - ] -addBuildConfiguration 'debug', 'linux', 'x64', - ['-O0', '-g', '-DDEBUG'] + commonLinuxCompilerArgs, - commonLinuxLinkerArgs - -addBuildConfiguration 'asan', 'linux', 'x64', - asanCompilerArgs + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + asanLinkerArgs, - asanEnv - -addBuildConfiguration 'tsan', 'linux', 'x64', - tsanCompilerArgs + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + tsanLinkerArgs, - tsanEnv - -addBuildConfiguration 'release', 'linux', 'arm64', - ['-O3', '-DNDEBUG', '-g'] + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + [ - "-Wl,-z,nodelete", - "-static-libstdc++", - "-static-libgcc", - "-Wl,--exclude-libs,ALL", - "-Wl,--gc-sections" - ] -addBuildConfiguration 'debug', 'linux', 'arm64', - ['-O0', '-g', '-DDEBUG'] + commonLinuxCompilerArgs, - commonLinuxLinkerArgs - -addBuildConfiguration 'asan', 'linux', 'arm64', - asanCompilerArgs + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + asanLinkerArgs, - asanEnv - -addBuildConfiguration 'tsan', 'linux', 'arm64', - tsanCompilerArgs + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + tsanLinkerArgs, - tsanEnv - -// Fuzzer configuration (requires clang with libFuzzer support) -// Note: Fuzzers are compiled separately and linked as executables, not shared libraries. -// The fuzzer build config provides common compiler flags; actual fuzzer linking happens -// in the fuzz subproject with -fsanitize=fuzzer added at link time. -def fuzzerCompilerArgs = hasFuzzer() ? [ - '-g', - // Generate debug information for crash analysis - '-DDEBUG', - // Enable debug mode and assertions - '-fPIC', - // Position-independent code - '-fsanitize=address', - // AddressSanitizer for memory errors - '-fsanitize=undefined', - // UndefinedBehaviorSanitizer for undefined behavior - '-fno-sanitize-recover=all', - // Halt on sanitizer errors for reliable crash detection - '-fno-omit-frame-pointer', - // Keep frame pointer for stack traces - '-fno-optimize-sibling-calls' - // Disable tail call optimization for better stack traces -] : [] - -def fuzzerLinkerArgs = hasFuzzer() ? ["-fsanitize=address", "-fsanitize=undefined", "-fno-omit-frame-pointer"] : [] - -def fuzzerEnv = hasFuzzer() ? - ["ASAN_OPTIONS" : "allocator_may_return_null=1:detect_stack_use_after_return=0:handle_segv=0:abort_on_error=1:symbolize=1:suppressions=${rootDir}/gradle/sanitizers/asan.supp", - "UBSAN_OPTIONS" : "halt_on_error=1:abort_on_error=1:print_stacktrace=1:suppressions=${rootDir}/gradle/sanitizers/ubsan.supp" - ] : [:] - -addBuildConfiguration 'fuzzer', 'linux', 'x64', - fuzzerCompilerArgs + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + fuzzerLinkerArgs, - fuzzerEnv - -addBuildConfiguration 'fuzzer', 'linux', 'arm64', - fuzzerCompilerArgs + commonLinuxCompilerArgs, - commonLinuxLinkerArgs + fuzzerLinkerArgs, - fuzzerEnv - -// MacOS -addBuildConfiguration 'release', 'macos', 'arm64', - commonMacosCompilerArgs + ['-O3', '-DNDEBUG'], - [] -addBuildConfiguration 'debug', 'macos', 'arm64', - commonMacosCompilerArgs + ['-O0', '-g', '-DDEBUG'], - [] - -// macOS fuzzer configuration -def fuzzerMacosCompilerArgs = hasFuzzer() ? [ - '-g', - '-DDEBUG', - '-fPIC', - '-fsanitize=address', - '-fsanitize=undefined', - '-fno-sanitize-recover=all', - '-fno-omit-frame-pointer', - '-fno-optimize-sibling-calls' -] : [] - -def fuzzerMacosLinkerArgs = hasFuzzer() ? ["-fsanitize=address", "-fsanitize=undefined"] : [] - -def fuzzerMacosEnv = hasFuzzer() ? - ["ASAN_OPTIONS" : "allocator_may_return_null=1:detect_stack_use_after_return=0:abort_on_error=1:symbolize=1", - "UBSAN_OPTIONS" : "halt_on_error=1:abort_on_error=1:print_stacktrace=1" - ] : [:] - -addBuildConfiguration 'fuzzer', 'macos', 'arm64', - fuzzerMacosCompilerArgs + commonMacosCompilerArgs, - fuzzerMacosLinkerArgs, - fuzzerMacosEnv \ No newline at end of file diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 000000000..b37a07660 --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,63 @@ +# Copyright 2026, Datadog, Inc + +[versions] +cpp-standard = "17" +kotlin = "1.9.22" +spotless = "6.11.0" + +# Testing +junit = "5.9.2" +junit-pioneer = "1.9.1" +slf4j = "1.7.32" + +# Profiler runtime +jmc = "8.1.0" +jol = "0.16" +lz4 = "1.8.0" +snappy = "1.1.10.1" +zstd = "1.5.5-4" + +# Code analysis +asm = "9.6" + +# Benchmarking +jmh = "1.36" + +[libraries] +# JUnit 5 +junit-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "junit" } +junit-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "junit" } +junit-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit" } +junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version.ref = "junit-pioneer" } + +# Logging +slf4j-simple = { module = "org.slf4j:slf4j-simple", version.ref = "slf4j" } + +# JFR and memory analysis +jmc-flightrecorder = { module = "org.openjdk.jmc:flightrecorder", version.ref = "jmc" } +jol-core = { module = "org.openjdk.jol:jol-core", version.ref = "jol" } + +# Compression libraries +lz4 = { module = "org.lz4:lz4-java", version.ref = "lz4" } +snappy = { module = "org.xerial.snappy:snappy-java", version.ref = "snappy" } +zstd = { module = "com.github.luben:zstd-jni", version.ref = "zstd" } + +# Bytecode analysis +asm = { module = "org.ow2.asm:asm", version.ref = "asm" } + +# JMH benchmarking +jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" } +jmh-annprocess = { module = "org.openjdk.jmh:jmh-generator-annprocess", version.ref = "jmh" } + +[bundles] +# Core testing framework (JUnit + logging) +testing = ["junit-api", "junit-engine", "junit-params", "junit-pioneer", "slf4j-simple"] + +# Profiler runtime dependencies (JFR analysis + compression) +profiler-runtime = ["jmc-flightrecorder", "jol-core", "lz4", "snappy", "zstd"] + +# JMH benchmarking +jmh = ["jmh-core", "jmh-annprocess"] + +[plugins] +spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } diff --git a/gradle/spotless.gradle b/gradle/spotless.gradle deleted file mode 100644 index 3db49fb22..000000000 --- a/gradle/spotless.gradle +++ /dev/null @@ -1,96 +0,0 @@ -apply from: rootProject.file('common.gradle') -apply plugin: 'com.diffplug.spotless' - -// This definition is needed since the spotless file is used from stand alone projects -def configPath = rootProject.hasProperty('sharedConfigDirectory') ? sharedConfigDirectory : project.rootProject.rootDir.path + '/gradle' -// This is necessary for some projects that set a special groovy target which can't coexist with excludeJava -boolean groovySkipJavaExclude = project.hasProperty('groovySkipJavaExclude') ? groovySkipJavaExclude : false - -spotless { - if (project.plugins.hasPlugin('java')) { - java { - toggleOffOn() - // set explicit target to workaround https://github.com/diffplug/spotless/issues/1163 - target 'src/**/*.java' - // ignore embedded test projects - targetExclude 'src/test/resources/**' - // This is the last Google Java Format version that supports Java 8 - googleJavaFormat('1.7') - } - } - - groovyGradle { - toggleOffOn() - // same as groovy, but for .gradle (defaults to '*.gradle') - target '*.gradle', 'gradle/**/*.gradle' - greclipse().configFile(configPath + '/enforcement/spotless-groovy.properties') - } - - kotlinGradle { - toggleOffOn() - // same as kotlin, but for .gradle.kts files (defaults to '*.gradle.kts') - target '*.gradle.kts' - // ktfmt('0.40').kotlinlangStyle() // needs Java 11+ - // Newer versions do not work well with the older version of kotlin in this build - ktlint('0.41.0').userData(['indent_size': '2', 'continuation_indent_size': '2']) - } - - if (project.plugins.hasPlugin('groovy')) { - groovy { - toggleOffOn() - if (!groovySkipJavaExclude) { - excludeJava() // excludes all Java sources within the Groovy source dirs from formatting - // the Groovy Eclipse formatter extends the Java Eclipse formatter, - // so it formats Java files by default (unless `excludeJava` is used). - } - greclipse().configFile(configPath + '/enforcement/spotless-groovy.properties') - } - } - - if (project.plugins.hasPlugin('scala')) { - scala { - toggleOffOn() - scalafmt('2.7.5').configFile(configPath + '/enforcement/spotless-scalafmt.conf') - } - } - if (project.plugins.hasPlugin('cpp-library') || project.plugins.hasPlugin('cpp-application')) { - cpp { - def clangVersion = null - try { - def checkClangFormat = ['clang-format', '--version'].execute() - checkClangFormat.waitFor() - if (checkClangFormat.exitValue() == 0) { - def versionOutput = checkClangFormat.text.trim() - // Extract the version number from the output using regex - def versionMatcher = (versionOutput =~ /clang-format version (\d+\.\d+\.\d+.*)/) - if (versionMatcher) { - clangVersion = versionMatcher[0][1] // The version is captured in group 1 - } - } - } catch (Throwable t) { - throw new GradleException("Clang-format is not available. Please install Clang 11.") - } - - // Check if the version is 11.0.0 or higher - if (clangVersion && !clangVersion.startsWith("11")) { - throw new GradleException("Clang version ${clangVersion} is not supported. Please install Clang 11.") - } - - target 'src/main/cpp/**' - clangFormat(clangVersion).configFile(configPath + '/enforcement/.clang-format') - } - } - - - format 'misc', { - toggleOffOn() - target '.gitignore', '*.md', '.github/**/*.md', 'src/**/*.md', 'application/**/*.md', '*.sh', 'tooling/*.sh', '.circleci/*.sh' - indentWithSpaces() - trimTrailingWhitespace() - endWithNewline() - } -} - -check.configure { - dependsOn 'spotlessCheck' -} diff --git a/malloc-shim/build.gradle b/malloc-shim/build.gradle deleted file mode 100644 index 229f5bcb8..000000000 --- a/malloc-shim/build.gradle +++ /dev/null @@ -1,24 +0,0 @@ -plugins { - id 'cpp-library' -} - -group = 'com.datadoghq' -version = '0.1' - -tasks.withType(CppCompile).configureEach { - compilerArgs.addAll( - [ - "-O3", - "-fno-omit-frame-pointer", - "-fvisibility=hidden", - "-std=c++17", - "-DPROFILER_VERSION=\"${project.getProperty('version')}\"" - ] - ) -} - -library { - baseName = "debug" - targetMachines = [machines.linux.x86_64] - linkage = [Linkage.SHARED] -} \ No newline at end of file diff --git a/malloc-shim/build.gradle.kts b/malloc-shim/build.gradle.kts new file mode 100644 index 000000000..1a819db48 --- /dev/null +++ b/malloc-shim/build.gradle.kts @@ -0,0 +1,36 @@ +/* + * Memory allocation interceptor for malloc debugging (Linux only). + */ + +import com.datadoghq.native.model.Platform +import com.datadoghq.native.util.PlatformUtils + +plugins { + base + id("com.datadoghq.simple-native-lib") +} + +group = "com.datadoghq" +version = "0.1" + +simpleNativeLib { + // malloc-shim is Linux-only + enabled.set(PlatformUtils.currentPlatform == Platform.LINUX) + + libraryName.set("debug") + sourceDir.set(file("src/main/cpp")) + includeDirs.set(listOf(file("src/main/public").absolutePath)) + + compilerArgs.set( + listOf( + "-O3", + "-fno-omit-frame-pointer", + "-fvisibility=hidden", + "-std=c++17", + "-DPROFILER_VERSION=\"${project.version}\"", + "-fPIC" + ) + ) + + linkerArgs.set(listOf("-ldl")) +} diff --git a/malloc-shim/settings.gradle b/malloc-shim/settings.gradle deleted file mode 100644 index 90f543d4c..000000000 --- a/malloc-shim/settings.gradle +++ /dev/null @@ -1 +0,0 @@ -rootProject.name = "malloc-shim" \ No newline at end of file diff --git a/malloc-shim/settings.gradle.kts b/malloc-shim/settings.gradle.kts new file mode 100644 index 000000000..0b00a8c6e --- /dev/null +++ b/malloc-shim/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "malloc-shim" diff --git a/settings.gradle b/settings.gradle deleted file mode 100644 index 8f4bae0d7..000000000 --- a/settings.gradle +++ /dev/null @@ -1,8 +0,0 @@ -include ':ddprof-lib' -include ':ddprof-lib:gtest' -include ':ddprof-lib:fuzz' -include ':ddprof-lib:benchmarks' -include ':ddprof-test-tracer' -include ':ddprof-test' -include ':malloc-shim' -include ':ddprof-stresstest' diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 000000000..c4881dc3b --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,23 @@ +pluginManagement { + includeBuild("build-logic") +} + +// Centralized dependency resolution - subprojects should not define their own repositories +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.PREFER_SETTINGS) + repositories { + mavenCentral() + gradlePluginPortal() + } +} + +rootProject.name = "java-profiler" + +include(":ddprof-lib") +include(":ddprof-lib:fuzz") +include(":ddprof-lib:benchmarks") +include(":ddprof-test-tracer") +include(":ddprof-test") +include(":ddprof-test-native") +include(":malloc-shim") +include(":ddprof-stresstest")