feat(profiling): add metric on sample count

Author: KowalskiThomas

ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt

@@ -54,6 +54,8 @@ add_library(
     src/code_provenance_interface.cpp
     src/ddup_interface.cpp
     src/profile.cpp
+    src/profile_borrow.cpp
+    src/profiler_stats.cpp
     src/sample.cpp
     src/sample_manager.cpp
     src/static_sample_pool.cpp

ddtrace/internal/datadog/profiling/dd_wrapper/include/ddup_interface.hpp

@@ -1,14 +1,13 @@
 #pragma once
 
-#include <stddef.h>
-#include <stdint.h>
+#include <cstdint>
 #include <string_view>
 #include <unordered_map>
 
 // Forward decl of the return pointer
 namespace Datadog {
 class Sample;
-}
+} // namespace Datadog
 
 #ifdef __cplusplus
 extern "C"
@@ -68,6 +67,10 @@ extern "C"
                          int64_t line);
     void ddup_push_absolute_ns(Datadog::Sample* sample, int64_t timestamp_ns);
     void ddup_push_monotonic_ns(Datadog::Sample* sample, int64_t monotonic_ns);
+
+    void ddup_increment_sampling_event_count();
+    void ddup_increment_sample_count();
+
     void ddup_flush_sample(Datadog::Sample* sample);
     // Stack v2 specific flush, which reverses the locations
     void ddup_flush_sample_v2(Datadog::Sample* sample);

ddtrace/internal/datadog/profiling/dd_wrapper/include/profile.hpp

@@ -1,13 +1,11 @@
 #pragma once
 
 #include "constants.hpp"
+#include "profiler_stats.hpp"
 #include "types.hpp"
 
 #include <atomic>
-#include <memory>
 #include <mutex>
-#include <string>
-#include <string_view>
 #include <vector>
 
 extern "C"
@@ -17,9 +15,13 @@ extern "C"
 
 namespace Datadog {
 
+class ProfileBorrow;
+
 // Serves to collect individual samples, as well as lengthen the scope of string data
 class Profile
 {
+    friend class ProfileBorrow;
+
   private:
     // Serialization for static state
     // - string table
@@ -45,6 +47,12 @@ class Profile
     // cannot be used until it's initialized by libdatadog
     ddog_prof_Profile cur_profile{};
 
+    Datadog::ProfilerStats profiler_stats{};
+
+    // Internal access methods - not for direct use
+    ddog_prof_Profile& profile_borrow_internal();
+    void profile_release();
+
   public:
     // State management
     void one_time_init(SampleType type, unsigned int _max_nframes);
@@ -53,8 +61,8 @@ class Profile
 
     // Getters
     size_t get_sample_type_length();
-    ddog_prof_Profile& profile_borrow();
-    void profile_release();
+
+    ProfileBorrow borrow();
 
     // constref getters
     const ValueIndex& val();

ddtrace/internal/datadog/profiling/dd_wrapper/include/profile_borrow.hpp

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "profile.hpp"
+
+namespace Datadog {
+
+// Forward declaration
+class Profile;
+
+// RAII wrapper for borrowing both profile and stats under a single lock
+class ProfileBorrow
+{
+  private:
+    Profile* profile_ptr;
+
+  public:
+    explicit ProfileBorrow(Profile& profile);
+    ~ProfileBorrow();
+
+    // Disable copy
+    ProfileBorrow(const ProfileBorrow&) = delete;
+    ProfileBorrow& operator=(const ProfileBorrow&) = delete;
+
+    // Enable move
+    ProfileBorrow(ProfileBorrow&& other) noexcept;
+    ProfileBorrow& operator=(ProfileBorrow&& other) noexcept;
+
+    // Accessors
+    ddog_prof_Profile& profile();
+    ProfilerStats& stats();
+};
+
+} // namespace Datadog

ddtrace/internal/datadog/profiling/dd_wrapper/include/profiler_stats.hpp

@@ -0,0 +1,47 @@
+#pragma once
+
+#include <cstddef>
+
+#include <string>
+#include <string_view>
+
+namespace Datadog {
+
+/*
+ProfilerStats holds statistics around Profiling to be sent along
+with the actual Profiles.
+
+None of its methods are thread-safe and it should typically used with
+a mutex to protect access to the data.
+*/
+class ProfilerStats
+{
+  private:
+    std::string internal_metadata_json;
+
+    // Number of samples collected (one per thread)
+    size_t sample_count = 0;
+
+    // Number of sampling events (one per collection cycle)
+    size_t sampling_event_count = 0;
+
+  public:
+    ProfilerStats() = default;
+    ~ProfilerStats() = default;
+
+    void increment_sample_count(size_t k_sample_count = 1);
+    size_t get_sample_count();
+
+    void increment_sampling_event_count(size_t k_sampling_event_count = 1);
+    size_t get_sampling_event_count();
+
+    // Returns a JSON string containing relevant Profiler Stats to be included
+    // in the libdatadog payload.
+    // The function returned a string_view to a statically allocated string that
+    // is updated every time the function is called.
+    std::string_view get_internal_metadata_json();
+
+    void reset_state();
+};
+
+} // namespace Datadog

ddtrace/internal/datadog/profiling/dd_wrapper/include/sample.hpp

@@ -2,6 +2,7 @@
 
 #include "libdatadog_helpers.hpp"
 #include "profile.hpp"
+#include "profile_borrow.hpp"
 #include "types.hpp"
 
 #include <string>
@@ -134,8 +135,7 @@ class Sample
     // Flushes the current buffer, clearing it
     bool flush_sample(bool reverse_locations = false);
 
-    static ddog_prof_Profile& profile_borrow();
-    static void profile_release();
+    static ProfileBorrow profile_borrow();
     static void postfork_child();
     Sample(SampleType _type_mask, unsigned int _max_nframes);
 

ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader.hpp

@@ -1,11 +1,10 @@
 #pragma once
 
-#include "sample.hpp"
-#include "types.hpp"
+#include "profiler_stats.hpp"
 
 #include <atomic>
-#include <memory>
 #include <mutex>
+#include <string>
 
 extern "C"
 {
@@ -24,10 +23,10 @@ class Uploader
     std::string output_filename;
     ddog_prof_ProfileExporter ddog_exporter{ .inner = nullptr };
 
-    bool export_to_file(ddog_prof_EncodedProfile* encoded);
+    bool export_to_file(ddog_prof_EncodedProfile* encoded, std::string_view internal_metadata_json);
 
   public:
-    bool upload(ddog_prof_Profile& profile);
+    bool upload(ddog_prof_Profile& profile, Datadog::ProfilerStats& profiler_stats);
     static void cancel_inflight();
     static void lock();
     static void unlock();

ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader_builder.hpp

@@ -1,8 +1,8 @@
 #pragma once
 
+#include "constants.hpp"
 #include "uploader.hpp"
 
-#include <mutex>
 #include <string>
 #include <string_view>
 #include <unordered_map>

ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp

@@ -1,8 +1,7 @@
 #include "ddup_interface.hpp"
 
-#include "code_provenance.hpp"
 #include "libdatadog_helpers.hpp"
-#include "profile.hpp"
+#include "profiler_stats.hpp"
 #include "sample.hpp"
 #include "sample_manager.hpp"
 #include "uploader.hpp"
@@ -304,6 +303,20 @@ ddup_push_monotonic_ns(Datadog::Sample* sample, int64_t monotonic_ns) // cppchec
     sample->push_monotonic_ns(monotonic_ns);
 }
 
+void
+ddup_increment_sampling_event_count() // cppcheck-suppress unusedFunction
+{
+    auto borrowed = Datadog::Sample::profile_borrow();
+    borrowed.stats().increment_sampling_event_count();
+}
+
+void
+ddup_increment_sample_count() // cppcheck-suppress unusedFunction
+{
+    auto borrowed = Datadog::Sample::profile_borrow();
+    borrowed.stats().increment_sample_count();
+}
+
 void
 ddup_flush_sample(Datadog::Sample* sample) // cppcheck-suppress unusedFunction
 {
@@ -351,17 +364,19 @@ ddup_upload() // cppcheck-suppress unusedFunction
     //  be modified.  It gets released and cleared after uploading.
     // * Uploading cancels inflight uploads. There are better ways to do this, but this is what
     //   we have for now.
-    uploader.upload(Datadog::Sample::profile_borrow());
-    Datadog::Sample::profile_release();
-    return true;
+    auto borrowed = Datadog::Sample::profile_borrow();
+    bool success = uploader.upload(borrowed.profile(), borrowed.stats());
+    borrowed.stats().reset_state();
+    return success;
 }
 
 void
 ddup_profile_set_endpoints(
   std::unordered_map<int64_t, std::string_view> span_ids_to_endpoints) // cppcheck-suppress unusedFunction
 {
     static bool already_warned = false; // cppcheck-suppress threadsafety-threadsafety
-    ddog_prof_Profile& profile = Datadog::Sample::profile_borrow();
+    auto borrowed = Datadog::Sample::profile_borrow();
+    ddog_prof_Profile& profile = borrowed.profile();
     for (const auto& [span_id, trace_endpoint] : span_ids_to_endpoints) {
         ddog_CharSlice trace_endpoint_slice = Datadog::to_slice(trace_endpoint);
         auto res = ddog_prof_Profile_set_endpoint(&profile, span_id, trace_endpoint_slice);
@@ -375,14 +390,14 @@ ddup_profile_set_endpoints(
             ddog_Error_drop(&err);
         }
     }
-    Datadog::Sample::profile_release();
 }
 
 void
 ddup_profile_add_endpoint_counts(std::unordered_map<std::string_view, int64_t> trace_endpoints_to_counts)
 {
     static bool already_warned = false; // cppcheck-suppress threadsafety-threadsafety
-    ddog_prof_Profile& profile = Datadog::Sample::profile_borrow();
+    auto borrowed = Datadog::Sample::profile_borrow();
+    ddog_prof_Profile& profile = borrowed.profile();
     for (const auto& [trace_endpoint, count] : trace_endpoints_to_counts) {
         ddog_CharSlice trace_endpoint_slice = Datadog::to_slice(trace_endpoint);
         auto res = ddog_prof_Profile_add_endpoint_count(&profile, trace_endpoint_slice, count);
@@ -396,5 +411,4 @@ ddup_profile_add_endpoint_counts(std::unordered_map<std::string_view, int64_t> t
             ddog_Error_drop(&err);
         }
     }
-    Datadog::Sample::profile_release();
 }

ddtrace/internal/datadog/profiling/dd_wrapper/src/profile.cpp

@@ -1,6 +1,8 @@
 #include "profile.hpp"
 
 #include "libdatadog_helpers.hpp"
+#include "profile_borrow.hpp"
+#include "profiler_stats.hpp"
 
 #include <functional>
 #include <iostream>
@@ -55,6 +57,8 @@ Datadog::Profile::reset_profile()
         ddog_Error_drop(&err);
         return false;
     }
+
+    profiler_stats.reset_state();
     return true;
 }
 
@@ -127,11 +131,16 @@ Datadog::Profile::get_sample_type_length()
     return samplers.size();
 }
 
+Datadog::ProfileBorrow
+Datadog::Profile::borrow()
+{
+    return ProfileBorrow(*this);
+}
+
 ddog_prof_Profile&
-Datadog::Profile::profile_borrow()
+Datadog::Profile::profile_borrow_internal()
 {
-    // We could wrap this in an object for better RAII, but since this
-    // sequence is only used in a single place, we'll hold off on that sidequest.
+    // Note: Caller is responsible for ensuring profile_release() is called
     profile_mtx.lock();
     return cur_profile;
 }
@@ -219,5 +228,6 @@ Datadog::Profile::postfork_child()
 {
     new (&profile_mtx) std::mutex();
     // Reset the profile to clear any samples collected in the parent process
+    profiler_stats.reset_state();
     reset_profile();
 }