diff --git a/Android.bp b/Android.bp index ee13d5b81ca..cf513ef396b 100644 --- a/Android.bp +++ b/Android.bp @@ -698,8 +698,10 @@ cc_library_shared { ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ], host_supported: true, export_include_dirs: [ @@ -787,6 +789,7 @@ cc_library_shared { "liblog", "libpcre2", "libz", + "libzstd", ], static_libs: [ "perfetto_flags_c_lib", @@ -796,6 +799,7 @@ cc_library_shared { static_libs: [ "liblog", "libz", + "libzstd", ], whole_static_libs: [ "libpcre2", @@ -969,8 +973,10 @@ cc_library { ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_src_tracing_system_backend", ], host_supported: true, @@ -1050,6 +1056,7 @@ cc_library { ], cflags: [ "-DPERFETTO_FORCE_DISABLE_ZLIB", + "-DPERFETTO_FORCE_DISABLE_ZSTD", "-DPERFETTO_REGEX_FORCE_STD", "-DPERFETTO_SHLIB_SDK_IMPLEMENTATION", "-DZLIB_IMPLEMENTATION", @@ -1195,8 +1202,10 @@ cc_library_static { ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_src_tracing_system_backend", ], shared_libs: [ @@ -1348,6 +1357,7 @@ cc_library_static { ], cflags: [ "-DPERFETTO_FORCE_DISABLE_ZLIB", + "-DPERFETTO_FORCE_DISABLE_ZSTD", "-DPERFETTO_REGEX_FORCE_STD", "-DZLIB_IMPLEMENTATION", ], @@ -1992,7 +2002,10 @@ cc_library_static { ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", + ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_test_test_helper", "test/cts/art_module/heapprofd_java_test_cts.cc", "test/cts/device_feature_test_cts.cc", @@ -2006,6 +2019,7 @@ cc_library_static { "liblog", "libpcre2", "libz", + "libzstd", ], static_libs: [ "libgmock", @@ -2370,11 +2384,16 @@ cc_library_static { ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", + ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_test_test_helper", ], shared_libs: [ "libpcre2", + "libz", + "libzstd", ], static_libs: [ "libgmock", @@ -2561,6 +2580,9 @@ cc_library_static { defaults: [ "perfetto_defaults", ], + cflags: [ + "-DZLIB_IMPLEMENTATION", + ], } // GN: //gn:default_deps @@ -3184,8 +3206,10 @@ cc_test { ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_producer_relay", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_src_tracing_system_backend", ":perfetto_src_tracing_test_api_test_support", ":perfetto_src_tracing_test_client_api_integrationtests", @@ -3206,6 +3230,7 @@ cc_test { "libunwindstack", "libutils", "libz", + "libzstd", ], static_libs: [ "libgmock", @@ -22169,6 +22194,11 @@ filegroup { ], } +// GN: //src/tracing/service:packet_compressor_common +filegroup { + name: "perfetto_src_tracing_service_packet_compressor_common", +} + // GN: //src/tracing/service:service filegroup { name: "perfetto_src_tracing_service_service", @@ -22189,12 +22219,12 @@ filegroup { filegroup { name: "perfetto_src_tracing_service_unittests", srcs: [ + "src/tracing/service/compressor_unittest.cc", "src/tracing/service/histogram_unittest.cc", "src/tracing/service/packet_stream_validator_unittest.cc", "src/tracing/service/trace_buffer_v1_unittest.cc", "src/tracing/service/trace_buffer_v2_unittest.cc", "src/tracing/service/tracing_service_impl_unittest.cc", - "src/tracing/service/zlib_compressor_unittest.cc", ], } @@ -22206,6 +22236,14 @@ filegroup { ], } +// GN: //src/tracing/service:zstd_compressor +filegroup { + name: "perfetto_src_tracing_service_zstd_compressor", + srcs: [ + "src/tracing/service/zstd_compressor.cc", + ], +} + // GN: //src/tracing:system_backend filegroup { name: "perfetto_src_tracing_system_backend", @@ -23621,9 +23659,11 @@ cc_test { ":perfetto_src_tracing_ipc_producer_relay", ":perfetto_src_tracing_ipc_service_service", ":perfetto_src_tracing_ipc_unittests", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", ":perfetto_src_tracing_service_unittests", ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_src_tracing_system_backend", ":perfetto_src_tracing_test_test_support", ":perfetto_src_tracing_unittests", @@ -23641,6 +23681,7 @@ cc_test { "libunwindstack", "libutils", "libz", + "libzstd", ], static_libs: [ "libgmock", @@ -24066,11 +24107,16 @@ cc_library_static { ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", ":perfetto_src_tracing_ipc_service_service", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", + ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", ":perfetto_test_test_helper", ], shared_libs: [ "libpcre2", + "libz", + "libzstd", ], static_libs: [ "libgmock", @@ -24257,6 +24303,9 @@ cc_library_static { defaults: [ "perfetto_defaults", ], + cflags: [ + "-DZLIB_IMPLEMENTATION", + ], } // GN: //src/protovm/compiler:protovm_compiler @@ -25589,7 +25638,10 @@ cc_binary { ":perfetto_src_tracing_ipc_common", ":perfetto_src_tracing_ipc_default_socket", ":perfetto_src_tracing_ipc_producer_producer", + ":perfetto_src_tracing_service_packet_compressor_common", ":perfetto_src_tracing_service_service", + ":perfetto_src_tracing_service_zlib_compressor", + ":perfetto_src_tracing_service_zstd_compressor", "src/profiling/perf/main.cc", ], shared_libs: [ @@ -25598,6 +25650,8 @@ cc_binary { "libpcre2", "libprocinfo", "libunwindstack", + "libz", + "libzstd", ], static_libs: [ "perfetto_flags_c_lib", @@ -25675,6 +25729,9 @@ cc_binary { defaults: [ "perfetto_defaults", ], + cflags: [ + "-DZLIB_IMPLEMENTATION", + ], include_dirs: [ "bionic/libc/kernel", ], diff --git a/BUILD b/BUILD index 196853dd86f..ae76acd292e 100644 --- a/BUILD +++ b/BUILD @@ -182,8 +182,10 @@ perfetto_cc_library( ":src_tracing_ipc_default_socket", ":src_tracing_ipc_producer_producer", ":src_tracing_ipc_service_service", + ":src_tracing_service_packet_compressor_common", ":src_tracing_service_service", ":src_tracing_service_zlib_compressor", + ":src_tracing_service_zstd_compressor", ":src_tracing_system_backend", ], hdrs = [ @@ -208,77 +210,78 @@ perfetto_cc_library( ], visibility = PERFETTO_CONFIG.public_visibility, deps = [ - ":perfetto_ipc", - ":protos_perfetto_common_cpp", - ":protos_perfetto_common_semantic_type_cpp", - ":protos_perfetto_common_semantic_type_zero", - ":protos_perfetto_common_zero", - ":protos_perfetto_config_android_cpp", - ":protos_perfetto_config_android_zero", - ":protos_perfetto_config_cpp", - ":protos_perfetto_config_ftrace_cpp", - ":protos_perfetto_config_ftrace_zero", - ":protos_perfetto_config_gpu_cpp", - ":protos_perfetto_config_gpu_zero", - ":protos_perfetto_config_inode_file_cpp", - ":protos_perfetto_config_inode_file_zero", - ":protos_perfetto_config_interceptors_cpp", - ":protos_perfetto_config_interceptors_zero", - ":protos_perfetto_config_linux_cpp", - ":protos_perfetto_config_linux_zero", - ":protos_perfetto_config_power_cpp", - ":protos_perfetto_config_power_zero", - ":protos_perfetto_config_priority_boost_cpp", - ":protos_perfetto_config_priority_boost_zero", - ":protos_perfetto_config_process_stats_cpp", - ":protos_perfetto_config_process_stats_zero", - ":protos_perfetto_config_profiling_cpp", - ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_protovm_cpp", - ":protos_perfetto_config_protovm_zero", - ":protos_perfetto_config_qnx_cpp", - ":protos_perfetto_config_qnx_zero", - ":protos_perfetto_config_statsd_cpp", - ":protos_perfetto_config_statsd_zero", - ":protos_perfetto_config_sys_stats_cpp", - ":protos_perfetto_config_sys_stats_zero", - ":protos_perfetto_config_system_info_cpp", - ":protos_perfetto_config_system_info_zero", - ":protos_perfetto_config_track_event_cpp", - ":protos_perfetto_config_track_event_zero", - ":protos_perfetto_config_zero", - ":protos_perfetto_ipc_cpp", - ":protos_perfetto_ipc_ipc", - ":protos_perfetto_protovm_cpp", - ":protos_perfetto_protovm_zero", - ":protos_perfetto_trace_android_zero", - ":protos_perfetto_trace_chrome_zero", - ":protos_perfetto_trace_etw_zero", - ":protos_perfetto_trace_filesystem_zero", - ":protos_perfetto_trace_ftrace_zero", - ":protos_perfetto_trace_generic_kernel_zero", - ":protos_perfetto_trace_gpu_zero", - ":protos_perfetto_trace_interned_data_zero", - ":protos_perfetto_trace_linux_zero", - ":protos_perfetto_trace_minimal_zero", - ":protos_perfetto_trace_non_minimal_zero", - ":protos_perfetto_trace_perfetto_zero", - ":protos_perfetto_trace_power_zero", - ":protos_perfetto_trace_profiling_zero", - ":protos_perfetto_trace_ps_zero", - ":protos_perfetto_trace_statsd_zero", - ":protos_perfetto_trace_sys_stats_zero", - ":protos_perfetto_trace_system_info_zero", - ":protos_perfetto_trace_track_event_cpp", - ":protos_perfetto_trace_track_event_zero", - ":protos_perfetto_trace_translation_zero", - ":protozero", - ":src_base_base", - ":src_base_clock_snapshots", - ":src_base_regex_regex", - ":src_base_version", - ":src_protovm_protovm", - ] + PERFETTO_CONFIG.deps.zlib, + ":perfetto_ipc", + ":protos_perfetto_common_cpp", + ":protos_perfetto_common_semantic_type_cpp", + ":protos_perfetto_common_semantic_type_zero", + ":protos_perfetto_common_zero", + ":protos_perfetto_config_android_cpp", + ":protos_perfetto_config_android_zero", + ":protos_perfetto_config_cpp", + ":protos_perfetto_config_ftrace_cpp", + ":protos_perfetto_config_ftrace_zero", + ":protos_perfetto_config_gpu_cpp", + ":protos_perfetto_config_gpu_zero", + ":protos_perfetto_config_inode_file_cpp", + ":protos_perfetto_config_inode_file_zero", + ":protos_perfetto_config_interceptors_cpp", + ":protos_perfetto_config_interceptors_zero", + ":protos_perfetto_config_linux_cpp", + ":protos_perfetto_config_linux_zero", + ":protos_perfetto_config_power_cpp", + ":protos_perfetto_config_power_zero", + ":protos_perfetto_config_priority_boost_cpp", + ":protos_perfetto_config_priority_boost_zero", + ":protos_perfetto_config_process_stats_cpp", + ":protos_perfetto_config_process_stats_zero", + ":protos_perfetto_config_profiling_cpp", + ":protos_perfetto_config_profiling_zero", + ":protos_perfetto_config_protovm_cpp", + ":protos_perfetto_config_protovm_zero", + ":protos_perfetto_config_qnx_cpp", + ":protos_perfetto_config_qnx_zero", + ":protos_perfetto_config_statsd_cpp", + ":protos_perfetto_config_statsd_zero", + ":protos_perfetto_config_sys_stats_cpp", + ":protos_perfetto_config_sys_stats_zero", + ":protos_perfetto_config_system_info_cpp", + ":protos_perfetto_config_system_info_zero", + ":protos_perfetto_config_track_event_cpp", + ":protos_perfetto_config_track_event_zero", + ":protos_perfetto_config_zero", + ":protos_perfetto_ipc_cpp", + ":protos_perfetto_ipc_ipc", + ":protos_perfetto_protovm_cpp", + ":protos_perfetto_protovm_zero", + ":protos_perfetto_trace_android_zero", + ":protos_perfetto_trace_chrome_zero", + ":protos_perfetto_trace_etw_zero", + ":protos_perfetto_trace_filesystem_zero", + ":protos_perfetto_trace_ftrace_zero", + ":protos_perfetto_trace_generic_kernel_zero", + ":protos_perfetto_trace_gpu_zero", + ":protos_perfetto_trace_interned_data_zero", + ":protos_perfetto_trace_linux_zero", + ":protos_perfetto_trace_minimal_zero", + ":protos_perfetto_trace_non_minimal_zero", + ":protos_perfetto_trace_perfetto_zero", + ":protos_perfetto_trace_power_zero", + ":protos_perfetto_trace_profiling_zero", + ":protos_perfetto_trace_ps_zero", + ":protos_perfetto_trace_statsd_zero", + ":protos_perfetto_trace_sys_stats_zero", + ":protos_perfetto_trace_system_info_zero", + ":protos_perfetto_trace_track_event_cpp", + ":protos_perfetto_trace_track_event_zero", + ":protos_perfetto_trace_translation_zero", + ":protozero", + ":src_base_base", + ":src_base_clock_snapshots", + ":src_base_regex_regex", + ":src_base_version", + ":src_protovm_protovm", + ] + PERFETTO_CONFIG.deps.zlib + + PERFETTO_CONFIG.deps.zstd, linkstatic = True, ) @@ -1176,8 +1179,10 @@ perfetto_cc_library( ":src_tracing_ipc_common", ":src_tracing_ipc_default_socket", ":src_tracing_ipc_service_service", + ":src_tracing_service_packet_compressor_common", ":src_tracing_service_service", ":src_tracing_service_zlib_compressor", + ":src_tracing_service_zstd_compressor", ] + select({ "@platforms//os:windows": [], "//conditions:default": [ @@ -1233,79 +1238,80 @@ perfetto_cc_library( ":include_perfetto_tracing_tracing", ], deps = [ - ":perfetto_ipc", - ":protos_perfetto_common_cpp", - ":protos_perfetto_common_semantic_type_cpp", - ":protos_perfetto_common_semantic_type_zero", - ":protos_perfetto_common_zero", - ":protos_perfetto_config_android_cpp", - ":protos_perfetto_config_android_zero", - ":protos_perfetto_config_cpp", - ":protos_perfetto_config_ftrace_cpp", - ":protos_perfetto_config_ftrace_zero", - ":protos_perfetto_config_gpu_cpp", - ":protos_perfetto_config_gpu_zero", - ":protos_perfetto_config_inode_file_cpp", - ":protos_perfetto_config_inode_file_zero", - ":protos_perfetto_config_interceptors_cpp", - ":protos_perfetto_config_interceptors_zero", - ":protos_perfetto_config_linux_cpp", - ":protos_perfetto_config_linux_zero", - ":protos_perfetto_config_power_cpp", - ":protos_perfetto_config_power_zero", - ":protos_perfetto_config_priority_boost_cpp", - ":protos_perfetto_config_priority_boost_zero", - ":protos_perfetto_config_process_stats_cpp", - ":protos_perfetto_config_process_stats_zero", - ":protos_perfetto_config_profiling_cpp", - ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_protovm_cpp", - ":protos_perfetto_config_protovm_zero", - ":protos_perfetto_config_qnx_cpp", - ":protos_perfetto_config_qnx_zero", - ":protos_perfetto_config_statsd_cpp", - ":protos_perfetto_config_statsd_zero", - ":protos_perfetto_config_sys_stats_cpp", - ":protos_perfetto_config_sys_stats_zero", - ":protos_perfetto_config_system_info_cpp", - ":protos_perfetto_config_system_info_zero", - ":protos_perfetto_config_track_event_cpp", - ":protos_perfetto_config_track_event_zero", - ":protos_perfetto_config_zero", - ":protos_perfetto_ipc_cpp", - ":protos_perfetto_ipc_ipc", - ":protos_perfetto_protovm_cpp", - ":protos_perfetto_protovm_zero", - ":protos_perfetto_trace_android_zero", - ":protos_perfetto_trace_chrome_zero", - ":protos_perfetto_trace_etw_zero", - ":protos_perfetto_trace_filesystem_zero", - ":protos_perfetto_trace_ftrace_zero", - ":protos_perfetto_trace_generic_kernel_zero", - ":protos_perfetto_trace_gpu_zero", - ":protos_perfetto_trace_interned_data_zero", - ":protos_perfetto_trace_linux_zero", - ":protos_perfetto_trace_minimal_zero", - ":protos_perfetto_trace_non_minimal_zero", - ":protos_perfetto_trace_perfetto_zero", - ":protos_perfetto_trace_power_zero", - ":protos_perfetto_trace_profiling_zero", - ":protos_perfetto_trace_ps_zero", - ":protos_perfetto_trace_statsd_cpp", - ":protos_perfetto_trace_statsd_zero", - ":protos_perfetto_trace_sys_stats_zero", - ":protos_perfetto_trace_system_info_zero", - ":protos_perfetto_trace_track_event_cpp", - ":protos_perfetto_trace_track_event_zero", - ":protos_perfetto_trace_translation_zero", - ":protos_third_party_statsd_config_zero", - ":protozero", - ":src_base_base", - ":src_base_clock_snapshots", - ":src_base_regex_regex", - ":src_base_version", - ":src_protovm_protovm", - ] + PERFETTO_CONFIG.deps.zlib, + ":perfetto_ipc", + ":protos_perfetto_common_cpp", + ":protos_perfetto_common_semantic_type_cpp", + ":protos_perfetto_common_semantic_type_zero", + ":protos_perfetto_common_zero", + ":protos_perfetto_config_android_cpp", + ":protos_perfetto_config_android_zero", + ":protos_perfetto_config_cpp", + ":protos_perfetto_config_ftrace_cpp", + ":protos_perfetto_config_ftrace_zero", + ":protos_perfetto_config_gpu_cpp", + ":protos_perfetto_config_gpu_zero", + ":protos_perfetto_config_inode_file_cpp", + ":protos_perfetto_config_inode_file_zero", + ":protos_perfetto_config_interceptors_cpp", + ":protos_perfetto_config_interceptors_zero", + ":protos_perfetto_config_linux_cpp", + ":protos_perfetto_config_linux_zero", + ":protos_perfetto_config_power_cpp", + ":protos_perfetto_config_power_zero", + ":protos_perfetto_config_priority_boost_cpp", + ":protos_perfetto_config_priority_boost_zero", + ":protos_perfetto_config_process_stats_cpp", + ":protos_perfetto_config_process_stats_zero", + ":protos_perfetto_config_profiling_cpp", + ":protos_perfetto_config_profiling_zero", + ":protos_perfetto_config_protovm_cpp", + ":protos_perfetto_config_protovm_zero", + ":protos_perfetto_config_qnx_cpp", + ":protos_perfetto_config_qnx_zero", + ":protos_perfetto_config_statsd_cpp", + ":protos_perfetto_config_statsd_zero", + ":protos_perfetto_config_sys_stats_cpp", + ":protos_perfetto_config_sys_stats_zero", + ":protos_perfetto_config_system_info_cpp", + ":protos_perfetto_config_system_info_zero", + ":protos_perfetto_config_track_event_cpp", + ":protos_perfetto_config_track_event_zero", + ":protos_perfetto_config_zero", + ":protos_perfetto_ipc_cpp", + ":protos_perfetto_ipc_ipc", + ":protos_perfetto_protovm_cpp", + ":protos_perfetto_protovm_zero", + ":protos_perfetto_trace_android_zero", + ":protos_perfetto_trace_chrome_zero", + ":protos_perfetto_trace_etw_zero", + ":protos_perfetto_trace_filesystem_zero", + ":protos_perfetto_trace_ftrace_zero", + ":protos_perfetto_trace_generic_kernel_zero", + ":protos_perfetto_trace_gpu_zero", + ":protos_perfetto_trace_interned_data_zero", + ":protos_perfetto_trace_linux_zero", + ":protos_perfetto_trace_minimal_zero", + ":protos_perfetto_trace_non_minimal_zero", + ":protos_perfetto_trace_perfetto_zero", + ":protos_perfetto_trace_power_zero", + ":protos_perfetto_trace_profiling_zero", + ":protos_perfetto_trace_ps_zero", + ":protos_perfetto_trace_statsd_cpp", + ":protos_perfetto_trace_statsd_zero", + ":protos_perfetto_trace_sys_stats_zero", + ":protos_perfetto_trace_system_info_zero", + ":protos_perfetto_trace_track_event_cpp", + ":protos_perfetto_trace_track_event_zero", + ":protos_perfetto_trace_translation_zero", + ":protos_third_party_statsd_config_zero", + ":protozero", + ":src_base_base", + ":src_base_clock_snapshots", + ":src_base_regex_regex", + ":src_base_version", + ":src_protovm_protovm", + ] + PERFETTO_CONFIG.deps.zlib + + PERFETTO_CONFIG.deps.zstd, linkstatic = True, ) @@ -6569,6 +6575,14 @@ perfetto_filegroup( ], ) +# GN target: //src/tracing/service:packet_compressor_common +perfetto_filegroup( + name = "src_tracing_service_packet_compressor_common", + srcs = [ + "src/tracing/service/packet_compressor_common.h", + ], +) + # GN target: //src/tracing/service:service perfetto_filegroup( name = "src_tracing_service_service", @@ -6607,6 +6621,15 @@ perfetto_filegroup( ], ) +# GN target: //src/tracing/service:zstd_compressor +perfetto_filegroup( + name = "src_tracing_service_zstd_compressor", + srcs = [ + "src/tracing/service/zstd_compressor.cc", + "src/tracing/service/zstd_compressor.h", + ], +) + # GN target: //src/tracing:client_api_without_backends perfetto_filegroup( name = "src_tracing_client_api_without_backends", @@ -10956,8 +10979,10 @@ perfetto_cc_library( ":src_tracing_ipc_default_socket", ":src_tracing_ipc_producer_producer", ":src_tracing_ipc_service_service", + ":src_tracing_service_packet_compressor_common", ":src_tracing_service_service", ":src_tracing_service_zlib_compressor", + ":src_tracing_service_zstd_compressor", ":src_tracing_system_backend", ], hdrs = [ @@ -10979,77 +11004,78 @@ perfetto_cc_library( "//visibility:public", ], deps = [ - ":perfetto_ipc", - ":protos_perfetto_common_cpp", - ":protos_perfetto_common_semantic_type_cpp", - ":protos_perfetto_common_semantic_type_zero", - ":protos_perfetto_common_zero", - ":protos_perfetto_config_android_cpp", - ":protos_perfetto_config_android_zero", - ":protos_perfetto_config_cpp", - ":protos_perfetto_config_ftrace_cpp", - ":protos_perfetto_config_ftrace_zero", - ":protos_perfetto_config_gpu_cpp", - ":protos_perfetto_config_gpu_zero", - ":protos_perfetto_config_inode_file_cpp", - ":protos_perfetto_config_inode_file_zero", - ":protos_perfetto_config_interceptors_cpp", - ":protos_perfetto_config_interceptors_zero", - ":protos_perfetto_config_linux_cpp", - ":protos_perfetto_config_linux_zero", - ":protos_perfetto_config_power_cpp", - ":protos_perfetto_config_power_zero", - ":protos_perfetto_config_priority_boost_cpp", - ":protos_perfetto_config_priority_boost_zero", - ":protos_perfetto_config_process_stats_cpp", - ":protos_perfetto_config_process_stats_zero", - ":protos_perfetto_config_profiling_cpp", - ":protos_perfetto_config_profiling_zero", - ":protos_perfetto_config_protovm_cpp", - ":protos_perfetto_config_protovm_zero", - ":protos_perfetto_config_qnx_cpp", - ":protos_perfetto_config_qnx_zero", - ":protos_perfetto_config_statsd_cpp", - ":protos_perfetto_config_statsd_zero", - ":protos_perfetto_config_sys_stats_cpp", - ":protos_perfetto_config_sys_stats_zero", - ":protos_perfetto_config_system_info_cpp", - ":protos_perfetto_config_system_info_zero", - ":protos_perfetto_config_track_event_cpp", - ":protos_perfetto_config_track_event_zero", - ":protos_perfetto_config_zero", - ":protos_perfetto_ipc_cpp", - ":protos_perfetto_ipc_ipc", - ":protos_perfetto_protovm_cpp", - ":protos_perfetto_protovm_zero", - ":protos_perfetto_trace_android_zero", - ":protos_perfetto_trace_chrome_zero", - ":protos_perfetto_trace_etw_zero", - ":protos_perfetto_trace_filesystem_zero", - ":protos_perfetto_trace_ftrace_zero", - ":protos_perfetto_trace_generic_kernel_zero", - ":protos_perfetto_trace_gpu_zero", - ":protos_perfetto_trace_interned_data_zero", - ":protos_perfetto_trace_linux_zero", - ":protos_perfetto_trace_minimal_zero", - ":protos_perfetto_trace_non_minimal_zero", - ":protos_perfetto_trace_perfetto_zero", - ":protos_perfetto_trace_power_zero", - ":protos_perfetto_trace_profiling_zero", - ":protos_perfetto_trace_ps_zero", - ":protos_perfetto_trace_statsd_zero", - ":protos_perfetto_trace_sys_stats_zero", - ":protos_perfetto_trace_system_info_zero", - ":protos_perfetto_trace_track_event_cpp", - ":protos_perfetto_trace_track_event_zero", - ":protos_perfetto_trace_translation_zero", - ":protozero", - ":src_base_base", - ":src_base_clock_snapshots", - ":src_base_regex_regex", - ":src_base_version", - ":src_protovm_protovm", - ] + PERFETTO_CONFIG.deps.zlib, + ":perfetto_ipc", + ":protos_perfetto_common_cpp", + ":protos_perfetto_common_semantic_type_cpp", + ":protos_perfetto_common_semantic_type_zero", + ":protos_perfetto_common_zero", + ":protos_perfetto_config_android_cpp", + ":protos_perfetto_config_android_zero", + ":protos_perfetto_config_cpp", + ":protos_perfetto_config_ftrace_cpp", + ":protos_perfetto_config_ftrace_zero", + ":protos_perfetto_config_gpu_cpp", + ":protos_perfetto_config_gpu_zero", + ":protos_perfetto_config_inode_file_cpp", + ":protos_perfetto_config_inode_file_zero", + ":protos_perfetto_config_interceptors_cpp", + ":protos_perfetto_config_interceptors_zero", + ":protos_perfetto_config_linux_cpp", + ":protos_perfetto_config_linux_zero", + ":protos_perfetto_config_power_cpp", + ":protos_perfetto_config_power_zero", + ":protos_perfetto_config_priority_boost_cpp", + ":protos_perfetto_config_priority_boost_zero", + ":protos_perfetto_config_process_stats_cpp", + ":protos_perfetto_config_process_stats_zero", + ":protos_perfetto_config_profiling_cpp", + ":protos_perfetto_config_profiling_zero", + ":protos_perfetto_config_protovm_cpp", + ":protos_perfetto_config_protovm_zero", + ":protos_perfetto_config_qnx_cpp", + ":protos_perfetto_config_qnx_zero", + ":protos_perfetto_config_statsd_cpp", + ":protos_perfetto_config_statsd_zero", + ":protos_perfetto_config_sys_stats_cpp", + ":protos_perfetto_config_sys_stats_zero", + ":protos_perfetto_config_system_info_cpp", + ":protos_perfetto_config_system_info_zero", + ":protos_perfetto_config_track_event_cpp", + ":protos_perfetto_config_track_event_zero", + ":protos_perfetto_config_zero", + ":protos_perfetto_ipc_cpp", + ":protos_perfetto_ipc_ipc", + ":protos_perfetto_protovm_cpp", + ":protos_perfetto_protovm_zero", + ":protos_perfetto_trace_android_zero", + ":protos_perfetto_trace_chrome_zero", + ":protos_perfetto_trace_etw_zero", + ":protos_perfetto_trace_filesystem_zero", + ":protos_perfetto_trace_ftrace_zero", + ":protos_perfetto_trace_generic_kernel_zero", + ":protos_perfetto_trace_gpu_zero", + ":protos_perfetto_trace_interned_data_zero", + ":protos_perfetto_trace_linux_zero", + ":protos_perfetto_trace_minimal_zero", + ":protos_perfetto_trace_non_minimal_zero", + ":protos_perfetto_trace_perfetto_zero", + ":protos_perfetto_trace_power_zero", + ":protos_perfetto_trace_profiling_zero", + ":protos_perfetto_trace_ps_zero", + ":protos_perfetto_trace_statsd_zero", + ":protos_perfetto_trace_sys_stats_zero", + ":protos_perfetto_trace_system_info_zero", + ":protos_perfetto_trace_track_event_cpp", + ":protos_perfetto_trace_track_event_zero", + ":protos_perfetto_trace_translation_zero", + ":protozero", + ":src_base_base", + ":src_base_clock_snapshots", + ":src_base_regex_regex", + ":src_base_version", + ":src_protovm_protovm", + ] + PERFETTO_CONFIG.deps.zlib + + PERFETTO_CONFIG.deps.zstd, linkstatic = True, ) diff --git a/CHANGELOG b/CHANGELOG index e0c9c341387..5aa3d400c69 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,7 @@ Unreleased: Tracing service and probes: + * Added Zstd as a trace compression option alongside deflate, selected via + the new TraceConfig.compression field (which supersedes compression_type). * The procfs scraper (process_stats data source) now writes an explicit Thread message for a process' main thread, instead of leaving it implied by the process entry. This fixes missing main thread names when a trace diff --git a/MODULE.bazel b/MODULE.bazel index 9e639ddfee6..10941b7fc8d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -26,13 +26,14 @@ git_override( remote = "https://github.com/abseil/abseil-cpp.git", ) -bazel_dep(name = "re2", version = "2024-07-02.bcr.1") bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "platforms", version = "0.0.10") bazel_dep(name = "protobuf", version = "31.1", repo_name = "com_google_protobuf") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1") bazel_dep(name = "rules_python", version = "1.0.0") bazel_dep(name = "rules_android", version = "0.6.6") bazel_dep(name = "rules_cc", version = "0.2.19") +bazel_dep(name = "zstd", version = "1.5.7.bcr.1") remote_android_extensions = use_extension( "@rules_android//bzlmod_extensions:android_extensions.bzl", diff --git a/bazel/deps.bzl b/bazel/deps.bzl index 17029e52506..e99f126a0f8 100644 --- a/bazel/deps.bzl +++ b/bazel/deps.bzl @@ -97,6 +97,16 @@ def perfetto_deps(): build_file = "//bazel:zlib.BUILD", ) + # Zstd, for the legacy WORKSPACE path (bzlmod uses the BCR module, see + # MODULE.bazel). Upstream ships no Bazel BUILD (facebook/zstd#3123). + _add_repo_if_not_existing( + new_git_repository, + name = "zstd", + remote = "https://chromium.googlesource.com/external/github.com/facebook/zstd.git", + commit = "ac66b19e6bd6b83238bf008eecc1298105298532", # refs/tags/upstream/v1.5.7 + build_file = "//bazel:zstd.BUILD", + ) + _add_repo_if_not_existing( http_archive, name = "perfetto_dep_llvm_demangle", diff --git a/bazel/standalone/perfetto_cfg.bzl b/bazel/standalone/perfetto_cfg.bzl index f055955eae0..d9c96892a8e 100644 --- a/bazel/standalone/perfetto_cfg.bzl +++ b/bazel/standalone/perfetto_cfg.bzl @@ -45,6 +45,7 @@ PERFETTO_CONFIG = struct( base_platform = ["//:perfetto_base_default_platform"], zlib = ["@perfetto_dep_zlib//:zlib"], + zstd = ["@zstd//:zstd"], re2 = ["@re2//:re2"], expat = ["@perfetto_dep_expat//:expat"], linenoise = ["@perfetto_dep_linenoise//:linenoise"], @@ -97,6 +98,7 @@ PERFETTO_CONFIG = struct( # initialized with the Perfetto build files (i.e. via perfetto_deps()). deps_copts = struct( zlib = [], + zstd = [], expat = [], linenoise = [], sqlite = [], diff --git a/bazel/zstd.BUILD b/bazel/zstd.BUILD new file mode 100644 index 00000000000..55ed7044f22 --- /dev/null +++ b/bazel/zstd.BUILD @@ -0,0 +1,52 @@ +# Copyright (C) 2026 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Overlay for the legacy WORKSPACE path only; bzlmod uses the BCR module (see +# MODULE.bazel). Needed because upstream zstd ships no Bazel BUILD +# (facebook/zstd#3123), so new_git_repository has no //:zstd target. + +load("@perfetto_cfg//:perfetto_cfg.bzl", "PERFETTO_CONFIG") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +cc_library( + name = "zstd", + srcs = glob([ + "lib/common/*.c", + "lib/common/*.h", + "lib/compress/*.c", + "lib/compress/*.h", + "lib/decompress/*.c", + "lib/decompress/*.h", + "lib/decompress/*.S", + "lib/dictBuilder/*.c", + "lib/dictBuilder/*.h", + ]), + hdrs = [ + "lib/zdict.h", + "lib/zstd.h", + "lib/zstd_errors.h", + ], + copts = [ + "-Wno-unused-function", + ] + PERFETTO_CONFIG.deps_copts.zstd, + # Disable the inline assembly to match the standalone GN build (which sets + # ZSTD_DISABLE_ASM on x64) and keep the build portable across toolchains. + defines = [ + "ZSTD_DISABLE_ASM", + ], + includes = [ + "lib", + ], + visibility = ["//visibility:public"], +) diff --git a/buildtools/BUILD.gn b/buildtools/BUILD.gn index 74631e2cb52..222112bfaf4 100644 --- a/buildtools/BUILD.gn +++ b/buildtools/BUILD.gn @@ -1915,7 +1915,7 @@ config("zstd_config") { visibility = _buildtools_visibility cflags = [ perfetto_isystem_cflag, - rebase_path("zstd", root_build_dir), + rebase_path("zstd/lib", root_build_dir), ] if (current_cpu == "x64") { defines = [ "ZSTD_DISABLE_ASM" ] diff --git a/gn/BUILD.gn b/gn/BUILD.gn index 6f2ad9fdf55..60ce0cbf611 100644 --- a/gn/BUILD.gn +++ b/gn/BUILD.gn @@ -101,6 +101,7 @@ action("gen_buildflags") { "PERFETTO_TP_INSTRUMENTS=$enable_perfetto_trace_processor_mac_instruments", "PERFETTO_LOCAL_SYMBOLIZER=$perfetto_local_symbolizer", "PERFETTO_ZLIB=$enable_perfetto_zlib", + "PERFETTO_ZSTD=$enable_perfetto_zstd", "PERFETTO_TRACED_PERF=$enable_perfetto_traced_perf", "PERFETTO_HEAPPROFD=$enable_perfetto_heapprofd", "PERFETTO_STDERR_CRASH_DUMP=$enable_perfetto_stderr_crash_dump", @@ -460,6 +461,19 @@ if (enable_perfetto_zlib) { } } +# Zstd is used both by trace_processor and by the tracing service. +if (enable_perfetto_zstd) { + group("zstd") { + if (perfetto_root_path == "//") { + public_configs = [ "//buildtools:zstd_config" ] + public_deps = [ "//buildtools:zstd" ] + } else { + public_configs = [ "//third_party/zstd:zstd_config" ] + public_deps = [ "//third_party/zstd" ] + } + } +} + if (enable_perfetto_llvm_demangle) { group("llvm_demangle") { public_deps = [ "//buildtools:llvm_demangle" ] diff --git a/gn/perfetto.gni b/gn/perfetto.gni index 7def73a91b9..11f4dd67542 100644 --- a/gn/perfetto.gni +++ b/gn/perfetto.gni @@ -389,6 +389,12 @@ declare_args() { enable_perfetto_zlib = enable_perfetto_trace_processor || enable_perfetto_platform_services + # Enables Zstd support. This is used to compress traces (by the tracing + # service) and to decompress traces (by trace_processor), as an alternative + # to Zlib/deflate. + enable_perfetto_zstd = + enable_perfetto_trace_processor || enable_perfetto_platform_services + # Enables PCRE2 support. Individual modules that should not pull libpcre2 # in (e.g. libperfetto_client_experimental) can opt out by setting # -DPERFETTO_REGEX_FORCE_STD on their compile, which makes base::Regex diff --git a/include/perfetto/base/build_config.h b/include/perfetto/base/build_config.h index 1ed224ad9ab..0c519a313b8 100644 --- a/include/perfetto/base/build_config.h +++ b/include/perfetto/base/build_config.h @@ -252,16 +252,29 @@ #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB() 0 #endif +#undef PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD +#if defined(PERFETTO_SDK_ENABLE_ZSTD) && (PERFETTO_SDK_ENABLE_ZSTD == 1) +#define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD() 1 +#else +#define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD() 0 +#endif + #endif // PERFETTO_BUILDFLAG(PERFETTO_AMALGAMATED_SDK) -// Unlike PERFETTO_SDK_ENABLE_ZLIB above (amalgamated SDK opt-in), this is a -// file-scope opt-out that applies to every build. Define it to compile out the -// optional zlib compressor. Static libs like libperfetto_client_experimental -// get linked into many binaries, so keeping the compressor out means those -// binaries don't have to link libz. See tools/gen_android_bp. +// Unlike the PERFETTO_SDK_ENABLE_* opt-ins above (amalgamated SDK), these are +// file-scope opt-outs that apply to every build. Define them to compile out the +// optional zlib/zstd compressors. Static libs like +// libperfetto_client_experimental get linked into many binaries, so keeping a +// compressor out means those binaries don't have to link libz/libzstd. See +// tools/gen_android_bp. #if defined(PERFETTO_FORCE_DISABLE_ZLIB) #undef PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB() 0 #endif +#if defined(PERFETTO_FORCE_DISABLE_ZSTD) +#undef PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD +#define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD() 0 +#endif + #endif // INCLUDE_PERFETTO_BASE_BUILD_CONFIG_H_ diff --git a/include/perfetto/base/build_configs/android_tree/perfetto_build_flags.h b/include/perfetto/base/build_configs/android_tree/perfetto_build_flags.h index 8e556ee112c..d2f6745a225 100644 --- a/include/perfetto/base/build_configs/android_tree/perfetto_build_flags.h +++ b/include/perfetto/base/build_configs/android_tree/perfetto_build_flags.h @@ -44,6 +44,7 @@ #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_INSTRUMENTS() (0) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_LOCAL_SYMBOLIZER() (0) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB() (1) +#define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD() (1) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TRACED_PERF() (1) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_HEAPPROFD() (1) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_STDERR_CRASH_DUMP() (0) @@ -84,6 +85,7 @@ static const struct PerfettoBuildFlag kPerfettoBuildFlags[] = { {"PERFETTO_TP_HTTPD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_HTTPD()}, {"PERFETTO_TP_INSTRUMENTS", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_INSTRUMENTS()}, {"PERFETTO_ZLIB", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB()}, + {"PERFETTO_ZSTD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD()}, {"PERFETTO_TRACED_PERF", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TRACED_PERF()}, {"PERFETTO_HEAPPROFD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_HEAPPROFD()}, {"PERFETTO_STDERR_CRASH_DUMP", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_STDERR_CRASH_DUMP()}, diff --git a/include/perfetto/base/build_configs/bazel/perfetto_build_flags.h b/include/perfetto/base/build_configs/bazel/perfetto_build_flags.h index 4dfa944f314..a5519e4de61 100644 --- a/include/perfetto/base/build_configs/bazel/perfetto_build_flags.h +++ b/include/perfetto/base/build_configs/bazel/perfetto_build_flags.h @@ -44,6 +44,7 @@ #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_INSTRUMENTS() (0) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_LOCAL_SYMBOLIZER() (PERFETTO_BUILDFLAG_DEFINE_PERFETTO_OS_LINUX() || PERFETTO_BUILDFLAG_DEFINE_PERFETTO_OS_FREEBSD() || PERFETTO_BUILDFLAG_DEFINE_PERFETTO_OS_MAC() ||PERFETTO_BUILDFLAG_DEFINE_PERFETTO_OS_WIN()) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB() (1) +#define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD() (1) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TRACED_PERF() (0) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_HEAPPROFD() (0) #define PERFETTO_BUILDFLAG_DEFINE_PERFETTO_STDERR_CRASH_DUMP() (0) @@ -84,6 +85,7 @@ static const struct PerfettoBuildFlag kPerfettoBuildFlags[] = { {"PERFETTO_TP_HTTPD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_HTTPD()}, {"PERFETTO_TP_INSTRUMENTS", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TP_INSTRUMENTS()}, {"PERFETTO_ZLIB", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZLIB()}, + {"PERFETTO_ZSTD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_ZSTD()}, {"PERFETTO_TRACED_PERF", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_TRACED_PERF()}, {"PERFETTO_HEAPPROFD", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_HEAPPROFD()}, {"PERFETTO_STDERR_CRASH_DUMP", PERFETTO_BUILDFLAG_DEFINE_PERFETTO_STDERR_CRASH_DUMP()}, diff --git a/include/perfetto/ext/tracing/core/tracing_service.h b/include/perfetto/ext/tracing/core/tracing_service.h index aa10c5798e9..c0eb3762d34 100644 --- a/include/perfetto/ext/tracing/core/tracing_service.h +++ b/include/perfetto/ext/tracing/core/tracing_service.h @@ -307,12 +307,6 @@ class PERFETTO_EXPORT_COMPONENT ConsumerEndpoint { }; // class ConsumerEndpoint. struct PERFETTO_EXPORT_COMPONENT TracingServiceInitOpts { - // Function used by tracing service to compress packets. Takes a pointer to - // a vector of TracePackets and replaces the packets in the vector with - // compressed ones. - using CompressorFn = void (*)(std::vector*); - CompressorFn compressor_fn = nullptr; - // An (optional) list of proto extension descriptors to dump into each trace // recorded. This is to support injecting protos that are known by the // embedder (e.g. the Android vendor image) but not by the upstream perfetto. @@ -385,8 +379,7 @@ class PERFETTO_EXPORT_COMPONENT TracingService { kDisabled }; - // Implemented in src/core/tracing_service_impl.cc . CompressorFn can be - // nullptr, in which case TracingService will not support compression. + // Implemented in src/core/tracing_service_impl.cc . static std::unique_ptr CreateInstance( std::unique_ptr, base::TaskRunner*, diff --git a/protos/perfetto/config/perfetto_config.proto b/protos/perfetto/config/perfetto_config.proto index 2fd8b7a6ce2..a0714ed9406 100644 --- a/protos/perfetto/config/perfetto_config.proto +++ b/protos/perfetto/config/perfetto_config.proto @@ -4968,7 +4968,7 @@ message DataSourceConfig { // It contains the general config for the logging buffer(s) and the configs for // all the data source being enabled. // -// Next id: 47. +// Next id: 48. message TraceConfig { message BufferConfig { optional uint32 size_kb = 1; @@ -5427,12 +5427,39 @@ message TraceConfig { // with this key. optional string unique_session_name = 22; - // Compress trace with the given method. Best effort. + // Deprecated: use |compression| (CompressionConfig) instead. Legacy selector + // that only ever selects deflate. Kept for backwards compatibility with + // Perfetto <= v57, whose services predate |compression|. Best effort. enum CompressionType { COMPRESSION_TYPE_UNSPECIFIED = 0; COMPRESSION_TYPE_DEFLATE = 1; } - optional CompressionType compression_type = 24; + optional CompressionType compression_type = 24 [deprecated = true]; + + // Selects and configures the compression codec, superseding + // |compression_type|. The codec is chosen by which sub-message is set; when + // several are set the service picks the highest-numbered one it supports and + // ignores codecs it doesn't know, so a single config can target old and new + // services. Falls back to |compression_type| (deflate-only), if given. + // + // Introduced in: perfetto v58. + // Supported on: Android 26Q3+. + message CompressionConfig { + // Deflate (zlib), equivalent to the legacy `COMPRESSION_TYPE_DEFLATE`. + message Deflate {} + message Zstd { + // Compression level, 1 (fast) to 22 (smallest); 0/unset uses zstd's + // default (3). Levels above the max are clamped; negatives are valid + // faster, lower-ratio modes. + optional int32 level = 1; + } + // If a config sets more than one of these, the service picks the one with + // the highest field number (see the rule above): e.g. with deflate (1) and + // zstd (2) both set, zstd wins. + optional Deflate deflate = 1; + optional Zstd zstd = 2; + } + optional CompressionConfig compression = 47; // DEPRECATED, was compress_from_cli. reserved 37; diff --git a/protos/perfetto/config/trace_config.proto b/protos/perfetto/config/trace_config.proto index 37a9c635a7a..fe965f9154c 100644 --- a/protos/perfetto/config/trace_config.proto +++ b/protos/perfetto/config/trace_config.proto @@ -28,7 +28,7 @@ import "protos/perfetto/config/priority_boost/priority_boost_config.proto"; // It contains the general config for the logging buffer(s) and the configs for // all the data source being enabled. // -// Next id: 47. +// Next id: 48. message TraceConfig { message BufferConfig { optional uint32 size_kb = 1; @@ -487,12 +487,39 @@ message TraceConfig { // with this key. optional string unique_session_name = 22; - // Compress trace with the given method. Best effort. + // Deprecated: use |compression| (CompressionConfig) instead. Legacy selector + // that only ever selects deflate. Kept for backwards compatibility with + // Perfetto <= v57, whose services predate |compression|. Best effort. enum CompressionType { COMPRESSION_TYPE_UNSPECIFIED = 0; COMPRESSION_TYPE_DEFLATE = 1; } - optional CompressionType compression_type = 24; + optional CompressionType compression_type = 24 [deprecated = true]; + + // Selects and configures the compression codec, superseding + // |compression_type|. The codec is chosen by which sub-message is set; when + // several are set the service picks the highest-numbered one it supports and + // ignores codecs it doesn't know, so a single config can target old and new + // services. Falls back to |compression_type| (deflate-only), if given. + // + // Introduced in: perfetto v58. + // Supported on: Android 26Q3+. + message CompressionConfig { + // Deflate (zlib), equivalent to the legacy `COMPRESSION_TYPE_DEFLATE`. + message Deflate {} + message Zstd { + // Compression level, 1 (fast) to 22 (smallest); 0/unset uses zstd's + // default (3). Levels above the max are clamped; negatives are valid + // faster, lower-ratio modes. + optional int32 level = 1; + } + // If a config sets more than one of these, the service picks the one with + // the highest field number (see the rule above): e.g. with deflate (1) and + // zstd (2) both set, zstd wins. + optional Deflate deflate = 1; + optional Zstd zstd = 2; + } + optional CompressionConfig compression = 47; // DEPRECATED, was compress_from_cli. reserved 37; diff --git a/protos/perfetto/trace/perfetto_trace.proto b/protos/perfetto/trace/perfetto_trace.proto index 5da7a11b9ea..4fa57f0bd38 100644 --- a/protos/perfetto/trace/perfetto_trace.proto +++ b/protos/perfetto/trace/perfetto_trace.proto @@ -4968,7 +4968,7 @@ message DataSourceConfig { // It contains the general config for the logging buffer(s) and the configs for // all the data source being enabled. // -// Next id: 47. +// Next id: 48. message TraceConfig { message BufferConfig { optional uint32 size_kb = 1; @@ -5427,12 +5427,39 @@ message TraceConfig { // with this key. optional string unique_session_name = 22; - // Compress trace with the given method. Best effort. + // Deprecated: use |compression| (CompressionConfig) instead. Legacy selector + // that only ever selects deflate. Kept for backwards compatibility with + // Perfetto <= v57, whose services predate |compression|. Best effort. enum CompressionType { COMPRESSION_TYPE_UNSPECIFIED = 0; COMPRESSION_TYPE_DEFLATE = 1; } - optional CompressionType compression_type = 24; + optional CompressionType compression_type = 24 [deprecated = true]; + + // Selects and configures the compression codec, superseding + // |compression_type|. The codec is chosen by which sub-message is set; when + // several are set the service picks the highest-numbered one it supports and + // ignores codecs it doesn't know, so a single config can target old and new + // services. Falls back to |compression_type| (deflate-only), if given. + // + // Introduced in: perfetto v58. + // Supported on: Android 26Q3+. + message CompressionConfig { + // Deflate (zlib), equivalent to the legacy `COMPRESSION_TYPE_DEFLATE`. + message Deflate {} + message Zstd { + // Compression level, 1 (fast) to 22 (smallest); 0/unset uses zstd's + // default (3). Levels above the max are clamped; negatives are valid + // faster, lower-ratio modes. + optional int32 level = 1; + } + // If a config sets more than one of these, the service picks the one with + // the highest field number (see the rule above): e.g. with deflate (1) and + // zstd (2) both set, zstd wins. + optional Deflate deflate = 1; + optional Zstd zstd = 2; + } + optional CompressionConfig compression = 47; // DEPRECATED, was compress_from_cli. reserved 37; @@ -17660,7 +17687,7 @@ message UiState { // See the [Buffers and Dataflow](/docs/concepts/buffers.md) doc for details. // // Next reserved id: 14 (up to 15). -// Next id: 133. +// Next id: 134. message TracePacket { // Encapsulates the state and configuration of the ProtoVM instances running // when the trace was snapshotted. This allows TP to re-instantiate the VMs @@ -17799,6 +17826,9 @@ message TracePacket { // sizes) should be less than 512KB. bytes compressed_packets = 50; + // The zstd counterpart of |compressed_packets|. + bytes zstd_compressed_packets = 133; + // Data sources can extend the trace proto with custom extension protos (see // docs/instrumentation/extensions.md). When they do that, the descriptor of // their extension proto descriptor is serialized in this packet. This diff --git a/protos/perfetto/trace/trace_packet.proto b/protos/perfetto/trace/trace_packet.proto index 09d1b534c60..ccc3063c01e 100644 --- a/protos/perfetto/trace/trace_packet.proto +++ b/protos/perfetto/trace/trace_packet.proto @@ -109,7 +109,7 @@ package perfetto.protos; // See the [Buffers and Dataflow](/docs/concepts/buffers.md) doc for details. // // Next reserved id: 14 (up to 15). -// Next id: 133. +// Next id: 134. message TracePacket { // Encapsulates the state and configuration of the ProtoVM instances running // when the trace was snapshotted. This allows TP to re-instantiate the VMs @@ -248,6 +248,9 @@ message TracePacket { // sizes) should be less than 512KB. bytes compressed_packets = 50; + // The zstd counterpart of |compressed_packets|. + bytes zstd_compressed_packets = 133; + // Data sources can extend the trace proto with custom extension protos (see // docs/instrumentation/extensions.md). When they do that, the descriptor of // their extension proto descriptor is serialized in this packet. This diff --git a/src/traced/service/BUILD.gn b/src/traced/service/BUILD.gn index cbfca417476..2b88ecd0734 100644 --- a/src/traced/service/BUILD.gn +++ b/src/traced/service/BUILD.gn @@ -55,9 +55,6 @@ source_set("service") { "../../tracing/ipc/service", "../../tracing/service:service", ] - if (enable_perfetto_zlib) { - deps += [ "../../tracing/service:zlib_compressor" ] - } sources = [ "service.cc" ] diff --git a/src/traced/service/service.cc b/src/traced/service/service.cc index 25776c6432c..2e1cdaf2030 100644 --- a/src/traced/service/service.cc +++ b/src/traced/service/service.cc @@ -39,10 +39,6 @@ #include #endif -#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) -#include "src/tracing/service/zlib_compressor.h" -#endif - namespace perfetto { namespace { void PrintUsage(const char* prog_name) { @@ -155,9 +151,6 @@ int PERFETTO_EXPORT_ENTRYPOINT ServiceMain(int argc, char** argv) { base::MaybeLockFreeTaskRunner task_runner; std::unique_ptr svc; TracingService::InitOpts init_opts = {}; -#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) - init_opts.compressor_fn = &ZlibCompressFn; -#endif #if PERFETTO_BUILDFLAG(PERFETTO_OS_ANDROID) // See /rfcs/0017-out-of-tree-protos.md . std::vector extension_descriptor_mmaps; diff --git a/src/tracing/BUILD.gn b/src/tracing/BUILD.gn index 730ccc9ebc1..acd6bbf95bd 100644 --- a/src/tracing/BUILD.gn +++ b/src/tracing/BUILD.gn @@ -205,9 +205,6 @@ source_set("in_process_backend") { "core", "service", ] - if (enable_perfetto_zlib) { - deps += [ "service:zlib_compressor" ] - } sources = [ "internal/in_process_tracing_backend.cc" ] } diff --git a/src/tracing/internal/in_process_tracing_backend.cc b/src/tracing/internal/in_process_tracing_backend.cc index a88be1e7e71..7405833e135 100644 --- a/src/tracing/internal/in_process_tracing_backend.cc +++ b/src/tracing/internal/in_process_tracing_backend.cc @@ -25,10 +25,6 @@ #include "src/tracing/core/in_process_shared_memory.h" -#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) -#include "src/tracing/service/zlib_compressor.h" -#endif - // TODO(primiano): When the in-process backend is used, we should never end up // in a situation where the thread where the TracingService and Producer live // writes a packet and hence can get into the GetNewChunk() stall. @@ -74,13 +70,6 @@ TracingService* InProcessTracingBackend::GetOrCreateService( std::unique_ptr shm( new InProcessSharedMemory::Factory()); TracingService::InitOpts init_opts = {}; -#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) - // Wire the zlib compressor so TraceConfig.compression_type = - // COMPRESSION_TYPE_DEFLATE takes effect on the in-process backend's - // service, mirroring what src/traced/service/service.cc does for the - // system backend. - init_opts.compressor_fn = &ZlibCompressFn; -#endif service_ = TracingService::CreateInstance(std::move(shm), task_runner, init_opts); service_->SetSMBScrapingEnabled(true); diff --git a/src/tracing/service/BUILD.gn b/src/tracing/service/BUILD.gn index d8d090e7e88..13ebe344e9c 100644 --- a/src/tracing/service/BUILD.gn +++ b/src/tracing/service/BUILD.gn @@ -72,11 +72,34 @@ source_set("service") { "../../android_internal:lazy_library_loader", ] } + + # The service compresses ReadBuffers output using the codecs below, which + # pull in the optional compression libraries. Depend on them only when + # enabled, so libz/libzstd stay out of builds that don't compress. + if (enable_perfetto_zlib) { + deps += [ ":zlib_compressor" ] + } + if (enable_perfetto_zstd) { + deps += [ ":zstd_compressor" ] + } +} + +# Shared by the zlib and zstd packet compressors below; header-only. +if (enable_perfetto_zlib || enable_perfetto_zstd) { + source_set("packet_compressor_common") { + public_deps = [ + "../../../gn:default_deps", + "../../../include/perfetto/ext/tracing/core", + "../../protozero", + ] + sources = [ "packet_compressor_common.h" ] + } } if (enable_perfetto_zlib) { source_set("zlib_compressor") { deps = [ + ":packet_compressor_common", "../../../gn:default_deps", "../../../gn:zlib", "../../../include/perfetto/tracing", @@ -89,6 +112,22 @@ if (enable_perfetto_zlib) { } } +if (enable_perfetto_zstd) { + source_set("zstd_compressor") { + deps = [ + ":packet_compressor_common", + "../../../gn:default_deps", + "../../../gn:zstd", + "../../../include/perfetto/tracing", + "../core", + ] + sources = [ + "zstd_compressor.cc", + "zstd_compressor.h", + ] + } +} + perfetto_unittest_source_set("unittests") { testonly = true deps = [ @@ -116,6 +155,13 @@ perfetto_unittest_source_set("unittests") { ] } + if (enable_perfetto_zstd) { + deps += [ + ":zstd_compressor", + "../../../gn:zstd", + ] + } + sources = [ "histogram_unittest.cc", "packet_stream_validator_unittest.cc", @@ -123,8 +169,9 @@ perfetto_unittest_source_set("unittests") { "trace_buffer_v2_unittest.cc", ] - if (enable_perfetto_zlib) { - sources += [ "zlib_compressor_unittest.cc" ] + if (enable_perfetto_zlib || enable_perfetto_zstd) { + deps += [ ":packet_compressor_common" ] + sources += [ "compressor_unittest.cc" ] } # These tests rely on test_task_runner.h which @@ -143,14 +190,29 @@ if (enable_perfetto_benchmarks) { "../../../gn:default_deps", "../../../protos/perfetto/trace:zero", "../../../protos/perfetto/trace/ftrace:zero", + "../../../protos/perfetto/trace/track_event:zero", + "../../../protos/third_party/android/art:art_heap_graph_zero", "../../protozero", "../core", "../test:test_support", ] sources = [ + "compression_benchmark.cc", "packet_stream_validator_benchmark.cc", "trace_buffer_benchmark.cc", ] + if (enable_perfetto_zlib) { + deps += [ + ":zlib_compressor", + "../../../gn:zlib", + ] + } + if (enable_perfetto_zstd) { + deps += [ + ":zstd_compressor", + "../../../gn:zstd", + ] + } } } diff --git a/src/tracing/service/compression_benchmark.cc b/src/tracing/service/compression_benchmark.cc new file mode 100644 index 00000000000..e266934c1d8 --- /dev/null +++ b/src/tracing/service/compression_benchmark.cc @@ -0,0 +1,263 @@ +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include +#include + +#include "perfetto/base/build_config.h" +#include "perfetto/ext/tracing/core/slice.h" +#include "perfetto/ext/tracing/core/trace_packet.h" +#include "perfetto/protozero/packed_repeated_fields.h" +#include "perfetto/protozero/scattered_heap_buffer.h" + +#include "protos/perfetto/trace/ftrace/ftrace_event.pbzero.h" +#include "protos/perfetto/trace/ftrace/ftrace_event_bundle.pbzero.h" +#include "protos/perfetto/trace/ftrace/sched.pbzero.h" +#include "protos/perfetto/trace/test_event.pbzero.h" +#include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "protos/perfetto/trace/track_event/track_event.pbzero.h" +#include "protos/third_party/android/art/heap_graph.pbzero.h" + +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +#include "src/tracing/service/zlib_compressor.h" +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +#include "src/tracing/service/zstd_compressor.h" +#endif + +namespace perfetto { +namespace { + +// Workload policies: Make(i) returns one serialized TracePacket, varied by `i` +// so a batch isn't trivially deduped away (zstd dedupes across its window) and +// the measured ratios stay representative. + +// Structured, highly repetitive: a sched_switch ftrace bundle (~64 events), the +// bulk of a system trace. +struct FtraceSched { + static std::vector Make(uint32_t i) { + protozero::HeapBuffered packet; + auto* bundle = packet->set_ftrace_events(); + bundle->set_cpu(i % 8); + uint64_t ts = static_cast(i) * 1000000; + for (uint32_t e = 0; e < 64; e++) { + auto* evt = bundle->add_event(); + evt->set_pid(1000 + ((i + e) % 256)); + evt->set_timestamp(ts + e * 1000); + auto* ss = evt->set_sched_switch(); + ss->set_prev_comm("thread-" + std::to_string((i + e) % 50)); + ss->set_prev_pid(1000 + ((i + e) % 256)); + ss->set_prev_state(e % 4); + ss->set_next_comm("thread-" + std::to_string((i + e + 1) % 50)); + ss->set_next_pid(1000 + ((i + e + 1) % 256)); + } + return packet.SerializeAsArray(); + } +}; + +// Java heap dump: a HeapGraph with types and cross-referencing objects (mostly +// packed varint ids). heap_graph is an out-of-tree field (number 56), not +// declared on TracePacket, so append it by field number. +struct JavaHeapDump { + static std::vector Make(uint32_t i) { + namespace art = com::android::art::tracing::pbzero; + protozero::HeapBuffered hg; + hg->set_pid(static_cast(1000 + i)); + for (uint32_t t = 0; t < 8; t++) { + auto* type = hg->add_types(); + type->set_id(t + 1); + type->set_class_name("com.example.Class" + std::to_string((i + t) % 64)); + type->set_object_size(16 * (t + 1)); + } + for (uint32_t o = 0; o < 64; o++) { + uint64_t id = (static_cast(i) << 16) + o + 1; + auto* obj = hg->add_objects(); + obj->set_id(id); + obj->set_type_id((o % 8) + 1); + obj->set_self_size(16 + (o % 4) * 8); + protozero::PackedVarInt refs; + refs.Append(id + 1); + refs.Append(id + 7); + obj->set_reference_object_id(refs); + } + std::vector heap_graph = hg.SerializeAsArray(); + + protozero::HeapBuffered packet; + // 56 = HeapGraph heap_graph (out-of-tree field, see heap_graph.proto). + packet->AppendBytes(56, heap_graph.data(), heap_graph.size()); + return packet.SerializeAsArray(); + } +}; + +// String-heavy: a TrackEvent slice with a name and categories. +struct TrackEvent { + static std::vector Make(uint32_t i) { + protozero::HeapBuffered packet; + packet->set_timestamp(static_cast(i) * 1000); + auto* te = packet->set_track_event(); + te->set_track_uuid(i % 32); + te->add_categories("category-" + std::to_string(i % 16)); + te->set_name("SomeEventName::DoWork_" + std::to_string(i % 128)); + return packet.SerializeAsArray(); + } +}; + +// Worst case: incompressible random bytes (e.g. an already-compressed payload). +struct Incompressible { + static std::vector Make(uint32_t i) { + std::minstd_rand rng(i + 1); + std::string s(1024, '\0'); + for (char& c : s) + c = static_cast(rng()); + protozero::HeapBuffered packet; + packet->set_for_testing()->set_str(s); + return packet.SerializeAsArray(); + } +}; + +// Backend policies: each runs one compile-time-selected codec at the given +// level. +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +struct Zlib { + static void Compress(std::vector* packets, int /*level*/) { + ZlibCompressFn(packets); + } +}; +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +struct Zstd { + static void Compress(std::vector* packets, int level) { + ZstdCompressFn(packets, level); + } +}; +#endif + +TracePacket MakePacket(const std::vector& buf) { + Slice slice = Slice::Allocate(buf.size()); + memcpy(slice.own_data(), buf.data(), buf.size()); + TracePacket packet; + packet.AddSlice(std::move(slice)); + return packet; +} + +// The compressor consumes (clears) its input, so each iteration needs a fresh +// copy of the packets. +std::vector CopyPackets(const std::vector& src) { + std::vector out; + out.reserve(src.size()); + for (const TracePacket& packet : src) { + TracePacket copy; + for (const Slice& slice : packet.slices()) { + Slice new_slice = Slice::Allocate(slice.size); + memcpy(new_slice.own_data(), slice.start, slice.size); + copy.AddSlice(std::move(new_slice)); + } + out.push_back(std::move(copy)); + } + return out; +} + +bool IsBenchmarkFunctionalOnly() { + return getenv("BENCHMARK_FUNCTIONAL_TEST_ONLY") != nullptr; +} + +// Sweep the block size (packets per compress call) at the codec's default +// level. +static void BlockSizeArgs(benchmark::internal::Benchmark* b) { + b->Unit(benchmark::kMicrosecond); + b->ArgNames({"packets", "level"}); + if (IsBenchmarkFunctionalOnly()) { + b->Args({1, 0})->Iterations(1); + return; + } + b->RangeMultiplier(8)->Ranges({{1, 4096}, {0, 0}}); +} + +// Sweep the zstd level (fastest..max) at a fixed, representative block size. +static void ZstdLevelArgs(benchmark::internal::Benchmark* b) { + b->Unit(benchmark::kMicrosecond); + b->ArgNames({"packets", "level"}); + if (IsBenchmarkFunctionalOnly()) { + b->Args({1, 0})->Iterations(1); + return; + } + for (int level : {1, 3, 9, 19, 22}) + b->Args({512, level}); +} + +// Compresses a batch of `state.range(0)` packets per iteration. The reported +// time is the per-batch compression latency; SetBytesProcessed gives throughput +// and the counters report the compression ratio. +template +static void BM_Compress(benchmark::State& state) { + const size_t num_packets = static_cast(state.range(0)); + const int level = static_cast(state.range(1)); + std::vector packets; + size_t total_in = 0; + for (size_t i = 0; i < num_packets; i++) { + packets.push_back(MakePacket(Workload::Make(static_cast(i)))); + total_in += packets.back().size(); + } + + size_t total_out = 0; + for (auto _ : state) { + state.PauseTiming(); + std::vector input = CopyPackets(packets); + state.ResumeTiming(); + + Backend::Compress(&input, level); + + total_out = 0; + for (const TracePacket& packet : input) + total_out += packet.size(); + benchmark::DoNotOptimize(total_out); + } + + state.counters["packets"] = static_cast(num_packets); + state.counters["in_bytes"] = static_cast(total_in); + state.counters["out_bytes"] = static_cast(total_out); + state.counters["ratio"] = + total_out ? static_cast(total_in) / static_cast(total_out) + : 0; + state.SetBytesProcessed(static_cast(state.iterations()) * + static_cast(total_in)); +} + +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +BENCHMARK_TEMPLATE(BM_Compress, Zlib, FtraceSched)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zlib, JavaHeapDump)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zlib, TrackEvent)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zlib, Incompressible)->Apply(BlockSizeArgs); +#endif + +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +BENCHMARK_TEMPLATE(BM_Compress, Zstd, FtraceSched)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, JavaHeapDump)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, TrackEvent)->Apply(BlockSizeArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, Incompressible)->Apply(BlockSizeArgs); + +// Level sweep (zstd only): how ratio and latency trade off across levels. +BENCHMARK_TEMPLATE(BM_Compress, Zstd, FtraceSched)->Apply(ZstdLevelArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, JavaHeapDump)->Apply(ZstdLevelArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, TrackEvent)->Apply(ZstdLevelArgs); +BENCHMARK_TEMPLATE(BM_Compress, Zstd, Incompressible)->Apply(ZstdLevelArgs); +#endif + +} // namespace +} // namespace perfetto diff --git a/src/tracing/service/compressor_unittest.cc b/src/tracing/service/compressor_unittest.cc new file mode 100644 index 00000000000..8c0ec399b6f --- /dev/null +++ b/src/tracing/service/compressor_unittest.cc @@ -0,0 +1,373 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "perfetto/base/build_config.h" + +#include +#include +#include +#include +#include + +#include "protos/perfetto/trace/test_event.gen.h" +#include "protos/perfetto/trace/trace.gen.h" +#include "protos/perfetto/trace/trace_packet.gen.h" +#include "src/tracing/service/packet_compressor_common.h" +#include "src/tracing/service/tracing_service_impl.h" +#include "test/gtest_and_gmock.h" + +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +#include +#include "src/tracing/service/zlib_compressor.h" +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +#include +#include "src/tracing/service/zstd_compressor.h" +#endif + +namespace perfetto { +namespace { + +using ::testing::Each; +using ::testing::ElementsAre; +using ::testing::Field; +using ::testing::IsEmpty; +using ::testing::Le; +using ::testing::Not; +using ::testing::Property; +using ::testing::SizeIs; +using tracing_service::TracingServiceImpl; + +// The compressors cap their output slices at the service's max slice size. +static_assert(packet_compressor::kCompressSliceSize == + TracingServiceImpl::kMaxTracePacketSliceSize); + +// One backend per compile-time-selected compressor. Each provides the compress +// entrypoint and a matching decompressor for the test to verify the output. +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +struct ZlibBackend { + static constexpr const char* kName = "Zlib"; + static void Compress(std::vector* packets, int /*level*/ = 0) { + ZlibCompressFn(packets); + } + static std::string Decompress(const std::string& data) { + uint8_t out[1024]; + z_stream stream{}; + stream.next_in = reinterpret_cast(const_cast(data.data())); + stream.avail_in = static_cast(data.size()); + EXPECT_EQ(inflateInit(&stream), Z_OK); + std::string s; + int ret; + do { + stream.next_out = out; + stream.avail_out = sizeof(out); + ret = inflate(&stream, Z_NO_FLUSH); + EXPECT_NE(ret, Z_STREAM_ERROR); + EXPECT_NE(ret, Z_NEED_DICT); + EXPECT_NE(ret, Z_DATA_ERROR); + EXPECT_NE(ret, Z_MEM_ERROR); + s.append(reinterpret_cast(out), sizeof(out) - stream.avail_out); + } while (ret != Z_STREAM_END); + inflateEnd(&stream); + return s; + } + static const std::string& CompressedField(const protos::gen::TracePacket& p) { + return p.compressed_packets(); + } +}; +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB) + +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +struct ZstdBackend { + static constexpr const char* kName = "Zstd"; + static void Compress(std::vector* packets, int level = 0) { + ZstdCompressFn(packets, level); + } + static std::string Decompress(const std::string& data) { + ZSTD_DStream* stream = ZSTD_createDStream(); + EXPECT_NE(stream, nullptr); + ZSTD_initDStream(stream); + uint8_t out[1024]; + ZSTD_inBuffer in = {data.data(), data.size(), 0}; + std::string s; + size_t ret = 0; + do { + ZSTD_outBuffer out_buf = {out, sizeof(out), 0}; + ret = ZSTD_decompressStream(stream, &out_buf, &in); + EXPECT_FALSE(ZSTD_isError(ret)); + s.append(reinterpret_cast(out), out_buf.pos); + } while (ret != 0 && in.pos < in.size); + ZSTD_freeDStream(stream); + return s; + } + static const std::string& CompressedField(const protos::gen::TracePacket& p) { + return p.zstd_compressed_packets(); + } +}; +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + +using Backends = ::testing::Types< +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) + ZlibBackend +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) && PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + , +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + ZstdBackend +#endif + >; + +class BackendNames { + public: + template + static std::string GetName(int) { + return T::kName; + } +}; + +template +class CompressorTest : public ::testing::Test {}; +TYPED_TEST_SUITE(CompressorTest, Backends, BackendNames); + +template +TracePacket CreateTracePacket(F fill_function) { + protos::gen::TracePacket msg; + fill_function(&msg); + std::vector buf = msg.SerializeAsArray(); + Slice slice = Slice::Allocate(buf.size()); + memcpy(slice.own_data(), buf.data(), buf.size()); + TracePacket packet; + packet.AddSlice(std::move(slice)); + return packet; +} + +// Returns a copy of `old` that owns its slices' data. +TracePacket CopyTracePacket(const TracePacket& old) { + TracePacket ret; + for (const Slice& slice : old.slices()) { + Slice new_slice = Slice::Allocate(slice.size); + memcpy(new_slice.own_data(), slice.start, slice.size); + ret.AddSlice(std::move(new_slice)); + } + return ret; +} + +std::vector CopyTracePackets(const std::vector& old) { + std::vector ret; + ret.reserve(old.size()); + for (const TracePacket& packet : old) + ret.push_back(CopyTracePacket(packet)); + return ret; +} + +// Distinct random bytes on every call: incompressible, and crucially never +// repeating across packets, so zstd (which dedupes across its window) can't +// collapse the MaxSliceSize input to nothing. +std::string RandomString(size_t size) { + static uint32_t seed = 0; + std::default_random_engine rnd(seed++); + std::uniform_int_distribution<> dist(0, 255); + std::string s(size, '\0'); + for (char& c : s) + c = static_cast(dist(rnd)); + return s; +} + +TYPED_TEST(CompressorTest, Empty) { + std::vector packets; + + TypeParam::Compress(&packets); + + EXPECT_THAT(packets, IsEmpty()); +} + +TYPED_TEST(CompressorTest, End2EndCompressAndDecompress) { + std::vector packets; + packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str("abc"); + })); + packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str("def"); + })); + + TypeParam::Compress(&packets); + + ASSERT_THAT(packets, SizeIs(1)); + protos::gen::TracePacket compressed_packet_proto; + ASSERT_TRUE(compressed_packet_proto.ParseFromString( + packets[0].GetRawBytesForTesting())); + const std::string& data = TypeParam::CompressedField(compressed_packet_proto); + EXPECT_THAT(data, Not(IsEmpty())); + protos::gen::Trace subtrace; + ASSERT_TRUE(subtrace.ParseFromString(TypeParam::Decompress(data))); + EXPECT_THAT( + subtrace.packet(), + ElementsAre(Property(&protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, "abc")), + Property(&protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, "def")))); +} + +TYPED_TEST(CompressorTest, MaxSliceSize) { + constexpr size_t kStopOutputSize = + TracingServiceImpl::kMaxTracePacketSliceSize + 2000; + + std::vector packets; + TracePacket compressed_packet; + while (compressed_packet.size() < kStopOutputSize) { + packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str(RandomString(65536)); + })); + std::vector packets_copy = CopyTracePackets(packets); + TypeParam::Compress(&packets_copy); + ASSERT_THAT(packets_copy, SizeIs(1)); + compressed_packet = std::move(packets_copy[0]); + } + + EXPECT_GE(compressed_packet.slices().size(), 2u); + ASSERT_GT(compressed_packet.size(), + TracingServiceImpl::kMaxTracePacketSliceSize); + EXPECT_THAT(compressed_packet.slices(), + Each(Field(&Slice::size, + Le(TracingServiceImpl::kMaxTracePacketSliceSize)))); +} + +// Round-trips many distinct packets and checks they come back in order. This +// exercises the per-packet framing/tokenization beyond the two-packet case. +TYPED_TEST(CompressorTest, ManyPacketsPreserveOrder) { + constexpr size_t kNumPackets = 100; + std::vector packets; + for (size_t i = 0; i < kNumPackets; i++) { + packets.push_back(CreateTracePacket([i](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str("packet-" + std::to_string(i)); + })); + } + + TypeParam::Compress(&packets); + + ASSERT_THAT(packets, SizeIs(1)); + protos::gen::TracePacket compressed; + ASSERT_TRUE(compressed.ParseFromString(packets[0].GetRawBytesForTesting())); + protos::gen::Trace subtrace; + ASSERT_TRUE(subtrace.ParseFromString( + TypeParam::Decompress(TypeParam::CompressedField(compressed)))); + ASSERT_THAT(subtrace.packet(), SizeIs(kNumPackets)); + for (size_t i = 0; i < kNumPackets; i++) { + EXPECT_EQ(subtrace.packet()[i].for_testing().str(), + "packet-" + std::to_string(i)); + } +} + +// Compresses enough incompressible data that the output spans multiple slices, +// then decompresses and checks the exact content survives. End2End only covers +// single-slice output; MaxSliceSize checks slice sizes but never round-trips. +TYPED_TEST(CompressorTest, MultiSliceOutputRoundTrip) { + std::vector expected; + std::vector packets; + for (int i = 0; i < 4; i++) { + std::string payload = RandomString(65536); + expected.push_back(payload); + packets.push_back(CreateTracePacket([&](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str(payload); + })); + } + + TypeParam::Compress(&packets); + + ASSERT_THAT(packets, SizeIs(1)); + EXPECT_GE(packets[0].slices().size(), 2u); // Output spans multiple slices. + protos::gen::TracePacket compressed; + ASSERT_TRUE(compressed.ParseFromString(packets[0].GetRawBytesForTesting())); + protos::gen::Trace subtrace; + ASSERT_TRUE(subtrace.ParseFromString( + TypeParam::Decompress(TypeParam::CompressedField(compressed)))); + ASSERT_THAT(subtrace.packet(), SizeIs(expected.size())); + for (size_t i = 0; i < expected.size(); i++) + EXPECT_EQ(subtrace.packet()[i].for_testing().str(), expected[i]); +} + +// A TracePacket whose payload is split across several input slices (as the +// service produces) must round-trip. Exercises the multi-slice loop in +// PushPacket, which the other tests (single-slice packets) never hit. +TYPED_TEST(CompressorTest, MultiSliceInputPacket) { + protos::gen::TracePacket msg; + msg.mutable_for_testing()->set_str("multi-slice-input-payload"); + std::vector buf = msg.SerializeAsArray(); + ASSERT_GT(buf.size(), 8u); + + TracePacket packet; + constexpr size_t kInSliceSize = 4; // Tiny, to force several input slices. + for (size_t pos = 0; pos < buf.size(); pos += kInSliceSize) { + size_t n = std::min(kInSliceSize, buf.size() - pos); + Slice slice = Slice::Allocate(n); + memcpy(slice.own_data(), &buf[pos], n); + packet.AddSlice(std::move(slice)); + } + ASSERT_GT(packet.slices().size(), 1u); + + std::vector packets; + packets.push_back(std::move(packet)); + + TypeParam::Compress(&packets); + + ASSERT_THAT(packets, SizeIs(1)); + protos::gen::TracePacket compressed; + ASSERT_TRUE(compressed.ParseFromString(packets[0].GetRawBytesForTesting())); + protos::gen::Trace subtrace; + ASSERT_TRUE(subtrace.ParseFromString( + TypeParam::Decompress(TypeParam::CompressedField(compressed)))); + ASSERT_THAT(subtrace.packet(), SizeIs(1)); + EXPECT_EQ(subtrace.packet()[0].for_testing().str(), + "multi-slice-input-payload"); +} + +// An explicit level must reach the codec: the output still round-trips, and on +// compressible data a higher level is never worse than a lower one (catching +// the level being silently dropped). +TYPED_TEST(CompressorTest, CompressionLevelIsHonored) { + auto make_packets = [] { + std::vector packets; + for (int i = 0; i < 200; i++) { + packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { + msg->mutable_for_testing()->set_str(std::string(1024, 'a')); + })); + } + return packets; + }; + + auto compress_and_measure = [](std::vector packets, + int level) -> size_t { + TypeParam::Compress(&packets, level); + EXPECT_THAT(packets, SizeIs(1)); + protos::gen::TracePacket compressed; + EXPECT_TRUE(compressed.ParseFromString(packets[0].GetRawBytesForTesting())); + protos::gen::Trace subtrace; + EXPECT_TRUE(subtrace.ParseFromString( + TypeParam::Decompress(TypeParam::CompressedField(compressed)))); + EXPECT_THAT(subtrace.packet(), SizeIs(200)); + return packets[0].size(); + }; + + size_t low_level_size = compress_and_measure(make_packets(), 1); + size_t high_level_size = compress_and_measure(make_packets(), 9); + + EXPECT_GT(low_level_size, 0u); + EXPECT_LE(high_level_size, low_level_size); +} + +} // namespace +} // namespace perfetto diff --git a/src/tracing/service/packet_compressor_common.h b/src/tracing/service/packet_compressor_common.h new file mode 100644 index 00000000000..e1b98a8108f --- /dev/null +++ b/src/tracing/service/packet_compressor_common.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACING_SERVICE_PACKET_COMPRESSOR_COMMON_H_ +#define SRC_TRACING_SERVICE_PACKET_COMPRESSOR_COMMON_H_ + +#include +#include +#include + +#include "perfetto/base/logging.h" +#include "perfetto/ext/tracing/core/slice.h" +#include "perfetto/protozero/proto_utils.h" + +namespace perfetto { +namespace packet_compressor { + +// TODO(sashwinbalaji): Extract the output-slice buffering and packet assembly +// shared by the zlib and zstd compressors into this header, so each codec is +// just its own streaming loop. + +// Size of each compressed output slice. Mirrors the service's +// kMaxTracePacketSliceSize. +inline constexpr size_t kCompressSliceSize = 128 * 1024 - 512; + +// Builds the proto preamble (field tag + length) that the zlib and zstd packet +// compressors prefix to each TracePacket, so the compressed stream itself +// parses as a valid Trace proto. +struct Preamble { + uint32_t size; + std::array buf; +}; + +template +Preamble GetPreamble(size_t sz) { + Preamble preamble{}; + uint8_t* ptr = preamble.buf.data(); + constexpr uint32_t tag = protozero::proto_utils::MakeTagLengthDelimited(id); + ptr = protozero::proto_utils::WriteVarInt(tag, ptr); + ptr = protozero::proto_utils::WriteVarInt(sz, ptr); + preamble.size = + static_cast(reinterpret_cast(ptr) - + reinterpret_cast(preamble.buf.data())); + PERFETTO_DCHECK(preamble.size < preamble.buf.size()); + return preamble; +} + +inline Slice PreambleToSlice(const Preamble& preamble) { + Slice slice = Slice::Allocate(preamble.size); + memcpy(slice.own_data(), preamble.buf.data(), preamble.size); + return slice; +} + +} // namespace packet_compressor +} // namespace perfetto + +#endif // SRC_TRACING_SERVICE_PACKET_COMPRESSOR_COMMON_H_ diff --git a/src/tracing/service/packet_stream_validator.cc b/src/tracing/service/packet_stream_validator.cc index 86c5a3e567f..7572b05ce9c 100644 --- a/src/tracing/service/packet_stream_validator.cc +++ b/src/tracing/service/packet_stream_validator.cc @@ -38,6 +38,7 @@ const uint32_t kReservedFieldIds[] = { protos::pbzero::TracePacket::kTraceConfigFieldNumber, protos::pbzero::TracePacket::kTraceStatsFieldNumber, protos::pbzero::TracePacket::kCompressedPacketsFieldNumber, + protos::pbzero::TracePacket::kZstdCompressedPacketsFieldNumber, protos::pbzero::TracePacket::kSynchronizationMarkerFieldNumber, protos::pbzero::TracePacket::kTrustedPidFieldNumber, protos::pbzero::TracePacket::kMachineIdFieldNumber, diff --git a/src/tracing/service/tracing_service_impl.cc b/src/tracing/service/tracing_service_impl.cc index 7495c705aca..2a19742fd21 100644 --- a/src/tracing/service/tracing_service_impl.cc +++ b/src/tracing/service/tracing_service_impl.cc @@ -116,6 +116,12 @@ #include "src/tracing/service/tracing_service_endpoints_impl.h" #include "src/tracing/service/tracing_service_session.h" #include "src/tracing/service/tracing_service_structs.h" +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +#include "src/tracing/service/zlib_compressor.h" +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +#include "src/tracing/service/zstd_compressor.h" +#endif #include "protos/perfetto/common/builtin_clock.gen.h" #include "protos/perfetto/common/builtin_clock.pbzero.h" @@ -1165,15 +1171,14 @@ base::Status TracingServiceImpl::EnableTracing(ConsumerEndpointImpl* consumer, cfg.fflush_post_write() == TraceConfig::FFLUSH_ENABLED; } - if (cfg.compression_type() == TraceConfig::COMPRESSION_TYPE_DEFLATE) { - if (init_opts_.compressor_fn) { - tracing_session->compress_deflate = true; - } else { - PERFETTO_LOG( - "COMPRESSION_TYPE_DEFLATE is not supported in the current build " - "configuration. Skipping compression"); - } +#if !PERFETTO_BUILDFLAG(PERFETTO_ZLIB) && !PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + if (cfg.compression_type() != TraceConfig::COMPRESSION_TYPE_UNSPECIFIED || + cfg.has_compression()) { + PERFETTO_LOG( + "Compression was requested but this build has no compressor. " + "Skipping compression"); } +#endif // Initialize the log buffers. bool did_allocate_all_buffers = true; @@ -2909,12 +2914,30 @@ void TracingServiceImpl::MaybeFilterPackets(TracingSession* tracing_session, void TracingServiceImpl::MaybeCompressPackets( TracingSession* tracing_session, - std::vector* packets) { - if (!tracing_session->compress_deflate) { + [[maybe_unused]] std::vector* packets) { + // Compress with the codec the config selects, preferring the newest (highest + // proto field number) this build supports. Leaves the packets uncompressed if + // none is available. + // + // The branches below run highest-field-number-first, so a new codec's branch + // goes at the top. + [[maybe_unused]] const auto& compression = + tracing_session->config.compression(); +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + if (compression.has_zstd()) { + ZstdCompressFn(packets, compression.zstd().level()); return; } - - init_opts_.compressor_fn(packets); +#endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) + // Deflate also serves the legacy compression_type = DEFLATE, so configs + // predating `compression` still get compressed. + if (compression.has_deflate() || tracing_session->config.compression_type() == + TraceConfig::COMPRESSION_TYPE_DEFLATE) { + ZlibCompressFn(packets); + return; + } +#endif } bool TracingServiceImpl::WriteIntoFile(TracingSession* tracing_session, @@ -4840,7 +4863,6 @@ base::Status TracingServiceImpl::FinishCloneSession( cloned_session->flushes_requested = src->flushes_requested; cloned_session->flushes_succeeded = src->flushes_succeeded; cloned_session->flushes_failed = src->flushes_failed; - cloned_session->compress_deflate = src->compress_deflate; if (src->trace_filter && !skip_trace_filter) { // Copy the trace filter, unless it's a clone-for-bugreport (b/317065412). cloned_session->trace_filter.reset( diff --git a/src/tracing/service/tracing_service_impl_unittest.cc b/src/tracing/service/tracing_service_impl_unittest.cc index bde0170654d..3737ca609e1 100644 --- a/src/tracing/service/tracing_service_impl_unittest.cc +++ b/src/tracing/service/tracing_service_impl_unittest.cc @@ -88,6 +88,11 @@ #include "src/tracing/service/zlib_compressor.h" #endif +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +#include +#include "src/tracing/service/zstd_compressor.h" +#endif + using ::testing::_; using ::testing::AssertionFailure; using ::testing::AssertionResult; @@ -165,7 +170,7 @@ MATCHER_P(LowerCase, } #if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) -std::string Decompress(const std::string& data) { +std::string DecompressZlib(const std::string& data) { uint8_t out[1024]; z_stream stream{}; @@ -191,7 +196,7 @@ std::string Decompress(const std::string& data) { return s; } -std::vector DecompressTrace( +std::vector DecompressTraceZlib( const std::vector compressed) { std::vector decompressed; @@ -201,7 +206,7 @@ std::vector DecompressTrace( continue; } - std::string s = Decompress(c.compressed_packets()); + std::string s = DecompressZlib(c.compressed_packets()); protos::gen::Trace t; EXPECT_TRUE(t.ParseFromString(s)); decompressed.insert(decompressed.end(), t.packet().begin(), @@ -211,6 +216,42 @@ std::vector DecompressTrace( } #endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB) +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +std::string DecompressZstd(const std::string& data) { + ZSTD_DStream* stream = ZSTD_createDStream(); + ZSTD_initDStream(stream); + uint8_t out[1024]; + ZSTD_inBuffer in = {data.data(), data.size(), 0}; + std::string s; + size_t ret = 0; + do { + ZSTD_outBuffer out_buf = {out, sizeof(out), 0}; + ret = ZSTD_decompressStream(stream, &out_buf, &in); + EXPECT_FALSE(ZSTD_isError(ret)); + s.append(reinterpret_cast(out), out_buf.pos); + } while (ret != 0 && in.pos < in.size); + ZSTD_freeDStream(stream); + return s; +} + +std::vector DecompressTraceZstd( + const std::vector compressed) { + std::vector decompressed; + for (const protos::gen::TracePacket& c : compressed) { + if (c.zstd_compressed_packets().empty()) { + decompressed.push_back(c); + continue; + } + std::string s = DecompressZstd(c.zstd_compressed_packets()); + protos::gen::Trace t; + EXPECT_TRUE(t.ParseFromString(s)); + decompressed.insert(decompressed.end(), t.packet().begin(), + t.packet().end()); + } + return decompressed; +} +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + std::vector GetReceivedTriggers( const std::vector& trace) { std::vector triggers; @@ -2119,11 +2160,11 @@ TEST_F(TracingServiceImplTest, ReconnectProducerWhileTracing) { producer->WaitForDataSourceStart("data_source"); } +// With no codec compiled in, a config that asks for compression must degrade to +// an uncompressed trace rather than dropping data. +#if !PERFETTO_BUILDFLAG(PERFETTO_ZLIB) && !PERFETTO_BUILDFLAG(PERFETTO_ZSTD) TEST_F(TracingServiceImplTest, CompressionConfiguredButUnsupported) { - // Initialize the service without support for compression. - TracingService::InitOpts init_opts; - init_opts.compressor_fn = nullptr; - InitializeSvcWithOpts(init_opts); + InitializeSvcWithOpts({}); std::unique_ptr consumer = CreateMockConsumer(); consumer->Connect(svc.get()); @@ -2176,11 +2217,12 @@ TEST_F(TracingServiceImplTest, CompressionConfiguredButUnsupported) { Property(&protos::gen::TestEvent::str, Eq("payload-2"))))); } +#endif // !PERFETTO_BUILDFLAG(PERFETTO_ZLIB) && + // !PERFETTO_BUILDFLAG(PERFETTO_ZSTD) #if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) TEST_F(TracingServiceImplTest, CompressionReadIpc) { TracingService::InitOpts init_opts; - init_opts.compressor_fn = ZlibCompressFn; InitializeSvcWithOpts(init_opts); std::unique_ptr consumer = CreateMockConsumer(); @@ -2227,7 +2269,7 @@ TEST_F(TracingServiceImplTest, CompressionReadIpc) { Each(Property(&protos::gen::TracePacket::compressed_packets, Not(IsEmpty())))); std::vector decompressed_packets = - DecompressTrace(compressed_packets); + DecompressTraceZlib(compressed_packets); EXPECT_THAT(decompressed_packets, Contains(Property( &protos::gen::TracePacket::for_testing, @@ -2238,9 +2280,164 @@ TEST_F(TracingServiceImplTest, CompressionReadIpc) { Property(&protos::gen::TestEvent::str, Eq("payload-2"))))); } +// A config that selects zstd via `compression` but is served by a build without +// zstd must fall back to the legacy compression_type (DEFLATE), not emit an +// uncompressed trace. The output is therefore deflate-compressed. +#if !PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +TEST_F(TracingServiceImplTest, CompressionConfigZstdFallsBackToLegacyDeflate) { + InitializeSvcWithOpts({}); + + std::unique_ptr consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + trace_config.set_compression_type(TraceConfig::COMPRESSION_TYPE_DEFLATE); + trace_config.mutable_compression()->mutable_zstd(); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + std::vector compressed_packets = + consumer->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::compressed_packets, + Not(IsEmpty())))); + // Decodes with the zlib (deflate) decompressor, proving the fall-through. + std::vector decompressed_packets = + DecompressTraceZlib(compressed_packets); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-1"))))); +} +#endif // !PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + +// Deflate can be selected directly via the new config (compression.deflate), +// with no legacy compression_type set. The output is deflate-compressed. +TEST_F(TracingServiceImplTest, CompressionConfigDeflate) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + std::unique_ptr consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + trace_config.mutable_compression()->mutable_deflate(); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + std::vector compressed_packets = + consumer->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::compressed_packets, + Not(IsEmpty())))); + std::vector decompressed_packets = + DecompressTraceZlib(compressed_packets); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-1"))))); +} + +// A compression field naming no codec this service understands (e.g. a newer +// config with a future codec) must not crash: it degrades to no compression. +TEST_F(TracingServiceImplTest, CompressionConfigUnknownCodecDoesNotCrash) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + std::unique_ptr consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + // compression is set but names no codec this service understands (as + // an old service would observe a future codec's sub-message). No legacy + // compression_type is set, so this degrades to no compression, not a crash. + trace_config.mutable_compression(); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + // No compression was applied; the packet is readable as-is (no crash). + std::vector packets = consumer->ReadBuffers(); + EXPECT_THAT(packets, Contains(Property(&protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, + Eq("payload-1"))))); +} + TEST_F(TracingServiceImplTest, CompressionWriteIntoFile) { TracingService::InitOpts init_opts; - init_opts.compressor_fn = ZlibCompressFn; InitializeSvcWithOpts(init_opts); std::unique_ptr consumer = CreateMockConsumer(); @@ -2292,7 +2489,7 @@ TEST_F(TracingServiceImplTest, CompressionWriteIntoFile) { Each(Property(&protos::gen::TracePacket::compressed_packets, Not(IsEmpty())))); std::vector decompressed_packets = - DecompressTrace(trace.packet()); + DecompressTraceZlib(trace.packet()); EXPECT_THAT(decompressed_packets, Contains(Property( &protos::gen::TracePacket::for_testing, @@ -2303,6 +2500,245 @@ TEST_F(TracingServiceImplTest, CompressionWriteIntoFile) { Property(&protos::gen::TestEvent::str, Eq("payload-2"))))); } +TEST_F(TracingServiceImplTest, CloneSessionWithCompression) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + // The consumer the creates the initial tracing session. + std::unique_ptr consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + // The consumer that clones it and reads back the data. + std::unique_ptr consumer2 = CreateMockConsumer(); + consumer2->Connect(svc.get()); + + std::unique_ptr producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + + producer->RegisterDataSource("ds_1"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(32); + auto* ds_cfg = trace_config.add_data_sources()->mutable_config(); + ds_cfg->set_name("ds_1"); + trace_config.set_compression_type(TraceConfig::COMPRESSION_TYPE_DEFLATE); + + consumer->EnableTracing(trace_config); + producer->WaitForTracingSetup(); + + producer->WaitForDataSourceSetup("ds_1"); + + producer->WaitForDataSourceStart("ds_1"); + + std::unique_ptr writer = producer->CreateTraceWriter("ds_1"); + + // Add some data. + static constexpr size_t kNumTestPackets = 20; + for (size_t i = 0; i < kNumTestPackets; i++) { + auto tp = writer->NewTracePacket(); + std::string payload("payload" + std::to_string(i)); + tp->set_for_testing()->set_str(payload.c_str(), payload.size()); + tp->set_timestamp(static_cast(i)); + } + + auto clone_done = task_runner.CreateCheckpoint("clone_done"); + EXPECT_CALL(*consumer2, OnSessionCloned(_)) + .WillOnce( + [clone_done](const Consumer::OnSessionClonedArgs&) { clone_done(); }); + consumer2->CloneSession(1); + // CloneSession() will implicitly issue a flush. Linearize with that. + FlushFlags expected_flags(FlushFlags::Initiator::kTraced, + FlushFlags::Reason::kTraceClone); + producer->ExpectFlush(writer.get(), /*reply=*/true, expected_flags); + task_runner.RunUntilCheckpoint("clone_done"); + + // Delete the initial tracing session. + consumer->DisableTracing(); + consumer->FreeBuffers(); + producer->WaitForDataSourceStop("ds_1"); + consumer->WaitForTracingDisabled(); + + // Read back the cloned trace and check that it's compressed + std::vector compressed_packets = + consumer2->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::compressed_packets, + Not(IsEmpty())))); +} + +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB) + +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +TEST_F(TracingServiceImplTest, CompressionZstdReadIpc) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + std::unique_ptr consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + trace_config.mutable_compression()->mutable_zstd(); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-2"); + } + + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + std::vector compressed_packets = + consumer->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::zstd_compressed_packets, + Not(IsEmpty())))); + std::vector decompressed_packets = + DecompressTraceZstd(compressed_packets); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-1"))))); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-2"))))); +} + +// An explicit zstd compression level from the TraceConfig must be plumbed +// through to the compressor and still produce a decodable trace. +TEST_F(TracingServiceImplTest, CompressionZstdConfiguredLevel) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + std::unique_ptr consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + trace_config.mutable_compression()->mutable_zstd()->set_level(19); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + std::vector compressed_packets = + consumer->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::zstd_compressed_packets, + Not(IsEmpty())))); + std::vector decompressed_packets = + DecompressTraceZstd(compressed_packets); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-1"))))); +} +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + +#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) && PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +// When both compressors are available, the zstd compression is used in +// preference to the legacy compression_type (deflate). The output is therefore +// zstd-compressed. +TEST_F(TracingServiceImplTest, CompressionConfigZstdUsedWhenAvailable) { + TracingService::InitOpts init_opts; + InitializeSvcWithOpts(init_opts); + + std::unique_ptr consumer = CreateMockConsumer(); + consumer->Connect(svc.get()); + + std::unique_ptr producer = CreateMockProducer(); + producer->Connect(svc.get(), "mock_producer"); + producer->RegisterDataSource("data_source"); + + TraceConfig trace_config; + trace_config.add_buffers()->set_size_kb(4096); + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("data_source"); + ds_config->set_target_buffer(0); + trace_config.set_compression_type(TraceConfig::COMPRESSION_TYPE_DEFLATE); + trace_config.mutable_compression()->mutable_zstd(); + consumer->EnableTracing(trace_config); + + producer->WaitForTracingSetup(); + producer->WaitForDataSourceSetup("data_source"); + producer->WaitForDataSourceStart("data_source"); + + std::unique_ptr writer = + producer->CreateTraceWriter("data_source"); + { + auto tp = writer->NewTracePacket(); + tp->set_for_testing()->set_str("payload-1"); + } + writer->Flush(); + writer.reset(); + + consumer->DisableTracing(); + producer->WaitForDataSourceStop("data_source"); + consumer->WaitForTracingDisabled(); + + std::vector compressed_packets = + consumer->ReadBuffers(); + EXPECT_THAT(compressed_packets, Not(IsEmpty())); + EXPECT_THAT(compressed_packets, + Each(Property(&protos::gen::TracePacket::zstd_compressed_packets, + Not(IsEmpty())))); + // Decodes with the zstd decompressor, proving the preference won. + std::vector decompressed_packets = + DecompressTraceZstd(compressed_packets); + EXPECT_THAT(decompressed_packets, + Contains(Property( + &protos::gen::TracePacket::for_testing, + Property(&protos::gen::TestEvent::str, Eq("payload-1"))))); +} +#endif // PERFETTO_ZLIB && PERFETTO_ZSTD + TEST_F(TracingServiceImplTest, FlushStrategies) { constexpr uint32_t kDefaultWriteIntoFilePeriodMs = 5000; @@ -2414,76 +2850,6 @@ TEST_F(TracingServiceImplTest, FlushStrategies) { } } -TEST_F(TracingServiceImplTest, CloneSessionWithCompression) { - TracingService::InitOpts init_opts; - init_opts.compressor_fn = ZlibCompressFn; - InitializeSvcWithOpts(init_opts); - - // The consumer the creates the initial tracing session. - std::unique_ptr consumer = CreateMockConsumer(); - consumer->Connect(svc.get()); - - // The consumer that clones it and reads back the data. - std::unique_ptr consumer2 = CreateMockConsumer(); - consumer2->Connect(svc.get()); - - std::unique_ptr producer = CreateMockProducer(); - producer->Connect(svc.get(), "mock_producer"); - - producer->RegisterDataSource("ds_1"); - - TraceConfig trace_config; - trace_config.add_buffers()->set_size_kb(32); - auto* ds_cfg = trace_config.add_data_sources()->mutable_config(); - ds_cfg->set_name("ds_1"); - trace_config.set_compression_type(TraceConfig::COMPRESSION_TYPE_DEFLATE); - - consumer->EnableTracing(trace_config); - producer->WaitForTracingSetup(); - - producer->WaitForDataSourceSetup("ds_1"); - - producer->WaitForDataSourceStart("ds_1"); - - std::unique_ptr writer = producer->CreateTraceWriter("ds_1"); - - // Add some data. - static constexpr size_t kNumTestPackets = 20; - for (size_t i = 0; i < kNumTestPackets; i++) { - auto tp = writer->NewTracePacket(); - std::string payload("payload" + std::to_string(i)); - tp->set_for_testing()->set_str(payload.c_str(), payload.size()); - tp->set_timestamp(static_cast(i)); - } - - auto clone_done = task_runner.CreateCheckpoint("clone_done"); - EXPECT_CALL(*consumer2, OnSessionCloned(_)) - .WillOnce( - [clone_done](const Consumer::OnSessionClonedArgs&) { clone_done(); }); - consumer2->CloneSession(1); - // CloneSession() will implicitly issue a flush. Linearize with that. - FlushFlags expected_flags(FlushFlags::Initiator::kTraced, - FlushFlags::Reason::kTraceClone); - producer->ExpectFlush(writer.get(), /*reply=*/true, expected_flags); - task_runner.RunUntilCheckpoint("clone_done"); - - // Delete the initial tracing session. - consumer->DisableTracing(); - consumer->FreeBuffers(); - producer->WaitForDataSourceStop("ds_1"); - consumer->WaitForTracingDisabled(); - - // Read back the cloned trace and check that it's compressed - std::vector compressed_packets = - consumer2->ReadBuffers(); - EXPECT_THAT(compressed_packets, Not(IsEmpty())); - EXPECT_THAT(compressed_packets, - Each(Property(&protos::gen::TracePacket::compressed_packets, - Not(IsEmpty())))); -} - -#endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB) - // Note: file_write_period_ms is set to a large enough to have exactly one flush // of the tracing buffers (and therefore at most one synchronization section), // unless the test runs unrealistically slowly, or the implementation of the diff --git a/src/tracing/service/tracing_service_session.h b/src/tracing/service/tracing_service_session.h index 9bf2e9f91f0..c9fbb146564 100644 --- a/src/tracing/service/tracing_service_session.h +++ b/src/tracing/service/tracing_service_session.h @@ -159,9 +159,6 @@ struct TracingSession { // Whether we emitted the ProtoVM instances. bool did_emit_protovm_instances_ = false; - // Whether we should compress TracePackets after reading them. - bool compress_deflate = false; - // The number of received triggers we've emitted into the trace output. size_t num_triggers_emitted_into_trace = 0; diff --git a/src/tracing/service/zlib_compressor.cc b/src/tracing/service/zlib_compressor.cc index a8ad6171f3a..e4da016c2db 100644 --- a/src/tracing/service/zlib_compressor.cc +++ b/src/tracing/service/zlib_compressor.cc @@ -25,35 +25,16 @@ #include "protos/perfetto/trace/trace.pbzero.h" #include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "src/tracing/service/packet_compressor_common.h" namespace perfetto { namespace { -struct Preamble { - uint32_t size; - std::array buf; -}; - -template -Preamble GetPreamble(size_t sz) { - Preamble preamble; - uint8_t* ptr = preamble.buf.data(); - constexpr uint32_t tag = protozero::proto_utils::MakeTagLengthDelimited(id); - ptr = protozero::proto_utils::WriteVarInt(tag, ptr); - ptr = protozero::proto_utils::WriteVarInt(sz, ptr); - preamble.size = - static_cast(reinterpret_cast(ptr) - - reinterpret_cast(preamble.buf.data())); - PERFETTO_DCHECK(preamble.size < preamble.buf.size()); - return preamble; -} - -Slice PreambleToSlice(const Preamble& preamble) { - Slice slice = Slice::Allocate(preamble.size); - memcpy(slice.own_data(), preamble.buf.data(), preamble.size); - return slice; -} +using packet_compressor::GetPreamble; +using packet_compressor::kCompressSliceSize; +using packet_compressor::Preamble; +using packet_compressor::PreambleToSlice; // A compressor for `TracePacket`s that uses zlib. The class is exposed for // testing. @@ -139,16 +120,16 @@ TracePacket ZlibPacketCompressor::Finish() { void ZlibPacketCompressor::NewOutputSlice() { PushCurSlice(); - cur_slice_ = std::make_unique(kZlibCompressSliceSize); + cur_slice_ = std::make_unique(kCompressSliceSize); stream_.next_out = reinterpret_cast(cur_slice_.get()); - stream_.avail_out = kZlibCompressSliceSize; + stream_.avail_out = kCompressSliceSize; } void ZlibPacketCompressor::PushCurSlice() { if (cur_slice_) { - total_new_slices_size_ += kZlibCompressSliceSize - stream_.avail_out; + total_new_slices_size_ += kCompressSliceSize - stream_.avail_out; new_slices_.push_back(Slice::TakeOwnership( - std::move(cur_slice_), kZlibCompressSliceSize - stream_.avail_out)); + std::move(cur_slice_), kCompressSliceSize - stream_.avail_out)); } } diff --git a/src/tracing/service/zlib_compressor.h b/src/tracing/service/zlib_compressor.h index b2b1dd9011b..45af806338c 100644 --- a/src/tracing/service/zlib_compressor.h +++ b/src/tracing/service/zlib_compressor.h @@ -25,9 +25,6 @@ namespace perfetto { #if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) -// Matches TracingServiceImpl::kMaxTracePacketSliceSize. Exposed for testing. -static constexpr size_t kZlibCompressSliceSize = 128 * 1024 - 512; - void ZlibCompressFn(std::vector*); #endif diff --git a/src/tracing/service/zlib_compressor_unittest.cc b/src/tracing/service/zlib_compressor_unittest.cc deleted file mode 100644 index 3532f56a1a7..00000000000 --- a/src/tracing/service/zlib_compressor_unittest.cc +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "src/tracing/service/zlib_compressor.h" - -#include - -#include - -#include "protos/perfetto/trace/test_event.gen.h" -#include "protos/perfetto/trace/trace.gen.h" -#include "protos/perfetto/trace/trace_packet.gen.h" -#include "src/tracing/service/tracing_service_impl.h" -#include "test/gtest_and_gmock.h" - -namespace perfetto { -namespace { - -using ::testing::Each; -using ::testing::ElementsAre; -using ::testing::Field; -using ::testing::IsEmpty; -using ::testing::Le; -using ::testing::Not; -using ::testing::Property; -using ::testing::SizeIs; -using tracing_service::TracingServiceImpl; - -template -TracePacket CreateTracePacket(F fill_function) { - protos::gen::TracePacket msg; - fill_function(&msg); - std::vector buf = msg.SerializeAsArray(); - Slice slice = Slice::Allocate(buf.size()); - memcpy(slice.own_data(), buf.data(), buf.size()); - perfetto::TracePacket packet; - packet.AddSlice(std::move(slice)); - return packet; -} - -// Return a copy of the `old` trace packets that owns its own slices data. -TracePacket CopyTracePacket(const TracePacket& old) { - TracePacket ret; - for (const Slice& slice : old.slices()) { - auto new_slice = Slice::Allocate(slice.size); - memcpy(new_slice.own_data(), slice.start, slice.size); - ret.AddSlice(std::move(new_slice)); - } - return ret; -} - -std::vector CopyTracePackets(const std::vector& old) { - std::vector ret; - ret.reserve(old.size()); - for (const TracePacket& trace_packet : old) { - ret.push_back(CopyTracePacket(trace_packet)); - } - return ret; -} -std::string RandomString(size_t size) { - std::default_random_engine rnd(0); - std::uniform_int_distribution<> dist(0, 255); - std::string s; - s.resize(size); - for (size_t i = 0; i < s.size(); i++) - s[i] = static_cast(dist(rnd)); - return s; -} - -std::string Decompress(const std::string& data) { - uint8_t out[1024]; - - z_stream stream{}; - stream.next_in = reinterpret_cast(const_cast(data.data())); - stream.avail_in = static_cast(data.size()); - - EXPECT_EQ(inflateInit(&stream), Z_OK); - std::string s; - - int ret; - do { - stream.next_out = out; - stream.avail_out = sizeof(out); - ret = inflate(&stream, Z_NO_FLUSH); - EXPECT_NE(ret, Z_STREAM_ERROR); - EXPECT_NE(ret, Z_NEED_DICT); - EXPECT_NE(ret, Z_DATA_ERROR); - EXPECT_NE(ret, Z_MEM_ERROR); - s.append(reinterpret_cast(out), sizeof(out) - stream.avail_out); - } while (ret != Z_STREAM_END); - - inflateEnd(&stream); - return s; -} - -static_assert(kZlibCompressSliceSize == - TracingServiceImpl::kMaxTracePacketSliceSize); - -TEST(ZlibCompressFnTest, Empty) { - std::vector packets; - - ZlibCompressFn(&packets); - - EXPECT_THAT(packets, IsEmpty()); -} - -TEST(ZlibCompressFnTest, End2EndCompressAndDecompress) { - std::vector packets; - - packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { - auto* for_testing = msg->mutable_for_testing(); - for_testing->set_str("abc"); - })); - packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { - auto* for_testing = msg->mutable_for_testing(); - for_testing->set_str("def"); - })); - - ZlibCompressFn(&packets); - - ASSERT_THAT(packets, SizeIs(1)); - protos::gen::TracePacket compressed_packet_proto; - ASSERT_TRUE(compressed_packet_proto.ParseFromString( - packets[0].GetRawBytesForTesting())); - const std::string& data = compressed_packet_proto.compressed_packets(); - EXPECT_THAT(data, Not(IsEmpty())); - protos::gen::Trace subtrace; - ASSERT_TRUE(subtrace.ParseFromString(Decompress(data))); - EXPECT_THAT( - subtrace.packet(), - ElementsAre(Property(&protos::gen::TracePacket::for_testing, - Property(&protos::gen::TestEvent::str, "abc")), - Property(&protos::gen::TracePacket::for_testing, - Property(&protos::gen::TestEvent::str, "def")))); -} - -TEST(ZlibCompressFnTest, MaxSliceSize) { - std::vector packets; - - constexpr size_t kStopOutputSize = - TracingServiceImpl::kMaxTracePacketSliceSize + 2000; - - TracePacket compressed_packet; - while (compressed_packet.size() < kStopOutputSize) { - packets.push_back(CreateTracePacket([](protos::gen::TracePacket* msg) { - auto* for_testing = msg->mutable_for_testing(); - for_testing->set_str(RandomString(65536)); - })); - { - std::vector packets_copy = CopyTracePackets(packets); - ZlibCompressFn(&packets_copy); - ASSERT_THAT(packets_copy, SizeIs(1)); - compressed_packet = std::move(packets_copy[0]); - } - } - - EXPECT_GE(compressed_packet.slices().size(), 2u); - ASSERT_GT(compressed_packet.size(), - TracingServiceImpl::kMaxTracePacketSliceSize); - EXPECT_THAT(compressed_packet.slices(), - Each(Field(&Slice::size, - Le(TracingServiceImpl::kMaxTracePacketSliceSize)))); -} - -} // namespace -} // namespace perfetto diff --git a/src/tracing/service/zstd_compressor.cc b/src/tracing/service/zstd_compressor.cc new file mode 100644 index 00000000000..752c4f8cad5 --- /dev/null +++ b/src/tracing/service/zstd_compressor.cc @@ -0,0 +1,185 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/tracing/service/zstd_compressor.h" + +#include "perfetto/base/build_config.h" + +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) + +#include + +#include + +#include "perfetto/base/logging.h" +#include "perfetto/ext/base/scoped_file.h" +#include "protos/perfetto/trace/trace.pbzero.h" +#include "protos/perfetto/trace/trace_packet.pbzero.h" +#include "src/tracing/service/packet_compressor_common.h" + +namespace perfetto { + +namespace { + +using packet_compressor::GetPreamble; +using packet_compressor::kCompressSliceSize; +using packet_compressor::Preamble; +using packet_compressor::PreambleToSlice; + +// ScopedResource's close function must return int; ZSTD_freeCStream returns +// size_t, so wrap it. It only fails for a static-allocated stream, not the +// heap one ZSTD_createCStream gives us, so returning 0 is safe. +inline int ZstdFreeCStream(ZSTD_CStream* cstream) { + ZSTD_freeCStream(cstream); + return 0; +} +using ScopedZstdCStream = base::ScopedResource; + +// A compressor for `TracePacket`s that uses zstd's streaming API: data is fed +// in with ZSTD_compressStream2(ZSTD_e_continue) and the frame is finalized with +// ZSTD_e_end, emitting output in fixed-size slices (kCompressSliceSize). +// zstd API reference: https://facebook.github.io/zstd/zstd_manual.html +class ZstdPacketCompressor { + public: + explicit ZstdPacketCompressor(int level); + + // Can be called multiple times, before Finish() is called. + void PushPacket(const TracePacket& packet); + + // Returns the compressed data. Can be called at most once. After this call, + // the object is unusable (PushPacket should not be called) and must be + // destroyed. + TracePacket Finish(); + + private: + void PushData(const void* data, uint32_t size); + void NewOutputSlice(); + void PushCurSlice(); + + ScopedZstdCStream cstream_; + // Points into `cur_slice_`. Zero-initialized so that the first compression + // call observes a full output buffer and allocates the initial slice. + ZSTD_outBuffer out_{/*dst=*/nullptr, /*size=*/0, /*pos=*/0}; + size_t total_new_slices_size_ = 0; + std::vector new_slices_; + std::unique_ptr cur_slice_; +}; + +ZstdPacketCompressor::ZstdPacketCompressor(int level) { + cstream_.reset(ZSTD_createCStream()); + PERFETTO_CHECK(cstream_); + // zstd maps 0 to its default level, clamps levels above its max, and treats + // negatives as fast modes, so any int is safe to pass through here. + size_t rc = + ZSTD_CCtx_setParameter(cstream_.get(), ZSTD_c_compressionLevel, level); + PERFETTO_CHECK(!ZSTD_isError(rc)); +} + +void ZstdPacketCompressor::PushPacket(const TracePacket& packet) { + // Prefix each packet with its proto preamble so the compressed stream itself + // parses as a valid Trace proto, and its packets can be tokenized back out. + Preamble preamble = + GetPreamble(packet.size()); + PushData(preamble.buf.data(), preamble.size); + for (const Slice& slice : packet.slices()) { + PushData(slice.start, static_cast(slice.size)); + } +} + +void ZstdPacketCompressor::PushData(const void* data, uint32_t size) { + // ZSTD_e_continue hands data to the encoder, which buffers and emits at its + // own discretion. It may not consume all input in one call (e.g. when the + // output slice fills), so loop until `in` is drained, giving it a fresh slice + // whenever `out_` is full. + ZSTD_inBuffer in = {/*src=*/data, /*size=*/size, /*pos=*/0}; + while (in.pos < in.size) { + if (out_.pos == out_.size) { + NewOutputSlice(); + } + size_t rc = + ZSTD_compressStream2(cstream_.get(), &out_, &in, ZSTD_e_continue); + PERFETTO_CHECK(!ZSTD_isError(rc)); + } +} + +TracePacket ZstdPacketCompressor::Finish() { + // ZSTD_e_end flushes buffered data and writes the frame epilogue. Per the + // contract, keep calling (draining `out_` into new slices) until it reports 0 + // bytes remaining. + size_t remaining; + do { + if (out_.pos == out_.size) { + NewOutputSlice(); + } + ZSTD_inBuffer in = {/*src=*/nullptr, /*size=*/0, /*pos=*/0}; + remaining = ZSTD_compressStream2(cstream_.get(), &out_, &in, ZSTD_e_end); + PERFETTO_CHECK(!ZSTD_isError(remaining)); + } while (remaining != 0); + + PushCurSlice(); + + TracePacket packet; + packet.AddSlice(PreambleToSlice( + GetPreamble< + protos::pbzero::TracePacket::kZstdCompressedPacketsFieldNumber>( + total_new_slices_size_))); + for (auto& slice : new_slices_) { + packet.AddSlice(std::move(slice)); + } + return packet; +} + +void ZstdPacketCompressor::NewOutputSlice() { + PushCurSlice(); + cur_slice_ = std::make_unique(kCompressSliceSize); + out_.dst = cur_slice_.get(); + out_.size = kCompressSliceSize; + out_.pos = 0; +} + +void ZstdPacketCompressor::PushCurSlice() { + if (!cur_slice_) { + return; + } + total_new_slices_size_ += out_.pos; + new_slices_.push_back(Slice::TakeOwnership(std::move(cur_slice_), out_.pos)); +} + +} // namespace + +void ZstdCompressFn(std::vector* packets, int level) { + if (packets->empty()) { + return; + } + + ZstdPacketCompressor stream(level); + + for (const TracePacket& packet : *packets) { + stream.PushPacket(packet); + } + + TracePacket packet = stream.Finish(); + + packets->clear(); + packets->push_back(std::move(packet)); +} + +} // namespace perfetto + +#endif // PERFETTO_BUILDFLAG(PERFETTO_ZSTD) diff --git a/src/tracing/service/zstd_compressor.h b/src/tracing/service/zstd_compressor.h new file mode 100644 index 00000000000..88d7eb3aa1f --- /dev/null +++ b/src/tracing/service/zstd_compressor.h @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SRC_TRACING_SERVICE_ZSTD_COMPRESSOR_H_ +#define SRC_TRACING_SERVICE_ZSTD_COMPRESSOR_H_ + +#include + +#include "perfetto/base/build_config.h" +#include "perfetto/ext/tracing/core/trace_packet.h" + +namespace perfetto { + +#if PERFETTO_BUILDFLAG(PERFETTO_ZSTD) +// Compresses `packets` with zstd at the given compression `level`. +void ZstdCompressFn(std::vector*, int level); +#endif + +} // namespace perfetto + +#endif // SRC_TRACING_SERVICE_ZSTD_COMPRESSOR_H_ diff --git a/test/cts/Android.bp b/test/cts/Android.bp index 2b9fcc47867..faee307f614 100644 --- a/test/cts/Android.bp +++ b/test/cts/Android.bp @@ -40,6 +40,7 @@ cc_test { "liblog", "libpcre2", "libz", + "libzstd", ], test_suites: [ "cts", diff --git a/test/cts/art_module/Android.bp b/test/cts/art_module/Android.bp index edd32e69723..18961d5f776 100644 --- a/test/cts/art_module/Android.bp +++ b/test/cts/art_module/Android.bp @@ -32,6 +32,8 @@ cc_test { "libandroid", "liblog", "libpcre2", + "libz", + "libzstd", ], test_suites: [ "cts", diff --git a/test/cts/producer/jni/Android.bp b/test/cts/producer/jni/Android.bp index 88583bf508b..0eba1f742b4 100644 --- a/test/cts/producer/jni/Android.bp +++ b/test/cts/producer/jni/Android.bp @@ -22,6 +22,8 @@ cc_library_shared { "libandroid", "liblog", "libpcre2", + "libz", + "libzstd", ], compile_multilib: "both", stl: "libc++_static", diff --git a/test/cts/reporter/Android.bp b/test/cts/reporter/Android.bp index de17355d631..1fb8125545e 100644 --- a/test/cts/reporter/Android.bp +++ b/test/cts/reporter/Android.bp @@ -34,6 +34,8 @@ cc_test { "libandroid", "liblog", "libpcre2", + "libz", + "libzstd", ], test_suites: [ "cts", diff --git a/test/cts/test_apps/jni/Android.bp b/test/cts/test_apps/jni/Android.bp index b9bcd8c5f02..dd8e9f231a7 100644 --- a/test/cts/test_apps/jni/Android.bp +++ b/test/cts/test_apps/jni/Android.bp @@ -35,6 +35,8 @@ cc_library_shared { "libandroid", "liblog", "libpcre2", + "libz", + "libzstd", ], compile_multilib: "both", stl: "libc++_static", diff --git a/tools/gen_amalgamated b/tools/gen_amalgamated index 75eeb5b6374..007bd460a22 100755 --- a/tools/gen_amalgamated +++ b/tools/gen_amalgamated @@ -64,9 +64,10 @@ c_sdk_header_filter = r'^include/perfetto/public/' # line). gn_args = ' '.join([ 'enable_perfetto_ipc=true', - # Compile zlib and re2 into the amalgamation; the consumer opts in at + # Compile zlib, zstd and re2 into the amalgamation; the consumer opts in at # use-site (see perfetto_amalgamated_sdk). 'enable_perfetto_zlib=true', + 'enable_perfetto_zstd=true', 'enable_perfetto_re2=true', 'enable_perfetto_pcre2=false', 'is_debug=false', diff --git a/tools/gen_android_bp b/tools/gen_android_bp index 51b0d675fa4..ab29ae88385 100755 --- a/tools/gen_android_bp +++ b/tools/gen_android_bp @@ -362,10 +362,13 @@ additional_args = { # SDK is minimal, so std::regex is an acceptable tradeoff. # See b/498785154 for the binary-size regression that motivated this. # - # Same idea for zlib: the compressor would otherwise pull in libz, and - # consumers that don't link it fail with undefined deflate* symbols. - ('cflags', - {'-DPERFETTO_REGEX_FORCE_STD', '-DPERFETTO_FORCE_DISABLE_ZLIB'}), + # Same idea for the compressors: they would otherwise pull in + # libz/libzstd, and consumers that don't link them fail with undefined + # symbols. + ('cflags', { + '-DPERFETTO_REGEX_FORCE_STD', '-DPERFETTO_FORCE_DISABLE_ZLIB', + '-DPERFETTO_FORCE_DISABLE_ZSTD' + }), ], 'libperfetto_c': [ ('min_sdk_version', '30'), @@ -386,10 +389,13 @@ additional_args = { # SDK is minimal, so std::regex is an acceptable tradeoff. # See b/498785154 for the binary-size regression that motivated this. # - # Same idea for zlib: the compressor would otherwise pull in libz, and - # consumers that don't link it fail with undefined deflate* symbols. - ('cflags', - {'-DPERFETTO_REGEX_FORCE_STD', '-DPERFETTO_FORCE_DISABLE_ZLIB'}), + # Same idea for the compressors: they would otherwise pull in + # libz/libzstd, and consumers that don't link them fail with undefined + # symbols. + ('cflags', { + '-DPERFETTO_REGEX_FORCE_STD', '-DPERFETTO_FORCE_DISABLE_ZLIB', + '-DPERFETTO_FORCE_DISABLE_ZSTD' + }), ], 'perfetto_trace_protos': [ ('apex_available', { @@ -500,6 +506,20 @@ def enable_pcre2(module): module.shared_libs.add('libpcre2') +def enable_zstd(module): + # Modules built with PERFETTO_FORCE_DISABLE_ZSTD compile out the compressor + # and don't reference any zstd symbols, so skip the libzstd dep for them. + if '-DPERFETTO_FORCE_DISABLE_ZSTD' in module.cflags: + return + if module.type == 'cc_binary_host': + module.static_libs.add('libzstd') + elif module.host_supported: + module.android.shared_libs.add('libzstd') + module.host.static_libs.add('libzstd') + else: + module.shared_libs.add('libzstd') + + def enable_sqlite(module): if module.type == 'cc_binary_host': module.static_libs.add('libsqlite_static_noicu') @@ -593,6 +613,8 @@ builtin_deps = { enable_sqlite, '//gn:zlib': enable_zlib, + '//gn:zstd': + enable_zstd, '//gn:expat': enable_expat, '//gn:bionic_kernel_uapi_headers': diff --git a/tools/gen_bazel b/tools/gen_bazel index e67c8b509a9..235d00fd214 100755 --- a/tools/gen_bazel +++ b/tools/gen_bazel @@ -201,6 +201,7 @@ external_deps = { 'PERFETTO_CONFIG.deps.sqlite_ext_percentile' ], '//gn:zlib': ['PERFETTO_CONFIG.deps.zlib'], + '//gn:zstd': ['PERFETTO_CONFIG.deps.zstd'], '//gn:llvm_demangle': ['PERFETTO_CONFIG.deps.llvm_demangle'], '//src/trace_processor:demangle': ['PERFETTO_CONFIG.deps.demangle_wrapper'], gn_utils.GEN_VERSION_TARGET: ['PERFETTO_CONFIG.deps.version_header'],