Skip to content

Commit

Permalink
feat: implement API for clearing rules profiling data.
Browse files Browse the repository at this point in the history
Also renames `most_expensive_rules` to the more straightforward `slowest_rules`.
  • Loading branch information
plusvic committed Dec 10, 2024
1 parent 50180d8 commit d7487c8
Show file tree
Hide file tree
Showing 11 changed files with 125 additions and 63 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/coverage.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
uses: taiki-e/install-action@cargo-llvm-cov

- name: Generate code coverage
run: cargo llvm-cov --features=magic-module --workspace --lib --lcov --output-path lcov.info
run: cargo llvm-cov --features=magic-module,rules-profiling --workspace --lib --lcov --output-path lcov.info

- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
Expand Down
10 changes: 5 additions & 5 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,21 @@ jobs:
- build: msrv
os: ubuntu-latest
rust: 1.78.0
args: "--features=magic-module"
args: "--features=magic-module,rules-profiling"
rust_flags: "-Awarnings"
experimental: false

- build: stable
os: ubuntu-latest
rust: stable
args: "--features=magic-module"
args: "--features=magic-module,rules-profiling"
rust_flags: "-Awarnings"
experimental: false

- build: nightly
os: ubuntu-latest
rust: nightly
args: "--features=magic-module"
args: "--features=magic-module,rules-profiling"
# Link is currently failing with rust-lld (rust-lang/rust#124129)
# Disable rust-lld with -Zlinker-features=-lld
# See: https://github.com/dtolnay/linkme/commit/d13709bfd2c1278b4c8b6c846e2017b623923c0c
Expand All @@ -50,14 +50,14 @@ jobs:
- build: macos
os: macos-latest
rust: stable
args: ""
args: "--features=rules-profiling"
rust_flags: "-Awarnings"
experimental: false

- build: win-msvc
os: windows-latest
rust: stable
args: ""
args: "--features=rules-profiling"
rust_flags: "-Awarnings"
experimental: false

Expand Down
28 changes: 14 additions & 14 deletions capi/include/yara_x.h
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,7 @@ typedef void (*YRX_RULE_CALLBACK)(const struct YRX_RULE *rule,
typedef void (*YRX_IMPORT_CALLBACK)(const char *module_name,
void *user_data);

// Callback function passed to [`yrx_scanner_iter_most_expensive_rules`].
// Callback function passed to [`yrx_scanner_iter_slowest_rules`].
//
// The callback function receives pointers to the namespace and rule name,
// and two float numbers with the time spent by the rule matching patterns
Expand All @@ -240,11 +240,11 @@ typedef void (*YRX_IMPORT_CALLBACK)(const char *module_name,
// data owned by the user.
//
// Requires the `rules-profiling` feature.
typedef void (*YRX_MOST_EXPENSIVE_RULES_CALLBACK)(const char *namespace,
const char *rule,
double pattern_matching_time,
double condition_exec_time,
void *user_data);
typedef void (*YRX_SLOWEST_RULES_CALLBACK)(const char *namespace,
const char *rule,
double pattern_matching_time,
double condition_exec_time,
void *user_data);

// Returns the error message for the most recent function in this API
// invoked by the current thread.
Expand Down Expand Up @@ -708,15 +708,15 @@ enum YRX_RESULT yrx_scanner_set_global_float(struct YRX_SCANNER *scanner,
const char *ident,
double value);

// Iterates over the top N most expensive rules, calling the callback for
// each rule.
// Iterates over the slowest N rules, calling the callback for each rule.
//
// Requires the `rules-profiling` feature, otherwise the
// Requires the `rules-profiling` feature, otherwise returns
// [`YRX_RESULT::NOT_SUPPORTED`]
//
// See [`YRX_MOST_EXPENSIVE_RULES_CALLBACK`] for more details.
enum YRX_RESULT yrx_scanner_iter_most_expensive_rules(struct YRX_SCANNER *scanner,
size_t n,
YRX_MOST_EXPENSIVE_RULES_CALLBACK callback,
void *user_data);
// See [`YRX_SLOWEST_RULES_CALLBACK`] for more details.
enum YRX_RESULT yrx_scanner_iter_slowest_rules(struct YRX_SCANNER *scanner,
size_t n,
YRX_SLOWEST_RULES_CALLBACK callback,
void *user_data);

#endif /* YARA_X */
18 changes: 9 additions & 9 deletions capi/src/scanner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@ unsafe fn slice_from_ptr_and_len<'a>(
Some(data)
}

/// Callback function passed to [`yrx_scanner_iter_most_expensive_rules`].
/// Callback function passed to [`yrx_scanner_iter_slowest_rules`].
///
/// The callback function receives pointers to the namespace and rule name,
/// and two float numbers with the time spent by the rule matching patterns
Expand All @@ -314,26 +314,26 @@ unsafe fn slice_from_ptr_and_len<'a>(
/// data owned by the user.
///
/// Requires the `rules-profiling` feature.
pub type YRX_MOST_EXPENSIVE_RULES_CALLBACK = extern "C" fn(
pub type YRX_SLOWEST_RULES_CALLBACK = extern "C" fn(
namespace: *const c_char,
rule: *const c_char,
pattern_matching_time: f64,
condition_exec_time: f64,
user_data: *mut c_void,
) -> ();

/// Iterates over the top N most expensive rules, calling the callback for
/// each rule.
/// Iterates over the slowest N rules, calling the callback for each rule.
///
/// Requires the `rules-profiling` feature, otherwise the
/// Requires the `rules-profiling` feature, otherwise returns
/// [`YRX_RESULT::NOT_SUPPORTED`]
///
/// See [`YRX_MOST_EXPENSIVE_RULES_CALLBACK`] for more details.
/// See [`YRX_SLOWEST_RULES_CALLBACK`] for more details.
#[no_mangle]
#[allow(unused_variables)]
pub unsafe extern "C" fn yrx_scanner_iter_most_expensive_rules(
pub unsafe extern "C" fn yrx_scanner_iter_slowest_rules(
scanner: *mut YRX_SCANNER,
n: usize,
callback: YRX_MOST_EXPENSIVE_RULES_CALLBACK,
callback: YRX_SLOWEST_RULES_CALLBACK,
user_data: *mut c_void,
) -> YRX_RESULT {
#[cfg(not(feature = "rules-profiling"))]
Expand All @@ -346,7 +346,7 @@ pub unsafe extern "C" fn yrx_scanner_iter_most_expensive_rules(
None => return YRX_RESULT::INVALID_ARGUMENT,
};

for profiling_info in scanner.inner.most_expensive_rules(n) {
for profiling_info in scanner.inner.slowest_rules(n) {
let namespace = CString::new(profiling_info.namespace).unwrap();
let rule = CString::new(profiling_info.rule).unwrap();

Expand Down
24 changes: 13 additions & 11 deletions cli/src/commands/scan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -354,8 +354,7 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
};

#[cfg(feature = "rules-profiling")]
let most_expensive_rules: Mutex<Vec<ProfilingData>> =
Mutex::new(Vec::new());
let slowest_rules: Mutex<Vec<ProfilingData>> = Mutex::new(Vec::new());

w.walk(
state,
Expand Down Expand Up @@ -447,17 +446,20 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {
|scanner, _| {
#[cfg(feature = "rules-profiling")]
if profiling {
let mut mer = most_expensive_rules.lock().unwrap();
for er in scanner.most_expensive_rules(1000) {
let mut mer = slowest_rules.lock().unwrap();
for profiling_data in scanner.slowest_rules(1000) {
if let Some(r) = mer.iter_mut().find(|r| {
r.rule == er.rule && r.namespace == er.namespace
r.rule == profiling_data.rule
&& r.namespace == profiling_data.namespace
}) {
r.condition_exec_time += er.condition_exec_time;
r.pattern_matching_time += er.pattern_matching_time;
r.total_time +=
er.condition_exec_time + er.pattern_matching_time;
r.condition_exec_time +=
profiling_data.condition_exec_time;
r.pattern_matching_time +=
profiling_data.pattern_matching_time;
r.total_time += profiling_data.condition_exec_time
+ profiling_data.pattern_matching_time;
} else {
mer.push(er.into());
mer.push(profiling_data.into());
}
}
}
Expand Down Expand Up @@ -495,7 +497,7 @@ pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> {

#[cfg(feature = "rules-profiling")]
if profiling {
let mut mer = most_expensive_rules.lock().unwrap();
let mut mer = slowest_rules.lock().unwrap();

println!("\n«««««««««««« PROFILING INFORMATION »»»»»»»»»»»»");

Expand Down
34 changes: 17 additions & 17 deletions go/scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,17 @@ package yara_x
// return yrx_scanner_on_matching_rule(scanner, callback, (void*) user_data);
// }
//
// enum YRX_RESULT static inline _yrx_scanner_iter_most_expensive_rules(
// enum YRX_RESULT static inline _yrx_scanner_iter_slowest_rules(
// struct YRX_SCANNER *scanner,
// size_t n,
// YRX_MOST_EXPENSIVE_RULES_CALLBACK callback,
// uintptr_t most_expensive_rules_handle)
// YRX_SLOWEST_RULES_CALLBACK callback,
// uintptr_t slowest_rules_handle)
// {
// return yrx_scanner_iter_most_expensive_rules(scanner, n, callback, (void*) most_expensive_rules_handle);
// return yrx_scanner_iter_slowest_rules(scanner, n, callback, (void*) slowest_rules_handle);
// }
//
// extern void onMatchingRule(YRX_RULE*, uintptr_t);
// extern void mostExpensiveRulesCallback(char*, char*, double, double, uintptr_t);
// extern void slowestRulesCallback(char*, char*, double, double, uintptr_t);
import "C"

import (
Expand Down Expand Up @@ -261,10 +261,10 @@ type ProfilingInfo struct {
ConditionExecTime time.Duration
}

// This is the callback called by yrx_rule_iter_patterns.
// This is the callback called by yrx_scanner_iter_slowest_rules.
//
//export mostExpensiveRulesCallback
func mostExpensiveRulesCallback(
//export slowestRulesCallback
func slowestRulesCallback(
namespace *C.char,
rule *C.char,
patternMatchingTime C.double,
Expand All @@ -283,29 +283,29 @@ func mostExpensiveRulesCallback(
})
}

// MostExpensiveRules returns information about the slowest rules and how much
// SlowestRules returns information about the slowest rules and how much
// time they spent matching patterns and executing their conditions.
//
// In order to use this function the YARA-X C library must be built with
// support for rules profiling, which is done by enabling the `rules-profiling`
// feature. Otherwise, calling this function will cause a panic.
func (s *Scanner) MostExpensiveRules(n int) []ProfilingInfo {
func (s *Scanner) SlowestRules(n int) []ProfilingInfo {
profilingInfo := make([]ProfilingInfo, 0)
mostExpensiveRules := cgo.NewHandle(&profilingInfo)
defer mostExpensiveRules.Delete()
slowestRules := cgo.NewHandle(&profilingInfo)
defer slowestRules.Delete()

result := C._yrx_scanner_iter_most_expensive_rules(
result := C._yrx_scanner_iter_slowest_rules(
s.cScanner,
C.size_t(n),
C.YRX_MOST_EXPENSIVE_RULES_CALLBACK(C.mostExpensiveRulesCallback),
C.uintptr_t(mostExpensiveRules))
C.YRX_SLOWEST_RULES_CALLBACK(C.slowestRulesCallback),
C.uintptr_t(slowestRules))

if result == C.NOT_SUPPORTED {
panic("MostExpensiveRules requires that the YARA-X C library is built with the `rules-profiling` feature")
panic("SlowestRules requires that the YARA-X C library is built with the `rules-profiling` feature")
}

if result != C.SUCCESS {
panic("yrx_scanner_iter_most_expensive_rules failed")
panic("yrx_scanner_slowest_rules failed")
}

return profilingInfo
Expand Down
4 changes: 4 additions & 0 deletions lib/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ exclude = [
"src/modules/**/*.out"
]

[package.metadata.docs.rs]
all-features = true
rustdoc-args = ["--cfg", "docsrs"]

[features]
# Enables constant folding. When constant folding is enabled, expressions
# like `2+2+2` and `true or false`, whose value can be determined at compile
Expand Down
1 change: 1 addition & 0 deletions lib/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ assert_eq!(results.matching_rules().len(), 1);
*/

#![deny(missing_docs)]
#![cfg_attr(docsrs, feature(doc_auto_cfg))]

pub use compiler::compile;
pub use compiler::Compiler;
Expand Down
13 changes: 11 additions & 2 deletions lib/src/scanner/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,11 @@ pub(crate) struct ScanContext<'r> {

#[cfg(feature = "rules-profiling")]
impl<'r> ScanContext<'r> {
/// Returns the top N most expensive rules.
pub fn most_expensive_rules(&self, n: usize) -> Vec<ProfilingData> {
/// Returns the slowest N rules.
///
/// Profiling has an accumulative effect. When the scanner is used for
/// scanning multiple files the times add up.
pub fn slowest_rules(&self, n: usize) -> Vec<ProfilingData> {
debug_assert_eq!(
self.compiled_rules.num_rules(),
self.time_spent_in_rule.len()
Expand Down Expand Up @@ -190,6 +193,12 @@ impl<'r> ScanContext<'r> {
result.truncate(n);
result
}

/// Clears profiling information.
pub fn clear_profiling_data(&mut self) {
self.time_spent_in_rule.fill(0);
self.time_spent_in_pattern.clear();
}
}

impl ScanContext<'_> {
Expand Down
23 changes: 19 additions & 4 deletions lib/src/scanner/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -527,10 +527,25 @@ impl<'r> Scanner<'r> {
)
}

/// Returns the top N most expensive rules.
/// Returns profiling data for the slowest N rules.
///
/// The profiling data reflects the cumulative execution time of each rule
/// across all scanned files. This information is useful for identifying
/// performance bottlenecks. To reset the profiling data and start fresh
/// for subsequent scans, use [`Scanner::clear_profiling_data`].
#[cfg(feature = "rules-profiling")]
pub fn slowest_rules(&self, n: usize) -> Vec<ProfilingData> {
self.wasm_store.data().slowest_rules(n)
}

/// Clears all accumulated profiling data.
///
/// This method resets the profiling data collected during rule execution
/// across scanned files. Use this to start a new profiling session, ensuring
/// the results reflect only the data gathered after this method is called.
#[cfg(feature = "rules-profiling")]
pub fn most_expensive_rules(&self, n: usize) -> Vec<ProfilingData> {
self.wasm_store.data().most_expensive_rules(n)
pub fn clear_profiling_data(&mut self) {
self.wasm_store.data_mut().clear_profiling_data()
}
}

Expand Down Expand Up @@ -757,7 +772,7 @@ impl<'r> Scanner<'r> {

#[cfg(all(feature = "rules-profiling", feature = "logging"))]
{
let most_expensive_rules = self.most_expensive_rules(10);
let most_expensive_rules = self.slowest_rules(10);
if !most_expensive_rules.is_empty() {
log::info!("Most expensive rules:");
for profiling_data in most_expensive_rules {
Expand Down
Loading

0 comments on commit d7487c8

Please sign in to comment.