Skip to content

Commit

Permalink
Fail on issues with the custom mutator for FuzzTest.
Browse files Browse the repository at this point in the history 
With this, we no longer need to perform the "domain setup check" as part of
executing an input. The check happens implicitly during mutation. If the
mutation fails, either due to an ineffective domain or other reasons, Centipede
itself fails.

PiperOrigin-RevId: 726191912
  • Loading branch information
@fniksic @copybara-github
fniksic authored and copybara-github committed Feb 20, 2025
1 parent 1d61720 commit 87db27a
Show file tree
Hide file tree
Showing 8 changed files with 244 additions and 35 deletions.
1 change: 1 addition & 0 deletions centipede/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1828,6 +1828,7 @@ cc_test(
"@com_google_fuzztest//centipede/testing:abort_fuzz_target",
"@com_google_fuzztest//centipede/testing:expensive_startup_fuzz_target",
"@com_google_fuzztest//centipede/testing:fuzz_target_with_config",
"@com_google_fuzztest//centipede/testing:fuzz_target_with_custom_mutator",
"@com_google_fuzztest//centipede/testing:seeded_fuzz_target",
"@com_google_fuzztest//centipede/testing:test_fuzz_target",
"@com_google_fuzztest//centipede/testing:test_input_filter",
Expand Down
30 changes: 22 additions & 8 deletions centipede/centipede_default_callbacks.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include <cstddef>
#include <string>
#include <string_view>
#include <type_traits>
#include <vector>

#include "absl/log/check.h"
Expand Down Expand Up @@ -77,31 +78,44 @@ void CentipedeDefaultCallbacks::Mutate(
std::vector<ByteArray> &mutants) {
mutants.resize(num_mutants);
if (num_mutants == 0) return;
if (env_.mutator_type == MutatorType::kBuiltIn) {
CentipedeCallbacks::Mutate(inputs, num_mutants, mutants);
return;
}
// Try to use the custom mutator if it hasn't been disabled.
if (custom_mutator_is_usable_.value_or(true)) {
if (MutateViaExternalBinary(env_.binary, inputs, mutants)) {
if (!custom_mutator_is_usable_.has_value()) {
LOG(INFO) << "Custom mutator detected: will use it";
LOG_IF(INFO, env_.mutator_type == MutatorType::kAuto)
<< "Custom mutator detected: will use it";
custom_mutator_is_usable_ = true;
}
if (!mutants.empty()) return;
LOG_FIRST_N(WARNING, 5)
<< "Custom mutator returned no mutants: falling back to internal "
"default mutator";
LOG_IF(FATAL, env_.mutator_type == MutatorType::kCustom)
<< "Custom mutator returned no mutants.";
LOG_FIRST_N(WARNING, 5) << "Custom mutator returned no mutants: falling "
"back to the built-in mutator";
} else if (ShouldStop()) {
LOG(WARNING) << "Custom mutator failed, but ignored since the stop "
"condition it met. Possibly what triggered the stop "
"condition also interrupted the mutator.";
return;
} else {
LOG(WARNING) << "Custom mutator undetected or misbehaving:";
CHECK(!custom_mutator_is_usable_.has_value())
<< "Custom mutator is unreliable, aborting";
LOG(WARNING) << "Falling back to internal default mutator";
LOG_IF(FATAL, env_.mutator_type == MutatorType::kCustom ||
custom_mutator_is_usable_.has_value())
<< "Custom mutator failed, and it was expected to succeed.";
// TODO(b/397945452): We want to distinguish between an undetected and a
// misbehaving custom mutator: fall back in the first case, but crash in
// the second case.
LOG(WARNING) << "Custom mutator undetected or misbehaving. Falling back "
"to the built-in mutator.";
custom_mutator_is_usable_ = false;
}
}
// Fallback of the internal mutator.
CHECK(env_.mutator_type == MutatorType::kAuto)
<< "Internal error: unexpected mutator type "
<< std::underlying_type_t<MutatorType>(env_.mutator_type);
CentipedeCallbacks::Mutate(inputs, num_mutants, mutants);
}

Expand Down
148 changes: 129 additions & 19 deletions centipede/centipede_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
#include "./common/test_util.h"

namespace centipede {

namespace {

using ::testing::HasSubstr;
Expand Down Expand Up @@ -137,8 +136,6 @@ class MockFactory : public CentipedeCallbacksFactory {
CentipedeCallbacks &cb_;
};

} // namespace

TEST(Centipede, MockTest) {
TempCorpusDir tmp_dir{test_info_->name()};
Environment env;
Expand Down Expand Up @@ -365,6 +362,29 @@ class MutateCallbacks : public CentipedeCallbacks {
using CentipedeCallbacks::MutateViaExternalBinary;
};

// Maintains `TemporaryLocalDirPath()` during the lifetime.
//
// This is necessary in tests with Centipede callbacks where some functions
// actually manipulate test binaries (e.g., they rely on real implementations of
// `Execute()`, `Mutate()`, etc.), and the callbacks are used directly instead
// of being passed to `CentipedeMain()` (which would normally set up
// `TemporaryLocalDirPath()`).
//
// Should be instantiated before calling binary-manipulating functions on the
// callbacks instance.
class TemporaryLocalDirForCentipedeCallbacks {
public:
TemporaryLocalDirForCentipedeCallbacks() {
std::filesystem::path tmp_dir = TemporaryLocalDirPath();
std::filesystem::remove_all(tmp_dir);
std::filesystem::create_directory(tmp_dir);
}

~TemporaryLocalDirForCentipedeCallbacks() {
std::filesystem::remove_all(TemporaryLocalDirPath());
}
};

TEST(Centipede, MutateViaExternalBinary) {
// This binary contains a test-friendly custom mutator.
const std::string binary_with_custom_mutator =
Expand Down Expand Up @@ -408,6 +428,7 @@ TEST(Centipede, MutateViaExternalBinary) {
{
Environment env;
MutateCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

// Expect to fail on the binary w/o a custom mutator.
mutants.resize(1);
Expand All @@ -431,6 +452,8 @@ TEST(Centipede, MutateViaExternalBinary) {
Environment env_no_crossover;
env_no_crossover.crossover_level = 0;
MutateCallbacks callbacks_no_crossover(env_no_crossover);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

mutants.resize(10000);
EXPECT_TRUE(callbacks_no_crossover.MutateViaExternalBinary(
binary_with_custom_mutator, GetMutationInputRefsFromDataInputs(inputs),
Expand Down Expand Up @@ -628,8 +651,6 @@ TEST(Centipede, FunctionFilter) {
}
}

namespace {

struct Crash {
std::string binary;
unsigned char input = 0;
Expand Down Expand Up @@ -701,8 +722,6 @@ MATCHER_P(HasFilesWithContents, expected_files_and_contents, "") {
result_listener);
}

} // namespace

// Tests --extra_binaries.
// Executes one main binary (--binary) and 3 extra ones (--extra_binaries).
// Expects the main binary and two extra ones to generate one crash each.
Expand Down Expand Up @@ -746,8 +765,6 @@ TEST(Centipede, ExtraBinaries) {
FileAndContents{Hash({50}), "b3-crash"})));
}

namespace {

// A mock for UndetectedCrashingInput test.
class UndetectedCrashingInputMock : public CentipedeCallbacks {
public:
Expand Down Expand Up @@ -815,8 +832,6 @@ class UndetectedCrashingInputMock : public CentipedeCallbacks {
bool first_pass_ = true;
};

} // namespace

// Test for preserving a crashing batch when 1-by-1 exec fails to reproduce.
// Executes one main binary (--binary).
// Expects the binary to crash once and 1-by-1 reproduction to fail.
Expand Down Expand Up @@ -883,6 +898,8 @@ TEST(Centipede, GetsSeedInputs) {
env.binary =
GetDataDependencyFilepath("centipede/testing/seeded_fuzz_target");
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

std::vector<ByteArray> seeds;
EXPECT_EQ(callbacks.GetSeeds(10, seeds), 10);
EXPECT_THAT(seeds, testing::ContainerEq(std::vector<ByteArray>{
Expand All @@ -900,6 +917,8 @@ TEST(Centipede, GetsSerializedTargetConfig) {
env.binary =
GetDataDependencyFilepath("centipede/testing/fuzz_target_with_config");
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const auto serialized_config = callbacks.GetSerializedTargetConfig();
ASSERT_TRUE(serialized_config.ok());
EXPECT_EQ(*serialized_config, "fake serialized config");
Expand All @@ -912,6 +931,8 @@ TEST(Centipede, GetSerializedTargetConfigProducesFailure) {
.c_str(),
" --simulate_failure");
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const auto serialized_config = callbacks.GetSerializedTargetConfig();
EXPECT_FALSE(serialized_config.ok());
}
Expand All @@ -921,6 +942,8 @@ TEST(Centipede, CleansUpMetadataAfterStartup) {
env.binary = GetDataDependencyFilepath(
"centipede/testing/expensive_startup_fuzz_target");
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

BatchResult batch_result;
const std::vector<ByteArray> inputs = {{0}};
ASSERT_TRUE(callbacks.Execute(env.binary, inputs, batch_result));
Expand All @@ -934,8 +957,6 @@ TEST(Centipede, CleansUpMetadataAfterStartup) {
EXPECT_FALSE(found_startup_cmp_entry);
}

namespace {

class FakeCentipedeCallbacksForThreadChecking : public CentipedeCallbacks {
public:
FakeCentipedeCallbacksForThreadChecking(const Environment &env,
Expand All @@ -962,8 +983,6 @@ class FakeCentipedeCallbacksForThreadChecking : public CentipedeCallbacks {
bool thread_check_passed_ = true;
};

} // namespace

TEST(Centipede, RunsExecuteCallbackInTheCurrentThreadWhenFuzzingWithOneThread) {
TempDir temp_dir{test_info_->name()};
Environment env;
Expand All @@ -985,6 +1004,8 @@ TEST(Centipede, DetectsStackOverflow) {
env.binary = GetDataDependencyFilepath("centipede/testing/test_fuzz_target");
env.stack_limit_kb = 64;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

BatchResult batch_result;
const std::vector<ByteArray> inputs = {ByteArray{'s', 't', 'k'}};

Expand All @@ -993,8 +1014,6 @@ TEST(Centipede, DetectsStackOverflow) {
EXPECT_EQ(batch_result.failure_description(), "stack-limit-exceeded");
}

namespace {

class SetupFailureCallbacks : public CentipedeCallbacks {
public:
using CentipedeCallbacks::CentipedeCallbacks;
Expand All @@ -1013,8 +1032,6 @@ class SetupFailureCallbacks : public CentipedeCallbacks {
}
};

} // namespace

TEST(Centipede, AbortsOnSetupFailure) {
TempDir temp_dir{test_info_->name()};
Environment env;
Expand All @@ -1028,4 +1045,97 @@ TEST(Centipede, AbortsOnSetupFailure) {
"Terminating Centipede due to setup failure in the test.");
}

TEST(Centipede, UsesCustomMutatorWithCustomMutatorType) {
Environment env;
env.binary = GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator");
env.mutator_type = MutatorType::kCustom;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The custom mutator just returns the original inputs as mutants.
EXPECT_EQ(inputs, mutants);
}

TEST(Centipede, FailsOnMisbehavingMutatorWithCustomMutatorType) {
Environment env;
env.binary =
absl::StrCat(GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator")
.c_str(),
" --simulate_failure");
env.mutator_type = MutatorType::kCustom;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
EXPECT_DEATH(callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs),
inputs.size(), mutants),
"Custom mutator failed");
}

TEST(Centipede, DetectsCustomMutatorWithAutoMutatorType) {
Environment env;
env.binary = GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator");
env.mutator_type = MutatorType::kAuto;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The custom mutator just returns the original inputs as mutants.
EXPECT_EQ(inputs, mutants);
}

TEST(Centipede, FallsBackToBuiltInMutatorWithAutoMutatorType) {
Environment env;
env.binary =
absl::StrCat(GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator")
.c_str(),
" --simulate_failure");
env.mutator_type = MutatorType::kAuto;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The built-in mutator performs non-trivial mutations.
EXPECT_EQ(inputs.size(), mutants.size());
EXPECT_NE(inputs, mutants);
}

TEST(Centipede, UsesBuiltInMutatorWithBuiltInMutatorType) {
Environment env;
env.binary = GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator");
env.mutator_type = MutatorType::kBuiltIn;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The built-in mutator performs non-trivial mutations, while the custom
// mutator just returns the original inputs as mutants.
EXPECT_EQ(inputs.size(), mutants.size());
EXPECT_NE(inputs, mutants);
}

} // namespace
} // namespace centipede
8 changes: 8 additions & 0 deletions centipede/environment.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,14 @@ void Environment::UpdateWithTargetConfig(
LOG(INFO) << "Overriding the default max_num_crash_reports to "
<< max_num_crash_reports << " for FuzzTest.";
}

// Use the custom mutator when running with FuzzTest to surface any errors
// with the mutation (e.g., ineffective filter domain).
if (mutator_type == Default().mutator_type) {
mutator_type = MutatorType::kCustom;
LOG(INFO) << "Switching to the custom mutator for FuzzTest.";
}

if (config.jobs != 0) {
CHECK(j == Default().j || j == config.jobs)
<< "Value for --j is inconsistent with the value for jobs in the "
Expand Down
11 changes: 11 additions & 0 deletions centipede/environment.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,16 @@

namespace centipede {

enum class MutatorType {
// Detect whether the binary supports custom mutator and use it if it does,
// otherwise fall back to the built-in mutator.
kAuto,
// Use the built-in ByteArray mutator.
kBuiltIn,
// Use the custom mutator provided by the binary.
kCustom
};

// Fuzzing environment controlling the behavior of
// CentipedeMain(). Centipede binaries are creating Environment instances using
// the flags defined in environment_flags.cc, while other users can use
Expand All @@ -52,6 +62,7 @@ struct Environment {
size_t batch_size = 1000;
size_t mutate_batch_size = 2;
bool use_legacy_default_mutator = false;
MutatorType mutator_type = MutatorType::kAuto;
size_t load_other_shard_frequency = 10;
bool serialize_shard_loads = false;
size_t seed = 0;
Expand Down
Loading

0 comments on commit 87db27a

Please sign in to comment.