Skip to content

Commit

Permalink
Add a flag that forces failure on issues with the custom mutator.
Browse files Browse the repository at this point in the history 
With this flag, we no longer need to perform the "domain setup check" as part
of executing an input. The check happens implicitly during mutation. If the
mutation fails, either due to an ineffective domain or other reasons, Centipede
itself fails.

PiperOrigin-RevId: 726191912
  • Loading branch information
@fniksic @copybara-github
fniksic authored and copybara-github committed Feb 19, 2025
1 parent 1d61720 commit e88e49e
Show file tree
Hide file tree
Showing 9 changed files with 286 additions and 35 deletions.
1 change: 1 addition & 0 deletions centipede/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1828,6 +1828,7 @@ cc_test(
"@com_google_fuzztest//centipede/testing:abort_fuzz_target",
"@com_google_fuzztest//centipede/testing:expensive_startup_fuzz_target",
"@com_google_fuzztest//centipede/testing:fuzz_target_with_config",
"@com_google_fuzztest//centipede/testing:fuzz_target_with_custom_mutator",
"@com_google_fuzztest//centipede/testing:seeded_fuzz_target",
"@com_google_fuzztest//centipede/testing:test_fuzz_target",
"@com_google_fuzztest//centipede/testing:test_input_filter",
Expand Down
26 changes: 18 additions & 8 deletions centipede/centipede_default_callbacks.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,31 +77,41 @@ void CentipedeDefaultCallbacks::Mutate(
std::vector<ByteArray> &mutants) {
mutants.resize(num_mutants);
if (num_mutants == 0) return;
if (env_.mutator_type == MutatorType::kBuiltIn) {
CentipedeCallbacks::Mutate(inputs, num_mutants, mutants);
return;
}
// Try to use the custom mutator if it hasn't been disabled.
if (custom_mutator_is_usable_.value_or(true)) {
if (MutateViaExternalBinary(env_.binary, inputs, mutants)) {
if (!custom_mutator_is_usable_.has_value()) {
LOG(INFO) << "Custom mutator detected: will use it";
LOG_IF(INFO, env_.mutator_type == MutatorType::kAuto)
<< "Custom mutator detected: will use it";
custom_mutator_is_usable_ = true;
}
if (!mutants.empty()) return;
LOG_FIRST_N(WARNING, 5)
<< "Custom mutator returned no mutants: falling back to internal "
"default mutator";
LOG_IF(FATAL, env_.mutator_type == MutatorType::kCustom)
<< "Custom mutator returned no mutants.";
LOG_FIRST_N(WARNING, 5) << "Custom mutator returned no mutants: falling "
"back to the built-in mutator";
} else if (ShouldStop()) {
LOG(WARNING) << "Custom mutator failed, but ignored since the stop "
"condition it met. Possibly what triggered the stop "
"condition also interrupted the mutator.";
return;
} else {
LOG(WARNING) << "Custom mutator undetected or misbehaving:";
CHECK(!custom_mutator_is_usable_.has_value())
<< "Custom mutator is unreliable, aborting";
LOG(WARNING) << "Falling back to internal default mutator";
LOG_IF(FATAL, env_.mutator_type == MutatorType::kCustom ||
custom_mutator_is_usable_.has_value())
<< "Custom mutator failed, and it was expected to succeed.";
LOG(WARNING) << "Custom mutator undetected or misbehaving. Falling back "
"to the built-in mutator.";
custom_mutator_is_usable_ = false;
}
}
// Fallback of the internal mutator.
CHECK(env_.mutator_type == MutatorType::kAuto)
<< "Internal error: unexpected mutator type "
<< AbslUnparseFlag(env_.mutator_type);
CentipedeCallbacks::Mutate(inputs, num_mutants, mutants);
}

Expand Down
148 changes: 129 additions & 19 deletions centipede/centipede_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
#include "./common/test_util.h"

namespace centipede {

namespace {

using ::testing::HasSubstr;
Expand Down Expand Up @@ -137,8 +136,6 @@ class MockFactory : public CentipedeCallbacksFactory {
CentipedeCallbacks &cb_;
};

} // namespace

TEST(Centipede, MockTest) {
TempCorpusDir tmp_dir{test_info_->name()};
Environment env;
Expand Down Expand Up @@ -365,6 +362,29 @@ class MutateCallbacks : public CentipedeCallbacks {
using CentipedeCallbacks::MutateViaExternalBinary;
};

// Maintains `TemporaryLocalDirPath()` during the lifetime.
//
// This is necessary in tests with Centipede callbacks where some functions
// actually manipulate test binaries (e.g., they rely on real implementations of
// `Execute()`, `Mutate()`, etc.), and the callbacks are used directly instead
// of being passed to `CentipedeMain()` (which would normally set up
// `TemporaryLocalDirPath()`).
//
// Should be instantiated before calling binary-manipulating functions on the
// callbacks instance.
class TemporaryLocalDirForCentipedeCallbacks {
public:
TemporaryLocalDirForCentipedeCallbacks() {
std::filesystem::path tmp_dir = TemporaryLocalDirPath();
std::filesystem::remove_all(tmp_dir);
std::filesystem::create_directory(tmp_dir);
}

~TemporaryLocalDirForCentipedeCallbacks() {
std::filesystem::remove_all(TemporaryLocalDirPath());
}
};

TEST(Centipede, MutateViaExternalBinary) {
// This binary contains a test-friendly custom mutator.
const std::string binary_with_custom_mutator =
Expand Down Expand Up @@ -408,6 +428,7 @@ TEST(Centipede, MutateViaExternalBinary) {
{
Environment env;
MutateCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

// Expect to fail on the binary w/o a custom mutator.
mutants.resize(1);
Expand All @@ -431,6 +452,8 @@ TEST(Centipede, MutateViaExternalBinary) {
Environment env_no_crossover;
env_no_crossover.crossover_level = 0;
MutateCallbacks callbacks_no_crossover(env_no_crossover);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

mutants.resize(10000);
EXPECT_TRUE(callbacks_no_crossover.MutateViaExternalBinary(
binary_with_custom_mutator, GetMutationInputRefsFromDataInputs(inputs),
Expand Down Expand Up @@ -628,8 +651,6 @@ TEST(Centipede, FunctionFilter) {
}
}

namespace {

struct Crash {
std::string binary;
unsigned char input = 0;
Expand Down Expand Up @@ -701,8 +722,6 @@ MATCHER_P(HasFilesWithContents, expected_files_and_contents, "") {
result_listener);
}

} // namespace

// Tests --extra_binaries.
// Executes one main binary (--binary) and 3 extra ones (--extra_binaries).
// Expects the main binary and two extra ones to generate one crash each.
Expand Down Expand Up @@ -746,8 +765,6 @@ TEST(Centipede, ExtraBinaries) {
FileAndContents{Hash({50}), "b3-crash"})));
}

namespace {

// A mock for UndetectedCrashingInput test.
class UndetectedCrashingInputMock : public CentipedeCallbacks {
public:
Expand Down Expand Up @@ -815,8 +832,6 @@ class UndetectedCrashingInputMock : public CentipedeCallbacks {
bool first_pass_ = true;
};

} // namespace

// Test for preserving a crashing batch when 1-by-1 exec fails to reproduce.
// Executes one main binary (--binary).
// Expects the binary to crash once and 1-by-1 reproduction to fail.
Expand Down Expand Up @@ -883,6 +898,8 @@ TEST(Centipede, GetsSeedInputs) {
env.binary =
GetDataDependencyFilepath("centipede/testing/seeded_fuzz_target");
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

std::vector<ByteArray> seeds;
EXPECT_EQ(callbacks.GetSeeds(10, seeds), 10);
EXPECT_THAT(seeds, testing::ContainerEq(std::vector<ByteArray>{
Expand All @@ -900,6 +917,8 @@ TEST(Centipede, GetsSerializedTargetConfig) {
env.binary =
GetDataDependencyFilepath("centipede/testing/fuzz_target_with_config");
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const auto serialized_config = callbacks.GetSerializedTargetConfig();
ASSERT_TRUE(serialized_config.ok());
EXPECT_EQ(*serialized_config, "fake serialized config");
Expand All @@ -912,6 +931,8 @@ TEST(Centipede, GetSerializedTargetConfigProducesFailure) {
.c_str(),
" --simulate_failure");
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const auto serialized_config = callbacks.GetSerializedTargetConfig();
EXPECT_FALSE(serialized_config.ok());
}
Expand All @@ -921,6 +942,8 @@ TEST(Centipede, CleansUpMetadataAfterStartup) {
env.binary = GetDataDependencyFilepath(
"centipede/testing/expensive_startup_fuzz_target");
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

BatchResult batch_result;
const std::vector<ByteArray> inputs = {{0}};
ASSERT_TRUE(callbacks.Execute(env.binary, inputs, batch_result));
Expand All @@ -934,8 +957,6 @@ TEST(Centipede, CleansUpMetadataAfterStartup) {
EXPECT_FALSE(found_startup_cmp_entry);
}

namespace {

class FakeCentipedeCallbacksForThreadChecking : public CentipedeCallbacks {
public:
FakeCentipedeCallbacksForThreadChecking(const Environment &env,
Expand All @@ -962,8 +983,6 @@ class FakeCentipedeCallbacksForThreadChecking : public CentipedeCallbacks {
bool thread_check_passed_ = true;
};

} // namespace

TEST(Centipede, RunsExecuteCallbackInTheCurrentThreadWhenFuzzingWithOneThread) {
TempDir temp_dir{test_info_->name()};
Environment env;
Expand All @@ -985,6 +1004,8 @@ TEST(Centipede, DetectsStackOverflow) {
env.binary = GetDataDependencyFilepath("centipede/testing/test_fuzz_target");
env.stack_limit_kb = 64;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

BatchResult batch_result;
const std::vector<ByteArray> inputs = {ByteArray{'s', 't', 'k'}};

Expand All @@ -993,8 +1014,6 @@ TEST(Centipede, DetectsStackOverflow) {
EXPECT_EQ(batch_result.failure_description(), "stack-limit-exceeded");
}

namespace {

class SetupFailureCallbacks : public CentipedeCallbacks {
public:
using CentipedeCallbacks::CentipedeCallbacks;
Expand All @@ -1013,8 +1032,6 @@ class SetupFailureCallbacks : public CentipedeCallbacks {
}
};

} // namespace

TEST(Centipede, AbortsOnSetupFailure) {
TempDir temp_dir{test_info_->name()};
Environment env;
Expand All @@ -1028,4 +1045,97 @@ TEST(Centipede, AbortsOnSetupFailure) {
"Terminating Centipede due to setup failure in the test.");
}

TEST(Centipede, UsesCustomMutatorWithCustomMutatorFlag) {
Environment env;
env.binary = GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator");
env.mutator_type = MutatorType::kCustom;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The custom mutator just returns the original inputs as mutants.
EXPECT_EQ(inputs, mutants);
}

TEST(Centipede, FailsOnMisbehavingMutatorWithCustomMutatorFlag) {
Environment env;
env.binary =
absl::StrCat(GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator")
.c_str(),
" --simulate_failure");
env.mutator_type = MutatorType::kCustom;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
EXPECT_DEATH(callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs),
inputs.size(), mutants),
"Custom mutator failed");
}

TEST(Centipede, DetectsCustomMutatorWithAutoMutatorFlag) {
Environment env;
env.binary = GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator");
env.mutator_type = MutatorType::kAuto;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The custom mutator just returns the original inputs as mutants.
EXPECT_EQ(inputs, mutants);
}

TEST(Centipede, FallsBackToBuiltInMutatorWithAutoMutatorFlag) {
Environment env;
env.binary =
absl::StrCat(GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator")
.c_str(),
" --simulate_failure");
env.mutator_type = MutatorType::kAuto;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The built-in mutator performs non-trivial mutations.
EXPECT_EQ(inputs.size(), mutants.size());
EXPECT_NE(inputs, mutants);
}

TEST(Centipede, UsesBuiltInMutatorWithBuiltInMutatorFlag) {
Environment env;
env.binary = GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator");
env.mutator_type = MutatorType::kBuiltIn;
CentipedeDefaultCallbacks callbacks(env);
[[maybe_unused]] TemporaryLocalDirForCentipedeCallbacks tmp_dir;

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The built-in mutator performs non-trivial mutations, while the custom
// mutator just returns the original inputs as mutants.
EXPECT_EQ(inputs.size(), mutants.size());
EXPECT_NE(inputs, mutants);
}

} // namespace
} // namespace centipede
38 changes: 38 additions & 0 deletions centipede/environment.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,35 @@ size_t ComputeTimeoutPerBatch(size_t timeout_per_input, size_t batch_size) {

} // namespace

bool AbslParseFlag(std::string_view text, MutatorType *type,
std::string *error) {
if (text == "auto") {
*type = MutatorType::kAuto;
return true;
}
if (text == "built-in") {
*type = MutatorType::kBuiltIn;
return true;
}
if (text == "custom") {
*type = MutatorType::kCustom;
return true;
}
*error = "unknown value for mutator type";
return false;
}

std::string AbslUnparseFlag(MutatorType type) {
switch (type) {
case MutatorType::kAuto:
return "auto";
case MutatorType::kBuiltIn:
return "built-in";
case MutatorType::kCustom:
return "custom";
}
}

const Environment &Environment::Default() {
static absl::NoDestructor<Environment> default_env;
return *default_env;
Expand Down Expand Up @@ -250,6 +279,15 @@ void Environment::UpdateWithTargetConfig(
LOG(INFO) << "Overriding the default max_num_crash_reports to "
<< max_num_crash_reports << " for FuzzTest.";
}

// Use the custom mutator when running with FuzzTest to surface any errors
// with the mutation (e.g., ineffective filter domain).
if (mutator_type == Default().mutator_type) {
mutator_type = MutatorType::kCustom;
LOG(INFO) << "Overriding the default mutator_type to '"
<< AbslUnparseFlag(mutator_type) << "' for FuzzTest.";
}

if (config.jobs != 0) {
CHECK(j == Default().j || j == config.jobs)
<< "Value for --j is inconsistent with the value for jobs in the "
Expand Down
Loading

0 comments on commit e88e49e

Please sign in to comment.