Skip to content

Commit

Permalink
Fail on issues with the custom mutator for FuzzTest.
Browse files Browse the repository at this point in the history 
With this, we no longer need to perform the "domain setup check" as part of
executing an input. The check happens implicitly during mutation. If the
mutation fails, either due to an ineffective domain or other reasons, Centipede
itself fails.

PiperOrigin-RevId: 726191912
  • Loading branch information
@fniksic @copybara-github
fniksic authored and copybara-github committed Feb 20, 2025
1 parent 409e519 commit 9685b83
Show file tree
Hide file tree
Showing 9 changed files with 253 additions and 41 deletions.
1 change: 1 addition & 0 deletions centipede/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1829,6 +1829,7 @@ cc_test(
"@com_google_fuzztest//centipede/testing:abort_fuzz_target",
"@com_google_fuzztest//centipede/testing:expensive_startup_fuzz_target",
"@com_google_fuzztest//centipede/testing:fuzz_target_with_config",
"@com_google_fuzztest//centipede/testing:fuzz_target_with_custom_mutator",
"@com_google_fuzztest//centipede/testing:seeded_fuzz_target",
"@com_google_fuzztest//centipede/testing:test_fuzz_target",
"@com_google_fuzztest//centipede/testing:test_input_filter",
Expand Down
30 changes: 22 additions & 8 deletions centipede/centipede_default_callbacks.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include <cstddef>
#include <string>
#include <string_view>
#include <type_traits>
#include <vector>

#include "absl/log/check.h"
Expand Down Expand Up @@ -77,31 +78,44 @@ void CentipedeDefaultCallbacks::Mutate(
std::vector<ByteArray> &mutants) {
mutants.resize(num_mutants);
if (num_mutants == 0) return;
if (env_.mutator_type == MutatorType::kBuiltIn) {
CentipedeCallbacks::Mutate(inputs, num_mutants, mutants);
return;
}
// Try to use the custom mutator if it hasn't been disabled.
if (custom_mutator_is_usable_.value_or(true)) {
if (MutateViaExternalBinary(env_.binary, inputs, mutants)) {
if (!custom_mutator_is_usable_.has_value()) {
LOG(INFO) << "Custom mutator detected: will use it";
LOG_IF(INFO, env_.mutator_type == MutatorType::kAuto)
<< "Custom mutator detected: will use it";
custom_mutator_is_usable_ = true;
}
if (!mutants.empty()) return;
LOG_FIRST_N(WARNING, 5)
<< "Custom mutator returned no mutants: falling back to internal "
"default mutator";
LOG_IF(FATAL, env_.mutator_type == MutatorType::kCustom)
<< "Custom mutator returned no mutants.";
LOG_FIRST_N(WARNING, 5) << "Custom mutator returned no mutants: falling "
"back to the built-in mutator";
} else if (ShouldStop()) {
LOG(WARNING) << "Custom mutator failed, but ignored since the stop "
"condition it met. Possibly what triggered the stop "
"condition also interrupted the mutator.";
return;
} else {
LOG(WARNING) << "Custom mutator undetected or misbehaving:";
CHECK(!custom_mutator_is_usable_.has_value())
<< "Custom mutator is unreliable, aborting";
LOG(WARNING) << "Falling back to internal default mutator";
LOG_IF(FATAL, env_.mutator_type == MutatorType::kCustom ||
custom_mutator_is_usable_.has_value())
<< "Custom mutator failed, and it was expected to succeed.";
// TODO(b/397945452): We want to distinguish between an undetected and a
// misbehaving custom mutator: fall back in the first case, but crash in
// the second case.
LOG(WARNING) << "Custom mutator undetected or misbehaving. Falling back "
"to the built-in mutator.";
custom_mutator_is_usable_ = false;
}
}
// Fallback of the internal mutator.
CHECK(env_.mutator_type == MutatorType::kAuto)
<< "Internal error: unexpected mutator type "
<< std::underlying_type_t<MutatorType>(env_.mutator_type);
CentipedeCallbacks::Mutate(inputs, num_mutants, mutants);
}

Expand Down
151 changes: 126 additions & 25 deletions centipede/centipede_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
#include "./common/test_util.h"

namespace centipede {

namespace {

using ::testing::HasSubstr;
Expand Down Expand Up @@ -137,8 +136,6 @@ class MockFactory : public CentipedeCallbacksFactory {
CentipedeCallbacks &cb_;
};

} // namespace

TEST(Centipede, MockTest) {
TempCorpusDir tmp_dir{test_info_->name()};
Environment env;
Expand Down Expand Up @@ -365,7 +362,28 @@ class MutateCallbacks : public CentipedeCallbacks {
using CentipedeCallbacks::MutateViaExternalBinary;
};

TEST(Centipede, MutateViaExternalBinary) {
// Maintains `TemporaryLocalDirPath()` during the lifetime.
//
// Some parts of Centipede rely on `TemporaryLocalDirPath()` being set up as a
// global resource. Tests that exercise such parts of Centipede should use this
// fixture.
//
// TODO(b/391433873): Get rid of this once the design of
// `TemporaryLocalDirPath()` is revisited.
class CentipedeWithTemporaryLocalDir : public testing::Test {
public:
CentipedeWithTemporaryLocalDir() {
std::filesystem::path tmp_dir = TemporaryLocalDirPath();
std::filesystem::remove_all(tmp_dir);
std::filesystem::create_directory(tmp_dir);
}

~CentipedeWithTemporaryLocalDir() override {
std::filesystem::remove_all(TemporaryLocalDirPath());
}
};

TEST_F(CentipedeWithTemporaryLocalDir, MutateViaExternalBinary) {
// This binary contains a test-friendly custom mutator.
const std::string binary_with_custom_mutator =
GetDataDependencyFilepath("centipede/testing/test_fuzz_target");
Expand Down Expand Up @@ -431,6 +449,7 @@ TEST(Centipede, MutateViaExternalBinary) {
Environment env_no_crossover;
env_no_crossover.crossover_level = 0;
MutateCallbacks callbacks_no_crossover(env_no_crossover);

mutants.resize(10000);
EXPECT_TRUE(callbacks_no_crossover.MutateViaExternalBinary(
binary_with_custom_mutator, GetMutationInputRefsFromDataInputs(inputs),
Expand Down Expand Up @@ -628,8 +647,6 @@ TEST(Centipede, FunctionFilter) {
}
}

namespace {

struct Crash {
std::string binary;
unsigned char input = 0;
Expand Down Expand Up @@ -701,8 +718,6 @@ MATCHER_P(HasFilesWithContents, expected_files_and_contents, "") {
result_listener);
}

} // namespace

// Tests --extra_binaries.
// Executes one main binary (--binary) and 3 extra ones (--extra_binaries).
// Expects the main binary and two extra ones to generate one crash each.
Expand Down Expand Up @@ -746,8 +761,6 @@ TEST(Centipede, ExtraBinaries) {
FileAndContents{Hash({50}), "b3-crash"})));
}

namespace {

// A mock for UndetectedCrashingInput test.
class UndetectedCrashingInputMock : public CentipedeCallbacks {
public:
Expand Down Expand Up @@ -815,8 +828,6 @@ class UndetectedCrashingInputMock : public CentipedeCallbacks {
bool first_pass_ = true;
};

} // namespace

// Test for preserving a crashing batch when 1-by-1 exec fails to reproduce.
// Executes one main binary (--binary).
// Expects the binary to crash once and 1-by-1 reproduction to fail.
Expand Down Expand Up @@ -878,11 +889,12 @@ TEST(Centipede, UndetectedCrashingInput) {
EXPECT_EQ(suspect_only_mock.num_inputs_triaged(), 1);
}

TEST(Centipede, GetsSeedInputs) {
TEST_F(CentipedeWithTemporaryLocalDir, GetsSeedInputs) {
Environment env;
env.binary =
GetDataDependencyFilepath("centipede/testing/seeded_fuzz_target");
CentipedeDefaultCallbacks callbacks(env);

std::vector<ByteArray> seeds;
EXPECT_EQ(callbacks.GetSeeds(10, seeds), 10);
EXPECT_THAT(seeds, testing::ContainerEq(std::vector<ByteArray>{
Expand All @@ -895,32 +907,36 @@ TEST(Centipede, GetsSeedInputs) {
{0}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}}));
}

TEST(Centipede, GetsSerializedTargetConfig) {
TEST_F(CentipedeWithTemporaryLocalDir, GetsSerializedTargetConfig) {
Environment env;
env.binary =
GetDataDependencyFilepath("centipede/testing/fuzz_target_with_config");
CentipedeDefaultCallbacks callbacks(env);

const auto serialized_config = callbacks.GetSerializedTargetConfig();
ASSERT_TRUE(serialized_config.ok());
EXPECT_EQ(*serialized_config, "fake serialized config");
}

TEST(Centipede, GetSerializedTargetConfigProducesFailure) {
TEST_F(CentipedeWithTemporaryLocalDir,
GetSerializedTargetConfigProducesFailure) {
Environment env;
env.binary = absl::StrCat(
GetDataDependencyFilepath("centipede/testing/fuzz_target_with_config")
.c_str(),
" --simulate_failure");
CentipedeDefaultCallbacks callbacks(env);

const auto serialized_config = callbacks.GetSerializedTargetConfig();
EXPECT_FALSE(serialized_config.ok());
}

TEST(Centipede, CleansUpMetadataAfterStartup) {
TEST_F(CentipedeWithTemporaryLocalDir, CleansUpMetadataAfterStartup) {
Environment env;
env.binary = GetDataDependencyFilepath(
"centipede/testing/expensive_startup_fuzz_target");
CentipedeDefaultCallbacks callbacks(env);

BatchResult batch_result;
const std::vector<ByteArray> inputs = {{0}};
ASSERT_TRUE(callbacks.Execute(env.binary, inputs, batch_result));
Expand All @@ -934,8 +950,6 @@ TEST(Centipede, CleansUpMetadataAfterStartup) {
EXPECT_FALSE(found_startup_cmp_entry);
}

namespace {

class FakeCentipedeCallbacksForThreadChecking : public CentipedeCallbacks {
public:
FakeCentipedeCallbacksForThreadChecking(const Environment &env,
Expand All @@ -962,8 +976,6 @@ class FakeCentipedeCallbacksForThreadChecking : public CentipedeCallbacks {
bool thread_check_passed_ = true;
};

} // namespace

TEST(Centipede, RunsExecuteCallbackInTheCurrentThreadWhenFuzzingWithOneThread) {
TempDir temp_dir{test_info_->name()};
Environment env;
Expand All @@ -980,11 +992,12 @@ TEST(Centipede, RunsExecuteCallbackInTheCurrentThreadWhenFuzzingWithOneThread) {
EXPECT_TRUE(callbacks.thread_check_passed());
}

TEST(Centipede, DetectsStackOverflow) {
TEST_F(CentipedeWithTemporaryLocalDir, DetectsStackOverflow) {
Environment env;
env.binary = GetDataDependencyFilepath("centipede/testing/test_fuzz_target");
env.stack_limit_kb = 64;
CentipedeDefaultCallbacks callbacks(env);

BatchResult batch_result;
const std::vector<ByteArray> inputs = {ByteArray{'s', 't', 'k'}};

Expand All @@ -993,8 +1006,6 @@ TEST(Centipede, DetectsStackOverflow) {
EXPECT_EQ(batch_result.failure_description(), "stack-limit-exceeded");
}

namespace {

class SetupFailureCallbacks : public CentipedeCallbacks {
public:
using CentipedeCallbacks::CentipedeCallbacks;
Expand All @@ -1013,8 +1024,6 @@ class SetupFailureCallbacks : public CentipedeCallbacks {
}
};

} // namespace

TEST(Centipede, AbortsOnSetupFailure) {
TempDir temp_dir{test_info_->name()};
Environment env;
Expand All @@ -1028,4 +1037,96 @@ TEST(Centipede, AbortsOnSetupFailure) {
"Terminating Centipede due to setup failure in the test.");
}

TEST_F(CentipedeWithTemporaryLocalDir, UsesCustomMutatorWithCustomMutatorType) {
Environment env;
env.binary = GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator");
env.mutator_type = MutatorType::kCustom;
CentipedeDefaultCallbacks callbacks(env);

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The custom mutator just returns the original inputs as mutants.
EXPECT_EQ(inputs, mutants);
}

TEST_F(CentipedeWithTemporaryLocalDir,
FailsOnMisbehavingMutatorWithCustomMutatorType) {
Environment env;
env.binary =
absl::StrCat(GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator")
.c_str(),
" --simulate_failure");
env.mutator_type = MutatorType::kCustom;
CentipedeDefaultCallbacks callbacks(env);

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
EXPECT_DEATH(callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs),
inputs.size(), mutants),
"Custom mutator failed");
}

TEST_F(CentipedeWithTemporaryLocalDir,
DetectsCustomMutatorWithAutoMutatorType) {
Environment env;
env.binary = GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator");
env.mutator_type = MutatorType::kAuto;
CentipedeDefaultCallbacks callbacks(env);

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The custom mutator just returns the original inputs as mutants.
EXPECT_EQ(inputs, mutants);
}

TEST_F(CentipedeWithTemporaryLocalDir,
FallsBackToBuiltInMutatorWithAutoMutatorType) {
Environment env;
env.binary =
absl::StrCat(GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator")
.c_str(),
" --simulate_failure");
env.mutator_type = MutatorType::kAuto;
CentipedeDefaultCallbacks callbacks(env);

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The built-in mutator performs non-trivial mutations.
EXPECT_EQ(inputs.size(), mutants.size());
EXPECT_NE(inputs, mutants);
}

TEST_F(CentipedeWithTemporaryLocalDir,
UsesBuiltInMutatorWithBuiltInMutatorType) {
Environment env;
env.binary = GetDataDependencyFilepath(
"centipede/testing/fuzz_target_with_custom_mutator");
env.mutator_type = MutatorType::kBuiltIn;
CentipedeDefaultCallbacks callbacks(env);

const std::vector<ByteArray> inputs = {{1}, {2}, {3}, {4}, {5}, {6}};
std::vector<ByteArray> mutants;
callbacks.Mutate(GetMutationInputRefsFromDataInputs(inputs), inputs.size(),
mutants);

// The built-in mutator performs non-trivial mutations, while the custom
// mutator just returns the original inputs as mutants.
EXPECT_EQ(inputs.size(), mutants.size());
EXPECT_NE(inputs, mutants);
}

} // namespace
} // namespace centipede
8 changes: 8 additions & 0 deletions centipede/environment.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,14 @@ void Environment::UpdateWithTargetConfig(
LOG(INFO) << "Overriding the default max_num_crash_reports to "
<< max_num_crash_reports << " for FuzzTest.";
}

// Use the custom mutator when running with FuzzTest to surface any errors
// with the mutation (e.g., ineffective filter domain).
if (mutator_type == Default().mutator_type) {
mutator_type = MutatorType::kCustom;
LOG(INFO) << "Switching to the custom mutator for FuzzTest.";
}

if (config.jobs != 0) {
CHECK(j == Default().j || j == config.jobs)
<< "Value for --j is inconsistent with the value for jobs in the "
Expand Down
Loading

0 comments on commit 9685b83

Please sign in to comment.