centipede/centipede_interface.cc

// Copyright 2022 The Centipede Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "./centipede/centipede_interface.h"

#include <unistd.h>

#include <algorithm>
#include <atomic>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <filesystem>  // NOLINT
#include <iostream>
#include <memory>
#include <string>
#include <string_view>
#include <utility>
#include <vector>

#include "absl/cleanup/cleanup.h"
#include "absl/container/flat_hash_map.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/ascii.h"
#include "absl/strings/escaping.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_join.h"
#include "absl/strings/str_replace.h"
#include "absl/time/clock.h"
#include "absl/time/time.h"
#include "absl/types/span.h"
#include "./centipede/analyze_corpora.h"
#include "./centipede/binary_info.h"
#include "./centipede/centipede.h"
#include "./centipede/centipede_callbacks.h"
#include "./centipede/command.h"
#include "./centipede/coverage.h"
#include "./centipede/crash_deduplication.h"
#include "./centipede/crash_summary.h"
#include "./centipede/distill.h"
#include "./centipede/environment.h"
#include "./centipede/minimize_crash.h"
#include "./centipede/pc_info.h"
#include "./centipede/periodic_action.h"
#include "./centipede/runner_result.h"
#include "./centipede/seed_corpus_maker_lib.h"
#include "./centipede/stats.h"
#include "./centipede/stop.h"
#include "./centipede/thread_pool.h"
#include "./centipede/util.h"
#include "./centipede/workdir.h"
#include "./common/bazel.h"
#include "./common/blob_file.h"
#include "./common/defs.h"
#include "./common/hash.h"
#include "./common/logging.h"
#include "./common/remote_file.h"
#include "./common/status_macros.h"
#include "./fuzztest/internal/configuration.h"

namespace fuzztest::internal {

namespace {

constexpr absl::Duration kDefaultRegressionTtl = absl::Hours(24 * 7);

// Runs env.for_each_blob on every blob extracted from env.args.
// Returns EXIT_SUCCESS on success, EXIT_FAILURE otherwise.
void ForEachBlob(const Environment& env, StopCondition& stop_condition) {
  auto tmpdir = TemporaryLocalDirPath();
  CreateLocalDirRemovedAtExit(tmpdir);
  std::string tmpfile = std::filesystem::path(tmpdir).append("t");

  for (const auto& arg : env.args) {
    FUZZTEST_LOG(INFO) << "Running '" << env.for_each_blob << "' on " << arg;
    auto blob_reader = DefaultBlobFileReaderFactory();
    absl::Status open_status = blob_reader->Open(arg);
    if (!open_status.ok()) {
      const std::string stop_reason =
          absl::StrCat("Failed to open ", arg, ": ", open_status);
      FUZZTEST_LOG(WARNING) << stop_reason;
      stop_condition.RequestStop(EXIT_FAILURE, stop_reason);
      return;
    }
    ByteSpan blob;
    while (blob_reader->Read(blob) == absl::OkStatus()) {
      WriteToLocalFile(tmpfile, blob);
      std::string command_line = absl::StrReplaceAll(
          env.for_each_blob, {{"%P", tmpfile}, {"%H", Hash(blob)}});
      Command cmd(command_line);
      // TODO(kcc): [as-needed] this creates one process per blob.
      // If this flag gets active use, we may want to define special cases,
      // e.g. if for_each_blob=="cp %P /some/where" we can do it in-process.
      cmd.Execute();
      if (stop_condition.ShouldStop()) return;
    }
  }
}

void MinimizeCrash(const Environment& env,
                   CentipedeCallbacksFactory& callbacks_factory,
                   StopCondition& stop_condition) {
  ByteArray crashy_input;
  ReadFromLocalFile(env.minimize_crash_file_path, crashy_input);

  BatchResult batch_result;
  {
    ScopedCentipedeCallbacks scoped_callbacks(callbacks_factory, env,
                                              stop_condition);
    CreateLocalDirRemovedAtExit(TemporaryLocalDirPath());
    if (scoped_callbacks.callbacks()->Execute(env.binary, {crashy_input},
                                              batch_result)) {
      FUZZTEST_LOG(ERROR) << "The original crashy input did not crash; exiting";
      stop_condition.RequestStop(EXIT_FAILURE,
                                 "The original crashy input did not crash");
      return;
    }
  }

  const auto result =
      MinimizeCrash(crashy_input, env, callbacks_factory,
                    batch_result.failure_signature(), stop_condition);
  if (!result.has_value()) {
    stop_condition.RequestStop(EXIT_FAILURE, "No smaller crash found");
    return;
  }

  const auto output_dir =
      std::filesystem::path{WorkDir{env}.CrashReproducerDirPaths().MyShard()};
  const auto mkdir_status = RemoteMkdir(output_dir.string());
  if (!mkdir_status.ok()) {
    const auto error_msg =
        absl::StrCat("Failed to create the minimized crasher dir ",
                     output_dir.string(), ": ", mkdir_status);
    FUZZTEST_LOG(ERROR) << error_msg;
    stop_condition.RequestStop(EXIT_FAILURE, error_msg);
    return;
  }
  const auto result_input_hash = Hash(result->input);
  const auto output_path = (output_dir / result_input_hash).string();
  const auto write_status = RemoteFileSetContents(output_path, result->input);
  if (!write_status.ok()) {
    const auto error_msg =
        absl::StrCat("Failed to write the minimized crasher to ", output_path,
                     ": ", write_status);
    FUZZTEST_LOG(ERROR) << error_msg;
    stop_condition.RequestStop(EXIT_FAILURE, error_msg);
    return;
  }
}

// Loads corpora from work dirs provided in `env.args`, if there are two args
// provided, analyzes differences. If there is one arg provided, reports the
// function coverage. Returns EXIT_SUCCESS on success, EXIT_FAILURE otherwise.
int Analyze(const Environment& env) {
  FUZZTEST_LOG(INFO) << "Analyze " << absl::StrJoin(env.args, ",");
  FUZZTEST_CHECK(!env.binary.empty()) << "--binary must be used";
  if (env.args.size() == 1) {
    const CoverageResults coverage_results =
        GetCoverage(env.binary_name, env.binary_hash, env.args[0]);
    WorkDir workdir{env};
    const std::string coverage_report_path =
        workdir.CoverageReportPath(/*annotation=*/"");
    DumpCoverageReport(coverage_results, coverage_report_path);
  } else if (env.args.size() == 2) {
    AnalyzeCorporaToLog(env.binary_name, env.binary_hash, env.args[0],
                        env.args[1]);
  } else {
    FUZZTEST_LOG(FATAL)
        << "for now, --analyze supports only 1 or 2 work dirs; got "
        << env.args.size();
  }
  return EXIT_SUCCESS;
}

void SavePCTableToFile(const PCTable& pc_table, std::string_view file_path) {
  WriteToLocalFile(file_path, AsByteSpan(pc_table));
}

BinaryInfo PopulateBinaryInfoAndSavePCsIfNecessary(
    const Environment& env, CentipedeCallbacksFactory& callbacks_factory,
    std::string& pcs_file_path, StopCondition& stop_condition) {
  BinaryInfo binary_info;
  // Some fuzz targets have coverage not based on instrumenting binaries.
  // For those target, we should not populate binary info.
  if (env.populate_binary_info) {
    ScopedCentipedeCallbacks scoped_callbacks(callbacks_factory, env,
                                              stop_condition);
    scoped_callbacks.callbacks()->PopulateBinaryInfo(binary_info);
    if (stop_condition.ShouldStop()) {
      return binary_info;
    }
  }
  if (env.save_binary_info) {
    const std::string binary_info_dir = WorkDir{env}.BinaryInfoDirPath();
    FUZZTEST_CHECK_OK(RemoteMkdir(binary_info_dir));
    FUZZTEST_LOG(INFO) << "Serializing binary info to: " << binary_info_dir;
    binary_info.Write(binary_info_dir);
  }
  if (binary_info.uses_legacy_trace_pc_instrumentation) {
    pcs_file_path = std::filesystem::path(TemporaryLocalDirPath()) / "pcs";
    SavePCTableToFile(binary_info.pc_table, pcs_file_path);
  }
  if (env.use_pcpair_features) {
    FUZZTEST_CHECK(!binary_info.pc_table.empty())
        << "--use_pcpair_features requires non-empty pc_table";
  }
  return binary_info;
}

std::vector<Environment> CreateEnvironmentsForThreads(
    const Environment& origin_env, std::string_view pcs_file_path) {
  std::vector<Environment> envs(origin_env.num_threads, origin_env);
  size_t thread_idx = 0;
  for (auto& env : envs) {
    env.my_shard_index += thread_idx++;
    env.UpdateForExperiment();
    env.pcs_file_path = pcs_file_path;
  }
  return envs;
}

void Fuzz(const Environment& env, const BinaryInfo& binary_info,
          std::string_view pcs_file_path,
          CentipedeCallbacksFactory& callbacks_factory,
          StopCondition& stop_condition) {
  CoverageLogger coverage_logger(binary_info.pc_table, binary_info.symbols);

  std::vector<Environment> envs =
      CreateEnvironmentsForThreads(env, pcs_file_path);
  std::vector<std::atomic<Stats>> stats_vec(env.num_threads);

  // Start periodic stats dumping and, optionally, logging.
  std::vector<PeriodicAction> stats_reporters;
  stats_reporters.emplace_back(
      [csv_appender = StatsCsvFileAppender{stats_vec, envs}]() mutable {
        csv_appender.ReportCurrStats();
      },
      PeriodicAction::Options{
          /*sleep_before_each=*/
          [](size_t iteration) {
            return absl::Minutes(std::clamp(iteration, 0UL, 10UL));
          },
      });
  if (!envs.front().experiment.empty() || FUZZTEST_VLOG_IS_ON(1)) {
    stats_reporters.emplace_back(
        [logger = StatsLogger{stats_vec, envs}]() mutable {
          logger.ReportCurrStats();
        },
        PeriodicAction::Options{
            /*sleep_before_each=*/
            [](size_t iteration) {
              return absl::Seconds(std::clamp(iteration, 5UL, 600UL));
            },
        });
  }

  auto fuzzing_worker = [&env, &callbacks_factory, &binary_info,
                         &coverage_logger, &stop_condition](
                            Environment& my_env, std::atomic<Stats>& stats,
                            bool create_tmpdir) {
    if (create_tmpdir) CreateLocalDirRemovedAtExit(TemporaryLocalDirPath());
    // Uses TID, call in this thread.
    my_env.seed = GetRandomSeed(env.seed);

    if (env.dry_run) return;

    ScopedCentipedeCallbacks scoped_callbacks(callbacks_factory, my_env,
                                              stop_condition);
    Centipede centipede(my_env, *scoped_callbacks.callbacks(), binary_info,
                        coverage_logger, stats, stop_condition);
    centipede.FuzzingLoop();
  };

  if (env.num_threads == 1) {
    // When fuzzing with one thread, run fuzzing loop in the current
    // thread. This is because FuzzTest/Centipede's single-process
    // fuzzing requires the test body, which is invoked by the fuzzing
    // loop, to run in the main thread.
    //
    // Here, the fuzzing worker should not re-create the tmpdir since the path
    // is thread-local and it has been created in the current function.
    fuzzing_worker(envs[0], stats_vec[0], /*create_tmpdir=*/false);
  } else {
    ThreadPool fuzzing_worker_threads{static_cast<int>(env.num_threads)};
    for (size_t thread_idx = 0; thread_idx < env.num_threads; thread_idx++) {
      Environment& my_env = envs[thread_idx];
      std::atomic<Stats>& my_stats = stats_vec[thread_idx];
      fuzzing_worker_threads.Schedule([&fuzzing_worker, &my_env, &my_stats]() {
        fuzzing_worker(my_env, my_stats, /*create_tmpdir=*/true);
      });
    }  // All `fuzzing_worker_threads` join here.
  }

  for (auto& reporter : stats_reporters) {
    // Nudge one final update and stop the reporting thread.
    reporter.Nudge();
    reporter.Stop();
  }

  if (!env.knobs_file.empty()) PrintRewardValues(stats_vec, std::cerr);
}

TestShard SetUpTestSharding() {
  TestShard test_shard = GetBazelTestShard();
  // Update the shard status file to indicate that we support test sharding.
  // It suffices to update the file's modification time, but we clear the
  // contents for simplicity. This is also what the GoogleTest framework does.
  if (const char* test_shard_status_file =
          std::getenv("TEST_SHARD_STATUS_FILE");
      test_shard_status_file != nullptr) {
    ClearLocalFileContents(test_shard_status_file);
  }
  return test_shard;
}

// Seeds the corpus files in `env.workdir` with the inputs in `regression_dir`
// (always used) and the previously distilled corpus files from `coverage_dir`
// (used if non-empty).
SeedCorpusConfig GetSeedCorpusConfig(const Environment& env,
                                     std::string_view regression_dir,
                                     std::string_view coverage_dir) {
  const WorkDir workdir{env};
  SeedCorpusSource regression;
  regression.dir_glob = std::string(regression_dir);
  regression.num_recent_dirs = 1;
  regression.individual_input_rel_glob = "*";
  regression.sampled_fraction_or_count = 1.0f;
  std::vector<SeedCorpusSource> sources = {std::move(regression)};
  if (!coverage_dir.empty()) {
    SeedCorpusSource coverage;
    coverage.dir_glob = std::string(coverage_dir);
    coverage.num_recent_dirs = 1;
    // We're using the previously distilled corpus files as seeds.
    coverage.shard_rel_glob =
        std::filesystem::path{
            workdir.DistilledCorpusFilePaths().AllShardsGlob()}
            .filename();
    coverage.individual_input_rel_glob = "*";
    coverage.sampled_fraction_or_count = 1.0f;
    sources.push_back(std::move(coverage));
  }
  SeedCorpusDestination destination;
  destination.dir_path = env.workdir;
  // We're seeding the current corpus files.
  destination.shard_rel_glob =
      std::filesystem::path{workdir.CorpusFilePaths().AllShardsGlob()}
          .filename();
  destination.shard_index_digits = WorkDir::kDigitsInShardIndex;
  destination.num_shards = static_cast<uint32_t>(env.num_threads);
  return {
      std::move(sources),
      std::move(destination),
  };
}

absl::Duration ReadFuzzingTime(std::string_view fuzzing_time_file) {
  std::string fuzzing_time_str;
  FUZZTEST_CHECK_OK(RemoteFileGetContents(fuzzing_time_file, fuzzing_time_str));
  absl::Duration fuzzing_time;
  if (!absl::ParseDuration(absl::StripAsciiWhitespace(fuzzing_time_str),
                           &fuzzing_time)) {
    FUZZTEST_LOG(WARNING)
        << "Failed to parse fuzzing time of a resuming fuzz test: '"
        << fuzzing_time_str << "'. Assuming no fuzzing time so far.";
    return absl::ZeroDuration();
  }
  return fuzzing_time;
}

PeriodicAction RecordFuzzingTime(std::string_view fuzzing_time_file,
                                 absl::Time start_time) {
  return {[=] {
            absl::Status status = RemoteFileSetContents(
                fuzzing_time_file,
                absl::FormatDuration(absl::Now() - start_time));
            FUZZTEST_LOG_IF(WARNING, !status.ok())
                << "Failed to write fuzzing time: " << status;
          },
          PeriodicAction::ZeroDelayConstInterval(absl::Seconds(15))};
}

struct DatabasePaths {
  std::filesystem::path fuzztest_db_path;
  std::filesystem::path regression_dir;
  std::filesystem::path coverage_dir;
  std::filesystem::path crashing_dir;
  std::filesystem::path incubating_dir;
};

DatabasePaths GetDatabasePaths(const Environment& env) {
  const auto fuzztest_db_path =
      std::filesystem::path(env.fuzztest_corpus_database) /
      env.fuzztest_binary_identifier / env.test_name;
  return {
      fuzztest_db_path,
      fuzztest_db_path / "regression",
      fuzztest_db_path / "coverage",
      fuzztest_db_path / "crashing",
      fuzztest_db_path / "incubating",
  };
}

void RecordFuzzingResults(const Environment& env, const DatabasePaths& db_paths,
                          CentipedeCallbacksFactory& callbacks_factory,
                          StopCondition& stop_condition) {
  const WorkDir workdir{env};

  const bool update_crashing_db =
      env.fuzztest_update_corpus_database.value_or(!env.fuzztest_only_replay);
  const bool update_coverage_db =
      update_crashing_db && !env.fuzztest_only_replay;
  if (!update_crashing_db && !env.report_crash_summary) {
    return;
  }

  // Deduplicate and optionally update the crashing inputs.
  CrashSummary crash_summary{env.fuzztest_binary_identifier, env.test_name};
  const absl::flat_hash_map<std::string, CrashDetails> crashes_by_signature =
      GetCrashesFromWorkdir(workdir, env.total_shards);
  if (update_crashing_db) {
    const absl::Status status = OrganizeCrashingInputs(
        db_paths.regression_dir, db_paths.crashing_dir, db_paths.incubating_dir,
        env, callbacks_factory, crashes_by_signature, crash_summary,
        stop_condition, kDefaultRegressionTtl);
    FUZZTEST_LOG_IF(ERROR, !status.ok())
        << "Failed to organize crashing inputs: " << status;
    if (env.report_crash_summary) crash_summary.Report(&std::cerr);
  } else if (env.report_crash_summary) {
    // Just report the crashes.
    for (const auto& [crash_signature, crash_details] : crashes_by_signature) {
      crash_summary.AddCrash({/*id=*/crash_details.input_signature,
                              /*category=*/crash_details.description,
                              crash_signature, crash_details.description});
    }
    crash_summary.Report(&std::cerr);
  }

  if (update_coverage_db) {
    // Distill and store the coverage corpus.
    Distill(env);
    if (RemotePathExists(db_paths.coverage_dir.c_str())) {
      // In the future, we will store k latest coverage corpora for some k, but
      // for now we only keep the latest one.
      FUZZTEST_CHECK_OK(RemotePathDelete(db_paths.coverage_dir.c_str(),
                                         /*recursively=*/true));
    }
    FUZZTEST_CHECK_OK(RemoteMkdir(db_paths.coverage_dir.c_str()));
    std::vector<std::string> distilled_corpus_files;
    FUZZTEST_CHECK_OK(
        RemoteGlobMatch(workdir.DistilledCorpusFilePaths().AllShardsGlob(),
                        distilled_corpus_files));
    for (const std::string& corpus_file : distilled_corpus_files) {
      const std::string file_name =
          std::filesystem::path(corpus_file).filename();
      FUZZTEST_CHECK_OK(RemoteFileRename(
          corpus_file, (db_paths.coverage_dir / file_name).c_str()));
    }
  }
}

void UpdateCorpusDatabase(Environment env,
                          CentipedeCallbacksFactory& callbacks_factory,
                          StopCondition& stop_condition) {
  FUZZTEST_LOG(INFO) << "Starting the update of the corpus database for:"
                     << "\nFuzz test: " << env.test_name
                     << "\nBinary: " << env.binary
                     << "\nCorpus database: " << env.fuzztest_corpus_database;

  // Step 1: Preliminary set up of test sharding, binary info, etc.
  const auto [test_shard_index, total_test_shards] = SetUpTestSharding();
  const auto corpus_database_path =
      std::filesystem::path(env.fuzztest_corpus_database) /
      env.fuzztest_binary_identifier;
  const auto stats_root_path =
      env.fuzztest_stats_root.empty()
          ? std::filesystem::path()
          : std::filesystem::path(env.fuzztest_stats_root) /
                env.fuzztest_binary_identifier;
  const auto workdir_root_path =
      env.fuzztest_workdir_root.empty()
          ? corpus_database_path
          : std::filesystem::path(env.fuzztest_workdir_root) /
                env.fuzztest_binary_identifier;
  const auto execution_stamp = [] {
    std::string stamp =
        absl::FormatTime("%Y-%m-%d-%H-%M-%S", absl::Now(), absl::UTCTimeZone());
    return stamp;
  }();

  const bool is_workdir_specified = !env.workdir.empty();
  // When env.workdir is empty, the full workdir paths will be formed by
  // appending the fuzz test names to the base workdir path. We use different
  // path when only replaying to avoid replaying an unfinished fuzzing sessions.
  const auto base_workdir_path =
      is_workdir_specified
          ? std::filesystem::path{}  // Will not be used.
          : workdir_root_path /
                absl::StrFormat("workdir%s.%03d",
                                env.fuzztest_only_replay ? "-replay" : "",
                                test_shard_index);
  // There's no point in saving the binary info to the workdir, since the
  // workdir is deleted at the end.
  env.save_binary_info = false;
  std::string pcs_file_path;
  BinaryInfo binary_info = PopulateBinaryInfoAndSavePCsIfNecessary(
      env, callbacks_factory, pcs_file_path, stop_condition);
  if (stop_condition.StopRequested()) return;

  FUZZTEST_LOG(INFO) << "Test shard index: " << test_shard_index
                     << " Total test shards: " << total_test_shards;

  StopCondition::StopRequest stop_request;

  // Step 2: Run the fuzz test.

  // Unset stop time. stop_time will be set later.
  stop_condition.SetStopTime(absl::InfiniteFuture());

  if (!is_workdir_specified) {
    env.workdir = base_workdir_path / env.test_name;
  }
  const auto execution_id_path =
      (base_workdir_path / absl::StrCat(env.test_name, ".execution_id"))
          .string();

  bool is_resuming = false;
  if (!is_workdir_specified && !env.fuzztest_execution_id.empty()) {
    // Use the execution IDs to resume or skip tests.
    const bool execution_id_matched = [&] {
      if (!RemotePathExists(execution_id_path)) return false;
      FUZZTEST_CHECK(!RemotePathIsDirectory(execution_id_path));
      std::string prev_execution_id;
      FUZZTEST_CHECK_OK(
          RemoteFileGetContents(execution_id_path, prev_execution_id));
      return prev_execution_id == env.fuzztest_execution_id;
    }();
    if (execution_id_matched) {
      // If execution IDs match but the previous coverage is missing, it means
      // the test was previously finished, and we skip running for the test.
      if (!RemotePathExists(WorkDir{env}.CoverageDirPath())) {
        FUZZTEST_LOG(INFO) << "Skipping running the fuzz test "
                           << env.test_name;
        return;
      }
      // If execution IDs match and the previous coverage exists, it means
      // the same workflow got interrupted when running the test. So we resume
      // the test.
      is_resuming = true;
      FUZZTEST_LOG(INFO) << "Resuming running the fuzz test " << env.test_name;
    } else {
      // If the execution IDs mismatch, we start a new run.
      is_resuming = false;
      FUZZTEST_LOG(INFO) << "Starting a new run of the fuzz test "
                         << env.test_name;
    }
  }

  if (!is_workdir_specified && RemotePathExists(env.workdir) && !is_resuming) {
    // This could be a workdir from a failed run that used a different version
    // of the binary. We delete it so that we don't have to deal with
    // the assumptions under which it is safe to reuse an old workdir.
    FUZZTEST_CHECK_OK(RemotePathDelete(env.workdir, /*recursively=*/true));
  }
  const WorkDir workdir{env};
  FUZZTEST_CHECK_OK(RemoteMkdir(
      workdir.CoverageDirPath()));  // Implicitly creates the workdir

  // Updating execution ID must be after creating the coverage dir. Otherwise
  // if it fails to create coverage dir after updating execution ID, next
  // attempt would skip this test.
  if (!is_workdir_specified && !env.fuzztest_execution_id.empty() &&
      !is_resuming) {
    FUZZTEST_CHECK_OK(
        RemoteFileSetContents(execution_id_path, env.fuzztest_execution_id));
  }

  absl::Cleanup clean_up_workdir = [is_workdir_specified, &env,
                                    &stop_condition] {
    if (!is_workdir_specified && !stop_condition.StopRequested()) {
      FUZZTEST_CHECK_OK(RemotePathDelete(env.workdir, /*recursively=*/true));
    }
  };

  const DatabasePaths db_paths = GetDatabasePaths(env);

  // Seed the fuzzing session with the latest coverage corpus and regression
  // inputs from the previous fuzzing session.
  if (!is_resuming) {
    FUZZTEST_CHECK_OK(GenerateSeedCorpusFromConfig(
        GetSeedCorpusConfig(env, db_paths.regression_dir.c_str(),
                            env.fuzztest_replay_coverage_inputs
                                ? db_paths.coverage_dir.c_str()
                                : ""),
        env.binary_name, env.binary_hash))
        << "while generating the seed corpus";
  }

  absl::Duration time_limit = env.fuzztest_time_limit_per_test;
  absl::Duration time_spent = absl::ZeroDuration();
  const std::string fuzzing_time_file =
      std::filesystem::path(env.workdir) / "fuzzing_time";
  if (is_resuming && RemotePathExists(fuzzing_time_file)) {
    time_spent = ReadFuzzingTime(fuzzing_time_file);
    time_limit = std::max(time_limit - time_spent, absl::ZeroDuration());
  }
  is_resuming = false;

  if (stop_condition.StopRequested(&stop_request)) {
    if (stop_request.exit_code != EXIT_SUCCESS) {
      FUZZTEST_LOG(ERROR) << "Early stop requested for test " << env.test_name
                          << " with failure exit code "
                          << stop_request.exit_code;
    } else {
      FUZZTEST_LOG(INFO) << "Skipping test " << env.test_name
                         << " due to early stop requested without failure.";
    }
    return;
  }

  FUZZTEST_LOG(INFO) << (env.fuzztest_only_replay ? "Replaying " : "Fuzzing ")
                     << env.test_name << " for " << time_limit
                     << "\n\tTest binary: " << env.binary;

  const absl::Time start_time = absl::Now();
  stop_condition.SetStopTime(start_time + time_limit);
  PeriodicAction record_fuzzing_time =
      RecordFuzzingTime(fuzzing_time_file, start_time - time_spent);
  Fuzz(env, binary_info, pcs_file_path, callbacks_factory, stop_condition);
  record_fuzzing_time.Nudge();
  record_fuzzing_time.Stop();

  if (!stats_root_path.empty()) {
    const auto stats_dir = stats_root_path / env.test_name;
    FUZZTEST_CHECK_OK(RemoteMkdir(stats_dir.c_str()));
    FUZZTEST_CHECK_OK(RemoteFileRename(
        workdir.FuzzingStatsPath(),
        (stats_dir / absl::StrCat("fuzzing_stats_", execution_stamp)).c_str()));
  }

  if (stop_condition.StopRequested(&stop_request)) {
    if (stop_request.exit_code != EXIT_SUCCESS) {
      FUZZTEST_LOG(ERROR) << "Early stop requested for test " << env.test_name
                          << " with failure exit code "
                          << stop_request.exit_code;
    } else {
      FUZZTEST_LOG(INFO) << "Skip updating corpus database due to early stop "
                            "requested without failure.";
    }
    return;
  }

  // The test time limit does not apply for updating the corpus database.
  stop_condition.SetStopTime(absl::InfiniteFuture());
  RecordFuzzingResults(env, db_paths, callbacks_factory, stop_condition);
}

int ListCrashIds(const Environment& env) {
  FUZZTEST_CHECK(!env.list_crash_ids_file.empty())
      << "Need list_crash_ids_file to be set for listing crash IDs";
  FUZZTEST_CHECK(!env.test_name.empty());
  std::vector<std::string> crash_paths;
  // TODO: b/406003594 - move the path construction to a library.
  const auto crash_dir = GetDatabasePaths(env).crashing_dir;
  if (RemotePathExists(crash_dir.string())) {
    FUZZTEST_CHECK(RemotePathIsDirectory(crash_dir.string()))
        << "Crash dir " << crash_dir << " in the corpus database "
        << env.fuzztest_corpus_database << " is not a directory";
    crash_paths =
        ValueOrDie(RemoteListFiles(crash_dir.string(), /*recursively=*/false));
  }
  std::vector<std::string> results;
  results.reserve(crash_paths.size());
  for (const auto& crash_path : crash_paths) {
    std::string crash_id = std::filesystem::path{crash_path}.filename();
    results.push_back(std::move(crash_id));
  }
  FUZZTEST_CHECK_OK(RemoteFileSetContents(env.list_crash_ids_file,
                                          absl::StrJoin(results, "\n")));
  return EXIT_SUCCESS;
}

void ReplayCrash(const Environment& env,
                 CentipedeCallbacksFactory& callbacks_factory,
                 StopCondition& stop_condition) {
  FUZZTEST_CHECK(!env.crash_id.empty())
      << "Need crash_id to be set for replay a crash";
  FUZZTEST_CHECK(!env.test_name.empty())
      << "Expecting exactly one test for replay_crash";
  // TODO: b/406003594 - move the path construction to a library.
  const auto crash_dir = std::filesystem::path(env.fuzztest_corpus_database) /
                         env.fuzztest_binary_identifier / env.test_name /
                         "crashing";
  const WorkDir workdir{env};
  SeedCorpusSource crash_corpus_source;
  crash_corpus_source.dir_glob = crash_dir;
  crash_corpus_source.num_recent_dirs = 1;
  crash_corpus_source.individual_input_rel_glob = env.crash_id;
  crash_corpus_source.sampled_fraction_or_count = 1.0f;
  const SeedCorpusConfig crash_corpus_config = {
      /*sources=*/{crash_corpus_source},
      /*destination=*/{
          /*dir_path=*/env.workdir,
          /*shard_rel_glob=*/
          std::filesystem::path{workdir.CorpusFilePaths().AllShardsGlob()}
              .filename(),
          /*shard_index_digits=*/WorkDir::kDigitsInShardIndex,
          /*num_shards=*/1}};
  FUZZTEST_CHECK_OK(GenerateSeedCorpusFromConfig(
      crash_corpus_config, env.binary_name, env.binary_hash));
  Environment run_crash_env = env;
  run_crash_env.load_shards_only = true;
  run_crash_env.persistent_mode = false;
  Fuzz(run_crash_env, {}, "", callbacks_factory, stop_condition);
  if (env.report_crash_summary) {
    CrashSummary crash_summary{env.fuzztest_binary_identifier, env.test_name};
    const absl::flat_hash_map<std::string, CrashDetails> crashes_by_signature =
        GetCrashesFromWorkdir(workdir, /*total_shards=*/1);
    FUZZTEST_CHECK_LE(crashes_by_signature.size(), 1);
    for (const auto& [signature, crash_details] : crashes_by_signature) {
      crash_summary.AddCrash({env.crash_id,
                              /*category=*/crash_details.description, signature,
                              crash_details.description});
    }
    crash_summary.Report(&std::cerr);
  }
}

int ExportCrash(const Environment& env) {
  FUZZTEST_CHECK(!env.crash_id.empty())
      << "Need crash_id to be set for exporting a crash";
  FUZZTEST_CHECK(!env.export_crash_file.empty())
      << "Need export_crash_file to be set for exporting a crash";
  FUZZTEST_CHECK(!env.test_name.empty())
      << "Expecting exactly one test for exporting a crash";
  // TODO: b/406003594 - move the path construction to a library.
  const auto crash_dir = std::filesystem::path(env.fuzztest_corpus_database) /
                         env.fuzztest_binary_identifier / env.test_name /
                         "crashing";
  std::string crash_contents;
  const auto read_status =
      RemoteFileGetContents((crash_dir / env.crash_id).c_str(), crash_contents);
  if (!read_status.ok()) {
    FUZZTEST_LOG(ERROR) << "Failed reading the crash " << env.crash_id
                        << " from " << crash_dir.c_str() << ": " << read_status;
    return EXIT_FAILURE;
  }
  const auto write_status =
      RemoteFileSetContents(env.export_crash_file, crash_contents);
  if (!write_status.ok()) {
    FUZZTEST_LOG(ERROR) << "Failed write the crash " << env.crash_id << " to "
                        << env.export_crash_file << ": " << write_status;
    return EXIT_FAILURE;
  }
  return EXIT_SUCCESS;
}

}  // namespace

int CentipedeMain(const Environment& env,
                  CentipedeCallbacksFactory& callbacks_factory,
                  StopCondition* stop_condition) {
  StopCondition default_stop_condition;
  if (stop_condition == nullptr) {
    stop_condition = &default_stop_condition;
  }
  stop_condition->SetStopTime(env.stop_at);

  auto SaveStopReasonAndGetExitCode = [&]() -> int {
    StopCondition::StopRequest stop_request;
    const bool stop_requested = stop_condition->StopRequested(&stop_request);
    if (!stop_requested) return EXIT_SUCCESS;
    if (!env.stop_reason_file.empty()) {
      WriteToLocalFile(env.stop_reason_file, stop_request.reason);
    }
    return stop_request.exit_code;
  };

  if (!env.corpus_to_files.empty()) {
    Centipede::CorpusToFiles(env, env.corpus_to_files);
    return EXIT_SUCCESS;
  }

  if (!env.crashes_to_files.empty()) {
    const auto status = Centipede::CrashesToFiles(env, env.crashes_to_files);
    if (status.ok()) return EXIT_SUCCESS;
    FUZZTEST_LOG(ERROR) << "Got error when exporting crashes to files: "
                        << status;
    return EXIT_FAILURE;
  }

  if (!env.for_each_blob.empty()) {
    ForEachBlob(env, *stop_condition);
    return SaveStopReasonAndGetExitCode();
  }

  if (!env.minimize_crash_file_path.empty()) {
    MinimizeCrash(env, callbacks_factory, *stop_condition);
    return SaveStopReasonAndGetExitCode();
  }

  // Just export the corpus from a local dir and exit.
  if (!env.corpus_from_files.empty()) {
    Centipede::CorpusFromFiles(env, env.corpus_from_files);
    return EXIT_SUCCESS;
  }

  // Export the corpus from a local dir and then fuzz.
  if (!env.corpus_dir.empty()) {
    for (size_t i = 0; i < env.corpus_dir.size(); ++i) {
      const auto& corpus_dir = env.corpus_dir[i];
      if (i > 0 || !env.first_corpus_dir_output_only)
        Centipede::CorpusFromFiles(env, corpus_dir);
    }
  }

  if (env.distill) return Distill(env);

  // Create the local temporary dir once, before creating any threads. The
  // temporary dir must typically exist before `CentipedeCallbacks` can be used.
  const auto tmpdir = TemporaryLocalDirPath();
  CreateLocalDirRemovedAtExit(tmpdir);

  // Enter the update corpus database mode only if we have a binary to invoke
  // and a corpus database to update.
  // We don't update the corpus database for standalone binaries (i.e., when
  // `env.has_input_wildcards` is true).
  if (!env.binary.empty() && !env.has_input_wildcards) {
    const auto serialized_target_config = [&]() -> absl::StatusOr<std::string> {
      // TODO: b/410051414 Use Centipede flags to pass necessary information
      // instead of passing the entirely serialized Configuration once switched
      // to the unified execution model.
      if (env.fuzztest_configuration == "(null)") {
        return "";
      } else if (!env.fuzztest_configuration.empty()) {
        std::string result;
        FUZZTEST_CHECK(
            absl::WebSafeBase64Unescape(env.fuzztest_configuration, &result));
        return result;
      }
      ScopedCentipedeCallbacks scoped_callbacks(callbacks_factory, env,
                                                *stop_condition);
      return scoped_callbacks.callbacks()->GetSerializedTargetConfig();
    }();
    Environment updated_env = env;
    if (updated_env.fuzztest_corpus_database.empty()) {
      if (!serialized_target_config.ok()) {
        FUZZTEST_LOG(ERROR) << "Failed to get the serialized target config: "
                            << serialized_target_config.status();
        return EXIT_FAILURE;
      }
      if (!serialized_target_config->empty()) {
        const auto target_config =
            fuzztest::internal::Configuration::Deserialize(
                *serialized_target_config);
        FUZZTEST_CHECK_OK(target_config.status())
            << "Failed to deserialize target configuration";
        if (!target_config->corpus_database.empty()) {
          updated_env.UpdateWithTargetConfig(*target_config);
        }
      }
    }
    if (!updated_env.fuzztest_corpus_database.empty()) {
      FUZZTEST_LOG_IF(
          FATAL, env.list_crash_ids + env.replay_crash + env.export_crash > 1)
          << "At most one of list_crash_ids/replay_crash/export_crash can "
             "be set, but seeing list_crash_ids: "
          << env.list_crash_ids << ", replay_crash: " << env.replay_crash
          << ", export_crash: " << env.export_crash;
      if (env.list_crash_ids) {
        return ListCrashIds(updated_env);
      }
      if (env.replay_crash) {
        ReplayCrash(updated_env, callbacks_factory, *stop_condition);
        return SaveStopReasonAndGetExitCode();
      }
      if (env.export_crash) {
        return ExportCrash(updated_env);
      }

      FUZZTEST_CHECK(!updated_env.fuzztest_multi_test_mode_soon_to_be_removed)
          << "Multi-test mode is no longer supported.";
      FUZZTEST_CHECK(!updated_env.test_name.empty())
          << "Centipede requires the test name for FuzzTest.";
      FUZZTEST_CHECK(updated_env.fuzztest_time_limit_per_test >=
                     absl::Seconds(1))
          << "Time limit per fuzz test must be at least 1 second.";
      UpdateCorpusDatabase(updated_env, callbacks_factory, *stop_condition);
      return SaveStopReasonAndGetExitCode();
    }
  }

  // Create the remote coverage dirs once, before creating any threads.
  const auto coverage_dir = WorkDir{env}.CoverageDirPath();
  FUZZTEST_CHECK_OK(RemoteMkdir(coverage_dir));
  FUZZTEST_LOG(INFO) << "Coverage dir: " << coverage_dir
                     << "; temporary dir: " << tmpdir;

  std::string pcs_file_path;
  BinaryInfo binary_info = PopulateBinaryInfoAndSavePCsIfNecessary(
      env, callbacks_factory, pcs_file_path, *stop_condition);
  if (stop_condition->StopRequested()) {
    return SaveStopReasonAndGetExitCode();
  }

  if (env.analyze) return Analyze(env);

  Fuzz(env, binary_info, pcs_file_path, callbacks_factory, *stop_condition);
  return SaveStopReasonAndGetExitCode();

  // TODO: fniksic - Report the crash summary here if requested. What are the
  // binary identifier and the fuzz test name here?
}

}  // namespace fuzztest::internal