| // Copyright 2022 The Centipede Authors. | |
| // | |
| // Licensed under the Apache License, Version 2.0 (the "License"); | |
| // you may not use this file except in compliance with the License. | |
| // You may obtain a copy of the License at | |
| // | |
| // https://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| #include "./centipede/stats.h" | |
| #include <atomic> | |
| #include <cstddef> | |
| #include <cstdint> | |
| #include <filesystem> // NOLINT | |
| #include <sstream> | |
| #include <string> | |
| #include <string_view> | |
| #include <vector> | |
| #include "gmock/gmock.h" | |
| #include "gtest/gtest.h" | |
| #include "absl/log/log_entry.h" | |
| #include "absl/log/log_sink.h" | |
| #include "absl/log/log_sink_registry.h" | |
| #include "absl/strings/str_cat.h" | |
| #include "absl/strings/str_join.h" | |
| #include "absl/strings/str_split.h" | |
| #include "absl/strings/string_view.h" | |
| #include "absl/time/civil_time.h" | |
| #include "absl/time/time.h" | |
| #include "./centipede/environment.h" | |
| #include "./centipede/util.h" | |
| #include "./common/logging.h" // IWYU pragma: keep | |
| #include "./common/test_util.h" | |
| namespace fuzztest::internal { | |
| namespace { | |
| using ::testing::ElementsAreArray; | |
| namespace { | |
| class LogCapture : public absl::LogSink { | |
| public: | |
| LogCapture() { absl::AddLogSink(this); } | |
| ~LogCapture() override { absl::RemoveLogSink(this); } | |
| void Send(const absl::LogEntry &entry) override { | |
| captured_log_.emplace_back(entry.text_message()); | |
| } | |
| std::vector<std::string> CapturedLogLines() const { | |
| // Join->Split normalizes multi-line messages. | |
| return absl::StrSplit(absl::StrJoin(captured_log_, "\n"), '\n'); | |
| } | |
| private: | |
| std::vector<std::string> captured_log_; | |
| }; | |
| uint64_t CivilTimeToUnixMicros( // | |
| int64_t y, int64_t m, int64_t d, int64_t hh, int64_t mm, int64_t ss) { | |
| return absl::ToUnixMicros(absl::FromCivil( | |
| absl::CivilSecond{y, m, d, hh, mm, ss}, absl::LocalTimeZone())); | |
| } | |
| Environment CreateEnv(std::string_view workdir, | |
| std::string_view experiment_name = "", | |
| std::string_view experiment_flags = "") { | |
| Environment env; | |
| env.workdir = std::string(workdir); | |
| env.experiment_name = std::string(experiment_name); | |
| env.experiment_flags = std::string(experiment_flags); | |
| return env; | |
| } | |
| } // namespace | |
| TEST(Stats, PrintStatsToLog) { | |
| std::vector<std::atomic<Stats>> stats_vec(4); | |
| for (uint64_t i = 0; i < stats_vec.size(); ++i) { | |
| const uint64_t j = i + 1; | |
| CovStats cov_stats = {/*num_covered_pcs=*/21 * j, | |
| /*num_8bit_counter_features=*/22 * j, | |
| /*num_data_flow_features=*/23 * j, | |
| /*num_cmp_features=*/24 * j, | |
| /*num_call_stack_features=*/25 * j, | |
| /*num_bounded_path_features=*/26 * j, | |
| /*num_pc_pair_features=*/27 * j, | |
| /*num_user_features=*/28 * j}; | |
| // NOTE: These two don't add up to `num_user_features`, but that's | |
| // ok for the purposes of this test. | |
| cov_stats.num_user0_features = 28 * j + 1; | |
| cov_stats.num_user10_features = 28 * j + 2; | |
| cov_stats.num_unknown_features = 29 * j; | |
| cov_stats.num_funcs_in_frontier = 31 * j; | |
| stats_vec[i].store(Stats{ | |
| StatsMeta{/*timestamp_unix_micros=*/CivilTimeToUnixMicros(1970, 1, 1, 0, | |
| 0, i)}, | |
| ExecStats{ | |
| /*fuzz_time_sec=*/10 * j, | |
| /*num_executions=*/12 * j, | |
| /*num_target_crashes=*/13 * j, | |
| }, | |
| cov_stats, | |
| CorpusStats{ | |
| /*active_corpus_size=*/101 * j, | |
| /*total_corpus_size=*/102 * j, | |
| /*max_corpus_element_size=*/103 * j, | |
| /*avg_corpus_element_size=*/104 * j, | |
| }, | |
| RusageStats{ | |
| /*engine_rusage_avg_millicores=*/201 * j, | |
| /*engine_rusage_cpu_percent=*/202 * j, | |
| /*engine_rusage_rss_mb=*/203 * j, | |
| /*engine_rusage_vsize_mb=*/204 * j, | |
| }, | |
| }); | |
| } | |
| const std::vector<Environment> env_vec = { | |
| CreateEnv( | |
| /*workdir=*/"", /*experiment_name=*/"Experiment A", | |
| /*experiment_flags=*/"AAA"), | |
| CreateEnv(/*workdir=*/"", /*experiment_name=*/"Experiment B", | |
| /*experiment_flags=*/"BBB"), | |
| CreateEnv( | |
| /*workdir=*/"", /*experiment_name=*/"Experiment A", | |
| /*experiment_flags=*/"AAA"), | |
| CreateEnv(/*workdir=*/"", /*experiment_name=*/"Experiment B", | |
| /*experiment_flags=*/"BBB")}; | |
| StatsLogger stats_logger{stats_vec, env_vec}; | |
| { | |
| const std::vector<std::string_view> kExpectedLogLines = { | |
| "Current stats:", | |
| "Coverage:", | |
| " Experiment A: min:\t21\tmax:\t63\tavg:\t42.0\t--\t21\t63", | |
| " Experiment B: min:\t42\tmax:\t84\tavg:\t63.0\t--\t42\t84", | |
| "Number of executions:", | |
| " Experiment A: min:\t12\tmax:\t36\tavg:\t24.0\t--\t12\t36", | |
| " Experiment B: min:\t24\tmax:\t48\tavg:\t36.0\t--\t24\t48", | |
| "Active corpus size:", | |
| " Experiment A: min:\t101\tmax:\t303\tavg:\t202.0\t--\t101\t303", | |
| " Experiment B: min:\t202\tmax:\t404\tavg:\t303.0\t--\t202\t404", | |
| "Max element size:", | |
| " Experiment A: min:\t103\tmax:\t309\tavg:\t206.0\t--\t103\t309", | |
| " Experiment B: min:\t206\tmax:\t412\tavg:\t309.0\t--\t206\t412", | |
| "Avg element size:", | |
| " Experiment A: min:\t104\tmax:\t312\tavg:\t208.0\t--\t104\t312", | |
| " Experiment B: min:\t208\tmax:\t416\tavg:\t312.0\t--\t208\t416", | |
| "Fuzz time (sec):", | |
| " Experiment A: min:\t10\tmax:\t30\tavg:\t20.0\t--\t10\t30", | |
| " Experiment B: min:\t20\tmax:\t40\tavg:\t30.0\t--\t20\t40", | |
| "Num proxy crashes:", | |
| " Experiment A: min:\t13\tmax:\t39\tsum:\t52\t--\t13\t39", | |
| " Experiment B: min:\t26\tmax:\t52\tsum:\t78\t--\t26\t52", | |
| "Total corpus size:", | |
| " Experiment A: min:\t102\tmax:\t306\tsum:\t408\t--\t102\t306", | |
| " Experiment B: min:\t204\tmax:\t408\tsum:\t612\t--\t204\t408", | |
| "Num 8-bit counter features:", | |
| " Experiment A: min:\t22\tmax:\t66\tavg:\t44.0\t--\t22\t66", | |
| " Experiment B: min:\t44\tmax:\t88\tavg:\t66.0\t--\t44\t88", | |
| "Num data flow features:", | |
| " Experiment A: min:\t23\tmax:\t69\tavg:\t46.0\t--\t23\t69", | |
| " Experiment B: min:\t46\tmax:\t92\tavg:\t69.0\t--\t46\t92", | |
| "Num cmp features:", | |
| " Experiment A: min:\t24\tmax:\t72\tavg:\t48.0\t--\t24\t72", | |
| " Experiment B: min:\t48\tmax:\t96\tavg:\t72.0\t--\t48\t96", | |
| "Num call stack features:", | |
| " Experiment A: min:\t25\tmax:\t75\tavg:\t50.0\t--\t25\t75", | |
| " Experiment B: min:\t50\tmax:\t100\tavg:\t75.0\t--\t50\t100", | |
| "Num bounded path features:", | |
| " Experiment A: min:\t26\tmax:\t78\tavg:\t52.0\t--\t26\t78", | |
| " Experiment B: min:\t52\tmax:\t104\tavg:\t78.0\t--\t52\t104", | |
| "Num PC pair features:", | |
| " Experiment A: min:\t27\tmax:\t81\tavg:\t54.0\t--\t27\t81", | |
| " Experiment B: min:\t54\tmax:\t108\tavg:\t81.0\t--\t54\t108", | |
| "Num user features:", | |
| " Experiment A: min:\t28\tmax:\t84\tavg:\t56.0\t--\t28\t84", | |
| " Experiment B: min:\t56\tmax:\t112\tavg:\t84.0\t--\t56\t112", | |
| "Num unknown features:", | |
| " Experiment A: min:\t29\tmax:\t87\tavg:\t58.0\t--\t29\t87", | |
| " Experiment B: min:\t58\tmax:\t116\tavg:\t87.0\t--\t58\t116", | |
| "Num funcs in frontier:", | |
| " Experiment A: min:\t31\tmax:\t93\tavg:\t62.0\t--\t31\t93", | |
| " Experiment B: min:\t62\tmax:\t124\tavg:\t93.0\t--\t62\t124", | |
| "Num user0 features:", | |
| " Experiment A: min:\t29\tmax:\t85\tavg:\t57.0\t--\t29\t85", | |
| " Experiment B: min:\t57\tmax:\t113\tavg:\t85.0\t--\t57\t113", | |
| "Num user1 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user2 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user3 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user4 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user5 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user6 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user7 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user8 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user9 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user10 features:", | |
| " Experiment A: min:\t30\tmax:\t86\tavg:\t58.0\t--\t30\t86", | |
| " Experiment B: min:\t58\tmax:\t114\tavg:\t86.0\t--\t58\t114", | |
| "Num user11 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user12 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user13 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user14 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user15 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Flags:", | |
| " Experiment A: AAA", | |
| " Experiment B: BBB", | |
| }; | |
| LogCapture log_capture; | |
| stats_logger.ReportCurrStats(); | |
| const auto log_lines = log_capture.CapturedLogLines(); | |
| EXPECT_THAT(log_lines, ElementsAreArray(kExpectedLogLines)) | |
| << "\nActual:" << absl::StrJoin(log_lines, "\n") << "\nExpected:\n" | |
| << absl::StrJoin(kExpectedLogLines, "\n"); | |
| } | |
| { | |
| for (auto &stats : stats_vec) { | |
| auto new_stats = stats.load(); | |
| new_stats.timestamp_unix_micros += 1000000; | |
| new_stats.fuzz_time_sec += 1; | |
| new_stats.num_executions += 1; | |
| new_stats.num_target_crashes += 1; | |
| new_stats.num_covered_pcs += 1; | |
| new_stats.num_8bit_counter_features += 1; | |
| new_stats.num_data_flow_features += 1; | |
| new_stats.num_cmp_features += 1; | |
| new_stats.num_call_stack_features += 1; | |
| new_stats.num_bounded_path_features += 1; | |
| new_stats.num_pc_pair_features += 1; | |
| new_stats.num_user_features += 1; | |
| new_stats.num_user0_features += 1; | |
| new_stats.num_user10_features += 1; | |
| new_stats.num_unknown_features += 1; | |
| new_stats.num_funcs_in_frontier += 1; | |
| new_stats.active_corpus_size += 1; | |
| new_stats.total_corpus_size += 1; | |
| new_stats.max_corpus_element_size += 1; | |
| new_stats.avg_corpus_element_size += 1; | |
| new_stats.engine_rusage_avg_millicores += 1; | |
| new_stats.engine_rusage_cpu_percent += 1; | |
| new_stats.engine_rusage_rss_mb += 1; | |
| new_stats.engine_rusage_vsize_mb += 1; | |
| stats.store(new_stats); | |
| } | |
| const std::vector<std::string_view> kExpectedLogLines = { | |
| "Current stats:", | |
| "Coverage:", | |
| " Experiment A: min:\t22\tmax:\t64\tavg:\t43.0\t--\t22\t64", | |
| " Experiment B: min:\t43\tmax:\t85\tavg:\t64.0\t--\t43\t85", | |
| "Number of executions:", | |
| " Experiment A: min:\t13\tmax:\t37\tavg:\t25.0\t--\t13\t37", | |
| " Experiment B: min:\t25\tmax:\t49\tavg:\t37.0\t--\t25\t49", | |
| "Active corpus size:", | |
| " Experiment A: min:\t102\tmax:\t304\tavg:\t203.0\t--\t102\t304", | |
| " Experiment B: min:\t203\tmax:\t405\tavg:\t304.0\t--\t203\t405", | |
| "Max element size:", | |
| " Experiment A: min:\t104\tmax:\t310\tavg:\t207.0\t--\t104\t310", | |
| " Experiment B: min:\t207\tmax:\t413\tavg:\t310.0\t--\t207\t413", | |
| "Avg element size:", | |
| " Experiment A: min:\t105\tmax:\t313\tavg:\t209.0\t--\t105\t313", | |
| " Experiment B: min:\t209\tmax:\t417\tavg:\t313.0\t--\t209\t417", | |
| "Fuzz time (sec):", | |
| " Experiment A: min:\t11\tmax:\t31\tavg:\t21.0\t--\t11\t31", | |
| " Experiment B: min:\t21\tmax:\t41\tavg:\t31.0\t--\t21\t41", | |
| "Num proxy crashes:", | |
| " Experiment A: min:\t14\tmax:\t40\tsum:\t54\t--\t14\t40", | |
| " Experiment B: min:\t27\tmax:\t53\tsum:\t80\t--\t27\t53", | |
| "Total corpus size:", | |
| " Experiment A: min:\t103\tmax:\t307\tsum:\t410\t--\t103\t307", | |
| " Experiment B: min:\t205\tmax:\t409\tsum:\t614\t--\t205\t409", | |
| "Num 8-bit counter features:", | |
| " Experiment A: min:\t23\tmax:\t67\tavg:\t45.0\t--\t23\t67", | |
| " Experiment B: min:\t45\tmax:\t89\tavg:\t67.0\t--\t45\t89", | |
| "Num data flow features:", | |
| " Experiment A: min:\t24\tmax:\t70\tavg:\t47.0\t--\t24\t70", | |
| " Experiment B: min:\t47\tmax:\t93\tavg:\t70.0\t--\t47\t93", | |
| "Num cmp features:", | |
| " Experiment A: min:\t25\tmax:\t73\tavg:\t49.0\t--\t25\t73", | |
| " Experiment B: min:\t49\tmax:\t97\tavg:\t73.0\t--\t49\t97", | |
| "Num call stack features:", | |
| " Experiment A: min:\t26\tmax:\t76\tavg:\t51.0\t--\t26\t76", | |
| " Experiment B: min:\t51\tmax:\t101\tavg:\t76.0\t--\t51\t101", | |
| "Num bounded path features:", | |
| " Experiment A: min:\t27\tmax:\t79\tavg:\t53.0\t--\t27\t79", | |
| " Experiment B: min:\t53\tmax:\t105\tavg:\t79.0\t--\t53\t105", | |
| "Num PC pair features:", | |
| " Experiment A: min:\t28\tmax:\t82\tavg:\t55.0\t--\t28\t82", | |
| " Experiment B: min:\t55\tmax:\t109\tavg:\t82.0\t--\t55\t109", | |
| "Num user features:", | |
| " Experiment A: min:\t29\tmax:\t85\tavg:\t57.0\t--\t29\t85", | |
| " Experiment B: min:\t57\tmax:\t113\tavg:\t85.0\t--\t57\t113", | |
| "Num unknown features:", | |
| " Experiment A: min:\t30\tmax:\t88\tavg:\t59.0\t--\t30\t88", | |
| " Experiment B: min:\t59\tmax:\t117\tavg:\t88.0\t--\t59\t117", | |
| "Num funcs in frontier:", | |
| " Experiment A: min:\t32\tmax:\t94\tavg:\t63.0\t--\t32\t94", | |
| " Experiment B: min:\t63\tmax:\t125\tavg:\t94.0\t--\t63\t125", | |
| "Num user0 features:", | |
| " Experiment A: min:\t30\tmax:\t86\tavg:\t58.0\t--\t30\t86", | |
| " Experiment B: min:\t58\tmax:\t114\tavg:\t86.0\t--\t58\t114", | |
| "Num user1 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user2 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user3 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user4 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user5 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user6 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user7 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user8 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user9 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user10 features:", | |
| " Experiment A: min:\t31\tmax:\t87\tavg:\t59.0\t--\t31\t87", | |
| " Experiment B: min:\t59\tmax:\t115\tavg:\t87.0\t--\t59\t115", | |
| "Num user11 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user12 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user13 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user14 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Num user15 features:", | |
| " Experiment A: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| " Experiment B: min:\t0\tmax:\t0\tavg:\t0.0\t--\t0\t0", | |
| "Flags:", | |
| " Experiment A: AAA", | |
| " Experiment B: BBB", | |
| }; | |
| LogCapture log_capture; | |
| stats_logger.ReportCurrStats(); | |
| const auto log_lines = log_capture.CapturedLogLines(); | |
| EXPECT_THAT(log_lines, ElementsAreArray(kExpectedLogLines)); | |
| } | |
| } | |
| TEST(Stats, DumpStatsToCsvFile) { | |
| const std::filesystem::path workdir = GetTestTempDir(test_info_->name()); | |
| std::vector<std::atomic<Stats>> stats_vec(4); | |
| for (uint64_t i = 0; i < stats_vec.size(); ++i) { | |
| const uint64_t j = i + 1; | |
| CovStats cov_stats{/*num_covered_pcs=*/ | |
| 21 * j, | |
| /*num_8bit_counter_features=*/22 * j, | |
| /*num_data_flow_features=*/23 * j, | |
| /*num_cmp_features=*/24 * j, | |
| /*num_call_stack_features=*/25 * j, | |
| /*num_bounded_path_features=*/26 * j, | |
| /*num_pc_pair_features=*/27 * j, | |
| /*num_user_features=*/28 * j}; | |
| // NOTE: These two don't add up to `num_user_features`, but that's | |
| // ok for the purposes of this test. | |
| cov_stats.num_user0_features = 28 * j + 1; | |
| cov_stats.num_user10_features = 28 * j + 2; | |
| cov_stats.num_unknown_features = 29 * j; | |
| cov_stats.num_funcs_in_frontier = 31 * j; | |
| stats_vec[i].store(Stats{ | |
| StatsMeta{/*timestamp_unix_micros=*/1000000 * j}, | |
| ExecStats{ | |
| /*fuzz_time_sec=*/10 * j, | |
| /*.num_executions=*/12 * j, | |
| /*.num_target_crashes=*/13 * j, | |
| }, | |
| cov_stats, | |
| CorpusStats{ | |
| /*active_corpus_size=*/101 * j, | |
| /*total_corpus_size=*/102 * j, | |
| /*max_corpus_element_size=*/103 * j, | |
| /*avg_corpus_element_size=*/104 * j, | |
| }, | |
| RusageStats{ | |
| /*engine_rusage_avg_millicores=*/201 * j, | |
| /*engine_rusage_cpu_percent=*/202 * j, | |
| /*engine_rusage_rss_mb=*/203 * j, | |
| /*engine_rusage_vsize_mb=*/204 * j, | |
| }, | |
| }); | |
| } | |
| const std::vector<Environment> env_vec = { | |
| CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentA", | |
| /*experiment_flags=*/"AAA"), | |
| CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentB", | |
| /*experiment_flags=*/"BBB"), | |
| CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentA", | |
| /*experiment_flags=*/"AAA"), | |
| CreateEnv(workdir.c_str(), /*experiment_name=*/"ExperimentB", | |
| /*experiment_flags=*/"BBB")}; | |
| { | |
| StatsCsvFileAppender stats_csv_appender{stats_vec, env_vec}; | |
| stats_csv_appender.ReportCurrStats(); | |
| // Emulate progress in shard #2 of each experiment. In the second line of | |
| // each CSV, min's shouldn't change, max's should increase by 1, avg's | |
| // should increase by 0.5. | |
| for (int i = 2; i < stats_vec.size(); ++i) { | |
| auto &stats = stats_vec[i]; | |
| auto new_stats = stats.load(); | |
| new_stats.timestamp_unix_micros += 1; | |
| new_stats.fuzz_time_sec += 1; | |
| new_stats.num_executions += 1; | |
| new_stats.num_target_crashes += 1; | |
| new_stats.num_covered_pcs += 1; | |
| new_stats.num_8bit_counter_features += 1; | |
| new_stats.num_data_flow_features += 1; | |
| new_stats.num_cmp_features += 1; | |
| new_stats.num_call_stack_features += 1; | |
| new_stats.num_bounded_path_features += 1; | |
| new_stats.num_pc_pair_features += 1; | |
| new_stats.num_user_features += 1; | |
| new_stats.num_user0_features += 1; | |
| new_stats.num_user10_features += 1; | |
| new_stats.num_unknown_features += 1; | |
| new_stats.num_funcs_in_frontier += 1; | |
| new_stats.active_corpus_size += 1; | |
| new_stats.total_corpus_size += 1; | |
| new_stats.max_corpus_element_size += 1; | |
| new_stats.avg_corpus_element_size += 1; | |
| new_stats.engine_rusage_avg_millicores += 1; | |
| new_stats.engine_rusage_cpu_percent += 1; | |
| new_stats.engine_rusage_rss_mb += 1; | |
| new_stats.engine_rusage_vsize_mb += 1; | |
| stats.store(new_stats); | |
| } | |
| stats_csv_appender.ReportCurrStats(); | |
| } | |
| const std::vector<std::string> kExpectedCsvs = { | |
| workdir / "fuzzing-stats-.000000.ExperimentA.csv", | |
| workdir / "fuzzing-stats-.000000.ExperimentB.csv", | |
| }; | |
| const std::vector<std::vector<std::string>> kExpectedCsvLines = { | |
| // CSV #1. | |
| { | |
| // clang-format off | |
| // Header. | |
| "NumCoveredPcs_Min,NumCoveredPcs_Max,NumCoveredPcs_Avg," | |
| "NumExecs_Min,NumExecs_Max,NumExecs_Avg," | |
| "ActiveCorpusSize_Min,ActiveCorpusSize_Max,ActiveCorpusSize_Avg," | |
| "MaxEltSize_Min,MaxEltSize_Max,MaxEltSize_Avg," | |
| "AvgEltSize_Min,AvgEltSize_Max,AvgEltSize_Avg," | |
| "UnixMicros_Min,UnixMicros_Max," | |
| "FuzzTimeSec_Min,FuzzTimeSec_Max,FuzzTimeSec_Avg," | |
| "NumProxyCrashes_Min,NumProxyCrashes_Max,NumProxyCrashes_Sum," | |
| "TotalCorpusSize_Min,TotalCorpusSize_Max,TotalCorpusSize_Sum," | |
| "Num8BitCounterFts_Min,Num8BitCounterFts_Max,Num8BitCounterFts_Avg," | |
| "NumDataFlowFts_Min,NumDataFlowFts_Max,NumDataFlowFts_Avg," | |
| "NumCmpFts_Min,NumCmpFts_Max,NumCmpFts_Avg," | |
| "NumCallStackFts_Min,NumCallStackFts_Max,NumCallStackFts_Avg," | |
| "NumBoundedPathFts_Min,NumBoundedPathFts_Max,NumBoundedPathFts_Avg," | |
| "NumPcPairFts_Min,NumPcPairFts_Max,NumPcPairFts_Avg," | |
| "NumUserFts_Min,NumUserFts_Max,NumUserFts_Avg," | |
| "NumUnknownFts_Min,NumUnknownFts_Max,NumUnknownFts_Avg," | |
| "NumFuncsInFrontier_Min,NumFuncsInFrontier_Max,NumFuncsInFrontier_Avg," // NOLINT | |
| "EngineRusageAvgCores_Max,EngineRusageCpuPct_Max," | |
| "EngineRusageRssMb_Max,EngineRusageVSizeMb_Max," | |
| "NumUser0Fts_Min,NumUser0Fts_Max,NumUser0Fts_Avg," | |
| "NumUser1Fts_Min,NumUser1Fts_Max,NumUser1Fts_Avg," | |
| "NumUser2Fts_Min,NumUser2Fts_Max,NumUser2Fts_Avg," | |
| "NumUser3Fts_Min,NumUser3Fts_Max,NumUser3Fts_Avg," | |
| "NumUser4Fts_Min,NumUser4Fts_Max,NumUser4Fts_Avg," | |
| "NumUser5Fts_Min,NumUser5Fts_Max,NumUser5Fts_Avg," | |
| "NumUser6Fts_Min,NumUser6Fts_Max,NumUser6Fts_Avg," | |
| "NumUser7Fts_Min,NumUser7Fts_Max,NumUser7Fts_Avg," | |
| "NumUser8Fts_Min,NumUser8Fts_Max,NumUser8Fts_Avg," | |
| "NumUser9Fts_Min,NumUser9Fts_Max,NumUser9Fts_Avg," | |
| "NumUser10Fts_Min,NumUser10Fts_Max,NumUser10Fts_Avg," | |
| "NumUser11Fts_Min,NumUser11Fts_Max,NumUser11Fts_Avg," | |
| "NumUser12Fts_Min,NumUser12Fts_Max,NumUser12Fts_Avg," | |
| "NumUser13Fts_Min,NumUser13Fts_Max,NumUser13Fts_Avg," | |
| "NumUser14Fts_Min,NumUser14Fts_Max,NumUser14Fts_Avg," | |
| "NumUser15Fts_Min,NumUser15Fts_Max,NumUser15Fts_Avg,", | |
| // Line 1. | |
| "21,63,42.0," | |
| "12,36,24.0," | |
| "101,303,202.0," | |
| "103,309,206.0," | |
| "104,312,208.0," | |
| "1000000,3000000," | |
| "10,30,20.0," | |
| "13,39,52," | |
| "102,306,408," | |
| "22,66,44.0," | |
| "23,69,46.0," | |
| "24,72,48.0," | |
| "25,75,50.0," | |
| "26,78,52.0," | |
| "27,81,54.0," | |
| "28,84,56.0," | |
| "29,87,58.0," | |
| "31,93,62.0," | |
| "603,606," | |
| "609,612," | |
| "29,85,57.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "30,86,58.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0,", | |
| // Line 2. | |
| "21,64,42.5," | |
| "12,37,24.5," | |
| "101,304,202.5," | |
| "103,310,206.5," | |
| "104,313,208.5," | |
| "1000000,3000001," | |
| "10,31,20.5," | |
| "13,40,53," | |
| "102,307,409," | |
| "22,67,44.5," | |
| "23,70,46.5," | |
| "24,73,48.5," | |
| "25,76,50.5," | |
| "26,79,52.5," | |
| "27,82,54.5," | |
| "28,85,56.5," | |
| "29,88,58.5," | |
| "31,94,62.5," | |
| "604,607," | |
| "610,613," | |
| "29,86,57.5," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "30,87,58.5," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0,", | |
| "", // empty line at EOF | |
| // clang-format on | |
| }, | |
| // CSV #2. | |
| { | |
| // clang-format off | |
| // Header. | |
| "NumCoveredPcs_Min,NumCoveredPcs_Max,NumCoveredPcs_Avg," | |
| "NumExecs_Min,NumExecs_Max,NumExecs_Avg," | |
| "ActiveCorpusSize_Min,ActiveCorpusSize_Max,ActiveCorpusSize_Avg," | |
| "MaxEltSize_Min,MaxEltSize_Max,MaxEltSize_Avg," | |
| "AvgEltSize_Min,AvgEltSize_Max,AvgEltSize_Avg," | |
| "UnixMicros_Min,UnixMicros_Max," | |
| "FuzzTimeSec_Min,FuzzTimeSec_Max,FuzzTimeSec_Avg," | |
| "NumProxyCrashes_Min,NumProxyCrashes_Max,NumProxyCrashes_Sum," | |
| "TotalCorpusSize_Min,TotalCorpusSize_Max,TotalCorpusSize_Sum," | |
| "Num8BitCounterFts_Min,Num8BitCounterFts_Max,Num8BitCounterFts_Avg," | |
| "NumDataFlowFts_Min,NumDataFlowFts_Max,NumDataFlowFts_Avg," | |
| "NumCmpFts_Min,NumCmpFts_Max,NumCmpFts_Avg," | |
| "NumCallStackFts_Min,NumCallStackFts_Max,NumCallStackFts_Avg," | |
| "NumBoundedPathFts_Min,NumBoundedPathFts_Max,NumBoundedPathFts_Avg," | |
| "NumPcPairFts_Min,NumPcPairFts_Max,NumPcPairFts_Avg," | |
| "NumUserFts_Min,NumUserFts_Max,NumUserFts_Avg," | |
| "NumUnknownFts_Min,NumUnknownFts_Max,NumUnknownFts_Avg," | |
| "NumFuncsInFrontier_Min,NumFuncsInFrontier_Max,NumFuncsInFrontier_Avg," // NOLINT | |
| "EngineRusageAvgCores_Max,EngineRusageCpuPct_Max," | |
| "EngineRusageRssMb_Max,EngineRusageVSizeMb_Max," | |
| "NumUser0Fts_Min,NumUser0Fts_Max,NumUser0Fts_Avg," | |
| "NumUser1Fts_Min,NumUser1Fts_Max,NumUser1Fts_Avg," | |
| "NumUser2Fts_Min,NumUser2Fts_Max,NumUser2Fts_Avg," | |
| "NumUser3Fts_Min,NumUser3Fts_Max,NumUser3Fts_Avg," | |
| "NumUser4Fts_Min,NumUser4Fts_Max,NumUser4Fts_Avg," | |
| "NumUser5Fts_Min,NumUser5Fts_Max,NumUser5Fts_Avg," | |
| "NumUser6Fts_Min,NumUser6Fts_Max,NumUser6Fts_Avg," | |
| "NumUser7Fts_Min,NumUser7Fts_Max,NumUser7Fts_Avg," | |
| "NumUser8Fts_Min,NumUser8Fts_Max,NumUser8Fts_Avg," | |
| "NumUser9Fts_Min,NumUser9Fts_Max,NumUser9Fts_Avg," | |
| "NumUser10Fts_Min,NumUser10Fts_Max,NumUser10Fts_Avg," | |
| "NumUser11Fts_Min,NumUser11Fts_Max,NumUser11Fts_Avg," | |
| "NumUser12Fts_Min,NumUser12Fts_Max,NumUser12Fts_Avg," | |
| "NumUser13Fts_Min,NumUser13Fts_Max,NumUser13Fts_Avg," | |
| "NumUser14Fts_Min,NumUser14Fts_Max,NumUser14Fts_Avg," | |
| "NumUser15Fts_Min,NumUser15Fts_Max,NumUser15Fts_Avg,", | |
| // Line 1. | |
| "42,84,63.0," | |
| "24,48,36.0," | |
| "202,404,303.0," | |
| "206,412,309.0," | |
| "208,416,312.0," | |
| "2000000,4000000," | |
| "20,40,30.0," | |
| "26,52,78," | |
| "204,408,612," | |
| "44,88,66.0," | |
| "46,92,69.0," | |
| "48,96,72.0," | |
| "50,100,75.0," | |
| "52,104,78.0," | |
| "54,108,81.0," | |
| "56,112,84.0," | |
| "58,116,87.0," | |
| "62,124,93.0," | |
| "804,808," | |
| "812,816," | |
| "57,113,85.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "58,114,86.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0,", | |
| // Line 2. | |
| "42,85,63.5," | |
| "24,49,36.5," | |
| "202,405,303.5," | |
| "206,413,309.5," | |
| "208,417,312.5," | |
| "2000000,4000001," | |
| "20,41,30.5," | |
| "26,53,79," | |
| "204,409,613," | |
| "44,89,66.5," | |
| "46,93,69.5," | |
| "48,97,72.5," | |
| "50,101,75.5," | |
| "52,105,78.5," | |
| "54,109,81.5," | |
| "56,113,84.5," | |
| "58,117,87.5," | |
| "62,125,93.5," | |
| "805,809," | |
| "813,817," | |
| "57,114,85.5," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "58,115,86.5," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0," | |
| "0,0,0.0,", | |
| "", // empty line at EOF | |
| // clang-format on | |
| }, | |
| }; | |
| for (int i = 0; i < 2; ++i) { | |
| ASSERT_TRUE(std::filesystem::exists(kExpectedCsvs[i])); | |
| std::string csv_contents; | |
| ReadFromLocalFile(kExpectedCsvs[i], csv_contents); | |
| const std::vector<std::string_view> csv_lines = | |
| absl::StrSplit(csv_contents, '\n'); | |
| ASSERT_EQ(csv_lines.size(), kExpectedCsvLines[i].size()); | |
| for (size_t j = 0; j < csv_lines.size(); ++j) { | |
| EXPECT_EQ(csv_lines[j], kExpectedCsvLines[i][j]) << VV(i) << VV(j); | |
| } | |
| } | |
| } | |
| TEST(Stats, DumpStatsToExistingCsvFile) { | |
| class TestStatsCsvFileAppender : public StatsCsvFileAppender { | |
| using StatsCsvFileAppender::StatsCsvFileAppender; | |
| private: | |
| std::string GetBackupFilename(const std::string &filename) const override { | |
| return absl::StrCat(filename, ".bak"); | |
| } | |
| }; | |
| const std::filesystem::path workdir = GetTestTempDir(test_info_->name()); | |
| const std::vector<Environment> env_vec = {CreateEnv(workdir.c_str()), | |
| CreateEnv(workdir.c_str())}; | |
| std::vector<std::atomic<Stats>> stats_vec(2); | |
| stats_vec[0].store({StatsMeta{/*timestamp_unix_micros=*/1000000}}); | |
| stats_vec[1].store({StatsMeta{/*timestamp_unix_micros=*/2000000}}); | |
| const std::string kExpectedCsv = workdir / "fuzzing-stats-.000000.csv"; | |
| const std::string kExpectedCsvBak = workdir / "fuzzing-stats-.000000.csv.bak"; | |
| // `StatsCsvFileAppender` creates a brand-new fuzzing-stats file and writes | |
| // the CSV header and a 1st stats line to it. | |
| { | |
| TestStatsCsvFileAppender stats_csv_appender{stats_vec, env_vec}; | |
| stats_csv_appender.ReportCurrStats(); | |
| ASSERT_TRUE(std::filesystem::exists(kExpectedCsv)); | |
| ASSERT_FALSE(std::filesystem::exists(kExpectedCsvBak)); | |
| std::string contents; | |
| ReadFromLocalFile(kExpectedCsv, contents); | |
| const std::vector<std::string> lines = absl::StrSplit(contents, '\n'); | |
| // Header + 1 stats line + empty line at EOF. | |
| EXPECT_EQ(lines.size(), 3); | |
| } | |
| // `StatsCsvFileAppender` finds an existing file with a CSV header matching | |
| // the current version and appends a 2nd stats line to it. | |
| { | |
| stats_vec[0].store({StatsMeta{/*timestamp_unix_micros=*/3000000}}); | |
| stats_vec[1].store({StatsMeta{/*timestamp_unix_micros=*/4000000}}); | |
| TestStatsCsvFileAppender stats_csv_appender{stats_vec, env_vec}; | |
| stats_csv_appender.ReportCurrStats(); | |
| ASSERT_TRUE(std::filesystem::exists(kExpectedCsv)); | |
| ASSERT_FALSE(std::filesystem::exists(kExpectedCsvBak)); | |
| std::string contents; | |
| ReadFromLocalFile(kExpectedCsv, contents); | |
| const std::vector<std::string> lines = absl::StrSplit(contents, '\n'); | |
| // Header + 2 stats lines + empty line at EOF. | |
| EXPECT_EQ(lines.size(), 4); | |
| } | |
| // `StatsCsvFileAppender` finds an existing file with a CSV header not | |
| // matching the current version (ostensibly created by a previous version of | |
| // Centipede), creates a backup copy of it, and starts a new file scratch, | |
| // writing the new CSV header and a 1st line to it. | |
| { | |
| // Fake a CSV with an outdated header and contents. | |
| const std::vector<std::string> kFakeOldCsvLines = { | |
| "Old,Csv,Header,", "0,1,2,", "10,11,12,", "20,21,22,", "", | |
| }; | |
| WriteToLocalFile(kExpectedCsv, absl::StrJoin(kFakeOldCsvLines, "\n")); | |
| stats_vec[0].store({StatsMeta{/*timestamp_unix_micros=*/5000000}}); | |
| stats_vec[1].store({StatsMeta{/*timestamp_unix_micros=*/6000000}}); | |
| TestStatsCsvFileAppender stats_csv_appender{stats_vec, env_vec}; | |
| stats_csv_appender.ReportCurrStats(); | |
| ASSERT_TRUE(std::filesystem::exists(kExpectedCsv)); | |
| ASSERT_TRUE(std::filesystem::exists(kExpectedCsvBak)); | |
| std::string contents; | |
| ReadFromLocalFile(kExpectedCsv, contents); | |
| const std::vector<std::string> lines = absl::StrSplit(contents, '\n'); | |
| // Header + 1 stats line + empty line at EOF. | |
| EXPECT_EQ(lines.size(), 3); | |
| std::string contents_bak; | |
| ReadFromLocalFile(kExpectedCsvBak, contents_bak); | |
| const std::vector<std::string> lines_bak = | |
| absl::StrSplit(contents_bak, '\n'); | |
| FUZZTEST_CHECK_NE(lines.size(), | |
| kFakeOldCsvLines.size()); // Prevent false positives | |
| EXPECT_EQ(lines_bak.size(), kFakeOldCsvLines.size()); | |
| } | |
| } | |
| TEST(Stats, PrintRewardValues) { | |
| std::stringstream ss; | |
| std::vector<std::atomic<Stats>> stats_vec(4); | |
| stats_vec[0].store( | |
| {StatsMeta{}, ExecStats{}, CovStats{/*num_covered_pcs=*/15}}); | |
| stats_vec[1].store( | |
| {StatsMeta{}, ExecStats{}, CovStats{/*num_covered_pcs=*/10}}); | |
| stats_vec[2].store( | |
| {StatsMeta{}, ExecStats{}, CovStats{/*num_covered_pcs=*/40}}); | |
| stats_vec[3].store( | |
| {StatsMeta{}, ExecStats{}, CovStats{/*num_covered_pcs=*/25}}); | |
| PrintRewardValues(stats_vec, ss); | |
| const char *expected = | |
| "REWARD_MAX 40\n" | |
| "REWARD_SECOND_MAX 25\n" | |
| "REWARD_MIN 10\n" | |
| "REWARD_MEDIAN 25\n" | |
| "REWARD_AVERAGE 22.5\n"; | |
| EXPECT_EQ(ss.str(), expected); | |
| } | |
| } // namespace | |
| } // namespace fuzztest::internal |