ml_benchmark/json_serializer_test.cc

// Copyright 2020 The ChromiumOS Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "ml_benchmark/json_serializer.h"

#include <gmock/gmock.h>
#include <gtest/gtest.h>

#include <optional>
#include <string>
#include <utility>

#include <base/check.h>

using chrome::ml_benchmark::BenchmarkResults;
using chrome::ml_benchmark::Metric;
using ::testing::Eq;
using ::testing::Pointee;

namespace ml_benchmark {

TEST(BenchmarkResultsToJson, Basics) {
  BenchmarkResults results;
  results.set_status(chrome::ml_benchmark::RUNTIME_ERROR);
  results.set_results_message("Test error");

  const std::optional<base::Value::Dict> json =
      ml_benchmark::BenchmarkResultsToJson(results);
  ASSERT_TRUE(json);
  EXPECT_EQ(json->FindInt("status"), chrome::ml_benchmark::RUNTIME_ERROR);
  EXPECT_THAT(json->FindString("results_message"), Pointee(Eq("Test error")));
}

TEST(BenchmarkResultsToJson, Percentiles) {
  BenchmarkResults results;
  auto& latency_map = *results.mutable_percentile_latencies_in_us();
  latency_map[50] = 1000;
  latency_map[90] = 2000;
  latency_map[95] = 3000;
  latency_map[99] = 4000;

  const std::optional<base::Value::Dict> json =
      ml_benchmark::BenchmarkResultsToJson(results);
  ASSERT_TRUE(json);
  const base::Value::Dict* latencies =
      json->FindDict("percentile_latencies_in_us");
  ASSERT_TRUE(json);
  EXPECT_EQ(latencies->FindInt("50"), 1000);
  EXPECT_EQ(latencies->FindInt("90"), 2000);
  EXPECT_EQ(latencies->FindInt("95"), 3000);
  EXPECT_EQ(latencies->FindInt("99"), 4000);
}

TEST(BenchmarkResultsToJson, Metrics) {
  BenchmarkResults results;

  {
    Metric* m = results.add_metrics();
    m->set_name("Multiple ms metric");
    m->set_units(Metric::MS);
    m->set_cardinality(Metric::MULTIPLE);

    m->add_values(1);
    m->add_values(2);
    m->add_values(3);
  }

  {
    Metric* m = results.add_metrics();
    m->set_name("Single unitless metric");
    m->set_direction(Metric::BIGGER_IS_BETTER);
    // UNITLESS + Cardinality::SINGLE by default.
    m->add_values(42);
  }

  const std::optional<base::Value::Dict> json =
      ml_benchmark::BenchmarkResultsToJson(results);
  ASSERT_TRUE(json);
  const base::Value::List* metrics = json->FindList("metrics");
  EXPECT_EQ(metrics->size(), 2);

  {
    assert((*metrics)[0].is_dict());
    const auto& m = (*metrics)[0].GetDict();
    EXPECT_THAT(m.FindString("name"), Pointee(Eq("Multiple ms metric")));
    EXPECT_THAT(m.FindString("units"), Pointee(Eq("ms")));
    EXPECT_THAT(m.FindString("improvement_direction"),
                Pointee(Eq("smaller_is_better")));
    EXPECT_THAT(m.FindString("cardinality"), Pointee(Eq("multiple")));

    const base::Value::List* values = m.FindList("values");
    ASSERT_TRUE(values);
    EXPECT_EQ(values->size(), 3);
    EXPECT_EQ((*values)[0].GetIfDouble(), 1);
    EXPECT_EQ((*values)[1].GetIfDouble(), 2);
    EXPECT_EQ((*values)[2].GetIfDouble(), 3);
  }

  {
    assert((*metrics)[1].is_dict());
    const auto& m = (*metrics)[1].GetDict();
    EXPECT_THAT(m.FindString("name"), Pointee(Eq("Single unitless metric")));
    EXPECT_THAT(m.FindString("units"), Pointee(Eq("unitless")));
    EXPECT_THAT(m.FindString("improvement_direction"),
                Pointee(Eq("bigger_is_better")));
    EXPECT_THAT(m.FindString("cardinality"), Pointee(Eq("single")));

    const base::Value::List* values = m.FindList("values");
    ASSERT_TRUE(values);
    EXPECT_EQ(values->size(), 1);
    EXPECT_EQ((*values)[0].GetIfDouble(), 42);
  }
}

TEST(BenchmarkResultsToJson, MetricsCardinality) {
  auto get_metrics_size =
      [](const BenchmarkResults& results) -> std::optional<size_t> {
    const std::optional<base::Value::Dict> json =
        ml_benchmark::BenchmarkResultsToJson(results);
    if (!json)
      return std::nullopt;

    const base::Value::List* metrics = json->FindList("metrics");
    CHECK(metrics);
    CHECK(!metrics->empty());
    CHECK((*metrics)[0].is_dict());
    const auto& m = (*metrics)[0].GetDict();
    const base::Value::List* values = m.FindList("values");
    CHECK(values);
    return values->size();
  };

  {
    BenchmarkResults results;
    Metric* m = results.add_metrics();
    m->set_cardinality(Metric::MULTIPLE);
    m->add_values(1);
    m->add_values(2);
    m->add_values(3);
    EXPECT_EQ(get_metrics_size(results), 3);
  }

  {
    BenchmarkResults results;
    Metric* m = results.add_metrics();
    m->set_cardinality(Metric::MULTIPLE);
    // No results is OK here.
    EXPECT_EQ(get_metrics_size(results), 0);
  }

  {
    BenchmarkResults results;
    Metric* m = results.add_metrics();
    m->set_cardinality(Metric::SINGLE);
    m->add_values(1);
    EXPECT_EQ(get_metrics_size(results), 1);
  }

  {
    BenchmarkResults results;
    Metric* m = results.add_metrics();
    m->set_cardinality(Metric::SINGLE);
    // Three results instead of a single one is not OK.
    m->add_values(1);
    m->add_values(2);
    m->add_values(3);
    EXPECT_EQ(get_metrics_size(results), std::nullopt);
  }

  {
    BenchmarkResults results;
    Metric* m = results.add_metrics();
    m->set_cardinality(Metric::SINGLE);
    // No results instead of a single one is not OK.
    EXPECT_EQ(get_metrics_size(results), std::nullopt);
  }
}

TEST(BenchmarkResultsToJson, PowerNormalizationFactor) {
  BenchmarkResults results;
  results.set_power_normalization_factor(100);

  const std::optional<base::Value::Dict> json =
      ml_benchmark::BenchmarkResultsToJson(results);
  ASSERT_TRUE(json);
  EXPECT_EQ(json->FindDouble("power_normalization_factor"), 100);
}

}  // namespace ml_benchmark