blob: 902fbfdedf493354e72dc5ce97d1335ccf061a2d [file] [log] [blame]
/* Copyright 2019 Google LLC. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef RUY_RUY_TEST_H_
#define RUY_RUY_TEST_H_
#include <math.h>
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <ctime>
#include <iostream>
#include <iterator>
#include <limits>
#include <memory>
#include <random>
#include <set>
#include <sstream>
#include <string>
#include <tuple>
#include <type_traits>
#include <vector>
#include "ruy/allocator.h"
#include "ruy/context.h"
#include "ruy/context_get_ctx.h"
#include "ruy/ctx.h"
#include "ruy/gtest_wrapper.h" // IWYU pragma: export
#include "ruy/matrix.h" // IWYU pragma: export
#include "ruy/mul_params.h" // IWYU pragma: export
#include "ruy/platform.h"
#include "ruy/pmu.h"
#include "ruy/reference_mul.h"
#include "ruy/ruy.h"
#include "ruy/time.h"
#ifdef RUY_TEST_EXTERNAL_PATHS
#define EIGEN_USE_THREADS
#define EIGEN_USE_CUSTOM_THREAD_POOL
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#include "third_party/eigen3/Eigen/Core"
#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
#pragma GCC diagnostic pop
#include "third_party/gemmlowp/public/gemmlowp.h"
#include "third_party/lapack/blas.h"
#endif
#ifdef RUY_PROFILER
#include "ruy/profiler/profiler.h"
#endif
namespace ruy {
const float kClampRatio = 0.1f;
enum class ExternalPath {
kNone,
kReference,
kGemmlowp,
kEigen,
kEigenTensor,
kOpenBlas
};
inline std::vector<std::string>* CoveredPaths() {
static std::vector<std::string> covered_paths;
return &covered_paths;
}
inline const char* PathName(Path path) {
#define RUY_PATHNAME_CASE(NAME) \
case Path::NAME: \
return #NAME;
switch (path) {
RUY_PATHNAME_CASE(kStandardCpp)
#if RUY_PLATFORM_NEON
RUY_PATHNAME_CASE(kNeon)
RUY_PATHNAME_CASE(kNeonDotprod)
#elif RUY_PLATFORM_X86
RUY_PATHNAME_CASE(kSse42)
RUY_PATHNAME_CASE(kAvx2)
RUY_PATHNAME_CASE(kAvx512)
RUY_PATHNAME_CASE(kAvxVnni)
#endif
default:
RUY_CHECK(false);
return nullptr;
}
#undef RUY_PATHNAME_CASE
}
inline const char* TuningName(Tuning tuning) {
#define RUY_SUBPATHNAME_CASE(NAME) \
case Tuning::NAME: \
return #NAME;
switch (tuning) {
RUY_SUBPATHNAME_CASE(kInOrder)
RUY_SUBPATHNAME_CASE(kOutOfOrder)
default:
RUY_CHECK(false);
return nullptr;
}
#undef RUY_SUBPATHNAME_CASE
}
inline const char* PathName(ExternalPath path) {
#define RUY_PATHNAME_CASE(NAME) \
case ExternalPath::NAME: \
return #NAME;
switch (path) {
RUY_PATHNAME_CASE(kReference)
RUY_PATHNAME_CASE(kGemmlowp)
RUY_PATHNAME_CASE(kEigen)
RUY_PATHNAME_CASE(kEigenTensor)
RUY_PATHNAME_CASE(kOpenBlas)
default:
RUY_CHECK(false);
return nullptr;
}
#undef RUY_PATHNAME_CASE
}
inline std::ostream& operator<<(std::ostream& stream, Path path) {
return stream << PathName(path);
}
inline std::ostream& operator<<(std::ostream& stream,
ExternalPath external_path) {
return stream << PathName(external_path);
}
template <typename ContainerType>
std::string Join(const ContainerType& container) {
if (container.empty()) {
return "<empty>";
}
std::ostringstream stream;
auto it = container.begin();
stream << *it++;
for (; it != container.end(); ++it) {
stream << ", ";
stream << *it;
}
return stream.str();
}
struct LogCoveredPathsOnDestruction final {
~LogCoveredPathsOnDestruction() {
std::cerr << "Covered paths: " << Join(*CoveredPaths()) << std::endl;
// When we know that it would be abnormal for some path not to be covered,
// we check it here. Accidentally disabling SIMD paths has occurred in the
// past and is one of the biggest performance regressions imaginable.
//
// TODO: we should be able to require some x86 paths as well, at least
// SSE4.2.
#if RUY_PLATFORM_ARM
// When testing on ARM32 or ARM64, make sure that we covered the NEON path.
// NEON is always available on ARM64, and we treat it as always available
// also on ARM32.
bool found_neon = false;
for (const std::string& covered_path : *CoveredPaths()) {
if (covered_path == "kNeon") {
found_neon = true;
}
}
if (!found_neon) {
std::cerr
<< "Error: we haven't tested the kNeon path as we should have.\n"
<< std::endl;
abort();
}
#endif
// When testing on ARM64 ChromiumOS emulator, make sure that we covered
// the dotprod path. We're getting such coverage at the moment thanks to
// using a sufficiently recent emulator, and we don't want to regress that.
#if RUY_PLATFORM_ARM_64 && defined RUY_TESTING_ON_CHROMIUMOS
bool found_dotprod = false;
for (const std::string& covered_path : *CoveredPaths()) {
if (covered_path == "kNeonDotprod") {
found_dotprod = true;
}
}
if (!found_dotprod) {
std::cerr
<< "Error: we haven't tested the kNeonDotprod path as we should "
"have. At the moment, this is required on ChromiumOS as this is "
"what we run emulator tests in, that currently supports "
"dot-product "
"instructions, and we care very much about not regressing that. "
"If this test was run in an emulator, please upgrade to a newer "
"emulator version. If this test was run on an actual device, and "
"you need to be able to run ruy tests on devices not supporting "
"dot-product instructions, get in touch with us.\n"
<< std::endl;
abort();
}
#endif
}
static void Singleton() { static LogCoveredPathsOnDestruction singleton; }
};
enum class RandomRange {
kGeneral,
kAvoidMinValue,
kOffCenterAvoidMinValue,
kReasonableSrcZeroPoint,
kReasonableDstZeroPoint,
kBias
};
template <typename Scalar,
bool IsFloatingPoint = std::is_floating_point<Scalar>::value>
struct RandomRangeBounds {};
template <typename Scalar>
struct RandomRangeBounds<Scalar, true> {
static Scalar GetMinBound(RandomRange range) {
switch (range) {
case RandomRange::kGeneral:
return -1;
case RandomRange::kAvoidMinValue:
return -1;
case RandomRange::kOffCenterAvoidMinValue:
return -1;
case RandomRange::kReasonableSrcZeroPoint:
return 0;
case RandomRange::kReasonableDstZeroPoint:
return 0;
case RandomRange::kBias:
return -1;
default:
RUY_CHECK(false);
return 0;
}
}
static Scalar GetMaxBound(RandomRange range) {
switch (range) {
case RandomRange::kGeneral:
return 1;
case RandomRange::kAvoidMinValue:
return 1;
case RandomRange::kOffCenterAvoidMinValue:
return 1;
case RandomRange::kReasonableSrcZeroPoint:
return 0;
case RandomRange::kReasonableDstZeroPoint:
return 0;
case RandomRange::kBias:
return 1;
default:
RUY_CHECK(false);
return 0;
}
}
};
template <typename Scalar>
Scalar WeightedSum(Scalar s1, float weight1, Scalar s2, float weight2) {
float sum = s1 * weight1 + s2 * weight2;
float clamped = std::min<float>(
std::numeric_limits<Scalar>::max(),
std::max<float>(std::numeric_limits<Scalar>::lowest(), sum));
return static_cast<Scalar>(clamped);
}
template <typename Scalar>
Scalar Parametrized(float param) {
return WeightedSum(std::numeric_limits<Scalar>::max(), param,
std::numeric_limits<Scalar>::lowest(), 1 - param);
}
template <typename Scalar>
struct RandomRangeBounds<Scalar, false> {
static Scalar GetMinBound(RandomRange range) {
static constexpr double offcenteredness =
0.02; // Shift lower limit by about 5 for range of 255.
switch (range) {
case RandomRange::kGeneral:
return std::numeric_limits<Scalar>::lowest();
case RandomRange::kAvoidMinValue:
return 1 + std::numeric_limits<Scalar>::lowest();
case RandomRange::kOffCenterAvoidMinValue:
return 1 + std::numeric_limits<Scalar>::lowest() +
static_cast<Scalar>(
offcenteredness * std::numeric_limits<Scalar>::max() -
offcenteredness *
(std::numeric_limits<Scalar>::lowest() + 1));
case RandomRange::kReasonableSrcZeroPoint:
return std::numeric_limits<Scalar>::lowest();
case RandomRange::kReasonableDstZeroPoint:
return Parametrized<Scalar>(0.4);
case RandomRange::kBias:
return std::is_same<Scalar, std::int32_t>::value
? static_cast<Scalar>(-10000)
: 0;
default:
RUY_CHECK(false);
return 0;
}
}
static Scalar GetMaxBound(RandomRange range) {
switch (range) {
case RandomRange::kGeneral:
return std::numeric_limits<Scalar>::max();
case RandomRange::kAvoidMinValue:
return std::numeric_limits<Scalar>::max();
case RandomRange::kOffCenterAvoidMinValue:
return std::numeric_limits<Scalar>::max();
case RandomRange::kReasonableSrcZeroPoint:
return std::numeric_limits<Scalar>::max();
case RandomRange::kReasonableDstZeroPoint:
return Parametrized<Scalar>(0.6);
case RandomRange::kBias:
return std::is_same<Scalar, std::int32_t>::value
? static_cast<Scalar>(10000)
: 0;
default:
RUY_CHECK(false);
return 0;
}
}
};
inline std::default_random_engine& global_random_engine() {
static std::default_random_engine engine;
return engine;
}
template <typename Scalar>
struct UniformRandomDistribution {
UniformRandomDistribution(RandomRange range)
: dist(RandomRangeBounds<Scalar>::GetMinBound(range),
RandomRangeBounds<Scalar>::GetMaxBound(range)) {}
Scalar Get() { return dist(global_random_engine()); }
// std::uniform_int_distribution is specified not to support char types,
// only short and wider types. MSVC actually generates an error on
// std::uniform_int_distribution<std::int8_t>.
using StdDistType = typename std::conditional<
std::is_floating_point<Scalar>::value,
std::uniform_real_distribution<Scalar>,
std::uniform_int_distribution<std::int32_t>>::type;
StdDistType dist;
};
template <typename Scalar>
void MakeRandomScalar(UniformRandomDistribution<Scalar>* uniform_dist,
Scalar* dst) {
*dst = uniform_dist->Get();
}
template <typename Scalar>
void MakeRandomVector(UniformRandomDistribution<Scalar>* uniform_dist, int size,
std::vector<Scalar>* dst) {
dst->resize(size);
for (auto& x : *dst) {
MakeRandomScalar(uniform_dist, &x);
}
}
template <typename Scalar>
void MakeRandomScalar(RandomRange range, Scalar* dst) {
UniformRandomDistribution<Scalar> dist(range);
*dst = dist.Get();
if (range == RandomRange::kReasonableDstZeroPoint ||
range == RandomRange::kReasonableSrcZeroPoint) {
if (global_random_engine()() & 1) {
*dst = SymmetricZeroPoint<Scalar>();
}
}
}
template <typename Scalar>
void MakeRandomVector(RandomRange range, int size, std::vector<Scalar>* dst) {
UniformRandomDistribution<Scalar> dist(range);
dst->resize(size);
for (auto& x : *dst) {
MakeRandomScalar(&dist, &x);
}
}
enum class LayoutStyle { kUnstridedLinear, kLinear };
inline void MakeLayout(int rows, int cols, Order order,
LayoutStyle layout_style, Layout* layout) {
layout->set_rows(rows);
layout->set_cols(cols);
layout->set_order(order);
const int min_stride = order == Order::kColMajor ? rows : cols;
RUY_CHECK(layout_style == LayoutStyle::kUnstridedLinear ||
layout_style == LayoutStyle::kLinear);
if (layout_style == LayoutStyle::kUnstridedLinear) {
layout->set_stride(min_stride);
} else {
layout->set_stride(min_stride + 1);
}
}
template <typename Scalar>
struct StorageMatrix {
StorageMatrix() = default;
StorageMatrix(const StorageMatrix&) = delete;
void operator=(const StorageMatrix&) = delete;
std::vector<Scalar> data;
Matrix<Scalar> matrix;
};
inline bool IsUnstrided(const Layout& layout) {
if (layout.order() == Order::kColMajor) {
return layout.stride() == layout.rows();
} else {
return layout.stride() == layout.cols();
}
}
inline bool IsRowMajor(const Layout& layout) {
return layout.order() == Order::kRowMajor;
}
inline bool IsColMajor(const Layout& layout) {
return layout.order() == Order::kColMajor;
}
inline int FlatSize(const Layout& layout) {
const int outerdim =
layout.order() == Order::kColMajor ? layout.cols() : layout.rows();
return layout.stride() * outerdim;
}
template <typename Scalar>
void VerifyConsistentFields(const StorageMatrix<Scalar>& storage_matrix) {
if (storage_matrix.data.empty()) {
RUY_CHECK_EQ(storage_matrix.matrix.data(), nullptr);
RUY_CHECK_EQ(storage_matrix.matrix.layout().rows(), 0);
RUY_CHECK_EQ(storage_matrix.matrix.layout().cols(), 0);
} else {
RUY_CHECK_EQ(storage_matrix.matrix.data(), storage_matrix.data.data());
RUY_CHECK_EQ(FlatSize(storage_matrix.matrix.layout()),
static_cast<int>(storage_matrix.data.size()));
}
}
template <typename Scalar>
void MakeRandom(int rows, int cols, Order order, Scalar zero_point,
LayoutStyle layout_style, RandomRange range,
StorageMatrix<Scalar>* storage_matrix) {
MakeLayout(rows, cols, order, layout_style,
storage_matrix->matrix.mutable_layout());
storage_matrix->matrix.set_zero_point(zero_point);
UniformRandomDistribution<Scalar> data_dist(range);
MakeRandomVector(&data_dist, FlatSize(storage_matrix->matrix.layout()),
&storage_matrix->data);
storage_matrix->matrix.set_data(storage_matrix->data.data());
VerifyConsistentFields(*storage_matrix);
}
template <typename Scalar>
struct TestResult {
void operator=(const TestResult&) = delete;
void operator=(const TestResult&&) = delete;
StorageMatrix<Scalar> storage_matrix;
Path path = Path::kNone;
Tuning tuning = Tuning::kAuto;
ExternalPath external_path = ExternalPath::kNone;
float latency;
float l1_refill_rate;
float l2_refill_rate;
float l3_refill_rate;
float l1tlb_refill_rate;
float l2tlb_refill_rate;
float mispred_rate;
float frontend_stall_rate;
float backend_stall_rate;
};
template <typename Scalar>
std::string PathName(const TestResult<Scalar>& result) {
std::string pathname;
if (result.path != Path::kNone) {
pathname.assign(PathName(result.path));
} else if (result.external_path != ExternalPath::kNone) {
pathname.assign(PathName(result.external_path));
} else {
RUY_CHECK(false);
}
if (result.tuning != Tuning::kAuto) {
pathname.append("/");
pathname.append(TuningName(result.tuning));
}
return pathname;
}
enum class ExpectedOutcome { kSuccess, kDeath };
template <typename tLhsScalar, typename tRhsScalar, typename SpecType>
struct TestSet final {
using LhsScalar = tLhsScalar;
using RhsScalar = tRhsScalar;
using AccumScalar = typename SpecType::AccumScalar;
using DstScalar = typename SpecType::DstScalar;
using MulParamsType = SpecType;
using TestResultType = TestResult<DstScalar>;
void Run() {
MakeZeroPoints();
MakeLhsRhs();
MakeMulParams();
MakeOtherParams();
MakeResultPaths();
Eval();
Verify();
}
private:
void MakeZeroPoints();
void MakeLhsRhs();
void MakeMulParams();
void MakeResultPaths();
void MakeOtherParams();
void EvalAndVerify();
void Eval();
void Verify();
void EvalResult(TestResultType* result);
void EvalRuy(TestResultType* result);
void DoMul(TestResultType* result);
void Benchmark(TestResultType* result);
void VerifyTestResults() const;
public:
enum class LifeStage {
kInitial,
kHasZeroPoints,
kHasLhsRhs,
kHasMulParams,
kHasOtherParams,
kHasResultPaths,
kEvaluated,
kFinal
};
~TestSet();
LifeStage life_stage = LifeStage::kInitial;
int rows = 0;
int cols = 0;
int depth = 0;
Order lhs_order = Order::kRowMajor;
Order rhs_order = Order::kColMajor;
Order dst_order = Order::kColMajor;
LayoutStyle layout_style = LayoutStyle::kUnstridedLinear;
ExpectedOutcome expected_outcome = ExpectedOutcome::kSuccess;
bool use_specified_zero_points = false;
LhsScalar lhs_zero_point = 0;
RhsScalar rhs_zero_point = 0;
DstScalar dst_zero_point = 0;
std::vector<AccumScalar> per_channel_multiplier_fixedpoint;
std::vector<int> per_channel_multiplier_exponent;
StorageMatrix<LhsScalar> lhs;
StorageMatrix<RhsScalar> rhs;
MulParamsType mul_params;
std::vector<AccumScalar> bias_data;
std::vector<std::unique_ptr<TestResultType>> results;
std::vector<Path> paths;
std::vector<ExternalPath> external_paths;
bool benchmark = false;
bool perchannel = false;
int max_num_threads = 0;
bool cache_lhs = false;
bool cache_rhs = false;
};
inline PmuEvents& GlobalPmuEvents() {
static PmuEvents pmu;
return pmu;
}
inline Context& GlobalContext() {
// Ensure that GlobalPmuEvents is constructed before we create any context.
// This ensures that pmu counters are opened before we create any worker
// thread, which is necessary to count events from worker threads.
GlobalPmuEvents();
static Context context;
return context;
}
template <typename LhsScalar, typename RhsScalar, typename SpecType>
TestSet<LhsScalar, RhsScalar, SpecType>::~TestSet() {
RUY_CHECK_EQ(life_stage, LifeStage::kFinal);
LogCoveredPathsOnDestruction::Singleton();
GlobalContext().ClearPrepackedCache();
}
#if defined(__has_feature)
#if __has_feature(thread_sanitizer)
#define RUY_TSAN
#endif
#if __has_feature(address_sanitizer)
#define RUY_ASAN
#endif
#endif // defined(__has_feature)
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::DoMul(TestResultType* result) {
Mul<kAllPaths>(lhs.matrix, rhs.matrix, mul_params, &GlobalContext(),
&result->storage_matrix.matrix);
}
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::EvalRuy(TestResultType* result) {
GlobalContext().set_explicit_tuning(result->tuning);
if (max_num_threads) {
GlobalContext().set_max_num_threads(max_num_threads);
} else if (benchmark) {
GlobalContext().set_max_num_threads(1);
} else {
GlobalContext().set_max_num_threads(1 + global_random_engine()() % 8);
}
get_ctx(&GlobalContext())->SetRuntimeEnabledPaths(result->path);
if (expected_outcome == ExpectedOutcome::kSuccess) {
DoMul(result);
// If enabling caching, Mul is stateful, so we run it a second time to get
// coverage of these aspects.
if (cache_lhs || cache_rhs) {
DoMul(result);
}
RUY_CHECK_EQ(GlobalContext().last_used_path(), result->path);
} else if (expected_outcome == ExpectedOutcome::kDeath) {
// TODO(benoitjacob) TSan and ASan seem to be breaking ASSERT_DEATH.
// Report a bug?
#if (!defined NDEBUG) && (!defined RUY_ASAN) && (!defined RUY_TSAN)
RUY_ASSERT_DEATH(DoMul(result), "");
#endif
} else {
RUY_CHECK(false);
}
GlobalContext().set_explicit_tuning(Tuning::kAuto);
GlobalContext().set_max_num_threads(1);
}
#ifdef RUY_TEST_EXTERNAL_PATHS
template <typename Scalar, gemmlowp::MapOrder tOrder>
void WrapGemmlowp(const Matrix<Scalar>& src,
gemmlowp::MatrixMap<const Scalar, tOrder>* dst) {
RUY_CHECK(src.layout().order() == (tOrder == gemmlowp::MapOrder::ColMajor
? Order::kColMajor
: Order::kRowMajor));
*dst = gemmlowp::MatrixMap<const Scalar, tOrder>(
src.data(), src.layout().rows(), src.layout().cols(),
src.layout().stride());
}
template <typename Scalar, gemmlowp::MapOrder tOrder>
void WrapGemmlowpMutable(Matrix<Scalar>* src,
gemmlowp::MatrixMap<Scalar, tOrder>* dst) {
RUY_CHECK(src->layout().order() == (tOrder == gemmlowp::MapOrder::ColMajor
? Order::kColMajor
: Order::kRowMajor));
*dst = gemmlowp::MatrixMap<Scalar, tOrder>(src->data(), src->layout().rows(),
src->layout().cols(),
src->layout().stride());
}
template <Order tOrder>
struct GemmlowpOrder {};
template <>
struct GemmlowpOrder<Order::kColMajor> {
static constexpr gemmlowp::MapOrder kValue = gemmlowp::MapOrder::ColMajor;
};
template <>
struct GemmlowpOrder<Order::kRowMajor> {
static constexpr gemmlowp::MapOrder kValue = gemmlowp::MapOrder::RowMajor;
};
inline gemmlowp::GemmContext& GlobalGemmlowpContext() {
static gemmlowp::GemmContext context;
return context;
}
template <Order LhsOrder, Order RhsOrder, Order DstOrder, typename LhsScalar,
typename RhsScalar, typename DstScalar, typename MulParamsType>
void EvalGemmlowp(const Matrix<LhsScalar>& lhs, const Matrix<RhsScalar>& rhs,
const MulParamsType& mul_params, int max_num_threads,
Matrix<DstScalar>* dst) {
static constexpr gemmlowp::MapOrder kGemmlowpLhsOrder =
GemmlowpOrder<LhsOrder>::kValue;
static constexpr gemmlowp::MapOrder kGemmlowpRhsOrder =
GemmlowpOrder<RhsOrder>::kValue;
static constexpr gemmlowp::MapOrder kGemmlowpDstOrder =
GemmlowpOrder<DstOrder>::kValue;
gemmlowp::MatrixMap<const LhsScalar, kGemmlowpLhsOrder> gemmlowp_lhs;
gemmlowp::MatrixMap<const RhsScalar, kGemmlowpRhsOrder> gemmlowp_rhs;
gemmlowp::MatrixMap<DstScalar, kGemmlowpDstOrder> gemmlowp_dst;
WrapGemmlowp(lhs, &gemmlowp_lhs);
WrapGemmlowp(rhs, &gemmlowp_rhs);
WrapGemmlowpMutable(dst, &gemmlowp_dst);
gemmlowp::OutputStageScaleInt32ByFixedPointAndExponent quantize_down_stage;
quantize_down_stage.result_offset_after_shift = dst->zero_point();
quantize_down_stage.result_fixedpoint_multiplier =
mul_params.multiplier_fixedpoint();
quantize_down_stage.result_exponent = mul_params.multiplier_exponent();
gemmlowp::OutputStageScaleInt32ByFixedPointAndExponentPC<
gemmlowp::VectorShape::Col>
quantize_down_stage_pc;
quantize_down_stage_pc.result_offset_after_shift = dst->zero_point();
using ColVectorMap =
gemmlowp::VectorMap<const std::int32_t, gemmlowp::VectorShape::Col>;
quantize_down_stage_pc.result_fixedpoint_multiplier = ColVectorMap(
mul_params.multiplier_fixedpoint_perchannel(), lhs.layout().rows());
quantize_down_stage_pc.result_exponent = ColVectorMap(
mul_params.multiplier_exponent_perchannel(), lhs.layout().rows());
gemmlowp::OutputStageClamp clamp_stage;
clamp_stage.min = mul_params.clamp_min();
clamp_stage.max = mul_params.clamp_max();
using OutputStageSaturatingCast = typename std::conditional<
std::is_same<DstScalar, std::uint8_t>::value,
gemmlowp::OutputStageSaturatingCastToUint8,
gemmlowp::OutputStageSaturatingCastToInt16>::type;
OutputStageSaturatingCast saturating_cast_stage;
GlobalGemmlowpContext().set_max_num_threads(max_num_threads ? max_num_threads
: 1);
if (mul_params.bias()) {
using ColVectorMap =
gemmlowp::VectorMap<const std::int32_t, gemmlowp::VectorShape::Col>;
gemmlowp::OutputStageBiasAddition<ColVectorMap> bias_add_stage;
bias_add_stage.bias_vector =
ColVectorMap(mul_params.bias(), dst->layout().rows());
#ifndef GEMMLOWP_SSE4 // gemmlowp perchannel stuff does not build on SSE
if (mul_params.multiplier_exponent_perchannel()) {
const auto& output_pipeline =
std::make_tuple(bias_add_stage, quantize_down_stage_pc, clamp_stage,
saturating_cast_stage);
gemmlowp::GemmWithOutputPipeline<
LhsScalar, DstScalar, gemmlowp::L8R8WithLhsNonzeroBitDepthParams>(
&GlobalGemmlowpContext(), gemmlowp_lhs, gemmlowp_rhs, &gemmlowp_dst,
-lhs.zero_point(), -rhs.zero_point(), output_pipeline);
} else // NOLINT[readability/braces]
#endif
{
const auto& output_pipeline =
std::make_tuple(bias_add_stage, quantize_down_stage, clamp_stage,
saturating_cast_stage);
gemmlowp::GemmWithOutputPipeline<
LhsScalar, DstScalar, gemmlowp::L8R8WithLhsNonzeroBitDepthParams>(
&GlobalGemmlowpContext(), gemmlowp_lhs, gemmlowp_rhs, &gemmlowp_dst,
-lhs.zero_point(), -rhs.zero_point(), output_pipeline);
}
} else {
#ifndef GEMMLOWP_SSE4 // gemmlowp perchannel stuff does not build on SSE
if (mul_params.multiplier_exponent_perchannel()) {
const auto& output_pipeline = std::make_tuple(
quantize_down_stage_pc, clamp_stage, saturating_cast_stage);
gemmlowp::GemmWithOutputPipeline<
LhsScalar, DstScalar, gemmlowp::L8R8WithLhsNonzeroBitDepthParams>(
&GlobalGemmlowpContext(), gemmlowp_lhs, gemmlowp_rhs, &gemmlowp_dst,
-lhs.zero_point(), -rhs.zero_point(), output_pipeline);
} else // NOLINT[readability/braces]
#endif
{
const auto& output_pipeline = std::make_tuple(
quantize_down_stage, clamp_stage, saturating_cast_stage);
gemmlowp::GemmWithOutputPipeline<
LhsScalar, DstScalar, gemmlowp::L8R8WithLhsNonzeroBitDepthParams>(
&GlobalGemmlowpContext(), gemmlowp_lhs, gemmlowp_rhs, &gemmlowp_dst,
-lhs.zero_point(), -rhs.zero_point(), output_pipeline);
}
}
}
inline constexpr int Mash(Order LhsOrder, Order RhsOrder, Order DstOrder) {
return (LhsOrder == Order::kRowMajor ? 4 : 0) +
(RhsOrder == Order::kRowMajor ? 2 : 0) +
(DstOrder == Order::kRowMajor ? 1 : 0);
}
template <typename LhsScalar, typename RhsScalar, typename DstScalar,
typename MulParamsType>
void EvalGemmlowp(const Matrix<LhsScalar>& lhs, const Matrix<RhsScalar>& rhs,
const MulParamsType& mul_params, int max_num_threads,
Matrix<DstScalar>* dst) {
int index =
Mash(lhs.layout().order(), rhs.layout().order(), dst->layout().order());
switch (index) {
#define EVALGEMMLOWP_CASE3(LHS, RHS, DST) \
case Mash(LHS, RHS, DST): \
return EvalGemmlowp<LHS, RHS, DST>(lhs, rhs, mul_params, max_num_threads, \
dst);
#define EVALGEMMLOWP_CASE2(LHS, RHS) \
EVALGEMMLOWP_CASE3(LHS, RHS, Order::kColMajor) \
EVALGEMMLOWP_CASE3(LHS, RHS, Order::kRowMajor)
#define EVALGEMMLOWP_CASE1(LHS) \
EVALGEMMLOWP_CASE2(LHS, Order::kColMajor) \
EVALGEMMLOWP_CASE2(LHS, Order::kRowMajor)
EVALGEMMLOWP_CASE1(Order::kColMajor)
EVALGEMMLOWP_CASE1(Order::kRowMajor)
#undef EVALGEMMLOWP_CASE1
#undef EVALGEMMLOWP_CASE2
#undef EVALGEMMLOWP_CASE3
default:
RUY_CHECK(false);
}
}
template <Order tOrder>
struct EigenOrder {};
template <>
struct EigenOrder<Order::kColMajor> {
static constexpr int kValue = Eigen::ColMajor;
};
template <>
struct EigenOrder<Order::kRowMajor> {
static constexpr int kValue = Eigen::RowMajor;
};
template <Order LhsOrder, Order RhsOrder, Order DstOrder, typename LhsScalar,
typename RhsScalar, typename DstScalar, typename MulParamsType>
void EvalEigen(const Matrix<LhsScalar>& lhs, const Matrix<RhsScalar>& rhs,
const MulParamsType& mul_params, int max_num_threads,
Matrix<DstScalar>* dst) {
RUY_CHECK_EQ(lhs.zero_point(), 0);
RUY_CHECK_EQ(rhs.zero_point(), 0);
RUY_CHECK_EQ(dst->zero_point(), 0);
RUY_CHECK_EQ(mul_params.multiplier_fixedpoint(), 0);
RUY_CHECK_EQ(mul_params.multiplier_exponent(), 0);
static constexpr int kEigenLhsOrder = EigenOrder<LhsOrder>::kValue;
static constexpr int kEigenRhsOrder = EigenOrder<RhsOrder>::kValue;
static constexpr int kEigenDstOrder = EigenOrder<DstOrder>::kValue;
using EigenLhsType = typename Eigen::Matrix<LhsScalar, Eigen::Dynamic,
Eigen::Dynamic, kEigenLhsOrder>::
template StridedConstMapType<Eigen::OuterStride<Eigen::Dynamic>>::type;
using EigenRhsType = typename Eigen::Matrix<RhsScalar, Eigen::Dynamic,
Eigen::Dynamic, kEigenRhsOrder>::
template StridedConstMapType<Eigen::OuterStride<Eigen::Dynamic>>::type;
using EigenDstType = typename Eigen::Matrix<DstScalar, Eigen::Dynamic,
Eigen::Dynamic, kEigenDstOrder>::
template StridedMapType<Eigen::OuterStride<Eigen::Dynamic>>::type;
using EigenBiasType =
typename Eigen::Matrix<DstScalar, Eigen::Dynamic, 1>::ConstMapType;
EigenLhsType eigen_lhs(
lhs.data(), lhs.layout().rows(), lhs.layout().cols(),
Eigen::OuterStride<Eigen::Dynamic>(lhs.layout().stride()));
EigenRhsType eigen_rhs(
rhs.data(), rhs.layout().rows(), rhs.layout().cols(),
Eigen::OuterStride<Eigen::Dynamic>(rhs.layout().stride()));
EigenDstType eigen_dst(
dst->data(), dst->layout().rows(), dst->layout().cols(),
Eigen::OuterStride<Eigen::Dynamic>(dst->layout().stride()));
Eigen::setNbThreads(max_num_threads ? max_num_threads : 1);
if (mul_params.bias()) {
EigenBiasType eigen_bias(mul_params.bias(), dst->layout().rows());
if (mul_params.clamp_max() == std::numeric_limits<DstScalar>::infinity() &&
mul_params.clamp_min() == -std::numeric_limits<DstScalar>::infinity()) {
eigen_dst.noalias() = (eigen_lhs * eigen_rhs).colwise() + eigen_bias;
} else {
eigen_dst.noalias() = ((eigen_lhs * eigen_rhs).colwise() + eigen_bias)
.cwiseMin(mul_params.clamp_max())
.cwiseMax(mul_params.clamp_min());
}
} else {
if (mul_params.clamp_max() == std::numeric_limits<DstScalar>::infinity() &&
mul_params.clamp_min() == -std::numeric_limits<DstScalar>::infinity()) {
eigen_dst.noalias() = eigen_lhs * eigen_rhs;
} else {
eigen_dst.noalias() = (eigen_lhs * eigen_rhs)
.cwiseMin(mul_params.clamp_max())
.cwiseMax(mul_params.clamp_min());
}
}
}
template <typename LhsScalar, typename RhsScalar, typename DstScalar,
typename MulParamsType>
void EvalEigen(const Matrix<LhsScalar>& lhs, const Matrix<RhsScalar>& rhs,
const MulParamsType& mul_params, int max_num_threads,
Matrix<DstScalar>* dst) {
int index =
Mash(lhs.layout().order(), rhs.layout().order(), dst->layout().order());
switch (index) {
#define EVALEIGEN_CASE3(LHS, RHS, DST) \
case Mash(LHS, RHS, DST): \
return EvalEigen<LHS, RHS, DST>(lhs, rhs, mul_params, max_num_threads, dst);
#define EVALEIGEN_CASE2(LHS, RHS) \
EVALEIGEN_CASE3(LHS, RHS, Order::kColMajor) \
EVALEIGEN_CASE3(LHS, RHS, Order::kRowMajor)
#define EVALEIGEN_CASE1(LHS) \
EVALEIGEN_CASE2(LHS, Order::kColMajor) \
EVALEIGEN_CASE2(LHS, Order::kRowMajor)
EVALEIGEN_CASE1(Order::kColMajor)
EVALEIGEN_CASE1(Order::kRowMajor)
#undef EVALEIGEN_CASE1
#undef EVALEIGEN_CASE2
#undef EVALEIGEN_CASE3
default:
RUY_CHECK(false);
}
}
template <Order LhsOrder, Order RhsOrder, Order DstOrder, typename Scalar,
typename MulParamsType>
void EvalEigenTensor(const Matrix<Scalar>& lhs, const Matrix<Scalar>& rhs,
const MulParamsType& mul_params, int max_num_threads,
Matrix<Scalar>* dst) {
RUY_CHECK_EQ(lhs.zero_point(), 0);
RUY_CHECK_EQ(rhs.zero_point(), 0);
RUY_CHECK_EQ(dst->zero_point(), 0);
RUY_CHECK_EQ(mul_params.multiplier_fixedpoint(), 0);
RUY_CHECK_EQ(mul_params.multiplier_exponent(), 0);
// Eigen::TensorMap only supports unstrided layouts
RUY_CHECK(IsUnstrided(lhs.layout()));
RUY_CHECK(IsUnstrided(rhs.layout()));
RUY_CHECK(IsUnstrided(dst->layout()));
using TensorLhsType =
Eigen::TensorMap<Eigen::Tensor<const Scalar, 2, Eigen::ColMajor>>;
using TensorRhsType =
Eigen::TensorMap<Eigen::Tensor<const Scalar, 2, Eigen::ColMajor>>;
using TensorDstType =
Eigen::TensorMap<Eigen::Tensor<Scalar, 2, Eigen::ColMajor>>;
using TensorBiasType =
Eigen::TensorMap<Eigen::Tensor<const Scalar, 1, Eigen::ColMajor>>;
const bool tr = DstOrder == Order::kRowMajor;
const auto& contract_lhs = tr ? rhs : lhs;
const auto& contract_rhs = tr ? lhs : rhs;
TensorLhsType tensor_lhs(
contract_lhs.data(),
LhsOrder == Order::kColMajor ? contract_lhs.layout().rows()
: contract_lhs.layout().cols(),
LhsOrder == Order::kColMajor ? contract_lhs.layout().cols()
: contract_lhs.layout().rows());
TensorRhsType tensor_rhs(
contract_rhs.data(),
RhsOrder == Order::kColMajor ? contract_rhs.layout().rows()
: contract_rhs.layout().cols(),
RhsOrder == Order::kColMajor ? contract_rhs.layout().cols()
: contract_rhs.layout().rows());
TensorDstType tensor_dst(dst->data(),
DstOrder == Order::kColMajor ? dst->layout().rows()
: dst->layout().cols(),
DstOrder == Order::kColMajor ? dst->layout().cols()
: dst->layout().rows());
using DimPair =
typename Eigen::Tensor<Scalar, 1, 0, Eigen::Index>::DimensionPair;
Eigen::array<DimPair, 1> contract_dims(
{DimPair((LhsOrder == Order::kColMajor) ? 1 : 0,
(RhsOrder == Order::kColMajor) ? 0 : 1)});
Eigen::array<int, 2> shuffle(DstOrder == Order::kColMajor ? 0 : 1,
DstOrder == Order::kColMajor ? 1 : 0);
static Eigen::ThreadPool pool(max_num_threads ? max_num_threads : 1);
static Eigen::ThreadPoolDevice device(&pool, pool.NumThreads());
if (mul_params.bias()) {
TensorBiasType tensor_bias(mul_params.bias(), dst->layout().rows());
Eigen::array<int, 2> bias_2d_shape(tr ? 1 : dst->layout().rows(),
tr ? dst->layout().rows() : 1);
Eigen::array<int, 2> bcast(tr ? dst->layout().cols() : 1,
tr ? 1 : dst->layout().cols());
if (mul_params.clamp_max() == std::numeric_limits<Scalar>::infinity() &&
mul_params.clamp_min() == -std::numeric_limits<Scalar>::infinity()) {
tensor_dst.device(device) =
tensor_lhs.contract(tensor_rhs, contract_dims);
} else {
tensor_dst.device(device) =
(tensor_lhs.contract(tensor_rhs, contract_dims) +
tensor_bias.reshape(bias_2d_shape).broadcast(bcast))
.cwiseMin(mul_params.clamp_max())
.cwiseMax(mul_params.clamp_min());
}
} else {
if (mul_params.clamp_max() == std::numeric_limits<Scalar>::infinity() &&
mul_params.clamp_min() == -std::numeric_limits<Scalar>::infinity()) {
tensor_dst.device(device) =
tensor_lhs.contract(tensor_rhs, contract_dims);
} else {
tensor_dst.device(device) = tensor_lhs.contract(tensor_rhs, contract_dims)
.cwiseMin(mul_params.clamp_max())
.cwiseMax(mul_params.clamp_min());
}
}
}
template <typename Scalar, typename MulParamsType>
void EvalEigenTensor(const Matrix<Scalar>& lhs, const Matrix<Scalar>& rhs,
const MulParamsType& mul_params, int max_num_threads,
Matrix<Scalar>* dst) {
int index =
Mash(lhs.layout().order(), rhs.layout().order(), dst->layout().order());
switch (index) {
#define EVALEIGENTENSOR_CASE3(LHS, RHS, DST) \
case Mash(LHS, RHS, DST): \
return EvalEigenTensor<LHS, RHS, DST>(lhs, rhs, mul_params, \
max_num_threads, dst);
#define EVALEIGENTENSOR_CASE2(LHS, RHS) \
EVALEIGENTENSOR_CASE3(LHS, RHS, Order::kColMajor) \
EVALEIGENTENSOR_CASE3(LHS, RHS, Order::kRowMajor)
#define EVALEIGENTENSOR_CASE1(LHS) \
EVALEIGENTENSOR_CASE2(LHS, Order::kColMajor) \
EVALEIGENTENSOR_CASE2(LHS, Order::kRowMajor)
EVALEIGENTENSOR_CASE1(Order::kColMajor)
EVALEIGENTENSOR_CASE1(Order::kRowMajor)
#undef EVALEIGENTENSOR_CASE1
#undef EVALEIGENTENSOR_CASE2
#undef EVALEIGENTENSOR_CASE3
default:
RUY_CHECK(false);
}
}
template <typename Scalar>
struct GenericBlasGemm {};
template <>
struct GenericBlasGemm<lapack::doublereal> {
static void Run(char* transa, char* transb, lapack::integer* m,
lapack::integer* n, lapack::integer* k,
lapack::doublereal* alpha, lapack::doublereal* a,
lapack::integer* lda, lapack::doublereal* b,
lapack::integer* ldb, lapack::doublereal* beta,
lapack::doublereal* c, lapack::integer* ldc) {
dgemm_(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);
}
};
template <>
struct GenericBlasGemm<lapack::real> {
static void Run(char* transa, char* transb, lapack::integer* m,
lapack::integer* n, lapack::integer* k, lapack::real* alpha,
lapack::real* a, lapack::integer* lda, lapack::real* b,
lapack::integer* ldb, lapack::real* beta, lapack::real* c,
lapack::integer* ldc) {
sgemm_(transa, transb, m, n, k, alpha, a, lda, b, ldb, beta, c, ldc);
}
};
inline void TransposeLayout(Layout* layout) {
layout->set_order((layout->order() == Order::kRowMajor) ? Order::kColMajor
: Order::kRowMajor);
int tmp_rows = layout->rows();
layout->set_rows(layout->cols());
layout->set_cols(tmp_rows);
}
template <typename Scalar>
void Transpose(Matrix<Scalar>* matrix) {
TransposeLayout(matrix->mutable_layout());
}
template <typename Scalar, typename MulParamsType>
void EvalOpenBlas(const Matrix<Scalar>& lhs, const Matrix<Scalar>& rhs,
const MulParamsType& mul_params, int max_num_threads,
Matrix<Scalar>* dst) {
RUY_CHECK_EQ(lhs.zero_point(), 0);
RUY_CHECK_EQ(rhs.zero_point(), 0);
RUY_CHECK_EQ(dst->zero_point(), 0);
RUY_CHECK_EQ(mul_params.multiplier_fixedpoint(), 0);
RUY_CHECK_EQ(mul_params.multiplier_exponent(), 0);
Matrix<Scalar> gemm_lhs;
Matrix<Scalar> gemm_rhs;
Matrix<Scalar> gemm_dst;
gemm_dst = *dst;
// Use Transpose to reduce to the all-column-major case.
// Notice that ruy::Matrix merely holds a pointer, does not own data,
// so Transpose is cheap -- no actual matrix data is being transposed here.
if (dst->layout().order() == Order::kColMajor) {
gemm_lhs = lhs;
gemm_rhs = rhs;
} else {
gemm_lhs = rhs;
gemm_rhs = lhs;
Transpose(&gemm_lhs);
Transpose(&gemm_rhs);
Transpose(&gemm_dst);
}
bool transposed_lhs = false;
bool transposed_rhs = false;
if (gemm_lhs.layout().order() == Order::kRowMajor) {
Transpose(&gemm_lhs);
transposed_lhs = true;
}
if (gemm_rhs.layout().order() == Order::kRowMajor) {
Transpose(&gemm_rhs);
transposed_rhs = true;
}
RUY_CHECK_EQ(gemm_lhs.layout().order(), Order::kColMajor);
RUY_CHECK_EQ(gemm_rhs.layout().order(), Order::kColMajor);
RUY_CHECK_EQ(gemm_dst.layout().order(), Order::kColMajor);
char transa = transposed_lhs ? 'T' : 'N';
char transb = transposed_rhs ? 'T' : 'N';
int m = gemm_lhs.layout().rows();
int n = gemm_rhs.layout().cols();
int k = gemm_lhs.layout().cols();
float alpha = 1;
Scalar* a = gemm_lhs.data();
int lda = gemm_lhs.layout().stride();
Scalar* b = gemm_rhs.data();
int ldb = gemm_rhs.layout().stride();
float beta = 0;
Scalar* c = gemm_dst.data();
int ldc = gemm_dst.layout().stride();
GenericBlasGemm<Scalar>::Run(&transa, &transb, &m, &n, &k, &alpha, a, &lda, b,
&ldb, &beta, c, &ldc);
// BLAS does not allow us to express the bias-addition and clamping, so
// we use Eigen for that.
using EigenDstType =
typename Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic>::
template StridedMapType<Eigen::OuterStride<Eigen::Dynamic>>::type;
using EigenBiasType =
typename Eigen::Matrix<Scalar, Eigen::Dynamic, 1>::ConstMapType;
EigenDstType eigen_dst(
gemm_dst.data(), gemm_dst.layout().rows(), gemm_dst.layout().cols(),
Eigen::OuterStride<Eigen::Dynamic>(gemm_dst.layout().stride()));
Eigen::setNbThreads(max_num_threads ? max_num_threads : 1);
if (mul_params.bias()) {
EigenBiasType eigen_bias(mul_params.bias(), dst->layout().rows());
if (mul_params.clamp_max() == std::numeric_limits<Scalar>::infinity() &&
mul_params.clamp_min() == -std::numeric_limits<Scalar>::infinity()) {
eigen_dst.noalias() = eigen_dst.colwise() + eigen_bias;
} else {
eigen_dst.noalias() = (eigen_dst.colwise() + eigen_bias)
.cwiseMin(mul_params.clamp_max())
.cwiseMax(mul_params.clamp_min());
}
} else {
if (mul_params.clamp_max() == std::numeric_limits<Scalar>::infinity() &&
mul_params.clamp_min() == -std::numeric_limits<Scalar>::infinity()) {
} else {
eigen_dst.noalias() = eigen_dst.cwiseMin(mul_params.clamp_max())
.cwiseMax(mul_params.clamp_min());
}
}
}
template <typename TestSetType>
struct SupportsGemmlowp {
static constexpr bool kValue =
std::is_same<typename TestSetType::LhsScalar, std::uint8_t>::value &&
std::is_same<typename TestSetType::RhsScalar, std::uint8_t>::value;
};
template <typename TestSetType>
struct UsesSingleScalarType {
static constexpr bool kValue =
std::is_same<typename TestSetType::DstScalar,
typename TestSetType::LhsScalar>::value &&
std::is_same<typename TestSetType::DstScalar,
typename TestSetType::RhsScalar>::value &&
std::is_same<typename TestSetType::DstScalar,
typename TestSetType::AccumScalar>::value;
};
template <typename TestSetType,
bool IsFloatingPoint =
std::is_floating_point<typename TestSetType::AccumScalar>::value,
bool EnableGemmlowp = SupportsGemmlowp<TestSetType>::kValue,
bool SingleScalarType = UsesSingleScalarType<TestSetType>::kValue>
struct EvalExternalPathImpl {
using DstScalar = typename TestSetType::DstScalar;
static void Run(TestSetType*, TestResult<DstScalar>*) { RUY_CHECK(false); }
};
template <typename TestSetType>
struct EvalExternalPathImpl<TestSetType, true, false, true> {
using DstScalar = typename TestSetType::DstScalar;
static void Run(TestSetType* test_set, TestResult<DstScalar>* test_result) {
if (test_result->external_path == ExternalPath::kEigen) {
EvalEigen(test_set->lhs.matrix, test_set->rhs.matrix,
test_set->mul_params, test_set->max_num_threads,
&test_result->storage_matrix.matrix);
} else if (test_result->external_path == ExternalPath::kEigenTensor) {
EvalEigenTensor(test_set->lhs.matrix, test_set->rhs.matrix,
test_set->mul_params, test_set->max_num_threads,
&test_result->storage_matrix.matrix);
} else if (test_result->external_path == ExternalPath::kOpenBlas) {
EvalOpenBlas(test_set->lhs.matrix, test_set->rhs.matrix,
test_set->mul_params, test_set->max_num_threads,
&test_result->storage_matrix.matrix);
} else {
RUY_CHECK(false);
}
}
};
template <typename TestSetType, bool SingleScalarType>
struct EvalExternalPathImpl<TestSetType, false, true, SingleScalarType> {
using DstScalar = typename TestSetType::DstScalar;
static void Run(TestSetType* test_set, TestResult<DstScalar>* test_result) {
if (test_result->external_path == ExternalPath::kGemmlowp) {
EvalGemmlowp(test_set->lhs.matrix, test_set->rhs.matrix,
test_set->mul_params, test_set->max_num_threads,
&test_result->storage_matrix.matrix);
} else {
RUY_CHECK(false);
}
}
};
#endif // RUY_TEST_EXTERNAL_PATHS
template <typename TestSetType>
void EvalExternalPath(
TestSetType* test_set,
TestResult<typename TestSetType::DstScalar>* test_result) {
if (test_result->external_path == ExternalPath::kReference) {
// kReference is special because it's always available (the implementation
// is provided by ruy) and supports all cases (quantized and float).
ruy::ReferenceMul(test_set->lhs.matrix, test_set->rhs.matrix,
test_set->mul_params,
&test_result->storage_matrix.matrix);
} else {
#ifdef RUY_TEST_EXTERNAL_PATHS
EvalExternalPathImpl<TestSetType>::Run(test_set, test_result);
#endif // RUY_TEST_EXTERNAL_PATHS
}
}
template <typename Scalar>
bool Agree(const Matrix<Scalar>& matrix1, const Matrix<Scalar>& matrix2,
int depth) {
RUY_CHECK_EQ(matrix1.layout().rows(), matrix2.layout().rows());
RUY_CHECK_EQ(matrix1.layout().cols(), matrix2.layout().cols());
RUY_CHECK_EQ(matrix1.zero_point(), matrix2.zero_point());
const int size = matrix1.layout().rows() * matrix1.layout().cols();
double tolerated_max_diff = 0;
double tolerated_mean_diff = 0;
if (std::is_floating_point<Scalar>::value) {
// TODO: replace hardcoded 100 by something more sensible, probably
// roughly sqrt(depth) based on central limit theorem.
double max_abs_val = 0;
for (int row = 0; row < matrix1.layout().rows(); row++) {
for (int col = 0; col < matrix1.layout().cols(); col++) {
max_abs_val =
std::max(max_abs_val,
std::abs(static_cast<double>(Element(matrix1, row, col))));
max_abs_val =
std::max(max_abs_val,
std::abs(static_cast<double>(Element(matrix2, row, col))));
}
}
tolerated_max_diff = max_abs_val * std::numeric_limits<Scalar>::epsilon() *
64 * std::sqrt(static_cast<float>(depth));
tolerated_mean_diff = tolerated_max_diff / std::sqrt(size);
} else if (RUY_OPT(NATIVE_ROUNDING)) {
tolerated_max_diff = 1;
// totally empirical
tolerated_mean_diff = std::min(1.0, 2.0 * std::pow(size, -0.2));
}
double sum_diff = 0;
for (int row = 0; row < matrix1.layout().rows(); row++) {
for (int col = 0; col < matrix1.layout().cols(); col++) {
double elem1 = Element(matrix1, row, col);
double elem2 = Element(matrix2, row, col);
double diff = elem1 - elem2;
sum_diff += diff;
// Test (std::abs(diff) > tolerated_max_diff), but also true if diff is
// NaN.
if (!(std::abs(diff) <= tolerated_max_diff)) {
return false;
}
}
}
double mean_diff = sum_diff / size;
if (std::abs(mean_diff) > tolerated_mean_diff) {
return false;
}
return true;
}
template <typename Scalar>
bool Agree(const StorageMatrix<Scalar>& storage_matrix1,
const StorageMatrix<Scalar>& storage_matrix2, int depth) {
VerifyConsistentFields(storage_matrix1);
VerifyConsistentFields(storage_matrix2);
return Agree(storage_matrix1.matrix, storage_matrix2.matrix, depth);
}
template <typename Scalar>
bool Agree(const TestResult<Scalar>& result1, const TestResult<Scalar>& result2,
int depth) {
return Agree(result1.storage_matrix, result2.storage_matrix, depth);
}
struct Stats {
double median;
double mean;
double min;
double max;
};
inline std::string StatsAsString(const Stats& stats) {
char buf[256];
snprintf(buf, sizeof(buf), "(median = %g, mean = %g, min = %g, max = %g)",
stats.median, stats.mean, stats.min, stats.max);
return std::string(buf);
}
template <typename Scalar>
void GetMatrixStats(const Matrix<Scalar>& matrix, Stats* stats) {
double min = std::numeric_limits<double>::infinity();
double max = -std::numeric_limits<double>::infinity();
double sum = 0;
std::vector<double> allvals;
for (int row = 0; row < matrix.layout().rows(); row++) {
for (int col = 0; col < matrix.layout().cols(); col++) {
double val = Element(matrix, row, col);
min = std::min(min, val);
max = std::max(max, val);
sum += val;
allvals.push_back(val);
}
}
std::sort(allvals.begin(), allvals.end());
stats->min = min;
stats->max = max;
stats->mean = sum / allvals.size();
stats->median = allvals[allvals.size() / 2];
}
struct ErrorAnalysis {
Stats stats_good;
Stats stats_bad;
// The below is to help document departure from bit exactness. It's probably
// not going to be relevant to floating-point.
std::set<int> error_rows;
std::set<int> error_cols;
int row_of_first_error = 0;
int col_of_first_error = 0;
double first_error_good_value = 0;
double first_error_bad_value = 0;
};
template <typename TestSetType>
void AnalyzeTestError(const TestSetType& test_set, int first_bad_result_index,
ErrorAnalysis* error_analysis) {
const auto& good_matrix = test_set.results[0]->storage_matrix.matrix;
const auto& bad_matrix =
test_set.results[first_bad_result_index]->storage_matrix.matrix;
GetMatrixStats(good_matrix, &error_analysis->stats_good);
GetMatrixStats(bad_matrix, &error_analysis->stats_bad);
bool found_first_error = false;
for (int row = 0; row < good_matrix.layout().rows(); row++) {
for (int col = 0; col < good_matrix.layout().cols(); col++) {
if (Element(good_matrix, row, col) != Element(bad_matrix, row, col)) {
if (!found_first_error) {
found_first_error = true;
error_analysis->row_of_first_error = row;
error_analysis->col_of_first_error = col;
error_analysis->first_error_good_value =
Element(good_matrix, row, col);
error_analysis->first_error_bad_value = Element(bad_matrix, row, col);
}
error_analysis->error_rows.insert(row);
error_analysis->error_cols.insert(col);
}
}
}
}
template <typename TestSetType>
void ComputeReasonableMultiplier(
const Matrix<typename TestSetType::LhsScalar>& lhs,
const Matrix<typename TestSetType::RhsScalar>&, double* multiplier) {
using LhsScalar = typename TestSetType::LhsScalar;
using RhsScalar = typename TestSetType::RhsScalar;
using DstScalar = typename TestSetType::DstScalar;
if (std::is_floating_point<DstScalar>::value ||
std::is_same<DstScalar, std::int32_t>::value) {
*multiplier = 0;
return;
}
*multiplier = static_cast<double>(std::numeric_limits<DstScalar>::max()) /
(static_cast<double>(lhs.layout().cols()) *
std::numeric_limits<LhsScalar>::max() *
std::numeric_limits<RhsScalar>::max());
}
inline void QuantizeMultiplier(double multiplier_double,
std::int32_t* multiplier_fixedpoint,
int* multiplier_exponent) {
RUY_CHECK_GT(multiplier_double, 0);
if (multiplier_double == 0.) {
*multiplier_fixedpoint = 0;
*multiplier_exponent = 0;
return;
}
const double q = std::frexp(multiplier_double, multiplier_exponent);
auto q_fixed = static_cast<std::int64_t>(std::round(q * (1ll << 31)));
RUY_CHECK_LE(q_fixed, (1ll << 31));
if (q_fixed == (1ll << 31)) {
q_fixed /= 2;
++*multiplier_exponent;
}
RUY_CHECK_LE(q_fixed, std::numeric_limits<std::int32_t>::max());
*multiplier_fixedpoint = static_cast<std::int32_t>(q_fixed);
}
template <typename TestSetType>
void SwitchMultiplierToPerChannel(TestSetType* test_set) {
test_set->per_channel_multiplier_fixedpoint.resize(test_set->rows);
test_set->per_channel_multiplier_exponent.resize(test_set->rows);
for (int i = 0; i < test_set->rows; i++) {
// multipliers typically range in [2^30 ; 2^31 - 1].
// Values in [0, 2^30 - 1] are normally unused, but harmless.
// Thus a good way to randomize multipliers is to subtract from them
// a random value smaller than 2^30 but still significant compared to it.
std::int32_t nudged_multiplier =
test_set->mul_params.multiplier_fixedpoint() -
(global_random_engine()() % (1 << 26));
int nudged_exponent = test_set->mul_params.multiplier_exponent() - 1 +
(global_random_engine()() % 4);
test_set->per_channel_multiplier_fixedpoint[i] = nudged_multiplier;
test_set->per_channel_multiplier_exponent[i] = nudged_exponent;
}
test_set->mul_params.set_multiplier_fixedpoint_perchannel(
test_set->per_channel_multiplier_fixedpoint.data());
test_set->mul_params.set_multiplier_exponent_perchannel(
test_set->per_channel_multiplier_exponent.data());
test_set->mul_params.set_multiplier_fixedpoint(0);
test_set->mul_params.set_multiplier_exponent(0);
}
template <
typename TestSetType,
bool IsApplicable =
std::is_same<typename TestSetType::AccumScalar, std::int32_t>::value &&
!std::is_same<typename TestSetType::DstScalar, std::int32_t>::value>
struct MakeSpecMultiplierFieldsImpl {};
template <typename TestSetType>
struct MakeSpecMultiplierFieldsImpl<TestSetType, true> {
static void Run(TestSetType* test_set) {
double multiplier;
ComputeReasonableMultiplier<TestSetType>(test_set->lhs.matrix,
test_set->rhs.matrix, &multiplier);
typename TestSetType::AccumScalar multiplier_fixedpoint;
int multiplier_exponent;
QuantizeMultiplier(multiplier, &multiplier_fixedpoint,
&multiplier_exponent);
test_set->mul_params.set_multiplier_fixedpoint(multiplier_fixedpoint);
test_set->mul_params.set_multiplier_exponent(multiplier_exponent);
if (!test_set->benchmark) {
test_set->perchannel = global_random_engine()() & 1;
}
if (test_set->perchannel) {
SwitchMultiplierToPerChannel(test_set);
}
}
};
template <typename TestSetType>
struct MakeSpecMultiplierFieldsImpl<TestSetType, false> {
static void Run(TestSetType* test_set) {
test_set->mul_params.set_multiplier_fixedpoint(0);
test_set->mul_params.set_multiplier_exponent(0);
}
};
template <typename MulParamsType>
void MakeSpecClampFields(MulParamsType* mul_params) {
using AccumScalar = typename MulParamsType::AccumScalar;
using DstScalar = typename MulParamsType::DstScalar;
if (std::is_same<AccumScalar, std::int32_t>::value) {
// Returning raw accumulators, clamping is not supported.
mul_params->set_clamp_min(std::numeric_limits<DstScalar>::lowest());
mul_params->set_clamp_max(std::numeric_limits<DstScalar>::max());
return;
}
if (getenv("BENCHMARK_ONLY_MATMUL")) {
if (std::is_floating_point<DstScalar>::value) {
mul_params->set_clamp_min(-std::numeric_limits<DstScalar>::infinity());
mul_params->set_clamp_max(std::numeric_limits<DstScalar>::infinity());
} else {
mul_params->set_clamp_min(std::numeric_limits<DstScalar>::lowest());
mul_params->set_clamp_max(std::numeric_limits<DstScalar>::max());
}
return;
}
mul_params->set_clamp_min(std::numeric_limits<DstScalar>::lowest() + 1);
mul_params->set_clamp_max(std::numeric_limits<DstScalar>::max() - 1);
}
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::MakeZeroPoints() {
RUY_CHECK_EQ(life_stage, LifeStage::kInitial);
if (!benchmark && !use_specified_zero_points) {
MakeRandomScalar(RandomRange::kReasonableSrcZeroPoint, &lhs_zero_point);
MakeRandomScalar(RandomRange::kReasonableSrcZeroPoint, &rhs_zero_point);
// If destination is std::int32_t, no dst_zero_point is necessary.
if (std::is_same<DstScalar, std::int32_t>::value) {
dst_zero_point = 0;
} else {
MakeRandomScalar(RandomRange::kReasonableDstZeroPoint, &dst_zero_point);
}
}
life_stage = LifeStage::kHasZeroPoints;
}
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::MakeLhsRhs() {
RUY_CHECK_EQ(life_stage, LifeStage::kHasZeroPoints);
MakeRandom(rows, depth, lhs_order, lhs_zero_point, layout_style,
RandomRange::kOffCenterAvoidMinValue, &lhs);
MakeRandom(depth, cols, rhs_order, rhs_zero_point, layout_style,
RandomRange::kGeneral, &rhs);
if (!benchmark) {
cache_lhs = (global_random_engine()() & 0xf) == 0;
cache_rhs = (global_random_engine()() & 0xf) == 0;
}
if (cache_lhs) {
lhs.matrix.set_cache_policy(CachePolicy::kAlwaysCache);
}
if (cache_rhs) {
rhs.matrix.set_cache_policy(CachePolicy::kAlwaysCache);
}
life_stage = LifeStage::kHasLhsRhs;
}
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::MakeMulParams() {
RUY_CHECK_EQ(life_stage, LifeStage::kHasLhsRhs);
if (!getenv("BENCHMARK_ONLY_MATMUL") &&
(benchmark || (global_random_engine()() & 1))) {
MakeRandomVector(RandomRange::kBias, rows, &bias_data);
mul_params.set_bias(bias_data.data());
}
if (lhs.matrix.zero_point() == std::numeric_limits<LhsScalar>::lowest() &&
rhs.matrix.zero_point() == std::numeric_limits<RhsScalar>::lowest()) {
lhs.matrix.set_zero_point(lhs.matrix.zero_point() + 1);
}
MakeSpecMultiplierFieldsImpl<TestSet>::Run(this);
MakeSpecClampFields(&mul_params);
life_stage = LifeStage::kHasMulParams;
}
inline int GetIntEnvVarOrZero(const char* name) {
const char* val = getenv(name);
if (!val) {
return 0;
}
return std::stoi(val);
}
inline float GetFloatEnvVarOrZero(const char* name) {
const char* val = getenv(name);
if (!val) {
return 0;
}
return std::stof(val);
}
inline int GetHexIntEnvVarOrZero(const char* name) {
const char* val = getenv(name);
if (!val) {
return 0;
}
return std::stoi(val, nullptr, 16);
}
inline bool GetBoolEnvVarOrFalse(const char* name) {
return static_cast<bool>(GetIntEnvVarOrZero(name));
}
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::MakeOtherParams() {
RUY_CHECK_EQ(life_stage, LifeStage::kHasMulParams);
if (max_num_threads == 0) {
max_num_threads = GetIntEnvVarOrZero("THREADS");
}
life_stage = LifeStage::kHasOtherParams;
}
inline std::vector<Path> PathsBitfieldAsVector(Path paths_bitfield) {
std::vector<Path> result;
std::uint32_t remaining_paths = static_cast<std::uint32_t>(paths_bitfield);
std::uint32_t test_bit = 1;
while (remaining_paths) {
if (remaining_paths & test_bit) {
result.push_back(static_cast<Path>(test_bit));
}
remaining_paths &= ~test_bit;
test_bit <<= 1;
}
return result;
}
inline std::vector<Tuning> EnumerateTuningsForPath(Path path, bool benchmark) {
if (benchmark) {
return {Tuning::kAuto};
}
#if RUY_PLATFORM_ARM
if (path == Path::kNeon || path == Path::kNeonDotprod) {
return {Tuning::kInOrder, Tuning::kOutOfOrder, Tuning::kAuto};
}
#endif
(void)path;
return {Tuning::kAuto};
}
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::MakeResultPaths() {
RUY_CHECK_EQ(life_stage, LifeStage::kHasOtherParams);
Path paths_bitfield = static_cast<Path>(GetHexIntEnvVarOrZero("PATHS"));
if (paths_bitfield == Path::kNone) {
// Use a dummy Context just to perform the resolution of specific runtime
// enabled paths.
Context context;
paths_bitfield = get_ctx(&context)->GetRuntimeEnabledPaths();
}
// Trim bits that don't correspond to a compiled path,
// to allow specifying e.g. ffff to mean 'all paths' regardless of whether all
// those bits exist as actual paths.
paths_bitfield = paths_bitfield & kAllPaths;
RUY_CHECK_NE(paths_bitfield, Path::kNone);
paths = PathsBitfieldAsVector(paths_bitfield);
// kReference is a special 'external path' that's always available.
external_paths.push_back(ExternalPath::kReference);
#ifdef RUY_TEST_EXTERNAL_PATHS
using TestSetType = TestSet<LhsScalar, RhsScalar, SpecType>;
if (!GetBoolEnvVarOrFalse("NOEXT")) {
if (SupportsGemmlowp<TestSetType>::kValue) {
#ifdef GEMMLOWP_SSE4
const bool gemmlowp_supported =
!mul_params.multiplier_fixedpoint_perchannel();
#else
const bool gemmlowp_supported = true;
#endif
if (gemmlowp_supported) {
external_paths.push_back(ExternalPath::kGemmlowp);
}
}
if (UsesSingleScalarType<TestSetType>::kValue &&
std::is_floating_point<AccumScalar>::value) {
external_paths.push_back(ExternalPath::kEigen);
if (layout_style == LayoutStyle::kUnstridedLinear) {
external_paths.push_back(ExternalPath::kEigenTensor);
}
// We link against a generic BLAS target that only maps to OpenBLAS on specific
// architectures.
#if RUY_PLATFORM_ARM_32 || RUY_PLATFORM_ARM_64
// OpenBLAS multi-threading is disabled, so avoid mixing single-threaded
// and multi-threaded benchmark results.
if (max_num_threads == 1 && !getenv("NO_OPENBLAS")) {
external_paths.push_back(ExternalPath::kOpenBlas);
}
#endif
}
}
#endif // RUY_TEST_EXTERNAL_PATHS
for (Path path : paths) {
for (Tuning tuning : EnumerateTuningsForPath(path, benchmark)) {
results.emplace_back(new TestResultType);
TestResultType& result = *results.back();
result.path = path;
result.tuning = tuning;
MakeRandom(rows, cols, dst_order, dst_zero_point, layout_style,
RandomRange::kGeneral, &result.storage_matrix);
}
}
for (ExternalPath external_path : external_paths) {
results.emplace_back(new TestResultType);
TestResultType& result = *results.back();
result.external_path = external_path;
MakeRandom(rows, cols, dst_order, dst_zero_point, layout_style,
RandomRange::kGeneral, &result.storage_matrix);
}
life_stage = LifeStage::kHasResultPaths;
}
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::EvalResult(
TestResult<typename SpecType::DstScalar>* result) {
RUY_CHECK(result->path != Path::kNone ||
result->external_path != ExternalPath::kNone);
if (result->path != Path::kNone) {
EvalRuy(result);
} else {
EvalExternalPath(this, result);
}
const std::string& pathname = PathName(*result);
if (std::find(CoveredPaths()->begin(), CoveredPaths()->end(), pathname) ==
CoveredPaths()->end()) {
CoveredPaths()->push_back(pathname);
}
}
using f32 = float;
using f64 = double;
using u8 = std::uint8_t;
using i8 = std::int8_t;
using u16 = std::uint16_t;
using i16 = std::int16_t;
using u32 = std::uint32_t;
using i32 = std::int32_t;
using u64 = std::uint64_t;
using i64 = std::int64_t;
template <typename Scalar>
const char* TypeName() {
return nullptr;
}
#define RUY_TYPENAME(TYPE) \
template <> \
const char* TypeName<TYPE>() { \
return #TYPE; \
}
RUY_TYPENAME(f32)
RUY_TYPENAME(f64)
RUY_TYPENAME(u8)
RUY_TYPENAME(i8)
RUY_TYPENAME(u16)
RUY_TYPENAME(i16)
RUY_TYPENAME(u32)
RUY_TYPENAME(i32)
RUY_TYPENAME(u64)
RUY_TYPENAME(i64)
#undef RUY_TYPENAME
template <typename Scalar>
const char* SymmetryName(const Matrix<Scalar>& matrix) {
if (matrix.zero_point() == SymmetricZeroPoint<Scalar>()) {
return "symm";
} else {
return "asymm";
}
}
template <typename Scalar>
int StorageSize(const Matrix<Scalar>& matrix) {
return sizeof(Scalar) * FlatSize(matrix.layout());
}
// Helper that replicates a buffer and gives out pointers to the replicas.
// This is useful when one wants to traverse data so that it is cold in cache.
// By having a sufficiently large value of num_repeats, one can ensure that the
// working set covered by the replicas is greater than the cache size.
template <typename T>
class RepeatedBuffer {
public:
RepeatedBuffer() = default;
void Init(const T* elems, int num_elems, int num_repeats) {
buffers_.clear();
allocator_.FreeAll();
for (int i = 0; i < num_repeats; i++) {
T* p;
allocator_.Allocate(num_elems, &p);
memcpy(p, elems, num_elems * sizeof(T));
buffers_.push_back(p);
}
}
T* Next() {
T* ret = buffers_[current_];
current_ = (current_ + 1) % buffers_.size();
return ret;
}
private:
Allocator allocator_;
std::vector<T*> buffers_;
int current_ = 0;
};
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::Benchmark(
TestResult<typename SpecType::DstScalar>* result) {
using DstScalar = typename SpecType::DstScalar;
const bool cold = getenv("RUY_BENCHMARK_COLD");
LhsScalar* orig_lhs_data = lhs.matrix.data();
RhsScalar* orig_rhs_data = rhs.matrix.data();
DstScalar* orig_dst_data = result->storage_matrix.matrix.data();
int num_matmul_sets = 0;
RepeatedBuffer<LhsScalar> cold_lhs;
RepeatedBuffer<RhsScalar> cold_rhs;
RepeatedBuffer<DstScalar> cold_dst;
if (cold) {
const int kWorkingSetSize = 100 << 20;
const int each_matmul_set_size = StorageSize(lhs.matrix) +
StorageSize(rhs.matrix) +
StorageSize(result->storage_matrix.matrix);
num_matmul_sets =
(kWorkingSetSize + each_matmul_set_size - 1) / each_matmul_set_size;
cold_lhs.Init(lhs.matrix.data(), FlatSize(lhs.matrix.layout()),
num_matmul_sets);
cold_rhs.Init(rhs.matrix.data(), FlatSize(rhs.matrix.layout()),
num_matmul_sets);
cold_dst.Init(result->storage_matrix.matrix.data(),
FlatSize(result->storage_matrix.matrix.layout()),
num_matmul_sets);
}
const bool record_pmu = GetBoolEnvVarOrFalse("RUY_BENCHMARK_PMU");
int repeats = GetIntEnvVarOrZero("RUY_BENCHMARK_REPEATS");
if (!repeats) {
repeats = 4;
}
float benchmark_min_secs = GetFloatEnvVarOrZero("RUY_BENCHMARK_MIN_SECS");
if (!benchmark_min_secs) {
benchmark_min_secs = 0.5;
}
#ifdef RUY_PROFILER
{
const char* lhstype = TypeName<LhsScalar>();
const char* lhssymm = SymmetryName(lhs.matrix);
const char* rhstype = TypeName<RhsScalar>();
const char* rhssymm = SymmetryName(rhs.matrix);
printf("Profiling path=%s shape=(%dx%dx%d) lhs=(%s,%s) rhs=(%s,%s)\n",
PathName(*result).c_str(), rows, depth, cols, lhstype, lhssymm,
rhstype, rhssymm);
ruy::profiler::ScopeProfile profile;
#endif
float latency = std::numeric_limits<float>::infinity();
float l1_refill_rate = std::numeric_limits<float>::infinity();
float l2_refill_rate = std::numeric_limits<float>::infinity();
float l3_refill_rate = std::numeric_limits<float>::infinity();
float l1tlb_refill_rate = std::numeric_limits<float>::infinity();
float l2tlb_refill_rate = std::numeric_limits<float>::infinity();
float mispred_rate = std::numeric_limits<float>::infinity();
float frontend_stall_rate = std::numeric_limits<float>::infinity();
float backend_stall_rate = std::numeric_limits<float>::infinity();
for (int repeat = 0; repeat < repeats; repeat++) {
auto& pmu_events = GlobalPmuEvents();
if (record_pmu) {
pmu_events.StartRecording();
}
TimePoint time_start = Now();
TimePoint t = time_start;
int iters = 0;
int iters_at_a_time = 1;
while (ToFloatSeconds(t - time_start) < benchmark_min_secs) {
for (int i = 0; i < iters_at_a_time; i++) {
if (cold) {
lhs.matrix.set_data(cold_lhs.Next());
rhs.matrix.set_data(cold_rhs.Next());
result->storage_matrix.matrix.set_data(cold_dst.Next());
}
EvalResult(result);
iters++;
}
iters_at_a_time *= 2;
t = Now();
}
latency = std::min(
latency, static_cast<float>(ToFloatSeconds(t - time_start) / iters));
if (record_pmu) {
pmu_events.StopRecording();
const float normalization_factor =
1.0f / (static_cast<float>(iters) * rows * cols * depth);
l1_refill_rate = std::min(
l1_refill_rate, pmu_events.L1RefillCount() * normalization_factor);
l2_refill_rate = std::min(
l2_refill_rate, pmu_events.L2RefillCount() * normalization_factor);
l3_refill_rate = std::min(
l3_refill_rate, pmu_events.L3RefillCount() * normalization_factor);
l1tlb_refill_rate =
std::min(l1tlb_refill_rate,
pmu_events.L1TLBRefillCount() * normalization_factor);
l2tlb_refill_rate =
std::min(l2tlb_refill_rate,
pmu_events.L2TLBRefillCount() * normalization_factor);
mispred_rate =
std::min(mispred_rate, pmu_events.BranchMispredictionCount() *
normalization_factor);
frontend_stall_rate =
std::min(frontend_stall_rate,
pmu_events.FrontendStallCount() * normalization_factor);
backend_stall_rate =
std::min(backend_stall_rate,
pmu_events.BackendStallCount() * normalization_factor);
}
}
result->latency = latency;
if (record_pmu) {
result->l1_refill_rate = l1_refill_rate;
result->l2_refill_rate = l2_refill_rate;
result->l3_refill_rate = l3_refill_rate;
result->l1tlb_refill_rate = l1tlb_refill_rate;
result->l2tlb_refill_rate = l2tlb_refill_rate;
result->mispred_rate = mispred_rate;
result->frontend_stall_rate = frontend_stall_rate;
result->backend_stall_rate = backend_stall_rate;
}
#ifdef RUY_PROFILER
}
fflush(stdout);
#endif
if (cold) {
lhs.matrix.set_data(orig_lhs_data);
rhs.matrix.set_data(orig_rhs_data);
memcpy(orig_dst_data, result->storage_matrix.matrix.data(),
StorageSize(result->storage_matrix.matrix));
result->storage_matrix.matrix.set_data(orig_dst_data);
}
}
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::Eval() {
RUY_CHECK_EQ(life_stage, LifeStage::kHasResultPaths);
for (auto& result : results) {
if (benchmark) {
Benchmark(result.get());
} else {
EvalResult(result.get());
}
}
life_stage = LifeStage::kEvaluated;
}
template <typename Scalar>
std::string DumpRegion(const Matrix<Scalar>& matrix, int center_row,
int center_col) {
static constexpr int kRadius = 20;
int first_row = std::max(0, center_row - kRadius);
int last_row = std::min(matrix.layout().rows() - 1, center_row + kRadius);
int first_col = std::max(0, center_col - kRadius);
int last_col = std::min(matrix.layout().cols() - 1, center_col + kRadius);
std::ostringstream stream;
for (int row = first_row; row <= last_row; row++) {
for (int col = first_col; col <= last_col; col++) {
stream << static_cast<double>(Element(matrix, row, col)) << " ";
}
stream << "\n";
}
return stream.str();
}
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::VerifyTestResults() const {
const int depth = lhs.matrix.layout().cols();
for (int i = 0; i < static_cast<int>(results.size()) - 1; i++) {
if (!Agree(*results[i], *results[i + 1], depth)) {
std::string paths_in_agreement;
paths_in_agreement.append(PathName(*results[0]));
for (int j = 1; j <= i; j++) {
paths_in_agreement.append(", ");
paths_in_agreement.append(PathName(*results[j]));
}
ErrorAnalysis error_analysis;
AnalyzeTestError(*this, i + 1, &error_analysis);
std::cerr << "Error: path (" << PathName(*results[i + 1])
<< ") disagrees with the other paths (" << paths_in_agreement
<< "), which agree with each other." << std::endl;
std::cerr << "Shape: rows = " << rows << ", cols = " << cols
<< ", depth = " << depth << std::endl;
std::cerr << "Stats of the good result matrix: "
<< StatsAsString(error_analysis.stats_good) << std::endl;
std::cerr << "Stats of the bad result matrix: "
<< StatsAsString(error_analysis.stats_bad) << std::endl;
if (static_cast<int>(error_analysis.error_rows.size()) < rows) {
std::cerr << "Rows containing errors: "
<< Join(error_analysis.error_rows) << std::endl;
} else {
std::cerr << "Errors found in ALL rows." << std::endl;
}
if (static_cast<int>(error_analysis.error_cols.size()) < cols) {
std::cerr << "Cols containing errors: "
<< Join(error_analysis.error_cols) << std::endl;
} else {
std::cerr << "Errors found in ALL cols." << std::endl;
}
std::cerr << "The first error occurs at row "
<< error_analysis.row_of_first_error << ", col "
<< error_analysis.col_of_first_error << std::endl;
std::cerr << "Good value: " << error_analysis.first_error_good_value
<< std::endl;
std::cerr << "Bad value : " << error_analysis.first_error_bad_value
<< std::endl;
std::cerr << "Region of Good result matrix around first error:\n\n"
<< DumpRegion(results[0]->storage_matrix.matrix,
error_analysis.row_of_first_error,
error_analysis.col_of_first_error)
<< std::endl;
std::cerr << "Region of Bad result matrix around first error:\n\n"
<< DumpRegion(results[i + 1]->storage_matrix.matrix,
error_analysis.row_of_first_error,
error_analysis.col_of_first_error)
<< std::endl;
RUY_CHECK(false);
}
}
}
template <typename LhsScalar, typename RhsScalar, typename SpecType>
void TestSet<LhsScalar, RhsScalar, SpecType>::Verify() {
RUY_CHECK_EQ(life_stage, LifeStage::kEvaluated);
if (expected_outcome == ExpectedOutcome::kSuccess) {
VerifyTestResults();
}
life_stage = LifeStage::kFinal;
}
template <typename TestSetType>
void TestRCC(int rows, int depth, int cols, ExpectedOutcome expected_outcome) {
TestSetType test_set;
test_set.rows = rows;
test_set.depth = depth;
test_set.cols = cols;
test_set.lhs_order = Order::kRowMajor;
test_set.rhs_order = Order::kColMajor;
test_set.dst_order = Order::kColMajor;
test_set.layout_style = LayoutStyle::kUnstridedLinear;
test_set.expected_outcome = expected_outcome;
test_set.Run();
}
template <typename TestSetType>
void TestRCC(int rows, int depth, int cols) {
TestRCC<TestSetType>(rows, depth, cols, ExpectedOutcome::kSuccess);
}
template <typename TestSetType>
void TestNonRCC(int rows, int depth, int cols,
ExpectedOutcome expected_outcome) {
TestSetType test_set;
test_set.rows = rows;
test_set.depth = depth;
test_set.cols = cols;
test_set.lhs_order = Order::kColMajor;
test_set.rhs_order = Order::kColMajor;
test_set.dst_order = Order::kColMajor;
test_set.layout_style = LayoutStyle::kUnstridedLinear;
test_set.expected_outcome = expected_outcome;
test_set.Run();
}
template <typename TestSetType>
void TestLinearAllOrders(int rows, int depth, int cols,
ExpectedOutcome expected_outcome) {
const std::vector<Order> orders{Order::kColMajor, Order::kRowMajor};
for (Order lhs_order : orders) {
for (Order rhs_order : orders) {
for (Order dst_order : orders) {
TestSetType test_set;
test_set.rows = rows;
test_set.depth = depth;
test_set.cols = cols;
test_set.lhs_order = lhs_order;
test_set.rhs_order = rhs_order;
test_set.dst_order = dst_order;
test_set.layout_style = LayoutStyle::kLinear;
test_set.expected_outcome = expected_outcome;
test_set.Run();
}
}
}
}
template <typename TestSetType>
void TestLinearAllOrders(int rows, int depth, int cols) {
TestLinearAllOrders<TestSetType>(rows, depth, cols,
ExpectedOutcome::kSuccess);
}
} // namespace ruy
#endif // RUY_RUY_TEST_H_