blob: efd731298261e6e8a0609e485e94f3b3fd4e99f9 [file] [log] [blame]
// Copyright (c) 2012 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include <stdint.h>
#include "base/format_macros.h"
#include "base/strings/stringprintf.h"
#include "gpu/config/gpu_test_expectations_parser.h"
#include "testing/gtest/include/gtest/gtest.h"
namespace gpu {
struct TestOSEntry {
const char* name;
GPUTestConfig::OS os;
};
static const TestOSEntry kOsFamilyWin = { "WIN", GPUTestConfig::kOsWin };
static const TestOSEntry kOsFamilyMac = { "MAC", GPUTestConfig::kOsMac };
static const struct TestOsWithFamily {
TestOSEntry version;
TestOSEntry family;
} kOSVersionsWithFamily[] = {
{{"XP", GPUTestConfig::kOsWinXP}, kOsFamilyWin},
{{"VISTA", GPUTestConfig::kOsWinVista}, kOsFamilyWin},
{{"WIN7", GPUTestConfig::kOsWin7}, kOsFamilyWin},
{{"WIN8", GPUTestConfig::kOsWin8}, kOsFamilyWin},
{{"WIN10", GPUTestConfig::kOsWin10}, kOsFamilyWin},
{{"LEOPARD", GPUTestConfig::kOsMacLeopard}, kOsFamilyMac},
{{"SNOWLEOPARD", GPUTestConfig::kOsMacSnowLeopard}, kOsFamilyMac},
{{"LION", GPUTestConfig::kOsMacLion}, kOsFamilyMac},
{{"MOUNTAINLION", GPUTestConfig::kOsMacMountainLion}, kOsFamilyMac},
{{"MAVERICKS", GPUTestConfig::kOsMacMavericks}, kOsFamilyMac},
{{"YOSEMITE", GPUTestConfig::kOsMacYosemite}, kOsFamilyMac},
{{"ELCAPITAN", GPUTestConfig::kOsMacElCapitan}, kOsFamilyMac},
{{"SIERRA", GPUTestConfig::kOsMacSierra}, kOsFamilyMac},
{{"HIGHSIERRA", GPUTestConfig::kOsMacHighSierra}, kOsFamilyMac},
{{"MOJAVE", GPUTestConfig::kOsMacMojave}, kOsFamilyMac},
{{"CATALINA", GPUTestConfig::kOsMacCatalina}, kOsFamilyMac},
{{"BIGSUR", GPUTestConfig::kOsMacBigSur}, kOsFamilyMac},
{{"MONTEREY", GPUTestConfig::kOsMacMonterey}, kOsFamilyMac},
{{"LINUX", GPUTestConfig::kOsLinux}, {"LINUX", GPUTestConfig::kOsLinux}},
{{"CHROMEOS", GPUTestConfig::kOsChromeOS},
{"CHROMEOS", GPUTestConfig::kOsChromeOS}},
{{"ANDROID", GPUTestConfig::kOsAndroid},
{"ANDROID", GPUTestConfig::kOsAndroid}}};
TestOSEntry GetUnrelatedOS(const TestOSEntry& os) {
return (os.os & kOsFamilyWin.os) ? kOsFamilyMac : kOsFamilyWin;
}
// Prints test parameter details.
std::ostream& operator << (std::ostream& out, const TestOsWithFamily& os) {
out << "{ os_name: \"" << os.version.name
<< "\", os_flag: " << os.version.os
<< ", os_family: \"" << os.family.name
<< "\", os_family_flag: " << os.family.os
<< " }";
return out;
}
class GPUTestExpectationsParserTest : public testing::Test {
public:
GPUTestExpectationsParserTest() = default;
~GPUTestExpectationsParserTest() override = default;
const GPUTestBotConfig& bot_config() const {
return bot_config_;
}
protected:
void SetUp() override {
bot_config_.set_os(GPUTestConfig::kOsWin7);
bot_config_.set_build_type(GPUTestConfig::kBuildTypeRelease);
bot_config_.AddGPUVendor(0x10de);
bot_config_.set_gpu_device_id(0x0640);
bot_config_.set_api(GPUTestConfig::kAPID3D11);
bot_config_.set_command_decoder(GPUTestConfig::kCommandDecoderPassthrough);
ASSERT_TRUE(bot_config_.IsValid());
}
void TearDown() override {}
void set_os(int32_t os) {
bot_config_.set_os(os);
ASSERT_TRUE(bot_config_.IsValid());
}
private:
GPUTestBotConfig bot_config_;
};
class GPUTestExpectationsParserParamTest
: public GPUTestExpectationsParserTest,
public testing::WithParamInterface<TestOsWithFamily> {
public:
GPUTestExpectationsParserParamTest() = default;
~GPUTestExpectationsParserParamTest() override = default;
protected:
const GPUTestBotConfig& GetBotConfig() {
set_os(GetParam().version.os);
return bot_config();
}
private:
// Restrict access to bot_config() function.
// GetBotConfig() should be used instead.
using GPUTestExpectationsParserTest::bot_config;
};
TEST_F(GPUTestExpectationsParserTest, CommentOnly) {
const std::string text =
" \n"
"// This is just some comment\n"
"";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
parser.GetTestExpectation("some_test", bot_config()));
}
TEST_P(GPUTestExpectationsParserParamTest, ValidFullEntry) {
const std::string text =
base::StringPrintf("BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
GetParam().version.name);
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
parser.GetTestExpectation("MyTest", GetBotConfig()));
}
TEST_P(GPUTestExpectationsParserParamTest, ValidPartialEntry) {
const std::string text =
base::StringPrintf("BUG12345 %s NVIDIA : MyTest = TIMEOUT",
GetParam().family.name);
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestTimeout,
parser.GetTestExpectation("MyTest", GetBotConfig()));
}
TEST_P(GPUTestExpectationsParserParamTest, ValidUnrelatedOsEntry) {
const std::string text = base::StringPrintf(
"BUG12345 %s : MyTest = TIMEOUT",
GetUnrelatedOS(GetParam().version).name);
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
parser.GetTestExpectation("MyTest", GetBotConfig()));
}
TEST_F(GPUTestExpectationsParserTest, ValidUnrelatedTestEntry) {
const std::string text =
"BUG12345 WIN7 RELEASE NVIDIA 0x0640 : AnotherTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
parser.GetTestExpectation("MyTest", bot_config()));
}
TEST_F(GPUTestExpectationsParserTest, AllModifiers) {
const std::string text =
"BUG12345 XP VISTA WIN7 WIN8 WIN10 LEOPARD SNOWLEOPARD LION MOUNTAINLION "
"MAVERICKS LINUX CHROMEOS ANDROID "
"NVIDIA INTEL AMD VMWARE RELEASE DEBUG : MyTest = "
"PASS FAIL FLAKY TIMEOUT SKIP";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
GPUTestExpectationsParser::kGpuTestFail |
GPUTestExpectationsParser::kGpuTestFlaky |
GPUTestExpectationsParser::kGpuTestTimeout |
GPUTestExpectationsParser::kGpuTestSkip,
parser.GetTestExpectation("MyTest", bot_config()));
}
TEST_P(GPUTestExpectationsParserParamTest, DuplicateModifiers) {
const std::string text = base::StringPrintf(
"BUG12345 %s %s RELEASE NVIDIA 0x0640 : MyTest = FAIL",
GetParam().version.name,
GetParam().version.name);
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.LoadTestExpectations(text));
EXPECT_NE(0u, parser.GetErrorMessages().size());
}
TEST_F(GPUTestExpectationsParserTest, AllModifiersLowerCase) {
const std::string text =
"BUG12345 xp vista win7 win8 win10 leopard snowleopard lion linux "
"chromeos android nvidia intel amd vmware release debug : MyTest = "
"pass fail flaky timeout skip";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass |
GPUTestExpectationsParser::kGpuTestFail |
GPUTestExpectationsParser::kGpuTestFlaky |
GPUTestExpectationsParser::kGpuTestTimeout |
GPUTestExpectationsParser::kGpuTestSkip,
parser.GetTestExpectation("MyTest", bot_config()));
}
TEST_F(GPUTestExpectationsParserTest, MissingColon) {
const std::string text =
"BUG12345 XP MyTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.LoadTestExpectations(text));
EXPECT_NE(0u, parser.GetErrorMessages().size());
}
TEST_F(GPUTestExpectationsParserTest, MissingEqual) {
const std::string text =
"BUG12345 XP : MyTest FAIL";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.LoadTestExpectations(text));
EXPECT_NE(0u, parser.GetErrorMessages().size());
}
TEST_F(GPUTestExpectationsParserTest, IllegalModifier) {
const std::string text =
"BUG12345 XP XXX : MyTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.LoadTestExpectations(text));
EXPECT_NE(0u, parser.GetErrorMessages().size());
}
TEST_P(GPUTestExpectationsParserParamTest, OsConflicts) {
const std::string text = base::StringPrintf("BUG12345 %s %s : MyTest = FAIL",
GetParam().version.name,
GetParam().family.name);
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.LoadTestExpectations(text));
EXPECT_NE(0u, parser.GetErrorMessages().size());
}
TEST_F(GPUTestExpectationsParserTest, InvalidModifierCombination) {
const std::string text =
"BUG12345 XP NVIDIA INTEL 0x0640 : MyTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.LoadTestExpectations(text));
EXPECT_NE(0u, parser.GetErrorMessages().size());
}
TEST_F(GPUTestExpectationsParserTest, BadGpuDeviceID) {
const std::string text =
"BUG12345 XP NVIDIA 0xU07X : MyTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.LoadTestExpectations(text));
EXPECT_NE(0u, parser.GetErrorMessages().size());
}
TEST_F(GPUTestExpectationsParserTest, MoreThanOneGpuDeviceID) {
const std::string text =
"BUG12345 XP NVIDIA 0x0640 0x0641 : MyTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.LoadTestExpectations(text));
EXPECT_NE(0u, parser.GetErrorMessages().size());
}
TEST_P(GPUTestExpectationsParserParamTest, MultipleEntriesConflicts) {
const std::string text = base::StringPrintf(
"BUG12345 %s RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
"BUG12345 %s : MyTest = FAIL",
GetParam().version.name,
GetParam().family.name);
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.LoadTestExpectations(text));
EXPECT_NE(0u, parser.GetErrorMessages().size());
}
TEST_F(GPUTestExpectationsParserTest, MultipleTests) {
const std::string text =
"BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
"BUG12345 WIN : AnotherTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
}
TEST_F(GPUTestExpectationsParserTest, ValidMultipleEntries) {
const std::string text =
"BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest = FAIL\n"
"BUG12345 LINUX : MyTest = TIMEOUT";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
parser.GetTestExpectation("MyTest", bot_config()));
}
TEST_F(GPUTestExpectationsParserTest, StarMatching) {
const std::string text =
"BUG12345 WIN7 RELEASE NVIDIA 0x0640 : MyTest* = FAIL";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
parser.GetTestExpectation("MyTest0", bot_config()));
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
parser.GetTestExpectation("OtherTest", bot_config()));
}
TEST_F(GPUTestExpectationsParserTest, ValidAPI) {
const std::string text =
"BUG12345 WIN7 NVIDIA D3D9 D3D11 OPENGL GLES : MyTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
parser.GetTestExpectation("MyTest", bot_config()));
}
TEST_F(GPUTestExpectationsParserTest, MultipleAPIsConflict) {
const std::string text = "BUG12345 WIN7 NVIDIA D3D9 D3D9 : MyTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.LoadTestExpectations(text));
EXPECT_NE(0u, parser.GetErrorMessages().size());
}
TEST_F(GPUTestExpectationsParserTest, PassthroughCommandDecoder) {
const std::string text = "BUG12345 PASSTHROUGH : MyTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestFail,
parser.GetTestExpectation("MyTest", bot_config()));
}
TEST_F(GPUTestExpectationsParserTest, ValidatingCommandDecoder) {
const std::string text = "BUG12345 VALIDATING : MyTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_TRUE(parser.LoadTestExpectations(text));
EXPECT_EQ(0u, parser.GetErrorMessages().size());
EXPECT_EQ(GPUTestExpectationsParser::kGpuTestPass,
parser.GetTestExpectation("MyTest", bot_config()));
}
TEST_F(GPUTestExpectationsParserTest, MultipleCommandDecodersConflict) {
const std::string text = "BUG12345 VALIDATING VALIDATING : MyTest = FAIL";
GPUTestExpectationsParser parser;
EXPECT_FALSE(parser.LoadTestExpectations(text));
EXPECT_NE(0u, parser.GetErrorMessages().size());
}
INSTANTIATE_TEST_SUITE_P(GPUTestExpectationsParser,
GPUTestExpectationsParserParamTest,
::testing::ValuesIn(kOSVersionsWithFamily));
} // namespace gpu