| // Copyright 2018 The Chromium OS Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| package graphics |
| |
| import ( |
| "context" |
| "os" |
| "path/filepath" |
| "strings" |
| |
| "chromiumos/tast/common/testexec" |
| "chromiumos/tast/errors" |
| "chromiumos/tast/local/graphics" |
| "chromiumos/tast/testing" |
| ) |
| |
| func init() { |
| testing.AddTest(&testing.Test{ |
| Func: DEQP, |
| Desc: "Runs a pre-CQ-suitable subset of the drawElements Quality Program test suite shipped with test images", |
| Contacts: []string{"andrescj@chromium.org", "ihf@chromium.org", "chromeos-gfx@google.com"}, |
| SoftwareDeps: []string{"no_qemu"}, |
| Attr: []string{"group:mainline"}, |
| Fixture: "gpuWatchHangs", |
| }) |
| } |
| |
| // deqpTests contains the names of the DEQP tests to run. Some may be skipped depending on the supported graphics APIs. |
| var deqpTests = []string{ |
| "dEQP-GLES2.info.vendor", |
| "dEQP-GLES2.info.renderer", |
| "dEQP-GLES2.info.version", |
| "dEQP-GLES2.info.shading_language_version", |
| "dEQP-GLES2.info.extensions", |
| "dEQP-GLES2.info.render_target", |
| "dEQP-GLES2.functional.prerequisite.state_reset", |
| "dEQP-GLES2.functional.prerequisite.clear_color", |
| "dEQP-GLES2.functional.prerequisite.read_pixels", |
| "dEQP-GLES3.info.vendor", |
| "dEQP-GLES3.info.renderer", |
| "dEQP-GLES3.info.version", |
| "dEQP-GLES3.info.shading_language_version", |
| "dEQP-GLES3.info.extensions", |
| "dEQP-GLES3.info.render_target", |
| "dEQP-GLES3.functional.prerequisite.state_reset", |
| "dEQP-GLES3.functional.prerequisite.clear_color", |
| "dEQP-GLES3.functional.prerequisite.read_pixels", |
| "dEQP-GLES31.info.vendor", |
| "dEQP-GLES31.info.renderer", |
| "dEQP-GLES31.info.version", |
| "dEQP-GLES31.info.shading_language_version", |
| "dEQP-GLES31.info.extensions", |
| "dEQP-GLES31.info.render_target", |
| "dEQP-VK.info.build", |
| "dEQP-VK.info.device", |
| "dEQP-VK.info.platform", |
| "dEQP-VK.info.memory_limits", |
| "dEQP-VK.api.smoke.create_sampler", |
| "dEQP-VK.api.smoke.create_shader", |
| // TODO(ihf): reenable when smoke.triangle is not flaky anymore on coral/boxton b/149329522 |
| //"dEQP-VK.api.smoke.triangle", |
| //"dEQP-VK.api.smoke.triangle_ext_structs", |
| //"dEQP-VK.api.smoke.asm_triangle", |
| //"dEQP-VK.api.smoke.asm_triangle_no_opname", |
| "dEQP-VK.api.smoke.unused_resolve_attachment", |
| } |
| |
| // testNameToAPI extracts the graphics API that should be used based on a DEQP |
| // test name. An error is returned if the name is invalid. This is a port of the |
| // _translate_name_to_api() method in |
| // autotest/files/client/site_tests/graphics_dEQP/graphics_dEQP.py. |
| // TODO(andrescj): consider moving this to utils.go and adding tests for it. |
| func testNameToAPI(test string) (graphics.APIType, error) { |
| deqpPrefixToAPIMap := map[string]graphics.APIType{ |
| "dEQP-EGL": graphics.EGL, |
| "dEQP-GLES2": graphics.GLES2, |
| "dEQP-GLES3": graphics.GLES3, |
| "dEQP-GLES31": graphics.GLES31, |
| "dEQP-VK": graphics.VK, |
| } |
| if api, ok := deqpPrefixToAPIMap[strings.Split(test, ".")[0]]; ok { |
| return api, nil |
| } |
| return graphics.UnknownAPI, errors.Errorf("%q is not a valid test name", test) |
| } |
| |
| // canRunTest returns true iff the DEQP test named test can be run according to |
| // the supported graphics APIs (passed as apis). If the graphics API cannot be |
| // inferred from the test name, an error is returned. This function is based on |
| // the _can_run() method of graphics_dEQP in |
| // autotest/files/client/site_tests/graphics_dEQP/graphics_dEQP.py. |
| // TODO(andrescj): consider moving this to utils.go and adding tests for it. |
| func canRunTest(test string, apis []graphics.APIType) (bool, error) { |
| testAPI, err := testNameToAPI(test) |
| if err != nil { |
| return false, err |
| } |
| for _, a := range apis { |
| if testAPI == a { |
| return true, nil |
| } |
| } |
| return false, nil |
| } |
| |
| // runSingleTest runs a single DEQP test named test, e.g., |
| // "dEQP-GLES2.info.vendor" in a child process (which means, e.g., a new |
| // graphics context for the test). env lists the environment variables to set |
| // when running the test, e.g., "SHELL=/bin/bash". The test's log is written to |
| // a file named <test>.log within logDir. |
| // |
| // This function returns the outcome of the test determined by the result of |
| // parsing the test's log file. If an unrecoverable parsing error occurs, |
| // "parsefailed" is returned. |
| // |
| // This function is based on multiple places: |
| // |
| // - Initialization of graphics_dEQP in |
| // autotest/files/client/site_tests/graphics_dEQP/graphics_dEQP.py. |
| // |
| // - The _run_tests_individually() method of graphics_dEQP in |
| // autotest/files/client/site_tests/graphics_dEQP/graphics_dEQP.py. |
| // |
| // - The _get_executable() method of graphics_dEQP in |
| // autotest/files/client/site_tests/graphics_dEQP/graphics_dEQP.py. |
| func runSingleTest(ctx context.Context, s *testing.State, test string, env []string, logDir string) string { |
| // Get the path to the DEQP binary to run for the test. |
| api, err := testNameToAPI(test) |
| if err != nil { |
| s.Fatalf("Could not infer the API for %q: %v", test, err) |
| } |
| p, err := graphics.DEQPExecutable(api) |
| if err != nil { |
| s.Fatalf("Could not get the executable for %q: %v", api, err) |
| } |
| |
| // Prepare the command. Note that --deqp-surface-type is either "fbo" or |
| // "pbuffer". The latter avoids DEQP assumptions. The --deqp-surface-width |
| // and --deqp-surface-height should be the smallest for which all tests |
| // run/pass. |
| logFile := filepath.Join(logDir, test+".log") |
| cmd := testexec.CommandContext(ctx, p, |
| "--deqp-case="+test, |
| "--deqp-surface-type=pbuffer", |
| "--deqp-gl-config-name=rgba8888d24s8ms0", |
| "--deqp-log-images=disable", |
| "--deqp-watchdog=enable", |
| "--deqp-surface-width=256", |
| "--deqp-surface-height=256", |
| "--deqp-log-filename="+logFile) |
| // We should be in the executable's directory when running it so that it can |
| // find its test data files. |
| cmd.Dir = filepath.Dir(p) |
| cmd.Env = env |
| s.Log("Command: ", cmd.Args) |
| |
| // Run the test. Note that we don't care about the exit status code of the |
| // command. For example, even if the DEQP test fails, the command can return |
| // 0. We base our determination of the outcome entirely on the parsing of |
| // the detailed log file. |
| // TODO(andrescj): since cmd.Run() returns an error, maybe we should look |
| // into it and fatally fail for unexpected errors not related to dEQP. |
| cmd.Run() |
| if ctx.Err() == context.DeadlineExceeded { |
| // In the original code, we impose a timeout on the command (on top of |
| // the DEQP watchdog). Here, the deadline is imposed by the overall Tast |
| // test timeout. That means that if enough DEQP tests time out, we will |
| // just stop running the Tast test. |
| cmd.DumpLog(ctx) |
| s.Fatal("Absolute timeout! Things seem too broken") |
| } |
| |
| stats, nonFailed, err := graphics.ParseDEQPOutput(logFile) |
| if err != nil { |
| // An unrecoverable error occurred during parsing. An unrecoverable |
| // error implies that the log file wasn't processed fully. Since we ran |
| // a single test, we can attribute the error to that test. Hence, we'll |
| // count this error under the "parsefailed" outcome. |
| s.Logf("Unrecoverable parsing error for %v: %v", logFile, err) |
| return "parsefailed" |
| } |
| |
| // Do some validity checks on the parsing results. |
| if len(stats) != 1 { |
| s.Fatalf("Unexpected parsing result for %v: got %v stats; want 1", logFile, len(stats)) |
| } |
| if len(nonFailed) > 1 { |
| s.Fatalf("Unexpected parsing result for %v: got %v non-failed tests; want at most 1", logFile, len(nonFailed)) |
| } |
| var outcome string |
| for r, c := range stats { |
| if c != 1 { |
| s.Fatalf("Unexpected parsing result for %v: got %v tests for outcome %v; want 1", logFile, c, r) |
| } |
| outcome = r |
| } |
| return outcome |
| } |
| |
| func DEQP(ctx context.Context, s *testing.State) { |
| // Start of setup code - this is a port from multiple places: |
| // |
| // - Initialization of GraphicsApiHelper in |
| // autotest/files/client/cros/graphics/graphics_utils.py. |
| // |
| // - Initialization of graphics_dEQP in |
| // autotest/files/client/site_tests/graphics_dEQP/graphics_dEQP.py. |
| // |
| // - The run_once() method of graphics_dEQP in |
| // autotest/files/client/site_tests/graphics_dEQP/graphics_dEQP.py. |
| |
| // TODO(andrescj): port GraphicsTest initialization and clean up from |
| // autotest/files/client/cros/graphics/graphics_utils.py to prepare for the |
| // test and clean up at the end. |
| |
| // Step 1: query the supported graphics APIs. |
| glMajor, glMinor, err := graphics.GLESVersion(ctx) |
| if err != nil { |
| s.Fatal("Could not obtain the OpenGL version: ", err) |
| } |
| s.Logf("Found gles%d.%d", glMajor, glMinor) |
| |
| hasVulkan, err := graphics.SupportsVulkanForDEQP(ctx) |
| if err != nil { |
| s.Fatal("Could not check for Vulkan support: ", err) |
| } |
| s.Log("Vulkan support: ", hasVulkan) |
| |
| apis := graphics.SupportedAPIs(glMajor, glMinor, hasVulkan) |
| s.Log("Supported APIs: ", apis) |
| |
| // TODO(andrescj): also extract/log the following in the configuration per |
| // graphics_dEQP initialization: board, CPU type, and GPU type. Right now, |
| // the board and CPU type seem to be used only for logging. The GPU type is |
| // used to deduce test expectations (tests that we expect to pass/fail |
| // depending on the GPU). |
| |
| // Step 2: get the environment for the DEQP binaries. |
| env := graphics.DEQPEnvironment(os.Environ()) |
| s.Logf("Using environment: %q", env) |
| |
| // Step 3: create a location for storing detailed logs. |
| logDir := filepath.Join(s.OutDir(), "dEQP-results") |
| if err := os.Mkdir(logDir, 0700); err != nil { |
| s.Fatalf("Could not create %v: %v", logDir, err) |
| } |
| |
| // TODO(andrescj): stop services per graphics_dEQP initialization - ui and |
| // powerd. Restore after tests are done. |
| |
| // End of setup code |
| |
| // Step 4: get the list of tests to execute and run them. This is based on |
| // the _run_once() and _run_tests_individually() methods of graphics_dEQP in |
| // autotest/files/client/site_tests/graphics_dEQP/graphics_dEQP.py. |
| for i, t := range deqpTests { |
| s.Logf("[%d/%d] Test: %v", i+1, len(deqpTests), t) |
| |
| // Check if the test is supported in the DUT. |
| if canRun, err := canRunTest(t, apis); err != nil { |
| s.Fatal("Could not check if test is supported: ", err) |
| } else if !canRun { |
| // TODO(andrescj): add the GPU type to this log message. |
| s.Logf("Skipping %q due to unsupported API", t) |
| continue |
| } |
| |
| // Actually run the test. |
| if o := runSingleTest(ctx, s, t, env, logDir); graphics.DEQPOutcomeIsFailure(o) { |
| s.Errorf("Result for %q: %v", t, strings.ToUpper(o)) |
| } else { |
| s.Logf("Result for %q: %v", t, strings.ToUpper(o)) |
| } |
| } |
| |
| // TODO(andrescj): maybe output some counts, like # passes, # failures, |
| // #skipped per the run_once() method of graphics_dEQP in |
| // autotest/files/client/site_tests/graphics_dEQP/graphics_dEQP.py. |
| } |
