| // Copyright 2020 The Chromium OS Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| // Package video provides common code to run ARC binary tests for video encoding. |
| package video |
| |
| import ( |
| "context" |
| "fmt" |
| "io/ioutil" |
| "os" |
| "path/filepath" |
| "strings" |
| "time" |
| |
| "chromiumos/tast/common/perf" |
| "chromiumos/tast/common/testexec" |
| "chromiumos/tast/ctxutil" |
| "chromiumos/tast/errors" |
| "chromiumos/tast/local/arc" |
| "chromiumos/tast/local/bundles/cros/arc/c2e2etest" |
| "chromiumos/tast/local/cpu" |
| mediacpu "chromiumos/tast/local/media/cpu" |
| "chromiumos/tast/local/media/encoding" |
| "chromiumos/tast/local/media/logging" |
| "chromiumos/tast/local/media/videotype" |
| "chromiumos/tast/local/power/setup" |
| "chromiumos/tast/testing" |
| ) |
| |
| // EncoderBlocklistVM is the list of devices on which the ARCVM HW encoder is not enabled. |
| // TODO(b/155138175): Remove devices from this list once the ARCVM HW encoder is enabled. |
| var EncoderBlocklistVM = []string{ |
| "grunt-arc-r", |
| } |
| |
| // EncoderAllowlistVPxVM is the list of devices on which the ARCVM VP8/9 HW encoder is enabled. |
| // TODO(b/155138243): Remove allowlist once VP8/9 HW encoding is enabled on all devices. |
| // Note: Combining this allowlist with the above blocklist works fine, entries in this list won't |
| // override entries in the blocklist. |
| var EncoderAllowlistVPxVM = []string{ |
| "hatch", |
| } |
| |
| // cpuLog is the name of log file recording CPU usage. |
| const cpuLog = "cpu.log" |
| |
| // powerLog is the name of lof file recording power consumption. |
| const powerLog = "power.log" |
| |
| // binArgs is the arguments and the modes for executing video_encode_accelerator_unittest binary. |
| type binArgs struct { |
| // testFilter specifies test pattern in googletest style for the unittest to run and will be passed with "--gtest_filter" (see go/gtest-running-subset). |
| // If unspecified, the unittest runs all tests. |
| testFilter string |
| // extraArgs is the additional arguments to pass video_encode_accelerator_unittest, for example, "--run_at_fps" |
| extraArgs []string |
| // measureUsage indicates whether to measure CPU usage and power consumption while running binary and save as perf metrics. |
| measureUsage bool |
| // measureDuration specifies how long to measure CPU usage and power consumption when measureUsage is set. |
| measureDuration time.Duration |
| } |
| |
| // EncoderType represents the type of video encoder that can be used. |
| type EncoderType int |
| |
| const ( |
| // HardwareEncoder is the encoder type that uses hardware encoding. |
| HardwareEncoder EncoderType = iota |
| // SoftwareEncoder is the encoder type that uses software encoding. |
| SoftwareEncoder |
| ) |
| |
| // EncodeTestOptions contains all options for the video encoder test. |
| type EncodeTestOptions struct { |
| // Profile specifies the codec profile to use when encoding. |
| Profile videotype.CodecProfile |
| // Params contains the test parameters for the e2e video encode test. |
| Params encoding.StreamParams |
| // PixelFormat is the format of the raw input video data. |
| PixelFormat videotype.PixelFormat |
| // EncoderType indicates whether a HW or SW encoder will be used. |
| EncoderType EncoderType |
| // BatteryDischargeMode specifies battery usage during a test. |
| BatteryDischargeMode setup.BatteryDischargeMode |
| } |
| |
| // testMode represents the test's running mode. |
| type testMode int |
| |
| const ( |
| // functionalTest indicates a functional test. |
| functionalTest testMode = iota |
| // performanceTest indicates a performance test. CPU scaling should be adujst to performance. |
| performanceTest |
| ) |
| |
| // runARCVideoEncoderTest runs arcvideoencoder_test in ARC. |
| // It pushes the binary files with different ABI and testing video data into ARC, and runs each binary for each binArgs. |
| // pv is optional value, passed when we run performance test and record measurement value. |
| // Note: pv must be provided when measureUsage is set at binArgs. |
| func runARCVideoEncoderTest(ctx context.Context, s *testing.State, a *arc.ARC, |
| opts EncodeTestOptions, pullEncodedVideo, cacheExtractedVideo bool, pv *perf.Values, bas ...binArgs) { |
| // Install the test APK and grant permissions |
| apkName, err := c2e2etest.ApkNameForArch(ctx, a) |
| if err != nil { |
| s.Fatal("Failed to get apk: ", err) |
| } |
| |
| s.Log("Installing APK ", apkName) |
| if err := a.Install(ctx, s.DataPath(apkName)); err != nil { |
| s.Fatal("Failed installing app: ", err) |
| } |
| |
| s.Log("Granting storage permissions") |
| if err := c2e2etest.GrantApkPermissions(ctx, a); err != nil { |
| s.Fatal("Failed granting storage permission: ", err) |
| } |
| |
| // Prepare video stream. |
| params := opts.Params |
| streamPath, err := encoding.PrepareYUV(ctx, s.DataPath(params.Name), opts.PixelFormat, params.Size) |
| if err != nil { |
| s.Fatal("Failed to prepare YUV file: ", err) |
| } |
| if !cacheExtractedVideo { |
| defer os.Remove(streamPath) |
| } |
| |
| // Push video stream file to ARC container. |
| if err := a.Command(ctx, "mkdir", "-p", arcFilePath).Run(testexec.DumpLogOnError); err != nil { |
| s.Fatal("Failed creating test data dir: ", err) |
| } |
| defer a.Command(ctx, "rm", "-rf", arcFilePath).Run() |
| |
| if err := a.PushFile(ctx, streamPath, arcFilePath); err != nil { |
| s.Fatal("Failed to push video stream to ARC: ", err) |
| } |
| |
| ctx, cancel := ctxutil.Shorten(ctx, 10*time.Second) |
| defer cancel() |
| |
| encodeOutFile := strings.TrimSuffix(params.Name, ".vp9.webm") |
| if opts.Profile == videotype.H264MainProf { |
| encodeOutFile = encodeOutFile + ".h264" |
| } else if opts.Profile == videotype.VP8Prof { |
| encodeOutFile = encodeOutFile + ".vp8.ivf" |
| } else if opts.Profile == videotype.VP9Prof { |
| encodeOutFile = encodeOutFile + ".vp9.ivf" |
| } else { |
| s.Fatalf("Profile (%d) is not supported", opts.Profile) |
| } |
| outPath := filepath.Join(arcFilePath, encodeOutFile) |
| |
| commonArgs := []string{ |
| encoding.CreateStreamDataArg(params, opts.Profile, opts.PixelFormat, arcFilePath+"/"+filepath.Base(streamPath), outPath), |
| } |
| for _, ba := range bas { |
| if opts.EncoderType == SoftwareEncoder { |
| ba.extraArgs = append(ba.extraArgs, "--use_sw_encoder") |
| } |
| if err := runARCBinaryWithArgs(ctx, s, a, commonArgs, ba, pv); err != nil { |
| s.Errorf("Failed to run test with %v: %v", ba, err) |
| } |
| } |
| |
| if pullEncodedVideo { |
| err := pullVideo(ctx, a, s.OutDir(), outPath) |
| if err != nil { |
| s.Fatal("Failed to pull encoded video: ", err) |
| } |
| } |
| } |
| |
| // runARCBinaryWithArgs runs arcvideoencoder_test binary with one binary argument. |
| // pv is optional value, passed when we run performance test and record measurement value. |
| // Note: pv must be provided when measureUsage is set at binArgs. |
| func runARCBinaryWithArgs(ctx context.Context, s *testing.State, a *arc.ARC, commonArgs []string, ba binArgs, pv *perf.Values) error { |
| cr := s.FixtValue().(*arc.PreData).Chrome |
| |
| tconn, err := cr.TestAPIConn(ctx) |
| if err != nil { |
| s.Fatal("Failed to create Test API connection: ", err) |
| } |
| |
| nowStr := time.Now().Format("20060102-150405") |
| outputLogFileName := fmt.Sprintf("output_%s.log", nowStr) |
| outputXMLFileName := fmt.Sprintf("output_%s.xml", nowStr) |
| |
| s.Log("Starting APK main activity") |
| act, err := arc.NewActivity(a, c2e2etest.Pkg, c2e2etest.ActivityName) |
| if err != nil { |
| s.Fatal("Failed to create new activity: ", err) |
| } |
| defer act.Close() |
| args := append([]string(nil), commonArgs...) |
| args = append(args, ba.extraArgs...) |
| args = append(args, "--gtest_filter="+ba.testFilter) |
| args = append(args, "--gtest_output=xml:"+arcFilePath+outputXMLFileName) |
| |
| if err := act.Start(ctx, tconn, |
| arc.WithWaitForLaunch(), |
| arc.WithExtraBool("do-encode", true), |
| arc.WithExtraStringArray("test-args", args), |
| arc.WithExtraString("log-file", arcFilePath+outputLogFileName), |
| ); err != nil { |
| s.Fatal("Failed starting APK main activity: ", err) |
| } |
| |
| const schemaName = "c2e2etest" |
| if ba.measureUsage { |
| // We need to manually stop the activity after measuring CPU usage. |
| defer func() { |
| s.Log("Stopping activity") |
| if err := act.Stop(ctx, tconn); err != nil { |
| s.Log("Warning: Failed to stop activity: ", err) |
| } |
| }() |
| |
| if pv == nil { |
| return errors.New("pv should not be nil when measuring CPU usage and power consumption") |
| } |
| |
| s.Log("Initializing test and loading test data") |
| if err := c2e2etest.WaitForCodecReady(ctx, a); err != nil { |
| return errors.Wrap(err, "failed to initialize test") |
| } |
| |
| s.Log("Starting CPU measurements") |
| measurements, err := mediacpu.MeasureUsage(ctx, ba.measureDuration) |
| if err != nil { |
| return errors.Wrapf(err, "failed to run (measure CPU and power consumption): %v", err) |
| } |
| cpuUsage := measurements["cpu"] |
| // TODO(b/143190876): Don't write value to disk, as this can increase test flakiness. |
| cpuLogPath := filepath.Join(s.OutDir(), cpuLog) |
| if err := ioutil.WriteFile(cpuLogPath, []byte(fmt.Sprintf("%f", cpuUsage)), 0644); err != nil { |
| return errors.Wrap(err, "failed to write CPU usage to file") |
| } |
| |
| if err := reportCPUUsage(ctx, pv, schemaName, cpuLogPath); err != nil { |
| return errors.Wrap(err, "failed to report CPU usage") |
| } |
| |
| powerConsumption, ok := measurements["power"] |
| if ok { |
| // TODO(b/143190876): Don't write value to disk, as this can increase test flakiness. |
| powerLogPath := filepath.Join(s.OutDir(), powerLog) |
| if err := ioutil.WriteFile(powerLogPath, []byte(fmt.Sprintf("%f", powerConsumption)), 0644); err != nil { |
| return errors.Wrap(err, "failed to write power consumption to file") |
| } |
| |
| if err := reportPowerConsumption(ctx, pv, schemaName, powerLogPath); err != nil { |
| return errors.Wrap(err, "failed to report power consumption") |
| } |
| } |
| } else { |
| s.Log("Initializing test and loading test data") |
| if err := c2e2etest.WaitForCodecReady(ctx, a); err != nil { |
| return errors.Wrap(err, "failed to initialize test") |
| } |
| |
| s.Log("Waiting for activity to finish") |
| if err := act.WaitForFinished(ctx, 0*time.Second); err != nil { |
| s.Fatal("Failed to wait for activity: ", err) |
| } |
| |
| localOutputLogFile, localXMLLogFile, err := c2e2etest.PullLogs(ctx, a, arcFilePath, s.OutDir(), "", outputLogFileName, outputXMLFileName) |
| |
| if err != nil { |
| s.Fatal("Failed to pull logs: ", err) |
| } |
| |
| if err := c2e2etest.ValidateXMLLogs(localXMLLogFile); err != nil { |
| s.Fatal("Failed to validate logs: ", err) |
| } |
| |
| // Parse the performance result. |
| if pv != nil { |
| if err := reportFPS(ctx, pv, schemaName, localOutputLogFile); err != nil { |
| return errors.Wrap(err, "failed to report FPS value") |
| } |
| |
| if err := reportEncodeLatency(ctx, pv, schemaName, localOutputLogFile); err != nil { |
| return errors.Wrap(err, "failed to report encode latency") |
| } |
| } |
| } |
| return nil |
| } |
| |
| // RunARCVideoTest runs all non-perf tests of arcvideoencoder_test in ARC. |
| func RunARCVideoTest(ctx context.Context, s *testing.State, a *arc.ARC, |
| opts EncodeTestOptions, pullEncodedVideo, cacheExtractedVideo bool) { |
| vl, err := logging.NewVideoLogger() |
| if err != nil { |
| s.Fatal("Failed to set values for verbose logging: ", err) |
| } |
| defer vl.Close() |
| |
| runARCVideoEncoderTest(ctx, s, a, opts, pullEncodedVideo, cacheExtractedVideo, |
| nil, binArgs{testFilter: "C2VideoEncoderE2ETest.Test*"}) |
| } |
| |
| // RunARCPerfVideoTest runs all perf tests of arcvideoencoder_test in ARC. |
| func RunARCPerfVideoTest(ctx context.Context, s *testing.State, a *arc.ARC, |
| opts EncodeTestOptions, cacheExtractedVideo bool) { |
| const ( |
| // duration of the interval during which CPU usage will be measured. |
| measureDuration = 10 * time.Second |
| // time reserved for cleanup. |
| cleanupTime = 5 * time.Second |
| ) |
| |
| cleanUpBenchmark, err := mediacpu.SetUpBenchmark(ctx) |
| if err != nil { |
| s.Fatal("Failed to set up benchmark mode: ", err) |
| } |
| defer cleanUpBenchmark(ctx) |
| |
| // Leave a bit of time to clean up benchmark mode. |
| ctx, cancel := ctxutil.Shorten(ctx, cleanupTime) |
| defer cancel() |
| |
| if err := cpu.WaitUntilIdle(ctx); err != nil { |
| s.Fatal("Failed waiting for CPU to become idle: ", err) |
| } |
| |
| pv := perf.NewValues() |
| runARCVideoEncoderTest(ctx, s, a, opts, false, cacheExtractedVideo, pv, |
| // Measure FPS and latency. |
| binArgs{ |
| testFilter: "C2VideoEncoderE2ETest.Perf*", |
| }, |
| // Measure CPU usage. |
| binArgs{ |
| testFilter: "C2VideoEncoderE2ETest.TestSimpleEncode", |
| extraArgs: []string{"--run_at_fps", "--num_encoded_frames=10000"}, |
| measureUsage: true, |
| measureDuration: measureDuration, |
| }) |
| pv.Save(s.OutDir()) |
| } |
| |
| // pullVideo downloads the video encoded by the e2e video encode test. |
| func pullVideo(ctx context.Context, a *arc.ARC, localFilePath, videoPath string) error { |
| testing.ContextLogf(ctx, "Pulling encoded video file %s", videoPath) |
| localVideoPath := filepath.Join(localFilePath, filepath.Base(videoPath)) |
| if err := a.PullFile(ctx, videoPath, localVideoPath); err != nil { |
| return errors.Wrapf(err, "failed fo pull %s", localVideoPath) |
| } |
| return nil |
| } |