recipe_engine/test_result.proto

// Copyright 2017 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

syntax = "proto3";

package recipe_engine;

// Result of running recipe tests.
message TestResult {
  // Version of the format. Currently equal to 1.
  // May change if a backwards-incompatible change must be done for the proto.
  int32 version = 1;

  // Whether the test results can be considered valid and complete
  // (e.g. lack of internal failures).
  bool valid = 2;

  // Failure caused by incomplete code coverage.
  message CoverageFailure {
    // List of lines that lack code coverage.
    repeated int64 uncovered_lines = 1;
  }
  // Map absolute native paths to detailed failure info.
  map<string, CoverageFailure> coverage_failures = 3;

  // Indicates recorded steps differ from results of actual recipe simulation.
  // Contents of the diff are deliberately ommitted to keep output reasonably
  // small.
  message DiffFailure {
  }

  // Indicates one of the post-process checks has failed.
  message CheckFailure {
    // Name of the check.
    string name = 1;

    // Name of the function performing the check.
    string func = 2;

    // Positional arguments to the function.
    repeated string args = 3;

    // Keyword arguments to the function.
    map<string, string> kwargs = 4;

    // Filename where the check was added.
    string filename = 5;

    // Line number where the check was added.
    int64 lineno = 6;
  }

  // Indicates recipe execution has crashed (in the user code).
  message CrashFailure {
    string error = 1;
  }

  // Indicates that the test was badly formed in some way.
  message BadTestFailure {
    string error = 1;
  }

  // An internal failure (either ctrl-c interruption, or an exception
  // within test runner while running the test).
  message InternalFailure {
  }

  message TestFailure {
    oneof failures {
      DiffFailure diff_failure = 1;

      CheckFailure check_failure = 2;

      InternalFailure internal_failure = 3;

      CrashFailure crash_failure = 4;

      BadTestFailure bad_test_failure = 5;
    }
  }

  message TestFailures {
    repeated TestFailure failures = 1;
  }
  // Maps test names to failure details.
  // TODO(iannucci): this is crazy; it should be:
  //
  //   message TestFailures {
  //     DiffFailure diff = 1;
  //     InternalFailure internal = 2;
  //     CrashFailure crash = 3;
  //     BadTestFailure bad_test = 3;
  //     repeated CheckFailure check = 4;
  //   }
  //   map<string, TestFailures> test_failures = 4;
  //
  // The current proto generates really obtuse JSONPB.
  map<string, TestFailures> test_failures = 4;

  // Module names completely lacking test coverage.
  repeated string uncovered_modules = 5;

  // Absolute native paths of unused recipe expectation files.
  repeated string unused_expectations = 6;
}