xref: /aosp_15_r20/tools/tradefederation/core/proto/resultdb/test_result.proto

/*
 * Copyright (C) 2024 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

syntax = "proto3";

package luci.resultdb.v1;

import "google/api/field_behavior.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/struct.proto";
import "google/protobuf/timestamp.proto";
import public "tools/tradefederation/core/proto/resultdb/common.proto";
import public "tools/tradefederation/core/proto/resultdb/test_metadata.proto";
import public "tools/tradefederation/core/proto/resultdb/failure_reason.proto";

option go_package = "go.chromium.org/luci/resultdb/proto/v1;resultpb";
option java_package = "com.android.resultdb.proto";
option java_multiple_files = true;

// A result of a functional test case.
// Often a single test case is executed multiple times and has multiple results,
// a single test suite has multiple test cases,
// and the same test suite can be executed in different variants
// (OS, GPU, compile flags, etc).
//
// This message does not specify the test id.
// It should be available in the message that embeds this message.
//
// Next id: 17.
message TestResult {
  reserved 11;  // test_location

  // Can be used to refer to this test result, e.g. in ResultDB.GetTestResult
  // RPC.
  // Format:
  // "invocations/{INVOCATION_ID}/tests/{URL_ESCAPED_TEST_ID}/results/{RESULT_ID}".
  // where URL_ESCAPED_TEST_ID is test_id escaped with
  // https://golang.org/pkg/net/url/#PathEscape See also https://aip.dev/122.
  //
  // Output only.
  string name = 1 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = IMMUTABLE
  ];

  // Test id, a unique identifier of the test in a LUCI project.
  // Regex: ^[[::print::]]{1,512}$
  //
  // If two tests have a common test id prefix that ends with a
  // non-alphanumeric character, they considered a part of a group. Examples:
  // - "a/b/c"
  // - "a/b/d"
  // - "a/b/e:x"
  // - "a/b/e:y"
  // - "a/f"
  // This defines the following groups:
  // - All items belong to one group because of the common prefix "a/"
  // - Within that group, the first 4 form a sub-group because of the common
  //   prefix "a/b/"
  // - Within that group, "a/b/e:x" and "a/b/e:y" form a sub-group because of
  //   the common prefix "a/b/e:".
  // This can be used in UI.
  // LUCI does not interpret test ids in any other way.
  string test_id = 2 [(google.api.field_behavior) = IMMUTABLE];

  // Identifies a test result in a given invocation and test id.
  // Regex: ^[a-z0-9\-_.]{1,32}$
  string result_id = 3 [
    (google.api.field_behavior) = IMMUTABLE,
    (google.api.field_behavior) = REQUIRED
  ];

  // Description of one specific way of running the test,
  // e.g. a specific bucket, builder and a test suite.
  Variant variant = 4 [(google.api.field_behavior) = IMMUTABLE];

  // Whether the result of test case execution is expected.
  // In a typical Chromium CL, 99%+ of test results are expected.
  // Users are typically interested only in the unexpected results.
  //
  // An unexpected result != test case failure. There are test cases that are
  // expected to fail/skip/crash. The test harness compares the actual status
  // with the expected one(s) and this field is the result of the comparison.
  bool expected = 5 [(google.api.field_behavior) = IMMUTABLE];

  // Machine-readable status of the test case.
  // MUST NOT be STATUS_UNSPECIFIED.
  TestStatus status = 6 [(google.api.field_behavior) = IMMUTABLE];

  // Human-readable explanation of the result, in HTML.
  // MUST be sanitized before rendering in the browser.
  //
  // The size of the summary must be equal to or smaller than 4096 bytes in
  // UTF-8.
  //
  // Supports artifact embedding using custom tags:
  // * <text-artifact> renders contents of an artifact as text.
  //   Usage:
  //   * To embed result level artifact: <text-artifact
  //   artifact-id="<artifact_id>">
  //   * To embed invocation level artifact: <text-artifact
  //   artifact-id="<artifact_id>" inv-level>
  string summary_html = 7 [(google.api.field_behavior) = IMMUTABLE];

  // The point in time when the test case started to execute.
  google.protobuf.Timestamp start_time = 8
      [(google.api.field_behavior) = IMMUTABLE];

  // Duration of the test case execution.
  // MUST be equal to or greater than 0.
  google.protobuf.Duration duration = 9
      [(google.api.field_behavior) = IMMUTABLE];

  // Metadata for this test result.
  // It might describe this particular execution or the test case.
  // A key can be repeated.
  repeated StringPair tags = 10 [(google.api.field_behavior) = IMMUTABLE];

  // Hash of the variant.
  // hex(sha256(sorted(''.join('%s:%s\n' for k, v in variant.items())))).
  //
  // Output only.
  string variant_hash = 12 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = IMMUTABLE
  ];

  // Information about the test at the time of its execution.
  TestMetadata test_metadata = 13;

  // Information about the test failure. Only present if the test failed.
  FailureReason failure_reason = 14;

  // Arbitrary JSON object that contains structured, domain-specific properties
  // of the test result.
  //
  // The serialized size must be <= 8 KB.
  google.protobuf.Struct properties = 15;

  // Whether the test result has been masked so that it includes only metadata.
  // The metadata fields for a TestResult are:
  // * name
  // * test_id
  // * result_id
  // * expected
  // * status
  // * start_time
  // * duration
  // * variant_hash
  // * failure_reason.primary_error_message (truncated to 140 characters)
  // * skip_reason
  //
  // Output only.
  bool is_masked = 16 [(google.api.field_behavior) = OUTPUT_ONLY];

  // Reasoning behind a test skip, in machine-readable form.
  // Used to assist downstream analyses, such as automatic bug-filing.
  // MUST not be set unless status is SKIP.
  SkipReason skip_reason = 18;
}

// Machine-readable status of a test result.
enum TestStatus {
  // Status was not specified.
  // Not to be used in actual test results; serves as a default value for an
  // unset field.
  STATUS_UNSPECIFIED = 0;

  // The test case has passed.
  PASS = 1;

  // The test case has failed.
  // Suggests that the code under test is incorrect, but it is also possible
  // that the test is incorrect or it is a flake.
  FAIL = 2;

  // The test case has crashed during execution.
  // The outcome is inconclusive: the code under test might or might not be
  // correct, but the test+code is incorrect.
  CRASH = 3;

  // The test case has started, but was aborted before finishing.
  // A common reason: timeout.
  ABORT = 4;

  // The test case did not execute.
  // Examples:
  // - The execution of the collection of test cases, such as a test
  //   binary, was aborted prematurely and execution of some test cases was
  //   skipped.
  // - The test harness configuration specified that the test case MUST be
  //   skipped.
  SKIP = 5;
}

// Machine-readable reason that a test execution was skipped.
// Only reasons actually used are listed here, if you need a new reason
// please add it here and send a CL to the OWNERS.
enum SkipReason {
  // Skip reason was not specified.
  // This represents an unset field which should be used for non-skip test
  // result statuses.  It can also be used if none of the other statuses
  // apply.
  SKIP_REASON_UNSPECIFIED = 0;

  // Disabled automatically in response to a test skipping policy that skips
  // flaky tests.
  // Used for ChromeOS CQ test filtering.
  AUTOMATICALLY_DISABLED_FOR_FLAKINESS = 1;
}

// Indicates the test subject (e.g. a CL) is absolved from blame
// for an unexpected result of a test variant.
// For example, the test variant fails both with and without CL, so it is not
// CL's fault.
message TestExoneration {
  // Can be used to refer to this test exoneration, e.g. in
  // ResultDB.GetTestExoneration RPC.
  // Format:
  // invocations/{INVOCATION_ID}/tests/{URL_ESCAPED_TEST_ID}/exonerations/{EXONERATION_ID}.
  // URL_ESCAPED_TEST_ID is test_variant.test_id escaped with
  // https://golang.org/pkg/net/url/#PathEscape See also https://aip.dev/122.
  //
  // Output only.
  string name = 1 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = IMMUTABLE
  ];

  // Test identifier, see TestResult.test_id.
  string test_id = 2;

  // Description of the variant of the test, see Variant type.
  // Unlike TestResult.extra_variant_pairs, this one must be a full definition
  // of the variant, i.e. it is not combined with Invocation.base_test_variant.
  Variant variant = 3;

  // Identifies an exoneration in a given invocation and test id.
  // It is server-generated.
  string exoneration_id = 4 [
    (google.api.field_behavior) = OUTPUT_ONLY,
    (google.api.field_behavior) = IMMUTABLE
  ];

  // Reasoning behind the exoneration, in HTML.
  // MUST be sanitized before rendering in the browser.
  string explanation_html = 5 [(google.api.field_behavior) = IMMUTABLE];

  // Hash of the variant.
  // hex(sha256(sorted(''.join('%s:%s\n' for k, v in variant.items())))).
  string variant_hash = 6 [(google.api.field_behavior) = IMMUTABLE];

  // Reasoning behind the exoneration, in machine-readable form.
  // Used to assist downstream analyses, such as automatic bug-filing.
  // This allow detection of e.g. critical tests failing in presubmit,
  // even if they are being exonerated because they fail on other CLs.
  ExonerationReason reason = 7 [(google.api.field_behavior) = IMMUTABLE];

  // Whether the test exoneration has been masked so that it includes only
  // metadata. The metadata fields for a TestExoneration are:
  // * name
  // * test_id
  // * exoneration_id
  // * variant_hash
  // * explanation_html
  // * reason
  //
  // Output only.
  bool is_masked = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// Reason why a test variant was exonerated.
enum ExonerationReason {
  // Reason was not specified.
  // Not to be used in actual test exonerations; serves as a default value for
  // an unset field.
  EXONERATION_REASON_UNSPECIFIED = 0;

  // Similar unexpected results were observed on a mainline branch
  // (i.e. against a build without unsubmitted changes applied).
  // (For avoidance of doubt, this includes both flakily and
  // deterministically occurring unexpected results.)
  // Applies to unexpected results in presubmit/CQ runs only.
  OCCURS_ON_MAINLINE = 1;

  // Similar unexpected results were observed in presubmit run(s) for other,
  // unrelated CL(s). (This is suggestive of the issue being present
  // on mainline but is not confirmed as there are possible confounding
  // factors, like how tests are run on CLs vs how tests are run on
  // mainline branches.)
  // Applies to unexpected results in presubmit/CQ runs only.
  OCCURS_ON_OTHER_CLS = 2;

  // The tests are not critical to the test subject (e.g. CL) passing.
  // This could be because more data is being collected to determine if
  // the tests are stable enough to be made critical (as is often the
  // case for experimental test suites).
  // If information exists indicating the tests are producing unexpected
  // results, and the tests are not critical for that reason,
  // prefer more specific reasons OCCURS_ON_MAINLINE or OCCURS_ON_OTHER_CLS.
  NOT_CRITICAL = 3;

  // The test result was an unexpected pass. (Note that such an exoneration is
  // not automatically created for unexpected passes, unless the option is
  // specified to ResultSink or the project manually creates one).
  UNEXPECTED_PASS = 4;
}