infra/config/lib/try.star

# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Library for defining try builders.

The `try_builder` function defined in this module enables defining a builder and
the tryjob verifier for it in the same location. It can also be accessed through
`try_.builder`.

The `tryjob` function specifies the verifier details for a builder. It can also
be accessed through `try_.job`.

The `defaults` struct provides module-level defaults for the arguments to
`try_builder`. The parameters that support module-level defaults have a
corresponding attribute on `defaults` that is a `lucicfg.var` that can be used
to set the default value. Can also be accessed through `try_.defaults`.
"""

load("@stdlib//internal/graph.star", "graph")
load("@stdlib//internal/luci/common.star", "keys")
load("./args.star", "args")
load("./branches.star", "branches")
load("./builders.star", "builders")

DEFAULT_EXCLUDE_REGEXPS = [
    # Contains documentation that doesn't affect the outputs
    ".+/[+]/docs/.+",
    # Contains configuration files that aren't active until after committed
    ".+/[+]/infra/config/.+",
]

defaults = args.defaults(
    extends = builders.defaults,
    add_to_list_view = False,
    cq_group = None,
    list_view = args.COMPUTE,
    main_list_view = None,
)

def declare_bucket(milestone_vars, *, branch_selector = branches.MAIN_ONLY):
    if not branches.matches(branch_selector):
        return

    luci.bucket(
        name = milestone_vars.try_bucket,
        acls = [
            acl.entry(
                roles = acl.BUILDBUCKET_READER,
                groups = "all",
            ),
            acl.entry(
                roles = acl.BUILDBUCKET_TRIGGERER,
                users = [
                    "findit-for-me@appspot.gserviceaccount.com",
                    "tricium-prod@appspot.gserviceaccount.com",
                ],
                groups = [
                    "project-chromium-tryjob-access",
                    # Allow Pinpoint to trigger builds for bisection
                    "service-account-chromeperf",
                    "service-account-cq",
                ],
                projects = milestone_vars.try_triggering_projects,
            ),
            acl.entry(
                roles = acl.BUILDBUCKET_OWNER,
                groups = "service-account-chromium-tryserver",
            ),
        ],
    )

    luci.cq_group(
        name = milestone_vars.cq_group,
        retry_config = cq.RETRY_ALL_FAILURES,
        tree_status_host = milestone_vars.tree_status_host,
        watch = cq.refset(
            repo = "https://chromium.googlesource.com/chromium/src",
            refs = [milestone_vars.cq_ref_regexp],
        ),
        acls = [
            acl.entry(
                acl.CQ_COMMITTER,
                groups = "project-chromium-committers",
            ),
            acl.entry(
                acl.CQ_DRY_RUNNER,
                groups = "project-chromium-tryjob-access",
            ),
        ],
    )

    try_.list_view(
        name = milestone_vars.main_list_view_name,
        branch_selector = branch_selector,
        title = milestone_vars.main_list_view_title,
    )

def set_defaults(milestone_vars, **kwargs):
    default_values = dict(
        add_to_list_view = milestone_vars.is_master,
        bucket = milestone_vars.try_bucket,
        build_numbers = True,
        caches = [
            swarming.cache(
                name = "win_toolchain",
                path = "win_toolchain",
            ),
        ],
        configure_kitchen = True,
        cores = 8,
        cpu = builders.cpu.X86_64,
        cq_group = milestone_vars.cq_group,
        executable = "recipe:chromium_trybot",
        execution_timeout = 4 * time.hour,
        # Max. pending time for builds. CQ considers builds pending >2h as timed
        # out: http://shortn/_8PaHsdYmlq. Keep this in sync.
        expiration_timeout = 2 * time.hour,
        os = builders.os.LINUX_DEFAULT,
        pool = "luci.chromium.try",
        service_account = "chromium-try-builder@chops-service-accounts.iam.gserviceaccount.com",
        swarming_tags = ["vpython:native-python-wrapper"],
        task_template_canary_percentage = 5,
    )
    default_values.update(kwargs)
    for k, v in default_values.items():
        getattr(defaults, k).set(v)

def _sorted_list_view_graph_key(console_name):
    return graph.key("@chromium", "", "sorted_list_view", console_name)

def _sorted_list_view_impl(ctx, *, console_name):
    key = _sorted_list_view_graph_key(console_name)
    graph.add_node(key)
    graph.add_edge(keys.project(), key)
    return graph.keyset(key)

_sorted_list_view = lucicfg.rule(impl = _sorted_list_view_impl)

def _sort_console_entries(ctx):
    milo = ctx.output["luci-milo.cfg"]
    consoles = []
    for console in milo.consoles:
        if not console.builders:
            continue
        graph_key = _sorted_list_view_graph_key(console.id)
        node = graph.node(graph_key)
        if node:
            console.builders = sorted(console.builders, lambda b: b.name)
        consoles.append(console)

lucicfg.generator(_sort_console_entries)

def list_view(*, name, branch_selector = branches.MAIN_ONLY, **kwargs):
    if not branches.matches(branch_selector):
        return

    luci.list_view(
        name = name,
        **kwargs
    )

    _sorted_list_view(
        console_name = name,
    )

def tryjob(
        *,
        disable_reuse = None,
        experiment_percentage = None,
        location_regexp = None,
        location_regexp_exclude = None,
        cancel_stale = None,
        add_default_excludes = True):
    """Specifies the details of a tryjob verifier.

    See https://chromium.googlesource.com/infra/luci/luci-go/+/refs/heads/master/lucicfg/doc/README.md#luci.cq_tryjob_verifier
    for details on the most of the arguments.

    Arguments:
      add_default_excludes - A bool indicating whether to add exclude regexps
        for certain directories that would have no impact when building chromium
        with the patch applied (docs, config files that don't take effect until
        landing, etc., see DEFAULT_EXCLUDE_REGEXPS).

    Returns:
      A struct that can be passed to the `tryjob` argument of `try_.builder` to
      enable the builder for CQ.
    """
    if add_default_excludes:
        location_regexp_exclude = DEFAULT_EXCLUDE_REGEXPS + (location_regexp_exclude or [])
    return struct(
        disable_reuse = disable_reuse,
        experiment_percentage = experiment_percentage,
        location_regexp = location_regexp,
        location_regexp_exclude = location_regexp_exclude,
        cancel_stale = cancel_stale,
    )

def try_builder(
        *,
        name,
        branch_selector = branches.MAIN_ONLY,
        add_to_list_view = args.DEFAULT,
        cq_group = args.DEFAULT,
        list_view = args.DEFAULT,
        main_list_view = args.DEFAULT,
        tryjob = None,
        **kwargs):
    """Define a try builder.

    Arguments:
      name - name of the builder, will show up in UIs and logs. Required.
      branch_selector - A branch selector value controlling whether the
        builder definition is executed. See branches.star for more
        information.
      add_to_list_view - A bool indicating whether an entry should be
        created for the builder in the console identified by
        `list_view`. Supports a module-level default that defaults to
        False.
      cq_group - The CQ group to add the builder to. If tryjob is None, it will
        be added as includable_only.
      list_view - A string identifying the ID of the list view to
        add an entry to. Supports a module-level default that defaults to
        the group of the builder, if provided. An entry will be added
        only if `add_to_list_view` is True.
      main_console_view - A string identifying the ID of the main list
        view to add an entry to. Supports a module-level default that
        defaults to None. Note that `add_to_list_view` has no effect on
        creating an entry to the main list view.
      tryjob - A struct containing the details of the tryjob verifier for the
        builder, obtained by calling the `tryjob` function.
    """
    if not branches.matches(branch_selector):
        return

    # Define the builder first so that any validation of luci.builder arguments
    # (e.g. bucket) occurs before we try to use it
    builders.builder(
        name = name,
        branch_selector = branch_selector,
        resultdb_bigquery_exports = [resultdb.export_test_results(
            bq_table = "luci-resultdb.chromium.try_test_results",
        )],
        **kwargs
    )

    bucket = defaults.get_value_from_kwargs("bucket", kwargs)
    builder = "{}/{}".format(bucket, name)
    cq_group = defaults.get_value("cq_group", cq_group)
    if tryjob != None:
        luci.cq_tryjob_verifier(
            builder = builder,
            cq_group = cq_group,
            disable_reuse = tryjob.disable_reuse,
            experiment_percentage = tryjob.experiment_percentage,
            location_regexp = tryjob.location_regexp,
            location_regexp_exclude = tryjob.location_regexp_exclude,
            cancel_stale = tryjob.cancel_stale,
        )
    else:
        # Allow CQ to trigger this builder if user opts in via CQ-Include-Trybots.
        luci.cq_tryjob_verifier(
            builder = builder,
            cq_group = cq_group,
            includable_only = True,
        )

    add_to_list_view = defaults.get_value("add_to_list_view", add_to_list_view)
    if add_to_list_view:
        list_view = defaults.get_value("list_view", list_view)
        if list_view == args.COMPUTE:
            list_view = defaults.get_value_from_kwargs("builder_group", kwargs)

        if list_view:
            add_to_list_view = defaults.get_value(
                "add_to_list_view",
                add_to_list_view,
            )

            luci.list_view_entry(
                builder = builder,
                list_view = list_view,
            )

    main_list_view = defaults.get_value("main_list_view", main_list_view)
    if main_list_view:
        luci.list_view_entry(
            builder = builder,
            list_view = main_list_view,
        )

def blink_builder(*, name, goma_backend = None, **kwargs):
    return try_builder(
        name = name,
        builder_group = "tryserver.blink",
        goma_backend = goma_backend,
        **kwargs
    )

def blink_mac_builder(
        *,
        name,
        os = builders.os.MAC_ANY,
        builderless = True,
        **kwargs):
    return blink_builder(
        name = name,
        cores = None,
        goma_backend = builders.goma.backend.RBE_PROD,
        os = os,
        builderless = builderless,
        ssd = True,
        **kwargs
    )

def chromium_builder(*, name, **kwargs):
    return try_builder(
        name = name,
        builder_group = "tryserver.chromium",
        builderless = True,
        goma_backend = builders.goma.backend.RBE_PROD,
        **kwargs
    )

def chromium_android_builder(*, name, **kwargs):
    return try_builder(
        name = name,
        builder_group = "tryserver.chromium.android",
        goma_backend = builders.goma.backend.RBE_PROD,
        **kwargs
    )

def chromium_angle_builder(*, name, **kwargs):
    return try_builder(
        name = name,
        builder_group = "tryserver.chromium.angle",
        builderless = False,
        goma_backend = builders.goma.backend.RBE_PROD,
        goma_jobs = builders.goma.jobs.J150,
        service_account = "chromium-try-gpu-builder@chops-service-accounts.iam.gserviceaccount.com",
        **kwargs
    )

def chromium_chromiumos_builder(*, name, **kwargs):
    return try_builder(
        name = name,
        builder_group = "tryserver.chromium.chromiumos",
        goma_backend = builders.goma.backend.RBE_PROD,
        **kwargs
    )

def chromium_dawn_builder(*, name, **kwargs):
    return try_builder(
        name = name,
        builder_group = "tryserver.chromium.dawn",
        builderless = False,
        cores = None,
        goma_backend = builders.goma.backend.RBE_PROD,
        service_account = "chromium-try-gpu-builder@chops-service-accounts.iam.gserviceaccount.com",
        **kwargs
    )

def chromium_linux_builder(*, name, goma_backend = builders.goma.backend.RBE_PROD, **kwargs):
    return try_builder(
        name = name,
        builder_group = "tryserver.chromium.linux",
        goma_backend = goma_backend,
        **kwargs
    )

def chromium_mac_builder(
        *,
        name,
        builderless = True,
        cores = None,
        goma_backend = builders.goma.backend.RBE_PROD,
        os = builders.os.MAC_ANY,
        **kwargs):
    return try_builder(
        name = name,
        builder_group = "tryserver.chromium.mac",
        cores = cores,
        goma_backend = goma_backend,
        os = os,
        builderless = builderless,
        ssd = True,
        **kwargs
    )

def chromium_mac_ios_builder(
        *,
        name,
        caches = None,
        executable = "recipe:ios/try",
        goma_backend = builders.goma.backend.RBE_PROD,
        os = builders.os.MAC_10_15,
        properties = None,
        **kwargs):
    if not caches:
        caches = [builders.xcode_cache.x12a8189n]
    if not properties:
        properties = {
            "xcode_build_version": "12a8189n",
        }
    return try_builder(
        name = name,
        builder_group = "tryserver.chromium.mac",
        caches = caches,
        cores = None,
        executable = executable,
        goma_backend = goma_backend,
        os = os,
        properties = properties,
        **kwargs
    )

def chromium_swangle_builder(*, name, pinned = True, **kwargs):
    builder_args = dict(kwargs)
    builder_args.update(
        name = name,
        builder_group = "tryserver.chromium.swangle",
        builderless = True,
        service_account = "chromium-try-gpu-builder@chops-service-accounts.iam.gserviceaccount.com",
    )
    if pinned:
        builder_args.update(executable = "recipe:angle_chromium_trybot")
    return try_builder(**builder_args)

def chromium_swangle_linux_builder(*, name, **kwargs):
    return chromium_swangle_builder(
        name = name,
        goma_backend = builders.goma.backend.RBE_PROD,
        os = builders.os.LINUX_DEFAULT,
        **kwargs
    )

def chromium_swangle_mac_builder(*, name, **kwargs):
    return chromium_swangle_builder(
        name = name,
        cores = None,
        goma_backend = builders.goma.backend.RBE_PROD,
        os = builders.os.MAC_ANY,
        **kwargs
    )

def chromium_swangle_windows_builder(*, name, **kwargs):
    return chromium_swangle_builder(
        name = name,
        goma_backend = builders.goma.backend.RBE_PROD,
        os = builders.os.WINDOWS_DEFAULT,
        **kwargs
    )

def chromium_win_builder(
        *,
        name,
        builderless = True,
        goma_backend = builders.goma.backend.RBE_PROD,
        os = builders.os.WINDOWS_DEFAULT,
        **kwargs):
    return try_builder(
        name = name,
        builder_group = "tryserver.chromium.win",
        builderless = builderless,
        goma_backend = goma_backend,
        os = os,
        **kwargs
    )

def gpu_try_builder(*, name, builderless = False, execution_timeout = 6 * time.hour, **kwargs):
    return try_builder(
        name = name,
        builderless = builderless,
        execution_timeout = execution_timeout,
        service_account = "chromium-try-gpu-builder@chops-service-accounts.iam.gserviceaccount.com",
        **kwargs
    )

def gpu_chromium_android_builder(*, name, **kwargs):
    return gpu_try_builder(
        name = name,
        builder_group = "tryserver.chromium.android",
        goma_backend = builders.goma.backend.RBE_PROD,
        **kwargs
    )

def gpu_chromium_linux_builder(*, name, **kwargs):
    return gpu_try_builder(
        name = name,
        builder_group = "tryserver.chromium.linux",
        goma_backend = builders.goma.backend.RBE_PROD,
        **kwargs
    )

def gpu_chromium_mac_builder(*, name, **kwargs):
    return gpu_try_builder(
        name = name,
        builder_group = "tryserver.chromium.mac",
        cores = None,
        goma_backend = builders.goma.backend.RBE_PROD,
        os = builders.os.MAC_ANY,
        **kwargs
    )

def gpu_chromium_win_builder(*, name, os = builders.os.WINDOWS_ANY, **kwargs):
    return gpu_try_builder(
        name = name,
        builder_group = "tryserver.chromium.win",
        goma_backend = builders.goma.backend.RBE_PROD,
        os = os,
        **kwargs
    )

try_ = struct(
    defaults = defaults,
    builder = try_builder,
    declare_bucket = declare_bucket,
    job = tryjob,
    list_view = list_view,
    set_defaults = set_defaults,
    blink_builder = blink_builder,
    blink_mac_builder = blink_mac_builder,
    chromium_builder = chromium_builder,
    chromium_android_builder = chromium_android_builder,
    chromium_angle_builder = chromium_angle_builder,
    chromium_chromiumos_builder = chromium_chromiumos_builder,
    chromium_dawn_builder = chromium_dawn_builder,
    chromium_linux_builder = chromium_linux_builder,
    chromium_mac_builder = chromium_mac_builder,
    chromium_mac_ios_builder = chromium_mac_ios_builder,
    chromium_swangle_linux_builder = chromium_swangle_linux_builder,
    chromium_swangle_mac_builder = chromium_swangle_mac_builder,
    chromium_swangle_windows_builder = chromium_swangle_windows_builder,
    chromium_win_builder = chromium_win_builder,
    gpu_chromium_android_builder = gpu_chromium_android_builder,
    gpu_chromium_linux_builder = gpu_chromium_linux_builder,
    gpu_chromium_mac_builder = gpu_chromium_mac_builder,
    gpu_chromium_win_builder = gpu_chromium_win_builder,
)