| # Copyright 2020 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| load("//lib/try.star", "location_filters_without_defaults", "try_") |
| load("//outages/config.star", outages_config = "config") |
| |
| _MD_HEADER = """\ |
| <!-- Auto-generated by lucicfg (via cq-builders-md.star). --> |
| <!-- Do not modify manually. --> |
| |
| # List of CQ builders |
| |
| [TOC] |
| |
| Changes that only modify files in //infra/config, which do not change any build |
| outputs, only require the chromium_presubmit builder to pass before landing. |
| |
| Each builder name links to that builder on Milo. The "matching builders" links |
| point to the file used to determine which configurations a builder should copy |
| when running. These links might 404 or error; they are hard-coded right now, |
| using common assumptions about how builders are configured. |
| """ |
| |
| _REQUIRED_HEADER = """\ |
| These builders must pass before a CL may land that affects files outside of |
| //docs and //infra/config. |
| """ |
| |
| _OPTIONAL_HEADER = """\ |
| These builders optionally run, depending on the files in a CL. For example, a CL |
| which touches `//gpu/BUILD.gn` would trigger the builder |
| `android_optional_gpu_tests_rel`, due to the `location_filters` values for that |
| builder. |
| """ |
| |
| _EXPERIMENTAL_HEADER = """\ |
| These builders are run on some percentage of builds. Their results are ignored |
| by CQ. These are often used to test new configurations before they are added |
| as required builders. |
| """ + ("""\ |
| |
| These builders are currently disabled due to the cq_disable_experiments outages |
| setting. See //infra/config/outages/README.md for more information. |
| """ if outages_config.disable_cq_experiments else "") |
| |
| _MEGA_MODE_HEADER = """\ |
| These builders run when the "Mega" CQ mode is triggered. This mode runs all the |
| builders required in the standard CQ, plus a large amount of optional builders. |
| """ |
| |
| _TRY_BUILDER_VIEW_URL = "https://ci.chromium.org/p/{project}/builders/{bucket}/{builder}" |
| |
| def _get_main_config_group_builders(ctx): |
| cq_cfg = ctx.output["luci/commit-queue.cfg"] |
| |
| for c in cq_cfg.config_groups: |
| if len(c.gerrit) != 1: |
| continue |
| |
| gerrit = c.gerrit[0] |
| if len(gerrit.projects) != 1: |
| continue |
| |
| project = gerrit.projects[0] |
| if len(project.ref_regexp) != 1: |
| continue |
| |
| if (project.name == "chromium/src" and |
| # Repeated proto fields have an internal type that won't compare equal |
| # to a list, so convert it |
| list(project.ref_regexp) == ["refs/heads/.+"]): |
| return c.verifiers.tryjob.builders |
| |
| fail("Could not find the main CQ group") |
| |
| def _normalize_builder(builder): |
| location_filters = location_filters_without_defaults(builder) |
| |
| return struct( |
| name = builder.name, |
| experiment_percentage = builder.experiment_percentage, |
| includable_only = builder.includable_only, |
| location_filters = location_filters, |
| mode_allowlist = builder.mode_allowlist, |
| equivalent_to = proto.clone(builder).equivalent_to, |
| ) |
| |
| def _group_builders_by_section(builders): |
| required = [] |
| experimental = [] |
| optional = [] |
| mega = [] |
| |
| for builder in builders: |
| builder = _normalize_builder(builder) |
| if builder.experiment_percentage: |
| experimental.append(builder) |
| elif builder.location_filters: |
| optional.append(builder) |
| elif builder.includable_only: |
| continue |
| elif try_.MEGA_CQ_FULL_RUN_NAME in builder.mode_allowlist: |
| mega.append(builder) |
| else: |
| required.append(builder) |
| |
| return struct( |
| required = required, |
| experimental = experimental, |
| optional = optional, |
| mega = mega, |
| ) |
| |
| def _codesearch_query(url, *atoms): |
| query = ["{}/search?q=".format(url)] |
| for atom in atoms: |
| query.append("+") |
| query.append(atom) |
| return "".join(query) |
| |
| def _public_codesearch_query(*atoms): |
| return _codesearch_query("https://cs.chromium.org", *atoms) |
| |
| def _internal_codesearch_query(*atoms): |
| return _codesearch_query("https://source.corp.google.com", *atoms) |
| |
| def _get_location_filter_details(f): |
| if f.gerrit_host_regexp != ".*" or f.gerrit_project_regexp != ".*": |
| fail("cq-builders.md generator needs updating to support gerrit host/project regexp") |
| |
| regex = f.path_regexp |
| title = "//" + regex.lstrip("/") |
| if regex.endswith(".+"): |
| regex = regex[:-len(".+")] |
| |
| url = _public_codesearch_query("file:" + regex) |
| |
| # If the regex doesn't have any interesting characters that might be part of a |
| # regex, assume the regex is targeting a single path and direct link to it |
| # Equals sign and dashes used by layout tests |
| if all([c.isalnum() or c in "/-_=" for c in regex.codepoints()]): |
| url = "https://cs.chromium.org/chromium/src/" + regex |
| |
| return struct( |
| prefix = "exclude: " if f.exclude else "", |
| title = title, |
| url = url, |
| ) |
| |
| def _generate_cq_builders_md(ctx): |
| builders = _get_main_config_group_builders(ctx) |
| |
| builders_by_section = _group_builders_by_section(builders) |
| |
| lines = [_MD_HEADER] |
| |
| for title, header, section in ( |
| ("Required builders", _REQUIRED_HEADER, "required"), |
| ("Optional builders", _OPTIONAL_HEADER, "optional"), |
| ("Experimental builders", _EXPERIMENTAL_HEADER, "experimental"), |
| ("Mega CQ builders", _MEGA_MODE_HEADER, "mega"), |
| ): |
| builders = getattr(builders_by_section, section) |
| if not builders: |
| continue |
| |
| lines.append("## %s" % title) |
| lines.append(header) |
| |
| printed_projects = set() |
| for b in builders: |
| project, bucket, name = b.name.split("/") |
| if project not in ("chrome", "chromium"): |
| fail("unexpected project added to the CQ: {}".format(project)) |
| if project not in printed_projects: |
| lines.append("### %s" % project) |
| printed_projects = printed_projects.union([project]) |
| builder_url = _TRY_BUILDER_VIEW_URL.format( |
| project = project, |
| bucket = bucket, |
| builder = name, |
| ) |
| |
| # Some builders share a common prefix (android-marshmallow-x86-rel |
| # and android-marshmallow-x86-rel-non-cq for example). The quotes |
| # below ensure codesearch links find the exact builder name, rather |
| # than everything with a common prefix. Two sets of quotes are |
| # needed because the first set is interpreted by codesearch. |
| quoted_name = "\"\"{name}\"\"".format(name = name) |
| if project == "chrome": |
| codesearch_query = _internal_codesearch_query("file:/try/.*\\.star$") |
| else: |
| codesearch_query = _public_codesearch_query("file:/try/.*\\.star$") |
| lines.append(( |
| "* [{name}]({try_builder_view}) " + |
| "([definition]({definition_query}+{quoted_name}))" |
| ).format( |
| name = name, |
| quoted_name = quoted_name, |
| try_builder_view = builder_url, |
| definition_query = codesearch_query, |
| )) |
| |
| if b.experiment_percentage: |
| lines.append(" * Experiment percentage: {percentage}".format( |
| percentage = b.experiment_percentage, |
| )) |
| |
| if b.location_filters: |
| lines.append("") |
| lines.append(" Location filters:") |
| for f in b.location_filters: |
| details = _get_location_filter_details(f) |
| lines.append(" * {prefix}[`{title}`]({url})".format( |
| prefix = details.prefix, |
| title = details.title, |
| url = details.url, |
| )) |
| if getattr(b, "equivalent_to") and b.equivalent_to.name: |
| eq_project, eq_bucket, eq_name = b.equivalent_to.name.split("/") |
| eq_builder_url = _TRY_BUILDER_VIEW_URL.format( |
| project = eq_project, |
| bucket = eq_bucket, |
| builder = eq_name, |
| ) |
| lines.append("") |
| s = (" * Replaced with builder: [{name}]({builder}) \ |
| when CL owner is in group \ |
| [{group}](https://chrome-infra-auth.appspot.com/auth/lookup?p={group})".format( |
| name = eq_name, |
| builder = eq_builder_url, |
| group = b.equivalent_to.owner_whitelist_group, |
| )) |
| if b.equivalent_to.percentage != 100: |
| s = s + (" with percentage {p}".format( |
| p = b.equivalent_to.percentage, |
| )) |
| lines.append(s) |
| |
| lines.append("") |
| |
| lines.append("") |
| |
| ctx.output["cq-builders.md"] = "\n".join(lines) |
| |
| lucicfg.generator(_generate_cq_builders_md) |