| #!/usr/bin/env python |
| |
| """Wrapper script for running jobs in TaskCluster |
| |
| This is intended for running test jobs in TaskCluster. The script |
| takes a two positional arguments which are the name of the test job |
| and the script to actually run. |
| |
| The name of the test job is used to determine whether the script should be run |
| for this push (this is in lieu of having a proper decision task). There are |
| several ways that the script can be scheduled to run |
| |
| 1. The output of wpt test-jobs includes the job name |
| 2. The job name is included in a job declaration (see below) |
| 3. The string "all" is included in the job declaration |
| 4. The job name is set to "all" |
| |
| A job declaration is a line appearing in the pull request body (for |
| pull requests) or first commit message (for pushes) of the form: |
| |
| tc-jobs: job1,job2,[...] |
| |
| In addition, there are a number of keyword arguments used to set options for the |
| environment in which the jobs run. Documentation for these is in the command help. |
| |
| As well as running the script, the script sets two environment variables; |
| GITHUB_BRANCH which is the branch that the commits will merge into (if it's a PR) |
| or the branch that the commits are on (if it's a push), and GITHUB_PULL_REQUEST |
| which is the string "false" if the event triggering this job wasn't a pull request |
| or the pull request number if it was. The semantics of these variables are chosen |
| to match the corresponding TRAVIS_* variables. |
| |
| Note: for local testing in the Docker image the script ought to still work, but |
| full functionality requires that the TASK_EVENT environment variable is set to |
| the serialization of a GitHub event payload. |
| """ |
| |
| import argparse |
| import json |
| import os |
| import re |
| import subprocess |
| import sys |
| try: |
| from urllib2 import urlopen |
| except ImportError: |
| # Python 3 case |
| from urllib.request import urlopen |
| |
| |
| QUEUE_BASE = "https://queue.taskcluster.net/v1/task" |
| |
| |
| root = os.path.abspath( |
| os.path.join(os.path.dirname(__file__), |
| os.pardir, |
| os.pardir)) |
| |
| |
| def run(cmd, return_stdout=False, **kwargs): |
| print(" ".join(cmd)) |
| if return_stdout: |
| f = subprocess.check_output |
| else: |
| f = subprocess.check_call |
| return f(cmd, **kwargs) |
| |
| |
| def start(cmd): |
| print(" ".join(cmd)) |
| subprocess.Popen(cmd) |
| |
| |
| def get_parser(): |
| p = argparse.ArgumentParser() |
| p.add_argument("--oom-killer", |
| action="store_true", |
| default=False, |
| help="Run userspace OOM killer") |
| p.add_argument("--hosts", |
| dest="hosts_file", |
| action="store_true", |
| default=True, |
| help="Setup wpt entries in hosts file") |
| p.add_argument("--no-hosts", |
| dest="hosts_file", |
| action="store_false", |
| help="Don't setup wpt entries in hosts file") |
| p.add_argument("--browser", |
| action="append", |
| default=[], |
| help="Browsers that will be used in the job") |
| p.add_argument("--channel", |
| default=None, |
| choices=["experimental", "dev", "nightly", "beta", "stable"], |
| help="Chrome browser channel") |
| p.add_argument("--xvfb", |
| action="store_true", |
| help="Start xvfb") |
| p.add_argument("--checkout", |
| help="Revision to checkout before starting job") |
| p.add_argument("job", |
| help="Name of the job associated with the current event") |
| p.add_argument("script", |
| help="Script to run for the job") |
| p.add_argument("script_args", |
| nargs=argparse.REMAINDER, |
| help="Additional arguments to pass to the script") |
| return p |
| |
| |
| def start_userspace_oom_killer(): |
| # Start userspace OOM killer: https://github.com/rfjakob/earlyoom |
| # It will report memory usage every minute and prefer to kill browsers. |
| start(["sudo", "earlyoom", "-p", "-r", "60", "--prefer=(chrome|firefox)", "--avoid=python"]) |
| |
| |
| def make_hosts_file(): |
| subprocess.check_call(["sudo", "sh", "-c", "./wpt make-hosts-file >> /etc/hosts"]) |
| |
| |
| def checkout_revision(rev): |
| subprocess.check_call(["git", "checkout", "--quiet", rev]) |
| |
| |
| def install_chrome(channel): |
| if channel in ("experimental", "dev", "nightly"): |
| deb_archive = "google-chrome-unstable_current_amd64.deb" |
| elif channel == "beta": |
| deb_archive = "google-chrome-beta_current_amd64.deb" |
| elif channel == "stable": |
| deb_archive = "google-chrome-stable_current_amd64.deb" |
| else: |
| raise ValueError("Unrecognized release channel: %s" % channel) |
| |
| dest = os.path.join("/tmp", deb_archive) |
| resp = urlopen("https://dl.google.com/linux/direct/%s" % deb_archive) |
| with open(dest, "w") as f: |
| f.write(resp.read()) |
| |
| run(["sudo", "apt-get", "-qqy", "update"]) |
| run(["sudo", "gdebi", "-qn", "/tmp/%s" % deb_archive]) |
| |
| |
| def start_xvfb(): |
| start(["sudo", "Xvfb", os.environ["DISPLAY"], "-screen", "0", |
| "%sx%sx%s" % (os.environ["SCREEN_WIDTH"], |
| os.environ["SCREEN_HEIGHT"], |
| os.environ["SCREEN_DEPTH"])]) |
| start(["sudo", "fluxbox", "-display", os.environ["DISPLAY"]]) |
| |
| |
| def get_extra_jobs(event): |
| body = None |
| jobs = set() |
| if "commits" in event and event["commits"]: |
| body = event["commits"][0]["message"] |
| elif "pull_request" in event: |
| body = event["pull_request"]["body"] |
| |
| if not body: |
| return jobs |
| |
| regexp = re.compile(r"\s*tc-jobs:(.*)$") |
| |
| for line in body.splitlines(): |
| m = regexp.match(line) |
| if m: |
| items = m.group(1) |
| for item in items.split(","): |
| jobs.add(item.strip()) |
| break |
| return jobs |
| |
| |
| def set_variables(event): |
| # Set some variables that we use to get the commits on the current branch |
| ref_prefix = "refs/heads/" |
| pull_request = "false" |
| branch = None |
| if "pull_request" in event: |
| pull_request = str(event["pull_request"]["number"]) |
| # Note that this is the branch that a PR will merge to, |
| # not the branch name for the PR |
| branch = event["pull_request"]["base"]["ref"] |
| elif "ref" in event: |
| branch = event["ref"] |
| if branch.startswith(ref_prefix): |
| branch = branch[len(ref_prefix):] |
| |
| os.environ["GITHUB_PULL_REQUEST"] = pull_request |
| if branch: |
| os.environ["GITHUB_BRANCH"] = branch |
| |
| |
| def include_job(job): |
| # Special case things that unconditionally run on pushes, |
| # assuming a higher layer is filtering the required list of branches |
| if (os.environ["GITHUB_PULL_REQUEST"] == "false" and |
| job == "run-all"): |
| return True |
| |
| jobs_str = run([os.path.join(root, "wpt"), |
| "test-jobs"], return_stdout=True) |
| print(jobs_str) |
| return job in set(jobs_str.splitlines()) |
| |
| |
| def setup_environment(args): |
| if args.hosts_file: |
| make_hosts_file() |
| |
| if "chrome" in args.browser: |
| assert args.channel is not None |
| install_chrome(args.channel) |
| |
| if args.xvfb: |
| start_xvfb() |
| |
| if args.oom_killer: |
| start_userspace_oom_killer() |
| |
| if args.checkout: |
| checkout_revision(args.checkout) |
| |
| |
| def setup_repository(): |
| if os.environ.get("GITHUB_PULL_REQUEST", "false") != "false": |
| parents = run(["git", "show", "--no-patch", "--format=%P", "task_head"], return_stdout=True).strip().split() |
| if len(parents) == 2: |
| base_head = parents[0] |
| pr_head = parents[1] |
| |
| run(["git", "branch", "base_head", base_head]) |
| run(["git", "branch", "pr_head", pr_head]) |
| else: |
| print("ERROR: Pull request HEAD wasn't a 2-parent merge commit; " |
| "expected to test the merge of PR into the base") |
| commit = run(["git", "show", "--no-patch", "--format=%H", "task_head"], return_stdout=True).strip() |
| print("HEAD: %s" % commit) |
| print("Parents: %s" % ", ".join(parents)) |
| sys.exit(1) |
| |
| branch = os.environ.get("GITHUB_BRANCH") |
| if branch: |
| # Ensure that the remote base branch exists |
| # TODO: move this somewhere earlier in the task |
| run(["git", "fetch", "--quiet", "origin", "%s:%s" % (branch, branch)]) |
| |
| |
| def fetch_event_data(): |
| try: |
| task_id = os.environ["TASK_ID"] |
| except KeyError: |
| print("WARNING: Missing TASK_ID environment variable") |
| # For example under local testing |
| return None |
| |
| resp = urlopen("%s/%s" % (QUEUE_BASE, task_id)) |
| |
| task_data = json.load(resp) |
| event_data = task_data.get("extra", {}).get("github_event") |
| if event_data is not None: |
| return json.loads(event_data) |
| |
| |
| def main(): |
| args = get_parser().parse_args() |
| |
| if "TASK_EVENT" in os.environ: |
| event = json.loads(os.environ["TASK_EVENT"]) |
| else: |
| event = fetch_event_data() |
| |
| if event: |
| set_variables(event) |
| |
| setup_repository() |
| |
| extra_jobs = get_extra_jobs(event) |
| |
| job = args.job |
| |
| print("Job %s" % job) |
| |
| run_if = [(lambda: job == "all", "job set to 'all'"), |
| (lambda:"all" in extra_jobs, "Manually specified jobs includes 'all'"), |
| (lambda:job in extra_jobs, "Manually specified jobs includes '%s'" % job), |
| (lambda:include_job(job), "CI required jobs includes '%s'" % job)] |
| |
| for fn, msg in run_if: |
| if fn(): |
| print(msg) |
| break |
| else: |
| print("Job not scheduled for this push") |
| return |
| |
| # Run the job |
| setup_environment(args) |
| os.chdir(root) |
| cmd = [args.script] + args.script_args |
| print(" ".join(cmd)) |
| sys.exit(subprocess.call(cmd)) |
| |
| |
| if __name__ == "__main__": |
| main() |