DevTools FE uploader: create checker instance to validate uploads
Runs on a separate GCE instance from the uploader and periodically checks
that earlier commits have been uploaded to the devtools frontend appspot instance.
This runs a simple HTTP server in a separate daemon thread so GCP/Stackdriver
can do a periodic uptime check. Ideally, because the HTTP server and checker thread
run in the same process, they should have similar uptime.
BUG=none
R=dgozman@chromium.org
Review URL: https://codereview.chromium.org/2358303002 .
diff --git a/gce/checker.py b/gce/checker.py
new file mode 100755
index 0000000..04d2317
--- /dev/null
+++ b/gce/checker.py
@@ -0,0 +1,114 @@
+#!/usr/bin/python
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+import logging
+import logging.handlers
+from os import path
+import subprocess
+import sys
+import threading
+import time
+import urllib2
+
+import uploader_mail
+
+# How many seconds to wait after a cycle.
+CYCLE_DELAY = 1800
+
+# Verifies a commit this long ago has been uploaded to devtools-frontend appspot
+CHECKER_DELAY = "12 hours"
+
+CYCLES_PER_24_HOURS = (86400 / CYCLE_DELAY)
+
+CHECKER_PATH = path.join(path.dirname(path.abspath(__file__)), '..', '..')
+CHROMIUM_CHECKOUT_PATH = path.join(CHECKER_PATH, 'src')
+LOG_PATH = path.join(CHECKER_PATH, 'logs', 'checker.log')
+
+consecutive_success = 0
+
+def main():
+ logfile = init_logger()
+ start_uptime_check_server()
+ while True:
+ logfile.doRollover()
+ logging.info('Starting iteration cycle')
+ call(['git', 'pull', 'origin', 'master'])
+ yesterday_commit_hash = call(['git', 'rev-list', '-n1', '--before={}'.format(CHECKER_DELAY), '--first-parent', 'HEAD'])
+ check_commit(yesterday_commit_hash.strip())
+ logging.info('Finished iteration cycle')
+ time.sleep(CYCLE_DELAY)
+
+def init_logger():
+ logger = logging.getLogger()
+ logger.setLevel(logging.INFO)
+
+ console = logging.StreamHandler()
+ console.setLevel(logging.INFO)
+ logger.addHandler(console)
+
+ logfile = logging.handlers.RotatingFileHandler(LOG_PATH, backupCount=30)
+ logfile.setLevel(logging.INFO)
+ logger.addHandler(logfile)
+
+ formatter = logging.Formatter('%(asctime)s:%(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S')
+ console.setFormatter(formatter)
+ logfile.setFormatter(formatter)
+ return logfile
+
+def check_commit(commit_hash):
+ global consecutive_success
+ url = 'https://chrome-devtools-frontend.appspot.com/serve_file/@{}/inspector.html'.format(commit_hash)
+ request = urllib2.Request(url)
+ try:
+ response = urllib2.urlopen(request)
+ summary = format_summary('Success', commit_hash, response.getcode())
+ logging.info(summary)
+ # Send an email on first success after start-up or getting an error
+ if consecutive_success == 0:
+ send_email(subject=summary, body='Response body:\n' + response.read())
+ consecutive_success += 1
+ if consecutive_success >= CYCLES_PER_24_HOURS:
+ consecutive_success = 0
+ except urllib2.HTTPError as error:
+ summary = format_summary('Error', commit_hash, error.code)
+ logging.info(summary)
+ send_email(subject=summary, body='Error message:\n' + error.reason)
+ consecutive_success = 0
+
+def send_email(subject, body):
+ mail_config = uploader_mail.ParseMailConfig(call(
+ ['gsutil', 'cat', 'gs://chrome-devtools-frontend/mail_config'], log_output=False))
+ uploader_mail.SendMail(mail_config, subject, body)
+ logging.info('Sent email with subject: {} and body: {}'.format(subject, body))
+
+def call(args, log_output=True, cwd=CHROMIUM_CHECKOUT_PATH):
+ process = subprocess.Popen(args, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, cwd=cwd)
+ out, err = process.communicate()
+ if process.returncode != 0:
+ logging.info('Error {} from {}'.format(process.returncode, args))
+ if log_output:
+ logging.info(out)
+ return out
+
+def format_summary(status, commit_hash, status_code):
+ return '[devtools-frontend-checker] {} for commit {} - HTTP status {}'.format(
+ status, commit_hash, status_code)
+
+def start_uptime_check_server():
+ class Handler(BaseHTTPRequestHandler):
+ def do_GET(self):
+ self.send_response(200)
+ self.send_header('Content-Type', 'text/html')
+ self.end_headers()
+ server = HTTPServer(('', 80), Handler)
+ thread = threading.Thread(target=server.serve_forever)
+ thread.daemon = True
+ thread.start()
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/gce/setup_checker.sh b/gce/setup_checker.sh
new file mode 100644
index 0000000..22c07d3
--- /dev/null
+++ b/gce/setup_checker.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+sudo apt-get install -y git
+cd ~/home
+sudo mkdir checker
+cd checker
+sudo git clone https://chromium.googlesource.com/chromium/tools/chrome-devtools-frontend
+sudo git clone https://chromium.googlesource.com/chromium/src --depth 500
+sudo mkdir logs
+cd chrome-devtools-frontend/gce && sudo ./start_checker.sh
diff --git a/gce/start_checker.sh b/gce/start_checker.sh
new file mode 100755
index 0000000..4643ed6
--- /dev/null
+++ b/gce/start_checker.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+# Copyright 2016 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+echo "Killing previous instance of checker.py (if applicable)"
+pkill -f checker.py
+nohup ./checker.py >/dev/null 2>&1 &
+echo "Started checker process in background"
