Create tryserver appengine app.
This appengine app is a step towards decentralized job control. It accepts POSTs
of json blobs describing buildbot jobs to its 'push' endpoint. Then, requests to
the 'pull' endpoint will be served not-yet-picked-up jobs. The poller can then
POST to the 'accept' endpoint to indicate that it has successfully picked up and
started a job. The 'peek' endpoint serves to inspect the job queue without
marking any jobs as potentially taken.
This is intended to interact with 'git try' on the client side to push blobs to
the app, and a new buildbot poller and scheduler to consume blobs and start
jobs.
R=cmp@google.com
Review URL: https://codereview.chromium.org/23093011
git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/chromium-jobqueue@219250 0039d316-1c4b-4281-b951-d872f2087c98
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..9fbb630
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+# Ignore Python bytecode generated by testing
+*.pyc
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..3d0f7d3
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,27 @@
+// Copyright (c) 2013 The Chromium Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/app.py b/app.py
new file mode 100644
index 0000000..3a0c042
--- /dev/null
+++ b/app.py
@@ -0,0 +1,130 @@
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import datetime
+import json
+
+import webapp2
+from google.appengine.ext import ndb
+
+
+DEFAULT_NAMESPACE = 'default'
+DEFAULT_JOBS_TO_SERVE = 20
+
+
+class Job(ndb.Model):
+ """Represents a single build job description.
+
+ Attributes:
+ description: A JSON blob representing the job itself.
+ created: A timestamp for when this job was posted.
+ last_served: A timestamp for the last time this job was served, usually
+ to a polling buildbot. Default is epoch 0, so new jobs are "old".
+ taken: A boolean signalling that this job has been successfully picked
+ up by a poller and can be dropped.
+ """
+ description = ndb.JsonProperty()
+ last_served = ndb.DateTimeProperty(
+ default=datetime.datetime.utcfromtimestamp(0))
+ taken = ndb.BooleanProperty(default=False)
+
+
+class MainHandler(webapp2.RequestHandler):
+
+ def get(self):
+ self.response.write("""
+<html>
+ <body>
+ <form action="/default/push" method="post">
+ <div><textarea name="job" rows="3" cols="60"></textarea></div>
+ <div><input type="submit" value="Add Job"></div>
+ </form>
+ <form action="/default/accept" method="post">
+ <div><textarea name="job" rows="1" cols="60"></textarea></div>
+ <div><input type="submit" value="Accept Job"></div>
+ </form>
+ </body>
+</html>
+""")
+
+
+class PushHandler(webapp2.RequestHandler):
+
+ def post(self, project):
+ job = Job(description=self.request.get('job'), namespace=project)
+ job.put()
+
+
+class PullHandler(webapp2.RequestHandler):
+
+ def post(self, project):
+ # Get the jobs we'd like to serve.
+ time_threshold = datetime.datetime.utcnow() - datetime.timedelta(seconds=30)
+ query = Job.query(namespace=project)
+ query = query.filter(Job.last_served < time_threshold)
+ query = query.filter(Job.taken == False)
+ query = query.order(Job.last_served)
+ jobs = query.fetch(DEFAULT_JOBS_TO_SERVE)
+
+ # Mark them as served.
+ for job in jobs:
+ job.last_served = datetime.datetime.utcnow()
+ job.put()
+
+ # Serve them.
+ result = []
+ for job in jobs:
+ job_blob = json.loads(job.description)
+ job_blob.update({'job_key': job.key.urlsafe()})
+ result.append(job_blob)
+ self.response.headers['Content-Type'] = 'application/json'
+ self.response.write(json.dumps(result))
+
+
+class PeekHandler(webapp2.RequestHandler):
+
+ def get(self, project, job):
+ # Get the jobs we'd like to serve.
+ if job:
+ job_key = ndb.Key(urlsafe=job, namespace=project)
+ job = job_key.get()
+ jobs = [job]
+ else:
+ time_threshold = (datetime.datetime.utcnow() -
+ datetime.timedelta(seconds=30))
+ query = Job.query(namespace=project)
+ query = query.filter(Job.last_served < time_threshold)
+ query = query.filter(Job.taken == False)
+ query = query.order(Job.last_served)
+ jobs = query.fetch(DEFAULT_JOBS_TO_SERVE)
+
+ # Serve them.
+ result = []
+ for job in jobs:
+ job_blob = json.loads(job.description)
+ job_blob.update({'job_key': job.key.urlsafe()})
+ result.append(job_blob)
+ self.response.headers['Content-Type'] = 'application/json'
+ self.response.write(json.dumps(result))
+
+
+class AcceptHandler(webapp2.RequestHandler):
+
+ def post(self, project, job):
+ job_key = ndb.Key(urlsafe=job, namespace=project)
+ job = job_key.get()
+ if job.taken:
+ # This job has been previously accepted by someone else.
+ self.response.set_status(409)
+ job.taken = True
+ job.put()
+
+
+app = webapp2.WSGIApplication([
+ ('/', MainHandler),
+ ('/(.*)/push', PushHandler),
+ ('/(.*)/pull', PullHandler),
+ ('/(.*)/peek/?(.*)', PeekHandler),
+ ('/(.*)/accept/(.*)', AcceptHandler),
+])
diff --git a/app.yaml b/app.yaml
new file mode 100644
index 0000000..3e1804f
--- /dev/null
+++ b/app.yaml
@@ -0,0 +1,30 @@
+application: chromium-jobqueue
+version: 1
+runtime: python27
+api_version: 1
+threadsafe: true
+
+libraries:
+- name: webapp2
+ version: latest
+
+handlers:
+- url: /.*
+ script: app.app
+
+builtins:
+- appstats: on
+- deferred: on
+
+skip_files:
+- ^(.*/)?app\.yaml
+- ^(.*/)?app\.yml
+- ^(.*/)?index\.yaml
+- ^(.*/)?index\.yml
+- ^(.*/)?#.*#
+- ^(.*/)?.*~
+- ^(.*/)?.*\.py[co]
+- ^(.*/)?.*/RCS/.*
+- ^(.*/)?\..*
+- ^(.*/)?.*\.bak$
+- tests/(.*/)?.*
