blob: 6e1bd3e37995b2bdc77824e7f9b4a1d2500fbd09 [file] [log] [blame]
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This file implements various time-based expiration.
This includes:
- expiration of leases
- expiration of builds after a hard timeout
- deletion of very old builds
"""
import datetime
import logging
from google.appengine.ext import ndb
import webapp2
from components import decorators
from components import utils
from go.chromium.org.luci.buildbucket.proto import common_pb2
import events
import model
class CronExpireBuildLeases(webapp2.RequestHandler): # pragma: no cover
@decorators.require_cronjob
def get(self):
expire_build_leases()
def expire_build_leases():
"""Finds builds with expired lease and resets their lease and status."""
@ndb.transactional_tasklet
def txn_async(build_key):
now = utils.utcnow()
build = yield build_key.get_async()
if not build: # pragma: no cover
raise ndb.Return(False, build)
if not build.is_leased: # pragma: no cover
yield build.put_async() # ensure computed `is_leased` is committed
raise ndb.Return(False, build)
is_expired = build.lease_expiration_date <= now
if not is_expired: # pragma: no cover
raise ndb.Return(False, build)
assert not build.is_ended, 'Completed build is leased'
build.clear_lease()
build.proto.status = common_pb2.SCHEDULED
build.status_changed_time = now
build.url = None
yield build.put_async(), events.on_build_resetting_async(build)
raise ndb.Return(True, build)
@ndb.tasklet
def update_async(build_key):
# This is the only yield in this function, but it is not
# performance-critical.
updated, build = yield txn_async(build_key)
if updated: # pragma: no branch
events.on_expired_build_reset(build)
q = model.Build.query(
model.Build.is_leased == True,
model.Build.lease_expiration_date <= datetime.datetime.utcnow(),
)
q.map_async(update_async, keys_only=True).get_result()
class CronExpireBuilds(webapp2.RequestHandler): # pragma: no cover
@decorators.require_cronjob
def get(self):
expire_builds()
@ndb.tasklet
def expire_builds():
"""Finds old incomplete builds and marks them as TIMEOUT."""
expected_statuses = (common_pb2.SCHEDULED, common_pb2.STARTED)
@ndb.transactional_tasklet
def txn_async(build_key):
now = utils.utcnow()
build = yield build_key.get_async()
if not build or build.status not in expected_statuses:
raise ndb.Return(False, build) # pragma: no cover
build.clear_lease()
build.proto.status = common_pb2.INFRA_FAILURE
build.proto.status_details.timeout.SetInParent()
build.proto.end_time.FromDatetime(now)
build.status_changed_time = now
yield build.put_async(), events.on_build_completing_async(build)
raise ndb.Return(True, build)
@ndb.tasklet
def update_async(build_key):
# This is the only yield in this function, but it is not
# performance-critical.
updated, build = yield txn_async(build_key)
if updated: # pragma: no branch
events.on_build_completed(build)
# Utilize time-based build keys.
id_low, _ = model.build_id_range(None, utils.utcnow() - model.BUILD_TIMEOUT)
q = model.Build.query(
model.Build.key > ndb.Key(model.Build, id_low),
# Cannot use >1 inequality filters per query.
model.Build.status.IN(expected_statuses),
)
q.map_async(update_async, keys_only=True).get_result()
class CronDeleteBuilds(webapp2.RequestHandler): # pragma: no cover
@decorators.require_cronjob
def get(self):
delete_builds()
@ndb.tasklet
def delete_builds():
"""Finds very old builds and deletes them and their children.
Very old is defined by model.BUILD_STORAGE_DURATION.
"""
@ndb.transactional_tasklet
def txn_async(build_key):
to_delete = [build_key]
for clazz in model.BUILD_CHILD_CLASSES:
keys = yield clazz.query(ancestor=build_key).fetch_async(keys_only=True)
to_delete.extend(keys)
yield ndb.delete_multi_async(to_delete)
# Utilize time-based build keys.
id_low, _ = model.build_id_range(
None,
utils.utcnow() - model.BUILD_STORAGE_DURATION
)
q = model.Build.query(model.Build.key > ndb.Key(model.Build, id_low))
nones = q.map_async(txn_async, keys_only=True, limit=1000).get_result()
logging.info('Deleted %d builds', len(nones))