blob: a18b5d2bbda32b95123d33e93c81fa883358ecee [file] [log] [blame]
# Copyright (c) 2009-2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""A CherryPy-based webserver to host factory installation."""
# This a special fork from devserver for serving factory and may be executed in
# a very limited environment without full CrOS source tree.
import glob
import optparse
import os
import shutil
import socket
import subprocess
import sys
import threading
import time
import urllib
import zipfile
import cherrypy
import miniomaha_engine
def _LogUpdateMessage(message):
cherrypy.log(message, 'UPDATE')
def _GetConfig(opts):
"""Returns the configuration for the miniomaha."""
base_config = { 'global':
{ 'server.log_request_headers': True,
'server.protocol_version': 'HTTP/1.1',
'::' if socket.has_ipv6 else '',
'server.socket_port': int(opts.port),
'server.socket_timeout': 6000,
'response.timeout': 6000,
# Gets rid of cherrypy parsing post file for args.
'request.process_request_body': False,
# Gets rid of cherrypy parsing post file for args.
'request.process_request_body': False,
'response.timeout': 10000,
# Sets up the static dir for file hosting.
{ 'tools.staticdir.dir': 'static',
'tools.staticdir.on': True,
'response.timeout': 10000,
return base_config
class BoardNotFoundException(Exception):
class OmahaPreparer(object):
"""Class for preparing all the necessary files for mini-omaha server."""
conf_filename = 'miniomaha.conf'
def __init__(self, script_dir, cache_dir, config_path=None):
self.script_dir = script_dir
self.cache_dir = cache_dir
self.boards_to_update = None
self.version_offset = None
self.omaha_config_path = config_path
def set_boards_to_update(self, _boards_to_update):
self.boards_to_update = _boards_to_update
def set_version_offset(self, _version_offset):
self.version_offset = _version_offset
def _read_config(self, config_path):
output = {}
with open(config_path, 'r') as f:
exec(, output)
return output['config']
def generate_files_from_image(self, board_name):
"""Generate all the files and config with respect to a new image."""
# create data directory if necessary
data_dir = os.path.join(self.cache_dir, board_name)
if os.path.exists(data_dir):
# unzip the image of the target board
for cached_file in os.listdir(self.cache_dir):
if '%s_' % board_name in cached_file:
zip_file = zipfile.ZipFile(os.path.join(self.cache_dir, cached_file))
# find the unzipped image
file_path = glob.glob(os.path.join(data_dir, '*.bin'))[0]
# call, the result is stored in data_dir
return_value =
[os.path.join(self.script_dir, ''),
'--board=%s' % board_name,
'--release=%s' % file_path,
'--omaha_data_dir=%s' % data_dir])
if return_value:
sys.exit("Failed to run")
# read and delete the temporary config file
config_path = os.path.join(data_dir, self.conf_filename)
config = self._read_config(config_path)
# modify fields into the miniomaha readable form
new_config = {}
if config:
new_config = config[0]
for keys in new_config:
if keys.endswith('_image'):
dir_name = board_name
if self.version_offset:
dir_name = os.path.join(self.version_offset, board_name)
new_config[keys] = os.path.join(dir_name, new_config[keys])
return new_config
def generate_miniomaha_files(self):
"""Generate files for the updated boards."""
config_path = os.path.join(self.cache_dir, self.conf_filename)
if os.path.exists(config_path):
factory_configs = self._read_config(config_path)
factory_configs = []
# remove the old information of updated boards from config
for board in self.boards_to_update:
factory_configs[:] = [config for config in factory_configs
if board not in config['qual_ids']]
# generate the new configs and file for updated boards
for board in self.boards_to_update:
config = self.generate_files_from_image(board)
if not config:
sys.exit("Failed to generate config files")
with open(config_path, 'w') as f:
f.write('config=%s\n' % factory_configs)
def setup_miniomaha_files(self):
"""Move the updated files to mini-omaha static directory."""
omaha_dir = os.path.join(self.script_dir, 'static')
if self.version_offset:
omaha_dir = os.path.join(omaha_dir, self.version_offset)
if not os.path.isdir(omaha_dir):
for board in self.boards_to_update:
target_dir = os.path.join(omaha_dir, board)
if os.path.isdir(target_dir):
os.rename(os.path.join(self.cache_dir, board), target_dir)
omaha_config_path = (self.omaha_config_path or
os.path.join(omaha_dir, self.conf_filename))
shutil.copy(os.path.join(self.cache_dir, self.conf_filename),
class ImageUpdater(object):
"""Class for requesting the latest stable image from server."""
conf_url = ''
def __init__(self):
self.board_images = []
def _get_version_info(self):
"""Download and analyze the version information from server."""
urllib.urlretrieve(self.conf_url, 'recovery.conf')
with open('recovery.conf', 'r') as f:
current_version = ''
for line in f:
stanza = line.strip().split('=')
if stanza[0] == 'url':
if (current_version, stanza[1]) not in self.board_images:
self.board_images.append((current_version, stanza[1]))
elif stanza[0] == 'version':
current_version = stanza[1]
def update_image(self, board_name, cache_dir):
"""Check and update an image of the give board name."""
update_filename = ''
update_url = ''
for version, url in self.board_images:
if '%s_' % board_name in url:
update_filename = '' % (board_name, version)
update_url = url
if not update_filename:
raise BoardNotFoundException
need_update = True
for cached_file in os.listdir(cache_dir):
if '%s_' % board_name in cached_file:
if cached_file == update_filename:
need_update = False
os.remove(os.path.join(cache_dir, cached_file))
if need_update:
urllib.urlretrieve(update_url, os.path.join(cache_dir, update_filename))
return need_update
class UpdateChecker(object):
"""Class for doing peridically update check."""
def __init__(self, opts, script_dir, cache_dir, _updater, lock):
self.opts = opts
self.script_dir = script_dir
self.cache_dir = cache_dir
self.updater = _updater
self.update_lock = lock
self.timer = None
self.base_dir = os.path.realpath(self.opts.data_dir)
self.next_version = 1
if opts.boards:
def _CleanUpConfig(self):
"""Put the updated files into initial position"""
if self.updater.GetActiveConfigIndex() == 1:
# No update
initial_config = self.updater.GetConfig(0)
last_config = self.updater.GetConfig(self.updater.GetActiveConfigIndex())
# Parse initial dir for initial_dir/board/release_image
initial_dir = os.path.dirname(initial_config[0]['release_image'])
initial_dir = os.path.dirname(initial_dir)
initial_dir = os.path.join(self.base_dir, initial_dir)
# Move the final version of each board into initial dir
for board_conf in last_config:
board_dir = os.path.dirname(board_conf['release_image'])
board_dir = os.path.join(self.base_dir, board_dir)
board_name = os.path.basename(board_dir)
target_dir = os.path.join(initial_dir, board_name)
if os.path.samefile(board_dir, target_dir):
if os.path.isdir(target_dir):
shutil.move(board_dir, target_dir)
# Correct the file path and pop unnecessary keys
for key in board_conf.copy().iterkeys():
if key.endswith('_image'):
board_conf[key] = os.path.join(target_dir,
elif key.endswith('_size'):
# Overwrite the config file
with open(self.opts.factory_config, 'w') as file_handle:
file_handle.write('config=%s\n' % last_config)
# Remove _ver*/
for version in glob.glob(os.path.join(self.base_dir, '_ver*')):
def _UpdateCheck(self, boards=None):
"""Do update check periodically."""
# Initialize preparer and updater
if not os.path.exists(self.cache_dir):
image_updater = ImageUpdater()
image_preparer = OmahaPreparer(self.script_dir, self.cache_dir)
# If boards is set, do an initial update setup for those boards
if boards:
init_preparer = OmahaPreparer(
self.script_dir, self.cache_dir,
for board in list(boards):
updated = image_updater.update_image(board, self.cache_dir)
except BoardNotFoundException:
_LogUpdateMessage('WARNING: No board named %s is found, ignored' %
self.updater.ImportFactoryConfigFile(self.opts.factory_config, False)
# Try to update all boards in config
updated_boards = []
active_config = self.updater.GetConfig(self.updater.GetActiveConfigIndex())
for board_conf in active_config:
# The format in config is qual_id: set(['board'])
for board in board_conf['qual_ids']:
updated = image_updater.update_image(board, self.cache_dir)
if updated:
_LogUpdateMessage('Detect update for board %s' % board)
if not updated_boards:
_LogUpdateMessage('Everything up-to-date, update check finished')
_LogUpdateMessage('Start updating')
version_offset = '_ver%s' % self.next_version
self.next_version += 1
# Prepare the files for the newly downloaded boards
data_dir = self.base_dir
# Change config, critical session
with self.update_lock:
# Read config
config_dir = os.path.join(data_dir, version_offset)
config_path = os.path.join(config_dir, 'miniomaha.conf')
self.updater.ImportFactoryConfigFile(config_path, False)
# Restart timers
# Time interval between each update check, by seconds
self.timer = threading.Timer(self.opts.interval, self._UpdateCheck)
self.timer.daemon = True
def cleanup(self):
class ApiRoot(object):
"""RESTful API for Dev Server information."""
exposed = True
def hostinfo(self, ip):
"""Returns a JSON dictionary containing information about the given ip.
Not all information may be known at the time the request is made. The
possible keys are:
last_event_type: int
Last update event type received.
last_event_status: int
Last update event status received.
last_known_version: string
Last known version recieved for update ping.
forced_update_label: string
Update label to force next update ping to use. Set by setnextupdate.
See the OmahaEvent class in update_engine/omaha_request_action.h for status
code definitions. If the ip does not exist an empty string is returned."""
return updater.HandleHostInfoPing(ip)
def setnextupdate(self, ip):
"""Allows the response to the next update ping from a host to be set.
Takes the IP of the host and an update label as normally provided to the
/update command."""
body_length = int(cherrypy.request.headers['Content-Length'])
label =
if label:
label = label.strip()
if label:
return updater.HandleSetUpdatePing(ip, label)
raise cherrypy.HTTPError(400, 'No label provided.')
class DevServerRoot(object):
"""The Root Class for the Dev Server.
CherryPy works as follows:
For each method in this class, cherrpy interprets root/path
as a call to an instance of DevServerRoot->method_name. For example,
a call to http://myhost/build will call build. CherryPy automatically
parses http args and places them as keyword arguments in each method.
For paths http://myhost/update/dir1/dir2, you can use *args so that
cherrypy uses the update method and puts the extra paths in args.
api = ApiRoot()
fail_msg = 'Previous session from %s, uuid: %s, start at %s did not complete'
time_string = '%d/%b/%Y %H:%M:%S'
def __init__(self, lock, auto_update):
self.client_table = {}
self.update_lock = lock
self.auto_update = auto_update
def GetClientConfigIndex(self, ip):
return self.client_table[ip]['config_index']
def SetClientConfigIndex(self, ip, index):
self.client_table[ip]['config_index'] = index
def GetClientStartTime(self, ip):
return self.client_table[ip]['start_time']
def SetClientStartTime(self, ip, start_time):
self.client_table[ip]['start_time'] = start_time
def index(self):
return 'Welcome to the Dev Server!'
def update(self):
client_ip = cherrypy.request.remote.ip.split(':')[-1]
body_length = int(cherrypy.request.headers['Content-Length'])
data =
# For backward compatibility of old install shim.
# Updater should work anyway.
if client_ip not in self.client_table:
if self.auto_update:
'WARNING: Detect unrecorded ip: %s. '
'If you are using an old factory install shim, '
'there may be unexpected outcome in --auto_update mode' %
return updater.HandleUpdatePing(data, updater.GetActiveConfigIndex())
return updater.HandleUpdatePing(data,
def greetings(self, label, uuid):
# Temporarily use ip as identifier.
# This may be changed if we found better session ids
client_ip = cherrypy.request.remote.ip.split(':')[-1]
if label != 'hello' and client_ip not in self.client_table:
_LogUpdateMessage('Unexpected %s notification from %s, uuid: %s' %
(label, client_ip, uuid))
return 'Wrong notification'
if label == 'hello':
if client_ip in self.client_table:
# previous session did not complete, print error to log
start_time = time.strftime(
_LogUpdateMessage(self.fail_msg % (client_ip, uuid, start_time))
self.client_table[client_ip] = {}
self.SetClientStartTime(client_ip, time.time())
_LogUpdateMessage('Start a install session for %s, uuid: %s' %
(client_ip, uuid))
with self.update_lock:
self.SetClientConfigIndex(client_ip, updater.GetActiveConfigIndex())
return 'hello'
elif label == 'download_complete':
'Session from %s, uuid: %s, '
'successfully downloaded all necessary files' %
(client_ip, uuid))
return 'download complete'
elif label == 'goodbye':
elapse_time = time.time() - self.GetClientStartTime(client_ip)
'Session from %s, uuid: %s, '
'has been completed, elapse time is %s seconds' %
(client_ip, uuid, elapse_time))
return 'goodbye'
#TODO(chunyen): move this to cherrypy exit callback
def __del__(self):
# Write log for those incomplete session
for client_ip in self.client_table:
start_time = time.strftime(
_LogUpdateMessage(self.fail_msg % (client_ip, start_time))
if __name__ == '__main__':
base_path = os.path.dirname(os.path.abspath(sys.argv[0]))
usage = 'usage: %prog [options]'
parser = optparse.OptionParser(usage)
parser.add_option('--data_dir', dest='data_dir',
help='Writable directory where static lives',
default=os.path.join(base_path, 'static'))
parser.add_option('--factory_config', dest='factory_config',
help='Config file for serving images from factory floor.',
parser.add_option('--port', default=8080,
help='Port for the dev server to use.')
parser.add_option('--proxy_port', default=None,
help='Port to have the client connect to (testing support)')
parser.add_option('--validate_factory_config', action="store_true",
help='Validate factory config file, then exit.')
parser.add_option('--log', dest='log_path',
help='Path for server execution log',
default=os.path.join(base_path, 'miniomaha.log'))
parser.add_option('--auto_update', action='store_true', dest='auto_update',
help='Enable auto updating image from server')
parser.add_option('--cache', dest='cache_dir', default=None,
help='Cache_dir for auto update images')
parser.add_option('--interval', dest='interval', default=1800, type=int,
help='Interval between each update check')
parser.add_option('--boards', dest='boards', default=None,
help='Name of boards to track in auto-update mode, '
'split by comma.')
(options, _) = parser.parse_args()
static_dir = os.path.realpath(options.data_dir)
os.system('mkdir -p %s' % static_dir)
cherrypy.log('Data dir is %s' % options.data_dir, 'DEVSERVER')
cherrypy.log('Serving from %s' % static_dir, 'DEVSERVER')
updater = miniomaha_engine.ServerEngine(
updater_lock = threading.Lock()
# Auto update is not support in validate factory config mode
if options.validate_factory_config and options.auto_update:
cherrypy.log('Auto update is not support when validating factory config',
options.auto_update = False
# --boards only works in auto update mode
if options.boards and not options.auto_update:
cherrypy.log('--boards only works in auto_update mode')
options.boards = None
# Sanity-check for use of validate_factory_config.
# In previous version, the default configuration file is in base_path,
# but now it is in data_dir,
# so we want to check both for backward compatibility.
if not options.factory_config:
config_files = (os.path.join(base_path, 'miniomaha.conf'),
os.path.join(options.data_dir, 'miniomaha.conf'))
exists = map(os.path.exists, config_files)
# When boards is set, we always use the current default data dir
if options.boards:
options.factory_config = config_files[1]
elif all(exists):
parser.error('Confusing factory config files.\n'
'Please remove the old config file in %s' % base_path)
elif any(exists):
options.factory_config = config_files[exists.index(True)]
parser.error('No factory files found')
# When boards is set, we should import config after the first update check.
if not options.boards:
# We've done validating factory config, exit now!
if options.validate_factory_config:
if options.auto_update:
# Set up cache directory.
options.cache_dir = (options.cache_dir or
os.path.join(base_path, 'cache_dir'))
# Ensure that the configure file in cache directory is the same as that
# in data directory.
if os.path.exists(options.factory_config):
shutil.copy(options.factory_config, options.cache_dir)
update_checker = UpdateChecker(options, base_path, options.cache_dir,
updater, updater_lock)
# Since cheerypy need an existing file to append log,
# here we make sure the log file path exist and ready for writing.
with open(options.log_path, 'a'):
cherrypy.log.screen = True
cherrypy.log.access_file = options.log_path
cherrypy.log.error_file = options.log_path
cherrypy.quickstart(DevServerRoot(updater_lock, options.auto_update),
if options.auto_update:
# Sync the config file again to avoid error if user run
shutil.copy(options.factory_config, options.cache_dir)