| #!/usr/bin/python |
| |
| # Copyright 2011 Google Inc. All Rights Reserved. |
| |
| import datetime |
| import os |
| import re |
| import threading |
| import time |
| import traceback |
| from results_cache import Result |
| from utils import logger |
| from utils import command_executer |
| from autotest_runner import AutotestRunner |
| from perf_processor import PerfProcessor |
| from results_cache import ResultsCache |
| |
| |
| STATUS_FAILED = "FAILED" |
| STATUS_SUCCEEDED = "SUCCEEDED" |
| STATUS_IMAGING = "IMAGING" |
| STATUS_RUNNING = "RUNNING" |
| STATUS_WAITING = "WAITING" |
| STATUS_PENDING = "PENDING" |
| |
| |
| class BenchmarkRun(threading.Thread): |
| def __init__(self, name, benchmark_name, autotest_name, autotest_args, |
| label_name, chromeos_root, chromeos_image, board, iteration, |
| cache_conditions, outlier_range, profile_counters, profile_type, |
| machine_manager, |
| logger_to_use): |
| threading.Thread.__init__(self) |
| self.name = name |
| self._logger = logger_to_use |
| self.benchmark_name = benchmark_name |
| self.autotest_name = autotest_name |
| self.autotest_args = autotest_args |
| self.label_name = label_name |
| self.chromeos_root = chromeos_root |
| self.chromeos_image = os.path.expanduser(chromeos_image) |
| self.board = board |
| self.iteration = iteration |
| self.result = None |
| self.results = {} |
| self.terminated = False |
| self.retval = None |
| self.status = STATUS_PENDING |
| self.run_completed = False |
| self.outlier_range = outlier_range |
| self.profile_counters = profile_counters |
| self.profile_type = profile_type |
| self.machine_manager = machine_manager |
| self.cache = ResultsCache() |
| self.autotest_runner = AutotestRunner(self._logger) |
| self.perf_processor = None |
| self.machine = None |
| self.full_name = self.autotest_name |
| self.cache_conditions = cache_conditions |
| self.runs_complete = 0 |
| self.cache_hit = False |
| self.perf_results = None |
| self.failure_reason = "" |
| self._ce = command_executer.GetCommandExecuter(self._logger) |
| |
| def ProcessResults(self): |
| # Generate results from the output file. |
| self.full_name = os.path.basename(self.results_dir) |
| self.results = self.result.keyvals |
| |
| # Store the autotest output in the cache also. |
| if not self.cache_hit: |
| self.cache.StoreResult(self.result) |
| self.cache.StoreAutotestOutput(self.results_dir) |
| |
| self.perf_processor = PerfProcessor(self.results_dir, |
| self.chromeos_root, |
| self.board, |
| self._logger) |
| # Generate a perf report and cache it. |
| if self.profile_type: |
| if self.cache_hit: |
| self.perf_results = self.cache.ReadPerfResults() |
| else: |
| self.perf_results = self.perf_processor.GeneratePerfResults() |
| self.cache.StorePerfResults(self.perf_results) |
| |
| # If there are valid results from perf stat, combine them with the |
| # autotest results. |
| if self.perf_results: |
| stat_results = self.perf_processor.ParseStatResults(self.perf_results) |
| self.results = dict(self.results.items() + stat_results.items()) |
| |
| def _GetResultsDir(self, output): |
| mo = re.search("Results placed in (\S+)", output) |
| if mo: |
| result = mo.group(1) |
| return result |
| raise Exception("Could not find results directory.") |
| |
| def run(self): |
| try: |
| # Just use the first machine for running the cached version, |
| # without locking it. |
| self.cache.Init(self.chromeos_image, |
| self.chromeos_root, |
| self.autotest_name, |
| self.iteration, |
| self.autotest_args, |
| self.machine_manager.GetMachines()[0].name, |
| self.board, |
| self.cache_conditions, |
| self._logger) |
| |
| self.result = self.cache.ReadResult() |
| self.cache_hit = (self.result is not None) |
| |
| if self.result: |
| self._logger.LogOutput("%s: Cache hit." % self.name) |
| self._logger.LogOutput(self.result.out + "\n" + self.result.err) |
| self.results_dir = self._GetResultsDir(self.result.out) |
| else: |
| self._logger.LogOutput("%s: No cache hit." % self.name) |
| self.status = STATUS_WAITING |
| # Try to acquire a machine now. |
| self.machine = self.AcquireMachine() |
| self.cache.remote = self.machine.name |
| self.result = self.RunTest(self.machine) |
| |
| if self.terminated: |
| return |
| |
| if not self.result.retval: |
| self.status = STATUS_SUCCEEDED |
| else: |
| if self.status != STATUS_FAILED: |
| self.status = STATUS_FAILED |
| self.failure_reason = "Return value of autotest was non-zero." |
| |
| self.ProcessResults() |
| |
| except Exception, e: |
| self._logger.LogError("Benchmark run: '%s' failed: %s" % (self.name, e)) |
| traceback.print_exc() |
| if self.status != STATUS_FAILED: |
| self.status = STATUS_FAILED |
| self.failure_reason = str(e) |
| finally: |
| if self.machine: |
| self._logger.LogOutput("Releasing machine: %s" % self.machine.name) |
| self.machine_manager.ReleaseMachine(self.machine) |
| self._logger.LogOutput("Released machine: %s" % self.machine.name) |
| |
| def Terminate(self): |
| self.terminated = True |
| self.autotest_runner.Terminate() |
| if self.status != STATUS_FAILED: |
| self.status = STATUS_FAILED |
| self.failure_reason = "Thread terminated." |
| |
| def AcquireMachine(self): |
| while True: |
| if self.terminated: |
| raise Exception("Thread terminated while trying to acquire machine.") |
| machine = self.machine_manager.AcquireMachine(self.chromeos_image) |
| if machine: |
| self._logger.LogOutput("%s: Machine %s acquired at %s" % |
| (self.name, |
| machine.name, |
| datetime.datetime.now())) |
| break |
| else: |
| sleep_duration = 10 |
| time.sleep(sleep_duration) |
| return machine |
| |
| def RunTest(self, machine): |
| self.status = STATUS_IMAGING |
| self.machine_manager.ImageMachine(machine, |
| self.chromeos_image, |
| self.board) |
| self.status = "%s %s" % (STATUS_RUNNING, self.autotest_name) |
| [retval, out, err] = self.autotest_runner.Run(machine.name, |
| self.chromeos_root, |
| self.board, |
| self.autotest_name, |
| self.autotest_args, |
| self.profile_counters, |
| self.profile_type) |
| self.run_completed = True |
| |
| # Include the keyvals in the result. |
| self.results_dir = self._GetResultsDir(out) |
| keyvals = self._GetKeyvals() |
| keyvals["retval"] = retval |
| |
| result = Result(out, err, retval, keyvals) |
| |
| return result |
| |
| def _GetKeyvals(self): |
| full_results_dir = os.path.join(self.chromeos_root, |
| "chroot", |
| self.results_dir.lstrip("/")) |
| command = "find %s -regex .*results/keyval$" % full_results_dir |
| [ret, out, err] = self._ce.RunCommand(command, return_output=True) |
| keyvals_dict = {} |
| for f in out.splitlines(): |
| keyvals = open(f, "r").read() |
| keyvals_dict.update(self._ParseKeyvals(keyvals)) |
| |
| return keyvals_dict |
| |
| def _ParseKeyvals(self, keyvals): |
| keyval_dict = {} |
| for l in keyvals.splitlines(): |
| l = l.strip() |
| if l: |
| key, val = l.split("=") |
| keyval_dict[key] = val |
| return keyval_dict |
| |
| def SetCacheConditions(self, cache_conditions): |
| self.cache_conditions = cache_conditions |
