| # Lint as: python2, python3 |
| from __future__ import absolute_import |
| from __future__ import division |
| from __future__ import print_function |
| import os, re, db, sys, datetime |
| import common |
| from autotest_lib.client.common_lib import kernel_versions |
| from six.moves import map |
| |
| MAX_RECORDS = 50000 |
| MAX_CELLS = 500000 |
| |
| tko = os.path.dirname(os.path.realpath(os.path.abspath(__file__))) |
| root_url_file = os.path.join(tko, '.root_url') |
| if os.path.exists(root_url_file): |
| html_root = open(root_url_file, 'r').readline().rstrip() |
| else: |
| html_root = '/results/' |
| |
| |
| class status_cell: |
| # One cell in the matrix of status data. |
| def __init__(self): |
| # Count is a dictionary: status -> count of tests with status |
| self.status_count = {} |
| self.reasons_list = [] |
| self.job_tag = None |
| self.job_tag_count = 0 |
| |
| |
| def add(self, status, count, job_tags, reasons = None): |
| assert count > 0 |
| |
| self.job_tag = job_tags |
| self.job_tag_count += count |
| if self.job_tag_count > 1: |
| self.job_tag = None |
| |
| self.status_count[status] = count |
| ### status == 6 means 'GOOD' |
| if status != 6: |
| ## None implies sorting problems and extra CRs in a cell |
| if reasons: |
| self.reasons_list.append(reasons) |
| |
| |
| class status_data: |
| def __init__(self, sql_rows, x_field, y_field, query_reasons = False): |
| data = {} |
| y_values = set() |
| |
| # Walk through the query, filing all results by x, y info |
| for row in sql_rows: |
| if query_reasons: |
| (x,y, status, count, job_tags, reasons) = row |
| else: |
| (x,y, status, count, job_tags) = row |
| reasons = None |
| if x not in data: |
| data[x] = {} |
| if y not in data[x]: |
| y_values.add(y) |
| data[x][y] = status_cell() |
| data[x][y].add(status, count, job_tags, reasons) |
| |
| # 2-d hash of data - [x-value][y-value] |
| self.data = data |
| # List of possible columns (x-values) |
| self.x_values = smart_sort(list(data.keys()), x_field) |
| # List of rows columns (y-values) |
| self.y_values = smart_sort(list(y_values), y_field) |
| nCells = len(self.y_values)*len(self.x_values) |
| if nCells > MAX_CELLS: |
| msg = 'Exceeded allowed number of cells in a table' |
| raise db.MySQLTooManyRows(msg) |
| |
| |
| def get_matrix_data(db_obj, x_axis, y_axis, where = None, |
| query_reasons = False): |
| # Searches on the test_view table - x_axis and y_axis must both be |
| # column names in that table. |
| x_field = test_view_field_dict[x_axis] |
| y_field = test_view_field_dict[y_axis] |
| query_fields_list = [x_field, y_field, 'status','COUNT(status)'] |
| query_fields_list.append("LEFT(GROUP_CONCAT(job_tag),100)") |
| if query_reasons: |
| query_fields_list.append( |
| "LEFT(GROUP_CONCAT(DISTINCT reason SEPARATOR '|'),500)" |
| ) |
| fields = ','.join(query_fields_list) |
| |
| group_by = '%s, %s, status' % (x_field, y_field) |
| rows = db_obj.select(fields, 'tko_test_view', |
| where=where, group_by=group_by, max_rows = MAX_RECORDS) |
| return status_data(rows, x_field, y_field, query_reasons) |
| |
| |
| # Dictionary used simply for fast lookups from short reference names for users |
| # to fieldnames in test_view |
| test_view_field_dict = { |
| 'kernel' : 'kernel_printable', |
| 'hostname' : 'machine_hostname', |
| 'test' : 'test', |
| 'label' : 'job_label', |
| 'machine_group' : 'machine_group', |
| 'reason' : 'reason', |
| 'tag' : 'job_tag', |
| 'user' : 'job_username', |
| 'status' : 'status_word', |
| 'time' : 'test_finished_time', |
| 'start_time' : 'test_started_time', |
| 'time_daily' : 'DATE(test_finished_time)' |
| } |
| |
| |
| def smart_sort(list, field): |
| if field == 'kernel_printable': |
| def kernel_encode(kernel): |
| return kernel_versions.version_encode(kernel) |
| list.sort(key = kernel_encode, reverse = True) |
| return list |
| ## old records may contain time=None |
| ## make None comparable with timestamp datetime or date |
| elif field == 'test_finished_time': |
| def convert_None_to_datetime(date_time): |
| if not date_time: |
| return datetime.datetime(1970, 1, 1, 0, 0, 0) |
| else: |
| return date_time |
| list = list(map(convert_None_to_datetime, list)) |
| elif field == 'DATE(test_finished_time)': |
| def convert_None_to_date(date): |
| if not date: |
| return datetime.date(1970, 1, 1) |
| else: |
| return date |
| list = list(map(convert_None_to_date, list)) |
| list.sort() |
| return list |
| |
| |
| class group: |
| @classmethod |
| def select(klass, db): |
| """Return all possible machine groups""" |
| rows = db.select('distinct machine_group', 'tko_machines', |
| 'machine_group is not null') |
| groupnames = sorted([row[0] for row in rows]) |
| return [klass(db, groupname) for groupname in groupnames] |
| |
| |
| def __init__(self, db, name): |
| self.name = name |
| self.db = db |
| |
| |
| def machines(self): |
| return machine.select(self.db, { 'machine_group' : self.name }) |
| |
| |
| def tests(self, where = {}): |
| values = [self.name] |
| sql = 't inner join tko_machines m on m.machine_idx=t.machine_idx' |
| sql += ' where m.machine_group=%s' |
| for key in where.keys(): |
| sql += ' and %s=%%s' % key |
| values.append(where[key]) |
| return test.select_sql(self.db, sql, values) |
| |
| |
| class machine: |
| @classmethod |
| def select(klass, db, where = {}): |
| fields = ['machine_idx', 'hostname', 'machine_group', 'owner'] |
| machines = [] |
| for row in db.select(','.join(fields), 'tko_machines', where): |
| machines.append(klass(db, *row)) |
| return machines |
| |
| |
| def __init__(self, db, idx, hostname, group, owner): |
| self.db = db |
| self.idx = idx |
| self.hostname = hostname |
| self.group = group |
| self.owner = owner |
| |
| |
| class kernel: |
| @classmethod |
| def select(klass, db, where = {}): |
| fields = ['kernel_idx', 'kernel_hash', 'base', 'printable'] |
| rows = db.select(','.join(fields), 'tko_kernels', where) |
| return [klass(db, *row) for row in rows] |
| |
| |
| def __init__(self, db, idx, hash, base, printable): |
| self.db = db |
| self.idx = idx |
| self.hash = hash |
| self.base = base |
| self.printable = printable |
| self.patches = [] # THIS SHOULD PULL IN PATCHES! |
| |
| |
| class test: |
| @classmethod |
| def select(klass, db, where={}, distinct=False): |
| fields = ['test_idx', 'job_idx', 'test', 'subdir', |
| 'kernel_idx', 'status', 'reason', 'machine_idx'] |
| tests = [] |
| for row in db.select(','.join(fields), 'tko_tests', where, |
| distinct): |
| tests.append(klass(db, *row)) |
| return tests |
| |
| |
| @classmethod |
| def select_sql(klass, db, sql, values): |
| fields = ['test_idx', 'job_idx', 'test', 'subdir', |
| 'kernel_idx', 'status', 'reason', 'machine_idx'] |
| fields = ['t.'+field for field in fields] |
| rows = db.select_sql(','.join(fields), 'tko_tests', sql, values) |
| return [klass(db, *row) for row in rows] |
| |
| |
| def __init__(self, db, test_idx, job_idx, testname, subdir, kernel_idx, |
| status_num, reason, machine_idx): |
| self.idx = test_idx |
| self.job = job(db, job_idx) |
| self.testname = testname |
| self.subdir = subdir |
| self.kernel_idx = kernel_idx |
| self.__kernel = None |
| self.__iterations = None |
| self.machine_idx = machine_idx |
| self.__machine = None |
| self.status_num = status_num |
| self.status_word = db.status_word[status_num] |
| self.reason = reason |
| self.db = db |
| if self.subdir: |
| self.url = html_root + self.job.tag + '/' + self.subdir |
| else: |
| self.url = None |
| |
| |
| def iterations(self): |
| """ |
| Caching function for iterations |
| """ |
| if not self.__iterations: |
| self.__iterations = {} |
| # A dictionary - dict{key} = [value1, value2, ....] |
| where = {'test_idx' : self.idx} |
| for i in iteration.select(self.db, where): |
| if i.key in self.__iterations: |
| self.__iterations[i.key].append(i.value) |
| else: |
| self.__iterations[i.key] = [i.value] |
| return self.__iterations |
| |
| |
| def kernel(self): |
| """ |
| Caching function for kernels |
| """ |
| if not self.__kernel: |
| where = {'kernel_idx' : self.kernel_idx} |
| self.__kernel = kernel.select(self.db, where)[0] |
| return self.__kernel |
| |
| |
| def machine(self): |
| """ |
| Caching function for kernels |
| """ |
| if not self.__machine: |
| where = {'machine_idx' : self.machine_idx} |
| self.__machine = machine.select(self.db, where)[0] |
| return self.__machine |
| |
| |
| class job: |
| def __init__(self, db, job_idx): |
| where = {'job_idx' : job_idx} |
| rows = db.select('tag, machine_idx', 'tko_jobs', where) |
| if rows: |
| self.tag, self.machine_idx = rows[0] |
| self.job_idx = job_idx |
| |
| |
| class iteration: |
| @classmethod |
| def select(klass, db, where): |
| fields = ['iteration', 'attribute', 'value'] |
| iterations = [] |
| rows = db.select(','.join(fields), 'tko_iteration_result', where) |
| for row in rows: |
| iterations.append(klass(*row)) |
| return iterations |
| |
| |
| def __init__(self, iteration, key, value): |
| self.iteration = iteration |
| self.key = key |
| self.value = value |
| |
| # class patch: |
| # def __init__(self): |
| # self.spec = None |