server/site_tests/suites/control.regression

# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

AUTHOR = "Chrome OS Team"
NAME = "Regression"
TIME = "LONG"
TEST_CATEGORY = "Functional"
TEST_CLASS = "suite"
TEST_TYPE = "server"

DOC = """
This test suite runs automated tests that should all pass and that, together
with the control.bvt suite, verify that all major features in the build are
fully functional.  These are meant to run in less than 8 hours.
"""

from autotest_lib.server import site_host_attributes

# List of client tests with parameters
TESTS = [
  ('compilebench', {}),
  ('dbench', {}),
  ('desktopui_FontCache', {}),
  ('desktopui_GTK2Config', {}),
  ('desktopui_IBusTest', {}),
  ('desktopui_ImeTest', {}),
  ('desktopui_SunSpiderBench', {}),
  ('desktopui_V8Bench', {}),
  ('desktopui_WindowManagerHotkeys', {}),
  ('disktest', {}),
  ('fsx', {}),
  ('graphics_GLAPICheck', {}),
  ('graphics_GLBench', {}),
  ('graphics_Piglit', {}),
  ('graphics_SanAngeles', {'creds': '$backdoor'}),
  ('graphics_WindowManagerGraphicsCapture', {}),
  ('hackbench', {}),
  ('hardware_Backlight', {}),
  ('hardware_DiskSize', {}),
  ('hardware_MemoryThroughput', {'num_iteration': 1,
                                 'test_list': '18'}),
  ('hardware_MemoryTotalSize', {}),
  ('hardware_SAT', {'seconds': 20}),
  ('hardware_StorageFio', {}),
  ('login_BadAuthentication', {'tag' : 'not_verified',
                               'error_string' : 'NotVerified'}),
  ('login_BadAuthentication', {'tag' : 'no_terms',
                               'error_string' : 'TermsNotAgreed'}),
  ('login_BadAuthentication', {'tag' : 'account_deleted',
                               'error_string' : 'AccountDeleted'}),
  ('login_BadAuthentication', {'tag' : 'account_disabled',
                               'error_string' : 'AccountDisabled'}),
  ('login_BadAuthentication', {'tag' : 'service_disabled',
                               'error_string' : 'ServiceDisabled'}),
  ('login_BadAuthentication', {'tag' : 'service_unavailable',
                               'error_string' : 'ServiceUnavailable'}),
  ('login_BadAuthentication', {'tag' : 'unknown',
                               'error_string' : 'Unknown'}),
  ('login_OwnershipApi', {}),
  ('login_DBusCalls', {}),
# TODO(kmixter): Fix log volume usage, or change test constraints. See http://crosbug.com/11350
#  ('logging_LogVolume', {'constraints': ['bytes_unexpected <= 100000',
#                                         'files_unexpected <= 50',
#                                         'percent_stateful_used <= 50',
#                                         'percent_unused_patterns <= 70']}),
#
# TODO: Until ltp can pass on chromiumos
#  ('ltp', {}),
# TODO(stanlyew): Re-open network_ConnmanCromoCrash when crosbug.com/13272
#                 is fixed.
#  ('network_ConnmanCromoCrash', {}),
  ('network_UdevRename', {}),
  ('platform_AccurateTime', {'constraints': ['seconds_offset < 60']}),
  ('platform_CleanShutdown', {}),
  ('platform_DaemonsRespawn', {}),
  ('platform_FileNum', {}),
  ('platform_FileSize', {}),
  ('platform_KernelVersion', {}),
  ('platform_MemCheck', {}),
  ('platform_MiniJailCmdLine', {}),
  ('platform_MiniJailPidNamespace', {}),
  ('platform_MiniJailRootCapabilities', {}),
  ('platform_MiniJailVfsNamespace', {}),
  ('platform_ProcessPrivileges', {}),
  ('platform_Rootdev', {}),
  ('platform_SessionManagerTerm', {}),
  ('platform_StackProtector', {}),
  ('platform_TempFS', {}),
# TODO(dalecurtis): AudioBench does not work; see http://crosbug.com/12647
#  ('realtimecomm_GTalkAudioBench', {}),
  ('realtimecomm_GTalkunittest', {}),
  ('unixbench', {}),
]

# A dict which holds {machine: [(test, argv), (test, argv), ...]}
test_dict = {}

# Assign tests with its argv to machines in a round-robin way.
index = 0
for test_with_argv in TESTS:
  m_index = index % len(machines)
  test_list = test_dict.setdefault(machines[m_index], [])
  test_list.append(test_with_argv)
  index += 1


def run_assigned_tests(machine):
  client = hosts.create_host(machine)
  client_at = autotest.Autotest(client)
  test_list = test_dict[machine]
  for test, argv in test_list:
    client_at.run_test(test, **argv)

  client_attributes = site_host_attributes.HostAttributes(machine)
  if client_attributes.has_ssd:
    client_at.run_test('hardware_SsdDetection')
  if client_attributes.has_bluetooth:
    client_at.run_test('network_DisableInterface',
                       iface_name='hci0', tag='hci0')
  if client_attributes.has_80211n:
    client_at.run_test('network_WiFiCaps')
  if not client_attributes.has_resume_bug:
    client_at.run_test('power_Resume')

  job.run_test('platform_KernelErrorPaths', host=client)
  if client_attributes.has_chromeos_firmware:
    job.run_test('logging_KernelCrashServer', host=client)

  # This test stops tcsd which is known to cause issues with cryptohome and
  # possibly other services. Specifically, cros_ui_test based tests will fail;
  # so always run it last. See http://crosbug.com/14265
  if client_attributes.has_chromeos_firmware:
    client_at.run_test('hardware_TPMCheck')


# Run server-side suites.
job.run_control('server/site_tests/suites/control.security')

# Some of the tests in the main regression suite must be run last to prevent
# cross-test issues. Always add new suites above this line.
job.parallel_on_machines(run_assigned_tests, machines)