[moblab] create check for lxc base container

Mobmonitor check that attempts to clone and start a new image
off of the base container. If it's unable to do this, tests
that rely on the base container will also likely not be able
to. Give the ability to repair the base container on a failed
mobmonitor check by forced redownloading it.

Refactored config reading that is duplicated in several places
into a util/config module.

BUG=chromium:837463
TEST=base_container_check_unittest.py cloud_storage_speedtest_unittest.py
	config_unittest.py

Change-Id: I4c2b2b453b24a8d1a0372f9b9613c44e962597f9
Reviewed-on: https://chromium-review.googlesource.com/1062800
Commit-Ready: Matt Mallett <mattmallett@chromium.org>
Tested-by: Matt Mallett <mattmallett@chromium.org>
Reviewed-by: Keith Haddow <haddowk@chromium.org>
diff --git a/src/mobmonitor/README b/src/mobmonitor/README
index 86e5ff4..6f08f13 100644
--- a/src/mobmonitor/README
+++ b/src/mobmonitor/README
@@ -86,8 +86,8 @@
 
 Health checks can (optionally) also define the following attributes:
 
-  - CHECK_INTERVAL: Defines the interval (in seconds) between health check
-                    executions. This defaults to 30 seconds if not defined.
+  - CHECK_INTERVAL_SEC: Defines the interval (in seconds) between health check
+                    executions. This defaults to 10 seconds if not defined.
 
 
 A check file may contain as many health checks as the writer feels is
diff --git a/src/mobmonitor/checkfiles/__init__.py b/src/mobmonitor/checkfiles/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/mobmonitor/checkfiles/__init__.py
diff --git a/src/mobmonitor/checkfiles/devserver/gs_check.py b/src/mobmonitor/checkfiles/devserver/gs_check.py
index 495e1a5..fc04ea7 100644
--- a/src/mobmonitor/checkfiles/devserver/gs_check.py
+++ b/src/mobmonitor/checkfiles/devserver/gs_check.py
@@ -8,15 +8,11 @@
 
 import netifaces
 import os
-import ConfigParser
 
 from util import osutils
+from util import config
 
 
-GLOBAL_CONFIG = '/usr/local/autotest/global_config.ini'
-MOBLAB_CONFIG = '/usr/local/autotest/moblab_config.ini'
-SHADOW_CONFIG = '/usr/local/autotest/shadow_config.ini'
-
 GSUTIL_TIMEOUT_SEC = 5
 GSUTIL_USER = 'moblab'
 GSUTIL_INTERNAL_BUCKETS = ['gs://chromeos-image-archive']
@@ -84,21 +80,8 @@
       -2 if the configuration files did not contain the appropriate
         information.
     """
-    config_files = [GLOBAL_CONFIG, MOBLAB_CONFIG, SHADOW_CONFIG]
-    for f in config_files:
-      if not os.path.exists(f):
-        return -1
 
-    config = ConfigParser.ConfigParser()
-    config.read(config_files)
-
-    gs_url = None
-
-    for section in config.sections():
-      for option, value in config.items(section):
-        if 'image_storage_server' == option:
-          gs_url = value
-          break
+    gs_url = config.Config().get('image_storage_server')
 
     if not (gs_url and self.BucketReachable(gs_url)):
       self.bucket = gs_url
diff --git a/src/mobmonitor/checkfiles/moblab/base_container_check.py b/src/mobmonitor/checkfiles/moblab/base_container_check.py
new file mode 100644
index 0000000..05962ae
--- /dev/null
+++ b/src/mobmonitor/checkfiles/moblab/base_container_check.py
@@ -0,0 +1,52 @@
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""LXC base container checks."""
+
+from __future__ import print_function
+
+import common
+import moblab_actions
+
+from util import osutils
+from util import config
+
+class BASE_CONTAINER_CODES:
+  OK = 0
+  BASE_CONTAINER_MISSING = -1
+
+class BaseContainerCheck(object):
+  """Verifies that the base container exists."""
+
+  CONTAINER_PATH = '/mnt/moblab/containers'
+
+  def Check(self):
+    """Verifies that moblab has a properly configured base lxc container
+
+    Returns:
+      BASE_CONTAINER_CODES
+        OK if the base container is ok
+        BASE_CONTAINER_MISSING if the container files are not present on moblab
+    """
+    container_base_name = config.Config().get('container_base_name')
+
+    try:
+      # Check that the base container is present at all in the filesystem
+      check_container_path = '%s/%s' % (self.CONTAINER_PATH,
+          container_base_name)
+      check_container_rootfs = '%s/rootfs' % check_container_path
+      osutils.run_command(['test', '-e', check_container_path])
+      osutils.run_command(['test', '-e', check_container_rootfs])
+    except osutils.RunCommandError:
+      return BASE_CONTAINER_CODES.BASE_CONTAINER_MISSING
+
+    return BASE_CONTAINER_CODES.OK
+
+
+  def Diagnose(self, errcode):
+    msg = ('Base lxc container is missing. This container is required to run'
+            ' most tests. Please redownload and reinstall the base lxc'
+            ' container (about 400MB)')
+    if BASE_CONTAINER_CODES.BASE_CONTAINER_MISSING == errcode:
+      return (msg, [moblab_actions.RepairBaseContainer()])
diff --git a/src/mobmonitor/checkfiles/moblab/base_container_check_unittest.py b/src/mobmonitor/checkfiles/moblab/base_container_check_unittest.py
new file mode 100644
index 0000000..2daef71
--- /dev/null
+++ b/src/mobmonitor/checkfiles/moblab/base_container_check_unittest.py
@@ -0,0 +1,49 @@
+# Copyright 2015 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+import mock
+import sys
+
+sys.path.append('../..')
+
+import base_container_check
+from util import osutils
+
+class TestBaseContainerCheck(unittest.TestCase):
+
+    def setUp(self):
+        config_patch = mock.patch('base_container_check.config')
+        config_patch.start()
+        config_mock = mock.MagicMock()
+        config_mock.get.return_value = 'test_base_container'
+        base_container_check.config.Config.return_value = config_mock
+
+        osutils_patch = mock.patch('base_container_check.osutils')
+        osutils_patch.start()
+        base_container_check.osutils.RunCommandError = osutils.RunCommandError
+
+    def test_check(self):
+        check = base_container_check.BaseContainerCheck()
+        self.assertEquals(base_container_check.BASE_CONTAINER_CODES.OK,
+            check.Check())
+
+    def test_check_missing(self):
+        base_container_check.osutils.run_command.side_effect = \
+            osutils.RunCommandError(1, 'error')
+        check = base_container_check.BaseContainerCheck()
+        self.assertEquals(
+            base_container_check.BASE_CONTAINER_CODES.BASE_CONTAINER_MISSING,
+            check.Check())
+
+    def test_diagnose(self):
+        check = base_container_check.BaseContainerCheck()
+
+        diagnosis_missing = check.Diagnose(
+            base_container_check.BASE_CONTAINER_CODES.BASE_CONTAINER_MISSING)
+        self.assertTrue('missing' in diagnosis_missing[0])
+
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file
diff --git a/src/mobmonitor/checkfiles/moblab/moblab_actions.py b/src/mobmonitor/checkfiles/moblab/moblab_actions.py
index 21a0de2..9b471bc 100644
--- a/src/mobmonitor/checkfiles/moblab/moblab_actions.py
+++ b/src/mobmonitor/checkfiles/moblab/moblab_actions.py
@@ -94,3 +94,16 @@
     def run(self, params):
         cmd = ['start', self.service]
         osutils.sudo_run_command(cmd)
+
+
+class RepairBaseContainer(AbstractAction):
+    action = 'Repair Base Container'
+    info = """Redownload and reinstall the base lxc container (about 400MB)"""
+    param_info = {}
+
+    def run(self, params):
+        rebuild_container = \
+                ['python', '/usr/local/autotest/site_utils/lxc.py', '-s', '-f']
+        osutils.sudo_run_command(rebuild_container)
+        init = ['start', 'moblab-base-container-init']
+        osutils.sudo_run_command(init)
\ No newline at end of file
diff --git a/src/mobmonitor/diagnostic_checks/base_container_clone.py b/src/mobmonitor/diagnostic_checks/base_container_clone.py
new file mode 100644
index 0000000..d566f4c
--- /dev/null
+++ b/src/mobmonitor/diagnostic_checks/base_container_clone.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import sys
+
+sys.path.append('..')
+from abstract_diagnostic_check import AbstractDiagnosticCheck
+from diagnostic_error import DiagnosticError
+from util import osutils
+from util import config
+
+class BaseContainerClone(AbstractDiagnosticCheck):
+    """
+    Attempt to clone and start a new container from the base
+    container
+    """
+
+    CONTAINER_PATH = '/mnt/moblab/containers'
+
+    category = 'lxc'
+
+    name = 'Base Container Clone'
+
+    description = ('Test that the base lxc container is valid and '
+        'able to be cloned')
+
+    def _run_command(self, cmd):
+        print_cmd = ' '.join(cmd)
+        self.output += print_cmd + '\n'
+        cmd_out = osutils.sudo_run_command(cmd)
+        self.output += cmd_out
+        self.output += 'OK\n\n'
+
+
+    def run(self):
+        self.output = ''
+        container_base_name = config.Config().get('container_base_name')
+        try:
+            clone_cmd = ['lxc-copy',
+                '-P', self.CONTAINER_PATH, # note uppercase P and lowercase p
+                '-p', self.CONTAINER_PATH,
+                '-n', container_base_name,
+                '-N', 'base_check']
+
+            start_cmd = ['lxc-start',
+                '-P', self.CONTAINER_PATH,
+                '-n', 'base_check',
+                '-d']
+
+            stop_cmd = ['lxc-stop',
+                '-P', self.CONTAINER_PATH,
+                '-n', 'base_check']
+
+            self._run_command(clone_cmd)
+            self._run_command(start_cmd)
+            self._run_command(stop_cmd)
+
+        except osutils.RunCommandError as e:
+            self.output += str(e) + '\n\n'
+        finally:
+            try:
+                destroy_cmd = ['lxc-destroy',
+                '-P', self.CONTAINER_PATH,
+                '-n', 'base_check',
+                '--force']
+                self._run_command(destroy_cmd)
+            except osutils.RunCommandError as e:
+                self.output += str(e) + '\n\n'
+
+        return self.output
diff --git a/src/mobmonitor/diagnostic_checks/base_container_clone_unittest.py b/src/mobmonitor/diagnostic_checks/base_container_clone_unittest.py
new file mode 100644
index 0000000..f99aea8
--- /dev/null
+++ b/src/mobmonitor/diagnostic_checks/base_container_clone_unittest.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unittests for the base container clone diagnostic."""
+
+import unittest
+import mock
+import sys
+
+sys.path.append('..')
+from util import osutils
+
+import base_container_clone
+import diagnostic_error
+
+class TestBaseContainerClone(unittest.TestCase):
+
+    def setUp(self):
+        os_patch = mock.patch('base_container_clone.osutils')
+        os_patch.start()
+        base_container_clone.osutils.sudo_run_command.return_value = ''
+        base_container_clone.osutils.RunCommandError = osutils.RunCommandError
+
+        config_patch = mock.patch('base_container_clone.config')
+        config_patch.start()
+        config_mock = mock.MagicMock()
+        config_mock.get.return_value = 'test'
+        base_container_clone.config.Config.return_value = config_mock
+
+    def testRun(self):
+        cloner = base_container_clone.BaseContainerClone()
+        expect_list = [
+            'lxc-copy',
+            'lxc-start',
+            'lxc-stop',
+            'lxc-destroy'
+        ]
+        result = cloner.run()
+        for expect in expect_list:
+            self.assertIn(expect, result)
+
+    def testRunError(self):
+        base_container_clone.osutils.sudo_run_command.side_effect = \
+            osutils.RunCommandError(1, 'error')
+        cloner = base_container_clone.BaseContainerClone()
+
+        # expect first command to fail, but finally clause to execute
+        expect_list = [
+            'lxc-copy',
+            'lxc-destroy'
+        ]
+        not_expect_list = [
+            'lxc-start',
+            'lxc-stop'
+        ]
+
+        result = cloner.run()
+        for expect in expect_list:
+            self.assertIn(expect, result)
+        for not_expect in not_expect_list:
+            self.assertNotIn(not_expect, result)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/src/mobmonitor/diagnostic_checks/cloud_storage_speedtest.py b/src/mobmonitor/diagnostic_checks/cloud_storage_speedtest.py
index 7b57df6..6ddf396 100644
--- a/src/mobmonitor/diagnostic_checks/cloud_storage_speedtest.py
+++ b/src/mobmonitor/diagnostic_checks/cloud_storage_speedtest.py
@@ -3,7 +3,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import ConfigParser
 import json
 import sys
 
@@ -11,6 +10,7 @@
 from abstract_diagnostic_check import AbstractDiagnosticCheck
 from diagnostic_error import DiagnosticError
 from util import osutils
+from util import config
 
 class CloudStorageSpeedTest(AbstractDiagnosticCheck):
     """
@@ -18,9 +18,6 @@
     moblab and the configured gs bucket
     """
 
-    GLOBAL_CONFIG = '/usr/local/autotest/global_config.ini'
-    MOBLAB_CONFIG = '/usr/local/autotest/moblab_config.ini'
-    SHADOW_CONFIG = '/usr/local/autotest/shadow_config.ini'
     GSUTIL_USER = 'moblab'
 
     category = 'Cloud Storage'
@@ -30,19 +27,6 @@
     description = ('Test the speed of the connection between the moblab and the'
     ' cloud storage bucket')
 
-    def _get_bucket_url(self):
-        # load the bucket location from moblab config
-        # TODO haddowk/mattmallett refactor this to use the "autotest"
-        #   interface when that work is completed
-        config = ConfigParser.ConfigParser()
-        config.read(
-            [self.GLOBAL_CONFIG, self.MOBLAB_CONFIG, self.SHADOW_CONFIG])
-        for section in config.sections():
-            for option, value in config.items(section):
-                if 'image_storage_server' == option:
-                    return value
-        return None
-
     def _process_results(self, data):
         # search through the results json file to get the throughput numbers
         json_data = json.loads(data)
@@ -59,7 +43,7 @@
 
 
     def run(self):
-        bucket_url = self._get_bucket_url()
+        bucket_url = config.Config().get('image_storage_server')
         if bucket_url is None:
             raise DiagnosticError('Bucket URL is not configured')
 
diff --git a/src/mobmonitor/diagnostic_checks/cloud_storage_speedtest_unittest.py b/src/mobmonitor/diagnostic_checks/cloud_storage_speedtest_unittest.py
index 4092094..89b639e 100644
--- a/src/mobmonitor/diagnostic_checks/cloud_storage_speedtest_unittest.py
+++ b/src/mobmonitor/diagnostic_checks/cloud_storage_speedtest_unittest.py
@@ -13,30 +13,6 @@
 
 class TestCloudStorageSpeedTest(unittest.TestCase):
 
-    def setUp(self):
-        config_patch = mock.patch('cloud_storage_speedtest.ConfigParser')
-        config_patch.start()
-        config_mock = mock.MagicMock()
-        config_mock.sections.return_value = [1]
-        config_mock.items.return_value = \
-                [('image_storage_server', 'gs://bucket/')]
-        cloud_storage_speedtest.ConfigParser.ConfigParser.return_value = \
-                config_mock
-
-        self.config_mock = config_mock
-
-    def testGetBucketUrl(self):
-        expect = 'gs://bucket/'
-        speed_test = cloud_storage_speedtest.CloudStorageSpeedTest()
-        result = speed_test._get_bucket_url()
-        self.assertEqual(expect, result)
-
-    def testMissingGetBucketUrl(self):
-        self.config_mock.items.return_value = [('something', 'not bucket')]
-        speed_test = cloud_storage_speedtest.CloudStorageSpeedTest()
-        result = speed_test._get_bucket_url()
-        self.assertEqual(None, result)
-
     def testProcessResults(self):
         # test 10mbps down 5mbps up
         data = """{
diff --git a/src/mobmonitor/diagnostic_checks/disk_info.py b/src/mobmonitor/diagnostic_checks/disk_info.py
index 0be8429..cbd9b12 100644
--- a/src/mobmonitor/diagnostic_checks/disk_info.py
+++ b/src/mobmonitor/diagnostic_checks/disk_info.py
@@ -3,8 +3,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import ConfigParser
-import json
 import sys
 
 sys.path.append('..')
diff --git a/src/mobmonitor/diagnostic_checks/manager.py b/src/mobmonitor/diagnostic_checks/manager.py
index 65dc3b1..29fb468 100644
--- a/src/mobmonitor/diagnostic_checks/manager.py
+++ b/src/mobmonitor/diagnostic_checks/manager.py
@@ -3,8 +3,13 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import logging
+LOGGER = logging.getLogger(__name__)
+
 from cloud_storage_speedtest import CloudStorageSpeedTest
 from disk_info import DiskInfo
+from base_container_clone import BaseContainerClone
+from repair_base_container import RepairBaseContainer
 from diagnostic_error import DiagnosticError
 
 class DiagnosticCheckManager:
@@ -27,7 +32,9 @@
     # All checks must inherit from AbstractDiagnosticCheck
     CHECKS = [
         CloudStorageSpeedTest,
-        DiskInfo
+        DiskInfo,
+        BaseContainerClone,
+        RepairBaseContainer
     ]
 
     def __init__(self):
@@ -94,9 +101,13 @@
             DiagnosticError if the check is not found, or encounters an error
         """
         try:
+            LOGGER.info('running diagnostic %s %s' % (category, name))
             check = self.diagnostic_checks[category][name]
             if check is None:
                 raise DiagnosticError('Check not found')
             return check.run()
         except KeyError:
             raise DiagnosticError('Check not found')
+        except DiagnosticError as e:
+            LOGGER.error('failed to run diagnostic check: %s', str(e))
+            raise e
diff --git a/src/mobmonitor/diagnostic_checks/repair_base_container.py b/src/mobmonitor/diagnostic_checks/repair_base_container.py
new file mode 100644
index 0000000..8acded8
--- /dev/null
+++ b/src/mobmonitor/diagnostic_checks/repair_base_container.py
@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import sys
+
+sys.path.append('..')
+from abstract_diagnostic_check import AbstractDiagnosticCheck
+from diagnostic_error import DiagnosticError
+from util import osutils
+
+from checkfiles.moblab import moblab_actions
+
+class RepairBaseContainer(AbstractDiagnosticCheck):
+    """
+    Attempt to redownload the base lxc container
+    """
+
+    category = 'lxc'
+
+    def __init__(self):
+        self.action = moblab_actions.RepairBaseContainer()
+        self.name = self.action.action
+        self.description = self.action.info
+
+    def run(self):
+        try:
+            self.action.run({})
+            return 'base lxc container repaired successfully'
+        except osutils.RunCommandError as e:
+            raise DiagnosticError('Failed repair on ' + str(e))
+
diff --git a/src/mobmonitor/diagnostic_checks/repair_base_container_unittest.py b/src/mobmonitor/diagnostic_checks/repair_base_container_unittest.py
new file mode 100644
index 0000000..ec4bc5a
--- /dev/null
+++ b/src/mobmonitor/diagnostic_checks/repair_base_container_unittest.py
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unittests for the repair base container diagnostic."""
+
+import unittest
+import mock
+import sys
+
+sys.path.append('..')
+from util import osutils
+
+import repair_base_container
+import diagnostic_error
+
+class TestRepairBaseContainer(unittest.TestCase):
+
+    def setUp(self):
+        action_patch = mock.patch('repair_base_container.moblab_actions')
+        action_patch.start()
+
+    def testRun(self):
+        repair = repair_base_container.RepairBaseContainer()
+        expect = 'base lxc container repaired successfully'
+        self.assertEqual(expect, repair.run())
+
+    def testRunError(self):
+        repair = repair_base_container.RepairBaseContainer()
+        repair.action.run.side_effect = osutils.RunCommandError(1, 'error')
+
+        with self.assertRaises(diagnostic_error.DiagnosticError):
+            repair.run()
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/src/mobmonitor/util/config.py b/src/mobmonitor/util/config.py
new file mode 100644
index 0000000..a225c6d
--- /dev/null
+++ b/src/mobmonitor/util/config.py
@@ -0,0 +1,40 @@
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import ConfigParser
+
+class Config:
+    """
+    Provide an interface to read and get config values from
+    moblab
+    """
+
+    GLOBAL_CONFIG = '/usr/local/autotest/global_config.ini'
+    MOBLAB_CONFIG = '/usr/local/autotest/moblab_config.ini'
+    SHADOW_CONFIG = '/usr/local/autotest/shadow_config.ini'
+
+    def __init__(self):
+        self.config = ConfigParser.ConfigParser()
+        self.config.read(
+            [self.GLOBAL_CONFIG, self.MOBLAB_CONFIG, self.SHADOW_CONFIG])
+
+    def get(self, key):
+        """
+        Get the specified config value
+
+        Args:
+            key: string which config value to get
+
+        Return:
+            string containing the config value, or None if no value is set
+        """
+        # TODO haddowk/mattmallett refactor this to use the "autotest"
+        #   interface when that work is completed
+        for section in self.config.sections():
+            for option, value in self.config.items(section):
+                if key == option:
+                    return value
+        return None
+
+
diff --git a/src/mobmonitor/util/config_unittest.py b/src/mobmonitor/util/config_unittest.py
new file mode 100644
index 0000000..788c78a
--- /dev/null
+++ b/src/mobmonitor/util/config_unittest.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+import mock
+
+import config
+
+class TestConfig(unittest.TestCase):
+
+    def setUp(self):
+        config_patch = mock.patch('config.ConfigParser')
+        config_patch.start()
+        config_mock = mock.MagicMock()
+        config_mock.sections.return_value = [1]
+        config_mock.items.return_value = \
+            [('testkey', 'testval'), ('testkey2', 'testval2')]
+        config.ConfigParser.ConfigParser.return_value = config_mock
+
+
+    def test_config_get(self):
+        test_config = config.Config()
+        self.assertEqual('testval', test_config.get('testkey'))
+        self.assertEqual('testval2', test_config.get('testkey2'))
+
+    def test_config_get_missing(self):
+        test_config = config.Config()
+        self.assertIsNone(test_config.get('testmissingkey'))
+
+
+if __name__ == '__main__':
+    unittest.main()