factory-archiver: handle the error cases of tar command
Implement the logic for error cases of tar command.
In addition,
- fix and improve the clean-up and in the unittest.
- add "gpg -k" to trigger the first initialization.
BUG=chrome-os-partner:27567
TEST=make lint LINT_WHITELIST=\
"py/lumberjack/archiver.py \
py/lumberjack/archiver_cli.py \
py/lumberjack/archiver_config.py \
py/lumberjack/archiver_exception.py \
py/lumberjack/common.py \
py/lumberjack/archiver_unittest.py "
TEST=make test
Change-Id: I61069c7e7115f78f59991e55417ab269c229be0a
Reviewed-on: https://chromium-review.googlesource.com/198160
Tested-by: Chun-ta Lin <itspeter@chromium.org>
Reviewed-by: Ricky Liang <jcliang@chromium.org>
Commit-Queue: Chun-ta Lin <itspeter@chromium.org>
diff --git a/py/lumberjack/archiver.py b/py/lumberjack/archiver.py
index 803105f..266fc75 100644
--- a/py/lumberjack/archiver.py
+++ b/py/lumberjack/archiver.py
@@ -8,6 +8,7 @@
import hashlib
import logging
import os
+import pprint
import re
import shutil
import tempfile
@@ -15,6 +16,7 @@
import uuid
import yaml
+from archiver_exception import ArchiverFieldError
from subprocess import PIPE, Popen
from twisted.internet import reactor
@@ -28,6 +30,10 @@
'archiver_cli.py', 'archiver_config.py']
# Global variable to keep locked file open during process life-cycle
locks = []
+# Global variable to postpone the time (i.e. give few more archiving cycle
+# a chance to recover automatically) of raising exception
+archive_failures = []
+MAX_ALOOWED_FAILURES = 5
def _ListEligibleFiles(dir_path):
@@ -401,10 +407,16 @@
config.archived_dir, _GenerateArchiveName(config))
tmp_archive = generated_archive + '.part'
logging.info('Compressing %r into %r', tmp_dir, tmp_archive)
- # TODO(itspeter): Handle the failure cases.
- output_tuple = Popen( # pylint: disable=W0612
- ['tar', '-cvJf', tmp_archive, '-C', tmp_dir, '.'],
- stdout=PIPE, stderr=PIPE).communicate()
+ cmd_line = ['tar', '-cvJf', tmp_archive, '-C', tmp_dir, '.']
+ p = Popen(cmd_line, stdout=PIPE, stderr=PIPE)
+ stdout, stderr = p.communicate()
+ if p.returncode != 0:
+ error_msg = ('Command %r failed. retcode[%r]\nstdout:\n%s\n\n'
+ 'stderr:\n%s\n' % (cmd_line, p.returncode, stdout, stderr))
+ logging.error(error_msg)
+ archive_failures.append(error_msg)
+ raise ArchiverFieldError(error_msg)
+
# Remove .part suffix.
os.rename(tmp_archive, generated_archive)
return generated_archive
@@ -431,9 +443,19 @@
if len(archive_metadata['files']) > 0:
_UpdateMetadata(archive_metadata, started_time) # Write other info
# TODO(itspeter): Handle the .zip compress_format.
- generated_archive = _CompressIntoTarXz(tmp_dir, archive_metadata, config)
- # Update metadata data for archived files.
- _UpdateArchiverMetadata(archive_metadata['files'], config)
+ try:
+ generated_archive = _CompressIntoTarXz(tmp_dir,
+ archive_metadata, config)
+ # Update metadata data for archived files only when compression
+ # succeed.
+ _UpdateArchiverMetadata(archive_metadata['files'], config)
+ except ArchiverFieldError:
+ # Do not raise error since it will stop the archiver. We would like to
+ # give few more retires in coming cycle.
+ if len(archive_failures) > MAX_ALOOWED_FAILURES:
+ raise ArchiverFieldError(
+ 'Archiver failed %d times. The error messages are\n%s\n' %
+ pprint.pformat(archive_failures))
else:
logging.info('No data available for archiving for %r', config.data_type)
diff --git a/py/lumberjack/archiver_config.py b/py/lumberjack/archiver_config.py
index 6abe8c7..d907ab4 100644
--- a/py/lumberjack/archiver_config.py
+++ b/py/lumberjack/archiver_config.py
@@ -16,7 +16,7 @@
from archiver import locks
from archiver_exception import ArchiverFieldError
from common import CheckAndLockFile, CheckExecutableExist, TryMakeDirs
-from subprocess import PIPE, Popen
+from subprocess import check_call, PIPE, Popen
ALLOWED_DATA_TYPE = set(['eventlog', 'reports', 'regcode'])
@@ -260,6 +260,11 @@
if not CheckExecutableExist('gpg'):
raise ArchiverFieldError(
'GnuPG(gpg) is not callable. It is required for encryption.')
+ # List the existing keys via "gpg -k". This step is to make sure local
+ # gpg initializes its database so following commands can be run wihtout
+ # issues.
+ check_call(['gpg', '-k'])
+
# Check if the public key's format and recipient are valid.
# Since we don't have the private key, we can only verify if the public
# key is working properly with gpg.
diff --git a/py/lumberjack/archiver_unittest.py b/py/lumberjack/archiver_unittest.py
index f3223c2..761bc79 100755
--- a/py/lumberjack/archiver_unittest.py
+++ b/py/lumberjack/archiver_unittest.py
@@ -49,24 +49,25 @@
def tearDown(self):
os.chdir(self.pwd)
directories_to_delete = [
- os.path.join(TEST_DATA_PATH, 'archives'),
- os.path.join(TEST_DATA_PATH, 'raw/report'),
- os.path.join(TEST_DATA_PATH, 'raw/regcode'),
+ 'archives', 'raw/report', 'raw/regcode',
# Clean-up to make git status cleaner
- os.path.join(TEST_DATA_PATH, 'raw/eventlog/20140406/.archiver'),
- os.path.join(TEST_DATA_PATH, 'raw/eventlog/20140419/.archiver'),
- os.path.join(TEST_DATA_PATH, 'raw/eventlog/20140420/.archiver'),
- os.path.join(TEST_DATA_PATH, 'raw/eventlog/20140421/.archiver')]
+ 'raw/eventlog/20140406/.archiver', 'raw/eventlog/20140419/.archiver',
+ 'raw/eventlog/20140420/.archiver', 'raw/eventlog/20140421/.archiver']
for directory in directories_to_delete:
try:
- shutil.rmtree(directory)
+ shutil.rmtree(os.path.join(TEST_DATA_PATH, directory))
except: # pylint: disable=W0702
pass
+
# Delete lock file
- try:
- os.unlink(os.path.join(TEST_DATA_PATH, 'raw/eventlog/.archiver.lock'))
- except: # pylint: disable=W0702
- pass
+ lock_file_to_delete = [
+ 'raw/eventlog/.archiver.lock',
+ 'raw/.mocked_regcode.csv.archiver.lock']
+ for lock_file in lock_file_to_delete:
+ try:
+ os.unlink(os.path.join(TEST_DATA_PATH, lock_file))
+ except: # pylint: disable=W0702
+ pass
def testYAMLConfigNonExist(self):
argv = ['dry-run', 'nonexist.yaml']
