commit dd5cfd07e7802fcac79c20afd66ddb3d0d9316d0
author Shuhei Takahashi <nya@chromium.org> Tue Oct 24 09:31:24 2017
committer chrome-bot <chrome-bot@chromium.org> Wed Oct 25 05:47:54 2017
tree 3ecdc566d0c008e95c77a5c1a835e878da144e3b
parent ed00935597f44fdaec2d59e7f67ab4d70f494fd3

Add a config to disable table dumps.

Table dumps often take time and not desired on local testing.

BUG=None
TEST=bin/run_tests && bin/run_yapf -i && bin/run_lint

Change-Id: Iac3533decf01b805305a312937d00be1999382b9
Reviewed-on: https://chromium-review.googlesource.com/734982
Commit-Ready: Shuhei Takahashi <nya@chromium.org>
Tested-by: Shuhei Takahashi <nya@chromium.org>
Reviewed-by: Prathmesh Prabhu <pprabhu@chromium.org>

ci_results_archiver/archive_builder.py

diff --git a/ci_results_archiver/archive_builder.py b/ci_results_archiver/archive_builder.py
index 64df62a..d6b6af8 100644
--- a/ci_results_archiver/archive_builder.py
+++ b/ci_results_archiver/archive_builder.py
@@ -109,7 +109,7 @@
       importer: AbstractImporter object.
       modifier: AbstractModifier object.
       exporter: BigQueryExporter object.
-      dumper: BigQueryDumper object.
+      dumper: BigQueryDumper object or None.
       checkpointer: Checkpointer object.
     """
     self._table_type = table_type
@@ -155,7 +155,7 @@
           updated_table_suffixes.update(exported_table_suffixes)
 
       # Dump updated tables.
-      if updated_table_suffixes:
+      if updated_table_suffixes and self._dumper:
         with timer.Step('dump'):
           self._dumper.DumpTables(list(updated_table_suffixes))
 

ci_results_archiver/archive_builder_factory.py

diff --git a/ci_results_archiver/archive_builder_factory.py b/ci_results_archiver/archive_builder_factory.py
index bb755c5..e8a5dec 100644
--- a/ci_results_archiver/archive_builder_factory.py
+++ b/ci_results_archiver/archive_builder_factory.py
@@ -79,10 +79,14 @@
   exporter = bigquery_exporter.BigQueryExporter(
       bigquery_tables=bigquery_tables, table_spec=table_spec)
 
-  dumper = bigquery_dumper.BigQueryDumper(
-      bigquery_tables=bigquery_tables,
-      table_spec=table_spec,
-      bucket_name=persistent_bucket.name)
+  dump_enabled = configs['tables'][table_type.value].get('dump', True)
+  if dump_enabled:
+    dumper = bigquery_dumper.BigQueryDumper(
+        bigquery_tables=bigquery_tables,
+        table_spec=table_spec,
+        bucket_name=persistent_bucket.name)
+  else:
+    dumper = None
 
   checkpointer = checkpointer_lib.Checkpointer(
       bucket=persistent_bucket,

ci_results_archiver/archive_builder_factory_unittest.py

diff --git a/ci_results_archiver/archive_builder_factory_unittest.py b/ci_results_archiver/archive_builder_factory_unittest.py
index 3e83edc..33494d8 100644
--- a/ci_results_archiver/archive_builder_factory_unittest.py
+++ b/ci_results_archiver/archive_builder_factory_unittest.py
@@ -117,7 +117,7 @@
     self.assertIsInstance(builder._importer,
                           cidb_build_importer.CidbBuildImporter)
     self.assertIs(builder._modifier, None)
-    self.assertIsInstance(builder._dumper, bigquery_dumper.BigQueryDumper)
+    self.assertIs(builder._dumper, None)
     self.assertIsInstance(builder._exporter, bigquery_exporter.BigQueryExporter)
     self.assertIsInstance(builder._checkpointer, checkpointer.Checkpointer)
 

ci_results_archiver/test/config_example.yaml

diff --git a/ci_results_archiver/test/config_example.yaml b/ci_results_archiver/test/config_example.yaml
index 73f7495..61d4cb2 100644
--- a/ci_results_archiver/test/config_example.yaml
+++ b/ci_results_archiver/test/config_example.yaml
@@ -7,14 +7,17 @@
     max_entries: 1000
     grace_period:
      timeout: 10s
+    dump: true
   tko_jobs:
     max_entries: 2000
     grace_period:
       capacity: 200
+    dump: true
   cidb_builds:
     max_entries: 3000
     grace_period:
       timeout: 30s
+    dump: false
 
 databases:
   afe: