[Findit] Detect occurrences of cq hidden flakes and update flake data.

In this version of change, reuses the current FlakeOccurrence to store cq hidden flake occurrences.
Only query hidden flakes every 2 hours because of the big volume.

Bug: 896004
Change-Id: I43bbdcb8c2968c2062248edd641c1acc247da963
Reviewed-on: https://chromium-review.googlesource.com/c/1389283
Commit-Queue: Chan Li <chanli@chromium.org>
Reviewed-by: Shuotao Gao <stgao@chromium.org>
Cr-Commit-Position: refs/heads/master@{#19878}
diff --git a/appengine/findit/handlers/flake/detection/test/flake_detection_utils_test.py b/appengine/findit/handlers/flake/detection/test/flake_detection_utils_test.py
index 7c61a4f..d3ea1e7 100644
--- a/appengine/findit/handlers/flake/detection/test/flake_detection_utils_test.py
+++ b/appengine/findit/handlers/flake/detection/test/flake_detection_utils_test.py
@@ -151,6 +151,11 @@
                 'impacted_cl_count': 0,
                 'occurrence_count': 0
             },
+            {
+                'flake_type': 'cq hidden flake',
+                'impacted_cl_count': 0,
+                'occurrence_count': 0
+            },
         ],
         'flake_score_last_week':
             0,
@@ -338,6 +343,11 @@
                 'impacted_cl_count': 0,
                 'occurrence_count': 0
             },
+            {
+                'flake_type': 'cq hidden flake',
+                'impacted_cl_count': 0,
+                'occurrence_count': 0
+            },
         ],
         'flake_score_last_week':
             0,
diff --git a/appengine/findit/handlers/flake/detection/test/rank_flakes_test.py b/appengine/findit/handlers/flake/detection/test/rank_flakes_test.py
index d47f7ab..5499283 100644
--- a/appengine/findit/handlers/flake/detection/test/rank_flakes_test.py
+++ b/appengine/findit/handlers/flake/detection/test/rank_flakes_test.py
@@ -109,6 +109,11 @@
               'impacted_cl_count': 0,
               'occurrence_count': 0
           },
+          {
+              'flake_type': 'cq hidden flake',
+              'impacted_cl_count': 0,
+              'occurrence_count': 0
+          }
       ]
 
   @mock.patch.object(
@@ -139,7 +144,8 @@
             'error_message':
                 None,
             'flake_weights': [('cq false rejection', 100),
-                              ('cq retry with patch', 10)]
+                              ('cq retry with patch', 10),
+                              ('cq hidden flake', 1)]
         },
                    default=str), response.body)
 
@@ -203,7 +209,8 @@
             'error_message':
                 None,
             'flake_weights': [('cq false rejection', 100),
-                              ('cq retry with patch', 10)]
+                              ('cq retry with patch', 10),
+                              ('cq hidden flake', 1)]
         },
                    default=str), response.body)
 
@@ -238,7 +245,8 @@
             'error_message':
                 None,
             'flake_weights': [('cq false rejection', 100),
-                              ('cq retry with patch', 10)]
+                              ('cq retry with patch', 10),
+                              ('cq hidden flake', 1)]
         },
                    default=str), response.body)
 
@@ -273,7 +281,8 @@
             'error_message':
                 None,
             'flake_weights': [('cq false rejection', 100),
-                              ('cq retry with patch', 10)]
+                              ('cq retry with patch', 10),
+                              ('cq hidden flake', 1)]
         },
                    default=str), response.body)
 
@@ -308,6 +317,7 @@
             'error_message':
                 None,
             'flake_weights': [('cq false rejection', 100),
-                              ('cq retry with patch', 10)]
+                              ('cq retry with patch', 10),
+                              ('cq hidden flake', 1)]
         },
                    default=str), response.body)
diff --git a/appengine/findit/handlers/flake/detection/test/show_flake_test.py b/appengine/findit/handlers/flake/detection/test/show_flake_test.py
index 8690058..a6ef387 100644
--- a/appengine/findit/handlers/flake/detection/test/show_flake_test.py
+++ b/appengine/findit/handlers/flake/detection/test/show_flake_test.py
@@ -152,6 +152,11 @@
                 'impacted_cl_count': 0,
                 'occurrence_count': 0
             },
+            {
+                'flake_type': 'cq hidden flake',
+                'impacted_cl_count': 0,
+                'occurrence_count': 0
+            },
         ],
         'flake_score_last_week':
             0,
@@ -197,7 +202,7 @@
             'show_all_occurrences':
                 '',
             'weights': [('cq false rejection', 100),
-                        ('cq retry with patch', 10)]
+                        ('cq retry with patch', 10), ('cq hidden flake', 1)]
         },
                    default=str,
                    sort_keys=True,
diff --git a/appengine/findit/handlers/flake/reporting/test/component_report_test.py b/appengine/findit/handlers/flake/reporting/test/component_report_test.py
index e6c87a3..0fd3864 100644
--- a/appengine/findit/handlers/flake/reporting/test/component_report_test.py
+++ b/appengine/findit/handlers/flake/reporting/test/component_report_test.py
@@ -65,6 +65,11 @@
             'impacted_cl_count': 0,
             'occurrence_count': 0
         },
+        {
+            'flake_type': 'cq hidden flake',
+            'impacted_cl_count': 0,
+            'occurrence_count': 0
+        },
     ]
 
     luci_project = 'chromium'
diff --git a/appengine/findit/model/flake/flake_type.py b/appengine/findit/model/flake/flake_type.py
index df243bf..a4b9a75 100644
--- a/appengine/findit/model/flake/flake_type.py
+++ b/appengine/findit/model/flake/flake_type.py
@@ -19,10 +19,14 @@
   # services/flake_detection/flaky_tests.retry_with_patch.sql.
   RETRY_WITH_PATCH = 2
 
+  # A flaky test that failed some test runs then pass.
+  CQ_HIDDEN_FLAKE = 3
+
 
 FLAKE_TYPE_DESCRIPTIONS = {
     FlakeType.CQ_FALSE_REJECTION: 'cq false rejection',
-    FlakeType.RETRY_WITH_PATCH: 'cq retry with patch'
+    FlakeType.RETRY_WITH_PATCH: 'cq retry with patch',
+    FlakeType.CQ_HIDDEN_FLAKE: 'cq hidden flake'
 }
 
 # Weights for each type of flakes.
@@ -30,5 +34,6 @@
 # See goo.gl/y5awC5 for the comparison.
 FLAKE_TYPE_WEIGHT = {
     FlakeType.CQ_FALSE_REJECTION: 100,
-    FlakeType.RETRY_WITH_PATCH: 10
+    FlakeType.RETRY_WITH_PATCH: 10,
+    FlakeType.CQ_HIDDEN_FLAKE: 1
 }
diff --git a/appengine/findit/services/flake_detection/detect_flake_occurrences.py b/appengine/findit/services/flake_detection/detect_flake_occurrences.py
index 57da7b1..54c194d 100644
--- a/appengine/findit/services/flake_detection/detect_flake_occurrences.py
+++ b/appengine/findit/services/flake_detection/detect_flake_occurrences.py
@@ -4,11 +4,14 @@
 
 import ast
 import collections
+from datetime import datetime
+from datetime import timedelta
 import json
 import logging
 import os
 import re
 
+from google.appengine.api import memcache
 from google.appengine.ext import ndb
 
 from common.findit_http_client import FinditHttpClient
@@ -36,7 +39,11 @@
     FlakeType.RETRY_WITH_PATCH:
         os.path.realpath(
             os.path.join(__file__, os.path.pardir,
-                         'flaky_tests.retry_with_patch.sql'))
+                         'flaky_tests.retry_with_patch.sql')),
+    FlakeType.CQ_HIDDEN_FLAKE:
+        os.path.realpath(
+            os.path.join(__file__, os.path.pardir,
+                         'flaky_tests.hidden_flakes.sql'))
 }
 
 # Url to the file with the mapping from the directories to crbug components.
@@ -63,6 +70,15 @@
     'source',
 )
 
+# Runs query for cq hidden flakes every 2 hours.
+_CQ_HIDDEN_FLAKE_QUERY_HOUR_INTERVAL = 2
+
+# Roughly estimated max run time of a build.
+_ROUGH_MAX_BUILD_CYCLE_HOURS = 2
+
+# Overlap between queries.
+_CQ_HIDDEN_FLAKE_QUERY_OVERLAP_MINUTES = 20
+
 
 def _CreateFlakeFromRow(row):
   """Creates a Flake entity from a row fetched from BigQuery."""
@@ -385,16 +401,78 @@
       flake.put()
 
 
+def _GetLastCQHiddenFlakeQueryTimeCacheKey(namespace):
+  return '{}-{}'.format(namespace, 'last_cq_hidden_flake_query_time')
+
+
+def _CacheLastCQHiddenFlakeQueryTime(last_cq_hidden_flake_query_time,
+                                     namespace='chromium/src'):
+  """Saves last_cq_hidden_flake_query_time to memcache.
+
+  Once cached, the value will never expires until next time this function is
+  called.
+  """
+  if not last_cq_hidden_flake_query_time or not isinstance(
+      last_cq_hidden_flake_query_time, datetime):
+    return
+  memcache.set(
+      key=_GetLastCQHiddenFlakeQueryTimeCacheKey(namespace),
+      value=last_cq_hidden_flake_query_time)
+
+
+def _GetLastCQHiddenFlakeQueryTime(namespace='chromium/src'):
+  return memcache.get(_GetLastCQHiddenFlakeQueryTimeCacheKey(namespace))
+
+
+def _GetCQHiddenFlakeQueryStartTime():
+  """Gets the latest happen time of cq hidden flakes.
+
+  Uses this time to decide if we should run the query for cq hidden flakes.
+  And also uses this time to decides the start time of the query.
+
+  Returns:
+    (str): String representation of a datetime in the format
+      %Y-%m-%d %H:%M:%S UTC.
+  """
+  last_query_time_right_bourndary = time_util.GetUTCNow() - timedelta(
+      hours=_CQ_HIDDEN_FLAKE_QUERY_HOUR_INTERVAL)
+  hidden_flake_query_start_time = time_util.FormatDatetime(time_util.GetUTCNow(
+  ) - timedelta(
+      hours=_CQ_HIDDEN_FLAKE_QUERY_HOUR_INTERVAL + _ROUGH_MAX_BUILD_CYCLE_HOURS,
+      minutes=_CQ_HIDDEN_FLAKE_QUERY_OVERLAP_MINUTES))
+  hidden_flake_query_end_time = time_util.FormatDatetime(
+      time_util.GetUTCNow() -
+      timedelta(hours=_CQ_HIDDEN_FLAKE_QUERY_HOUR_INTERVAL))
+
+  last_query_time = _GetLastCQHiddenFlakeQueryTime()
+
+  if not last_query_time:
+    # Only before the first time of running the query.
+    return hidden_flake_query_start_time, hidden_flake_query_end_time
+  return ((hidden_flake_query_start_time, hidden_flake_query_end_time) if
+          last_query_time <= last_query_time_right_bourndary else (None, None))
+
+
 def QueryAndStoreFlakes(flake_type_enum):
   """Runs the query to fetch flake occurrences and store them."""
+
+  parameters = None
+  if flake_type_enum == FlakeType.CQ_HIDDEN_FLAKE:
+    start_time_string, end_time_string = _GetCQHiddenFlakeQueryStartTime()
+    if not start_time_string:
+      # Only runs this query every 2 hours.
+      return
+    parameters = [('hidden_flake_query_start_time', 'TIMESTAMP',
+                   start_time_string),
+                  ('hidden_flake_query_end_time', 'TIMESTAMP', end_time_string)]
+
   path = _MAP_FLAKY_TESTS_QUERY_PATH[flake_type_enum]
   flake_type_desc = flake_type.FLAKE_TYPE_DESCRIPTIONS.get(
       flake_type_enum, 'N/A')
   with open(path) as f:
     query = f.read()
-
   success, rows = bigquery_helper.ExecuteQuery(
-      appengine_util.GetApplicationId(), query)
+      appengine_util.GetApplicationId(), query, parameters=parameters)
 
   if not success:
     logging.error('Failed executing the query to detect %s flakes.',
@@ -413,5 +491,10 @@
   ]
   new_occurrences = _StoreMultipleLocalEntities(local_flake_occurrences)
   _UpdateFlakeMetadata(new_occurrences)
+
+  if flake_type_enum == FlakeType.CQ_HIDDEN_FLAKE:
+    # Updates memecache for the new last_cq_hidden_flake_query_time.
+    _CacheLastCQHiddenFlakeQueryTime(time_util.GetUTCNow())
+
   monitoring.OnFlakeDetectionDetectNewOccurrences(
       flake_type=flake_type_desc, num_occurrences=len(new_occurrences))
diff --git a/appengine/findit/services/flake_detection/flaky_tests.hidden_flakes.sql b/appengine/findit/services/flake_detection/flaky_tests.hidden_flakes.sql
index 25804f5..583b97d 100644
--- a/appengine/findit/services/flake_detection/flaky_tests.hidden_flakes.sql
+++ b/appengine/findit/services/flake_detection/flaky_tests.hidden_flakes.sql
@@ -51,8 +51,8 @@
     UNNEST(ca.contributing_buildbucket_ids) AS build_id
   WHERE
     # cq_events table is not partitioned.
-    ca.attempt_start_usec >= UNIX_MICROS(
-      TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 day))
+    ca.attempt_start_usec >= UNIX_MICROS(@hidden_flake_query_start_time)
+    AND ca.attempt_start_usec < UNIX_MICROS(@hidden_flake_query_end_time)
     AND ca.cq_name IN (
       'chromium/chromium/src'
       #, 'chromium/angle/angle'
diff --git a/appengine/findit/services/flake_detection/test/detect_flake_occurrences_test.py b/appengine/findit/services/flake_detection/test/detect_flake_occurrences_test.py
index d154a37..28684b5 100644
--- a/appengine/findit/services/flake_detection/test/detect_flake_occurrences_test.py
+++ b/appengine/findit/services/flake_detection/test/detect_flake_occurrences_test.py
@@ -8,6 +8,7 @@
 import textwrap
 
 from dto.test_location import TestLocation as DTOTestLocation
+from libs import time_util
 from model.flake.detection.flake_occurrence import BuildConfiguration
 from model.flake.detection.flake_occurrence import FlakeOccurrence
 from model.flake.flake import Flake
@@ -546,3 +547,122 @@
     flake = flake_key.get()
     self.assertEqual(flake.last_occurred_time, datetime(2018, 1, 1, 2))
     self.assertEqual(flake.tags, ['tag1::v1', 'tag2::v2'])
+
+  @mock.patch.object(detect_flake_occurrences, '_UpdateFlakeMetadata')
+  @mock.patch.object(
+      time_util, 'GetUTCNow', return_value=datetime(2018, 12, 20))
+  @mock.patch.object(bigquery_helper, 'ExecuteQuery')
+  def testDetectCQHiddenFlakesShouldSkip(self, mock_query, *_):
+    flake = Flake.Create(
+        luci_project='luci_project',
+        normalized_step_name='s',
+        normalized_test_name='t',
+        test_label_name='t')
+    flake.put()
+    existing_occurrence = FlakeOccurrence.Create(
+        flake_type=FlakeType.CQ_HIDDEN_FLAKE,
+        build_id=123,
+        step_ui_name='s',
+        test_name='t',
+        luci_project='luci_project',
+        luci_bucket='luci_bucket',
+        luci_builder='luci_builder',
+        legacy_master_name='legacy_master_name',
+        legacy_build_number=123,
+        time_happened=datetime(2018, 12, 19, 23),
+        gerrit_cl_id=654321,
+        parent_flake_key=flake.key,
+        tags=[])
+    existing_occurrence.time_detected = datetime(2018, 12, 19, 23, 30)
+    existing_occurrence.put()
+
+    mock_query.side_effect = [(False, []), (True, [])]
+
+    QueryAndStoreFlakes(FlakeType.CQ_HIDDEN_FLAKE)
+    self.assertIsNone(detect_flake_occurrences._GetLastCQHiddenFlakeQueryTime())
+    QueryAndStoreFlakes(FlakeType.CQ_HIDDEN_FLAKE)
+    self.assertEqual(
+        datetime(2018, 12, 20),
+        detect_flake_occurrences._GetLastCQHiddenFlakeQueryTime())
+    QueryAndStoreFlakes(FlakeType.CQ_HIDDEN_FLAKE)
+
+  @mock.patch.object(
+      time_util, 'GetUTCNow', return_value=datetime(2018, 12, 20))
+  def testDetectCQHiddenFlakesShouldRun(self, _):
+    flake = Flake.Create(
+        luci_project='luci_project',
+        normalized_step_name='s',
+        normalized_test_name='t',
+        test_label_name='t')
+    flake.put()
+    existing_occurrence = FlakeOccurrence.Create(
+        flake_type=FlakeType.CQ_HIDDEN_FLAKE,
+        build_id=123,
+        step_ui_name='s',
+        test_name='t',
+        luci_project='luci_project',
+        luci_bucket='luci_bucket',
+        luci_builder='luci_builder',
+        legacy_master_name='legacy_master_name',
+        legacy_build_number=123,
+        time_happened=datetime(2018, 12, 19, 20),
+        gerrit_cl_id=654321,
+        parent_flake_key=flake.key,
+        tags=[])
+    existing_occurrence.time_detected = datetime(2018, 12, 19, 20)
+    existing_occurrence.put()
+
+    self.assertEqual(('2018-12-19 19:40:00 UTC', '2018-12-19 22:00:00 UTC'),
+                     detect_flake_occurrences._GetCQHiddenFlakeQueryStartTime())
+
+  @mock.patch.object(
+      time_util, 'GetUTCNow', return_value=datetime(2018, 12, 20))
+  @mock.patch.object(
+      detect_flake_occurrences, '_GetTestLocation', return_value=None)
+  @mock.patch.object(
+      detect_flake_occurrences,
+      '_GetChromiumDirectoryToComponentMapping',
+      return_value={})
+  @mock.patch.object(
+      detect_flake_occurrences, '_GetChromiumWATCHLISTS', return_value={})
+  @mock.patch.object(bigquery_helper, '_GetBigqueryClient')
+  def testDetectCQHiddenFlakes(self, mocked_get_client, *_):
+    query_response = self._GetEmptyQueryResponse()
+    test_name1 = 'suite.test'
+    test_name2 = 'suite.test_1'
+
+    self._AddRowToQueryResponse(
+        query_response=query_response,
+        step_ui_name='step_ui_name',
+        test_name=test_name1,
+        gerrit_cl_id='10000')
+
+    self._AddRowToQueryResponse(
+        query_response=query_response,
+        step_ui_name='step_ui_name',
+        test_name=test_name1,
+        gerrit_cl_id='10001',
+        build_id='124',
+        test_start_msec='1')
+
+    self._AddRowToQueryResponse(
+        query_response=query_response,
+        luci_builder='another_builder',
+        step_ui_name='step_ui_name',
+        test_name=test_name1,
+        gerrit_cl_id='10001',
+        build_id='125')
+
+    self._AddRowToQueryResponse(
+        query_response=query_response,
+        step_ui_name='step_ui_name',
+        test_name=test_name2,
+        gerrit_cl_id='10000')
+    mocked_client = mock.Mock()
+    mocked_get_client.return_value = mocked_client
+    mocked_client.jobs().query().execute.return_value = query_response
+
+    QueryAndStoreFlakes(FlakeType.CQ_HIDDEN_FLAKE)
+
+    all_flake_occurrences = FlakeOccurrence.query().fetch()
+    self.assertEqual(4, len(all_flake_occurrences))
diff --git a/appengine/findit/services/flake_detection/test/update_flake_counts_service_test.py b/appengine/findit/services/flake_detection/test/update_flake_counts_service_test.py
index 97fe75a..9c93be0 100644
--- a/appengine/findit/services/flake_detection/test/update_flake_counts_service_test.py
+++ b/appengine/findit/services/flake_detection/test/update_flake_counts_service_test.py
@@ -40,6 +40,7 @@
     flake2.last_occurred_time = datetime(2017, 9, 1)
     flake2.false_rejection_count_last_week = 5
     flake2.impacted_cl_count_last_week = 3
+    flake2.flake_score_last_week = 10
     flake2.put()
     flake2_key = flake2.key
 
@@ -54,6 +55,15 @@
     flake3.put()
     flake3_key = flake3.key
 
+    flake4 = Flake.Create(
+        luci_project=luci_project,
+        normalized_step_name=normalized_step_name,
+        normalized_test_name='normalized_test_name_4',
+        test_label_name='test_label4')
+    flake4.last_occurred_time = datetime(2018, 9, 1)
+    flake4.put()
+    flake4_key = flake4.key
+
     luci_bucket = 'try'
     luci_builder = 'luci builder'
     legacy_master_name = 'buildbot master'
@@ -164,6 +174,38 @@
         parent_flake_key=flake1_key)
     occurrence7.put()
 
+    for i in xrange(98760, 98790):
+      occurrence = FlakeOccurrence.Create(
+          flake_type=FlakeType.CQ_HIDDEN_FLAKE,
+          build_id=i,
+          step_ui_name=step_ui_name,
+          test_name='t1',
+          luci_project=luci_project,
+          luci_bucket=luci_bucket,
+          luci_builder=luci_builder,
+          legacy_master_name=legacy_master_name,
+          legacy_build_number=i,
+          time_happened=datetime(2018, 8, 31),
+          gerrit_cl_id=i,
+          parent_flake_key=flake3_key)
+      occurrence.put()
+
+    for i in xrange(98761, 98765):
+      occurrence = FlakeOccurrence.Create(
+          flake_type=FlakeType.CQ_HIDDEN_FLAKE,
+          build_id=i,
+          step_ui_name=step_ui_name,
+          test_name='t4',
+          luci_project=luci_project,
+          luci_bucket=luci_bucket,
+          luci_builder=luci_builder,
+          legacy_master_name=legacy_master_name,
+          legacy_build_number=i,
+          time_happened=datetime(2018, 8, 31),
+          gerrit_cl_id=i,
+          parent_flake_key=flake4_key)
+      occurrence.put()
+
     UpdateFlakeCounts()
 
     flake1 = flake1_key.get()
@@ -190,6 +232,19 @@
         FlakeCountsByType(
             flake_type=FlakeType.CQ_FALSE_REJECTION,
             impacted_cl_count=1,
-            occurrence_count=1)
+            occurrence_count=1),
+        FlakeCountsByType(
+            flake_type=FlakeType.CQ_HIDDEN_FLAKE,
+            impacted_cl_count=29,
+            occurrence_count=30)
     ], flake3.flake_counts_last_week)
-    self.assertEqual(0, flake3.flake_score_last_week)
+    self.assertEqual(129, flake3.flake_score_last_week)
+
+    flake4 = flake4_key.get()
+    self.assertEqual([
+        FlakeCountsByType(
+            flake_type=FlakeType.CQ_HIDDEN_FLAKE,
+            impacted_cl_count=4,
+            occurrence_count=4)
+    ], flake4.flake_counts_last_week)
+    self.assertEqual(0, flake4.flake_score_last_week)
diff --git a/appengine/findit/services/flake_detection/update_flake_counts_service.py b/appengine/findit/services/flake_detection/update_flake_counts_service.py
index 37d4230..05a974b 100644
--- a/appengine/findit/services/flake_detection/update_flake_counts_service.py
+++ b/appengine/findit/services/flake_detection/update_flake_counts_service.py
@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import copy
+
 from google.appengine.ext import ndb
 
 from libs import time_util
@@ -12,12 +14,17 @@
 from model.flake.flake_type import FLAKE_TYPE_WEIGHT
 from services import constants
 
+# Minimum number of distinct impacted CLs for a flake's false rejections or
+# retry with patch occurrences to calculate the flake score.
+_MIN_NON_HIDDEN_DISTINCT_CL_NUMBER = 3
+
 # Minimum number of distinct impacted CLs for a flake to calculate the flake
-# score.
-_MIN_DISTINCT_CL_NUMBER = 3
+# score, including occurrences of false rejection, retry with patch flakes and
+# hidden flakes.
+_MIN_TOTAL_DISTINCT_CL_NUMBER = 20
 
 
-def _GetsTypedFlakeCounts(flake, start_date, flake_type, counted_gerrit_cl_ids):
+def _GetTypedFlakeCounts(flake, start_date, flake_type, counted_gerrit_cl_ids):
   """Gets the counts of a type of occurrences for a flakes within a time range.
 
   Args:
@@ -90,7 +97,7 @@
     # Counts the occurrences/impacted CLs of the flake from the type with the
     # highest impact to the type with the lowest impact.
     # So that we don't count the same CL multiple times.
-    typed_counts, counted_gerrit_cl_ids = _GetsTypedFlakeCounts(
+    typed_counts, counted_gerrit_cl_ids = _GetTypedFlakeCounts(
         flake, start_date, flake_type, counted_gerrit_cl_ids)
     if not typed_counts:
       continue
@@ -104,7 +111,17 @@
     flake.false_rejection_count_last_week += typed_counts.occurrence_count
     flake.impacted_cl_count_last_week += typed_counts.impacted_cl_count
 
-  if len(counted_gerrit_cl_ids) < _MIN_DISTINCT_CL_NUMBER:
+  # Store CL ids that are impacted by false rejection or retry with patch.
+  non_hidden_flake_gerrit_cl_ids = copy.deepcopy(counted_gerrit_cl_ids)
+
+  # Count hidden flake occurrences.
+  typed_counts, counted_gerrit_cl_ids = _GetTypedFlakeCounts(
+      flake, start_date, FlakeType.CQ_HIDDEN_FLAKE, counted_gerrit_cl_ids)
+  if typed_counts:
+    flake.flake_counts_last_week.append(typed_counts)
+
+  if (len(non_hidden_flake_gerrit_cl_ids) < _MIN_NON_HIDDEN_DISTINCT_CL_NUMBER
+      and len(counted_gerrit_cl_ids) < _MIN_TOTAL_DISTINCT_CL_NUMBER):
     # If there is not enough occurrences for the flake, bail out.
     return
 
diff --git a/appengine/findit/services/flake_reporting/test/component_test.py b/appengine/findit/services/flake_reporting/test/component_test.py
index b64c698..8ce7228 100644
--- a/appengine/findit/services/flake_reporting/test/component_test.py
+++ b/appengine/findit/services/flake_reporting/test/component_test.py
@@ -196,7 +196,8 @@
 
     expected_report_counts = {
         FlakeType.CQ_FALSE_REJECTION: (7, 3),
-        FlakeType.RETRY_WITH_PATCH: (1, 0)
+        FlakeType.RETRY_WITH_PATCH: (1, 0),
+        FlakeType.CQ_HIDDEN_FLAKE: (0, 0)
     }
 
     for occurrence_count in report.occurrence_counts:
@@ -213,7 +214,8 @@
 
     expected_A_counts = {
         FlakeType.CQ_FALSE_REJECTION: (5, 3),
-        FlakeType.RETRY_WITH_PATCH: (1, 0)
+        FlakeType.RETRY_WITH_PATCH: (1, 0),
+        FlakeType.CQ_HIDDEN_FLAKE: (0, 0)
     }
 
     for occurrence_count in component_report_A.occurrence_counts:
@@ -231,7 +233,8 @@
 
     expected_Unknown_counts = {
         FlakeType.CQ_FALSE_REJECTION: (1, 1),
-        FlakeType.RETRY_WITH_PATCH: (0, 0)
+        FlakeType.RETRY_WITH_PATCH: (0, 0),
+        FlakeType.CQ_HIDDEN_FLAKE: (0, 0)
     }
 
     for occurrence_count in component_report_unknown.occurrence_counts:
@@ -249,7 +252,8 @@
 
     expected_A_B_counts = {
         FlakeType.CQ_FALSE_REJECTION: (2, 2),
-        FlakeType.RETRY_WITH_PATCH: (0, 0)
+        FlakeType.RETRY_WITH_PATCH: (0, 0),
+        FlakeType.CQ_HIDDEN_FLAKE: (0, 0)
     }
 
     for occurrence_count in component_test_report_A_B.occurrence_counts:
@@ -272,7 +276,8 @@
 
     expected_report_counts = {
         FlakeType.CQ_FALSE_REJECTION: (7, 3),
-        FlakeType.RETRY_WITH_PATCH: (1, 0)
+        FlakeType.RETRY_WITH_PATCH: (1, 0),
+        FlakeType.CQ_HIDDEN_FLAKE: (0, 0)
     }
 
     for occurrence_count in report.occurrence_counts:
@@ -289,7 +294,8 @@
 
     expected_A_counts = {
         FlakeType.CQ_FALSE_REJECTION: (5, 3),
-        FlakeType.RETRY_WITH_PATCH: (1, 0)
+        FlakeType.RETRY_WITH_PATCH: (1, 0),
+        FlakeType.CQ_HIDDEN_FLAKE: (0, 0)
     }
 
     for occurrence_count in component_report_A.occurrence_counts:
@@ -307,7 +313,8 @@
 
     expected_Unknown_counts = {
         FlakeType.CQ_FALSE_REJECTION: (1, 1),
-        FlakeType.RETRY_WITH_PATCH: (0, 0)
+        FlakeType.RETRY_WITH_PATCH: (0, 0),
+        FlakeType.CQ_HIDDEN_FLAKE: (0, 0)
     }
 
     for occurrence_count in component_report_unknown.occurrence_counts: