blob: e2939a2f57a1254220ece9b4a7c15ae63067f190 [file] [log] [blame]
# Copyright 2018 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Abstract wrapper for deduplicating previously processed work."""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
def CheckpointSequence(already_processed, save, sequence, limit=None):
"""Yields elements from |sequence| that haven't been processed, saving after.
already_processed: A callback which returns whether the item was already
save: A callback which checkpoints the given element.
limit: The maximum number of elements to yield. Note this is not
equivalent to itertools.islice(limit, _Checkpoint(...)) because the
last element of that sequence would not be checkpointed.
sequence: A sequence of elements to yield from
i = 0
for item in sequence:
if limit is not None and i >= limit:
if not already_processed(item):
yield item
i += 1