| # Copyright 2018 The Chromium OS Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Abstract wrapper for deduplicating previously processed work.""" |
| |
| from __future__ import print_function |
| from __future__ import absolute_import |
| from __future__ import division |
| |
| |
| def CheckpointSequence(already_processed, save, sequence, limit=None): |
| """Yields elements from |sequence| that haven't been processed, saving after. |
| |
| Args: |
| already_processed: A callback which returns whether the item was already |
| processed. |
| save: A callback which checkpoints the given element. |
| limit: The maximum number of elements to yield. Note this is not |
| equivalent to itertools.islice(limit, _Checkpoint(...)) because the |
| last element of that sequence would not be checkpointed. |
| sequence: A sequence of elements to yield from |
| """ |
| i = 0 |
| for item in sequence: |
| if limit is not None and i >= limit: |
| return |
| if not already_processed(item): |
| yield item |
| save(item) |
| i += 1 |
