diff --git a/monai/data/iterable_dataset.py b/monai/data/iterable_dataset.py index 3ee70bf69e..978780a846 100644 --- a/monai/data/iterable_dataset.py +++ b/monai/data/iterable_dataset.py @@ -278,4 +278,5 @@ def __iter__(self): data=self._flattened(), transform=self.transform, buffer_size=self.buffer_size, seed=self.seed ) yield from buffer - yield from IterableDataset(data=self._flattened(), transform=self.transform) + else: + yield from IterableDataset(data=self._flattened(), transform=self.transform) diff --git a/tests/data/test_csv_iterable_dataset.py b/tests/data/test_csv_iterable_dataset.py index ed71cb39d4..95a8e95aec 100644 --- a/tests/data/test_csv_iterable_dataset.py +++ b/tests/data/test_csv_iterable_dataset.py @@ -247,6 +247,17 @@ def prepare_csv_file(data, filepath): for i in iters: i.close() + def test_shuffle_no_duplicates(self): + """Shuffling a re-iterable stream source yields each item once per epoch.""" + chunks = [ + pd.DataFrame({"subject_id": ["s0", "s1"], "label": [0, 1]}), + pd.DataFrame({"subject_id": ["s2", "s3"], "label": [2, 3]}), + ] + dataset = CSVIterableDataset(src=[chunks], buffer_size=4, shuffle=True, seed=0) + items = [item["subject_id"] for item in dataset] + self.assertEqual(len(items), 4) + self.assertSetEqual(set(items), {"s0", "s1", "s2", "s3"}) + if __name__ == "__main__": unittest.main()