From cc19bd9920f1e4ba634c05baed83cfbb68d35fff Mon Sep 17 00:00:00 2001 From: Mehedi Hasan <43575088+Mahdi-Hasan@users.noreply.github.com> Date: Wed, 4 Nov 2026 12:37:56 +0600 Subject: [PATCH 1/2] Fix CSVIterableDataset yielding data twice when shuffle=True With shuffle=True, __iter__ yielded the shuffle buffer and then fell through to also yield an unshuffled pass. File sources hid this because the pandas readers were already exhausted. Re-iterable stream sources returned every item twice per epoch. Fixes #7869 Co-Authored-By: Claude Opus 5.5 Signed-off-by: Mehedi Hasan <43575088+Mahdi-Hasan@users.noreply.github.com> --- monai/data/iterable_dataset.py | 3 ++- tests/data/test_csv_iterable_dataset.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/monai/data/iterable_dataset.py b/monai/data/iterable_dataset.py index 3ee70bf69eb..978780a846a 100644 --- a/monai/data/iterable_dataset.py +++ b/monai/data/iterable_dataset.py @@ -278,4 +278,5 @@ def __iter__(self): data=self._flattened(), transform=self.transform, buffer_size=self.buffer_size, seed=self.seed ) yield from buffer - yield from IterableDataset(data=self._flattened(), transform=self.transform) + else: + yield from IterableDataset(data=self._flattened(), transform=self.transform) diff --git a/tests/data/test_csv_iterable_dataset.py b/tests/data/test_csv_iterable_dataset.py index ed71cb39d44..637ea2292c9 100644 --- a/tests/data/test_csv_iterable_dataset.py +++ b/tests/data/test_csv_iterable_dataset.py @@ -247,6 +247,17 @@ def prepare_csv_file(data, filepath): for i in iters: i.close() + def test_shuffle_no_duplicates(self): + # a re-iterable stream source must be read only once per epoch when shuffling + chunks = [ + pd.DataFrame({"subject_id": ["s0", "s1"], "label": [0, 1]}), + pd.DataFrame({"subject_id": ["s2", "s3"], "label": [2, 3]}), + ] + dataset = CSVIterableDataset(src=[chunks], buffer_size=4, shuffle=True, seed=0) + items = [item["subject_id"] for item in dataset] + self.assertEqual(len(items), 4) + self.assertSetEqual(set(items), {"s0", "s1", "s2", "s3"}) + if __name__ == "__main__": unittest.main() From 1c81d337d66a1f00451fe4bcd716cafc20abe6cf Mon Sep 17 00:00:00 2001 From: Mehedi Hasan <43575088+Mahdi-Hasan@users.noreply.github.com> Date: Wed, 4 Nov 2026 13:09:38 +0600 Subject: [PATCH 2/2] Add docstring to test_shuffle_no_duplicates Co-Authored-By: Claude Opus 5.5 Signed-off-by: Mehedi Hasan <43575088+Mahdi-Hasan@users.noreply.github.com> --- tests/data/test_csv_iterable_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/data/test_csv_iterable_dataset.py b/tests/data/test_csv_iterable_dataset.py index 637ea2292c9..95a8e95aecf 100644 --- a/tests/data/test_csv_iterable_dataset.py +++ b/tests/data/test_csv_iterable_dataset.py @@ -248,7 +248,7 @@ def prepare_csv_file(data, filepath): i.close() def test_shuffle_no_duplicates(self): - # a re-iterable stream source must be read only once per epoch when shuffling + """Shuffling a re-iterable stream source yields each item once per epoch.""" chunks = [ pd.DataFrame({"subject_id": ["s0", "s1"], "label": [0, 1]}), pd.DataFrame({"subject_id": ["s2", "s3"], "label": [2, 3]}),