Skip to content

Commit 64c410a

Browse files
committed
Remove shuffle_after_epoch table and point each reader's docstring the the relevant page
Signed-off-by: Rostan Tabet <[email protected]>
1 parent 6cd41cf commit 64c410a

9 files changed

Lines changed: 26 additions & 30 deletions

‎dali/operators/reader/coco_reader_op.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,9 @@ This readers produces the following outputs::
9393
.AddOptionalArg("shuffle_after_epoch",
9494
R"code(If set to True, the reader shuffles the entire dataset after each epoch.
9595
96-
`stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.)code",
96+
`stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.
97+
98+
See the Sharding and Shuffling documentation for details.)code",
9799
false)
98100
.AddOptionalArg<int64_t>("shuffle_after_epoch_seed",
99101
R"code(Random seed for the dataset shuffling performed after each epoch.

‎dali/operators/reader/file_reader_op.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,9 @@ This argument is mutually exclusive with `files`.)", nullptr)
9696
.AddOptionalArg("shuffle_after_epoch",
9797
R"(If set to True, the reader shuffles the entire dataset after each epoch.
9898
99-
`stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.)",
99+
`stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.
100+
101+
See the Sharding and Shuffling documentation for details.)",
100102
false)
101103
.AddOptionalArg<int64_t>("shuffle_after_epoch_seed",
102104
R"(Random seed for the dataset shuffling performed after each epoch.

‎dali/operators/reader/fits_reader_op.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,9 @@ This argument is mutually exclusive with `files`.)",
6969
.AddOptionalArg("shuffle_after_epoch",
7070
R"(If set to True, the reader shuffles the entire dataset after each epoch.
7171
72-
`stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.)",
72+
`stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.
73+
74+
See the Sharding and Shuffling documentation for details.)",
7375
false)
7476
.AddOptionalArg<int64_t>("shuffle_after_epoch_seed",
7577
R"(Random seed for the dataset shuffling performed after each epoch.

‎dali/operators/reader/mxnet_reader_op.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@ This keeps I/O access patterns sequential — only the order in which whole file
4040
changes between epochs. ``random_shuffle`` can be combined with this option to additionally
4141
shuffle samples within the pipeline's prefetch buffer.
4242
43-
``stick_to_shard`` cannot be used when this argument is set to True.)code",
43+
``stick_to_shard`` cannot be used when this argument is set to True.
44+
45+
See the Sharding and Shuffling documentation for details.)code",
4446
false)
4547
.AddOptionalArg<int64_t>("shuffle_after_epoch_seed",
4648
R"code(Random seed for the file-order shuffling performed after each epoch.

‎dali/operators/reader/nemo_asr_reader_op.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,9 @@ This reader produces between 1 and 3 outputs:
8686
.AddOptionalArg("shuffle_after_epoch",
8787
R"code(If set to True, the reader shuffles the entire dataset after each epoch.
8888
89-
`stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.)code",
89+
`stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.
90+
91+
See the Sharding and Shuffling documentation for details.)code",
9092
false)
9193
.AddOptionalArg<int64_t>("shuffle_after_epoch_seed",
9294
R"code(Random seed for the dataset shuffling performed after each epoch.

‎dali/operators/reader/numpy_reader_op.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,9 @@ This argument is mutually exclusive with `files`.)", nullptr)
135135
.AddOptionalArg("shuffle_after_epoch",
136136
R"(If set to True, the reader shuffles the entire dataset after each epoch.
137137
138-
`stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.)",
138+
`stick_to_shard` and `random_shuffle` cannot be used when this argument is set to True.
139+
140+
See the Sharding and Shuffling documentation for details.)",
139141
false)
140142
.AddOptionalArg<int64_t>("shuffle_after_epoch_seed",
141143
R"(Random seed for the dataset shuffling performed after each epoch.

‎dali/operators/reader/tfrecord_reader_op.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,9 @@ This keeps I/O access patterns sequential — only the order in which whole file
6262
changes between epochs. ``random_shuffle`` can be combined with this option to additionally
6363
shuffle samples within the pipeline's prefetch buffer.
6464
65-
``stick_to_shard`` cannot be used when this argument is set to True.)code",
65+
``stick_to_shard`` cannot be used when this argument is set to True.
66+
67+
See the Sharding and Shuffling documentation for details.)code",
6668
false)
6769
.AddOptionalArg<int64_t>("shuffle_after_epoch_seed",
6870
R"code(Random seed for the file-order shuffling performed after each epoch.

‎dali/operators/reader/webdataset_reader_op.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,9 @@ This keeps I/O access patterns sequential — only the order in which whole shar
162162
changes between epochs. ``random_shuffle`` can be combined with this option to additionally
163163
shuffle samples within the pipeline's prefetch buffer.
164164
165-
``stick_to_shard`` cannot be used when this argument is set to True.)code",
165+
``stick_to_shard`` cannot be used when this argument is set to True.
166+
167+
See the Sharding and Shuffling documentation for details.)code",
166168
false)
167169
.AddOptionalArg<int64_t>("shuffle_after_epoch_seed",
168170
R"code(Random seed for the shard-order shuffling performed after each epoch.

‎docs/advanced_topics_sharding.rst‎

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -33,28 +33,8 @@ With ``shuffle_after_epoch=True``, each pipeline keeps its shard index while the
3333
changes which samples belong to that shard. This conflicts with ``stick_to_shard=True``, which
3434
keeps shard membership stable across epochs, so these options cannot be enabled together.
3535

36-
Whether ``random_shuffle`` can be combined with ``shuffle_after_epoch`` depends on what the reader
37-
reshuffles:
38-
39-
.. list-table::
40-
:header-rows: 1
41-
:widths: 3,3,2
42-
43-
* - Readers
44-
- ``shuffle_after_epoch`` reshuffles
45-
- With ``random_shuffle``
46-
* - :func:`~nvidia.dali.fn.readers.file`, :func:`~nvidia.dali.fn.readers.coco`,
47-
:func:`~nvidia.dali.fn.readers.numpy`,
48-
:func:`~nvidia.dali.fn.experimental.readers.fits`,
49-
:func:`~nvidia.dali.fn.readers.nemo_asr`
50-
- Samples
51-
- Not supported
52-
* - :func:`~nvidia.dali.fn.readers.tfrecord`, :func:`~nvidia.dali.fn.readers.mxnet`
53-
- Source files
54-
- Supported
55-
* - :func:`~nvidia.dali.fn.readers.webdataset`
56-
- Tar archives
57-
- Supported
36+
Whether ``random_shuffle`` can be combined with ``shuffle_after_epoch`` depends on the reader.
37+
See the reader's ``shuffle_after_epoch`` argument documentation for supported combinations.
5838

5939
Framework iterator configuration
6040
--------------------------------

0 commit comments

Comments
 (0)