Skip to content

Commit 17beeb5

Browse files
Change CLI to employ multithreading by default (#4211)
* Change CLI to employ multithreading by default * Document changes to benchmarking, print number of threads for display level >= 4, and add lower bound of 1 for the default number of threads
1 parent 709be6c commit 17beeb5

3 files changed

Lines changed: 39 additions & 33 deletions

File tree

‎programs/fileio_common.h‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,14 @@ extern "C" {
2828
#define GB *(1U<<30)
2929
#undef MAX
3030
#define MAX(a,b) ((a)>(b) ? (a) : (b))
31+
#undef MIN /* in case it would be already defined */
32+
#define MIN(a,b) ((a) < (b) ? (a) : (b))
3133

3234
extern FIO_display_prefs_t g_display_prefs;
3335

34-
#define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
35-
#define DISPLAYOUT(...) fprintf(stdout, __VA_ARGS__)
36+
#define DISPLAY_F(f, ...) fprintf((f), __VA_ARGS__)
37+
#define DISPLAYOUT(...) DISPLAY_F(stdout, __VA_ARGS__)
38+
#define DISPLAY(...) DISPLAY_F(stderr, __VA_ARGS__)
3639
#define DISPLAYLEVEL(l, ...) { if (g_display_prefs.displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
3740

3841
extern UTIL_time_t g_displayClock;
@@ -56,10 +59,6 @@ extern UTIL_time_t g_displayClock;
5659
#define DISPLAYUPDATE_PROGRESS(...) { if (SHOULD_DISPLAY_PROGRESS()) { DISPLAYUPDATE(1, __VA_ARGS__); }}
5760
#define DISPLAY_SUMMARY(...) { if (SHOULD_DISPLAY_SUMMARY()) { DISPLAYLEVEL(1, __VA_ARGS__); } }
5861

59-
#undef MIN /* in case it would be already defined */
60-
#define MIN(a,b) ((a) < (b) ? (a) : (b))
61-
62-
6362
#define EXM_THROW(error, ...) \
6463
{ \
6564
DISPLAYLEVEL(1, "zstd: "); \

‎programs/zstd.1.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,7 +343,7 @@ If the value of `ZSTD_CLEVEL` is not a valid integer, it will be ignored with a
343343

344344
`ZSTD_NBTHREADS` can be used to set the number of threads `zstd` will attempt to use during compression.
345345
If the value of `ZSTD_NBTHREADS` is not a valid unsigned integer, it will be ignored with a warning message.
346-
`ZSTD_NBTHREADS` has a default value of (`1`), and is capped at ZSTDMT_NBWORKERS_MAX==200.
346+
`ZSTD_NBTHREADS` has a default value of `max(1, min(4, nbCores/4))`, and is capped at ZSTDMT_NBWORKERS_MAX==200.
347347
`zstd` must be compiled with multithread support for this variable to have any effect.
348348

349349
They can both be overridden by corresponding command line arguments:
@@ -664,9 +664,12 @@ BENCHMARK
664664
The `zstd` CLI provides a benchmarking mode that can be used to easily find suitable compression parameters, or alternatively to benchmark a computer's performance.
665665
`zstd -b [FILE(s)]` will benchmark `zstd` for both compression and decompression using default compression level.
666666
Note that results are very dependent on the content being compressed.
667+
667668
It's possible to pass multiple files to the benchmark, and even a directory with `-r DIRECTORY`.
668669
When no `FILE` is provided, the benchmark will use a procedurally generated `lorem ipsum` text.
669670

671+
Benchmarking will employ `max(1, min(4, nbCores/4))` worker threads by default in order to match the behavior of the normal CLI I/O.
672+
670673
* `-b#`:
671674
benchmark file(s) using compression level #
672675
* `-e#`:

‎programs/zstdcli.c‎

Lines changed: 30 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,6 @@
88
* You may select, at your option, one of the above-listed licenses.
99
*/
1010

11-
12-
/*-************************************
13-
* Tuning parameters
14-
**************************************/
15-
#ifndef ZSTDCLI_CLEVEL_DEFAULT
16-
# define ZSTDCLI_CLEVEL_DEFAULT 3
17-
#endif
18-
19-
#ifndef ZSTDCLI_CLEVEL_MAX
20-
# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */
21-
#endif
22-
23-
#ifndef ZSTDCLI_NBTHREADS_DEFAULT
24-
# define ZSTDCLI_NBTHREADS_DEFAULT 1
25-
#endif
26-
2711
/*-************************************
2812
* Dependencies
2913
**************************************/
@@ -46,6 +30,23 @@
4630
#endif
4731
#include "../lib/zstd.h" /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
4832
#include "fileio_asyncio.h"
33+
#include "fileio_common.h"
34+
35+
/*-************************************
36+
* Tuning parameters
37+
**************************************/
38+
#ifndef ZSTDCLI_CLEVEL_DEFAULT
39+
# define ZSTDCLI_CLEVEL_DEFAULT 3
40+
#endif
41+
42+
#ifndef ZSTDCLI_CLEVEL_MAX
43+
# define ZSTDCLI_CLEVEL_MAX 19 /* without using --ultra */
44+
#endif
45+
46+
#ifndef ZSTDCLI_NBTHREADS_DEFAULT
47+
#define ZSTDCLI_NBTHREADS_DEFAULT MAX(1, MIN(4, UTIL_countLogicalCores() / 4))
48+
#endif
49+
4950

5051

5152
/*-************************************
@@ -99,9 +100,7 @@ typedef enum { cover, fastCover, legacy } dictType;
99100
/*-************************************
100101
* Display Macros
101102
**************************************/
102-
#define DISPLAY_F(f, ...) fprintf((f), __VA_ARGS__)
103-
#define DISPLAYOUT(...) DISPLAY_F(stdout, __VA_ARGS__)
104-
#define DISPLAY(...) DISPLAY_F(stderr, __VA_ARGS__)
103+
#undef DISPLAYLEVEL
105104
#define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
106105
static int g_displayLevel = DISPLAY_LEVEL_DEFAULT; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
107106

@@ -760,7 +759,7 @@ static int init_cLevel(void) {
760759
}
761760

762761
#ifdef ZSTD_MULTITHREAD
763-
static unsigned init_nbThreads(void) {
762+
static unsigned default_nbThreads(void) {
764763
const char* const env = getenv(ENV_NBTHREADS);
765764
if (env != NULL) {
766765
const char* ptr = env;
@@ -855,7 +854,7 @@ int main(int argCount, const char* argv[])
855854
ZSTD_paramSwitch_e mmapDict=ZSTD_ps_auto;
856855
ZSTD_paramSwitch_e useRowMatchFinder = ZSTD_ps_auto;
857856
FIO_compressionType_t cType = FIO_zstdCompression;
858-
unsigned nbWorkers = 0;
857+
int nbWorkers = -1; /* -1 means unset */
859858
double compressibility = -1.0; /* lorem ipsum generator */
860859
unsigned bench_nbSeconds = 3; /* would be better if this value was synchronized from bench */
861860
size_t blockSize = 0;
@@ -896,17 +895,13 @@ int main(int argCount, const char* argv[])
896895
#endif
897896
ZSTD_paramSwitch_e literalCompressionMode = ZSTD_ps_auto;
898897

899-
900898
/* init */
901899
checkLibVersion();
902900
(void)recursive; (void)cLevelLast; /* not used when ZSTD_NOBENCH set */
903901
(void)memLimit;
904902
assert(argCount >= 1);
905903
if ((filenames==NULL) || (file_of_names==NULL)) { DISPLAYLEVEL(1, "zstd: allocation error \n"); exit(1); }
906904
programName = lastNameFromPath(programName);
907-
#ifdef ZSTD_MULTITHREAD
908-
nbWorkers = init_nbThreads();
909-
#endif
910905

911906
/* preset behaviors */
912907
if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0, singleThread=0;
@@ -1298,7 +1293,7 @@ int main(int argCount, const char* argv[])
12981293
DISPLAYLEVEL(3, WELCOME_MESSAGE);
12991294

13001295
#ifdef ZSTD_MULTITHREAD
1301-
if ((operation==zom_decompress) && (!singleThread) && (nbWorkers > 1)) {
1296+
if ((operation==zom_decompress) && (nbWorkers > 1)) {
13021297
DISPLAYLEVEL(2, "Warning : decompression does not support multi-threading\n");
13031298
}
13041299
if ((nbWorkers==0) && (!singleThread)) {
@@ -1311,6 +1306,15 @@ int main(int argCount, const char* argv[])
13111306
DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
13121307
}
13131308
}
1309+
/* Resolve to default if nbWorkers is still unset */
1310+
if (nbWorkers == -1) {
1311+
if (operation == zom_decompress) {
1312+
nbWorkers = 1;
1313+
} else {
1314+
nbWorkers = default_nbThreads();
1315+
}
1316+
}
1317+
DISPLAYLEVEL(4, "Compressing with %u worker threads \n", nbWorkers);
13141318
#else
13151319
(void)singleThread; (void)nbWorkers; (void)defaultLogicalCores;
13161320
#endif

0 commit comments

Comments
 (0)