Skip to content
This repository was archived by the owner on Nov 17, 2023. It is now read-only.

Commit ff328ef

Browse files
DickJC123DickJC123
committed
Temp log all CI seeds to troubleshoot unwanted seed determinism
1 parent bfba74a commit ff328ef

File tree

1 file changed

+33
-33
lines changed

1 file changed

+33
-33
lines changed

‎ci/docker/runtime_functions.sh‎

Lines changed: 33 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -899,8 +899,8 @@ cd_unittest_ubuntu() {
899899

900900
local mxnet_variant=${1:?"This function requires a mxnet variant as the first argument"}
901901

902-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -n 4 --durations=50 --verbose tests/python/unittest
903-
pytest -m 'serial' --durations=50 --verbose tests/python/unittest
902+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/unittest
903+
pytest -m 'serial' --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/unittest
904904

905905
# https://github.com/apache/incubator-mxnet/issues/11801
906906
# if [[ ${mxnet_variant} = "cpu" ]] || [[ ${mxnet_variant} = "mkl" ]]; then
@@ -910,10 +910,10 @@ cd_unittest_ubuntu() {
910910
if [[ ${mxnet_variant} = cu* ]]; then
911911
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
912912
MXNET_ENGINE_TYPE=NaiveEngine \
913-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/gpu
913+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/gpu
914914
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
915-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose tests/python/gpu
916-
pytest -m 'serial' --durations=50 --verbose tests/python/gpu
915+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/gpu
916+
pytest -m 'serial' --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/gpu
917917

918918
# TODO(szha): fix and reenable the hanging issue. tracked in #18098
919919
# integrationtest_ubuntu_gpu_dist_kvstore
@@ -922,7 +922,7 @@ cd_unittest_ubuntu() {
922922
fi
923923

924924
if [[ ${mxnet_variant} = *mkl ]]; then
925-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 --durations=50 --verbose tests/python/mkl
925+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/mkl
926926
fi
927927
}
928928

@@ -934,10 +934,10 @@ unittest_ubuntu_python3_cpu() {
934934
export MXNET_SUBGRAPH_VERBOSE=0
935935
export MXNET_ENABLE_CYTHON=0
936936
export DMLC_LOG_STACK_TRACE_DEPTH=10
937-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
937+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose -s --log-cli-level=DEBUG tests/python/unittest
938938
MXNET_ENGINE_TYPE=NaiveEngine \
939-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
940-
pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
939+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/unittest
940+
pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/unittest
941941
}
942942

943943
unittest_ubuntu_python3_cpu_mkldnn() {
@@ -948,11 +948,11 @@ unittest_ubuntu_python3_cpu_mkldnn() {
948948
export MXNET_SUBGRAPH_VERBOSE=0
949949
export MXNET_ENABLE_CYTHON=0
950950
export DMLC_LOG_STACK_TRACE_DEPTH=10
951-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
951+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose -s --log-cli-level=DEBUG tests/python/unittest
952952
MXNET_ENGINE_TYPE=NaiveEngine \
953-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
954-
pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
955-
pytest --durations=50 --cov-report xml:tests_mkl.xml --verbose tests/python/mkl
953+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/unittest
954+
pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/unittest
955+
pytest --durations=50 --cov-report xml:tests_mkl.xml --verbose -s --log-cli-level=DEBUG tests/python/mkl
956956
}
957957

958958
unittest_ubuntu_python3_gpu() {
@@ -965,11 +965,11 @@ unittest_ubuntu_python3_gpu() {
965965
export MXNET_ENABLE_CYTHON=0
966966
export DMLC_LOG_STACK_TRACE_DEPTH=10
967967
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
968-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
968+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose -s --log-cli-level=DEBUG tests/python/gpu
969969
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
970970
MXNET_ENGINE_TYPE=NaiveEngine \
971-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
972-
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
971+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
972+
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
973973
}
974974

975975
unittest_ubuntu_python3_gpu_cython() {
@@ -984,11 +984,11 @@ unittest_ubuntu_python3_gpu_cython() {
984984
export DMLC_LOG_STACK_TRACE_DEPTH=10
985985
check_cython
986986
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
987-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
987+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose -s --log-cli-level=DEBUG tests/python/gpu
988988
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
989989
MXNET_ENGINE_TYPE=NaiveEngine \
990-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
991-
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
990+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
991+
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
992992
}
993993

994994
unittest_ubuntu_python3_gpu_nocudnn() {
@@ -1000,11 +1000,11 @@ unittest_ubuntu_python3_gpu_nocudnn() {
10001000
export MXNET_ENABLE_CYTHON=0
10011001
export DMLC_LOG_STACK_TRACE_DEPTH=10
10021002
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
1003-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
1003+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose -s --log-cli-level=DEBUG tests/python/gpu
10041004
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
10051005
MXNET_ENGINE_TYPE=NaiveEngine \
1006-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1007-
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1006+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
1007+
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
10081008
}
10091009

10101010
unittest_cpp() {
@@ -1016,11 +1016,11 @@ unittest_centos7_cpu() {
10161016
set -ex
10171017
source /opt/rh/rh-python36/enable
10181018
cd /work/mxnet
1019-
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
1019+
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose -s --log-cli-level=DEBUG tests/python/unittest
10201020
MXNET_ENGINE_TYPE=NaiveEngine \
1021-
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
1022-
python -m pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
1023-
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -n 4 --durations=50 --cov-report xml:tests_train.xml --verbose tests/python/train
1021+
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/unittest
1022+
python -m pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/unittest
1023+
OMP_NUM_THREADS=$(expr $(nproc) / 4) python -m pytest -n 4 --durations=50 --cov-report xml:tests_train.xml --verbose -s --log-cli-level=DEBUG tests/python/train
10241024
}
10251025

10261026
unittest_centos7_gpu() {
@@ -1030,11 +1030,11 @@ unittest_centos7_gpu() {
10301030
export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}
10311031
export DMLC_LOG_STACK_TRACE_DEPTH=10
10321032
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
1033-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1033+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
10341034
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
10351035
MXNET_ENGINE_TYPE=NaiveEngine \
1036-
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1037-
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1036+
OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
1037+
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
10381038
}
10391039

10401040
integrationtest_ubuntu_cpu_onnx() {
@@ -1120,10 +1120,10 @@ test_ubuntu_cpu_python3() {
11201120
cd /work/mxnet/python
11211121
pip3 install -e .
11221122
cd /work/mxnet
1123-
OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose tests/python/unittest
1123+
OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/unittest
11241124
MXNET_ENGINE_TYPE=NaiveEngine \
1125-
OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/unittest
1126-
python3 -m pytest -m 'serial' --durations=50 --verbose tests/python/unittest
1125+
OMP_NUM_THREADS=$(expr $(nproc) / 4) python3 -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/unittest
1126+
python3 -m pytest -m 'serial' --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/unittest
11271127

11281128
popd
11291129
}
@@ -1137,7 +1137,7 @@ unittest_ubuntu_python3_arm() {
11371137
export MXNET_SUBGRAPH_VERBOSE=0
11381138
export MXNET_ENABLE_CYTHON=0
11391139
export DMLC_LOG_STACK_TRACE_DEPTH=10
1140-
python3 -m pytest -n 2 --verbose tests/python/unittest/test_engine.py
1140+
python3 -m pytest -n 2 --verbose -s --log-cli-level=DEBUG tests/python/unittest/test_engine.py
11411141
}
11421142

11431143
# Functions that run the nightly Tests:

0 commit comments

Comments
 (0)