Skip to content
This repository was archived by the owner on Nov 17, 2023. It is now read-only.

Commit f60eff2

Browse files
DickJC123DickJC123
committed
Temporarily log all CI seeds to troubleshoot seed non-determinism
1 parent a282eb5 commit f60eff2

File tree

1 file changed

+32
-32
lines changed

1 file changed

+32
-32
lines changed

‎ci/docker/runtime_functions.sh‎

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -899,8 +899,8 @@ cd_unittest_ubuntu() {
899899

900900
local mxnet_variant=${1:?"This function requires a mxnet variant as the first argument"}
901901

902-
pytest -m 'not serial' -n 4 --durations=50 --verbose tests/python/unittest
903-
pytest -m 'serial' --durations=50 --verbose tests/python/unittest
902+
pytest -m 'not serial' -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/unittest
903+
pytest -m 'serial' --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/unittest
904904

905905
# https://github.com/apache/incubator-mxnet/issues/11801
906906
# if [[ ${mxnet_variant} = "cpu" ]] || [[ ${mxnet_variant} = "mkl" ]]; then
@@ -910,10 +910,10 @@ cd_unittest_ubuntu() {
910910
if [[ ${mxnet_variant} = cu* ]]; then
911911
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
912912
MXNET_ENGINE_TYPE=NaiveEngine \
913-
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/gpu
913+
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/gpu
914914
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
915-
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose tests/python/gpu
916-
pytest -m 'serial' --durations=50 --verbose tests/python/gpu
915+
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/gpu
916+
pytest -m 'serial' --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/gpu
917917

918918
# TODO(szha): fix and reenable the hanging issue. tracked in #18098
919919
# integrationtest_ubuntu_gpu_dist_kvstore
@@ -922,7 +922,7 @@ cd_unittest_ubuntu() {
922922
fi
923923

924924
if [[ ${mxnet_variant} = *mkl ]]; then
925-
pytest -n 4 --durations=50 --verbose tests/python/mkl
925+
pytest -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/mkl
926926
fi
927927
}
928928

@@ -934,10 +934,10 @@ unittest_ubuntu_python3_cpu() {
934934
export MXNET_SUBGRAPH_VERBOSE=0
935935
export MXNET_ENABLE_CYTHON=0
936936
export DMLC_LOG_STACK_TRACE_DEPTH=10
937-
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
937+
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose -s --log-cli-level=DEBUG tests/python/unittest
938938
MXNET_ENGINE_TYPE=NaiveEngine \
939-
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
940-
pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
939+
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/unittest
940+
pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/unittest
941941
}
942942

943943
unittest_ubuntu_python3_cpu_serial() {
@@ -949,7 +949,7 @@ unittest_ubuntu_python3_cpu_serial() {
949949
export MXNET_SUBGRAPH_VERBOSE=0
950950
export MXNET_ENABLE_CYTHON=0
951951
export DMLC_LOG_STACK_TRACE_DEPTH=10
952-
pytest --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
952+
pytest --durations=50 --cov-report xml:tests_unittest.xml --verbose -s --log-cli-level=DEBUG tests/python/unittest
953953
}
954954

955955
unittest_ubuntu_python3_cpu_mkldnn() {
@@ -961,8 +961,8 @@ unittest_ubuntu_python3_cpu_mkldnn() {
961961
export MXNET_ENABLE_CYTHON=0
962962
export DMLC_LOG_STACK_TRACE_DEPTH=10
963963
# TODO(szha): enable parallel testing and naive engine for ops once #18244 is fixed
964-
pytest --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
965-
pytest --durations=50 --cov-report xml:tests_mkl.xml --verbose tests/python/mkl
964+
pytest --durations=50 --cov-report xml:tests_unittest.xml --verbose -s --log-cli-level=DEBUG tests/python/unittest
965+
pytest --durations=50 --cov-report xml:tests_mkl.xml --verbose -s --log-cli-level=DEBUG tests/python/mkl
966966
}
967967

968968
unittest_ubuntu_python3_gpu() {
@@ -975,11 +975,11 @@ unittest_ubuntu_python3_gpu() {
975975
export MXNET_ENABLE_CYTHON=0
976976
export DMLC_LOG_STACK_TRACE_DEPTH=10
977977
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
978-
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
978+
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose -s --log-cli-level=DEBUG tests/python/gpu
979979
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
980980
MXNET_ENGINE_TYPE=NaiveEngine \
981-
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
982-
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
981+
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append -s --verbose --log-cli-level=DEBUG tests/python/gpu
982+
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
983983
}
984984

985985
unittest_ubuntu_python3_gpu_cython() {
@@ -994,11 +994,11 @@ unittest_ubuntu_python3_gpu_cython() {
994994
export DMLC_LOG_STACK_TRACE_DEPTH=10
995995
check_cython
996996
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
997-
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
997+
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose -s --log-cli-level=DEBUG tests/python/gpu
998998
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
999999
MXNET_ENGINE_TYPE=NaiveEngine \
1000-
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1001-
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1000+
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
1001+
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
10021002
}
10031003

10041004
unittest_ubuntu_python3_gpu_nocudnn() {
@@ -1010,11 +1010,11 @@ unittest_ubuntu_python3_gpu_nocudnn() {
10101010
export MXNET_ENABLE_CYTHON=0
10111011
export DMLC_LOG_STACK_TRACE_DEPTH=10
10121012
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
1013-
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
1013+
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --verbose -s --log-cli-level=DEBUG tests/python/gpu
10141014
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
10151015
MXNET_ENGINE_TYPE=NaiveEngine \
1016-
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1017-
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1016+
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
1017+
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
10181018
}
10191019

10201020
unittest_cpp() {
@@ -1026,11 +1026,11 @@ unittest_centos7_cpu() {
10261026
set -ex
10271027
source /opt/rh/rh-python36/enable
10281028
cd /work/mxnet
1029-
python -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
1029+
python -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --verbose -s --log-cli-level=DEBUG tests/python/unittest
10301030
MXNET_ENGINE_TYPE=NaiveEngine \
1031-
python -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
1032-
python -m pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
1033-
python -m pytest -n 4 --durations=50 --cov-report xml:tests_train.xml --verbose tests/python/train
1031+
python -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/unittest
1032+
python -m pytest -m 'serial' --durations=50 --cov-report xml:tests_unittest.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/unittest
1033+
python -m pytest -n 4 --durations=50 --cov-report xml:tests_train.xml --verbose -s --log-cli-level=DEBUG tests/python/train
10341034
}
10351035

10361036
unittest_centos7_gpu() {
@@ -1040,11 +1040,11 @@ unittest_centos7_gpu() {
10401040
export CUDNN_VERSION=${CUDNN_VERSION:-7.0.3}
10411041
export DMLC_LOG_STACK_TRACE_DEPTH=10
10421042
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
1043-
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1043+
pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
10441044
MXNET_GPU_MEM_POOL_TYPE=Unpooled \
10451045
MXNET_ENGINE_TYPE=NaiveEngine \
1046-
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1047-
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
1046+
pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
1047+
pytest -m 'serial' --durations=50 --cov-report xml:tests_gpu.xml --cov-append --verbose -s --log-cli-level=DEBUG tests/python/gpu
10481048
}
10491049

10501050
integrationtest_ubuntu_cpu_onnx() {
@@ -1130,10 +1130,10 @@ test_ubuntu_cpu_python3() {
11301130
cd /work/mxnet/python
11311131
pip3 install -e .
11321132
cd /work/mxnet
1133-
python3 -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose tests/python/unittest
1133+
python3 -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/unittest
11341134
MXNET_ENGINE_TYPE=NaiveEngine \
1135-
python3 -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose tests/python/unittest
1136-
python3 -m pytest -m 'serial' --durations=50 --verbose tests/python/unittest
1135+
python3 -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/unittest
1136+
python3 -m pytest -m 'serial' --durations=50 --verbose -s --log-cli-level=DEBUG tests/python/unittest
11371137

11381138
popd
11391139
}
@@ -1147,7 +1147,7 @@ unittest_ubuntu_python3_arm() {
11471147
export MXNET_SUBGRAPH_VERBOSE=0
11481148
export MXNET_ENABLE_CYTHON=0
11491149
export DMLC_LOG_STACK_TRACE_DEPTH=10
1150-
python3 -m pytest -n 2 --verbose tests/python/unittest/test_engine.py
1150+
python3 -m pytest -n 2 --verbose -s --log-cli-level=DEBUG tests/python/unittest/test_engine.py
11511151
}
11521152

11531153
# Functions that run the nightly Tests:

0 commit comments

Comments
 (0)