From e1ba3164dc16ffd90eb234be55874c34efb9b04e Mon Sep 17 00:00:00 2001 From: driazati <9407960+driazati@users.noreply.github.com> Date: Thu, 25 Aug 2022 10:55:58 -0700 Subject: [PATCH] [ci] Default to n=2 for test parallelism (#12414) * Revert "[skip ci] Revert "[ci] Default to n=2 for test parallelism (#12376)" (#12413)" This reverts commit 478b672f2b7bb37f529fa6477b3c4ac353217b7a. * [ci] Default to n=2 for test parallelism This is attempt #2 of #12376 which was reverted in #12413. The changes in `plugin.py` should keep all the tests on the same node so sporadic failures don't happen due to scheduling. Co-authored-by: driazati --- Jenkinsfile | 60 +++++++++++++++++++++++++++++-- ci/jenkins/Jenkinsfile.j2 | 2 +- ci/jenkins/macros.j2 | 3 ++ python/tvm/testing/plugin.py | 42 ++++++++++++++++++++++ tests/scripts/setup-pytest-env.sh | 8 ++++- 5 files changed, 111 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 15cd4927d0ba..8c1ce9ed5020 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-08-15T16:55:31.189354 +// Generated at 2022-08-19T15:38:38.311410 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -97,7 +97,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS' +docker_run = 'docker/bash.sh --env CI --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 @@ -610,6 +610,7 @@ def lint() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'TVM_NUM_SHARDS=2', + 'TEST_STEP_NAME=Lint', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh ( @@ -629,6 +630,7 @@ def lint() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'TVM_NUM_SHARDS=2', + 'TEST_STEP_NAME=Lint', 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh ( @@ -1225,6 +1227,7 @@ def shard_run_unittest_GPU_1_of_3() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=unittest: GPU', 'TVM_NUM_SHARDS=3', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -1340,6 +1343,7 @@ def shard_run_unittest_GPU_2_of_3() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=unittest: GPU', 'TVM_NUM_SHARDS=3', 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -1421,6 +1425,7 @@ def shard_run_unittest_GPU_3_of_3() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=unittest: GPU', 'TVM_NUM_SHARDS=3', 'TVM_SHARD_INDEX=2', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -1499,6 +1504,7 @@ def shard_run_integration_CPU_1_of_10() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', + 'TEST_STEP_NAME=integration: CPU', 'TVM_NUM_SHARDS=10', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -1574,6 +1580,7 @@ def shard_run_integration_CPU_2_of_10() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', + 'TEST_STEP_NAME=integration: CPU', 'TVM_NUM_SHARDS=10', 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -1649,6 +1656,7 @@ def shard_run_integration_CPU_3_of_10() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', + 'TEST_STEP_NAME=integration: CPU', 'TVM_NUM_SHARDS=10', 'TVM_SHARD_INDEX=2', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -1724,6 +1732,7 @@ def shard_run_integration_CPU_4_of_10() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', + 'TEST_STEP_NAME=integration: CPU', 'TVM_NUM_SHARDS=10', 'TVM_SHARD_INDEX=3', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -1799,6 +1808,7 @@ def shard_run_integration_CPU_5_of_10() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', + 'TEST_STEP_NAME=integration: CPU', 'TVM_NUM_SHARDS=10', 'TVM_SHARD_INDEX=4', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -1874,6 +1884,7 @@ def shard_run_integration_CPU_6_of_10() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', + 'TEST_STEP_NAME=integration: CPU', 'TVM_NUM_SHARDS=10', 'TVM_SHARD_INDEX=5', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -1949,6 +1960,7 @@ def shard_run_integration_CPU_7_of_10() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', + 'TEST_STEP_NAME=integration: CPU', 'TVM_NUM_SHARDS=10', 'TVM_SHARD_INDEX=6', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2024,6 +2036,7 @@ def shard_run_integration_CPU_8_of_10() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', + 'TEST_STEP_NAME=integration: CPU', 'TVM_NUM_SHARDS=10', 'TVM_SHARD_INDEX=7', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2099,6 +2112,7 @@ def shard_run_integration_CPU_9_of_10() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', + 'TEST_STEP_NAME=integration: CPU', 'TVM_NUM_SHARDS=10', 'TVM_SHARD_INDEX=8', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2174,6 +2188,7 @@ def shard_run_integration_CPU_10_of_10() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cpu', + 'TEST_STEP_NAME=integration: CPU', 'TVM_NUM_SHARDS=10', 'TVM_SHARD_INDEX=9', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2250,6 +2265,7 @@ def shard_run_python_i386_1_of_5() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=i386', + 'TEST_STEP_NAME=python: i386', 'TVM_NUM_SHARDS=5', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2325,6 +2341,7 @@ def shard_run_python_i386_2_of_5() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=i386', + 'TEST_STEP_NAME=python: i386', 'TVM_NUM_SHARDS=5', 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2400,6 +2417,7 @@ def shard_run_python_i386_3_of_5() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=i386', + 'TEST_STEP_NAME=python: i386', 'TVM_NUM_SHARDS=5', 'TVM_SHARD_INDEX=2', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2474,6 +2492,7 @@ def shard_run_python_i386_4_of_5() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=i386', + 'TEST_STEP_NAME=python: i386', 'TVM_NUM_SHARDS=5', 'TVM_SHARD_INDEX=3', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2548,6 +2567,7 @@ def shard_run_python_i386_5_of_5() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=i386', + 'TEST_STEP_NAME=python: i386', 'TVM_NUM_SHARDS=5', 'TVM_SHARD_INDEX=4', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2623,6 +2643,7 @@ def shard_run_test_Hexagon_1_of_7() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', + 'TEST_STEP_NAME=test: Hexagon', 'TVM_NUM_SHARDS=7', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2697,6 +2718,7 @@ def shard_run_test_Hexagon_2_of_7() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', + 'TEST_STEP_NAME=test: Hexagon', 'TVM_NUM_SHARDS=7', 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2770,6 +2792,7 @@ def shard_run_test_Hexagon_3_of_7() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', + 'TEST_STEP_NAME=test: Hexagon', 'TVM_NUM_SHARDS=7', 'TVM_SHARD_INDEX=2', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2843,6 +2866,7 @@ def shard_run_test_Hexagon_4_of_7() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', + 'TEST_STEP_NAME=test: Hexagon', 'TVM_NUM_SHARDS=7', 'TVM_SHARD_INDEX=3', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2916,6 +2940,7 @@ def shard_run_test_Hexagon_5_of_7() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', + 'TEST_STEP_NAME=test: Hexagon', 'TVM_NUM_SHARDS=7', 'TVM_SHARD_INDEX=4', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -2989,6 +3014,7 @@ def shard_run_test_Hexagon_6_of_7() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', + 'TEST_STEP_NAME=test: Hexagon', 'TVM_NUM_SHARDS=7', 'TVM_SHARD_INDEX=5', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3062,6 +3088,7 @@ def shard_run_test_Hexagon_7_of_7() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=hexagon', + 'TEST_STEP_NAME=test: Hexagon', 'TVM_NUM_SHARDS=7', 'TVM_SHARD_INDEX=6', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3136,6 +3163,7 @@ def shard_run_integration_aarch64_1_of_4() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', + 'TEST_STEP_NAME=integration: aarch64', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3210,6 +3238,7 @@ def shard_run_integration_aarch64_2_of_4() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', + 'TEST_STEP_NAME=integration: aarch64', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3284,6 +3313,7 @@ def shard_run_integration_aarch64_3_of_4() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', + 'TEST_STEP_NAME=integration: aarch64', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=2', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3358,6 +3388,7 @@ def shard_run_integration_aarch64_4_of_4() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', + 'TEST_STEP_NAME=integration: aarch64', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=3', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3433,6 +3464,7 @@ def shard_run_topi_GPU_1_of_4() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=topi: GPU', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3506,6 +3538,7 @@ def shard_run_topi_GPU_2_of_4() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=topi: GPU', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3579,6 +3612,7 @@ def shard_run_topi_GPU_3_of_4() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=topi: GPU', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=2', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3652,6 +3686,7 @@ def shard_run_topi_GPU_4_of_4() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=topi: GPU', 'TVM_NUM_SHARDS=4', 'TVM_SHARD_INDEX=3', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3726,6 +3761,7 @@ def shard_run_frontend_GPU_1_of_6() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=frontend: GPU', 'TVM_NUM_SHARDS=6', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3799,6 +3835,7 @@ def shard_run_frontend_GPU_2_of_6() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=frontend: GPU', 'TVM_NUM_SHARDS=6', 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3872,6 +3909,7 @@ def shard_run_frontend_GPU_3_of_6() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=frontend: GPU', 'TVM_NUM_SHARDS=6', 'TVM_SHARD_INDEX=2', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -3945,6 +3983,7 @@ def shard_run_frontend_GPU_4_of_6() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=frontend: GPU', 'TVM_NUM_SHARDS=6', 'TVM_SHARD_INDEX=3', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4018,6 +4057,7 @@ def shard_run_frontend_GPU_5_of_6() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=frontend: GPU', 'TVM_NUM_SHARDS=6', 'TVM_SHARD_INDEX=4', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4091,6 +4131,7 @@ def shard_run_frontend_GPU_6_of_6() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=gpu', + 'TEST_STEP_NAME=frontend: GPU', 'TVM_NUM_SHARDS=6', 'TVM_SHARD_INDEX=5', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4165,6 +4206,7 @@ def shard_run_topi_aarch64_1_of_2() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', + 'TEST_STEP_NAME=topi: aarch64', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4243,6 +4285,7 @@ def shard_run_topi_aarch64_2_of_2() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', + 'TEST_STEP_NAME=topi: aarch64', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4321,6 +4364,7 @@ def shard_run_frontend_aarch64_1_of_2() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', + 'TEST_STEP_NAME=frontend: aarch64', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4394,6 +4438,7 @@ def shard_run_frontend_aarch64_2_of_2() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=arm', + 'TEST_STEP_NAME=frontend: aarch64', 'TVM_NUM_SHARDS=2', 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4468,6 +4513,7 @@ def shard_run_test_Cortex_M_1_of_8() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', + 'TEST_STEP_NAME=test: Cortex-M', 'TVM_NUM_SHARDS=8', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4546,6 +4592,7 @@ def shard_run_test_Cortex_M_2_of_8() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', + 'TEST_STEP_NAME=test: Cortex-M', 'TVM_NUM_SHARDS=8', 'TVM_SHARD_INDEX=1', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4619,6 +4666,7 @@ def shard_run_test_Cortex_M_3_of_8() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', + 'TEST_STEP_NAME=test: Cortex-M', 'TVM_NUM_SHARDS=8', 'TVM_SHARD_INDEX=2', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4692,6 +4740,7 @@ def shard_run_test_Cortex_M_4_of_8() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', + 'TEST_STEP_NAME=test: Cortex-M', 'TVM_NUM_SHARDS=8', 'TVM_SHARD_INDEX=3', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4765,6 +4814,7 @@ def shard_run_test_Cortex_M_5_of_8() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', + 'TEST_STEP_NAME=test: Cortex-M', 'TVM_NUM_SHARDS=8', 'TVM_SHARD_INDEX=4', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4838,6 +4888,7 @@ def shard_run_test_Cortex_M_6_of_8() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', + 'TEST_STEP_NAME=test: Cortex-M', 'TVM_NUM_SHARDS=8', 'TVM_SHARD_INDEX=5', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4911,6 +4962,7 @@ def shard_run_test_Cortex_M_7_of_8() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', + 'TEST_STEP_NAME=test: Cortex-M', 'TVM_NUM_SHARDS=8', 'TVM_SHARD_INDEX=6', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -4984,6 +5036,7 @@ def shard_run_test_Cortex_M_8_of_8() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=cortexm', + 'TEST_STEP_NAME=test: Cortex-M', 'TVM_NUM_SHARDS=8', 'TVM_SHARD_INDEX=7', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -5058,6 +5111,7 @@ def shard_run_test_RISC_V_1_of_1() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM=riscv', + 'TEST_STEP_NAME=test: RISC-V', 'TVM_NUM_SHARDS=1', 'TVM_SHARD_INDEX=0', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -5361,6 +5415,7 @@ stage('Test') { docker_init(ci_cpu) init_git() withEnv(['PLATFORM=cpu', + 'TEST_STEP_NAME=unittest: CPU', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: """ @@ -5435,6 +5490,7 @@ stage('Test') { docker_init(ci_cpu) init_git() withEnv(['PLATFORM=cpu', + 'TEST_STEP_NAME=frontend: CPU', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { sh( script: """ diff --git a/ci/jenkins/Jenkinsfile.j2 b/ci/jenkins/Jenkinsfile.j2 index 4960d4f0fa57..be2776c6d9e3 100644 --- a/ci/jenkins/Jenkinsfile.j2 +++ b/ci/jenkins/Jenkinsfile.j2 @@ -85,7 +85,7 @@ properties([ upstream_revision = null // command to start a docker container -docker_run = 'docker/bash.sh --env CI --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS' +docker_run = 'docker/bash.sh --env CI --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME' docker_build = 'docker/build.sh' // timeout in minutes max_time = 180 diff --git a/ci/jenkins/macros.j2 b/ci/jenkins/macros.j2 index dbd6ac551db4..9d02ad68d6da 100644 --- a/ci/jenkins/macros.j2 +++ b/ci/jenkins/macros.j2 @@ -44,6 +44,7 @@ def {{ method_name }}() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'PLATFORM={{ platform }}', + 'TEST_STEP_NAME={{ name }}', 'TVM_NUM_SHARDS={{ num_shards }}', 'TVM_SHARD_INDEX={{ shard_index - 1 }}', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { @@ -75,6 +76,7 @@ def {{ method_name }}() { timeout(time: max_time, unit: 'MINUTES') { withEnv([ 'TVM_NUM_SHARDS={{ num_shards }}', + 'TEST_STEP_NAME={{ name }}', 'TVM_SHARD_INDEX={{ shard_index - 1 }}', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { {{ caller() | trim | indent(width=6) }} @@ -121,6 +123,7 @@ def {{ method_name }}() { docker_init({{ docker_image }}) init_git() withEnv(['PLATFORM={{ platform }}', + 'TEST_STEP_NAME={{ name }}', "SKIP_SLOW_TESTS=${skip_slow_tests}"], { {{ caller() | indent(width=12) | trim }} }) diff --git a/python/tvm/testing/plugin.py b/python/tvm/testing/plugin.py index 1f4f983b7210..2d845b70ff11 100644 --- a/python/tvm/testing/plugin.py +++ b/python/tvm/testing/plugin.py @@ -37,6 +37,13 @@ import tvm from tvm.testing import utils +try: + from xdist.scheduler.loadscope import LoadScopeScheduling + + HAVE_XDIST = True +except ImportError: + HAVE_XDIST = False + MARKERS = { "gpu": "mark a test as requiring a gpu", @@ -319,3 +326,38 @@ def _parametrize_correlated_parameters(metafunc): names = ",".join(name for name, values in params) value_sets = zip(*[values for name, values in params]) metafunc.parametrize(names, value_sets, indirect=True, ids=ids) + + +# pytest-xdist isn't required but is used in CI, so guard on its presence +if HAVE_XDIST: + + def pytest_xdist_make_scheduler(config, log): + """ + Serialize certain tests for pytest-xdist that have inter-test + dependencies + """ + + class TvmTestScheduler(LoadScopeScheduling): + """ + Scheduler to serializer tests + """ + + def _split_scope(self, nodeid): + """ + Returns a specific string for classes of nodeids + """ + # NOTE: these tests contain inter-test dependencies and must be + # serialized + items = { + "test_tvm_testing_features": "functional-tests", + "tests/python/unittest/test_crt": "crt-tests", + "tests/python/driver/tvmc": "tvmc-tests", + } + + for nodeid_pattern, suite_name in items.items(): + if nodeid_pattern in nodeid: + return suite_name + + return nodeid + + return TvmTestScheduler(config, log) diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh index afb759c09356..d6c49a42819a 100755 --- a/tests/scripts/setup-pytest-env.sh +++ b/tests/scripts/setup-pytest-env.sh @@ -74,8 +74,14 @@ function run_pytest() { suite_name="${test_suite_name}-${current_shard}-${ffi_type}" + # Some test environments don't play well with parallelism + DEFAULT_PARALLELISM=2 + if [[ "${TEST_STEP_NAME:-default}" == "frontend: GPU"* ]] || [[ "${TEST_STEP_NAME:-default}" == "test: Hexagon"* ]]; then + DEFAULT_PARALLELISM=1 + fi + if [ ! "${extra_args[@]}" == *" -n"* ] && [! "${extra_args[@]}" == *" -dist"* ]; then - extra_args+=("-n=1") + extra_args+=("-n=$DEFAULT_PARALLELISM") fi exit_code=0