clearer definition of flaky tests (#9693)

qgis · Apr 3, 2019 · 7410fb0 · 7410fb0
1 parent e043954
commit 7410fb0
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 15 deletions.
diff --git a/.ci/travis/linux/scripts/docker-qgis-test.sh b/.ci/travis/linux/scripts/docker-qgis-test.sh
@@ -65,7 +65,15 @@ EOT
 CURRENT_TIME=$(date +%s)
 TIMEOUT=$((( TRAVIS_AVAILABLE_TIME - TRAVIS_UPLOAD_TIME) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
 echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
-timeout ${TIMEOUT}s python3 /root/QGIS/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V -E "$(cat /root/QGIS/.ci/travis/linux/scripts/test_blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S /root/QGIS/.ci/travis/travis.ctest --output-on-failure
+EXCLUDE_TESTS=$(cat /root/QGIS/.ci/travis/linux/scripts/test_blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)
+if ! [[ ${RUN_FLAKY_TESTS} =~ ^true$ ]]; then
+  echo "Flaky tests are skipped!"
+  EXCLUDE_TESTS=${EXCLUDE_TESTS}"|"$(cat /root/QGIS/.ci/travis/linux/scripts/test_flaky.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)
+else
+  echo "Flaky tests are run!"
+fi
+echo "List of skipped tests: $EXCLUDE_TESTS"
+timeout ${TIMEOUT}s python3 /root/QGIS/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V -E "${EXCLUDE_TESTS}" -S /root/QGIS/.ci/travis/travis.ctest --output-on-failure
 rv=$?
 if [ $rv -eq 124 ] ; then
     printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'

diff --git a/.ci/travis/linux/scripts/test_blacklist.txt b/.ci/travis/linux/scripts/test_blacklist.txt
@@ -11,20 +11,6 @@ qgis_sipify
 qgis_sip_include
 qgis_sip_uptodate
 
-# flaky
-qgis_filedownloader
-qgis_openstreetmaptest
-qgis_wcsprovidertest
-PyQgsWFSProviderGUI
-qgis_ziplayertest
-
-# Flaky, see https://dash.orfeo-toolbox.org/testDetails.php?test=63061783&build=297405
-PyQgsSpatialiteProvider
-
-# Flaky, the ms odbc driver crashes a lot on the ubuntu docker image. Retest when
-# the docker base image is upgraded
-PyQgsMssqlProvider
-
 # Need a local postgres installation
 PyQgsAuthManagerPKIPostgresTest
 PyQgsAuthManagerPasswordPostgresTest

diff --git a/.ci/travis/linux/scripts/test_flaky.txt b/.ci/travis/linux/scripts/test_flaky.txt
@@ -0,0 +1,13 @@
+# flaky
+qgis_filedownloader
+qgis_wcsprovidertest
+PyQgsWFSProviderGUI
+qgis_ziplayertest
+
+# Flaky, see https://dash.orfeo-toolbox.org/testDetails.php?test=63061783&build=297405
+PyQgsSpatialiteProvider
+
+# Flaky, the ms odbc driver crashes a lot on the ubuntu docker image. Retest when
+# the docker base image is upgraded
+PyQgsMssqlProvider
+