[TRAVIS] run flaky test when label is defined (#9509)

3nids · web-flow · commit 7fb752e0ff00 · 2019-03-27T09:45:34.000-05:00
to declare a test as flaky:

* for cpp, use

```
if ( !QgsTest::runFlakyTests() )
    QSKIP( "This test is disabled on Travis CI environment" );
```

* for Python, you can use `RUN_FLAKY_TEST` environment variable
diff --git a/.ci/travis/linux/docker-variables.env b/.ci/travis/linux/docker-variables.env
@@ -1,6 +1,6 @@
 
 # Env variables for Docker
-# These without assignment are taken from the host env variabbles
+# These without assignment are taken from the host env variables
 
 # TRAVIS variables
 TRAVIS_AVAILABLE_TIME
@@ -11,6 +11,7 @@ TRAVIS_PULL_REQUEST
 TRAVIS_OS_NAME
 TRAVIS_CONFIG
 TRAVIS
+RUN_FLAKY_TESTS
 
 # CTEST
 LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
diff --git a/.ci/travis/linux/script.sh b/.ci/travis/linux/script.sh
@@ -16,6 +16,8 @@
 
 set -e
 
+echo "Running flaky test: ${RUN_FLAKY_TESTS}"
+
 # build QGIS in docker
 echo "travis_fold:start:docker_build_qgis"
 echo "${bold}Docker build QGIS${endbold}"
diff --git a/.ci/travis/scripts/pr_has_label.py b/.ci/travis/scripts/pr_has_label.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+
+import sys
+import json
+from urllib.request import urlopen # using urllib since it is a standard module (vs. requests)
+from urllib.error import URLError
+import argparse
+
+parser = argparse.ArgumentParser(description='Determines if a pull request has a defined label')
+parser.add_argument('pull_request', type=int,
+                    help='pull request id')
+parser.add_argument('label', type=int,
+                    help='label ID')
+
+args = parser.parse_args()
+
+url = "https://api.github.com/repos/qgis/QGIS/pulls/{}".format(args.pull_request)
+
+try:
+    data = urlopen(url).read().decode('utf-8')
+except URLError as err:
+    print("URLError: ".format(err.reason))
+    sys.exit(1)
+
+obj = json.loads(data)
+
+for label in obj['labels']:
+    if label["id"] == args.label:
+        print("true")
+        sys.exit(0)
+
+print("label not found")
+sys.exit(1)
diff --git a/.travis.yml b/.travis.yml
@@ -11,7 +11,7 @@ cache:
     - ${HOME}/.ccache_docker_build_cosmic
     - ${HOME}/.ccache_docker_build_bionic
   timeout: 1000
-if: NOT branch =~ /^(cherry-pick-)?backport-\d+-on-/ AND NOT branch =~ /-patch-\d+$/ 
+if: NOT branch =~ /^(cherry-pick-)?backport-\d+-on-/ AND NOT branch =~ /-patch-\d+$/
 
 env:
   global:
@@ -46,6 +46,8 @@ matrix:
         - CCACHE_DIR=${HOME}/.ccache_testing
         - DOCKER_TAG=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && echo $TRAVIS_BRANCH | sed 's/master/latest/' || echo "latest" )
         - DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile
+        # Label ID can be found here https://api.github.com/repos/qgis/QGIS/labels
+        - RUN_FLAKY_TESTS=$(.ci/travis/scripts/pr_has_label.py $TRAVIS_PULL_REQUEST 1271248184)
 
       ##########################################################
       # CODE LAYOUT
diff --git a/src/test/qgstest.h b/src/test/qgstest.h
@@ -83,6 +83,11 @@ namespace QgsTest
   {
     return qgetenv( "TRAVIS" ) == QStringLiteral( "true" );
   }
+
+  bool runFlakyTests()
+  {
+    return qgetenv( "RUN_FLAKY_TESTS" ) == QStringLiteral( "true" );
+  }
 }
 
 #endif // QGSTEST_H
diff --git a/tests/src/providers/testqgsogrprovider.cpp b/tests/src/providers/testqgsogrprovider.cpp
@@ -187,7 +187,7 @@ void TestQgsOgrProvider::testThread()
   // Disabled by @m-kuhn
   // This test is flaky
   // See https://travis-ci.org/qgis/QGIS/jobs/505008602#L6464-L7108
-  if ( QgsTest::isTravis() )
+  if ( !QgsTest::runFlakyTests() )
     QSKIP( "This test is disabled on Travis CI environment" );
 
   // After reading a QgsVectorLayer (getFeatures) from another thread the QgsOgrConnPoolGroup starts

Original file line number	Diff line number	Diff line change
`@@ -83,6 +83,11 @@ namespace QgsTest`
`83`	`83`	`{`
`84`	`84`	`return qgetenv( "TRAVIS" ) == QStringLiteral( "true" );`
`85`	`85`	`}`
	`86`	`+`
	`87`	`+ bool runFlakyTests()`
	`88`	`+ {`
	`89`	`+ return qgetenv( "RUN_FLAKY_TESTS" ) == QStringLiteral( "true" );`
	`90`	`+ }`
`86`	`91`	`}`
`87`	`92`
`88`	`93`	`#endif // QGSTEST_H`