Skip to content

Commit

Permalink
Backport #9387 on release-3_6 (#9391)
Browse files Browse the repository at this point in the history 
* simplify CI scripts

by separating the testing and docker image building scripts
there was almost no redundany in the scripts, it is much clearer now

* update headers

* fix script path

* start container
  • Loading branch information
@backporting @3nids
backporting[bot] authored and 3nids committed Mar 5, 2019
1 parent 03fcded commit 2ca9575
Show file tree
Hide file tree
Showing 9 changed files with 164 additions and 89 deletions.
16 changes: 16 additions & 0 deletions .ci/travis/docker_image/before_install.sh
View file
@@ -0,0 +1,16 @@
#!/usr/bin/env bash
###########################################################################
# before_install.sh
# ---------------------
# Date : March 2019
# Copyright : (C) 2019 by Denis Rouzaud
# Email : denis@opengis.ch
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################

36 changes: 36 additions & 0 deletions .ci/travis/docker_image/before_script.sh
View file
@@ -0,0 +1,36 @@
#!/usr/bin/env bash
###########################################################################
# before_install.sh
# ---------------------
# Date : March 2019
# Copyright : (C) 2019 by Denis Rouzaud
# Email : denis@opengis.ch
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################


set -e

.ci/travis/scripts/echo_travis_var.sh

pushd .docker



echo "travis_fold:start:docker_build"
echo "${bold}Docker build deps${endbold}"
docker --version

docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .

echo "travis_fold:end:docker_build"


popd
17 changes: 17 additions & 0 deletions .ci/travis/docker_image/install.sh
View file
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
###########################################################################
# before_install.sh
# ---------------------
# Date : March 2019
# Copyright : (C) 2019 by Denis Rouzaud
# Email : denis@opengis.ch
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################


59 changes: 59 additions & 0 deletions .ci/travis/docker_image/script.sh
View file
@@ -0,0 +1,59 @@
#!/usr/bin/env bash
###########################################################################
# before_install.sh
# ---------------------
# Date : March 2019
# Copyright : (C) 2019 by Denis Rouzaud
# Email : denis@opengis.ch
###########################################################################
# #
# This program is free software; you can redistribute it and/or modify #
# it under the terms of the GNU General Public License as published by #
# the Free Software Foundation; either version 2 of the License, or #
# (at your option) any later version. #
# #
###########################################################################


set -e

mkdir -p "CCACHE_DIR_IMAGE_BUILD"

# copy ccache dir within QGIS source so it can be accessed from docker
cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
# building docker images
DIR=$(git rev-parse --show-toplevel)/.docker
pushd "${DIR}"
echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
--cache-from "qgis/qgis:${DOCKER_TAG}" \
-t "qgis/qgis:${DOCKER_TAG}" \
-f qgis.dockerfile ..
echo "${bold}Pushing image to docker hub...${endbold}"
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
docker push "qgis/qgis:${DOCKER_TAG}"
echo "Copy build cache from Docker container to Travis cache directory"
rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
docker start ${container_id}
docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
popd
echo "Trigger build of PyQGIS Documentation"
body='{
"request": {
"branch":"master",
"message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
"config": {
"merge_mode": "deep_merge",
"matrix": {
"include": {
"env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
}
}
}
}
}'
body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
-H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests
1 change: 0 additions & 1 deletion .ci/travis/linux/before_install.sh
View file
Expand Up @@ -14,4 +14,3 @@
# #
###########################################################################

#pip3 install termcolor
32 changes: 6 additions & 26 deletions .ci/travis/linux/before_script.sh
View file
Expand Up @@ -16,39 +16,19 @@

set -e

pushd .docker

DOCKER_DEPS_PUSH=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && [[ "${TRAVIS_EVENT_TYPE}" != "pull_request" ]] && echo "true" || echo "false" )
DOCKER_DEPS_IMAGE_REBUILD=$( [[ $TRAVIS_COMMIT_MESSAGE =~ '[docker] update dependencies' ]] && echo "true" || echo "false" )

echo "travis_fold:start:travis_env"
echo "${bold}Travis environment variables${endbold}"
echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
echo "DOCKER_TAG: $DOCKER_TAG"
echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
echo "DOCKER_DEPS_IMAGE_REBUILD: $DOCKER_DEPS_IMAGE_REBUILD"
echo "DOCKER_BUILD_QGIS_IMAGE: $DOCKER_BUILD_QGIS_IMAGE"
echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
echo "travis_fold:end:travis_env"
.ci/travis/scripts/echo_travis_var.sh

pushd .docker

echo "travis_fold:start:docker_build"
echo "${bold}Docker build deps${endbold}"
docker --version
if [[ $DOCKER_BUILD_QGIS_IMAGE =~ false ]]; then
docker-compose --version
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
fi

if [[ $DOCKER_DEPS_IMAGE_REBUILD =~ true ]]; then
docker build --no-cache -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
else
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
fi
docker-compose --version
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
echo "travis_fold:end:docker_build"

echo "travis_fold:start:docker_push"
Expand Down
73 changes: 16 additions & 57 deletions .ci/travis/linux/script.sh
View file
Expand Up @@ -16,62 +16,21 @@

set -e

mkdir -p "CCACHE_DIR_IMAGE_BUILD"
# running QGIS tests
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps

if [[ ${DOCKER_BUILD_QGIS_IMAGE} =~ true ]]; then
# copy ccache dir within QGIS source so it can be accessed from docker
cp -r ${CCACHE_DIR_IMAGE_BUILD} ${TRAVIS_BUILD_DIR}/.ccache_image_build
# building docker images
DIR=$(git rev-parse --show-toplevel)/.docker
pushd "${DIR}"
echo "${bold}Building QGIS Docker image '${DOCKER_TAG}'...${endbold}"
docker build --build-arg DOCKER_TAG="${DOCKER_TAG}" \
--cache-from "qgis/qgis:${DOCKER_TAG}" \
-t "qgis/qgis:${DOCKER_TAG}" \
-f qgis.dockerfile ..
echo "${bold}Pushing image to docker hub...${endbold}"
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
docker push "qgis/qgis:${DOCKER_TAG}"
echo "Copy build cache from Docker container to Travis cache directory"
rm -rf "${CCACHE_DIR_IMAGE_BUILD:?}/"*
container_id=$(docker images -q "qgis/qgis:${DOCKER_TAG}")
docker cp ${container_id}:/usr/src/QGIS/.ccache_image_build ${CCACHE_DIR_IMAGE_BUILD}
popd
echo "Trigger build of PyQGIS Documentation"
body='{
"request": {
"branch":"master",
"message": "Trigger PyQGIS doc build after release of new Docker image as __DOCKER_TAG__",
"config": {
"merge_mode": "deep_merge",
"matrix": {
"include": {
"env": ["QGIS_VERSION_BRANCH=__QGIS_VERSION_BRANCH__"]
}
}
}
}
}'
body=$(sed "s/__QGIS_VERSION_BRANCH__/${TRAVIS_BRANCH}/; s/__DOCKER_TAG__/${DOCKER_TAG}/" <<< $body)
curl -s -X POST -H "Content-Type: application/json" -H "Accept: application/json" \
-H "Travis-API-Version: 3" -H "Authorization: token $TRAVIS_TOKEN" -d "$body" \
https://api.travis-ci.org/repo/qgis%2Fpyqgis/requests
else
# running QGIS tests
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
# running tests for the python test runner
docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
sleep 10 # Wait for xvfb to finish starting
# Temporary workaround until docker images are built
docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
# Run tests in the docker
# Passing cases:
TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
# Failing cases:
[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]

# running tests for the python test runner
docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
sleep 10 # Wait for xvfb to finish starting
# Temporary workaround until docker images are built
docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
# Run tests in the docker
# Passing cases:
TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
# Failing cases:
[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
fi
12 changes: 12 additions & 0 deletions .ci/travis/scripts/echo_travis_var.sh
View file
@@ -0,0 +1,12 @@
#!/usr/bin/env bash

echo "travis_fold:start:travis_env"
echo "${bold}Travis environment variables${endbold}"
echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
echo "DOCKER_TAG: $DOCKER_TAG"
echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
echo "travis_fold:end:travis_env"
7 changes: 2 additions & 5 deletions .travis.yml
View file
Expand Up @@ -43,7 +43,6 @@ matrix:
env:
- TRAVIS_CONFIG=linux
- DOCKER_TAG=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && echo $TRAVIS_BRANCH | sed 's/master/latest/' || echo "latest" )
- DOCKER_BUILD_QGIS_IMAGE=false
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile

##########################################################
Expand Down Expand Up @@ -104,9 +103,8 @@ matrix:
if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
services: docker
env:
- TRAVIS_CONFIG=linux
- TRAVIS_CONFIG=docker_image
- DOCKER_TAG=$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )
- DOCKER_BUILD_QGIS_IMAGE=true
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps.dockerfile

##########################################################
Expand All @@ -117,9 +115,8 @@ matrix:
if: repo = qgis/QGIS AND (tag IS PRESENT OR type = cron)
services: docker
env:
- TRAVIS_CONFIG=linux
- TRAVIS_CONFIG=docker_image
- DOCKER_TAG="$( echo $TRAVIS_BRANCH | sed 's/master/latest/' )_cosmic"
- DOCKER_BUILD_QGIS_IMAGE=true
- DOCKER_BUILD_DEPS_FILE=qgis3-build-deps-cosmic.dockerfile


Expand Down

0 comments on commit 2ca9575

Please sign in to comment.