Skip to content

Commit 23e3b58

Browse files
3nids3nids
committedMar 21, 2019
Refactoring of Docker builds
·
final-3_6_3final-3_6_1
1 parent ec3b47a commit 23e3b58

16 files changed

+198
-161
lines changed
 

‎.ci/travis/linux/before_script.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ echo "travis_fold:start:docker_build"
2626
echo "${bold}Docker build deps${endbold}"
2727
docker --version
2828
docker-compose --version
29-
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml config
29+
docker-compose -f ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-compose.travis.yml config
3030
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
3131
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" -f ${DOCKER_BUILD_DEPS_FILE} .
3232
echo "travis_fold:end:docker_build"

‎.ci/travis/linux/docker-compose.travis.yml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
version: '3'
2+
services:
3+
postgres:
4+
image: kartoza/postgis:9.5-2.2
5+
environment:
6+
- ALLOW_IP_RANGE="172.18.0.0/16"
7+
8+
mssql:
9+
image: microsoft/mssql-server-linux:2017-latest
10+
environment:
11+
ACCEPT_EULA: Y
12+
SA_PASSWORD: <YourStrong!Passw0rd>
13+
14+
qgis-deps:
15+
tty: true
16+
image: qgis_image
17+
volumes:
18+
- ${TRAVIS_BUILD_DIR}:/root/QGIS
19+
links:
20+
- postgres
21+
- mssql
22+
env_file:
23+
- docker-variables.env
24+

‎.ci/travis/linux/docker-variables.env

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
2+
# Env variables for Docker
3+
# These without assignment are taken from the host env variabbles
4+
5+
# TRAVIS variables
6+
TRAVIS_AVAILABLE_TIME
7+
TRAVIS_TIMESTAMP
8+
TRAVIS_UPLOAD_TIME
9+
TRAVIS_BRANCH
10+
TRAVIS_PULL_REQUEST
11+
TRAVIS_OS_NAME
12+
TRAVIS_CONFIG
13+
TRAVIS
14+
15+
# CTEST
16+
LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
17+
SEGFAULT_SIGNALS="abrt segv"
18+
CTEST_BUILD_COMMAND=/usr/bin/ninja
19+
CTEST_PARALLEL_LEVEL=1
20+
CTEST_BUILD_DIR=/root/QGIS
21+
22+
# Other var
23+
QGIS_NO_OVERRIDE_IMPORT=1

‎.ci/travis/linux/docker_test.sh

Lines changed: 0 additions & 28 deletions
This file was deleted.

‎.ci/travis/linux/script.sh

Lines changed: 56 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,21 +16,63 @@
1616

1717
set -e
1818

19-
# running QGIS tests
20-
docker-compose -f ${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml run --rm qgis-deps
19+
# build QGIS in docker
20+
echo "travis_fold:start:docker_build_qgis"
21+
echo "${bold}Docker build QGIS${endbold}"
22+
docker run -t --name qgis_container \
23+
-v ${TRAVIS_BUILD_DIR}:/root/QGIS \
24+
-v ${CCACHE_DIR}:/root/.ccache \
25+
--env-file ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-variables.env \
26+
qgis/qgis3-build-deps:${DOCKER_TAG} \
27+
/root/QGIS/.ci/travis/linux/scripts/docker-qgis-build.sh
28+
29+
# commit container
30+
docker commit qgis_container qgis_image
31+
echo "travis_fold:end:docker_build_qgis"
32+
33+
# running QGIS tests in commited image
34+
echo "travis_fold:start:docker_test_qgis"
35+
echo "${bold}Docker run tests${endbold}"
36+
docker-compose -f ${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker-compose.travis.yml run qgis-deps /root/QGIS/.ci/travis/linux/scripts/docker-qgis-test.sh
37+
echo "travis_fold:end:docker_test_qgis"
2138

2239
# running tests for the python test runner
23-
docker run -d --name qgis-testing-environment -v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory -e DISPLAY=:99 "qgis/qgis:${DOCKER_TAG}"
24-
sleep 10 # Wait for xvfb to finish starting
25-
# Temporary workaround until docker images are built
26-
docker cp ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner/qgis_testrunner.sh qgis-testing-environment:/usr/bin/qgis_testrunner.sh
27-
# Run tests in the docker
40+
echo "travis_fold:start:docker_test_runners"
41+
echo "${bold}Docker test QGIS runners${endbold}"
42+
docker run -d --name qgis-testing-environment \
43+
-v ${TRAVIS_BUILD_DIR}:/root/QGIS \
44+
-v ${TRAVIS_BUILD_DIR}/tests/src/python:/tests_directory \
45+
-v ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/test_runner:/usr/bin/test_runner \
46+
-v ${TRAVIS_BUILD_DIR}/.docker/qgis_resources/supervisor:/etc/supervisor \
47+
-e QGIS_BUILD_PATH=/root/QGIS/build/output/bin/qgis \
48+
-e TEST_RUNNER_PATH=/usr/bin/test_runner/qgis_testrunner.py \
49+
-e DISPLAY=:99 \
50+
qgis_image \
51+
/usr/bin/supervisord -c /etc/supervisor/supervisord.conf
52+
53+
# Wait for xvfb to finish starting
54+
printf "Waiting for the docker...🐳..."
55+
sleep 10
56+
echo " done 🥩"
57+
58+
declare -A testrunners
2859
# Passing cases:
29-
TEST_SCRIPT_PATH=${TRAVIS_BUILD_DIR}/.ci/travis/linux/docker_test.sh
30-
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_passing) -eq '0' ]]
31-
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_skipped_and_passing) -eq '0' ]]
60+
testrunners["test_testrunner.run_passing"]=0
61+
testrunners["test_testrunner.run_skipped_and_passing"]=0
3262
# Failing cases:
33-
[[ $(${TEST_SCRIPT_PATH} test_testrunner) -eq '1' ]]
34-
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_all) -eq '1' ]]
35-
[[ $(${TEST_SCRIPT_PATH} test_testrunner.run_failing) -eq '1' ]]
36-
63+
testrunners["test_testrunner"]=1
64+
testrunners["test_testrunner.run_all"]=1
65+
testrunners["test_testrunner.run_failing"]=1
66+
set +e # do not exit on error
67+
# Run tests in the docker
68+
for i in "${!testrunners[@]}"
69+
do
70+
echo "travis_fold:start:docker_test_runner_${i}"
71+
echo "test ${i}..."
72+
docker exec -it qgis-testing-environment sh -c "cd /tests_directory && /usr/bin/test_runner/qgis_testrunner.sh ${i}"
73+
[[ $? -eq "${testrunners[$i]}" ]] && echo "success" || exit 1
74+
echo "travis_fold:end:docker_test_runner_${i}"
75+
done
76+
set -e # switch back
77+
docker stop qgis-testing-environment
78+
echo "travis_fold:end:docker_test_runners"

‎.ci/travis/linux/docker-build-test.sh renamed to ‎.ci/travis/linux/scripts/docker-qgis-build.sh

Lines changed: 10 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,6 @@ ccache -M 1G
1212
# export CCACHE_LOGFILE=/tmp/cache.debug
1313
ccache -z
1414

15-
############################
16-
# Setup the (c)test environment
17-
############################
18-
export LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
19-
export SEGFAULT_SIGNALS="abrt segv"
20-
export CTEST_BUILD_COMMAND="/usr/bin/ninja"
21-
export CTEST_PARALLEL_LEVEL=1
22-
2315
##############################
2416
# Variables for output styling
2517
##############################
@@ -75,84 +67,17 @@ echo "travis_fold:end:cmake"
7567
# Travis will kill the job after approx 150 minutes, we subtract 5 minutes for
7668
# uploading and subtract the bootstrapping time from that.
7769
# Hopefully clocks are in sync :)
78-
TRAVIS_TIME=150
79-
UPLOAD_TIME=5
70+
8071
CURRENT_TIME=$(date +%s)
81-
TIMEOUT=$((( TRAVIS_TIME - UPLOAD_TIME ) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
72+
TIMEOUT=$((( TRAVIS_AVAILABLE_TIME - TRAVIS_UPLOAD_TIME ) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
8273
TIMEOUT=$(( TIMEOUT < 300 ? 300 : TIMEOUT ))
8374
echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
8475

8576
# echo "travis_fold:start:ninja-build.1"
8677
echo "${bold}Building QGIS...${endbold}"
8778
timeout ${TIMEOUT}s ${CTEST_BUILD_COMMAND}
8879
# echo "travis_fold:end:ninja-build.1"
89-
90-
rv=$?
91-
if [ $rv -eq 124 ] ; then
92-
printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
93-
exit #$rv
94-
fi
95-
96-
# Temporarily uncomment to debug ccache issues
97-
# echo "travis_fold:start:ccache-debug"
98-
# cat /tmp/cache.debug
99-
# echo "travis_fold:end:ccache-debug"
100-
101-
############################
102-
# Restore postgres test data
103-
############################
104-
printf "[qgis_test]\nhost=postgres\nport=5432\ndbname=qgis_test\nuser=docker\npassword=docker" > ~/.pg_service.conf
105-
export PGUSER=docker
106-
export PGHOST=postgres
107-
export PGPASSWORD=docker
108-
export PGDATABASE=qgis_test
109-
110-
pushd /root/QGIS > /dev/null
111-
/root/QGIS/tests/testdata/provider/testdata_pg.sh
112-
popd > /dev/null # /root/QGIS
113-
114-
##############################
115-
# Restore SQL Server test data
116-
##############################
117-
118-
echo "Importing SQL Server test data..."
119-
120-
export SQLUSER=sa
121-
export SQLHOST=mssql
122-
export SQLPORT=1433
123-
export SQLPASSWORD='<YourStrong!Passw0rd>'
124-
export SQLDATABASE=qgis_test
125-
126-
export PATH=$PATH:/opt/mssql-tools/bin
127-
128-
pushd /root/QGIS > /dev/null
129-
/root/QGIS/tests/testdata/provider/testdata_mssql.sh
130-
popd > /dev/null # /root/QGIS
131-
132-
echo "Setting up DSN for test SQL Server"
133-
134-
cat <<EOT > /etc/odbc.ini
135-
[ODBC Data Sources]
136-
testsqlserver = ODBC Driver 17 for SQL Server
137-
138-
[testsqlserver]
139-
Driver = ODBC Driver 17 for SQL Server
140-
Description = Test SQL Server
141-
Server = mssql
142-
EOT
143-
144-
###########
145-
# Run tests
146-
###########
147-
CURRENT_TIME=$(date +%s)
148-
TIMEOUT=$((( TRAVIS_TIME - UPLOAD_TIME) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
149-
echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
150-
timeout ${TIMEOUT}s python3 /root/QGIS/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V -E "$(cat /root/QGIS/.ci/travis/linux/blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S /root/QGIS/.ci/travis/travis.ctest --output-on-failure
15180
rv=$?
152-
if [ $rv -eq 124 ] ; then
153-
printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
154-
exit #$rv
155-
fi
15681

15782
########################
15883
# Show ccache statistics
@@ -166,3 +91,11 @@ popd > /dev/null # build
16691
popd > /dev/null # /root/QGIS
16792

16893
[ -r /tmp/ctest-important.log ] && cat /tmp/ctest-important.log || true
94+
95+
############################
96+
# Exit with error if timeout
97+
############################
98+
if [ $rv -eq 124 ] ; then
99+
printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
100+
exit #$rv
101+
fi

‎.ci/travis/linux/scripts/docker-qgis-test.sh

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
# Temporarily uncomment to debug ccache issues
6+
# echo "travis_fold:start:ccache-debug"
7+
# cat /tmp/cache.debug
8+
# echo "travis_fold:end:ccache-debug"
9+
10+
############################
11+
# Restore postgres test data
12+
############################
13+
printf "[qgis_test]\nhost=postgres\nport=5432\ndbname=qgis_test\nuser=docker\npassword=docker" > ~/.pg_service.conf
14+
export PGUSER=docker
15+
export PGHOST=postgres
16+
export PGPASSWORD=docker
17+
export PGDATABASE=qgis_test
18+
19+
# wait for the DB to be available
20+
echo "Wait a moment while loading the database."
21+
while ! PGPASSWORD='docker' psql -h postgres -U docker -p 5432 -l &> /dev/null
22+
do
23+
printf "🐘"
24+
sleep 1
25+
done
26+
echo " done 🥩"
27+
28+
pushd /root/QGIS > /dev/null
29+
/root/QGIS/tests/testdata/provider/testdata_pg.sh
30+
popd > /dev/null # /root/QGIS
31+
32+
##############################
33+
# Restore SQL Server test data
34+
##############################
35+
36+
echo "Importing SQL Server test data..."
37+
38+
export SQLUSER=sa
39+
export SQLHOST=mssql
40+
export SQLPORT=1433
41+
export SQLPASSWORD='<YourStrong!Passw0rd>'
42+
export SQLDATABASE=qgis_test
43+
44+
export PATH=$PATH:/opt/mssql-tools/bin
45+
46+
pushd /root/QGIS > /dev/null
47+
/root/QGIS/tests/testdata/provider/testdata_mssql.sh
48+
popd > /dev/null # /root/QGIS
49+
50+
echo "Setting up DSN for test SQL Server"
51+
52+
cat <<EOT > /etc/odbc.ini
53+
[ODBC Data Sources]
54+
testsqlserver = ODBC Driver 17 for SQL Server
55+
56+
[testsqlserver]
57+
Driver = ODBC Driver 17 for SQL Server
58+
Description = Test SQL Server
59+
Server = mssql
60+
EOT
61+
62+
###########
63+
# Run tests
64+
###########
65+
CURRENT_TIME=$(date +%s)
66+
TIMEOUT=$((( TRAVIS_AVAILABLE_TIME - TRAVIS_UPLOAD_TIME) * 60 - CURRENT_TIME + TRAVIS_TIMESTAMP))
67+
echo "Timeout: ${TIMEOUT}s (started at ${TRAVIS_TIMESTAMP}, current: ${CURRENT_TIME})"
68+
timeout ${TIMEOUT}s python3 /root/QGIS/.ci/travis/scripts/ctest2travis.py xvfb-run ctest -V -E "$(cat /root/QGIS/.ci/travis/linux/scripts/test_blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S /root/QGIS/.ci/travis/travis.ctest --output-on-failure
69+
rv=$?
70+
if [ $rv -eq 124 ] ; then
71+
printf '\n\n${bold}Build and test timeout. Please restart the build for meaningful results.${endbold}\n'
72+
exit #$rv
73+
fi
74+
75+
76+

‎.ci/travis/scripts/echo_travis_var.sh

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ echo "TRAVIS_BRANCH: $TRAVIS_BRANCH"
66
echo "TRAVIS_EVENT_TYPE: $TRAVIS_EVENT_TYPE"
77
echo "DOCKER_TAG: $DOCKER_TAG"
88
echo "TRAVIS_COMMIT_MESSAGE: $TRAVIS_COMMIT_MESSAGE"
9-
echo "DOCKER_DEPS_PUSH: $DOCKER_DEPS_PUSH"
109
echo "DOCKER_BUILD_DEPS_FILE: $DOCKER_BUILD_DEPS_FILE"
1110
echo "TRAVIS_TIMESTAMP: $TRAVIS_TIMESTAMP"
1211
echo "travis_fold:end:travis_env"

‎.docker/docker-compose.travis.yml

Lines changed: 0 additions & 35 deletions
This file was deleted.

‎.docker/qgis.dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ RUN cmake \
5353
# Python testing environment setup
5454

5555
# Add QGIS test runner
56-
COPY .docker/qgis_resources/test_runner/qgis_* /usr/bin/
56+
COPY .docker/qgis_resources/test_runner/qgis_* /usr/bin
5757

5858
# Make all scripts executable
5959
RUN chmod +x /usr/bin/qgis_*

‎.docker/qgis3-build-deps-cosmic.dockerfile

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,5 +135,3 @@ ENV CXX=/usr/lib/ccache/clang++
135135
ENV QT_SELECT=5
136136
ENV LANG=C.UTF-8
137137
ENV PATH="/usr/local/bin:${PATH}"
138-
139-
CMD /root/QGIS/.ci/travis/linux/docker-build-test.sh

‎.docker/qgis3-build-deps.dockerfile

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -149,4 +149,3 @@ ENV QT_SELECT=5
149149
ENV LANG=C.UTF-8
150150
ENV PATH="/usr/local/bin:${PATH}"
151151

152-
CMD /root/QGIS/.ci/travis/linux/docker-build-test.sh

‎.docker/qgis_resources/test_runner/qgis_testrunner.sh

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,12 @@
1010

1111
TEST_NAME=$1
1212

13+
TEST_RUNNER_PATH=${TEST_RUNNER_PATH:-/usr/bin/qgis_testrunner.py}
14+
QGIS_BUILD_PATH=${QGIS_BUILD_PATH:-qgis}
15+
1316
echo "Running test $1 ..."
14-
OUTPUT=$(QGIS_TEST_MODULE=${TEST_NAME} unbuffer qgis --version-migration --nologo --code /usr/bin/qgis_testrunner.py "$TEST_NAME" 2>/dev/null | tee /dev/tty)
17+
OUTPUT=$(QGIS_TEST_MODULE=${TEST_NAME} unbuffer ${QGIS_BUILD_PATH} --version-migration --nologo --code ${TEST_RUNNER_PATH} "$TEST_NAME" 2>/dev/null | tee /dev/tty)
18+
1519
EXIT_CODE="$?"
1620
if [ -z "$OUTPUT" ]; then
1721
echo "ERROR: no output from the test runner! (exit code: ${EXIT_CODE})"

‎.travis.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ if: NOT branch =~ /^(cherry-pick-)?backport-\d+-on-/
1616
env:
1717
global:
1818
- TRAVIS_TIMESTAMP=$(date +%s)
19+
- TRAVIS_AVAILABLE_TIME=150 # in minutes
20+
- TRAVIS_UPLOAD_TIME=5 # time considered to start the machine and the container (minutes)
1921
# Docker hub username and passowrd
2022
- secure: "b7eMDIolaAnq1voGKC1ez7Kcf+/A0WZDJEHBvNwk2KubBfrGOE83GMDrFNF4NqjIprqIAvVKj+TrX1ckCvs24re3IqUJo71TaF1IgxzDDPwSsmNh5UMmvZkeiJys9bWjqDO9wYR5ietNmIE18qyMc8ToJk8oKm6AXuAG2n6znmM="
2123
- secure: "PHCp7F3nApp38Mz6b4/OLxgfBiikRGzPQDHg3R5LX+SQOll24c/DMtwpPwizNuFEiCFcRmJ9uc1t0HWEerIZe5uqm7AtE/nMXBsvDZ+hj4Tz/fEBF98a1k4WLYheN1exFidVkJgdAeiwMOOUQXw5KuIX62bxBdzsdcd0QGwxiXo="

0 commit comments

Comments
 (0)
Please sign in to comment.