Skip to content

Commit

Permalink
use a matrix strategy to build with Qt 5.14 (#41416)
Browse files Browse the repository at this point in the history
  • Loading branch information
@3nids
3nids committed Feb 15, 2021
1 parent 3de13ad commit 972e41b
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 121 deletions.
74 changes: 41 additions & 33 deletions .docker/docker-qgis-build.sh
View file
Expand Up @@ -33,40 +33,47 @@ echo "::group::cmake"
export CC=/usr/lib/ccache/clang
export CXX=/usr/lib/ccache/clang++

cmake \
-GNinja \
-DUSE_CCACHE=OFF \
-DWITH_QUICK=OFF \
-DWITH_3D=ON \
-DWITH_STAGED_PLUGINS=ON \
-DWITH_GRASS=OFF \
-DSUPPRESS_QT_WARNINGS=ON \
-DENABLE_TESTS=ON \
-DENABLE_MODELTEST=ON \
-DENABLE_PGTEST=ON \
-DENABLE_SAGA_TESTS=ON \
-DENABLE_MSSQLTEST=ON \
-DWITH_QSPATIALITE=ON \
-DWITH_QWTPOLAR=OFF \
-DWITH_APIDOC=OFF \
-DWITH_ASTYLE=OFF \
-DWITH_DESKTOP=ON \
-DWITH_BINDINGS=ON \
-DWITH_SERVER=ON \
-DWITH_ORACLE=ON \
-DORACLE_INCLUDEDIR="/instantclient_19_9/sdk/include/" \
-DORACLE_LIBDIR="/instantclient_19_9/" \
-DDISABLE_DEPRECATED=ON \
-DPYTHON_TEST_WRAPPER="timeout -sSIGSEGV 55s"\
-DCXX_EXTRA_FLAGS="${CLANG_WARNINGS}" \
-DWERROR=TRUE \
-DADD_CLAZY_CHECKS=ON \
-DQT5_3DEXTRA_LIBRARY="/usr/lib/x86_64-linux-gnu/libQt53DExtras.so" \
-DQT5_3DEXTRA_INCLUDE_DIR="/root/QGIS/external/qt3dextra-headers" \
-DCMAKE_PREFIX_PATH="/root/QGIS/external/qt3dextra-headers/cmake" \
-DQt53DExtras_DIR="/root/QGIS/external/qt3dextra-headers/cmake/Qt53DExtras" \
..
CMAKE_ARGS=(
"-GNinja"
"-DUSE_CCACHE=OFF"
"-DWITH_QUICK=OFF"
"-DWITH_3D=ON"
"-DWITH_STAGED_PLUGINS=ON"
"-DWITH_GRASS=OFF"
"-DSUPPRESS_QT_WARNINGS=ON"
"-DENABLE_TESTS=ON"
"-DENABLE_MODELTEST=ON"
"-DENABLE_PGTEST=ON"
"-DENABLE_SAGA_TESTS=ON"
"-DENABLE_MSSQLTEST=ON"
"-DPUSH_TO_CDASH=${PUSH_TO_CDASH}"
"-DWITH_QSPATIALITE=ON"
"-DWITH_QWTPOLAR=OFF"
"-DWITH_APIDOC=OFF"
"-DWITH_ASTYLE=OFF"
"-DWITH_DESKTOP=ON"
"-DWITH_BINDINGS=ON"
"-DWITH_SERVER=ON"
"-DWITH_ORACLE=ON"
"-DORACLE_INCLUDEDIR=/instantclient_19_9/sdk/include/"
"-DORACLE_LIBDIR=/instantclient_19_9/"
"-DDISABLE_DEPRECATED=ON"
"-DPYTHON_TEST_WRAPPER=\"timeout -sSIGSEGV 55s\""
"-DCXX_EXTRA_FLAGS=\"${CLANG_WARNINGS}\""
"-DWERROR=TRUE"
"-DADD_CLAZY_CHECKS=ON"
)
if [[ ${PATCH_QT_3D} == "true" ]]; then
CMAKE_ARGS+=(
"-DQT5_3DEXTRA_LIBRARY=/usr/lib/x86_64-linux-gnu/libQt53DExtras.so"
"-DQT5_3DEXTRA_INCLUDE_DIR=/root/QGIS/external/qt3dextra-headers"
"-DCMAKE_PREFIX_PATH=/root/QGIS/external/qt3dextra-headers/cmake"
"-DQt53DExtras_DIR=/root/QGIS/external/qt3dextra-headers/cmake/Qt53DExtras"
)
fi

echo "Running cmake ${CMAKE_ARGS[*]} .."
cmake ${CMAKE_ARGS[*]} ..
echo "::endgroup::"

#######
Expand All @@ -87,3 +94,4 @@ popd > /dev/null # build
popd > /dev/null # /root/QGIS

[ -r /tmp/ctest-important.log ] && cat /tmp/ctest-important.log || true

50 changes: 26 additions & 24 deletions .docker/docker-qgis-test.sh
View file
Expand Up @@ -35,30 +35,31 @@ popd > /dev/null # /root/QGIS
# Restore Oracle test data
##############################

echo "${bold}Load Oracle database...🙏${endbold}"

export ORACLE_HOST="oracle"
export QGIS_ORACLETEST_DBNAME="${ORACLE_HOST}/XEPDB1"
export QGIS_ORACLETEST_DB="host=${QGIS_ORACLETEST_DBNAME} port=1521 user='QGIS' password='qgis'"

echo "Wait a moment while loading Oracle database."
COUNT=0
while ! echo exit | sqlplus -L SYSTEM/adminpass@$QGIS_ORACLETEST_DBNAME &> /dev/null
do
printf "🙏"
sleep 5
if [[ $(( COUNT++ )) -eq 200 ]]; then
break
fi
done
if [[ ${COUNT} -eq 201 ]]; then
echo "timeout, no oracle, no 🙏"
else
echo " done 👀"
pushd /root/QGIS > /dev/null
/root/QGIS/tests/testdata/provider/testdata_oracle.sh $ORACLE_HOST
popd > /dev/null # /root/QGIS
fi
# echo "${bold}Load Oracle database...🙏${endbold}"
#
# export ORACLE_HOST="oracle"
# export QGIS_ORACLETEST_DBNAME="${ORACLE_HOST}/XEPDB1"
# export QGIS_ORACLETEST_DB="host=${QGIS_ORACLETEST_DBNAME} port=1521 user='QGIS' password='qgis'"
#
# echo "Wait a moment while loading Oracle database."
# COUNT=0
# while ! echo exit | sqlplus -L SYSTEM/adminpass@$QGIS_ORACLETEST_DBNAME &> /dev/null
# do
# printf "🙏"
# sleep 5
# if [[ $(( COUNT++ )) -eq 200 ]]; then
# break
# fi
# done
# if [[ ${COUNT} -eq 201 ]]; then
# echo "timeout, no oracle, no 🙏"
# else
# echo " done 👀"
# pushd /root/QGIS > /dev/null
# /root/QGIS/tests/testdata/provider/testdata_oracle.sh $ORACLE_HOST
# popd > /dev/null # /root/QGIS
# fi
# this is proving very flaky:

##############################
# Restore SQL Server test data
Expand Down Expand Up @@ -102,3 +103,4 @@ else
fi
echo "List of skipped tests: $EXCLUDE_TESTS"
python3 /root/QGIS/.ci/ctest2ci.py xvfb-run ctest -V -E "${EXCLUDE_TESTS}" -S /root/QGIS/.ci/config.ctest --output-on-failure

6 changes: 5 additions & 1 deletion .docker/docker-variables.env
View file
Expand Up @@ -8,8 +8,12 @@ CTEST_BUILD_DIR=/root/QGIS
# CTEST vars defined in workflow
CTEST_BUILD_NAME

# Other var
# Other vars
QGIS_NO_OVERRIDE_IMPORT=1
HANA_TESTS_ENABLED=OFF

# Defined in workflow
PATCH_QT_3D

# This is used in some tests to be skipped
QGIS_CONTINUOUS_INTEGRATION_RUN=true
Expand Down
5 changes: 4 additions & 1 deletion .docker/qgis3-build-deps.dockerfile
View file
@@ -1,4 +1,7 @@
FROM ubuntu:20.04

ARG UBUNTU_BASE=20.04

FROM ubuntu:${UBUNTU_BASE}
MAINTAINER Denis Rouzaud <denis@opengis.ch>

LABEL Description="Docker container with QGIS dependencies" Vendor="QGIS.org" Version="1.0"
Expand Down
138 changes: 76 additions & 62 deletions .github/workflows/run-tests.yml
View file
Expand Up @@ -34,9 +34,16 @@ jobs:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
GH_WORKSPACE: ${{ github.workspace }} # used in docker compose
DEFAULT_UBUNTU_BASE: '20.04'

runs-on: ubuntu-latest

strategy:
matrix:
# tests run on 20.04 (Qt 5.12), compile test on 20.10 (Qt 5.14)
ubuntu-base: ['20.04', '20.10']
fail-fast: false

outputs:
compile_outcome: ${{ steps.compile.outcome }}
tests_failing: ${{ steps.tests.outputs.TESTS_FAILING }}
Expand All @@ -52,22 +59,28 @@ jobs:
GITHUB_EVENT_NAME: ${{ github.event_name }}
GITHUB_REF: ${{ github.ref }}
GITHUB_PR_NUMBER: ${{github.event.number}}
UBUNTU_BASE: ${{ matrix.ubuntu-base }}
run: |
DOCKER_TAG=$(echo $( [[ ${GITHUB_EVENT_NAME} =~ ^pull_request$ ]] && echo ${GITHUB_BASE_REF} || echo ${GITHUB_REF##*/} ) | sed 's/^master$/latest/')
DOCKER_TAG=$(echo $( [[ ${GITHUB_EVENT_NAME} =~ ^pull_request$ ]] && echo ${GITHUB_BASE_REF} || echo ${GITHUB_REF##*/} ) | sed 's/^master$/latest/')$( [[ ${UBUNTU_BASE} != ${DEFAULT_UBUNTU_BASE} ]] && echo "_${UBUNTU_BASE}" || echo "" )
CTEST_BUILD_NAME=$( [[ ${GITHUB_EVENT_NAME} =~ ^pull_request$ ]] && echo "PR${GITHUB_PR_NUMBER}" || echo ${GITHUB_REF##*/} )"_${GITHUB_SHA}"
[[ ${UBUNTU_BASE} == "20.04" ]] && PATCH_QT_3D=true || PATCH_QT_3D=false
echo "DOCKER_TAG=${DOCKER_TAG}" >> $GITHUB_ENV
echo "CTEST_BUILD_NAME=${CTEST_BUILD_NAME}" >> $GITHUB_ENV
echo "PATCH_QT_3D=${PATCH_QT_3D}" >> $GITHUB_ENV
- name: Print vars
run: |
echo DOCKER_TAG: ${DOCKER_TAG}
echo CTEST_BUILD_NAME: ${CTEST_BUILD_NAME}
echo PATCH_QT_3D: ${PATCH_QT_3D}
- name: Build deps
env:
UBUNTU_BASE: ${{ matrix.ubuntu-base }}
run: |
pushd .docker
docker pull qgis/qgis3-build-deps:${DOCKER_TAG}
docker build --cache-from qgis/qgis3-build-deps:${DOCKER_TAG} -t qgis/qgis3-build-deps:${DOCKER_TAG} -f qgis3-build-deps.dockerfile .
docker pull qgis/qgis3-build-deps:${DOCKER_TAG} || true
docker build --build-arg UBUNTU_BASE=${UBUNTU_BASE} --cache-from qgis/qgis3-build-deps:${DOCKER_TAG} -t qgis/qgis3-build-deps:${DOCKER_TAG} -f qgis3-build-deps.dockerfile .
popd
- name: Push deps image
Expand All @@ -81,13 +94,13 @@ jobs:
if: github.event_name == 'pull_request'
with:
path: /home/runner/QGIS/.ccache
key: build-ccache-${{ github.head_ref }}-${{ github.sha }}
key: build-ccache-${{ matrix.ubuntu-base }}-${{ github.actor }}-${{ github.head_ref }}-${{ github.sha }}
# The head_ref or source branch of the pull request in a workflow run.
# The base_ref or target branch of the pull request in a workflow run.
restore-keys: |
build-ccache-${{ github.head_ref }}-
build-ccache-${{ github.base_ref }}-
build-ccache-refs/heads/master-
build-ccache-${{ matrix.ubuntu-base }}-${{ github.actor }}-${{ github.head_ref }}-
build-ccache-${{ matrix.ubuntu-base }}-${{ github.base_ref }}-
build-ccache-${{ matrix.ubuntu-base }}-refs/heads/master-
- name: Prepare build cache for branch/tag
# use a fork of actions/cache@v2 to upload cache even when the build or test failed
Expand All @@ -96,10 +109,10 @@ jobs:
with:
path: /home/runner/QGIS/.ccache
# The branch or tag ref that triggered the workflow run. For branches this in the format refs/heads/<branch_name>, and for tags it is refs/tags/<tag_name>
key: build-ccache-${{ github.ref }}-${{ github.sha }}
key: build-ccache-${{ matrix.ubuntu-base }}-${{ github.ref }}-${{ github.sha }}
restore-keys: |
build-ccache-${{ github.ref }}-
build-ccache-refs/heads/master-
build-ccache-${{ matrix.ubuntu-base }}-${{ github.ref }}-
build-ccache-${{ matrix.ubuntu-base }}-refs/heads/master-
- name: Compile QGIS
id: compile
Expand All @@ -124,10 +137,12 @@ jobs:
- name: Run unit tests
id: tests
if: ${{ matrix.ubuntu-base == env.DEFAULT_UBUNTU_BASE }}
run: docker-compose -f .docker/docker-compose-testing.yml run qgis-deps /root/QGIS/.docker/docker-qgis-test.sh

# - name: Test QGIS runners
# id: runners
# if: ${{ matrix.ubuntu-base == env.DEFAULT_UBUNTU_BASE }}
# run: |
# docker run -d --name qgis-testing-environment \
# -v $(pwd):/root/QGIS \
Expand Down Expand Up @@ -166,55 +181,54 @@ jobs:
# docker stop qgis-testing-environment



tests-report-comment:
name: Write tests report in a comment
needs: build
runs-on: ubuntu-latest
if: always() && ( needs.build.result == 'failure' || needs.build.result == 'success' ) && github.event_name == 'pull_request'
steps:
- name: Find Comment
uses: peter-evans/find-comment@v1
id: find-comment
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: 'github-actions[bot]'
body-includes: Tests report

- name: Create comment
if: ${{ steps.find-comment.outputs.comment-id == 0 }}
uses: peter-evans/create-or-update-comment@v1
id: create-comment
with:
issue-number: ${{ github.event.pull_request.number }}
body: |
**Tests report**
- name: Process
id: process-vars
env:
COMMENT_FOUND: ${{ steps.find-comment.outputs.comment-id }}
COMMENT_CREATED: ${{ steps.create-comment.outputs.comment-id }}
COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
CDASH_URL: ${{ needs.build.outputs.cdash_url }}
COMPILE_OUTCOME: ${{ needs.build.outputs.compile_outcome }}
TESTS_FAILING: ${{ needs.build.outputs.tests_failing }}
RUNNERS_OUTCOME: ${{ needs.build.outputs.runners_outcome }}
run: |
echo "::set-output name=COMMENT_ID::"$([[ "${COMMENT_FOUND}" -eq "0" ]] && echo ${COMMENT_CREATED} || echo ${COMMENT_FOUND})
if [[ ${COMPILE_OUTCOME} != "success" ]]; then
echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :scream: compilation failed"
elif [[ ${TESTS_FAILING} != "0" ]]; then
echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :fire: ${TESTS_FAILING} unit-tests are failing ${CDASH_URL}"
elif [[ ${RUNNERS_OUTCOME} != "success" ]]; then
echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :broken_heart: QGIS runners test failed"
else
echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :sunglasses: unit-tests succeeded"
fi
- name: Update comment
uses: peter-evans/create-or-update-comment@v1
with:
comment-id: ${{ steps.process-vars.outputs.COMMENT_ID }}
edit-mode: append
body: ${{ steps.process-vars.outputs.COMMENT_BODY }}
# tests-report-comment:
# name: Write tests report in a comment
# needs: build
# runs-on: ubuntu-latest
# if: always() && ( needs.build.result == 'failure' || needs.build.result == 'success' ) && github.event_name == 'pull_request'
# steps:
# - name: Find Comment
# uses: peter-evans/find-comment@v1
# id: find-comment
# with:
# issue-number: ${{ github.event.pull_request.number }}
# comment-author: 'github-actions[bot]'
# body-includes: Tests report
#
# - name: Create comment
# if: ${{ steps.find-comment.outputs.comment-id == 0 }}
# uses: peter-evans/create-or-update-comment@v1
# id: create-comment
# with:
# issue-number: ${{ github.event.pull_request.number }}
# body: |
# **Tests report**
#
# - name: Process
# id: process-vars
# env:
# COMMENT_FOUND: ${{ steps.find-comment.outputs.comment-id }}
# COMMENT_CREATED: ${{ steps.create-comment.outputs.comment-id }}
# COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
# CDASH_URL: ${{ needs.build.outputs.cdash_url }}
# COMPILE_OUTCOME: ${{ needs.build.outputs.compile_outcome }}
# TESTS_FAILING: ${{ needs.build.outputs.tests_failing }}
# RUNNERS_OUTCOME: ${{ needs.build.outputs.runners_outcome }}
# run: |
# echo "::set-output name=COMMENT_ID::"$([[ "${COMMENT_FOUND}" -eq "0" ]] && echo ${COMMENT_CREATED} || echo ${COMMENT_FOUND})
# if [[ ${COMPILE_OUTCOME} != "success" ]]; then
# echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :scream: compilation failed"
# elif [[ ${TESTS_FAILING} != "0" ]]; then
# echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :fire: ${TESTS_FAILING} unit-tests are failing ${CDASH_URL}"
# elif [[ ${RUNNERS_OUTCOME} != "success" ]]; then
# echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :broken_heart: QGIS runners test failed"
# else
# echo "::set-output name=COMMENT_BODY::${COMMIT_SHA} :sunglasses: unit-tests succeeded"
# fi
#
# - name: Update comment
# uses: peter-evans/create-or-update-comment@v1
# with:
# comment-id: ${{ steps.process-vars.outputs.COMMENT_ID }}
# edit-mode: append
# body: ${{ steps.process-vars.outputs.COMMENT_BODY }}

0 comments on commit 972e41b

Please sign in to comment.