Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
use docker for newer dependencies on Travis
  • Loading branch information
3nids authored and m-kuhn committed Sep 16, 2017
1 parent c2162b0 commit 0092ce1
Show file tree
Hide file tree
Showing 9 changed files with 205 additions and 140 deletions.
4 changes: 0 additions & 4 deletions .ci/travis/linux/after_script.sh
Expand Up @@ -12,7 +12,3 @@
# (at your option) any later version. #
# #
###########################################################################

ccache -s

[ -r /tmp/ctest-important.log ] && cat /tmp/ctest-important.log
30 changes: 1 addition & 29 deletions .ci/travis/linux/before_install.sh
Expand Up @@ -13,33 +13,5 @@
# #
###########################################################################

##################################################
#
# Get precompiled dependencies
#
##################################################

pushd ${HOME}

# fetching data from github should be just as fast as S3
curl -s -S -L https://github.com/opengisch/osgeo4travis/archive/qt55bin.tar.gz | tar --strip-components=1 -xz -C /home/travis &
SETUP_OSGEO4W_PID=$!

mkdir /home/travis/osgeo4travis

# other dependencies live in a cached folder
pushd depcache
# Download newer version of cmake than in the repository
[[ -f cmake-3.5.0-Linux-x86_64.tar.gz ]] || curl -s -S -O https://cmake.org/files/v3.5/cmake-3.5.0-Linux-x86_64.tar.gz
tar --strip-components=1 -zx -f cmake-3.5.0-Linux-x86_64.tar.gz -C /home/travis/osgeo4travis

# Download OTB package for Processing tests
[[ -f OTB-5.6.0-Linux64.run ]] || curl -s -S -O https://www.orfeo-toolbox.org/packages/archives/OTB/OTB-5.6.0-Linux64.run
sh ./OTB-5.6.0-Linux64.run

wait $SETUP_OSGEO4W_PID

popd
popd

pip install psycopg2 numpy nose2 pyyaml mock future termcolor
#pip3 install termcolor
21 changes: 18 additions & 3 deletions .ci/travis/linux/before_script.sh
Expand Up @@ -13,7 +13,22 @@
# #
###########################################################################

printf "[qgis_test]\nhost=localhost\ndbname=qgis_test\nuser=postgres" > ~/.pg_service.conf
set -e

export PGUSER=postgres
$TRAVIS_BUILD_DIR/tests/testdata/provider/testdata_pg.sh
pushd .docker

docker --version
docker-compose --version
docker-compose -f $DOCKER_COMPOSE config
#docker pull ubuntu:16.04
docker pull "qgis/qgis3-build-deps:${DOCKER_TAG}" || true
docker build --cache-from "qgis/qgis3-build-deps:${DOCKER_TAG}" -t "qgis/qgis3-build-deps:${DOCKER_TAG}" .
# image should be pushed even if QGIS build fails
# but push is achieved only on branches (not for PRs)
if [[ $DOCKER_PUSH =~ true ]]; then
docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
#docker tag "qgis/qgis3-build-deps:${DOCKER_TAG}" "qgis/qgis3-build-deps:latest"
docker push "qgis/qgis3-build-deps:${DOCKER_TAG}"
fi

popd
47 changes: 47 additions & 0 deletions .ci/travis/linux/docker-build-test.sh
@@ -0,0 +1,47 @@
#!/usr/bin/env bash

set -e

export CTEST_PARALLEL_LEVEL=1
export CCACHE_TEMPDIR=/tmp
ccache -M 500M
ccache -z

cd /root/QGIS

mkdir -p build-docker &&

pushd build-docker

cmake \
-GNinja \
-DWITH_STAGED_PLUGINS=ON \
-DWITH_GRASS=OFF \
-DSUPPRESS_QT_WARNINGS=ON \
-DENABLE_MODELTEST=ON \
-DENABLE_PGTEST=ON \
-DWITH_QSPATIALITE=ON \
-DWITH_QWTPOLAR=OFF \
-DWITH_APIDOC=OFF \
-DWITH_ASTYLE=OFF \
-DWITH_DESKTOP=ON \
-DWITH_BINDINGS=ON \
-DDISABLE_DEPRECATED=ON \
-DCXX_EXTRA_FLAGS=${CLANG_WARNINGS} ..

# printf "[qgis_test]\nhost=postgres\nport=5432\ndbname=docker\nuser=docker\npassword=docker" > ~/.pg_service.conf
# export PGUSER=docker
# export PGHOST=postgres
# export PGPASSWORD=docker
#
# /root/QGIS/tests/testdata/provider/testdata_pg.sh

python3 /root/QGIS/.ci/travis/scripts/ctest2travis.py \
xvfb-run ctest -V -E "$(cat /root/QGIS/.ci/travis/linux/blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S /root/QGIS/.ci/travis/travis.ctest --output-on-failure

ccache -s

popd


[ -r /tmp/ctest-important.log ] && cat /tmp/ctest-important.log
65 changes: 2 additions & 63 deletions .ci/travis/linux/install.sh
Expand Up @@ -13,66 +13,5 @@
# #
###########################################################################

mkdir build
cd build

ln -s ${HOME}/osgeo4travis/bin/ccache ${HOME}/osgeo4travis/bin/clang++-${LLVM_VERSION}
ln -s ${HOME}/osgeo4travis/bin/ccache ${HOME}/osgeo4travis/bin/clang-${LLVM_VERSION}

ccache -s
ccache -z

export CXX="clang++-${LLVM_VERSION}"
export CC="clang-${LLVM_VERSION}"
#export CXX="g++-6"
#export CC="gcc-6"
export PATH=${HOME}/osgeo4travis/bin:${PATH}
export PYTHONPATH=${HOME}/osgeo4travis/lib/python3.3/site-packages/

cmake --version
${CC} --version
${CXX} --version

# CLANG_WARNINGS="-Wimplicit-fallthrough"
CLANG_WARNINGS=""

# Include this line for debug reasons
# -DCMAKE_BUILD_TYPE=RelWithDebInfo \
#
CMAKE_FLAGS="
-DCMAKE_PREFIX_PATH=/home/travis/osgeo4travis
-DWITH_STAGED_PLUGINS=ON
-DWITH_GRASS=ON
-DWITH_GRASS7=ON
-DGRASS_PREFIX7=/home/travis/osgeo4travis/grass-7.0.4
-DSUPPRESS_QT_WARNINGS=ON
-DENABLE_MODELTEST=ON
-DENABLE_PGTEST=ON
-DWITH_QSPATIALITE=ON
-DWITH_QWTPOLAR=OFF
-DWITH_APIDOC=OFF
-DWITH_ASTYLE=OFF
-DDISABLE_DEPRECATED=ON
-DCXX_EXTRA_FLAGS=${CLANG_WARNINGS}
"

# The following options trigger a minimalized build to
# reduce the travis build time so we don't time out and
# have a chance of slowly filling the ccache.
if [ "$CACHE_WARMING" = true ] ; then
CMAKE_FLAGS="
${CMAKE_FLAGS}
-DWITH_DESKTOP=OFF
-DWITH_SERVER=OFF
-DWITH_CUSTOM_WIDGETS=OFF
"
else
CMAKE_FLAGS="
${CMAKE_FLAGS}
-DWITH_DESKTOP=ON
-DWITH_SERVER=ON
-DWITH_CUSTOM_WIDGETS=ON
"
fi

cmake $CMAKE_FLAGS ..
shopt -s expand_aliases
alias python=python3
28 changes: 5 additions & 23 deletions .ci/travis/linux/script.sh
Expand Up @@ -15,31 +15,13 @@

set -e

export PYTHONPATH=${HOME}/osgeo4travis/lib/python3.3/site-packages/
export PATH=${HOME}/osgeo4travis/bin:${HOME}/osgeo4travis/sbin:${HOME}/OTB-5.6.0-Linux64/bin:${PATH}
export LD_LIBRARY_PATH=${HOME}/osgeo4travis/lib
export CTEST_PARALLEL_LEVEL=1
export CCACHE_TEMPDIR=/tmp
ccache -M 500M
ccache -z
DIR=$(git rev-parse --show-toplevel)/.docker

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
docker-compose -f $DOCKER_COMPOSE run --rm qgis-deps

# Set OTB application path (installed in before_install.sh script)
export OTB_APPLICATION_PATH=${HOME}/OTB-5.6.0-Linux64/lib/otb/applications
export LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
ccachedir=${HOME}/.ccache
mkdir -p $ccachedir

export CTEST_BUILD_COMMAND="/usr/bin/make -j3 -i"
docker-compose -f $DOCKER_COMPOSE run --rm qgis-deps

# This works around an issue where travis would timeout because
# when make is run inside ctest no output is generated. At the current time
# nobody know why, but at least this workaround gets travis results
# back. Better approaches VERY welcome.
pushd build
echo "travis_fold:start:qgis_build"
$CTEST_BUILD_COMMAND
echo "travis_fold:end:qgis_build"
popd

python ${TRAVIS_BUILD_DIR}/.ci/travis/scripts/ctest2travis.py \
xvfb-run ctest -V -E "$(cat ${DIR}/blacklist.txt | sed -r '/^(#.*?)?$/d' | paste -sd '|' -)" -S ${DIR}/../travis.ctest --output-on-failure
106 changes: 106 additions & 0 deletions .docker/Dockerfile
@@ -0,0 +1,106 @@


FROM ubuntu:16.04
MAINTAINER Denis Rouzaud <denis.rouzaud@gmail.com>

LABEL Description="Docker container with QGIS dependencies" Vendor="QGIS.org" Version="1.0"

ENV CORES 8

RUN apt-get update \
&& apt-get install -y software-properties-common \
&& add-apt-repository ppa:ubuntugis/ubuntugis-unstable \
&& apt-get update \
&& apt-get install -y \
bison \
ca-certificates \
ccache \
clang \
cmake \
dh-python \
flex \
gdal-bin \
graphviz \
grass-dev \
libexpat1-dev \
libfcgi-dev \
libgdal-dev \
libgeos-dev \
libgsl-dev \
libpq-dev \
libproj-dev \
libqca-qt5-2-dev \
libqca-qt5-2-plugins \
libqt5opengl5-dev \
libqt5scintilla2-dev \
libqt5sql5-sqlite \
libqt5svg5-dev \
libqt5webkit5-dev \
libqt5xmlpatterns5-dev \
libqwt-qt5-dev \
libspatialindex-dev \
libspatialite-dev \
libsqlite3-dev \
libsqlite3-mod-spatialite \
libzip-dev \
lighttpd \
locales \
ninja-build \
pkg-config \
poppler-utils \
postgresql-client \
pyqt5-dev \
pyqt5-dev-tools \
pyqt5.qsci-dev \
python3-all-dev \
python3-dev \
python3-future \
python3-gdal \
python3-mock \
python3-nose2 \
python3-pip \
python3-psycopg2 \
python3-pyqt5 \
python3-pyqt5.qsci \
python3-pyqt5.qtsql \
python3-pyqt5.qtsvg \
python3-sip \
python3-sip-dev \
python3-termcolor \
python3-yaml \
qt5keychain-dev \
qtbase5-dev \
qtpositioning5-dev \
qtscript5-dev \
qttools5-dev \
qttools5-dev-tools \
spawn-fcgi \
txt2tags \
xauth \
xfonts-100dpi \
xfonts-75dpi \
xfonts-base \
xfonts-scalable \
xvfb \
&& pip3 install \
psycopg2 \
numpy \
nose2 \
pyyaml \
mock \
future \
termcolor \
&& apt-get autoremove -y python3-pip python2.7 \
&& apt-get clean

RUN echo "alias python=python3" >> ~/.bash_aliases

ENV CC=/usr/lib/ccache/clang
ENV CXX=/usr/lib/ccache/clang++
ENV QT_SELECT=5

# RUN cmake --version
# RUN ${CC} --version
# RUN ${CXX} --version

CMD /root/QGIS/.ci/travis/linux/docker-build-test.sh
19 changes: 19 additions & 0 deletions .docker/docker-compose.travis.yml
@@ -0,0 +1,19 @@
version: '3'
services:
postgres:
image: kartoza/postgis:9.5-2.2
command: sh -c "echo \"host all all 172.18.0.0/16 md5\" >> /etc/postgresql/9.5/main/pg_hba.conf && /start-postgis.sh"

qgis-deps:
image: qgis/qgis3-build-deps:${DOCKER_TAG}
volumes:
- ${TRAVIS_BUILD_DIR}:/root/QGIS
- $HOME/.ccache:/root/.ccache
links:
- postgres
environment:
- TRAVIS_BUILD_DIR=${TRAVIS_BUILD_DIR}
- TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST}
- TRAVIS_PULL_REQUEST_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH}
- TRAVIS_COMMIT_RANGE=${TRAVIS_COMMIT_RANGE}
- TRAVIS_OS_NAME=${TRAVIS_OS_NAME}
25 changes: 7 additions & 18 deletions .travis.yml
Expand Up @@ -3,39 +3,28 @@ matrix:
include:
# QT5 based build with Python 3 // using container based builds and prebuild binary dependencies in osgeo4travis
- os: linux
services: docker
language: python # This lets us use newer python versions from virtualenv
env:
- LLVM_VERSION=3.8
- TRAVIS_CONFIG=linux
dist: precise
- DOCKER_COMPOSE=${TRAVIS_BUILD_DIR}/.docker/docker-compose.travis.yml
- DOCKER_TAG=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && echo $TRAVIS_BRANCH | sed 's/master/latest/' || echo "latest" )
- DOCKER_PUSH=$( [[ $TRAVIS_REPO_SLUG =~ qgis/QGIS ]] && [[ $TRAVIS_PULL_REQUEST =~ false ]] && echo "true" || echo "false" )

dist: trusty
sudo: false
cache:
apt: true
pip: true
directories:
- $HOME/.ccache
- $HOME/depcache
timeout: 1000
compiler: clang
python: "3.3"
python: "3.5"
addons:
postgresql: "9.4"
apt:
sources:
- llvm-toolchain-precise-3.8
- ubuntu-toolchain-r-test
packages:
- bison
- flex
- libpq-dev
- libfcgi-dev
- libfftw3-3
- libgsl0-dev
- pkg-config
- poppler-utils
- xvfb
- clang-3.8
- libzip-dev


- os: linux
Expand Down

0 comments on commit 0092ce1

Please sign in to comment.