Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #8475 from elpaso/docker-testing-env
Added QGIS testing environment to dockers
  • Loading branch information
elpaso committed Nov 17, 2018
2 parents d77d832 + 112e9d7 commit fb1d670
Show file tree
Hide file tree
Showing 11 changed files with 357 additions and 4 deletions.
4 changes: 4 additions & 0 deletions .ci/travis/linux/blacklist.txt
Expand Up @@ -31,3 +31,7 @@ PyQgsAuthManagerPKIPostgresTest
PyQgsAuthManagerPasswordPostgresTest
PyQgsAuthManagerOgrPostgresTest
PyQgsDbManagerPostgis

# Needs an OpenCL device, the library is not enough
qgis_openclutilstest

29 changes: 28 additions & 1 deletion .docker/qgis.dockerfile
Expand Up @@ -8,6 +8,9 @@ ARG CACHE_DIR
FROM qgis/qgis3-build-deps:${DOCKER_TAG}
MAINTAINER Denis Rouzaud <denis@opengis.ch>

LABEL Description="Docker container with QGIS" Vendor="QGIS.org" Version="1.1"


ENV CC=/usr/lib/ccache/clang
ENV CXX=/usr/lib/ccache/clang++
ENV QT_SELECT=5
Expand All @@ -25,7 +28,7 @@ RUN cmake \
-GNinja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=/usr \
-DWITH_DESKTOP=OFF \
-DWITH_DESKTOP=ON \
-DWITH_SERVER=ON \
-DWITH_3D=ON \
-DWITH_BINDINGS=ON \
Expand All @@ -42,4 +45,28 @@ RUN cmake \
&& ninja install \
&& rm -rf /usr/src/QGIS

################################################################################
# Python testing environment setup

# Add QGIS test runner
COPY .docker/qgis_resources/test_runner/qgis_* /usr/bin/

# Make all scripts executable
RUN chmod +x /usr/bin/qgis_*

# Add supervisor service configuration script
COPY .docker/qgis_resources/supervisor/supervisord.conf /etc/supervisor/
COPY .docker/qgis_resources/supervisor/supervisor.xvfb.conf /etc/supervisor/supervisor.d/

# Python paths are for
# - kartoza images (compiled)
# - deb installed
# - built from git
# needed to find PyQt wrapper provided by QGIS
ENV PYTHONPATH=/usr/share/qgis/python/:/usr/lib/python3/dist-packages/qgis:/usr/share/qgis/python/qgis


WORKDIR /

# Run supervisor
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"]
17 changes: 15 additions & 2 deletions .docker/qgis3-build-deps.dockerfile
@@ -1,7 +1,7 @@
FROM ubuntu:18.04
MAINTAINER Denis Rouzaud <denis@opengis.ch>

LABEL Description="Docker container with QGIS dependencies" Vendor="QGIS.org" Version="1.0"
LABEL Description="Docker container with QGIS dependencies" Vendor="QGIS.org" Version="1.1"

# && echo "deb http://ppa.launchpad.net/ubuntugis/ubuntugis-unstable/ubuntu xenial main" >> /etc/apt/sources.list \
# && echo "deb-src http://ppa.launchpad.net/ubuntugis/ubuntugis-unstable/ubuntu xenial main" >> /etc/apt/sources.list \
Expand All @@ -11,7 +11,8 @@ LABEL Description="Docker container with QGIS dependencies" Vendor="QGIS.org" Ve
RUN apt-get update \
&& apt-get install -y software-properties-common \
&& apt-get update \
&& apt-get install -y \
&& DEBIAN_FRONTEND=noninteractive \
apt-get install -y \
apt-transport-https \
bison \
ca-certificates \
Expand Down Expand Up @@ -77,6 +78,7 @@ RUN apt-get update \
python3-pyqt5.qsci \
python3-pyqt5.qtsql \
python3-pyqt5.qtsvg \
python3-pyqt5.qtwebkit \
python3-sip \
python3-sip-dev \
python3-termcolor \
Expand All @@ -103,6 +105,11 @@ RUN apt-get update \
xfonts-base \
xfonts-scalable \
xvfb \
opencl-headers \
ocl-icd-libopencl1 \
ocl-icd-opencl-dev \
supervisor \
expect \
&& pip3 install \
psycopg2 \
numpy \
Expand All @@ -114,6 +121,12 @@ RUN apt-get update \
owslib \
oauthlib \
pyopenssl \
pep8 \
pexpect \
capturer \
sphinx \
requests \
six \
&& apt-get clean


Expand Down
7 changes: 7 additions & 0 deletions .docker/qgis_resources/requirements.txt
@@ -0,0 +1,7 @@
pep8
pexpect
capturer
sphinx
requests
future
six
4 changes: 4 additions & 0 deletions .docker/qgis_resources/supervisor/supervisor.xvfb.conf
@@ -0,0 +1,4 @@
; Supervisor config file for Xvfb

[program:Xvfb]
command=/usr/bin/Xvfb :99 -screen 0 1024x768x24 -ac +extension GLX +render -noreset -nolisten tcp
13 changes: 13 additions & 0 deletions .docker/qgis_resources/supervisor/supervisord.conf
@@ -0,0 +1,13 @@
; Supervisor config file.

[supervisord]
nodaemon=true
logfile=/var/log/supervisor/supervisord.log
logfile_maxbytes=50MB
logfile_backups=10
loglevel=info
pidfile=/var/run/supervisord.pid
childlogdir=/var/log

[include]
files = /etc/supervisor/supervisor.d/*.conf
61 changes: 61 additions & 0 deletions .docker/qgis_resources/test_runner/qgis_setup.sh
@@ -0,0 +1,61 @@
#!/bin/bash
# Setup QGIS for the automated tests
# This is normally called from Travis or rundockertests.sh
# before running the tests for a particular plugin
#
# Note: on QGIS3 assumes the default profile for root user
#
# - create the folders
# - install startup.py monkey patches
# - disable tips
# - enable the plugin

PLUGIN_NAME=$1
CONF_MASTER_FOLDER="/root/.local/share/QGIS/QGIS3/profiles/default/QGIS/"
CONF_MASTER_FILE="${CONF_MASTER_FOLDER}/QGIS3.ini"

QGIS_MASTER_FOLDER="/root/.local/share/QGIS/QGIS3/profiles/default"
PLUGIN_MASTER_FOLDER="${QGIS_MASTER_FOLDER}/python/plugins"

STARTUP_MASTER_FOLDER="/root/.local/share/QGIS/QGIS3/"

# Creates the config file
mkdir -p $CONF_MASTER_FOLDER
if [ -e "$CONF_MASTER_FILE" ]; then
rm -f $CONF_MASTER_FILE
fi
touch $CONF_MASTER_FILE

# Creates plugin folder
mkdir -p $PLUGIN_MASTER_FOLDER
mkdir -p $STARTUP_MASTER_FOLDER

# Install the monkey patches to prevent modal stacktrace on python errors
cp /usr/bin/qgis_startup.py ${STARTUP_MASTER_FOLDER}/startup.py

# Disable tips
printf "[qgis]\n" >> $CONF_MASTER_FILE
SHOW_TIPS=$(qgis --help 2>&1 | head -2 | grep 'QGIS - ' | perl -npe 'chomp; s/QGIS - (\d+)\.(\d+).*/showTips\1\2=false/')
printf "%s\n\n" "$SHOW_TIPS" >> $CONF_MASTER_FILE

if [ -n "$PLUGIN_NAME" ]; then
# Enable plugin
printf '[PythonPlugins]\n' >> $CONF_MASTER_FILE
printf "%s=true\n\n" "$PLUGIN_NAME" >> $CONF_MASTER_FILE
fi

# Disable firstRunVersionFlag for master
{
printf
"\n[migration]\n"
"fileVersion=2\n"
"firstRunVersionFlag=30500\n"
"settings=true\n\n"
} >> $CONF_MASTER_FILE


# Install the plugin
if [ ! -d "${PLUGIN_MASTER_FOLDER}/${PLUGIN_NAME}" ]; then
ln -s "/tests_directory/${PLUGIN_NAME}" "${PLUGIN_MASTER_FOLDER}"
echo "Plugin master folder linked in ${PLUGIN_MASTER_FOLDER}/${PLUGIN_NAME}"
fi
26 changes: 26 additions & 0 deletions .docker/qgis_resources/test_runner/qgis_startup.py
@@ -0,0 +1,26 @@
"""
Disable QGIS modal error dialog.
This script is meant to be run automatically when QGIS starts.
Is should be renamed to `startup.py` and placed into
~/.qgis3/python/startup.py
"""
from qgis import utils
import traceback


def _showException(type, value, tb, msg, messagebar=False):
print(msg)
logmessage = ''
for s in traceback.format_exception(type, value, tb):
logmessage += s.decode('utf-8', 'replace') if hasattr(s, 'decode') else s
print(logmessage)


def _open_stack_dialog(type, value, tb, msg, pop_error=True):
print(msg)


utils.showException = _showException
utils.open_stack_dialog = _open_stack_dialog
170 changes: 170 additions & 0 deletions .docker/qgis_resources/test_runner/qgis_testrunner.py
@@ -0,0 +1,170 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
***************************************************************************
Launches a unit test inside QGIS and exit the application.
Arguments:
accepts a single argument with the package name in python dotted notation,
the program tries first to load the module and launch the `run_all`
function of the module, if that fails it considers the last part of
the dotted path to be the function name and the previous part to be the
module.
Extra options for QGIS command line can be passed in the env var
QGIS_EXTRA_OPTIONS
Example run:
# Will load geoserverexplorer.test.catalogtests and run `run_all`
QGIS_EXTRA_OPTIONS='--optionspath .' \
GSHOSTNAME=localhost \
python qgis_testrunner.py geoserverexplorer.test.catalogtests
GSHOSTNAME=localhost \
python qgis_testrunner.py geoserverexplorer.test.catalogtests.run_my
---------------------
Date : May 2016
Copyright : (C) 2016 by Alessandro Pasotti
Email : apasotti at boundlessgeo dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************
"""

__author__ = 'Alessandro Pasotti'
__date__ = 'May 2016'

import os
import re
import sys
import traceback
import signal
import importlib
from pexpect import run
from pipes import quote

from qgis.utils import iface


def eprint(text):
sys.__stderr__.write(text + "\n")


def __get_test_function(test_module_name):
"""
Load the test module and return the test function
"""
print("QGIS Test Runner - Trying to import %s" % test_module_name)
try:
test_module = importlib.import_module(test_module_name)
function_name = 'run_all'
except ImportError as e:
traceback.print_exc(file=sys.stdout)
# Strip latest name
pos = test_module_name.rfind('.')
if pos <= 0:
raise e
test_module_name, function_name = test_module_name[:pos], test_module_name[pos + 1:]
print("QGIS Test Runner - Trying to import %s" % test_module_name)
sys.stdout.flush()
try:
test_module = importlib.import_module(test_module_name)
except ImportError as e:
traceback.print_exc(file=sys.stdout)
raise e
return getattr(test_module, function_name, None)


if iface is None:
"""
Launch QGIS and passes itself as an init script
"""
sys.path.append(os.getcwd())
test_module_name = sys.argv[-1]
if __get_test_function(test_module_name) is None:
print("QGIS Test Runner - [ERROR] cannot load test function from %s" % test_module_name)
sys.exit(1)
try:
me = __file__
except NameError:
me = sys.argv[0]
os.environ['QGIS_DEBUG'] = '1'
args = [
'qgis',
os.environ.get('QGIS_EXTRA_OPTIONS', ''),
'--nologo',
'--noversioncheck',
'--code',
me,
test_module_name, # Must be the last one!
]
command_line = ' '.join(args)
print("QGIS Test Runner - launching QGIS as %s ..." % command_line)
out, returncode = run("sh -c " + quote(command_line), withexitstatus=1)
assert returncode is not None
print("QGIS Test Runner - QGIS exited.")
ok = out.find('(failures=') < 0 and \
len(re.findall(r'Ran \d+ tests in\s',
out, re.MULTILINE)) > 0
print('=' * 60)
if not ok:
print(out)
else:
eprint(out)
if len(out) == 0:
print("QGIS Test Runner - [WARNING] subprocess returned no output")
print('=' * 60)

print("QGIS Test Runner - %s bytes returned and finished with exit code: %s" % (len(out), 0 if ok else 1))
sys.exit(0 if ok else 1)

else: # We are inside QGIS!
# Start as soon as the initializationCompleted signal is fired
from qgis.core import QgsApplication, QgsProjectBadLayerHandler, QgsProject
from PyQt.QtCore import QDir
from qgis.utils import iface

class QgsProjectBadLayerDefaultHandler(QgsProjectBadLayerHandler):
def handleBadLayers(self, layers, dom):
pass

# Monkey patch QGIS Python console
from console.console_output import writeOut

def _write(self, m):
sys.__stdout__.write(m)
writeOut.write = _write

# Add current working dir to the python path
sys.path.append(QDir.current().path())

def __run_test():
"""
Run the test specified as last argument in the command line.
"""
# Disable modal handler for bad layers
QgsProject.instance().setBadLayerHandler(QgsProjectBadLayerDefaultHandler())
eprint("QGIS Test Runner Inside - starting the tests ...")
try:
test_module_name = QgsApplication.instance().arguments()[-1]
function_name = __get_test_function(test_module_name)
eprint("QGIS Test Runner Inside - executing function %s" % function_name)
function_name()
except Exception as e:
eprint("QGIS Test Runner Inside - [FAILED] Exception: %s" % e)
# Print tb
traceback.print_exc(file=sys.stdout)
app = QgsApplication.instance()
os.kill(app.applicationPid(), signal.SIGTERM)
iface.initializationCompleted.connect(__run_test)

0 comments on commit fb1d670

Please sign in to comment.