Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Create runtime profiles for various startup/provider initialisation
tasks in processing

Attempting to diagnose occasional very slow QGIS startup times
  • Loading branch information
nyalldawson committed May 18, 2020
1 parent a990adf commit c6690c2
Show file tree
Hide file tree
Showing 10 changed files with 205 additions and 162 deletions.
14 changes: 8 additions & 6 deletions python/plugins/processing/algs/gdal/GdalAlgorithmProvider.py
Expand Up @@ -27,7 +27,8 @@

from qgis.PyQt.QtCore import QCoreApplication
from qgis.core import (QgsApplication,
QgsProcessingProvider)
QgsProcessingProvider,
QgsRuntimeProfiler)
from processing.core.ProcessingConfig import ProcessingConfig, Setting
from .GdalUtils import GdalUtils

Expand Down Expand Up @@ -103,11 +104,12 @@ def __init__(self):
QgsApplication.processingRegistry().addAlgorithmAlias('qgis:buildvirtualvector', 'gdal:buildvirtualvector')

def load(self):
ProcessingConfig.settingIcons[self.name()] = self.icon()
ProcessingConfig.addSetting(Setting(self.name(), 'ACTIVATE_GDAL',
self.tr('Activate'), True))
ProcessingConfig.readSettings()
self.refreshAlgorithms()
with QgsRuntimeProfiler.profile('GDAL Provider'):
ProcessingConfig.settingIcons[self.name()] = self.icon()
ProcessingConfig.addSetting(Setting(self.name(), 'ACTIVATE_GDAL',
self.tr('Activate'), True))
ProcessingConfig.readSettings()
self.refreshAlgorithms()
return True

def unload(self):
Expand Down
79 changes: 41 additions & 38 deletions python/plugins/processing/algs/grass7/Grass7AlgorithmProvider.py
Expand Up @@ -28,7 +28,8 @@
QgsProcessingProvider,
QgsVectorFileWriter,
QgsMessageLog,
QgsProcessingUtils)
QgsProcessingUtils,
QgsRuntimeProfiler)
from processing.core.ProcessingConfig import (ProcessingConfig, Setting)
from .Grass7Utils import Grass7Utils
from .Grass7Algorithm import Grass7Algorithm
Expand All @@ -47,45 +48,47 @@ def __init__(self):
self.algs = []

def load(self):
ProcessingConfig.settingIcons[self.name()] = self.icon()
if self.activateSetting:
ProcessingConfig.addSetting(Setting(self.name(), self.activateSetting,
self.tr('Activate'), True))
if isMac():
with QgsRuntimeProfiler.profile('Grass Provider'):
ProcessingConfig.settingIcons[self.name()] = self.icon()
if self.activateSetting:
ProcessingConfig.addSetting(Setting(self.name(), self.activateSetting,
self.tr('Activate'), True))
if isMac():
ProcessingConfig.addSetting(Setting(
self.name(),
Grass7Utils.GRASS_FOLDER, self.tr('GRASS7 folder'),
Grass7Utils.grassPath(), valuetype=Setting.FOLDER))
ProcessingConfig.addSetting(Setting(
self.name(),
Grass7Utils.GRASS_LOG_COMMANDS,
self.tr('Log execution commands'), False))
ProcessingConfig.addSetting(Setting(
self.name(),
Grass7Utils.GRASS_LOG_CONSOLE,
self.tr('Log console output'), False))
ProcessingConfig.addSetting(Setting(
self.name(),
Grass7Utils.GRASS_FOLDER, self.tr('GRASS7 folder'),
Grass7Utils.grassPath(), valuetype=Setting.FOLDER))
ProcessingConfig.addSetting(Setting(
self.name(),
Grass7Utils.GRASS_LOG_COMMANDS,
self.tr('Log execution commands'), False))
ProcessingConfig.addSetting(Setting(
self.name(),
Grass7Utils.GRASS_LOG_CONSOLE,
self.tr('Log console output'), False))
ProcessingConfig.addSetting(Setting(
self.name(),
Grass7Utils.GRASS_HELP_PATH,
self.tr('Location of GRASS docs'),
Grass7Utils.grassHelpPath()))
# Add settings for using r.external/v.external instead of r.in.gdal/v.in.ogr
# but set them to False by default because the {r,v}.external implementations
# have some bugs on windows + there are algorithms that can't be used with
# external data (need a solid r.in.gdal/v.in.ogr).
# For more info have a look at e.g. https://trac.osgeo.org/grass/ticket/3927
ProcessingConfig.addSetting(Setting(
self.name(),
Grass7Utils.GRASS_USE_REXTERNAL,
self.tr('For raster layers, use r.external (faster) instead of r.in.gdal'),
False))
ProcessingConfig.addSetting(Setting(
self.name(),
Grass7Utils.GRASS_USE_VEXTERNAL,
self.tr('For vector layers, use v.external (faster) instead of v.in.ogr'),
False))
ProcessingConfig.readSettings()
self.refreshAlgorithms()
Grass7Utils.GRASS_HELP_PATH,
self.tr('Location of GRASS docs'),
Grass7Utils.grassHelpPath()))
# Add settings for using r.external/v.external instead of r.in.gdal/v.in.ogr
# but set them to False by default because the {r,v}.external implementations
# have some bugs on windows + there are algorithms that can't be used with
# external data (need a solid r.in.gdal/v.in.ogr).
# For more info have a look at e.g. https://trac.osgeo.org/grass/ticket/3927
ProcessingConfig.addSetting(Setting(
self.name(),
Grass7Utils.GRASS_USE_REXTERNAL,
self.tr('For raster layers, use r.external (faster) instead of r.in.gdal'),
False))
ProcessingConfig.addSetting(Setting(
self.name(),
Grass7Utils.GRASS_USE_VEXTERNAL,
self.tr('For vector layers, use v.external (faster) instead of v.in.ogr'),
False))
ProcessingConfig.readSettings()
self.refreshAlgorithms()

return True

def unload(self):
Expand Down
84 changes: 45 additions & 39 deletions python/plugins/processing/algs/otb/OtbAlgorithmProvider.py
Expand Up @@ -27,7 +27,11 @@
import re
from qgis.PyQt.QtGui import QIcon
from qgis.PyQt.QtCore import QCoreApplication
from qgis.core import (Qgis, QgsApplication, QgsProcessingProvider, QgsMessageLog)
from qgis.core import (Qgis,
QgsApplication,
QgsProcessingProvider,
QgsMessageLog,
QgsRuntimeProfiler)
from qgis import utils

from processing.core.ProcessingConfig import ProcessingConfig, Setting
Expand All @@ -44,44 +48,46 @@ def __init__(self):
self.version = '6.6.0'

def load(self):
group = self.name()
ProcessingConfig.settingIcons[group] = self.icon()
ProcessingConfig.addSetting(Setting(group, OtbUtils.ACTIVATE, self.tr('Activate'), True))
ProcessingConfig.addSetting(Setting(group, OtbUtils.FOLDER,
self.tr("OTB folder"),
OtbUtils.otbFolder(),
valuetype=Setting.FOLDER,
validator=self.validateOtbFolder
))
ProcessingConfig.addSetting(Setting(group, OtbUtils.APP_FOLDER,
self.tr("OTB application folder"),
OtbUtils.appFolder(),
valuetype=Setting.MULTIPLE_FOLDERS,
validator=self.validateAppFolders
))
ProcessingConfig.addSetting(Setting(group, OtbUtils.SRTM_FOLDER,
self.tr("SRTM tiles folder"),
OtbUtils.srtmFolder(),
valuetype=Setting.FOLDER
))
ProcessingConfig.addSetting(Setting(group, OtbUtils.GEOID_FILE,
self.tr("Geoid file"),
OtbUtils.geoidFile(),
valuetype=Setting.FOLDER
))
ProcessingConfig.addSetting(Setting(group, OtbUtils.MAX_RAM_HINT,
self.tr("Maximum RAM to use"),
OtbUtils.maxRAMHint(),
valuetype=Setting.STRING
))
ProcessingConfig.addSetting(Setting(group, OtbUtils.LOGGER_LEVEL,
self.tr("Logger level"),
OtbUtils.loggerLevel(),
valuetype=Setting.STRING,
validator=self.validateLoggerLevel
))
ProcessingConfig.readSettings()
self.refreshAlgorithms()
with QgsRuntimeProfiler.profile('OTB Provider'):
group = self.name()
ProcessingConfig.settingIcons[group] = self.icon()
ProcessingConfig.addSetting(Setting(group, OtbUtils.ACTIVATE, self.tr('Activate'), True))
ProcessingConfig.addSetting(Setting(group, OtbUtils.FOLDER,
self.tr("OTB folder"),
OtbUtils.otbFolder(),
valuetype=Setting.FOLDER,
validator=self.validateOtbFolder
))
ProcessingConfig.addSetting(Setting(group, OtbUtils.APP_FOLDER,
self.tr("OTB application folder"),
OtbUtils.appFolder(),
valuetype=Setting.MULTIPLE_FOLDERS,
validator=self.validateAppFolders
))
ProcessingConfig.addSetting(Setting(group, OtbUtils.SRTM_FOLDER,
self.tr("SRTM tiles folder"),
OtbUtils.srtmFolder(),
valuetype=Setting.FOLDER
))
ProcessingConfig.addSetting(Setting(group, OtbUtils.GEOID_FILE,
self.tr("Geoid file"),
OtbUtils.geoidFile(),
valuetype=Setting.FOLDER
))
ProcessingConfig.addSetting(Setting(group, OtbUtils.MAX_RAM_HINT,
self.tr("Maximum RAM to use"),
OtbUtils.maxRAMHint(),
valuetype=Setting.STRING
))
ProcessingConfig.addSetting(Setting(group, OtbUtils.LOGGER_LEVEL,
self.tr("Logger level"),
OtbUtils.loggerLevel(),
valuetype=Setting.STRING,
validator=self.validateLoggerLevel
))
ProcessingConfig.readSettings()
self.refreshAlgorithms()

return True

def unload(self):
Expand Down
12 changes: 7 additions & 5 deletions python/plugins/processing/algs/qgis/QgisAlgorithmProvider.py
Expand Up @@ -24,7 +24,8 @@
import os

from qgis.core import (QgsApplication,
QgsProcessingProvider)
QgsProcessingProvider,
QgsRuntimeProfiler)

from PyQt5.QtCore import QCoreApplication

Expand Down Expand Up @@ -196,11 +197,12 @@ def loadAlgorithms(self):
self.addAlgorithm(a)

def load(self):
success = super().load()
with QgsRuntimeProfiler.profile('QGIS Python Provider'):
success = super().load()

if success:
self.parameterTypeFieldsMapping = FieldsMapper.ParameterFieldsMappingType()
QgsApplication.instance().processingRegistry().addParameterType(self.parameterTypeFieldsMapping)
if success:
self.parameterTypeFieldsMapping = FieldsMapper.ParameterFieldsMappingType()
QgsApplication.instance().processingRegistry().addParameterType(self.parameterTypeFieldsMapping)

return success

Expand Down
33 changes: 18 additions & 15 deletions python/plugins/processing/algs/saga/SagaAlgorithmProvider.py
Expand Up @@ -28,7 +28,8 @@
QgsProcessingProvider,
QgsProcessingUtils,
QgsApplication,
QgsMessageLog)
QgsMessageLog,
QgsRuntimeProfiler)
from processing.core.ProcessingConfig import ProcessingConfig, Setting
from processing.tools.system import isWindows, isMac

Expand All @@ -50,20 +51,22 @@ def __init__(self):
self.algs = []

def load(self):
ProcessingConfig.settingIcons[self.name()] = self.icon()
ProcessingConfig.addSetting(Setting("SAGA", 'ACTIVATE_SAGA',
self.tr('Activate'), True))
ProcessingConfig.addSetting(Setting("SAGA",
SagaUtils.SAGA_IMPORT_EXPORT_OPTIMIZATION,
self.tr('Enable SAGA Import/Export optimizations'), False))
ProcessingConfig.addSetting(Setting("SAGA",
SagaUtils.SAGA_LOG_COMMANDS,
self.tr('Log execution commands'), True))
ProcessingConfig.addSetting(Setting("SAGA",
SagaUtils.SAGA_LOG_CONSOLE,
self.tr('Log console output'), True))
ProcessingConfig.readSettings()
self.refreshAlgorithms()
with QgsRuntimeProfiler.profile('SAGA Provider'):
ProcessingConfig.settingIcons[self.name()] = self.icon()
ProcessingConfig.addSetting(Setting("SAGA", 'ACTIVATE_SAGA',
self.tr('Activate'), True))
ProcessingConfig.addSetting(Setting("SAGA",
SagaUtils.SAGA_IMPORT_EXPORT_OPTIMIZATION,
self.tr('Enable SAGA Import/Export optimizations'), False))
ProcessingConfig.addSetting(Setting("SAGA",
SagaUtils.SAGA_LOG_COMMANDS,
self.tr('Log execution commands'), True))
ProcessingConfig.addSetting(Setting("SAGA",
SagaUtils.SAGA_LOG_CONSOLE,
self.tr('Log console output'), True))
ProcessingConfig.readSettings()
self.refreshAlgorithms()

return True

def unload(self):
Expand Down
66 changes: 41 additions & 25 deletions python/plugins/processing/core/Processing.py
Expand Up @@ -40,7 +40,8 @@
QgsProcessingOutputRasterLayer,
QgsProcessingOutputMapLayer,
QgsProcessingOutputMultipleLayers,
QgsProcessingFeedback)
QgsProcessingFeedback,
QgsRuntimeProfiler)

import processing
from processing.core.ProcessingConfig import ProcessingConfig
Expand All @@ -51,12 +52,25 @@
from processing.script import ScriptUtils
from processing.tools import dataobjects

from processing.algs.qgis.QgisAlgorithmProvider import QgisAlgorithmProvider # NOQA
from processing.algs.grass7.Grass7AlgorithmProvider import Grass7AlgorithmProvider
from processing.algs.gdal.GdalAlgorithmProvider import GdalAlgorithmProvider # NOQA
from processing.algs.otb.OtbAlgorithmProvider import OtbAlgorithmProvider # NOQA
from processing.algs.saga.SagaAlgorithmProvider import SagaAlgorithmProvider # NOQA
from processing.script.ScriptAlgorithmProvider import ScriptAlgorithmProvider # NOQA
with QgsRuntimeProfiler.profile('Import QGIS Provider'):
from processing.algs.qgis.QgisAlgorithmProvider import QgisAlgorithmProvider # NOQA

with QgsRuntimeProfiler.profile('Import GRASS Provider'):
from processing.algs.grass7.Grass7AlgorithmProvider import Grass7AlgorithmProvider

with QgsRuntimeProfiler.profile('Import GDAL Provider'):
from processing.algs.gdal.GdalAlgorithmProvider import GdalAlgorithmProvider # NOQA

with QgsRuntimeProfiler.profile('Import OTB Provider'):
from processing.algs.otb.OtbAlgorithmProvider import OtbAlgorithmProvider # NOQA

with QgsRuntimeProfiler.profile('Import SAGA Provider'):
from processing.algs.saga.SagaAlgorithmProvider import SagaAlgorithmProvider # NOQA

with QgsRuntimeProfiler.profile('Import Script Provider'):
from processing.script.ScriptAlgorithmProvider import ScriptAlgorithmProvider # NOQA


# from processing.preconfigured.PreconfiguredAlgorithmProvider import PreconfiguredAlgorithmProvider # NOQA

# should be loaded last - ensures that all dependent algorithms are available when loading models
Expand All @@ -83,24 +97,26 @@ def activateProvider(providerOrName, activate=True):
def initialize():
if "model" in [p.id() for p in QgsApplication.processingRegistry().providers()]:
return
# Add the basic providers
for c in [
QgisAlgorithmProvider,
Grass7AlgorithmProvider,
GdalAlgorithmProvider,
OtbAlgorithmProvider,
SagaAlgorithmProvider,
ScriptAlgorithmProvider,
ModelerAlgorithmProvider,
ProjectProvider
]:
p = c()
if QgsApplication.processingRegistry().addProvider(p):
Processing.BASIC_PROVIDERS.append(p)
# And initialize
ProcessingConfig.initialize()
ProcessingConfig.readSettings()
RenderingStyles.loadStyles()

with QgsRuntimeProfiler.profile('Initialize'):
# Add the basic providers
for c in [
QgisAlgorithmProvider,
Grass7AlgorithmProvider,
GdalAlgorithmProvider,
OtbAlgorithmProvider,
SagaAlgorithmProvider,
ScriptAlgorithmProvider,
ModelerAlgorithmProvider,
ProjectProvider
]:
p = c()
if QgsApplication.processingRegistry().addProvider(p):
Processing.BASIC_PROVIDERS.append(p)
# And initialize
ProcessingConfig.initialize()
ProcessingConfig.readSettings()
RenderingStyles.loadStyles()

@staticmethod
def deinitialize():
Expand Down

0 comments on commit c6690c2

Please sign in to comment.