[processing][FEATURE] Add "Save Log to File" algorithm for models

This algorithm saves the contents of the execution log (right up to the point in the model at which the 'save log' algorithm executes) to a file. It can be used to automatically store the debugging log when running models for later reference and transparency.
qgis · Mar 31, 2020 · 60af337 · 60af337
1 parent 0684d41
commit 60af337
Show file tree

Hide file tree

Showing 11 changed files with 224 additions and 11 deletions.
diff --git a/python/core/auto_generated/processing/qgsprocessingalgorithm.sip.in b/python/core/auto_generated/processing/qgsprocessingalgorithm.sip.in
@@ -48,6 +48,7 @@ Abstract base class for processing algorithms.
       FlagKnownIssues,
       FlagCustomException,
       FlagPruneModelBranchesBasedOnAlgorithmResults,
+      FlagSkipGenericModelLogging,
       FlagDeprecated,
     };
     typedef QFlags<QgsProcessingAlgorithm::Flag> Flags;

diff --git a/python/core/auto_generated/processing/qgsprocessingfeedback.sip.in b/python/core/auto_generated/processing/qgsprocessingfeedback.sip.in
@@ -108,7 +108,7 @@ Pushes a summary of the QGIS (and underlying library) version information to the
 .. versionadded:: 3.4.7
 %End
 
-    QString htmlLog() const;
+    virtual QString htmlLog() const;
 %Docstring
 Returns the HTML formatted contents of the log, which contains all messages pushed to the feedback object.
 
@@ -117,7 +117,7 @@ Returns the HTML formatted contents of the log, which contains all messages push
 .. versionadded:: 3.14
 %End
 
-    QString textLog() const;
+    virtual QString textLog() const;
 %Docstring
 Returns the plain text contents of the log, which contains all messages pushed to the feedback object.
 
@@ -172,6 +172,9 @@ to scale the current progress to account for progress through the overall proces
 
     virtual void pushConsoleInfo( const QString &info );
 
+    virtual QString htmlLog() const;
+
+    virtual QString textLog() const;
 
 };
 

diff --git a/src/analysis/CMakeLists.txt b/src/analysis/CMakeLists.txt
@@ -131,6 +131,7 @@ SET(QGIS_ANALYSIS_SRCS
   processing/qgsalgorithmreverselinedirection.cpp
   processing/qgsalgorithmrotate.cpp
   processing/qgsalgorithmruggedness.cpp
+  processing/qgsalgorithmsavelog.cpp
   processing/qgsalgorithmsaveselectedfeatures.cpp
   processing/qgsalgorithmsegmentize.cpp
   processing/qgsalgorithmserviceareafromlayer.cpp

diff --git a/src/analysis/processing/qgsalgorithmsavelog.cpp b/src/analysis/processing/qgsalgorithmsavelog.cpp
@@ -0,0 +1,93 @@
+/***************************************************************************
+                         qgsalgorithmsavelog.cpp
+                         ---------------------
+    begin                : March 2020
+    copyright            : (C) 2020 by Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsalgorithmsavelog.h"
+
+///@cond PRIVATE
+
+QString QgsSaveLogToFileAlgorithm::name() const
+{
+  return QStringLiteral( "savelog" );
+}
+
+QgsProcessingAlgorithm::Flags QgsSaveLogToFileAlgorithm::flags() const
+{
+  return QgsProcessingAlgorithm::flags() | FlagHideFromToolbox | FlagSkipGenericModelLogging;
+}
+
+QString QgsSaveLogToFileAlgorithm::displayName() const
+{
+  return QObject::tr( "Save log to file" );
+}
+
+QStringList QgsSaveLogToFileAlgorithm::tags() const
+{
+  return QObject::tr( "record,messages,logged" ).split( ',' );
+}
+
+QString QgsSaveLogToFileAlgorithm::group() const
+{
+  return QObject::tr( "Modeler tools" );
+}
+
+QString QgsSaveLogToFileAlgorithm::groupId() const
+{
+  return QStringLiteral( "modelertools" );
+}
+
+QString QgsSaveLogToFileAlgorithm::shortHelpString() const
+{
+  return QObject::tr( "This algorithm saves the model's execution log to a file.\n"
+                      "Optionally, the log can be saved in a HTML formatted version." );
+}
+
+QString QgsSaveLogToFileAlgorithm::shortDescription() const
+{
+  return QObject::tr( "Saves the model's log contents to a file." );
+}
+
+QgsSaveLogToFileAlgorithm *QgsSaveLogToFileAlgorithm::createInstance() const
+{
+  return new QgsSaveLogToFileAlgorithm();
+}
+
+void QgsSaveLogToFileAlgorithm::initAlgorithm( const QVariantMap & )
+{
+  addParameter( new QgsProcessingParameterFileDestination( QStringLiteral( "OUTPUT" ), QObject::tr( "Log file" ), QObject::tr( "Text files (*.txt);;HTML files (*.html *.HTML)" ) ) );
+  addParameter( new QgsProcessingParameterBoolean( QStringLiteral( "USE_HTML" ), QObject::tr( "Use HTML formatting" ), false ) );
+}
+
+QVariantMap QgsSaveLogToFileAlgorithm::processAlgorithm( const QVariantMap &parameters, QgsProcessingContext &context, QgsProcessingFeedback *feedback )
+{
+  const QString file = parameterAsFile( parameters, QStringLiteral( "OUTPUT" ), context );
+  const bool useHtml = parameterAsBool( parameters, QStringLiteral( "USE_HTML" ), context );
+  if ( !file.isEmpty() )
+  {
+    QFile exportFile( file );
+    if ( !exportFile.open( QIODevice::WriteOnly | QIODevice::Truncate ) )
+    {
+      throw QgsProcessingException( QObject::tr( "Could not save log to file %1" ).arg( file ) );
+    }
+    QTextStream fout( &exportFile );
+    fout << ( useHtml ? feedback->htmlLog() : feedback->textLog() );
+  }
+  QVariantMap res;
+  res.insert( QStringLiteral( "OUTPUT" ), file );
+  return res;
+}
+
+///@endcond
diff --git a/src/analysis/processing/qgsalgorithmsavelog.h b/src/analysis/processing/qgsalgorithmsavelog.h
@@ -0,0 +1,57 @@
+/***************************************************************************
+                         qgsalgorithmsavelog.h
+                         ---------------------
+    begin                : March 2020
+    copyright            : (C) 2020 by Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+ ***************************************************************************/
+
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSALGORITHMSAVELOG_H
+#define QGSALGORITHMSAVELOG_H
+
+#define SIP_NO_FILE
+
+#include "qgis_sip.h"
+#include "qgsprocessingalgorithm.h"
+#include "qgsapplication.h"
+
+///@cond PRIVATE
+
+/**
+ * Native save log to file algorithm.
+ */
+class QgsSaveLogToFileAlgorithm : public QgsProcessingAlgorithm
+{
+  public:
+    QgsSaveLogToFileAlgorithm() = default;
+    void initAlgorithm( const QVariantMap &configuration = QVariantMap() ) override;
+    Flags flags() const override;
+    QString name() const override;
+    QString displayName() const override;
+    QStringList tags() const override;
+    QString group() const override;
+    QString groupId() const override;
+    QString shortHelpString() const override;
+    QString shortDescription() const override;
+    QgsSaveLogToFileAlgorithm *createInstance() const override SIP_FACTORY;
+
+  protected:
+
+    QVariantMap processAlgorithm( const QVariantMap &parameters,
+                                  QgsProcessingContext &context, QgsProcessingFeedback * ) override;
+
+};
+
+
+///@endcond PRIVATE
+
+#endif // QGSALGORITHMSAVELOG_H
diff --git a/src/analysis/processing/qgsnativealgorithms.cpp b/src/analysis/processing/qgsnativealgorithms.cpp
@@ -125,6 +125,7 @@
 #include "qgsalgorithmreverselinedirection.h"
 #include "qgsalgorithmrotate.h"
 #include "qgsalgorithmruggedness.h"
+#include "qgsalgorithmsavelog.h"
 #include "qgsalgorithmsaveselectedfeatures.h"
 #include "qgsalgorithmsegmentize.h"
 #include "qgsalgorithmserviceareafromlayer.h"
@@ -331,6 +332,7 @@ void QgsNativeAlgorithms::loadAlgorithms()
   addAlgorithm( new QgsReverseLineDirectionAlgorithm() );
   addAlgorithm( new QgsRotateFeaturesAlgorithm() );
   addAlgorithm( new QgsRuggednessAlgorithm() );
+  addAlgorithm( new QgsSaveLogToFileAlgorithm() );
   addAlgorithm( new QgsSaveSelectedFeatures() );
   addAlgorithm( new QgsSegmentizeByMaximumAngleAlgorithm() );
   addAlgorithm( new QgsSegmentizeByMaximumDistanceAlgorithm() );

diff --git a/src/core/processing/models/qgsprocessingmodelalgorithm.cpp b/src/core/processing/models/qgsprocessingmodelalgorithm.cpp
@@ -282,18 +282,21 @@ QVariantMap QgsProcessingModelAlgorithm::processAlgorithm( const QVariantMap &pa
         continue;
 
       executedAlg = true;
-      if ( feedback )
-        feedback->pushDebugInfo( QObject::tr( "Prepare algorithm: %1" ).arg( childId ) );
 
       const QgsProcessingModelChildAlgorithm &child = mChildAlgorithms[ childId ];
+      std::unique_ptr< QgsProcessingAlgorithm > childAlg( child.algorithm()->create( child.configuration() ) );
+
+      const bool skipGenericLogging = childAlg->flags() & QgsProcessingAlgorithm::FlagSkipGenericModelLogging;
+      if ( feedback && !skipGenericLogging )
+        feedback->pushDebugInfo( QObject::tr( "Prepare algorithm: %1" ).arg( childId ) );
 
       QgsExpressionContext expContext = baseContext;
       expContext << QgsExpressionContextUtils::processingAlgorithmScope( child.algorithm(), parameters, context )
                  << createExpressionContextScopeForChildAlgorithm( childId, context, parameters, childResults );
       context.setExpressionContext( expContext );
 
       QVariantMap childParams = parametersForChildAlgorithm( child, parameters, childResults, expContext );
-      if ( feedback )
+      if ( feedback && !skipGenericLogging )
         feedback->setProgressText( QObject::tr( "Running %1 [%2/%3]" ).arg( child.description() ).arg( executed.count() + 1 ).arg( toExecute.count() ) );
 
       QStringList params;
@@ -303,7 +306,7 @@ QVariantMap QgsProcessingModelAlgorithm::processAlgorithm( const QVariantMap &pa
                child.algorithm()->parameterDefinition( childParamIt.key() )->valueAsPythonString( childParamIt.value(), context ) );
       }
 
-      if ( feedback )
+      if ( feedback && !skipGenericLogging )
       {
         feedback->pushInfo( QObject::tr( "Input Parameters:" ) );
         feedback->pushCommandInfo( QStringLiteral( "{ %1 }" ).arg( params.join( QStringLiteral( ", " ) ) ) );
@@ -313,7 +316,6 @@ QVariantMap QgsProcessingModelAlgorithm::processAlgorithm( const QVariantMap &pa
       childTime.start();
 
       bool ok = false;
-      std::unique_ptr< QgsProcessingAlgorithm > childAlg( child.algorithm()->create( child.configuration() ) );
       QVariantMap results = childAlg->run( childParams, context, &modelFeedback, &ok, child.configuration() );
       if ( !ok )
       {
@@ -389,7 +391,7 @@ QVariantMap QgsProcessingModelAlgorithm::processAlgorithm( const QVariantMap &pa
 
       childAlg.reset( nullptr );
       modelFeedback.setCurrentStep( executed.count() );
-      if ( feedback )
+      if ( feedback && !skipGenericLogging )
         feedback->pushInfo( QObject::tr( "OK. Execution took %1 s (%2 outputs)." ).arg( childTime.elapsed() / 1000.0 ).arg( results.count() ) );
     }
 

diff --git a/src/core/processing/qgsprocessingalgorithm.h b/src/core/processing/qgsprocessingalgorithm.h
@@ -78,6 +78,7 @@ class CORE_EXPORT QgsProcessingAlgorithm
       FlagKnownIssues = 1 << 9, //!< Algorithm has known issues
       FlagCustomException = 1 << 10, //!< Algorithm raises custom exception notices, don't use the standard ones
       FlagPruneModelBranchesBasedOnAlgorithmResults = 1 << 11, //!< Algorithm results will cause remaining model branches to be pruned based on the results of running the algorithm
+      FlagSkipGenericModelLogging = 1 << 12, //!< When running as part of a model, the generic algorithm setup and results logging should be skipped
       FlagDeprecated = FlagHideFromToolbox | FlagHideFromModeler, //!< Algorithm is deprecated
     };
     Q_DECLARE_FLAGS( Flags, Flag )

diff --git a/src/core/processing/qgsprocessingfeedback.cpp b/src/core/processing/qgsprocessingfeedback.cpp
@@ -104,6 +104,16 @@ void QgsProcessingFeedback::pushVersionInfo( const QgsProcessingProvider *provid
   }
 }
 
+QString QgsProcessingFeedback::htmlLog() const
+{
+  return mHtmlLog;
+}
+
+QString QgsProcessingFeedback::textLog() const
+{
+  return mTextLog;
+}
+
 
 QgsProcessingMultiStepFeedback::QgsProcessingMultiStepFeedback( int childAlgorithmCount, QgsProcessingFeedback *feedback )
   : mChildSteps( childAlgorithmCount )
@@ -149,6 +159,16 @@ void QgsProcessingMultiStepFeedback::pushConsoleInfo( const QString &info )
   mFeedback->pushConsoleInfo( info );
 }
 
+QString QgsProcessingMultiStepFeedback::htmlLog() const
+{
+  return mFeedback->htmlLog();
+}
+
+QString QgsProcessingMultiStepFeedback::textLog() const
+{
+  return mFeedback->textLog();
+}
+
 void QgsProcessingMultiStepFeedback::updateOverallProgress( double progress )
 {
   double baseProgress = 100.0 * static_cast< double >( mCurrentStep ) / mChildSteps;

diff --git a/src/core/processing/qgsprocessingfeedback.h b/src/core/processing/qgsprocessingfeedback.h
@@ -113,15 +113,15 @@ class CORE_EXPORT QgsProcessingFeedback : public QgsFeedback
      * \see textLog()
      * \since QGIS 3.14
      */
-    QString htmlLog() const { return mHtmlLog; }
+    virtual QString htmlLog() const;
 
     /**
      * Returns the plain text contents of the log, which contains all messages pushed to the feedback object.
      *
      * \see htmlLog()
      * \since QGIS 3.14
      */
-    QString textLog() const { return mTextLog; }
+    virtual QString textLog() const;
 
   private:
     bool mLogFeedback = true;
@@ -168,7 +168,8 @@ class CORE_EXPORT QgsProcessingMultiStepFeedback : public QgsProcessingFeedback
     void pushCommandInfo( const QString &info ) override;
     void pushDebugInfo( const QString &info ) override;
     void pushConsoleInfo( const QString &info ) override;
-
+    QString htmlLog() const override;
+    QString textLog() const override;
   private slots:
 
     void updateOverallProgress( double progress );

diff --git a/tests/src/analysis/testqgsprocessingalgs.cpp b/tests/src/analysis/testqgsprocessingalgs.cpp
@@ -114,6 +114,8 @@ class TestQgsProcessingAlgs: public QObject
 
     void filterByLayerType();
 
+    void saveLog();
+
   private:
 
     QString mPointLayerPath;
@@ -2379,5 +2381,35 @@ void TestQgsProcessingAlgs::filterByLayerType()
   QVERIFY( !results.contains( QStringLiteral( "VECTOR" ) ) );
 }
 
+void TestQgsProcessingAlgs::saveLog()
+{
+  std::unique_ptr< QgsProcessingAlgorithm > alg( QgsApplication::processingRegistry()->createAlgorithmById( QStringLiteral( "native:savelog" ) ) );
+  QVERIFY( alg != nullptr );
+
+  QVariantMap parameters;
+  parameters.insert( QStringLiteral( "OUTPUT" ), QgsProcessing::TEMPORARY_OUTPUT );
+
+  bool ok = false;
+  std::unique_ptr< QgsProcessingContext > context = qgis::make_unique< QgsProcessingContext >();
+  QgsProcessingFeedback feedback;
+  feedback.reportError( QStringLiteral( "test" ) );
+  QVariantMap results;
+  results = alg->run( parameters, *context, &feedback, &ok );
+  QVERIFY( ok );
+
+  QVERIFY( !results.value( QStringLiteral( "OUTPUT" ) ).toString().isEmpty() );
+  QFile file( results.value( QStringLiteral( "OUTPUT" ) ).toString() );
+  QVERIFY( file.open( QFile::ReadOnly  | QIODevice::Text ) );
+  QCOMPARE( file.readAll(), QStringLiteral( "test\n" ) );
+
+  parameters.insert( QStringLiteral( "USE_HTML" ), true );
+  results = alg->run( parameters, *context, &feedback, &ok );
+  QVERIFY( ok );
+  QVERIFY( !results.value( QStringLiteral( "OUTPUT" ) ).toString().isEmpty() );
+  QFile file2( results.value( QStringLiteral( "OUTPUT" ) ).toString() );
+  QVERIFY( file2.open( QFile::ReadOnly  | QIODevice::Text ) );
+  QCOMPARE( file2.readAll(), QStringLiteral( "<span style=\"color:red\">test</span><br/>" ) );
+}
+
 QGSTEST_MAIN( TestQgsProcessingAlgs )
 #include "testqgsprocessingalgs.moc"