Fix xyx OSM tiles too many downloads

Fixes #34813 by introducing a Prefetch rendering flag and raster provider capability that can be set by a provider when prefetching of out-of-viewport tiles is allowed. For now it is only denied (hardcoded) for OSM but the implementation allows per-layer setting when a GUI element to set the flag will been created.
qgis · May 13, 2020 · f029b1f · f029b1f
1 parent 59e1dce
commit f029b1f
Show file tree

Hide file tree

Showing 15 changed files with 60 additions and 27 deletions.
diff --git a/python/core/auto_generated/qgsmapsettings.sip.in b/python/core/auto_generated/qgsmapsettings.sip.in
@@ -344,6 +344,7 @@ Gets color that is used for drawing of selected vector features
       RenderPartialOutput,
       RenderPreviewJob,
       RenderBlocking,
+      Prefetch,
       // TODO: ignore scale-based visibility (overview)
     };
     typedef QFlags<QgsMapSettings::Flag> Flags;

diff --git a/python/core/auto_generated/qgsrendercontext.sip.in b/python/core/auto_generated/qgsrendercontext.sip.in
@@ -43,6 +43,7 @@ to be rendered etc.
       RenderPartialOutput,
       RenderPreviewJob,
       RenderBlocking,
+      Prefetch,
     };
     typedef QFlags<QgsRenderContext::Flag> Flags;
 

diff --git a/python/core/auto_generated/raster/qgsrasterinterface.sip.in b/python/core/auto_generated/raster/qgsrasterinterface.sip.in
@@ -161,6 +161,7 @@ Base class for processing filters like renderers, reprojector, resampler etc.
       IdentifyText,
       IdentifyHtml,
       IdentifyFeature,
+      Prefetch,
     };
 
     QgsRasterInterface( QgsRasterInterface *input = 0 );
@@ -179,7 +180,7 @@ Returns a bitmask containing the supported capabilities
 
     QString capabilitiesString() const;
 %Docstring
-Returns the above in friendly format.
+Returns the raster interface capabilities in friendly format.
 %End
 
     virtual Qgis::DataType dataType( int bandNo ) const = 0;

diff --git a/src/core/providers/gdal/qgsgdalprovider.cpp b/src/core/providers/gdal/qgsgdalprovider.cpp
@@ -1233,7 +1233,8 @@ int QgsGdalProvider::capabilities() const
                    | QgsRasterDataProvider::Size
                    | QgsRasterDataProvider::BuildPyramids
                    | QgsRasterDataProvider::Create
-                   | QgsRasterDataProvider::Remove;
+                   | QgsRasterDataProvider::Remove
+                   | QgsRasterDataProvider::Prefetch;
   if ( mDriverName != QLatin1String( "WMS" ) )
   {
     capability |= QgsRasterDataProvider::Size;

diff --git a/src/core/qgsmapsettings.h b/src/core/qgsmapsettings.h
@@ -310,6 +310,7 @@ class CORE_EXPORT QgsMapSettings : public QgsTemporalRangeObject
       RenderPartialOutput      = 0x200, //!< Whether to make extra effort to update map image with partially rendered layers (better for interactive map canvas). Added in QGIS 3.0
       RenderPreviewJob         = 0x400, //!< Render is a 'canvas preview' render, and shortcuts should be taken to ensure fast rendering
       RenderBlocking           = 0x800, //!< Render and load remote sources in the same thread to ensure rendering remote sources (svg and images). WARNING: this flag must NEVER be used from GUI based applications (like the main QGIS application) or crashes will result. Only for use in external scripts or QGIS server.
+      Prefetch                 = 0x1000, //!< The render is a prefetched image, not visible in the current view extent
       // TODO: ignore scale-based visibility (overview)
     };
     Q_DECLARE_FLAGS( Flags, Flag )

diff --git a/src/core/qgsrendercontext.cpp b/src/core/qgsrendercontext.cpp
@@ -187,6 +187,7 @@ QgsRenderContext QgsRenderContext::fromMapSettings( const QgsMapSettings &mapSet
   ctx.setFlag( RenderPartialOutput, mapSettings.testFlag( QgsMapSettings::RenderPartialOutput ) );
   ctx.setFlag( RenderPreviewJob, mapSettings.testFlag( QgsMapSettings::RenderPreviewJob ) );
   ctx.setFlag( RenderBlocking, mapSettings.testFlag( QgsMapSettings::RenderBlocking ) );
+  ctx.setFlag( Prefetch, mapSettings.testFlag( QgsMapSettings::Prefetch ) );
   ctx.setScaleFactor( mapSettings.outputDpi() / 25.4 ); // = pixels per mm
   ctx.setRendererScale( mapSettings.scale() );
   ctx.setExpressionContext( mapSettings.expressionContext() );

diff --git a/src/core/qgsrendercontext.h b/src/core/qgsrendercontext.h
@@ -79,6 +79,7 @@ class CORE_EXPORT QgsRenderContext : public QgsTemporalRangeObject
       RenderPartialOutput      = 0x100, //!< Whether to make extra effort to update map image with partially rendered layers (better for interactive map canvas). Added in QGIS 3.0
       RenderPreviewJob         = 0x200, //!< Render is a 'canvas preview' render, and shortcuts should be taken to ensure fast rendering
       RenderBlocking           = 0x400, //!< Render and load remote sources in the same thread to ensure rendering remote sources (svg and images). WARNING: this flag must NEVER be used from GUI based applications (like the main QGIS application) or crashes will result. Only for use in external scripts or QGIS server.
+      Prefetch                 = 0x800, //!< The render is a prefetched image, not visible in the current view extent
     };
     Q_DECLARE_FLAGS( Flags, Flag )
 

diff --git a/src/core/raster/qgsrasterinterface.h b/src/core/raster/qgsrasterinterface.h
@@ -192,6 +192,7 @@ class CORE_EXPORT QgsRasterInterface
       IdentifyText     = 1 << 7, // WMS text
       IdentifyHtml     = 1 << 8, // WMS HTML
       IdentifyFeature  = 1 << 9, // WMS GML -> feature
+      Prefetch         = 1 << 10, // allow prefetching of out-of-view images
     };
 
     QgsRasterInterface( QgsRasterInterface *input = nullptr );
@@ -208,7 +209,7 @@ class CORE_EXPORT QgsRasterInterface
     }
 
     /**
-     *  Returns the above in friendly format.
+     *  Returns the raster interface capabilities in friendly format.
      */
     QString capabilitiesString() const;
 

diff --git a/src/core/raster/qgsrasterlayerrenderer.cpp b/src/core/raster/qgsrasterlayerrenderer.cpp
@@ -64,6 +64,7 @@ void QgsRasterLayerRendererFeedback::onNewData()
 ///
 QgsRasterLayerRenderer::QgsRasterLayerRenderer( QgsRasterLayer *layer, QgsRenderContext &rendererContext )
   : QgsMapLayerRenderer( layer->id(), &rendererContext )
+  , mLayer( layer )
   , mFeedback( new QgsRasterLayerRendererFeedback( this ) )
 {
   QgsMapToPixel mapToPixel = rendererContext.mapToPixel();
@@ -264,10 +265,12 @@ QgsRasterLayerRenderer::~QgsRasterLayerRenderer()
 
 bool QgsRasterLayerRenderer::render()
 {
-  if ( !mRasterViewPort )
-    return true; // outside of layer extent - nothing to do
-
-  //R->draw( mPainter, mRasterViewPort, &mMapToPixel );
+  // Skip rendering of out of view tiles (xyz)
+  if ( !mRasterViewPort || ( mLayer && renderContext()->testFlag( QgsRenderContext::Flag::Prefetch ) &&
+                             mLayer->dataProvider() &&
+                             !( mLayer->dataProvider()->capabilities() &
+                                QgsRasterInterface::Capability::Prefetch ) ) )
+    return true;
 
   QElapsedTimer time;
   time.start();

diff --git a/src/core/raster/qgsrasterlayerrenderer.h b/src/core/raster/qgsrasterlayerrenderer.h
@@ -80,6 +80,8 @@ class CORE_EXPORT QgsRasterLayerRenderer : public QgsMapLayerRenderer
 
     QgsRasterPipe *mPipe = nullptr;
 
+    QgsRasterLayer *mLayer;
+
     //! feedback class for cancellation and preview generation
     QgsRasterLayerRendererFeedback *mFeedback = nullptr;
 

diff --git a/src/gui/qgsmapcanvas.cpp b/src/gui/qgsmapcanvas.cpp
@@ -2559,6 +2559,10 @@ void QgsMapCanvas::startPreviewJob( int number )
   jobSettings.setFlag( QgsMapSettings::DrawLabeling, false );
   jobSettings.setFlag( QgsMapSettings::RenderPreviewJob, true );
 
+  // Tell the renderer that this image is out of current view extent
+  if ( number != 4 )
+    jobSettings.setFlag( QgsMapSettings::Flag::Prefetch );
+
   // truncate preview layers to fast layers
   const QList<QgsMapLayer *> layers = jobSettings.layers();
   QList< QgsMapLayer * > previewLayers;

diff --git a/src/providers/arcgisrest/qgsamsprovider.h b/src/providers/arcgisrest/qgsamsprovider.h
@@ -90,7 +90,7 @@ class QgsAmsProvider : public QgsRasterDataProvider
 
     /* Inherited from QgsRasterInterface */
     int bandCount() const override { return 1; }
-    int capabilities() const override { return Identify | IdentifyText | IdentifyFeature; }
+    int capabilities() const override { return Identify | IdentifyText | IdentifyFeature | Prefetch; }
 
     /* Inherited from QgsRasterDataProvider */
     QgsRectangle extent() const override { return mExtent; }

diff --git a/src/providers/postgres/raster/qgspostgresrasterprovider.cpp b/src/providers/postgres/raster/qgspostgresrasterprovider.cpp
@@ -783,7 +783,8 @@ int QgsPostgresRasterProvider::capabilities() const
                          | QgsRasterDataProvider::Size
                          // TODO:| QgsRasterDataProvider::BuildPyramids
                          | QgsRasterDataProvider::Create
-                         | QgsRasterDataProvider::Remove;
+                         | QgsRasterDataProvider::Remove
+                         | QgsRasterDataProvider::Prefetch;
   return capability;
 }
 

diff --git a/src/providers/wcs/qgswcsprovider.cpp b/src/providers/wcs/qgswcsprovider.cpp
@@ -1220,6 +1220,7 @@ int QgsWcsProvider::capabilities() const
   int capability = NoCapabilities;
   capability |= QgsRasterDataProvider::Identify;
   capability |= QgsRasterDataProvider::IdentifyValue;
+  capability |= QgsRasterDataProvider::Prefetch;
 
   if ( mHasSize )
   {

diff --git a/src/providers/wms/qgswmsprovider.cpp b/src/providers/wms/qgswmsprovider.cpp
@@ -743,20 +743,24 @@ QImage *QgsWmsProvider::draw( QgsRectangle const &viewExtent, int pixelWidth, in
       return image;
     }
 
-    QgsDebugMsgLevel( QStringLiteral( "layer extent: %1,%2 %3x%4" )
-                      .arg( qgsDoubleToString( mLayerExtent.xMinimum() ),
-                            qgsDoubleToString( mLayerExtent.yMinimum() ) )
-                      .arg( mLayerExtent.width() )
-                      .arg( mLayerExtent.height() ), 3
-                    );
+    QgsDebugMsg( QStringLiteral( "layer extent: %1,%2,%3,%4 %5x%6" )
+                 .arg( qgsDoubleToString( mLayerExtent.xMinimum() ),
+                       qgsDoubleToString( mLayerExtent.yMinimum() ) )
+                 .arg( qgsDoubleToString( mLayerExtent.xMaximum() ),
+                       qgsDoubleToString( mLayerExtent.yMaximum() ) )
+                 .arg( mLayerExtent.width() )
+                 .arg( mLayerExtent.height() )
+               );
 
-    QgsDebugMsgLevel( QStringLiteral( "view extent: %1,%2 %3x%4  res:%5" )
-                      .arg( qgsDoubleToString( viewExtent.xMinimum() ),
-                            qgsDoubleToString( viewExtent.yMinimum() ) )
-                      .arg( viewExtent.width() )
-                      .arg( viewExtent.height() )
-                      .arg( vres, 0, 'f' ), 3
-                    );
+    QgsDebugMsg( QStringLiteral( "view extent: %1,%2,%3,%4 %5x%6  res:%7" )
+                 .arg( qgsDoubleToString( viewExtent.xMinimum() ),
+                       qgsDoubleToString( viewExtent.yMinimum() ) )
+                 .arg( qgsDoubleToString( viewExtent.xMaximum() ),
+                       qgsDoubleToString( viewExtent.yMaximum() ) )
+                 .arg( viewExtent.width() )
+                 .arg( viewExtent.height() )
+                 .arg( vres, 0, 'f' )
+               );
 
     QgsDebugMsgLevel( QStringLiteral( "tile matrix %1,%2 res:%3 tilesize:%4x%5 matrixsize:%6x%7 id:%8" )
                       .arg( tm->topLeft.x() ).arg( tm->topLeft.y() ).arg( tm->tres )
@@ -885,7 +889,7 @@ QImage *QgsWmsProvider::draw( QgsRectangle const &viewExtent, int pixelWidth, in
       p.setRenderHint( QPainter::SmoothPixmapTransform, false );  // let's not waste time with bilinear filtering
 
       QList<TileImage> lowerResTiles, lowerResTiles2, higherResTiles;
-      // first we check lower resolution tiles: one level back, then two levels back (if there is still some are not covered),
+      // first we check lower resolution tiles: one level back, then two levels back (if there is still some area not covered),
       // finally (in the worst case we use one level higher resolution tiles). This heuristic should give
       // good overviews while not spending too much time drawing cached tiles from resolutions far away.
       fetchOtherResTiles( tileMode, viewExtent, image->width(), missing, tm->tres, 1, lowerResTiles );
@@ -1918,7 +1922,6 @@ int QgsWmsProvider::capabilities() const
   int capability = NoCapabilities;
   bool canIdentify = false;
 
-
   if ( mSettings.mTiled && mTileLayer )
   {
     QgsDebugMsgLevel( QStringLiteral( "Tiled." ), 2 );
@@ -1950,10 +1953,17 @@ int QgsWmsProvider::capabilities() const
     capability = mCaps.identifyCapabilities();
     if ( capability )
     {
-      capability |= Identify;
+      capability |= Capability::Identify;
     }
   }
 
+  // Prevent prefetch of XYZ openstreetmap images
+  // See: https://github.com/qgis/QGIS/issues/34813
+  if ( !( mSettings.mTiled && mSettings.mXyz && dataSourceUri().contains( QStringLiteral( "openstreetmap.org" ) ) ) )
+  {
+    capability |= Capability::Prefetch;
+  }
+
   QgsDebugMsgLevel( QStringLiteral( "capability = %1" ).arg( capability ), 2 );
   return capability;
 }
@@ -2942,16 +2952,20 @@ QgsRasterIdentifyResult QgsWmsProvider::identify( const QgsPointXY &point, QgsRa
     tres = it.key();
     tm = &it.value();
 
-    QgsDebugMsg( QStringLiteral( "layer extent: %1,%2 %3x%4" )
+    QgsDebugMsg( QStringLiteral( "layer extent: %1,%2,%3,%4 %5x%6" )
                  .arg( qgsDoubleToString( mLayerExtent.xMinimum() ),
                        qgsDoubleToString( mLayerExtent.yMinimum() ) )
+                 .arg( qgsDoubleToString( mLayerExtent.xMaximum() ),
+                       qgsDoubleToString( mLayerExtent.yMaximum() ) )
                  .arg( mLayerExtent.width() )
                  .arg( mLayerExtent.height() )
                );
 
-    QgsDebugMsg( QStringLiteral( "view extent: %1,%2 %3x%4  res:%5" )
+    QgsDebugMsg( QStringLiteral( "view extent: %1,%2,%3,%4 %5x%6  res:%7" )
                  .arg( qgsDoubleToString( boundingBox.xMinimum() ),
                        qgsDoubleToString( boundingBox.yMinimum() ) )
+                 .arg( qgsDoubleToString( boundingBox.xMaximum() ),
+                       qgsDoubleToString( boundingBox.yMaximum() ) )
                  .arg( boundingBox.width() )
                  .arg( boundingBox.height() )
                  .arg( vres, 0, 'f' )