[WFS provider] Fix excessive memory consumption on big layers (refs #…

…1870) There was a pseudo memory leak in qgsgml.cpp And the WFS downloader could also have to process big replies, causing a lot of features to be instanciated at once. Was seen on the 'portugal_addresses' layer of http://www.naturalgis.pt/cgi-bin/opendata/mapserv?
qgis · May 21, 2018 · 0addae5 · 0addae5
1 parent 19b3325
commit 0addae5
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 3 deletions.
diff --git a/src/core/qgsgml.cpp b/src/core/qgsgml.cpp
@@ -844,7 +844,7 @@ void QgsGmlStreamingParser::endElement( const XML_Char *el )
   const int localNameLen = ( pszSep ) ? ( int )( elLen - nsLen ) - 1 : elLen;
   ParseMode parseMode( mParseModeStack.isEmpty() ? None : mParseModeStack.top() );
 
-  mDimension = mDimensionStack.isEmpty() ? 0 : mDimensionStack.top();
+  mDimension = mDimensionStack.isEmpty() ? 0 : mDimensionStack.pop();
 
   const bool isGMLNS = ( nsLen == mGMLNameSpaceURI.size() && mGMLNameSpaceURIPtr && memcmp( el, mGMLNameSpaceURIPtr, nsLen ) == 0 );
 

diff --git a/src/providers/wfs/qgswfsfeatureiterator.cpp b/src/providers/wfs/qgswfsfeatureiterator.cpp
@@ -456,6 +456,7 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
   int pagingIter = 1;
   QString gmlIdFirstFeatureFirstIter;
   bool disablePaging = false;
+  // Top level loop to do feature paging in WFS 2
   while ( true )
   {
     success = true;
@@ -476,9 +477,14 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
              false /* cache */ );
 
     int featureCountForThisResponse = 0;
+    bool bytesStillAvailableInReply = false;
+    // Loop until there is no data coming from the current requst
     while ( true )
     {
-      loop.exec( QEventLoop::ExcludeUserInputEvents );
+      if ( !bytesStillAvailableInReply )
+      {
+        loop.exec( QEventLoop::ExcludeUserInputEvents );
+      }
       if ( mStop )
       {
         interrupted = true;
@@ -495,7 +501,10 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
       bool finished = false;
       if ( mReply )
       {
-        data = mReply->readAll();
+        // Limit the number of bytes to process at once, to avoid the GML parser to
+        // create too many objects.
+        data = mReply->read( 10 * 1024 * 1024 );
+        bytesStillAvailableInReply = mReply->bytesAvailable() > 0;
       }
       else
       {

diff --git a/src/providers/wfs/qgswfsrequest.cpp b/src/providers/wfs/qgswfsrequest.cpp
@@ -25,6 +25,8 @@
 #include <QNetworkCacheMetaData>
 #include <QCryptographicHash> // just for testin file:// fake_qgis_http_endpoint hack
 
+const qint64 READ_BUFFER_SIZE_HINT = 1024 * 1024;
+
 QgsWfsRequest::QgsWfsRequest( const QgsWFSDataSourceURI &uri )
   : mUri( uri )
   , mErrorCode( QgsWfsRequest::NoError )
@@ -125,6 +127,7 @@ bool QgsWfsRequest::sendGET( const QUrl &url, bool synchronous, bool forceRefres
   }
 
   mReply = QgsNetworkAccessManager::instance()->get( request );
+  mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
   if ( !mUri.auth().setAuthorizationReply( mReply ) )
   {
     mErrorCode = QgsWfsRequest::NetworkError;
@@ -176,6 +179,7 @@ bool QgsWfsRequest::sendPOST( const QUrl &url, const QString &contentTypeHeader,
   request.setHeader( QNetworkRequest::ContentTypeHeader, contentTypeHeader );
 
   mReply = QgsNetworkAccessManager::instance()->post( request, data );
+  mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
   if ( !mUri.auth().setAuthorizationReply( mReply ) )
   {
     mErrorCode = QgsWfsRequest::NetworkError;
@@ -266,6 +270,7 @@ void QgsWfsRequest::replyFinished()
 
           QgsDebugMsg( QString( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ) );
           mReply = QgsNetworkAccessManager::instance()->get( request );
+          mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
           if ( !mUri.auth().setAuthorizationReply( mReply ) )
           {
             mResponse.clear();