[WFS provider] Fix excessive memory consumption on big layers (refs #18740)

rouault · rouault · commit 1121565d05c2 · 2018-05-22T23:22:50.000+02:00
There was a pseudo memory leak in qgsgml.cpp And the WFS downloader could also have to process big replies, causing a lot of features to be instanciated at once. Was seen on the 'portugal_addresses' layer of http://www.naturalgis.pt/cgi-bin/opendata/mapserv? Backport of 0addae5 c3c5c97 6cf1c50
diff --git a/src/core/qgsgml.cpp b/src/core/qgsgml.cpp
@@ -856,7 +856,7 @@ void QgsGmlStreamingParser::endElement( const XML_Char* el )
   const int localNameLen = ( pszSep ) ? ( int )( elLen - nsLen ) - 1 : elLen;
   ParseMode theParseMode( mParseModeStack.isEmpty() ? none : mParseModeStack.top() );
 
-  mDimension = mDimensionStack.isEmpty() ? 0 : mDimensionStack.top() ;
+  mDimension = mDimensionStack.isEmpty() ? 0 : mDimensionStack.pop() ;
 
   const bool isGMLNS = ( nsLen == mGMLNameSpaceURI.size() && mGMLNameSpaceURIPtr && memcmp( el, mGMLNameSpaceURIPtr, nsLen ) == 0 );
 
diff --git a/src/providers/wfs/qgswfsfeatureiterator.cpp b/src/providers/wfs/qgswfsfeatureiterator.cpp
@@ -436,6 +436,7 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
   int pagingIter = 1;
   QString gmlIdFirstFeatureFirstIter;
   bool disablePaging = false;
+  // Top level loop to do feature paging in WFS 2
   while ( true )
   {
     success = true;
@@ -456,9 +457,14 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
              false /* cache */ );
 
     int featureCountForThisResponse = 0;
+    bool bytesStillAvailableInReply = false;
+    // Loop until there is no data coming from the current request
     while ( true )
     {
-      loop.exec( QEventLoop::ExcludeUserInputEvents );
+      if ( !bytesStillAvailableInReply )
+      {
+        loop.exec( QEventLoop::ExcludeUserInputEvents );
+      }
       if ( mStop )
       {
         interrupted = true;
@@ -475,7 +481,10 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
       bool finished = false;
       if ( mReply )
       {
-        data = mReply->readAll();
+        // Limit the number of bytes to process at once, to avoid the GML parser to
+        // create too many objects.
+        data = mReply->read( 10 * 1024 * 1024 );
+        bytesStillAvailableInReply = mReply->bytesAvailable() > 0;
       }
       else
       {
diff --git a/src/providers/wfs/qgswfsrequest.cpp b/src/providers/wfs/qgswfsrequest.cpp
@@ -24,6 +24,8 @@
 #include <QNetworkCacheMetaData>
 #include <QCryptographicHash> // just for testin file:// fake_qgis_http_endpoint hack
 
+const qint64 READ_BUFFER_SIZE_HINT = 1024 * 1024;
+
 QgsWFSRequest::QgsWFSRequest( const QString& theUri )
     : mUri( theUri )
     , mReply( nullptr )
@@ -116,6 +118,7 @@ bool QgsWFSRequest::sendGET( const QUrl& url, bool synchronous, bool forceRefres
   }
 
   mReply = QgsNetworkAccessManager::instance()->get( request );
+  mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
   if ( !mUri.auth().setAuthorizationReply( mReply ) )
   {
     mErrorCode = QgsWFSRequest::NetworkError;
@@ -167,6 +170,7 @@ bool QgsWFSRequest::sendPOST( const QUrl& url, const QString& contentTypeHeader,
   request.setHeader( QNetworkRequest::ContentTypeHeader, contentTypeHeader );
 
   mReply = QgsNetworkAccessManager::instance()->post( request, data );
+  mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
   if ( !mUri.auth().setAuthorizationReply( mReply ) )
   {
     mErrorCode = QgsWFSRequest::NetworkError;
@@ -257,6 +261,7 @@ void QgsWFSRequest::replyFinished()
 
           QgsDebugMsg( QString( "redirected: %1 forceRefresh=%2" ).arg( redirect.toString() ).arg( mForceRefresh ) );
           mReply = QgsNetworkAccessManager::instance()->get( request );
+          mReply->setReadBufferSize( READ_BUFFER_SIZE_HINT );
           if ( !mUri.auth().setAuthorizationReply( mReply ) )
           {
             mResponse.clear();

Original file line number	Diff line number	Diff line change
`@@ -436,6 +436,7 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )`
`436`	`436`	`int pagingIter = 1;`
`437`	`437`	`QString gmlIdFirstFeatureFirstIter;`
`438`	`438`	`bool disablePaging = false;`
	`439`	`+ // Top level loop to do feature paging in WFS 2`
`439`	`440`	`while ( true )`
`440`	`441`	`{`
`441`	`442`	`success = true;`
`@@ -456,9 +457,14 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )`
`456`	`457`	`false /* cache */ );`
`457`	`458`
`458`	`459`	`int featureCountForThisResponse = 0;`
	`460`	`+ bool bytesStillAvailableInReply = false;`
	`461`	`+ // Loop until there is no data coming from the current request`
`459`	`462`	`while ( true )`
`460`	`463`	`{`
`461`		`- loop.exec( QEventLoop::ExcludeUserInputEvents );`
	`464`	`+ if ( !bytesStillAvailableInReply )`
	`465`	`+ {`
	`466`	`+ loop.exec( QEventLoop::ExcludeUserInputEvents );`
	`467`	`+ }`
`462`	`468`	`if ( mStop )`
`463`	`469`	`{`
`464`	`470`	`interrupted = true;`
`@@ -475,7 +481,10 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )`
`475`	`481`	`bool finished = false;`
`476`	`482`	`if ( mReply )`
`477`	`483`	`{`
`478`		`- data = mReply->readAll();`
	`484`	`+ // Limit the number of bytes to process at once, to avoid the GML parser to`
	`485`	`+ // create too many objects.`
	`486`	`+ data = mReply->read( 10 * 1024 * 1024 );`
	`487`	`+ bytesStillAvailableInReply = mReply->bytesAvailable() > 0;`
`479`	`488`	`}`
`480`	`489`	`else`
`481`	`490`	`{`