Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
QgsVectorLayerExporter: improve to use QgsFeature::approximateMemoryU…
…sage()

Flush when we have ~ 100 MB of features accumulated
  • Loading branch information
rouault authored and nyalldawson committed Oct 18, 2020
1 parent a04fcce commit d191aba
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 7 deletions.
8 changes: 3 additions & 5 deletions src/core/qgsvectorlayerexporter.cpp
Expand Up @@ -56,7 +56,6 @@ QgsVectorLayerExporter::QgsVectorLayerExporter( const QString &uri,
QgsFeatureSink::SinkFlags sinkFlags )
: mErrorCount( 0 )
, mAttributeCount( -1 )
, mFeatureSizeBuffer( 200 ) // Default value. May be overridden below

{
mProvider = nullptr;
Expand All @@ -68,7 +67,6 @@ QgsVectorLayerExporter::QgsVectorLayerExporter( const QString &uri,
( options[ QStringLiteral( "driverName" ) ].toString().compare( QLatin1String( "GPKG" ), Qt::CaseInsensitive ) == 0 ||
options[ QStringLiteral( "driverName" ) ].toString().compare( QLatin1String( "SQLite" ), Qt::CaseInsensitive ) == 0 ) )
{
mFeatureSizeBuffer = 10 * 1000; // Aggressively bump to reduce the number of SQLite transactions
if ( geometryType != QgsWkbTypes::NoGeometry )
{
// For GPKG/Spatialite, we explicitly ask not to create a spatial index at
Expand Down Expand Up @@ -228,10 +226,9 @@ bool QgsVectorLayerExporter::addFeature( QgsFeature &feat, Flags )
}

mFeatureBuffer.append( newFeat );
mFeatureBufferMemoryUsage += newFeat.approximateMemoryUsage();

// A better strategy would be to use an hypothetical QgsFeature::RAMUsage()
// method to decide when to flush.
if ( mFeatureBuffer.count() >= mFeatureSizeBuffer )
if ( mFeatureBufferMemoryUsage >= 100 * 1000 * 1000 )
{
return flushBuffer();
}
Expand All @@ -246,6 +243,7 @@ QString QgsVectorLayerExporter::lastError() const

bool QgsVectorLayerExporter::flushBuffer()
{
mFeatureBufferMemoryUsage = 0;
if ( mFeatureBuffer.count() <= 0 )
return true;

Expand Down
3 changes: 1 addition & 2 deletions src/core/qgsvectorlayerexporter.h
Expand Up @@ -163,11 +163,10 @@ class CORE_EXPORT QgsVectorLayerExporter : public QgsFeatureSink
int mAttributeCount;

QgsFeatureList mFeatureBuffer;
int mFeatureBufferMemoryUsage = 0;

bool mCreateSpatialIndex = true;

int mFeatureSizeBuffer;

#ifdef SIP_RUN
QgsVectorLayerExporter( const QgsVectorLayerExporter &rh );
#endif
Expand Down

0 comments on commit d191aba

Please sign in to comment.