Make it possible to load multiple terrain tiles at the same time

Also adds some event tracing support to better understand what is going on under the hood
qgis · Nov 22, 2019 · adad95e · adad95e
1 parent e86d283
commit adad95e
Show file tree

Hide file tree

Showing 11 changed files with 99 additions and 61 deletions.
diff --git a/src/3d/chunks/qgschunkedentity_p.cpp b/src/3d/chunks/qgschunkedentity_p.cpp
@@ -81,7 +81,7 @@ QgsChunkedEntity::~QgsChunkedEntity()
   // derived classes have to make sure that any pending active job has finished / been canceled
   // before getting to this destructor - here it would be too late to cancel them
   // (e.g. objects required for loading/updating have been deleted already)
-  Q_ASSERT( !mActiveJob );
+  Q_ASSERT( mActiveJobs.isEmpty() );
 
   // clean up any pending load requests
   while ( !mChunkLoaderQueue->isEmpty() )
@@ -167,8 +167,7 @@ void QgsChunkedEntity::update( const SceneState &state )
   }
 
   // start a job from queue if there is anything waiting
-  if ( !mActiveJob )
-    startJob();
+  startJobs();
 
   mNeedsUpdate = false;  // just updated
 
@@ -205,7 +204,7 @@ void QgsChunkedEntity::updateNodes( const QList<QgsChunkNode *> &nodes, QgsChunk
     }
     else if ( node->state() == QgsChunkNode::Updating )
     {
-      cancelActiveJob();  // we have currently just one active job so that must be it
+      cancelActiveJob( node->updater() );
     }
 
     Q_ASSERT( node->state() == QgsChunkNode::Loaded );
@@ -216,13 +215,12 @@ void QgsChunkedEntity::updateNodes( const QList<QgsChunkNode *> &nodes, QgsChunk
   }
 
   // trigger update
-  if ( !mActiveJob )
-    startJob();
+  startJobs();
 }
 
 int QgsChunkedEntity::pendingJobsCount() const
 {
-  return mChunkLoaderQueue->count() + ( mActiveJob ? 1 : 0 );
+  return mChunkLoaderQueue->count() + mActiveJobs.count();
 }
 
 
@@ -323,7 +321,7 @@ void QgsChunkedEntity::onActiveJobFinished()
 
   QgsChunkQueueJob *job = qobject_cast<QgsChunkQueueJob *>( sender() );
   Q_ASSERT( job );
-  Q_ASSERT( job == mActiveJob );
+  Q_ASSERT( mActiveJobs.contains( job ) );
 
   QgsChunkNode *node = job->chunk();
 
@@ -332,8 +330,7 @@ void QgsChunkedEntity::onActiveJobFinished()
     Q_ASSERT( node->state() == QgsChunkNode::Loading );
     Q_ASSERT( node->loader() == loader );
 
-    QgsEventTracing::addEvent( QgsEventTracing::End, QStringLiteral( "3D" ),
-                               QStringLiteral( "Load %1/%2/%3" ).arg( node->tileZ() ).arg( node->tileX() ).arg( node->tileY() ) );
+    QgsEventTracing::addEvent( QgsEventTracing::AsyncEnd, QStringLiteral( "3D" ), QStringLiteral( "Load " ) + node->tileId().text(), node->tileId().text() );
 
     QgsEventTracing::ScopedEvent e( "3D", QString( "create" ) );
     // mark as loaded + create entity
@@ -358,82 +355,96 @@ void QgsChunkedEntity::onActiveJobFinished()
   else
   {
     Q_ASSERT( node->state() == QgsChunkNode::Updating );
-    QgsEventTracing::addEvent( QgsEventTracing::End, QStringLiteral( "3D" ),
-                               QStringLiteral( "Update %1/%2/%3" ).arg( node->tileZ() ).arg( node->tileX() ).arg( node->tileY() ) );
+    QgsEventTracing::addEvent( QgsEventTracing::AsyncEnd, QStringLiteral( "3D" ), QStringLiteral( "Update" ), node->tileId().text() );
     node->setUpdated();
   }
 
   // cleanup the job that has just finished
-  mActiveJob->deleteLater();
-  mActiveJob = nullptr;
+  mActiveJobs.removeOne( job );
+  job->deleteLater();
 
   // start another job - if any
-  startJob();
+  startJobs();
 
   if ( pendingJobsCount() != oldJobsCount )
     emit pendingJobsCountChanged();
 }
 
-void QgsChunkedEntity::startJob()
+void QgsChunkedEntity::startJobs()
 {
-  Q_ASSERT( !mActiveJob );
-  if ( mChunkLoaderQueue->isEmpty() )
-    return;
+  while ( mActiveJobs.count() < 4 )
+  {
+    if ( mChunkLoaderQueue->isEmpty() )
+      return;
 
-  QgsChunkListEntry *entry = mChunkLoaderQueue->takeFirst();
-  Q_ASSERT( entry );
-  QgsChunkNode *node = entry->chunk;
-  delete entry;
+    QgsChunkListEntry *entry = mChunkLoaderQueue->takeFirst();
+    Q_ASSERT( entry );
+    QgsChunkNode *node = entry->chunk;
+    delete entry;
 
+    QgsChunkQueueJob *job = startJob( node );
+    mActiveJobs.append( job );
+  }
+}
+
+QgsChunkQueueJob *QgsChunkedEntity::startJob( QgsChunkNode *node )
+{
   if ( node->state() == QgsChunkNode::QueuedForLoad )
   {
-    QgsEventTracing::addEvent( QgsEventTracing::Begin, QStringLiteral( "3D" ),
-                               QStringLiteral( "Load %1/%2/%3" ).arg( node->tileZ() ).arg( node->tileX() ).arg( node->tileY() ) );
+    QgsEventTracing::addEvent( QgsEventTracing::AsyncBegin, QStringLiteral( "3D" ), QStringLiteral( "Load " ) + node->tileId().text(), node->tileId().text() );
 
     QgsChunkLoader *loader = mChunkLoaderFactory->createChunkLoader( node );
     connect( loader, &QgsChunkQueueJob::finished, this, &QgsChunkedEntity::onActiveJobFinished );
     node->setLoading( loader );
-    mActiveJob = loader;
+    return loader;
   }
   else if ( node->state() == QgsChunkNode::QueuedForUpdate )
   {
-    QgsEventTracing::addEvent( QgsEventTracing::Begin, QStringLiteral( "3D" ),
-                               QStringLiteral( "Update %1/%2/%3" ).arg( node->tileZ() ).arg( node->tileX() ).arg( node->tileY() ) );
+    QgsEventTracing::addEvent( QgsEventTracing::AsyncBegin, QStringLiteral( "3D" ), QStringLiteral( "Update" ), node->tileId().text() );
 
     node->setUpdating();
     connect( node->updater(), &QgsChunkQueueJob::finished, this, &QgsChunkedEntity::onActiveJobFinished );
-    mActiveJob = node->updater();
+    return node->updater();
   }
   else
+  {
     Q_ASSERT( false );  // not possible
+    return nullptr;
+  }
 }
 
-void QgsChunkedEntity::cancelActiveJob()
+void QgsChunkedEntity::cancelActiveJob( QgsChunkQueueJob *job )
 {
-  Q_ASSERT( mActiveJob );
+  Q_ASSERT( job );
 
-  QgsChunkNode *node = mActiveJob->chunk();
+  QgsChunkNode *node = job->chunk();
 
-  if ( qobject_cast<QgsChunkLoader *>( mActiveJob ) )
+  if ( qobject_cast<QgsChunkLoader *>( job ) )
   {
     // return node back to skeleton
     node->cancelLoading();
 
-    QgsEventTracing::addEvent( QgsEventTracing::End, QStringLiteral( "3D" ),
-                               QStringLiteral( "Load %1/%2/%3" ).arg( node->tileZ() ).arg( node->tileX() ).arg( node->tileY() ) );
+    QgsEventTracing::addEvent( QgsEventTracing::AsyncEnd, QStringLiteral( "3D" ), QStringLiteral( "Load " ) + node->tileId().text(), node->tileId().text() );
   }
   else
   {
     // return node back to loaded state
     node->cancelUpdating();
 
-    QgsEventTracing::addEvent( QgsEventTracing::End, QStringLiteral( "3D" ),
-                               QStringLiteral( "Update %1/%2/%3" ).arg( node->tileZ() ).arg( node->tileX() ).arg( node->tileY() ) );
+    QgsEventTracing::addEvent( QgsEventTracing::AsyncEnd, QStringLiteral( "3D" ), QStringLiteral( "Update" ), node->tileId().text() );
   }
 
-  mActiveJob->cancel();
-  mActiveJob->deleteLater();
-  mActiveJob = nullptr;
+  job->cancel();
+  mActiveJobs.removeOne( job );
+  job->deleteLater();
+}
+
+void QgsChunkedEntity::cancelActiveJobs()
+{
+  while ( !mActiveJobs.isEmpty() )
+  {
+    cancelActiveJob( mActiveJobs.takeFirst() );
+  }
 }
 
 /// @endcond
diff --git a/src/3d/chunks/qgschunkedentity_p.h b/src/3d/chunks/qgschunkedentity_p.h
@@ -87,7 +87,8 @@ class QgsChunkedEntity : public Qt3DCore::QEntity
 
   protected:
     //! Cancels the background job that is currently in progress
-    void cancelActiveJob();
+    void cancelActiveJob( QgsChunkQueueJob *job );
+    void cancelActiveJobs();
     //! Sets whether the entity needs to get active nodes updated
     void setNeedsUpdate( bool needsUpdate ) { mNeedsUpdate = needsUpdate; }
 
@@ -97,7 +98,8 @@ class QgsChunkedEntity : public Qt3DCore::QEntity
     //! make sure that the chunk will be loaded soon (if not loaded yet) and not unloaded anytime soon (if loaded already)
     void requestResidency( QgsChunkNode *node );
 
-    void startJob();
+    void startJobs();
+    QgsChunkQueueJob *startJob( QgsChunkNode *node );
 
   private slots:
     void onActiveJobFinished();
@@ -136,8 +138,8 @@ class QgsChunkedEntity : public Qt3DCore::QEntity
     //! Entity that shows bounding boxes of active chunks (NULLPTR if not enabled)
     QgsChunkBoundsEntity *mBboxesEntity = nullptr;
 
-    //! job that is currently being processed (asynchronously in a worker thread)
-    QgsChunkQueueJob *mActiveJob = nullptr;
+    //! jobs that are currently being processed (asynchronously in worker threads)
+    QList<QgsChunkQueueJob *> mActiveJobs;
 };
 
 /// @endcond

diff --git a/src/3d/chunks/qgschunknode_p.h b/src/3d/chunks/qgschunknode_p.h
@@ -42,6 +42,19 @@ class QgsChunkQueueJob;
 class QgsChunkQueueJobFactory;
 
 
+//! Helper class to store X,Y,Z integer coordinates of a node
+struct QgsChunkNodeId
+{
+  //! Constructs node ID
+  QgsChunkNodeId( int _x = -1, int _y = -1, int _z = -1 )
+    : x( _x ), y( _y ), z( _z ) {}
+
+  int x, y, z;
+
+  //! Returns textual representation of the node ID in form of "Z/X/Y"
+  QString text() const { return QStringLiteral( "%1/%2/%3" ).arg( z ).arg( x ).arg( y ); }
+};
+
 /**
  * \ingroup 3d
  * Data structure for keeping track of chunks of data for 3D entities that use "out of core" rendering,
@@ -102,6 +115,8 @@ class QgsChunkNode
     int tileY() const { return mTileY; }
     //! Returns chunk tile Z coordinate of the tiling scheme
     int tileZ() const { return mTileZ; }
+    //! Returns chunk tile coordinates of the tiling scheme
+    QgsChunkNodeId tileId() const { return QgsChunkNodeId( mTileX, mTileY, mTileZ ); }
     //! Returns pointer to the parent node. Parent is NULLPTR in the root node
     QgsChunkNode *parent() const { return mParent; }
     //! Returns array of the four children. Children may be NULLPTR if they were not created yet

diff --git a/src/3d/terrain/qgsdemterraintileloader_p.cpp b/src/3d/terrain/qgsdemterraintileloader_p.cpp
@@ -100,7 +100,7 @@ Qt3DCore::QEntity *QgsDemTerrainTileLoader::createEntity( Qt3DCore::QEntity *par
   double half = side / 2;
 
 
-  QgsTerrainTileEntity *entity = new QgsTerrainTileEntity;
+  QgsTerrainTileEntity *entity = new QgsTerrainTileEntity( mNode->tileId() );
 
   // create geometry renderer
 
@@ -164,12 +164,10 @@ QgsDemHeightMapGenerator::~QgsDemHeightMapGenerator()
   delete mClonedProvider;
 }
 
-#include <QElapsedTimer>
 
 static QByteArray _readDtmData( QgsRasterDataProvider *provider, const QgsRectangle &extent, int res, const QgsCoordinateReferenceSystem &destCrs )
 {
-  QElapsedTimer t;
-  t.start();
+  QgsEventTracing::ScopedEvent e( QStringLiteral( "3D" ), QStringLiteral( "DEM" ) );
 
   // TODO: use feedback object? (but GDAL currently does not support cancellation anyway)
   QgsRasterInterface *input = provider;
@@ -214,9 +212,9 @@ static QByteArray _readOnlineDtm( QgsTerrainDownloader *downloader, const QgsRec
 
 int QgsDemHeightMapGenerator::render( int x, int y, int z )
 {
-  Q_ASSERT( mJobs.isEmpty() );  // should be always just one active job...
+  QgsChunkNodeId tileId( x, y, z );
 
-  QgsEventTracing::addEvent( QgsEventTracing::Begin, QStringLiteral( "3D" ), QStringLiteral( "DEM" ) );
+  QgsEventTracing::addEvent( QgsEventTracing::AsyncBegin, QStringLiteral( "3D" ), QStringLiteral( "DEM" ), tileId.text() );
 
   // extend the rect by half-pixel on each side? to get the values in "corners"
   QgsRectangle extent = mTilingScheme.tileToExtent( x, y, z );
@@ -228,6 +226,7 @@ int QgsDemHeightMapGenerator::render( int x, int y, int z )
 
   JobData jd;
   jd.jobId = ++mLastJobId;
+  jd.tileId = tileId;
   jd.extent = extent;
   jd.timer.start();
   // make a clone of the data provider so it is safe to use in worker thread
@@ -303,7 +302,7 @@ void QgsDemHeightMapGenerator::onFutureFinished()
   mJobs.remove( fw );
   fw->deleteLater();
 
-  QgsEventTracing::addEvent( QgsEventTracing::End, QStringLiteral( "3D" ), QStringLiteral( "DEM" ) );
+  QgsEventTracing::addEvent( QgsEventTracing::AsyncEnd, QStringLiteral( "3D" ), QStringLiteral( "DEM" ), jobData.tileId.text() );
 
   QByteArray data = jobData.future.result();
   emit heightMapReady( jobData.jobId, data );

diff --git a/src/3d/terrain/qgsdemterraintileloader_p.h b/src/3d/terrain/qgsdemterraintileloader_p.h
@@ -31,6 +31,7 @@
 #include <QFutureWatcher>
 #include <QElapsedTimer>
 
+#include "qgschunknode_p.h"
 #include "qgsrectangle.h"
 #include "qgsterraintileloader_p.h"
 #include "qgstilingscheme.h"
@@ -121,6 +122,7 @@ class QgsDemHeightMapGenerator : public QObject
     struct JobData
     {
       int jobId;
+      QgsChunkNodeId tileId;
       QgsRectangle extent;
       QFuture<QByteArray> future;
       QFutureWatcher<QByteArray> *fw;

diff --git a/src/3d/terrain/qgsflatterraingenerator.cpp b/src/3d/terrain/qgsflatterraingenerator.cpp
@@ -37,7 +37,7 @@ FlatTerrainChunkLoader::FlatTerrainChunkLoader( QgsTerrainEntity *terrain, QgsCh
 
 Qt3DCore::QEntity *FlatTerrainChunkLoader::createEntity( Qt3DCore::QEntity *parent )
 {
-  QgsTerrainTileEntity *entity = new QgsTerrainTileEntity;
+  QgsTerrainTileEntity *entity = new QgsTerrainTileEntity( mNode->tileId() );
 
   // make geometry renderer
 

diff --git a/src/3d/terrain/qgsterrainentity_p.cpp b/src/3d/terrain/qgsterrainentity_p.cpp
@@ -89,8 +89,7 @@ QgsTerrainEntity::QgsTerrainEntity( int maxLevel, const Qgs3DMapSettings &map, Q
 QgsTerrainEntity::~QgsTerrainEntity()
 {
   // cancel / wait for jobs
-  if ( mActiveJob )
-    cancelActiveJob();
+  cancelActiveJobs();
 
   delete mTextureGenerator;
   delete mTerrainToMapTransform;
@@ -193,7 +192,7 @@ TerrainMapUpdateJob::TerrainMapUpdateJob( QgsTerrainTextureGenerator *textureGen
 {
   QgsTerrainTileEntity *entity = qobject_cast<QgsTerrainTileEntity *>( node->entity() );
   connect( textureGenerator, &QgsTerrainTextureGenerator::tileReady, this, &TerrainMapUpdateJob::onTileReady );
-  mJobId = textureGenerator->render( entity->textureImage()->imageExtent(), entity->textureImage()->imageDebugText() );
+  mJobId = textureGenerator->render( entity->textureImage()->imageExtent(), node->tileId(), entity->textureImage()->imageDebugText() );
 }
 
 void TerrainMapUpdateJob::cancel()

diff --git a/src/3d/terrain/qgsterraintexturegenerator_p.cpp b/src/3d/terrain/qgsterraintexturegenerator_p.cpp
@@ -33,19 +33,20 @@ QgsTerrainTextureGenerator::QgsTerrainTextureGenerator( const Qgs3DMapSettings &
 {
 }
 
-int QgsTerrainTextureGenerator::render( const QgsRectangle &extent, const QString &debugText )
+int QgsTerrainTextureGenerator::render( const QgsRectangle &extent, QgsChunkNodeId tileId, const QString &debugText )
 {
   QgsMapSettings mapSettings( baseMapSettings() );
   mapSettings.setExtent( extent );
 
-  QgsEventTracing::addEvent( QgsEventTracing::Begin, QStringLiteral( "3D" ), QStringLiteral( "Texture" ) );
+  QgsEventTracing::addEvent( QgsEventTracing::AsyncBegin, QStringLiteral( "3D" ), QStringLiteral( "Texture" ), tileId.text() );
 
   QgsMapRendererSequentialJob *job = new QgsMapRendererSequentialJob( mapSettings );
   connect( job, &QgsMapRendererJob::finished, this, &QgsTerrainTextureGenerator::onRenderingFinished );
   job->start();
 
   JobData jobData;
   jobData.jobId = ++mLastJobId;
+  jobData.tileId = tileId;
   jobData.job = job;
   jobData.extent = extent;
   jobData.debugText = debugText;
@@ -125,7 +126,7 @@ void QgsTerrainTextureGenerator::onRenderingFinished()
 
   //qDebug() << "finished job " << jobData.jobId << "  ... in queue: " << jobs.count();
 
-  QgsEventTracing::addEvent( QgsEventTracing::End, QStringLiteral( "3D" ), QStringLiteral( "Texture" ) );
+  QgsEventTracing::addEvent( QgsEventTracing::AsyncEnd, QStringLiteral( "3D" ), QStringLiteral( "Texture" ), jobData.tileId.text() );
 
   // pass QImage further
   emit tileReady( jobData.jobId, img );