Add method to retrieve original metadata from point cloud files,

and expose some more useful metadata in layer properties (las version, software, creation date)
qgis · Dec 7, 2020 · 0708f81 · 0708f81
1 parent f21ca86
commit 0708f81
Show file tree

Hide file tree

Showing 11 changed files with 141 additions and 0 deletions.
diff --git a/python/core/auto_generated/pointcloud/qgspointclouddataprovider.sip.in b/python/core/auto_generated/pointcloud/qgspointclouddataprovider.sip.in
@@ -76,6 +76,14 @@ will be returned.
 
 This method will not attempt to calculate the data bounds, rather it will return only whatever precomputed bounds
 are included in the data source's metadata.
+%End
+
+    virtual QVariantMap originalMetadata() const;
+%Docstring
+Returns a representation of the original metadata included in a point cloud dataset.
+
+This is a free-form dictionary of values, the contents and structure of which will vary by provider and
+dataset.
 %End
 
     virtual QgsPointCloudRenderer *createRenderer( const QVariantMap &configuration = QVariantMap() ) const /Factory/;

diff --git a/src/core/pointcloud/qgspointclouddataprovider.cpp b/src/core/pointcloud/qgspointclouddataprovider.cpp
@@ -41,6 +41,11 @@ QgsGeometry QgsPointCloudDataProvider::polygonBounds() const
   return QgsGeometry::fromRect( extent() );
 }
 
+QVariantMap QgsPointCloudDataProvider::originalMetadata() const
+{
+  return QVariantMap();
+}
+
 QgsPointCloudRenderer *QgsPointCloudDataProvider::createRenderer( const QVariantMap & ) const
 {
   return nullptr;

diff --git a/src/core/pointcloud/qgspointclouddataprovider.h b/src/core/pointcloud/qgspointclouddataprovider.h
@@ -100,6 +100,14 @@ class CORE_EXPORT QgsPointCloudDataProvider: public QgsDataProvider
      */
     virtual QgsGeometry polygonBounds() const;
 
+    /**
+     * Returns a representation of the original metadata included in a point cloud dataset.
+     *
+     * This is a free-form dictionary of values, the contents and structure of which will vary by provider and
+     * dataset.
+     */
+    virtual QVariantMap originalMetadata() const;
+
     /**
      * Creates a new 2D point cloud renderer, using provider backend specific information.
      *

diff --git a/src/core/pointcloud/qgspointcloudlayer.cpp b/src/core/pointcloud/qgspointcloudlayer.cpp
@@ -422,6 +422,51 @@ QString QgsPointCloudLayer::htmlMetadata() const
                 + ( pointCount < 0 ? tr( "unknown" ) : locale.toString( static_cast<qlonglong>( pointCount ) ) )
                 + QStringLiteral( "</td></tr>\n" );
 
+  const QVariantMap originalMetadata = mDataProvider ? mDataProvider->originalMetadata() : QVariantMap();
+
+  if ( originalMetadata.value( QStringLiteral( "creation_year" ) ).toInt() > 0 && originalMetadata.contains( QStringLiteral( "creation_doy" ) ) )
+  {
+    QDate creationDate( originalMetadata.value( QStringLiteral( "creation_year" ) ).toInt(), 1, 1 );
+    creationDate = creationDate.addDays( originalMetadata.value( QStringLiteral( "creation_doy" ) ).toInt() );
+
+    myMetadata += QStringLiteral( "<tr><td class=\"highlight\">" )
+                  + tr( "Creation date" ) + QStringLiteral( "</td><td>" )
+                  + creationDate.toString( Qt::ISODate )
+                  + QStringLiteral( "</td></tr>\n" );
+  }
+  if ( originalMetadata.contains( QStringLiteral( "major_version" ) ) && originalMetadata.contains( QStringLiteral( "minor_version" ) ) )
+  {
+    myMetadata += QStringLiteral( "<tr><td class=\"highlight\">" )
+                  + tr( "Version" ) + QStringLiteral( "</td><td>" )
+                  + QStringLiteral( "%1.%2" ).arg( originalMetadata.value( QStringLiteral( "major_version" ) ).toString(),
+                      originalMetadata.value( QStringLiteral( "minor_version" ) ).toString() )
+                  + QStringLiteral( "</td></tr>\n" );
+  }
+
+  if ( !originalMetadata.value( QStringLiteral( "project_id" ) ).toString().isEmpty() )
+  {
+    myMetadata += QStringLiteral( "<tr><td class=\"highlight\">" )
+                  + tr( "Project ID" ) + QStringLiteral( "</td><td>" )
+                  + originalMetadata.value( QStringLiteral( "project_id" ) ).toString()
+                  + QStringLiteral( "</td></tr>\n" );
+  }
+
+  if ( !originalMetadata.value( QStringLiteral( "system_id" ) ).toString().isEmpty() )
+  {
+    myMetadata += QStringLiteral( "<tr><td class=\"highlight\">" )
+                  + tr( "System ID" ) + QStringLiteral( "</td><td>" )
+                  + originalMetadata.value( QStringLiteral( "system_id" ) ).toString()
+                  + QStringLiteral( "</td></tr>\n" );
+  }
+
+  if ( !originalMetadata.value( QStringLiteral( "software_id" ) ).toString().isEmpty() )
+  {
+    myMetadata += QStringLiteral( "<tr><td class=\"highlight\">" )
+                  + tr( "Software ID" ) + QStringLiteral( "</td><td>" )
+                  + originalMetadata.value( QStringLiteral( "software_id" ) ).toString()
+                  + QStringLiteral( "</td></tr>\n" );
+  }
+
   // End Provider section
   myMetadata += QLatin1String( "</table>\n<br><br>" );
 

diff --git a/src/core/providers/ept/qgseptpointcloudindex.cpp b/src/core/providers/ept/qgseptpointcloudindex.cpp
@@ -181,6 +181,40 @@ bool QgsEptPointCloudIndex::load( const QString &fileName )
   }
   setAttributes( attributes );
 
+  // try to import the metadata too!
+
+  QFile manifestFile( mDirectory + QStringLiteral( "/ept-sources/manifest.json" ) );
+  if ( manifestFile.open( QIODevice::ReadOnly ) )
+  {
+    const QByteArray manifestJson = manifestFile.readAll();
+    const QJsonDocument manifestDoc = QJsonDocument::fromJson( manifestJson, &err );
+    if ( err.error == QJsonParseError::NoError )
+    {
+      const QJsonArray manifestArray = manifestDoc.array();
+      // TODO how to handle multiple?
+      if ( ! manifestArray.empty() )
+      {
+        const QJsonObject sourceObject = manifestArray.at( 0 ).toObject();
+        const QString metadataPath = sourceObject.value( QStringLiteral( "metadataPath" ) ).toString();
+        QFile metadataFile( mDirectory + QStringLiteral( "/ept-sources/" ) + metadataPath );
+        if ( metadataFile.open( QIODevice::ReadOnly ) )
+        {
+          const QByteArray metadataJson = metadataFile.readAll();
+          const QJsonDocument metadataDoc = QJsonDocument::fromJson( metadataJson, &err );
+          if ( err.error == QJsonParseError::NoError )
+          {
+            const QJsonObject metadataObject = metadataDoc.object().value( QStringLiteral( "metadata" ) ).toObject();
+            if ( !metadataObject.empty() )
+            {
+              const QJsonObject sourceMetadata = metadataObject.constBegin().value().toObject();
+              mOriginalMetadata = sourceMetadata.toVariantMap();
+            }
+          }
+        }
+      }
+    }
+  }
+
   // save mRootBounds
 
   // bounds (cube - octree volume)

diff --git a/src/core/providers/ept/qgseptpointcloudindex.h b/src/core/providers/ept/qgseptpointcloudindex.h
@@ -53,6 +53,8 @@ class QgsEptPointCloudIndex: public QgsPointCloudIndex
     QVariantList metadataClasses( const QString &attribute ) const;
     QVariant metadataClassStatistic( const QString &attribute, const QVariant &value, QgsStatisticalSummary::Statistic statistic ) const;
 
+    QVariantMap originalMetadata() const { return mOriginalMetadata; }
+
   private:
     bool loadHierarchy();
 
@@ -75,6 +77,7 @@ class QgsEptPointCloudIndex: public QgsPointCloudIndex
     QMap< QString, AttributeStatistics > mMetadataStats;
 
     QMap< QString, QMap< int, int > > mAttributeClasses;
+    QVariantMap mOriginalMetadata;
 };
 
 ///@endcond

diff --git a/src/core/providers/ept/qgseptprovider.cpp b/src/core/providers/ept/qgseptprovider.cpp
@@ -93,6 +93,11 @@ QVariant QgsEptProvider::metadataClassStatistic( const QString &attribute, const
   return mIndex->metadataClassStatistic( attribute, value, statistic );
 }
 
+QVariantMap QgsEptProvider::originalMetadata() const
+{
+  return mIndex->originalMetadata();
+}
+
 QVariant QgsEptProvider::metadataStatistic( const QString &attribute, QgsStatisticalSummary::Statistic statistic ) const
 {
   return mIndex->metadataStatistic( attribute, statistic );

diff --git a/src/core/providers/ept/qgseptprovider.h b/src/core/providers/ept/qgseptprovider.h
@@ -56,6 +56,7 @@ class QgsEptProvider: public QgsPointCloudDataProvider
     QVariant metadataStatistic( const QString &attribute, QgsStatisticalSummary::Statistic statistic ) const override;
     QVariantList metadataClasses( const QString &attribute ) const override;
     QVariant metadataClassStatistic( const QString &attribute, const QVariant &value, QgsStatisticalSummary::Statistic statistic ) const override;
+    QVariantMap originalMetadata() const override;
 
   private:
     std::unique_ptr<QgsEptPointCloudIndex> mIndex;

diff --git a/src/providers/pdal/qgspdalprovider.cpp b/src/providers/pdal/qgspdalprovider.cpp
@@ -22,6 +22,7 @@
 #include "qgsapplication.h"
 #include "qgslogger.h"
 #include "qgsmessagelog.h"
+#include "qgsjsonutils.h"
 
 #include <pdal/io/LasReader.hpp>
 #include <pdal/io/LasHeader.hpp>
@@ -66,6 +67,11 @@ int QgsPdalProvider::pointCount() const
   return mPointCount;
 }
 
+QVariantMap QgsPdalProvider::originalMetadata() const
+{
+  return mOriginalMetadata;
+}
+
 bool QgsPdalProvider::isValid() const
 {
   return mIsValid;
@@ -100,6 +106,12 @@ bool QgsPdalProvider::load( const QString &uri )
     las_reader.prepare( table );
     pdal::LasHeader las_header = las_reader.header();
 
+    const std::string tableMetadata = pdal::Utils::toJSON( table.metadata() );
+    const QVariantMap readerMetadata = QgsJsonUtils::parseJson( tableMetadata ).toMap().value( QStringLiteral( "root" ) ).toMap();
+    // source metadata is only value present here!
+    if ( !readerMetadata.empty() )
+      mOriginalMetadata = readerMetadata.constBegin().value().toMap();
+
     // extent
     /*
     double scale_x = las_header.scaleX();

diff --git a/src/providers/pdal/qgspdalprovider.h b/src/providers/pdal/qgspdalprovider.h
@@ -38,6 +38,7 @@ class QgsPdalProvider: public QgsPointCloudDataProvider
     QgsRectangle extent() const override;
     QgsPointCloudAttributeCollection attributes() const override;
     int pointCount() const override;
+    QVariantMap originalMetadata() const override;
 
     bool isValid() const override;
 
@@ -53,6 +54,7 @@ class QgsPdalProvider: public QgsPointCloudDataProvider
     QgsRectangle mExtent;
     bool mIsValid = false;
     int mPointCount = 0;
+    QVariantMap mOriginalMetadata;
 };
 
 class QgsPdalProviderMetadata : public QgsProviderMetadata

diff --git a/tests/src/python/test_qgspointcloudprovider.py b/tests/src/python/test_qgspointcloudprovider.py
@@ -94,6 +94,24 @@ def testMetadataClassStatistics(self):
         self.assertEqual(layer.dataProvider().metadataClassStatistic('Classification', 5, QgsStatisticalSummary.Count),
                          3)
 
+    @unittest.skipIf('ept' not in QgsProviderRegistry.instance().providerList(), 'EPT provider not available')
+    def testOriginalMetadataEpt(self):
+        layer = QgsPointCloudLayer(unitTestDataPath() + '/point_clouds/ept/sunshine-coast/ept.json', 'test', 'ept')
+        self.assertEqual(layer.dataProvider().originalMetadata()['major_version'], 1.0)
+        self.assertEqual(layer.dataProvider().originalMetadata()['minor_version'], 2.0)
+        self.assertEqual(layer.dataProvider().originalMetadata()['software_id'], 'PDAL 2.1.0 (Releas)')
+        self.assertEqual(layer.dataProvider().originalMetadata()['creation_year'], 2020.0)
+        self.assertEqual(layer.dataProvider().originalMetadata()['creation_doy'], 309.0)
+
+    @unittest.skipIf('pdal' not in QgsProviderRegistry.instance().providerList(), 'PDAL provider not available')
+    def testOriginalMetadataPdal(self):
+        layer = QgsPointCloudLayer(unitTestDataPath() + '/point_clouds/las/cloud.las', 'test', 'pdal')
+        self.assertEqual(layer.dataProvider().originalMetadata()['major_version'], 1.0)
+        self.assertEqual(layer.dataProvider().originalMetadata()['minor_version'], 2.0)
+        self.assertEqual(layer.dataProvider().originalMetadata()['software_id'], 'PDAL 2.1.0 (Releas)')
+        self.assertEqual(layer.dataProvider().originalMetadata()['creation_year'], 2020.0)
+        self.assertEqual(layer.dataProvider().originalMetadata()['creation_doy'], 309.0)
+
 
 if __name__ == '__main__':
     unittest.main()