Memory dataset group and possibility to persist it (#37389)

[FEATURE] Introduces memory dataset groups for mesh layer. These dataset groups are temporary and are not kept when the project is closed. Memory dataset groups can be created from the mesh calculator with a new option. Allows the possibility to remove or save these memory dataset groups to a file with specified driver.
qgis · Jul 1, 2020 · 6be16a5 · 6be16a5
1 parent 6068c64
commit 6be16a5
Show file tree

Hide file tree

Showing 84 changed files with 5,154 additions and 1,292 deletions.
diff --git a/external/mdal/api/mdal.h b/external/mdal/api/mdal.h
@@ -163,6 +163,14 @@ MDAL_EXPORT bool MDAL_DR_meshLoadCapability( MDAL_DriverH driver );
  */
 MDAL_EXPORT bool MDAL_DR_writeDatasetsCapability( MDAL_DriverH driver, MDAL_DataLocation location );
 
+/**
+ * Returns the file suffix used to write datasets on file
+ * not thread-safe and valid only till next call
+ *
+ * \since MDAL 0.7.0
+ */
+MDAL_EXPORT const char *MDAL_DR_writeDatasetsSuffix( MDAL_DriverH driver );
+
 /**
  * Returns whether driver has capability to save mesh
  */
@@ -208,8 +216,10 @@ MDAL_EXPORT MDAL_MeshH MDAL_LoadMesh( const char *uri );
  * not thread-safe and valid only till next call
  *
  * Parameter uri can be in format:
- *  - <drivername>:"meshfile" - function then returns uris with provided driver and meshfile
- *  - "meshfile" or meshfile  - function then finds proper driver and returns uris with it
+ *
+ * - <drivername>:"meshfile" - function then returns uris with provided driver and meshfile
+ * - "meshfile" or meshfile  - function then finds proper driver and returns uris with it
+ *
  * The uris can be used directly in MDAL_LoadMesh to load particular meshes
  *
  * \since MDAL 0.6.0
@@ -530,6 +540,14 @@ MDAL_EXPORT const char *MDAL_G_referenceTime( MDAL_DatasetGroupH group );
  */
 MDAL_EXPORT bool MDAL_G_isTemporal( MDAL_DatasetGroupH group );
 
+/**
+ * Returns dataset group uri
+ * not thread-safe and valid only till next call
+ *
+ * \since MDAL 0.7.0
+ */
+MDAL_EXPORT const char *MDAL_G_uri( MDAL_DatasetGroupH group );
+
 ///////////////////////////////////////////////////////////////////////////////////////
 /// DATASETS
 ///////////////////////////////////////////////////////////////////////////////////////

diff --git a/external/mdal/frmts/mdal_2dm.cpp b/external/mdal/frmts/mdal_2dm.cpp
@@ -22,19 +22,11 @@
 
 #define DRIVER_NAME "2DM"
 
-MDAL::Mesh2dm::Mesh2dm( size_t verticesCount,
-                        size_t edgesCount,
-                        size_t facesCount,
-                        size_t faceVerticesMaximumCount,
-                        MDAL::BBox extent,
+MDAL::Mesh2dm::Mesh2dm( size_t faceVerticesMaximumCount,
                         const std::string &uri,
                         const std::map<size_t, size_t> vertexIDtoIndex )
   : MemoryMesh( DRIVER_NAME,
-                verticesCount,
-                edgesCount,
-                facesCount,
                 faceVerticesMaximumCount,
-                extent,
                 uri )
   , mVertexIDtoIndex( vertexIDtoIndex )
 {
@@ -125,6 +117,8 @@ std::unique_ptr<MDAL::Mesh> MDAL::Driver2dm::load( const std::string &meshFile,
   size_t faceCount = 0;
   size_t vertexCount = 0;
   size_t edgesCount = 0;
+  size_t materialCount = 0;
+  bool hasMaterialsDefinitionsForElements = false;
 
   // Find out how many nodes and elements are contained in the .2dm mesh file
   while ( std::getline( in, line ) )
@@ -150,15 +144,21 @@ std::unique_ptr<MDAL::Mesh> MDAL::Driver2dm::load( const std::string &meshFile,
       MDAL::Log::warning( MDAL_Status::Err_UnsupportedElement, name(),  "found unsupported element" );
       return nullptr;
     }
+    // If specified, update the number of materials of the mesh
+    else if ( startsWith( line, "NUM_MATERIALS_PER_ELEM" ) )
+    {
+      hasMaterialsDefinitionsForElements = true;
+      materialCount = MDAL::toSizeT( split( line, ' ' )[1] );
+    }
   }
 
   // Allocate memory
   Vertices vertices( vertexCount );
   Edges edges( edgesCount );
   Faces faces( faceCount );
 
-  // Basement 3.x supports definition of elevation for cell centers
-  std::vector<double> elementCenteredElevation;
+  // .2dm mesh files may have any number of material ID columns
+  std::vector<std::vector<double>> faceMaterials;
 
   in.clear();
   in.seekg( 0, std::ios::beg );
@@ -187,26 +187,46 @@ std::unique_ptr<MDAL::Mesh> MDAL::Driver2dm::load( const std::string &meshFile,
       face.resize( faceVertexCount );
 
       // chunks format here
-      // E** id vertex_id1, vertex_id2, ... material_id (elevation - optional)
+      // E** id vertex_id1, vertex_id2, vertex_id3, material_id [, aux_column_1, aux_column_2, ...]
       // vertex ids are numbered from 1
       // Right now we just store node IDs here - we will convert them to node indices afterwards
       assert( chunks.size() > faceVertexCount + 1 );
 
       for ( size_t i = 0; i < faceVertexCount; ++i )
         face[i] = MDAL::toSizeT( chunks[i + 2] ) - 1; // 2dm is numbered from 1
 
-      // OK, now find out if there is optional cell elevation (BASEMENT 3.x)
-      if ( chunks.size() == faceVertexCount + 4 )
+      // NUM_MATERIALS_PER_ELEM tag provided, use new MATID parser
+      if ( hasMaterialsDefinitionsForElements )
       {
+        // This assertion will fail if a mesh has fewer material ID columns than
+        // promised by the NUM_MATERIAL_PER_ELEM tag.
+        assert( chunks.size() - 5 >= materialCount );
+
+        if ( faceMaterials.empty() ) // Initialize dataset if still empty
+        {
+          faceMaterials = std::vector<std::vector<double>>( materialCount, std::vector<double>(
+                            faceCount, std::numeric_limits<double>::quiet_NaN() ) );
+        }
+
+        // Add material ID values
+        for ( size_t i = 0; i < materialCount; ++i )
+        {
+          // Offset of 2 for E** tag and element ID
+          faceMaterials[i][faceIndex] = MDAL::toDouble( chunks[ faceVertexCount + 2 + i] );
+        }
+      }
 
-        // initialize dataset if it is still empty
-        if ( elementCenteredElevation.empty() )
+      // No NUM_MATERIALS_PER_ELEM tag provided, use legacy MATID parser
+      else if ( chunks.size() == faceVertexCount + 4 )
+      {
+        if ( faceMaterials.empty() ) // Initialize dataset if still empty
         {
-          elementCenteredElevation = std::vector<double>( faceCount, std::numeric_limits<double>::quiet_NaN() );
+          // Add a single vector dataset for the "Bed Elevation (Face)" dataset
+          faceMaterials = std::vector<std::vector<double>>( 1, std::vector<double>(
+                            faceCount, std::numeric_limits<double>::quiet_NaN() ) );
         }
 
-        // add Bed Elevation (Face) value
-        elementCenteredElevation[faceIndex] = MDAL::toDouble( chunks[ faceVertexCount + 3 ] );
+        faceMaterials[0][faceIndex] = MDAL::toDouble( chunks[ faceVertexCount + 3 ] );
       }
 
       faceIndex++;
@@ -276,22 +296,39 @@ std::unique_ptr<MDAL::Mesh> MDAL::Driver2dm::load( const std::string &meshFile,
 
   std::unique_ptr< Mesh2dm > mesh(
     new Mesh2dm(
-      vertices.size(),
-      edges.size(),
-      faces.size(),
       MAX_VERTICES_PER_FACE_2DM,
-      computeExtent( vertices ),
       mMeshFile,
       vertexIDtoIndex
     )
   );
-  mesh->faces = faces;
-  mesh->vertices = vertices;
-  mesh->edges = edges;
+  mesh->setFaces( std::move( faces ) );
+  mesh->setVertices( std::move( vertices ) );
+  mesh->setEdges( std::move( edges ) );
+
+  // Add Bed Elevation
+  MDAL::addBedElevationDatasetGroup( mesh.get(), mesh->vertices() );
+
+  // Add material IDs
+  if ( hasMaterialsDefinitionsForElements )
+  {
+    // New MATID parser: Add all MATID dataset groups
+    std::string dataSetName;
+    for ( size_t i = 0; i < materialCount; ++i )
+    {
+      // The first two columns get special names for convenience
+      if ( i == 0 ) dataSetName = "Material ID";
+      else if ( i == 1 ) dataSetName = "Bed Elevation (Face)";
+      else dataSetName = "Auxiliary Material ID " + std::to_string( i - 1 );
 
-  // Add Bed Elevations
-  MDAL::addFaceScalarDatasetGroup( mesh.get(), elementCenteredElevation, "Bed Elevation (Face)" );
-  MDAL::addBedElevationDatasetGroup( mesh.get(), vertices );
+      MDAL::addFaceScalarDatasetGroup( mesh.get(), faceMaterials[i], dataSetName );
+    }
+  }
+  // Add "Bed Elevation (Face)"
+  else if ( !faceMaterials.empty() )
+  {
+    // Legacy MATID parser: "Bed Elevation (Face)" dataset group only
+    MDAL::addFaceScalarDatasetGroup( mesh.get(), faceMaterials[0], "Bed Elevation (Face)" );
+  }
 
   return std::unique_ptr<Mesh>( mesh.release() );
 }

diff --git a/external/mdal/frmts/mdal_2dm.hpp b/external/mdal/frmts/mdal_2dm.hpp
@@ -22,11 +22,7 @@ namespace MDAL
   class Mesh2dm: public MemoryMesh
   {
     public:
-      Mesh2dm( size_t verticesCount,
-               size_t edgesCount,
-               size_t facesCount,
-               size_t faceVerticesMaximumCount,
-               BBox extent,
+      Mesh2dm( size_t faceVerticesMaximumCount,
                const std::string &uri,
                const std::map<size_t, size_t> vertexIDtoIndex
              );
@@ -63,12 +59,15 @@ namespace MDAL
    *
    * full specification here: https://www.xmswiki.com/wiki/SMS:2D_Mesh_Files_*.2dm
    *
-   * Exception for the official specification is for recognition of cell-centered
-   * elevation values supported by BASEMENT 3.x releases
-   * If face definition has extra column, it is parsed and recognized as
-   * elevation, e.g. format for triangle
-   * E3T id 1 2 3 mat_id elevation
-   * and added automatically as "Bed Elevation (Face)"
+   * This will process as many material IDs as promised by the NUM_MATERIALS_PER_ELEM tag and add them as face
+   * dataset groups. The naming for these groups is "Material ID" for the first, "Bed Elevation (Face)" for the
+   * second, and finally "Auxiliary Material ID <X>" for any subsequent materials, X being a counter to ensure
+   * unique group names:
+   * E** id 1 2 3 [Material ID] [Bed Elevation (Face)] [Auxiliary Material ID 1] [Auxiliary Material ID 2] ...
+   * If the NUM_MATERIALS_PER_ELEM tag is not provided, a fallback mode is used that will only check for the
+   * second MATID column and add it under the name "Bed Elevation (Face)" if found.
+   * Noe that this is purely a compatibility mode for BASEMENT 3.x releases; NUM_MATERIALS_... is a required
+   * tag according to the 2DM specification.
    *
    * Note that some 2dm formats do have some extra columns after mat_id column with
    * data with unknown origin/name (e.g. tests/data/2dm/regular_grid.2dm)

diff --git a/external/mdal/frmts/mdal_3di.cpp b/external/mdal/frmts/mdal_3di.cpp
@@ -268,11 +268,13 @@ void MDAL::Driver3Di::parseNetCDFVariableMetadata( int varid,
     std::string &name,
     bool *is_vector,
     bool *isPolar,
+    bool *invertedDirection,
     bool *is_x )
 {
   *is_vector = false;
   *is_x = true;
   *isPolar = false;
+  MDAL_UNUSED( invertedDirection )
 
   std::string long_name = mNcFile->getAttrStr( "long_name", varid );
   if ( long_name.empty() )

diff --git a/external/mdal/frmts/mdal_3di.hpp b/external/mdal/frmts/mdal_3di.hpp
@@ -58,6 +58,7 @@ namespace MDAL
                                         std::string &name,
                                         bool *is_vector,
                                         bool *isPolar,
+                                        bool *invertedDirection,
                                         bool *is_x ) override;
       std::vector<std::pair<double, double>> parseClassification( int varid ) const override;
 

diff --git a/external/mdal/frmts/mdal_ascii_dat.cpp b/external/mdal/frmts/mdal_ascii_dat.cpp
@@ -485,7 +485,9 @@ bool MDAL::DriverAsciiDat::persist( MDAL::DatasetGroup *group )
   if ( !MDAL::contains( uri, "_els" ) && group->dataLocation() != MDAL_DataLocation::DataOnVertices )
   {
     // Should contain _els in name for edges/faces dataset but it does not
-    uri.insert( uri.size() - 4, "_els" );
+    int pos = uri.size() - 4;
+    uri.insert( std::max( 0, pos ), "_els" );
+    group->replaceUri( uri );
   }
 
   if ( ( mesh->facesCount() > 0 ) && ( mesh->edgesCount() > 0 ) )
@@ -561,3 +563,8 @@ bool MDAL::DriverAsciiDat::persist( MDAL::DatasetGroup *group )
 
   return false;
 }
+
+std::string MDAL::DriverAsciiDat::writeDatasetOnFileSuffix() const
+{
+  return "dat";
+}
diff --git a/external/mdal/frmts/mdal_ascii_dat.hpp b/external/mdal/frmts/mdal_ascii_dat.hpp
@@ -59,6 +59,8 @@ namespace MDAL
       void load( const std::string &datFile, Mesh *mesh ) override;
       bool persist( DatasetGroup *group ) override;
 
+      std::string writeDatasetOnFileSuffix() const override;
+
     private:
       bool canReadOldFormat( const std::string &line ) const;
       bool canReadNewFormat( const std::string &line ) const;

diff --git a/external/mdal/frmts/mdal_binary_dat.cpp b/external/mdal/frmts/mdal_binary_dat.cpp
@@ -491,3 +491,8 @@ bool MDAL::DriverBinaryDat::persist( MDAL::DatasetGroup *group )
 
   return false;
 }
+
+std::string MDAL::DriverBinaryDat::writeDatasetOnFileSuffix() const
+{
+  return "dat";
+}
diff --git a/external/mdal/frmts/mdal_binary_dat.hpp b/external/mdal/frmts/mdal_binary_dat.hpp
@@ -31,6 +31,8 @@ namespace MDAL
       void load( const std::string &datFile, Mesh *mesh ) override;
       bool persist( DatasetGroup *group ) override;
 
+      std::string writeDatasetOnFileSuffix() const override;
+
     private:
       bool readVertexTimestep( const Mesh *mesh,
                                std::shared_ptr<DatasetGroup> group,