Skip to content

Commit

Permalink
push MDAL 0.4.93 RC1 for 3.12 release with fixed bug in reading of re…
Browse files Browse the repository at this point in the history 
…ference time
  • Loading branch information
@PeterPetrik @nyalldawson
PeterPetrik authored and nyalldawson committed Dec 13, 2019
1 parent c2cde01 commit 6e2ef15
Show file tree
Hide file tree
Showing 35 changed files with 941 additions and 265 deletions.
9 changes: 3 additions & 6 deletions external/mdal/api/mdal.h
View file
Expand Up @@ -345,7 +345,7 @@ MDAL_EXPORT void MDAL_G_minimumMaximum( DatasetGroupH group, double *min, double
* Only for 2D datasets
*
* \param group parent group handle
* \param time time for dataset
* \param time time for dataset (hours)
* \param values For scalar data on vertices, the size must be vertex count
* For scalar data on faces, the size must be faces count
* For vector data on vertices, the size must be vertex count * 2 (x1, y1, x2, y2, ..., xN, yN)
Expand All @@ -371,10 +371,7 @@ MDAL_EXPORT bool MDAL_G_isInEditMode( DatasetGroupH group );
*/
MDAL_EXPORT void MDAL_G_closeEditMode( DatasetGroupH group );

/**
* Returns reference time for dataset group
* If returned value begins with word JULIAN, following number represents date in Julian format
*/
//! Returns reference time for dataset group expressed in date with ISO8601 format, return "" if reference time is not defined
MDAL_EXPORT const char *MDAL_G_referenceTime( DatasetGroupH group );

///////////////////////////////////////////////////////////////////////////////////////
Expand All @@ -384,7 +381,7 @@ MDAL_EXPORT const char *MDAL_G_referenceTime( DatasetGroupH group );
//! Returns dataset parent group
MDAL_EXPORT DatasetGroupH MDAL_D_group( DatasetH dataset );

//! Returns dataset time
//! Returns dataset time (hours)
MDAL_EXPORT double MDAL_D_time( DatasetH dataset );

//! Returns volumes count for the mesh (for 3D meshes)
Expand Down
2 changes: 1 addition & 1 deletion external/mdal/frmts/mdal_3di.cpp
View file
Expand Up @@ -139,7 +139,7 @@ void MDAL::Driver3Di::addBedElevation( MemoryMesh *mesh )
group->setIsScalar( true );

std::shared_ptr<MDAL::MemoryDataset2D> dataset = std::make_shared< MemoryDataset2D >( group.get() );
dataset->setTime( 0.0 );
dataset->setTime( MDAL::RelativeTimestamp() );
for ( size_t i = 0; i < faceCount; ++i )
{
dataset->setScalarValue( i, MDAL::safeValue( coordZ[i], fillZ ) );
Expand Down
45 changes: 11 additions & 34 deletions external/mdal/frmts/mdal_ascii_dat.cpp
View file
Expand Up @@ -116,7 +116,8 @@ void MDAL::DriverAsciiDat::loadOldFormat( std::ifstream &in,
}
else if ( cardType == "TS" && items.size() >= 2 )
{
double t = toDouble( items[ 1 ] );
double rawTime = toDouble( items[ 1 ] );
MDAL::RelativeTimestamp t( rawTime, MDAL::RelativeTimestamp::hours );
readVertexTimestep( mesh, group, t, isVector, false, in );
}
else
Expand Down Expand Up @@ -147,7 +148,7 @@ void MDAL::DriverAsciiDat::loadNewFormat(
std::shared_ptr<DatasetGroup> group; // DAT outputs data
std::string groupName( MDAL::baseName( mDatFile ) );
std::string line;
std::string referenceTime;
MDAL::DateTime referenceTime;
// see if it contains face-centered results - supported by BASEMENT
bool faceCentered = false;
if ( contains( groupName, "_els" ) )
Expand Down Expand Up @@ -231,7 +232,7 @@ void MDAL::DriverAsciiDat::loadNewFormat(
}
else if ( cardType == "RT_JULIAN" && items.size() >= 2 )
{
referenceTime = "JULIAN " + items[1];
referenceTime = DateTime( MDAL::toDouble( items[1] ), DateTime::JulianDay );
}
else if ( cardType == "TIMEUNITS" && items.size() >= 2 )
{
Expand All @@ -245,8 +246,8 @@ void MDAL::DriverAsciiDat::loadNewFormat(
}
else if ( cardType == "TS" && items.size() >= 3 )
{
double t = toDouble( items[2] );
t = convertTimeDataToHours( t, group->getMetadata( "TIMEUNITS" ) );
double rawTime = toDouble( items[2] );
MDAL::RelativeTimestamp t( rawTime, MDAL::parseDurationTimeUnit( group->getMetadata( "TIMEUNITS" ) ) );

if ( faceCentered )
{
Expand Down Expand Up @@ -329,7 +330,7 @@ void MDAL::DriverAsciiDat::load( const std::string &datFile, MDAL::Mesh *mesh, M
void MDAL::DriverAsciiDat::readVertexTimestep(
const MDAL::Mesh *mesh,
std::shared_ptr<DatasetGroup> group,
double t,
MDAL::RelativeTimestamp t,
bool isVector,
bool hasStatus,
std::ifstream &stream ) const
Expand Down Expand Up @@ -398,7 +399,7 @@ void MDAL::DriverAsciiDat::readVertexTimestep(
void MDAL::DriverAsciiDat::readFaceTimestep(
const MDAL::Mesh *mesh,
std::shared_ptr<DatasetGroup> group,
double t,
MDAL::RelativeTimestamp t,
bool isVector,
std::ifstream &stream ) const
{
Expand Down Expand Up @@ -477,17 +478,11 @@ bool MDAL::DriverAsciiDat::persist( MDAL::DatasetGroup *group )
out << "ND " << nodeCount << "\n";
out << "NC " << elemCount << "\n";
out << "NAME " "\"" << group->name() << "\"" "\n";
std::string referenceTimeStr = group->referenceTime();
std::string referenceTimeStr = group->referenceTime().toJulianDayString();

if ( !referenceTimeStr.empty() )
{
// Cutting of the JULIAN prefix
std::vector<std::string> referenceTimeStrWords = split( referenceTimeStr, ' ' );

if ( referenceTimeStrWords.size() > 1 )
out << "RT_JULIAN " << referenceTimeStrWords[1] << "\n";
else
out << "RT_JULIAN " << referenceTimeStr << "\n";
out << "RT_JULIAN " << referenceTimeStr << "\n";
}

out << "TIMEUNITS " << 0 << "\n";
Expand All @@ -498,7 +493,7 @@ bool MDAL::DriverAsciiDat::persist( MDAL::DatasetGroup *group )
= std::dynamic_pointer_cast<MDAL::MemoryDataset2D>( group->datasets[time_index] );

bool hasActiveStatus = isOnVertices && dataset->supportsActiveFlag();
out << "TS " << hasActiveStatus << " " << std::to_string( dataset->time() ) << "\n";
out << "TS " << hasActiveStatus << " " << std::to_string( dataset->time( RelativeTimestamp::hours ) ) << "\n";

if ( hasActiveStatus )
{
Expand Down Expand Up @@ -528,21 +523,3 @@ bool MDAL::DriverAsciiDat::persist( MDAL::DatasetGroup *group )

return false;
}

double MDAL::DriverAsciiDat::convertTimeDataToHours( double time, const std::string &originalTimeDataUnit ) const
{
if ( originalTimeDataUnit == "se" || originalTimeDataUnit == "2" || originalTimeDataUnit == "Seconds"
|| originalTimeDataUnit.empty() )
{
time /= 3600.0;
}
else if ( originalTimeDataUnit == "mi" || originalTimeDataUnit == "1" || originalTimeDataUnit == "Minutes" )
{
time /= 60.0;
}
else if ( originalTimeDataUnit == "days" )
{
time *= 24;
}
return time;
}
26 changes: 11 additions & 15 deletions external/mdal/frmts/mdal_ascii_dat.hpp
View file
Expand Up @@ -70,22 +70,18 @@ namespace MDAL
//! maximum native index of the vertex in defined in the mesh
size_t maximumId( const Mesh *mesh ) const;

void readVertexTimestep(
const Mesh *mesh,
std::shared_ptr<DatasetGroup> group,
double t,
bool isVector,
bool hasStatus,
std::ifstream &stream ) const;
void readVertexTimestep( const Mesh *mesh,
std::shared_ptr<DatasetGroup> group,
RelativeTimestamp t,
bool isVector,
bool hasStatus,
std::ifstream &stream ) const;

void readFaceTimestep(
const Mesh *mesh,
std::shared_ptr<DatasetGroup> group,
double t,
bool isVector,
std::ifstream &stream ) const;

double convertTimeDataToHours( double time, const std::string &originalTimeDataUnit ) const;
void readFaceTimestep( const Mesh *mesh,
std::shared_ptr<DatasetGroup> group,
RelativeTimestamp t,
bool isVector,
std::ifstream &stream ) const;

std::string mDatFile;
};
Expand Down
34 changes: 7 additions & 27 deletions external/mdal/frmts/mdal_binary_dat.cpp
View file
Expand Up @@ -251,7 +251,7 @@ void MDAL::DriverBinaryDat::load( const std::string &datFile, MDAL::Mesh *mesh,
return exit_with_error( status, MDAL_Status::Err_UnknownFormat, "unable to read reference time" );

referenceTime = static_cast<double>( time );
group->setReferenceTime( "JULIAN " + std::to_string( referenceTime ) );
group->setReferenceTime( DateTime( referenceTime, DateTime::JulianDay ) );
break;

case CT_TIMEUNITS:
Expand Down Expand Up @@ -288,8 +288,8 @@ void MDAL::DriverBinaryDat::load( const std::string &datFile, MDAL::Mesh *mesh,
if ( read( in, reinterpret_cast< char * >( &time ), 4 ) )
return exit_with_error( status, MDAL_Status::Err_UnknownFormat, "Invalid time step" );

double t = static_cast<double>( time );
t = convertTimeDataToHours( t, timeUnit );
double rawTime = static_cast<double>( time );
MDAL::RelativeTimestamp t( rawTime, MDAL::parseDurationTimeUnit( timeUnitStr ) );

if ( readVertexTimestep( mesh, group, groupMax, t, istat, sflg, in ) )
return exit_with_error( status, MDAL_Status::Err_UnknownFormat, "Unable to read vertex timestep" );
Expand All @@ -315,7 +315,7 @@ bool MDAL::DriverBinaryDat::readVertexTimestep(
const MDAL::Mesh *mesh,
std::shared_ptr<DatasetGroup> group,
std::shared_ptr<DatasetGroup> groupMax,
double time,
MDAL::RelativeTimestamp time,
bool hasStatus,
int sflg,
std::ifstream &in )
Expand Down Expand Up @@ -364,15 +364,15 @@ bool MDAL::DriverBinaryDat::readVertexTimestep(
}
}

if ( MDAL::equals( time, 99999.0 ) ) // Special TUFLOW dataset with maximus
if ( MDAL::equals( time.value( MDAL::RelativeTimestamp::hours ), 99999.0 ) ) // Special TUFLOW dataset with maximus
{
dataset->setTime( time );
dataset->setStatistics( MDAL::calculateStatistics( dataset ) );
groupMax->datasets.push_back( dataset );
}
else
{
dataset->setTime( time ); // TODO read TIMEUNITS
dataset->setTime( time );
dataset->setStatistics( MDAL::calculateStatistics( dataset ) );
group->datasets.push_back( dataset );
}
Expand Down Expand Up @@ -458,7 +458,7 @@ bool MDAL::DriverBinaryDat::persist( MDAL::DatasetGroup *group )

writeRawData( out, reinterpret_cast< const char * >( &CT_TS ), 4 );
writeRawData( out, reinterpret_cast< const char * >( &istat ), 1 );
float ftime = static_cast<float>( dataset->time() );
float ftime = static_cast<float>( dataset->time( RelativeTimestamp::hours ) );
writeRawData( out, reinterpret_cast< const char * >( &ftime ), 4 );

if ( istat )
Expand Down Expand Up @@ -493,23 +493,3 @@ bool MDAL::DriverBinaryDat::persist( MDAL::DatasetGroup *group )

return false;
}

double MDAL::DriverBinaryDat::convertTimeDataToHours( double time, int originalTimeDataUnit )
{
switch ( originalTimeDataUnit )
{
case 1:
time /= 60.0;
break;
case 2:
time /= 3600.0;
break;
case 4:
time *= 24;
break;
case 0:
default:
break;
}
return time;
}
3 changes: 1 addition & 2 deletions external/mdal/frmts/mdal_binary_dat.hpp
View file
Expand Up @@ -35,12 +35,11 @@ namespace MDAL
bool readVertexTimestep( const Mesh *mesh,
std::shared_ptr<DatasetGroup> group,
std::shared_ptr<DatasetGroup> groupMax,
double time,
RelativeTimestamp time,
bool hasStatus,
int sflg,
std::ifstream &in );

double convertTimeDataToHours( double time, int originalTimeDataUnit );
std::string mDatFile;
};

Expand Down
33 changes: 20 additions & 13 deletions external/mdal/frmts/mdal_cf.cpp
View file
Expand Up @@ -164,7 +164,7 @@ static void populate_vals( bool is_vector, double *vals, size_t i,
}
}

void MDAL::DriverCF::addDatasetGroups( MDAL::Mesh *mesh, const std::vector<double> &times, const MDAL::cfdataset_info_map &dsinfo_map )
void MDAL::DriverCF::addDatasetGroups( MDAL::Mesh *mesh, const std::vector<RelativeTimestamp> &times, const MDAL::cfdataset_info_map &dsinfo_map, const MDAL::DateTime &referenceTime )
{
/* PHASE 2 - add dataset groups */
for ( const auto &it : dsinfo_map )
Expand Down Expand Up @@ -206,7 +206,6 @@ void MDAL::DriverCF::addDatasetGroups( MDAL::Mesh *mesh, const std::vector<doubl
for ( size_t ts = 0; ts < dsi.nTimesteps; ++ts )
{
std::shared_ptr<MDAL::Dataset> dataset;
double time = times[ts];
if ( dsi.outputType == CFDimensions::Volume3D )
{
dataset = create3DDataset(
Expand All @@ -223,7 +222,7 @@ void MDAL::DriverCF::addDatasetGroups( MDAL::Mesh *mesh, const std::vector<doubl

if ( dataset )
{
dataset->setTime( time );
dataset->setTime( times[ts] );
group->datasets.push_back( dataset );
}
}
Expand All @@ -232,30 +231,38 @@ void MDAL::DriverCF::addDatasetGroups( MDAL::Mesh *mesh, const std::vector<doubl
if ( !group->datasets.empty() )
{
group->setStatistics( MDAL::calculateStatistics( group ) );
group->setReferenceTime( referenceTime );
mesh->datasetGroups.push_back( group );
}
}
}

void MDAL::DriverCF::parseTime( std::vector<double> &times )
MDAL::DateTime MDAL::DriverCF::parseTime( std::vector<RelativeTimestamp> &times )
{

size_t nTimesteps = mDimensions.size( CFDimensions::Time );
if ( 0 == nTimesteps )
{
//if no time dimension is present creates only one time step to store the potential time-independent variable
nTimesteps = 1;
times = std::vector<double>( 1, 0 );
return;
times = std::vector<RelativeTimestamp>( 1, RelativeTimestamp() );
return MDAL::DateTime();
}
const std::string timeArrName = getTimeVariableName();
times = mNcFile->readDoubleArr( timeArrName, nTimesteps );
std::string units = mNcFile->getAttrStr( timeArrName, "units" );
double div_by = MDAL::parseTimeUnits( units );
std::vector<double> rawTimes = mNcFile->readDoubleArr( timeArrName, nTimesteps );

std::string timeUnitInformation = mNcFile->getAttrStr( timeArrName, "units" );
std::string calendar = mNcFile->getAttrStr( timeArrName, "calendar" );
MDAL::DateTime referenceTime = parseCFReferenceTime( timeUnitInformation, calendar );
MDAL::RelativeTimestamp::Unit unit = parseCFTimeUnit( timeUnitInformation );

times = std::vector<RelativeTimestamp>( nTimesteps );
for ( size_t i = 0; i < nTimesteps; ++i )
{
times[i] /= div_by;
times[i] = RelativeTimestamp( rawTimes[i], unit );
}

return referenceTime;
}

std::shared_ptr<MDAL::Dataset> MDAL::DriverCF::create2DDataset( std::shared_ptr<MDAL::DatasetGroup> group, size_t ts, const MDAL::CFDatasetGroupInfo &dsi, double fill_val_x, double fill_val_y )
Expand Down Expand Up @@ -371,7 +378,7 @@ std::unique_ptr< MDAL::Mesh > MDAL::DriverCF::load( const std::string &fileName,
if ( status ) *status = MDAL_Status::None;

//Dimensions dims;
std::vector<double> times;
std::vector<MDAL::RelativeTimestamp> times;

try
{
Expand Down Expand Up @@ -401,13 +408,13 @@ std::unique_ptr< MDAL::Mesh > MDAL::DriverCF::load( const std::string &fileName,
setProjection( mesh.get() );

// Parse time array
parseTime( times );
MDAL::DateTime referenceTime = parseTime( times );

// Parse dataset info
cfdataset_info_map dsinfo_map = parseDatasetGroupInfo();

// Create datasets
addDatasetGroups( mesh.get(), times, dsinfo_map );
addDatasetGroups( mesh.get(), times, dsinfo_map, referenceTime );

return std::unique_ptr<Mesh>( mesh.release() );
}
Expand Down
6 changes: 3 additions & 3 deletions external/mdal/frmts/mdal_cf.hpp
View file
Expand Up @@ -138,10 +138,10 @@ namespace MDAL

void setProjection( MDAL::Mesh *m );
cfdataset_info_map parseDatasetGroupInfo();
void parseTime( std::vector<double> &times );
DateTime parseTime( std::vector<RelativeTimestamp> &times ); //Return the reference time
void addDatasetGroups( Mesh *mesh,
const std::vector<double> &times,
const cfdataset_info_map &dsinfo_map );
const std::vector<RelativeTimestamp> &times,
const cfdataset_info_map &dsinfo_map, const DateTime &referenceTime );

std::string mFileName;
std::shared_ptr<NetCDFFile> mNcFile;
Expand Down
2 changes: 1 addition & 1 deletion external/mdal/frmts/mdal_driver.cpp
View file
Expand Up @@ -82,7 +82,7 @@ void MDAL::Driver::createDatasetGroup( MDAL::Mesh *mesh, const std::string &grou
mesh->datasetGroups.push_back( grp );
}

void MDAL::Driver::createDataset( MDAL::DatasetGroup *group, double time, const double *values, const int *active )
void MDAL::Driver::createDataset( MDAL::DatasetGroup *group, MDAL::RelativeTimestamp time, const double *values, const int *active )
{
bool supportsActiveFlag = ( active != nullptr );
std::shared_ptr<MDAL::MemoryDataset2D> dataset = std::make_shared< MemoryDataset2D >( group, supportsActiveFlag );
Expand Down

0 comments on commit 6e2ef15

Please sign in to comment.