Skip to content

Commit

Permalink
add unitest
Browse files Browse the repository at this point in the history
  • Loading branch information
@NEDJIMAbelgacem @wonder-sk
NEDJIMAbelgacem authored and wonder-sk committed Feb 1, 2022
1 parent 1dbc4a2 commit 2bb4432
Show file tree
Hide file tree
Showing 5 changed files with 185 additions and 135 deletions.
137 changes: 2 additions & 135 deletions src/core/pointcloud/qgseptdecoder.cpp
View file
Expand Up @@ -32,8 +32,6 @@

#include <zstd.h>

#include "laz-perf/io.hpp"
#include "laz-perf/common/common.hpp"

///@cond PRIVATE

Expand Down Expand Up @@ -269,137 +267,6 @@ QgsPointCloudBlock *QgsEptDecoder::decompressZStandard( const QByteArray &data,

/* *************************************************************************************** */

struct ExtraBytesAttributeDetails
{
ExtraBytesAttributeDetails( const QString &attribute, QgsPointCloudAttribute::DataType type, int size, int offset )
: attribute( attribute )
, type( type )
, size( size )
, offset( offset )
{}

QString attribute;
QgsPointCloudAttribute::DataType type;
int size;
int offset;
};

template<typename FileType>
QVector<ExtraBytesAttributeDetails> readExtraByteAttributes( laszip::io::reader::basic_file<FileType> &f, FileType &file )
{
auto pastFilePos = file.tellg();

// Read VLR stuff

struct VlrHeader
{
unsigned short reserved;
char user_id[16];
unsigned short record_id;
unsigned short record_length;
char desc[32];
};

struct ExtraByteDescriptor
{
unsigned char reserved[2]; // 2 bytes
unsigned char data_type; // 1 byte
unsigned char options; // 1 byte
char name[32]; // 32 bytes
unsigned char unused[4]; // 4 bytes
unsigned char no_data[8]; // 8 bytes
unsigned char deprecated1[16]; // 16 bytes
unsigned char min[8]; // 8 bytes
unsigned char deprecated2[16]; // 16 bytes
unsigned char max[8]; // 8 bytes
unsigned char deprecated3[16]; // 16 bytes
unsigned char scale[8]; // 8 bytes
unsigned char deprecated4[16]; // 16 bytes
double offset; // 8 bytes
unsigned char deprecated5[16]; // 16 bytes
char description[32]; // 32 bytes
};

QVector<ExtraByteDescriptor> extraBytes;
QVector<ExtraBytesAttributeDetails> extrabytesAttr;

VlrHeader extraBytesVlrHeader;
int extraBytesDescriptorsOffset = -1;

file.seekg( f.get_header().header_size );
for ( unsigned int i = 0; i < f.get_header().vlr_count && file.good() && !file.eof(); ++i )
{
VlrHeader vlrHeader;
file.read( ( char * )&vlrHeader, sizeof( VlrHeader ) );
file.seekg( vlrHeader.record_length, std::ios::cur );
if ( std::equal( vlrHeader.user_id, vlrHeader.user_id + 9, "LASF_Spec" ) && vlrHeader.record_id == 4 )
{
extraBytesVlrHeader = vlrHeader;
extraBytesDescriptorsOffset = f.get_header().header_size + sizeof( VlrHeader );
}
}

// Read VLR fields
if ( extraBytesDescriptorsOffset != -1 )
{
file.seekg( extraBytesDescriptorsOffset );
int n_descriptors = extraBytesVlrHeader.record_length / sizeof( ExtraByteDescriptor );
for ( int i = 0; i < n_descriptors; ++i )
{
ExtraByteDescriptor ebd;
file.read( ( char * )&ebd, sizeof( ExtraByteDescriptor ) );
extraBytes.push_back( ebd );
}
}

for ( ExtraByteDescriptor &eb : extraBytes )
{
int accOffset = extrabytesAttr.empty() ? 0 : extrabytesAttr.back().offset + extrabytesAttr.back().size;
// TODO: manage other data types
switch ( eb.data_type )
{
case 0:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Char, eb.options, accOffset ) );
break;
case 1:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Char, 1, accOffset ) );
break;
case 2:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Char, 1, accOffset ) );
break;
case 3:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::UShort, 2, accOffset ) );
break;
case 4:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Short, 2, accOffset ) );
break;
case 5:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Int32, 4, accOffset ) );
break;
case 6:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Int32, 4, accOffset ) );
break;
case 7:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Int32, 8, accOffset ) );
break;
case 8:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Int32, 8, accOffset ) );
break;
case 9:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Float, 4, accOffset ) );
break;
case 10:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Double, 8, accOffset ) );
break;
default:
break;
}
}

file.seekg( pastFilePos );

return extrabytesAttr;
}

template<typename FileType>
QgsPointCloudBlock *__decompressLaz( FileType &file, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &_scale, const QgsVector3D &_offset )
Expand Down Expand Up @@ -470,7 +337,7 @@ QgsPointCloudBlock *__decompressLaz( FileType &file, const QgsPointCloudAttribut
int offset; // Used in case the attribute is an extra byte attribute
};

QVector<ExtraBytesAttributeDetails> extrabytesAttr = readExtraByteAttributes<FileType>( f, file );
QVector<QgsEptDecoder::ExtraBytesAttributeDetails> extrabytesAttr = QgsEptDecoder::readExtraByteAttributes<FileType>( file );

std::vector< RequestedAttributeDetails > requestedAttributeDetails;
requestedAttributeDetails.reserve( requestedAttributesVector.size() );
Expand Down Expand Up @@ -543,7 +410,7 @@ QgsPointCloudBlock *__decompressLaz( FileType &file, const QgsPointCloudAttribut
else
{
bool foundAttr = false;
for ( ExtraBytesAttributeDetails &eba : extrabytesAttr )
for ( QgsEptDecoder::ExtraBytesAttributeDetails &eba : extrabytesAttr )
{
if ( requestedAttribute.name().compare( eba.attribute.trimmed() ) )
{
Expand Down
143 changes: 143 additions & 0 deletions src/core/pointcloud/qgseptdecoder.h
View file
Expand Up @@ -24,19 +24,162 @@
#include "qgspointcloudblock.h"
#include "qgspointcloudattribute.h"

#include "laz-perf/io.hpp"
#include "laz-perf/common/common.hpp"

///@cond PRIVATE
#define SIP_NO_FILE

#include <QString>

namespace QgsEptDecoder
{
struct ExtraBytesAttributeDetails
{
ExtraBytesAttributeDetails( const QString &attribute, QgsPointCloudAttribute::DataType type, int size, int offset )
: attribute( attribute )
, type( type )
, size( size )
, offset( offset )
{}

QString attribute;
QgsPointCloudAttribute::DataType type;
int size;
int offset;
};

QgsPointCloudBlock *decompressBinary( const QString &filename, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset );
QgsPointCloudBlock *decompressBinary( const QByteArray &data, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset );
QgsPointCloudBlock *decompressZStandard( const QString &filename, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset );
QgsPointCloudBlock *decompressZStandard( const QByteArray &data, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset );
QgsPointCloudBlock *decompressLaz( const QString &filename, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset );
QgsPointCloudBlock *decompressLaz( const QByteArray &data, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset );

//! Returns the list of extrabytes attributes with their type, size and offsets represented in the LAS file
template<typename FileType>
QVector<QgsEptDecoder::ExtraBytesAttributeDetails> readExtraByteAttributes( FileType &file )
{
if ( !file.good() )
return QVector<QgsEptDecoder::ExtraBytesAttributeDetails>();

auto pastFilePos = file.tellg();

file.seekg( 0 );

laszip::io::reader::basic_file<FileType> f( file );

// Read VLR stuff

struct VlrHeader
{
unsigned short reserved;
char user_id[16];
unsigned short record_id;
unsigned short record_length;
char desc[32];
};

struct ExtraByteDescriptor
{
unsigned char reserved[2]; // 2 bytes
unsigned char data_type; // 1 byte
unsigned char options; // 1 byte
char name[32]; // 32 bytes
unsigned char unused[4]; // 4 bytes
unsigned char no_data[8]; // 8 bytes
unsigned char deprecated1[16]; // 16 bytes
unsigned char min[8]; // 8 bytes
unsigned char deprecated2[16]; // 16 bytes
unsigned char max[8]; // 8 bytes
unsigned char deprecated3[16]; // 16 bytes
unsigned char scale[8]; // 8 bytes
unsigned char deprecated4[16]; // 16 bytes
double offset; // 8 bytes
unsigned char deprecated5[16]; // 16 bytes
char description[32]; // 32 bytes
};

QVector<ExtraByteDescriptor> extraBytes;
QVector<ExtraBytesAttributeDetails> extrabytesAttr;

VlrHeader extraBytesVlrHeader;
int extraBytesDescriptorsOffset = -1;

file.seekg( f.get_header().header_size );
for ( unsigned int i = 0; i < f.get_header().vlr_count && file.good() && !file.eof(); ++i )
{
VlrHeader vlrHeader;
file.read( ( char * )&vlrHeader, sizeof( VlrHeader ) );
file.seekg( vlrHeader.record_length, std::ios::cur );
if ( std::equal( vlrHeader.user_id, vlrHeader.user_id + 9, "LASF_Spec" ) && vlrHeader.record_id == 4 )
{
extraBytesVlrHeader = vlrHeader;
extraBytesDescriptorsOffset = f.get_header().header_size + sizeof( VlrHeader );
}
}

// Read VLR fields
if ( extraBytesDescriptorsOffset != -1 )
{
file.seekg( extraBytesDescriptorsOffset );
int n_descriptors = extraBytesVlrHeader.record_length / sizeof( ExtraByteDescriptor );
for ( int i = 0; i < n_descriptors; ++i )
{
ExtraByteDescriptor ebd;
file.read( ( char * )&ebd, sizeof( ExtraByteDescriptor ) );
extraBytes.push_back( ebd );
}
}

for ( ExtraByteDescriptor &eb : extraBytes )
{
int accOffset = extrabytesAttr.empty() ? 0 : extrabytesAttr.back().offset + extrabytesAttr.back().size;
// TODO: manage other data types
switch ( eb.data_type )
{
case 0:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Char, eb.options, accOffset ) );
break;
case 1:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Char, 1, accOffset ) );
break;
case 2:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Char, 1, accOffset ) );
break;
case 3:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::UShort, 2, accOffset ) );
break;
case 4:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Short, 2, accOffset ) );
break;
case 5:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Int32, 4, accOffset ) );
break;
case 6:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Int32, 4, accOffset ) );
break;
case 7:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Int32, 8, accOffset ) );
break;
case 8:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Int32, 8, accOffset ) );
break;
case 9:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Float, 4, accOffset ) );
break;
case 10:
extrabytesAttr.push_back( ExtraBytesAttributeDetails( QString::fromStdString( eb.name ), QgsPointCloudAttribute::Double, 8, accOffset ) );
break;
default:
break;
}
}

file.seekg( pastFilePos );

return extrabytesAttr;
}
};

///@endcond
Expand Down
40 changes: 40 additions & 0 deletions tests/src/providers/testqgseptprovider.cpp
View file
Expand Up @@ -22,6 +22,8 @@
#include <QApplication>
#include <QFileInfo>
#include <QDir>
#include <fstream>
#include <QVector>

//qgis includes...
#include "qgis.h"
Expand All @@ -33,6 +35,7 @@
#include "qgspointcloudlayerelevationproperties.h"
#include "qgsprovidersublayerdetails.h"
#include "qgsgeometry.h"
#include "qgseptdecoder.h"

/**
* \ingroup UnitTests
Expand Down Expand Up @@ -62,6 +65,7 @@ class TestQgsEptProvider : public QObject
void calculateZRange();
void testIdentify_data();
void testIdentify();
void testExtraBytesAttributes();

private:
QString mTestDataDir;
Expand Down Expand Up @@ -435,6 +439,42 @@ void TestQgsEptProvider::testIdentify()
}
}

void TestQgsEptProvider::testExtraBytesAttributes()
{
{
QString dataPath = mTestDataDir + QStringLiteral( "point_clouds/las/extrabytes-dataset.laz" );
std::ifstream file( dataPath.toStdString(), std::ios::binary );
QVector<QgsEptDecoder::ExtraBytesAttributeDetails> attributes = QgsEptDecoder::readExtraByteAttributes<std::ifstream>( file );
QVERIFY( attributes.size() == 4 );

QVERIFY( attributes[0].attribute == QStringLiteral( "ClassFlags" ) );
QVERIFY( attributes[1].attribute == QStringLiteral( "Amplitude" ) );
QVERIFY( attributes[2].attribute == QStringLiteral( "Reflectance" ) );
QVERIFY( attributes[3].attribute == QStringLiteral( "Deviation" ) );

QVERIFY( attributes[0].type == 0 );
QVERIFY( attributes[1].type == 5 );
QVERIFY( attributes[2].type == 5 );
QVERIFY( attributes[3].type == 1 );

QVERIFY( attributes[0].size == 1 );
QVERIFY( attributes[1].size == 8 );
QVERIFY( attributes[2].size == 8 );
QVERIFY( attributes[3].size == 2 );

QVERIFY( attributes[0].offset == 0 );
QVERIFY( attributes[1].offset == 1 );
QVERIFY( attributes[2].offset == 9 );
QVERIFY( attributes[3].offset == 17 );
}

{
QString dataPath = mTestDataDir + QStringLiteral( "point_clouds/las/no-extrabytes-dataset.laz" );
std::ifstream file( dataPath.toStdString(), std::ios::binary );
QVector<QgsEptDecoder::ExtraBytesAttributeDetails> attributes = QgsEptDecoder::readExtraByteAttributes<std::ifstream>( file );
QVERIFY( attributes.size() == 0 );
}
}

QGSTEST_MAIN( TestQgsEptProvider )
#include "testqgseptprovider.moc"
Binary file added tests/testdata/point_clouds/las/extrabytes-dataset.laz
View file
Binary file not shown.
Binary file added tests/testdata/point_clouds/las/no-extrabytes-dataset.laz
View file
Binary file not shown.

0 comments on commit 2bb4432

Please sign in to comment.