Skip to content

Commit

Permalink
add API to pass a filterRect to point cloud requests (#51063)
Browse files Browse the repository at this point in the history 
* add API to pass a filterRect in pointcloud requests

* update docstring

* clang-tidy, cleanup includes

* tests added

* more clang-tidy

* avoid checking filterRect if X or Y attribute is missing

* rename private mExtent to mFilterRect

* init pointers
  • Loading branch information
@uclaros
uclaros committed Dec 22, 2022
1 parent d7ad954 commit 29fd311
Show file tree
Hide file tree
Showing 17 changed files with 346 additions and 136 deletions.
6 changes: 3 additions & 3 deletions src/core/pointcloud/qgscopcpointcloudblockrequest.cpp
View file
Expand Up @@ -30,9 +30,9 @@

QgsCopcPointCloudBlockRequest::QgsCopcPointCloudBlockRequest( const IndexedPointCloudNode &node, const QString &uri,
const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes,
const QgsVector3D &scale, const QgsVector3D &offset, const QgsPointCloudExpression &filterExpression,
const QgsVector3D &scale, const QgsVector3D &offset, const QgsPointCloudExpression &filterExpression, const QgsRectangle &filterRect,
uint64_t blockOffset, int32_t blockSize, int pointCount, const QgsLazInfo &lazInfo )
: QgsPointCloudBlockRequest( node, uri, attributes, requestedAttributes, scale, offset, filterExpression ),
: QgsPointCloudBlockRequest( node, uri, attributes, requestedAttributes, scale, offset, filterExpression, filterRect ),
mBlockOffset( blockOffset ), mBlockSize( blockSize ), mPointCount( pointCount ), mLazInfo( lazInfo )
{
QNetworkRequest nr( mUri );
Expand Down Expand Up @@ -60,7 +60,7 @@ void QgsCopcPointCloudBlockRequest::blockFinishedLoading()
{
try
{
mBlock = QgsLazDecoder::decompressCopc( mTileDownloadManagerReply->data(), mLazInfo, mPointCount, mRequestedAttributes, mFilterExpression );
mBlock = QgsLazDecoder::decompressCopc( mTileDownloadManagerReply->data(), mLazInfo, mPointCount, mRequestedAttributes, mFilterExpression, mFilterRect );
}
catch ( std::exception &e )
{
Expand Down
2 changes: 1 addition & 1 deletion src/core/pointcloud/qgscopcpointcloudblockrequest.h
View file
Expand Up @@ -48,7 +48,7 @@ class CORE_EXPORT QgsCopcPointCloudBlockRequest : public QgsPointCloudBlockReque
*/
QgsCopcPointCloudBlockRequest( const IndexedPointCloudNode &node, const QString &Uri,
const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes,
const QgsVector3D &scale, const QgsVector3D &offset, const QgsPointCloudExpression &filterExpression,
const QgsVector3D &scale, const QgsVector3D &offset, const QgsPointCloudExpression &filterExpression, const QgsRectangle &filterRect,
uint64_t blockOffset, int32_t blockSize, int pointCount, const QgsLazInfo &lazInfo );

~QgsCopcPointCloudBlockRequest() = default;
Expand Down
3 changes: 2 additions & 1 deletion src/core/pointcloud/qgscopcpointcloudindex.cpp
View file
Expand Up @@ -162,8 +162,9 @@ QgsPointCloudBlock *QgsCopcPointCloudIndex::nodeData( const IndexedPointCloudNod
QgsDebugMsg( QStringLiteral( "Could not read file %1" ).arg( mFileName ) );
return nullptr;
}
QgsRectangle filterRect = request.filterRect();

return QgsLazDecoder::decompressCopc( rawBlockData, *mLazInfo.get(), pointCount, requestAttributes, filterExpression );
return QgsLazDecoder::decompressCopc( rawBlockData, *mLazInfo.get(), pointCount, requestAttributes, filterExpression, filterRect );
}

QgsPointCloudBlockRequest *QgsCopcPointCloudIndex::asyncNodeData( const IndexedPointCloudNode &n, const QgsPointCloudRequest &request )
Expand Down
70 changes: 42 additions & 28 deletions src/core/pointcloud/qgseptdecoder.cpp
View file
Expand Up @@ -16,30 +16,20 @@
***************************************************************************/

#include "qgseptdecoder.h"
#include "qgseptpointcloudindex.h"
#include "qgslazdecoder.h"
#include "qgspointcloudattribute.h"
#include "qgsvector3d.h"
#include "qgsconfig.h"
#include "qgslogger.h"
#include "qgspointcloudexpression.h"
#include "qgsrectangle.h"

#include <QFile>
#include <QDir>
#include <iostream>
#include <memory>
#include <cstring>
#include <QElapsedTimer>
#include <QTemporaryFile>
#include <string>

#include <zstd.h>

#include "lazperf/las.hpp"


///@cond PRIVATE

QgsPointCloudBlock *_decompressBinary( const QByteArray &dataUncompressed, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression )
QgsPointCloudBlock *decompressBinary_( const QByteArray &dataUncompressed, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression, QgsRectangle &filterRect )
{
const std::size_t pointRecordSize = attributes.pointRecordSize( );
const std::size_t requestedPointRecordSize = requestedAttributes.pointRecordSize();
Expand Down Expand Up @@ -99,37 +89,61 @@ QgsPointCloudBlock *_decompressBinary( const QByteArray &dataUncompressed, const
return block.release();
}

int xAttributeOffset, yAttributeOffset;
const QgsPointCloudAttribute *attributeX = nullptr;
const QgsPointCloudAttribute *attributeY = nullptr;
const bool hasFilterRect = !filterRect.isEmpty();
if ( hasFilterRect )
{
attributeX = requestedAttributes.find( QLatin1String( "X" ), xAttributeOffset );
attributeY = requestedAttributes.find( QLatin1String( "Y" ), yAttributeOffset );
filterRect.setXMinimum( ( filterRect.xMinimum() - offset.x() ) / scale.x() );
filterRect.setXMaximum( ( filterRect.xMaximum() - offset.x() ) / scale.x() );
filterRect.setYMinimum( ( filterRect.yMinimum() - offset.y() ) / scale.y() );
filterRect.setYMaximum( ( filterRect.yMaximum() - offset.y() ) / scale.y() );
}

// now loop through points
size_t outputOffset = 0;
for ( int i = 0; i < count; ++i )
{
for ( const AttributeData &attribute : attributeData )
{
_lazSerialize( destinationBuffer, outputOffset,
lazSerialize_( destinationBuffer, outputOffset,
attribute.requestedType, s,
attribute.inputType, attribute.inputSize, i * pointRecordSize + attribute.inputOffset );

outputOffset += attribute.requestedSize;
}

// check if point needs to be filtered out
if ( filterIsValid )
bool skipThisPoint = false;
if ( hasFilterRect && attributeX && attributeY )
{
const double x = attributeX->convertValueToDouble( destinationBuffer + outputOffset - requestedPointRecordSize + xAttributeOffset );
const double y = attributeY->convertValueToDouble( destinationBuffer + outputOffset - requestedPointRecordSize + yAttributeOffset );
if ( !filterRect.contains( x, y ) )
skipThisPoint = true;
}
if ( !skipThisPoint && filterIsValid )
{
// we're always evaluating the last written point in the buffer
double eval = filterExpression.evaluate( i - skippedPoints );
if ( !eval || std::isnan( eval ) )
{
// if the point is filtered out, rewind the offset so the next point is written over it
outputOffset -= requestedPointRecordSize;
++skippedPoints;
}
skipThisPoint = true;
}
if ( skipThisPoint )
{
// if the point is filtered out, rewind the offset so the next point is written over it
outputOffset -= requestedPointRecordSize;
++skippedPoints;
}
}
block->setPointCount( count - skippedPoints );
return block.release();
}

QgsPointCloudBlock *QgsEptDecoder::decompressBinary( const QString &filename, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression )
QgsPointCloudBlock *QgsEptDecoder::decompressBinary( const QString &filename, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression, QgsRectangle &filterRect )
{
if ( ! QFile::exists( filename ) )
return nullptr;
Expand All @@ -140,12 +154,12 @@ QgsPointCloudBlock *QgsEptDecoder::decompressBinary( const QString &filename, co
return nullptr;

const QByteArray dataUncompressed = f.read( f.size() );
return _decompressBinary( dataUncompressed, attributes, requestedAttributes, scale, offset, filterExpression );
return decompressBinary_( dataUncompressed, attributes, requestedAttributes, scale, offset, filterExpression, filterRect );
}

QgsPointCloudBlock *QgsEptDecoder::decompressBinary( const QByteArray &data, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression )
QgsPointCloudBlock *QgsEptDecoder::decompressBinary( const QByteArray &data, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression, QgsRectangle &filterRect )
{
return _decompressBinary( data, attributes, requestedAttributes, scale, offset, filterExpression );
return decompressBinary_( data, attributes, requestedAttributes, scale, offset, filterExpression, filterRect );
}

/* *************************************************************************************** */
Expand Down Expand Up @@ -178,7 +192,7 @@ QByteArray decompressZtdStream( const QByteArray &dataCompressed )
return dataUncompressed;
}

QgsPointCloudBlock *QgsEptDecoder::decompressZStandard( const QString &filename, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression )
QgsPointCloudBlock *QgsEptDecoder::decompressZStandard( const QString &filename, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression, QgsRectangle &filterRect )
{
if ( ! QFile::exists( filename ) )
return nullptr;
Expand All @@ -190,13 +204,13 @@ QgsPointCloudBlock *QgsEptDecoder::decompressZStandard( const QString &filename,

const QByteArray dataCompressed = f.readAll();
const QByteArray dataUncompressed = decompressZtdStream( dataCompressed );
return _decompressBinary( dataUncompressed, attributes, requestedAttributes, scale, offset, filterExpression );
return decompressBinary_( dataUncompressed, attributes, requestedAttributes, scale, offset, filterExpression, filterRect );
}

QgsPointCloudBlock *QgsEptDecoder::decompressZStandard( const QByteArray &data, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression )
QgsPointCloudBlock *QgsEptDecoder::decompressZStandard( const QByteArray &data, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression, QgsRectangle &filterRect )
{
const QByteArray dataUncompressed = decompressZtdStream( data );
return _decompressBinary( dataUncompressed, attributes, requestedAttributes, scale, offset, filterExpression );
return decompressBinary_( dataUncompressed, attributes, requestedAttributes, scale, offset, filterExpression, filterRect );
}

/* *************************************************************************************** */
Expand Down
15 changes: 6 additions & 9 deletions src/core/pointcloud/qgseptdecoder.h
View file
Expand Up @@ -19,27 +19,24 @@
#define QGSEPTDECODER_H


#include "qgis_core.h"
#include "qgis_sip.h"
#include "qgspointcloudblock.h"
#include "qgspointcloudattribute.h"

#include "lazperf/lazperf.hpp"
#include "lazperf/readers.hpp"
#include <QString>

///@cond PRIVATE
#define SIP_NO_FILE

#include <QString>

class QgsPointCloudExpression;
class QgsRectangle;

namespace QgsEptDecoder
{
QgsPointCloudBlock *decompressBinary( const QString &filename, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression );
QgsPointCloudBlock *decompressBinary( const QByteArray &data, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression );
QgsPointCloudBlock *decompressZStandard( const QString &filename, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression );
QgsPointCloudBlock *decompressZStandard( const QByteArray &data, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression );
QgsPointCloudBlock *decompressBinary( const QString &filename, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression, QgsRectangle &filterRect );
QgsPointCloudBlock *decompressBinary( const QByteArray &data, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression, QgsRectangle &filterRect );
QgsPointCloudBlock *decompressZStandard( const QString &filename, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression, QgsRectangle &filterRect );
QgsPointCloudBlock *decompressZStandard( const QByteArray &data, const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes, const QgsVector3D &scale, const QgsVector3D &offset, QgsPointCloudExpression &filterExpression, QgsRectangle &filterRect );
};

///@endcond
Expand Down
10 changes: 5 additions & 5 deletions src/core/pointcloud/qgseptpointcloudblockrequest.cpp
View file
Expand Up @@ -31,8 +31,8 @@

QgsEptPointCloudBlockRequest::QgsEptPointCloudBlockRequest( const IndexedPointCloudNode &node, const QString &uri, const QString &dataType,
const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes,
const QgsVector3D &scale, const QgsVector3D &offset, const QgsPointCloudExpression &filterExpression )
: QgsPointCloudBlockRequest( node, uri, attributes, requestedAttributes, scale, offset, filterExpression ),
const QgsVector3D &scale, const QgsVector3D &offset, const QgsPointCloudExpression &filterExpression, const QgsRectangle &filterRect )
: QgsPointCloudBlockRequest( node, uri, attributes, requestedAttributes, scale, offset, filterExpression, filterRect ),
mDataType( dataType )
{
QNetworkRequest nr( mUri );
Expand All @@ -53,15 +53,15 @@ void QgsEptPointCloudBlockRequest::blockFinishedLoading()
mBlock = nullptr;
if ( mDataType == QLatin1String( "binary" ) )
{
mBlock = QgsEptDecoder::decompressBinary( mTileDownloadManagerReply->data(), mAttributes, mRequestedAttributes, mScale, mOffset, mFilterExpression );
mBlock = QgsEptDecoder::decompressBinary( mTileDownloadManagerReply->data(), mAttributes, mRequestedAttributes, mScale, mOffset, mFilterExpression, mFilterRect );
}
else if ( mDataType == QLatin1String( "zstandard" ) )
{
mBlock = QgsEptDecoder::decompressZStandard( mTileDownloadManagerReply->data(), mAttributes, mRequestedAttributes, mScale, mOffset, mFilterExpression );
mBlock = QgsEptDecoder::decompressZStandard( mTileDownloadManagerReply->data(), mAttributes, mRequestedAttributes, mScale, mOffset, mFilterExpression, mFilterRect );
}
else if ( mDataType == QLatin1String( "laszip" ) )
{
mBlock = QgsLazDecoder::decompressLaz( mTileDownloadManagerReply->data(), mRequestedAttributes, mFilterExpression );
mBlock = QgsLazDecoder::decompressLaz( mTileDownloadManagerReply->data(), mRequestedAttributes, mFilterExpression, mFilterRect );
}
else
{
Expand Down
2 changes: 1 addition & 1 deletion src/core/pointcloud/qgseptpointcloudblockrequest.h
View file
Expand Up @@ -47,7 +47,7 @@ class CORE_EXPORT QgsEptPointCloudBlockRequest : public QgsPointCloudBlockReques
*/
QgsEptPointCloudBlockRequest( const IndexedPointCloudNode &node, const QString &Uri, const QString &dataType,
const QgsPointCloudAttributeCollection &attributes, const QgsPointCloudAttributeCollection &requestedAttributes,
const QgsVector3D &scale, const QgsVector3D &offset, const QgsPointCloudExpression &filterExpression );
const QgsVector3D &scale, const QgsVector3D &offset, const QgsPointCloudExpression &filterExpression, const QgsRectangle &filterRect );

~QgsEptPointCloudBlockRequest() = default;
private:
Expand Down
7 changes: 4 additions & 3 deletions src/core/pointcloud/qgseptpointcloudindex.cpp
View file
Expand Up @@ -315,21 +315,22 @@ QgsPointCloudBlock *QgsEptPointCloudIndex::nodeData( const IndexedPointCloudNode
QgsPointCloudExpression filterExpression = mFilterExpression;
QgsPointCloudAttributeCollection requestAttributes = request.attributes();
requestAttributes.extend( attributes(), filterExpression.referencedAttributes() );
QgsRectangle filterRect = request.filterRect();

if ( mDataType == QLatin1String( "binary" ) )
{
const QString filename = QStringLiteral( "%1/ept-data/%2.bin" ).arg( mDirectory, n.toString() );
return QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression );
return QgsEptDecoder::decompressBinary( filename, attributes(), requestAttributes, scale(), offset(), filterExpression, filterRect );
}
else if ( mDataType == QLatin1String( "zstandard" ) )
{
const QString filename = QStringLiteral( "%1/ept-data/%2.zst" ).arg( mDirectory, n.toString() );
return QgsEptDecoder::decompressZStandard( filename, attributes(), request.attributes(), scale(), offset(), filterExpression );
return QgsEptDecoder::decompressZStandard( filename, attributes(), request.attributes(), scale(), offset(), filterExpression, filterRect );
}
else if ( mDataType == QLatin1String( "laszip" ) )
{
const QString filename = QStringLiteral( "%1/ept-data/%2.laz" ).arg( mDirectory, n.toString() );
return QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression );
return QgsLazDecoder::decompressLaz( filename, requestAttributes, filterExpression, filterRect );
}
else
{
Expand Down

0 comments on commit 29fd311

Please sign in to comment.