Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Tiles cache for PG raster provider
  • Loading branch information
elpaso committed Jan 10, 2020
1 parent 30fbed4 commit 849a295
Show file tree
Hide file tree
Showing 8 changed files with 740 additions and 356 deletions.
5 changes: 4 additions & 1 deletion src/providers/postgres/CMakeLists.txt
Expand Up @@ -56,6 +56,7 @@ INCLUDE_DIRECTORIES(SYSTEM
${POSTGRES_INCLUDE_DIR}
${QCA_INCLUDE_DIR}
${QTKEYCHAIN_INCLUDE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}
)

IF (WITH_GUI)
Expand Down Expand Up @@ -96,7 +97,9 @@ ENDIF (WITH_GUI)
# Postgres Raster

SET(PGRASTER_SRCS
qgspostgresrasterprovider.cpp
raster/qgspostgresrasterprovider.cpp
raster/qgspostgresrastershareddata.cpp
raster/qgspostgresrasterutils.cpp
qgspostgresconn.cpp
qgspostgresconnpool.cpp
)
Expand Down
2 changes: 1 addition & 1 deletion src/providers/postgres/qgspostgresprovider.cpp
Expand Up @@ -71,7 +71,7 @@ static bool tableExists( QgsPostgresConn &conn, const QString &name )
}

QgsPostgresPrimaryKeyType
static QgsPostgresProvider::pkType( const QgsField &f ) const
QgsPostgresProvider::pkType( const QgsField &f ) const
{
switch ( f.type() )
{
Expand Down

Large diffs are not rendered by default.

Expand Up @@ -21,9 +21,14 @@
#include "qgsprovidermetadata.h"
#include "qgspostgresconn.h"
#include "qgspostgresprovider.h"
#include "qgsogrutils.h"
#include "qgspostgresrastershareddata.h"

#include <exception>

/**
* The QgsPostgresRasterProvider class implements a raster data provider for PostGIS
*/
class QgsPostgresRasterProvider : public QgsRasterDataProvider
{

Expand All @@ -36,8 +41,9 @@ class QgsPostgresRasterProvider : public QgsRasterDataProvider

virtual ~QgsPostgresRasterProvider() override;

// QgsDataProvider interface
public:

// QgsDataProvider interface
virtual QgsCoordinateReferenceSystem crs() const override;
virtual QgsRectangle extent() const override;
virtual bool isValid() const override;
Expand All @@ -54,17 +60,17 @@ class QgsPostgresRasterProvider : public QgsRasterDataProvider
//! Gets block size
virtual int xBlockSize() const;
virtual int yBlockSize() const;

virtual QgsRasterBandStats bandStatistics(int bandNo, int stats, const QgsRectangle& extent, int sampleSize, QgsRasterBlockFeedback* feedback) override;

// QgsRasterDataProvider interface
virtual QString htmlMetadata() override;
virtual QString lastErrorTitle() override;
virtual QString lastError() override;
int capabilities() const override;

// Utility functions
//! Parses a WKB raster and returns information as a variant map
static QVariantMap parseWkb( const QByteArray &wkb );
// QgsRasterInterface interface
int xSize() const override;
int ySize() const override;

static const QString PG_RASTER_PROVIDER_KEY;
static const QString PG_RASTER_PROVIDER_DESCRIPTION;
Expand Down Expand Up @@ -100,7 +106,7 @@ class QgsPostgresRasterProvider : public QgsRasterDataProvider
//! Data size in bytes for each band
std::vector<int> mDataSizes;
//! Store overviews
QMap<int, QString> mOverViews;
QMap<unsigned int, QString> mOverViews;
//! Band count
int mBandCount = 0;
//! If is tiled
Expand All @@ -118,9 +124,9 @@ class QgsPostgresRasterProvider : public QgsRasterDataProvider
//! Raster tile size y
int mTileHeight = 0;
//! Scale x
int mScaleX = 0;
double mScaleX = 0;
//! Scale y
int mScaleY = 0;
double mScaleY = 0;

QString mDetectedSrid; //!< Spatial reference detected in the database
QString mRequestedSrid; //!< Spatial reference requested in the uri
Expand All @@ -137,16 +143,20 @@ class QgsPostgresRasterProvider : public QgsRasterDataProvider
*/
QList<QString> mPrimaryKeyAttrs;

//! Mutable data shared between provider and feature sources
std::shared_ptr<QgsPostgresRasterSharedData> mShared;


QgsPostgresConn *connectionRO() const;
QgsPostgresConn *connectionRW();

bool hasSufficientPermsAndCapabilities();
void disconnectDb();
//! Get SRID and data type, FALSE if it's not a valid raster table
bool getDetails();
//! Initialize the raster by fetching metadata and creating spatial indexes.
bool init();
//! Search for overviews and store a map
void findOverviews();
//! Initialize spatial indexes

static QString quotedIdentifier( const QString &ident ) { return QgsPostgresConn::quotedIdentifier( ident ); }
static QString quotedValue( const QVariant &value ) { return QgsPostgresConn::quotedValue( value ); }
Expand All @@ -168,14 +178,8 @@ class QgsPostgresRasterProvider : public QgsRasterDataProvider
*/
void determinePrimaryKeyFromUriKeyColumn();


// QgsRasterInterface interface
public:
int xSize() const override;
int ySize() const override;
};


struct QgsPostgresRasterProviderException: public std::exception
{
QgsPostgresRasterProviderException( const QString &msg )
Expand Down
73 changes: 72 additions & 1 deletion src/providers/postgres/raster/qgspostgresrastershareddata.cpp
Expand Up @@ -13,9 +13,80 @@
* (at your option) any later version. *
* *
***************************************************************************/

#include "qgspostgresrastershareddata.h"
#include "qgspostgresrasterutils.h"

QgsPostgresRasterSharedData::~QgsPostgresRasterSharedData()
{
for ( auto idx : mSpatialIndexes )
{
delete idx.second;
}
}

QList<QgsPostgresRasterSharedData::Tile *> QgsPostgresRasterSharedData::tiles( unsigned int overviewFactor, const QgsRectangle &extent )
{
QList<QgsPostgresRasterSharedData::Tile *> result;
QMutexLocker locker( &mMutex );
Q_ASSERT( mSpatialIndexes.find( overviewFactor ) != mSpatialIndexes.end() );
mSpatialIndexes[ overviewFactor ]->intersects( extent, [ &result ]( Tile * tilePtr ) -> bool
{
result.push_back( tilePtr );
return true;
} );
return result;
}

QgsPostgresRasterSharedData::QgsPostgresRasterSharedData()
QgsPostgresRasterSharedData::Tile *QgsPostgresRasterSharedData::addToIndex( unsigned int overviewFactor, QgsPostgresRasterSharedData::Tile *tile )
{
QMutexLocker locker( &mMutex );
Q_ASSERT( mSpatialIndexes.find( overviewFactor ) != mSpatialIndexes.end() );
mTiles.push_back( std::unique_ptr<Tile>( tile ) );
mSpatialIndexes[ overviewFactor ]->insert( tile, tile->extent );
return tile;
}

void QgsPostgresRasterSharedData::initIndexes( const std::list<unsigned int> &overviewFactors )
{
QMutexLocker locker( &mMutex );
if ( mSpatialIndexes.size() == 0 )
{
// Create index for full resolution data
mSpatialIndexes.emplace( 1, new QgsGenericSpatialIndex<Tile>() );
// Create indexes for overviews
for ( const auto &ovFactor : qgis::as_const( overviewFactors ) )
{
mSpatialIndexes.emplace( ovFactor, new QgsGenericSpatialIndex<Tile>() );
}
}
}

bool QgsPostgresRasterSharedData::indexIsEmpty( unsigned int overviewFactor )
{
QMutexLocker locker( &mMutex );
Q_ASSERT( mSpatialIndexes.find( overviewFactor ) != mSpatialIndexes.end() );
return mSpatialIndexes[ overviewFactor ]->isEmpty();
}

QgsPostgresRasterSharedData::Tile::Tile( long tileId, int srid, QgsRectangle extent, double upperLeftX, double upperLeftY, long width, long height, double scaleX, double scaleY, double skewX, double skewY, int numBands )
: tileId( tileId )
, srid( srid )
, extent( extent )
, upperLeftX( upperLeftX )
, upperLeftY( upperLeftY )
, width( width )
, height( height )
, scaleX( scaleX )
, scaleY( scaleY )
, skewX( skewX )
, skewY( skewY )
, numBands( numBands )
{

}

QByteArray QgsPostgresRasterSharedData::Tile::bandData( int bandNo )
{
return QgsPostgresRasterUtils::parseWkb( data, bandNo )[ QStringLiteral( "data" )].toByteArray();
}
84 changes: 83 additions & 1 deletion src/providers/postgres/raster/qgspostgresrastershareddata.h
Expand Up @@ -16,11 +16,93 @@
#ifndef QGSPOSTGRESRASTERSHAREDDATA_H
#define QGSPOSTGRESRASTERSHAREDDATA_H

#include <QMutex>

#include "qgsrectangle.h"
#include "qgsgenericspatialindex.h"

/**
* The QgsPostgresRasterSharedData class is thread safe.
* The cache owns the tiles and manages its memory.
*/
class QgsPostgresRasterSharedData
{
public:
QgsPostgresRasterSharedData();

/**
* The Tile struct represents a raster tile with metadata and data (intially NULL).
*/
struct Tile
{
Tile( long int tileId,
int srid,
QgsRectangle extent,
double upperLeftX,
double upperLeftY,
long int width,
long int height,
double scaleX,
double scaleY,
double skewX,
double skewY,
int numBands );

long int tileId;
int srid;
QgsRectangle extent;
double upperLeftX;
double upperLeftY;
long int width;
long int height;
double scaleX ;
double scaleY;
double skewX;
double skewY;
int numBands;
QByteArray data;

/**
* Returns data for specified \a bandNo
*/
QByteArray bandData( int bandNo );
};

~QgsPostgresRasterSharedData( );


/**
* Returns tiles (possibly with NULL data) for a given \a extent and \a overviewFactor.
*/
QList<Tile *> tiles( unsigned int overviewFactor, const QgsRectangle &extent );

/**
* Adds a \a tile to the index corresponding to the \a overviewFactor, the index takes ownership of the tile.
*/
Tile *addToIndex( unsigned int overviewFactor, Tile *tile );

/**
* Creates spatial indexes for full resolution data and (possibly empty) \a overviewFactors
*/
void initIndexes( const std::list<unsigned int> &overviewFactors );

/**
* Checks if the index \a overviewFactor is empty
*/
bool indexIsEmpty( unsigned int overviewFactor );


private:

//! Protect access to tile indexes
QMutex mMutex;

// Note: cannot be a smart pointer because spatial index cannot be copied
//! Tile caches, index is the overview factor (1 is the full resolution data)
std::map<unsigned int, QgsGenericSpatialIndex<Tile>*> mSpatialIndexes;

//! Memory manager for owned tiles
std::list<std::unique_ptr<Tile>> mTiles;

};

#endif // QGSPOSTGRESRASTERSHAREDDATA_H

0 comments on commit 849a295

Please sign in to comment.