Add writing of vector tiles to MBTiles container

qgis · Apr 25, 2020 · 6f746b5 · 6f746b5
1 parent 5e70067
commit 6f746b5
Show file tree

Hide file tree

Showing 11 changed files with 419 additions and 90 deletions.
diff --git a/python/core/auto_generated/qgstiles.sip.in b/python/core/auto_generated/qgstiles.sip.in
@@ -111,6 +111,16 @@ Please note that we follow the XYZ convention of X/Y axes, i.e. top-left tile ha
     static QgsTileMatrix fromWebMercator( int mZoomLevel );
 %Docstring
 Returns a tile matrix for the usual web mercator
+%End
+
+    int matrixWidth() const;
+%Docstring
+Returns number of columns of the tile matrix
+%End
+
+    int matrixHeight() const;
+%Docstring
+Returns number of rows of the tile matrix
 %End
 
     QgsRectangle tileExtent( QgsTileXYZ id ) const;

diff --git a/python/core/auto_generated/vectortile/qgsvectortilewriter.sip.in b/python/core/auto_generated/vectortile/qgsvectortilewriter.sip.in
@@ -26,8 +26,8 @@ the "url" key is normally the path. Currently supported types:
 - "xyz" - tile data written as local files, using a template where {x},{y},{z}
 are replaced by the actual tile column, row and zoom level numbers, e.g.:
 file:///home/qgis/tiles/{z}/{x}/{y}.pbf
-
-(More types such as "mbtiles" or "gpkg" may be added later.)
+- "mbtiles" - tile data written to a new MBTiles file, the "url" key should
+be ordinary file system path, e.g.: /home/qgis/output.mbtiles
 
 Currently the writer only support MVT encoding of data.
 

diff --git a/src/core/qgsmbtilesreader.cpp b/src/core/qgsmbtilesreader.cpp
@@ -18,8 +18,11 @@
 #include "qgslogger.h"
 #include "qgsrectangle.h"
 
+#include <QFile>
 #include <QImage>
 
+#include <zlib.h>
+
 
 QgsMBTilesReader::QgsMBTilesReader( const QString &filename )
   : mFilename( filename )
@@ -46,6 +49,37 @@ bool QgsMBTilesReader::isOpen() const
   return bool( mDatabase );
 }
 
+bool QgsMBTilesReader::create()
+{
+  if ( mDatabase )
+    return false;
+
+  if ( QFile::exists( mFilename ) )
+    return false;
+
+  sqlite3_database_unique_ptr database;
+  int result = mDatabase.open_v2( mFilename, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr );
+  if ( result != SQLITE_OK )
+  {
+    QgsDebugMsg( QStringLiteral( "Can't create MBTiles database: %1" ).arg( database.errorMessage() ) );
+    return false;
+  }
+
+  QString sql = \
+                "CREATE TABLE metadata (name text, value text);" \
+                "CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob);" \
+                "CREATE UNIQUE INDEX tile_index on tiles (zoom_level, tile_column, tile_row);";
+  QString errorMessage;
+  result = mDatabase.exec( sql, errorMessage );
+  if ( result != SQLITE_OK )
+  {
+    QgsDebugMsg( QStringLiteral( "Failed to initialize MBTiles database: " ) + errorMessage );
+    return false;
+  }
+
+  return true;
+}
+
 QString QgsMBTilesReader::metadataValue( const QString &key )
 {
   if ( !mDatabase )
@@ -72,6 +106,30 @@ QString QgsMBTilesReader::metadataValue( const QString &key )
   return preparedStatement.columnAsText( 0 );
 }
 
+void QgsMBTilesReader::setMetadataValue( const QString &key, const QString &value )
+{
+  if ( !mDatabase )
+  {
+    QgsDebugMsg( QStringLiteral( "MBTiles database not open: " ) + mFilename );
+    return;
+  }
+
+  int result;
+  QString sql = QStringLiteral( "insert into metadata values (%1, %2)" ).arg( QgsSqliteUtils::quotedValue( key ), QgsSqliteUtils::quotedValue( value ) );
+  sqlite3_statement_unique_ptr preparedStatement = mDatabase.prepare( sql, result );
+  if ( result != SQLITE_OK )
+  {
+    QgsDebugMsg( QStringLiteral( "MBTile failed to prepare statement: " ) + sql );
+    return;
+  }
+
+  if ( preparedStatement.step() != SQLITE_DONE )
+  {
+    QgsDebugMsg( QStringLiteral( "MBTile metadata value failed to be set: " ) + key );
+    return;
+  }
+}
+
 QgsRectangle QgsMBTilesReader::extent()
 {
   QString boundsStr = metadataValue( "bounds" );
@@ -122,3 +180,127 @@ QImage QgsMBTilesReader::tileDataAsImage( int z, int x, int y )
   }
   return tileImage;
 }
+
+void QgsMBTilesReader::setTileData( int z, int x, int y, const QByteArray &data )
+{
+  if ( !mDatabase )
+  {
+    QgsDebugMsg( QStringLiteral( "MBTiles database not open: " ) + mFilename );
+    return;
+  }
+
+  int result;
+  QString sql = QStringLiteral( "insert into tiles values (%1, %2, %3, ?)" ).arg( z ).arg( x ).arg( y );
+  sqlite3_statement_unique_ptr preparedStatement = mDatabase.prepare( sql, result );
+  if ( result != SQLITE_OK )
+  {
+    QgsDebugMsg( QStringLiteral( "MBTile failed to prepare statement: " ) + sql );
+    return;
+  }
+
+  sqlite3_bind_blob( preparedStatement.get(), 1, data.constData(), data.size(), SQLITE_TRANSIENT );
+
+  if ( preparedStatement.step() != SQLITE_DONE )
+  {
+    QgsDebugMsg( QStringLiteral( "MBTile tile failed to be set: %1,%2,%3" ).arg( z ).arg( x ).arg( y ) );
+    return;
+  }
+}
+
+bool QgsMBTilesReader::decodeGzip( const QByteArray &bytesIn, QByteArray &bytesOut )
+{
+  unsigned char *bytesInPtr = reinterpret_cast<unsigned char *>( const_cast<char *>( bytesIn.constData() ) );
+  uint bytesInLeft = static_cast<uint>( bytesIn.count() );
+
+  const uint CHUNK = 16384;
+  unsigned char out[CHUNK];
+  const int DEC_MAGIC_NUM_FOR_GZIP = 16;
+
+  // allocate inflate state
+  z_stream strm;
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+  strm.avail_in = 0;
+  strm.next_in = Z_NULL;
+
+  int ret = inflateInit2( &strm, MAX_WBITS + DEC_MAGIC_NUM_FOR_GZIP );
+  if ( ret != Z_OK )
+    return false;
+
+  while ( ret != Z_STREAM_END ) // done when inflate() says it's done
+  {
+    // prepare next chunk
+    uint bytesToProcess = std::min( CHUNK, bytesInLeft );
+    strm.next_in = bytesInPtr;
+    strm.avail_in = bytesToProcess;
+    bytesInPtr += bytesToProcess;
+    bytesInLeft -= bytesToProcess;
+
+    if ( bytesToProcess == 0 )
+      break;  // we end with an error - no more data but inflate() wants more data
+
+    // run inflate() on input until output buffer not full
+    do
+    {
+      strm.avail_out = CHUNK;
+      strm.next_out = out;
+      ret = inflate( &strm, Z_NO_FLUSH );
+      Q_ASSERT( ret != Z_STREAM_ERROR ); // state not clobbered
+      if ( ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR )
+      {
+        inflateEnd( &strm );
+        return false;
+      }
+      unsigned have = CHUNK - strm.avail_out;
+      bytesOut.append( QByteArray::fromRawData( reinterpret_cast<const char *>( out ), static_cast<int>( have ) ) );
+    }
+    while ( strm.avail_out == 0 );
+  }
+
+  inflateEnd( &strm );
+  return ret == Z_STREAM_END;
+}
+
+
+bool QgsMBTilesReader::encodeGzip( const QByteArray &bytesIn, QByteArray &bytesOut )
+{
+  unsigned char *bytesInPtr = reinterpret_cast<unsigned char *>( const_cast<char *>( bytesIn.constData() ) );
+  uint bytesInLeft = static_cast<uint>( bytesIn.count() );
+
+  const uint CHUNK = 16384;
+  unsigned char out[CHUNK];
+  const int DEC_MAGIC_NUM_FOR_GZIP = 16;
+
+  // allocate deflate state
+  z_stream strm;
+  strm.zalloc = Z_NULL;
+  strm.zfree = Z_NULL;
+  strm.opaque = Z_NULL;
+
+  int ret = deflateInit2( &strm, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + DEC_MAGIC_NUM_FOR_GZIP, 8, Z_DEFAULT_STRATEGY );
+  if ( ret != Z_OK )
+    return false;
+
+  strm.avail_in = bytesInLeft;
+  strm.next_in = bytesInPtr;
+
+  // run deflate() on input until output buffer not full, finish
+  // compression if all of source has been read in
+  do
+  {
+    strm.avail_out = CHUNK;
+    strm.next_out = out;
+    ret = deflate( &strm, Z_FINISH );  // no bad return value
+    Q_ASSERT( ret != Z_STREAM_ERROR ); // state not clobbered
+
+    unsigned have = CHUNK - strm.avail_out;
+    bytesOut.append( QByteArray::fromRawData( reinterpret_cast<const char *>( out ), static_cast<int>( have ) ) );
+  }
+  while ( strm.avail_out == 0 );
+  Q_ASSERT( ret == Z_STREAM_END );      // stream will be complete
+
+  // clean up and return
+  deflateEnd( &strm );
+  return true;
+}
diff --git a/src/core/qgsmbtilesreader.h b/src/core/qgsmbtilesreader.h
@@ -44,9 +44,22 @@ class CORE_EXPORT QgsMBTilesReader
     //! Returns whether the MBTiles file is currently opened
     bool isOpen() const;
 
+    /**
+     * Creates a new MBTiles file and initializes it with metadata and tiles tables.
+     * It is up to the caller to set appropriate metadata entries and add tiles afterwards.
+     * Returns true on success. If the file exists already, returns false.
+     */
+    bool create();
+
     //! Requests metadata value for the given key
     QString metadataValue( const QString &key );
 
+    /**
+     * Sets metadata value for the given key. Does not overwrite existing entries.
+     * \note the database has to be opened in read-write mode (currently only when opened with create()
+     */
+    void setMetadataValue( const QString &key, const QString &value );
+
     //! Returns bounding box from metadata, given in WGS 84 (if available)
     QgsRectangle extent();
 
@@ -56,6 +69,17 @@ class CORE_EXPORT QgsMBTilesReader
     //! Returns tile decoded as a raster image (if stored in a known format like JPG or PNG)
     QImage tileDataAsImage( int z, int x, int y );
 
+    /**
+     * Adds tile data for the given tile coordinates. Does not overwrite existing entries.
+     * \note the database has to be opened in read-write mode (currently only when opened with create()
+     */
+    void setTileData( int z, int x, int y, const QByteArray &data );
+
+    //! Decodes gzip byte stream, returns true on success. Useful for reading vector tiles.
+    static bool decodeGzip( const QByteArray &bytesIn, QByteArray &bytesOut );
+    //! Encodes gzip byte stream, returns true on success. Useful for writing vector tiles.
+    static bool encodeGzip( const QByteArray &bytesIn, QByteArray &bytesOut );
+
   private:
     QString mFilename;
     sqlite3_database_unique_ptr mDatabase;

diff --git a/src/core/qgstiles.h b/src/core/qgstiles.h
@@ -106,6 +106,12 @@ class CORE_EXPORT QgsTileMatrix
     //! Returns a tile matrix for the usual web mercator
     static QgsTileMatrix fromWebMercator( int mZoomLevel );
 
+    //! Returns number of columns of the tile matrix
+    int matrixWidth() const { return mMatrixWidth; }
+
+    //! Returns number of rows of the tile matrix
+    int matrixHeight() const { return mMatrixHeight; }
+
     //! Returns extent of the given tile in this matrix
     QgsRectangle tileExtent( QgsTileXYZ id ) const;
 

diff --git a/src/core/vectortile/qgsvectortilelayer.cpp b/src/core/vectortile/qgsvectortilelayer.cpp
@@ -68,6 +68,13 @@ bool QgsVectorTileLayer::loadDataSource()
       return false;
     }
 
+    QString format = reader.metadataValue( QStringLiteral( "format" ) );
+    if ( format != QStringLiteral( "pbf" ) )
+    {
+      QgsDebugMsg( QStringLiteral( "Cannot open MBTiles for vector tiles. Format = " ) + format );
+      return false;
+    }
+
     QgsDebugMsgLevel( QStringLiteral( "name: " ) + reader.metadataValue( QStringLiteral( "name" ) ), 2 );
     bool minZoomOk, maxZoomOk;
     int minZoom = reader.metadataValue( QStringLiteral( "minzoom" ) ).toInt( &minZoomOk );

diff --git a/src/core/vectortile/qgsvectortileloader.cpp b/src/core/vectortile/qgsvectortileloader.cpp
@@ -17,8 +17,6 @@
 
 #include <QEventLoop>
 
-#include <zlib.h>
-
 #include "qgsblockingnetworkrequest.h"
 #include "qgslogger.h"
 #include "qgsmbtilesreader.h"
@@ -200,10 +198,8 @@ QByteArray QgsVectorTileLoader::loadFromMBTiles( const QgsTileXYZ &id, QgsMBTile
     return QByteArray();
   }
 
-  // TODO: check format is "pbf"
-
   QByteArray data;
-  if ( !decodeGzip( gzippedTileData, data ) )
+  if ( !QgsMBTilesReader::decodeGzip( gzippedTileData, data ) )
   {
     QgsDebugMsg( QStringLiteral( "Failed to decompress tile " ) + id.toString() );
     return QByteArray();
@@ -212,59 +208,3 @@ QByteArray QgsVectorTileLoader::loadFromMBTiles( const QgsTileXYZ &id, QgsMBTile
   QgsDebugMsgLevel( QStringLiteral( "Tile blob size %1 -> uncompressed size %2" ).arg( gzippedTileData.size() ).arg( data.size() ), 2 );
   return data;
 }
-
-
-bool QgsVectorTileLoader::decodeGzip( const QByteArray &bytesIn, QByteArray &bytesOut )
-{
-  unsigned char *bytesInPtr = reinterpret_cast<unsigned char *>( const_cast<char *>( bytesIn.constData() ) );
-  uint bytesInLeft = static_cast<uint>( bytesIn.count() );
-
-  const uint CHUNK = 16384;
-  unsigned char out[CHUNK];
-  const int DEC_MAGIC_NUM_FOR_GZIP = 16;
-
-  // allocate inflate state
-  z_stream strm;
-  strm.zalloc = Z_NULL;
-  strm.zfree = Z_NULL;
-  strm.opaque = Z_NULL;
-  strm.avail_in = 0;
-  strm.next_in = Z_NULL;
-
-  int ret = inflateInit2( &strm, MAX_WBITS + DEC_MAGIC_NUM_FOR_GZIP );
-  if ( ret != Z_OK )
-    return false;
-
-  while ( ret != Z_STREAM_END ) // done when inflate() says it's done
-  {
-    // prepare next chunk
-    uint bytesToProcess = std::min( CHUNK, bytesInLeft );
-    strm.next_in = bytesInPtr;
-    strm.avail_in = bytesToProcess;
-    bytesInPtr += bytesToProcess;
-    bytesInLeft -= bytesToProcess;
-
-    if ( bytesToProcess == 0 )
-      break;  // we end with an error - no more data but inflate() wants more data
-
-    // run inflate() on input until output buffer not full
-    do
-    {
-      strm.avail_out = CHUNK;
-      strm.next_out = out;
-      ret = inflate( &strm, Z_NO_FLUSH );
-      Q_ASSERT( ret != Z_STREAM_ERROR ); // state not clobbered
-      if ( ret == Z_NEED_DICT || ret == Z_DATA_ERROR || ret == Z_MEM_ERROR )
-      {
-        inflateEnd( &strm );
-        return false;
-      }
-      unsigned have = CHUNK - strm.avail_out;
-      bytesOut.append( QByteArray::fromRawData( reinterpret_cast<const char *>( out ), static_cast<int>( have ) ) );
-    }
-    while ( strm.avail_out == 0 );
-  }
-
-  inflateEnd( &strm );
-  return ret == Z_STREAM_END;
-}
diff --git a/src/core/vectortile/qgsvectortileloader.h b/src/core/vectortile/qgsvectortileloader.h
@@ -65,8 +65,6 @@ class QgsVectorTileLoader : public QObject
     static QByteArray loadFromNetwork( const QgsTileXYZ &id, const QString &requestUrl );
     //! Returns raw tile data for a single tile loaded from MBTiles file
     static QByteArray loadFromMBTiles( const QgsTileXYZ &id, QgsMBTilesReader &mbTileReader );
-    //! Decodes gzip byte stream, returns true on success
-    static bool decodeGzip( const QByteArray &bytesIn, QByteArray &bytesOut );
 
     //
     // non-static stuff