[FEATURE] Binary blob support for OGR provider

Instead of converting binary fields to truncated strings, we instead store their contents as QByteArray values, allowing the original binary content to be retrieved. This allows for plugins and scripts to utilise binary fields, such as extracting their contents.
qgis · Nov 12, 2018 · 5c27b7d · 5c27b7d
1 parent 8ae1880
commit 5c27b7d
Show file tree

Hide file tree

Showing 56 changed files with 59 additions and 1 deletion.
diff --git a/src/core/qgsfield.cpp b/src/core/qgsfield.cpp
@@ -259,6 +259,10 @@ QString QgsField::displayString( const QVariant &v ) const
     QJsonDocument doc = QJsonDocument::fromVariant( v );
     return QString::fromUtf8( doc.toJson().data() );
   }
+  else if ( d->type == QVariant::ByteArray )
+  {
+    return QObject::tr( "BLOB" );
+  }
   // Fallback if special rules do not apply
   return v.toString();
 }

diff --git a/src/core/qgsogrutils.cpp b/src/core/qgsogrutils.cpp
@@ -221,6 +221,17 @@ QVariant QgsOgrUtils::getOgrFeatureAttribute( OGRFeatureH ogrFet, const QgsField
           value = QDateTime( QDate( year, month, day ), QTime( hour, minute, second ) );
       }
       break;
+
+      case QVariant::ByteArray:
+      {
+        int size = 0;
+        const GByte *b = OGR_F_GetFieldAsBinary( ogrFet, attIndex, &size );
+        QByteArray ba = QByteArray::fromRawData( reinterpret_cast<const char *>( b ), size );
+        ba.detach();
+        value = ba;
+        break;
+      }
+
       default:
         Q_ASSERT_X( false, "QgsOgrUtils::getOgrFeatureAttribute", "unsupported field type" );
         if ( ok )

diff --git a/src/providers/ogr/qgsogrprovider.cpp b/src/providers/ogr/qgsogrprovider.cpp
@@ -1028,6 +1028,11 @@ void QgsOgrProvider::loadFields()
       case OFTDateTime:
         varType = QVariant::DateTime;
         break;
+
+      case OFTBinary:
+        varType = QVariant::ByteArray;
+        break;
+
       case OFTString:
       default:
         varType = QVariant::String; // other unsupported, leave it as a string
@@ -1504,6 +1509,13 @@ bool QgsOgrProvider::addFeaturePrivate( QgsFeature &f, Flags flags )
           OGR_F_SetFieldString( feature.get(), ogrAttId, textEncoding()->fromUnicode( attrVal.toString() ).constData() );
           break;
 
+        case OFTBinary:
+        {
+          const QByteArray ba = attrVal.toByteArray();
+          OGR_F_SetFieldBinary( feature.get(), ogrAttId, ba.size(), const_cast< GByte * >( reinterpret_cast< const GByte * >( ba.data() ) ) );
+          break;
+        }
+
         default:
           QgsMessageLog::logMessage( tr( "type %1 for attribute %2 not found" ).arg( type ).arg( qgisAttId ), tr( "OGR" ) );
           break;
@@ -2059,6 +2071,14 @@ bool QgsOgrProvider::changeAttributeValues( const QgsChangedAttributesMap &attr_
           case OFTString:
             OGR_F_SetFieldString( of.get(), f, textEncoding()->fromUnicode( it2->toString() ).constData() );
             break;
+
+          case OFTBinary:
+          {
+            const QByteArray ba = it2->toByteArray();
+            OGR_F_SetFieldBinary( of.get(), f, ba.size(), const_cast< GByte * >( reinterpret_cast< const GByte * >( ba.data() ) ) );
+            break;
+          }
+
           default:
             pushError( tr( "Type %1 of attribute %2 of feature %3 unknown." ).arg( type ).arg( fid ).arg( f ) );
             break;

diff --git a/tests/src/core/testqgsfield.cpp b/tests/src/core/testqgsfield.cpp
@@ -384,6 +384,11 @@ void TestQgsField::displayString()
   QCOMPARE( longField.displayString( 5 ), QString( "5" ) );
   QCOMPARE( longField.displayString( 599999898999LL ), QString( "599999898999" ) );
 
+  // binary field
+  QgsField binaryField( QStringLiteral( "binary" ), QVariant::ByteArray, QStringLiteral( "Binary" ) );
+  QString testBAString( QStringLiteral( "test string" ) );
+  QByteArray testBA( testBAString.toLocal8Bit() );
+  QCOMPARE( binaryField.displayString( testBA ), QStringLiteral( "BLOB" ) );
 }
 
 void TestQgsField::convertCompatible()

diff --git a/tests/src/python/test_provider_ogr.py b/tests/src/python/test_provider_ogr.py
@@ -16,9 +16,10 @@
 import shutil
 import sys
 import tempfile
+import hashlib
 
 from osgeo import gdal, ogr  # NOQA
-from qgis.PyQt.QtCore import QVariant
+from qgis.PyQt.QtCore import QVariant, QByteArray
 from qgis.core import (QgsApplication,
                        QgsRectangle,
                        QgsProviderRegistry,
@@ -489,6 +490,23 @@ def testOSM(self):
         self.assertEqual(f.id(), 8)
         del it
 
+    def testBinaryField(self):
+        source = os.path.join(TEST_DATA_DIR, 'attachments.gdb')
+        vl = QgsVectorLayer(source + "|layername=points__ATTACH")
+        self.assertTrue(vl.isValid())
+
+        fields = vl.fields()
+        data_field = fields[fields.lookupField('DATA')]
+        self.assertEqual(data_field.type(), QVariant.ByteArray)
+        self.assertEqual(data_field.typeName(), 'Binary')
+
+        features = {f['ATTACHMENTID']: f for f in vl.getFeatures()}
+        self.assertEqual(len(features), 2)
+        self.assertIsInstance(features[1]['DATA'], QByteArray)
+        self.assertEqual(hashlib.md5(features[1]['DATA'].data()).hexdigest(), 'ef3dbc530cc39a545832a6c82aac57b6')
+        self.assertIsInstance(features[2]['DATA'], QByteArray)
+        self.assertEqual(hashlib.md5(features[2]['DATA'].data()).hexdigest(), '4b952b80e4288ca5111be2f6dd5d6809')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/testdata/attachments.gdb/a00000001.TablesByName.atx b/tests/testdata/attachments.gdb/a00000001.TablesByName.atx
diff --git a/tests/testdata/attachments.gdb/a00000001.freelist b/tests/testdata/attachments.gdb/a00000001.freelist
diff --git a/tests/testdata/attachments.gdb/a00000001.gdbindexes b/tests/testdata/attachments.gdb/a00000001.gdbindexes
diff --git a/tests/testdata/attachments.gdb/a00000001.gdbtable b/tests/testdata/attachments.gdb/a00000001.gdbtable
diff --git a/tests/testdata/attachments.gdb/a00000001.gdbtablx b/tests/testdata/attachments.gdb/a00000001.gdbtablx
diff --git a/tests/testdata/attachments.gdb/a00000002.gdbtable b/tests/testdata/attachments.gdb/a00000002.gdbtable
diff --git a/tests/testdata/attachments.gdb/a00000002.gdbtablx b/tests/testdata/attachments.gdb/a00000002.gdbtablx
diff --git a/tests/testdata/attachments.gdb/a00000003.gdbindexes b/tests/testdata/attachments.gdb/a00000003.gdbindexes
diff --git a/tests/testdata/attachments.gdb/a00000003.gdbtable b/tests/testdata/attachments.gdb/a00000003.gdbtable
diff --git a/tests/testdata/attachments.gdb/a00000003.gdbtablx b/tests/testdata/attachments.gdb/a00000003.gdbtablx
diff --git a/tests/testdata/attachments.gdb/a00000004.CatItemsByPhysicalName.atx b/tests/testdata/attachments.gdb/a00000004.CatItemsByPhysicalName.atx
diff --git a/tests/testdata/attachments.gdb/a00000004.CatItemsByType.atx b/tests/testdata/attachments.gdb/a00000004.CatItemsByType.atx
diff --git a/tests/testdata/attachments.gdb/a00000004.FDO_UUID.atx b/tests/testdata/attachments.gdb/a00000004.FDO_UUID.atx
diff --git a/tests/testdata/attachments.gdb/a00000004.freelist b/tests/testdata/attachments.gdb/a00000004.freelist
diff --git a/tests/testdata/attachments.gdb/a00000004.gdbindexes b/tests/testdata/attachments.gdb/a00000004.gdbindexes
diff --git a/tests/testdata/attachments.gdb/a00000004.gdbtable b/tests/testdata/attachments.gdb/a00000004.gdbtable
diff --git a/tests/testdata/attachments.gdb/a00000004.gdbtablx b/tests/testdata/attachments.gdb/a00000004.gdbtablx
diff --git a/tests/testdata/attachments.gdb/a00000004.spx b/tests/testdata/attachments.gdb/a00000004.spx
diff --git a/tests/testdata/attachments.gdb/a00000005.CatItemTypesByName.atx b/tests/testdata/attachments.gdb/a00000005.CatItemTypesByName.atx
diff --git a/tests/testdata/attachments.gdb/a00000005.CatItemTypesByParentTypeID.atx b/tests/testdata/attachments.gdb/a00000005.CatItemTypesByParentTypeID.atx
diff --git a/tests/testdata/attachments.gdb/a00000005.CatItemTypesByUUID.atx b/tests/testdata/attachments.gdb/a00000005.CatItemTypesByUUID.atx
diff --git a/tests/testdata/attachments.gdb/a00000005.gdbindexes b/tests/testdata/attachments.gdb/a00000005.gdbindexes
diff --git a/tests/testdata/attachments.gdb/a00000005.gdbtable b/tests/testdata/attachments.gdb/a00000005.gdbtable
diff --git a/tests/testdata/attachments.gdb/a00000005.gdbtablx b/tests/testdata/attachments.gdb/a00000005.gdbtablx
diff --git a/tests/testdata/attachments.gdb/a00000006.CatRelsByDestinationID.atx b/tests/testdata/attachments.gdb/a00000006.CatRelsByDestinationID.atx
diff --git a/tests/testdata/attachments.gdb/a00000006.CatRelsByOriginID.atx b/tests/testdata/attachments.gdb/a00000006.CatRelsByOriginID.atx
diff --git a/tests/testdata/attachments.gdb/a00000006.CatRelsByType.atx b/tests/testdata/attachments.gdb/a00000006.CatRelsByType.atx
diff --git a/tests/testdata/attachments.gdb/a00000006.FDO_UUID.atx b/tests/testdata/attachments.gdb/a00000006.FDO_UUID.atx
diff --git a/tests/testdata/attachments.gdb/a00000006.freelist b/tests/testdata/attachments.gdb/a00000006.freelist
diff --git a/tests/testdata/attachments.gdb/a00000006.gdbindexes b/tests/testdata/attachments.gdb/a00000006.gdbindexes
diff --git a/tests/testdata/attachments.gdb/a00000006.gdbtable b/tests/testdata/attachments.gdb/a00000006.gdbtable
diff --git a/tests/testdata/attachments.gdb/a00000006.gdbtablx b/tests/testdata/attachments.gdb/a00000006.gdbtablx
diff --git a/tests/testdata/attachments.gdb/a00000007.CatRelTypesByBackwardLabel.atx b/tests/testdata/attachments.gdb/a00000007.CatRelTypesByBackwardLabel.atx
diff --git a/tests/testdata/attachments.gdb/a00000007.CatRelTypesByDestItemTypeID.atx b/tests/testdata/attachments.gdb/a00000007.CatRelTypesByDestItemTypeID.atx
diff --git a/tests/testdata/attachments.gdb/a00000007.CatRelTypesByForwardLabel.atx b/tests/testdata/attachments.gdb/a00000007.CatRelTypesByForwardLabel.atx
diff --git a/tests/testdata/attachments.gdb/a00000007.CatRelTypesByName.atx b/tests/testdata/attachments.gdb/a00000007.CatRelTypesByName.atx
diff --git a/tests/testdata/attachments.gdb/a00000007.CatRelTypesByOriginItemTypeID.atx b/tests/testdata/attachments.gdb/a00000007.CatRelTypesByOriginItemTypeID.atx
diff --git a/tests/testdata/attachments.gdb/a00000007.CatRelTypesByUUID.atx b/tests/testdata/attachments.gdb/a00000007.CatRelTypesByUUID.atx
diff --git a/tests/testdata/attachments.gdb/a00000007.gdbindexes b/tests/testdata/attachments.gdb/a00000007.gdbindexes
diff --git a/tests/testdata/attachments.gdb/a00000007.gdbtable b/tests/testdata/attachments.gdb/a00000007.gdbtable
diff --git a/tests/testdata/attachments.gdb/a00000007.gdbtablx b/tests/testdata/attachments.gdb/a00000007.gdbtablx
diff --git a/tests/testdata/attachments.gdb/a00000016.gdbindexes b/tests/testdata/attachments.gdb/a00000016.gdbindexes
diff --git a/tests/testdata/attachments.gdb/a00000016.gdbtable b/tests/testdata/attachments.gdb/a00000016.gdbtable
diff --git a/tests/testdata/attachments.gdb/a00000016.gdbtablx b/tests/testdata/attachments.gdb/a00000016.gdbtablx
diff --git a/tests/testdata/attachments.gdb/a00000016.spx b/tests/testdata/attachments.gdb/a00000016.spx
diff --git a/tests/testdata/attachments.gdb/a00000017.GDB_5_REL_OBJECTID.atx b/tests/testdata/attachments.gdb/a00000017.GDB_5_REL_OBJECTID.atx
diff --git a/tests/testdata/attachments.gdb/a00000017.gdbindexes b/tests/testdata/attachments.gdb/a00000017.gdbindexes
diff --git a/tests/testdata/attachments.gdb/a00000017.gdbtable b/tests/testdata/attachments.gdb/a00000017.gdbtable
diff --git a/tests/testdata/attachments.gdb/a00000017.gdbtablx b/tests/testdata/attachments.gdb/a00000017.gdbtablx
diff --git a/tests/testdata/attachments.gdb/gdb b/tests/testdata/attachments.gdb/gdb
diff --git a/tests/testdata/attachments.gdb/timestamps b/tests/testdata/attachments.gdb/timestamps