When required text codecs aren't available on the system, proxy

the conversion over to GDAL's API Resolves missing text codecs like CP852 on windows builds. These were previously available, but then Qt upstream dropped the ICU library from their windows builds, and accordingly a whole bunch of older text codecs are no longer available by default on the windows builds. Fixes #36871
qgis · Oct 20, 2020 · c3ca85e · c3ca85e
1 parent c5af382
commit c3ca85e
Show file tree

Hide file tree

Showing 12 changed files with 212 additions and 5 deletions.
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
@@ -362,6 +362,7 @@ SET(QGIS_CORE_SRCS
   qgsobjectcustomproperties.cpp
   qgsofflineediting.cpp
   qgsogcutils.cpp
+  qgsogrproxytextcodec.cpp
   qgsogrutils.cpp
   qgsoptionalexpression.cpp
   qgsowsconnection.cpp
@@ -933,6 +934,7 @@ SET(QGIS_CORE_HDRS
   qgsobjectcustomproperties.h
   qgsofflineediting.h
   qgsogcutils.h
+  qgsogrproxytextcodec.h
   qgsogrutils.h
   qgsoptional.h
   qgsoptionalexpression.h

diff --git a/src/core/qgsogrproxytextcodec.cpp b/src/core/qgsogrproxytextcodec.cpp
@@ -0,0 +1,98 @@
+/***************************************************************************
+                             qgsogrproxytextcodec.cpp
+                             -------------
+    begin                : June 2020
+    copyright            : (C) 2020 Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsogrproxytextcodec.h"
+#include "qgslogger.h"
+#include <cpl_string.h>
+#include <mutex>
+
+QgsOgrProxyTextCodec::QgsOgrProxyTextCodec( const QByteArray &name )
+  : mName( name )
+{
+
+}
+
+QString QgsOgrProxyTextCodec::convertToUnicode( const char *chars, int, ConverterState * ) const
+{
+  if ( !chars )
+    return QString();
+
+  char *res = CPLRecode( chars, mName.constData(), CPL_ENC_UTF8 );
+  if ( !res )
+  {
+    QgsDebugMsg( "convertToUnicode failed" );
+    return QString();
+  }
+
+  const QString result = QString::fromUtf8( res );
+  CPLFree( res );
+  return result;
+}
+
+QByteArray QgsOgrProxyTextCodec::convertFromUnicode( const QChar *unicode, int length, ConverterState * ) const
+{
+  if ( !unicode )
+    return QByteArray();
+
+  const QString src = QString( unicode, length );
+  char *res = CPLRecode( src.toUtf8().constData(), CPL_ENC_UTF8, mName.constData() );
+  if ( !res )
+  {
+    QgsDebugMsg( "convertFromUnicode failed" );
+    return QByteArray();
+  }
+
+  const QByteArray result = QByteArray( res );
+  CPLFree( res );
+  return result;
+}
+
+// MY 5 YEAR OLD DAUGHTER WROTE THIS LINE, REMOVE AT YOUR OWN RISK!!!
+// i don't want this to be here
+
+QByteArray QgsOgrProxyTextCodec::name() const
+{
+  return mName;
+}
+
+QList<QByteArray> QgsOgrProxyTextCodec::aliases() const
+{
+  return QList<QByteArray>();
+}
+
+int QgsOgrProxyTextCodec::mibEnum() const
+{
+  // doesn't seem required in this case
+  return 0;
+}
+
+QStringList QgsOgrProxyTextCodec::supportedCodecs()
+{
+  static QStringList codecs;
+  static std::once_flag initialized;
+  std::call_once( initialized, [&]( )
+  {
+    // there's likely others that are supported by GDAL, but we're primarily concerned here
+    // with codecs used by the shapefile driver, and which are no longer supported on the
+    // windows Qt builds (due to removal of ICU support in windows Qt builds)
+    // see https://github.com/qgis/QGIS/issues/36871
+    for ( int i = 437; i <= 950; ++i )
+      codecs << QStringLiteral( "CP%1" ).arg( i );
+    for ( int i = 1250; i <= 1258; ++i )
+      codecs << QStringLiteral( "CP%1" ).arg( i );
+    codecs << QStringLiteral( "CP1251" );
+  } );
+  return codecs;
+}
diff --git a/src/core/qgsogrproxytextcodec.h b/src/core/qgsogrproxytextcodec.h
@@ -0,0 +1,58 @@
+/***************************************************************************
+                             qgsogrproxytextcodec.h
+                             -------------
+    begin                : June 2020
+    copyright            : (C) 2020 Nyall Dawson
+    email                : nyall dot dawson at gmail dot com
+ ***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSOGRPROXYTEXTCODEC_H
+#define QGSOGRPROXYTEXTCODEC_H
+
+#define SIP_NO_FILE
+
+#include "qgis_core.h"
+
+#include <QTextCodec>
+
+/**
+ * \ingroup core
+ * \class QgsOgrProxyTextCodec
+ * \brief A QTextCodec implementation which relies on OGR to do the text conversion.
+ * \note not available in Python bindings
+ * \since QGIS 3.14
+ */
+class CORE_EXPORT QgsOgrProxyTextCodec: public QTextCodec
+{
+  public:
+
+    /**
+     * Constructor for QgsOgrProxyTextCodec, for the specified encoding \a name.
+     */
+    QgsOgrProxyTextCodec( const QByteArray &name );
+    ~QgsOgrProxyTextCodec() override = default;
+
+    QString convertToUnicode( const char *in, int length, ConverterState *state ) const override;
+    QByteArray convertFromUnicode( const QChar *in, int length, ConverterState *state ) const override;
+    QByteArray name() const override;
+    QList<QByteArray> aliases() const override;
+    int mibEnum() const override;
+
+    /**
+     * Returns a list of supported text codecs.
+     */
+    static QStringList supportedCodecs();
+
+  private:
+
+    QByteArray mName;
+};
+
+#endif // QGSOGRPROXYTEXTCODEC_H
diff --git a/src/core/qgsvectordataprovider.cpp b/src/core/qgsvectordataprovider.cpp
@@ -33,6 +33,7 @@
 #include "qgslogger.h"
 #include "qgsmessagelog.h"
 #include "qgssettings.h"
+#include "qgsogrproxytextcodec.h"
 #include <mutex>
 
 QgsVectorDataProvider::QgsVectorDataProvider( const QString &uri, const ProviderOptions &options,
@@ -199,16 +200,27 @@ QgsVectorDataProvider::Capabilities QgsVectorDataProvider::capabilities() const
   return QgsVectorDataProvider::NoCapabilities;
 }
 
-
 void QgsVectorDataProvider::setEncoding( const QString &e )
 {
   mEncoding = QTextCodec::codecForName( e.toLocal8Bit().constData() );
-
   if ( !mEncoding && e != QLatin1String( "System" ) )
   {
     if ( !e.isEmpty() )
-      QgsMessageLog::logMessage( tr( "Codec %1 not found. Falling back to system locale" ).arg( e ) );
-    mEncoding = QTextCodec::codecForName( "System" );
+    {
+      // can we use the OGR proxy codec?
+      if ( QgsOgrProxyTextCodec::supportedCodecs().contains( e, Qt::CaseInsensitive ) )
+      {
+        //from the Qt docs (https://doc.qt.io/qt-5/qtextcodec.html#QTextCodec-1)
+        // "The QTextCodec should always be constructed on the heap (i.e. with new).
+        // Qt takes ownership and will delete it when the application terminates."
+        mEncoding = new QgsOgrProxyTextCodec( e.toLocal8Bit() );
+      }
+      else
+      {
+        QgsMessageLog::logMessage( tr( "Codec %1 not found. Falling back to system locale" ).arg( e ) );
+        mEncoding = QTextCodec::codecForName( "System" );
+      }
+    }
   }
 
   if ( !mEncoding )

diff --git a/tests/src/core/testqgsogrutils.cpp b/tests/src/core/testqgsogrutils.cpp
@@ -27,6 +27,7 @@
 #include "qgsogrutils.h"
 #include "qgsapplication.h"
 #include "qgspoint.h"
+#include "qgsogrproxytextcodec.h"
 
 class TestQgsOgrUtils: public QObject
 {
@@ -47,6 +48,7 @@ class TestQgsOgrUtils: public QObject
     void readOgrFields();
     void stringToFeatureList();
     void stringToFields();
+    void textCodec();
 
   private:
 
@@ -498,7 +500,17 @@ void TestQgsOgrUtils::stringToFields()
   QCOMPARE( fields.at( 1 ).type(), QVariant::Double );
 }
 
-
+void TestQgsOgrUtils::textCodec()
+{
+  QVERIFY( QgsOgrProxyTextCodec::supportedCodecs().contains( QStringLiteral( "CP852" ) ) );
+  QVERIFY( !QgsOgrProxyTextCodec::supportedCodecs().contains( QStringLiteral( "xxx" ) ) );
+
+  // The QTextCodec should always be constructed on the heap. Qt takes ownership and will delete it when the application terminates.
+  QgsOgrProxyTextCodec *codec = new QgsOgrProxyTextCodec( "CP852" );
+  QCOMPARE( codec->toUnicode( codec->fromUnicode( "abcŐ" ) ), QStringLiteral( "abcŐ" ) );
+  QCOMPARE( codec->toUnicode( codec->fromUnicode( "" ) ), QString() );
+  // cppcheck-suppress memleak
+}
 
 QGSTEST_MAIN( TestQgsOgrUtils )
 #include "testqgsogrutils.moc"
diff --git a/tests/src/python/test_provider_shapefile.py b/tests/src/python/test_provider_shapefile.py
@@ -1054,6 +1054,28 @@ def testReadingLayerGeometryTypes(self):
 
             osgeo.ogr.GetDriverByName('ESRI Shapefile').DeleteDataSource(tmpfile)
 
+    def testEncoding(self):
+        """ Test that CP852 shapefile is read/written correctly """
+
+        tmpdir = tempfile.mkdtemp()
+        self.dirs_to_cleanup.append(tmpdir)
+        for file in glob.glob(os.path.join(TEST_DATA_DIR, 'test_852.*')):
+            shutil.copy(os.path.join(TEST_DATA_DIR, file), tmpdir)
+        datasource = os.path.join(tmpdir, 'test_852.shp')
+
+        vl = QgsVectorLayer(datasource, 'test')
+        self.assertTrue(vl.isValid())
+        self.assertEqual([f.attributes() for f in vl.dataProvider().getFeatures()], [['abcŐ']])
+
+        f = QgsFeature()
+        f.setAttributes(['abcŐabcŐabcŐ'])
+        self.assertTrue(vl.dataProvider().addFeature(f))
+
+        # read it back in
+        vl = QgsVectorLayer(datasource, 'test')
+        self.assertTrue(vl.isValid())
+        self.assertEqual([f.attributes() for f in vl.dataProvider().getFeatures()], [['abcŐ'], ['abcŐabcŐabcŐ']])
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/tests/testdata/test_852.cpg b/tests/testdata/test_852.cpg
@@ -0,0 +1 @@
+852
diff --git a/tests/testdata/test_852.dbf b/tests/testdata/test_852.dbf
diff --git a/tests/testdata/test_852.prj b/tests/testdata/test_852.prj
@@ -0,0 +1 @@
+PROJCS["WGS_1984_UTM_Zone_33N",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",15.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]]
diff --git a/tests/testdata/test_852.qpj b/tests/testdata/test_852.qpj
@@ -0,0 +1 @@
+PROJCS["WGS 84 / UTM zone 33N",GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",15],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1,AUTHORITY["EPSG","9001"]],AXIS["Easting",EAST],AXIS["Northing",NORTH],AUTHORITY["EPSG","32633"]]
diff --git a/tests/testdata/test_852.shp b/tests/testdata/test_852.shp
diff --git a/tests/testdata/test_852.shx b/tests/testdata/test_852.shx