Workaround issue in Qt detection of encoding for html QByteArrays

nyalldawson · nyalldawson · commit ade5b65e792e · 2014-10-03T23:03:36.000+10:00
diff --git a/src/core/qgsnetworkcontentfetcher.cpp b/src/core/qgsnetworkcontentfetcher.cpp
@@ -67,8 +67,47 @@ QString QgsNetworkContentFetcher::contentAsString() const
   QByteArray array = mReply->readAll();
 
   //correctly encode reply as unicode
-  QString content = QTextCodec::codecForHtml( array )->toUnicode( array );
-  return content;
+  QTextCodec* codec = codecForHtml( array );
+  return codec->toUnicode( array );
+}
+
+QTextCodec* QgsNetworkContentFetcher::codecForHtml( QByteArray& array ) const
+{
+  //QTextCodec::codecForHtml fails to detect "<meta charset="utf-8"/>" type tags
+  //see https://bugreports.qt-project.org/browse/QTBUG-41011
+  //so test for that ourselves
+
+  //basic check
+  QTextCodec* codec = QTextCodec::codecForUtfText( array, 0 );
+  if ( codec )
+  {
+    return codec;
+  }
+
+  //check for meta charset tag
+  QByteArray header = array.left( 1024 ).toLower();
+  int pos = header.indexOf( "meta charset=" );
+  if ( pos != -1 )
+  {
+    pos += int( strlen( "meta charset=" ) ) + 1;
+    int pos2 = header.indexOf( '\"', pos );
+    QByteArray cs = header.mid( pos, pos2 - pos );
+    codec = QTextCodec::codecForName( cs );
+    if ( codec )
+    {
+      return codec;
+    }
+  }
+
+  //fallback to QTextCodec::codecForHtml
+  codec = QTextCodec::codecForHtml( array, codec );
+  if ( codec )
+  {
+    return codec;
+  }
+
+  //no luck, default to utf-8
+  return QTextCodec::codecForName( "UTF-8" );
 }
 
 void QgsNetworkContentFetcher::contentLoaded( bool ok )
diff --git a/src/core/qgsnetworkcontentfetcher.h b/src/core/qgsnetworkcontentfetcher.h
@@ -69,6 +69,12 @@ class CORE_EXPORT QgsNetworkContentFetcher : public QObject
 
     bool mContentLoaded;
 
+    /**Tries to create a text codec for decoding html content. Works around bugs in Qt's built in method.
+     * @param array input html byte array
+     * @returns QTextCodec for html content, if detected
+    */
+    QTextCodec *codecForHtml( QByteArray &array ) const;
+
   private slots:
 
     /**Called when fetchUrlContent has finished loading a url. If
diff --git a/tests/src/core/testqgsnetworkcontentfetcher.cpp b/tests/src/core/testqgsnetworkcontentfetcher.cpp
@@ -31,6 +31,7 @@ class TestQgsNetworkContentFetcher: public QObject
     void fetchEmptyUrl(); //test fetching blank url
     void fetchBadUrl(); //test fetching bad url
     void fetchUrlContent(); //test fetching url content
+    void fetchEncodedContent(); //test fetching url content encoded as utf-8
 
     void contentLoaded();
 
@@ -107,6 +108,24 @@ void TestQgsNetworkContentFetcher::fetchUrlContent()
   QVERIFY( mFetchedHtml.contains( QString( "QGIS" ) ) );
 }
 
+void TestQgsNetworkContentFetcher::fetchEncodedContent()
+{
+  QgsNetworkContentFetcher fetcher;
+  //test fetching content from the QGIS homepage
+  mLoaded = false;
+  fetcher.fetchContent( QUrl::fromLocalFile( QString( TEST_DATA_DIR ) + QDir::separator() +  "encoded_html.html" ) );
+  connect( &fetcher, SIGNAL( finished() ), this, SLOT( contentLoaded() ) );
+  while ( !mLoaded )
+  {
+    qApp->processEvents();
+  }
+  QVERIFY( fetcher.reply()->error() == QNetworkReply::NoError );
+
+  //test retrieved content and check for correct detection of encoding
+  QString mFetchedHtml = fetcher.contentAsString();
+  QVERIFY( mFetchedHtml.contains( QChar( 6040 ) ) );
+}
+
 void TestQgsNetworkContentFetcher::contentLoaded()
 {
   mLoaded = true;
diff --git a/tests/testdata/encoded_html.html b/tests/testdata/encoded_html.html
@@ -0,0 +1,11 @@
+<!doctype html>
+<html>
+<head>
+<meta http-equiv="Content-Type" content="text/html;">
+<meta charset="UTF-8">
+<title>test</title>
+</head>
+<body>
+<p>សាលា ម៉ាត</p>
+</body></html>
+
