Merge pull request #37457 from elpaso/bugfix-gh37449-raster-float64-q…

…uantile-histogram Fix raster quantile with float
qgis · Jun 29, 2020 · fb9df9f · fb9df9f
2 parents b18b927 + 090e5da
commit fb9df9f
Showing 1 changed file with 7 additions and 6 deletions.
diff --git a/src/core/raster/qgsrasterinterface.cpp b/src/core/raster/qgsrasterinterface.cpp
@@ -306,7 +306,7 @@ void QgsRasterInterface::initHistogram( QgsRasterHistogram &histogram,
   {
     // Calc resolution from theSampleSize
     double xRes, yRes;
-    xRes = yRes = std::sqrt( ( finalExtent.width() * finalExtent.height() ) / sampleSize );
+    xRes = yRes = std::sqrt( ( static_cast<double>( finalExtent.width( ) ) * finalExtent.height() ) / sampleSize );
 
     // But limit by physical resolution
     if ( capabilities() & Size )
@@ -336,7 +336,7 @@ void QgsRasterInterface::initHistogram( QgsRasterHistogram &histogram,
   }
   QgsDebugMsgLevel( QStringLiteral( "theHistogram.width = %1 histogram.height = %2" ).arg( histogram.width ).arg( histogram.height ), 4 );
 
-  int myBinCount = binCount;
+  qint64 myBinCount = binCount;
   if ( myBinCount == 0 )
   {
     // TODO: this was OK when stats/histogram were calced in provider,
@@ -359,16 +359,17 @@ void QgsRasterInterface::initHistogram( QgsRasterHistogram &histogram,
              mySrcDataType == Qgis::Int16 || mySrcDataType == Qgis::Int32 ||
              mySrcDataType == Qgis::UInt16 || mySrcDataType == Qgis::UInt32 ) )
       {
-        myBinCount = std::min( histogram.width * histogram.height, static_cast<int>( std::ceil( histogram.maximum - histogram.minimum + 1 ) ) );
+        myBinCount = std::min( static_cast<qint64>( histogram.width ) * histogram.height, static_cast<qint64>( std::ceil( histogram.maximum - histogram.minimum + 1 ) ) );
       }
       else
       {
-        // This is for not integer types:
-        myBinCount = std::min( 2000, histogram.width * histogram.height );
+        // This is for not integer types
+        myBinCount = static_cast<qint64>( histogram.width ) * static_cast<qint64>( histogram.height );
       }
     }
   }
-  histogram.binCount = myBinCount;
+  // Hard limit 10'000'000
+  histogram.binCount = static_cast<int>( std::min( 10000000LL, myBinCount ) );
   QgsDebugMsgLevel( QStringLiteral( "theHistogram.binCount = %1" ).arg( histogram.binCount ), 4 );
 }