[GRASS] fixed hanging import of larger polygon layers on Linux

blazek · blazek · commit 272dd9c1cba9 · 2015-09-26T22:12:36.000+02:00
diff --git a/src/providers/grass/qgis.v.in.cpp b/src/providers/grass/qgis.v.in.cpp
@@ -221,16 +221,21 @@ int main( int argc, char **argv )
   QgsFeature feature;
   struct line_cats *cats = Vect_new_cats_struct();
 
-  qint32 featureCount = 0;
+  qint32 featureCount;
+  stdinStream >> featureCount;
+
+  qint32 count = 0;
   while ( true )
   {
     exitIfCanceled( stdinStream );
     stdinStream >> feature;
     checkStream( stdinStream );
 #ifndef Q_OS_WIN
     // cannot be used on Windows, see notes in qgis.r.in
+//#if 0
     stdoutStream << true; // feature received
     stdoutFile.flush();
+//#endif
 #endif
     if ( !feature.isValid() )
     {
@@ -310,7 +315,8 @@ int main( int argc, char **argv )
         G_fatal_error( "Cannot insert: %s", e.what() );
       }
     }
-    featureCount++;
+    count++;
+    G_percent( count, featureCount, 1 );
   }
   db_commit_transaction( driver );
   db_close_database_shutdown_driver( driver );
@@ -395,16 +401,20 @@ int main( int argc, char **argv )
       centroids.insert( area, feature );
       spatialIndex.insertFeature( feature );
     }
+
     G_message( "Attaching input polygons to cleaned areas" );
     // read once more to assign centroids to polygons
+    count = 0;
     while ( true )
     {
       exitIfCanceled( stdinStream );
       stdinStream >> feature;
       checkStream( stdinStream );
 #ifndef Q_OS_WIN
+#if 0
       stdoutStream << true; // feature received
       stdoutFile.flush();
+#endif
 #endif
       if ( !feature.isValid() )
       {
@@ -426,12 +436,17 @@ int main( int argc, char **argv )
           centroid.setAttributes( attr );
         }
       }
+      count++;
+      G_percent( count, featureCount, 1 );
     }
 
+    G_message( "Copying lines from temporary map" );
     Vect_copy_map_lines( tmpMap, finalMap );
     Vect_close( tmpMap );
     Vect_delete( tmpName.toUtf8().data() );
 
+    int centroidsCount = centroids.size();
+    count = 0;
     foreach ( QgsFeature centroid, centroids.values() )
     {
       QgsPoint point = centroid.geometry()->asPoint();
@@ -445,12 +460,16 @@ int main( int argc, char **argv )
         }
         writePoint( finalMap, GV_CENTROID, point, cats );
       }
+      G_percent( count, centroidsCount, 1 );
     }
   }
 
+  G_message( "Building final map topology" );
   Vect_build( finalMap );
   Vect_close( finalMap );
 
+  G_message( "Done" );
+
   stdoutStream << true; // to keep caller waiting until finished
   stdoutFile.flush();
   // TODO history
diff --git a/src/providers/grass/qgsgrassimport.cpp b/src/providers/grass/qgsgrassimport.cpp
@@ -588,9 +588,11 @@ bool QgsGrassVectorImport::import()
 
   outStream << mProvider->fields();
 
+  qint32 featureCount = mProvider->featureCount();
+  outStream << featureCount;
+
   QgsFeatureIterator iterator = mProvider->getFeatures();
   QgsFeature feature;
-  mProgress->setRange( 1, mProvider->featureCount() );
   mProgress->append( tr( "Writing features" ) );
   for ( int i = 0; i < ( isPolygon ? 2 : 1 ); i++ ) // two cycles with polygons
   {
@@ -600,10 +602,13 @@ bool QgsGrassVectorImport::import()
       iterator = mProvider->getFeatures();
     }
     QgsDebugMsg( "send features" );
+    // Better to get real progress from module (without buffer)
+#if 0
+    mProgress->setRange( 1, featureCount );
+#endif
     int count = 0;
     while ( iterator.nextFeature( feature ) )
     {
-      mProgress->setValue( count + 1 );
       if ( !feature.isValid() )
       {
         continue;
@@ -625,11 +630,26 @@ bool QgsGrassVectorImport::import()
 
 #ifndef Q_OS_WIN
       // wait until the feature is written to allow quick cancel (don't send data to buffer)
+
+      // Feedback disabled because it was sometimes hanging on Linux, for example, when importing polygons
+      // the features were written ok in the first run, but after cleaning of polygons, which takes some time
+      // it was hanging here for few seconds after each feature, but data did not arrive to the modulee anyway,
+      // QFSFileEnginePrivate::nativeRead() was hanging on fgetc()
+
+      // TODO: inspect what is happening in QProcess, if there is some buffer and how to disable it
+#if 0
       mProcess->waitForReadyRead();
       bool result;
       outStream >> result;
+#endif
 #endif
       count++;
+#if 0
+      if ( count % 100 == 0 )
+      {
+        mProgress->setValue( count );
+      }
+#endif
       // get some feedback for large datasets
       if ( count % 10000 == 0 )
       {
@@ -644,9 +664,12 @@ bool QgsGrassVectorImport::import()
     mProcess->waitForBytesWritten( -1 );
     QgsDebugMsg( "features sent" );
 #ifndef Q_OS_WIN
+#if 0
     mProcess->waitForReadyRead();
     bool result;
     outStream >> result;
+    QgsDebugMsg( "got feedback" );
+#endif
 #endif
   }
   iterator.close();

Original file line number	Diff line number	Diff line change
`@@ -221,16 +221,21 @@ int main( int argc, char **argv )`
`221`	`221`	`QgsFeature feature;`
`222`	`222`	`struct line_cats *cats = Vect_new_cats_struct();`
`223`	`223`
`224`		`- qint32 featureCount = 0;`
	`224`	`+ qint32 featureCount;`
	`225`	`+ stdinStream >> featureCount;`
	`226`	`+`
	`227`	`+ qint32 count = 0;`
`225`	`228`	`while ( true )`
`226`	`229`	`{`
`227`	`230`	`exitIfCanceled( stdinStream );`
`228`	`231`	`stdinStream >> feature;`
`229`	`232`	`checkStream( stdinStream );`
`230`	`233`	`#ifndef Q_OS_WIN`
`231`	`234`	`// cannot be used on Windows, see notes in qgis.r.in`
	`235`	`+//#if 0`
`232`	`236`	`stdoutStream << true; // feature received`
`233`	`237`	`stdoutFile.flush();`
	`238`	`+//#endif`
`234`	`239`	`#endif`
`235`	`240`	`if ( !feature.isValid() )`
`236`	`241`	`{`
`@@ -310,7 +315,8 @@ int main( int argc, char **argv )`
`310`	`315`	`G_fatal_error( "Cannot insert: %s", e.what() );`
`311`	`316`	`}`
`312`	`317`	`}`
`313`		`- featureCount++;`
	`318`	`+ count++;`
	`319`	`+ G_percent( count, featureCount, 1 );`
`314`	`320`	`}`
`315`	`321`	`db_commit_transaction( driver );`
`316`	`322`	`db_close_database_shutdown_driver( driver );`
`@@ -395,16 +401,20 @@ int main( int argc, char **argv )`
`395`	`401`	`centroids.insert( area, feature );`
`396`	`402`	`spatialIndex.insertFeature( feature );`
`397`	`403`	`}`
	`404`	`+`
`398`	`405`	`G_message( "Attaching input polygons to cleaned areas" );`
`399`	`406`	`// read once more to assign centroids to polygons`
	`407`	`+ count = 0;`
`400`	`408`	`while ( true )`
`401`	`409`	`{`
`402`	`410`	`exitIfCanceled( stdinStream );`
`403`	`411`	`stdinStream >> feature;`
`404`	`412`	`checkStream( stdinStream );`
`405`	`413`	`#ifndef Q_OS_WIN`
	`414`	`+#if 0`
`406`	`415`	`stdoutStream << true; // feature received`
`407`	`416`	`stdoutFile.flush();`
	`417`	`+#endif`
`408`	`418`	`#endif`
`409`	`419`	`if ( !feature.isValid() )`
`410`	`420`	`{`
`@@ -426,12 +436,17 @@ int main( int argc, char **argv )`
`426`	`436`	`centroid.setAttributes( attr );`
`427`	`437`	`}`
`428`	`438`	`}`
	`439`	`+ count++;`
	`440`	`+ G_percent( count, featureCount, 1 );`
`429`	`441`	`}`
`430`	`442`
	`443`	`+ G_message( "Copying lines from temporary map" );`
`431`	`444`	`Vect_copy_map_lines( tmpMap, finalMap );`
`432`	`445`	`Vect_close( tmpMap );`
`433`	`446`	`Vect_delete( tmpName.toUtf8().data() );`
`434`	`447`
	`448`	`+ int centroidsCount = centroids.size();`
	`449`	`+ count = 0;`
`435`	`450`	`foreach ( QgsFeature centroid, centroids.values() )`
`436`	`451`	`{`
`437`	`452`	`QgsPoint point = centroid.geometry()->asPoint();`
`@@ -445,12 +460,16 @@ int main( int argc, char **argv )`
`445`	`460`	`}`
`446`	`461`	`writePoint( finalMap, GV_CENTROID, point, cats );`
`447`	`462`	`}`
	`463`	`+ G_percent( count, centroidsCount, 1 );`
`448`	`464`	`}`
`449`	`465`	`}`
`450`	`466`
	`467`	`+ G_message( "Building final map topology" );`
`451`	`468`	`Vect_build( finalMap );`
`452`	`469`	`Vect_close( finalMap );`
`453`	`470`
	`471`	`+ G_message( "Done" );`
	`472`	`+`
`454`	`473`	`stdoutStream << true; // to keep caller waiting until finished`
`455`	`474`	`stdoutFile.flush();`
`456`	`475`	`// TODO history`