[GRASS] blocking vector import data stream

qgis · Jun 24, 2015 · a3441d1 · a3441d1
1 parent a54c2d0
commit a3441d1
Show file tree

Hide file tree

Showing 4 changed files with 139 additions and 26 deletions.
diff --git a/src/providers/grass/CMakeLists.txt b/src/providers/grass/CMakeLists.txt
@@ -30,6 +30,7 @@ MACRO(ADD_GRASSLIB GRASS_BUILD_VERSION)
 
     SET (GRASS_LIBRARY_SRCS
       ../qgsgrass.cpp 
+      ../qgsgrassdatafile.cpp 
       ../qgsgrassfeatureiterator.cpp 
       ../qgsgrassprovider.cpp 
       ../qgsgrassimport.cpp

diff --git a/src/providers/grass/qgis.v.in.cpp b/src/providers/grass/qgis.v.in.cpp
@@ -46,6 +46,7 @@ extern "C"
 #include "qgsrasterblock.h"
 #include "qgsspatialindex.h"
 #include "qgsgrass.h"
+#include "qgsgrassdatafile.h"
 
 static struct line_pnts *line = Vect_new_line_struct();
 
@@ -66,24 +67,46 @@ void writePolyline( struct Map_info* map, int type, QgsPolyline polyline, struct
   Vect_write_line( map, type, line, cats );
 }
 
-void exitIfCanceled( QDataStream& stdinStream, bool isPolygon,
-                     const QString & tmpName, struct Map_info * tmpMap,
-                     const QString & finalName, struct Map_info * finalMap )
+static struct Map_info *finalMap = 0;
+static struct Map_info *tmpMap = 0;
+static QString finalName;
+static QString tmpName;
+
+void closeMaps()
+{
+  if ( tmpMap )
+  {
+    Vect_close( tmpMap );
+    Vect_delete( tmpName.toUtf8().data() );
+  }
+  if ( finalMap )
+  {
+    Vect_close( finalMap );
+    Vect_delete( finalName.toUtf8().data() );
+  }
+  G_warning( "import canceled -> maps deleted" );
+}
+
+// check stream status or exit
+void checkStream( QDataStream& stdinStream )
+{
+  if ( stdinStream.status() != QDataStream::Ok )
+  {
+    closeMaps();
+    G_fatal_error( "Cannot read data stream" );
+  }
+}
+
+void exitIfCanceled( QDataStream& stdinStream )
 {
   bool isCanceled;
   stdinStream >> isCanceled;
+  checkStream( stdinStream );
   if ( !isCanceled )
   {
     return;
   }
-  if ( isPolygon )
-  {
-    Vect_close( tmpMap );
-    Vect_delete( tmpName.toUtf8().data() );
-  }
-  Vect_close( finalMap );
-  Vect_delete( finalName.toUtf8().data() );
-  G_warning( "import canceled -> maps deleted" );
+  closeMaps();
   exit( EXIT_SUCCESS );
 }
 
@@ -107,39 +130,42 @@ int main( int argc, char **argv )
 #ifdef Q_OS_WIN32
   _setmode( _fileno( stdin ), _O_BINARY );
   _setmode( _fileno( stdout ), _O_BINARY );
-  setvbuf( stdin, NULL, _IONBF, BUFSIZ );
-  setvbuf( stdout, NULL, _IONBF, BUFSIZ );
 #endif
-  QFile stdinFile;
-  stdinFile.open( stdin, QIODevice::ReadOnly );
+  //QFile stdinFile;
+  QgsGrassDataFile stdinFile;
+  stdinFile.open( stdin, QIODevice::ReadOnly | QIODevice::Unbuffered );
   QDataStream stdinStream( &stdinFile );
 
   QFile stdoutFile;
-  stdoutFile.open( stdout, QIODevice::WriteOnly );
+  stdoutFile.open( stdout, QIODevice::WriteOnly | QIODevice::Unbuffered );
   QDataStream stdoutStream( &stdoutFile );
 
+  // global finalName, tmpName are used by checkStream()
+  finalName = QString( mapOption->answer );
+  QDateTime now = QDateTime::currentDateTime();
+  tmpName = QString( "qgis_import_tmp_%1_%2" ).arg( mapOption->answer ).arg( now.toString( "yyyyMMddhhmmss" ) );
+
   qint32 typeQint32;
   stdinStream >> typeQint32;
+  checkStream( stdinStream );
   QGis::WkbType wkbType = ( QGis::WkbType )typeQint32;
   QGis::WkbType wkbFlatType = QGis::flatType( wkbType );
   bool isPolygon = QGis::singleType( wkbFlatType ) == QGis::WKBPolygon;
 
-  QString finalName = QString( mapOption->answer );
-  struct Map_info *finalMap = QgsGrass::vectNewMapStruct();
-  struct Map_info *tmpMap = QgsGrass::vectNewMapStruct();
+  finalMap = QgsGrass::vectNewMapStruct();
   Vect_open_new( finalMap, mapOption->answer, 0 );
   struct Map_info * map = finalMap;
-  QDateTime now = QDateTime::currentDateTime();
   // keep tmp name in sync with QgsGrassMapsetItem::createChildren
-  QString tmpName = QString( "qgis_import_tmp_%1_%2" ).arg( mapOption->answer ).arg( now.toString( "yyyyMMddhhmmss" ) );
   if ( isPolygon )
   {
+    tmpMap = QgsGrass::vectNewMapStruct();
     Vect_open_new( tmpMap, tmpName.toUtf8().data(), 0 );
     map = tmpMap;
   }
 
   QgsFields srcFields;
   stdinStream >> srcFields;
+  checkStream( stdinStream );
   // TODO: find (in QgsGrassVectorImport) if there is unique 'id' or 'cat' field and use it as cat
   int keyNum = 1;
   QString key;
@@ -191,11 +217,12 @@ int main( int argc, char **argv )
   qint32 featureCount = 0;
   while ( true )
   {
-    exitIfCanceled( stdinStream, isPolygon, tmpName, tmpMap, finalName, finalMap );
+    exitIfCanceled( stdinStream );
     stdinStream >> feature;
+    checkStream( stdinStream );
 #ifndef Q_OS_WIN
     // cannot be used on Windows, see notes in qgis.r.in
-    stdoutStream << ( bool )true; // feature received
+    stdoutStream << true; // feature received
     stdoutFile.flush();
 #endif
     if ( !feature.isValid() )
@@ -354,10 +381,11 @@ int main( int argc, char **argv )
     // read once more to assign centroids to polygons
     while ( true )
     {
-      exitIfCanceled( stdinStream, isPolygon, tmpName, tmpMap, finalName, finalMap );
+      exitIfCanceled( stdinStream );
       stdinStream >> feature;
+      checkStream( stdinStream );
 #ifndef Q_OS_WIN
-      stdoutStream << ( bool )true; // feature received
+      stdoutStream << true; // feature received
       stdoutFile.flush();
 #endif
       if ( !feature.isValid() )
@@ -405,7 +433,7 @@ int main( int argc, char **argv )
   Vect_build( finalMap );
   Vect_close( finalMap );
 
-  stdoutStream << ( bool )true; // to keep caller waiting until finished
+  stdoutStream << true; // to keep caller waiting until finished
   stdoutFile.flush();
   // TODO history
 

diff --git a/src/providers/grass/qgsgrassdatafile.cpp b/src/providers/grass/qgsgrassdatafile.cpp
@@ -0,0 +1,45 @@
+/***************************************************************************
+                            qgsgrassdatafile.cpp
+                             -------------------
+    begin                : June, 2015
+    copyright            : (C) 2015 Radim Blazek
+    email                : radim.blazek@gmail.com
+ ***************************************************************************/
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#include "qgsgrassdatafile.h"
+
+QgsGrassDataFile::QgsGrassDataFile( QObject *parent ) :
+    QFile( parent )
+{
+}
+
+qint64 QgsGrassDataFile::readData( char * data, qint64 len )
+{
+  qint64 readSoFar = 0;
+  forever
+  {
+    qint64 read = QFile::readData( data + readSoFar, len - readSoFar );
+    if ( read == -1 )
+    {
+      return -1;
+    }
+    readSoFar += read;
+    if ( readSoFar == len )
+    {
+      break;
+    }
+    // Should we sleep or select()? QFile has no waitForReadyRead() implementation.
+    // Even without sleep there was not observed CPU consuming looping on Linux.
+
+    //G_warning("len = %d readSoFar = %d", (int)len, (int)readSoFar);
+  }
+  return readSoFar;
+}
diff --git a/src/providers/grass/qgsgrassdatafile.h b/src/providers/grass/qgsgrassdatafile.h
@@ -0,0 +1,39 @@
+/***************************************************************************
+                            qgsgrassdatafile.h
+                             -------------------
+    begin                : June, 2015
+    copyright            : (C) 2015 Radim Blazek
+    email                : radim.blazek@gmail.com
+ ***************************************************************************/
+/***************************************************************************
+ *                                                                         *
+ *   This program is free software; you can redistribute it and/or modify  *
+ *   it under the terms of the GNU General Public License as published by  *
+ *   the Free Software Foundation; either version 2 of the License, or     *
+ *   (at your option) any later version.                                   *
+ *                                                                         *
+ ***************************************************************************/
+
+#ifndef QGSGRASSDATAFILE_H
+#define QGSGRASSDATAFILE_H
+
+#include <QFile>
+
+/*
+ * This class does blocking reading which is necessary for QgsDataFile.
+ * It reimplements QFile::readData to always read requested size of data.
+ * I found the blocking readData in combination with QgsGrassDataFile opened
+ * in QIODevice::Unbuffered mode the only way to get communication between
+ * QgsGrassVectorImport and qgis.v.in module working on Linux. On Windows
+ * it _seemed_ to work even without QgsDataFile.
+ */
+class GRASS_LIB_EXPORT QgsGrassDataFile : public QFile
+{
+  public:
+    explicit QgsGrassDataFile( QObject *parent = 0 );
+    virtual ~QgsGrassDataFile() {};
+    // Block until all data are read
+    virtual qint64 readData( char * data, qint64 len ) override;
+};
+
+#endif // QGSGRASSDATAFILE_H