Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[GRASS] blocking raster import data stream
  • Loading branch information
blazek committed Jun 25, 2015
1 parent 39f8bd9 commit 26588b5
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 21 deletions.
30 changes: 24 additions & 6 deletions src/providers/grass/qgis.r.in.cpp
Expand Up @@ -39,6 +39,7 @@ extern "C"
#include "qgsrectangle.h"
#include "qgsrasterblock.h"
#include "qgsgrass.h"
#include "qgsgrassdatafile.h"

#if GRASS_VERSION_MAJOR >= 7
#define G_allocate_raster_buf Rast_allocate_buf
Expand All @@ -58,12 +59,25 @@ extern "C"
#define G_unopen_cell Rast_unopen
#endif

static int cf = -1;

void checkStream( QDataStream& stream )
{
if ( stream.status() != QDataStream::Ok )
{
if ( cf != -1 )
{
G_unopen_cell( cf );
G_fatal_error( "Cannot read data stream" );
}
}
}

int main( int argc, char **argv )
{
char *name;
struct Option *map;
struct Cell_head window;
int cf;

G_gisinit( argv[0] );

Expand All @@ -79,22 +93,23 @@ int main( int argc, char **argv )
#ifdef Q_OS_WIN32
_setmode( _fileno( stdin ), _O_BINARY );
_setmode( _fileno( stdout ), _O_BINARY );
setvbuf( stdin, NULL, _IONBF, BUFSIZ );
//setvbuf( stdin, NULL, _IONBF, BUFSIZ );
// setting _IONBF on stdout works on windows correctly, data written immediately even without fflush(stdout)
setvbuf( stdout, NULL, _IONBF, BUFSIZ );
//setvbuf( stdout, NULL, _IONBF, BUFSIZ );
#endif

QFile stdinFile;
stdinFile.open( stdin, QIODevice::ReadOnly );
QgsGrassDataFile stdinFile;
stdinFile.open( stdin );
QDataStream stdinStream( &stdinFile );

QFile stdoutFile;
stdoutFile.open( stdout, QIODevice::WriteOnly );
stdoutFile.open( stdout, QIODevice::WriteOnly | QIODevice::Unbuffered );
QDataStream stdoutStream( &stdoutFile );

QgsRectangle extent;
qint32 rows, cols;
stdinStream >> extent >> cols >> rows;
checkStream( stdinStream );

QString err = QgsGrass::setRegion( &window, extent, rows, cols );
if ( !err.isEmpty() )
Expand All @@ -107,6 +122,7 @@ int main( int argc, char **argv )
QGis::DataType qgis_type;
qint32 type;
stdinStream >> type;
checkStream( stdinStream );
qgis_type = ( QGis::DataType )type;

RASTER_MAP_TYPE grass_type;
Expand Down Expand Up @@ -141,11 +157,13 @@ int main( int argc, char **argv )
for ( int row = 0; row < rows; row++ )
{
stdinStream >> isCanceled;
checkStream( stdinStream );
if ( isCanceled )
{
break;
}
stdinStream >> byteArray;
checkStream( stdinStream );

if ( byteArray.size() != expectedSize )
{
Expand Down
5 changes: 2 additions & 3 deletions src/providers/grass/qgis.v.in.cpp
Expand Up @@ -113,7 +113,7 @@ void exitIfCanceled( QDataStream& stdinStream )
// G_set_percent_routine only works in GRASS >= 7
//int percent_routine (int)
//{
// TODO: use it to interrupt cleaning functions
// TODO: use it to interrupt cleaning funct //stdinFile.open( stdin, QIODevice::ReadOnly | QIODevice::Unbuffered );ions
//}

int main( int argc, char **argv )
Expand All @@ -131,9 +131,8 @@ int main( int argc, char **argv )
_setmode( _fileno( stdin ), _O_BINARY );
_setmode( _fileno( stdout ), _O_BINARY );
#endif
//QFile stdinFile;
QgsGrassDataFile stdinFile;
stdinFile.open( stdin, QIODevice::ReadOnly | QIODevice::Unbuffered );
stdinFile.open( stdin );
QDataStream stdinStream( &stdinFile );

QFile stdoutFile;
Expand Down
55 changes: 49 additions & 6 deletions src/providers/grass/qgsgrassdatafile.cpp
Expand Up @@ -13,33 +13,76 @@
* (at your option) any later version. *
* *
***************************************************************************/
#ifdef Q_OS_UNIX
#include <sys/select.h>
#endif

#include "qgsgrassdatafile.h"

QgsGrassDataFile::QgsGrassDataFile( QObject *parent ) :
QFile( parent )
QgsGrassDataFile::QgsGrassDataFile( QObject *parent )
: QFile( parent )
, mFh( 0 )
{
}

bool QgsGrassDataFile::open( FILE * fh )
{
bool ret = QFile::open( fh, QIODevice::ReadOnly | QIODevice::Unbuffered );
if ( ret )
{
mFh = fh;
}
return ret;
}

qint64 QgsGrassDataFile::readData( char * data, qint64 len )
{
qint64 readSoFar = 0;
forever
{
// QFile::readData should return -1 when pipe is closed, but it does not
// and error is not set (at least with QProcess::closeWriteChannel).
// In fact, qfsfileengine_unix.cpp QFSFileEnginePrivate::nativeRead returns -1
// if (readBytes == 0 && !feof(fh)).
//
// feof(stdin) works (tested on Linux) but it is impossible to get FILE* from QFile
// ( fdopen(handle(),"rb") returns FILE*, but it doesn't have eof set until read() is used on it)
// => store FILE* in open()

qint64 read = QFile::readData( data + readSoFar, len - readSoFar );
if ( read == -1 )
{
return -1;
}
readSoFar += read;

//fprintf(stderr, "len = %d readSoFar = %d feof = %d", (int)len, (int)readSoFar, (int)feof(mFh) );
if ( readSoFar == len )
{
break;
}
// Should we sleep or select()? QFile has no waitForReadyRead() implementation.
// Even without sleep there was not observed CPU consuming looping on Linux.

//G_warning("len = %d readSoFar = %d", (int)len, (int)readSoFar);
if ( feof( mFh ) )
{
return -1;
}
// Should we select()? QFile has no waitForReadyRead() implementation.
// QFile::readData() seems to be blocking until there are data on Linux if pipe was not closed.
// If pipe was closed, QFile::readData() does not block and returns 0 (instead of -1) but
// we catch closed pipe above (feof) so select probably is not necessary, normally (on Linux) it is not reached.
// TODO: verify what happens on Windows and possibly port select().
#ifdef Q_OS_UNIX
if ( read == 0 )
{
fd_set readFds;
FD_ZERO( &readFds );
struct timeval tv;
tv.tv_sec = 0;
tv.tv_usec = 10000; // we could also wait for ever
int sel = select( 0, &readFds, NULL, NULL, &tv );
Q_UNUSED( sel );
//fprintf(stderr, "sel = %d", sel);
}
#endif
}
return readSoFar;
}
13 changes: 11 additions & 2 deletions src/providers/grass/qgsgrassdatafile.h
Expand Up @@ -24,16 +24,25 @@
* It reimplements QFile::readData to always read requested size of data.
* I found the blocking readData in combination with QgsGrassDataFile opened
* in QIODevice::Unbuffered mode the only way to get communication between
* QgsGrassVectorImport and qgis.v.in module working on Linux. On Windows
* it _seemed_ to work even without QgsDataFile.
* QgsGrassVectorImport and qgis.v.in module working on Linux.
* Note that it may seem to work OK even without QgsGrassDataFile and Unbuffered
* but then some imports randomply fails, so be careful and test well all changes.
*
* On Windows it _seemed_ to work even without QgsDataFile.
*
*/
class GRASS_LIB_EXPORT QgsGrassDataFile : public QFile
{
public:
explicit QgsGrassDataFile( QObject *parent = 0 );
virtual ~QgsGrassDataFile() {};
// We need FILE* to be able to test feof but QFile::open(FILE *, OpenMode) is not virtual
bool open( FILE * fh );
// Block until all data are read
virtual qint64 readData( char * data, qint64 len ) override;

private:
FILE *mFh;
};

#endif // QGSGRASSDATAFILE_H
9 changes: 5 additions & 4 deletions src/providers/grass/qgsgrassimport.cpp
Expand Up @@ -13,6 +13,7 @@
* (at your option) any later version. *
* *
***************************************************************************/

#include <QByteArray>
#include <QtConcurrentRun>

Expand Down Expand Up @@ -309,10 +310,10 @@ bool QgsGrassRasterImport::import()
QString stdoutString = process->readAllStandardOutput().data();
QString stderrString = process->readAllStandardError().data();

QString processResult = QString( "exitStatus=%1, exitCode=%2, errorCode=%3, error=%4 stdout=%5, stderr=%6" )
QString processResult = QString( "exitStatus=%1, exitCode=%2, error=%3, errorString=%4 stdout=%5, stderr=%6" )
.arg( process->exitStatus() ).arg( process->exitCode() )
.arg( process->error() ).arg( process->errorString() )
.arg( stdoutString ).arg( stderrString );
.arg( stdoutString.replace( "\n", ", " ) ).arg( stderrString.replace( "\n", ", " ) );
QgsDebugMsg( "processResult: " + processResult );

if ( process->exitStatus() != QProcess::NormalExit )
Expand Down Expand Up @@ -536,10 +537,10 @@ bool QgsGrassVectorImport::import()
QString stdoutString = process->readAllStandardOutput().data();
QString stderrString = process->readAllStandardError().data();

QString processResult = QString( "exitStatus=%1, exitCode=%2, errorCode=%3, error=%4 stdout=%5, stderr=%6" )
QString processResult = QString( "exitStatus=%1, exitCode=%2, error=%3, errorString=%4 stdout=%5, stderr=%6" )
.arg( process->exitStatus() ).arg( process->exitCode() )
.arg( process->error() ).arg( process->errorString() )
.arg( stdoutString ).arg( stderrString );
.arg( stdoutString.replace( "\n", ", " ) ).arg( stderrString.replace( "\n", ", " ) );
QgsDebugMsg( "processResult: " + processResult );

if ( process->exitStatus() != QProcess::NormalExit )
Expand Down

0 comments on commit 26588b5

Please sign in to comment.