Skip to content

Commit

Permalink
Merge branch 'master' of github.com:qgis/QGIS
Browse files Browse the repository at this point in the history
  • Loading branch information
@rouault
rouault committed Jun 11, 2016
2 parents 5839a3a + 7a3a38b commit 7ed1a7f
Show file tree
Hide file tree
Showing 13 changed files with 490 additions and 77 deletions.
108 changes: 95 additions & 13 deletions src/core/qgsgml.cpp
View file
Expand Up @@ -32,6 +32,8 @@
#include <QSettings>
#include <QUrl>

#include "ogr_api.h"

#include <limits>

static const char NS_SEPARATOR = '?';
Expand Down Expand Up @@ -288,6 +290,7 @@ QgsGmlStreamingParser::QgsGmlStreamingParser( const QString& typeName,
, mInvertAxisOrientation( invertAxisOrientation )
, mNumberReturned( -1 )
, mNumberMatched( -1 )
, mFoundUnhandledGeometryElement( false )
{
mThematicAttributes.clear();
for ( int i = 0; i < fields.size(); i++ )
Expand Down Expand Up @@ -348,6 +351,7 @@ QgsGmlStreamingParser::QgsGmlStreamingParser( const QList<LayerProperties>& laye
, mInvertAxisOrientation( invertAxisOrientation )
, mNumberReturned( -1 )
, mNumberMatched( -1 )
, mFoundUnhandledGeometryElement( false )
{
mThematicAttributes.clear();
for ( int i = 0; i < fields.size(); i++ )
Expand Down Expand Up @@ -411,6 +415,8 @@ QgsGmlStreamingParser::~QgsGmlStreamingParser()
{
delete featPair.first;
}

delete mCurrentFeature;
}

bool QgsGmlStreamingParser::processData( const QByteArray& data, bool atEnd )
Expand Down Expand Up @@ -476,6 +482,24 @@ void QgsGmlStreamingParser::startElement( const XML_Char* el, const XML_Char** a

const bool isGMLNS = ( nsLen == mGMLNameSpaceURI.size() && mGMLNameSpaceURIPtr && memcmp( el, mGMLNameSpaceURIPtr, nsLen ) == 0 );
bool isGeom = false;

if ( theParseMode == geometry || theParseMode == coordinate || theParseMode == posList ||
theParseMode == multiPoint || theParseMode == multiLine || theParseMode == multiPolygon )
{
mGeometryString.append( "<", 1 );
mGeometryString.append( pszLocalName, localNameLen );
mGeometryString.append( " ", 1 );
for ( const XML_Char** attrIter = attr; attrIter && *attrIter; attrIter += 2 )
{
mGeometryString.append( attrIter[0] );
mGeometryString.append( "=\"", 2 );
mGeometryString.append( attrIter[1] );
mGeometryString.append( "\" ", 2 );

}
mGeometryString.append( ">", 1 );
}

if ( isGMLNS && LOCALNAME_EQUALS( "coordinates" ) )
{
mParseModeStack.push( coordinate );
Expand Down Expand Up @@ -513,6 +537,8 @@ void QgsGmlStreamingParser::startElement( const XML_Char* el, const XML_Char** a
memcmp( pszLocalName, mGeometryAttributePtr, localNameLen ) == 0 )
{
mParseModeStack.push( QgsGmlStreamingParser::geometry );
mFoundUnhandledGeometryElement = false;
mGeometryString.clear();
}
//else if ( mParseModeStack.size() == 0 && elementName == mGMLNameSpaceURI + NS_SEPARATOR + "boundedBy" )
else if ( isGMLNS && LOCALNAME_EQUALS( "boundedBy" ) )
Expand Down Expand Up @@ -737,6 +763,27 @@ void QgsGmlStreamingParser::startElement( const XML_Char* el, const XML_Char** a
// e.g: http://services.cuzk.cz/wfs/inspire-cp-wfs.asp?SERVICE=WFS&REQUEST=GetFeature&VERSION=2.0.0&TYPENAMES=cp:CadastralParcel
mTruncatedResponse = true;
}
else if ( !mGeometryString.empty() &&
!LOCALNAME_EQUALS( "exterior" ) &&
!LOCALNAME_EQUALS( "interior" ) &&
!LOCALNAME_EQUALS( "innerBoundaryIs" ) &&
!LOCALNAME_EQUALS( "outerBoundaryIs" ) &&
!LOCALNAME_EQUALS( "LinearRing" ) &&
!LOCALNAME_EQUALS( "pointMember" ) &&
!LOCALNAME_EQUALS( "curveMember" ) &&
!LOCALNAME_EQUALS( "lineStringMember" ) &&
!LOCALNAME_EQUALS( "polygonMember" ) &&
!LOCALNAME_EQUALS( "surfaceMember" ) &&
!LOCALNAME_EQUALS( "Curve" ) &&
!LOCALNAME_EQUALS( "segments" ) &&
!LOCALNAME_EQUALS( "LineStringSegment" ) )
{
//QgsDebugMsg( "Found unhandled geometry element " + QString::fromUtf8( pszLocalName, localNameLen ) );
mFoundUnhandledGeometryElement = true;
}

if ( !mGeometryString.empty() )
isGeom = true;

if ( mDimension == 0 && isGeom )
{
Expand Down Expand Up @@ -805,6 +852,29 @@ void QgsGmlStreamingParser::endElement( const XML_Char* el )
memcmp( pszLocalName, mGeometryAttributePtr, localNameLen ) == 0 )
{
mParseModeStack.pop();
if ( mFoundUnhandledGeometryElement )
{
OGRGeometryH hGeom = OGR_G_CreateFromGML( mGeometryString.c_str() );
if ( hGeom )
{
const int wkbSize = OGR_G_WkbSize( hGeom );
unsigned char* pabyBuffer = new unsigned char[ wkbSize ];
#if GDAL_VERSION_MAJOR >= 2
OGR_G_ExportToIsoWkb( hGeom, wkbNDR, pabyBuffer );
#else
OGR_G_ExportToWkb( hGeom, wkbNDR, pabyBuffer );
#endif
QgsGeometry *g = new QgsGeometry();
g->fromWkb( pabyBuffer, wkbSize );
if ( mInvertAxisOrientation )
{
g->transform( QTransform( 0, 1, 1, 0, 0, 0 ) );
}
mCurrentFeature->setGeometry( g );
OGR_G_DestroyGeometry( hGeom );
}
}
mGeometryString.clear();
}
else if ( theParseMode == boundingBox && isGMLNS && LOCALNAME_EQUALS( "boundedBy" ) )
{
Expand Down Expand Up @@ -859,20 +929,19 @@ void QgsGmlStreamingParser::endElement( const XML_Char* el )
memcmp( pszLocalName, mTypeNamePtr, mTypeName.size() ) == 0 ) )
{
Q_ASSERT( mCurrentFeature );
if ( mCurrentWKB.size() > 0 )
if ( !mCurrentFeature->geometry() )
{
QgsGeometry *g = new QgsGeometry();
g->fromWkb( mCurrentWKB, mCurrentWKB.size() );
mCurrentFeature->setGeometry( g );
mCurrentWKB = QgsWkbPtr( nullptr, 0 );
}
else if ( !mCurrentExtent.isEmpty() )
{
mCurrentFeature->setGeometry( QgsGeometry::fromRect( mCurrentExtent ) );
}
else
{
mCurrentFeature->setGeometry( nullptr );
if ( mCurrentWKB.size() > 0 )
{
QgsGeometry *g = new QgsGeometry();
g->fromWkb( mCurrentWKB, mCurrentWKB.size() );
mCurrentFeature->setGeometry( g );
mCurrentWKB = QgsWkbPtr( nullptr, 0 );
}
else if ( !mCurrentExtent.isEmpty() )
{
mCurrentFeature->setGeometry( QgsGeometry::fromRect( mCurrentExtent ) );
}
}
mCurrentFeature->setValid( true );

Expand Down Expand Up @@ -1031,6 +1100,14 @@ void QgsGmlStreamingParser::endElement( const XML_Char* el )
mExceptionText = mStringCash;
mParseModeStack.pop();
}

if ( !mGeometryString.empty() )
{
mGeometryString.append( "</", 2 );
mGeometryString.append( pszLocalName, localNameLen );
mGeometryString.append( ">", 1 );
}

}

void QgsGmlStreamingParser::characters( const XML_Char* chars, int len )
Expand All @@ -1041,6 +1118,11 @@ void QgsGmlStreamingParser::characters( const XML_Char* chars, int len )
return;
}

if ( !mGeometryString.empty() )
{
mGeometryString.append( chars, len );
}

QgsGmlStreamingParser::ParseMode theParseMode = mParseModeStack.top();
if ( theParseMode == QgsGmlStreamingParser::attribute ||
theParseMode == QgsGmlStreamingParser::attributeTuple ||
Expand Down
6 changes: 6 additions & 0 deletions src/core/qgsgml.h
View file
Expand Up @@ -34,6 +34,8 @@
#include <QStack>
#include <QVector>

#include <string>

/** This class builds features from GML data in a streaming way. The caller must call processData()
* as soon it has new content from the source. At any point, it can call
* getAndStealReadyFeatures() to collect the features that have been completely
Expand Down Expand Up @@ -296,6 +298,10 @@ class CORE_EXPORT QgsGmlStreamingParser
int mNumberReturned;
/** WFS 2.0 "numberMatched" attribute, or -1 if invalid/not found */
int mNumberMatched;
/** XML blob containing geometry */
std::string mGeometryString;
/** Whether we found a unhandled geometry element */
bool mFoundUnhandledGeometryElement;
};


Expand Down
2 changes: 1 addition & 1 deletion src/core/qgslogger.cpp
View file
Expand Up @@ -95,7 +95,7 @@ void QgsLogger::debug( const QString& msg, int debuglevel, const char* file, con
}

#ifndef _MSC_VER
m.prepend( file + sPrefixLength );
m.prepend( file + ( file[0] == '/' ? sPrefixLength : 0 ) );
#else
m.prepend( file );
#endif
Expand Down
7 changes: 6 additions & 1 deletion src/providers/wfs/qgswfscapabilities.cpp
View file
Expand Up @@ -72,6 +72,7 @@ void QgsWFSCapabilities::Capabilities::clear()
setAllTypenames.clear();
mapUnprefixedTypenameToPrefixedTypename.clear();
setAmbiguousUnprefixedTypename.clear();
useEPSGColumnFormat = false;
}

QString QgsWFSCapabilities::Capabilities::addPrefixIfNeeded( const QString& name ) const
Expand Down Expand Up @@ -283,7 +284,11 @@ void QgsWFSCapabilities::capabilitiesReplyFinished()
defaultCRSList = featureTypeElem.elementsByTagName( "DefaultCRS" );
if ( defaultCRSList.length() > 0 )
{
featureType.crslist.append( NormalizeSRSName( defaultCRSList.at( 0 ).toElement().text() ) );
QString srsname( defaultCRSList.at( 0 ).toElement().text() );
// Some servers like Geomedia advertize EPSG:XXXX even in WFS 1.1 or 2.0
if ( srsname.startsWith( "EPSG:" ) )
mCaps.useEPSGColumnFormat = true;
featureType.crslist.append( NormalizeSRSName( srsname ) );
}

//OtherSRS
Expand Down
4 changes: 4 additions & 0 deletions src/providers/wfs/qgswfscapabilities.h
View file
Expand Up @@ -35,6 +35,9 @@ class QgsWFSCapabilities : public QgsWFSRequest
//! description of a vector layer
struct FeatureType
{
//! Default constructor
FeatureType() : insertCap( false ), updateCap( false ), deleteCap( false ) {}

QString name;
QString title;
QString abstract;
Expand Down Expand Up @@ -92,6 +95,7 @@ class QgsWFSCapabilities : public QgsWFSRequest
QList<FeatureType> featureTypes;
QList<Function> spatialPredicatesList;
QList<Function> functionList;
bool useEPSGColumnFormat; // whether to use EPSG:XXXX srsname

QSet< QString > setAllTypenames;
QMap< QString, QString> mapUnprefixedTypenameToPrefixedTypename;
Expand Down
60 changes: 50 additions & 10 deletions src/providers/wfs/qgswfsfeatureiterator.cpp
View file
Expand Up @@ -302,11 +302,15 @@ QUrl QgsWFSFeatureDownloader::buildURL( int startIndex, int maxFeatures, bool fo
{
invertAxis = !invertAxis;
}
getFeatureUrl.addQueryItem( "BBOX", QString(( invertAxis ) ? "%2,%1,%4,%3" : "%1,%2,%3,%4" )
.arg( qgsDoubleToString( mShared->mRect.xMinimum() ),
qgsDoubleToString( mShared->mRect.yMinimum() ),
qgsDoubleToString( mShared->mRect.xMaximum() ),
qgsDoubleToString( mShared->mRect.yMaximum() ) ) );
QString bbox( QString(( invertAxis ) ? "%2,%1,%4,%3" : "%1,%2,%3,%4" )
.arg( qgsDoubleToString( mShared->mRect.xMinimum() ),
qgsDoubleToString( mShared->mRect.yMinimum() ),
qgsDoubleToString( mShared->mRect.xMaximum() ),
qgsDoubleToString( mShared->mRect.yMaximum() ) ) );
// Some servers like Geomedia need the srsname to be explictly appended
// otherwise they are confused and do not interpret it properly
bbox += "," + mShared->srsName();
getFeatureUrl.addQueryItem( "BBOX", bbox );
}
else if ( !mShared->mWFSFilter.isEmpty() )
{
Expand Down Expand Up @@ -395,6 +399,9 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
const int maxRetry = s.value( "/qgis/defaultTileMaxRetry", "3" ).toInt();
int retryIter = 0;
int lastValidTotalDownloadedFeatureCount = 0;
int pagingIter = 1;
QString gmlIdFirstFeatureFirstIter;
bool disablePaging = false;
while ( true )
{
success = true;
Expand All @@ -414,7 +421,7 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
true, /* forceRefresh */
false /* cache */ );

int featureCount = 0;
int featureCountForThisResponse = 0;
while ( true )
{
loop.exec( QEventLoop::ExcludeUserInputEvents );
Expand Down Expand Up @@ -524,7 +531,6 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
QVector<QgsGmlStreamingParser::QgsGmlFeaturePtrGmlIdPair> featurePtrList =
parser->getAndStealReadyFeatures();

featureCount += featurePtrList.size();
mTotalDownloadedFeatureCount += featurePtrList.size();

if ( !mStop )
Expand All @@ -538,7 +544,29 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
for ( int i = 0;i < featurePtrList.size();i++ )
{
QgsGmlStreamingParser::QgsGmlFeaturePtrGmlIdPair& featPair = featurePtrList[i];
featureList.push_back( QgsWFSFeatureGmlIdPair( *( featPair.first ), featPair.second ) );
const QgsFeature& f = *( featPair.first );
QString gmlId( featPair.second );
if ( gmlId.isEmpty() )
{
// Should normally not happen on sane WFS sources, but can happen with
// Geomedia
gmlId = QgsWFSUtils::getMD5( f );
if ( !mShared->mHasWarnedAboutMissingFeatureId )
{
QgsDebugMsg( "Server returns features without fid/gml:id. Computing a fake one using feature attributes" );
mShared->mHasWarnedAboutMissingFeatureId = true;
}
}
if ( pagingIter == 1 && featureCountForThisResponse == 0 )
{
gmlIdFirstFeatureFirstIter = gmlId;
}
else if ( pagingIter == 2 && featureCountForThisResponse == 0 && gmlIdFirstFeatureFirstIter == gmlId )
{
disablePaging = true;
QgsDebugMsg( "Server does not seem to properly support paging since it returned the same first feature for 2 different page requests. Disabling paging" );
}
featureList.push_back( QgsWFSFeatureGmlIdPair( f, gmlId ) );
delete featPair.first;
if (( i > 0 && ( i % 1000 ) == 0 ) || i + 1 == featurePtrList.size() )
{
Expand All @@ -557,6 +585,8 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )

featureList.clear();
}

featureCountForThisResponse ++;
}
}

Expand All @@ -580,7 +610,7 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
if ( ++retryIter <= maxRetry )
{
QgsMessageLog::logMessage( tr( "Retrying request %1: %2/%3" ).arg( url.toString() ).arg( retryIter ).arg( maxRetry ), tr( "WFS" ) );
featureCount = 0;
featureCountForThisResponse = 0;
mTotalDownloadedFeatureCount = lastValidTotalDownloadedFeatureCount;
continue;
}
Expand All @@ -596,8 +626,18 @@ void QgsWFSFeatureDownloader::run( bool serializeFeatures, int maxFeatures )
if ( maxFeatures == 1 )
break;
// Detect if we are at the last page
if (( mShared->mMaxFeatures > 0 && featureCount < mShared->mMaxFeatures ) || featureCount == 0 )
if (( mShared->mMaxFeatures > 0 && featureCountForThisResponse < mShared->mMaxFeatures ) || featureCountForThisResponse == 0 )
break;
++ pagingIter;
if ( disablePaging )
{
mSupportsPaging = mShared->mCaps.supportsPaging = false;
mTotalDownloadedFeatureCount = 0;
if ( mShared->mMaxFeaturesWasSetFromDefaultForPaging )
{
mShared->mMaxFeatures = 0;
}
}
}

mStop = true;
Expand Down

0 comments on commit 7ed1a7f

Please sign in to comment.