Skip to content

Commit e2f09fa

Browse files
nyalldawsonnyalldawson
committedApr 1, 2018
Add a QgsTask for fetching network content
Provides a simple method for fetching remote HTTP content in a QgsTask. Utilises QgsNetworkContentFetcher so Url redirects and progress reports are automatically handled.
·
ltr-3_40final-3_2_0
1 parent bcf57c3 commit e2f09fa

10 files changed

+354
-8
lines changed
 

‎python/core/core_auto.sip

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,7 @@
332332
%Include qgsmessageoutput.sip
333333
%Include qgsnetworkaccessmanager.sip
334334
%Include qgsnetworkcontentfetcher.sip
335+
%Include qgsnetworkcontentfetchertask.sip
335336
%Include qgsofflineediting.sip
336337
%Include qgspluginlayer.sip
337338
%Include qgspointxy.sip

‎python/core/qgsnetworkcontentfetcher.sip.in

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111

1212

1313

14-
1514
class QgsNetworkContentFetcher : QObject
1615
{
1716
%Docstring
@@ -20,6 +19,8 @@ and converting the content to standard formats. Url redirects are automatically
2019
handled.
2120

2221
.. versionadded:: 2.5
22+
23+
.. seealso:: :py:class:`QgsNetworkContentFetcherTask`
2324
%End
2425

2526
%TypeHeaderCode
@@ -69,6 +70,13 @@ Returns the fetched content as a string
6970
void finished();
7071
%Docstring
7172
Emitted when content has loaded
73+
%End
74+
75+
void downloadProgress( qint64 bytesReceived, qint64 bytesTotal );
76+
%Docstring
77+
Emitted when data is received.
78+
79+
.. versionadded:: 3.2
7280
%End
7381

7482
};

‎python/core/qgsnetworkcontentfetchertask.sip.in

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/************************************************************************
2+
* This file has been generated automatically from *
3+
* *
4+
* src/core/qgsnetworkcontentfetchertask.h *
5+
* *
6+
* Do not edit manually ! Edit header and run scripts/sipify.pl again *
7+
************************************************************************/
8+
9+
10+
11+
12+
13+
14+
class QgsNetworkContentFetcherTask : QgsTask
15+
{
16+
%Docstring
17+
Handles HTTP network content fetching in a background task.
18+
19+
Provides a simple method for fetching remote HTTP content in a QgsTask.
20+
Url redirects are automatically handled.
21+
22+
After constructing a QgsNetworkContentFetcherTask, callers should
23+
connect to the QgsNetworkContentFetcherTask.fetched signal. They can
24+
then safely access the network reply() from the connected slot
25+
without danger of the task being first removed by the QgsTaskManager.
26+
27+
.. seealso:: :py:class:`QgsNetworkContentFetcher`
28+
29+
.. versionadded:: 3.2
30+
%End
31+
32+
%TypeHeaderCode
33+
#include "qgsnetworkcontentfetchertask.h"
34+
%End
35+
public:
36+
37+
QgsNetworkContentFetcherTask( const QUrl &url );
38+
%Docstring
39+
Constructor for a QgsNetworkContentFetcherTask which fetches
40+
the specified ``url``.
41+
%End
42+
43+
QgsNetworkContentFetcherTask( const QNetworkRequest &request );
44+
%Docstring
45+
Constructor for a QgsNetworkContentFetcherTask which fetches
46+
the specified network ``request``.
47+
%End
48+
49+
~QgsNetworkContentFetcherTask();
50+
51+
virtual bool run();
52+
53+
54+
QNetworkReply *reply();
55+
%Docstring
56+
Returns the network reply. Ownership is not transferred.
57+
58+
May return None if the request has not yet completed.
59+
%End
60+
61+
signals:
62+
63+
void fetched();
64+
%Docstring
65+
Emitted when the network content has been fetched, regardless
66+
of whether the fetch was successful or not.
67+
68+
Users of QgsNetworkContentFetcherTask should connect to this signal,
69+
and from the associated slot they can then safely access the network reply()
70+
without danger of the task being first removed by the :py:class:`QgsTaskManager`.
71+
%End
72+
73+
};
74+
75+
/************************************************************************
76+
* This file has been generated automatically from *
77+
* *
78+
* src/core/qgsnetworkcontentfetchertask.h *
79+
* *
80+
* Do not edit manually ! Edit header and run scripts/sipify.pl again *
81+
************************************************************************/

‎src/core/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,7 @@ SET(QGIS_CORE_SRCS
238238
qgsnetworkaccessmanager.cpp
239239
qgsnetworkdiskcache.cpp
240240
qgsnetworkcontentfetcher.cpp
241+
qgsnetworkcontentfetchertask.cpp
241242
qgsnetworkreplyparser.cpp
242243
qgsobjectcustomproperties.cpp
243244
qgsofflineediting.cpp
@@ -614,6 +615,7 @@ SET(QGIS_CORE_MOC_HDRS
614615
qgsnetworkaccessmanager.h
615616
qgsnetworkdiskcache.h
616617
qgsnetworkcontentfetcher.h
618+
qgsnetworkcontentfetchertask.h
617619
qgsnetworkreplyparser.h
618620
qgsofflineediting.h
619621
qgsowsconnection.h

‎src/core/qgsnetworkcontentfetcher.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ void QgsNetworkContentFetcher::fetchContent( const QNetworkRequest &request )
5555

5656
mReply = QgsNetworkAccessManager::instance()->get( request );
5757
connect( mReply, &QNetworkReply::finished, this, [ = ] { contentLoaded(); } );
58+
connect( mReply, &QNetworkReply::downloadProgress, this, &QgsNetworkContentFetcher::downloadProgress );
5859
}
5960

6061
QNetworkReply *QgsNetworkContentFetcher::reply()

‎src/core/qgsnetworkcontentfetcher.h

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,14 @@
2626
#include "qgis_core.h"
2727

2828
/**
29-
\class QgsNetworkContentFetcher
30-
\ingroup core
31-
\brief HTTP network content fetcher. A simple method for fetching remote HTTP content
32-
and converting the content to standard formats. Url redirects are automatically
33-
handled.
34-
\since 2.5
29+
* \class QgsNetworkContentFetcher
30+
* \ingroup core
31+
* \brief HTTP network content fetcher. A simple method for fetching remote HTTP content
32+
* and converting the content to standard formats. Url redirects are automatically
33+
* handled.
34+
* \since QGIS 2.5
35+
* \see QgsNetworkContentFetcherTask
3536
*/
36-
3737
class CORE_EXPORT QgsNetworkContentFetcher : public QObject
3838
{
3939
Q_OBJECT
@@ -81,6 +81,12 @@ class CORE_EXPORT QgsNetworkContentFetcher : public QObject
8181
*/
8282
void finished();
8383

84+
/**
85+
* Emitted when data is received.
86+
* \since QGIS 3.2
87+
*/
88+
void downloadProgress( qint64 bytesReceived, qint64 bytesTotal );
89+
8490
private:
8591

8692
QNetworkReply *mReply = nullptr;

‎src/core/qgsnetworkcontentfetchertask.cpp

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/***************************************************************************
2+
qgsnetworkcontentfetchertask.cpp
3+
-------------------
4+
begin : March 2018
5+
copyright : (C) 2018 by Nyall Dawson
6+
email : nyall dot dawson at gmail dot com
7+
8+
***************************************************************************/
9+
10+
/***************************************************************************
11+
* *
12+
* This program is free software; you can redistribute it and/or modify *
13+
* it under the terms of the GNU General Public License as published by *
14+
* the Free Software Foundation; either version 2 of the License, or *
15+
* (at your option) any later version. *
16+
* *
17+
***************************************************************************/
18+
19+
#include "qgsnetworkcontentfetchertask.h"
20+
#include "qgsnetworkcontentfetcher.h"
21+
22+
QgsNetworkContentFetcherTask::QgsNetworkContentFetcherTask( const QUrl &url )
23+
: QgsNetworkContentFetcherTask( QNetworkRequest( url ) )
24+
{
25+
}
26+
27+
QgsNetworkContentFetcherTask::QgsNetworkContentFetcherTask( const QNetworkRequest &request )
28+
: QgsTask( tr( "Fetching %1" ).arg( request.url().toString() ) )
29+
, mRequest( request )
30+
{
31+
}
32+
33+
QgsNetworkContentFetcherTask::~QgsNetworkContentFetcherTask()
34+
{
35+
if ( mFetcher )
36+
mFetcher->deleteLater();
37+
}
38+
39+
bool QgsNetworkContentFetcherTask::run()
40+
{
41+
mFetcher = new QgsNetworkContentFetcher();
42+
QEventLoop loop;
43+
connect( mFetcher, &QgsNetworkContentFetcher::finished, &loop, &QEventLoop::quit );
44+
connect( mFetcher, &QgsNetworkContentFetcher::downloadProgress, this, [ = ]( qint64 bytesReceived, qint64 bytesTotal )
45+
{
46+
if ( bytesTotal > 0 )
47+
{
48+
setProgress( ( bytesReceived * 100 ) / bytesTotal );
49+
}
50+
} );
51+
mFetcher->fetchContent( mRequest );
52+
loop.exec();
53+
emit fetched();
54+
return true;
55+
}
56+
57+
QNetworkReply *QgsNetworkContentFetcherTask::reply()
58+
{
59+
return mFetcher ? mFetcher->reply() : nullptr;
60+
}

‎src/core/qgsnetworkcontentfetchertask.h

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/***************************************************************************
2+
qgsnetworkcontentfetchertask.h
3+
-------------------
4+
begin : March, 2018
5+
copyright : (C) 2018 by Nyall Dawson
6+
email : nyall dot dawson at gmail dot com
7+
8+
***************************************************************************/
9+
10+
/***************************************************************************
11+
* *
12+
* This program is free software; you can redistribute it and/or modify *
13+
* it under the terms of the GNU General Public License as published by *
14+
* the Free Software Foundation; either version 2 of the License, or *
15+
* (at your option) any later version. *
16+
* *
17+
***************************************************************************/
18+
19+
20+
#ifndef QGSNETWORKCONTENTFETCHERTASK_H
21+
#define QGSNETWORKCONTENTFETCHERTASK_H
22+
23+
#include "qgstaskmanager.h"
24+
#include "qgis_core.h"
25+
#include <QNetworkRequest>
26+
27+
class QgsNetworkContentFetcher;
28+
class QNetworkReply;
29+
30+
/**
31+
* \class QgsNetworkContentFetcherTask
32+
* \ingroup core
33+
* \brief Handles HTTP network content fetching in a background task.
34+
*
35+
* Provides a simple method for fetching remote HTTP content in a QgsTask.
36+
* Url redirects are automatically handled.
37+
*
38+
* After constructing a QgsNetworkContentFetcherTask, callers should
39+
* connect to the QgsNetworkContentFetcherTask::fetched signal. They can
40+
* then safely access the network reply() from the connected slot
41+
* without danger of the task being first removed by the QgsTaskManager.
42+
*
43+
* \see QgsNetworkContentFetcher
44+
*
45+
* \since QGIS 3.2
46+
*/
47+
class CORE_EXPORT QgsNetworkContentFetcherTask : public QgsTask
48+
{
49+
Q_OBJECT
50+
51+
public:
52+
53+
/**
54+
* Constructor for a QgsNetworkContentFetcherTask which fetches
55+
* the specified \a url.
56+
*/
57+
QgsNetworkContentFetcherTask( const QUrl &url );
58+
59+
/**
60+
* Constructor for a QgsNetworkContentFetcherTask which fetches
61+
* the specified network \a request.
62+
*/
63+
QgsNetworkContentFetcherTask( const QNetworkRequest &request );
64+
65+
~QgsNetworkContentFetcherTask();
66+
67+
bool run() override;
68+
69+
/**
70+
* Returns the network reply. Ownership is not transferred.
71+
*
72+
* May return nullptr if the request has not yet completed.
73+
*/
74+
QNetworkReply *reply();
75+
76+
signals:
77+
78+
/**
79+
* Emitted when the network content has been fetched, regardless
80+
* of whether the fetch was successful or not.
81+
*
82+
* Users of QgsNetworkContentFetcherTask should connect to this signal,
83+
* and from the associated slot they can then safely access the network reply()
84+
* without danger of the task being first removed by the QgsTaskManager.
85+
*/
86+
void fetched();
87+
88+
private:
89+
90+
QNetworkRequest mRequest;
91+
QgsNetworkContentFetcher *mFetcher = nullptr;
92+
93+
};
94+
95+
#endif //QGSNETWORKCONTENTFETCHERTASK_H

‎tests/src/python/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ ADD_PYTHON_TEST(PyQgsMetadataWidget test_qgsmetadatawidget.py)
114114
ADD_PYTHON_TEST(PyQgsMemoryProvider test_provider_memory.py)
115115
ADD_PYTHON_TEST(PyQgsMultiEditToolButton test_qgsmultiedittoolbutton.py)
116116
ADD_PYTHON_TEST(PyQgsNetworkContentFetcher test_qgsnetworkcontentfetcher.py)
117+
ADD_PYTHON_TEST(PyQgsNetworkContentFetcherTask test_qgsnetworkcontentfetchertask.py)
117118
ADD_PYTHON_TEST(PyQgsNullSymbolRenderer test_qgsnullsymbolrenderer.py)
118119
ADD_PYTHON_TEST(PyQgsNewGeoPackageLayerDialog test_qgsnewgeopackagelayerdialog.py)
119120
ADD_PYTHON_TEST(PyQgsNoApplication test_qgsnoapplication.py)

‎tests/src/python/test_qgsnetworkcontentfetchertask.py

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# -*- coding: utf-8 -*-
2+
"""QGIS Unit tests for QgsNetworkContentFetcherTask
3+
4+
.. note:: This program is free software; you can redistribute it and/or modify
5+
it under the terms of the GNU General Public License as published by
6+
the Free Software Foundation; either version 2 of the License, or
7+
(at your option) any later version.
8+
"""
9+
10+
from builtins import chr
11+
from builtins import str
12+
13+
__author__ = 'Nyall Dawson'
14+
__date__ = '29/03/2018'
15+
__copyright__ = 'Copyright 2018, The QGIS Project'
16+
# This will get replaced with a git SHA1 when you do a git archive
17+
__revision__ = '$Format:%H$'
18+
19+
import qgis # NOQA
20+
21+
import os
22+
from qgis.testing import unittest, start_app
23+
from qgis.core import QgsNetworkContentFetcher, QgsNetworkContentFetcherTask, QgsApplication
24+
from utilities import unitTestDataPath
25+
from qgis.PyQt.QtCore import QUrl
26+
from qgis.PyQt.QtNetwork import QNetworkReply, QNetworkRequest
27+
import socketserver
28+
import threading
29+
import http.server
30+
31+
app = start_app()
32+
33+
34+
class TestQgsNetworkContentFetcherTask(unittest.TestCase):
35+
36+
@classmethod
37+
def setUpClass(cls):
38+
# Bring up a simple HTTP server
39+
os.chdir(unitTestDataPath() + '')
40+
handler = http.server.SimpleHTTPRequestHandler
41+
42+
cls.httpd = socketserver.TCPServer(('localhost', 0), handler)
43+
cls.port = cls.httpd.server_address[1]
44+
45+
cls.httpd_thread = threading.Thread(target=cls.httpd.serve_forever)
46+
cls.httpd_thread.setDaemon(True)
47+
cls.httpd_thread.start()
48+
49+
def __init__(self, methodName):
50+
"""Run once on class initialization."""
51+
unittest.TestCase.__init__(self, methodName)
52+
53+
self.loaded = False
54+
55+
def contentLoaded(self):
56+
self.loaded = True
57+
58+
def testFetchBadUrl(self):
59+
fetcher = QgsNetworkContentFetcherTask(QUrl('http://x'))
60+
self.loaded = False
61+
62+
def check_reply():
63+
r = fetcher.reply()
64+
assert r.error() != QNetworkReply.NoError
65+
self.loaded = True
66+
67+
fetcher.fetched.connect(check_reply)
68+
QgsApplication.taskManager().addTask(fetcher)
69+
while not self.loaded:
70+
app.processEvents()
71+
72+
def testFetchUrlContent(self):
73+
fetcher = QgsNetworkContentFetcherTask(
74+
QUrl('http://localhost:' + str(self.port) + '/qgis_local_server/index.html'))
75+
self.loaded = False
76+
77+
def check_reply():
78+
r = fetcher.reply()
79+
assert r.error() == QNetworkReply.NoError, r.error()
80+
81+
assert b'QGIS' in r.readAll()
82+
self.loaded = True
83+
84+
fetcher.fetched.connect(check_reply)
85+
QgsApplication.taskManager().addTask(fetcher)
86+
while not self.loaded:
87+
app.processEvents()
88+
89+
90+
if __name__ == "__main__":
91+
unittest.main()

0 commit comments

Comments
 (0)
Please sign in to comment.