diff options
Diffstat (limited to 'rbutil/rbutilqt/httpget.cpp')
-rw-r--r-- | rbutil/rbutilqt/httpget.cpp | 199 |
1 files changed, 136 insertions, 63 deletions
diff --git a/rbutil/rbutilqt/httpget.cpp b/rbutil/rbutilqt/httpget.cpp index 40ef62f9c7..fcc2d4163c 100644 --- a/rbutil/rbutilqt/httpget.cpp +++ b/rbutil/rbutilqt/httpget.cpp | |||
@@ -29,13 +29,13 @@ QUrl HttpGet::m_globalProxy; //< global proxy value for new objects | |||
29 | HttpGet::HttpGet(QObject *parent) | 29 | HttpGet::HttpGet(QObject *parent) |
30 | : QObject(parent) | 30 | : QObject(parent) |
31 | { | 31 | { |
32 | m_usecache = false; | ||
33 | outputToBuffer = true; | 32 | outputToBuffer = true; |
34 | cached = false; | 33 | m_cached = false; |
34 | m_noHeaderCheck = false; | ||
35 | getRequest = -1; | 35 | getRequest = -1; |
36 | // if a request is cancelled before a reponse is available return some | 36 | // if a request is cancelled before a reponse is available return some |
37 | // hint about this in the http response instead of nonsense. | 37 | // hint about this in the http response instead of nonsense. |
38 | response = -1; | 38 | m_response = -1; |
39 | 39 | ||
40 | // default to global proxy / cache if not empty. | 40 | // default to global proxy / cache if not empty. |
41 | // proxy is automatically enabled, disable it by setting an empty proxy | 41 | // proxy is automatically enabled, disable it by setting an empty proxy |
@@ -63,7 +63,7 @@ void HttpGet::setCache(QDir d) | |||
63 | m_cachedir = d; | 63 | m_cachedir = d; |
64 | bool result; | 64 | bool result; |
65 | result = initializeCache(d); | 65 | result = initializeCache(d); |
66 | qDebug() << "HttpGet::setCache(QDir)" << d.absolutePath() << result; | 66 | qDebug() << "[HTTP]"<< __func__ << "(QDir)" << d.absolutePath() << result; |
67 | m_usecache = result; | 67 | m_usecache = result; |
68 | } | 68 | } |
69 | 69 | ||
@@ -73,7 +73,7 @@ void HttpGet::setCache(QDir d) | |||
73 | */ | 73 | */ |
74 | void HttpGet::setCache(bool c) | 74 | void HttpGet::setCache(bool c) |
75 | { | 75 | { |
76 | qDebug() << "setCache(bool)" << c; | 76 | qDebug() << "[HTTP]" << __func__ << "(bool) =" << c; |
77 | m_usecache = c; | 77 | m_usecache = c; |
78 | // make sure cache is initialized | 78 | // make sure cache is initialized |
79 | if(c) | 79 | if(c) |
@@ -100,6 +100,9 @@ bool HttpGet::initializeCache(const QDir& d) | |||
100 | } | 100 | } |
101 | 101 | ||
102 | 102 | ||
103 | /** @brief read all downloaded data into a buffer | ||
104 | * @return data | ||
105 | */ | ||
103 | QByteArray HttpGet::readAll() | 106 | QByteArray HttpGet::readAll() |
104 | { | 107 | { |
105 | return dataBuffer; | 108 | return dataBuffer; |
@@ -123,7 +126,7 @@ void HttpGet::httpProgress(int read, int total) | |||
123 | 126 | ||
124 | void HttpGet::setProxy(const QUrl &proxy) | 127 | void HttpGet::setProxy(const QUrl &proxy) |
125 | { | 128 | { |
126 | qDebug() << "HttpGet::setProxy(QUrl)" << proxy.toString(); | 129 | qDebug() << "[HTTP]" << __func__ << "(QUrl)" << proxy.toString(); |
127 | m_proxy = proxy; | 130 | m_proxy = proxy; |
128 | http.setProxy(m_proxy.host(), m_proxy.port(), m_proxy.userName(), m_proxy.password()); | 131 | http.setProxy(m_proxy.host(), m_proxy.port(), m_proxy.userName(), m_proxy.password()); |
129 | } | 132 | } |
@@ -131,7 +134,7 @@ void HttpGet::setProxy(const QUrl &proxy) | |||
131 | 134 | ||
132 | void HttpGet::setProxy(bool enable) | 135 | void HttpGet::setProxy(bool enable) |
133 | { | 136 | { |
134 | qDebug() << "HttpGet::setProxy(bool)" << enable; | 137 | qDebug() << "[HTTP]" << __func__ << "(bool)" << enable; |
135 | if(enable) | 138 | if(enable) |
136 | http.setProxy(m_proxy.host(), m_proxy.port(), m_proxy.userName(), m_proxy.password()); | 139 | http.setProxy(m_proxy.host(), m_proxy.port(), m_proxy.userName(), m_proxy.password()); |
137 | else | 140 | else |
@@ -143,7 +146,7 @@ void HttpGet::setFile(QFile *file) | |||
143 | { | 146 | { |
144 | outputFile = file; | 147 | outputFile = file; |
145 | outputToBuffer = false; | 148 | outputToBuffer = false; |
146 | qDebug() << "HttpGet::setFile" << outputFile->fileName(); | 149 | qDebug() << "[HTTP]" << __func__ << "(QFile*)" << outputFile->fileName(); |
147 | } | 150 | } |
148 | 151 | ||
149 | 152 | ||
@@ -158,40 +161,74 @@ void HttpGet::abort() | |||
158 | bool HttpGet::getFile(const QUrl &url) | 161 | bool HttpGet::getFile(const QUrl &url) |
159 | { | 162 | { |
160 | if (!url.isValid()) { | 163 | if (!url.isValid()) { |
161 | qDebug() << "Error: Invalid URL" << endl; | 164 | qDebug() << "[HTTP] Error: Invalid URL" << endl; |
162 | return false; | 165 | return false; |
163 | } | 166 | } |
164 | 167 | ||
165 | if (url.scheme() != "http") { | 168 | if (url.scheme() != "http") { |
166 | qDebug() << "Error: URL must start with 'http:'" << endl; | 169 | qDebug() << "[HTTP] Error: URL must start with 'http:'" << endl; |
167 | return false; | 170 | return false; |
168 | } | 171 | } |
169 | 172 | ||
170 | if (url.path().isEmpty()) { | 173 | if (url.path().isEmpty()) { |
171 | qDebug() << "Error: URL has no path" << endl; | 174 | qDebug() << "[HTTP] Error: URL has no path" << endl; |
172 | return false; | 175 | return false; |
173 | } | 176 | } |
174 | // if no output file was set write to buffer | 177 | // if no output file was set write to buffer |
175 | if(!outputToBuffer) { | 178 | if(!outputToBuffer) { |
176 | if (!outputFile->open(QIODevice::ReadWrite)) { | 179 | if (!outputFile->open(QIODevice::ReadWrite)) { |
177 | qDebug() << "Error: Cannot open " << qPrintable(outputFile->fileName()) | 180 | qDebug() << "[HTTP] Error: Cannot open " << qPrintable(outputFile->fileName()) |
178 | << " for writing: " << qPrintable(outputFile->errorString()) | 181 | << " for writing: " << qPrintable(outputFile->errorString()) |
179 | << endl; | 182 | << endl; |
180 | return false; | 183 | return false; |
181 | } | 184 | } |
182 | } | 185 | } |
183 | // put hash generation here so it can get reused later | 186 | qDebug() << "[HTTP] downloading" << url.toEncoded(); |
184 | QString hash = QCryptographicHash::hash(url.toEncoded(), QCryptographicHash::Md5).toHex(); | 187 | // create request |
185 | cachefile = m_cachedir.absolutePath() + "/rbutil-cache/" + hash; | 188 | http.setHost(url.host(), url.port(80)); |
189 | // construct query (if any) | ||
190 | QList<QPair<QString, QString> > qitems = url.queryItems(); | ||
191 | if(url.hasQuery()) { | ||
192 | m_query = "?"; | ||
193 | for(int i = 0; i < qitems.size(); i++) | ||
194 | m_query += QUrl::toPercentEncoding(qitems.at(i).first, "/") + "=" | ||
195 | + QUrl::toPercentEncoding(qitems.at(i).second, "/") + "&"; | ||
196 | } | ||
197 | |||
198 | // create hash used for caching | ||
199 | m_hash = QCryptographicHash::hash(url.toEncoded(), QCryptographicHash::Md5).toHex(); | ||
200 | m_path = QString(QUrl::toPercentEncoding(url.path(), "/")); | ||
201 | |||
202 | if(m_noHeaderCheck || !m_usecache) { | ||
203 | getFileFinish(); | ||
204 | } | ||
205 | else { | ||
206 | // request HTTP header | ||
207 | connect(this, SIGNAL(headerFinished()), this, SLOT(getFileFinish())); | ||
208 | headRequest = http.head(m_path + m_query); | ||
209 | } | ||
210 | |||
211 | return true; | ||
212 | } | ||
213 | |||
214 | |||
215 | void HttpGet::getFileFinish() | ||
216 | { | ||
217 | m_cachefile = m_cachedir.absolutePath() + "/rbutil-cache/" + m_hash; | ||
186 | if(m_usecache) { | 218 | if(m_usecache) { |
187 | // check if the file is present in cache | 219 | // check if the file is present in cache |
188 | qDebug() << "[HTTP] cache ENABLED for" << url.toEncoded(); | 220 | qDebug() << "[HTTP] cache ENABLED"; |
189 | if(QFileInfo(cachefile).isReadable() && QFileInfo(cachefile).size() > 0) { | 221 | QFileInfo cachefile = QFileInfo(m_cachefile); |
190 | qDebug() << "[HTTP] cached file found!" << cachefile; | 222 | if(cachefile.isReadable() |
223 | && cachefile.size() > 0 | ||
224 | && cachefile.lastModified() > m_serverTimestamp) { | ||
225 | |||
226 | qDebug() << "[HTTP] cached file found:" << m_cachefile; | ||
227 | |||
191 | getRequest = -1; | 228 | getRequest = -1; |
192 | QFile c(cachefile); | 229 | QFile c(m_cachefile); |
193 | if(!outputToBuffer) { | 230 | if(!outputToBuffer) { |
194 | qDebug() << outputFile->fileName(); | 231 | qDebug() << "[HTTP] copying cache file to output" << outputFile->fileName(); |
195 | c.open(QIODevice::ReadOnly); | 232 | c.open(QIODevice::ReadOnly); |
196 | outputFile->open(QIODevice::ReadWrite); | 233 | outputFile->open(QIODevice::ReadWrite); |
197 | outputFile->write(c.readAll()); | 234 | outputFile->write(c.readAll()); |
@@ -199,46 +236,45 @@ bool HttpGet::getFile(const QUrl &url) | |||
199 | c.close(); | 236 | c.close(); |
200 | } | 237 | } |
201 | else { | 238 | else { |
239 | qDebug() << "[HTTP] reading cache file into buffer"; | ||
202 | c.open(QIODevice::ReadOnly); | 240 | c.open(QIODevice::ReadOnly); |
203 | dataBuffer = c.readAll(); | 241 | dataBuffer = c.readAll(); |
204 | c.close(); | 242 | c.close(); |
205 | } | 243 | } |
206 | response = 200; // fake "200 OK" HTTP response | 244 | m_response = 200; // fake "200 OK" HTTP response |
207 | cached = true; | 245 | m_cached = true; |
208 | httpDone(false); // we're done now. This will emit the correct signal too. | 246 | httpDone(false); // we're done now. Fake http "done" signal. |
209 | return true; | 247 | return; |
248 | } | ||
249 | else { | ||
250 | if(cachefile.isReadable()) | ||
251 | qDebug() << "[HTTP] file in cache timestamp:" << cachefile.lastModified(); | ||
252 | else | ||
253 | qDebug() << "[HTTP] file not in cache."; | ||
254 | qDebug() << "[HTTP] server file timestamp:" << m_serverTimestamp; | ||
255 | qDebug() << "[HTTP] downloading file to" << m_cachefile; | ||
256 | // unlink old cache file | ||
257 | if(cachefile.isReadable()) | ||
258 | QFile(m_cachefile).remove(); | ||
210 | } | 259 | } |
211 | else qDebug() << "[HTTP] file not cached, downloading to" << cachefile; | ||
212 | 260 | ||
213 | } | 261 | } |
214 | else { | 262 | else { |
215 | qDebug() << "[HTTP] cache DISABLED"; | 263 | qDebug() << "[HTTP] cache DISABLED"; |
216 | } | 264 | } |
217 | 265 | ||
218 | http.setHost(url.host(), url.port(80)); | ||
219 | // construct query (if any) | ||
220 | QList<QPair<QString, QString> > qitems = url.queryItems(); | ||
221 | if(url.hasQuery()) { | ||
222 | query = "?"; | ||
223 | for(int i = 0; i < qitems.size(); i++) | ||
224 | query += QUrl::toPercentEncoding(qitems.at(i).first, "/") + "=" | ||
225 | + QUrl::toPercentEncoding(qitems.at(i).second, "/") + "&"; | ||
226 | qDebug() << query; | ||
227 | } | ||
228 | |||
229 | QString path; | ||
230 | path = QString(QUrl::toPercentEncoding(url.path(), "/")); | ||
231 | if(outputToBuffer) { | 266 | if(outputToBuffer) { |
232 | qDebug() << "[HTTP] downloading to buffer:" << url.toString(); | 267 | qDebug() << "[HTTP] downloading to buffer."; |
233 | getRequest = http.get(path + query); | 268 | getRequest = http.get(m_path + m_query); |
234 | } | 269 | } |
235 | else { | 270 | else { |
236 | qDebug() << "[HTTP] downloading to file:" << url.toString() << qPrintable(outputFile->fileName()); | 271 | qDebug() << "[HTTP] downloading to file:" |
237 | getRequest = http.get(path + query, outputFile); | 272 | << qPrintable(outputFile->fileName()); |
273 | getRequest = http.get(m_path + m_query, outputFile); | ||
238 | } | 274 | } |
239 | qDebug() << "[HTTP] request scheduled: GET" << getRequest; | 275 | qDebug() << "[HTTP] GET request scheduled, id:" << getRequest; |
240 | 276 | ||
241 | return true; | 277 | return; |
242 | } | 278 | } |
243 | 279 | ||
244 | 280 | ||
@@ -250,9 +286,9 @@ void HttpGet::httpDone(bool error) | |||
250 | if(!outputToBuffer) | 286 | if(!outputToBuffer) |
251 | outputFile->close(); | 287 | outputFile->close(); |
252 | 288 | ||
253 | if(m_usecache && !cached) { | 289 | if(m_usecache && !m_cached) { |
254 | qDebug() << "[HTTP] creating cache file" << cachefile; | 290 | qDebug() << "[HTTP] creating cache file" << m_cachefile; |
255 | QFile c(cachefile); | 291 | QFile c(m_cachefile); |
256 | c.open(QIODevice::ReadWrite); | 292 | c.open(QIODevice::ReadWrite); |
257 | if(!outputToBuffer) { | 293 | if(!outputToBuffer) { |
258 | outputFile->open(QIODevice::ReadOnly | QIODevice::Truncate); | 294 | outputFile->open(QIODevice::ReadOnly | QIODevice::Truncate); |
@@ -264,23 +300,60 @@ void HttpGet::httpDone(bool error) | |||
264 | 300 | ||
265 | c.close(); | 301 | c.close(); |
266 | } | 302 | } |
267 | emit done(error); | 303 | m_serverTimestamp = QDateTime(); |
304 | // take care of concurring requests. If there is still one running, | ||
305 | // don't emit done(). That request will call this slot again. | ||
306 | if(http.currentId() == 0 && !http.hasPendingRequests()) | ||
307 | emit done(error); | ||
268 | } | 308 | } |
269 | 309 | ||
270 | 310 | ||
271 | void HttpGet::httpFinished(int id, bool error) | 311 | void HttpGet::httpFinished(int id, bool error) |
272 | { | 312 | { |
273 | qDebug() << "HttpGet::httpFinished(int, bool) =" << id << error; | 313 | qDebug() << "[HTTP]" << __func__ << "(int, bool) =" << id << error; |
274 | if(id == getRequest) dataBuffer = http.readAll(); | 314 | if(id == getRequest) { |
275 | qDebug() << "pending:" << http.hasPendingRequests(); | 315 | dataBuffer = http.readAll(); |
276 | //if(!http.hasPendingRequests()) httpDone(error); | 316 | |
277 | emit requestFinished(id, error); | 317 | emit requestFinished(id, error); |
318 | } | ||
319 | qDebug() << "[HTTP] hasPendingRequests =" << http.hasPendingRequests(); | ||
320 | |||
278 | 321 | ||
322 | if(id == headRequest) { | ||
323 | QHttpResponseHeader h = http.lastResponse(); | ||
324 | |||
325 | QString date = h.value("Last-Modified").simplified(); | ||
326 | if(date.isEmpty()) { | ||
327 | m_serverTimestamp = QDateTime(); // no value = invalid | ||
328 | emit headerFinished(); | ||
329 | return; | ||
330 | } | ||
331 | // to successfully parse the date strip weekday and timezone | ||
332 | date.remove(0, date.indexOf(" ") + 1); | ||
333 | if(date.endsWith("GMT")) | ||
334 | date.truncate(date.indexOf(" GMT")); | ||
335 | // distinguish input formats (see RFC1945) | ||
336 | // RFC 850 | ||
337 | if(date.contains("-")) | ||
338 | m_serverTimestamp = QDateTime::fromString(date, "dd-MMM-yy hh:mm:ss"); | ||
339 | // asctime format | ||
340 | else if(date.at(0).isLetter()) | ||
341 | m_serverTimestamp = QDateTime::fromString(date, "MMM d hh:mm:ss yyyy"); | ||
342 | // RFC 822 | ||
343 | else | ||
344 | m_serverTimestamp = QDateTime::fromString(date, "dd MMM yyyy hh:mm:ss"); | ||
345 | qDebug() << "[HTTP] Header Request Date:" << date << ", parsed:" << m_serverTimestamp; | ||
346 | emit headerFinished(); | ||
347 | return; | ||
348 | } | ||
349 | if(id == getRequest) | ||
350 | emit requestFinished(id, error); | ||
279 | } | 351 | } |
280 | 352 | ||
281 | void HttpGet::httpStarted(int id) | 353 | void HttpGet::httpStarted(int id) |
282 | { | 354 | { |
283 | qDebug() << "HttpGet::httpStarted(int) =" << id; | 355 | qDebug() << "[HTTP]" << __func__ << "(int) =" << id; |
356 | qDebug() << "headRequest" << headRequest << "getRequest" << getRequest; | ||
284 | } | 357 | } |
285 | 358 | ||
286 | 359 | ||
@@ -294,9 +367,9 @@ void HttpGet::httpResponseHeader(const QHttpResponseHeader &resp) | |||
294 | { | 367 | { |
295 | // if there is a network error abort all scheduled requests for | 368 | // if there is a network error abort all scheduled requests for |
296 | // this download | 369 | // this download |
297 | response = resp.statusCode(); | 370 | m_response = resp.statusCode(); |
298 | if(response != 200) { | 371 | if(m_response != 200) { |
299 | qDebug() << "http response error:" << response << resp.reasonPhrase(); | 372 | qDebug() << "[HTTP] response error =" << m_response << resp.reasonPhrase(); |
300 | http.abort(); | 373 | http.abort(); |
301 | } | 374 | } |
302 | // 301 -- moved permanently | 375 | // 301 -- moved permanently |
@@ -304,17 +377,17 @@ void HttpGet::httpResponseHeader(const QHttpResponseHeader &resp) | |||
304 | // 303 -- see other | 377 | // 303 -- see other |
305 | // 307 -- moved temporarily | 378 | // 307 -- moved temporarily |
306 | // in all cases, header: location has the correct address so we can follow. | 379 | // in all cases, header: location has the correct address so we can follow. |
307 | if(response == 301 || response == 302 || response == 303 || response == 307) { | 380 | if(m_response == 301 || m_response == 302 || m_response == 303 || m_response == 307) { |
308 | // start new request with new url | 381 | // start new request with new url |
309 | qDebug() << "http response" << response << "- following"; | 382 | qDebug() << "[HTTP] response =" << m_response << "- following"; |
310 | getFile(resp.value("location") + query); | 383 | getFile(resp.value("location") + m_query); |
311 | } | 384 | } |
312 | } | 385 | } |
313 | 386 | ||
314 | 387 | ||
315 | int HttpGet::httpResponse() | 388 | int HttpGet::httpResponse() |
316 | { | 389 | { |
317 | return response; | 390 | return m_response; |
318 | } | 391 | } |
319 | 392 | ||
320 | 393 | ||
@@ -323,7 +396,7 @@ void HttpGet::httpState(int state) | |||
323 | QString s[] = {"Unconnected", "HostLookup", "Connecting", "Sending", | 396 | QString s[] = {"Unconnected", "HostLookup", "Connecting", "Sending", |
324 | "Reading", "Connected", "Closing"}; | 397 | "Reading", "Connected", "Closing"}; |
325 | if(state <= 6) | 398 | if(state <= 6) |
326 | qDebug() << "HttpGet::httpState() = " << s[state]; | 399 | qDebug() << "[HTTP]" << __func__ << "() = " << s[state]; |
327 | else qDebug() << "HttpGet::httpState() = " << state; | 400 | else qDebug() << "[HTTP]" << __func__ << "() = " << state; |
328 | } | 401 | } |
329 | 402 | ||