1/*
2    Copyright (C) 2008 Nokia Corporation and/or its subsidiary(-ies)
3    Copyright (C) 2007 Staikos Computing Services Inc.  <info@staikos.net>
4    Copyright (C) 2008 Holger Hans Peter Freyther
5
6    This library is free software; you can redistribute it and/or
7    modify it under the terms of the GNU Library General Public
8    License as published by the Free Software Foundation; either
9    version 2 of the License, or (at your option) any later version.
10
11    This library is distributed in the hope that it will be useful,
12    but WITHOUT ANY WARRANTY; without even the implied warranty of
13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14    Library General Public License for more details.
15
16    You should have received a copy of the GNU Library General Public License
17    along with this library; see the file COPYING.LIB.  If not, write to
18    the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
19    Boston, MA 02110-1301, USA.
20*/
21#include "config.h"
22#include "QNetworkReplyHandler.h"
23
24#include "HTTPParsers.h"
25#include "MIMETypeRegistry.h"
26#include "ResourceHandle.h"
27#include "ResourceHandleClient.h"
28#include "ResourceHandleInternal.h"
29#include "ResourceResponse.h"
30#include "ResourceRequest.h"
31#include <QDateTime>
32#include <QFile>
33#include <QFileInfo>
34#include <QNetworkReply>
35#include <QNetworkCookie>
36#include <qwebframe.h>
37#include <qwebpage.h>
38
39#include <wtf/text/CString.h>
40
41#include <QDebug>
42#include <QCoreApplication>
43
44// In Qt 4.8, the attribute for sending a request synchronously will be made public,
45// for now, use this hackish solution for setting the internal attribute.
46const QNetworkRequest::Attribute gSynchronousNetworkRequestAttribute = static_cast<QNetworkRequest::Attribute>(QNetworkRequest::HttpPipeliningWasUsedAttribute + 7);
47
48static const int gMaxRedirections = 10;
49
50namespace WebCore {
51
52// Take a deep copy of the FormDataElement
53FormDataIODevice::FormDataIODevice(FormData* data)
54    : m_formElements(data ? data->elements() : Vector<FormDataElement>())
55    , m_currentFile(0)
56    , m_currentDelta(0)
57    , m_fileSize(0)
58    , m_dataSize(0)
59{
60    setOpenMode(FormDataIODevice::ReadOnly);
61
62    if (!m_formElements.isEmpty() && m_formElements[0].m_type == FormDataElement::encodedFile)
63        openFileForCurrentElement();
64    computeSize();
65}
66
67FormDataIODevice::~FormDataIODevice()
68{
69    delete m_currentFile;
70}
71
72qint64 FormDataIODevice::computeSize()
73{
74    for (int i = 0; i < m_formElements.size(); ++i) {
75        const FormDataElement& element = m_formElements[i];
76        if (element.m_type == FormDataElement::data)
77            m_dataSize += element.m_data.size();
78        else {
79            QFileInfo fi(element.m_filename);
80            m_fileSize += fi.size();
81        }
82    }
83    return m_dataSize + m_fileSize;
84}
85
86void FormDataIODevice::moveToNextElement()
87{
88    if (m_currentFile)
89        m_currentFile->close();
90    m_currentDelta = 0;
91
92    m_formElements.remove(0);
93
94    if (m_formElements.isEmpty() || m_formElements[0].m_type == FormDataElement::data)
95        return;
96
97    openFileForCurrentElement();
98}
99
100void FormDataIODevice::openFileForCurrentElement()
101{
102    if (!m_currentFile)
103        m_currentFile = new QFile;
104
105    m_currentFile->setFileName(m_formElements[0].m_filename);
106    m_currentFile->open(QFile::ReadOnly);
107}
108
109// m_formElements[0] is the current item. If the destination buffer is
110// big enough we are going to read from more than one FormDataElement
111qint64 FormDataIODevice::readData(char* destination, qint64 size)
112{
113    if (m_formElements.isEmpty())
114        return -1;
115
116    qint64 copied = 0;
117    while (copied < size && !m_formElements.isEmpty()) {
118        const FormDataElement& element = m_formElements[0];
119        const qint64 available = size-copied;
120
121        if (element.m_type == FormDataElement::data) {
122            const qint64 toCopy = qMin<qint64>(available, element.m_data.size() - m_currentDelta);
123            memcpy(destination+copied, element.m_data.data()+m_currentDelta, toCopy);
124            m_currentDelta += toCopy;
125            copied += toCopy;
126
127            if (m_currentDelta == element.m_data.size())
128                moveToNextElement();
129        } else {
130            const QByteArray data = m_currentFile->read(available);
131            memcpy(destination+copied, data.constData(), data.size());
132            copied += data.size();
133
134            if (m_currentFile->atEnd() || !m_currentFile->isOpen())
135                moveToNextElement();
136        }
137    }
138
139    return copied;
140}
141
142qint64 FormDataIODevice::writeData(const char*, qint64)
143{
144    return -1;
145}
146
147bool FormDataIODevice::isSequential() const
148{
149    return true;
150}
151
152QNetworkReplyHandlerCallQueue::QNetworkReplyHandlerCallQueue(QNetworkReplyHandler* handler, bool deferSignals)
153    : m_replyHandler(handler)
154    , m_locks(0)
155    , m_deferSignals(deferSignals)
156    , m_flushing(false)
157{
158    Q_ASSERT(handler);
159}
160
161void QNetworkReplyHandlerCallQueue::push(EnqueuedCall method)
162{
163    m_enqueuedCalls.append(method);
164    flush();
165}
166
167void QNetworkReplyHandlerCallQueue::lock()
168{
169    ++m_locks;
170}
171
172void QNetworkReplyHandlerCallQueue::unlock()
173{
174    if (!m_locks)
175        return;
176
177    --m_locks;
178    flush();
179}
180
181void QNetworkReplyHandlerCallQueue::setDeferSignals(bool defer)
182{
183    m_deferSignals = defer;
184    flush();
185}
186
187void QNetworkReplyHandlerCallQueue::flush()
188{
189    if (m_flushing)
190        return;
191
192    m_flushing = true;
193
194    while (!m_deferSignals && !m_locks && !m_enqueuedCalls.isEmpty())
195        (m_replyHandler->*(m_enqueuedCalls.takeFirst()))();
196
197    m_flushing = false;
198}
199
200class QueueLocker {
201public:
202    QueueLocker(QNetworkReplyHandlerCallQueue* queue) : m_queue(queue) { m_queue->lock(); }
203    ~QueueLocker() { m_queue->unlock(); }
204private:
205    QNetworkReplyHandlerCallQueue* m_queue;
206};
207
208QNetworkReplyWrapper::QNetworkReplyWrapper(QNetworkReplyHandlerCallQueue* queue, QNetworkReply* reply, bool sniffMIMETypes, QObject* parent)
209    : QObject(parent)
210    , m_reply(reply)
211    , m_queue(queue)
212    , m_responseContainsData(false)
213    , m_sniffer(0)
214    , m_sniffMIMETypes(sniffMIMETypes)
215{
216    Q_ASSERT(m_reply);
217
218    connect(m_reply, SIGNAL(readyRead()), this, SLOT(receiveMetaData()));
219    connect(m_reply, SIGNAL(finished()), this, SLOT(receiveMetaData()));
220}
221
222QNetworkReplyWrapper::~QNetworkReplyWrapper()
223{
224    if (m_reply)
225        m_reply->deleteLater();
226    m_queue->clear();
227}
228
229QNetworkReply* QNetworkReplyWrapper::release()
230{
231    if (!m_reply)
232        return 0;
233
234    resetConnections();
235    QNetworkReply* reply = m_reply;
236    m_reply = 0;
237    m_sniffer = 0;
238
239    reply->setParent(0);
240    return reply;
241}
242
243void QNetworkReplyWrapper::resetConnections()
244{
245    if (m_reply)
246        m_reply->disconnect(this);
247    QCoreApplication::removePostedEvents(this, QEvent::MetaCall);
248}
249
250void QNetworkReplyWrapper::receiveMetaData()
251{
252    // This slot is only used to receive the first signal from the QNetworkReply object.
253    resetConnections();
254
255
256    WTF::String contentType = m_reply->header(QNetworkRequest::ContentTypeHeader).toString();
257    m_encoding = extractCharsetFromMediaType(contentType);
258    m_advertisedMIMEType = extractMIMETypeFromMediaType(contentType);
259
260    m_redirectionTargetUrl = m_reply->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
261    if (m_redirectionTargetUrl.isValid()) {
262        QueueLocker lock(m_queue);
263        m_queue->push(&QNetworkReplyHandler::sendResponseIfNeeded);
264        m_queue->push(&QNetworkReplyHandler::finish);
265        return;
266    }
267
268    if (!m_sniffMIMETypes) {
269        emitMetaDataChanged();
270        return;
271    }
272
273    bool isSupportedImageType = MIMETypeRegistry::isSupportedImageMIMEType(m_advertisedMIMEType);
274
275    Q_ASSERT(!m_sniffer);
276
277    m_sniffer = new QtMIMETypeSniffer(m_reply, m_advertisedMIMEType, isSupportedImageType);
278
279    if (m_sniffer->isFinished()) {
280        receiveSniffedMIMEType();
281        return;
282    }
283
284    connect(m_sniffer.get(), SIGNAL(finished()), this, SLOT(receiveSniffedMIMEType()));
285}
286
287void QNetworkReplyWrapper::receiveSniffedMIMEType()
288{
289    Q_ASSERT(m_sniffer);
290
291    m_sniffedMIMEType = m_sniffer->mimeType();
292    m_sniffer = 0;
293
294    emitMetaDataChanged();
295}
296
297void QNetworkReplyWrapper::emitMetaDataChanged()
298{
299    QueueLocker lock(m_queue);
300    m_queue->push(&QNetworkReplyHandler::sendResponseIfNeeded);
301
302    if (m_reply->bytesAvailable()) {
303        m_responseContainsData = true;
304        m_queue->push(&QNetworkReplyHandler::forwardData);
305    }
306
307    if (m_reply->isFinished()) {
308        m_queue->push(&QNetworkReplyHandler::finish);
309        return;
310    }
311
312    // If not finished, connect to the slots that will be used from this point on.
313    connect(m_reply, SIGNAL(readyRead()), this, SLOT(didReceiveReadyRead()));
314    connect(m_reply, SIGNAL(finished()), this, SLOT(didReceiveFinished()));
315}
316
317void QNetworkReplyWrapper::didReceiveReadyRead()
318{
319    if (m_reply->bytesAvailable())
320        m_responseContainsData = true;
321    m_queue->push(&QNetworkReplyHandler::forwardData);
322}
323
324void QNetworkReplyWrapper::didReceiveFinished()
325{
326    // Disconnecting will make sure that nothing will happen after emitting the finished signal.
327    resetConnections();
328    m_queue->push(&QNetworkReplyHandler::finish);
329}
330
331String QNetworkReplyHandler::httpMethod() const
332{
333    switch (m_method) {
334    case QNetworkAccessManager::GetOperation:
335        return "GET";
336    case QNetworkAccessManager::HeadOperation:
337        return "HEAD";
338    case QNetworkAccessManager::PostOperation:
339        return "POST";
340    case QNetworkAccessManager::PutOperation:
341        return "PUT";
342    case QNetworkAccessManager::DeleteOperation:
343        return "DELETE";
344    case QNetworkAccessManager::CustomOperation:
345        return m_resourceHandle->firstRequest().httpMethod();
346    default:
347        ASSERT_NOT_REACHED();
348        return "GET";
349    }
350}
351
352QNetworkReplyHandler::QNetworkReplyHandler(ResourceHandle* handle, LoadType loadType, bool deferred)
353    : QObject(0)
354    , m_replyWrapper(0)
355    , m_resourceHandle(handle)
356    , m_loadType(loadType)
357    , m_redirectionTries(gMaxRedirections)
358    , m_queue(this, deferred)
359{
360    const ResourceRequest &r = m_resourceHandle->firstRequest();
361
362    if (r.httpMethod() == "GET")
363        m_method = QNetworkAccessManager::GetOperation;
364    else if (r.httpMethod() == "HEAD")
365        m_method = QNetworkAccessManager::HeadOperation;
366    else if (r.httpMethod() == "POST")
367        m_method = QNetworkAccessManager::PostOperation;
368    else if (r.httpMethod() == "PUT")
369        m_method = QNetworkAccessManager::PutOperation;
370    else if (r.httpMethod() == "DELETE")
371        m_method = QNetworkAccessManager::DeleteOperation;
372    else
373        m_method = QNetworkAccessManager::CustomOperation;
374
375    QObject* originatingObject = 0;
376    if (m_resourceHandle->getInternal()->m_context)
377        originatingObject = m_resourceHandle->getInternal()->m_context->originatingObject();
378
379    m_request = r.toNetworkRequest(originatingObject);
380
381    m_queue.push(&QNetworkReplyHandler::start);
382}
383
384void QNetworkReplyHandler::abort()
385{
386    m_resourceHandle = 0;
387    if (QNetworkReply* reply = release()) {
388        reply->abort();
389        reply->deleteLater();
390    }
391    deleteLater();
392}
393
394QNetworkReply* QNetworkReplyHandler::release()
395{
396    if (!m_replyWrapper)
397        return 0;
398
399    QNetworkReply* reply = m_replyWrapper->release();
400    m_replyWrapper = 0;
401    return reply;
402}
403
404static bool shouldIgnoreHttpError(QNetworkReply* reply, bool receivedData)
405{
406    int httpStatusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
407
408    if (httpStatusCode == 401 || httpStatusCode == 407)
409        return true;
410
411    if (receivedData && (httpStatusCode >= 400 && httpStatusCode < 600))
412        return true;
413
414    return false;
415}
416
417void QNetworkReplyHandler::finish()
418{
419    ASSERT(m_replyWrapper && m_replyWrapper->reply() && !wasAborted());
420
421    ResourceHandleClient* client = m_resourceHandle->client();
422    if (!client) {
423        m_replyWrapper = 0;
424        return;
425    }
426
427    if (m_replyWrapper->wasRedirected()) {
428        m_replyWrapper = 0;
429        m_queue.push(&QNetworkReplyHandler::start);
430        return;
431    }
432
433    if (!m_replyWrapper->reply()->error() || shouldIgnoreHttpError(m_replyWrapper->reply(), m_replyWrapper->responseContainsData()))
434        client->didFinishLoading(m_resourceHandle, 0);
435    else {
436        QUrl url = m_replyWrapper->reply()->url();
437        int httpStatusCode = m_replyWrapper->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
438
439        if (httpStatusCode) {
440            ResourceError error("HTTP", httpStatusCode, url.toString(), m_replyWrapper->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toString());
441            client->didFail(m_resourceHandle, error);
442        } else {
443            ResourceError error("QtNetwork", m_replyWrapper->reply()->error(), url.toString(), m_replyWrapper->reply()->errorString());
444            client->didFail(m_resourceHandle, error);
445        }
446    }
447
448    m_replyWrapper = 0;
449}
450
451void QNetworkReplyHandler::sendResponseIfNeeded()
452{
453    ASSERT(m_replyWrapper && m_replyWrapper->reply() && !wasAborted());
454
455    if (m_replyWrapper->reply()->error() && m_replyWrapper->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).isNull())
456        return;
457
458    ResourceHandleClient* client = m_resourceHandle->client();
459    if (!client)
460        return;
461
462    WTF::String mimeType = m_replyWrapper->mimeType();
463
464    if (mimeType.isEmpty()) {
465        // let's try to guess from the extension
466        mimeType = MIMETypeRegistry::getMIMETypeForPath(m_replyWrapper->reply()->url().path());
467    }
468
469    KURL url(m_replyWrapper->reply()->url());
470    ResourceResponse response(url, mimeType.lower(),
471                              m_replyWrapper->reply()->header(QNetworkRequest::ContentLengthHeader).toLongLong(),
472                              m_replyWrapper->encoding(), String());
473
474    if (url.isLocalFile()) {
475        client->didReceiveResponse(m_resourceHandle, response);
476        return;
477    }
478
479    // The status code is equal to 0 for protocols not in the HTTP family.
480    int statusCode = m_replyWrapper->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
481
482    if (url.protocolInHTTPFamily()) {
483        String suggestedFilename = filenameFromHTTPContentDisposition(QString::fromLatin1(m_replyWrapper->reply()->rawHeader("Content-Disposition")));
484
485        if (!suggestedFilename.isEmpty())
486            response.setSuggestedFilename(suggestedFilename);
487        else
488            response.setSuggestedFilename(url.lastPathComponent());
489
490        response.setHTTPStatusCode(statusCode);
491        response.setHTTPStatusText(m_replyWrapper->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toByteArray().constData());
492
493        // Add remaining headers.
494        foreach (const QNetworkReply::RawHeaderPair& pair, m_replyWrapper->reply()->rawHeaderPairs())
495            response.setHTTPHeaderField(QString::fromLatin1(pair.first), QString::fromLatin1(pair.second));
496    }
497
498    QUrl redirection = m_replyWrapper->reply()->attribute(QNetworkRequest::RedirectionTargetAttribute).toUrl();
499    if (redirection.isValid()) {
500        redirect(response, redirection);
501        return;
502    }
503
504    client->didReceiveResponse(m_resourceHandle, response);
505}
506
507void QNetworkReplyHandler::redirect(ResourceResponse& response, const QUrl& redirection)
508{
509    QUrl newUrl = m_replyWrapper->reply()->url().resolved(redirection);
510
511    ResourceHandleClient* client = m_resourceHandle->client();
512    ASSERT(client);
513
514    int statusCode = m_replyWrapper->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
515
516    m_redirectionTries--;
517    if (!m_redirectionTries) {
518        ResourceError error(newUrl.host(), 400 /*bad request*/,
519                            newUrl.toString(),
520                            QCoreApplication::translate("QWebPage", "Redirection limit reached"));
521        client->didFail(m_resourceHandle, error);
522        m_replyWrapper = 0;
523        return;
524    }
525
526    //  Status Code 301 (Moved Permanently), 302 (Moved Temporarily), 303 (See Other):
527    //    - If original request is POST convert to GET and redirect automatically
528    //  Status Code 307 (Temporary Redirect) and all other redirect status codes:
529    //    - Use the HTTP method from the previous request
530    if ((statusCode >= 301 && statusCode <= 303) && m_resourceHandle->firstRequest().httpMethod() == "POST")
531        m_method = QNetworkAccessManager::GetOperation;
532
533    ResourceRequest newRequest = m_resourceHandle->firstRequest();
534    newRequest.setHTTPMethod(httpMethod());
535    newRequest.setURL(newUrl);
536
537    // Should not set Referer after a redirect from a secure resource to non-secure one.
538    if (!newRequest.url().protocolIs("https") && protocolIs(newRequest.httpReferrer(), "https"))
539        newRequest.clearHTTPReferrer();
540
541    client->willSendRequest(m_resourceHandle, newRequest, response);
542    if (wasAborted()) // Network error cancelled the request.
543        return;
544
545    QObject* originatingObject = 0;
546    if (m_resourceHandle->getInternal()->m_context)
547        originatingObject = m_resourceHandle->getInternal()->m_context->originatingObject();
548
549    m_request = newRequest.toNetworkRequest(originatingObject);
550}
551
552void QNetworkReplyHandler::forwardData()
553{
554    ASSERT(m_replyWrapper && m_replyWrapper->reply() && !wasAborted() && !m_replyWrapper->wasRedirected());
555
556    QByteArray data = m_replyWrapper->reply()->read(m_replyWrapper->reply()->bytesAvailable());
557
558    ResourceHandleClient* client = m_resourceHandle->client();
559    if (!client)
560        return;
561
562    // FIXME: https://bugs.webkit.org/show_bug.cgi?id=19793
563    // -1 means we do not provide any data about transfer size to inspector so it would use
564    // Content-Length headers or content size to show transfer size.
565    if (!data.isEmpty())
566        client->didReceiveData(m_resourceHandle, data.constData(), data.length(), -1);
567}
568
569void QNetworkReplyHandler::uploadProgress(qint64 bytesSent, qint64 bytesTotal)
570{
571    if (wasAborted())
572        return;
573
574    ResourceHandleClient* client = m_resourceHandle->client();
575    if (!client)
576        return;
577
578    client->didSendData(m_resourceHandle, bytesSent, bytesTotal);
579}
580
581QNetworkReply* QNetworkReplyHandler::sendNetworkRequest(QNetworkAccessManager* manager, const ResourceRequest& request)
582{
583    if (m_loadType == SynchronousLoad)
584        m_request.setAttribute(gSynchronousNetworkRequestAttribute, true);
585
586    if (!manager)
587        return 0;
588
589    const QUrl url = m_request.url();
590    const QString scheme = url.scheme();
591    // Post requests on files and data don't really make sense, but for
592    // fast/forms/form-post-urlencoded.html and for fast/forms/button-state-restore.html
593    // we still need to retrieve the file/data, which means we map it to a Get instead.
594    if (m_method == QNetworkAccessManager::PostOperation
595        && (!url.toLocalFile().isEmpty() || url.scheme() == QLatin1String("data")))
596        m_method = QNetworkAccessManager::GetOperation;
597
598    switch (m_method) {
599        case QNetworkAccessManager::GetOperation:
600            return manager->get(m_request);
601        case QNetworkAccessManager::PostOperation: {
602            FormDataIODevice* postDevice = new FormDataIODevice(request.httpBody());
603            // We may be uploading files so prevent QNR from buffering data
604            m_request.setHeader(QNetworkRequest::ContentLengthHeader, postDevice->getFormDataSize());
605            m_request.setAttribute(QNetworkRequest::DoNotBufferUploadDataAttribute, QVariant(true));
606            QNetworkReply* result = manager->post(m_request, postDevice);
607            postDevice->setParent(result);
608            return result;
609        }
610        case QNetworkAccessManager::HeadOperation:
611            return manager->head(m_request);
612        case QNetworkAccessManager::PutOperation: {
613            FormDataIODevice* putDevice = new FormDataIODevice(request.httpBody());
614            // We may be uploading files so prevent QNR from buffering data
615            m_request.setHeader(QNetworkRequest::ContentLengthHeader, putDevice->getFormDataSize());
616            m_request.setAttribute(QNetworkRequest::DoNotBufferUploadDataAttribute, QVariant(true));
617            QNetworkReply* result = manager->put(m_request, putDevice);
618            putDevice->setParent(result);
619            return result;
620        }
621        case QNetworkAccessManager::DeleteOperation: {
622            return manager->deleteResource(m_request);
623        }
624        case QNetworkAccessManager::CustomOperation:
625            return manager->sendCustomRequest(m_request, m_resourceHandle->firstRequest().httpMethod().latin1().data());
626        case QNetworkAccessManager::UnknownOperation:
627            ASSERT_NOT_REACHED();
628            return 0;
629    }
630    return 0;
631}
632
633void QNetworkReplyHandler::start()
634{
635    ResourceHandleInternal* d = m_resourceHandle->getInternal();
636    if (!d || !d->m_context)
637        return;
638
639    QNetworkReply* reply = sendNetworkRequest(d->m_context->networkAccessManager(), d->m_firstRequest);
640    if (!reply)
641        return;
642
643    m_replyWrapper = new QNetworkReplyWrapper(&m_queue, reply, m_resourceHandle->shouldContentSniff() && d->m_context->mimeSniffingEnabled(), this);
644
645    if (m_loadType == SynchronousLoad && m_replyWrapper->reply()->isFinished()) {
646        m_replyWrapper->synchronousLoad();
647        // If supported, a synchronous request will be finished at this point, no need to hook up the signals.
648        return;
649    }
650
651    if (m_resourceHandle->firstRequest().reportUploadProgress())
652        connect(m_replyWrapper->reply(), SIGNAL(uploadProgress(qint64, qint64)), this, SLOT(uploadProgress(qint64, qint64)));
653}
654
655}
656
657#include "moc_QNetworkReplyHandler.cpp"
658