WiP: Enable pipelining

This does not work yet, it still did not pipeline.
Need to analyze https://code.woboq.org/qt5/qtbase/src/network/access/qhttpnetworkconnection.cpp.html#_ZN29QHttpNetworkConnectionPrivate12fillPipelineEP15QAbstractSocket more

For #4454

src/libsync/connectionvalidator.cpp

@@ -205,6 +205,7 @@ void ConnectionValidator::slotAuthFailed(QNetworkReply *reply)
 void ConnectionValidator::slotAuthSuccess()
 {
     _errors.clear();
+    qDebug() << !_isCheckingServerAndAuth;
     if (!_isCheckingServerAndAuth) {
         reportResult(Connected);
         return;

src/libsync/creds/httpcredentials.cpp

@@ -16,6 +16,7 @@
 #include <QMutex>
 #include <QDebug>
 #include <QNetworkReply>
+#include <QAuthenticator>
 #include <QSettings>
 
 #include <keychain.h>
@@ -41,22 +42,6 @@ const char certifPasswdC[] = "certificatePasswd";
 const char authenticationFailedC[] = "owncloud-authentication-failed";
 } // ns
 
-class HttpCredentialsAccessManager : public AccessManager {
-public:
-    HttpCredentialsAccessManager(const HttpCredentials *cred, QObject* parent = 0)
-        : AccessManager(parent), _cred(cred) {}
-protected:
-    QNetworkReply *createRequest(Operation op, const QNetworkRequest &request, QIODevice *outgoingData) Q_DECL_OVERRIDE {
-        QByteArray credHash = QByteArray(_cred->user().toUtf8()+":"+_cred->password().toUtf8()).toBase64();
-        QNetworkRequest req(request);
-        req.setRawHeader(QByteArray("Authorization"), QByteArray("Basic ") + credHash);
-        //qDebug() << "Request for " << req.url() << "with authorization" << QByteArray::fromBase64(credHash);
-        return AccessManager::createRequest(op, req, outgoingData);
-    }
-private:
-    const HttpCredentials *_cred;
-};
-
 HttpCredentials::HttpCredentials()
     : _ready(false)
 {
@@ -121,7 +106,7 @@ void HttpCredentials::setAccount(Account* account)
 
 QNetworkAccessManager* HttpCredentials::getQNAM() const
 {
-    AccessManager* qnam = new HttpCredentialsAccessManager(this);
+    AccessManager* qnam = new AccessManager;
 
     connect( qnam, SIGNAL(authenticationRequired(QNetworkReply*, QAuthenticator*)),
              this, SLOT(slotAuthentication(QNetworkReply*,QAuthenticator*)));
@@ -175,10 +160,7 @@ void HttpCredentials::fetchFromKeychain()
 }
 bool HttpCredentials::stillValid(QNetworkReply *reply)
 {
-    return ((reply->error() != QNetworkReply::AuthenticationRequiredError)
-            // returned if user or password is incorrect
-            && (reply->error() != QNetworkReply::OperationCanceledError
-                || !reply->property(authenticationFailedC).toBool()));
+    return (reply->error() != QNetworkReply::AuthenticationRequiredError);
 }
 
 void HttpCredentials::slotReadJobDone(QKeychain::Job *job)
@@ -306,12 +288,18 @@ void HttpCredentials::slotWriteJobDone(QKeychain::Job *job)
 
 void HttpCredentials::slotAuthentication(QNetworkReply* reply, QAuthenticator* authenticator)
 {
-    Q_UNUSED(authenticator)
-    // Because of issue #4326, we need to set the login and password manually at every requests
-    // Thus, if we reach this signal, those credentials were invalid and we terminate.
-    qDebug() << "Stop request: Authentication failed for " << reply->url().toString();
+    if (reply->property(authenticationFailedC).toBool()) {
+        qDebug() << "Authentication failed for " << reply->url().toString();
+        return;
+    }
+
+    // QNAM sends the user and password in latin-1,  but the server expects UTF-8.
+    // So send mojibake on purpose
+    authenticator->setUser(QString::fromLatin1(user().toUtf8()));
+    authenticator->setPassword(QString::fromLatin1(password().toUtf8()));
+
+    // Set a property so we don't send the same password twice
     reply->setProperty(authenticationFailedC, true);
-    reply->close();
 }
 
 } // namespace OCC

src/libsync/owncloudpropagator.cpp

@@ -412,6 +412,11 @@ void OwncloudPropagator::start(const SyncFileItemVector& items)
 
     qDebug() << "Using QNAM/HTTP parallel code path";
 
+    // We want to clearAccessCache() here because we want QNAM to switch to sending the cookie
+    // only and not the things in the authenticator. Note this is only possible from
+    // server version [...] since only that one has DAV auth possible with cookies only
+    _account->resetNetworkAccessManager();
+
     QTimer::singleShot(0, this, SLOT(scheduleNextJob()));
 }
 
@@ -516,12 +521,26 @@ QString OwncloudPropagator::getFilePath(const QString& tmp_file_name) const
     return _localDir + tmp_file_name;
 }
 
+int OwncloudPropagator::_pipelinedRequests = 0;
 void OwncloudPropagator::scheduleNextJob()
 {
-    if (this->_activeJobs < maximumActiveJob()) {
+     int maximumJobAdjustment = 0; // should be -1 if low bandwidth or +2 if high bandwidth was detected
+
+     // If there is a GET in
+     bool currentJobsCouldPipelineMoreIn = false;
+
+     if (this->_activeJobs < maximumActiveJob() + maximumJobAdjustment) {
         if (_rootJob->scheduleNextJob()) {
+            // Something was scheduled, maybe we can schedule one more?
             QTimer::singleShot(0, this, SLOT(scheduleNextJob()));
         }
+    } else if (_pipelinedRequests > 0 && this->_activeJobs < maximumActiveJob()*3) {
+        // FIXME check if we don't exceed the amount of sockets?
+        // Try to pipeline in some more
+        qDebug() << "Pipeline more!";
+
+        _rootJob->scheduleNextJob();
+        _rootJob->scheduleNextJob();
     }
 }
 

src/libsync/owncloudpropagator.h

@@ -267,6 +267,8 @@ public:
     SyncJournalDb * const _journal;
     bool _finishedEmited; // used to ensure that finished is only emitted once
 
+    static int _pipelinedRequests;
+
 
 public:
     OwncloudPropagator(AccountPtr account, const QString &localDir,
@@ -281,7 +283,7 @@ public:
             , _activeJobs(0)
             , _anotherSyncNeeded(false)
             , _account(account)
-    { }
+    { _pipelinedRequests = 0; }
 
     ~OwncloudPropagator();
 

src/libsync/propagatedownload.cpp

@@ -67,6 +67,7 @@ GETFileJob::GETFileJob(AccountPtr account, const QString& path, QFile *device,
 , _bandwidthLimited(false), _bandwidthChoked(false), _bandwidthQuota(0), _bandwidthManager(0)
 , _hasEmittedFinishedSignal(false), _lastModified()
 {
+    _allowPipelining = false;
 }
 
 GETFileJob::GETFileJob(AccountPtr account, const QUrl& url, QFile *device,
@@ -79,6 +80,8 @@ GETFileJob::GETFileJob(AccountPtr account, const QUrl& url, QFile *device,
 , _bandwidthLimited(false), _bandwidthChoked(false), _bandwidthQuota(0), _bandwidthManager(0)
 , _hasEmittedFinishedSignal(false), _lastModified()
 {
+    _allowPipelining = false;
+
 }
 
 
@@ -93,6 +96,10 @@ void GETFileJob::start() {
     for(QMap<QByteArray, QByteArray>::const_iterator it = _headers.begin(); it != _headers.end(); ++it) {
         req.setRawHeader(it.key(), it.value());
     }
+    if (_allowPipelining) {
+        req.setAttribute(QNetworkRequest::HttpPipeliningAllowedAttribute, true);
+        qDebug() << path() << "pipelining ok!";
+    }
 
     if (_directDownloadUrl.isEmpty()) {
         setReply(davRequest("GET", path(), req));
@@ -129,6 +136,10 @@ void GETFileJob::slotMetaDataChanged()
 
     int httpStatus = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
 
+    if (reply()->attribute(QNetworkRequest::HttpPipeliningWasUsedAttribute).toBool()) {
+        qDebug() << reply()->url() << "MARKUS DID USE PIPELINING!";
+    }
+
     // If the status code isn't 2xx, don't write the reply body to the file.
     // For any error: handle it when the job is finished, not here.
     if (httpStatus / 100 != 2) {
@@ -396,6 +407,13 @@ void PropagateDownloadFileQNAM::start()
         _job = new GETFileJob(_propagator->account(),
                             _propagator->_remoteFolder + _item->_file,
                             &_tmpFile, headers, expectedEtagForResume, _resumeStart);
+
+        if (/*false &&*/ _item->_size < 1024*256) {
+            _job->setPipeliningAllowed(true);
+            OwncloudPropagator::_pipelinedRequests++;
+            qDebug() << "MARKUS incrementing" << OwncloudPropagator::_pipelinedRequests;
+
+        }
     } else {
         // We were provided a direct URL, use that one
         qDebug() << Q_FUNC_INFO << "directDownloadUrl given for " << _item->_file << _item->_directDownloadUrl;
@@ -437,6 +455,12 @@ void PropagateDownloadFileQNAM::slotGetFinished()
     GETFileJob *job = qobject_cast<GETFileJob *>(sender());
     Q_ASSERT(job);
 
+    if (job->reply()->request().attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool()) {
+        OwncloudPropagator::_pipelinedRequests--;
+        qDebug() << "MARKUS decreasing" << OwncloudPropagator::_pipelinedRequests;
+
+    }
+
     qDebug() << Q_FUNC_INFO << job->reply()->request().url() << "FINISHED WITH STATUS"
              << job->reply()->error()
              << (job->reply()->error() == QNetworkReply::NoError ? QLatin1String("") : job->reply()->errorString())

src/libsync/propagatedownload.h

@@ -93,6 +93,14 @@ public:
     quint64 resumeStart() { return _resumeStart; }
     time_t lastModified() { return _lastModified; }
 
+    bool _allowPipelining;
+    void setPipeliningAllowed(bool b) {
+        _allowPipelining = b;
+    }
+    bool isPipeliningAllowed() {
+        return _allowPipelining;
+    }
+
 
 signals:
     void finishedSignal();