Comparar commits
1 Commits
| Autor | SHA1 | Data | |
|---|---|---|---|
| 79011916d7 |
@@ -58,6 +58,7 @@ struct CmdOptions {
|
||||
QString password;
|
||||
QString proxy;
|
||||
bool silent;
|
||||
bool bundleRequests;
|
||||
bool trustSSL;
|
||||
bool useNetrc;
|
||||
bool interactive;
|
||||
@@ -157,6 +158,7 @@ void help()
|
||||
std::cout << std::endl;
|
||||
std::cout << "Options:" << std::endl;
|
||||
std::cout << " --silent, -s Don't be so verbose" << std::endl;
|
||||
std::cout << " --bundle-requests, -b Bundle Requests if supported" << std::endl;
|
||||
std::cout << " --httpproxy [proxy] Specify a http proxy to use." << std::endl;
|
||||
std::cout << " Proxy is http://server:port" << std::endl;
|
||||
std::cout << " --trust Trust the SSL certification." << std::endl;
|
||||
@@ -224,6 +226,8 @@ void parseOptions( const QStringList& app_args, CmdOptions *options )
|
||||
options->silent = true;
|
||||
} else if( option == "--trust") {
|
||||
options->trustSSL = true;
|
||||
} else if( option == "-b" || option == "--bundle-requests") {
|
||||
options->bundleRequests = true;
|
||||
} else if( option == "-n") {
|
||||
options->useNetrc = true;
|
||||
} else if( option == "-h") {
|
||||
@@ -292,6 +296,7 @@ int main(int argc, char **argv) {
|
||||
|
||||
CmdOptions options;
|
||||
options.silent = false;
|
||||
options.bundleRequests = false;
|
||||
options.trustSSL = false;
|
||||
options.useNetrc = false;
|
||||
options.interactive = true;
|
||||
@@ -416,6 +421,7 @@ int main(int argc, char **argv) {
|
||||
|
||||
loop.exec();
|
||||
#endif
|
||||
account->setBundleRequestsIfCapable(options.bundleRequests);
|
||||
|
||||
// much lower age than the default since this utility is usually made to be run right after a change in the tests
|
||||
SyncEngine::minimumFileAgeForUpload = 0;
|
||||
|
||||
@@ -128,6 +128,11 @@ QNetworkReply *AbstractNetworkJob::davRequest(const QByteArray &verb, const QUrl
|
||||
return addTimer(_account->davRequest(verb, url, req, data));
|
||||
}
|
||||
|
||||
QNetworkReply *AbstractNetworkJob::multipartRequest(const QString &relPath, QNetworkRequest req, QHttpMultiPart *multiPart)
|
||||
{
|
||||
return addTimer(_account->multipartRequest(relPath, req, multiPart));
|
||||
}
|
||||
|
||||
QNetworkReply* AbstractNetworkJob::getRequest(const QString &relPath)
|
||||
{
|
||||
return addTimer(_account->getRequest(relPath));
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
#include "accountfwd.h"
|
||||
|
||||
class QUrl;
|
||||
class QHttpMultiPart;
|
||||
|
||||
namespace OCC {
|
||||
|
||||
@@ -78,6 +79,7 @@ protected:
|
||||
QNetworkReply* headRequest(const QString &relPath);
|
||||
QNetworkReply* headRequest(const QUrl &url);
|
||||
QNetworkReply* deleteRequest(const QUrl &url);
|
||||
QNetworkReply* multipartRequest(const QString &relPath, QNetworkRequest req, QHttpMultiPart *multiPart);
|
||||
|
||||
int maxRedirects() const { return 10; }
|
||||
virtual bool finished() = 0;
|
||||
|
||||
@@ -39,6 +39,8 @@ Account::Account(QObject *parent)
|
||||
: QObject(parent)
|
||||
, _capabilities(QVariantMap())
|
||||
, _davPath( Theme::instance()->webDavPath() )
|
||||
, _wasMigrated(false)
|
||||
, _bundleRequests(true)
|
||||
{
|
||||
qRegisterMetaType<AccountPtr>("AccountPtr");
|
||||
}
|
||||
@@ -75,6 +77,13 @@ AccountPtr Account::sharedFromThis()
|
||||
return _sharedThis.toStrongRef();
|
||||
}
|
||||
|
||||
QString Account::davFilesPath() const
|
||||
{
|
||||
//TODO DO NOT HARCODE PATH, GET IT FROM THE SERVER!!!!
|
||||
QString dfp("remote.php/dav/files/");
|
||||
dfp.append(_credentials->user());
|
||||
return dfp;
|
||||
}
|
||||
|
||||
QString Account::displayName() const
|
||||
{
|
||||
@@ -228,6 +237,20 @@ QNetworkReply *Account::davRequest(const QByteArray &verb, const QUrl &url, QNet
|
||||
return _am->sendCustomRequest(req, verb, data);
|
||||
}
|
||||
|
||||
QNetworkReply *Account::multipartRequest(const QString &relPath, QNetworkRequest req, QHttpMultiPart *multiPart)
|
||||
{
|
||||
return multipartRequest(Utility::concatUrlPath(url(), relPath), req, multiPart);
|
||||
}
|
||||
|
||||
QNetworkReply *Account::multipartRequest(const QUrl &url, QNetworkRequest req, QHttpMultiPart *multiPart)
|
||||
{
|
||||
req.setUrl(url);
|
||||
#if QT_VERSION > QT_VERSION_CHECK(4, 8, 4)
|
||||
req.setSslConfiguration(this->getOrCreateSslConfig());
|
||||
#endif
|
||||
return _am->post(req, multiPart);
|
||||
}
|
||||
|
||||
void Account::setCertificate(const QByteArray certficate, const QString privateKey)
|
||||
{
|
||||
_pemCertificate=certficate;
|
||||
@@ -466,6 +489,13 @@ void Account::setNonShib(bool nonShib)
|
||||
}
|
||||
}
|
||||
|
||||
void Account::setBundleRequestsIfCapable(bool bundleRequests){
|
||||
_bundleRequests = bundleRequests;
|
||||
}
|
||||
|
||||
bool Account::bundledRequestsEnabled()
|
||||
{
|
||||
return (_bundleRequests && _capabilities.bundledRequest()) ? true : false;
|
||||
}
|
||||
|
||||
} // namespace OCC
|
||||
|
||||
+25
-9
@@ -33,6 +33,7 @@ class QSettings;
|
||||
class QNetworkReply;
|
||||
class QUrl;
|
||||
class QNetworkAccessManager;
|
||||
class QHttpMultiPart;
|
||||
|
||||
namespace OCC {
|
||||
|
||||
@@ -63,6 +64,26 @@ public:
|
||||
class OWNCLOUDSYNC_EXPORT Account : public QObject {
|
||||
Q_OBJECT
|
||||
public:
|
||||
/**
|
||||
* @brief The possibly themed dav path for the account. It has
|
||||
* a trailing slash.
|
||||
* @returns the (themeable) dav path for the account.
|
||||
*/
|
||||
QString davPath() const;
|
||||
|
||||
/**
|
||||
* @brief The possibly themed files dav path for the account. It has
|
||||
* a trailing slash.
|
||||
* @returns the (themeable) files dav path for the account.
|
||||
*/
|
||||
QString davFilesPath() const;
|
||||
void setDavPath(const QString&s) { _davPath = s; }
|
||||
void setNonShib(bool nonShib);
|
||||
|
||||
/// Functions for bundle support
|
||||
void setBundleRequestsIfCapable(bool bundleRequests);
|
||||
bool bundledRequestsEnabled();
|
||||
|
||||
static AccountPtr create();
|
||||
~Account();
|
||||
|
||||
@@ -78,15 +99,6 @@ public:
|
||||
void setUrl(const QUrl &url);
|
||||
QUrl url() const { return _url; }
|
||||
|
||||
/**
|
||||
* @brief The possibly themed dav path for the account. It has
|
||||
* a trailing slash.
|
||||
* @returns the (themeable) dav path for the account.
|
||||
*/
|
||||
QString davPath() const;
|
||||
void setDavPath(const QString&s) { _davPath = s; }
|
||||
void setNonShib(bool nonShib);
|
||||
|
||||
/** Returns webdav entry URL, based on url() */
|
||||
QUrl davUrl() const;
|
||||
|
||||
@@ -103,6 +115,8 @@ public:
|
||||
QNetworkReply* deleteRequest( const QUrl &url);
|
||||
QNetworkReply* davRequest(const QByteArray &verb, const QString &relPath, QNetworkRequest req, QIODevice *data = 0);
|
||||
QNetworkReply* davRequest(const QByteArray &verb, const QUrl &url, QNetworkRequest req, QIODevice *data = 0);
|
||||
QNetworkReply* multipartRequest(const QString &relPath, QNetworkRequest req, QHttpMultiPart *multiPart);
|
||||
QNetworkReply* multipartRequest(const QUrl &url, QNetworkRequest req, QHttpMultiPart *multiPart);
|
||||
|
||||
|
||||
/** The ssl configuration during the first connection */
|
||||
@@ -216,6 +230,8 @@ private:
|
||||
QByteArray _pemCertificate;
|
||||
QString _pemPrivateKey;
|
||||
QString _davPath; // defaults to value from theme, might be overwritten in brandings
|
||||
bool _wasMigrated;
|
||||
bool _bundleRequests;
|
||||
friend class AccountManager;
|
||||
};
|
||||
|
||||
|
||||
@@ -46,6 +46,12 @@ bool Capabilities::sharePublicLink() const
|
||||
}
|
||||
}
|
||||
|
||||
bool Capabilities::bundledRequest() const
|
||||
{
|
||||
bool bundle = _capabilities["dav"].toMap()["bundlerequest"].toBool();
|
||||
return bundle;
|
||||
}
|
||||
|
||||
bool Capabilities::sharePublicLinkAllowUpload() const
|
||||
{
|
||||
return _capabilities["files_sharing"].toMap()["public"].toMap()["upload"].toBool();
|
||||
|
||||
@@ -39,6 +39,7 @@ public:
|
||||
bool sharePublicLinkEnforceExpireDate() const;
|
||||
int sharePublicLinkExpireDateDays() const;
|
||||
bool shareResharing() const;
|
||||
bool bundledRequest() const;
|
||||
|
||||
/// returns true if the capabilities report notifications
|
||||
bool notificationsAvailable() const;
|
||||
|
||||
@@ -303,6 +303,13 @@ void OwncloudPropagator::start(const SyncFileItemVector& items)
|
||||
directories.push(qMakePair(QString(), _rootJob.data()));
|
||||
QVector<PropagatorJob*> directoriesToRemove;
|
||||
QString removedDirectory;
|
||||
|
||||
PropagateBundle *bundledUploadRequestsJob = new PropagateBundle(this);
|
||||
quint64 chunkSize = OwncloudPropagator::chunkSize();
|
||||
|
||||
// TODO: here we should also check somehow if bundle is not broken for specific sync - bug recovery to standard PUTs
|
||||
bool bundledRequestsEnabled = _account->bundledRequestsEnabled();
|
||||
|
||||
foreach(const SyncFileItemPtr &item, items) {
|
||||
|
||||
if (!removedDirectory.isEmpty() && item->_file.startsWith(removedDirectory)) {
|
||||
@@ -379,6 +386,12 @@ void OwncloudPropagator::start(const SyncFileItemVector& items)
|
||||
currentDirJob->append(dir);
|
||||
}
|
||||
directories.push(qMakePair(item->destination() + "/" , dir));
|
||||
} else if (bundledRequestsEnabled
|
||||
&& (item->_instruction == CSYNC_INSTRUCTION_NEW)
|
||||
&& (item->_direction == SyncFileItem::Up)
|
||||
&& (item->_size < chunkSize)) {
|
||||
//this will create list of bundle files to sync for that bundlejob
|
||||
bundledUploadRequestsJob->append(item);
|
||||
} else if (PropagateItemJob* current = createJob(item)) {
|
||||
if (item->_instruction == CSYNC_INSTRUCTION_TYPE_CHANGE) {
|
||||
// will delete directories, so defer execution
|
||||
@@ -394,6 +407,21 @@ void OwncloudPropagator::start(const SyncFileItemVector& items)
|
||||
_rootJob->append(it);
|
||||
}
|
||||
|
||||
// Root job is DirectoryJob with FullParallelizm, so we need to halt bundle till all other jobs are finished.
|
||||
// If there are no running jobs in root folder it means we can start with sending the bundle
|
||||
// If there are running jobs, we halt and wait for finish of the other job to trigger sending
|
||||
if (bundledRequestsEnabled && !bundledUploadRequestsJob->empty()) {
|
||||
bundledUploadRequestsJob->_item->_direction = SyncFileItem::Direction::Up;
|
||||
bundledUploadRequestsJob->_item->_type = SyncFileItem::Type::RequestsContainer;
|
||||
bundledUploadRequestsJob->_item->_instruction = CSYNC_INSTRUCTION_NEW;
|
||||
bundledUploadRequestsJob->_item->_size = bundledUploadRequestsJob->syncItemsSize();
|
||||
bundledUploadRequestsJob->_item->_originalFile = tr("%1 file(s)").arg(bundledUploadRequestsJob->syncItemsNumber());
|
||||
_rootJob->append(bundledUploadRequestsJob);
|
||||
}
|
||||
else{
|
||||
delete bundledUploadRequestsJob;
|
||||
}
|
||||
|
||||
connect(_rootJob.data(), SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)),
|
||||
this, SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));
|
||||
connect(_rootJob.data(), SIGNAL(progress(const SyncFileItem &,quint64)), this, SIGNAL(progress(const SyncFileItem &,quint64)));
|
||||
@@ -624,6 +652,7 @@ bool PropagateDirectory::scheduleNextJob()
|
||||
Q_ASSERT(_subJobs.at(i)->_state == Running);
|
||||
|
||||
auto paral = _subJobs.at(i)->parallelism();
|
||||
|
||||
if (paral == WaitForFinished) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -324,6 +324,10 @@ public:
|
||||
emitFinished(SyncFileItem::NormalError);
|
||||
}
|
||||
|
||||
int runningNowAtRootJob() const {
|
||||
return _rootJob->_runningNow;
|
||||
}
|
||||
|
||||
// timeout in seconds
|
||||
static int httpTimeout();
|
||||
|
||||
@@ -383,6 +387,7 @@ private:
|
||||
friend class PropagateLocalMkdir;
|
||||
friend class PropagateLocalRename;
|
||||
friend class PropagateRemoteMove;
|
||||
friend class PropagateBundle;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
||||
@@ -28,11 +28,13 @@ QString Progress::asResultString( const SyncFileItem& item)
|
||||
case CSYNC_INSTRUCTION_SYNC:
|
||||
case CSYNC_INSTRUCTION_NEW:
|
||||
case CSYNC_INSTRUCTION_TYPE_CHANGE:
|
||||
if (item._direction != SyncFileItem::Up) {
|
||||
return QCoreApplication::translate( "progress", "Downloaded");
|
||||
} else {
|
||||
return QCoreApplication::translate( "progress", "Uploaded");
|
||||
}
|
||||
if (item._type != SyncFileItem::Type::RequestsContainer) {
|
||||
if (item._direction != SyncFileItem::Up) {
|
||||
return QCoreApplication::translate( "progress", "Downloaded");
|
||||
} else
|
||||
return QCoreApplication::translate( "progress", "Uploaded");
|
||||
}
|
||||
return QCoreApplication::translate( "progress", "Processed Bundle");
|
||||
case CSYNC_INSTRUCTION_CONFLICT:
|
||||
return QCoreApplication::translate( "progress", "Server version downloaded, copied changed local file into conflict file");
|
||||
case CSYNC_INSTRUCTION_REMOVE:
|
||||
|
||||
@@ -28,10 +28,12 @@
|
||||
|
||||
#include <json.h>
|
||||
#include <QNetworkAccessManager>
|
||||
#include <QHttpMultiPart>
|
||||
#include <QFileInfo>
|
||||
#include <QDir>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <QXmlStreamReader>
|
||||
|
||||
#if QT_VERSION < QT_VERSION_CHECK(5, 4, 2)
|
||||
namespace {
|
||||
@@ -894,5 +896,601 @@ void PropagateUploadFile::abortWithError(SyncFileItem::Status status, const QStr
|
||||
done(status, error);
|
||||
}
|
||||
|
||||
void PropagateBundle::start()
|
||||
{
|
||||
if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!_itemsToChecksum.isEmpty()) {
|
||||
//this will add checksums and remove itself from activeJobList after is completed
|
||||
_preparingBundle = true;
|
||||
_propagator->_activeJobList.append(this);
|
||||
return slotComputeTransmissionChecksum();
|
||||
}
|
||||
|
||||
//this can generate itemDone(), dont add to activeJobList
|
||||
startBundle();
|
||||
}
|
||||
|
||||
void PropagateBundle::slotComputeTransmissionChecksum()
|
||||
{
|
||||
if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) {
|
||||
return;
|
||||
}
|
||||
|
||||
SyncFileItemPtr item = _itemsToChecksum.first();
|
||||
const QString filePath = _propagator->getFilePath(item->_file);
|
||||
|
||||
// remember the modtime before checksumming to be able to detect a file
|
||||
// change during the checksum calculation
|
||||
item->_modtime = FileSystem::getModTime(filePath);
|
||||
|
||||
QByteArray checksumType = contentChecksumType();
|
||||
|
||||
// Maybe the discovery already computed the checksum?
|
||||
if (item->_contentChecksumType == checksumType
|
||||
&& !item->_contentChecksum.isEmpty()) {
|
||||
// Reuse the content checksum as the transmission checksum if possible
|
||||
const auto supportedTransmissionChecksums =
|
||||
_propagator->account()->capabilities().supportedChecksumTypes();
|
||||
if (supportedTransmissionChecksums.contains(checksumType)) {
|
||||
slotStartUpload(checksumType, item->_contentChecksum);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Compute the transmission checksum.
|
||||
auto computeChecksum = new ComputeChecksum(this);
|
||||
if (uploadChecksumEnabled()) {
|
||||
computeChecksum->setChecksumType(_propagator->account()->capabilities().uploadChecksumType());
|
||||
} else {
|
||||
computeChecksum->setChecksumType(QByteArray());
|
||||
}
|
||||
|
||||
connect(computeChecksum, SIGNAL(done(QByteArray,QByteArray)),
|
||||
SLOT(slotStartUpload(QByteArray,QByteArray)));
|
||||
computeChecksum->start(filePath);
|
||||
}
|
||||
|
||||
void PropagateBundle::slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum)
|
||||
{
|
||||
SyncFileItemPtr item = _itemsToChecksum.takeFirst();
|
||||
|
||||
item->_contentChecksum = transmissionChecksum;
|
||||
item->_contentChecksumType = transmissionChecksumType;
|
||||
|
||||
// add this item to sync list, as it now has checksum computed
|
||||
_itemsToSync.append(item);
|
||||
|
||||
_currentBundleSize += item->_size;
|
||||
_currentRequestsNumber++;
|
||||
|
||||
// Remove ourselfs from the list of active job, before any posible call to itemDone()
|
||||
_propagator->_activeJobList.removeOne(this);
|
||||
|
||||
// check if we have anything to add to our bundle
|
||||
if (!_itemsToChecksum.empty()){
|
||||
SyncFileItemPtr nextItem = _itemsToChecksum.first();
|
||||
|
||||
// if next item will exceed bundle size, send the bundle now
|
||||
// otherwise, compute next checksum
|
||||
if (((_currentBundleSize + nextItem->_size) >= chunkSize())
|
||||
|| (_currentRequestsNumber >= checkBundledRequestsLimits())){
|
||||
_currentBundleSize = 0;
|
||||
_currentRequestsNumber = 0;
|
||||
startBundle();
|
||||
} else {
|
||||
start();
|
||||
}
|
||||
} else {
|
||||
//_itemsToChecksum is already empty, send what is left in _itemsToSync.
|
||||
startBundle();
|
||||
}
|
||||
}
|
||||
|
||||
void PropagateBundle::startBundle()
|
||||
{
|
||||
if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: use QHttpMultiPartIODevice as as Upload Device
|
||||
QHttpMultiPart *multiPart = new QHttpMultiPart(QHttpMultiPart::MixedType);
|
||||
|
||||
QVector<SyncFileItemPtr> syncItems;
|
||||
while (!_itemsToSync.isEmpty()){
|
||||
SyncFileItemPtr item = _itemsToSync.takeFirst();
|
||||
const QString fullFilePath = _propagator->getFilePath(item->_file);
|
||||
|
||||
if (!FileSystem::fileExists(fullFilePath)) {
|
||||
itemDone(item, SyncFileItem::SoftError, tr("File Removed"));
|
||||
continue;
|
||||
}
|
||||
|
||||
time_t prevModtime = item->_modtime; // the _item value was set in PropagateUploadFileQNAM::start()
|
||||
// but a potential checksum calculation could have taken some time during which the file could
|
||||
// have been changed again, so better check again here.
|
||||
|
||||
item->_modtime = FileSystem::getModTime(fullFilePath);
|
||||
if( prevModtime != item->_modtime ) {
|
||||
_propagator->_anotherSyncNeeded = true;
|
||||
itemDone(item, SyncFileItem::SoftError, tr("Local file changed during syncing. It will be resumed."));
|
||||
continue;
|
||||
}
|
||||
|
||||
quint64 fileSize = FileSystem::getSize(fullFilePath);
|
||||
item->_size = fileSize;
|
||||
|
||||
// But skip the file if the mtime is too close to 'now'!
|
||||
// That usually indicates a file that is still being changed
|
||||
// or not yet fully copied to the destination.
|
||||
if (fileIsStillChanging(*item)) {
|
||||
_propagator->_anotherSyncNeeded = true;
|
||||
itemDone(item, SyncFileItem::SoftError, tr("Local file changed during sync."));
|
||||
continue;
|
||||
}
|
||||
|
||||
//TODO use Upload Device to support bandwith limitation on the client
|
||||
QFile *file = new QFile(fullFilePath);
|
||||
if (! file->open(QIODevice::ReadOnly)) {
|
||||
qDebug() << "ERR: Could not prepare upload device: " << file->errorString();
|
||||
|
||||
// If the file is currently locked, we want to retry the sync
|
||||
// when it becomes available again.
|
||||
if (FileSystem::isFileLocked(fullFilePath)) {
|
||||
emit _propagator->seenLockedFile(fullFilePath);
|
||||
}
|
||||
|
||||
// Soft error because this is likely caused by the user modifying his files while syncing
|
||||
itemDone(item, SyncFileItem::SoftError, tr("ERR: Could not prepare upload device"));
|
||||
} else {
|
||||
QHttpPart bundleContent;
|
||||
bundleContent.setHeader(QNetworkRequest::ContentLengthHeader, QVariant(fileSize));
|
||||
|
||||
//TODO: dont use strings here!! Must change to variable in propagatorjobs.h!
|
||||
bundleContent.setRawHeader("X-OC-Method", QByteArray("PUT"));
|
||||
bundleContent.setRawHeader("X-OC-Path", getRemotePath(item->_file));
|
||||
bundleContent.setRawHeader("X-OC-Mtime", QByteArray::number(qint64(item->_modtime)));
|
||||
|
||||
if (!item->_contentChecksumType.isEmpty()) {
|
||||
bundleContent.setRawHeader(checkSumHeaderC,
|
||||
makeChecksumHeader(item->_contentChecksumType, item->_contentChecksum));
|
||||
}
|
||||
if(item->_file.contains(".sys.admin#recall#")) {
|
||||
// This is a file recall triggered by the admin. Note: the
|
||||
// recall list file created by the admin and downloaded by the
|
||||
// client (.sys.admin#recall#) also falls into this category
|
||||
// (albeit users are not supposed to mess up with it)
|
||||
|
||||
// We use a special tag header so that the server may decide to store this file away in some admin stage area
|
||||
// And not directly in the user's area (which would trigger redownloads etc).
|
||||
bundleContent.setRawHeader("OC-Tag", QByteArray(".sys.admin#recall#"));
|
||||
}
|
||||
|
||||
// File has to be written to multipart body immedietely, because we dont want 1000> open files till the reply comes.
|
||||
bundleContent.setBody(file->readAll());
|
||||
file->close();
|
||||
multiPart->append(bundleContent);
|
||||
syncItems.append(item);
|
||||
}
|
||||
delete file;
|
||||
}
|
||||
|
||||
if (!syncItems.isEmpty())
|
||||
{
|
||||
_propagator->_activeJobList.append(this);
|
||||
|
||||
const QString userPath = _propagator->account()->davFilesPath();
|
||||
|
||||
// job takes ownership of device via a QScopedPointer. Job deletes itself when finishing
|
||||
MultipartJob* job = new MultipartJob(_propagator->account(), userPath, multiPart, syncItems);
|
||||
job->_duration.start();
|
||||
|
||||
_jobs.append(job);
|
||||
connect(job, SIGNAL(finishedSignal()), this, SLOT(slotMultipartFinished()));
|
||||
connect(job, SIGNAL(destroyed(QObject*)), this, SLOT(slotJobDestroyed(QObject*)));
|
||||
job->start();
|
||||
_preparingBundle = false;
|
||||
|
||||
// check if there are any items to be bundled and if you can run another parallel job
|
||||
if (!_itemsToChecksum.empty() && (_propagator->_activeJobList.count() < _propagator->maximumActiveJob())) {
|
||||
start();
|
||||
}
|
||||
} else {
|
||||
delete multiPart;
|
||||
}
|
||||
}
|
||||
|
||||
//TODO this function is copied over from owncloudpropagator.cpp (not in owncloudpropagator.h)
|
||||
/** Updates, creates or removes a blacklist entry for the given item.
|
||||
*
|
||||
* Returns whether the file is in the blacklist now.
|
||||
*
|
||||
*/
|
||||
static bool blacklistCheck(SyncJournalDb* journal, const SyncFileItem& item)
|
||||
{
|
||||
SyncJournalErrorBlacklistRecord oldEntry = journal->errorBlacklistEntry(item._file);
|
||||
SyncJournalErrorBlacklistRecord newEntry = SyncJournalErrorBlacklistRecord::update(oldEntry, item);
|
||||
|
||||
if (newEntry.isValid()) {
|
||||
journal->updateErrorBlacklistEntry(newEntry);
|
||||
} else if (oldEntry.isValid()) {
|
||||
journal->wipeErrorBlacklistEntry(item._file);
|
||||
}
|
||||
|
||||
return newEntry.isValid();
|
||||
}
|
||||
|
||||
void PropagateBundle::itemDone(SyncFileItemPtr item, SyncFileItem::Status status, const QString &errorString)
|
||||
{
|
||||
if (item->_isRestoration) {
|
||||
if( status == SyncFileItem::Success || status == SyncFileItem::Conflict) {
|
||||
status = SyncFileItem::Restoration;
|
||||
} else {
|
||||
item->_errorString += tr("; Restoration Failed: %1").arg(errorString);
|
||||
}
|
||||
} else {
|
||||
if( item->_errorString.isEmpty() ) {
|
||||
item->_errorString = errorString;
|
||||
}
|
||||
}
|
||||
|
||||
if( _propagator->_abortRequested.fetchAndAddRelaxed(0) &&
|
||||
(status == SyncFileItem::NormalError || status == SyncFileItem::FatalError)) {
|
||||
// an abort request is ongoing. Change the status to Soft-Error
|
||||
status = SyncFileItem::SoftError;
|
||||
}
|
||||
|
||||
switch( status ) {
|
||||
case SyncFileItem::SoftError:
|
||||
case SyncFileItem::FatalError:
|
||||
// do not blacklist in case of soft error or fatal error.
|
||||
break;
|
||||
case SyncFileItem::NormalError:
|
||||
if (blacklistCheck(_propagator->_journal, *item) && item->_hasBlacklistEntry) {
|
||||
// do not error if the item was, and continues to be, blacklisted
|
||||
status = SyncFileItem::FileIgnored;
|
||||
item->_errorString.prepend(tr("Continue blacklisting:") + " ");
|
||||
}
|
||||
break;
|
||||
case SyncFileItem::Success:
|
||||
case SyncFileItem::Restoration:
|
||||
if( item->_hasBlacklistEntry ) {
|
||||
// wipe blacklist entry.
|
||||
_propagator->_journal->wipeErrorBlacklistEntry(item->_file);
|
||||
// remove a blacklist entry in case the file was moved.
|
||||
if( item->_originalFile != item->_file ) {
|
||||
_propagator->_journal->wipeErrorBlacklistEntry(item->_originalFile);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case SyncFileItem::Conflict:
|
||||
case SyncFileItem::FileIgnored:
|
||||
case SyncFileItem::NoStatus:
|
||||
// nothing
|
||||
break;
|
||||
}
|
||||
|
||||
item->_status = status;
|
||||
|
||||
emit itemCompleted(*item, *this);
|
||||
}
|
||||
|
||||
QByteArray PropagateBundle::getRemotePath(QString filePath){
|
||||
QString remotePath(_propagator->_remoteFolder+filePath);
|
||||
return remotePath.toStdString().c_str();
|
||||
}
|
||||
|
||||
void PropagateBundle::append(const SyncFileItemPtr &bundledFile){
|
||||
_size += bundledFile->_size;
|
||||
_itemsToChecksum.append(bundledFile);
|
||||
}
|
||||
|
||||
bool PropagateBundle::empty(){
|
||||
return _itemsToChecksum.empty();
|
||||
}
|
||||
|
||||
void PropagateBundle::slotMultipartFinished()
|
||||
{
|
||||
MultipartJob *job = qobject_cast<MultipartJob *>(sender());
|
||||
Q_ASSERT(job);
|
||||
slotJobDestroyed(job); // remove it from the _jobs list
|
||||
|
||||
qDebug() << Q_FUNC_INFO << job->reply()->request().url() << "FINISHED WITH STATUS"
|
||||
<< job->reply()->error()
|
||||
<< (job->reply()->error() == QNetworkReply::NoError ? QLatin1String("") : job->reply()->errorString())
|
||||
<< job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute)
|
||||
<< job->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute);
|
||||
|
||||
_propagator->_activeJobList.removeOne(this);
|
||||
|
||||
QNetworkReply::NetworkError err = job->reply()->error();
|
||||
|
||||
#if QT_VERSION < QT_VERSION_CHECK(5, 4, 2)
|
||||
if (job->reply()->error() == QNetworkReply::OperationCanceledError && job->reply()->property(owncloudShouldSoftCancelPropertyName).isValid()) {
|
||||
// Abort the job and try again later.
|
||||
// This works around a bug in QNAM wich might reuse a non-empty buffer for the next request.
|
||||
qDebug() << "Forcing job abort on HTTP connection reset with Qt < 5.4.2.";
|
||||
_propagator->_anotherSyncNeeded = true;
|
||||
abortWithError(SyncFileItem::SoftError, tr("Forcing job abort on HTTP connection reset with Qt < 5.4.2."));
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
if (err != QNetworkReply::NoError) {
|
||||
_item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
|
||||
|
||||
QByteArray replyContent = job->reply()->readAll();
|
||||
qDebug() << replyContent; // display the XML error in the debug
|
||||
QString errorString = errorMessage(job->errorString(), replyContent);
|
||||
|
||||
if (job->reply()->hasRawHeader("OC-ErrorString")) {
|
||||
errorString = job->reply()->rawHeader("OC-ErrorString");
|
||||
}
|
||||
|
||||
abortWithError(SyncFileItem::SoftError, errorString);
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse DAV response
|
||||
QXmlStreamReader reader(job->reply());
|
||||
reader.addExtraNamespaceDeclaration(QXmlStreamNamespaceDeclaration("d", "DAV:"));
|
||||
|
||||
QString currentHref;
|
||||
QString currentOcPath;
|
||||
QString expectedPath(_propagator->account()->davFilesPath());
|
||||
QMap<QString, QString> itemProperties;
|
||||
QMap<QString, QMap<QString, QString> > responseObjectsProperties;
|
||||
bool insidePropstat = false;
|
||||
bool insideProp = false;
|
||||
bool insideError = false;
|
||||
|
||||
while (!reader.atEnd()) {
|
||||
QXmlStreamReader::TokenType type = reader.readNext();
|
||||
QString name = reader.name().toString();
|
||||
// Start elements with DAV:
|
||||
if (type == QXmlStreamReader::StartElement && reader.namespaceUri() == QLatin1String("DAV:")) {
|
||||
if (name == QLatin1String("href")) {
|
||||
// We don't use URL encoding in our request URL (which is the expected path) (QNAM will do it for us)
|
||||
// but the result will have URL encoding..
|
||||
QString hrefString = QString::fromUtf8(QByteArray::fromPercentEncoding(reader.readElementText().toUtf8()));
|
||||
if (!hrefString.endsWith(expectedPath)) {
|
||||
qDebug() << "Invalid href" << hrefString << "expected ending with" << expectedPath;
|
||||
}
|
||||
currentHref = hrefString;
|
||||
} else if (name == QLatin1String("response")) {
|
||||
continue;
|
||||
} else if (name == QLatin1String("propstat")) {
|
||||
insidePropstat = true;
|
||||
} else if (name == QLatin1String("status") && insidePropstat) {
|
||||
QString httpStatus = reader.readElementText();
|
||||
itemProperties.insert(name, httpStatus);
|
||||
} else if (name == QLatin1String("prop")) {
|
||||
insideProp = true;
|
||||
continue;
|
||||
} else if (name == QLatin1String("multistatus")) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (type == QXmlStreamReader::StartElement && insidePropstat && insideProp) {
|
||||
if (name == QLatin1String("oc-path")){
|
||||
currentOcPath = reader.readElementText(QXmlStreamReader::SkipChildElements);
|
||||
} else if (name == QLatin1String("error")){
|
||||
insideError = true;
|
||||
} else if (name == QLatin1String("exception") && insideError){
|
||||
itemProperties.insert(name, reader.readElementText(QXmlStreamReader::SkipChildElements));
|
||||
} else if (name == QLatin1String("message") && insideError){
|
||||
itemProperties.insert(name, reader.readElementText(QXmlStreamReader::SkipChildElements));
|
||||
} else{
|
||||
itemProperties.insert(name, reader.readElementText(QXmlStreamReader::SkipChildElements));
|
||||
}
|
||||
}
|
||||
|
||||
// End elements with DAV:
|
||||
if (type == QXmlStreamReader::EndElement) {
|
||||
if (reader.namespaceUri() == QLatin1String("DAV:")) {
|
||||
if (reader.name() == "response") {
|
||||
currentHref.clear();
|
||||
} else if (reader.name() == "propstat") {
|
||||
insidePropstat = false;
|
||||
if (!currentOcPath.isEmpty()){
|
||||
responseObjectsProperties.insert(currentOcPath, QMap<QString,QString>(itemProperties));
|
||||
}
|
||||
currentOcPath.clear();
|
||||
itemProperties.clear();
|
||||
} else if (reader.name() == "prop") {
|
||||
insideProp = false;
|
||||
} else if (reader.name() == "error") {
|
||||
insideError = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (reader.hasError()) {
|
||||
// XML Parser error? Whatever had been emitted before will come as directoryListingIterated
|
||||
qDebug() << "ERROR" << reader.errorString();
|
||||
abortWithError(SyncFileItem::SoftError, tr("Cannot parse multistatus response!"));
|
||||
return;
|
||||
}
|
||||
|
||||
QVector<SyncFileItemPtr> syncItems = job->syncItems();
|
||||
|
||||
//get the total duration, and try to estimate sync time for single item in the bundle
|
||||
quint64 avgItemDuration = job->_duration.elapsed()/syncItems.count();
|
||||
|
||||
foreach(SyncFileItemPtr item, syncItems) {
|
||||
item->_requestDuration = avgItemDuration;
|
||||
item->_responseTimeStamp = job->responseTimestamp();
|
||||
slotItemFinished(item, responseObjectsProperties);
|
||||
}
|
||||
|
||||
// performance logging
|
||||
qDebug() << "*==* duration BUNDLE UPLOAD" << syncItems.count() << "items synced"
|
||||
<< "average duration" << avgItemDuration;
|
||||
// The job might stay alive for the whole sync, release this tiny bit of memory.
|
||||
|
||||
if (_jobs.empty() && _itemsToChecksum.isEmpty()){
|
||||
done(SyncFileItem::Success);
|
||||
} else if (!_preparingBundle){
|
||||
start();
|
||||
}
|
||||
}
|
||||
|
||||
void PropagateBundle::slotItemFinished(const SyncFileItemPtr &item, QMap<QString, QMap<QString, QString> > &responseObjectsProperties)
|
||||
{
|
||||
QString itemFilePath(getRemotePath(item->_file));
|
||||
QMap<QString, QString> fileProperties = responseObjectsProperties.value(itemFilePath);
|
||||
|
||||
item->_httpErrorCode = getHttpStatusCode(fileProperties.value("status"));
|
||||
|
||||
if (200 == item->_httpErrorCode){
|
||||
// Check if the file still exists
|
||||
const QString fullFilePath(_propagator->getFilePath(item->_file));
|
||||
if( !FileSystem::fileExists(fullFilePath) ) {
|
||||
_propagator->_anotherSyncNeeded = true;
|
||||
}
|
||||
|
||||
if (! FileSystem::verifyFileUnchanged(fullFilePath, item->_size, item->_modtime)) {
|
||||
_propagator->_anotherSyncNeeded = true;
|
||||
}
|
||||
|
||||
//OC-FileID section
|
||||
if (fileProperties.contains("oc-fileid")){
|
||||
QString fid = fileProperties.value("oc-fileid");
|
||||
if( !item->_fileId.isEmpty() && item->_fileId != fid ) {
|
||||
qDebug() << "WARN: File ID changed!" << item->_fileId << fid;
|
||||
}
|
||||
item->_fileId =fid.toStdString().c_str();
|
||||
}
|
||||
|
||||
//OC-ETag section
|
||||
QByteArray ocEtag = parseEtag(fileProperties.value("oc-etag").toStdString().c_str());
|
||||
QByteArray etag = parseEtag(fileProperties.value("etag").toStdString().c_str());
|
||||
item->_etag = ocEtag.isEmpty() ? etag : ocEtag;
|
||||
if (ocEtag.length() > 0 && ocEtag != etag) {
|
||||
qDebug() << "Quite peculiar, we have an etag != OC-Etag [no problem!]" << etag << ocEtag;
|
||||
}
|
||||
|
||||
if (fileProperties.value("x-oc-mtime") != "accepted"){
|
||||
// X-OC-MTime is supported since owncloud 5.0. But not when chunking.
|
||||
// Normally Owncloud 6 always puts X-OC-MTime
|
||||
qWarning() << "Server does not support X-OC-MTime" << fileProperties.value("x-oc-mtime");
|
||||
// Well, the mtime was not set
|
||||
itemDone(item, SyncFileItem::SoftError, "Server does not support X-OC-MTime");
|
||||
return;
|
||||
}
|
||||
} else{
|
||||
//We do not check for problems with shared folder since it is only CREATE
|
||||
//TODO: if the file will support update, ensure to override checkForProblemsWithShared()
|
||||
|
||||
qDebug() << Q_FUNC_INFO << item->_file << "ERROR WITH STATUS"
|
||||
<< fileProperties.value("status")
|
||||
<< fileProperties.value("exception")
|
||||
<< fileProperties.value("message");
|
||||
|
||||
if (412 == _item->_httpErrorCode) {
|
||||
// Precondition Failed: Maybe the bad etag is in the database, we need to clear the
|
||||
// parent folder etag so we won't read from DB next sync.
|
||||
_propagator->_journal->avoidReadFromDbOnNextSync(item->_file);
|
||||
_propagator->_anotherSyncNeeded = true;
|
||||
}
|
||||
|
||||
item->_errorString = fileProperties.value("message");
|
||||
item->_status = classifyError(QNetworkReply::ContentOperationNotPermittedError, item->_httpErrorCode,
|
||||
&_propagator->_anotherSyncNeeded);
|
||||
itemDone(item, item->_status, item->_errorString);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!_propagator->_journal->setFileRecord(SyncJournalFileRecord(*item, _propagator->getFilePath(item->_file)))) {
|
||||
itemDone(item, SyncFileItem::FatalError, tr("Error writing metadata to the database"));
|
||||
return;
|
||||
}
|
||||
|
||||
// Remove from the progress database:
|
||||
_propagator->_journal->setUploadInfo(item->_file, SyncJournalDb::UploadInfo());
|
||||
_propagator->_journal->commit("upload file start");
|
||||
itemDone(item, SyncFileItem::Success);
|
||||
}
|
||||
|
||||
int PropagateBundle::getHttpStatusCode(const QString &status){
|
||||
//if cannot read code, it means that server is not supported and raise 500 Internal Server Error
|
||||
int code = 500;
|
||||
QStringList statusList = status.split(QRegExp("\\s+"));
|
||||
if (2 < statusList.size()){
|
||||
//we expect status list to be at least 3 elements and second element to be the http error code
|
||||
code = statusList.at(1).toInt();
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
void PropagateBundle::slotJobDestroyed(QObject* job)
|
||||
{
|
||||
_jobs.erase(std::remove(_jobs.begin(), _jobs.end(), job) , _jobs.end());
|
||||
}
|
||||
|
||||
// This function is used whenever there is an error occuring and we need to raise status for whole the bundle
|
||||
void PropagateBundle::abortWithError(SyncFileItem::Status status, const QString &error)
|
||||
{
|
||||
foreach(auto *job, _jobs) {
|
||||
if (job->reply()) {
|
||||
qDebug() << Q_FUNC_INFO << job;
|
||||
job->reply()->abort();
|
||||
}
|
||||
}
|
||||
|
||||
done(status, error);
|
||||
}
|
||||
|
||||
MultipartJob::~MultipartJob()
|
||||
{
|
||||
// Make sure that we destroy the QNetworkReply before our _device of which it keeps an internal pointer.
|
||||
setReply(0);
|
||||
}
|
||||
|
||||
void MultipartJob::start() {
|
||||
QNetworkRequest req;
|
||||
setReply(multipartRequest(path(), req, _multipart));
|
||||
_multipart->setParent(reply()); // delete the multiPart with the job
|
||||
setupConnections(reply());
|
||||
|
||||
if( reply()->error() != QNetworkReply::NoError ) {
|
||||
qWarning() << Q_FUNC_INFO << " Network error: " << reply()->errorString();
|
||||
}
|
||||
|
||||
connect(this, SIGNAL(networkActivity()), account().data(), SIGNAL(propagatorNetworkActivity()));
|
||||
|
||||
//TODO this uses _device, and we have no access here to that
|
||||
// // For Qt versions not including https://codereview.qt-project.org/110150
|
||||
// // Also do the runtime check if compiled with an old Qt but running with fixed one.
|
||||
// // (workaround disabled on windows and mac because the binaries we ship have patched qt)
|
||||
//#if QT_VERSION < QT_VERSION_CHECK(4, 8, 7)
|
||||
// if (QLatin1String(qVersion()) < QLatin1String("4.8.7"))
|
||||
// connect(_device.data(), SIGNAL(wasReset()), this, SLOT(slotSoftAbort()));
|
||||
//#elif QT_VERSION > QT_VERSION_CHECK(5, 0, 0) && QT_VERSION < QT_VERSION_CHECK(5, 4, 2) && !defined Q_OS_WIN && !defined Q_OS_MAC
|
||||
// if (QLatin1String(qVersion()) < QLatin1String("5.4.2"))
|
||||
// connect(_device.data(), SIGNAL(wasReset()), this, SLOT(slotSoftAbort()));
|
||||
//#endif
|
||||
|
||||
AbstractNetworkJob::start();
|
||||
}
|
||||
|
||||
void MultipartJob::slotTimeout() {
|
||||
qDebug() << "Timeout" << (reply() ? reply()->request().url() : path());
|
||||
if (!reply())
|
||||
return;
|
||||
_errorString = tr("Connection Timeout");
|
||||
reply()->abort();
|
||||
}
|
||||
|
||||
#if QT_VERSION < QT_VERSION_CHECK(5, 4, 2)
|
||||
void MultipartJob::slotSoftAbort() {
|
||||
reply()->setProperty(owncloudShouldSoftCancelPropertyName, true);
|
||||
reply()->abort();
|
||||
}
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
#include <QFile>
|
||||
#include <QDebug>
|
||||
|
||||
|
||||
namespace OCC {
|
||||
class BandwidthManager;
|
||||
|
||||
@@ -123,6 +122,51 @@ private slots:
|
||||
#endif
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief The MultipartJob class
|
||||
* @ingroup libsync
|
||||
*/
|
||||
class MultipartJob : public AbstractNetworkJob {
|
||||
Q_OBJECT
|
||||
|
||||
private:
|
||||
QVector<SyncFileItemPtr> _syncItems;
|
||||
QHttpMultiPart* _multipart;
|
||||
QString _errorString;
|
||||
public:
|
||||
QElapsedTimer _duration;
|
||||
// Takes ownership of the device
|
||||
explicit MultipartJob(AccountPtr account, const QString& path, QHttpMultiPart* multiPart, const QVector<SyncFileItemPtr> &syncItems,QObject* parent = 0)
|
||||
: AbstractNetworkJob(account, path, parent), _syncItems(syncItems), _multipart(multiPart), _errorString(QString()) {}
|
||||
~MultipartJob();
|
||||
|
||||
virtual void start() Q_DECL_OVERRIDE;
|
||||
|
||||
virtual bool finished() Q_DECL_OVERRIDE {
|
||||
emit finishedSignal();
|
||||
return true;
|
||||
}
|
||||
|
||||
QString errorString() {
|
||||
return _errorString.isEmpty() ? reply()->errorString() : _errorString;
|
||||
}
|
||||
|
||||
QVector<SyncFileItemPtr> syncItems() const {
|
||||
return _syncItems;
|
||||
}
|
||||
|
||||
virtual void slotTimeout() Q_DECL_OVERRIDE;
|
||||
|
||||
|
||||
signals:
|
||||
void finishedSignal();
|
||||
|
||||
private slots:
|
||||
#if QT_VERSION < 0x050402
|
||||
void slotSoftAbort();
|
||||
#endif
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief This job implements the asynchronous PUT
|
||||
*
|
||||
@@ -222,5 +266,66 @@ private:
|
||||
void abortWithError(SyncFileItem::Status status, const QString &error);
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief The PropagateBundle class
|
||||
* @ingroup libsync
|
||||
*/
|
||||
class PropagateBundle : public PropagateItemJob {
|
||||
Q_OBJECT
|
||||
|
||||
private:
|
||||
QLinkedList<SyncFileItemPtr> _itemsToSync;
|
||||
QLinkedList<SyncFileItemPtr> _itemsToChecksum;
|
||||
QVector<MultipartJob*> _jobs; /// network jobs that are currently in transit
|
||||
bool _running; // Tells that all the jobs have been finished
|
||||
bool _preparingBundle; // Tells that all the jobs have been finished
|
||||
quint64 _size; // Tells what is the total size of _itemsToSync inside the bundle
|
||||
quint64 _currentBundleSize;
|
||||
quint64 _currentRequestsNumber;
|
||||
quint64 _totalFiles;
|
||||
// measure the performance of checksum calc and upload
|
||||
Utility::StopWatch _stopWatch;
|
||||
|
||||
public:
|
||||
PropagateBundle(OwncloudPropagator* propagator)
|
||||
: PropagateItemJob(propagator, SyncFileItemPtr(new SyncFileItem)), _running(false), _preparingBundle(true), _size(0), _currentBundleSize(0), _currentRequestsNumber(0) {}
|
||||
void start() Q_DECL_OVERRIDE;
|
||||
void startBundle();
|
||||
void append(const SyncFileItemPtr &bundledFile);
|
||||
QByteArray getRemotePath(QString filePath);
|
||||
bool empty();
|
||||
bool scheduleNextJob() Q_DECL_OVERRIDE {
|
||||
if (_running != true && _propagator->runningNowAtRootJob() == 1){
|
||||
_running = true;
|
||||
QMetaObject::invokeMethod(this, "start"); // We could be in a different thread (neon jobs)
|
||||
}
|
||||
if (_state == NotYetStarted){
|
||||
_state = Running;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
quint64 syncItemsSize() const { return _size; }
|
||||
quint64 syncItemsNumber() const { return _itemsToChecksum.count(); }
|
||||
private slots:
|
||||
void slotComputeTransmissionChecksum();
|
||||
void slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum);
|
||||
void slotMultipartFinished();
|
||||
void slotJobDestroyed(QObject *job);
|
||||
void abortWithError(SyncFileItem::Status status, const QString &error);
|
||||
|
||||
private:
|
||||
quint64 checkBundledRequestsLimits()
|
||||
{
|
||||
//TODO: obtain this value from the server or by other means
|
||||
quint64 maximumBundledFiles = 500;
|
||||
return (maximumBundledFiles/_propagator->maximumActiveJob());
|
||||
}
|
||||
quint64 chunkSize() const { return _propagator->chunkSize(); }
|
||||
int getHttpStatusCode(const QString &status);
|
||||
void slotItemFinished(const SyncFileItemPtr &item, QMap<QString, QMap<QString, QString> > &responseObjectsProperties);
|
||||
void itemDone(SyncFileItemPtr item, SyncFileItem::Status status, const QString &errorString = QString());
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ public:
|
||||
|
||||
enum Type {
|
||||
UnknownType = 0,
|
||||
RequestsContainer = 1,
|
||||
File = CSYNC_FTW_TYPE_FILE,
|
||||
Directory = CSYNC_FTW_TYPE_DIR,
|
||||
SoftLink = CSYNC_FTW_TYPE_SLINK
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
#include <QtTest>
|
||||
|
||||
static const QUrl sRootUrl("owncloud://somehost/owncloud/remote.php/webdav/");
|
||||
static const QUrl sBundleRootUrl("owncloud://somehost/remote.php/dav/files/");
|
||||
|
||||
inline QString generateEtag() {
|
||||
return QString::number(QDateTime::currentDateTime().toMSecsSinceEpoch(), 16);
|
||||
@@ -533,6 +534,7 @@ public:
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
class FakeErrorReply : public QNetworkReply
|
||||
{
|
||||
Q_OBJECT
|
||||
@@ -556,6 +558,169 @@ public:
|
||||
qint64 readData(char *, qint64) override { return 0; }
|
||||
};
|
||||
|
||||
class FakeBundlePOSTReply : public QNetworkReply
|
||||
{
|
||||
Q_OBJECT
|
||||
FileInfo *fileInfo;
|
||||
QByteArray payload;
|
||||
public:
|
||||
FakeBundlePOSTReply(FileInfo &remoteRootFileInfo, QNetworkAccessManager::Operation op, const QNetworkRequest &request, const QByteArray &postPayload, QObject *parent)
|
||||
: QNetworkReply{parent} {
|
||||
setRequest(request);
|
||||
QUrl rawUrl = request.url();
|
||||
QString bundlePath(sBundleRootUrl.path()+rawUrl.userName());
|
||||
setUrl(rawUrl);
|
||||
setOperation(op);
|
||||
open(QIODevice::ReadOnly);
|
||||
const QString davUri{QStringLiteral("DAV:")};
|
||||
const QString ocUri{QStringLiteral("http://owncloud.org/ns")};
|
||||
const QString sabUri{QStringLiteral("http://sabredav.org/ns")};
|
||||
QBuffer buffer{&payload};
|
||||
buffer.open(QIODevice::WriteOnly);
|
||||
QXmlStreamWriter xml( &buffer );
|
||||
xml.writeNamespace(davUri, "d");
|
||||
xml.writeNamespace(ocUri, "o");
|
||||
xml.writeNamespace(sabUri, "s");
|
||||
|
||||
auto writeFileResponse = [&](const FileInfo &fileInfo) {
|
||||
xml.writeStartElement(davUri, QStringLiteral("response"));
|
||||
|
||||
//TODO: no need for X-OC-PATH, href could contain that, fix client/server
|
||||
xml.writeTextElement(davUri, QStringLiteral("href"), bundlePath);
|
||||
xml.writeStartElement(davUri, QStringLiteral("propstat"));
|
||||
xml.writeStartElement(davUri, QStringLiteral("prop"));
|
||||
|
||||
xml.writeTextElement(davUri, QStringLiteral("oc-etag"), fileInfo.etag);
|
||||
xml.writeTextElement(davUri, QStringLiteral("etag"), fileInfo.etag);
|
||||
xml.writeTextElement(davUri, QStringLiteral("oc-fileid"), fileInfo.fileId);
|
||||
xml.writeTextElement(davUri, QStringLiteral("x-oc-mtime"), QStringLiteral("accepted"));
|
||||
|
||||
//TODO: this slash to be fixed on client/server
|
||||
xml.writeTextElement(davUri, QStringLiteral("oc-path"), "/"+fileInfo.path());
|
||||
xml.writeEndElement(); // prop
|
||||
xml.writeTextElement(davUri, QStringLiteral("status"), "HTTP/1.1 200 OK");
|
||||
xml.writeEndElement(); // propstat
|
||||
xml.writeEndElement(); // response
|
||||
};
|
||||
auto writeFileErrorResponse = [&](const FileInfo &fileInfo, const QString &exception, const QString &message, const QString &status) {
|
||||
xml.writeStartElement(davUri, QStringLiteral("response"));
|
||||
|
||||
//TODO: no need for X-OC-PATH, href could contain that, fix client/server
|
||||
xml.writeTextElement(davUri, QStringLiteral("href"), bundlePath);
|
||||
xml.writeStartElement(davUri, QStringLiteral("propstat"));
|
||||
xml.writeStartElement(davUri, QStringLiteral("prop"));
|
||||
xml.writeStartElement(davUri, QStringLiteral("error"));
|
||||
xml.writeTextElement(sabUri, QStringLiteral("exception"), exception);
|
||||
xml.writeTextElement(sabUri, QStringLiteral("message"), message);
|
||||
xml.writeEndElement(); // error
|
||||
|
||||
//TODO: this slash to be fixed on client/server
|
||||
xml.writeTextElement(davUri, QStringLiteral("oc-path"), "/"+fileInfo.path());
|
||||
xml.writeEndElement(); // prop
|
||||
xml.writeTextElement(davUri, QStringLiteral("status"), status);
|
||||
xml.writeEndElement(); // propstat
|
||||
xml.writeEndElement(); // response
|
||||
};
|
||||
|
||||
|
||||
if ("erroruser" == rawUrl.userName()) {
|
||||
xml.writeStartDocument();
|
||||
xml.writeStartElement(davUri, QStringLiteral("error"));
|
||||
xml.writeTextElement(sabUri, QStringLiteral("exception"), QStringLiteral("OCA\\DAV\\Connector\\Sabre\\Exception\\Forbidden"));
|
||||
xml.writeTextElement(sabUri, QStringLiteral("message"), QStringLiteral("URL endpoint has to be instance of \\OCA\\DAV\\Files\\FilesHome"));
|
||||
xml.writeTextElement(ocUri, QStringLiteral("retry"), QStringLiteral("false"));
|
||||
xml.writeTextElement(ocUri, QStringLiteral("reason"), QStringLiteral("URL endpoint has to be instance of \\OCA\\DAV\\Files\\FilesHome"));
|
||||
xml.writeEndElement(); // error
|
||||
xml.writeEndDocument();
|
||||
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 403);
|
||||
} else {
|
||||
Q_ASSERT(request.url().path().endsWith(bundlePath));
|
||||
xml.writeStartDocument();
|
||||
xml.writeStartElement(davUri, QStringLiteral("multistatus"));
|
||||
|
||||
//multipart parsing
|
||||
QString headerSectEnd = "\r\n\r\n";
|
||||
QString headerEnd = "\r\n";
|
||||
QString headerOcMethod = "X-OC-Method: ";
|
||||
QString headerConLen = "Content-Length: ";
|
||||
QString headerOcPath = "X-OC-Path: ";
|
||||
int indexOfBody = 0;
|
||||
QChar contentChar;
|
||||
|
||||
while(postPayload.indexOf(headerSectEnd,indexOfBody) + headerSectEnd.length() >=indexOfBody) {
|
||||
//find oc-method
|
||||
int indexOfheaderEnd = postPayload.indexOf(headerOcMethod,indexOfBody) + headerOcMethod.length();
|
||||
int indexOfheaderBodyEnd = postPayload.indexOf(headerEnd,indexOfheaderEnd);
|
||||
Q_ASSERT(postPayload.mid(indexOfheaderEnd,indexOfheaderBodyEnd-indexOfheaderEnd) == QString("PUT"));
|
||||
|
||||
//find oc-path
|
||||
indexOfheaderEnd = postPayload.indexOf(headerOcPath,indexOfBody) + headerOcPath.length();
|
||||
indexOfheaderBodyEnd = postPayload.indexOf(headerEnd,indexOfheaderEnd)-1;
|
||||
QString filePath(postPayload.mid(indexOfheaderEnd+1,indexOfheaderBodyEnd-indexOfheaderEnd));
|
||||
|
||||
//find content-length
|
||||
indexOfheaderEnd = postPayload.indexOf(headerConLen,indexOfBody) + headerConLen.length();
|
||||
indexOfheaderBodyEnd = postPayload.indexOf(headerEnd,indexOfheaderEnd);
|
||||
QString fileSize(postPayload.mid(indexOfheaderEnd,indexOfheaderBodyEnd-indexOfheaderEnd));
|
||||
|
||||
|
||||
//find body content and extract first letter
|
||||
indexOfheaderEnd = postPayload.indexOf(headerSectEnd,indexOfBody) + headerSectEnd.length();
|
||||
indexOfBody = indexOfheaderEnd+1;
|
||||
contentChar = postPayload.at(indexOfheaderEnd+1);
|
||||
|
||||
if ((fileInfo = remoteRootFileInfo.find(filePath))) {
|
||||
fileInfo->size = fileSize.toInt();
|
||||
fileInfo->contentChar = contentChar.toAscii();
|
||||
} else {
|
||||
// Assume that the file is filled with the same character
|
||||
fileInfo = remoteRootFileInfo.create(filePath, fileSize.toInt(), contentChar.toAscii());
|
||||
}
|
||||
|
||||
if (!fileInfo) {
|
||||
abort();
|
||||
return;
|
||||
}
|
||||
|
||||
if (filePath.endsWith("normalerrorfile")){
|
||||
writeFileErrorResponse(*fileInfo, QStringLiteral("Sabre\\DAV\\Exception\\BadRequest"), QStringLiteral("Method not allowed - file exists - update of the file is not supported!"), QStringLiteral("HTTP/1.1 400 Bad Request"));
|
||||
} else if (filePath.endsWith("fatalerrorfile")){
|
||||
writeFileErrorResponse(*fileInfo, QStringLiteral("Sabre\\DAV\\Exception\\ServiceUnavailable"), QStringLiteral("Failed to check file size"), QStringLiteral("HTTP/1.1 503 Service Unavailable"));
|
||||
} else if (filePath.endsWith("softerrorfile")){
|
||||
writeFileErrorResponse(*fileInfo, QStringLiteral("OCA\\DAV\\Connector\\Sabre\\Exception\\FileLocked"), QStringLiteral("Target file is locked by another process."), QStringLiteral("HTTP/1.1 423 Locked (WebDAV; RFC 4918)"));
|
||||
} else {
|
||||
writeFileResponse(*fileInfo);
|
||||
}
|
||||
}
|
||||
xml.writeEndElement(); // multistatus
|
||||
xml.writeEndDocument();
|
||||
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 207);
|
||||
setFinished(true);
|
||||
}
|
||||
|
||||
QMetaObject::invokeMethod(this, "respond", Qt::QueuedConnection);
|
||||
}
|
||||
|
||||
Q_INVOKABLE void respond() {
|
||||
setHeader(QNetworkRequest::ContentTypeHeader, "application/xml; charset=utf-8");
|
||||
setHeader(QNetworkRequest::ContentLengthHeader, payload.size());
|
||||
emit metaDataChanged();
|
||||
if (bytesAvailable())
|
||||
emit readyRead();
|
||||
emit finished();
|
||||
}
|
||||
|
||||
void abort() override { }
|
||||
|
||||
qint64 bytesAvailable() const override { return payload.size() + QIODevice::bytesAvailable(); }
|
||||
qint64 readData(char *data, qint64 maxlen) override {
|
||||
qint64 len = std::min(qint64{payload.size()}, maxlen);
|
||||
strncpy(data, payload.constData(), len);
|
||||
payload.remove(0, len);
|
||||
return len;
|
||||
}
|
||||
};
|
||||
|
||||
class FakeQNAM : public QNetworkAccessManager
|
||||
{
|
||||
FileInfo _remoteRootFileInfo;
|
||||
@@ -586,7 +751,9 @@ protected:
|
||||
return new FakeDeleteReply{_remoteRootFileInfo, op, request, this};
|
||||
else if (verb == QLatin1String("MOVE"))
|
||||
return new FakeMoveReply{_remoteRootFileInfo, op, request, this};
|
||||
else {
|
||||
else if (op == QNetworkAccessManager::PostOperation) {
|
||||
return new FakeBundlePOSTReply{_remoteRootFileInfo, op, request, outgoingData->readAll(), this};
|
||||
} else {
|
||||
qDebug() << verb << outgoingData;
|
||||
Q_UNREACHABLE();
|
||||
}
|
||||
@@ -703,6 +870,10 @@ public:
|
||||
execUntilFinished();
|
||||
}
|
||||
|
||||
OCC::AccountPtr getAccount() {
|
||||
return _account;
|
||||
}
|
||||
|
||||
private:
|
||||
static void toDisk(QDir &dir, const FileInfo &templateFi) {
|
||||
foreach (const FileInfo &child, templateFi.children) {
|
||||
|
||||
@@ -31,6 +31,16 @@ bool itemDidCompleteSuccessfully(const QSignalSpy &spy, const QString &path)
|
||||
return false;
|
||||
}
|
||||
|
||||
SyncFileItem::Status itemDidCompleteWithStatus(const QSignalSpy &spy, const QString &path)
|
||||
{
|
||||
for(const QList<QVariant> &args : spy) {
|
||||
SyncFileItem item = args[0].value<SyncFileItem>();
|
||||
if (item.destination() == path)
|
||||
return item._status;
|
||||
}
|
||||
return SyncFileItem::NoStatus;
|
||||
}
|
||||
|
||||
class TestSyncEngine : public QObject
|
||||
{
|
||||
Q_OBJECT
|
||||
@@ -54,6 +64,75 @@ private slots:
|
||||
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
|
||||
}
|
||||
|
||||
void testFileUploadBundled() {
|
||||
FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
|
||||
|
||||
QVariantMap capBundle;
|
||||
capBundle["bundlerequest"] = true;
|
||||
QVariantMap caps;
|
||||
caps["dav"] = capBundle;
|
||||
fakeFolder.getAccount()->setCapabilities(caps);
|
||||
|
||||
testFileUploadBundledAllFilesOK(fakeFolder);
|
||||
testFileUploadBundledErrorForFile(fakeFolder);
|
||||
|
||||
//TODO unfinished, cannot generate NetworkError
|
||||
//testFileUploadBundledNotHomeCollection(fakeFolder);
|
||||
}
|
||||
|
||||
void testFileUploadBundledAllFilesOK(FakeFolder &fakeFolder) {
|
||||
QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));
|
||||
fakeFolder.localModifier().insert("A/a3");
|
||||
fakeFolder.localModifier().insert("A/a4");
|
||||
fakeFolder.localModifier().insert("B/b0");
|
||||
fakeFolder.syncOnce();
|
||||
|
||||
//check separate files
|
||||
QVERIFY(itemDidCompleteSuccessfully(completeSpy, "A/a3"));
|
||||
QVERIFY(itemDidCompleteSuccessfully(completeSpy, "A/a4"));
|
||||
QVERIFY(itemDidCompleteSuccessfully(completeSpy, "B/b0"));
|
||||
|
||||
//check whole bundle
|
||||
QVERIFY(itemDidCompleteSuccessfully(completeSpy, ""));
|
||||
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
|
||||
}
|
||||
|
||||
void testFileUploadBundledErrorForFile(FakeFolder &fakeFolder) {
|
||||
QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));
|
||||
fakeFolder.localModifier().insert("A/a5");
|
||||
fakeFolder.localModifier().insert("A/normalerrorfile");
|
||||
fakeFolder.localModifier().insert("A/fatalerrorfile");
|
||||
fakeFolder.localModifier().insert("A/softerrorfile");
|
||||
fakeFolder.localModifier().insert("B/b3");
|
||||
fakeFolder.syncOnce();
|
||||
|
||||
//check separate files
|
||||
QVERIFY(itemDidCompleteSuccessfully(completeSpy, "A/a5"));
|
||||
QVERIFY(SyncFileItem::NormalError == itemDidCompleteWithStatus(completeSpy, "A/normalerrorfile"));
|
||||
QVERIFY(SyncFileItem::FatalError == itemDidCompleteWithStatus(completeSpy, "A/fatalerrorfile"));
|
||||
QVERIFY(SyncFileItem::SoftError == itemDidCompleteWithStatus(completeSpy, "A/softerrorfile"));
|
||||
QVERIFY(itemDidCompleteSuccessfully(completeSpy, "B/b3"));
|
||||
|
||||
//check whole bundle
|
||||
QVERIFY(itemDidCompleteSuccessfully(completeSpy, ""));
|
||||
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
|
||||
}
|
||||
|
||||
void testFileUploadBundledNotHomeCollection(FakeFolder &fakeFolder) {
|
||||
QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));
|
||||
fakeFolder.localModifier().insert("A/a7");
|
||||
fakeFolder.localModifier().insert("A/a8");
|
||||
fakeFolder.localModifier().insert("B/b4");
|
||||
|
||||
//add the user "erroruser" which is not a FilesHomeCollection
|
||||
fakeFolder.getAccount()->setUrl(QUrl(QStringLiteral("http://erroruser:admin@localhost/owncloud")));
|
||||
fakeFolder.syncOnce();
|
||||
|
||||
//check whole bundle
|
||||
QVERIFY(itemDidCompleteSuccessfully(completeSpy, ""));
|
||||
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
|
||||
}
|
||||
|
||||
void testDirDownload() {
|
||||
FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
|
||||
QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));
|
||||
|
||||
Referência em uma Nova Issue
Bloquear um usuário