Comparar commits

..

1 Commits

Autor SHA1 Mensagem Data
Piotr M 5aa1629440 remove excesive loops and add sync files scheduler 2016-10-25 00:23:50 +02:00
40 arquivos alterados com 517 adições e 1397 exclusões
-6
Ver Arquivo
@@ -58,7 +58,6 @@ struct CmdOptions {
QString password;
QString proxy;
bool silent;
bool bundleRequests;
bool trustSSL;
bool useNetrc;
bool interactive;
@@ -158,7 +157,6 @@ void help()
std::cout << std::endl;
std::cout << "Options:" << std::endl;
std::cout << " --silent, -s Don't be so verbose" << std::endl;
std::cout << " --bundle-requests, -b Bundle Requests if supported" << std::endl;
std::cout << " --httpproxy [proxy] Specify a http proxy to use." << std::endl;
std::cout << " Proxy is http://server:port" << std::endl;
std::cout << " --trust Trust the SSL certification." << std::endl;
@@ -226,8 +224,6 @@ void parseOptions( const QStringList& app_args, CmdOptions *options )
options->silent = true;
} else if( option == "--trust") {
options->trustSSL = true;
} else if( option == "-b" || option == "--bundle-requests") {
options->bundleRequests = true;
} else if( option == "-n") {
options->useNetrc = true;
} else if( option == "-h") {
@@ -296,7 +292,6 @@ int main(int argc, char **argv) {
CmdOptions options;
options.silent = false;
options.bundleRequests = false;
options.trustSSL = false;
options.useNetrc = false;
options.interactive = true;
@@ -421,7 +416,6 @@ int main(int argc, char **argv) {
loop.exec();
#endif
account->setBundleRequestsIfCapable(options.bundleRequests);
// much lower age than the default since this utility is usually made to be run right after a change in the tests
SyncEngine::minimumFileAgeForUpload = 0;
+12 -6
Ver Arquivo
@@ -44,7 +44,7 @@ AccountManager *AccountManager::instance()
bool AccountManager::restore()
{
auto settings = Utility::settingsWithGroup(QLatin1String(accountsC));
auto settings = Account::settingsWithGroup(QLatin1String(accountsC));
// If there are no accounts, check the old format.
if (settings->childGroups().isEmpty()
@@ -69,7 +69,9 @@ bool AccountManager::restore()
bool AccountManager::restoreFromLegacySettings()
{
// try to open the correctly themed settings
auto settings = Utility::settingsWithGroup(Theme::instance()->appName());
auto settings = Account::settingsWithGroup(Theme::instance()->appName());
bool migratedCreds = false;
// if the settings file could not be opened, the childKeys list is empty
// then try to load settings from a very old place
@@ -100,6 +102,7 @@ bool AccountManager::restoreFromLegacySettings()
qDebug() << "Migrate oC config if " << oCUrl << " == " << overrideUrl << ":"
<< (oCUrl == overrideUrl ? "Yes" : "No");
if( oCUrl == overrideUrl ) {
migratedCreds = true;
settings.reset( oCSettings );
} else {
delete oCSettings;
@@ -111,6 +114,9 @@ bool AccountManager::restoreFromLegacySettings()
// Try to load the single account.
if (!settings->childKeys().isEmpty()) {
if (auto acc = loadAccountHelper(*settings)) {
if (migratedCreds) {
acc->setMigrated(true);
}
addAccount(acc);
return true;
}
@@ -120,7 +126,7 @@ bool AccountManager::restoreFromLegacySettings()
void AccountManager::save(bool saveCredentials)
{
auto settings = Utility::settingsWithGroup(QLatin1String(accountsC));
auto settings = Account::settingsWithGroup(QLatin1String(accountsC));
settings->setValue(QLatin1String(versionC), 2);
foreach (const auto &acc, _accounts) {
settings->beginGroup(acc->account()->id());
@@ -136,7 +142,7 @@ void AccountManager::save(bool saveCredentials)
void AccountManager::saveAccount(Account* a)
{
qDebug() << "Saving account" << a->url().toString();
auto settings = Utility::settingsWithGroup(QLatin1String(accountsC));
auto settings = Account::settingsWithGroup(QLatin1String(accountsC));
settings->beginGroup(a->id());
saveAccountHelper(a, *settings, false); // don't save credentials they might not have been loaded yet
settings->endGroup();
@@ -148,7 +154,7 @@ void AccountManager::saveAccount(Account* a)
void AccountManager::saveAccountState(AccountState* a)
{
qDebug() << "Saving account state" << a->account()->url().toString();
auto settings = Utility::settingsWithGroup(QLatin1String(accountsC));
auto settings = Account::settingsWithGroup(QLatin1String(accountsC));
settings->beginGroup(a->account()->id());
a->writeToSettings(*settings);
settings->endGroup();
@@ -274,7 +280,7 @@ void AccountManager::deleteAccount(AccountState* account)
auto copy = *it; // keep a reference to the shared pointer so it does not delete it just yet
_accounts.erase(it);
auto settings = Utility::settingsWithGroup(QLatin1String(accountsC));
auto settings = Account::settingsWithGroup(QLatin1String(accountsC));
settings->remove(account->account()->id());
emit accountRemoved(account);
+3 -3
Ver Arquivo
@@ -321,7 +321,7 @@ void AccountSettings::slotFolderWizardAccepted()
// The user already accepted the selective sync dialog. everything is in the white list
f->journalDb()->setSelectiveSyncList(SyncJournalDb::SelectiveSyncWhiteList,
QStringList() << QLatin1String("/"));
folderMan->scheduleAllFolders();
folderMan->slotScheduleAllFolders();
emit folderChanged();
}
}
@@ -360,7 +360,7 @@ void AccountSettings::slotRemoveCurrentFolder()
return;
}
folderMan->removeFolder( folderMan->folder(alias) );
folderMan->slotRemoveFolder( folderMan->folder(alias) );
_model->removeRow(row);
// single folder fix to show add-button and hide remove-button
@@ -468,7 +468,7 @@ void AccountSettings::slotSyncCurrentFolderNow()
QString alias = _model->data( selected, FolderStatusDelegate::FolderAliasRole ).toString();
FolderMan *folderMan = FolderMan::instance();
folderMan->scheduleFolder(folderMan->folder(alias));
folderMan->slotScheduleSync(folderMan->folder(alias));
}
void AccountSettings::slotOpenOC()
+1 -1
Ver Arquivo
@@ -306,7 +306,7 @@ void AccountState::slotCredentialsAsked(AbstractCredentials* credentials)
std::unique_ptr<QSettings> AccountState::settings()
{
auto s = Utility::settingsWithGroup(QLatin1String("Accounts"));
auto s = _account->settingsWithGroup(QLatin1String("Accounts"));
s->beginGroup(_account->id());
return s;
}
+19 -4
Ver Arquivo
@@ -79,6 +79,21 @@ void ShibbolethCredentials::setAccount(Account* account)
}
}
bool ShibbolethCredentials::changed(AbstractCredentials* credentials) const
{
ShibbolethCredentials* other(qobject_cast< ShibbolethCredentials* >(credentials));
if (!other) {
return true;
}
if (_shibCookie != other->_shibCookie || _user != other->_user) {
return true;
}
return false;
}
QString ShibbolethCredentials::authType() const
{
return QString::fromLatin1("shibboleth");
@@ -127,7 +142,7 @@ void ShibbolethCredentials::fetchFromKeychain()
} else {
_url = _account->url();
ReadPasswordJob *job = new ReadPasswordJob(Theme::instance()->appName());
job->setSettings(Utility::settingsWithGroup(Theme::instance()->appName(), job).release());
job->setSettings(_account->settingsWithGroup(Theme::instance()->appName(), job).release());
job->setInsecureFallback(false);
job->setKey(keychainKey(_account->url().toString(), "shibAssertion"));
connect(job, SIGNAL(finished(QKeychain::Job*)), SLOT(slotReadJobDone(QKeychain::Job*)));
@@ -246,7 +261,7 @@ void ShibbolethCredentials::slotReadJobDone(QKeychain::Job *job)
addToCookieJar(_shibCookie);
}
// access
job->setSettings(Utility::settingsWithGroup(Theme::instance()->appName(), job).release());
job->setSettings(_account->settingsWithGroup(Theme::instance()->appName(), job).release());
_ready = true;
_stillValid = true;
@@ -305,7 +320,7 @@ QByteArray ShibbolethCredentials::shibCookieName()
void ShibbolethCredentials::storeShibCookie(const QNetworkCookie &cookie)
{
WritePasswordJob *job = new WritePasswordJob(Theme::instance()->appName());
job->setSettings(Utility::settingsWithGroup(Theme::instance()->appName(), job).release());
job->setSettings(_account->settingsWithGroup(Theme::instance()->appName(), job).release());
// we don't really care if it works...
//connect(job, SIGNAL(finished(QKeychain::Job*)), SLOT(slotWriteJobDone(QKeychain::Job*)));
job->setKey(keychainKey(_account->url().toString(), "shibAssertion"));
@@ -316,7 +331,7 @@ void ShibbolethCredentials::storeShibCookie(const QNetworkCookie &cookie)
void ShibbolethCredentials::removeShibCookie()
{
DeletePasswordJob *job = new DeletePasswordJob(Theme::instance()->appName());
job->setSettings(Utility::settingsWithGroup(Theme::instance()->appName(), job).release());
job->setSettings(_account->settingsWithGroup(Theme::instance()->appName(), job).release());
job->setKey(keychainKey(_account->url().toString(), "shibAssertion"));
job->start();
}
+1
Ver Arquivo
@@ -48,6 +48,7 @@ public:
ShibbolethCredentials(const QNetworkCookie &cookie);
void setAccount(Account* account) Q_DECL_OVERRIDE;
bool changed(AbstractCredentials* credentials) const Q_DECL_OVERRIDE;
QString authType() const Q_DECL_OVERRIDE;
QString user() const Q_DECL_OVERRIDE;
QNetworkAccessManager* getQNAM() const Q_DECL_OVERRIDE;
+53 -35
Ver Arquivo
@@ -58,6 +58,7 @@ Folder::Folder(const FolderDefinition& definition,
, _wipeDb(false)
, _proxyDirty(true)
, _lastSyncDuration(0)
, _forceSyncOnPollTimeout(false)
, _consecutiveFailingSyncs(0)
, _consecutiveFollowUpSyncs(0)
, _journal(definition.localPath)
@@ -111,11 +112,6 @@ Folder::Folder(const FolderDefinition& definition,
connect(_engine.data(), SIGNAL(seenLockedFile(QString)), FolderMan::instance(), SLOT(slotSyncOnceFileUnlocks(QString)));
connect(_engine.data(), SIGNAL(aboutToPropagate(SyncFileItemVector&)),
SLOT(slotLogPropagationStart()));
_scheduleSelfTimer.setSingleShot(true);
_scheduleSelfTimer.setInterval(SyncEngine::minimumFileAgeForUpload);
connect(&_scheduleSelfTimer, SIGNAL(timeout()),
SLOT(slotScheduleThisFolder()));
}
Folder::~Folder()
@@ -225,7 +221,7 @@ QString Folder::remotePath() const
QUrl Folder::remoteUrl() const
{
return Utility::concatUrlPath(_accountState->account()->davUrl(), remotePath());
return Account::concatUrlPath(_accountState->account()->davUrl(), remotePath());
}
bool Folder::syncPaused() const
@@ -279,7 +275,7 @@ void Folder::slotRunEtagJob()
AccountPtr account = _accountState->account();
if (_requestEtagJob) {
if (!_requestEtagJob.isNull()) {
qDebug() << Q_FUNC_INFO << remoteUrl().toString() << "has ETag job queued, not trying to sync";
return;
}
@@ -289,15 +285,47 @@ void Folder::slotRunEtagJob()
return;
}
// Do the ordinary etag check for the root folder and schedule a
// sync if it's different.
bool forceSyncIntervalExpired =
quint64(_timeSinceLastSyncDone.elapsed()) > ConfigFile().forceSyncInterval();
bool syncAgainAfterFail = _consecutiveFailingSyncs > 0 && _consecutiveFailingSyncs < 3;
_requestEtagJob = new RequestEtagJob(account, remotePath(), this);
_requestEtagJob->setTimeout(60*1000);
// check if the etag is different when retrieved
QObject::connect(_requestEtagJob, SIGNAL(etagRetreived(QString)), this, SLOT(etagRetreived(QString)));
FolderMan::instance()->slotScheduleETagJob(alias(), _requestEtagJob);
// The _requestEtagJob is auto deleting itself on finish. Our guard pointer _requestEtagJob will then be null.
// There are several conditions under which we trigger a full-discovery sync:
// * When a suitably long time has passed since the last sync finished
// * When the last sync failed (only a couple of times)
// * When the last sync requested another sync to be done (only a couple of times)
//
// Note that the etag check (see below) and the file watcher may also trigger
// syncs.
if (forceSyncIntervalExpired
|| _forceSyncOnPollTimeout
|| syncAgainAfterFail) {
if (forceSyncIntervalExpired) {
qDebug() << "** Force Sync, because it has been " << _timeSinceLastSyncDone.elapsed() << "ms "
<< "since the last sync";
}
if (_forceSyncOnPollTimeout) {
qDebug() << "** Force Sync, because it was requested";
}
if (syncAgainAfterFail) {
qDebug() << "** Force Sync, because the last"
<< _consecutiveFailingSyncs << "syncs failed, last status:"
<< _syncResult.statusString();
}
_forceSyncOnPollTimeout = false;
emit scheduleToSync(this);
} else {
// Do the ordinary etag check for the root folder and only schedule a real
// sync if it's different.
_requestEtagJob = new RequestEtagJob(account, remotePath(), this);
_requestEtagJob->setTimeout(60*1000);
// check if the etag is different
QObject::connect(_requestEtagJob, SIGNAL(etagRetreived(QString)), this, SLOT(etagRetreived(QString)));
FolderMan::instance()->slotScheduleETagJob(alias(), _requestEtagJob);
// The _requestEtagJob is auto deleting itself on finish. Our guard pointer _requestEtagJob will then be null.
}
}
void Folder::etagRetreived(const QString& etag)
@@ -310,7 +338,7 @@ void Folder::etagRetreived(const QString& etag)
if (_lastEtag != etag) {
qDebug() << "* Compare etag with previous etag: last:" << _lastEtag << ", received:" << etag << "-> CHANGED";
_lastEtag = etag;
slotScheduleThisFolder();
emit scheduleToSync(this);
}
_accountState->tagLastSuccessfullETagRequest();
@@ -570,10 +598,7 @@ void Folder::slotWatchedPathChanged(const QString& path)
}
emit watchedFileChangedExternally(path);
// Also schedule this folder for a sync, but only after some delay:
// The sync will not upload files that were changed too recently.
scheduleThisFolderSoon();
emit scheduleToSync(this);
}
void Folder::slotThreadTreeWalkResult(const SyncFileItemVector& items)
@@ -854,10 +879,11 @@ void Folder::slotSyncFinished(bool success)
// Maybe force a follow-up sync to take place, but only a couple of times.
if (anotherSyncNeeded && _consecutiveFollowUpSyncs <= 3)
{
// Sometimes another sync is requested because a local file is still
// changing, so wait at least a small amount of time before syncing
// the folder again.
scheduleThisFolderSoon();
_forceSyncOnPollTimeout = true;
// We will make sure that the poll timer occurs soon enough.
// delay 1s, 4s, 9s
int c = _consecutiveFollowUpSyncs;
QTimer::singleShot(c*c * 1000, this, SLOT(slotRunEtagJob() ));
}
}
@@ -938,17 +964,7 @@ void Folder::slotLogPropagationStart()
_fileLog->logLap("Propagation starts");
}
void Folder::slotScheduleThisFolder()
{
FolderMan::instance()->scheduleFolder(this);
}
void Folder::scheduleThisFolderSoon()
{
if (!_scheduleSelfTimer.isActive()) {
_scheduleSelfTimer.start();
}
}
void Folder::slotAboutToRemoveAllFiles(SyncFileItem::Direction, bool *cancel)
{
@@ -972,8 +988,10 @@ void Folder::slotAboutToRemoveAllFiles(SyncFileItem::Direction, bool *cancel)
*cancel = msgBox.clickedButton() == keepBtn;
if (*cancel) {
wipe();
// speed up next sync
_lastEtag.clear();
slotScheduleThisFolder();
_forceSyncOnPollTimeout = true;
QTimer::singleShot(50, this, SLOT(slotRunEtagJob()));
}
}
+2 -19
Ver Arquivo
@@ -178,7 +178,6 @@ public:
qint64 msecSinceLastSync() const { return _timeSinceLastSyncDone.elapsed(); }
qint64 msecLastSyncDuration() const { return _lastSyncDuration; }
int consecutiveFollowUpSyncs() const { return _consecutiveFollowUpSyncs; }
int consecutiveFailingSyncs() const { return _consecutiveFailingSyncs; }
/// Saves the folder data in the account's settings.
void saveToSettings() const;
@@ -195,21 +194,11 @@ public:
*/
bool isFileExcludedRelative(const QString& relativePath) const;
/** Calls schedules this folder on the FolderMan after a short delay.
*
* This should be used in situations where a sync should be triggered
* because a local file was modified. Syncs don't upload files that were
* modified too recently, and this delay ensures the modification is
* far enough in the past.
*
* The delay doesn't reset with subsequent calls.
*/
void scheduleThisFolderSoon();
signals:
void syncStateChange();
void syncStarted();
void syncFinished(const SyncResult &result);
void scheduleToSync(Folder*);
void progressInfo(const ProgressInfo& progress);
void newBigFolderDiscovered(const QString &); // A new folder bigger than the threshold was discovered
void syncPausedChanged(Folder*, bool paused);
@@ -277,11 +266,6 @@ private slots:
void slotLogPropagationStart();
/** Adds this folder to the list of scheduled folders in the
* FolderMan.
*/
void slotScheduleThisFolder();
private:
bool setIgnoredFiles();
@@ -318,6 +302,7 @@ private:
QElapsedTimer _timeSinceLastSyncDone;
QElapsedTimer _timeSinceLastSyncStart;
qint64 _lastSyncDuration;
bool _forceSyncOnPollTimeout;
/// The number of syncs that failed in a row.
/// Reset when a sync is successful.
@@ -332,8 +317,6 @@ private:
ClientProxy _clientProxy;
QScopedPointer<SyncRunFileLog> _fileLog;
QTimer _scheduleSelfTimer;
};
}
+45 -79
Ver Arquivo
@@ -64,17 +64,11 @@ FolderMan::FolderMan(QObject *parent) :
connect(&_startScheduledSyncTimer, SIGNAL(timeout()),
SLOT(slotStartScheduledFolderSync()));
_timeScheduler.setInterval(5000);
_timeScheduler.setSingleShot(false);
connect(&_timeScheduler, SIGNAL(timeout()),
SLOT(slotScheduleFolderByTime()));
_timeScheduler.start();
connect(AccountManager::instance(), SIGNAL(accountRemoved(AccountState*)),
SLOT(slotRemoveFoldersForAccount(AccountState*)));
connect(_lockWatcher.data(), SIGNAL(fileUnlocked(QString)),
SLOT(slotWatchedFileUnlocked(QString)));
SLOT(slotScheduleFolderOwningFile(QString)));
}
FolderMan *FolderMan::instance()
@@ -106,6 +100,8 @@ void FolderMan::unloadFolder( Folder *f )
}
_folderMap.remove( f->alias() );
disconnect(f, SIGNAL(scheduleToSync(Folder*)),
this, SLOT(slotScheduleSync(Folder*)));
disconnect(f, SIGNAL(syncStarted()),
this, SLOT(slotFolderSyncStarted()));
disconnect(f, SIGNAL(syncFinished(SyncResult)),
@@ -135,7 +131,7 @@ int FolderMan::unloadAndDeleteAllFolders()
}
_lastSyncFolder = 0;
_currentSyncFolder = 0;
_scheduledFolders.clear();
_scheduleQueue.clear();
emit scheduleQueueChanged();
Q_ASSERT(_folderMap.count() == 0);
@@ -191,7 +187,7 @@ int FolderMan::setupFolders()
{
unloadAndDeleteAllFolders();
auto settings = Utility::settingsWithGroup(QLatin1String("Accounts"));
auto settings = Account::settingsWithGroup(QLatin1String("Accounts"));
const auto accountsWithSettings = settings->childGroups();
if (accountsWithSettings.isEmpty()) {
int r = setupFoldersMigration();
@@ -215,7 +211,7 @@ int FolderMan::setupFolders()
if (FolderDefinition::load(*settings, folderAlias, &folderDefinition)) {
Folder* f = addFolderInternal(std::move(folderDefinition), account.data());
if (f) {
scheduleFolder(f);
slotScheduleSync(f);
emit folderSyncStateChange(f);
}
}
@@ -248,7 +244,7 @@ int FolderMan::setupFoldersMigration()
foreach ( const QString& alias, list ) {
Folder *f = setupFolderFromOldConfigFile( alias, accountState );
if( f ) {
scheduleFolder(f);
slotScheduleSync(f);
emit folderSyncStateChange(f);
}
}
@@ -426,7 +422,7 @@ void FolderMan::slotFolderSyncPaused( Folder *f, bool paused )
if (!paused) {
_disabledFolders.remove(f);
scheduleFolder(f);
slotScheduleSync(f);
} else {
_disabledFolders.insert(f);
}
@@ -466,11 +462,11 @@ Folder *FolderMan::folder( const QString& alias )
return 0;
}
void FolderMan::scheduleAllFolders()
void FolderMan::slotScheduleAllFolders()
{
foreach( Folder *f, _folderMap.values() ) {
if (f && f->canSync()) {
scheduleFolder( f );
slotScheduleSync( f );
}
}
}
@@ -490,7 +486,7 @@ void FolderMan::slotSyncOnceFileUnlocks(const QString& path)
* if a folder wants to be synced, it calls this slot and is added
* to the queue. The slot to actually start a sync is called afterwards.
*/
void FolderMan::scheduleFolder( Folder *f )
void FolderMan::slotScheduleSync( Folder *f )
{
if( !f ) {
qWarning() << "slotScheduleSync called with null folder";
@@ -500,7 +496,7 @@ void FolderMan::scheduleFolder( Folder *f )
qDebug() << "Schedule folder " << alias << " to sync!";
if( ! _scheduledFolders.contains(f) ) {
if( ! _scheduleQueue.contains(f) ) {
if( !f->canSync() ) {
qDebug() << "Folder is not ready to sync, not scheduled!";
_socketApi->slotUpdateFolderView(f);
@@ -508,7 +504,7 @@ void FolderMan::scheduleFolder( Folder *f )
}
f->prepareToSync();
emit folderSyncStateChange(f);
_scheduledFolders.enqueue(f);
_scheduleQueue.enqueue(f);
emit scheduleQueueChanged();
} else {
qDebug() << " II> Sync for folder " << alias << " already scheduled, do not enqueue!";
@@ -572,7 +568,7 @@ void FolderMan::slotAccountStateChanged()
if (f
&& f->canSync()
&& f->accountState() == accountState) {
scheduleFolder(f);
slotScheduleSync(f);
}
}
} else {
@@ -584,7 +580,7 @@ void FolderMan::slotAccountStateChanged()
_currentSyncFolder->slotTerminateSync();
}
QMutableListIterator<Folder*> it(_scheduledFolders);
QMutableListIterator<Folder*> it(_scheduleQueue);
while (it.hasNext()) {
Folder* f = it.next();
if (f->accountState() == accountState) {
@@ -599,7 +595,7 @@ void FolderMan::slotAccountStateChanged()
// this is not the same as Pause and Resume of folders.
void FolderMan::setSyncEnabled( bool enabled )
{
if (!_syncEnabled && enabled && !_scheduledFolders.isEmpty()) {
if (!_syncEnabled && enabled && !_scheduleQueue.isEmpty()) {
// We have things in our queue that were waiting for the connection to come back on.
startScheduledSyncSoon();
}
@@ -608,19 +604,19 @@ void FolderMan::setSyncEnabled( bool enabled )
emit( folderSyncStateChange(0) );
}
void FolderMan::startScheduledSyncSoon()
void FolderMan::startScheduledSyncSoon(qint64 msMinimumDelay)
{
if (_startScheduledSyncTimer.isActive()) {
return;
}
if (_scheduledFolders.empty()) {
if (_scheduleQueue.empty()) {
return;
}
if (_currentSyncFolder) {
return;
}
qint64 msDelay = 100; // 100ms minimum delay
qint64 msDelay = msMinimumDelay;
qint64 msSinceLastSync = 0;
// Require a pause based on the duration of the last sync run.
@@ -635,6 +631,15 @@ void FolderMan::startScheduledSyncSoon()
msDelay = qMax(msDelay, pause);
}
// Punish consecutive follow-up syncs with longer delays.
if (Folder* nextFolder = _scheduleQueue.head()) {
int followUps = nextFolder->consecutiveFollowUpSyncs();
if (followUps >= 2) {
// This is okay due to the 1min maximum delay limit below.
msDelay *= qPow(followUps, 2);
}
}
// Delays beyond one minute seem too big, particularly since there
// could be things later in the queue that shouldn't be punished by a
// long delay!
@@ -643,7 +648,11 @@ void FolderMan::startScheduledSyncSoon()
// Time since the last sync run counts against the delay
msDelay = qMax(1ll, msDelay - msSinceLastSync);
qDebug() << "Starting the next scheduled sync in" << (msDelay/1000) << "seconds";
// A minimum of delay here is essential as the sync will not upload
// files that were changed too recently.
msDelay = qMax(SyncEngine::minimumFileAgeForUpload, msDelay);
qDebug() << "Scheduling a sync in" << (msDelay/1000) << "seconds";
_startScheduledSyncTimer.start(msDelay);
}
@@ -664,15 +673,15 @@ void FolderMan::slotStartScheduledFolderSync()
return;
}
qDebug() << "XX slotScheduleFolderSync: folderQueue size: " << _scheduledFolders.count();
if( _scheduledFolders.isEmpty() ) {
qDebug() << "XX slotScheduleFolderSync: folderQueue size: " << _scheduleQueue.count();
if( _scheduleQueue.isEmpty() ) {
return;
}
// Find the first folder in the queue that can be synced.
Folder* f = 0;
while( !_scheduledFolders.isEmpty() ) {
f = _scheduledFolders.dequeue();
while( !_scheduleQueue.isEmpty() ) {
f = _scheduleQueue.dequeue();
Q_ASSERT(f);
if( f->canSync() ) {
@@ -702,7 +711,7 @@ void FolderMan::slotEtagPollTimerTimeout()
if (_currentSyncFolder == f) {
continue;
}
if (_scheduledFolders.contains(f)) {
if (_scheduleQueue.contains(f)) {
continue;
}
if (_disabledFolders.contains(f)) {
@@ -731,7 +740,7 @@ void FolderMan::slotRemoveFoldersForAccount(AccountState* accountState)
}
foreach (const auto &f, foldersToRemove) {
removeFolder(f);
slotRemoveFolder(f);
}
}
@@ -757,54 +766,10 @@ void FolderMan::slotServerVersionChanged(Account *account)
}
}
void FolderMan::slotWatchedFileUnlocked(const QString& path)
void FolderMan::slotScheduleFolderOwningFile(const QString& path)
{
if (Folder* f = folderForPath(path)) {
f->scheduleThisFolderSoon();
}
}
void FolderMan::slotScheduleFolderByTime()
{
foreach (auto& f, _folderMap) {
// Never schedule if syncing is disabled or when we're currently
// querying the server for etags
if (!f->canSync() || f->etagJob()) {
continue;
}
auto msecsSinceSync = f->msecSinceLastSync();
// Possibly it's just time for a new sync run
bool forceSyncIntervalExpired =
quint64(msecsSinceSync) > ConfigFile().forceSyncInterval();
if (forceSyncIntervalExpired) {
qDebug() << "** scheduling folder" << f->alias()
<< "because it has been" << msecsSinceSync << "ms "
<< "since the last sync";
scheduleFolder(f);
continue;
}
// Retry a couple of times after failure
bool syncAgainAfterFail = f->consecutiveFailingSyncs() > 0 && f->consecutiveFailingSyncs() < 3;
qint64 syncAgainAfterFailDelay = 10 * 1000; // 10s for the first retry-after-fail
if (f->consecutiveFailingSyncs() > 1)
syncAgainAfterFailDelay = 60 * 1000; // 60s for each further attempt
if (syncAgainAfterFail
&& msecsSinceSync > syncAgainAfterFailDelay) {
qDebug() << "** scheduling folder" << f->alias()
<< "because the last"
<< f->consecutiveFailingSyncs() << "syncs failed, last status:"
<< f->syncResult().statusString()
<< "time since last sync:" << msecsSinceSync;
scheduleFolder(f);
continue;
}
// Do we want to retry failing syncs or another-sync-needed runs more often?
slotScheduleSync(f);
}
}
@@ -862,6 +827,7 @@ Folder* FolderMan::addFolderInternal(FolderDefinition folderDefinition, AccountS
}
// See matching disconnects in unloadFolder().
connect(folder, SIGNAL(scheduleToSync(Folder*)), SLOT(slotScheduleSync(Folder*)));
connect(folder, SIGNAL(syncStarted()), SLOT(slotFolderSyncStarted()));
connect(folder, SIGNAL(syncFinished(SyncResult)), SLOT(slotFolderSyncFinished(SyncResult)));
connect(folder, SIGNAL(syncStateChange()), SLOT(slotForwardFolderSyncStateChange()));
@@ -914,7 +880,7 @@ QStringList FolderMan::findFileInLocalFolders( const QString& relPath, const Acc
return re;
}
void FolderMan::removeFolder( Folder *f )
void FolderMan::slotRemoveFolder( Folder *f )
{
if( !f ) {
qWarning() << "!! Can not remove null folder";
@@ -929,7 +895,7 @@ void FolderMan::removeFolder( Folder *f )
terminateSyncProcess();
}
if (_scheduledFolders.removeAll(f) > 0) {
if (_scheduleQueue.removeAll(f) > 0) {
emit scheduleQueueChanged();
}
@@ -1296,7 +1262,7 @@ void FolderMan::setIgnoreHiddenFiles(bool ignore)
QQueue<Folder*> FolderMan::scheduleQueue() const
{
return _scheduledFolders;
return _scheduleQueue;
}
Folder *FolderMan::currentSyncFolder() const
+43 -94
Ver Arquivo
@@ -37,26 +37,6 @@ class LockWatcher;
/**
* @brief The FolderMan class
* @ingroup gui
*
* The FolderMan knows about all loaded folders and is responsible for
* scheduling them when necessary.
*
* A folder is scheduled if:
* - The configured force-sync-interval has expired
* (_timeScheduler and slotScheduleFolderByTime())
*
* - A folder watcher receives a notification about a file change
* (_folderWatchers and Folder::slotWatchedPathChanged())
*
* - The folder etag on the server has changed
* (_etagPollTimer)
*
* - The locks of a monitored file are released
* (_lockWatcher and slotWatchedFileUnlocked())
*
* - There was a sync error or a follow-up sync is requested
* (_timeScheduler and slotScheduleFolderByTime()
* and Folder::slotSyncFinished())
*/
class FolderMan : public QObject
{
@@ -74,9 +54,6 @@ public:
*/
Folder* addFolder(AccountState* accountState, const FolderDefinition& folderDefinition);
/** Removes a folder */
void removeFolder( Folder* );
/** Returns the folder which the file or directory stored in path is in */
Folder* folderForPath(const QString& path);
@@ -149,24 +126,6 @@ public:
*/
Folder* currentSyncFolder() const;
/** Removes all folders */
int unloadAndDeleteAllFolders();
/**
* If enabled is set to false, no new folders will start to sync.
* The current one will finish.
*/
void setSyncEnabled( bool );
/** Queues a folder for syncing. */
void scheduleFolder(Folder*);
/** Queues all folders for syncing. */
void scheduleAllFolders();
void setDirtyProxy(bool value = true);
void setDirtyNetworkLimits();
signals:
/**
* signal to indicate a folder has changed its sync state.
@@ -180,12 +139,41 @@ signals:
*/
void scheduleQueueChanged();
/**
* Emitted whenever the list of configured folders changes.
*/
void folderListChanged(const Folder::Map &);
public slots:
void slotRemoveFolder( Folder* );
void slotFolderSyncPaused(Folder *, bool paused);
void slotFolderCanSyncChanged();
void slotFolderSyncStarted();
void slotFolderSyncFinished( const SyncResult& );
/**
* Terminates the current folder sync.
*
* It does not switch the folder to paused state.
*/
void terminateSyncProcess();
/* delete all folder objects */
int unloadAndDeleteAllFolders();
// if enabled is set to false, no new folders will start to sync.
// the current one will finish.
void setSyncEnabled( bool );
void slotScheduleAllFolders();
void setDirtyProxy(bool value = true);
void setDirtyNetworkLimits();
// slot to add a folder to the syncing queue
void slotScheduleSync(Folder*);
// slot to schedule an ETag job
void slotScheduleETagJob ( const QString &alias, RequestEtagJob *job);
void slotEtagJobDestroyed (QObject*);
void slotRunOneEtagJob();
/**
* Schedules folders of newly connected accounts, terminates and
@@ -202,22 +190,10 @@ public slots:
* Triggers a sync run once the lock on the given file is removed.
*
* Automatically detemines the folder that's responsible for the file.
* See slotWatchedFileUnlocked().
*/
void slotSyncOnceFileUnlocks(const QString& path);
// slot to schedule an ETag job (from Folder only)
void slotScheduleETagJob ( const QString &alias, RequestEtagJob *job);
private slots:
void slotFolderSyncPaused(Folder *, bool paused);
void slotFolderCanSyncChanged();
void slotFolderSyncStarted();
void slotFolderSyncFinished( const SyncResult& );
void slotRunOneEtagJob();
void slotEtagJobDestroyed (QObject*);
// slot to take the next folder from queue and start syncing.
void slotStartScheduledFolderSync();
void slotEtagPollTimerTimeout();
@@ -231,29 +207,12 @@ private slots:
void slotServerVersionChanged(Account* account);
/**
* A file whose locks were being monitored has become unlocked.
*
* This schedules the folder for synchronization that contains
* Schedules the folder for synchronization that contains
* the file with the given path.
*/
void slotWatchedFileUnlocked(const QString& path);
/**
* Schedules folders whose time to sync has come.
*
* Either because a long time has passed since the last sync or
* because of previous failures.
*/
void slotScheduleFolderByTime();
void slotScheduleFolderOwningFile(const QString& path);
private:
/**
* Terminates the current folder sync.
*
* It does not switch the folder to paused state.
*/
void terminateSyncProcess();
/** Adds a new folder, does not add it to the account settings and
* does not set an account on the new folder.
*/
@@ -263,7 +222,7 @@ private:
void unloadFolder( Folder * );
/** Will start a sync after a bit of delay. */
void startScheduledSyncSoon();
void startScheduledSyncSoon(qint64 msMinimumDelay = 0);
// finds all folder configuration files
// and create the folders
@@ -279,29 +238,19 @@ private:
Folder *_currentSyncFolder;
QPointer<Folder> _lastSyncFolder;
bool _syncEnabled;
QTimer _etagPollTimer;
QPointer<RequestEtagJob> _currentEtagJob; // alias of Folder running the current RequestEtagJob
/// Watching for file changes in folders
QMap<QString, FolderWatcher*> _folderWatchers;
/// Starts regular etag query jobs
QTimer _etagPollTimer;
/// The currently running etag query
QPointer<RequestEtagJob> _currentEtagJob;
/// Watches files that couldn't be synced due to locks
QScopedPointer<LockWatcher> _lockWatcher;
/// Occasionally schedules folders
QTimer _timeScheduler;
/// Scheduled folders that should be synced as soon as possible
QQueue<Folder*> _scheduledFolders;
/// Picks the next scheduled folder and starts the sync
QTimer _startScheduledSyncTimer;
QScopedPointer<SocketApi> _socketApi;
/** The aliases of folders that shall be synced. */
QQueue<Folder*> _scheduleQueue;
/** When the timer expires one of the scheduled syncs will be started. */
QTimer _startScheduledSyncTimer;
bool _appRestartRequired;
static FolderMan *_instance;
+2 -2
Ver Arquivo
@@ -785,7 +785,7 @@ void FolderStatusModel::slotApplySelectiveSync()
foreach(const auto &it, changes) {
folder->journalDb()->avoidReadFromDbOnNextSync(it);
}
FolderMan::instance()->scheduleFolder(folder);
FolderMan::instance()->slotScheduleSync(folder);
}
}
@@ -1099,7 +1099,7 @@ void FolderStatusModel::slotSyncAllPendingBigFolders()
foreach (const auto &it, undecidedList) {
folder->journalDb()->avoidReadFromDbOnNextSync(it);
}
FolderMan::instance()->scheduleFolder(folder);
FolderMan::instance()->slotScheduleSync(folder);
}
resetFolders();
+1 -1
Ver Arquivo
@@ -126,7 +126,7 @@ void IgnoreListEditor::slotUpdateLocalIgnoreList()
// ignored (because the remote etag did not change) (issue #3172)
foreach (Folder* folder, folderMan->map()) {
folder->journalDb()->forceRemoteDiscoveryNextSync();
folderMan->scheduleFolder(folder);
folderMan->slotScheduleSync(folder);
}
ExcludedFiles::instance().reloadExcludes();
+3 -3
Ver Arquivo
@@ -66,7 +66,7 @@ void OcsJob::start()
req.setRawHeader("Ocs-APIREQUEST", "true");
req.setRawHeader("Content-Type", "application/x-www-form-urlencoded");
QUrl url = Utility::concatUrlPath(account()->url(), path());
QUrl url = Account::concatUrlPath(account()->url(), path());
QBuffer *buffer = new QBuffer;
if (_verb == "GET") {
@@ -107,7 +107,7 @@ bool OcsJob::finished()
if (!success) {
qDebug() << "Could not parse reply to"
<< _verb
<< Utility::concatUrlPath(account()->url(), path())
<< Account::concatUrlPath(account()->url(), path())
<< _params
<< ":" << replyData;
}
@@ -117,7 +117,7 @@ bool OcsJob::finished()
if (!_passStatusCodes.contains(statusCode)) {
qDebug() << "Reply to"
<< _verb
<< Utility::concatUrlPath(account()->url(), path())
<< Account::concatUrlPath(account()->url(), path())
<< _params
<< "has unexpected status code:" << statusCode << replyData;
emit ocsError(statusCode, message);
+1 -1
Ver Arquivo
@@ -460,7 +460,7 @@ void SelectiveSyncDialog::accept()
_folder->journalDb()->avoidReadFromDbOnNextSync(it);
}
folderMan->scheduleFolder(_folder);
folderMan->slotScheduleSync(_folder);
}
QDialog::accept();
}
+2 -2
Ver Arquivo
@@ -344,12 +344,12 @@ QSharedPointer<LinkShare> ShareManager::parseLinkShare(const QVariantMap &data)
url = QUrl(data.value("url").toString());
} else if (_account->serverVersionInt() >= (8 << 16)) {
// From ownCloud server version 8 on, a different share link scheme is used.
url = QUrl(Utility::concatUrlPath(_account->url(), QString("index.php/s/%1").arg(data.value("token").toString())).toString());
url = QUrl(Account::concatUrlPath(_account->url(), QString("index.php/s/%1").arg(data.value("token").toString())).toString());
} else {
QList<QPair<QString, QString>> queryArgs;
queryArgs.append(qMakePair(QString("service"), QString("files")));
queryArgs.append(qMakePair(QString("t"), data.value("token").toString()));
url = QUrl(Utility::concatUrlPath(_account->url(), QLatin1String("public.php"), queryArgs).toString());
url = QUrl(Account::concatUrlPath(_account->url(), QLatin1String("public.php"), queryArgs).toString());
}
QDate expireDate;
-5
Ver Arquivo
@@ -128,11 +128,6 @@ QNetworkReply *AbstractNetworkJob::davRequest(const QByteArray &verb, const QUrl
return addTimer(_account->davRequest(verb, url, req, data));
}
QNetworkReply *AbstractNetworkJob::multipartRequest(const QString &relPath, QNetworkRequest req, QHttpMultiPart *multiPart)
{
return addTimer(_account->multipartRequest(relPath, req, multiPart));
}
QNetworkReply* AbstractNetworkJob::getRequest(const QString &relPath)
{
return addTimer(_account->getRequest(relPath));
-2
Ver Arquivo
@@ -26,7 +26,6 @@
#include "accountfwd.h"
class QUrl;
class QHttpMultiPart;
namespace OCC {
@@ -79,7 +78,6 @@ protected:
QNetworkReply* headRequest(const QString &relPath);
QNetworkReply* headRequest(const QUrl &url);
QNetworkReply* deleteRequest(const QUrl &url);
QNetworkReply* multipartRequest(const QString &relPath, QNetworkRequest req, QHttpMultiPart *multiPart);
int maxRedirects() const { return 10; }
virtual bool finished() = 0;
+80 -33
Ver Arquivo
@@ -40,7 +40,6 @@ Account::Account(QObject *parent)
, _capabilities(QVariantMap())
, _davPath( Theme::instance()->webDavPath() )
, _wasMigrated(false)
, _bundleRequests(true)
{
qRegisterMetaType<AccountPtr>("AccountPtr");
}
@@ -77,13 +76,6 @@ AccountPtr Account::sharedFromThis()
return _sharedThis.toStrongRef();
}
QString Account::davFilesPath() const
{
//TODO DO NOT HARCODE PATH, GET IT FROM THE SERVER!!!!
QString dfp("remote.php/dav/files/");
dfp.append(_credentials->user());
return dfp;
}
QString Account::displayName() const
{
@@ -102,6 +94,30 @@ QString Account::id() const
return _id;
}
static bool isEqualExceptProtocol(const QUrl &url1, const QUrl &url2)
{
return (url1.host() != url2.host() ||
url1.port() != url2.port() ||
url1.path() != url2.path());
}
bool Account::changed(AccountPtr other, bool ignoreUrlProtocol) const
{
if (!other) {
return false;
}
bool changes = false;
if (ignoreUrlProtocol) {
changes = isEqualExceptProtocol(_url, other->_url);
} else {
changes = (_url == other->_url);
}
changes |= _credentials->changed(other->credentials());
return changes;
}
AbstractCredentials *Account::credentials() const
{
return _credentials.data();
@@ -141,7 +157,12 @@ void Account::setCredentials(AbstractCredentials *cred)
QUrl Account::davUrl() const
{
return Utility::concatUrlPath(url(), davPath());
return concatUrlPath(url(), davPath());
}
QList<QNetworkCookie> Account::lastAuthCookies() const
{
return _am->cookieJar()->cookiesForUrl(_url);
}
void Account::clearCookieJar()
@@ -188,7 +209,7 @@ QNetworkAccessManager *Account::networkAccessManager()
QNetworkReply *Account::headRequest(const QString &relPath)
{
return headRequest(Utility::concatUrlPath(url(), relPath));
return headRequest(concatUrlPath(url(), relPath));
}
QNetworkReply *Account::headRequest(const QUrl &url)
@@ -202,7 +223,7 @@ QNetworkReply *Account::headRequest(const QUrl &url)
QNetworkReply *Account::getRequest(const QString &relPath)
{
return getRequest(Utility::concatUrlPath(url(), relPath));
return getRequest(concatUrlPath(url(), relPath));
}
QNetworkReply *Account::getRequest(const QUrl &url)
@@ -225,7 +246,7 @@ QNetworkReply *Account::deleteRequest( const QUrl &url)
QNetworkReply *Account::davRequest(const QByteArray &verb, const QString &relPath, QNetworkRequest req, QIODevice *data)
{
return davRequest(verb, Utility::concatUrlPath(davUrl(), relPath), req, data);
return davRequest(verb, concatUrlPath(davUrl(), relPath), req, data);
}
QNetworkReply *Account::davRequest(const QByteArray &verb, const QUrl &url, QNetworkRequest req, QIODevice *data)
@@ -237,20 +258,6 @@ QNetworkReply *Account::davRequest(const QByteArray &verb, const QUrl &url, QNet
return _am->sendCustomRequest(req, verb, data);
}
QNetworkReply *Account::multipartRequest(const QString &relPath, QNetworkRequest req, QHttpMultiPart *multiPart)
{
return multipartRequest(Utility::concatUrlPath(url(), relPath), req, multiPart);
}
QNetworkReply *Account::multipartRequest(const QUrl &url, QNetworkRequest req, QHttpMultiPart *multiPart)
{
req.setUrl(url);
#if QT_VERSION > QT_VERSION_CHECK(4, 8, 4)
req.setSslConfiguration(this->getOrCreateSslConfig());
#endif
return _am->post(req, multiPart);
}
void Account::setCertificate(const QByteArray certficate, const QString privateKey)
{
_pemCertificate=certficate;
@@ -333,6 +340,43 @@ void Account::setUrl(const QUrl &url)
_url = url;
}
QUrl Account::concatUrlPath(const QUrl &url, const QString &concatPath,
const QList< QPair<QString, QString> > &queryItems)
{
QString path = url.path();
if (! concatPath.isEmpty()) {
// avoid '//'
if (path.endsWith('/') && concatPath.startsWith('/')) {
path.chop(1);
} // avoid missing '/'
else if (!path.endsWith('/') && !concatPath.startsWith('/')) {
path += QLatin1Char('/');
}
path += concatPath; // put the complete path together
}
QUrl tmpUrl = url;
tmpUrl.setPath(path);
if( queryItems.size() > 0 ) {
tmpUrl.setQueryItems(queryItems);
}
return tmpUrl;
}
QString Account::_configFileName;
std::unique_ptr<QSettings> Account::settingsWithGroup(const QString& group, QObject *parent)
{
if (_configFileName.isEmpty()) {
// cache file name
ConfigFile cfg;
_configFileName = cfg.configFile();
}
std::unique_ptr<QSettings> settings(new QSettings(_configFileName, QSettings::IniFormat, parent));
settings->beginGroup(group);
return settings;
}
QVariant Account::credentialSetting(const QString &key) const
{
if (_credentials) {
@@ -431,6 +475,16 @@ void Account::handleInvalidCredentials()
emit invalidCredentials();
}
bool Account::wasMigrated()
{
return _wasMigrated;
}
void Account::setMigrated(bool mig)
{
_wasMigrated = mig;
}
const Capabilities &Account::capabilities() const
{
return _capabilities;
@@ -489,13 +543,6 @@ void Account::setNonShib(bool nonShib)
}
}
void Account::setBundleRequestsIfCapable(bool bundleRequests){
_bundleRequests = bundleRequests;
}
bool Account::bundledRequestsEnabled()
{
return (_bundleRequests && _capabilities.bundledRequest()) ? true : false;
}
} // namespace OCC
+30 -37
Ver Arquivo
@@ -33,7 +33,6 @@ class QSettings;
class QNetworkReply;
class QUrl;
class QNetworkAccessManager;
class QHttpMultiPart;
namespace OCC {
@@ -57,9 +56,6 @@ public:
/**
* @brief The Account class represents an account on an ownCloud Server
* @ingroup libsync
*
* The Account has a name and url. It also has information about credentials,
* SSL errors and certificates.
*/
class OWNCLOUDSYNC_EXPORT Account : public QObject {
Q_OBJECT
@@ -70,23 +66,13 @@ public:
* @returns the (themeable) dav path for the account.
*/
QString davPath() const;
/**
* @brief The possibly themed files dav path for the account. It has
* a trailing slash.
* @returns the (themeable) files dav path for the account.
*/
QString davFilesPath() const;
void setDavPath(const QString&s) { _davPath = s; }
void setNonShib(bool nonShib);
/// Functions for bundle support
void setBundleRequestsIfCapable(bool bundleRequests);
bool bundledRequestsEnabled();
static AccountPtr create();
~Account();
void setSharedThis(AccountPtr sharedThis);
AccountPtr sharedFromThis();
/// The name of the account as shown in the toolbar
@@ -95,6 +81,17 @@ public:
/// The internal id of the account.
QString id() const;
/**
* @brief Checks the Account instance is different from @param other
*
* @returns true, if credentials or url have changed, false otherwise
*/
bool changed(AccountPtr other, bool ignoreUrlProtocol) const;
/** Holds the accounts credentials */
AbstractCredentials* credentials() const;
void setCredentials(AbstractCredentials *cred);
/** Server url of the account */
void setUrl(const QUrl &url);
QUrl url() const { return _url; }
@@ -102,12 +99,14 @@ public:
/** Returns webdav entry URL, based on url() */
QUrl davUrl() const;
/** Holds the accounts credentials */
AbstractCredentials* credentials() const;
void setCredentials(AbstractCredentials *cred);
/** set and retrieve the migration flag: if an account of a branded
* client was migrated from a former ownCloud Account, this is true
*/
void setMigrated(bool mig);
bool wasMigrated();
QList<QNetworkCookie> lastAuthCookies() const;
// For creating various network requests
QNetworkReply* headRequest(const QString &relPath);
QNetworkReply* headRequest(const QUrl &url);
QNetworkReply* getRequest(const QString &relPath);
@@ -115,9 +114,6 @@ public:
QNetworkReply* deleteRequest( const QUrl &url);
QNetworkReply* davRequest(const QByteArray &verb, const QString &relPath, QNetworkRequest req, QIODevice *data = 0);
QNetworkReply* davRequest(const QByteArray &verb, const QUrl &url, QNetworkRequest req, QIODevice *data = 0);
QNetworkReply* multipartRequest(const QString &relPath, QNetworkRequest req, QHttpMultiPart *multiPart);
QNetworkReply* multipartRequest(const QUrl &url, QNetworkRequest req, QHttpMultiPart *multiPart);
/** The ssl configuration during the first connection */
QSslConfiguration getOrCreateSslConfig();
@@ -142,21 +138,25 @@ public:
// pluggable handler
void setSslErrorHandler(AbstractSslErrorHandler *handler);
// To be called by credentials only, for storing username and the like
// static helper function
static QUrl concatUrlPath(const QUrl &url, const QString &concatPath,
const QList< QPair<QString, QString> > &queryItems = (QList<QPair<QString, QString>>()));
/** Returns a new settings pre-set in a specific group. The Settings will be created
with the given parent. If no parent is specified, the caller must destroy the settings */
static std::unique_ptr<QSettings> settingsWithGroup(const QString& group, QObject* parent = 0);
// to be called by credentials only
QVariant credentialSetting(const QString& key) const;
void setCredentialSetting(const QString& key, const QVariant &value);
/** Assign a client certificate */
void setCertificate(const QByteArray certficate = QByteArray(), const QString privateKey = QString());
/** Access the server capabilities */
const Capabilities &capabilities() const;
void setCapabilities(const QVariantMap &caps);
/** Access the server version */
const Capabilities &capabilities() const;
void setServerVersion(const QString &version);
QString serverVersion() const;
int serverVersionInt() const;
void setServerVersion(const QString &version);
/** Whether the server is too old.
*
@@ -171,7 +171,6 @@ public:
bool serverVersionUnsupported() const;
// Fixed from 8.1 https://github.com/owncloud/client/issues/3730
/** Detects a specific bug in older server versions */
bool rootEtagChangesNotOnlySubFolderEtags();
void clearCookieJar();
@@ -180,16 +179,12 @@ public:
void resetNetworkAccessManager();
QNetworkAccessManager* networkAccessManager();
/// Called by network jobs on credential errors, emits invalidCredentials()
/// Called by network jobs on credential errors.
void handleInvalidCredentials();
signals:
/// Emitted whenever there's network activity
void propagatorNetworkActivity();
/// Triggered by handleInvalidCredentials()
void invalidCredentials();
void credentialsFetched(AbstractCredentials* credentials);
void credentialsAsked(AbstractCredentials* credentials);
@@ -208,7 +203,6 @@ protected Q_SLOTS:
private:
Account(QObject *parent = 0);
void setSharedThis(AccountPtr sharedThis);
QWeakPointer<Account> _sharedThis;
QString _id;
@@ -231,7 +225,6 @@ private:
QString _pemPrivateKey;
QString _davPath; // defaults to value from theme, might be overwritten in brandings
bool _wasMigrated;
bool _bundleRequests;
friend class AccountManager;
};
-6
Ver Arquivo
@@ -46,12 +46,6 @@ bool Capabilities::sharePublicLink() const
}
}
bool Capabilities::bundledRequest() const
{
bool bundle = _capabilities["dav"].toMap()["bundlerequest"].toBool();
return bundle;
}
bool Capabilities::sharePublicLinkAllowUpload() const
{
return _capabilities["files_sharing"].toMap()["public"].toMap()["upload"].toBool();
-1
Ver Arquivo
@@ -39,7 +39,6 @@ public:
bool sharePublicLinkEnforceExpireDate() const;
int sharePublicLinkExpireDateDays() const;
bool shareResharing() const;
bool bundledRequest() const;
/// returns true if the capabilities report notifications
bool notificationsAvailable() const;
+1
Ver Arquivo
@@ -41,6 +41,7 @@ public:
*/
virtual void setAccount(Account* account);
virtual bool changed(AbstractCredentials* credentials) const = 0;
virtual QString authType() const = 0;
virtual QString user() const = 0;
virtual QNetworkAccessManager* getQNAM() const = 0;
+7
Ver Arquivo
@@ -17,6 +17,13 @@
namespace OCC
{
bool DummyCredentials::changed(AbstractCredentials* credentials) const
{
DummyCredentials* dummy(dynamic_cast< DummyCredentials* >(credentials));
return dummy == 0;
}
QString DummyCredentials::authType() const
{
return QString::fromLatin1("dummy");
+1
Ver Arquivo
@@ -27,6 +27,7 @@ public:
QString _user;
QString _password;
bool changed(AbstractCredentials* credentials) const Q_DECL_OVERRIDE;
QString authType() const Q_DECL_OVERRIDE;
QString user() const Q_DECL_OVERRIDE;
QNetworkAccessManager* getQNAM() const Q_DECL_OVERRIDE;
+18 -3
Ver Arquivo
@@ -71,6 +71,21 @@ HttpCredentials::HttpCredentials(const QString& user, const QString& password, c
{
}
bool HttpCredentials::changed(AbstractCredentials* credentials) const
{
HttpCredentials* other(qobject_cast< HttpCredentials* >(credentials));
if (!other) {
return true;
}
if (!other || (other->user() != this->user())) {
return true;
}
return false;
}
QString HttpCredentials::authType() const
{
return QString::fromLatin1("http");
@@ -132,7 +147,7 @@ void HttpCredentials::fetchFromKeychain()
_certificatePath = _account->credentialSetting(QLatin1String(certifPathC)).toString();
_certificatePasswd = _account->credentialSetting(QLatin1String(certifPasswdC)).toString();
auto settings = Utility::settingsWithGroup(Theme::instance()->appName());
auto settings = _account->settingsWithGroup(Theme::instance()->appName());
const QString kck = keychainKey(_account->url().toString(), _user );
QString key = QString::fromLatin1( "%1/data" ).arg( kck );
@@ -214,7 +229,7 @@ void HttpCredentials::invalidateToken()
}
DeletePasswordJob *job = new DeletePasswordJob(Theme::instance()->appName());
auto settings = Utility::settingsWithGroup(Theme::instance()->appName());
auto settings = _account->settingsWithGroup(Theme::instance()->appName());
settings->setParent(job); // make the job parent to make setting deleted properly
job->setSettings(settings.release());
job->setInsecureFallback(true);
@@ -265,7 +280,7 @@ void HttpCredentials::persist()
_account->setCredentialSetting(QLatin1String(certifPathC), _certificatePath);
_account->setCredentialSetting(QLatin1String(certifPasswdC), _certificatePasswd);
WritePasswordJob *job = new WritePasswordJob(Theme::instance()->appName());
auto settings = Utility::settingsWithGroup(Theme::instance()->appName());
auto settings = _account->settingsWithGroup(Theme::instance()->appName());
settings->setParent(job); // make the job parent to make setting deleted properly
job->setSettings(settings.release());
+1
Ver Arquivo
@@ -38,6 +38,7 @@ public:
explicit HttpCredentials();
HttpCredentials(const QString& user, const QString& password, const QString& certificatePath, const QString& certificatePasswd);
bool changed(AbstractCredentials* credentials) const Q_DECL_OVERRIDE;
QString authType() const Q_DECL_OVERRIDE;
QNetworkAccessManager* getQNAM() const Q_DECL_OVERRIDE;
bool ready() const Q_DECL_OVERRIDE;
+11
Ver Arquivo
@@ -80,6 +80,17 @@ TokenCredentials::TokenCredentials(const QString& user, const QString& password,
{
}
bool TokenCredentials::changed(AbstractCredentials* credentials) const
{
TokenCredentials* other(dynamic_cast< TokenCredentials* >(credentials));
if (!other || (other->user() != this->user())) {
return true;
}
return false;
}
QString TokenCredentials::authType() const
{
return QString::fromLatin1("token");
+1
Ver Arquivo
@@ -40,6 +40,7 @@ public:
TokenCredentials();
TokenCredentials(const QString& user, const QString& password, const QString &token);
bool changed(AbstractCredentials* credentials) const Q_DECL_OVERRIDE;
QString authType() const Q_DECL_OVERRIDE;
QNetworkAccessManager* getQNAM() const Q_DECL_OVERRIDE;
bool ready() const Q_DECL_OVERRIDE;
+1 -1
Ver Arquivo
@@ -680,7 +680,7 @@ void JsonApiJob::start()
{
QNetworkRequest req;
req.setRawHeader("OCS-APIREQUEST", "true");
QUrl url = Utility::concatUrlPath(account()->url(), path());
QUrl url = Account::concatUrlPath(account()->url(), path());
QList<QPair<QString, QString> > params = _additionalParams;
params << qMakePair(QString::fromLatin1("format"), QString::fromLatin1("json"));
url.setQueryItems(params);
+93 -31
Ver Arquivo
@@ -303,13 +303,6 @@ void OwncloudPropagator::start(const SyncFileItemVector& items)
directories.push(qMakePair(QString(), _rootJob.data()));
QVector<PropagatorJob*> directoriesToRemove;
QString removedDirectory;
PropagateBundle *bundledUploadRequestsJob = new PropagateBundle(this);
quint64 chunkSize = OwncloudPropagator::chunkSize();
// TODO: here we should also check somehow if bundle is not broken for specific sync - bug recovery to standard PUTs
bool bundledRequestsEnabled = _account->bundledRequestsEnabled();
foreach(const SyncFileItemPtr &item, items) {
if (!removedDirectory.isEmpty() && item->_file.startsWith(removedDirectory)) {
@@ -386,17 +379,13 @@ void OwncloudPropagator::start(const SyncFileItemVector& items)
currentDirJob->append(dir);
}
directories.push(qMakePair(item->destination() + "/" , dir));
} else if (bundledRequestsEnabled
&& (item->_instruction == CSYNC_INSTRUCTION_NEW)
&& (item->_direction == SyncFileItem::Up)
&& (item->_size < chunkSize)) {
//this will create list of bundle files to sync for that bundlejob
bundledUploadRequestsJob->append(item);
} else if (PropagateItemJob* current = createJob(item)) {
if (item->_instruction == CSYNC_INSTRUCTION_TYPE_CHANGE) {
// will delete directories, so defer execution
directoriesToRemove.prepend(current);
removedDirectory = item->_file + "/";
} else if (item->_instruction == CSYNC_INSTRUCTION_NEW || item->_instruction == CSYNC_INSTRUCTION_SYNC){
directories.top().second->appendSyncJob(current);
} else {
directories.top().second->append(current);
}
@@ -407,21 +396,6 @@ void OwncloudPropagator::start(const SyncFileItemVector& items)
_rootJob->append(it);
}
// Root job is DirectoryJob with FullParallelizm, so we need to halt bundle till all other jobs are finished.
// If there are no running jobs in root folder it means we can start with sending the bundle
// If there are running jobs, we halt and wait for finish of the other job to trigger sending
if (bundledRequestsEnabled && !bundledUploadRequestsJob->empty()) {
bundledUploadRequestsJob->_item->_direction = SyncFileItem::Direction::Up;
bundledUploadRequestsJob->_item->_type = SyncFileItem::Type::RequestsContainer;
bundledUploadRequestsJob->_item->_instruction = CSYNC_INSTRUCTION_NEW;
bundledUploadRequestsJob->_item->_size = bundledUploadRequestsJob->syncItemsSize();
bundledUploadRequestsJob->_item->_originalFile = tr("%1 file(s)").arg(bundledUploadRequestsJob->syncItemsNumber());
_rootJob->append(bundledUploadRequestsJob);
}
else{
delete bundledUploadRequestsJob;
}
connect(_rootJob.data(), SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)),
this, SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));
connect(_rootJob.data(), SIGNAL(progress(const SyncFileItem &,quint64)), this, SIGNAL(progress(const SyncFileItem &,quint64)));
@@ -621,7 +595,11 @@ bool PropagateDirectory::scheduleNextJob()
_state = Running;
if (!_firstJob && _subJobs.isEmpty()) {
finalize();
if (!_scheduledSyncJobs && _syncJobsScheduler) {
scheduleSyncJobs();
} else {
finalize();
}
return true;
}
}
@@ -652,7 +630,6 @@ bool PropagateDirectory::scheduleNextJob()
Q_ASSERT(_subJobs.at(i)->_state == Running);
auto paral = _subJobs.at(i)->parallelism();
if (paral == WaitForFinished) {
return false;
}
@@ -686,12 +663,23 @@ void PropagateDirectory::slotSubJobFinished(SyncFileItem::Status status)
// check if we finished
if (_jobsFinished >= totalJobs) {
Q_ASSERT(!_runningNow); // how can we be finished if there are still jobs running now
finalize();
if (!_scheduledSyncJobs && _syncJobsScheduler) {
scheduleSyncJobs();
} else {
finalize();
}
} else {
emit ready();
}
}
void PropagateDirectory::scheduleSyncJobs() {
_scheduledSyncJobs = true;
//at this point, scheduler will be destroyed together with _subJobs
_subJobs.append(_syncJobsScheduler.data());
emit ready();
}
void PropagateDirectory::finalize()
{
bool ok = true;
@@ -740,6 +728,80 @@ qint64 PropagateDirectory::committedDiskSpace() const
return needed;
}
// ================================================================================
PropagatorJob::JobParallelism PropagateSyncItems::parallelism()
{
return FullParallelism;
}
bool PropagateSyncItems::scheduleNextJob()
{
if (_state == Finished) {
return false;
}
if (_state == NotYetStarted) {
_state = Running;
if (_syncJobs.isEmpty()) {
finalize();
return true;
}
}
while (!_syncJobs.isEmpty()) {
PropagatorJob *next = _syncJobs.takeFirst();
if (next->_state == Finished) {
continue;
}
if (possiblyRunNextJob(next)) {
return true;
}
Q_ASSERT(next->_state == Running);
}
return false;
}
void PropagateSyncItems::slotSubJobFinished(SyncFileItem::Status status)
{
if (status == SyncFileItem::FatalError) {
abort();
_hasError = status;
finalize();
return;
} else if (status == SyncFileItem::NormalError || status == SyncFileItem::SoftError) {
_hasError = status;
}
_jobsFinished++;
// We finished processing all the jobs
// check if we finished
if (_syncJobs.isEmpty() && (_syncJobsCount == _jobsFinished)) {
finalize();
} else {
emit ready();
}
}
void PropagateSyncItems::finalize()
{
_state = Finished;
emit finished(_hasError == SyncFileItem::NoStatus ? SyncFileItem::Success : _hasError);
}
qint64 PropagateSyncItems::committedDiskSpace() const
{
qint64 needed = 0;
foreach (PropagatorJob* job, _syncJobs) {
needed += job->committedDiskSpace();
}
return needed;
}
CleanupPollsJob::~CleanupPollsJob()
{}
+76 -8
Ver Arquivo
@@ -176,6 +176,59 @@ public slots:
virtual void start() = 0;
};
/**
* @brief Propagate sync items within a specific folder, and all its sub entries.
* @ingroup libsync
*/
class PropagateSyncItems : public PropagatorJob {
Q_OBJECT
public:
// all the sub files or sub directories.
QList<PropagatorJob *> _syncJobs;
int _jobsFinished; // number of jobs that have completed
int _syncJobsCount; // number of subJobs running right now
SyncFileItem::Status _hasError; // NoStatus, or NormalError / SoftError if there was an error
PropagateSyncItems(OwncloudPropagator *propagator)
: PropagatorJob(propagator)
, _jobsFinished(0), _syncJobsCount(0), _hasError(SyncFileItem::NoStatus)
{ }
virtual ~PropagateSyncItems() {
qDeleteAll(_syncJobs);
}
void append(PropagatorJob *subJob) {
_syncJobs.append(subJob);
_syncJobsCount++;
}
virtual bool scheduleNextJob() Q_DECL_OVERRIDE;
virtual JobParallelism parallelism() Q_DECL_OVERRIDE;
virtual void abort() Q_DECL_OVERRIDE {
foreach (PropagatorJob *j, _syncJobs)
j->abort();
}
void finalize();
qint64 committedDiskSpace() const Q_DECL_OVERRIDE;
private slots:
bool possiblyRunNextJob(PropagatorJob *next) {
if (next->_state == NotYetStarted) {
connect(next, SIGNAL(finished(SyncFileItem::Status)), this, SLOT(slotSubJobFinished(SyncFileItem::Status)), Qt::QueuedConnection);
connect(next, SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)),
this, SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));
connect(next, SIGNAL(progress(const SyncFileItem &,quint64)), this, SIGNAL(progress(const SyncFileItem &,quint64)));
connect(next, SIGNAL(ready()), this, SIGNAL(ready()));
}
return next->scheduleNextJob();
}
void slotSubJobFinished(SyncFileItem::Status status);
};
/**
* @brief Propagate a directory, and all its sub entries.
@@ -187,7 +240,12 @@ public:
// e.g: create the directory
QScopedPointer<PropagateItemJob>_firstJob;
// all the sub files or sub directories.
// all the new and changed files without conflicts scheduler class
// remark: do not QScopedPointer, since this class is either deleted via qDeleteAll(_subJobs) or usual delete,
// depending on ownership determined by flag _scheduledSyncJobs
QPointer<PropagateSyncItems> _syncJobsScheduler;
// all the other file operation or sub directories.
QVector<PropagatorJob *> _subJobs;
SyncFileItemPtr _item;
@@ -195,13 +253,18 @@ public:
int _jobsFinished; // number of jobs that have completed
int _runningNow; // number of subJobs running right now
SyncFileItem::Status _hasError; // NoStatus, or NormalError / SoftError if there was an error
bool _scheduledSyncJobs; // verify if already scheduled execution of files sync jobs
explicit PropagateDirectory(OwncloudPropagator *propagator, const SyncFileItemPtr &item = SyncFileItemPtr(new SyncFileItem))
: PropagatorJob(propagator)
, _firstJob(0), _item(item), _jobsFinished(0), _runningNow(0), _hasError(SyncFileItem::NoStatus)
, _firstJob(0), _syncJobsScheduler(0), _item(item), _jobsFinished(0), _runningNow(0), _hasError(SyncFileItem::NoStatus), _scheduledSyncJobs(false)
{ }
virtual ~PropagateDirectory() {
// check whether the owner of the pointer is _subJobs or PropagateDirectory class
if (!_scheduledSyncJobs){
delete _syncJobsScheduler;
}
qDeleteAll(_subJobs);
}
@@ -209,11 +272,20 @@ public:
_subJobs.append(subJob);
}
void appendSyncJob(PropagatorJob *subJob) {
if (!_syncJobsScheduler) {
_syncJobsScheduler = new PropagateSyncItems(_propagator);
}
_syncJobsScheduler->append(subJob);
}
virtual bool scheduleNextJob() Q_DECL_OVERRIDE;
virtual JobParallelism parallelism() Q_DECL_OVERRIDE;
virtual void abort() Q_DECL_OVERRIDE {
if (_firstJob)
_firstJob->abort();
if (_syncJobsScheduler)
_syncJobsScheduler->abort();
foreach (PropagatorJob *j, _subJobs)
j->abort();
}
@@ -224,6 +296,8 @@ public:
void finalize();
void scheduleSyncJobs();
qint64 committedDiskSpace() const Q_DECL_OVERRIDE;
private slots:
@@ -242,7 +316,6 @@ private slots:
void slotSubJobFinished(SyncFileItem::Status status);
};
/**
* @brief Dummy job that just mark it as completed and ignored
* @ingroup libsync
@@ -324,10 +397,6 @@ public:
emitFinished(SyncFileItem::NormalError);
}
int runningNowAtRootJob() const {
return _rootJob->_runningNow;
}
// timeout in seconds
static int httpTimeout();
@@ -387,7 +456,6 @@ private:
friend class PropagateLocalMkdir;
friend class PropagateLocalRename;
friend class PropagateRemoteMove;
friend class PropagateBundle;
#endif
};
+5 -7
Ver Arquivo
@@ -28,13 +28,11 @@ QString Progress::asResultString( const SyncFileItem& item)
case CSYNC_INSTRUCTION_SYNC:
case CSYNC_INSTRUCTION_NEW:
case CSYNC_INSTRUCTION_TYPE_CHANGE:
if (item._type != SyncFileItem::Type::RequestsContainer) {
if (item._direction != SyncFileItem::Up) {
return QCoreApplication::translate( "progress", "Downloaded");
} else
return QCoreApplication::translate( "progress", "Uploaded");
}
return QCoreApplication::translate( "progress", "Processed Bundle");
if (item._direction != SyncFileItem::Up) {
return QCoreApplication::translate( "progress", "Downloaded");
} else {
return QCoreApplication::translate( "progress", "Uploaded");
}
case CSYNC_INSTRUCTION_CONFLICT:
return QCoreApplication::translate( "progress", "Server version downloaded, copied changed local file into conflict file");
case CSYNC_INSTRUCTION_REMOVE:
-598
Ver Arquivo
@@ -28,12 +28,10 @@
#include <json.h>
#include <QNetworkAccessManager>
#include <QHttpMultiPart>
#include <QFileInfo>
#include <QDir>
#include <cmath>
#include <cstring>
#include <QXmlStreamReader>
#if QT_VERSION < QT_VERSION_CHECK(5, 4, 2)
namespace {
@@ -896,601 +894,5 @@ void PropagateUploadFile::abortWithError(SyncFileItem::Status status, const QStr
done(status, error);
}
void PropagateBundle::start()
{
if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) {
return;
}
if (!_itemsToChecksum.isEmpty()) {
//this will add checksums and remove itself from activeJobList after is completed
_preparingBundle = true;
_propagator->_activeJobList.append(this);
return slotComputeTransmissionChecksum();
}
//this can generate itemDone(), dont add to activeJobList
startBundle();
}
void PropagateBundle::slotComputeTransmissionChecksum()
{
if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) {
return;
}
SyncFileItemPtr item = _itemsToChecksum.first();
const QString filePath = _propagator->getFilePath(item->_file);
// remember the modtime before checksumming to be able to detect a file
// change during the checksum calculation
item->_modtime = FileSystem::getModTime(filePath);
QByteArray checksumType = contentChecksumType();
// Maybe the discovery already computed the checksum?
if (item->_contentChecksumType == checksumType
&& !item->_contentChecksum.isEmpty()) {
// Reuse the content checksum as the transmission checksum if possible
const auto supportedTransmissionChecksums =
_propagator->account()->capabilities().supportedChecksumTypes();
if (supportedTransmissionChecksums.contains(checksumType)) {
slotStartUpload(checksumType, item->_contentChecksum);
return;
}
}
// Compute the transmission checksum.
auto computeChecksum = new ComputeChecksum(this);
if (uploadChecksumEnabled()) {
computeChecksum->setChecksumType(_propagator->account()->capabilities().uploadChecksumType());
} else {
computeChecksum->setChecksumType(QByteArray());
}
connect(computeChecksum, SIGNAL(done(QByteArray,QByteArray)),
SLOT(slotStartUpload(QByteArray,QByteArray)));
computeChecksum->start(filePath);
}
void PropagateBundle::slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum)
{
SyncFileItemPtr item = _itemsToChecksum.takeFirst();
item->_contentChecksum = transmissionChecksum;
item->_contentChecksumType = transmissionChecksumType;
// add this item to sync list, as it now has checksum computed
_itemsToSync.append(item);
_currentBundleSize += item->_size;
_currentRequestsNumber++;
// Remove ourselfs from the list of active job, before any posible call to itemDone()
_propagator->_activeJobList.removeOne(this);
// check if we have anything to add to our bundle
if (!_itemsToChecksum.empty()){
SyncFileItemPtr nextItem = _itemsToChecksum.first();
// if next item will exceed bundle size, send the bundle now
// otherwise, compute next checksum
if (((_currentBundleSize + nextItem->_size) >= chunkSize())
|| (_currentRequestsNumber >= checkBundledRequestsLimits())){
_currentBundleSize = 0;
_currentRequestsNumber = 0;
startBundle();
} else {
start();
}
} else {
//_itemsToChecksum is already empty, send what is left in _itemsToSync.
startBundle();
}
}
void PropagateBundle::startBundle()
{
if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) {
return;
}
// TODO: use QHttpMultiPartIODevice as as Upload Device
QHttpMultiPart *multiPart = new QHttpMultiPart(QHttpMultiPart::MixedType);
QVector<SyncFileItemPtr> syncItems;
while (!_itemsToSync.isEmpty()){
SyncFileItemPtr item = _itemsToSync.takeFirst();
const QString fullFilePath = _propagator->getFilePath(item->_file);
if (!FileSystem::fileExists(fullFilePath)) {
itemDone(item, SyncFileItem::SoftError, tr("File Removed"));
continue;
}
time_t prevModtime = item->_modtime; // the _item value was set in PropagateUploadFileQNAM::start()
// but a potential checksum calculation could have taken some time during which the file could
// have been changed again, so better check again here.
item->_modtime = FileSystem::getModTime(fullFilePath);
if( prevModtime != item->_modtime ) {
_propagator->_anotherSyncNeeded = true;
itemDone(item, SyncFileItem::SoftError, tr("Local file changed during syncing. It will be resumed."));
continue;
}
quint64 fileSize = FileSystem::getSize(fullFilePath);
item->_size = fileSize;
// But skip the file if the mtime is too close to 'now'!
// That usually indicates a file that is still being changed
// or not yet fully copied to the destination.
if (fileIsStillChanging(*item)) {
_propagator->_anotherSyncNeeded = true;
itemDone(item, SyncFileItem::SoftError, tr("Local file changed during sync."));
continue;
}
//TODO use Upload Device to support bandwith limitation on the client
QFile *file = new QFile(fullFilePath);
if (! file->open(QIODevice::ReadOnly)) {
qDebug() << "ERR: Could not prepare upload device: " << file->errorString();
// If the file is currently locked, we want to retry the sync
// when it becomes available again.
if (FileSystem::isFileLocked(fullFilePath)) {
emit _propagator->seenLockedFile(fullFilePath);
}
// Soft error because this is likely caused by the user modifying his files while syncing
itemDone(item, SyncFileItem::SoftError, tr("ERR: Could not prepare upload device"));
} else {
QHttpPart bundleContent;
bundleContent.setHeader(QNetworkRequest::ContentLengthHeader, QVariant(fileSize));
//TODO: dont use strings here!! Must change to variable in propagatorjobs.h!
bundleContent.setRawHeader("X-OC-Method", QByteArray("PUT"));
bundleContent.setRawHeader("X-OC-Path", getRemotePath(item->_file));
bundleContent.setRawHeader("X-OC-Mtime", QByteArray::number(qint64(item->_modtime)));
if (!item->_contentChecksumType.isEmpty()) {
bundleContent.setRawHeader(checkSumHeaderC,
makeChecksumHeader(item->_contentChecksumType, item->_contentChecksum));
}
if(item->_file.contains(".sys.admin#recall#")) {
// This is a file recall triggered by the admin. Note: the
// recall list file created by the admin and downloaded by the
// client (.sys.admin#recall#) also falls into this category
// (albeit users are not supposed to mess up with it)
// We use a special tag header so that the server may decide to store this file away in some admin stage area
// And not directly in the user's area (which would trigger redownloads etc).
bundleContent.setRawHeader("OC-Tag", QByteArray(".sys.admin#recall#"));
}
// File has to be written to multipart body immedietely, because we dont want 1000> open files till the reply comes.
bundleContent.setBody(file->readAll());
file->close();
multiPart->append(bundleContent);
syncItems.append(item);
}
delete file;
}
if (!syncItems.isEmpty())
{
_propagator->_activeJobList.append(this);
const QString userPath = _propagator->account()->davFilesPath();
// job takes ownership of device via a QScopedPointer. Job deletes itself when finishing
MultipartJob* job = new MultipartJob(_propagator->account(), userPath, multiPart, syncItems);
job->_duration.start();
_jobs.append(job);
connect(job, SIGNAL(finishedSignal()), this, SLOT(slotMultipartFinished()));
connect(job, SIGNAL(destroyed(QObject*)), this, SLOT(slotJobDestroyed(QObject*)));
job->start();
_preparingBundle = false;
// check if there are any items to be bundled and if you can run another parallel job
if (!_itemsToChecksum.empty() && (_propagator->_activeJobList.count() < _propagator->maximumActiveJob())) {
start();
}
} else {
delete multiPart;
}
}
//TODO this function is copied over from owncloudpropagator.cpp (not in owncloudpropagator.h)
/** Updates, creates or removes a blacklist entry for the given item.
*
* Returns whether the file is in the blacklist now.
*
*/
static bool blacklistCheck(SyncJournalDb* journal, const SyncFileItem& item)
{
SyncJournalErrorBlacklistRecord oldEntry = journal->errorBlacklistEntry(item._file);
SyncJournalErrorBlacklistRecord newEntry = SyncJournalErrorBlacklistRecord::update(oldEntry, item);
if (newEntry.isValid()) {
journal->updateErrorBlacklistEntry(newEntry);
} else if (oldEntry.isValid()) {
journal->wipeErrorBlacklistEntry(item._file);
}
return newEntry.isValid();
}
void PropagateBundle::itemDone(SyncFileItemPtr item, SyncFileItem::Status status, const QString &errorString)
{
if (item->_isRestoration) {
if( status == SyncFileItem::Success || status == SyncFileItem::Conflict) {
status = SyncFileItem::Restoration;
} else {
item->_errorString += tr("; Restoration Failed: %1").arg(errorString);
}
} else {
if( item->_errorString.isEmpty() ) {
item->_errorString = errorString;
}
}
if( _propagator->_abortRequested.fetchAndAddRelaxed(0) &&
(status == SyncFileItem::NormalError || status == SyncFileItem::FatalError)) {
// an abort request is ongoing. Change the status to Soft-Error
status = SyncFileItem::SoftError;
}
switch( status ) {
case SyncFileItem::SoftError:
case SyncFileItem::FatalError:
// do not blacklist in case of soft error or fatal error.
break;
case SyncFileItem::NormalError:
if (blacklistCheck(_propagator->_journal, *item) && item->_hasBlacklistEntry) {
// do not error if the item was, and continues to be, blacklisted
status = SyncFileItem::FileIgnored;
item->_errorString.prepend(tr("Continue blacklisting:") + " ");
}
break;
case SyncFileItem::Success:
case SyncFileItem::Restoration:
if( item->_hasBlacklistEntry ) {
// wipe blacklist entry.
_propagator->_journal->wipeErrorBlacklistEntry(item->_file);
// remove a blacklist entry in case the file was moved.
if( item->_originalFile != item->_file ) {
_propagator->_journal->wipeErrorBlacklistEntry(item->_originalFile);
}
}
break;
case SyncFileItem::Conflict:
case SyncFileItem::FileIgnored:
case SyncFileItem::NoStatus:
// nothing
break;
}
item->_status = status;
emit itemCompleted(*item, *this);
}
QByteArray PropagateBundle::getRemotePath(QString filePath){
QString remotePath(_propagator->_remoteFolder+filePath);
return remotePath.toStdString().c_str();
}
void PropagateBundle::append(const SyncFileItemPtr &bundledFile){
_size += bundledFile->_size;
_itemsToChecksum.append(bundledFile);
}
bool PropagateBundle::empty(){
return _itemsToChecksum.empty();
}
void PropagateBundle::slotMultipartFinished()
{
MultipartJob *job = qobject_cast<MultipartJob *>(sender());
Q_ASSERT(job);
slotJobDestroyed(job); // remove it from the _jobs list
qDebug() << Q_FUNC_INFO << job->reply()->request().url() << "FINISHED WITH STATUS"
<< job->reply()->error()
<< (job->reply()->error() == QNetworkReply::NoError ? QLatin1String("") : job->reply()->errorString())
<< job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute)
<< job->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute);
_propagator->_activeJobList.removeOne(this);
QNetworkReply::NetworkError err = job->reply()->error();
#if QT_VERSION < QT_VERSION_CHECK(5, 4, 2)
if (job->reply()->error() == QNetworkReply::OperationCanceledError && job->reply()->property(owncloudShouldSoftCancelPropertyName).isValid()) {
// Abort the job and try again later.
// This works around a bug in QNAM wich might reuse a non-empty buffer for the next request.
qDebug() << "Forcing job abort on HTTP connection reset with Qt < 5.4.2.";
_propagator->_anotherSyncNeeded = true;
abortWithError(SyncFileItem::SoftError, tr("Forcing job abort on HTTP connection reset with Qt < 5.4.2."));
return;
}
#endif
if (err != QNetworkReply::NoError) {
_item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
QByteArray replyContent = job->reply()->readAll();
qDebug() << replyContent; // display the XML error in the debug
QString errorString = errorMessage(job->errorString(), replyContent);
if (job->reply()->hasRawHeader("OC-ErrorString")) {
errorString = job->reply()->rawHeader("OC-ErrorString");
}
abortWithError(SyncFileItem::SoftError, errorString);
return;
}
// Parse DAV response
QXmlStreamReader reader(job->reply());
reader.addExtraNamespaceDeclaration(QXmlStreamNamespaceDeclaration("d", "DAV:"));
QString currentHref;
QString currentOcPath;
QString expectedPath(_propagator->account()->davFilesPath());
QMap<QString, QString> itemProperties;
QMap<QString, QMap<QString, QString> > responseObjectsProperties;
bool insidePropstat = false;
bool insideProp = false;
bool insideError = false;
while (!reader.atEnd()) {
QXmlStreamReader::TokenType type = reader.readNext();
QString name = reader.name().toString();
// Start elements with DAV:
if (type == QXmlStreamReader::StartElement && reader.namespaceUri() == QLatin1String("DAV:")) {
if (name == QLatin1String("href")) {
// We don't use URL encoding in our request URL (which is the expected path) (QNAM will do it for us)
// but the result will have URL encoding..
QString hrefString = QString::fromUtf8(QByteArray::fromPercentEncoding(reader.readElementText().toUtf8()));
if (!hrefString.endsWith(expectedPath)) {
qDebug() << "Invalid href" << hrefString << "expected ending with" << expectedPath;
}
currentHref = hrefString;
} else if (name == QLatin1String("response")) {
continue;
} else if (name == QLatin1String("propstat")) {
insidePropstat = true;
} else if (name == QLatin1String("status") && insidePropstat) {
QString httpStatus = reader.readElementText();
itemProperties.insert(name, httpStatus);
} else if (name == QLatin1String("prop")) {
insideProp = true;
continue;
} else if (name == QLatin1String("multistatus")) {
continue;
}
}
if (type == QXmlStreamReader::StartElement && insidePropstat && insideProp) {
if (name == QLatin1String("oc-path")){
currentOcPath = reader.readElementText(QXmlStreamReader::SkipChildElements);
} else if (name == QLatin1String("error")){
insideError = true;
} else if (name == QLatin1String("exception") && insideError){
itemProperties.insert(name, reader.readElementText(QXmlStreamReader::SkipChildElements));
} else if (name == QLatin1String("message") && insideError){
itemProperties.insert(name, reader.readElementText(QXmlStreamReader::SkipChildElements));
} else{
itemProperties.insert(name, reader.readElementText(QXmlStreamReader::SkipChildElements));
}
}
// End elements with DAV:
if (type == QXmlStreamReader::EndElement) {
if (reader.namespaceUri() == QLatin1String("DAV:")) {
if (reader.name() == "response") {
currentHref.clear();
} else if (reader.name() == "propstat") {
insidePropstat = false;
if (!currentOcPath.isEmpty()){
responseObjectsProperties.insert(currentOcPath, QMap<QString,QString>(itemProperties));
}
currentOcPath.clear();
itemProperties.clear();
} else if (reader.name() == "prop") {
insideProp = false;
} else if (reader.name() == "error") {
insideError = false;
}
}
}
}
if (reader.hasError()) {
// XML Parser error? Whatever had been emitted before will come as directoryListingIterated
qDebug() << "ERROR" << reader.errorString();
abortWithError(SyncFileItem::SoftError, tr("Cannot parse multistatus response!"));
return;
}
QVector<SyncFileItemPtr> syncItems = job->syncItems();
//get the total duration, and try to estimate sync time for single item in the bundle
quint64 avgItemDuration = job->_duration.elapsed()/syncItems.count();
foreach(SyncFileItemPtr item, syncItems) {
item->_requestDuration = avgItemDuration;
item->_responseTimeStamp = job->responseTimestamp();
slotItemFinished(item, responseObjectsProperties);
}
// performance logging
qDebug() << "*==* duration BUNDLE UPLOAD" << syncItems.count() << "items synced"
<< "average duration" << avgItemDuration;
// The job might stay alive for the whole sync, release this tiny bit of memory.
if (_jobs.empty() && _itemsToChecksum.isEmpty()){
done(SyncFileItem::Success);
} else if (!_preparingBundle){
start();
}
}
void PropagateBundle::slotItemFinished(const SyncFileItemPtr &item, QMap<QString, QMap<QString, QString> > &responseObjectsProperties)
{
QString itemFilePath(getRemotePath(item->_file));
QMap<QString, QString> fileProperties = responseObjectsProperties.value(itemFilePath);
item->_httpErrorCode = getHttpStatusCode(fileProperties.value("status"));
if (200 == item->_httpErrorCode){
// Check if the file still exists
const QString fullFilePath(_propagator->getFilePath(item->_file));
if( !FileSystem::fileExists(fullFilePath) ) {
_propagator->_anotherSyncNeeded = true;
}
if (! FileSystem::verifyFileUnchanged(fullFilePath, item->_size, item->_modtime)) {
_propagator->_anotherSyncNeeded = true;
}
//OC-FileID section
if (fileProperties.contains("oc-fileid")){
QString fid = fileProperties.value("oc-fileid");
if( !item->_fileId.isEmpty() && item->_fileId != fid ) {
qDebug() << "WARN: File ID changed!" << item->_fileId << fid;
}
item->_fileId =fid.toStdString().c_str();
}
//OC-ETag section
QByteArray ocEtag = parseEtag(fileProperties.value("oc-etag").toStdString().c_str());
QByteArray etag = parseEtag(fileProperties.value("etag").toStdString().c_str());
item->_etag = ocEtag.isEmpty() ? etag : ocEtag;
if (ocEtag.length() > 0 && ocEtag != etag) {
qDebug() << "Quite peculiar, we have an etag != OC-Etag [no problem!]" << etag << ocEtag;
}
if (fileProperties.value("x-oc-mtime") != "accepted"){
// X-OC-MTime is supported since owncloud 5.0. But not when chunking.
// Normally Owncloud 6 always puts X-OC-MTime
qWarning() << "Server does not support X-OC-MTime" << fileProperties.value("x-oc-mtime");
// Well, the mtime was not set
itemDone(item, SyncFileItem::SoftError, "Server does not support X-OC-MTime");
return;
}
} else{
//We do not check for problems with shared folder since it is only CREATE
//TODO: if the file will support update, ensure to override checkForProblemsWithShared()
qDebug() << Q_FUNC_INFO << item->_file << "ERROR WITH STATUS"
<< fileProperties.value("status")
<< fileProperties.value("exception")
<< fileProperties.value("message");
if (412 == _item->_httpErrorCode) {
// Precondition Failed: Maybe the bad etag is in the database, we need to clear the
// parent folder etag so we won't read from DB next sync.
_propagator->_journal->avoidReadFromDbOnNextSync(item->_file);
_propagator->_anotherSyncNeeded = true;
}
item->_errorString = fileProperties.value("message");
item->_status = classifyError(QNetworkReply::ContentOperationNotPermittedError, item->_httpErrorCode,
&_propagator->_anotherSyncNeeded);
itemDone(item, item->_status, item->_errorString);
return;
}
if (!_propagator->_journal->setFileRecord(SyncJournalFileRecord(*item, _propagator->getFilePath(item->_file)))) {
itemDone(item, SyncFileItem::FatalError, tr("Error writing metadata to the database"));
return;
}
// Remove from the progress database:
_propagator->_journal->setUploadInfo(item->_file, SyncJournalDb::UploadInfo());
_propagator->_journal->commit("upload file start");
itemDone(item, SyncFileItem::Success);
}
int PropagateBundle::getHttpStatusCode(const QString &status){
//if cannot read code, it means that server is not supported and raise 500 Internal Server Error
int code = 500;
QStringList statusList = status.split(QRegExp("\\s+"));
if (2 < statusList.size()){
//we expect status list to be at least 3 elements and second element to be the http error code
code = statusList.at(1).toInt();
}
return code;
}
void PropagateBundle::slotJobDestroyed(QObject* job)
{
_jobs.erase(std::remove(_jobs.begin(), _jobs.end(), job) , _jobs.end());
}
// This function is used whenever there is an error occuring and we need to raise status for whole the bundle
void PropagateBundle::abortWithError(SyncFileItem::Status status, const QString &error)
{
foreach(auto *job, _jobs) {
if (job->reply()) {
qDebug() << Q_FUNC_INFO << job;
job->reply()->abort();
}
}
done(status, error);
}
MultipartJob::~MultipartJob()
{
// Make sure that we destroy the QNetworkReply before our _device of which it keeps an internal pointer.
setReply(0);
}
void MultipartJob::start() {
QNetworkRequest req;
setReply(multipartRequest(path(), req, _multipart));
_multipart->setParent(reply()); // delete the multiPart with the job
setupConnections(reply());
if( reply()->error() != QNetworkReply::NoError ) {
qWarning() << Q_FUNC_INFO << " Network error: " << reply()->errorString();
}
connect(this, SIGNAL(networkActivity()), account().data(), SIGNAL(propagatorNetworkActivity()));
//TODO this uses _device, and we have no access here to that
// // For Qt versions not including https://codereview.qt-project.org/110150
// // Also do the runtime check if compiled with an old Qt but running with fixed one.
// // (workaround disabled on windows and mac because the binaries we ship have patched qt)
//#if QT_VERSION < QT_VERSION_CHECK(4, 8, 7)
// if (QLatin1String(qVersion()) < QLatin1String("4.8.7"))
// connect(_device.data(), SIGNAL(wasReset()), this, SLOT(slotSoftAbort()));
//#elif QT_VERSION > QT_VERSION_CHECK(5, 0, 0) && QT_VERSION < QT_VERSION_CHECK(5, 4, 2) && !defined Q_OS_WIN && !defined Q_OS_MAC
// if (QLatin1String(qVersion()) < QLatin1String("5.4.2"))
// connect(_device.data(), SIGNAL(wasReset()), this, SLOT(slotSoftAbort()));
//#endif
AbstractNetworkJob::start();
}
void MultipartJob::slotTimeout() {
qDebug() << "Timeout" << (reply() ? reply()->request().url() : path());
if (!reply())
return;
_errorString = tr("Connection Timeout");
reply()->abort();
}
#if QT_VERSION < QT_VERSION_CHECK(5, 4, 2)
void MultipartJob::slotSoftAbort() {
reply()->setProperty(owncloudShouldSoftCancelPropertyName, true);
reply()->abort();
}
#endif
}
+1 -106
Ver Arquivo
@@ -20,6 +20,7 @@
#include <QFile>
#include <QDebug>
namespace OCC {
class BandwidthManager;
@@ -122,51 +123,6 @@ private slots:
#endif
};
/**
* @brief The MultipartJob class
* @ingroup libsync
*/
class MultipartJob : public AbstractNetworkJob {
Q_OBJECT
private:
QVector<SyncFileItemPtr> _syncItems;
QHttpMultiPart* _multipart;
QString _errorString;
public:
QElapsedTimer _duration;
// Takes ownership of the device
explicit MultipartJob(AccountPtr account, const QString& path, QHttpMultiPart* multiPart, const QVector<SyncFileItemPtr> &syncItems,QObject* parent = 0)
: AbstractNetworkJob(account, path, parent), _syncItems(syncItems), _multipart(multiPart), _errorString(QString()) {}
~MultipartJob();
virtual void start() Q_DECL_OVERRIDE;
virtual bool finished() Q_DECL_OVERRIDE {
emit finishedSignal();
return true;
}
QString errorString() {
return _errorString.isEmpty() ? reply()->errorString() : _errorString;
}
QVector<SyncFileItemPtr> syncItems() const {
return _syncItems;
}
virtual void slotTimeout() Q_DECL_OVERRIDE;
signals:
void finishedSignal();
private slots:
#if QT_VERSION < 0x050402
void slotSoftAbort();
#endif
};
/**
* @brief This job implements the asynchronous PUT
*
@@ -266,66 +222,5 @@ private:
void abortWithError(SyncFileItem::Status status, const QString &error);
};
/**
* @brief The PropagateBundle class
* @ingroup libsync
*/
class PropagateBundle : public PropagateItemJob {
Q_OBJECT
private:
QLinkedList<SyncFileItemPtr> _itemsToSync;
QLinkedList<SyncFileItemPtr> _itemsToChecksum;
QVector<MultipartJob*> _jobs; /// network jobs that are currently in transit
bool _running; // Tells that all the jobs have been finished
bool _preparingBundle; // Tells that all the jobs have been finished
quint64 _size; // Tells what is the total size of _itemsToSync inside the bundle
quint64 _currentBundleSize;
quint64 _currentRequestsNumber;
quint64 _totalFiles;
// measure the performance of checksum calc and upload
Utility::StopWatch _stopWatch;
public:
PropagateBundle(OwncloudPropagator* propagator)
: PropagateItemJob(propagator, SyncFileItemPtr(new SyncFileItem)), _running(false), _preparingBundle(true), _size(0), _currentBundleSize(0), _currentRequestsNumber(0) {}
void start() Q_DECL_OVERRIDE;
void startBundle();
void append(const SyncFileItemPtr &bundledFile);
QByteArray getRemotePath(QString filePath);
bool empty();
bool scheduleNextJob() Q_DECL_OVERRIDE {
if (_running != true && _propagator->runningNowAtRootJob() == 1){
_running = true;
QMetaObject::invokeMethod(this, "start"); // We could be in a different thread (neon jobs)
}
if (_state == NotYetStarted){
_state = Running;
return true;
}
return false;
}
quint64 syncItemsSize() const { return _size; }
quint64 syncItemsNumber() const { return _itemsToChecksum.count(); }
private slots:
void slotComputeTransmissionChecksum();
void slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum);
void slotMultipartFinished();
void slotJobDestroyed(QObject *job);
void abortWithError(SyncFileItem::Status status, const QString &error);
private:
quint64 checkBundledRequestsLimits()
{
//TODO: obtain this value from the server or by other means
quint64 maximumBundledFiles = 500;
return (maximumBundledFiles/_propagator->maximumActiveJob());
}
quint64 chunkSize() const { return _propagator->chunkSize(); }
int getHttpStatusCode(const QString &status);
void slotItemFinished(const SyncFileItemPtr &item, QMap<QString, QMap<QString, QString> > &responseObjectsProperties);
void itemDone(SyncFileItemPtr item, SyncFileItem::Status status, const QString &errorString = QString());
};
}
-1
Ver Arquivo
@@ -45,7 +45,6 @@ public:
enum Type {
UnknownType = 0,
RequestsContainer = 1,
File = CSYNC_FTW_TYPE_FILE,
Directory = CSYNC_FTW_TYPE_DIR,
SoftLink = CSYNC_FTW_TYPE_SLINK
-38
Ver Arquivo
@@ -16,7 +16,6 @@
#include "utility.h"
#include "version.h"
#include "configfile.h"
// Note: This file must compile without QtGui
#include <QCoreApplication>
@@ -587,41 +586,4 @@ void Utility::sortFilenames(QStringList& fileNames)
#endif
}
QUrl Utility::concatUrlPath(const QUrl &url, const QString &concatPath,
const QList< QPair<QString, QString> > &queryItems)
{
QString path = url.path();
if (! concatPath.isEmpty()) {
// avoid '//'
if (path.endsWith('/') && concatPath.startsWith('/')) {
path.chop(1);
} // avoid missing '/'
else if (!path.endsWith('/') && !concatPath.startsWith('/')) {
path += QLatin1Char('/');
}
path += concatPath; // put the complete path together
}
QUrl tmpUrl = url;
tmpUrl.setPath(path);
if( queryItems.size() > 0 ) {
tmpUrl.setQueryItems(queryItems);
}
return tmpUrl;
}
Q_GLOBAL_STATIC(QString, g_configFileName)
std::unique_ptr<QSettings> Utility::settingsWithGroup(const QString& group, QObject *parent)
{
if (g_configFileName()->isEmpty()) {
// cache file name
ConfigFile cfg;
*g_configFileName() = cfg.configFile();
}
std::unique_ptr<QSettings> settings(new QSettings(*g_configFileName(), QSettings::IniFormat, parent));
settings->beginGroup(group);
return settings;
}
} // namespace OCC
-12
Ver Arquivo
@@ -21,10 +21,6 @@
#include <QDateTime>
#include <QElapsedTimer>
#include <QMap>
#include <QUrl>
#include <memory>
class QSettings;
namespace OCC {
@@ -146,14 +142,6 @@ namespace Utility
*/
OWNCLOUDSYNC_EXPORT void sortFilenames(QStringList& fileNames);
/** Appends concatPath and queryItems to the url */
OWNCLOUDSYNC_EXPORT QUrl concatUrlPath(
const QUrl &url, const QString &concatPath,
const QList< QPair<QString, QString> > &queryItems = (QList<QPair<QString, QString>>()));
/** Returns a new settings pre-set in a specific group. The Settings will be created
with the given parent. If no parent is specified, the caller must destroy the settings */
OWNCLOUDSYNC_EXPORT std::unique_ptr<QSettings> settingsWithGroup(const QString& group, QObject* parent = 0);
}
/** @} */ // \addtogroup
+2 -172
Ver Arquivo
@@ -18,7 +18,6 @@
#include <QtTest>
static const QUrl sRootUrl("owncloud://somehost/owncloud/remote.php/webdav/");
static const QUrl sBundleRootUrl("owncloud://somehost/remote.php/dav/files/");
inline QString generateEtag() {
return QString::number(QDateTime::currentDateTime().toMSecsSinceEpoch(), 16);
@@ -534,7 +533,6 @@ public:
}
};
class FakeErrorReply : public QNetworkReply
{
Q_OBJECT
@@ -558,169 +556,6 @@ public:
qint64 readData(char *, qint64) override { return 0; }
};
class FakeBundlePOSTReply : public QNetworkReply
{
Q_OBJECT
FileInfo *fileInfo;
QByteArray payload;
public:
FakeBundlePOSTReply(FileInfo &remoteRootFileInfo, QNetworkAccessManager::Operation op, const QNetworkRequest &request, const QByteArray &postPayload, QObject *parent)
: QNetworkReply{parent} {
setRequest(request);
QUrl rawUrl = request.url();
QString bundlePath(sBundleRootUrl.path()+rawUrl.userName());
setUrl(rawUrl);
setOperation(op);
open(QIODevice::ReadOnly);
const QString davUri{QStringLiteral("DAV:")};
const QString ocUri{QStringLiteral("http://owncloud.org/ns")};
const QString sabUri{QStringLiteral("http://sabredav.org/ns")};
QBuffer buffer{&payload};
buffer.open(QIODevice::WriteOnly);
QXmlStreamWriter xml( &buffer );
xml.writeNamespace(davUri, "d");
xml.writeNamespace(ocUri, "o");
xml.writeNamespace(sabUri, "s");
auto writeFileResponse = [&](const FileInfo &fileInfo) {
xml.writeStartElement(davUri, QStringLiteral("response"));
//TODO: no need for X-OC-PATH, href could contain that, fix client/server
xml.writeTextElement(davUri, QStringLiteral("href"), bundlePath);
xml.writeStartElement(davUri, QStringLiteral("propstat"));
xml.writeStartElement(davUri, QStringLiteral("prop"));
xml.writeTextElement(davUri, QStringLiteral("oc-etag"), fileInfo.etag);
xml.writeTextElement(davUri, QStringLiteral("etag"), fileInfo.etag);
xml.writeTextElement(davUri, QStringLiteral("oc-fileid"), fileInfo.fileId);
xml.writeTextElement(davUri, QStringLiteral("x-oc-mtime"), QStringLiteral("accepted"));
//TODO: this slash to be fixed on client/server
xml.writeTextElement(davUri, QStringLiteral("oc-path"), "/"+fileInfo.path());
xml.writeEndElement(); // prop
xml.writeTextElement(davUri, QStringLiteral("status"), "HTTP/1.1 200 OK");
xml.writeEndElement(); // propstat
xml.writeEndElement(); // response
};
auto writeFileErrorResponse = [&](const FileInfo &fileInfo, const QString &exception, const QString &message, const QString &status) {
xml.writeStartElement(davUri, QStringLiteral("response"));
//TODO: no need for X-OC-PATH, href could contain that, fix client/server
xml.writeTextElement(davUri, QStringLiteral("href"), bundlePath);
xml.writeStartElement(davUri, QStringLiteral("propstat"));
xml.writeStartElement(davUri, QStringLiteral("prop"));
xml.writeStartElement(davUri, QStringLiteral("error"));
xml.writeTextElement(sabUri, QStringLiteral("exception"), exception);
xml.writeTextElement(sabUri, QStringLiteral("message"), message);
xml.writeEndElement(); // error
//TODO: this slash to be fixed on client/server
xml.writeTextElement(davUri, QStringLiteral("oc-path"), "/"+fileInfo.path());
xml.writeEndElement(); // prop
xml.writeTextElement(davUri, QStringLiteral("status"), status);
xml.writeEndElement(); // propstat
xml.writeEndElement(); // response
};
if ("erroruser" == rawUrl.userName()) {
xml.writeStartDocument();
xml.writeStartElement(davUri, QStringLiteral("error"));
xml.writeTextElement(sabUri, QStringLiteral("exception"), QStringLiteral("OCA\\DAV\\Connector\\Sabre\\Exception\\Forbidden"));
xml.writeTextElement(sabUri, QStringLiteral("message"), QStringLiteral("URL endpoint has to be instance of \\OCA\\DAV\\Files\\FilesHome"));
xml.writeTextElement(ocUri, QStringLiteral("retry"), QStringLiteral("false"));
xml.writeTextElement(ocUri, QStringLiteral("reason"), QStringLiteral("URL endpoint has to be instance of \\OCA\\DAV\\Files\\FilesHome"));
xml.writeEndElement(); // error
xml.writeEndDocument();
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 403);
} else {
Q_ASSERT(request.url().path().endsWith(bundlePath));
xml.writeStartDocument();
xml.writeStartElement(davUri, QStringLiteral("multistatus"));
//multipart parsing
QString headerSectEnd = "\r\n\r\n";
QString headerEnd = "\r\n";
QString headerOcMethod = "X-OC-Method: ";
QString headerConLen = "Content-Length: ";
QString headerOcPath = "X-OC-Path: ";
int indexOfBody = 0;
QChar contentChar;
while(postPayload.indexOf(headerSectEnd,indexOfBody) + headerSectEnd.length() >=indexOfBody) {
//find oc-method
int indexOfheaderEnd = postPayload.indexOf(headerOcMethod,indexOfBody) + headerOcMethod.length();
int indexOfheaderBodyEnd = postPayload.indexOf(headerEnd,indexOfheaderEnd);
Q_ASSERT(postPayload.mid(indexOfheaderEnd,indexOfheaderBodyEnd-indexOfheaderEnd) == QString("PUT"));
//find oc-path
indexOfheaderEnd = postPayload.indexOf(headerOcPath,indexOfBody) + headerOcPath.length();
indexOfheaderBodyEnd = postPayload.indexOf(headerEnd,indexOfheaderEnd)-1;
QString filePath(postPayload.mid(indexOfheaderEnd+1,indexOfheaderBodyEnd-indexOfheaderEnd));
//find content-length
indexOfheaderEnd = postPayload.indexOf(headerConLen,indexOfBody) + headerConLen.length();
indexOfheaderBodyEnd = postPayload.indexOf(headerEnd,indexOfheaderEnd);
QString fileSize(postPayload.mid(indexOfheaderEnd,indexOfheaderBodyEnd-indexOfheaderEnd));
//find body content and extract first letter
indexOfheaderEnd = postPayload.indexOf(headerSectEnd,indexOfBody) + headerSectEnd.length();
indexOfBody = indexOfheaderEnd+1;
contentChar = postPayload.at(indexOfheaderEnd+1);
if ((fileInfo = remoteRootFileInfo.find(filePath))) {
fileInfo->size = fileSize.toInt();
fileInfo->contentChar = contentChar.toAscii();
} else {
// Assume that the file is filled with the same character
fileInfo = remoteRootFileInfo.create(filePath, fileSize.toInt(), contentChar.toAscii());
}
if (!fileInfo) {
abort();
return;
}
if (filePath.endsWith("normalerrorfile")){
writeFileErrorResponse(*fileInfo, QStringLiteral("Sabre\\DAV\\Exception\\BadRequest"), QStringLiteral("Method not allowed - file exists - update of the file is not supported!"), QStringLiteral("HTTP/1.1 400 Bad Request"));
} else if (filePath.endsWith("fatalerrorfile")){
writeFileErrorResponse(*fileInfo, QStringLiteral("Sabre\\DAV\\Exception\\ServiceUnavailable"), QStringLiteral("Failed to check file size"), QStringLiteral("HTTP/1.1 503 Service Unavailable"));
} else if (filePath.endsWith("softerrorfile")){
writeFileErrorResponse(*fileInfo, QStringLiteral("OCA\\DAV\\Connector\\Sabre\\Exception\\FileLocked"), QStringLiteral("Target file is locked by another process."), QStringLiteral("HTTP/1.1 423 Locked (WebDAV; RFC 4918)"));
} else {
writeFileResponse(*fileInfo);
}
}
xml.writeEndElement(); // multistatus
xml.writeEndDocument();
setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 207);
setFinished(true);
}
QMetaObject::invokeMethod(this, "respond", Qt::QueuedConnection);
}
Q_INVOKABLE void respond() {
setHeader(QNetworkRequest::ContentTypeHeader, "application/xml; charset=utf-8");
setHeader(QNetworkRequest::ContentLengthHeader, payload.size());
emit metaDataChanged();
if (bytesAvailable())
emit readyRead();
emit finished();
}
void abort() override { }
qint64 bytesAvailable() const override { return payload.size() + QIODevice::bytesAvailable(); }
qint64 readData(char *data, qint64 maxlen) override {
qint64 len = std::min(qint64{payload.size()}, maxlen);
strncpy(data, payload.constData(), len);
payload.remove(0, len);
return len;
}
};
class FakeQNAM : public QNetworkAccessManager
{
FileInfo _remoteRootFileInfo;
@@ -751,9 +586,7 @@ protected:
return new FakeDeleteReply{_remoteRootFileInfo, op, request, this};
else if (verb == QLatin1String("MOVE"))
return new FakeMoveReply{_remoteRootFileInfo, op, request, this};
else if (op == QNetworkAccessManager::PostOperation) {
return new FakeBundlePOSTReply{_remoteRootFileInfo, op, request, outgoingData->readAll(), this};
} else {
else {
qDebug() << verb << outgoingData;
Q_UNREACHABLE();
}
@@ -765,6 +598,7 @@ class FakeCredentials : public OCC::AbstractCredentials
QNetworkAccessManager *_qnam;
public:
FakeCredentials(QNetworkAccessManager *qnam) : _qnam{qnam} { }
virtual bool changed(AbstractCredentials *) const { return false; }
virtual QString authType() const { return "test"; }
virtual QString user() const { return "admin"; }
virtual QNetworkAccessManager* getQNAM() const { return _qnam; }
@@ -870,10 +704,6 @@ public:
execUntilFinished();
}
OCC::AccountPtr getAccount() {
return _account;
}
private:
static void toDisk(QDir &dir, const FileInfo &templateFi) {
foreach (const FileInfo &child, templateFi.children) {
+1 -1
Ver Arquivo
@@ -50,7 +50,7 @@ private slots:
QFETCH(QueryItems, query);
QFETCH(QString, expected);
QUrl baseUrl("http://example.com" + base);
QUrl resultUrl = Utility::concatUrlPath(baseUrl, concat, query);
QUrl resultUrl = Account::concatUrlPath(baseUrl, concat, query);
QString result = QString::fromUtf8(resultUrl.toEncoded());
QString expectedFull = "http://example.com" + expected;
QCOMPARE(result, expectedFull);
-79
Ver Arquivo
@@ -31,16 +31,6 @@ bool itemDidCompleteSuccessfully(const QSignalSpy &spy, const QString &path)
return false;
}
SyncFileItem::Status itemDidCompleteWithStatus(const QSignalSpy &spy, const QString &path)
{
for(const QList<QVariant> &args : spy) {
SyncFileItem item = args[0].value<SyncFileItem>();
if (item.destination() == path)
return item._status;
}
return SyncFileItem::NoStatus;
}
class TestSyncEngine : public QObject
{
Q_OBJECT
@@ -64,75 +54,6 @@ private slots:
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
}
void testFileUploadBundled() {
FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
QVariantMap capBundle;
capBundle["bundlerequest"] = true;
QVariantMap caps;
caps["dav"] = capBundle;
fakeFolder.getAccount()->setCapabilities(caps);
testFileUploadBundledAllFilesOK(fakeFolder);
testFileUploadBundledErrorForFile(fakeFolder);
//TODO unfinished, cannot generate NetworkError
//testFileUploadBundledNotHomeCollection(fakeFolder);
}
void testFileUploadBundledAllFilesOK(FakeFolder &fakeFolder) {
QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));
fakeFolder.localModifier().insert("A/a3");
fakeFolder.localModifier().insert("A/a4");
fakeFolder.localModifier().insert("B/b0");
fakeFolder.syncOnce();
//check separate files
QVERIFY(itemDidCompleteSuccessfully(completeSpy, "A/a3"));
QVERIFY(itemDidCompleteSuccessfully(completeSpy, "A/a4"));
QVERIFY(itemDidCompleteSuccessfully(completeSpy, "B/b0"));
//check whole bundle
QVERIFY(itemDidCompleteSuccessfully(completeSpy, ""));
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
}
void testFileUploadBundledErrorForFile(FakeFolder &fakeFolder) {
QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));
fakeFolder.localModifier().insert("A/a5");
fakeFolder.localModifier().insert("A/normalerrorfile");
fakeFolder.localModifier().insert("A/fatalerrorfile");
fakeFolder.localModifier().insert("A/softerrorfile");
fakeFolder.localModifier().insert("B/b3");
fakeFolder.syncOnce();
//check separate files
QVERIFY(itemDidCompleteSuccessfully(completeSpy, "A/a5"));
QVERIFY(SyncFileItem::NormalError == itemDidCompleteWithStatus(completeSpy, "A/normalerrorfile"));
QVERIFY(SyncFileItem::FatalError == itemDidCompleteWithStatus(completeSpy, "A/fatalerrorfile"));
QVERIFY(SyncFileItem::SoftError == itemDidCompleteWithStatus(completeSpy, "A/softerrorfile"));
QVERIFY(itemDidCompleteSuccessfully(completeSpy, "B/b3"));
//check whole bundle
QVERIFY(itemDidCompleteSuccessfully(completeSpy, ""));
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
}
void testFileUploadBundledNotHomeCollection(FakeFolder &fakeFolder) {
QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));
fakeFolder.localModifier().insert("A/a7");
fakeFolder.localModifier().insert("A/a8");
fakeFolder.localModifier().insert("B/b4");
//add the user "erroruser" which is not a FilesHomeCollection
fakeFolder.getAccount()->setUrl(QUrl(QStringLiteral("http://erroruser:admin@localhost/owncloud")));
fakeFolder.syncOnce();
//check whole bundle
QVERIFY(itemDidCompleteSuccessfully(completeSpy, ""));
QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
}
void testDirDownload() {
FakeFolder fakeFolder{FileInfo::A12_B12_C12_S12()};
QSignalSpy completeSpy(&fakeFolder.syncEngine(), SIGNAL(itemCompleted(const SyncFileItem &, const PropagatorJob &)));