diff --git a/.gitmodules b/.gitmodules index 208d31911..b51a5f87a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "binary"] path = binary url = git://github.com/owncloud/owncloud-client-binary.git +[submodule "src/3rdparty/libcrashreporter-qt"] + path = src/3rdparty/libcrashreporter-qt + url = git://github.com/dschmidt/libcrashreporter-qt.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 20543bd8c..fcde73357 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,11 +17,24 @@ endif() set(PACKAGE "${APPLICATION_SHORTNAME}-client") set( CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules ) +if(NOT CRASHREPORTER_EXECUTABLE) + set(CRASHREPORTER_EXECUTABLE "${APPLICATION_EXECUTABLE}_crash_reporter") +endif() + include(Warnings) include(${CMAKE_SOURCE_DIR}/VERSION.cmake) include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR} "${CMAKE_CURRENT_BINARY_DIR}/src/mirall/") +# disable the crashrepoter if libcrashreporter-qt is not available or we're building for ARM +if( CMAKE_SYSTEM_PROCESSOR MATCHES "arm" OR NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/libcrashreporter-qt/CMakeLists.txt") + set( WITH_CRASHREPORTER OFF ) +endif() + +if(NOT WITH_CRASHREPORTER) + message(STATUS "Build of crashreporter disabled.") +endif() + ##### ## handle DBUS for Fdo notifications if( UNIX AND NOT APPLE ) diff --git a/CPackOptions.cmake.in b/CPackOptions.cmake.in index 01fab348c..c5941d4a3 100644 --- a/CPackOptions.cmake.in +++ b/CPackOptions.cmake.in @@ -17,6 +17,8 @@ else () include ( "${CMAKE_SOURCE_DIR}/OWNCLOUD.cmake" ) endif() +set( CRASHREPORTER_EXECUTABLE @CRASHREPORTER_EXECUTABLE@) + set( BUILD_OWNCLOUD_OSX_BUNDLE @BUILD_OWNCLOUD_OSX_BUNDLE@) if(APPLE AND NOT BUILD_OWNCLOUD_OSX_BUNDLE) message( FATAL_ERROR "You're trying to build a bundle although you haven't built mirall in bundle mode.\n Add -DBUILD_OWNCLOUD_OSX_BUNDLE=ON") diff --git a/OWNCLOUD.cmake b/OWNCLOUD.cmake index c17834c69..b45249683 100644 --- a/OWNCLOUD.cmake +++ b/OWNCLOUD.cmake @@ -10,3 +10,7 @@ set( APPLICATION_REV_DOMAIN "com.owncloud.desktopclient" ) set( WIN_SETUP_BITMAP_PATH "${CMAKE_SOURCE_DIR}/admin/win/nsi" ) # set( THEME_INCLUDE "${OEM_THEME_DIR}/mytheme.h" ) # set( APPLICATION_LICENSE "${OEM_THEME_DIR}/license.txt ) + +option( WITH_CRASHREPORTER "Build crashreporter" OFF ) +set( CRASHREPORTER_SUBMIT_URL "https://crash-reports.owncloud.org/submit" CACHE string "URL for crash repoter" ) +set( CRASHREPORTER_ICON ":/owncloud-icon.png" ) diff --git a/admin/win/nsi/l10n/French.nsh b/admin/win/nsi/l10n/French.nsh index 219f03b77..aa0b495bf 100644 --- a/admin/win/nsi/l10n/French.nsh +++ b/admin/win/nsi/l10n/French.nsh @@ -38,7 +38,7 @@ StrCpy $UNINSTALL_MESSAGEBOX "Il semble que ${APPLICATION_NAME} ne soit pas inst StrCpy $UNINSTALL_ABORT "Désinstallation interrompue par l'utilisateur" StrCpy $INIT_NO_QUICK_LAUNCH "Raccourci de lancement rapide (non disponible)" StrCpy $INIT_NO_DESKTOP "Raccourci bureau (remplace l’existant)" -StrCpy $UAC_ERROR_ELEVATE "Echec d'élévation, erreur :" +StrCpy $UAC_ERROR_ELEVATE "Échec d'élévation, erreur :" StrCpy $UAC_INSTALLER_REQUIRE_ADMIN "Cet installateur requiert les droits administrateur, essayez à nouveau" StrCpy $INIT_INSTALLER_RUNNING "Une installation est déjà en cours." StrCpy $UAC_UNINSTALLER_REQUIRE_ADMIN "Ce désinstallateur requiert les droits administrateur, essayez à nouveau" diff --git a/cmake/modules/NSIS.template.in b/cmake/modules/NSIS.template.in index f19a32a45..9bbf377bf 100644 --- a/cmake/modules/NSIS.template.in +++ b/cmake/modules/NSIS.template.in @@ -9,6 +9,8 @@ !define APPLICATION_LICENSE "@APPLICATION_LICENSE@" !define WIN_SETUP_BITMAP_PATH "@WIN_SETUP_BITMAP_PATH@" +!define CRASHREPORTER_EXECUTABLE "@CRASHREPORTER_EXECUTABLE@" + ;----------------------------------------------------------------------------- ; Some installer script options (comment-out options not required) ;----------------------------------------------------------------------------- @@ -394,6 +396,9 @@ Section "${APPLICATION_NAME}" SEC_APPLICATION File "${MING_SHARE}\qt5\translations\qtbase_*.qm" File "${MING_SHARE}\qt5\translations\qtkeychain_*.qm" + ;Add crash reporter if it was built + File /nonfatal "${BUILD_PATH}/bin/${CRASHREPORTER_EXECUTABLE}.exe" + SetOutPath "$INSTDIR\platforms" File "${PLATFORMS_DLL_PATH}\qwindows.dll" SetOutPath "$INSTDIR\accessible" diff --git a/cmake/modules/QtVersionAbstraction.cmake b/cmake/modules/QtVersionAbstraction.cmake index 957d792db..7b2e01257 100644 --- a/cmake/modules/QtVersionAbstraction.cmake +++ b/cmake/modules/QtVersionAbstraction.cmake @@ -70,10 +70,15 @@ endif() qt5_add_resources(${ARGN}) endmacro() +if(NOT TOKEN_AUTH_ONLY) find_package(Qt5LinguistTools REQUIRED) macro(qt_add_translation) qt5_add_translation(${ARGN}) endmacro() + else() + macro(qt_add_translation) + endmacro() +endif() macro(qt_add_dbus_interface) qt5_add_dbus_interface(${ARGN}) diff --git a/config.h.in b/config.h.in index d739a212a..48e9d8852 100644 --- a/config.h.in +++ b/config.h.in @@ -3,6 +3,9 @@ #cmakedefine USE_INOTIFY 1 #cmakedefine WITH_QTKEYCHAIN 1 +#cmakedefine WITH_CRASHREPORTER +#cmakedefine CRASHREPORTER_EXECUTABLE "@CRASHREPORTER_EXECUTABLE@" + #cmakedefine GIT_SHA1 "@GIT_SHA1@" #cmakedefine APPLICATION_DOMAIN @APPLICATION_DOMAIN@ diff --git a/csync/src/csync.c b/csync/src/csync.c index 2cd5fa7d4..976c38be3 100644 --- a/csync/src/csync.c +++ b/csync/src/csync.c @@ -177,6 +177,8 @@ int csync_init(CSYNC *ctx) { goto out; } + ctx->remote.root_perms = 0; + ctx->status = CSYNC_STATUS_INIT; /* initialize random generator */ @@ -559,6 +561,7 @@ static void _csync_clean_ctx(CSYNC *ctx) ctx->local.list = 0; SAFE_FREE(ctx->statedb.file); + SAFE_FREE(ctx->remote.root_perms); } int csync_commit(CSYNC *ctx) { diff --git a/csync/src/csync_owncloud.c b/csync/src/csync_owncloud.c index 837ef8868..5cf08c43f 100644 --- a/csync/src/csync_owncloud.c +++ b/csync/src/csync_owncloud.c @@ -730,6 +730,14 @@ csync_vio_file_stat_t *owncloud_readdir(CSYNC *ctx, csync_vio_handle_t *dhandle) SAFE_FREE( escaped_path ); return lfs; + } else { + /* The first item is the root item, memorize its permissions */ + if (!ctx->remote.root_perms) { + if (strlen(currResource->remotePerm) > 0) { + /* Only copy if permissions contain something. Empty string means server didn't return them */ + ctx->remote.root_perms = c_strdup(currResource->remotePerm); + } + } } /* This is the target URI */ diff --git a/csync/src/csync_owncloud_util.c b/csync/src/csync_owncloud_util.c index 5a5811fa6..8b017d0a8 100644 --- a/csync/src/csync_owncloud_util.c +++ b/csync/src/csync_owncloud_util.c @@ -375,6 +375,7 @@ void fill_webdav_properties_into_resource(struct resource* newres, const ne_prop if (directDownloadCookies) { newres->directDownloadCookies = c_strdup(directDownloadCookies); } + /* DEBUG_WEBDAV("fill_webdav_properties_into_resource %s >%p< ", newres->name, perm ); */ if (perm && !perm[0]) { // special meaning for our code: server returned permissions but are empty // meaning only reading is allowed for this resource diff --git a/csync/src/csync_private.h b/csync/src/csync_private.h index ecc4f772c..764491647 100644 --- a/csync/src/csync_private.h +++ b/csync/src/csync_private.h @@ -117,8 +117,10 @@ struct csync_s { c_list_t *list; enum csync_replica_e type; int read_from_db; + const char *root_perms; /* Permission of the root folder. (Since the root folder is not in the db tree, we need to keep a separate entry.) */ } remote; + #if defined(HAVE_ICONV) && defined(WITH_ICONV) struct { iconv_t iconv_cd; diff --git a/csync/src/csync_update.c b/csync/src/csync_update.c index 80635570e..2934d71ab 100644 --- a/csync/src/csync_update.c +++ b/csync/src/csync_update.c @@ -144,6 +144,14 @@ static int _csync_detect_update(CSYNC *ctx, const char *file, len = strlen(path); + /* This code should probably be in csync_exclude, but it does not have the fs parameter. + Keep it here for now and TODO also find out if we want this for Windows + https://github.com/owncloud/mirall/issues/2086 */ + if (fs->flags & CSYNC_VIO_FILE_FLAGS_HIDDEN) { + CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "file excluded because it is a hidden file: %s", path); + return 0; + } + /* Check if file is excluded */ excluded = csync_excluded(ctx, path,type); diff --git a/csync/src/vio/csync_vio_file_stat.h b/csync/src/vio/csync_vio_file_stat.h index 21c4eaf4c..2c0782d46 100644 --- a/csync/src/vio/csync_vio_file_stat.h +++ b/csync/src/vio/csync_vio_file_stat.h @@ -42,7 +42,7 @@ typedef struct csync_vio_file_stat_s csync_vio_file_stat_t; enum csync_vio_file_flags_e { CSYNC_VIO_FILE_FLAGS_NONE = 0, CSYNC_VIO_FILE_FLAGS_SYMLINK = 1 << 0, - CSYNC_VIO_FILE_FLAGS_LOCAL = 1 << 1 + CSYNC_VIO_FILE_FLAGS_HIDDEN = 1 << 1 }; enum csync_vio_file_type_e { diff --git a/csync/src/vio/csync_vio_local.c b/csync/src/vio/csync_vio_local.c index 75383a63e..19b90e64e 100644 --- a/csync/src/vio/csync_vio_local.c +++ b/csync/src/vio/csync_vio_local.c @@ -224,6 +224,7 @@ int csync_vio_local_stat(const char *uri, csync_vio_file_stat_t *buf) { buf->type = CSYNC_VIO_FILE_TYPE_REGULAR; break; } while (0); + /* TODO Do we want to parse for CSYNC_VIO_FILE_FLAGS_HIDDEN ? */ buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_FLAGS; buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_TYPE; @@ -321,6 +322,11 @@ int csync_vio_local_stat(const char *uri, csync_vio_file_stat_t *buf) { } else { buf->flags = CSYNC_VIO_FILE_FLAGS_NONE; } +#ifdef __APPLE__ + if (sb.st_flags & UF_HIDDEN) { + buf->flags |= CSYNC_VIO_FILE_FLAGS_HIDDEN; + } +#endif buf->fields |= CSYNC_VIO_FILE_STAT_FIELDS_FLAGS; buf->device = sb.st_dev; diff --git a/csync/tests/ownCloud/t3.pl b/csync/tests/ownCloud/t3.pl index a27409ef2..55742ec67 100755 --- a/csync/tests/ownCloud/t3.pl +++ b/csync/tests/ownCloud/t3.pl @@ -141,6 +141,16 @@ move( localDir() . '3.txt', localDir() . '3_bis.txt' ); system( "echo \"new file un\" > " . localDir() . '1.txt' ); system( "echo \"new file trois\" > " . localDir() . '3.txt' ); +#also add special file with special character for next sync +#and file with special characters +createLocalFile(localDir(). 'hêllo%20th@re.txt' , 1208 ); + +csync(); +assertLocalAndRemoteDir( '', 0); + +printInfo("Move a file containing special character"); + +move(localDir(). 'hêllo%20th@re.txt', localDir(). 'hêllo%20th@re.doc'); csync(); assertLocalAndRemoteDir( '', 0); diff --git a/shell_integration/testclient/window.cpp b/shell_integration/testclient/window.cpp index ec30b51cd..3b36cbd88 100644 --- a/shell_integration/testclient/window.cpp +++ b/shell_integration/testclient/window.cpp @@ -23,7 +23,7 @@ public: LogWindowHighlighter(QTextDocument *parent = 0); protected: - void highlightBlock(const QString &text); + void highlightBlock(const QString &text) Q_DECL_OVERRIDE; void highlightHelper(const QString& text, const QTextCharFormat &format, const QString &exp); }; diff --git a/src/3rdparty/libcrashreporter-qt b/src/3rdparty/libcrashreporter-qt new file mode 160000 index 000000000..9bac9c15b --- /dev/null +++ b/src/3rdparty/libcrashreporter-qt @@ -0,0 +1 @@ +Subproject commit 9bac9c15b474ab03d0ba80d89ff126ce20a9dff8 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 094d08372..311ccd20a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,6 @@ # TODO: OSX and LIB_ONLY seem to require this to go to binary dir only +if(NOT TOKEN_AUTH_ONLY) +endif() set(BIN_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") set(synclib_NAME ${APPLICATION_EXECUTABLE}sync) @@ -7,6 +9,11 @@ add_subdirectory(libsync) if (NOT BUILD_LIBRARIES_ONLY) add_subdirectory(gui) add_subdirectory(cmd) + + if (WITH_CRASHREPORTER) + add_subdirectory(3rdparty/libcrashreporter-qt) + add_subdirectory(crashreporter) + endif() endif(NOT BUILD_LIBRARIES_ONLY) find_program(KRAZY2_EXECUTABLE krazy2) diff --git a/src/cmd/CMakeLists.txt b/src/cmd/CMakeLists.txt index 6c488cdad..440948d85 100644 --- a/src/cmd/CMakeLists.txt +++ b/src/cmd/CMakeLists.txt @@ -25,7 +25,7 @@ if(NOT BUILD_LIBRARIES_ONLY) set_target_properties(${cmd_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${BIN_OUTPUT_DIRECTORY} ) set_target_properties(${cmd_NAME} PROPERTIES - INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${LIB_INSTALL_DIR}/${APPLICATION_EXECUTABLE}" ) + INSTALL_RPATH "${LIB_INSTALL_DIR}/${APPLICATION_EXECUTABLE}" ) target_link_libraries(${cmd_NAME} ${synclib_NAME}) endif() diff --git a/src/cmd/cmd.cpp b/src/cmd/cmd.cpp index a6e71f00b..ee0419878 100644 --- a/src/cmd/cmd.cpp +++ b/src/cmd/cmd.cpp @@ -115,7 +115,7 @@ public: _sslTrusted(false) {} - QString queryPassword(bool *ok) { + QString queryPassword(bool *ok) Q_DECL_OVERRIDE { if (ok) { *ok = true; } @@ -126,7 +126,7 @@ public: _sslTrusted = isTrusted; } - bool sslIsTrusted() { + bool sslIsTrusted() Q_DECL_OVERRIDE { return _sslTrusted; } diff --git a/src/crashreporter/CMakeLists.txt b/src/crashreporter/CMakeLists.txt new file mode 100644 index 000000000..dbdb48cbb --- /dev/null +++ b/src/crashreporter/CMakeLists.txt @@ -0,0 +1,53 @@ +PROJECT( CrashReporter ) +cmake_policy(SET CMP0017 NEW) + +list(APPEND crashreporter_SOURCES main.cpp) +list(APPEND crashreporter_RC resources.qrc) + +qt_wrap_ui( crashreporter_UI_HEADERS ${crashreporter_UI} ) +qt_add_resources( crashreporter_RC_RCC ${crashreporter_RC} ) + + +# TODO: differentiate release channel +# if(BUILD_RELEASE) +# set(CRASHREPORTER_RELEASE_CHANNEL "release") +# else() + set(CRASHREPORTER_RELEASE_CHANNEL "nightly") +# endif() + +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CrashReporterConfig.h.in + ${CMAKE_CURRENT_BINARY_DIR}/CrashReporterConfig.h) + + +include_directories(${CMAKE_CURRENT_BINARY_DIR} + "../3rdparty/libcrashreporter-qt/src/" +) + + +if(NOT BUILD_LIBRARIES_ONLY) + add_executable( ${CRASHREPORTER_EXECUTABLE} WIN32 + ${crashreporter_SOURCES} + ${crashreporter_HEADERS_MOC} + ${crashreporter_UI_HEADERS} + ${crashreporter_RC_RCC} + ) + + qt5_use_modules(${CRASHREPORTER_EXECUTABLE} Widgets Network) + + set_target_properties(${CRASHREPORTER_EXECUTABLE} PROPERTIES AUTOMOC ON) + set_target_properties(${CRASHREPORTER_EXECUTABLE} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${BIN_OUTPUT_DIRECTORY} ) + set_target_properties(${CRASHREPORTER_EXECUTABLE} PROPERTIES INSTALL_RPATH "${LIB_INSTALL_DIR}/${APPLICATION_EXECUTABLE}" ) + target_link_libraries(${CRASHREPORTER_EXECUTABLE} + crashreporter-gui + ${QT_LIBRARIES} + ) + + if(BUILD_OWNCLOUD_OSX_BUNDLE) + install(TARGETS ${CRASHREPORTER_EXECUTABLE} DESTINATION ${OWNCLOUD_OSX_BUNDLE}/Contents/MacOS) + elseif(NOT BUILD_LIBRARIES_ONLY) + install(TARGETS ${CRASHREPORTER_EXECUTABLE} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + endif() +endif() diff --git a/src/crashreporter/CrashReporterConfig.h.in b/src/crashreporter/CrashReporterConfig.h.in new file mode 100644 index 000000000..d17edc7c6 --- /dev/null +++ b/src/crashreporter/CrashReporterConfig.h.in @@ -0,0 +1,16 @@ +#ifndef CRASHREPORTERCONFIG_H +#define CRASHREPORTERCONFIG_H + +#define CRASHREPORTER_BUILD_ID "@CMAKE_DATESTAMP_YEAR@@CMAKE_DATESTAMP_MONTH@@CMAKE_DATESTAMP_DAY@000000" + +#define CRASHREPORTER_RELEASE_CHANNEL "@CRASHREPORTER_RELEASE_CHANNEL@" + +#define CRASHREPORTER_PRODUCT_NAME "@APPLICATION_NAME@" + +#define CRASHREPORTER_VERSION_STRING "@MIRALL_VERSION_STRING@" + +#define CRASHREPORTER_SUBMIT_URL "@CRASHREPORTER_SUBMIT_URL@" + +#define CRASHREPORTER_ICON "@CRASHREPORTER_ICON@" + +#endif // CRASHREPORTERCONFIG_H diff --git a/src/crashreporter/main.cpp b/src/crashreporter/main.cpp new file mode 100644 index 000000000..048b9ff0b --- /dev/null +++ b/src/crashreporter/main.cpp @@ -0,0 +1,87 @@ +/* + * Copyright (C) by Dominik Schmidt + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + + +#include "CrashReporterConfig.h" + +#include + +#include +#include + +int main( int argc, char* argv[] ) +{ + QApplication app( argc, argv ); + + if ( app.arguments().size() != 2 ) + { + qDebug() << "You need to pass the .dmp file path as only argument"; + return 1; + } + + // TODO: install socorro .... + CrashReporter reporter( QUrl( CRASHREPORTER_SUBMIT_URL ), app.arguments() ); + +#ifdef CRASHREPORTER_ICON + reporter.setLogo(QPixmap(CRASHREPORTER_ICON)); +#endif + reporter.setWindowTitle(CRASHREPORTER_PRODUCT_NAME); + reporter.setText("

Sorry! " CRASHREPORTER_PRODUCT_NAME " crashed. Please tell us about it! " CRASHREPORTER_PRODUCT_NAME " has created an error report for you that can help improve the stability in the future. You can now send this report directly to the " CRASHREPORTER_PRODUCT_NAME " developers.

"); + + reporter.setReportData( "BuildID", CRASHREPORTER_BUILD_ID ); + reporter.setReportData( "ProductName", CRASHREPORTER_PRODUCT_NAME ); + reporter.setReportData( "Version", CRASHREPORTER_VERSION_STRING ); + reporter.setReportData( "ReleaseChannel", CRASHREPORTER_RELEASE_CHANNEL); + + //reporter.setReportData( "timestamp", QByteArray::number( QDateTime::currentDateTime().toTime_t() ) ); + + + // add parameters + + // << Pair("InstallTime", "1357622062") + // << Pair("Theme", "classic/1.0") + // << Pair("Version", "30") + // << Pair("id", "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}") + // << Pair("Vendor", "Mozilla") + // << Pair("EMCheckCompatibility", "true") + // << Pair("Throttleable", "0") + // << Pair("URL", "http://code.google.com/p/crashme/") + // << Pair("version", "20.0a1") + // << Pair("CrashTime", "1357770042") + // << Pair("submitted_timestamp", "2013-01-09T22:21:18.646733+00:00") + // << Pair("buildid", "20130107030932") + // << Pair("timestamp", "1357770078.646789") + // << Pair("Notes", "OpenGL: NVIDIA Corporation -- GeForce 8600M GT/PCIe/SSE2 -- 3.3.0 NVIDIA 313.09 -- texture_from_pixmap\r\n") + // << Pair("StartupTime", "1357769913") + // << Pair("FramePoisonSize", "4096") + // << Pair("FramePoisonBase", "7ffffffff0dea000") + // << Pair("Add-ons", "%7B972ce4c6-7e08-4474-a285-3208198ce6fd%7D:20.0a1,crashme%40ted.mielczarek.org:0.4") + // << Pair("SecondsSinceLastCrash", "1831736") + // << Pair("ProductName", "WaterWolf") + // << Pair("legacy_processing", "0") + // << Pair("ProductID", "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}") + + ; + + // TODO: + // send log +// QFile logFile( INSERT_FILE_PATH_HERE ); +// logFile.open( QFile::ReadOnly ); +// reporter.setReportData( "upload_file_miralllog", qCompress( logFile.readAll() ), "application/x-gzip", QFileInfo( INSERT_FILE_PATH_HERE ).fileName().toUtf8()); +// logFile.close(); + + reporter.show(); + + return app.exec(); +} diff --git a/src/crashreporter/resources.qrc b/src/crashreporter/resources.qrc new file mode 100644 index 000000000..830281e77 --- /dev/null +++ b/src/crashreporter/resources.qrc @@ -0,0 +1,5 @@ + + + ../../theme/colored/owncloud-icon-128.png + + diff --git a/src/gui/CMakeLists.txt b/src/gui/CMakeLists.txt index 85702d503..ea01d9771 100644 --- a/src/gui/CMakeLists.txt +++ b/src/gui/CMakeLists.txt @@ -224,13 +224,23 @@ set_target_properties( ${APPLICATION_EXECUTABLE} PROPERTIES ) # Only relevant for Linux? On OS X it by default properly checks in the bundle directory next to the exe set_target_properties( ${APPLICATION_EXECUTABLE} PROPERTIES - INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${LIB_INSTALL_DIR}/${APPLICATION_EXECUTABLE}" ) + INSTALL_RPATH "${LIB_INSTALL_DIR}/${APPLICATION_EXECUTABLE}" ) target_link_libraries( ${APPLICATION_EXECUTABLE} ${QT_LIBRARIES} ) target_link_libraries( ${APPLICATION_EXECUTABLE} ${synclib_NAME} ) target_link_libraries( ${APPLICATION_EXECUTABLE} updater ) target_link_libraries( ${APPLICATION_EXECUTABLE} ${OS_SPECIFIC_LINK_LIBRARIES} ) +if(WITH_CRASHREPORTER) + target_link_libraries( ${APPLICATION_EXECUTABLE} crashreporter-handler) + include_directories( "../3rdparty/libcrashreporter-qt/src/" ) + + if(UNIX AND NOT MAC) + find_package(Threads REQUIRED) + target_link_libraries( ${APPLICATION_EXECUTABLE} ${CMAKE_THREAD_LIBS_INIT}) + endif() +endif() + install(TARGETS ${APPLICATION_EXECUTABLE} RUNTIME DESTINATION bin LIBRARY DESTINATION lib diff --git a/src/gui/application.cpp b/src/gui/application.cpp index cbff86ae3..80af5ee88 100644 --- a/src/gui/application.cpp +++ b/src/gui/application.cpp @@ -87,7 +87,8 @@ Application::Application(int &argc, char **argv) : _showLogWindow(false), _logExpire(0), _logFlush(false), - _userTriggeredConnect(false) + _userTriggeredConnect(false), + _debugMode(false) { // TODO: Can't set this without breaking current config pathes // setOrganizationName(QLatin1String(APPLICATION_VENDOR)); @@ -289,6 +290,11 @@ void Application::slotToggleFolderman(int state) } +void Application::slotCrash() +{ + Utility::crash(); +} + void Application::slotConnectionValidatorResult(ConnectionValidator::Status status) { qDebug() << "Connection Validator Result: " << _conValidator->statusString(status); @@ -408,6 +414,8 @@ void Application::parseOptions(const QStringList &options) } else { showHelp(); } + } else if (option == QLatin1String("--debug")) { + _debugMode = true; } else { setHelp(); break; @@ -459,6 +467,11 @@ void Application::showHelp() displayHelpText(helpText); } +bool Application::debugMode() +{ + return _debugMode; +} + void Application::setHelp() { _helpOnly = true; diff --git a/src/gui/application.h b/src/gui/application.h index 367141011..1f68eb806 100644 --- a/src/gui/application.h +++ b/src/gui/application.h @@ -48,6 +48,7 @@ public: bool giveHelp(); void showHelp(); + bool debugMode(); public slots: // TODO: this should not be public @@ -81,6 +82,7 @@ protected slots: void slotAccountChanged(Account *newAccount, Account *oldAccount = 0); void slotCredentialsFetched(); void slotToggleFolderman(int state); + void slotCrash(); private: void setHelp(); @@ -101,6 +103,7 @@ private: int _logExpire; bool _logFlush; bool _userTriggeredConnect; + bool _debugMode; ClientProxy _proxy; diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 237d1877b..ca0c696f0 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -850,6 +850,7 @@ void Folder::slotSyncFinished() // _watcher->setEventsEnabledDelayed(2000); + // This is for sync state calculation _stateLastSyncItemsWithError = _stateLastSyncItemsWithErrorNew; _stateLastSyncItemsWithErrorNew.clear(); @@ -892,6 +893,16 @@ void Folder::slotSyncFinished() // all come in. QTimer::singleShot(200, this, SLOT(slotEmitFinishedDelayed() )); + if (!anotherSyncNeeded) { + _pollTimer.start(); + _timeSinceLastSync.restart(); + } else { + // Another sync is required. We will make sure that the poll timer occurs soon enough + // and we clear the etag to force a sync + _lastEtag.clear(); + QTimer::singleShot(1000, this, SLOT(slotPollTimerTimeout() )); + } + _timeSinceLastSync.restart(); // Increment the follow-up sync counter if necessary. diff --git a/src/gui/generalsettings.cpp b/src/gui/generalsettings.cpp index 3d497b905..38a566d82 100644 --- a/src/gui/generalsettings.cpp +++ b/src/gui/generalsettings.cpp @@ -23,6 +23,8 @@ #include "updater/updater.h" #include "updater/ocupdater.h" +#include "config.h" + #include #include @@ -55,6 +57,11 @@ GeneralSettings::GeneralSettings(QWidget *parent) : // misc connect(_ui->monoIconsCheckBox, SIGNAL(toggled(bool)), SLOT(saveMiscSettings())); + connect(_ui->crashreporterCheckBox, SIGNAL(toggled(bool)), SLOT(saveMiscSettings())); + +#ifndef WITH_CRASHREPORTER + _ui->crashreporterCheckBox->setVisible(false); +#endif // OEM themes are not obliged to ship mono icons, so there // is no point in offering an option @@ -73,6 +80,7 @@ void GeneralSettings::loadMiscSettings() ConfigFile cfgFile; _ui->monoIconsCheckBox->setChecked(cfgFile.monoIcons()); _ui->desktopNotificationsCheckBox->setChecked(cfgFile.optionalDesktopNotifications()); + _ui->crashreporterCheckBox->setChecked(cfgFile.crashReporter()); } void GeneralSettings::slotUpdateInfo() @@ -96,6 +104,7 @@ void GeneralSettings::saveMiscSettings() bool isChecked = _ui->monoIconsCheckBox->isChecked(); cfgFile.setMonoIcons(isChecked); Theme::instance()->setSystrayUseMonoIcons(isChecked); + cfgFile.setCrashReporter(_ui->crashreporterCheckBox->isChecked()); } void GeneralSettings::slotToggleLaunchOnStartup(bool enable) diff --git a/src/gui/generalsettings.ui b/src/gui/generalsettings.ui index 9ad0963db..ed2a47f7a 100644 --- a/src/gui/generalsettings.ui +++ b/src/gui/generalsettings.ui @@ -41,6 +41,13 @@ + + + + Show crash reporter + + + diff --git a/src/gui/main.cpp b/src/gui/main.cpp index 2caed2563..6dab176b8 100644 --- a/src/gui/main.cpp +++ b/src/gui/main.cpp @@ -28,6 +28,13 @@ #include "updater/updater.h" + +#include "config.h" +#ifdef WITH_CRASHREPORTER + #include "mirallconfigfile.h" + #include +#endif + #include #include @@ -51,6 +58,14 @@ int main(int argc, char **argv) Mac::CocoaInitializer cocoaInit; // RIIA #endif OCC::Application app(argc, argv); + + +#ifdef WITH_CRASHREPORTER + CrashReporter::Handler* handler = new CrashReporter::Handler( QDir::tempPath(), true, CRASHREPORTER_EXECUTABLE ); + ConfigFile cfgFile; + handler->setActive(cfgFile.crashReporter()); +#endif + #ifndef Q_OS_WIN signal(SIGPIPE, SIG_IGN); #endif diff --git a/src/gui/owncloudgui.cpp b/src/gui/owncloudgui.cpp index bc85fcbf9..25a9f057c 100644 --- a/src/gui/owncloudgui.cpp +++ b/src/gui/owncloudgui.cpp @@ -357,6 +357,11 @@ void ownCloudGui::setupContextMenu() if (!Theme::instance()->helpUrl().isEmpty()) { _contextMenu->addAction(_actionHelp); } + + if(_actionCrash) { + _contextMenu->addAction(_actionCrash); + } + _contextMenu->addSeparator(); if (isConfigured && isConnected) { _contextMenu->addAction(_actionLogout); @@ -431,6 +436,13 @@ void ownCloudGui::setupActions() _actionLogout = new QAction(tr("Sign out"), this); connect(_actionLogout, SIGNAL(triggered()), _app, SLOT(slotLogout())); + if(_app->debugMode()) { + _actionCrash = new QAction(tr("Crash now"), this); + connect(_actionCrash, SIGNAL(triggered()), _app, SLOT(slotCrash())); + } else { + _actionCrash = 0; + } + } void ownCloudGui::slotRefreshQuotaDisplay( qint64 total, qint64 used ) diff --git a/src/gui/owncloudgui.h b/src/gui/owncloudgui.h index e6cba6993..fe6932b4b 100644 --- a/src/gui/owncloudgui.h +++ b/src/gui/owncloudgui.h @@ -99,6 +99,7 @@ private: QAction *_actionRecent; QAction *_actionHelp; QAction *_actionQuit; + QAction *_actionCrash; QList _recentItemsActions; diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index 80bdf765d..b9840c017 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -34,6 +34,7 @@ endif() set(libsync_SRCS account.cpp authenticationdialog.cpp + bandwidthmanager.cpp clientproxy.cpp connectionvalidator.cpp cookiejar.cpp @@ -48,7 +49,10 @@ set(libsync_SRCS progressdispatcher.cpp propagatorjobs.cpp propagator_legacy.cpp - propagator_qnam.cpp + propagatedownload.cpp + propagateupload.cpp + propagateremotedelete.cpp + propagateremotemove.cpp quotainfo.cpp syncengine.cpp syncfilestatus.cpp @@ -151,7 +155,7 @@ set_target_properties( ${synclib_NAME} PROPERTIES SOVERSION ${MIRALL_SOVERSION} ) set_target_properties( ${synclib_NAME} PROPERTIES - INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${LIB_INSTALL_DIR}/${APPLICATION_EXECUTABLE}" ) + INSTALL_RPATH "${LIB_INSTALL_DIR}/${APPLICATION_EXECUTABLE}" ) target_link_libraries(${synclib_NAME} ${libsync_LINK_TARGETS} ) diff --git a/src/libsync/accessmanager.cpp b/src/libsync/accessmanager.cpp index 39b1a65c4..c8a21faba 100644 --- a/src/libsync/accessmanager.cpp +++ b/src/libsync/accessmanager.cpp @@ -16,6 +16,8 @@ #include #include #include +#include +#include #include "authenticationdialog.h" #include "cookiejar.h" @@ -42,9 +44,27 @@ AccessManager::AccessManager(QObject* parent) } +void AccessManager::setRawCookie(const QByteArray &rawCookie, const QUrl &url) +{ + QNetworkCookie cookie(rawCookie.left(rawCookie.indexOf('=')), + rawCookie.mid(rawCookie.indexOf('=')+1)); + qDebug() << Q_FUNC_INFO << cookie.name() << cookie.value(); + QList cookieList; + cookieList.append(cookie); + + QNetworkCookieJar *jar = cookieJar(); + jar->setCookiesFromUrl(cookieList, url); +} + QNetworkReply* AccessManager::createRequest(QNetworkAccessManager::Operation op, const QNetworkRequest& request, QIODevice* outgoingData) { QNetworkRequest newRequest(request); + + if (newRequest.hasRawHeader("cookie")) { + // This will set the cookie into the QNetworkCookieJar which will then override the cookie header + setRawCookie(request.rawHeader("cookie"), request.url()); + } + newRequest.setRawHeader(QByteArray("User-Agent"), Utility::userAgentString()); QByteArray verb = newRequest.attribute(QNetworkRequest::CustomVerbAttribute).toByteArray(); // For PROPFIND (assumed to be a WebDAV op), set xml/utf8 as content type/encoding diff --git a/src/libsync/accessmanager.h b/src/libsync/accessmanager.h index 53e7b865b..943e3b45f 100644 --- a/src/libsync/accessmanager.h +++ b/src/libsync/accessmanager.h @@ -17,6 +17,9 @@ #include "owncloudlib.h" #include +class QByteArray; +class QUrl; + namespace OCC { @@ -27,6 +30,8 @@ class OWNCLOUDSYNC_EXPORT AccessManager : public QNetworkAccessManager public: AccessManager(QObject* parent = 0); + void setRawCookie(const QByteArray &rawCookie, const QUrl &url); + protected: QNetworkReply* createRequest(QNetworkAccessManager::Operation op, const QNetworkRequest& request, QIODevice* outgoingData = 0) Q_DECL_OVERRIDE; protected slots: diff --git a/src/libsync/bandwidthmanager.cpp b/src/libsync/bandwidthmanager.cpp new file mode 100644 index 000000000..e7662c267 --- /dev/null +++ b/src/libsync/bandwidthmanager.cpp @@ -0,0 +1,404 @@ +/* + * Copyright (C) by Markus Goetz + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#include "owncloudpropagator.h" +#include "propagatedownload.h" +#include "propagateupload.h" +#include "propagatorjobs.h" +#include "propagator_legacy.h" +#include "utility.h" + +#ifdef Q_OS_WIN +#include +#include +#endif + +#include +#include + +namespace Mirall { + +// Because of the many layers of buffering inside Qt (and probably the OS and the network) +// we cannot lower this value much more. If we do, the estimated bw will be very high +// because the buffers fill fast while the actual network algorithms are not relevant yet. +static qint64 relativeLimitMeasuringTimerIntervalMsec = 1000*2; +// See also WritingState in http://code.woboq.org/qt5/qtbase/src/network/access/qhttpprotocolhandler.cpp.html#_ZN20QHttpProtocolHandler11sendRequestEv + +// FIXME At some point: +// * Register device only after the QNR received its metaDataChanged() signal +// * Incorporate Qt buffer fill state (it's a negative absolute delta). +// * Incorporate SSL overhead (percentage) +// * For relative limiting, do less measuring and more delaying+giving quota +// * For relative limiting, smoothen measurements + +BandwidthManager::BandwidthManager(OwncloudPropagator *p) : QObject(), + _propagator(p), + _relativeLimitCurrentMeasuredDevice(0), + _relativeUploadLimitProgressAtMeasuringRestart(0), + _currentUploadLimit(0), + _relativeLimitCurrentMeasuredJob(0), + _currentDownloadLimit(0) +{ + _currentUploadLimit = _propagator->_uploadLimit.fetchAndAddAcquire(0); + _currentDownloadLimit = _propagator->_downloadLimit.fetchAndAddAcquire(0); + + QObject::connect(&_switchingTimer, SIGNAL(timeout()), this, SLOT(switchingTimerExpired())); + _switchingTimer.setInterval(10*1000); + _switchingTimer.start(); + QMetaObject::invokeMethod(this, "switchingTimerExpired", Qt::QueuedConnection); + + // absolute uploads/downloads + QObject::connect(&_absoluteLimitTimer, SIGNAL(timeout()), this, SLOT(absoluteLimitTimerExpired())); + _absoluteLimitTimer.setInterval(1000); + _absoluteLimitTimer.start(); + + // Relative uploads + QObject::connect(&_relativeUploadMeasuringTimer,SIGNAL(timeout()), + this, SLOT(relativeUploadMeasuringTimerExpired())); + _relativeUploadMeasuringTimer.setInterval(relativeLimitMeasuringTimerIntervalMsec); + _relativeUploadMeasuringTimer.start(); + _relativeUploadMeasuringTimer.setSingleShot(true); // will be restarted from the delay timer + QObject::connect(&_relativeUploadDelayTimer, SIGNAL(timeout()), + this, SLOT(relativeUploadDelayTimerExpired())); + _relativeUploadDelayTimer.setSingleShot(true); // will be restarted from the measuring timer + + // Relative downloads + QObject::connect(&_relativeDownloadMeasuringTimer,SIGNAL(timeout()), + this, SLOT(relativeDownloadMeasuringTimerExpired())); + _relativeDownloadMeasuringTimer.setInterval(relativeLimitMeasuringTimerIntervalMsec); + _relativeDownloadMeasuringTimer.start(); + _relativeDownloadMeasuringTimer.setSingleShot(true); // will be restarted from the delay timer + QObject::connect(&_relativeDownloadDelayTimer, SIGNAL(timeout()), + this, SLOT(relativeDownloadDelayTimerExpired())); + _relativeDownloadDelayTimer.setSingleShot(true); // will be restarted from the measuring timer +} + +void BandwidthManager::registerUploadDevice(UploadDevice *p) +{ + qDebug() << Q_FUNC_INFO << p; + _absoluteUploadDeviceList.append(p); + _relativeUploadDeviceList.append(p); + QObject::connect(p, SIGNAL(destroyed(QObject*)), this, SLOT(unregisterUploadDevice(QObject*))); + + if (usingAbsoluteUploadLimit()) { + p->setBandwidthLimited(true); + p->setChoked(false); + } else if (usingRelativeUploadLimit()) { + p->setBandwidthLimited(true); + p->setChoked(true); + } else { + p->setBandwidthLimited(false); + p->setChoked(false); + } +} + +void BandwidthManager::unregisterUploadDevice(QObject *o) +{ + UploadDevice *p = qobject_cast(o); + if (p) { + unregisterUploadDevice(p); + } +} + +void BandwidthManager::unregisterUploadDevice(UploadDevice* p) +{ + qDebug() << Q_FUNC_INFO << p; + _absoluteUploadDeviceList.removeAll(p); + _relativeUploadDeviceList.removeAll(p); + if (p == _relativeLimitCurrentMeasuredDevice) { + _relativeLimitCurrentMeasuredDevice = 0; + } +} + +void BandwidthManager::registerDownloadJob(GETFileJob* j) +{ + qDebug() << Q_FUNC_INFO << j; + _downloadJobList.append(j); + QObject::connect(j, SIGNAL(destroyed(QObject*)), this, SLOT(unregisterDownloadJob(QObject*))); + + if (usingAbsoluteDownloadLimit()) { + j->setBandwidthLimited(true); + j->setChoked(false); + } else if (usingRelativeDownloadLimit()) { + j->setBandwidthLimited(true); + j->setChoked(true); + } else { + j->setBandwidthLimited(false); + j->setChoked(false); + } +} + +void BandwidthManager::unregisterDownloadJob(GETFileJob* j) +{ + _downloadJobList.removeAll(j); +} + +void BandwidthManager::unregisterDownloadJob(QObject* o) +{ + GETFileJob *p = qobject_cast(o); + if (p) { + unregisterDownloadJob(p); + } +} + +void BandwidthManager::relativeUploadMeasuringTimerExpired() +{ + if (!usingRelativeUploadLimit() || _relativeUploadDeviceList.count() == 0) { + // Not in this limiting mode, just wait 1 sec to continue the cycle + _relativeUploadDelayTimer.setInterval(1000); + _relativeUploadDelayTimer.start(); + return; + } + if (_relativeLimitCurrentMeasuredDevice == 0) { + qDebug() << Q_FUNC_INFO << "No device set, just waiting 1 sec"; + _relativeUploadDelayTimer.setInterval(1000); + _relativeUploadDelayTimer.start(); + return; + } + + qDebug() << Q_FUNC_INFO << _relativeUploadDeviceList.count() << "Starting Delay"; + + qint64 relativeLimitProgressMeasured = (_relativeLimitCurrentMeasuredDevice->_readWithProgress + + _relativeLimitCurrentMeasuredDevice->_read) / 2; + qint64 relativeLimitProgressDifference = relativeLimitProgressMeasured - _relativeUploadLimitProgressAtMeasuringRestart; + qDebug() << Q_FUNC_INFO << _relativeUploadLimitProgressAtMeasuringRestart + << relativeLimitProgressMeasured << relativeLimitProgressDifference; + + qint64 speedkBPerSec = (relativeLimitProgressDifference / relativeLimitMeasuringTimerIntervalMsec*1000.0) / 1024.0; + qDebug() << Q_FUNC_INFO << relativeLimitProgressDifference/1024 <<"kB =>" << speedkBPerSec << "kB/sec on full speed (" + << _relativeLimitCurrentMeasuredDevice->_readWithProgress << _relativeLimitCurrentMeasuredDevice->_read + << qAbs(_relativeLimitCurrentMeasuredDevice->_readWithProgress + - _relativeLimitCurrentMeasuredDevice->_read) << ")"; + + qint64 uploadLimitPercent = -_currentUploadLimit; + // don't use too extreme values + uploadLimitPercent = qMin(uploadLimitPercent, qint64(90)); + uploadLimitPercent = qMax(qint64(10), uploadLimitPercent); + qint64 wholeTimeMsec = (100.0 / uploadLimitPercent) * relativeLimitMeasuringTimerIntervalMsec; + qint64 waitTimeMsec = wholeTimeMsec - relativeLimitMeasuringTimerIntervalMsec; + qint64 realWaitTimeMsec = waitTimeMsec + wholeTimeMsec; + qDebug() << Q_FUNC_INFO << waitTimeMsec << " - "<< realWaitTimeMsec << + " msec for " << uploadLimitPercent << "%"; + qDebug() << Q_FUNC_INFO << "XXXX" << uploadLimitPercent << relativeLimitMeasuringTimerIntervalMsec; + + // We want to wait twice as long since we want to give all + // devices the same quota we used now since we don't want + // any upload to timeout + _relativeUploadDelayTimer.setInterval(realWaitTimeMsec); + _relativeUploadDelayTimer.start(); + + int deviceCount = _relativeUploadDeviceList.count(); + qint64 quotaPerDevice = relativeLimitProgressDifference * (uploadLimitPercent / 100.0) / deviceCount + 1.0; + qDebug() << Q_FUNC_INFO << "YYYY" << relativeLimitProgressDifference << uploadLimitPercent << deviceCount; + Q_FOREACH(UploadDevice *ud, _relativeUploadDeviceList) { + ud->setBandwidthLimited(true); + ud->setChoked(false); + ud->giveBandwidthQuota(quotaPerDevice); + qDebug() << Q_FUNC_INFO << "Gave" << quotaPerDevice/1024.0 << "kB to" << ud; + } + _relativeLimitCurrentMeasuredDevice = 0; +} + +void BandwidthManager::relativeUploadDelayTimerExpired() +{ + // Switch to measuring state + _relativeUploadMeasuringTimer.start(); // always start to continue the cycle + + if (!usingRelativeUploadLimit()) { + return; // oh, not actually needed + } + + if (_relativeUploadDeviceList.isEmpty()) { + return; + } + + qDebug() << Q_FUNC_INFO << _relativeUploadDeviceList.count() << "Starting measuring"; + + // Take first device and then append it again (= we round robin all devices) + _relativeLimitCurrentMeasuredDevice = _relativeUploadDeviceList.takeFirst(); + _relativeUploadDeviceList.append(_relativeLimitCurrentMeasuredDevice); + + _relativeUploadLimitProgressAtMeasuringRestart = (_relativeLimitCurrentMeasuredDevice->_readWithProgress + + _relativeLimitCurrentMeasuredDevice->_read) / 2; + _relativeLimitCurrentMeasuredDevice->setBandwidthLimited(false); + _relativeLimitCurrentMeasuredDevice->setChoked(false); + + // choke all other UploadDevices + Q_FOREACH(UploadDevice *ud, _relativeUploadDeviceList) { + if (ud != _relativeLimitCurrentMeasuredDevice) { + ud->setBandwidthLimited(true); + ud->setChoked(true); + } + } + + // now we're in measuring state +} + +// for downloads: +void BandwidthManager::relativeDownloadMeasuringTimerExpired() +{ + if (!usingRelativeDownloadLimit() || _downloadJobList.count() == 0) { + // Not in this limiting mode, just wait 1 sec to continue the cycle + _relativeDownloadDelayTimer.setInterval(1000); + _relativeDownloadDelayTimer.start(); + return; + } + if (_relativeLimitCurrentMeasuredJob == 0) { + qDebug() << Q_FUNC_INFO << "No job set, just waiting 1 sec"; + _relativeDownloadDelayTimer.setInterval(1000); + _relativeDownloadDelayTimer.start(); + return; + } + + qDebug() << Q_FUNC_INFO << _downloadJobList.count() << "Starting Delay"; + + qint64 relativeLimitProgressMeasured = _relativeLimitCurrentMeasuredJob->currentDownloadPosition(); + qint64 relativeLimitProgressDifference = relativeLimitProgressMeasured - _relativeDownloadLimitProgressAtMeasuringRestart; + qDebug() << Q_FUNC_INFO << _relativeDownloadLimitProgressAtMeasuringRestart + << relativeLimitProgressMeasured << relativeLimitProgressDifference; + + qint64 speedkBPerSec = (relativeLimitProgressDifference / relativeLimitMeasuringTimerIntervalMsec*1000.0) / 1024.0; + qDebug() << Q_FUNC_INFO << relativeLimitProgressDifference/1024 <<"kB =>" << speedkBPerSec << "kB/sec on full speed (" + << _relativeLimitCurrentMeasuredJob->currentDownloadPosition() ; + + qint64 downloadLimitPercent = -_currentDownloadLimit; + // don't use too extreme values + downloadLimitPercent = qMin(downloadLimitPercent, qint64(90)); + downloadLimitPercent = qMax(qint64(10), downloadLimitPercent); + qint64 wholeTimeMsec = (100.0 / downloadLimitPercent) * relativeLimitMeasuringTimerIntervalMsec; + qint64 waitTimeMsec = wholeTimeMsec - relativeLimitMeasuringTimerIntervalMsec; + qint64 realWaitTimeMsec = waitTimeMsec + wholeTimeMsec; + qDebug() << Q_FUNC_INFO << waitTimeMsec << " - "<< realWaitTimeMsec << + " msec for " << downloadLimitPercent << "%"; + qDebug() << Q_FUNC_INFO << "XXXX" << downloadLimitPercent << relativeLimitMeasuringTimerIntervalMsec; + + // We want to wait twice as long since we want to give all + // devices the same quota we used now since we don't want + // any upload to timeout + _relativeDownloadDelayTimer.setInterval(realWaitTimeMsec); + _relativeDownloadDelayTimer.start(); + + int jobCount = _downloadJobList.count(); + qint64 quota = relativeLimitProgressDifference * (downloadLimitPercent / 100.0); +// if (quota > 20*1024) { +// qDebug() << "======== ADJUSTING QUOTA FROM " << quota << " TO " << quota - 20*1024; +// quota -= 20*1024; +// } + qint64 quotaPerJob = quota / jobCount + 1.0; + qDebug() << Q_FUNC_INFO << "YYYY" << relativeLimitProgressDifference << downloadLimitPercent << jobCount; + Q_FOREACH(GETFileJob *gfj, _downloadJobList) { + gfj->setBandwidthLimited(true); + gfj->setChoked(false); + gfj->giveBandwidthQuota(quotaPerJob); + qDebug() << Q_FUNC_INFO << "Gave" << quotaPerJob/1024.0 << "kB to" << gfj; + } + _relativeLimitCurrentMeasuredDevice = 0; +} + +void BandwidthManager::relativeDownloadDelayTimerExpired() +{ + // Switch to measuring state + _relativeDownloadMeasuringTimer.start(); // always start to continue the cycle + + if (!usingRelativeDownloadLimit()) { + return; // oh, not actually needed + } + + if (_downloadJobList.isEmpty()) { + //qDebug() << Q_FUNC_INFO << _downloadJobList.count() << "No jobs?"; + return; + } + + qDebug() << Q_FUNC_INFO << _downloadJobList.count() << "Starting measuring"; + + // Take first device and then append it again (= we round robin all devices) + _relativeLimitCurrentMeasuredJob = _downloadJobList.takeFirst(); + _downloadJobList.append(_relativeLimitCurrentMeasuredJob); + + _relativeDownloadLimitProgressAtMeasuringRestart = _relativeLimitCurrentMeasuredJob->currentDownloadPosition(); + _relativeLimitCurrentMeasuredJob->setBandwidthLimited(false); + _relativeLimitCurrentMeasuredJob->setChoked(false); + + // choke all other UploadDevices + Q_FOREACH(GETFileJob *gfj, _downloadJobList) { + if (gfj != _relativeLimitCurrentMeasuredJob) { + gfj->setBandwidthLimited(true); + gfj->setChoked(true); + } + } + + // now we're in measuring state +} + +// end downloads + +void BandwidthManager::switchingTimerExpired() { + qint64 newUploadLimit = _propagator->_uploadLimit.fetchAndAddAcquire(0); + if (newUploadLimit != _currentUploadLimit) { + qDebug() << Q_FUNC_INFO << "Upload Bandwidth limit changed" << _currentUploadLimit << newUploadLimit; + _currentUploadLimit = newUploadLimit; + Q_FOREACH(UploadDevice *ud, _relativeUploadDeviceList) { + if (newUploadLimit == 0) { + ud->setBandwidthLimited(false); + ud->setChoked(false); + } else if (newUploadLimit > 0) { + ud->setBandwidthLimited(true); + ud->setChoked(false); + } else if (newUploadLimit < 0) { + ud->setBandwidthLimited(true); + ud->setChoked(true); + } + } + } + qint64 newDownloadLimit = _propagator->_downloadLimit.fetchAndAddAcquire(0); + if (newDownloadLimit != _currentDownloadLimit) { + qDebug() << Q_FUNC_INFO << "Download Bandwidth limit changed" << _currentDownloadLimit << newDownloadLimit; + _currentDownloadLimit = newDownloadLimit; + Q_FOREACH(GETFileJob *j, _downloadJobList) { + if (usingAbsoluteDownloadLimit()) { + j->setBandwidthLimited(true); + j->setChoked(false); + } else if (usingRelativeDownloadLimit()) { + j->setBandwidthLimited(true); + j->setChoked(true); + } else { + j->setBandwidthLimited(false); + j->setChoked(false); + } + } + } +} + +void BandwidthManager::absoluteLimitTimerExpired() +{ + if (usingAbsoluteUploadLimit() && _absoluteUploadDeviceList.count() > 0) { + qint64 quotaPerDevice = _currentUploadLimit / qMax(1, _absoluteUploadDeviceList.count()); + qDebug() << Q_FUNC_INFO << quotaPerDevice << _absoluteUploadDeviceList.count() << _currentUploadLimit; + Q_FOREACH(UploadDevice *device, _absoluteUploadDeviceList) { + device->giveBandwidthQuota(quotaPerDevice); + qDebug() << Q_FUNC_INFO << "Gave " << quotaPerDevice/1024.0 << " kB to" << device; + } + } + if (usingAbsoluteDownloadLimit() && _downloadJobList.count() > 0) { + qint64 quotaPerJob = _currentDownloadLimit / qMax(1, _downloadJobList.count()); + qDebug() << Q_FUNC_INFO << quotaPerJob << _downloadJobList.count() << _currentDownloadLimit; + Q_FOREACH(GETFileJob *j, _downloadJobList) { + j->giveBandwidthQuota(quotaPerJob); + qDebug() << Q_FUNC_INFO << "Gave " << quotaPerJob/1024.0 << " kB to" << j; + } + } +} + + +} diff --git a/src/libsync/bandwidthmanager.h b/src/libsync/bandwidthmanager.h new file mode 100644 index 000000000..6b705f970 --- /dev/null +++ b/src/libsync/bandwidthmanager.h @@ -0,0 +1,83 @@ +/* + * Copyright (C) by Markus Goetz + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#ifndef BANDWIDTHMANAGER_H +#define BANDWIDTHMANAGER_H + +#include +#include +#include +#include + +namespace Mirall { + +class UploadDevice; +class GETFileJob; +class OwncloudPropagator; + +class BandwidthManager : public QObject { + Q_OBJECT +public: + BandwidthManager(OwncloudPropagator *p); + + bool usingAbsoluteUploadLimit() { return _currentUploadLimit > 0; } + bool usingRelativeUploadLimit() { return _currentUploadLimit < 0; } + bool usingAbsoluteDownloadLimit() { return _currentDownloadLimit > 0; } + bool usingRelativeDownloadLimit() { return _currentDownloadLimit < 0; } + + +public slots: + void registerUploadDevice(UploadDevice*); + void unregisterUploadDevice(UploadDevice*); + void unregisterUploadDevice(QObject*); + + void registerDownloadJob(GETFileJob*); + void unregisterDownloadJob(GETFileJob*); + void unregisterDownloadJob(QObject*); + + void absoluteLimitTimerExpired(); + void switchingTimerExpired(); + + void relativeUploadMeasuringTimerExpired(); + void relativeUploadDelayTimerExpired(); + + void relativeDownloadMeasuringTimerExpired(); + void relativeDownloadDelayTimerExpired(); + +private: + QTimer _switchingTimer; // for switching between absolute and relative bw limiting + OwncloudPropagator *_propagator; // FIXME this timer and this variable should be replaced + // by the propagator emitting the changed limit values to us as signal + + QTimer _absoluteLimitTimer; // for absolute up/down bw limiting + + QLinkedList _absoluteUploadDeviceList; + QLinkedList _relativeUploadDeviceList; // FIXME merge with list above ^^ + QTimer _relativeUploadMeasuringTimer; + QTimer _relativeUploadDelayTimer; // for relative bw limiting, we need to wait this amount before measuring again + UploadDevice *_relativeLimitCurrentMeasuredDevice; // the device measured + qint64 _relativeUploadLimitProgressAtMeasuringRestart; // for measuring how much progress we made at start + qint64 _currentUploadLimit; + + QLinkedList _downloadJobList; + QTimer _relativeDownloadMeasuringTimer; + QTimer _relativeDownloadDelayTimer; // for relative bw limiting, we need to wait this amount before measuring again + GETFileJob *_relativeLimitCurrentMeasuredJob; // the device measured + qint64 _relativeDownloadLimitProgressAtMeasuringRestart; // for measuring how much progress we made at start + qint64 _currentDownloadLimit; +}; + +} + +#endif diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp index a78da42b3..23df4db76 100644 --- a/src/libsync/configfile.cpp +++ b/src/libsync/configfile.cpp @@ -44,6 +44,7 @@ static const char caCertsKeyC[] = "CaCertificates"; static const char remotePollIntervalC[] = "remotePollInterval"; static const char forceSyncIntervalC[] = "forceSyncInterval"; static const char monoIconsC[] = "monoIcons"; +static const char crashReporterC[] = "crashReporter"; static const char optionalDesktopNoficationsC[] = "optionalDesktopNotifications"; static const char skipUpdateCheckC[] = "skipUpdateCheck"; static const char geometryC[] = "geometry"; @@ -553,4 +554,16 @@ void ConfigFile::setMonoIcons(bool useMonoIcons) settings.setValue(QLatin1String(monoIconsC), useMonoIcons); } +bool MirallConfigFile::crashReporter() const +{ + QSettings settings(configFile(), QSettings::IniFormat); + return settings.value(QLatin1String(crashReporterC), true).toBool(); +} + +void MirallConfigFile::setCrashReporter(bool enabled) +{ + QSettings settings(configFile(), QSettings::IniFormat); + settings.setValue(QLatin1String(crashReporterC), enabled); +} + } diff --git a/src/libsync/configfile.h b/src/libsync/configfile.h index a82ea8280..64d98df46 100644 --- a/src/libsync/configfile.h +++ b/src/libsync/configfile.h @@ -68,6 +68,9 @@ public: bool monoIcons() const; void setMonoIcons(bool); + bool crashReporter() const; + void setCrashReporter(bool enabled); + // proxy settings void setProxyType(int proxyType, const QString& host = QString(), diff --git a/src/libsync/creds/tokencredentials.cpp b/src/libsync/creds/tokencredentials.cpp index 4faae0d2a..7c8ccebec 100644 --- a/src/libsync/creds/tokencredentials.cpp +++ b/src/libsync/creds/tokencredentials.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include "account.h" #include "accessmanager.h" @@ -88,7 +89,9 @@ protected: QByteArray credHash = QByteArray(_cred->user().toUtf8()+":"+_cred->password().toUtf8()).toBase64(); req.setRawHeader(QByteArray("Authorization"), QByteArray("Basic ") + credHash); - req.setRawHeader("Cookie", _cred->_token.toUtf8()); // analogous to neon in syncContextPreStart + // A pre-authenticated cookie + QByteArray token = _cred->_token.toUtf8(); + setRawCookie(token, request.url()); return AccessManager::createRequest(op, req, outgoingData); } diff --git a/src/libsync/networkjobs.cpp b/src/libsync/networkjobs.cpp index 341e5dc4f..fa87b4475 100644 --- a/src/libsync/networkjobs.cpp +++ b/src/libsync/networkjobs.cpp @@ -44,6 +44,7 @@ bool AbstractNetworkJob::preOc7WasDetected = false; AbstractNetworkJob::AbstractNetworkJob(Account *account, const QString &path, QObject *parent) : QObject(parent) , _duration(0) + , _timedout(false) , _ignoreCredentialFailure(false) , _reply(0) , _account(account) @@ -216,6 +217,7 @@ void AbstractNetworkJob::start() void AbstractNetworkJob::slotTimeout() { + _timedout = true; qDebug() << this << "Timeout"; if (reply()) { reply()->abort(); diff --git a/src/libsync/networkjobs.h b/src/libsync/networkjobs.h index 779b7e862..fdd750014 100644 --- a/src/libsync/networkjobs.h +++ b/src/libsync/networkjobs.h @@ -93,10 +93,13 @@ protected: QString _responseTimestamp; QElapsedTimer _durationTimer; quint64 _duration; + bool _timedout; // set to true when the timeout slot is recieved +public: // Timeout workarounds (Because of PHP session locking) static bool preOc7WasDetected; + private slots: void slotFinished(); virtual void slotTimeout(); diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp index feb2e15ed..474c9d3dd 100644 --- a/src/libsync/owncloudpropagator.cpp +++ b/src/libsync/owncloudpropagator.cpp @@ -16,11 +16,15 @@ #include "owncloudpropagator.h" #include "syncjournaldb.h" #include "syncjournalfilerecord.h" -#include "propagator_qnam.h" +#include "propagatedownload.h" +#include "propagateupload.h" +#include "propagateremotedelete.h" +#include "propagateremotemove.h" #include "propagatorjobs.h" #include "propagator_legacy.h" #include "configfile.h" #include "utility.h" +#include #ifdef Q_OS_WIN #include @@ -30,11 +34,15 @@ #include #include #include +#include +#include +#include namespace OCC { /* The maximum number of active job in parallel */ -static int maximumActiveJob() { +int OwncloudPropagator::maximumActiveJob() +{ static int max = qgetenv("OWNCLOUD_MAX_PARALLEL").toUInt(); if (!max) { max = 3; //default @@ -191,7 +199,7 @@ PropagateItemJob* OwncloudPropagator::createJob(const SyncFileItem& item) { switch(item._instruction) { case CSYNC_INSTRUCTION_REMOVE: if (item._direction == SyncFileItem::Down) return new PropagateLocalRemove(this, item); - else return new PropagateRemoteRemove(this, item); + else return new PropagateRemoteDelete(this, item); case CSYNC_INSTRUCTION_NEW: if (item._isDirectory) { if (item._direction == SyncFileItem::Down) return new PropagateLocalMkdir(this, item); @@ -218,7 +226,7 @@ PropagateItemJob* OwncloudPropagator::createJob(const SyncFileItem& item) { } case CSYNC_INSTRUCTION_RENAME: if (item._direction == SyncFileItem::Up) { - return new PropagateRemoteRename(this, item); + return new PropagateRemoteMove(this, item); } else { return new PropagateLocalRename(this, item); } @@ -328,14 +336,31 @@ bool OwncloudPropagator::isInSharedDirectory(const QString& file) */ bool OwncloudPropagator::useLegacyJobs() { - if (_downloadLimit.fetchAndAddAcquire(0) != 0 || _uploadLimit.fetchAndAddAcquire(0) != 0) { - // QNAM does not support bandwith limiting + // Allow an environement variable for debugging + QByteArray env = qgetenv("OWNCLOUD_USE_LEGACY_JOBS"); + if (env=="true" || env =="1") { + qDebug() << "Force Legacy Propagator ACTIVATED"; return true; } - // Allow an environement variable for debugging - QByteArray env = qgetenv("OWNCLOUD_USE_LEGACY_JOBS"); - return env=="true" || env =="1"; + env = qgetenv("OWNCLOUD_NEW_BANDWIDTH_LIMITING"); + if (env=="true" || env =="1") { + qDebug() << "New Bandwidth Limiting Code ACTIVATED"; + // Only certain Qt versions support this at the moment. + // They need those Change-Ids: Idb1c2d5a382a704d8cc08fe03c55c883bfc95aa7 Iefbcb1a21d8aedef1eb11761232dd16a049018dc + // FIXME We need to check the Qt version and then also return false here as soon + // as mirall ships with those Qt versions on Windows and OS X + return false; + } + + if (_downloadLimit.fetchAndAddAcquire(0) != 0 || _uploadLimit.fetchAndAddAcquire(0) != 0) { + qDebug() << "Switching To Legacy Propagator Because Of Bandwidth Limit ACTIVATED"; + // QNAM does not support bandwith limiting + // in most Qt versions. + return true; + } + + return false; } int OwncloudPropagator::httpTimeout() @@ -443,7 +468,7 @@ void PropagateDirectory::slotSubJobReady() return; // Ignore the case when the _fistJob is ready and not yet finished if (_runningNow && _current >= 0 && _current < _subJobs.count()) { // there is a job running and the current one is not ready yet, we can't start new job - if (!_subJobs[_current]->_readySent || _propagator->_activeJobs >= maximumActiveJob()) + if (!_subJobs[_current]->_readySent || _propagator->_activeJobs >= _propagator->maximumActiveJob()) return; } @@ -476,4 +501,38 @@ void PropagateDirectory::slotSubJobReady() } } +void CleanupPollsJob::start() +{ + if (_pollInfos.empty()) { + emit finished(); + deleteLater(); + return; + } + + auto info = _pollInfos.first(); + _pollInfos.pop_front(); + SyncFileItem item; + item._file = info._file; + item._modtime = info._modtime; + PollJob *job = new PollJob(_account, info._url, item, _journal, _localPath, this); + connect(job, SIGNAL(finishedSignal()), SLOT(slotPollFinished())); + job->start(); +} + +void CleanupPollsJob::slotPollFinished() +{ + PollJob *job = qobject_cast(sender()); + Q_ASSERT(job); + if (job->_item._status == SyncFileItem::FatalError) { + emit aborted(job->_item._errorString); + return; + } else if (job->_item._status != SyncFileItem::Success) { + qDebug() << "There was an error with file " << job->_item._file << job->_item._errorString; + } else { + _journal->setFileRecord(SyncJournalFileRecord(job->_item, _localPath + job->_item._file)); + } + // Continue with the next entry, or finish + start(); +} + } diff --git a/src/libsync/owncloudpropagator.h b/src/libsync/owncloudpropagator.h index d9aaa72c5..a5aaae5bd 100644 --- a/src/libsync/owncloudpropagator.h +++ b/src/libsync/owncloudpropagator.h @@ -18,9 +18,16 @@ #include #include #include -#include +#include +#include +#include +#include +#include +#include #include "syncfileitem.h" +#include "syncjournaldb.h" +#include "bandwidthmanager.h" struct hbf_transfer_s; struct ne_session_s; @@ -29,6 +36,8 @@ typedef struct ne_prop_result_set_s ne_prop_result_set; namespace OCC { +class Account; + class SyncJournalDb; class OwncloudPropagator; @@ -174,6 +183,43 @@ public: } }; +class BandwidthManager; // fwd +class UploadDevice : public QIODevice { + Q_OBJECT +public: + QPointer _file; + qint64 _read; + qint64 _size; + qint64 _start; + BandwidthManager* _bandwidthManager; + + qint64 _bandwidthQuota; + qint64 _readWithProgress; + + UploadDevice(QIODevice *file, qint64 start, qint64 size, BandwidthManager *bwm); + ~UploadDevice(); + virtual qint64 writeData(const char* , qint64 ); + virtual qint64 readData(char* data, qint64 maxlen); + virtual bool atEnd() const; + virtual qint64 size() const; + qint64 bytesAvailable() const; + virtual bool isSequential() const; + virtual bool seek ( qint64 pos ); + + void setBandwidthLimited(bool); + bool isBandwidthLimited() { return _bandwidthLimited; } + void setChoked(bool); + bool isChoked() { return _choked; } + void giveBandwidthQuota(qint64 bwq); +private: + bool _bandwidthLimited; // if _bandwidthQuota will be used + bool _choked; // if upload is paused (readData() will return 0) +protected slots: + void slotJobUploadProgress(qint64 sent, qint64 t); +}; +//Q_DECLARE_METATYPE(UploadDevice); +//Q_DECLARE_METATYPE(QPointer); + class OwncloudPropagator : public QObject { Q_OBJECT @@ -195,6 +241,8 @@ public: SyncJournalDb * const _journal; bool _finishedEmited; // used to ensure that finished is only emit once + BandwidthManager _bandwidthManager; + public: OwncloudPropagator(ne_session_s *session, const QString &localDir, const QString &remoteDir, const QString &remoteFolder, SyncJournalDb *progressDb, QThread *neonThread) @@ -205,6 +253,7 @@ public: , _remoteFolder((remoteFolder.endsWith(QChar('/'))) ? remoteFolder : remoteFolder+'/' ) , _journal(progressDb) , _finishedEmited(false) + , _bandwidthManager(this) , _activeJobs(0) , _anotherSyncNeeded(false) { } @@ -222,6 +271,9 @@ public: /** We detected that another sync is required after this one */ bool _anotherSyncNeeded; + /* The maximum number of active job in parallel */ + int maximumActiveJob(); + bool isInSharedDirectory(const QString& file); bool localFileNameClash(const QString& relfile); QString getFilePath(const QString& tmp_file_name) const; @@ -258,6 +310,26 @@ signals: }; +// Job that wait for all the poll jobs to be completed +class CleanupPollsJob : public QObject { + Q_OBJECT + QVector< SyncJournalDb::PollInfo > _pollInfos; + Account *_account; + SyncJournalDb *_journal; + QString _localPath; +public: + explicit CleanupPollsJob(const QVector< SyncJournalDb::PollInfo > &pollInfos, Account *account, + SyncJournalDb *journal, const QString &localPath, QObject* parent = 0) + : QObject(parent), _pollInfos(pollInfos), _account(account), _journal(journal), _localPath(localPath) {} + + void start(); +signals: + void finished(); + void aborted(const QString &error); +private slots: + void slotPollFinished(); +}; + } #endif diff --git a/src/libsync/owncloudpropagator_p.h b/src/libsync/owncloudpropagator_p.h index 9f1ad960d..4a89fe395 100644 --- a/src/libsync/owncloudpropagator_p.h +++ b/src/libsync/owncloudpropagator_p.h @@ -15,6 +15,9 @@ #pragma once +#include +#include "syncfileitem.h" + namespace OCC { inline QByteArray parseEtag(const char *header) { @@ -28,5 +31,33 @@ inline QByteArray parseEtag(const char *header) { return arr; } +inline QByteArray getEtagFromReply(QNetworkReply *reply) +{ + QByteArray ret = parseEtag(reply->rawHeader("OC-ETag")); + if (ret.isEmpty()) { + ret = parseEtag(reply->rawHeader("ETag")); + } + return ret; +} + +/** + * Fiven an error from the network, map to a SyncFileItem::Status error + */ +inline SyncFileItem::Status classifyError(QNetworkReply::NetworkError nerror, int httpCode) { + Q_ASSERT (nerror != QNetworkReply::NoError); // we should only be called when there is an error + + if (nerror > QNetworkReply::NoError && nerror <= QNetworkReply::UnknownProxyError) { + // network error or proxy error -> fatal + return SyncFileItem::FatalError; + } + + if (httpCode == 412) { + // "Precondition Failed" + // Happens when the e-tag has changed + return SyncFileItem::SoftError; + } + + return SyncFileItem::NormalError; +} } diff --git a/src/libsync/propagator_qnam.cpp b/src/libsync/propagatedownload.cpp similarity index 51% rename from src/libsync/propagator_qnam.cpp rename to src/libsync/propagatedownload.cpp index 544c1971f..ea3c67102 100644 --- a/src/libsync/propagator_qnam.cpp +++ b/src/libsync/propagatedownload.cpp @@ -12,7 +12,8 @@ * for more details. */ -#include "propagator_qnam.h" +#include "owncloudpropagator_p.h" +#include "propagatedownload.h" #include "networkjobs.h" #include "account.h" #include "syncjournaldb.h" @@ -20,6 +21,7 @@ #include "utility.h" #include "filesystem.h" #include "propagatorjobs.h" +#include #include #include #include @@ -27,425 +29,6 @@ namespace OCC { -/** - * The mtime of a file must be at least this many milliseconds in - * the past for an upload to be started. Otherwise the propagator will - * assume it's still being changed and skip it. - * - * This value must be smaller than the msBetweenRequestAndSync in - * the folder manager. - * - * Two seconds has shown to be a good value in tests. - */ -static int minFileAgeForUpload = 2000; - -static qint64 chunkSize() { - static uint chunkSize; - if (!chunkSize) { - chunkSize = qgetenv("OWNCLOUD_CHUNK_SIZE").toUInt(); - if (chunkSize == 0) { - chunkSize = 20*1024*1024; // default to 20 MiB - } - } - return chunkSize; -} - -static QByteArray get_etag_from_reply(QNetworkReply *reply) -{ - QByteArray ret = parseEtag(reply->rawHeader("OC-ETag")); - if (ret.isEmpty()) { - ret = parseEtag(reply->rawHeader("ETag")); - } - return ret; -} - -/** - * Fiven an error from the network, map to a SyncFileItem::Status error - */ -static SyncFileItem::Status classifyError(QNetworkReply::NetworkError nerror, int httpCode) { - Q_ASSERT (nerror != QNetworkReply::NoError); // we should only be called when there is an error - - if (nerror > QNetworkReply::NoError && nerror <= QNetworkReply::UnknownProxyError) { - // network error or proxy error -> fatal - return SyncFileItem::FatalError; - } - - if (httpCode == 412) { - // "Precondition Failed" - // Happens when the e-tag has changed - return SyncFileItem::SoftError; - } - - return SyncFileItem::NormalError; -} - -void PUTFileJob::start() { - QNetworkRequest req; - for(QMap::const_iterator it = _headers.begin(); it != _headers.end(); ++it) { - req.setRawHeader(it.key(), it.value()); - } - - setReply(davRequest("PUT", path(), req, _device)); - _device->setParent(reply()); - setupConnections(reply()); - - if( reply()->error() != QNetworkReply::NoError ) { - qWarning() << Q_FUNC_INFO << " Network error: " << reply()->errorString(); - } - - connect(reply(), SIGNAL(uploadProgress(qint64,qint64)), this, SIGNAL(uploadProgress(qint64,qint64))); - connect(this, SIGNAL(networkActivity()), account(), SIGNAL(propagatorNetworkActivity())); - - AbstractNetworkJob::start(); -} - -void PUTFileJob::slotTimeout() { - _errorString = tr("Connection Timeout"); - reply()->abort(); -} - -void PropagateUploadFileQNAM::start() -{ - if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) - return; - - _file = new QFile(_propagator->getFilePath(_item._file), this); - if (!_file->open(QIODevice::ReadOnly)) { - done(SyncFileItem::NormalError, _file->errorString()); - delete _file; - return; - } - - // Update the mtime and size, it might have changed since discovery. - _item._modtime = FileSystem::getModTime(_file->fileName()); - quint64 fileSize = _file->size(); - _item._size = fileSize; - - // But skip the file if the mtime is too close to 'now'! - // That usually indicates a file that is still being changed - // or not yet fully copied to the destination. - QDateTime modtime = Utility::qDateTimeFromTime_t(_item._modtime); - if (modtime.msecsTo(QDateTime::currentDateTime()) < minFileAgeForUpload) { - _propagator->_anotherSyncNeeded = true; - done(SyncFileItem::SoftError, tr("Local file changed during sync.")); - delete _file; - return; - } - - _chunkCount = std::ceil(fileSize/double(chunkSize())); - _startChunk = 0; - _transferId = qrand() ^ _item._modtime ^ (_item._size << 16); - - const SyncJournalDb::UploadInfo progressInfo = _propagator->_journal->getUploadInfo(_item._file); - - if (progressInfo._valid && Utility::qDateTimeToTime_t(progressInfo._modtime) == _item._modtime ) { - _startChunk = progressInfo._chunk; - _transferId = progressInfo._transferid; - qDebug() << Q_FUNC_INFO << _item._file << ": Resuming from chunk " << _startChunk; - } - - _currentChunk = 0; - _duration.start(); - - _propagator->_activeJobs++; - emit progress(_item, 0); - emitReady(); - this->startNextChunk(); -} - -struct ChunkDevice : QIODevice { -public: - QPointer _file; - qint64 _read; - qint64 _size; - qint64 _start; - - ChunkDevice(QIODevice *file, qint64 start, qint64 size) - : QIODevice(file), _file(file), _read(0), _size(size), _start(start) { - _file = QPointer(file); - _file.data()->seek(start); - } - - virtual qint64 writeData(const char* , qint64 ) Q_DECL_OVERRIDE { - Q_ASSERT(!"write to read only device"); - return 0; - } - - virtual qint64 readData(char* data, qint64 maxlen) Q_DECL_OVERRIDE { - if (_file.isNull()) { - qDebug() << Q_FUNC_INFO << "Upload file object deleted during upload"; - close(); - return -1; - } - maxlen = qMin(maxlen, chunkSize() - _read); - if (maxlen == 0) - return 0; - qint64 ret = _file.data()->read(data, maxlen); - if (ret < 0) - return -1; - _read += ret; - return ret; - } - - virtual bool atEnd() const Q_DECL_OVERRIDE { - if (_file.isNull()) { - qDebug() << Q_FUNC_INFO << "Upload file object deleted during upload"; - return true; - } - return _read >= chunkSize() || _file.data()->atEnd(); - } - - virtual qint64 size() const Q_DECL_OVERRIDE{ - return _size; - } - - qint64 bytesAvailable() const Q_DECL_OVERRIDE - { - return _size - _read + QIODevice::bytesAvailable(); - } - - // random access, we can seek - virtual bool isSequential() const Q_DECL_OVERRIDE{ - return false; - } - - virtual bool seek ( qint64 pos ) Q_DECL_OVERRIDE { - if (_file.isNull()) { - qDebug() << Q_FUNC_INFO << "Upload file object deleted during upload"; - close(); - return false; - } - _read = pos; - return _file.data()->seek(pos + _start); - } -}; - -void PropagateUploadFileQNAM::startNextChunk() -{ - if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) - return; - - quint64 fileSize = _item._size; - QMap headers; - headers["OC-Total-Length"] = QByteArray::number(fileSize); - headers["Content-Type"] = "application/octet-stream"; - headers["X-OC-Mtime"] = QByteArray::number(qint64(_item._modtime)); - if (!_item._etag.isEmpty() && _item._etag != "empty_etag" && - _item._instruction != CSYNC_INSTRUCTION_NEW // On new files never send a If-Match - ) { - // We add quotes because the owncloud server always add quotes around the etag, and - // csync_owncloud.c's owncloud_file_id always strip the quotes. - headers["If-Match"] = '"' + _item._etag + '"'; - } - - QString path = _item._file; - QIODevice *device = 0; - if (_chunkCount > 1) { - int sendingChunk = (_currentChunk + _startChunk) % _chunkCount; - // XOR with chunk size to make sure everything goes well if chunk size change between runs - uint transid = _transferId ^ chunkSize(); - path += QString("-chunking-%1-%2-%3").arg(transid).arg(_chunkCount).arg(sendingChunk); - headers["OC-Chunked"] = "1"; - int currentChunkSize = chunkSize(); - if (sendingChunk == _chunkCount - 1) { // last chunk - currentChunkSize = (fileSize % chunkSize()); - if( currentChunkSize == 0 ) { // if the last chunk pretents to be 0, its actually the full chunk size. - currentChunkSize = chunkSize(); - } - } - device = new ChunkDevice(_file, chunkSize() * quint64(sendingChunk), currentChunkSize); - } else { - device = _file; - } - - bool isOpen = true; - if (!device->isOpen()) { - isOpen = device->open(QIODevice::ReadOnly); - } - - if( isOpen ) { - _job = new PUTFileJob(AccountManager::instance()->account(), _propagator->_remoteFolder + path, device, headers); - _job->setTimeout(_propagator->httpTimeout() * 1000); - connect(_job, SIGNAL(finishedSignal()), this, SLOT(slotPutFinished())); - connect(_job, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(slotUploadProgress(qint64,qint64))); - _job->start(); - } else { - qDebug() << "ERR: Could not open upload file: " << device->errorString(); - done( SyncFileItem::NormalError, device->errorString() ); - delete device; - return; - } -} - -void PropagateUploadFileQNAM::slotPutFinished() -{ - PUTFileJob *job = qobject_cast(sender()); - Q_ASSERT(job); - - qDebug() << Q_FUNC_INFO << job->reply()->request().url() << "FINISHED WITH STATUS" - << job->reply()->error() - << (job->reply()->error() == QNetworkReply::NoError ? QLatin1String("") : job->reply()->errorString()) - << job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute) - << job->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute); - - QNetworkReply::NetworkError err = job->reply()->error(); - if (err != QNetworkReply::NoError) { - _item._httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - _propagator->_activeJobs--; - if(checkForProblemsWithShared(_item._httpErrorCode, - tr("The file was edited locally but is part of a read only share. " - "It is restored and your edit is in the conflict file."))) { - return; - } - QString errorString = job->errorString(); - - QByteArray replyContent = job->reply()->readAll(); - qDebug() << replyContent; // display the XML error in the debug - QRegExp rx("(.*)"); // Issue #1366: display server exception - if (rx.indexIn(QString::fromUtf8(replyContent)) != -1) { - errorString += QLatin1String(" (") + rx.cap(1) + QLatin1Char(')'); - } - - if (_item._httpErrorCode == 412) { - // Precondition Failed: Maybe the bad etag is in the database, we need to clear the - // parent folder etag so we won't read from DB next sync. - _propagator->_journal->avoidReadFromDbOnNextSync(_item._file); - _propagator->_anotherSyncNeeded = true; - } - - done(classifyError(err, _item._httpErrorCode), errorString); - return; - } - - // Check the file again post upload. - // Two cases must be considered separately: If the upload is finished, - // the file is on the server and has a changed ETag. In that case, - // the etag has to be properly updated in the client journal, and because - // of that we can bail out here with an error. But we can reschedule a - // sync ASAP. - // But if the upload is ongoing, because not all chunks were uploaded - // yet, the upload can be stopped and an error can be displayed, because - // the server hasn't registered the new file yet. - bool finished = job->reply()->hasRawHeader("ETag") - || job->reply()->hasRawHeader("OC-ETag"); - - QFileInfo fi(_propagator->getFilePath(_item._file)); - - // Check if the file still exists - if( !fi.exists() ) { - if( !finished ) { - _propagator->_activeJobs--; - done(SyncFileItem::SoftError, tr("The local file was removed during sync.")); - return; - } else { - _propagator->_anotherSyncNeeded = true; - } - } - - // compare expected and real modification time of the file and size - const time_t new_mtime = FileSystem::getModTime(fi.absoluteFilePath()); - const quint64 new_size = static_cast(fi.size()); - if (new_mtime != _item._modtime || new_size != _item._size) { - qDebug() << "The local file has changed during upload:" - << "mtime: " << _item._modtime << "<->" << new_mtime - << ", size: " << _item._size << "<->" << new_size - << ", QFileInfo: " << Utility::qDateTimeToTime_t(fi.lastModified()) << fi.lastModified(); - _propagator->_anotherSyncNeeded = true; - if( !finished ) { - _propagator->_activeJobs--; - done(SyncFileItem::SoftError, tr("Local file changed during sync.")); - // FIXME: the legacy code was retrying for a few seconds. - // and also checking that after the last chunk, and removed the file in case of INSTRUCTION_NEW - return; - } - } - - if (!finished) { - // Proceed to next chunk. - _currentChunk++; - if (_currentChunk >= _chunkCount) { - _propagator->_activeJobs--; - done(SyncFileItem::NormalError, tr("The server did not acknowledge the last chunk. (No e-tag were present)")); - return; - } - - SyncJournalDb::UploadInfo pi; - pi._valid = true; - pi._chunk = (_currentChunk + _startChunk) % _chunkCount; // next chunk to start with - pi._transferid = _transferId; - pi._modtime = Utility::qDateTimeFromTime_t(_item._modtime); - _propagator->_journal->setUploadInfo(_item._file, pi); - _propagator->_journal->commit("Upload info"); - startNextChunk(); - return; - } - - // the following code only happens after all chunks were uploaded. - // - // the file id should only be empty for new files up- or downloaded - QByteArray fid = job->reply()->rawHeader("OC-FileID"); - if( !fid.isEmpty() ) { - if( !_item._fileId.isEmpty() && _item._fileId != fid ) { - qDebug() << "WARN: File ID changed!" << _item._fileId << fid; - } - _item._fileId = fid; - } - - QByteArray etag = get_etag_from_reply(job->reply()); - _item._etag = etag; - - _item._responseTimeStamp = job->responseTimestamp(); - - if (job->reply()->rawHeader("X-OC-MTime") != "accepted") { - // X-OC-MTime is supported since owncloud 5.0. But not when chunking. - // Normaly Owncloud 6 always put X-OC-MTime - qDebug() << "Server do not support X-OC-MTime"; - PropagatorJob *newJob = new UpdateMTimeAndETagJob(_propagator, _item); - QObject::connect(newJob, SIGNAL(completed(SyncFileItem)), this, SLOT(finalize(SyncFileItem))); - QMetaObject::invokeMethod(newJob, "start"); - return; - } - finalize(_item); -} - -void PropagateUploadFileQNAM::finalize(const SyncFileItem ©) -{ - // Normally, copy == _item, but when it comes from the UpdateMTimeAndETagJob, we need to do - // some updates - _item._etag = copy._etag; - _item._fileId = copy._fileId; - - _propagator->_activeJobs--; - - _item._requestDuration = _duration.elapsed(); - - _propagator->_journal->setFileRecord(SyncJournalFileRecord(_item, _propagator->getFilePath(_item._file))); - // Remove from the progress database: - _propagator->_journal->setUploadInfo(_item._file, SyncJournalDb::UploadInfo()); - _propagator->_journal->commit("upload file start"); - - done(SyncFileItem::Success); -} - -void PropagateUploadFileQNAM::slotUploadProgress(qint64 sent, qint64) -{ - int progressChunk = _currentChunk + _startChunk; - if (progressChunk >= _chunkCount) - progressChunk = _currentChunk; - emit progress(_item, sent + progressChunk * chunkSize()); -} - - -void PropagateUploadFileQNAM::abort() -{ - if (_job && _job->reply()) { - qDebug() << Q_FUNC_INFO << this->_item._file; - _job->reply()->abort(); - } -} - -/////////////////////////////////////////////////////////////////////////////////////////////////// - // DOES NOT take owncership of the device. GETFileJob::GETFileJob(Account* account, const QString& path, QFile *device, const QMap &headers, QByteArray expectedEtagForResume, @@ -453,6 +36,8 @@ GETFileJob::GETFileJob(Account* account, const QString& path, QFile *device, : AbstractNetworkJob(account, path, parent), _device(device), _headers(headers), _expectedEtagForResume(expectedEtagForResume), _resumeStart(_resumeStart) , _errorStatus(SyncFileItem::NoStatus) +, _bandwidthLimited(false), _bandwidthChoked(false), _bandwidthQuota(0), _bandwidthManager(0) +, _hasEmittedFinishedSignal(false) { } @@ -462,6 +47,8 @@ GETFileJob::GETFileJob(Account* account, const QUrl& url, QFile *device, : AbstractNetworkJob(account, url.toEncoded(), parent), _device(device), _headers(headers), _resumeStart(0), _errorStatus(SyncFileItem::NoStatus), _directDownloadUrl(url) +, _bandwidthLimited(false), _bandwidthChoked(false), _bandwidthQuota(0), _bandwidthManager(0) +, _hasEmittedFinishedSignal(false) { } @@ -485,7 +72,12 @@ void GETFileJob::start() { setReply(davRequest("GET", _directDownloadUrl, req)); } setupConnections(reply()); - reply()->setReadBufferSize(128 * 1024); + + reply()->setReadBufferSize(16 * 1024); // keep low so we can easier limit the bandwidth + qDebug() << Q_FUNC_INFO << _bandwidthManager << _bandwidthChoked << _bandwidthLimited; + if (_bandwidthManager) { + _bandwidthManager->registerDownloadJob(this); + } if( reply()->error() != QNetworkReply::NoError ) { qWarning() << Q_FUNC_INFO << " Network error: " << reply()->errorString(); @@ -501,6 +93,10 @@ void GETFileJob::start() { void GETFileJob::slotMetaDataChanged() { + // For some reason setting the read buffer in GETFileJob::start doesn't seem to go + // through the HTTP layer thread(?) + reply()->setReadBufferSize(16 * 1024); + int httpStatus = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); // If the status code isn't 2xx, don't write the reply body to the file. @@ -512,7 +108,7 @@ void GETFileJob::slotMetaDataChanged() if (reply()->error() != QNetworkReply::NoError) { return; } - _etag = get_etag_from_reply(reply()); + _etag = getEtagFromReply(reply()); if (!_directDownloadUrl.isEmpty() && !_etag.isEmpty()) { qDebug() << Q_FUNC_INFO << "Direct download used, ignoring server ETag" << _etag; @@ -564,13 +160,62 @@ void GETFileJob::slotMetaDataChanged() } +void GETFileJob::setBandwidthManager(BandwidthManager *bwm) +{ + _bandwidthManager = bwm; +} + +void GETFileJob::setChoked(bool c) +{ + _bandwidthChoked = c; + QMetaObject::invokeMethod(this, "slotReadyRead", Qt::QueuedConnection); +} + +void GETFileJob::setBandwidthLimited(bool b) +{ + _bandwidthLimited = b; + QMetaObject::invokeMethod(this, "slotReadyRead", Qt::QueuedConnection); +} + +void GETFileJob::giveBandwidthQuota(qint64 q) +{ + _bandwidthQuota = q; + qDebug() << Q_FUNC_INFO << "Got" << q << "bytes"; + QMetaObject::invokeMethod(this, "slotReadyRead", Qt::QueuedConnection); +} + +qint64 GETFileJob::currentDownloadPosition() +{ + if (_device && _device->pos() > 0 && _device->pos() > qint64(_resumeStart)) { + return _device->pos(); + } + return _resumeStart; +} + void GETFileJob::slotReadyRead() { int bufferSize = qMin(1024*8ll , reply()->bytesAvailable()); QByteArray buffer(bufferSize, Qt::Uninitialized); + //qDebug() << Q_FUNC_INFO << reply()->bytesAvailable() << reply()->isOpen() << reply()->isFinished(); + while(reply()->bytesAvailable() > 0) { - qint64 r = reply()->read(buffer.data(), bufferSize); + if (_bandwidthChoked) { + qDebug() << Q_FUNC_INFO << "Download choked"; + break; + } + qint64 toRead = bufferSize; + if (_bandwidthLimited) { + toRead = qMin(qint64(bufferSize), _bandwidthQuota); + if (toRead == 0) { + qDebug() << Q_FUNC_INFO << "Out of quota"; + break; + } + _bandwidthQuota -= toRead; + //qDebug() << Q_FUNC_INFO << "Reading" << toRead << "remaining" << _bandwidthQuota; + } + + qint64 r = reply()->read(buffer.data(), toRead); if (r < 0) { _errorString = reply()->errorString(); _errorStatus = SyncFileItem::NormalError; @@ -590,6 +235,19 @@ void GETFileJob::slotReadyRead() } } } + + //qDebug() << Q_FUNC_INFO << "END" << reply()->isFinished() << reply()->bytesAvailable() << _hasEmittedFinishedSignal; + if (reply()->isFinished() && reply()->bytesAvailable() == 0) { + qDebug() << Q_FUNC_INFO << "Actually finished!"; + if (_bandwidthManager) { + _bandwidthManager->unregisterDownloadJob(this); + } + if (!_hasEmittedFinishedSignal) { + emit finishedSignal(); + } + _hasEmittedFinishedSignal = true; + deleteLater(); + } } void GETFileJob::slotTimeout() @@ -697,6 +355,7 @@ void PropagateDownloadFileQNAM::start() url, &_tmpFile, headers); } + _job->setBandwidthManager(&_propagator->_bandwidthManager); _job->setTimeout(_propagator->httpTimeout() * 1000); connect(_job, SIGNAL(finishedSignal()), this, SLOT(slotGetFinished())); connect(_job, SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(slotDownloadProgress(qint64,qint64))); @@ -736,7 +395,6 @@ void PropagateDownloadFileQNAM::slotGetFinished() _propagator->_journal->setDownloadInfo(_item._file, SyncJournalDb::DownloadInfo()); } - _propagator->_activeJobs--; SyncFileItem::Status status = job->errorStatus(); if (status == SyncFileItem::NoStatus) { status = classifyError(err, _item._httpErrorCode); @@ -862,4 +520,5 @@ void PropagateDownloadFileQNAM::abort() _job->reply()->abort(); } + } diff --git a/src/libsync/propagatedownload.h b/src/libsync/propagatedownload.h new file mode 100644 index 000000000..a74ba68f7 --- /dev/null +++ b/src/libsync/propagatedownload.h @@ -0,0 +1,117 @@ +/* + * Copyright (C) by Olivier Goffart + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include "owncloudpropagator.h" +#include "networkjobs.h" + +#include +#include + +namespace Mirall { + +class GETFileJob : public AbstractNetworkJob { + Q_OBJECT + QFile* _device; + QMap _headers; + QString _errorString; + QByteArray _expectedEtagForResume; + quint64 _resumeStart; + SyncFileItem::Status _errorStatus; + QUrl _directDownloadUrl; + QByteArray _etag; + bool _bandwidthLimited; // if _bandwidthQuota will be used + bool _bandwidthChoked; // if download is paused (won't read on readyRead()) + qint64 _bandwidthQuota; + BandwidthManager *_bandwidthManager; + bool _hasEmittedFinishedSignal; +public: + + // DOES NOT take owncership of the device. + explicit GETFileJob(Account* account, const QString& path, QFile *device, + const QMap &headers, QByteArray expectedEtagForResume, + quint64 resumeStart, QObject* parent = 0); + // For directDownloadUrl: + explicit GETFileJob(Account* account, const QUrl& url, QFile *device, + const QMap &headers, + QObject* parent = 0); + virtual ~GETFileJob() { + if (_bandwidthManager) { + _bandwidthManager->unregisterDownloadJob(this); + } + } + + virtual void start() Q_DECL_OVERRIDE; + virtual bool finished() Q_DECL_OVERRIDE { + qDebug() << Q_FUNC_INFO << reply()->bytesAvailable() << _hasEmittedFinishedSignal; + if (reply()->bytesAvailable()) { + qDebug() << Q_FUNC_INFO << "Not all read yet because of bandwidth limits"; + return false; + } else { + if (_bandwidthManager) { + _bandwidthManager->unregisterDownloadJob(this); + } + if (!_hasEmittedFinishedSignal) { + emit finishedSignal(); + } + _hasEmittedFinishedSignal = true; + return true; // discard + } + } + + void setBandwidthManager(BandwidthManager *bwm); + void setChoked(bool c); + void setBandwidthLimited(bool b); + void giveBandwidthQuota(qint64 q); + qint64 currentDownloadPosition(); + + QString errorString() { + return _errorString.isEmpty() ? reply()->errorString() : _errorString; + } + + SyncFileItem::Status errorStatus() { return _errorStatus; } + + virtual void slotTimeout() Q_DECL_OVERRIDE; + + QByteArray &etag() { return _etag; } + quint64 resumeStart() { return _resumeStart; } + + +signals: + void finishedSignal(); + void downloadProgress(qint64,qint64); +private slots: + void slotReadyRead(); + void slotMetaDataChanged(); +}; + + +class PropagateDownloadFileQNAM : public PropagateItemJob { + Q_OBJECT + QPointer _job; + +// QFile *_file; + QFile _tmpFile; +public: + PropagateDownloadFileQNAM(OwncloudPropagator* propagator,const SyncFileItem& item) + : PropagateItemJob(propagator, item) {} + void start() Q_DECL_OVERRIDE; +private slots: + void slotGetFinished(); + void abort() Q_DECL_OVERRIDE; + void downloadFinished(); + void slotDownloadProgress(qint64,qint64); +}; + +} diff --git a/src/libsync/propagateremotedelete.cpp b/src/libsync/propagateremotedelete.cpp new file mode 100644 index 000000000..654328bc1 --- /dev/null +++ b/src/libsync/propagateremotedelete.cpp @@ -0,0 +1,116 @@ +/* + * Copyright (C) by Olivier Goffart + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#include "propagateremotedelete.h" +#include "owncloudpropagator_p.h" +#include "account.h" + +namespace Mirall { + +DeleteJob::DeleteJob(Account* account, const QString& path, QObject* parent) + : AbstractNetworkJob(account, path, parent) +{ } + + +void DeleteJob::start() +{ + QNetworkRequest req; + setReply(davRequest("DELETE", path(), req)); + setupConnections(reply()); + + if( reply()->error() != QNetworkReply::NoError ) { + qWarning() << Q_FUNC_INFO << " Network error: " << reply()->errorString(); + } + AbstractNetworkJob::start(); +} + + +QString DeleteJob::errorString() +{ + return _timedout ? tr("Connection timed out") : reply()->errorString(); +} + +bool DeleteJob::finished() +{ + emit finishedSignal(); + return true; +} + +void PropagateRemoteDelete::start() +{ + if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) + return; + + qDebug() << Q_FUNC_INFO << _item._file; + + _job = new DeleteJob(AccountManager::instance()->account(), + _propagator->_remoteFolder + _item._file, + this); + connect(_job, SIGNAL(finishedSignal()), this, SLOT(slotDeleteJobFinished())); + _propagator->_activeJobs ++; + _job->start(); + emitReady(); +} + +void PropagateRemoteDelete::abort() +{ + if (_job && _job->reply()) + _job->reply()->abort(); +} + +void PropagateRemoteDelete::slotDeleteJobFinished() +{ + _propagator->_activeJobs--; + + Q_ASSERT(_job); + + qDebug() << Q_FUNC_INFO << _job->reply()->request().url() << "FINISHED WITH STATUS" + << _job->reply()->error() + << (_job->reply()->error() == QNetworkReply::NoError ? QLatin1String("") : _job->reply()->errorString()); + + QNetworkReply::NetworkError err = _job->reply()->error(); + _item._httpErrorCode = _job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + + if (err != QNetworkReply::NoError) { + + if( checkForProblemsWithShared(_item._httpErrorCode, + tr("The file has been removed from a read only share. It was restored.")) ) { + return; + } + + SyncFileItem::Status status = classifyError(err, _item._httpErrorCode); + done(status, _job->errorString()); + return; + } + + _item._requestDuration = _job->duration(); + _item._responseTimeStamp = _job->responseTimestamp(); + + if (_item._httpErrorCode != 204 ) { + // Normaly we expect "204 No Content" + // If it is not the case, it might be because of a proxy or gateway intercepting the request, so we must + // throw an error. + done(SyncFileItem::NormalError, tr("Wrong HTTP code returned by server. Expected 204, but recieved \"%1 %2\".") + .arg(_item._httpErrorCode).arg(_job->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toString())); + return; + } + + _propagator->_journal->deleteFileRecord(_item._originalFile, _item._isDirectory); + _propagator->_journal->commit("Remote Remove"); + done(SyncFileItem::Success); +} + + +} + diff --git a/src/libsync/propagateremotedelete.h b/src/libsync/propagateremotedelete.h new file mode 100644 index 000000000..e82b6acc4 --- /dev/null +++ b/src/libsync/propagateremotedelete.h @@ -0,0 +1,49 @@ +/* + * Copyright (C) by Olivier Goffart + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include "owncloudpropagator.h" +#include "networkjobs.h" + +namespace Mirall { + +class DeleteJob : public AbstractNetworkJob { + Q_OBJECT +public: + explicit DeleteJob(Account* account, const QString& path, QObject* parent = 0); + + void start() Q_DECL_OVERRIDE; + bool finished() Q_DECL_OVERRIDE; + + QString errorString(); + bool timedOut() { return _timedout; } + +signals: + void finishedSignal(); +}; + +class PropagateRemoteDelete : public PropagateItemJob { + Q_OBJECT + QPointer _job; +public: + PropagateRemoteDelete (OwncloudPropagator* propagator,const SyncFileItem& item) + : PropagateItemJob(propagator, item) {} + void start() Q_DECL_OVERRIDE; + void abort() Q_DECL_OVERRIDE; +private slots: + void slotDeleteJobFinished(); + +}; + +} \ No newline at end of file diff --git a/src/libsync/propagateremotemove.cpp b/src/libsync/propagateremotemove.cpp new file mode 100644 index 000000000..06084cb02 --- /dev/null +++ b/src/libsync/propagateremotemove.cpp @@ -0,0 +1,145 @@ +/* + * Copyright (C) by Olivier Goffart + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#include "propagateremotemove.h" +#include "owncloudpropagator_p.h" +#include "account.h" +#include "syncjournalfilerecord.h" +#include + +namespace Mirall { + +MoveJob::MoveJob(Account* account, const QString& path, + const QString &destination, QObject* parent) + : AbstractNetworkJob(account, path, parent), _destination(destination) +{ } + + +void MoveJob::start() +{ + QNetworkRequest req; + req.setRawHeader("Destination", QUrl::toPercentEncoding(_destination, "/")); + setReply(davRequest("MOVE", path(), req)); + setupConnections(reply()); + + if( reply()->error() != QNetworkReply::NoError ) { + qWarning() << Q_FUNC_INFO << " Network error: " << reply()->errorString(); + } + AbstractNetworkJob::start(); +} + + +QString MoveJob::errorString() +{ + return _timedout ? tr("Connection timed out") : reply()->errorString(); +} + +bool MoveJob::finished() +{ + emit finishedSignal(); + return true; +} + +void PropagateRemoteMove::start() +{ + if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) + return; + + qDebug() << Q_FUNC_INFO << _item._file << _item._renameTarget; + + if (_item._file == _item._renameTarget) { + // The parents has been renamed already so there is nothing more to do. + finalize(); + return; + } else if (AbstractNetworkJob::preOc7WasDetected && _item._file == QLatin1String("Shared") ) { + // Check if it is the toplevel Shared folder and do not propagate it. + if( QFile::rename( _propagator->_localDir + _item._renameTarget, _propagator->_localDir + QLatin1String("Shared")) ) { + done(SyncFileItem::NormalError, tr("This folder must not be renamed. It is renamed back to its original name.")); + } else { + done(SyncFileItem::NormalError, tr("This folder must not be renamed. Please name it back to Shared.")); + } + return; + } else { + _job = new MoveJob(AccountManager::instance()->account(), + _propagator->_remoteFolder + _item._file, + _propagator->_remoteDir + _item._renameTarget, + this); + connect(_job, SIGNAL(finishedSignal()), this, SLOT(slotMoveJobFinished())); + _propagator->_activeJobs++; + _job->start(); + emitReady(); + } +} + +void PropagateRemoteMove::abort() +{ + if (_job && _job->reply()) + _job->reply()->abort(); +} + +void PropagateRemoteMove::slotMoveJobFinished() +{ + _propagator->_activeJobs--; + + Q_ASSERT(_job); + + qDebug() << Q_FUNC_INFO << _job->reply()->request().url() << "FINISHED WITH STATUS" + << _job->reply()->error() + << (_job->reply()->error() == QNetworkReply::NoError ? QLatin1String("") : _job->reply()->errorString()); + + QNetworkReply::NetworkError err = _job->reply()->error(); + _item._httpErrorCode = _job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + + if (err != QNetworkReply::NoError) { + + if( checkForProblemsWithShared(_item._httpErrorCode, + tr("The file was renamed but is part of a read only share. The original file was restored."))) { + return; + } + + SyncFileItem::Status status = classifyError(err, _item._httpErrorCode); + done(status, _job->errorString()); + return; + } + + _item._requestDuration = _job->duration(); + _item._responseTimeStamp = _job->responseTimestamp(); + + if (_item._httpErrorCode != 201 ) { + // Normaly we expect "201 Created" + // If it is not the case, it might be because of a proxy or gateway intercepting the request, so we must + // throw an error. + done(SyncFileItem::NormalError, tr("Wrong HTTP code returned by server. Expected 201, but recieved \"%1 %2\".") + .arg(_item._httpErrorCode).arg(_job->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toString())); + return; + } + + finalize(); + +} + +void PropagateRemoteMove::finalize() +{ + _propagator->_journal->deleteFileRecord(_item._originalFile); + SyncJournalFileRecord record(_item, _propagator->_localDir + _item._renameTarget); + record._path = _item._renameTarget; + + _propagator->_journal->setFileRecord(record); + _propagator->_journal->commit("Remote Rename"); + done(SyncFileItem::Success); +} + + +} + diff --git a/src/libsync/propagateremotemove.h b/src/libsync/propagateremotemove.h new file mode 100644 index 000000000..97cf5c0f0 --- /dev/null +++ b/src/libsync/propagateremotemove.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) by Olivier Goffart + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ +#pragma once + +#include "owncloudpropagator.h" +#include "networkjobs.h" + +namespace Mirall { + +class MoveJob : public AbstractNetworkJob { + Q_OBJECT + const QString _destination; +public: + explicit MoveJob(Account* account, const QString& path, const QString &destination, QObject* parent = 0); + + void start() Q_DECL_OVERRIDE; + bool finished() Q_DECL_OVERRIDE; + + QString errorString(); + bool timedOut() { return _timedout; } + +signals: + void finishedSignal(); +}; + + +class PropagateRemoteMove : public PropagateItemJob { + Q_OBJECT + QPointer _job; +public: + PropagateRemoteMove (OwncloudPropagator* propagator,const SyncFileItem& item) + : PropagateItemJob(propagator, item) {} + void start() Q_DECL_OVERRIDE; + void abort() Q_DECL_OVERRIDE; +private slots: + void slotMoveJobFinished(); + void finalize(); +}; + +} \ No newline at end of file diff --git a/src/libsync/propagateupload.cpp b/src/libsync/propagateupload.cpp new file mode 100644 index 000000000..d5ac518da --- /dev/null +++ b/src/libsync/propagateupload.cpp @@ -0,0 +1,654 @@ +/* + * Copyright (C) by Olivier Goffart + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * for more details. + */ + +#include "propagateupload.h" +#include "owncloudpropagator_p.h" +#include "networkjobs.h" +#include "account.h" +#include "syncjournaldb.h" +#include "syncjournalfilerecord.h" +#include "utility.h" +#include "filesystem.h" +#include "propagatorjobs.h" +#include +#include +#include +#include +#include + +namespace Mirall { + +/** + * The mtime of a file must be at least this many milliseconds in + * the past for an upload to be started. Otherwise the propagator will + * assume it's still being changed and skip it. + * + * This value must be smaller than the msBetweenRequestAndSync in + * the folder manager. + * + * Two seconds has shown to be a good value in tests. + */ +static int minFileAgeForUpload = 2000; + +static qint64 chunkSize() { + static uint chunkSize; + if (!chunkSize) { + chunkSize = qgetenv("OWNCLOUD_CHUNK_SIZE").toUInt(); + if (chunkSize == 0) { + chunkSize = 20*1024*1024; // default to 20 MiB + } + } + return chunkSize; +} + +void PUTFileJob::start() { + QNetworkRequest req; + for(QMap::const_iterator it = _headers.begin(); it != _headers.end(); ++it) { + req.setRawHeader(it.key(), it.value()); + } + + setReply(davRequest("PUT", path(), req, _device.data())); + setupConnections(reply()); + + if( reply()->error() != QNetworkReply::NoError ) { + qWarning() << Q_FUNC_INFO << " Network error: " << reply()->errorString(); + } + + connect(reply(), SIGNAL(uploadProgress(qint64,qint64)), this, SIGNAL(uploadProgress(qint64,qint64))); + connect(this, SIGNAL(networkActivity()), account(), SIGNAL(propagatorNetworkActivity())); + + AbstractNetworkJob::start(); +} + +void PUTFileJob::slotTimeout() { + _errorString = tr("Connection Timeout"); + reply()->abort(); +} + +void PollJob::start() +{ + setTimeout(120 * 1000); + QUrl accountUrl = account()->url(); + QUrl finalUrl = QUrl::fromUserInput(accountUrl.scheme() + QLatin1String("://") + accountUrl.authority() + + (path().startsWith('/') ? QLatin1String("") : QLatin1String("/")) + path()); + setReply(getRequest(finalUrl)); + setupConnections(reply()); + connect(reply(), SIGNAL(downloadProgress(qint64,qint64)), this, SLOT(resetTimeout())); + AbstractNetworkJob::start(); +} + +bool PollJob::finished() +{ + QNetworkReply::NetworkError err = reply()->error(); + if (err != QNetworkReply::NoError) { + _item._httpErrorCode = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + _item._status = classifyError(err, _item._httpErrorCode); + _item._errorString = reply()->errorString(); + if (_item._status == SyncFileItem::FatalError || _item._httpErrorCode >= 400) { + if (_item._status != SyncFileItem::FatalError + && _item._httpErrorCode != 503) { + SyncJournalDb::PollInfo info; + info._file = _item._file; + // no info._url removes it from the database + _journal->setPollInfo(info); + _journal->commit("remove poll info"); + + } + emit finishedSignal(); + return true; + } + start(); + return false; + } + + bool ok = false; + QByteArray jsonData = reply()->readAll().trimmed(); + qDebug() << Q_FUNC_INFO << ">" << jsonData << "<" << reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + QVariantMap status = QtJson::parse(QString::fromUtf8(jsonData), ok).toMap(); + if (!ok || status.isEmpty()) { + _item._errorString = tr("Invalid json reply from the poll URL"); + _item._status = SyncFileItem::NormalError; + emit finishedSignal(); + return true; + } + + if (status["unfinished"].isValid()) { + start(); + return false; + } + + _item._errorString = status["error"].toString(); + _item._status = _item._errorString.isEmpty() ? SyncFileItem::Success : SyncFileItem::NormalError; + _item._fileId = status["fileid"].toByteArray(); + _item._etag = status["etag"].toByteArray(); + _item._responseTimeStamp = responseTimestamp(); + + SyncJournalDb::PollInfo info; + info._file = _item._file; + // no info._url removes it from the database + _journal->setPollInfo(info); + _journal->commit("remove poll info"); + + emit finishedSignal(); + return true; +} + + +void PropagateUploadFileQNAM::start() +{ + if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) + return; + + _file = new QFile(_propagator->getFilePath(_item._file), this); + if (!_file->open(QIODevice::ReadOnly)) { + done(SyncFileItem::NormalError, _file->errorString()); + delete _file; + return; + } + + // Update the mtime and size, it might have changed since discovery. + _item._modtime = FileSystem::getModTime(_file->fileName()); + quint64 fileSize = _file->size(); + _item._size = fileSize; + + // But skip the file if the mtime is too close to 'now'! + // That usually indicates a file that is still being changed + // or not yet fully copied to the destination. + QDateTime modtime = Utility::qDateTimeFromTime_t(_item._modtime); + if (modtime.msecsTo(QDateTime::currentDateTime()) < minFileAgeForUpload) { + _propagator->_anotherSyncNeeded = true; + done(SyncFileItem::SoftError, tr("Local file changed during sync.")); + delete _file; + return; + } + + _chunkCount = std::ceil(fileSize/double(chunkSize())); + _startChunk = 0; + _transferId = qrand() ^ _item._modtime ^ (_item._size << 16); + + const SyncJournalDb::UploadInfo progressInfo = _propagator->_journal->getUploadInfo(_item._file); + + if (progressInfo._valid && Utility::qDateTimeToTime_t(progressInfo._modtime) == _item._modtime ) { + _startChunk = progressInfo._chunk; + _transferId = progressInfo._transferid; + qDebug() << Q_FUNC_INFO << _item._file << ": Resuming from chunk " << _startChunk; + } + + _currentChunk = 0; + _duration.start(); + + emit progress(_item, 0); + this->startNextChunk(); +} + +UploadDevice::UploadDevice(QIODevice *file, qint64 start, qint64 size, BandwidthManager *bwm) + : QIODevice(file), _file(file), _read(0), _size(size), _start(start), + _bandwidthManager(bwm), + _bandwidthQuota(0), + _readWithProgress(0), + _bandwidthLimited(false), _choked(false) +{ + qDebug() << Q_FUNC_INFO << start << size << chunkSize(); + _bandwidthManager->registerUploadDevice(this); + _file = QPointer(file); +} + + +UploadDevice::~UploadDevice() { + _bandwidthManager->unregisterUploadDevice(this); +} + +qint64 UploadDevice::writeData(const char* , qint64 ) { + Q_ASSERT(!"write to read only device"); + return 0; +} + +qint64 UploadDevice::readData(char* data, qint64 maxlen) { + if (_file.isNull()) { + qDebug() << Q_FUNC_INFO << "Upload file object deleted during upload"; + close(); + return -1; + } + _file.data()->seek(_start + _read); + //qDebug() << Q_FUNC_INFO << maxlen << _read << _size << _bandwidthQuota; + if (_size - _read <= 0) { + // at end + qDebug() << Q_FUNC_INFO << _read << _size << _bandwidthQuota << "at end"; + _bandwidthManager->unregisterUploadDevice(this); + return -1; + } + maxlen = qMin(maxlen, _size - _read); + if (maxlen == 0) { + return 0; + } + if (isChoked()) { + qDebug() << Q_FUNC_INFO << this << "Upload Choked"; + return 0; + } + if (isBandwidthLimited()) { + qDebug() << Q_FUNC_INFO << "BW LIMITED" << maxlen << _bandwidthQuota + << qMin(maxlen, _bandwidthQuota); + maxlen = qMin(maxlen, _bandwidthQuota); + if (maxlen <= 0) { // no quota + qDebug() << Q_FUNC_INFO << "no quota"; + return 0; + } + _bandwidthQuota -= maxlen; + } + qDebug() << Q_FUNC_INFO << "reading limited=" << isBandwidthLimited() + << "maxlen=" << maxlen << "quota=" << _bandwidthQuota; + qint64 ret = _file.data()->read(data, maxlen); + //qDebug() << Q_FUNC_INFO << "returning " << ret; + + if (ret < 0) + return -1; + _read += ret; + //qDebug() << Q_FUNC_INFO << "returning2 " << ret << _read; + + return ret; +} + +void UploadDevice::slotJobUploadProgress(qint64 sent, qint64 t) +{ + //qDebug() << Q_FUNC_INFO << sent << _read << t << _size << _bandwidthQuota; + if (sent == 0 || t == 0) { + return; + } + _readWithProgress = sent; +} + +bool UploadDevice::atEnd() const { + if (_file.isNull()) { + qDebug() << Q_FUNC_INFO << "Upload file object deleted during upload"; + return true; + } +// qDebug() << this << Q_FUNC_INFO << _read << chunkSize() +// << (_read >= chunkSize() || _file.data()->atEnd()) +// << (_read >= _size); + return _file.data()->atEnd() || (_read >= _size); +} + +qint64 UploadDevice::size() const{ +// qDebug() << this << Q_FUNC_INFO << _size; + return _size; +} + +qint64 UploadDevice::bytesAvailable() const +{ +// qDebug() << this << Q_FUNC_INFO << _size << _read << QIODevice::bytesAvailable() +// << _size - _read + QIODevice::bytesAvailable(); + return _size - _read + QIODevice::bytesAvailable(); +} + +// random access, we can seek +bool UploadDevice::isSequential() const{ + return false; +} + +bool UploadDevice::seek ( qint64 pos ) { + if (_file.isNull()) { + qDebug() << Q_FUNC_INFO << "Upload file object deleted during upload"; + close(); + return false; + } + qDebug() << this << Q_FUNC_INFO << pos << _read; + _read = pos; + return _file.data()->seek(pos + _start); +} + +void UploadDevice::giveBandwidthQuota(qint64 bwq) { +// qDebug() << Q_FUNC_INFO << bwq; + if (!atEnd()) { + _bandwidthQuota = bwq; +// qDebug() << Q_FUNC_INFO << bwq << "emitting readyRead()" << _read << _readWithProgress; + QMetaObject::invokeMethod(this, "readyRead", Qt::QueuedConnection); // tell QNAM that we have quota + } +} + +void UploadDevice::setBandwidthLimited(bool b) { + _bandwidthLimited = b; + QMetaObject::invokeMethod(this, "readyRead", Qt::QueuedConnection); +} + +void UploadDevice::setChoked(bool b) { + _choked = b; + if (!_choked) { + QMetaObject::invokeMethod(this, "readyRead", Qt::QueuedConnection); + } +} + +void PropagateUploadFileQNAM::startNextChunk() +{ + if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) + return; + + if (! _jobs.isEmpty() && _currentChunk + _startChunk >= _chunkCount - 1) { + // Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that + // https://github.com/owncloud/core/issues/11106 + // We return now and when the _jobs will be finished we will proceed the last chunk + return; + } + quint64 fileSize = _item._size; + QMap headers; + headers["OC-Total-Length"] = QByteArray::number(fileSize); + headers["OC-Async"] = "1"; + headers["Content-Type"] = "application/octet-stream"; + headers["X-OC-Mtime"] = QByteArray::number(qint64(_item._modtime)); + if (!_item._etag.isEmpty() && _item._etag != "empty_etag" && + _item._instruction != CSYNC_INSTRUCTION_NEW // On new files never send a If-Match + ) { + // We add quotes because the owncloud server always add quotes around the etag, and + // csync_owncloud.c's owncloud_file_id always strip the quotes. + headers["If-Match"] = '"' + _item._etag + '"'; + } + + QString path = _item._file; + UploadDevice *device = 0; + if (_chunkCount > 1) { + int sendingChunk = (_currentChunk + _startChunk) % _chunkCount; + // XOR with chunk size to make sure everything goes well if chunk size change between runs + uint transid = _transferId ^ chunkSize(); + path += QString("-chunking-%1-%2-%3").arg(transid).arg(_chunkCount).arg(sendingChunk); + headers["OC-Chunked"] = "1"; + int currentChunkSize = chunkSize(); + if (sendingChunk == _chunkCount - 1) { // last chunk + currentChunkSize = (fileSize % chunkSize()); + if( currentChunkSize == 0 ) { // if the last chunk pretents to be 0, its actually the full chunk size. + currentChunkSize = chunkSize(); + } + } + device = new UploadDevice(_file, chunkSize() * quint64(sendingChunk), currentChunkSize, &_propagator->_bandwidthManager); + } else { + device = new UploadDevice(_file, 0, fileSize, &_propagator->_bandwidthManager); + } + + bool isOpen = true; + if (!device->isOpen()) { + isOpen = device->open(QIODevice::ReadOnly); + } + + if( isOpen ) { + PUTFileJob* job = new PUTFileJob(AccountManager::instance()->account(), _propagator->_remoteFolder + path, device, headers, _currentChunk); + _jobs.append(job); + job->setTimeout(_propagator->httpTimeout() * 1000); + connect(job, SIGNAL(finishedSignal()), this, SLOT(slotPutFinished())); + connect(job, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(slotUploadProgress(qint64,qint64))); + connect(job, SIGNAL(uploadProgress(qint64,qint64)), device, SLOT(slotJobUploadProgress(qint64,qint64))); + connect(job, SIGNAL(destroyed(QObject*)), this, SLOT(slotJobDestroyed(QObject*))); + job->start(); + _propagator->_activeJobs++; + _currentChunk++; + + QByteArray env = qgetenv("OWNCLOUD_PARALLEL_CHUNK"); + bool parallelChunkUpload = env=="true" || env =="1";; + if (_currentChunk + _startChunk >= _chunkCount - 1) { + // Don't do parallel upload of chunk if this might be the last chunk because the server cannot handle that + // https://github.com/owncloud/core/issues/11106 + parallelChunkUpload = false; + } + + if (parallelChunkUpload && (_propagator->_activeJobs < _propagator->maximumActiveJob()) + && _currentChunk < _chunkCount ) { + startNextChunk(); + } + if (!parallelChunkUpload || _chunkCount - _currentChunk <= 0) { + emitReady(); + } + } else { + qDebug() << "ERR: Could not open upload file: " << device->errorString(); + done( SyncFileItem::NormalError, device->errorString() ); + delete device; + return; + } +} + +void PropagateUploadFileQNAM::slotPutFinished() +{ + PUTFileJob *job = qobject_cast(sender()); + Q_ASSERT(job); + slotJobDestroyed(job); // remove it from the _jobs list + + qDebug() << Q_FUNC_INFO << job->reply()->request().url() << "FINISHED WITH STATUS" + << job->reply()->error() + << (job->reply()->error() == QNetworkReply::NoError ? QLatin1String("") : job->reply()->errorString()) + << job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute) + << job->reply()->attribute(QNetworkRequest::HttpReasonPhraseAttribute); + + _propagator->_activeJobs--; + + if (_finished) { + // We have send the finished signal already. We don't need to handle any remaining jobs + return; + } + + QNetworkReply::NetworkError err = job->reply()->error(); + if (err != QNetworkReply::NoError) { + _item._httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if(checkForProblemsWithShared(_item._httpErrorCode, + tr("The file was edited locally but is part of a read only share. " + "It is restored and your edit is in the conflict file."))) { + return; + } + QString errorString = job->errorString(); + + QByteArray replyContent = job->reply()->readAll(); + qDebug() << replyContent; // display the XML error in the debug + QRegExp rx("(.*)"); // Issue #1366: display server exception + if (rx.indexIn(QString::fromUtf8(replyContent)) != -1) { + errorString += QLatin1String(" (") + rx.cap(1) + QLatin1Char(')'); + } + + if (_item._httpErrorCode == 412) { + // Precondition Failed: Maybe the bad etag is in the database, we need to clear the + // parent folder etag so we won't read from DB next sync. + _propagator->_journal->avoidReadFromDbOnNextSync(_item._file); + _propagator->_anotherSyncNeeded = true; + } + + done(classifyError(err, _item._httpErrorCode), errorString); + return; + } + + _item._httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + // The server needs some time to process the request and provide with a poll URL + if (_item._httpErrorCode == 202) { + _finished = true; + QString path = QString::fromUtf8(job->reply()->rawHeader("OC-Finish-Poll")); + if (path.isEmpty()) { + done(SyncFileItem::NormalError, tr("Poll URL missing")); + return; + } + startPollJob(path); + return; + } + + // Check the file again post upload. + // Two cases must be considered separately: If the upload is finished, + // the file is on the server and has a changed ETag. In that case, + // the etag has to be properly updated in the client journal, and because + // of that we can bail out here with an error. But we can reschedule a + // sync ASAP. + // But if the upload is ongoing, because not all chunks were uploaded + // yet, the upload can be stopped and an error can be displayed, because + // the server hasn't registered the new file yet. + bool finished = job->reply()->hasRawHeader("ETag") + || job->reply()->hasRawHeader("OC-ETag"); + + + QFileInfo fi(_propagator->getFilePath(_item._file)); + + // Check if the file still exists + if( !fi.exists() ) { + if (!finished) { + _finished = true; + done(SyncFileItem::SoftError, tr("The local file was removed during sync.")); + return; + } else { + _propagator->_anotherSyncNeeded = true; + } + } + + // compare expected and real modification time of the file and size + const time_t new_mtime = FileSystem::getModTime(fi.absoluteFilePath()); + const quint64 new_size = static_cast(fi.size()); + if (new_mtime != _item._modtime || new_size != _item._size) { + qDebug() << "The local file has changed during upload:" + << "mtime: " << _item._modtime << "<->" << new_mtime + << ", size: " << _item._size << "<->" << new_size + << ", QFileInfo: " << Utility::qDateTimeToTime_t(fi.lastModified()) << fi.lastModified(); + _propagator->_anotherSyncNeeded = true; + if( !finished ) { + _finished = true; + done(SyncFileItem::SoftError, tr("Local file changed during sync.")); + // FIXME: the legacy code was retrying for a few seconds. + // and also checking that after the last chunk, and removed the file in case of INSTRUCTION_NEW + return; + } + } + + if (!finished) { + // Proceed to next chunk. + if (_currentChunk >= _chunkCount) { + if (!_jobs.empty()) { + // just wait for the other job to finish. + return; + } + _finished = true; + done(SyncFileItem::NormalError, tr("The server did not acknowledge the last chunk. (No e-tag were present)")); + return; + } + + SyncJournalDb::UploadInfo pi; + pi._valid = true; + auto currentChunk = job->_chunk; + foreach (auto *job, _jobs) { + // Take the minimum finished one + currentChunk = qMin(currentChunk, job->_chunk); + } + pi._chunk = (currentChunk + _startChunk + 1) % _chunkCount ; // next chunk to start with + pi._transferid = _transferId; + pi._modtime = Utility::qDateTimeFromTime_t(_item._modtime); + _propagator->_journal->setUploadInfo(_item._file, pi); + _propagator->_journal->commit("Upload info"); + startNextChunk(); + return; + } + + // the following code only happens after all chunks were uploaded. + _finished = true; + // the file id should only be empty for new files up- or downloaded + QByteArray fid = job->reply()->rawHeader("OC-FileID"); + if( !fid.isEmpty() ) { + if( !_item._fileId.isEmpty() && _item._fileId != fid ) { + qDebug() << "WARN: File ID changed!" << _item._fileId << fid; + } + _item._fileId = fid; + } + + QByteArray etag = getEtagFromReply(job->reply()); + _item._etag = etag; + + _item._responseTimeStamp = job->responseTimestamp(); + + if (job->reply()->rawHeader("X-OC-MTime") != "accepted") { + // X-OC-MTime is supported since owncloud 5.0. But not when chunking. + // Normaly Owncloud 6 always put X-OC-MTime + qDebug() << "Server do not support X-OC-MTime"; + PropagatorJob *newJob = new UpdateMTimeAndETagJob(_propagator, _item); + QObject::connect(newJob, SIGNAL(completed(SyncFileItem)), this, SLOT(finalize(SyncFileItem))); + QMetaObject::invokeMethod(newJob, "start"); + return; + } + finalize(_item); +} + +void PropagateUploadFileQNAM::finalize(const SyncFileItem ©) +{ + // Normally, copy == _item, but when it comes from the UpdateMTimeAndETagJob, we need to do + // some updates + _item._etag = copy._etag; + _item._fileId = copy._fileId; + + _item._requestDuration = _duration.elapsed(); + + _propagator->_journal->setFileRecord(SyncJournalFileRecord(_item, _propagator->getFilePath(_item._file))); + // Remove from the progress database: + _propagator->_journal->setUploadInfo(_item._file, SyncJournalDb::UploadInfo()); + _propagator->_journal->commit("upload file start"); + + qDebug() << Q_FUNC_INFO << "msec=" <<_duration.elapsed(); + done(SyncFileItem::Success); +} + +void PropagateUploadFileQNAM::slotUploadProgress(qint64 sent, qint64) +{ + int progressChunk = _currentChunk + _startChunk - 1; + if (progressChunk >= _chunkCount) + progressChunk = _currentChunk - 1; + quint64 amount = progressChunk * chunkSize(); + sender()->setProperty("byteWritten", sent); + if (_jobs.count() > 1) { + amount += sent; + } else { + amount -= (_jobs.count() -1) * chunkSize(); + foreach (QObject *j, _jobs) { + amount += j->property("byteWritten").toULongLong(); + } + } + emit progress(_item, amount); +} + +void PropagateUploadFileQNAM::startPollJob(const QString& path) +{ + PollJob* job = new PollJob(AccountManager::instance()->account(), path, _item, + _propagator->_journal, _propagator->_localDir, this); + connect(job, SIGNAL(finishedSignal()), SLOT(slotPollFinished())); + SyncJournalDb::PollInfo info; + info._file = _item._file; + info._url = path; + info._modtime = _item._modtime; + _propagator->_journal->setPollInfo(info); + _propagator->_journal->commit("add poll info"); + job->start(); +} + +void PropagateUploadFileQNAM::slotPollFinished() +{ + PollJob *job = qobject_cast(sender()); + Q_ASSERT(job); + + if (job->_item._status != SyncFileItem::Success) { + done(job->_item._status, job->_item._errorString); + return; + } + + finalize(job->_item); +} + +void PropagateUploadFileQNAM::slotJobDestroyed(QObject* job) +{ + _jobs.erase(std::remove(_jobs.begin(), _jobs.end(), job) , _jobs.end()); +} + +void PropagateUploadFileQNAM::abort() +{ + foreach(auto *job, _jobs) { + if (job->reply()) { + qDebug() << Q_FUNC_INFO << job << this->_item._file; + job->reply()->abort(); + } + } +} + +} \ No newline at end of file diff --git a/src/libsync/propagator_qnam.h b/src/libsync/propagateupload.h similarity index 55% rename from src/libsync/propagator_qnam.h rename to src/libsync/propagateupload.h index 0548aaed4..c6a250cbc 100644 --- a/src/libsync/propagator_qnam.h +++ b/src/libsync/propagateupload.h @@ -1,6 +1,5 @@ /* * Copyright (C) by Olivier Goffart - * Copyright (C) by Klaas Freitag * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -14,13 +13,12 @@ */ #pragma once - #include "owncloudpropagator.h" -#include "owncloudpropagator_p.h" #include "networkjobs.h" #include #include +#include namespace OCC { @@ -51,15 +49,17 @@ public: class PUTFileJob : public AbstractNetworkJob { Q_OBJECT - QIODevice* _device; + QSharedPointer _device; QMap _headers; QString _errorString; public: // Takes ownership of the device explicit PUTFileJob(Account* account, const QString& path, QIODevice *device, - const QMap &headers, QObject* parent = 0) - : AbstractNetworkJob(account, path, parent), _device(device), _headers(headers) {} + const QMap &headers, int chunk, QObject* parent = 0) + : AbstractNetworkJob(account, path, parent), _device(device), _headers(headers), _chunk(chunk) {} + + int _chunk; virtual void start() Q_DECL_OVERRIDE; @@ -80,96 +80,56 @@ signals: void uploadProgress(qint64,qint64); }; +class PollJob : public AbstractNetworkJob { + Q_OBJECT + SyncJournalDb *_journal; + QString _localPath; +public: + SyncFileItem _item; + // Takes ownership of the device + explicit PollJob(Account* account, const QString &path, const SyncFileItem &item, + SyncJournalDb *journal, const QString &localPath, QObject *parent) + : AbstractNetworkJob(account, path, parent), _journal(journal), _localPath(localPath), _item(item) {} + + void start() Q_DECL_OVERRIDE; + bool finished() Q_DECL_OVERRIDE; + void slotTimeout() Q_DECL_OVERRIDE { +// emit finishedSignal(false); +// deleteLater(); + qDebug() << Q_FUNC_INFO; + reply()->abort(); + } + +signals: + void finishedSignal(); +}; + class PropagateUploadFileQNAM : public PropagateItemJob { Q_OBJECT - QPointer _job; QFile *_file; int _startChunk; int _currentChunk; int _chunkCount; int _transferId; QElapsedTimer _duration; + QVector _jobs; + bool _finished; public: PropagateUploadFileQNAM(OwncloudPropagator* propagator,const SyncFileItem& item) - : PropagateItemJob(propagator, item), _startChunk(0), _currentChunk(0), _chunkCount(0), _transferId(0) {} + : PropagateItemJob(propagator, item), _startChunk(0), _currentChunk(0), _chunkCount(0), _transferId(0), _finished(false) {} void start() Q_DECL_OVERRIDE; private slots: void slotPutFinished(); + void slotPollFinished(); void slotUploadProgress(qint64,qint64); void abort() Q_DECL_OVERRIDE; void startNextChunk(); void finalize(const SyncFileItem&); + void slotJobDestroyed(QObject *job); +private: + void startPollJob(const QString& path); }; - -class GETFileJob : public AbstractNetworkJob { - Q_OBJECT - QFile* _device; - QMap _headers; - QString _errorString; - QByteArray _expectedEtagForResume; - quint64 _resumeStart; - SyncFileItem::Status _errorStatus; - QUrl _directDownloadUrl; - QByteArray _etag; -public: - - // DOES NOT take owncership of the device. - explicit GETFileJob(Account* account, const QString& path, QFile *device, - const QMap &headers, QByteArray expectedEtagForResume, - quint64 resumeStart, QObject* parent = 0); - // For directDownloadUrl: - explicit GETFileJob(Account* account, const QUrl& url, QFile *device, - const QMap &headers, - QObject* parent = 0); - - virtual void start() Q_DECL_OVERRIDE; - virtual bool finished() Q_DECL_OVERRIDE { - emit finishedSignal(); - return true; - } - - QString errorString() { - return _errorString.isEmpty() ? reply()->errorString() : _errorString; - } - - SyncFileItem::Status errorStatus() { return _errorStatus; } - - virtual void slotTimeout() Q_DECL_OVERRIDE; - - QByteArray &etag() { return _etag; } - quint64 resumeStart() { return _resumeStart; } - - -signals: - void finishedSignal(); - void downloadProgress(qint64,qint64); -private slots: - void slotReadyRead(); - void slotMetaDataChanged(); -}; - - -class PropagateDownloadFileQNAM : public PropagateItemJob { - Q_OBJECT - QPointer _job; - -// QFile *_file; - QFile _tmpFile; -public: - PropagateDownloadFileQNAM(OwncloudPropagator* propagator,const SyncFileItem& item) - : PropagateItemJob(propagator, item) {} - void start() Q_DECL_OVERRIDE; -private slots: - void slotGetFinished(); - void abort() Q_DECL_OVERRIDE; - void downloadFinished(); - void slotDownloadProgress(qint64,qint64); - - -}; - - - } + diff --git a/src/libsync/propagator_legacy.cpp b/src/libsync/propagator_legacy.cpp index 3aba92122..8d9dda1d0 100644 --- a/src/libsync/propagator_legacy.cpp +++ b/src/libsync/propagator_legacy.cpp @@ -483,7 +483,7 @@ void PropagateDownloadFileLegacy::notify_status_cb(void* userdata, ne_session_st } } -extern QString makeConflictFileName(const QString &fn, const QDateTime &dt); // _qnam.cpp +extern QString makeConflictFileName(const QString &fn, const QDateTime &dt); // propagatedownload.cpp void PropagateDownloadFileLegacy::start() { diff --git a/src/libsync/propagatorjobs.cpp b/src/libsync/propagatorjobs.cpp index e9d6e510d..d284d66fe 100644 --- a/src/libsync/propagatorjobs.cpp +++ b/src/libsync/propagatorjobs.cpp @@ -135,38 +135,6 @@ void PropagateLocalMkdir::start() done(SyncFileItem::Success); } -void PropagateRemoteRemove::start() -{ - if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) - return; - - QScopedPointer uri( - ne_path_escape((_propagator->_remoteDir + _item._file).toUtf8())); - emit progress(_item, 0); - qDebug() << "** DELETE " << uri.data(); - int rc = ne_delete(_propagator->_session, uri.data()); - - QString errorString = QString::fromUtf8(ne_get_error(_propagator->_session)); - int httpStatusCode = errorString.mid(0, errorString.indexOf(QChar(' '))).toInt(); - if( checkForProblemsWithShared(httpStatusCode, - tr("The file has been removed from a read only share. It was restored.")) ) { - return; - } - - /* Ignore the error 404, it means it is already deleted */ - if (updateErrorFromSession(rc, 0, 404)) { - return; - } - - // Wed, 15 Nov 1995 06:25:24 GMT - QDateTime dt = QDateTime::currentDateTimeUtc(); - _item._responseTimeStamp = dt.toString("hh:mm:ss"); - - _propagator->_journal->deleteFileRecord(_item._originalFile, _item._isDirectory); - _propagator->_journal->commit("Remote Remove"); - done(SyncFileItem::Success); -} - /* The list of properties that is fetched in PropFind after a MKCOL */ static const ne_propname ls_props[] = { { "DAV:", "getetag"}, @@ -295,54 +263,6 @@ void PropagateLocalRename::start() done(SyncFileItem::Success); } -void PropagateRemoteRename::start() -{ - if (_propagator->_abortRequested.fetchAndAddRelaxed(0)) - return; - - if (_item._file == _item._renameTarget) { - // The parents has been renamed already so there is nothing more to do. - } else if (_item._file == QLatin1String("Shared") ) { - // Check if it is the toplevel Shared folder and do not propagate it. - if( QFile::rename( _propagator->_localDir + _item._renameTarget, _propagator->_localDir + QLatin1String("Shared")) ) { - done(SyncFileItem::NormalError, tr("This folder must not be renamed. It is renamed back to its original name.")); - } else { - done(SyncFileItem::NormalError, tr("This folder must not be renamed. Please name it back to Shared.")); - } - return; - } else { - emit progress(_item, 0); - - QScopedPointer uri1(ne_path_escape((_propagator->_remoteDir + _item._file).toUtf8())); - QScopedPointer uri2(ne_path_escape((_propagator->_remoteDir + _item._renameTarget).toUtf8())); - qDebug() << "MOVE on Server: " << uri1.data() << "->" << uri2.data(); - - int rc = ne_move(_propagator->_session, 1, uri1.data(), uri2.data()); - - QString errorString = QString::fromUtf8(ne_get_error(_propagator->_session)); - int httpStatusCode = errorString.mid(0, errorString.indexOf(QChar(' '))).toInt(); - if( checkForProblemsWithShared(httpStatusCode, - tr("The file was renamed but is part of a read only share. The original file was restored."))) { - return; - } - - if (updateErrorFromSession(rc)) { - return; - } - } - // Wed, 15 Nov 1995 06:25:24 GMT - QDateTime dt = QDateTime::currentDateTimeUtc(); - _item._responseTimeStamp = dt.toString("hh:mm:ss"); - - _propagator->_journal->deleteFileRecord(_item._originalFile); - SyncJournalFileRecord record(_item, _propagator->_localDir + _item._renameTarget); - record._path = _item._renameTarget; - - _propagator->_journal->setFileRecord(record); - _propagator->_journal->commit("Remote Rename"); - done(SyncFileItem::Success); -} - bool PropagateNeonJob::updateErrorFromSession(int neon_code, ne_request* req, int ignoreHttpCode) { if( neon_code != NE_OK ) { diff --git a/src/libsync/propagatorjobs.h b/src/libsync/propagatorjobs.h index 786357be9..d321cd663 100644 --- a/src/libsync/propagatorjobs.h +++ b/src/libsync/propagatorjobs.h @@ -83,12 +83,6 @@ public: void start() Q_DECL_OVERRIDE; }; -class PropagateRemoteRemove : public PropagateNeonJob { - Q_OBJECT -public: - PropagateRemoteRemove (OwncloudPropagator* propagator,const SyncFileItem& item) : PropagateNeonJob(propagator, item) {} - void start() Q_DECL_OVERRIDE; -}; class PropagateRemoteMkdir : public PropagateNeonJob { Q_OBJECT public: @@ -105,13 +99,6 @@ public: PropagateLocalRename (OwncloudPropagator* propagator,const SyncFileItem& item) : PropagateItemJob(propagator, item) {} void start() Q_DECL_OVERRIDE; }; -class PropagateRemoteRename : public PropagateNeonJob { - Q_OBJECT -public: - PropagateRemoteRename (OwncloudPropagator* propagator,const SyncFileItem& item) : PropagateNeonJob(propagator, item) {} - void start() Q_DECL_OVERRIDE; -}; - // To support older owncloud in the class UpdateMTimeAndETagJob : public PropagateNeonJob{ diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index 800899b92..76f0ca916 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -23,6 +23,7 @@ #include "creds/abstractcredentials.h" #include "csync_util.h" #include "syncfilestatus.h" +#include "csync_private.h" #ifdef Q_OS_WIN #include @@ -520,11 +521,31 @@ void SyncEngine::handleSyncError(CSYNC *ctx, const char *state) { void SyncEngine::startSync() { + if (_journal->exists()) { + QVector< SyncJournalDb::PollInfo > pollInfos = _journal->getPollInfos(); + if (!pollInfos.isEmpty()) { + qDebug() << "Finish Poll jobs before starting a sync"; + CleanupPollsJob *job = new CleanupPollsJob(pollInfos, AccountManager::instance()->account(), + _journal, _localPath, this); + connect(job, SIGNAL(finished()), this, SLOT(startSync())); + connect(job, SIGNAL(aborted(QString)), this, SLOT(slotCleanPollsJobAborted(QString))); + job->start(); + return; + } + } + Q_ASSERT(!_syncRunning); _syncRunning = true; Q_ASSERT(_csync_ctx); + if (!QDir(_localPath).exists()) { + // No _tr, it should only occur in non-mirall + emit csyncError("Unable to find local sync directory."); + finalize(); + return; + } + _syncedItems.clear(); _syncItemMap.clear(); _needsUpdate = false; @@ -654,6 +675,11 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) qDebug() << "Error in remote treewalk."; } + if (_csync_ctx->remote.root_perms) { + _remotePerms[QLatin1String("")] = _csync_ctx->remote.root_perms; + qDebug() << "Permissions of the root folder: " << _remotePerms[QLatin1String("")]; + } + // The map was used for merging trees, convert it to a list: _syncedItems = _syncItemMap.values().toVector(); @@ -729,6 +755,12 @@ void SyncEngine::slotDiscoveryJobFinished(int discoveryResult) _propagator->start(_syncedItems); } +void SyncEngine::slotCleanPollsJobAborted(const QString &error) +{ + csyncError(error); + finalize(); +} + void SyncEngine::setNetworkLimits(int upload, int download) { _uploadLimit = upload; diff --git a/src/libsync/syncengine.h b/src/libsync/syncengine.h index 6387334c8..848a1a6d1 100644 --- a/src/libsync/syncengine.h +++ b/src/libsync/syncengine.h @@ -103,6 +103,7 @@ private slots: void slotProgress(const SyncFileItem& item, quint64 curent); void slotAdjustTotalTransmissionSize(qint64 change); void slotDiscoveryJobFinished(int updateResult); + void slotCleanPollsJobAborted(const QString &error); private: void handleSyncError(CSYNC *ctx, const char *state); diff --git a/src/libsync/syncfileitem.h b/src/libsync/syncfileitem.h index 2dce6c340..426deefda 100644 --- a/src/libsync/syncfileitem.h +++ b/src/libsync/syncfileitem.h @@ -65,7 +65,10 @@ public: } QString destination() const { - return _instruction == CSYNC_INSTRUCTION_RENAME ? _renameTarget : _file; + if (!_renameTarget.isEmpty()) { + return _renameTarget; + } + return _file; } bool isEmpty() const { diff --git a/src/libsync/syncjournaldb.cpp b/src/libsync/syncjournaldb.cpp index 401ab3a18..d006816e6 100644 --- a/src/libsync/syncjournaldb.cpp +++ b/src/libsync/syncjournaldb.cpp @@ -227,6 +227,15 @@ bool SyncJournalDb::checkConnect() return sqlFail("Create table blacklist", createQuery); } + createQuery.prepare("CREATE TABLE IF NOT EXISTS poll(" + "path VARCHAR(4096)," + "modtime INTEGER(8)," + "pollpath VARCHAR(4096));"); + if (!createQuery.exec()) { + return sqlFail("Create table poll", createQuery); + } + + createQuery.prepare("CREATE TABLE IF NOT EXISTS version(" "major INTEGER(8)," "minor INTEGER(8)," @@ -1118,6 +1127,63 @@ void SyncJournalDb::updateBlacklistEntry( const SyncJournalBlacklistRecord& item } +QVector< SyncJournalDb::PollInfo > SyncJournalDb::getPollInfos() +{ + QMutexLocker locker(&_mutex); + + QVector< SyncJournalDb::PollInfo > res; + + if( !checkConnect() ) + return res; + + SqlQuery query("SELECT path, modtime, pollpath FROM poll",_db); + + if (!query.exec()) { + QString err = query.error(); + qDebug() << "Database error :" << query.lastQuery() << ", Error:" << err; + return res; + } + + while( query.next() ) { + PollInfo info; + info._file = query.stringValue(0); + info._modtime = query.int64Value(1); + info._url = query.stringValue(2); + res.append(info); + } + + query.finish(); + return res; +} + +void SyncJournalDb::setPollInfo(const SyncJournalDb::PollInfo& info) +{ + QMutexLocker locker(&_mutex); + if( !checkConnect() ) { + return; + } + + if (info._url.isEmpty()) { + SqlQuery query("DELETE FROM poll WHERE path=?", _db); + query.bindValue(0, info._file); + if( !query.exec() ) { + qDebug() << "SQL error in setPollInfo: "<< query.error(); + } else { + qDebug() << query.lastQuery() << info._file; + } + } else { + SqlQuery query("INSERT OR REPLACE INTO poll (path, modtime, pollpath) VALUES( ? , ? , ? )", _db); + query.bindValue(0, info._file); + query.bindValue(1, QString::number(info._modtime)); + query.bindValue(2, info._url); + if( !query.exec() ) { + qDebug() << "SQL error in setPollInfo: "<< query.error(); + } else { + qDebug() << query.lastQuery() << info._file << info._url; + } + } +} + void SyncJournalDb::avoidRenamesOnNextSync(const QString& path) { QMutexLocker locker(&_mutex); diff --git a/src/libsync/syncjournaldb.h b/src/libsync/syncjournaldb.h index 85608459f..c174ff6cb 100644 --- a/src/libsync/syncjournaldb.h +++ b/src/libsync/syncjournaldb.h @@ -69,6 +69,12 @@ public: bool _valid; }; + struct PollInfo { + QString _file; + QString _url; + time_t _modtime; + }; + DownloadInfo getDownloadInfo(const QString &file); void setDownloadInfo(const QString &file, const DownloadInfo &i); QVector getAndDeleteStaleDownloadInfos(const QSet& keep); @@ -82,6 +88,8 @@ public: bool deleteStaleBlacklistEntries(const QSet& keep); void avoidRenamesOnNextSync(const QString &path); + void setPollInfo(const PollInfo &); + QVector getPollInfos(); /** * Make sure that on the next sync, filName is not read from the DB but use the PROPFIND to diff --git a/src/libsync/syncjournalfilerecord.cpp b/src/libsync/syncjournalfilerecord.cpp index 91b6ae734..46adce528 100644 --- a/src/libsync/syncjournalfilerecord.cpp +++ b/src/libsync/syncjournalfilerecord.cpp @@ -98,7 +98,7 @@ static time_t getMaxBlacklistTime() bool SyncJournalBlacklistRecord::isValid() const { return ! _file.isEmpty() - && (!_lastTryEtag.isEmpty() || !_lastTryModtime == 0) + && (!_lastTryEtag.isEmpty() || _lastTryModtime != 0) && _lastTryTime > 0 && _ignoreDuration > 0; } diff --git a/src/libsync/utility.cpp b/src/libsync/utility.cpp index a91ee0e2c..db6c187ac 100644 --- a/src/libsync/utility.cpp +++ b/src/libsync/utility.cpp @@ -377,6 +377,11 @@ bool Utility::isLinux() #endif } +void Utility::crash() +{ + volatile int* a = (int*)(NULL); + *a = 1; +} static const char STOPWATCH_END_TAG[] = "_STOPWATCH_END"; diff --git a/src/libsync/utility.h b/src/libsync/utility.h index 849db9fab..abc172f92 100644 --- a/src/libsync/utility.h +++ b/src/libsync/utility.h @@ -90,6 +90,9 @@ namespace Utility OWNCLOUDSYNC_EXPORT bool isUnix(); OWNCLOUDSYNC_EXPORT bool isLinux(); // use with care + // crash helper for --debug + OWNCLOUDSYNC_EXPORT void crash(); + // Case preserving file system underneath? // if this function returns true, the file system is case preserving, // that means "test" means the same as "TEST" for filenames.