wellton/client
1
0
Fork 0
Código Issues Pull Requests Ações Pacotes Projetos Versões Wiki Atividade

Checksums: Don't reupload if size and checksum are unchanged #3235

Ver Código-Fonte 
* Compute the content checksum (in addition to the optional
  transmission checksum) during upload (.eml files only)

* Add hook to compute and compare the checksum in csync_update

* Add content checksum to database, remove transmission checksum
Esse commit está contido em:
Christian Kamm
2015-11-23 11:53:06 +01:00
pai bdf830f691
commit a25f094c4c
23 arquivos alterados com 316 adições e 142 exclusões
Baixar Arquivo de Patch Baixar Arquivo de Diff Expandir todos os arquivos Colapsar todos os arquivos
csync/src/csync.c
+1
Ver Arquivo
@@ -742,6 +742,7 @@ void csync_file_stat_free(csync_file_stat_t *st)
SAFE_FREE(st->directDownloadCookies);
SAFE_FREE(st->etag);
SAFE_FREE(st->destpath);
SAFE_FREE(st->checksum);
SAFE_FREE(st);
}
}
csync/src/csync.h
+6
Ver Arquivo
@@ -301,6 +301,12 @@ typedef void (*csync_vio_closedir_hook) (csync_vio_handle_t *dhhandle,
typedef int (*csync_vio_stat_hook) (csync_vio_handle_t *dhhandle,
void *userdata);
/* compute the checksum of the given \a checksumTypeId for \a path
* and return true if it's the same as \a checksum */
typedef bool (*csync_checksum_hook) (const char *path,
uint32_t checksumTypeId, const char *checksum,
void *userdata);
/**
* @brief Allocate a csync context.
*
csync/src/csync_private.h
+8
Ver Arquivo
@@ -96,6 +96,11 @@ struct csync_s {
csync_vio_readdir_hook remote_readdir_hook;
csync_vio_closedir_hook remote_closedir_hook;
void *vio_userdata;
/* hook for comparing checksums of files during discovery */
csync_checksum_hook checksum_hook;
void *checksum_userdata;
} callbacks;
c_strlist_t *excludes;
@@ -192,6 +197,9 @@ struct csync_file_stat_s {
char *directDownloadCookies;
char remotePerm[REMOTE_PERM_BUF_SIZE+1];
char *checksum;
uint32_t checksumTypeId;
CSYNC_STATUS error_status;
enum csync_instructions_e instruction; /* u32 */
csync/src/csync_statedb.c
+11 -4
Ver Arquivo
@@ -226,6 +226,8 @@ int csync_statedb_close(CSYNC *ctx) {
return rc;
}
#define METADATA_COLUMNS "phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId"
// This funciton parses a line from the metadata table into the given csync_file_stat
// structure which it is also allocating.
// Note that this function calls laso sqlite3_step to actually get the info from db and
@@ -286,6 +288,11 @@ static int _csync_file_stat_from_metadata_table( csync_file_stat_t **st, sqlite3
if(column_count > 13) {
(*st)->has_ignored_files = sqlite3_column_int(stmt, 13);
}
if(column_count > 15 && sqlite3_column_int(stmt, 15)) {
(*st)->checksum = c_strdup( (char*) sqlite3_column_text(stmt, 14));
(*st)->checksumTypeId = sqlite3_column_int(stmt, 15);
}
}
} else {
if( rc != SQLITE_DONE ) {
@@ -307,7 +314,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_hash(CSYNC *ctx,
}
if( ctx->statedb.by_hash_stmt == NULL ) {
const char *hash_query = "SELECT * FROM metadata WHERE phash=?1";
const char *hash_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE phash=?1";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, hash_query, strlen(hash_query), &ctx->statedb.by_hash_stmt, NULL));
ctx->statedb.lastReturnValue = rc;
@@ -350,7 +357,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_file_id(CSYNC *ctx,
}
if( ctx->statedb.by_fileid_stmt == NULL ) {
const char *query = "SELECT * FROM metadata WHERE fileid=?1";
const char *query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE fileid=?1";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, query, strlen(query), &ctx->statedb.by_fileid_stmt, NULL));
ctx->statedb.lastReturnValue = rc;
@@ -390,7 +397,7 @@ csync_file_stat_t *csync_statedb_get_stat_by_inode(CSYNC *ctx,
}
if( ctx->statedb.by_inode_stmt == NULL ) {
const char *inode_query = "SELECT * FROM metadata WHERE inode=?1";
const char *inode_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE inode=?1";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, inode_query, strlen(inode_query), &ctx->statedb.by_inode_stmt, NULL));
ctx->statedb.lastReturnValue = rc;
@@ -433,7 +440,7 @@ int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) {
* In other words, anything that is between path+'/' and path+'0',
* (because '0' follows '/' in ascii)
*/
const char *below_path_query = "SELECT phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote FROM metadata WHERE path > (?||'/') AND path < (?||'0')";
const char *below_path_query = "SELECT " METADATA_COLUMNS " FROM metadata WHERE path > (?||'/') AND path < (?||'0')";
SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, below_path_query, -1, &stmt, NULL));
ctx->statedb.lastReturnValue = rc;
if( rc != SQLITE_OK ) {
csync/src/csync_update.c
+35 -17
Ver Arquivo
@@ -270,25 +270,43 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
((int64_t) fs->mtime), ((int64_t) tmp->modtime),
fs->etag, tmp->etag, (uint64_t) fs->inode, (uint64_t) tmp->inode,
(uint64_t) fs->size, (uint64_t) tmp->size, fs->remotePerm, tmp->remotePerm, tmp->has_ignored_files );
if((ctx->current == REMOTE_REPLICA && !c_streq(fs->etag, tmp->etag ))
|| (ctx->current == LOCAL_REPLICA && (!_csync_mtime_equal(fs->mtime, tmp->modtime)
// zero size in statedb can happen during migration
|| (tmp->size != 0 && fs->size != tmp->size)
if (ctx->current == REMOTE_REPLICA && !c_streq(fs->etag, tmp->etag)) {
st->instruction = CSYNC_INSTRUCTION_EVAL;
goto out;
}
if (ctx->current == LOCAL_REPLICA &&
(!_csync_mtime_equal(fs->mtime, tmp->modtime)
// zero size in statedb can happen during migration
|| (tmp->size != 0 && fs->size != tmp->size)
#if 0
|| fs->inode != tmp->inode
/* Comparison of the local inode is disabled because people reported problems
* on windows with flacky inode values, see github bug #779
*
* The inode needs to be observed because:
* $> echo a > a.txt ; echo b > b.txt
* both files have the same mtime
* sync them.
* $> rm a.txt && mv b.txt a.txt
* makes b.txt appearing as a.txt yet a sync is not performed because
* both have the same modtime as mv does not change that.
*/
|| fs->inode != tmp->inode
#endif
))) {
/* Comparison of the local inode is disabled because people reported problems
* on windows with flacky inode values, see github bug #779
*
* The inode needs to be observed because:
* $> echo a > a.txt ; echo b > b.txt
* both files have the same mtime
* sync them.
* $> rm a.txt && mv b.txt a.txt
* makes b.txt appearing as a.txt yet a sync is not performed because
* both have the same modtime as mv does not change that.
*/
)) {
if (fs->size == tmp->size && tmp->checksumTypeId) {
bool checksumIdentical = false;
if (ctx->callbacks.checksum_hook) {
checksumIdentical = ctx->callbacks.checksum_hook(
file, tmp->checksumTypeId, tmp->checksum,
ctx->callbacks.checksum_userdata);
}
if (checksumIdentical) {
st->instruction = CSYNC_INSTRUCTION_NONE;
st->should_update_metadata = true;
goto out;
}
}
st->instruction = CSYNC_INSTRUCTION_EVAL;
goto out;
}
csync/tests/csync_tests/check_csync_update.c
+6
Ver Arquivo
@@ -41,6 +41,12 @@ static void statedb_create_metadata_table(sqlite3 *db)
"modtime INTEGER(8),"
"type INTEGER,"
"md5 VARCHAR(32),"
"fileid VARCHAR(128),"
"remotePerm VARCHAR(128),"
"filesize BIGINT,"
"ignoredChildrenRemote INT,"
"contentChecksum TEXT,"
"contentChecksumTypeId INTEGER,"
"PRIMARY KEY(phash));";
rc = sqlite3_exec(db, sql, NULL, NULL, NULL);
src/libsync/propagatedownload.cpp
+3 -11
Ver Arquivo
@@ -351,10 +351,7 @@ void PropagateDownloadFileQNAM::start()
if (_resumeStart == _item->_size) {
qDebug() << "File is already complete, no need to download";
_tmpFile.close();
// Unfortunately we lost the checksum header, if any...
QByteArray noChecksumData;
downloadFinished(noChecksumData, noChecksumData);
downloadFinished();
return;
}
}
@@ -537,7 +534,7 @@ void PropagateDownloadFileQNAM::slotGetFinished()
// as this is (still) also correct.
ValidateChecksumHeader *validator = new ValidateChecksumHeader(this);
connect(validator, SIGNAL(validated(QByteArray,QByteArray)),
SLOT(downloadFinished(QByteArray,QByteArray)));
SLOT(downloadFinished()));
connect(validator, SIGNAL(validationFailed(QString)),
SLOT(slotChecksumFail(QString)));
auto checksumHeader = job->reply()->rawHeader(checkSumHeaderC);
@@ -621,13 +618,8 @@ static void handleRecallFile(const QString &fn)
}
} // end namespace
void PropagateDownloadFileQNAM::downloadFinished(const QByteArray& checksumType, const QByteArray& checksum)
void PropagateDownloadFileQNAM::downloadFinished()
{
if (!checksumType.isEmpty()) {
_item->_transmissionChecksum = checksum;
_item->_transmissionChecksumType = checksumType;
}
QString fn = _propagator->getFilePath(_item->_file);
// In case of file name clash, report an error
src/libsync/propagatedownload.h
+1 -1
Ver Arquivo
@@ -117,7 +117,7 @@ public:
private slots:
void slotGetFinished();
void abort() Q_DECL_OVERRIDE;
void downloadFinished(const QByteArray& checksumType, const QByteArray& checksum);
void downloadFinished();
void slotDownloadProgress(qint64,qint64);
void slotChecksumFail( const QString& errMsg );
src/libsync/propagateremotemove.cpp
+2 -2
Ver Arquivo
@@ -158,8 +158,8 @@ void PropagateRemoteMove::finalize()
SyncJournalFileRecord record(*_item, _propagator->getFilePath(_item->_renameTarget));
record._path = _item->_renameTarget;
record._transmissionChecksum = oldRecord._transmissionChecksum;
record._transmissionChecksumType = oldRecord._transmissionChecksumType;
record._contentChecksum = oldRecord._contentChecksum;
record._contentChecksumType = oldRecord._contentChecksumType;
_propagator->_journal->setFileRecord(record);
_propagator->_journal->commit("Remote Rename");
src/libsync/propagateupload.cpp
+41 -29
Ver Arquivo
@@ -216,24 +216,38 @@ void PropagateUploadFileQNAM::start()
_stopWatch.start();
auto supportedChecksumTypes = _propagator->account()->capabilities().supportedChecksumTypes();
// If we already have a checksum header and the checksum type is supported
// by the server, we keep that - otherwise recompute.
//
// Note: Currently we *always* recompute because we usually only upload
// files that have changed and thus have a new checksum. But if an earlier
// phase computed a checksum, this is where we would make use of it.
if (!_item->_transmissionChecksumType.isEmpty()) {
if (supportedChecksumTypes.contains(_item->_transmissionChecksumType)) {
// TODO: We could validate the old checksum and thereby determine whether
// an upload is necessary or not.
slotStartUpload(_item->_transmissionChecksumType, _item->_transmissionChecksum);
return;
}
// Compute the content checksum.
auto computeChecksum = new ComputeChecksum(this);
// We currently only do content checksums for the particular .eml case
// This should be done more generally in the future!
if (filePath.endsWith(QLatin1String(".eml"), Qt::CaseInsensitive)) {
computeChecksum->setChecksumType("MD5");
} else {
computeChecksum->setChecksumType(QByteArray());
}
// Compute a new checksum.
connect(computeChecksum, SIGNAL(done(QByteArray,QByteArray)),
SLOT(slotComputeTransmissionChecksum(QByteArray,QByteArray)));
computeChecksum->start(filePath);
}
void PropagateUploadFileQNAM::slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum)
{
_item->_contentChecksum = contentChecksum;
_item->_contentChecksumType = contentChecksumType;
_stopWatch.addLapTime(QLatin1String("ContentChecksum"));
_stopWatch.start();
// Reuse the content checksum as the transmission checksum if possible
const auto supportedTransmissionChecksums =
_propagator->account()->capabilities().supportedChecksumTypes();
if (supportedTransmissionChecksums.contains(contentChecksumType)) {
slotStartUpload(contentChecksumType, contentChecksum);
return;
}
// Compute the transmission checksum.
auto computeChecksum = new ComputeChecksum(this);
if (uploadChecksumEnabled()) {
computeChecksum->setChecksumType(_propagator->account()->capabilities().preferredChecksumType());
@@ -243,19 +257,14 @@ void PropagateUploadFileQNAM::start()
connect(computeChecksum, SIGNAL(done(QByteArray,QByteArray)),
SLOT(slotStartUpload(QByteArray,QByteArray)));
const QString filePath = _propagator->getFilePath(_item->_file);
computeChecksum->start(filePath);
}
void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& checksumType, const QByteArray& checksum)
void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum)
{
// Store the computed checksum in the database, if different
if (checksumType != _item->_transmissionChecksumType
|| checksum != _item->_transmissionChecksum) {
_item->_transmissionChecksum = checksum;
_item->_transmissionChecksumType = checksumType;
_propagator->_journal->updateFileRecordChecksum(
_item->_file, checksum, checksumType);
}
_transmissionChecksum = transmissionChecksum;
_transmissionChecksumType = transmissionChecksumType;
const QString fullFilePath = _propagator->getFilePath(_item->_file);
@@ -263,7 +272,7 @@ void PropagateUploadFileQNAM::slotStartUpload(const QByteArray& checksumType, co
done(SyncFileItem::SoftError, tr("File Removed"));
return;
}
_stopWatch.addLapTime(QLatin1String("Checksum"));
_stopWatch.addLapTime(QLatin1String("TransmissionChecksum"));
time_t prevModtime = _item->_modtime; // the _item value was set in PropagateUploadFileQNAM::start()
// but a potential checksum calculation could have taken some time during which the file could
@@ -511,9 +520,9 @@ void PropagateUploadFileQNAM::startNextChunk()
isFinalChunk = true;
}
if (isFinalChunk && !_item->_transmissionChecksumType.isEmpty()) {
if (isFinalChunk && !_transmissionChecksumType.isEmpty()) {
headers[checkSumHeaderC] = makeChecksumHeader(
_item->_transmissionChecksumType, _item->_transmissionChecksum);
_transmissionChecksumType, _transmissionChecksum);
}
if (! device->prepareAndOpen(_propagator->getFilePath(_item->_file), chunkStart, currentChunkSize)) {
@@ -724,7 +733,10 @@ void PropagateUploadFileQNAM::slotPutFinished()
// performance logging
_item->_requestDuration = _stopWatch.stop();
qDebug() << "*==* duration UPLOAD" << _item->_size << _stopWatch.durationOfLap(QLatin1String("Checksum")) << _item->_requestDuration;
qDebug() << "*==* duration UPLOAD" << _item->_size
<< _stopWatch.durationOfLap(QLatin1String("ContentChecksum"))
<< _stopWatch.durationOfLap(QLatin1String("TransmissionChecksum"))
<< _item->_requestDuration;
finalize(*_item);
}
src/libsync/propagateupload.h
+5 -1
Ver Arquivo
@@ -183,6 +183,9 @@ private:
// measure the performance of checksum calc and upload
Utility::StopWatch _stopWatch;
QByteArray _transmissionChecksum;
QByteArray _transmissionChecksumType;
public:
PropagateUploadFileQNAM(OwncloudPropagator* propagator,const SyncFileItemPtr& item)
: PropagateItemJob(propagator, item), _startChunk(0), _currentChunk(0), _chunkCount(0), _transferId(0), _finished(false) {}
@@ -195,7 +198,8 @@ private slots:
void startNextChunk();
void finalize(const SyncFileItem&);
void slotJobDestroyed(QObject *job);
void slotStartUpload(const QByteArray& checksumType, const QByteArray& checksum);
void slotStartUpload(const QByteArray& transmissionChecksumType, const QByteArray& transmissionChecksum);
void slotComputeTransmissionChecksum(const QByteArray& contentChecksumType, const QByteArray& contentChecksum);
private:
void startPollJob(const QString& path);
src/libsync/propagatorjobs.cpp
+2 -2
Ver Arquivo
@@ -209,8 +209,8 @@ void PropagateLocalRename::start()
SyncJournalFileRecord record(*_item, targetFile);
record._path = _item->_renameTarget;
record._transmissionChecksum = oldRecord._transmissionChecksum;
record._transmissionChecksumType = oldRecord._transmissionChecksumType;
record._contentChecksum = oldRecord._contentChecksum;
record._contentChecksumType = oldRecord._contentChecksumType;
if (!_item->_isDirectory) { // Directories are saved at the end
_propagator->_journal->setFileRecord(record);
src/libsync/syncengine.cpp
+26 -13
Ver Arquivo
@@ -71,6 +71,7 @@ SyncEngine::SyncEngine(AccountPtr account, CSYNC *ctx, const QString& localPath,
, _uploadLimit(0)
, _downloadLimit(0)
, _newBigFolderSizeLimit(-1)
, _checksum_hook(journal)
, _anotherSyncNeeded(false)
{
qRegisterMetaType<SyncFileItem>("SyncFileItem");
@@ -466,10 +467,11 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
int re = 0;
switch(file->instruction) {
case CSYNC_INSTRUCTION_NONE:
case CSYNC_INSTRUCTION_NONE: {
if (remote && item->_should_update_metadata && !item->_isDirectory && item->_instruction == CSYNC_INSTRUCTION_NONE) {
// Update the database now already: New fileid or Etag or RemotePerm
// Update the database now already: New remote fileid or Etag or RemotePerm
// Or for files that were detected as "resolved conflict".
// Or a local inode/mtime change (see localMetadataUpdate below)
// In case of "resolved conflict": there should have been a conflict because they
// both were new, or both had their local mtime or remote etag modified, but the
@@ -496,21 +498,28 @@ int SyncEngine::treewalkFile( TREE_WALK_FILE *file, bool remote )
_journal->setFileRecordMetadata(SyncJournalFileRecord(*item, filePath));
item->_should_update_metadata = false;
}
if (item->_isDirectory && file->should_update_metadata) {
// Because we want to still update etags of directories
dir = SyncFileItem::None;
} else {
// No need to do anything.
if (file->other.instruction == CSYNC_INSTRUCTION_NONE
// Directories with ignored files does not count as 'None'
&& (file->type != CSYNC_FTW_TYPE_DIR || !file->has_ignored_files)) {
_hasNoneFiles = true;
}
// Technically we're done with this item. See localMetadataUpdate hack below.
_syncItemMap.remove(key);
}
// Any files that are instruction NONE?
if (!item->_isDirectory && file->other.instruction == CSYNC_INSTRUCTION_NONE) {
_hasNoneFiles = true;
}
// We want to still update etags of directories, other NONE
// items can be ignored.
bool directoryEtagUpdate = item->_isDirectory && file->should_update_metadata;
bool localMetadataUpdate = !remote && file->should_update_metadata;
if (!directoryEtagUpdate) {
if (localMetadataUpdate) {
// Hack, we want a local metadata update to happen, but only if the
// remote tree doesn't ask us to do some kind of propagation.
_syncItemMap.insert(key, item);
}
return re;
}
break;
}
case CSYNC_INSTRUCTION_RENAME:
dir = !remote ? SyncFileItem::Down : SyncFileItem::Up;
item->_renameTarget = renameTarget;
@@ -686,6 +695,10 @@ void SyncEngine::startSync()
csync_set_userdata(_csync_ctx, this);
// Set up checksumming hook
_csync_ctx->callbacks.checksum_hook = &CSyncChecksumHook::hook;
_csync_ctx->callbacks.checksum_userdata = &_checksum_hook;
_stopWatch.start();
qDebug() << "#### Discovery start #################################################### >>";
src/libsync/syncengine.h
+4
Ver Arquivo
@@ -37,6 +37,7 @@
#include "syncfilestatus.h"
#include "accountfwd.h"
#include "discoveryphase.h"
#include "transmissionchecksumvalidator.h"
class QProcess;
@@ -216,6 +217,9 @@ private:
// hash containing the permissions on the remote directory
QHash<QString, QByteArray> _remotePerms;
/// Hook for computing checksums from csync_update
CSyncChecksumHook _checksum_hook;
bool _anotherSyncNeeded;
};
src/libsync/syncfileitem.h
+2 -2
Ver Arquivo
@@ -165,8 +165,8 @@ public:
quint64 _inode;
QByteArray _fileId;
QByteArray _remotePerm;
QByteArray _transmissionChecksum;
QByteArray _transmissionChecksumType;
QByteArray _contentChecksum;
QByteArray _contentChecksumType;
QString _directDownloadUrl;
QString _directDownloadCookies;
src/libsync/syncjournaldb.cpp
+61 -28
Ver Arquivo
@@ -204,10 +204,13 @@ bool SyncJournalDb::checkConnect()
"modtime INTEGER(8),"
"type INTEGER,"
"md5 VARCHAR(32)," /* This is the etag. Called md5 for compatibility */
// updateDatabaseStructure() will add a fileid column
// updateDatabaseStructure() will add a remotePerm column
// updateDatabaseStructure() will add a transmissionChecksum column
// updateDatabaseStructure() will add a transmissionChecksumTypeId column
// updateDatabaseStructure() will add
// fileid
// remotePerm
// filesize
// ignoredChildrenRemote
// contentChecksum
// contentChecksumTypeId
"PRIMARY KEY(phash)"
");");
@@ -358,20 +361,20 @@ bool SyncJournalDb::checkConnect()
_getFileRecordQuery.reset(new SqlQuery(_db));
_getFileRecordQuery->prepare(
"SELECT path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize,"
" ignoredChildrenRemote, transmissionChecksum, checksumtype.name"
" ignoredChildrenRemote, contentChecksum, contentchecksumtype.name"
" FROM metadata"
" LEFT JOIN checksumtype ON metadata.transmissionChecksumTypeId == checksumtype.id"
" LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id"
" WHERE phash=?1" );
_setFileRecordQuery.reset(new SqlQuery(_db) );
_setFileRecordQuery->prepare("INSERT OR REPLACE INTO metadata "
"(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, transmissionChecksum, transmissionChecksumTypeId) "
"(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId) "
"VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7, ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16);" );
_setFileRecordChecksumQuery.reset(new SqlQuery(_db) );
_setFileRecordChecksumQuery->prepare(
"UPDATE metadata"
" SET transmissionChecksum = ?2, transmissionChecksumTypeId = ?3"
" SET contentChecksum = ?2, contentChecksumTypeId = ?3"
" WHERE phash == ?1;");
_getDownloadInfoQuery.reset(new SqlQuery(_db) );
@@ -426,6 +429,9 @@ bool SyncJournalDb::checkConnect()
_getChecksumTypeIdQuery.reset(new SqlQuery(_db));
_getChecksumTypeIdQuery->prepare("SELECT id FROM checksumtype WHERE name=?1");
_getChecksumTypeQuery.reset(new SqlQuery(_db));
_getChecksumTypeQuery->prepare("SELECT name FROM checksumtype WHERE id=?1");
_insertChecksumTypeQuery.reset(new SqlQuery(_db));
_insertChecksumTypeQuery->prepare("INSERT OR IGNORE INTO checksumtype (name) VALUES (?1)");
@@ -463,6 +469,7 @@ void SyncJournalDb::close()
_setErrorBlacklistQuery.reset(0);
_getSelectiveSyncListQuery.reset(0);
_getChecksumTypeIdQuery.reset(0);
_getChecksumTypeQuery.reset(0);
_insertChecksumTypeQuery.reset(0);
_db.close();
@@ -557,23 +564,23 @@ bool SyncJournalDb::updateMetadataTableStructure()
commitInternal("update database structure: add ignoredChildrenRemote col");
}
if( columns.indexOf(QLatin1String("transmissionChecksum")) == -1 ) {
if( columns.indexOf(QLatin1String("contentChecksum")) == -1 ) {
SqlQuery query(_db);
query.prepare("ALTER TABLE metadata ADD COLUMN transmissionChecksum TEXT;");
query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksum TEXT;");
if( !query.exec()) {
sqlFail("updateMetadataTableStructure: add transmissionChecksum column", query);
sqlFail("updateMetadataTableStructure: add contentChecksum column", query);
re = false;
}
commitInternal("update database structure: add transmissionChecksum col");
commitInternal("update database structure: add contentChecksum col");
}
if( columns.indexOf(QLatin1String("transmissionChecksumTypeId")) == -1 ) {
if( columns.indexOf(QLatin1String("contentChecksumTypeId")) == -1 ) {
SqlQuery query(_db);
query.prepare("ALTER TABLE metadata ADD COLUMN transmissionChecksumTypeId INTEGER;");
query.prepare("ALTER TABLE metadata ADD COLUMN contentChecksumTypeId INTEGER;");
if( !query.exec()) {
sqlFail("updateMetadataTableStructure: add transmissionChecksumTypeId column", query);
sqlFail("updateMetadataTableStructure: add contentChecksumTypeId column", query);
re = false;
}
commitInternal("update database structure: add transmissionChecksumTypeId col");
commitInternal("update database structure: add contentChecksumTypeId col");
}
@@ -677,7 +684,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record )
if( fileId.isEmpty() ) fileId = "";
QString remotePerm (record._remotePerm);
if (remotePerm.isEmpty()) remotePerm = QString(); // have NULL in DB (vs empty)
int checksumTypeId = mapChecksumType(record._transmissionChecksumType);
int contentChecksumTypeId = mapChecksumType(record._contentChecksumType);
_setFileRecordQuery->reset();
_setFileRecordQuery->bindValue(1, QString::number(phash));
_setFileRecordQuery->bindValue(2, plen);
@@ -693,8 +700,8 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record )
_setFileRecordQuery->bindValue(12, remotePerm );
_setFileRecordQuery->bindValue(13, record._fileSize );
_setFileRecordQuery->bindValue(14, record._serverHasIgnoredFiles ? 1:0);
_setFileRecordQuery->bindValue(15, record._transmissionChecksum );
_setFileRecordQuery->bindValue(16, checksumTypeId );
_setFileRecordQuery->bindValue(15, record._contentChecksum );
_setFileRecordQuery->bindValue(16, contentChecksumTypeId );
if( !_setFileRecordQuery->exec() ) {
qWarning() << "Error SQL statement setFileRecord: " << _setFileRecordQuery->lastQuery() << " :"
@@ -705,7 +712,7 @@ bool SyncJournalDb::setFileRecord( const SyncJournalFileRecord& _record )
qDebug() << _setFileRecordQuery->lastQuery() << phash << plen << record._path << record._inode
<< QString::number(Utility::qDateTimeToTime_t(record._modtime)) << QString::number(record._type)
<< record._etag << record._fileId << record._remotePerm << record._fileSize << (record._serverHasIgnoredFiles ? 1:0)
<< record._transmissionChecksum << record._transmissionChecksumType << checksumTypeId;
<< record._contentChecksum << record._contentChecksumType << contentChecksumTypeId;
_setFileRecordQuery->reset();
return true;
@@ -785,9 +792,9 @@ SyncJournalFileRecord SyncJournalDb::getFileRecord( const QString& filename )
rec._remotePerm = _getFileRecordQuery->baValue(9);
rec._fileSize = _getFileRecordQuery->int64Value(10);
rec._serverHasIgnoredFiles = (_getFileRecordQuery->intValue(11) > 0);
rec._transmissionChecksum = _getFileRecordQuery->baValue(12);
rec._contentChecksum = _getFileRecordQuery->baValue(12);
if( !_getFileRecordQuery->nullValue(13) ) {
rec._transmissionChecksumType = _getFileRecordQuery->baValue(13);
rec._contentChecksumType = _getFileRecordQuery->baValue(13);
}
} else {
QString err = _getFileRecordQuery->error();
@@ -878,8 +885,8 @@ int SyncJournalDb::getFileRecordCount()
}
bool SyncJournalDb::updateFileRecordChecksum(const QString& filename,
const QByteArray& transmisisonChecksum,
const QByteArray& transmissionChecksumType)
const QByteArray& contentChecksum,
const QByteArray& contentChecksumType)
{
QMutexLocker locker(&_mutex);
@@ -889,12 +896,12 @@ bool SyncJournalDb::updateFileRecordChecksum(const QString& filename,
return false;
}
int checksumTypeId = mapChecksumType(transmissionChecksumType);
int checksumTypeId = mapChecksumType(contentChecksumType);
auto & query = _setFileRecordChecksumQuery;
query->reset();
query->bindValue(1, QString::number(phash));
query->bindValue(2, transmisisonChecksum);
query->bindValue(2, contentChecksum);
query->bindValue(3, checksumTypeId);
if( !query->exec() ) {
@@ -904,8 +911,8 @@ bool SyncJournalDb::updateFileRecordChecksum(const QString& filename,
return false;
}
qDebug() << query->lastQuery() << phash << transmisisonChecksum
<< transmissionChecksumType << checksumTypeId;
qDebug() << query->lastQuery() << phash << contentChecksum
<< contentChecksumType << checksumTypeId;
query->reset();
return true;
@@ -1501,6 +1508,32 @@ void SyncJournalDb::forceRemoteDiscoveryNextSyncLocked()
}
}
QByteArray SyncJournalDb::getChecksumType(int checksumTypeId)
{
QMutexLocker locker(&_mutex);
if( !checkConnect() ) {
return QByteArray();
}
// Retrieve the id
auto & query = *_getChecksumTypeQuery;
query.reset();
query.bindValue(1, checksumTypeId);
if( !query.exec() ) {
qWarning() << "Error SQL statement getChecksumType: "
<< query.lastQuery() << " :"
<< query.error();
return 0;
}
if( !query.next() ) {
qDebug() << "No checksum type mapping found for" << checksumTypeId;
return 0;
}
return query.baValue(0);
}
int SyncJournalDb::mapChecksumType(const QByteArray& checksumType)
{
if (checksumType.isEmpty()) {
src/libsync/syncjournaldb.h
+8 -2
Ver Arquivo
@@ -47,8 +47,8 @@ public:
bool deleteFileRecord( const QString& filename, bool recursively = false );
int getFileRecordCount();
bool updateFileRecordChecksum(const QString& filename,
const QByteArray& transmisisonChecksum,
const QByteArray& transmissionChecksumType);
const QByteArray& contentChecksum,
const QByteArray& contentChecksumType);
bool exists();
void walCheckpoint();
@@ -146,6 +146,11 @@ public:
*/
bool isConnected();
/**
* Returns the checksum type for an id.
*/
QByteArray getChecksumType(int checksumTypeId);
private:
bool updateDatabaseStructure();
bool updateMetadataTableStructure();
@@ -186,6 +191,7 @@ private:
QScopedPointer<SqlQuery> _setErrorBlacklistQuery;
QScopedPointer<SqlQuery> _getSelectiveSyncListQuery;
QScopedPointer<SqlQuery> _getChecksumTypeIdQuery;
QScopedPointer<SqlQuery> _getChecksumTypeQuery;
QScopedPointer<SqlQuery> _insertChecksumTypeQuery;
/* This is the list of paths we called avoidReadFromDbOnNextSync on.
src/libsync/syncjournalfilerecord.cpp
+6 -6
Ver Arquivo
@@ -36,8 +36,8 @@ SyncJournalFileRecord::SyncJournalFileRecord(const SyncFileItem &item, const QSt
: _path(item._file), _modtime(Utility::qDateTimeFromTime_t(item._modtime)),
_type(item._type), _etag(item._etag), _fileId(item._fileId), _fileSize(item._size),
_remotePerm(item._remotePerm), _serverHasIgnoredFiles(item._serverHasIgnoredFiles),
_transmissionChecksum(item._transmissionChecksum),
_transmissionChecksumType(item._transmissionChecksumType)
_contentChecksum(item._contentChecksum),
_contentChecksumType(item._contentChecksumType)
{
// use the "old" inode coming with the item for the case where the
// filesystem stat fails. That can happen if the the file was removed
@@ -97,8 +97,8 @@ SyncFileItem SyncJournalFileRecord::toSyncFileItem()
item._size = _fileSize;
item._remotePerm = _remotePerm;
item._serverHasIgnoredFiles = _serverHasIgnoredFiles;
item._transmissionChecksum = _transmissionChecksum;
item._transmissionChecksumType = _transmissionChecksumType;
item._contentChecksum = _contentChecksum;
item._contentChecksumType = _contentChecksumType;
return item;
}
@@ -176,8 +176,8 @@ bool operator==(const SyncJournalFileRecord & lhs,
&& lhs._fileSize == rhs._fileSize
&& lhs._remotePerm == rhs._remotePerm
&& lhs._serverHasIgnoredFiles == rhs._serverHasIgnoredFiles
&& lhs._transmissionChecksum == rhs._transmissionChecksum
&& lhs._transmissionChecksumType == rhs._transmissionChecksumType;
&& lhs._contentChecksum == rhs._contentChecksum
&& lhs._contentChecksumType == rhs._contentChecksumType;
}
}
src/libsync/syncjournalfilerecord.h
+2 -2
Ver Arquivo
@@ -55,8 +55,8 @@ public:
qint64 _fileSize;
QByteArray _remotePerm;
bool _serverHasIgnoredFiles;
QByteArray _transmissionChecksum;
QByteArray _transmissionChecksumType;
QByteArray _contentChecksum;
QByteArray _contentChecksumType;
};
bool OWNCLOUDSYNC_EXPORT
src/libsync/transmissionchecksumvalidator.cpp
+47 -15
Ver Arquivo
@@ -77,36 +77,40 @@ QByteArray ComputeChecksum::checksumType() const
void ComputeChecksum::start(const QString& filePath)
{
const QString csType = checksumType();
// Calculate the checksum in a different thread first.
connect( &_watcher, SIGNAL(finished()),
this, SLOT(slotCalculationDone()),
Qt::UniqueConnection );
if( csType == checkSumMD5C ) {
_watcher.setFuture(QtConcurrent::run(FileSystem::calcMd5, filePath));
_watcher.setFuture(QtConcurrent::run(ComputeChecksum::computeNow, filePath, checksumType()));
}
} else if( csType == checkSumSHA1C ) {
_watcher.setFuture(QtConcurrent::run( FileSystem::calcSha1, filePath));
QByteArray ComputeChecksum::computeNow(const QString& filePath, const QByteArray& checksumType)
{
if( checksumType == checkSumMD5C ) {
return FileSystem::calcMd5(filePath);
} else if( checksumType == checkSumSHA1C ) {
return FileSystem::calcSha1(filePath);
}
#ifdef ZLIB_FOUND
else if( csType == checkSumAdlerC) {
_watcher.setFuture(QtConcurrent::run(FileSystem::calcAdler32, filePath));
else if( checksumType == checkSumAdlerC) {
return FileSystem::calcAdler32(filePath);
}
#endif
else {
// for an unknown checksum or no checksum, we're done right now
if( !csType.isEmpty() ) {
qDebug() << "Unknown checksum type:" << csType;
}
emit done(QByteArray(), QByteArray());
// for an unknown checksum or no checksum, we're done right now
if( !checksumType.isEmpty() ) {
qDebug() << "Unknown checksum type:" << checksumType;
}
return QByteArray();
}
void ComputeChecksum::slotCalculationDone()
{
QByteArray checksum = _watcher.future().result();
emit done(_checksumType, checksum);
if (!checksum.isNull()) {
emit done(_checksumType, checksum);
} else {
emit done(QByteArray(), QByteArray());
}
}
@@ -151,4 +155,32 @@ void ValidateChecksumHeader::slotChecksumCalculated(const QByteArray& checksumTy
emit validated(checksumType, checksum);
}
CSyncChecksumHook::CSyncChecksumHook(SyncJournalDb *journal)
: _journal(journal)
{
}
bool CSyncChecksumHook::hook(const char* path, uint32_t checksumTypeId, const char* checksum, void *this_obj)
{
CSyncChecksumHook* checksumHook = static_cast<CSyncChecksumHook*>(this_obj);
return checksumHook->check(QString::fromUtf8(path), checksumTypeId, QByteArray(checksum));
}
bool CSyncChecksumHook::check(const QString& path, int checksumTypeId, const QByteArray& checksum)
{
QByteArray checksumType = _journal->getChecksumType(checksumTypeId);
if (checksumType.isEmpty()) {
qDebug() << "Checksum type" << checksumTypeId << "not found";
return false;
}
QByteArray newChecksum = ComputeChecksum::computeNow(path, checksumType);
if (newChecksum.isNull()) {
qDebug() << "Failed to compute checksum" << checksumType << "for" << path;
return false;
}
return newChecksum == checksum;
}
}
src/libsync/transmissionchecksumvalidator.h
+32
Ver Arquivo
@@ -23,6 +23,8 @@
namespace OCC {
class SyncJournalDb;
/// Creates a checksum header from type and value.
QByteArray makeChecksumHeader(const QByteArray& checksumType, const QByteArray& checksum);
@@ -59,6 +61,11 @@ public:
*/
void start(const QString& filePath);
/**
* Computes the checksum synchronously.
*/
static QByteArray computeNow(const QString& filePath, const QByteArray& checksumType);
signals:
void done(const QByteArray& checksumType, const QByteArray& checksum);
@@ -103,4 +110,29 @@ private:
QByteArray _expectedChecksum;
};
/**
* Hooks checksum computations into csync.
* @ingroup libsync
*/
class OWNCLOUDSYNC_EXPORT CSyncChecksumHook : public QObject
{
Q_OBJECT
public:
explicit CSyncChecksumHook(SyncJournalDb* journal);
/**
* Returns true if the checksum for \a path is the same as the one provided.
*
* Called from csync, where a instance of CSyncChecksumHook
* has to be set as userdata.
*/
static bool hook(const char* path, uint32_t checksumTypeId, const char* checksum,
void* this_obj);
bool check(const QString& path, int checksumTypeId, const QByteArray& checksum);
private:
SyncJournalDb* _journal;
};
}
test/test_journal.db
BIN
Ver Arquivo
Arquivo binário não exibido.
test/testsyncjournaldb.h
+7 -7
Ver Arquivo
@@ -59,17 +59,17 @@ private slots:
record._fileId = "abcd";
record._remotePerm = "744";
record._fileSize = 213089055;
record._transmissionChecksum = "mychecksum";
record._transmissionChecksumType = "MD5";
record._contentChecksum = "mychecksum";
record._contentChecksumType = "MD5";
QVERIFY(_db.setFileRecord(record));
SyncJournalFileRecord storedRecord = _db.getFileRecord("foo");
QVERIFY(storedRecord == record);
// Update checksum
record._transmissionChecksum = "newchecksum";
record._transmissionChecksumType = "Adler32";
_db.updateFileRecordChecksum("foo", record._transmissionChecksum, record._transmissionChecksumType);
record._contentChecksum = "newchecksum";
record._contentChecksumType = "Adler32";
_db.updateFileRecordChecksum("foo", record._contentChecksum, record._contentChecksumType);
storedRecord = _db.getFileRecord("foo");
QVERIFY(storedRecord == record);
@@ -97,8 +97,8 @@ private slots:
SyncJournalFileRecord record;
record._path = "foo-checksum";
record._remotePerm = "744";
record._transmissionChecksum = "mychecksum";
record._transmissionChecksumType = "MD5";
record._contentChecksum = "mychecksum";
record._contentChecksumType = "MD5";
QVERIFY(_db.setFileRecord(record));
SyncJournalFileRecord storedRecord = _db.getFileRecord("foo-checksum");