Excludes: Use QRegularExpression for speedup WiP

csync/src/csync.h

@@ -39,6 +39,7 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <config_csync.h>
+#include "csync_exclude.h"
 
 #ifdef __cplusplus
 extern "C" {

csync/src/csync_exclude.c

@@ -172,6 +172,7 @@ out:
   return rc;
 }
 
+
 // See http://support.microsoft.com/kb/74496 and
 // https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
 // Additionally, we ignore '$Recycle.Bin', see https://github.com/owncloud/client/issues/2955
@@ -207,7 +208,7 @@ bool csync_is_windows_reserved_word(const char* filename) {
   return false;
 }
 
-static CSYNC_EXCLUDE_TYPE _csync_excluded_common(c_strlist_t *excludes, const char *path, int filetype, bool check_leading_dirs) {
+static CSYNC_EXCLUDE_TYPE _csync_excluded_common(c_strlist_t *excludes, const char *path, int filetype, bool check_leading_dirs, csync_exclude_traversal_hook hook, void *hookUserData) {
     size_t i = 0;
     const char *bname = NULL;
     size_t blen = 0;
@@ -300,6 +301,13 @@ static CSYNC_EXCLUDE_TYPE _csync_excluded_common(c_strlist_t *excludes, const ch
         SAFE_FREE(conflict);
     }
 
+
+    if (!check_leading_dirs && hook) {
+        //
+        match = hook(bname, filetype, hookUserData);
+        goto out;
+    }
+
     if( ! excludes ) {
         goto out;
     }
@@ -332,6 +340,8 @@ static CSYNC_EXCLUDE_TYPE _csync_excluded_common(c_strlist_t *excludes, const ch
         SAFE_FREE(path_split);
     }
 
+
+
     /* Loop over all exclude patterns and evaluate the given path */
     for (i = 0; match == CSYNC_NOT_EXCLUDED && i < excludes->count; i++) {
         bool match_dirs_only = false;
@@ -400,11 +410,12 @@ static CSYNC_EXCLUDE_TYPE _csync_excluded_common(c_strlist_t *excludes, const ch
     return match;
 }
 
-CSYNC_EXCLUDE_TYPE csync_excluded_traversal(c_strlist_t *excludes, const char *path, int filetype) {
-  return _csync_excluded_common(excludes, path, filetype, false);
+CSYNC_EXCLUDE_TYPE csync_excluded_traversal(c_strlist_t *excludes, const char *path, int filetype, csync_exclude_traversal_hook hook, void *hookUserData) {
+ //   return ctx->callbacks.excluded_traversal_hook(path, filetype);
+  return _csync_excluded_common(excludes, path, filetype, false, hook, hookUserData);
 }
 
 CSYNC_EXCLUDE_TYPE csync_excluded_no_ctx(c_strlist_t *excludes, const char *path, int filetype) {
-  return _csync_excluded_common(excludes, path, filetype, true);
+  return _csync_excluded_common(excludes, path, filetype, true, 0, 0);
 }
 

csync/src/csync_exclude.h

@@ -21,6 +21,8 @@
 #ifndef _CSYNC_EXCLUDE_H
 #define _CSYNC_EXCLUDE_H
 
+#include "std/c_string.h"
+
 enum csync_exclude_type_e {
   CSYNC_NOT_EXCLUDED   = 0,
   CSYNC_FILE_SILENTLY_EXCLUDED,
@@ -37,6 +39,10 @@ typedef enum csync_exclude_type_e CSYNC_EXCLUDE_TYPE;
 int _csync_exclude_add(c_strlist_t **inList, const char *string);
 #endif
 
+// Hook from mirall
+typedef CSYNC_EXCLUDE_TYPE (*csync_exclude_traversal_hook) (
+        const char *path, int filetype, void *userData);
+
 /**
  * @brief Load exclude list
  *
@@ -62,7 +68,7 @@ int csync_exclude_load(const char *fname, c_strlist_t **list);
  *
  * @return  2 if excluded and needs cleanup, 1 if excluded, 0 if not.
  */
-CSYNC_EXCLUDE_TYPE csync_excluded_traversal(c_strlist_t *excludes, const char *path, int filetype);
+CSYNC_EXCLUDE_TYPE csync_excluded_traversal(c_strlist_t *excludes, const char *path, int filetype, csync_exclude_traversal_hook hook, void *hookUserData);
 
 /**
  * @brief csync_excluded_no_ctx
@@ -72,7 +78,6 @@ CSYNC_EXCLUDE_TYPE csync_excluded_traversal(c_strlist_t *excludes, const char *p
  * @return
  */
 CSYNC_EXCLUDE_TYPE csync_excluded_no_ctx(c_strlist_t *excludes, const char *path, int filetype);
-#endif /* _CSYNC_EXCLUDE_H */
 
 /**
  * @brief Checks if filename is considered reserved by Windows
@@ -81,5 +86,5 @@ CSYNC_EXCLUDE_TYPE csync_excluded_no_ctx(c_strlist_t *excludes, const char *path
  */
 bool csync_is_windows_reserved_word(const char *file_name);
 
-
+#endif /* _CSYNC_EXCLUDE_H */
 /* vim: set ft=c.doxygen ts=8 sw=2 et cindent: */

csync/src/csync_private.h

@@ -101,6 +101,10 @@ struct csync_s {
       csync_checksum_hook checksum_hook;
       void *checksum_userdata;
 
+      /* Hook for the QRegExp based exclude checker */
+      csync_exclude_traversal_hook excluded_traversal_hook;
+      void *excluded_traversal_userdata;
+
   } callbacks;
   c_strlist_t *excludes;
 

csync/src/csync_statedb.c

@@ -466,7 +466,7 @@ int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) {
             /* Check for exclusion from the tree.
              * Note that this is only a safety net in case the ignore list changes
              * without a full remote discovery being triggered. */
-            CSYNC_EXCLUDE_TYPE excluded = csync_excluded_traversal(ctx->excludes, st->path, st->type);
+            CSYNC_EXCLUDE_TYPE excluded = csync_excluded_traversal(ctx->excludes, st->path, st->type, ctx->callbacks.excluded_traversal_hook, ctx->callbacks.excluded_traversal_userdata);
             if (excluded != CSYNC_NOT_EXCLUDED) {
                 CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded (%d)", st->path, excluded);
 

csync/src/csync_update.c

@@ -203,7 +203,7 @@ static int _csync_detect_update(CSYNC *ctx, const char *file,
       excluded =CSYNC_FILE_EXCLUDE_STAT_FAILED;
   } else {
     /* Check if file is excluded */
-    excluded = csync_excluded_traversal(ctx->excludes, path, type);
+    excluded = csync_excluded_traversal(ctx->excludes, path, type, ctx->callbacks.excluded_traversal_hook, ctx->callbacks.excluded_traversal_userdata);
   }
 
   if( excluded == CSYNC_NOT_EXCLUDED ) {

src/libsync/discoveryphase.cpp

@@ -610,6 +610,74 @@ void DiscoveryJob::remote_vio_closedir_hook (csync_vio_handle_t *dhandle,  void
     }
 }
 
+static QString convertToRegexpSyntax(char *exclude)
+{
+    //QString::fromUtf8(exclude);
+    QString s;
+    for (unsigned int i = 0; i < strlen(exclude); i++) {
+        char &c = exclude[i];
+        if (c == '*') s += ".*";
+        else if (c == '.') s += "\\.";
+        else if (c == '$') s += "\\$";
+        else if (c == '?') s += ".";
+        else s += c;
+        // FIXME: [ ] { } ( ) etc
+    }
+    qDebug() << s;
+    return s;
+}
+
+CSYNC_EXCLUDE_TYPE DiscoveryJob::excluded_traversal_hook (const char *path, int filetype, void *userdata)
+{
+    DiscoveryJob *discoveryJob = static_cast<DiscoveryJob*>(userdata);
+
+    // FIXME: csync_e
+
+    // FIXME file match vs directory match?
+
+    qDebug() << discoveryJob->_exclude_traversel_regexp_exclude.isValid() << path << discoveryJob->_exclude_traversel_regexp_exclude.errorString();
+
+    if (discoveryJob->_exclude_traversel_regexp_exclude.pattern().length() == 0) {
+        // env QT_ENABLE_REGEXP_JIT 1
+        discoveryJob->_exclude_traversel_regexp_exclude.setPatternOptions(QRegularExpression::OptimizeOnFirstUsageOption);
+        discoveryJob->_exclude_traversel_regexp_exclude_and_remove.setPatternOptions(QRegularExpression::OptimizeOnFirstUsageOption);
+        QString _exclude_traversel_regexp_exclude;
+        QString _exclude_traversel_regexp_exclude_and_remove;
+        for (unsigned int i = 0; i < discoveryJob->_csync_ctx->excludes->count; i++) {
+            char *exclude = discoveryJob->_csync_ctx->excludes->vector[i];
+            QString *builderToUse = & _exclude_traversel_regexp_exclude;
+            if (exclude[0] == '\n') continue; // empty line
+            if (exclude[0] == '\r') continue; // empty line
+            if (exclude[0] == ']'){
+                exclude++;
+                builderToUse = &_exclude_traversel_regexp_exclude_and_remove;
+            }
+            if (builderToUse->size() > 0) {
+                builderToUse->append("|");
+            }
+            builderToUse->append(convertToRegexpSyntax(exclude));
+        }
+        qDebug() << _exclude_traversel_regexp_exclude;
+        qDebug() << _exclude_traversel_regexp_exclude_and_remove;
+        discoveryJob->_exclude_traversel_regexp_exclude.setPattern(_exclude_traversel_regexp_exclude);
+        discoveryJob->_exclude_traversel_regexp_exclude_and_remove.setPattern(_exclude_traversel_regexp_exclude_and_remove);
+    }
+
+    QString p = QString::fromUtf8(path);
+    if (discoveryJob->_exclude_traversel_regexp_exclude.match(p).hasMatch()) {
+        qDebug() << "WOULD EXCLUDE";
+        return CSYNC_FILE_EXCLUDE_LIST;
+    }
+    // FIXME
+    if (discoveryJob->_exclude_traversel_regexp_exclude_and_remove.match(p).hasMatch()) {
+        qDebug() << "WOULD EXCLUDE AND REMOVE";
+        return CSYNC_FILE_EXCLUDE_AND_REMOVE;
+    }
+    // FIXME
+
+    return CSYNC_NOT_EXCLUDED;
+}
+
 void DiscoveryJob::start() {
     _selectiveSyncBlackList.sort();
     _selectiveSyncWhiteList.sort();
@@ -623,6 +691,9 @@ void DiscoveryJob::start() {
     _csync_ctx->callbacks.remote_closedir_hook = remote_vio_closedir_hook;
     _csync_ctx->callbacks.vio_userdata = this;
 
+    _csync_ctx->callbacks.excluded_traversal_hook = excluded_traversal_hook;
+    _csync_ctx->callbacks.excluded_traversal_userdata = this;
+
     csync_set_log_callback(_log_callback);
     csync_set_log_level(_log_level);
     csync_set_log_userdata(_log_userdata);

src/libsync/discoveryphase.h

@@ -23,6 +23,7 @@
 #include <QMutex>
 #include <QWaitCondition>
 #include <QLinkedList>
+#include <QRegularExpression>
 
 namespace OCC {
 
@@ -182,6 +183,10 @@ class DiscoveryJob : public QObject {
     QMutex _vioMutex;
     QWaitCondition _vioWaitCondition;
 
+    QRegularExpression _exclude_traversel_regexp_exclude;
+    QRegularExpression _exclude_traversel_regexp_exclude_and_remove;
+    static CSYNC_EXCLUDE_TYPE excluded_traversal_hook (const char *path, int filetype, void *userdata);
+
 
 public:
     explicit DiscoveryJob(CSYNC *ctx, QObject* parent = 0)