From 7a3dece44a44481953ec26a091bce0f584d72e98 Mon Sep 17 00:00:00 2001
From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com>
Date: Sat, 3 Jun 2023 23:22:36 +0200
Subject: [PATCH 01/31] Fix UI display of filtered files with wildcards.
The previous JS code incorrectly converted glob expressions to regexp which prevented them from showing the correct file states in the list.
This was only a UI bug and has no effect on the backend.
---
.../ngax/scripts/directives/sourceFolderPicker.js | 4 +++-
.../webroot/ngax/scripts/services/AppUtils.js | 15 ++++++++++++---
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js
index 27b77a6bd..afa88466c 100644
--- a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js
+++ b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js
@@ -178,7 +178,9 @@ backupApp.directive('sourceFolderPicker', function() {
for(var i = 0; i < (scope.ngFilters || []).length; i++) {
var f = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], dirsep);
if (f != null) {
- if (f[0] == '-path')
+ if (f[1].indexOf('?') != -1 || f[1].indexOf('*') != -1)
+ anySpecials = true;
+ else if (f[0] == '-path')
excludemap[compareablePath(f[1])] = true;
else if (f[0] == '-folder')
excludemap[compareablePath(f[1] + dirsep)] = true;
diff --git a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
index 0817690d2..95f2ce91e 100644
--- a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
+++ b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
@@ -422,6 +422,13 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer
return str.replace( new RegExp( "(" + this.preg_quote(pattern) + ")" , 'g' ), replacement );
};
+ this.globToRegexp = function (str) {
+ // Escape special chars, except ? and *
+ str = (str + '').replace(/([\\\.\+\[\^\]\$\(\)\{\}\=\!\<\>\|\:])/g, "\\$1");
+ // Replace ? and * with .? and .*
+ return str.replace(/(\?|\*)/g, ".$1");
+ };
+
this.format = function() {
if (arguments == null || arguments.length < 1)
return null;
@@ -609,7 +616,7 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer
if (rx)
filter = filter.substr(1, filter.length - 2);
else
- filter = this.replace_all(this.replace_all(this.preg_quote(filter), '*', '.*'), '?', '.');
+ filter = this.globToRegexp(filter);
try {
res.push([flag == '+', new RegExp(filter, caseSensitive ? 'g' : 'gi')]);
@@ -621,9 +628,11 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer
};
this.evalFilter = function(path, filters, include) {
- for(var i = 0; i < filters.length; i++) {
+ for (var i = 0; i < filters.length; i++) {
var m = path.match(filters[i][1]);
- if (m && m.length == 1 && m[0].length == path.length)
+ // Regex such as .* might match empty string at the end which is unwanted
+ // Check that the first match covers the full string
+ if (m && m.length >= 1 && m[0].length == path.length)
return filters[i][0];
}
From f4c04da31a4fd3e0df5eaa355b3b74b156a85e5c Mon Sep 17 00:00:00 2001
From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com>
Date: Sun, 4 Jun 2023 20:07:15 +0200
Subject: [PATCH 02/31] Fix bugs with simple edit filter UI.
No longer adds invisible dir separators to filter list.
---
.../scripts/directives/parseFilterType.js | 21 ++++++++++---------
.../webroot/ngax/scripts/services/AppUtils.js | 7 ++++++-
2 files changed, 17 insertions(+), 11 deletions(-)
diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/parseFilterType.js b/Duplicati/Server/webroot/ngax/scripts/directives/parseFilterType.js
index 0fc22b2fc..90aee7227 100644
--- a/Duplicati/Server/webroot/ngax/scripts/directives/parseFilterType.js
+++ b/Duplicati/Server/webroot/ngax/scripts/directives/parseFilterType.js
@@ -3,11 +3,13 @@ backupApp.directive('parseFilterType', function(AppUtils, SystemInfo) {
restrict: 'A',
require: ['ngModel'],
link: function(scope, element, attr, ctrl) {
- var body = null;
-
ctrl[0].$parsers.push(function(txt) {
var dirsep = SystemInfo.state.DirectorySeparator || '/';
- return AppUtils.buildFilter(txt, body, dirsep);
+ // Store type in scope for parseFilterContent directive
+ // This works because ng-repeat creates a child scope for each iteration
+ // It is still glitchy in some edge cases, but should be more consistent
+ scope.filterType = txt;
+ return AppUtils.buildFilter(txt, scope.filterBody, dirsep);
});
ctrl[0].$formatters.push(function(src) {
@@ -18,8 +20,7 @@ backupApp.directive('parseFilterType', function(AppUtils, SystemInfo) {
return null;
}
- body = parts[1];
-
+ scope.filterType = parts[0];
return parts[0];
});
}
@@ -31,23 +32,23 @@ backupApp.directive('parseFilterContent', function(AppUtils, SystemInfo) {
restrict: 'A',
require: ['ngModel'],
link: function(scope, element, attr, ctrl) {
-
- var type = null;
-
ctrl[0].$parsers.push(function(txt) {
var dirsep = SystemInfo.state.DirectorySeparator || '/';
- return AppUtils.buildFilter(type, txt, dirsep);
+ // Store type in scope for parseFilterType directive
+ scope.filterBody = txt;
+ return AppUtils.buildFilter(scope.filterType, txt, dirsep);
});
ctrl[0].$formatters.push(function(src) {
var dirsep = SystemInfo.state.DirectorySeparator || '/';
+ // Remove double trailing slashes
var parts = AppUtils.splitFilterIntoTypeAndBody(src, dirsep);
if (parts == null) {
type = null;
return null;
}
- type = parts[0];
+ scope.filterBody = parts[1];
return parts[1];
});
}
diff --git a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
index 95f2ce91e..242bb2290 100644
--- a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
+++ b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
@@ -138,11 +138,13 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer
key: '-folder',
prefix: '-',
suffix: '!',
+ stripSep: true,
rx: '\\-(.*)\\!'
}, {
name: gettextCatalog.getString('Exclude file'),
key: '-path',
prefix: '-',
+ stripSep: true,
exclude: ['*', '?', '{'],
rx: '\\-([^\\[\\{\\*\\?]+)'
}, {
@@ -597,7 +599,10 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer
if (pre.length >= 2 && pre[1] == '[') {
//Regexp encode body ....
}
-
+ if (f.stripSep == true && body[body.length - 1] == dirsep) {
+ // Remove trailing dirsep
+ body = body.slice(0, -1);
+ }
return pre + body + suf;
};
From b94734498e0a55dabe3a5312ac309b1b71810693 Mon Sep 17 00:00:00 2001
From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com>
Date: Sun, 4 Jun 2023 21:37:53 +0200
Subject: [PATCH 03/31] Fix filter options for web UI command line.
Only the last exclude or include option was sent to the backend.
Now all include and exclude options are used.
---
.../controllers/CommandlineController.js | 15 ++++++++----
.../webroot/ngax/scripts/services/AppUtils.js | 24 +++++++++++++++----
2 files changed, 31 insertions(+), 8 deletions(-)
diff --git a/Duplicati/Server/webroot/ngax/scripts/controllers/CommandlineController.js b/Duplicati/Server/webroot/ngax/scripts/controllers/CommandlineController.js
index 2bc18fd0c..38a38886e 100644
--- a/Duplicati/Server/webroot/ngax/scripts/controllers/CommandlineController.js
+++ b/Duplicati/Server/webroot/ngax/scripts/controllers/CommandlineController.js
@@ -54,11 +54,18 @@ backupApp.controller('CommandlineController', function($scope, $routeParams, $lo
combined.unshift($scope.Command);
- for(n in opts) {
- if (opts[n] == null)
+ for (n in opts) {
+ var value = opts[n];
+ if (n == 'include' || n == 'exclude') {
+ // Handle filters that appear multiple times
+ for (var i = 0; i < value.length; ++i) {
+ combined.push('--' + n + '=' + value[i]);
+ }
+ } else if (value == null) {
combined.push(n);
- else
- combined.push(n + '=' + opts[n]);
+ } else {
+ combined.push(n + '=' + value);
+ }
}
options = {
diff --git a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
index 242bb2290..50a3ea3bc 100644
--- a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
+++ b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
@@ -259,12 +259,14 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer
return new Date(dt);
};
- this.parseOptionStrings = function(val, dict, validateCallback) {
+ this.parseOptionStrings = function (val, dict, validateCallback) {
+ // Parse options and return a dict with dict['--key']=value options
+ // Include and exclude filters are returned as arrays in dict['include'], dict['exclude'] if present
dict = dict || {};
var lines = null;
- if (val != null && typeof(val) == typeof([]))
+ if (val != null && Array.isArray(val))
lines = val;
else
lines = this.replace_all(val || '', '\r', '\n').split('\n');
@@ -289,8 +291,18 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer
if (validateCallback)
if (!validateCallback(dict, key, value))
return null;
-
- dict['--' + key] = value;
+ // Special handing for --include and --exclude, which are the only options that can appear multiple times (compare FilterCollector.DoExtractOptions)
+ // In all other cases, the last value overrides the earlier values
+ var lower = key.toLowerCase();
+ if (lower == 'include' || lower == 'exclude') {
+ if (!Array.isArray(dict[lower])) {
+ dict[lower] = [value];
+ } else {
+ dict[lower].push(value);
+ }
+ } else {
+ dict['--' + key] = value;
+ }
}
}
@@ -302,6 +314,10 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer
if (d['--' + k] !== undefined) {
DialogService.dialog(gettextCatalog.getString('Error'), gettextCatalog.getString('Duplicate option {{opt}}', { opt: k }));
return false;
+ } else if (k == 'include' || k == 'exclude') {
+ // Cannot specify filters in extra options
+ DialogService.dialog(gettextCatalog.getString('Error'), gettextCatalog.getString('Cannot specify filter include or excludes in extra options'));
+ return false;
}
return true;
From e2569cca68e08c16c94804f3c1b962b3d5dd59b2 Mon Sep 17 00:00:00 2001
From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com>
Date: Sun, 4 Jun 2023 22:39:32 +0200
Subject: [PATCH 04/31] Improve filter UI type matching.
Instead of selecting the first matching in list, which does not recognize regexp without ? or *, choose the type with longest prefix and suffix.
---
.../webroot/ngax/scripts/services/AppUtils.js | 20 +++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
diff --git a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
index 50a3ea3bc..353ed5525 100644
--- a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
+++ b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js
@@ -593,13 +593,25 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer
return [type, body];
}
- for(var i in this.filterClasses) {
+ var m = [];
+ for (var i in this.filterClasses) {
var n = matches(src, this.filterClasses[i]);
if (n != null)
- return n;
+ m.push(n);
}
-
- return null;
+ if (m.length == 0) {
+ return null;
+ }
+ // Select match with shortest body, meaning longest prefix and suffix
+ var shortestIdx = 0;
+ var shortestLen = src.length;
+ for (i = 0; i < m.length; ++i) {
+ if (m[i][1].length < shortestLen) {
+ shortestIdx = i;
+ shortestLen = m[i][1].length;
+ }
+ }
+ return m[shortestIdx];
};
this.buildFilter = function(type, body, dirsep) {
From 9e9850113e74a0e48276b0003620c68a19aba37b Mon Sep 17 00:00:00 2001
From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com>
Date: Wed, 14 Jun 2023 20:34:00 +0200
Subject: [PATCH 05/31] Fix highlighting with wildcards in restore file picker.
Inputs were interpreted as regexp in the UI instead of globbing expressions which are used in the backend.
Closes #2682.
---
.../Server/webroot/ngax/scripts/directives/restoreFilePicker.js | 1 +
Duplicati/Server/webroot/ngax/templates/restorefilepicker.html | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js
index 3e71e8648..c5a6b026a 100644
--- a/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js
+++ b/Duplicati/Server/webroot/ngax/scripts/directives/restoreFilePicker.js
@@ -21,6 +21,7 @@ backupApp.directive('restoreFilePicker', function() {
$scope.systeminfo = SystemInfo.watch($scope);
$scope.treedata = [];
+ $scope.AppUtils = AppUtils;
function compareablePath(path) {
return scope.systeminfo.CaseSensitiveFilesystem ? path : path.toLowerCase();
diff --git a/Duplicati/Server/webroot/ngax/templates/restorefilepicker.html b/Duplicati/Server/webroot/ngax/templates/restorefilepicker.html
index d23d0466e..2141bfe06 100644
--- a/Duplicati/Server/webroot/ngax/templates/restorefilepicker.html
+++ b/Duplicati/Server/webroot/ngax/templates/restorefilepicker.html
@@ -5,7 +5,7 @@
-
+
From f7e435b1ddd5df3a42e7b637f2ae2006ed55c8ac Mon Sep 17 00:00:00 2001
From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com>
Date: Sun, 19 Nov 2023 20:04:37 +0100
Subject: [PATCH 06/31] Improve file backend list speed.
Directly list FileInfo instead of lookups by filename. This greatly speeds up backup verify on exFAT.
Closes #5061
---
Duplicati/Library/Backend/File/FileBackend.cs | 10 +----
Duplicati/Library/Common/IO/ISystemIO.cs | 6 +++
Duplicati/Library/Common/IO/SystemIOLinux.cs | 26 ++++++++++++-
.../Library/Common/IO/SystemIOWindows.cs | 38 +++++++++++++++++--
4 files changed, 65 insertions(+), 15 deletions(-)
diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs
index a2b005458..a49d0a05e 100644
--- a/Duplicati/Library/Backend/File/FileBackend.cs
+++ b/Duplicati/Library/Backend/File/FileBackend.cs
@@ -176,15 +176,7 @@ namespace Duplicati.Library.Backend
if (!systemIO.DirectoryExists(m_path))
throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path));
- foreach (string s in systemIO.EnumerateFiles(m_path))
- {
- yield return systemIO.FileEntry(s);
- }
-
- foreach (string s in systemIO.EnumerateDirectories(m_path))
- {
- yield return systemIO.DirectoryEntry(s);
- }
+ return systemIO.EnumerateFileEntries(m_path);
}
#if DEBUG_RETRY
diff --git a/Duplicati/Library/Common/IO/ISystemIO.cs b/Duplicati/Library/Common/IO/ISystemIO.cs
index 654e72449..c0fb6affc 100644
--- a/Duplicati/Library/Common/IO/ISystemIO.cs
+++ b/Duplicati/Library/Common/IO/ISystemIO.cs
@@ -28,6 +28,7 @@ namespace Duplicati.Library.Common.IO
public interface ISystemIO
{
IFileEntry DirectoryEntry(string path);
+ IFileEntry DirectoryEntry(DirectoryInfo dirInfo);
void DirectoryCreate(string path);
void DirectoryDelete(string path, bool recursive);
bool DirectoryExists(string path);
@@ -36,6 +37,7 @@ namespace Duplicati.Library.Common.IO
void DirectorySetCreationTimeUtc(string path, DateTime time);
IFileEntry FileEntry(string path);
+ IFileEntry FileEntry(FileInfo fileInfo);
void FileMove(string source, string target);
void FileDelete(string path);
void FileCopy(string source, string target, bool overwrite);
@@ -69,6 +71,10 @@ namespace Duplicati.Library.Common.IO
IEnumerable EnumerateFiles(string path, string searchPattern, SearchOption searchOption);
IEnumerable EnumerateDirectories(string path);
+ // Enumerate FileEntries of files and directories
+ // This is more efficient than enumerating file names when metadata is needed
+ IEnumerable EnumerateFileEntries(string path);
+
void SetMetadata(string path, Dictionary metdata, bool restorePermissions);
Dictionary GetMetadata(string path, bool isSymlink, bool followSymlink);
}
diff --git a/Duplicati/Library/Common/IO/SystemIOLinux.cs b/Duplicati/Library/Common/IO/SystemIOLinux.cs
index 1acb95b47..6ac013cc1 100644
--- a/Duplicati/Library/Common/IO/SystemIOLinux.cs
+++ b/Duplicati/Library/Common/IO/SystemIOLinux.cs
@@ -132,6 +132,22 @@ namespace Duplicati.Library.Common.IO
return Directory.EnumerateFiles(path, searchPattern, searchOption);
}
+ public IEnumerable EnumerateFileEntries(string path)
+ {
+ // For consistency with previous implementation, enumerate files first and directories after
+ DirectoryInfo dir = new DirectoryInfo(path);
+
+ foreach (FileInfo file in dir.EnumerateFiles())
+ {
+ yield return FileEntry(file);
+ }
+
+ foreach (DirectoryInfo d in dir.EnumerateDirectories())
+ {
+ yield return DirectoryEntry(d);
+ }
+ }
+
public string PathGetFileName(string path)
{
return Path.GetFileName(path);
@@ -293,7 +309,10 @@ namespace Duplicati.Library.Common.IO
public IFileEntry DirectoryEntry(string path)
{
- var dInfo = new DirectoryInfo(path);
+ return DirectoryEntry(new DirectoryInfo(path));
+ }
+ public IFileEntry DirectoryEntry(DirectoryInfo dInfo)
+ {
return new FileEntry(dInfo.Name, 0, dInfo.LastAccessTime, dInfo.LastWriteTime)
{
IsFolder = true
@@ -302,7 +321,10 @@ namespace Duplicati.Library.Common.IO
public IFileEntry FileEntry(string path)
{
- var fileInfo = new FileInfo(path);
+ return FileEntry(new FileInfo(path));
+ }
+ public IFileEntry FileEntry(FileInfo fileInfo)
+ {
return new FileEntry(fileInfo.Name, fileInfo.Length, fileInfo.LastAccessTime, fileInfo.LastWriteTime);
}
}
diff --git a/Duplicati/Library/Common/IO/SystemIOWindows.cs b/Duplicati/Library/Common/IO/SystemIOWindows.cs
index 343dcb78e..cb16831e6 100644
--- a/Duplicati/Library/Common/IO/SystemIOWindows.cs
+++ b/Duplicati/Library/Common/IO/SystemIOWindows.cs
@@ -75,7 +75,7 @@ namespace Duplicati.Library.Common.IO
}
}
}
-
+
///
/// Returns true if prefixed with @"\\" or @"//".
///
@@ -361,7 +361,7 @@ namespace Duplicati.Library.Common.IO
public IEnumerable EnumerateFiles(string path, string searchPattern, SearchOption searchOption)
{
- return System.IO.Directory.EnumerateFiles(AddExtendedDevicePathPrefix(path), searchPattern, searchOption).Select(RemoveExtendedDevicePathPrefix);
+ return System.IO.Directory.EnumerateFiles(AddExtendedDevicePathPrefix(path), searchPattern, searchOption).Select(RemoveExtendedDevicePathPrefix);
}
public string PathGetFileName(string path)
@@ -474,6 +474,30 @@ namespace Duplicati.Library.Common.IO
}
}
+ public IEnumerable EnumerateFileEntries(string path)
+ {
+ // For consistency with previous implementation, enumerate files first and directories after
+ DirectoryInfo dir;
+ if (IsPrefixedWithExtendedDevicePathPrefix(path))
+ {
+ dir = new DirectoryInfo(path);
+ }
+ else
+ {
+ dir = new DirectoryInfo(AddExtendedDevicePathPrefix(path));
+ }
+
+ foreach (FileInfo file in dir.EnumerateFiles())
+ {
+ yield return FileEntry(file);
+ }
+
+ foreach (DirectoryInfo d in dir.EnumerateDirectories())
+ {
+ yield return DirectoryEntry(d);
+ }
+ }
+
public void FileCopy(string source, string target, bool overwrite)
{
File.Copy(AddExtendedDevicePathPrefix(source), AddExtendedDevicePathPrefix(target), overwrite);
@@ -498,7 +522,10 @@ namespace Duplicati.Library.Common.IO
public IFileEntry DirectoryEntry(string path)
{
- var dInfo = new DirectoryInfo(AddExtendedDevicePathPrefix(path));
+ return DirectoryEntry(new DirectoryInfo(AddExtendedDevicePathPrefix(path)));
+ }
+ public IFileEntry DirectoryEntry(DirectoryInfo dInfo)
+ {
return new FileEntry(dInfo.Name, 0, dInfo.LastAccessTime, dInfo.LastWriteTime)
{
IsFolder = true
@@ -507,7 +534,10 @@ namespace Duplicati.Library.Common.IO
public IFileEntry FileEntry(string path)
{
- var fileInfo = new FileInfo(AddExtendedDevicePathPrefix(path));
+ return FileEntry(new FileInfo(AddExtendedDevicePathPrefix(path)));
+ }
+ public IFileEntry FileEntry(FileInfo fileInfo)
+ {
var lastAccess = new DateTime();
try
{
From af4fd5af059a22bfecaa3220a8a89e9ce7d694d7 Mon Sep 17 00:00:00 2001
From: gpatel-fr <44170243+gpatel-fr@users.noreply.github.com>
Date: Sun, 10 Dec 2023 00:23:16 +0100
Subject: [PATCH 07/31] change error reporting for full backup remote
verification (#5004)
Before the change, it could happen that when the postbackup test
was finding problems in the backend, the job would be reported
as having completed successfully, the found problems being displayed
only in the complete log. After the change, an error is logged
that cause the job to be failed.
To repro, the following procedure can be used in a test backup:
change a hash for a block in the block table database and run a backup having set
--full-remote-verification and --full-block-verification to true and
--backup-test-percentage to 100.
---
Duplicati/Library/Main/Operation/TestHandler.cs | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs
index 8c8163bef..9ebd1091a 100644
--- a/Duplicati/Library/Main/Operation/TestHandler.cs
+++ b/Duplicati/Library/Main/Operation/TestHandler.cs
@@ -189,6 +189,15 @@ namespace Duplicati.Library.Main.Operation
}
m_results.EndTime = DateTime.UtcNow;
+ var filtered = from n in m_results.Verifications where n.Value.Any() select n;
+ if (!filtered.Any())
+ {
+ Logging.Log.WriteInformationMessage(LOGTAG, "Test results", "Successfully verified {0} remote files", m_results.VerificationsActualLength);
+ }
+ else
+ {
+ Logging.Log.WriteErrorMessage(LOGTAG, "Test results", null, "Verified {0} remote files with {1} problem(s)", m_results.VerificationsActualLength, filtered.Count());
+ }
}
///
From 0de4cb7968e7441707bc3ccfe49b5048e172dd88 Mon Sep 17 00:00:00 2001
From: gpatel-fr <44170243+gpatel-fr@users.noreply.github.com>
Date: Sun, 10 Dec 2023 00:24:07 +0100
Subject: [PATCH 08/31] allow full-remote-verification to target only list and
index files (#5037)
The option is turned into an enum with values True, False, ListIndexes
True and False are for compatibility with the existing backups.
ListIndexes is equivalent to True but don't handle block volumes.
This is to make more workable a strategy of full remote testing
after each backup.
---
.../Main/Database/LocalTestDatabase.cs | 3 ++
.../Library/Main/Operation/TestHandler.cs | 2 +-
Duplicati/Library/Main/Options.cs | 41 +++++++++++++++++--
Duplicati/Library/Main/Strings.cs | 2 +-
4 files changed, 42 insertions(+), 6 deletions(-)
diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs
index 8149bfa0c..2d1ece41a 100644
--- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs
@@ -150,6 +150,9 @@ namespace Duplicati.Library.Main.Database
foreach(var f in FilterByVerificationCount(files, samples, max))
yield return f;
+ if (options.FullRemoteVerification == Options.RemoteTestStrategy.ListAndIndexes)
+ yield break;
+
//And finally some block files
files.Clear();
diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs
index 9ebd1091a..ac3245b44 100644
--- a/Duplicati/Library/Main/Operation/TestHandler.cs
+++ b/Duplicati/Library/Main/Operation/TestHandler.cs
@@ -68,7 +68,7 @@ namespace Duplicati.Library.Main.Operation
m_results.OperationProgressUpdater.UpdateProgress(0);
var progress = 0L;
- if (m_options.FullRemoteVerification)
+ if (m_options.FullRemoteVerification != Options.RemoteTestStrategy.False)
{
foreach(var vol in new AsyncDownloader(files, backend))
{
diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs
index 5fbbbc5e5..3b3e65abd 100644
--- a/Duplicati/Library/Main/Options.cs
+++ b/Duplicati/Library/Main/Options.cs
@@ -124,6 +124,27 @@ namespace Duplicati.Library.Main
Ignore
}
+ ///
+ /// The possible settings for the remote test strategy
+ ///
+ public enum RemoteTestStrategy
+ {
+ ///
+ /// test the remote volumes
+ ///
+ True,
+
+ ///
+ /// do not test the remote volumes
+ ///
+ False,
+
+ ///
+ /// test only the list and index volumes
+ ///
+ ListAndIndexes
+ }
+
///
/// The possible settings for the hardlink strategy
///
@@ -356,7 +377,8 @@ namespace Duplicati.Library.Main
new CommandLineArgument("no-backend-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.NobackendverificationShort, Strings.Options.NobackendverificationLong, "false"),
new CommandLineArgument("backup-test-samples", CommandLineArgument.ArgumentType.Integer, Strings.Options.BackendtestsamplesShort, Strings.Options.BackendtestsamplesLong("no-backend-verification"), "1"),
new CommandLineArgument("backup-test-percentage", CommandLineArgument.ArgumentType.Integer, Strings.Options.BackendtestpercentageShort, Strings.Options.BackendtestpercentageLong, "0"),
- new CommandLineArgument("full-remote-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.FullremoteverificationShort, Strings.Options.FullremoteverificationLong("no-backend-verification"), "false"),
+ new CommandLineArgument("full-remote-verification", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.FullremoteverificationShort, Strings.Options.FullremoteverificationLong("no-backend-verification"), Enum.GetName(typeof(RemoteTestStrategy), RemoteTestStrategy.False), null, Enum.GetNames(typeof(RemoteTestStrategy))),
+
new CommandLineArgument("dry-run", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DryrunShort, Strings.Options.DryrunLong, "false", new string[] { "dryrun" }),
new CommandLineArgument("block-hash-algorithm", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.BlockhashalgorithmShort, Strings.Options.BlockhashalgorithmLong, DEFAULT_BLOCK_HASH_ALGORITHM, null, GetSupportedHashes()),
@@ -1621,11 +1643,22 @@ namespace Duplicati.Library.Main
}
///
- /// Gets a flag indicating if the remote verification is deep
+ /// Gets a value indicating if the remote verification is deep
///
- public bool FullRemoteVerification
+ public RemoteTestStrategy FullRemoteVerification
{
- get { return Library.Utility.Utility.ParseBoolOption(m_options, "full-remote-verification"); }
+ get
+ {
+ string policy;
+ if (!m_options.TryGetValue("full-remote-verification", out policy))
+ policy = "False";
+
+ RemoteTestStrategy r;
+ if (!Enum.TryParse(policy, true, out r))
+ r = RemoteTestStrategy.True;
+
+ return r;
+ }
}
///
diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs
index 9ba71054f..71722ddcc 100644
--- a/Duplicati/Library/Main/Strings.cs
+++ b/Duplicati/Library/Main/Strings.cs
@@ -195,7 +195,7 @@ namespace Duplicati.Library.Main.Strings
public static string BackendtestpercentageShort { get { return LC.L(@"The percentage of samples to test after a backup"); } }
public static string BackendtestpercentageLong { get { return LC.L(@"After a backup is completed, some (dblock, dindex, dlist) files from the remote backend are selected for verification. Use this option to specify the percentage (between 0 and 100) of files to test. If the backup-test-samples option is also provided, the number of samples tested is the maximum implied by the two options. If the no-backend-verification option is provided, no remote files are verified."); } }
public static string FullremoteverificationShort { get { return LC.L(@"Activates in-depth verification of files"); } }
- public static string FullremoteverificationLong(string optionname) { return LC.L(@"After a backup is completed, some (dblock, dindex, dlist) files from the remote backend are selected for verification. Use this option to turn on full verification, which will decrypt the files and examine the insides of each volume, instead of simply verifying the external hash, If the option --{0} is set, no remote files are verified. This option is automatically set when then verification is performed directly.", optionname); }
+ public static string FullremoteverificationLong(string optionname) { return LC.L(@"After a backup is completed, some (dblock, dindex, dlist) files from the remote backend are selected for verification. Use this option to turn on full verification, which will decrypt the files and examine the insides of each volume, instead of simply verifying the external hash, If the option --{0} is set, no remote files are verified. This option is automatically set when then verification is performed directly. ListIndexes is like True but only dlist and index volumes are handled.", optionname); }
public static string FilereadbuffersizeShort { get { return LC.L(@"Size of the file read buffer"); } }
public static string FilereadbuffersizeLong { get { return LC.L(@"Use this size to control how many bytes are read from a file before processing"); } }
public static string AllowpassphrasechangeShort { get { return LC.L(@"Allow the passphrase to change"); } }
From ccfa4e204e6974ef073ed3a12e97176c6c38807b Mon Sep 17 00:00:00 2001
From: Jan <33495614+Jojo-1000@users.noreply.github.com>
Date: Sun, 10 Dec 2023 00:25:47 +0100
Subject: [PATCH 09/31] Fix restore search error in RestoreController (#4998)
* Fix restore search error in RestoreController.
TypeError: Cannot read Property 'Path' of undefined.
Add a break to for loop, thanks @tx0h (#4997).
* Remove empty '\' entries from restore picker.
The paths are split by dirsep, but directories end with it, which results in an empty part at the end.
---
.../webroot/ngax/scripts/controllers/RestoreController.js | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js b/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js
index 14ddedcf8..4ebec1005 100644
--- a/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js
+++ b/Duplicati/Server/webroot/ngax/scripts/controllers/RestoreController.js
@@ -246,6 +246,10 @@ backupApp.controller('RestoreController', function ($rootScope, $scope, $routePa
if (cp.indexOf(compareablePath(sn.Path)) == 0) {
var curpath = sn.Path;
var parts = p.substr(sn.Path.length).split(dirsep);
+ // Remove empty part if path had dirsep at end
+ if (parts[parts.length - 1].length == 0) {
+ parts.pop();
+ }
var col = sn;
for(var k in parts) {
@@ -261,6 +265,7 @@ backupApp.controller('RestoreController', function ($rootScope, $scope, $routePa
if (compareablePath(col.Children[m].Path) == compareablePath(curpath)) {
found = true;
col = col.Children[m];
+ break;
}
}
From 11d6364b4a6ddf6d0ea732526b3e335e648238c9 Mon Sep 17 00:00:00 2001
From: gpatel-fr <44170243+gpatel-fr@users.noreply.github.com>
Date: Sun, 10 Dec 2023 00:26:26 +0100
Subject: [PATCH 10/31] remove agsXmpp, replace it with Sharp.Xmpp (#5046)
also add a missing space in default body (fixes a small bug under Windows)
---
.../Duplicati.Library.Modules.Builtin.csproj | 9 +-
.../Modules/Builtin/SendJabberMessage.cs | 77 +---
.../Library/Modules/Builtin/packages.config | 2 +
Duplicati/License/Duplicati.License.csproj | 24 +-
thirdparty/ARSoft.Tools.Net/download.txt | 1 +
thirdparty/ARSoft.Tools.Net/license.txt | 201 +++++++++++
thirdparty/ARSoft.Tools.Net/licensedata.json | 7 +
thirdparty/Sharp.Xmpp/download.txt | 1 +
thirdparty/Sharp.Xmpp/license.txt | 10 +
thirdparty/Sharp.Xmpp/licensedata.json | 7 +
thirdparty/agsXMPP/agsXMPP.dll | Bin 524288 -> 0 bytes
thirdparty/agsXMPP/agsxmpp.tar.bz2 | Bin 4648960 -> 0 bytes
thirdparty/agsXMPP/download.txt | 1 -
thirdparty/agsXMPP/license.txt | 339 ------------------
thirdparty/agsXMPP/licensedata.json | 7 -
15 files changed, 272 insertions(+), 414 deletions(-)
create mode 100644 thirdparty/ARSoft.Tools.Net/download.txt
create mode 100644 thirdparty/ARSoft.Tools.Net/license.txt
create mode 100644 thirdparty/ARSoft.Tools.Net/licensedata.json
create mode 100644 thirdparty/Sharp.Xmpp/download.txt
create mode 100644 thirdparty/Sharp.Xmpp/license.txt
create mode 100644 thirdparty/Sharp.Xmpp/licensedata.json
delete mode 100644 thirdparty/agsXMPP/agsXMPP.dll
delete mode 100644 thirdparty/agsXMPP/agsxmpp.tar.bz2
delete mode 100644 thirdparty/agsXMPP/download.txt
delete mode 100644 thirdparty/agsXMPP/license.txt
delete mode 100644 thirdparty/agsXMPP/licensedata.json
diff --git a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj
index daf5390a8..440bc2118 100644
--- a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj
+++ b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj
@@ -67,6 +67,12 @@
..\..\..\..\packages\System.Threading.Tasks.Extensions.4.5.4\lib\netstandard2.0\System.Threading.Tasks.Extensions.dll
+
+ ..\..\..\..\packages\Sharp.Xmpp.1.0.2.2\lib\net45\Sharp.Xmpp.dll
+
+
+ ..\..\..\..\packages\ARSoft.Tools.Net.1.8.1\lib\ARSoft.Tools.Net.dll
+
@@ -74,9 +80,6 @@
-
- ..\..\..\..\thirdparty\agsXMPP\agsXMPP.dll
-
diff --git a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs
index 755ae54f9..d31412ec8 100644
--- a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs
+++ b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs
@@ -5,6 +5,7 @@ using Duplicati.Library.Interface;
using Duplicati.Library.Logging;
using System.Net.NetworkInformation;
using Duplicati.Library.Modules.Builtin.ResultSerialization;
+using Sharp.Xmpp.Client;
namespace Duplicati.Library.Modules.Builtin
{
@@ -64,7 +65,7 @@ namespace Duplicati.Library.Modules.Builtin
///
/// The default message body
///
- protected override string DEFAULT_BODY => string.Format("Duplicati %OPERATIONNAME% report for %backup-name%{0}{0}%RESULT%", Environment.NewLine);
+ protected override string DEFAULT_BODY => string.Format("Duplicati %OPERATIONNAME% report for %backup-name%{0}{0} %RESULT%", Environment.NewLine);
///
/// Don't use the subject for XMPP
///
@@ -174,81 +175,41 @@ namespace Duplicati.Library.Modules.Builtin
protected override void SendMessage(string subject, string body)
{
- Exception ex = null;
- var waitEvent = new System.Threading.ManualResetEvent(false);
var uri = new Library.Utility.Uri(m_username.Contains("://") ? m_username : "http://" + m_username);
- var con = new agsXMPP.XmppClientConnection(uri.Host, uri.Port == -1 ? (uri.Scheme == "https" ? 5223 :5222) : uri.Port);
- if (uri.Scheme == "https")
- con.UseSSL = true;
-
var resource = uri.Path ?? "";
if (resource.StartsWith("/", StringComparison.Ordinal))
resource = resource.Substring(1);
if (string.IsNullOrWhiteSpace(resource))
resource = "Duplicati";
-
- agsXMPP.ObjectHandler loginDelegate = (sender) =>
+ using (XmppClient client = new XmppClient(uri.Host, uri.Username, string.IsNullOrWhiteSpace(m_password) ? uri.Password : m_password, uri.Port == -1 ? (uri.Scheme == "https" ? 5223 :5222) : uri.Port))
{
+ client.Connect(resource);
try
{
foreach(var recipient in m_to.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries))
- con.Send(new agsXMPP.protocol.client.Message(recipient, agsXMPP.protocol.client.MessageType.chat, body));
- }
+ {
+ client.SendMessage(recipient, new Dictionary(){ {"en", body} });
+ // hack to work around a failure to send the second time
+ // (the xmpp server reports a read failure)
+ try
+ {
+ client.Ping(recipient);
+ }
+ catch {}
+ }
+ }
catch (Exception e)
{
Logging.Log.WriteWarningMessage(LOGTAG, "XMPPSendError", e, "Failed to send to XMPP messages: {0}", e.Message);
- ex = e;
}
- finally
- {
- waitEvent.Set();
- }
- };
-
- agsXMPP.ErrorHandler errorHandler = (sender, e) => {
- Logging.Log.WriteWarningMessage(LOGTAG, "XMPPError", e, "An error ocurred in XMPP: {0}", e.Message);
-
- ex = e;
- waitEvent.Set();
- };
-
- agsXMPP.XmppElementHandler loginErroHandler = (sender, e) => {
- Logging.Log.WriteWarningMessage(LOGTAG, "XMPPLoginError", null, "Failed to login to XMPP: {0}", e);
-
- ex = new Exception(string.Format("Failed to log in: {0}", e));
- waitEvent.Set();
- };
-
- con.OnLogin += loginDelegate;
- con.OnError += errorHandler;
- con.OnAuthError += loginErroHandler;
- //con.OnBinded += (sender) => {Console.WriteLine("Binded: {0}", sender);};
- //con.OnIq += (sender, iq) => {Console.WriteLine("Iq: {0}", iq);};
- //con.OnReadXml += (sender, xml) => {Console.WriteLine("ReadXml: {0}", xml);};
- //con.OnWriteXml += (sender, xml) => {Console.WriteLine("writeXml: {0}", xml);};;
- con.Open(uri.Username, string.IsNullOrWhiteSpace(m_password) ? uri.Password : m_password, resource);
-
- var timeout = !waitEvent.WaitOne(TimeSpan.FromSeconds(30), true);
-
- con.OnLogin -= loginDelegate;
- con.OnError -= errorHandler;
- con.OnAuthError -= loginErroHandler;
-
- try
- {
- con.Close();
- }
- catch (Exception lex)
- {
- Logging.Log.WriteExplicitMessage(LOGTAG, "CloseConnectionError", lex, "Failed to close XMPP connection: {0}", lex.Message);
+ finally
+ {
+ client.Close();
+ }
}
- if (ex != null)
- throw ex;
- if (timeout)
- throw new TimeoutException(Strings.SendJabberMessage.LoginTimeoutError);
}
}
}
diff --git a/Duplicati/Library/Modules/Builtin/packages.config b/Duplicati/Library/Modules/Builtin/packages.config
index b957ad6b3..a5d9ebf8c 100644
--- a/Duplicati/Library/Modules/Builtin/packages.config
+++ b/Duplicati/Library/Modules/Builtin/packages.config
@@ -9,4 +9,6 @@
+
+
diff --git a/Duplicati/License/Duplicati.License.csproj b/Duplicati/License/Duplicati.License.csproj
index 21ef5bcdd..e2dc06920 100644
--- a/Duplicati/License/Duplicati.License.csproj
+++ b/Duplicati/License/Duplicati.License.csproj
@@ -248,16 +248,28 @@
PreserveNewest
-
- licenses\agsXMPP\download.txt
+
+ licenses\Sharp.Xmpp\download.txt
PreserveNewest
-
- licenses\agsXMPP\license.txt
+
+ licenses\Sharp.Xmpp\license.txt
PreserveNewest
-
- licenses\agsXMPP\licensedata.json
+
+ licenses\Sharp.Xmpp\licensedata.json
+ PreserveNewest
+
+
+ licenses\ARSoft.Tools.Net\download.txt
+ PreserveNewest
+
+
+ licenses\ARSoft.Tools.Net\license.txt
+ PreserveNewest
+
+
+ licenses\ARSoft.Tools.Net\licensedata.json
PreserveNewest
diff --git a/thirdparty/ARSoft.Tools.Net/download.txt b/thirdparty/ARSoft.Tools.Net/download.txt
new file mode 100644
index 000000000..110c97495
--- /dev/null
+++ b/thirdparty/ARSoft.Tools.Net/download.txt
@@ -0,0 +1 @@
+https://github.com/alexreinert/ARSoft.Tools.Net
diff --git a/thirdparty/ARSoft.Tools.Net/license.txt b/thirdparty/ARSoft.Tools.Net/license.txt
new file mode 100644
index 000000000..094d8e444
--- /dev/null
+++ b/thirdparty/ARSoft.Tools.Net/license.txt
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/thirdparty/ARSoft.Tools.Net/licensedata.json b/thirdparty/ARSoft.Tools.Net/licensedata.json
new file mode 100644
index 000000000..ecff7c77d
--- /dev/null
+++ b/thirdparty/ARSoft.Tools.Net/licensedata.json
@@ -0,0 +1,7 @@
+{
+ "name": " ARSoft.Tools.Net",
+ "description": "Complete managed .Net SPF validation, SenderID validation and a dns client and dns server implementation",
+ "link": "https://github.com/pgstath/Sharp.Xmpp",
+ "license": "Apache 2.0",
+ "notes": ""
+}
diff --git a/thirdparty/Sharp.Xmpp/download.txt b/thirdparty/Sharp.Xmpp/download.txt
new file mode 100644
index 000000000..8c70e0beb
--- /dev/null
+++ b/thirdparty/Sharp.Xmpp/download.txt
@@ -0,0 +1 @@
+https://github.com/pgstath/Sharp.Xmpp
diff --git a/thirdparty/Sharp.Xmpp/license.txt b/thirdparty/Sharp.Xmpp/license.txt
new file mode 100644
index 000000000..2f79e5b4f
--- /dev/null
+++ b/thirdparty/Sharp.Xmpp/license.txt
@@ -0,0 +1,10 @@
+The MIT License
+
+Sharp.Xmpp Copyright (c) 2015 Panos Georgiou Stath. S22.Xmpp Copyright (c) 2013-2014 Torben Könke
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
diff --git a/thirdparty/Sharp.Xmpp/licensedata.json b/thirdparty/Sharp.Xmpp/licensedata.json
new file mode 100644
index 000000000..a8f1df81e
--- /dev/null
+++ b/thirdparty/Sharp.Xmpp/licensedata.json
@@ -0,0 +1,7 @@
+{
+ "name": "Sharp.Xmpp",
+ "description": "easy-to-use and well-documented .NET assembly for communicating with an XMPP server",
+ "link": "https://github.com/pgstath/Sharp.Xmpp",
+ "license": "MIT",
+ "notes": ""
+}
diff --git a/thirdparty/agsXMPP/agsXMPP.dll b/thirdparty/agsXMPP/agsXMPP.dll
deleted file mode 100644
index 09dc1f76b7145ce88160e3aaf045ee08aa9e1581..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001
literal 524288
zcmeZ`n!v!!z`(%5z`*eTKLf)K1_*F~PMTJ
z5W@ooB?WFq0|tf*>iNPrDQfUpAx1A_nqgW+us24;qY
zRxn2ZBq0E1LRlaRj0>9C82rI34h9`iAb{COgaQWx4-%JXrUC;4*riYn9108!9#A%j
z(kn@=C}Ch=cyI{nN048^?gYs~umS@^f?iQ^Q8EJq)NK&IF+5<0D1ed<3=9r>MX5RY
z$qW#;DnQkJz)3uYS#%k-Sn}6!GO$VNaX}aodLSVVE(SI(HeE(7rfOaW
z1{NMMUJfB<4RHnrE{FhIz61jUSIJyP1}1KHU8Z~5Fkwjs2CjQtoH}eA8d3}lTzA;3
zZCM#um{{1W?N}LD^0`26Vqjnt0NJn2pc}!!1&TB+CT129P*AgJF$fzlGng?jfX0CY
zS#;Ej&vS#E!cq$|hmExsWDLYI=6sMV+47~qt^$Wuz9S<8n-&Ayk1Pxff=nW{>1#P7s#|WFW|qu<%i1WME)rU=S4H65$r%Vbs>BJ1_qcbszGj%cgR;{VBq3qg7EagJYg129Tq_rU9gtx3=9mM;t>#2jKM-jAVPvH
z5bjnGSJc>0giDY`)R+@u6t{K-gdrx#lCQ+Tz{L&W3bKet;3AHw4lj}TpgfJ^a
zm2lfM89mp>*b?~GFRtJ)Sniqr;03u+&Ynd@I7a!naVAD{8WP5PA>&5^cixd>&
zf&`fk9~*>`ug1W@mB^b$Gb)%3C~u7bQLuW!oFA+N5N
z$sw<=S7dPzEM^ty3K{_oV_;yI!@$6h0;P+fG^jq~kdLu$V_;yg=m#a1B3F0P7H4
z*f{dFK*^O|Jc65@30zj#fBN_TKQv8(!z_}K0aVzbg_#LTn8i3qFfuT(OGH4-umKe{
zATyaDp~{QI7G~j;H!>~}WRW*A6%=HVH_%f8vGuep?lORCFL^UPT|pLkb3Ic*7Wp(i
zTZ?0$V3SvXIMf0(d%z~I1m)%c((mpcr72SBLWYz;gOf9w;U_<&BLU1zF@1BVAb`
zl_f(g0|Nsu0|S_rU|?X-j%Jg$f@skWV3W6jGF{o^?VwC6PI(h!5s<@;^^tSAwj!sz
zsj-Y8i+r-N7+cX)0-<~W5z0*%Z39zKUDg7s?-zg)KB(?vW@uz!VBiMTec;*~OMM1v
zw?OJMrh8HMxDXXs@dapgYtPES!p2(b0HTWbaWSyfI)W;4dh
zYEW!~RI+j8>x0V7ZTGmaxQmw|DIel4FQ#Gx1_rKLP#X}^?f|tPnHWIxWqhEpz;vP^
zI0+z`3Tf-HFo;S*%*=y$jv12tcwjtGC~09PZ%0t_?g05^5hQstfYJ;oee;9t$Cn%-
z7NDo^i_rA#1WMSfway?4lD=JNm%fR1D{A_NxRtdQ>{dP&h+|o5$#dvnNZ$gWQXAf@
zz)U=->3cpTeS^G9Abo=Z2$a4-v6*khz(8#JmSd`p1$7KTy(2I~j;T5hBp}CB9S@?R
zZ2?%@iix=vBn)q2@xq!|AWkYY!X2KgQOw6@lSvjWa6p$*A1z?68Q*|m#0L+kMs!oRq
zfEjX3)oCyRFhh>1Is+t7q|wR5z^22i(FLN|ij6?6A|~cqu%sMQbtXs=$RaR9j;T5e
zB;duwTnl0_>##G4)nfJlK9fdV84qy%IPm?6hhod*(-W2(-D(nTN|
zqyWs2W2(*v3CJ;37eMJkC|v@ji=lKmhz4l{Gvt`6%U}Xvh8$CMDNF#&kYlQ@fC+#Z
z;5cSx&;&Ipz-3A$0|N^e8#g<%hB2s6>8u&mK4>}viQmKf9U$a_7R~E
z5)bXD_Cdo1B#PgEAQhP52@=JxuLG-nAW>rVf&B+{Cq8{*;PC83wQnILJVBxy>>6y?
z!V_H|*ghikiG%IyLbcBeVjoD9gI$9in|!Oe5c@!)9PAn#*z7~s
z2eyw0eUf1NdQk22gxCiXLm#Sr&~ye8#h*SwDlqc{NEA~a6T?x+@GApT(SHVpf?7~>oe4a_0&a9enV{x2
zQxPj8LqRR5p$!!UHLaPX!EI+I(OOW8nMu4B)J|p+uLU)RnZ#>BjbtWqa7&h{xSxZ8
ztrpZ+WGWH`83k%cG8J2}F|gHwT8vB@(u@pjOiabRpaEV`JC%v4*bu}9wN;szifut`
zP16wVqt;)ny9Kg!JRts*wS}|#?0MGGgXoF@fX2H1m@(c`Y8VVrenL%?B
zY#RO`Wh@$jpzZ=f7f6Mc9TT(0YKXKP$P_syjjbSMVNA@(+MuFE(9R1JOD#B*SRo+<
z(J~3UBp(YT&>)I1O#t;qpvHq7V+3-yxW;afBchm?HH<<0>PQ9#7F{+bjY}Y5(AX+7
z!va`bGP7uyGBU8S4WMI-T2TiG2fJP@Yp~8_242%e29Xpm1kVw8I0|S?W
zuQ~&Ru?|ypGy?;RjtYeC%M>@1WMZn0VPIg<*a-?CjU8Ze8^{Gr)u5VPhZjOKvE*}r
zT^R*3fR72)wV-kP447NRYC%3>5~~F%Wzx`OWMIpW1?kn$05PpVJ}5`=fsPmxSgr&l
z4Cy)LD|0ijF==QqGO+3LF=^;9GO+2eF==c^xNjGeh9XE-n2AY49&EfUBLka;9VmvG
zAaYC^2qExzh$jOBgERw!phggQbR*xIfq_fRP=`${-v%_JQY6dBPyjIvH2K8E!^FX^
zQ3RS50tFirICSoD^D;4UYPc~nu-#(-^=&}oHXyIt+A?s1<~TrM0a=^OzyJwn9SaEE
zr=!orq_GQ>=s`ga4f*_TZU#0DP{M=>^ne7E!2S@a1*;U%0CT}}exUhS83qRMya32q
zeVO8sy*;1_Mu;suV!YZi9GtU+Sp->_^KC&1gp-|3kYy2@cmx-xcmyveM_;DVJtp>A
zkZ#a86*EH^Xr2Z<$0f#HYzG=VgP3Kx5HzL>PBeX%Q{il0F<}<%dQL%>S;DMrx*~$C
zTpVoq_Mky;w)`$`1~zW(7(v$jPHqM^HZW-{$|1-KQNS+93RV)qt!;s#Mqi2}-+_Sv
zZjvA?7tAnz6om!?pqVL>PH1?aTlF|Id-{$iToQZZ`=uvm<7hSL_7hvgbQ9FmMU8
zi3ziFbQq&@sRx
z_e=#jN|hNH7=^h6xpdeCx$@mWGq+&dAWq142U)|-A;_KY0kee*G>-^MBTNkU864yp
zML}sr*nov$IcO}MfdM+t#a86Z2%p?4N`$AwqFhFX0u4_F1}^B#R#736y7_Q6lZGlN
zi^#AQF*1SDFo=h)7cz*WBg0lC4pIXi(m_|lR;14aa|aV-%0j~lloa7nte^?*{4Q1oHpr+`ehrAJkq
zRDglUD_9snd0v5mK~NmoEN(WiS-fm;v;MGxQY%P+IsYGuxd&k8G9#JG3^o@Om(b+^
z#)?eUpf0Hn4}@j{r8zbx))LTIGq@&!tspQKWvT{sF+oKuh-P8|r4~>!VF3AEhXK50
zLQo9Dy-aYg9zt^xNZ)Y=1_njYxFKYng7s3+{*e5&oD6KGM;REHEJ5=+EC_z_7f^6A
zRfC2XKrR8%OlK{7Wo2N~k%q(t
zQ#GivEdIsHz{bPQk-v_UflZr-oh|=6D+3#t2XQ<*(?7PuFvqh(!V^bmfX0kKa~{ZH
z9L6Ni3w9b)HE5Uu6xJXb8rG03pbg0iMaH0X>;+niAjzb07F2IkM}X$`Sxfa97?|=G
zFfp)cF=^}rr4&dMO5%!(6n-u
zfq{vOO~;EZzmtuDE#DhdPiiZ&LyFNW;3ji5sLKONLm--oBmX)#0~0N6R_qQ=t>RUrnW{mZUy#c|
zG!tk3Wo`yG4)F*c4)(rGqr1!qlR=$pkjWq#VlrglSce~`h>b~O4=6@%K@yXMF&~q7
zEoktTN#i<593l)(W1x{1kfk6RmieQYszF@1Ff&q_z|4@xz`&peiYF!(4Kq+%M;zR^
zVkz+hdki#Fz?RPqntTL}$}?%?GBB{|8bC6Fjv|Cc$`Jfa8rdKfvXG1c%LWj2y6j9M
z8V5m6hO|LJ>oGv_!354ex>ig~#q*dL*lNL<2`t9+kL?;LarH1Uu;tGJmG3a&8z5n4
z1_rkLIiTzo$HdH{VGeRXYV#2sp9dKk7}P<2WMcd(FgQUPr
zCWuUaD<=b+7E5U;R28V_X{ifV1!6KmWb!*9s=`2p>I6`MPzqvlv*+6|GqCB%K!*2q
zM405cAw^ZOIx_<_;WAZ&hV^x{*z;{cYL(dYO_>?k#PVH0JP{^2hz7A@6Apw9&^W&(
zsIkDFZwFGw!NdZI9ww$DOD2Yb;uE0eBB(velr8=NO)dLjF>p-z=
z35sP6doT&=&@eHvfZNPy9S3mv#mW!}YPo^p9^4HmEn#F}(zPk-2l?(TBLkai(PU6z
zp2Wn!rplZTYC(Wkt#C21$xE_ZwyxyV6aS8+(+
z1Z+*dC@46%AzhDrDTrr5^&WCafTvgT&B4_NsMiW*f)YQ-^B_8Z2`I^nKr)0FB+rZ0
zg63_RY9$l1zC&KM}h>mRv1$?Xnq2u2}DCRMKM)_W*
z1c+vW77|fR)u2HOkT8g5V#)6X<+guJ8U~;?Asds1JJ=bZnHi9YAR1yKWU>p?u{s9!
zA*inp;++7+V-Zs|Xp8}*14Ki0KngW2KX~V!57eJo44NqlW74<@GKLw_F=qjHYP6g{
z$>A1AoK@p4$Xq$5>IER>pt&e8LyoC>AxJ=usd^EJ21$S!a!l1=gFymd2B=Mdm`7TI
zbvy+mia9QVJZ}Y3fjOQ662-4?DOUSHqQvS0j~7A5eemhi0Of`1WvKQof{cfOL^;?s
zxUr2-q3Z+NM}$7ee93ZD`=IjzAW{7D2_O|n;YqZ<6ue
zsP;kUAwZ(|=Py7iknAHupC&l|R-xKw4~ajJC)gFx;19Gv}cNHL_N$Oh?$g9{~ag@f5yUJt65psQ06
zopGF<=p-gc*$3$j6c>Tk>wxR2C{Q5@ZP`J*^AqHCqy`hnub{p(6R3Ivm4h;jkV*}t
z6VhIC2G!%bOyaem(v+zhRG@(hZx9VFyulTMcC`*BAV-U^6
zQVJ5z_W>!fW2#08H!CJ)4e;Qr#t%^NBF0`(R9ylEDQLKrsTvd)I((?6Fja#J1W;`Y
zqM-w$ptZ%U3^veq406-uBPfYrYq}^yn=Zcg
zA(0Cjiv*<$Q2nR_&Y$a0^QQ>7vsMEV4VfC8&U0p)&n3>{PhP&1(JP4=+gs-
z=O$G9mP5){kSGVch9I`^MArunPonj0#%do(6u*5S6-fSr+6NNFrVlhu1uD6i6w*W(
z7;3kG3UA2#)OIivJd?Bo#1yaH3!>yJYWIO?@!I_$3cOQ^Ny7%zq%DqNWMHe^4HB*f
zldAvNL8WN1A84>82&4oq4(fM8#6wWTL1iaIJPbt~)PpJZ0+|zmB5uXVz*fu-5|08E
z#a7UUuvqOLP}^0GsT$-X@IE3iLyoC>D@Y=VS%;Six`6;Jz$8)&rsbHbw}BLZ8o6MG
z98)zYB)~?18FEb3pzr_-fEjX3)jL62oBj<8~42D=6lN+4lSK!Wvy
z8FEb3pb!NMfEjX3)u6Bi3qTnXkp3Q<7HA(7XgpRQo+d#BA}AFyv#{pB2lZc&7bt?u
zD;6=<{CLm|3F@+0=zbj`8dp#~4C?zCfX6MttAKq#yFS>&AZ4Qjq)!JPlLpVo
za4|8lX}Ew!2f#xg-0V6MOzioJppg$yP%*LRD}&hJPy+WYA>*`$AoIapR4%ssFQ8c?
zPPY7SjG&ncafm>XA~Qn)L<~II&RwL+%uv7%u^BY{&cnpS0k&DoiiwE@+#G;#)j+ic
zsQax|#>89;iWjD8P#FuVAV4$|b3P9!p@7O}_{
zQWvp+$F0D5orzh)6`YGeeqqj+18J)Ul@L0zOkkQx!b(q?8TfN-LiszDW#
z4ljgeX31v(c_NCb8dMK}gh4ctFsKj*34>^6$Ydy_&IkF`2$VPev3Y_*Hvp7?kzK{j
z#3sVdUK|KoAP(|{RuRM(piw)pxAKEPJM%