From e53f5808d9eb6efb89055768b7ae9ed02e3832b6 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 24 Jun 2016 15:03:08 +0200 Subject: [PATCH 01/36] add ability to check server fingerprint --- .../Library/Backend/SSHv2/SSHv2Backend.cs | 29 +++++++++++++++++-- Duplicati/Library/Backend/SSHv2/Strings.cs | 3 ++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 883b7d89d..c5f96e431 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -31,6 +31,7 @@ namespace Duplicati.Library.Backend { public const string SSH_KEYFILE_OPTION = "ssh-keyfile"; public const string SSH_KEYFILE_INLINE = "ssh-key"; + public const string SSH_FINGERPRINT_OPTION = "ssh-fingerprint"; public const string KEYFILE_URI = "sshkey://"; Dictionary m_options; @@ -39,6 +40,7 @@ namespace Duplicati.Library.Backend private string m_path; private string m_username; private string m_password; + private string m_fingerprint; private int m_port = 22; @@ -59,10 +61,14 @@ namespace Duplicati.Library.Backend m_username = options["auth-username"]; if (options.ContainsKey("auth-password")) m_password = options["auth-password"]; - if (!string.IsNullOrEmpty(uri .Username)) + if (options.ContainsKey(SSH_FINGERPRINT_OPTION)) + m_fingerprint = options[SSH_FINGERPRINT_OPTION]; + if (!string.IsNullOrEmpty(uri.Username)) m_username = uri.Username; - if (!string.IsNullOrEmpty(uri .Password)) + if (!string.IsNullOrEmpty(uri.Password)) m_password = uri.Password; + if (uri.QueryParameters != null && uri.QueryParameters[SSH_FINGERPRINT_OPTION] != null) + m_fingerprint = uri.QueryParameters[SSH_FINGERPRINT_OPTION]; m_path = uri.Path; @@ -136,6 +142,7 @@ namespace Duplicati.Library.Backend return new List(new ICommandLineArgument[] { new CommandLineArgument("auth-password", CommandLineArgument.ArgumentType.Password, Strings.SSHv2Backend.DescriptionAuthPasswordShort, Strings.SSHv2Backend.DescriptionAuthPasswordLong), new CommandLineArgument("auth-username", CommandLineArgument.ArgumentType.String, Strings.SSHv2Backend.DescriptionAuthUsernameShort, Strings.SSHv2Backend.DescriptionAuthUsernameLong), + new CommandLineArgument(SSH_FINGERPRINT_OPTION, CommandLineArgument.ArgumentType.String, Strings.SSHv2Backend.DescriptionFingerprintShort, Strings.SSHv2Backend.DescriptionFingerprintLong), new CommandLineArgument(SSH_KEYFILE_OPTION, CommandLineArgument.ArgumentType.Path, Strings.SSHv2Backend.DescriptionSshkeyfileShort, Strings.SSHv2Backend.DescriptionSshkeyfileLong), new CommandLineArgument(SSH_KEYFILE_INLINE, CommandLineArgument.ArgumentType.Password, Strings.SSHv2Backend.DescriptionSshkeyShort, Strings.SSHv2Backend.DescriptionSshkeyLong(KEYFILE_URI)), }); @@ -205,6 +212,24 @@ namespace Duplicati.Library.Backend else con = new SftpClient(m_server, m_port, m_username, m_password); + con.HostKeyReceived += delegate (object sender, HostKeyEventArgs e) + { + e.CanTrust = false; + + if (string.IsNullOrEmpty(m_fingerprint)) + { + e.CanTrust = true; + return; + } + + string hostFingerprint = e.HostKeyName + " " + e.KeyLength.ToString() + " " + BitConverter.ToString(e.FingerPrint).Replace('-', ':'); + + if (hostFingerprint.ToLower() != m_fingerprint.ToLower()) + throw new Exception(Strings.SSHv2Backend.FingerprintNotMatchManagedError(hostFingerprint.ToLower())); + else + e.CanTrust = true; + }; + con.Connect(); try diff --git a/Duplicati/Library/Backend/SSHv2/Strings.cs b/Duplicati/Library/Backend/SSHv2/Strings.cs index e7b31352c..0cd6a6982 100644 --- a/Duplicati/Library/Backend/SSHv2/Strings.cs +++ b/Duplicati/Library/Backend/SSHv2/Strings.cs @@ -24,11 +24,14 @@ namespace Duplicati.Library.Backend.Strings { public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supplies the password used to connect to the server"); } } public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } } public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supplies the username used to connect to the server"); } } + public static string DescriptionFingerprintLong { get { return LC.L(@"The server fingerprint used for validation of server identity. Format is eg. ""ssh - rsa 4096 11:22:33:44:55:66:77:88:99:00:11:22:33:44:55:66""."); } } + public static string DescriptionFingerprintShort { get { return LC.L(@"Supplies server fingerprint used for validation of server identity"); } } public static string DescriptionSshkeyfileLong { get { return LC.L(@"Points to a valid OpenSSH keyfile. If the file is encrypted, the password supplied is used to decrypt the keyfile. If this option is supplied, the password is not used to authenticate. This option only works when using the managed SSH client."); } } public static string DescriptionSshkeyfileShort { get { return LC.L(@"Uses a SSH private key to authenticate"); } } public static string DescriptionSshkeyLong(string urlprefix) { return LC.L(@"An url-encoded SSH private key. The private key must be prefixed with {0}. If the file is encrypted, the password supplied is used to decrypt the keyfile. If this option is supplied, the password is not used to authenticate. This option only works when using the managed SSH client.", urlprefix); } public static string DescriptionSshkeyShort { get { return LC.L(@"Uses a SSH private key to authenticate"); } } public static string DisplayName { get { return LC.L(@"SFTP (SSH)"); } } public static string FolderNotFoundManagedError(string foldername, string message) { return LC.L(@"Unable to set folder to {0}, error message: {1}", foldername, message); } + public static string FingerprintNotMatchManagedError(string fingerprint) { return LC.L(@"Validation of server fingerprint failed. Server returned fingerprint ""{0}"". Cause of this message is either not correct configuration or Man-in-the-middle attack!", fingerprint); } } } From 9f0b3a004243710e31c4dd0c591dd2891606b618 Mon Sep 17 00:00:00 2001 From: Max Date: Mon, 27 Jun 2016 19:57:36 +0200 Subject: [PATCH 02/36] default behavior is that fingerprint needs to be specified, but can be overided --- Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs | 9 ++++++++- Duplicati/Library/Backend/SSHv2/Strings.cs | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index c5f96e431..addb961a9 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -32,6 +32,7 @@ namespace Duplicati.Library.Backend public const string SSH_KEYFILE_OPTION = "ssh-keyfile"; public const string SSH_KEYFILE_INLINE = "ssh-key"; public const string SSH_FINGERPRINT_OPTION = "ssh-fingerprint"; + public const string SSH_FINGERPRINT_ACCEPT_ANY_OPTION = "shh-accept-any-fingerprints"; public const string KEYFILE_URI = "sshkey://"; Dictionary m_options; @@ -41,6 +42,7 @@ namespace Duplicati.Library.Backend private string m_username; private string m_password; private string m_fingerprint; + private bool m_fingerprintallowall; private int m_port = 22; @@ -70,6 +72,8 @@ namespace Duplicati.Library.Backend if (uri.QueryParameters != null && uri.QueryParameters[SSH_FINGERPRINT_OPTION] != null) m_fingerprint = uri.QueryParameters[SSH_FINGERPRINT_OPTION]; + m_fingerprintallowall = Utility.Utility.ParseBoolOption(options, SSH_FINGERPRINT_ACCEPT_ANY_OPTION); + m_path = uri.Path; if (!string.IsNullOrWhiteSpace(m_path) && !m_path.EndsWith("/")) @@ -216,7 +220,7 @@ namespace Duplicati.Library.Backend { e.CanTrust = false; - if (string.IsNullOrEmpty(m_fingerprint)) + if (m_fingerprintallowall) { e.CanTrust = true; return; @@ -224,6 +228,9 @@ namespace Duplicati.Library.Backend string hostFingerprint = e.HostKeyName + " " + e.KeyLength.ToString() + " " + BitConverter.ToString(e.FingerPrint).Replace('-', ':'); + if (string.IsNullOrEmpty(m_fingerprint)) + throw new Exception(Strings.SSHv2Backend.FingerprintNotSpecifiedManagedError(hostFingerprint.ToLower())); + if (hostFingerprint.ToLower() != m_fingerprint.ToLower()) throw new Exception(Strings.SSHv2Backend.FingerprintNotMatchManagedError(hostFingerprint.ToLower())); else diff --git a/Duplicati/Library/Backend/SSHv2/Strings.cs b/Duplicati/Library/Backend/SSHv2/Strings.cs index 0cd6a6982..39da27636 100644 --- a/Duplicati/Library/Backend/SSHv2/Strings.cs +++ b/Duplicati/Library/Backend/SSHv2/Strings.cs @@ -33,5 +33,6 @@ namespace Duplicati.Library.Backend.Strings { public static string DisplayName { get { return LC.L(@"SFTP (SSH)"); } } public static string FolderNotFoundManagedError(string foldername, string message) { return LC.L(@"Unable to set folder to {0}, error message: {1}", foldername, message); } public static string FingerprintNotMatchManagedError(string fingerprint) { return LC.L(@"Validation of server fingerprint failed. Server returned fingerprint ""{0}"". Cause of this message is either not correct configuration or Man-in-the-middle attack!", fingerprint); } + public static string FingerprintNotSpecifiedManagedError(string fingerprint) { return LC.L(@"Please add --ssh-fingerprint=""{0}"" to trust this host. Or --shh-accept-any-fingerprints (NOT SECURE) only for test reasons!", fingerprint); } } } From 228ade0209c8bf750fbc86b37dab3dbfcd9a4a0c Mon Sep 17 00:00:00 2001 From: Vaporizer8 Date: Tue, 12 Jul 2016 21:33:28 +0200 Subject: [PATCH 03/36] Handle paths with leading / Paths with a leading / can cause issues with some WebDAV servers (IIS) as this results in https://example.com//foo/bar/ --- Duplicati/Library/Utility/Uri.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Utility/Uri.cs b/Duplicati/Library/Utility/Uri.cs index 541fbed9d..e655ffd3b 100644 --- a/Duplicati/Library/Utility/Uri.cs +++ b/Duplicati/Library/Utility/Uri.cs @@ -260,7 +260,7 @@ namespace Duplicati.Library.Utility if (!string.IsNullOrEmpty(path)) { - if (!string.IsNullOrEmpty(host)) + if (!string.IsNullOrEmpty(host) && !path.StartsWith("/")) s += "/"; s += path; } From 7127ebba382f0c6b04c734d2cb77879443dbb49b Mon Sep 17 00:00:00 2001 From: Vaporizer8 Date: Tue, 12 Jul 2016 21:37:52 +0200 Subject: [PATCH 04/36] Fix editing SSL enabled backends in GUI --- .../Server/webroot/ngax/scripts/services/EditUriBuiltins.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js index 14b1a5862..1eda500d7 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js @@ -518,6 +518,7 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, continuation(); }); }; + EditUriBackendConfig.validaters['ftps'] = EditUriBackendConfig.validaters['ftp']; EditUriBackendConfig.validaters['aftp'] = function (scope, continuation) { var res = @@ -543,8 +544,10 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, }; EditUriBackendConfig.validaters['webdav'] = EditUriBackendConfig.validaters['ssh']; + EditUriBackendConfig.validaters['webdavs'] = EditUriBackendConfig.validaters['webdav']; EditUriBackendConfig.validaters['cloudfiles'] = EditUriBackendConfig.validaters['ssh']; EditUriBackendConfig.validaters['tahoe'] = EditUriBackendConfig.validaters['ssh']; + EditUriBackendConfig.validaters['tahoes'] = EditUriBackendConfig.validaters['tahoe']; EditUriBackendConfig.validaters['onedrive'] = function(scope, continuation) { @@ -667,6 +670,7 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, checkLowerCase(); } }; + EditUriBackendConfig.validaters['s3s'] = EditUriBackendConfig.validaters['s3']; EditUriBackendConfig.validaters['b2'] = function(scope, continuation) { var res = @@ -689,4 +693,4 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, }; -}); \ No newline at end of file +}); From 77bb0feb042b4402c376cd846d906c1b5a0d7404 Mon Sep 17 00:00:00 2001 From: Vaporizer8 Date: Wed, 13 Jul 2016 22:11:18 +0200 Subject: [PATCH 05/36] Fix SSL checkbox style --- Duplicati/Server/webroot/ngax/templates/backends/generic.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/templates/backends/generic.html b/Duplicati/Server/webroot/ngax/templates/backends/generic.html index 69eb4e1c8..7af66fc1d 100644 --- a/Duplicati/Server/webroot/ngax/templates/backends/generic.html +++ b/Duplicati/Server/webroot/ngax/templates/backends/generic.html @@ -1,6 +1,6 @@ -
- +
+
From d124ff1a097eb4c4f98b6d647f261c5fc7237cff Mon Sep 17 00:00:00 2001 From: Vaporizer8 Date: Wed, 13 Jul 2016 22:20:19 +0200 Subject: [PATCH 06/36] Fix SSL checkbox style for S3 --- Duplicati/Server/webroot/ngax/templates/backends/s3.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/templates/backends/s3.html b/Duplicati/Server/webroot/ngax/templates/backends/s3.html index 671d874a8..7dbda893d 100644 --- a/Duplicati/Server/webroot/ngax/templates/backends/s3.html +++ b/Duplicati/Server/webroot/ngax/templates/backends/s3.html @@ -1,6 +1,6 @@ -
- +
+
From a45d21781aebf5578b3f903c6d63e9b91ddca485 Mon Sep 17 00:00:00 2001 From: Vaporizer8 Date: Wed, 13 Jul 2016 22:22:41 +0200 Subject: [PATCH 07/36] Fix SSL checkbox style --- Duplicati/Server/webroot/ngax/templates/backends/generic.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Server/webroot/ngax/templates/backends/generic.html b/Duplicati/Server/webroot/ngax/templates/backends/generic.html index 7af66fc1d..62366a44c 100644 --- a/Duplicati/Server/webroot/ngax/templates/backends/generic.html +++ b/Duplicati/Server/webroot/ngax/templates/backends/generic.html @@ -1,6 +1,6 @@
- +
From e15d41654e8ac9d2490b575bd89e1eed9764afce Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 4 Aug 2016 14:31:18 +0200 Subject: [PATCH 08/36] Added compact feature to the UI. This fixes #1867 --- .../Duplicati.Server.Serialization/Enums.cs | 1 + Duplicati/Server/Program.cs | 4 +++- Duplicati/Server/Runner.cs | 9 ++++++++- Duplicati/Server/Strings.cs | 3 ++- .../Server/WebServer/RESTMethods/Backup.cs | 17 +++++++++++++++-- .../ngax/scripts/controllers/HomeController.js | 4 ++++ .../Server/webroot/ngax/templates/home.html | 5 +++-- 7 files changed, 36 insertions(+), 7 deletions(-) diff --git a/Duplicati/Server/Duplicati.Server.Serialization/Enums.cs b/Duplicati/Server/Duplicati.Server.Serialization/Enums.cs index 2bd602fc9..dbe2d341c 100755 --- a/Duplicati/Server/Duplicati.Server.Serialization/Enums.cs +++ b/Duplicati/Server/Duplicati.Server.Serialization/Enums.cs @@ -46,6 +46,7 @@ namespace Duplicati.Server.Serialization Repair, RepairUpdate, Verify, + Compact, CreateReport } diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index e59bdfa7f..65d84b789 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -621,7 +621,9 @@ namespace Duplicati.Server return Strings.TaskType.ListActualFiles; case Duplicati.Server.Serialization.DuplicatiOperation.Verify: return Strings.TaskType.ListBackupEntries; - case Duplicati.Server.Serialization.DuplicatiOperation.Restore: + case Duplicati.Server.Serialization.DuplicatiOperation.Compact: + return Strings.TaskType.CompactRemoteFiles; + case Duplicati.Server.Serialization.DuplicatiOperation.Restore: return Strings.TaskType.ListBackups; default: return type.ToString(); diff --git a/Duplicati/Server/Runner.cs b/Duplicati/Server/Runner.cs index 213bbd799..3e89c7092 100644 --- a/Duplicati/Server/Runner.cs +++ b/Duplicati/Server/Runner.cs @@ -464,7 +464,14 @@ namespace Duplicati.Server return r; } - case DuplicatiOperation.CreateReport: + case DuplicatiOperation.Compact: + { + var r = controller.Compact(); + UpdateMetadata(backup, r); + return r; + } + + case DuplicatiOperation.CreateReport: { using(var tf = new Duplicati.Library.Utility.TempFile()) { diff --git a/Duplicati/Server/Strings.cs b/Duplicati/Server/Strings.cs index 460a42fe3..e615f42cb 100644 --- a/Duplicati/Server/Strings.cs +++ b/Duplicati/Server/Strings.cs @@ -38,7 +38,8 @@ Error message: {0}", error); } public static string ListActualFiles { get { return LC.L(@"List actual files"); } } public static string ListBackupEntries { get { return LC.L(@"List backup entries"); } } public static string ListBackups { get { return LC.L(@"List backups"); } } - } + public static string CompactRemoteFiles { get { return LC.L(@"Compact remote files"); } } + } internal static class Scheduler { public static string InvalidTimeSetupError(System.DateTime startdate, string interval, string alloweddays) { return LC.L(@"Unable to find a valid date, given the start date {0}, the repetition interval {1} and the allowed days {2}", startdate, interval, alloweddays); } } diff --git a/Duplicati/Server/WebServer/RESTMethods/Backup.cs b/Duplicati/Server/WebServer/RESTMethods/Backup.cs index 150fd5e96..7750b1d0d 100644 --- a/Duplicati/Server/WebServer/RESTMethods/Backup.cs +++ b/Duplicati/Server/WebServer/RESTMethods/Backup.cs @@ -215,7 +215,16 @@ namespace Duplicati.Server.WebServer.RESTMethods info.OutputOK(new {Status = "OK", ID = task.TaskID}); } - private void DoRepair(IBackup backup, RequestInfo info, bool repairUpdate) + private void Compact(IBackup backup, RequestInfo info) + { + var task = Runner.CreateTask(DuplicatiOperation.Compact, backup); + Program.WorkThread.AddTask(task); + Program.StatusEventNotifyer.SignalNewEvent(); + + info.OutputOK(new { Status = "OK", ID = task.TaskID }); + } + + private void DoRepair(IBackup backup, RequestInfo info, bool repairUpdate) { var input = info.Request.Form; string[] filters = null; @@ -415,7 +424,11 @@ namespace Duplicati.Server.WebServer.RESTMethods Verify(bk, info); return; - case "start": + case "compact": + Compact(bk, info); + return; + + case "start": case "run": RunBackup(bk, info); return; diff --git a/Duplicati/Server/webroot/ngax/scripts/controllers/HomeController.js b/Duplicati/Server/webroot/ngax/scripts/controllers/HomeController.js index 1a5a26dc5..f11cfb59f 100644 --- a/Duplicati/Server/webroot/ngax/scripts/controllers/HomeController.js +++ b/Duplicati/Server/webroot/ngax/scripts/controllers/HomeController.js @@ -17,6 +17,10 @@ backupApp.controller('HomeController', function ($scope, $location, BackupList, $location.path('/export/' + id); }; + $scope.doCompact = function(id) { + AppService.post('/backup/' + id + '/compact'); + }; + $scope.doDelete = function(id, name) { DialogService.dialog('Confirm delete', 'Do you really want to delete the backup: ' + name, ['No', 'Yes'], function(ix) { if (ix == 1) diff --git a/Duplicati/Server/webroot/ngax/templates/home.html b/Duplicati/Server/webroot/ngax/templates/home.html index dd27b8a8b..e11b1b9e6 100644 --- a/Duplicati/Server/webroot/ngax/templates/home.html +++ b/Duplicati/Server/webroot/ngax/templates/home.html @@ -56,8 +56,9 @@
Advanced:
-

Manage local database ...

-

Verify backup files

+

Manage database ...

+

Verify files

+

Compact now

Reporting:
From e1ab177e69ca9c5945bc0390f7f534cd1aa69dfc Mon Sep 17 00:00:00 2001 From: Vaporizer8 Date: Fri, 5 Aug 2016 23:08:24 +0200 Subject: [PATCH 09/36] Revert 7127ebb --- .../Server/webroot/ngax/scripts/services/EditUriBuiltins.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js index 1eda500d7..0f4fee136 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js @@ -518,7 +518,6 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, continuation(); }); }; - EditUriBackendConfig.validaters['ftps'] = EditUriBackendConfig.validaters['ftp']; EditUriBackendConfig.validaters['aftp'] = function (scope, continuation) { var res = @@ -544,10 +543,8 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, }; EditUriBackendConfig.validaters['webdav'] = EditUriBackendConfig.validaters['ssh']; - EditUriBackendConfig.validaters['webdavs'] = EditUriBackendConfig.validaters['webdav']; EditUriBackendConfig.validaters['cloudfiles'] = EditUriBackendConfig.validaters['ssh']; EditUriBackendConfig.validaters['tahoe'] = EditUriBackendConfig.validaters['ssh']; - EditUriBackendConfig.validaters['tahoes'] = EditUriBackendConfig.validaters['tahoe']; EditUriBackendConfig.validaters['onedrive'] = function(scope, continuation) { @@ -670,7 +667,6 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, checkLowerCase(); } }; - EditUriBackendConfig.validaters['s3s'] = EditUriBackendConfig.validaters['s3']; EditUriBackendConfig.validaters['b2'] = function(scope, continuation) { var res = From 4d323a945055b1f9475bf10b51afb88bced336d6 Mon Sep 17 00:00:00 2001 From: Vaporizer8 Date: Sat, 6 Aug 2016 22:37:40 +0200 Subject: [PATCH 10/36] Fix file size dropdown in step 4 --- Duplicati/Server/webroot/ngax/templates/addoredit.html | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/templates/addoredit.html b/Duplicati/Server/webroot/ngax/templates/addoredit.html index 775ab16e9..4922ee689 100755 --- a/Duplicati/Server/webroot/ngax/templates/addoredit.html +++ b/Duplicati/Server/webroot/ngax/templates/addoredit.html @@ -220,10 +220,7 @@
- - - - +
@@ -268,4 +265,4 @@
- \ No newline at end of file + From b841a617ce27eb1a0af084e4c06d07ce0d9113f2 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 10 Aug 2016 09:29:55 +0200 Subject: [PATCH 11/36] Version bump to v2.0.1.18-2.0.1.18_canary_2016-08-10 You can download this build from: Binaries: http://updates.duplicati.com/canary/duplicati-2.0.1.18_canary_2016-08-10.zip Signature file: http://updates.duplicati.com/canary/duplicati-2.0.1.18_canary_2016-08-10.zip.sig ASCII signature file: http://updates.duplicati.com/canary/duplicati-2.0.1.18_canary_2016-08-10.zip.sig.asc MD5: 3a1979d0b03038d0afa7a1e84a4fbec5 SHA1: 8bfdb0989d747960145ee670b0151f42f5f0c221 SHA256: 4f69467299cce78301d21547f05e6907faaa5684236dd5631127d5216a841fb5 --- Updates/build_version.txt | 2 +- changelog.txt | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/Updates/build_version.txt b/Updates/build_version.txt index 98d9bcb75..3c032078a 100644 --- a/Updates/build_version.txt +++ b/Updates/build_version.txt @@ -1 +1 @@ -17 +18 diff --git a/changelog.txt b/changelog.txt index 88d8cc80e..9b4d630f7 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,29 @@ +2016-08-10 +========== +* OneDrive for Business now uploads files in chunks to prevent timeout, thanks @ReneHezer +* Prevent using both --keep-version and --keep-time to avoid confusion +* Fixed a number of usability issues with the WebUI +* Fixed encoding of special characters for Amazon Cloud Drive, thanks @Xelio +* Fixed an issue with the recovery tool not respecting the restore folder target, thanks @mnaiman +* Fixed issues when including the "home" folder on Windows +* Fixed expanding environment variables and special folder names for filters +* Changed displaying the dates in the WebUI with the browser locale +* Added option to re-compress and re-encrypt from the recovery tool, thanks @mnaiman +* Updated AWS S3 library to latests version +* Added option to create an un-priviledged S3 user via IAM, if the current user has permissions +* Fixed an issue with the scheduler that cause high CPU usage for invalid schedules +* Fixed an issue with the WebUI forgetting SSL settings in some cases +* Upgraded support for sending email to use a new library with proper SSL support, thanks @mnaiman +* Fixed spurious failures in SSH connections, caused by leaking open connections, thanks @mnaiman +* Fixed a Mono dependency in the Debian package that has been renamed +* Added a DropBox backend, thanks @wvc +* Added option to serve the WebUI with an SSL certificate, thanks @mnaiman +* Improments to the recovery tool, thanks @mnaiman +* Fixed an issue where deleting a backup and creating a new could cause data to be pre-set from the delete backup on the new, thanks @4m1g0 +* Fixed an issue with LVM not working on newer versions of Ubuntu +* Fixed an issue where Mono could not be located by the OSX app bundle +* Fixes to WebUI layout, thansk @Vaporizer8 + 2016-05-14 ========== * Fixes for UI on Safari, thanks @alexfran From 9c1afa8d1587998e3489ff4fb6984b7c79509c56 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 10 Aug 2016 09:56:43 +0200 Subject: [PATCH 12/36] Fixed path to SharpAESCrypt from build script after upgrade to NuGet --- build-installers.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) mode change 100644 => 100755 build-installers.sh diff --git a/build-installers.sh b/build-installers.sh old mode 100644 new mode 100755 index 3997f62d7..01db8459f --- a/build-installers.sh +++ b/build-installers.sh @@ -137,7 +137,7 @@ if [ -f "${GPG_KEYFILE}" ]; then echo fi - GPGDATA=`mono thirdparty/SharpAESCrypt/SharpAESCrypt.exe d "${KEYFILE_PASSWORD}" "${GPG_KEYFILE}"` + GPGDATA=`mono BuildTools/AutoUpdateBuilder/bin/Debug/SharpAESCrypt.exe d "${KEYFILE_PASSWORD}" "${GPG_KEYFILE}"` if [ ! $? -eq 0 ]; then echo "Decrypting GPG keyfile failed" exit 1 @@ -159,6 +159,8 @@ aws ec2 start-instances --profile=duplicati-builder --instance-ids "${WINDOWS_IN echo "" echo "" echo "Building OSX package locally ..." +echo "" +echo "Enter local sudo password..." cd Installer/OSX bash "make-dmg.sh" "../../$1" From 1d81c202135866629ce2ee6eacb3d992a2bc3e02 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 10 Aug 2016 09:57:11 +0200 Subject: [PATCH 13/36] Fixed assembly version in DropBox module to prevent unwanted changes during the build process --- .../Library/Backend/Dropbox/Properties/AssemblyInfo.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Backend/Dropbox/Properties/AssemblyInfo.cs b/Duplicati/Library/Backend/Dropbox/Properties/AssemblyInfo.cs index 51b1265d6..18663113a 100644 --- a/Duplicati/Library/Backend/Dropbox/Properties/AssemblyInfo.cs +++ b/Duplicati/Library/Backend/Dropbox/Properties/AssemblyInfo.cs @@ -1,4 +1,4 @@ -using System.Reflection; +using System.Reflection; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -31,6 +31,6 @@ using System.Runtime.InteropServices; // // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] +// [assembly: AssemblyVersion("2.0.0.7")] +[assembly: AssemblyVersion("2.0.0.7")] +[assembly: AssemblyFileVersion("2.0.0.7")] From 584e0404beea4a5f2b226bfe7d78652ae9fccc74 Mon Sep 17 00:00:00 2001 From: agrajaghh Date: Fri, 12 Aug 2016 11:34:09 +0200 Subject: [PATCH 14/36] add issue template --- ISSUE_TEMPLATE.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 ISSUE_TEMPLATE.md diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..70ea6d8ce --- /dev/null +++ b/ISSUE_TEMPLATE.md @@ -0,0 +1,32 @@ + + +I have: +- [ ] searched open and closed issues for duplicates + +---------------------------------------- + +### Version info + +**Duplicati Version:** 0.0.0.0 +**Operating System:** +**Backend:** + +### Bug description +Describe here the issue that you are experiencing. + +### Steps to reproduce +- using hyphens as bullet points +- list the steps +- that reproduce the bug + +**Actual result:** Describe here what happens after you run the steps above (i.e. the buggy behaviour) +**Expected result:** Describe here what should happen after you run the steps above (i.e. what would be the correct behaviour) + +### Screenshots + + +### debug log + From ae8ad5e1b05185d9f978a7b813396dc519f20244 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 16 Aug 2016 11:08:21 +0200 Subject: [PATCH 15/36] Added DropBox backend as a reference so it is included in the builds, as mentioned in #1835 --- .../BackendTool/Duplicati.CommandLine.BackendTool.csproj | 4 ++++ Duplicati/CommandLine/Duplicati.CommandLine.csproj | 4 ++++ .../RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj | 4 ++++ .../GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj | 4 ++++ Duplicati/Server/Duplicati.Server.csproj | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/Duplicati/CommandLine/BackendTool/Duplicati.CommandLine.BackendTool.csproj b/Duplicati/CommandLine/BackendTool/Duplicati.CommandLine.BackendTool.csproj index 9741f7991..faffbc2ed 100644 --- a/Duplicati/CommandLine/BackendTool/Duplicati.CommandLine.BackendTool.csproj +++ b/Duplicati/CommandLine/BackendTool/Duplicati.CommandLine.BackendTool.csproj @@ -138,6 +138,10 @@ {3F47CB0C-CEE1-447F-9C26-7F8E419E3E3F} Duplicati.Library.Backend.AlternativeFTP + + {B20A7CEE-9C5B-47B9-8B76-BC85ADFE8493} + Duplicati.Library.Backend.Dropbox + diff --git a/Duplicati/CommandLine/Duplicati.CommandLine.csproj b/Duplicati/CommandLine/Duplicati.CommandLine.csproj index 08a63a356..bf8357ead 100644 --- a/Duplicati/CommandLine/Duplicati.CommandLine.csproj +++ b/Duplicati/CommandLine/Duplicati.CommandLine.csproj @@ -182,6 +182,10 @@ {BB014EA5-CE2C-4444-8D30-38983A0E8553} Duplicati.Library.UsageReporter + + {B20A7CEE-9C5B-47B9-8B76-BC85ADFE8493} + Duplicati.Library.Backend.Dropbox + diff --git a/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj b/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj index 98fa1c568..acc76a047 100644 --- a/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj +++ b/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj @@ -150,6 +150,10 @@ {3F47CB0C-CEE1-447F-9C26-7F8E419E3E3F} Duplicati.Library.Backend.AlternativeFTP + + {B20A7CEE-9C5B-47B9-8B76-BC85ADFE8493} + Duplicati.Library.Backend.Dropbox + diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj index 5a0048049..02ded6c0e 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj @@ -307,6 +307,10 @@ {3F47CB0C-CEE1-447F-9C26-7F8E419E3E3F} Duplicati.Library.Backend.AlternativeFTP + + {B20A7CEE-9C5B-47B9-8B76-BC85ADFE8493} + Duplicati.Library.Backend.Dropbox + diff --git a/Duplicati/Server/Duplicati.Server.csproj b/Duplicati/Server/Duplicati.Server.csproj index f926f2b0a..e942d31f6 100644 --- a/Duplicati/Server/Duplicati.Server.csproj +++ b/Duplicati/Server/Duplicati.Server.csproj @@ -284,6 +284,10 @@ {BB014EA5-CE2C-4444-8D30-38983A0E8553} Duplicati.Library.UsageReporter + + {B20A7CEE-9C5B-47B9-8B76-BC85ADFE8493} + Duplicati.Library.Backend.Dropbox + From ca5bb08c071e37be9613ce5df2bdfd6a9a2f8563 Mon Sep 17 00:00:00 2001 From: Max Date: Sat, 20 Aug 2016 00:22:33 +0200 Subject: [PATCH 16/36] Add files via upload Converted python script to powershell for machines where is not powershell installed --- Tools/Verification/DuplicatiVerify.ps1 | 94 ++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 Tools/Verification/DuplicatiVerify.ps1 diff --git a/Tools/Verification/DuplicatiVerify.ps1 b/Tools/Verification/DuplicatiVerify.ps1 new file mode 100644 index 000000000..08f52c47d --- /dev/null +++ b/Tools/Verification/DuplicatiVerify.ps1 @@ -0,0 +1,94 @@ +Set-StrictMode -Version 5 + +function HexToBase64 +{ + param + ( + [Parameter(Mandatory=$true, Position=0, ValueFromPipeline=$true)] + [string]$s + ) + [array]$return = @() + + for ($i = 0; $i -lt $s.Length ; $i += 2) { + $return += [Byte]::Parse($s.Substring($i, 2), [System.Globalization.NumberStyles]::HexNumber) + } + + Write-Output $([Convert]::ToBase64String($return)) +} + +function Verify-Hashes +{ + [CmdletBinding(PositionalBinding=$false)] + param + ( + [Parameter(Mandatory=$true)] + [ValidateNotNullOrEmpty()] + [string]$filename + ) + + [int]$errorCount = 0 + [int]$checked = 0 + + if(-not $(Test-Path $filename)) { + Write-Host "Specified file does not exist: $filename" + return -1 + } + + [string]$folder = Split-Path -Path $filename + + $remoteVolumes = (Get-Content $filename) -Join "`n" | ConvertFrom-Json + + foreach ($remoteVolume in $remoteVolumes) { + [string]$volFileName = $remoteVolume.Name + [long]$volFileSize = $remoteVolume.Size + [string]$volFilePath = Join-Path -Path $folder -ChildPath $volFileName + + if(-not $(Test-Path $volFilePath)) { + Write-Host "File missing: $volFilePath" -ForegroundColor Red + $errorCount++ + } else { + $checked++ + + Write-Host "Verifying file $volFileName" + + $shaHash = Get-FileHash -Path $volFilePath -Algorithm SHA256 + + if($remoteVolume.Hash -ne $(HexToBase64 $shaHash.Hash)) { + Write-Host "*** Hash check failed for file: $volFileName" -ForegroundColor Red + $errorCount++ + } + } + } + + if($errorCount -gt 0) { + Write-Host "Errors were found" -ForegroundColor Red + } else { + Write-Host "No errors found" -ForegroundColor Green + } + + return $errorCount +} + +[string]$argument = "" +if ($args.Length -eq 1) { $argument = $args[0] } +else { $argument = $PSScriptRoot } + + +if(-not $(Test-Path $argument)) { + Write-Host "No such file or directory: $argument" -ForegroundColor Red +} + +if(Test-Path $argument -PathType Leaf) { + Verify-Hashes -filename $argument +} else { + [int] $files = 0 + + Get-ChildItem $argument -Filter "*-verification.json" | + Foreach-Object { + $files++ + Write-Host "Verifying file: $_.Name" + Verify-Hashes -filename $_.FullName + } + + if ($files -eq 0) { Write-Host "No verification files in folder: $argument" } +} \ No newline at end of file From 861dd9cc8be71fd0ca7e482516c81e86d1250977 Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 21 Aug 2016 12:53:44 +0200 Subject: [PATCH 17/36] during recompress is neccessary to recalculate hashes of recompressed block file and update them in dindex files - fixed --- .../Duplicati.CommandLine.RecoveryTool.csproj | 6 + .../CommandLine/RecoveryTool/Recompress.cs | 159 +++++++++++------- .../CommandLine/RecoveryTool/packages.config | 4 + 3 files changed, 107 insertions(+), 62 deletions(-) create mode 100644 Duplicati/CommandLine/RecoveryTool/packages.config diff --git a/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj b/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj index 98fa1c568..6f8dcea00 100644 --- a/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj +++ b/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj @@ -37,6 +37,9 @@ + + ..\..\..\packages\Newtonsoft.Json.9.0.1\lib\net45\Newtonsoft.Json.dll + @@ -154,4 +157,7 @@ + + + \ No newline at end of file diff --git a/Duplicati/CommandLine/RecoveryTool/Recompress.cs b/Duplicati/CommandLine/RecoveryTool/Recompress.cs index bf745ac7a..7ae4b867a 100644 --- a/Duplicati/CommandLine/RecoveryTool/Recompress.cs +++ b/Duplicati/CommandLine/RecoveryTool/Recompress.cs @@ -5,6 +5,7 @@ using System.Linq; using Duplicati.Library.Main; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; +using Newtonsoft.Json.Linq; namespace Duplicati.CommandLine.RecoveryTool { @@ -89,14 +90,24 @@ namespace Duplicati.CommandLine.RecoveryTool bool reencrypt = Library.Utility.Utility.ParseBoolOption(options, "reencrypt"); bool reupload = Library.Utility.Utility.ParseBoolOption(options, "reupload"); - foreach (var entry in remotefiles) + // Needs order (Files or Blocks) and Indexes as last because indexes content will be adjusted based on recompressed blocks + var files = remotefiles.Where(a => a.FileType == RemoteVolumeType.Files).ToArray(); + var blocks = remotefiles.Where(a => a.FileType == RemoteVolumeType.Blocks).ToArray(); + var indexes = remotefiles.Where(a => a.FileType == RemoteVolumeType.Index).ToArray(); + + remotefiles = files.Concat(blocks).ToArray().Concat(indexes).ToArray(); + + foreach (var remoteFile in remotefiles) { try { - Console.Write("{0}: {1}", i, entry.File.Name); + Console.Write("{0}: {1}", ++i, remoteFile.File.Name); - var local = Path.Combine(targetfolder, entry.File.Name); - if (entry.EncryptionModule != null) + var localFileSource = Path.Combine(targetfolder, remoteFile.File.Name); + string localFileTarget; + string localFileSourceEncryption = ""; + + if (remoteFile.EncryptionModule != null) { if (string.IsNullOrWhiteSpace(m_Options.Passphrase)) { @@ -105,116 +116,142 @@ namespace Duplicati.CommandLine.RecoveryTool continue; } - local = local.Substring(0, local.Length - entry.EncryptionModule.Length - 1); + using (var m = Library.DynamicLoader.EncryptionLoader.GetModule(remoteFile.EncryptionModule, m_Options.Passphrase, options)) + localFileSourceEncryption = m.FilenameExtension; + + localFileSource = localFileSource.Substring(0, localFileSource.Length - localFileSourceEncryption.Length - 1); } - if (entry.CompressionModule == target_compr_module) - { - Console.WriteLine(" - compression types are same"); - continue; - } - - string localNew; - - if (entry.CompressionModule != null) - { - localNew = local.Substring(0, local.Length - entry.CompressionModule.Length - 1) + "." + target_compr_module; - - if (File.Exists(localNew)) - { - Console.WriteLine(" - target file already exist"); - continue; - } - } + if (remoteFile.CompressionModule != null) + localFileTarget = localFileSource.Substring(0, localFileSource.Length - remoteFile.CompressionModule.Length - 1) + "." + target_compr_module; else { Console.WriteLine(" - cannot detect compression type"); continue; + } + + if ((!reencrypt && File.Exists(localFileTarget)) || (reencrypt && File.Exists(localFileTarget + "." + localFileSourceEncryption))) + { + Console.WriteLine(" - target file already exist"); + continue; } - if (File.Exists(local)) - File.Delete(local); + if (File.Exists(localFileSource)) + File.Delete(localFileSource); - Console.Write(" - downloading ({0})...", Library.Utility.Utility.FormatSizeString(entry.File.Size)); + Console.Write(" - downloading ({0})...", Library.Utility.Utility.FormatSizeString(remoteFile.File.Size)); DateTime originLastWriteTime; FileInfo destinationFileInfo; - using (var tf = new Library.Utility.TempFile()) + using (var tf = new TempFile()) { - backend.Get(entry.File.Name, tf); + backend.Get(remoteFile.File.Name, tf); originLastWriteTime = new FileInfo(tf).LastWriteTime; downloaded++; - if (entry.EncryptionModule != null) + if (remoteFile.EncryptionModule != null) { Console.Write(" decrypting ..."); - using (var m = Library.DynamicLoader.EncryptionLoader.GetModule(entry.EncryptionModule, m_Options.Passphrase, options)) - using (var tf2 = new Library.Utility.TempFile()) + using (var m = Library.DynamicLoader.EncryptionLoader.GetModule(remoteFile.EncryptionModule, m_Options.Passphrase, options)) + using (var tf2 = new TempFile()) { m.Decrypt(tf, tf2); - File.Copy(tf2, local); + File.Copy(tf2, localFileSource); File.Delete(tf2); } } else - File.Copy(tf, local); + File.Copy(tf, localFileSource); File.Delete(tf); - destinationFileInfo = new FileInfo(local); + destinationFileInfo = new FileInfo(localFileSource); destinationFileInfo.LastWriteTime = originLastWriteTime; } - if (entry.CompressionModule != null) + if (remoteFile.CompressionModule != null) { Console.Write(" recompressing ..."); - using (var cmOld = Library.DynamicLoader.CompressionLoader.GetModule(entry.CompressionModule, local, options)) - using (var cmNew = Library.DynamicLoader.CompressionLoader.GetModule(target_compr_module, localNew, options)) + //Recompressing from eg. zip to zip + if (localFileSource == localFileTarget) + { + File.Move(localFileSource, localFileSource + ".same"); + localFileSource = localFileSource + ".same"; + } + + using (var cmOld = Library.DynamicLoader.CompressionLoader.GetModule(remoteFile.CompressionModule, localFileSource, options)) + using (var cmNew = Library.DynamicLoader.CompressionLoader.GetModule(target_compr_module, localFileTarget, options)) foreach (var cmfile in cmOld.ListFiles("")) { string cmfileNew = cmfile; - - if (entry.FileType == RemoteVolumeType.Index) + var cmFileVolume = VolumeBase.ParseFilename(cmfileNew); + + if (remoteFile.FileType == RemoteVolumeType.Index && cmFileVolume != null && cmFileVolume.FileType == RemoteVolumeType.Blocks) { - var cmFileVolume = Library.Main.Volumes.VolumeBase.ParseFilename(cmfileNew); - if (cmFileVolume != null) - { - cmfileNew = cmfileNew.Replace("." + cmFileVolume.CompressionModule, "." + target_compr_module); - if(!reencrypt) - cmfileNew = cmfileNew.Replace("." + cmFileVolume.EncryptionModule, ""); - } - } + // Correct inner filename extension to target compression type + cmfileNew = cmfileNew.Replace("." + cmFileVolume.CompressionModule, "." + target_compr_module); + if (!reencrypt) + cmfileNew = cmfileNew.Replace("." + cmFileVolume.EncryptionModule, ""); - using (var sourceStream = cmOld.OpenRead(cmfile)) - using (var cs = cmNew.CreateFile(cmfileNew, Duplicati.Library.Interface.CompressionHint.Compressible, cmOld.GetLastWriteTime(cmfile))) + //Because compression changes blocks file sizes - needs to be updated + string textJSON; + using (var sourceStream = cmOld.OpenRead(cmfile)) + using (var sourceStreamReader = new StreamReader(sourceStream)) + { + textJSON = sourceStreamReader.ReadToEnd(); + JToken token = JObject.Parse(textJSON); + var fileInfoBlocks = new FileInfo(Path.Combine(targetfolder, cmfileNew.Replace("vol/", ""))); + var filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_Options.FileHashAlgorithm); + + using (var fileStream = fileInfoBlocks.Open(FileMode.Open)) + { + fileStream.Position = 0; + token["volumehash"] = Convert.ToBase64String(filehasher.ComputeHash(fileStream)); + fileStream.Close(); + } + + token["volumesize"] = fileInfoBlocks.Length; + textJSON = token.ToString(); + } + + using (var sourceStream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(textJSON))) + using (var cs = cmNew.CreateFile(cmfileNew, Library.Interface.CompressionHint.Compressible, cmOld.GetLastWriteTime(cmfile))) Library.Utility.Utility.CopyStream(sourceStream, cs); + } + else + { + using (var sourceStream = cmOld.OpenRead(cmfile)) + using (var cs = cmNew.CreateFile(cmfileNew, Library.Interface.CompressionHint.Compressible, cmOld.GetLastWriteTime(cmfile))) + Library.Utility.Utility.CopyStream(sourceStream, cs); + } } - File.Delete(local); - destinationFileInfo = new FileInfo(localNew); + File.Delete(localFileSource); + destinationFileInfo = new FileInfo(localFileTarget); destinationFileInfo.LastWriteTime = originLastWriteTime; } - if (reencrypt && entry.EncryptionModule != null) + if (reencrypt && remoteFile.EncryptionModule != null) { Console.Write(" reencrypting ..."); - using (var m = Library.DynamicLoader.EncryptionLoader.GetModule(entry.EncryptionModule, m_Options.Passphrase, options)) + using (var m = Library.DynamicLoader.EncryptionLoader.GetModule(remoteFile.EncryptionModule, m_Options.Passphrase, options)) { - m.Encrypt(localNew, localNew + "." + m.FilenameExtension); - File.Delete(localNew); - localNew = localNew + "." + m.FilenameExtension; + m.Encrypt(localFileTarget, localFileTarget + "." + localFileSourceEncryption); + File.Delete(localFileTarget); + localFileTarget = localFileTarget + "." + localFileSourceEncryption; } - destinationFileInfo = new FileInfo(localNew); + destinationFileInfo = new FileInfo(localFileTarget); destinationFileInfo.LastWriteTime = originLastWriteTime; } if (reupload) { - backend.Put((new FileInfo(localNew)).Name, localNew); - backend.Delete(entry.File.Name); - File.Delete(localNew); + Console.Write(" reuploading ..."); + backend.Put((new FileInfo(localFileTarget)).Name, localFileTarget); + backend.Delete(remoteFile.File.Name); + File.Delete(localFileTarget); } Console.WriteLine(" done!"); @@ -225,8 +262,6 @@ namespace Duplicati.CommandLine.RecoveryTool Console.WriteLine(" error: {0}", ex.ToString()); errors++; } - - i++; } if (reupload) diff --git a/Duplicati/CommandLine/RecoveryTool/packages.config b/Duplicati/CommandLine/RecoveryTool/packages.config new file mode 100644 index 000000000..e1fae9c6a --- /dev/null +++ b/Duplicati/CommandLine/RecoveryTool/packages.config @@ -0,0 +1,4 @@ + + + + \ No newline at end of file From 7127970fd559964474a523d92ebaa8cb4db2898c Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 21 Aug 2016 21:12:48 +0200 Subject: [PATCH 18/36] better progress reporting --- Duplicati/CommandLine/RecoveryTool/Recompress.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Duplicati/CommandLine/RecoveryTool/Recompress.cs b/Duplicati/CommandLine/RecoveryTool/Recompress.cs index 7ae4b867a..e8865341c 100644 --- a/Duplicati/CommandLine/RecoveryTool/Recompress.cs +++ b/Duplicati/CommandLine/RecoveryTool/Recompress.cs @@ -49,7 +49,7 @@ namespace Duplicati.CommandLine.RecoveryTool var rawlist = backend.List(); - Console.WriteLine("Found {0} files", rawlist.Count); + Console.WriteLine("Found {0} files at remote storage", rawlist.Count); var i = 0; var downloaded = 0; @@ -97,11 +97,13 @@ namespace Duplicati.CommandLine.RecoveryTool remotefiles = files.Concat(blocks).ToArray().Concat(indexes).ToArray(); + Console.WriteLine("Found {0} files which belongs to backup with prefix {1}", remotefiles.Count(), m_Options.Prefix); + foreach (var remoteFile in remotefiles) { try { - Console.Write("{0}: {1}", ++i, remoteFile.File.Name); + Console.Write("{0}/{1}: {2}", ++i, remotefiles.Count(), remoteFile.File.Name); var localFileSource = Path.Combine(targetfolder, remoteFile.File.Name); string localFileTarget; From 5d57104a267a434e5e35d6e71a65ba418e9bfeff Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 27 Aug 2016 13:43:44 +0200 Subject: [PATCH 19/36] Changed the BackBlaze API endpoint as mentioned in #1889. This fixes #1889 --- Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs b/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs index 327047f0e..9b4a98817 100644 --- a/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs +++ b/Duplicati/Library/Backend/Backblaze/B2AuthHelper.cs @@ -27,7 +27,7 @@ namespace Duplicati.Library.Backend.Backblaze private readonly string m_credentials; private AuthResponse m_config; private DateTime m_configExpires; - private const string AUTH_URL = "https://api.backblaze.com/b2api/v1/b2_authorize_account"; + private const string AUTH_URL = "https://api.backblazeb2.com/b2api/v1/b2_authorize_account"; public B2AuthHelper(string userid, string password) : base() From 541b1f6ab7ed9c5784a37afe4801a6ec880e7acd Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 27 Aug 2016 15:10:09 +0200 Subject: [PATCH 20/36] Added a cleaning step to prevent old library versions from being detected and used in the build. --- build-release.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-release.sh b/build-release.sh index e38b1f8e6..6530d0dae 100644 --- a/build-release.sh +++ b/build-release.sh @@ -98,6 +98,8 @@ ${NUGET} restore Duplicati.sln ${XBUILD} /p:Configuration=Debug "BuildTools/AutoUpdateBuilder/AutoUpdateBuilder.sln" +${XBUILD} /p:Configuration=Release /target:Clean Duplicati.sln +find Duplicati -type d -name Release -exec rm -rf {} \; ${XBUILD} /p:Configuration=Release Duplicati.sln BUILD_STATUS=$? From 4d4661ce165838c8503eb5c5ba48d8d3f11fb93a Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 27 Aug 2016 15:10:59 +0200 Subject: [PATCH 21/36] Manually adding AlphaVSS files as they are no longer being picked up automatically, even though they are NuGet references. --- build-release.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build-release.sh b/build-release.sh index 6530d0dae..9943619f0 100644 --- a/build-release.sh +++ b/build-release.sh @@ -122,6 +122,12 @@ mkdir "${UPDATE_TARGET}" cp -R Duplicati/GUI/Duplicati.GUI.TrayIcon/bin/Release/* "${UPDATE_SOURCE}" cp -R Duplicati/Server/webroot "${UPDATE_SOURCE}" +# We copy some files for alphavss manually as they are not picked up by xbuild +mkdir "${UPDATE_SOURCE}/alphavss" +for FN in "Duplicati/Library/Snapshots/bin/Release/SnapshotQuery.exe" "Duplicati/Library/Snapshots/bin/Release/AlphaShadow.exe" Duplicati/Library/Snapshots/bin/Release/AlphaVSS.*.*.dll; do + cp "${FN}" "${UPDATE_SOURCE}/alphavss/" +done + if [ -e "${UPDATE_SOURCE}/control_dir" ]; then rm -rf "${UPDATE_SOURCE}/control_dir"; fi if [ -e "${UPDATE_SOURCE}/Duplicati-server.sqlite" ]; then rm "${UPDATE_SOURCE}/Duplicati-server.sqlite"; fi if [ -e "${UPDATE_SOURCE}/Duplicati.debug.log" ]; then rm "${UPDATE_SOURCE}/Duplicati.debug.log"; fi From 2b2d843063821dca86958826eda03c2d694cf118 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 27 Aug 2016 15:12:12 +0200 Subject: [PATCH 22/36] Removed some files that are no longer used --- .../Duplicati.Library.SharpExpect.csproj | 47 ------ .../SharpExpect/Properties/AssemblyInfo.cs | 54 ------- .../Library/SharpExpect/SharpExpectProcess.cs | 152 ------------------ 3 files changed, 253 deletions(-) delete mode 100644 Duplicati/Library/SharpExpect/Duplicati.Library.SharpExpect.csproj delete mode 100644 Duplicati/Library/SharpExpect/Properties/AssemblyInfo.cs delete mode 100644 Duplicati/Library/SharpExpect/SharpExpectProcess.cs diff --git a/Duplicati/Library/SharpExpect/Duplicati.Library.SharpExpect.csproj b/Duplicati/Library/SharpExpect/Duplicati.Library.SharpExpect.csproj deleted file mode 100644 index 0fc1090a5..000000000 --- a/Duplicati/Library/SharpExpect/Duplicati.Library.SharpExpect.csproj +++ /dev/null @@ -1,47 +0,0 @@ - - - Debug - AnyCPU - 8.0.50727 - 2.0 - {3526D1E7-64D1-4659-918C-5FC18FBD0905} - Library - Properties - Duplicati.Library.SharpExpect - Duplicati.Library.SharpExpect - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - - - - - - - - - - \ No newline at end of file diff --git a/Duplicati/Library/SharpExpect/Properties/AssemblyInfo.cs b/Duplicati/Library/SharpExpect/Properties/AssemblyInfo.cs deleted file mode 100644 index acad15f10..000000000 --- a/Duplicati/Library/SharpExpect/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,54 +0,0 @@ -#region Disclaimer / License -// Copyright (C) 2015, The Duplicati Team -// http://www.duplicati.com, info@duplicati.com -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// -#endregion -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("Duplicati.Library.SharpExpect")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("GEOGRAF A/S")] -[assembly: AssemblyProduct("Duplicati.Library.SharpExpect")] -[assembly: AssemblyCopyright("Copyright © GEOGRAF A/S 2009")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("786d822a-f3fa-402a-a0dc-0e30f88937ea")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Revision and Build Numbers -// by using the '*' as shown below: -[assembly: AssemblyVersion("2.0.0.7")] -[assembly: AssemblyFileVersion("2.0.0.7")] diff --git a/Duplicati/Library/SharpExpect/SharpExpectProcess.cs b/Duplicati/Library/SharpExpect/SharpExpectProcess.cs deleted file mode 100644 index 240339cf0..000000000 --- a/Duplicati/Library/SharpExpect/SharpExpectProcess.cs +++ /dev/null @@ -1,152 +0,0 @@ -#region Disclaimer / License -// Copyright (C) 2015, The Duplicati Team -// http://www.duplicati.com, info@duplicati.com -// -// This library is free software; you can redistribute it and/or -// modify it under the terms of the GNU Lesser General Public -// License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// -// This library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public -// License along with this library; if not, write to the Free Software -// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// -#endregion -using System; -using System.Collections.Generic; -using System.Text; - -namespace Duplicati.Library.SharpExpect -{ - public class SharpExpectProcess - { - private System.Diagnostics.Process m_process; - private object m_lock = new object(); - private System.Threading.Thread m_stdOutReader; - private System.Threading.Thread m_stdErrReader; - private Queue m_stdOut = new Queue(); - private Queue m_stdErr = new Queue(); - private System.Threading.AutoResetEvent m_event = new System.Threading.AutoResetEvent(false); - - public SharpExpectProcess(System.Diagnostics.Process process) - { - m_process = process; - - if (m_process.StartInfo.RedirectStandardError) - { - m_stdErrReader = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(StreamReader)); - m_stdErrReader.Start(m_process.StandardError); - } - - if (m_process.StartInfo.RedirectStandardOutput) - { - m_stdOutReader = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(StreamReader)); - m_stdOutReader.Start(m_process.StandardOutput); - } - } - - public string GetNextOutputLine(int millisecondsTimeout) - { - //Is output buffered? - lock (m_lock) - { - if (m_stdOut.Count > 0) - return m_stdOut.Dequeue(); - - if (m_stdErr.Count > 0) - return m_stdErr.Dequeue(); - } - - if (millisecondsTimeout > 0) - { - m_event.WaitOne(millisecondsTimeout); - return GetNextOutputLine(0); - } - else - return null; - } - - public int Expect(int millisecondsTimeout, params string[] possibilities) - { - List> lst = new List>(); - for (int i = 0; i < possibilities.Length; i++) - if (!string.IsNullOrEmpty(possibilities[i])) - lst.Add(new KeyValuePair(new System.Text.RegularExpressions.Regex(possibilities[i]), i)); - - return Expect(millisecondsTimeout, lst); - } - - public T Expect(int millisecondsTimeout, List> possibilities) - { - List> lst = new List>(); - for (int i = 0; i < possibilities.Count; i++) - if (!string.IsNullOrEmpty(possibilities[i].Key)) - lst.Add(new KeyValuePair(new System.Text.RegularExpressions.Regex(possibilities[i].Key), possibilities[i].Value)); - - return Expect(millisecondsTimeout, possibilities); - } - - public T Expect(int millisecondsTimeout, List> possibilities) - { - if (possibilities == null || possibilities.Count == 0) - return default(T); - - string line = GetNextOutputLine(millisecondsTimeout); - if (line == null) - return default(T); - - foreach (KeyValuePair expr in possibilities) - if (expr.Key.Match(line).Success) - return expr.Value; - - return default(T); - } - - public void Sendline(string line) - { - m_process.StandardInput.WriteLine(line); - } - - private void StreamReader(object input) - { - char[] buf = new char[1024]; - System.IO.StreamReader sr = (System.IO.StreamReader)input; - Queue queue = sr == m_process.StandardError ? m_stdErr : m_stdOut; - - while (!m_process.HasExited) - { - int r = sr.Read(buf, 0, buf.Length); - if (r > 0) - { - lock (m_lock) - { - queue.Enqueue(new string(buf, 0, r)); - m_event.Set(); - } - } - } - } - - public System.Diagnostics.Process Process { get { return m_process; } } - - - public static SharpExpectProcess Spawn(string FileName, string Arguments) - { - System.Diagnostics.ProcessStartInfo pi = new System.Diagnostics.ProcessStartInfo(FileName, Arguments); - pi.RedirectStandardError = true; - pi.RedirectStandardOutput = true; - pi.RedirectStandardInput = true; - return Spawn(pi); - } - - public static SharpExpectProcess Spawn(System.Diagnostics.ProcessStartInfo startInfo) - { - return new SharpExpectProcess(System.Diagnostics.Process.Start(startInfo)); - } - } -} From bf0bc4e889792629215e3c8c36292f239281ac71 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sat, 27 Aug 2016 15:19:31 +0200 Subject: [PATCH 23/36] Version bump to v2.0.1.19-2.0.1.19_canary_2016-08-27 You can download this build from: Binaries: http://updates.duplicati.com/canary/duplicati-2.0.1.19_canary_2016-08-27.zip Signature file: http://updates.duplicati.com/canary/duplicati-2.0.1.19_canary_2016-08-27.zip.sig ASCII signature file: http://updates.duplicati.com/canary/duplicati-2.0.1.19_canary_2016-08-27.zip.sig.asc MD5: 8a773c19f2ac1f0b674bb14e6ab4d160 SHA1: 7943c32e5e30ea96659c7b0db868a1aaa843a62b SHA256: 488b1540fdd638dd1be7a3cebd58952c50a81c2421d182057636df1a45866d88 --- Updates/build_version.txt | 2 +- changelog.txt | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Updates/build_version.txt b/Updates/build_version.txt index 3c032078a..d6b24041c 100644 --- a/Updates/build_version.txt +++ b/Updates/build_version.txt @@ -1 +1 @@ -18 +19 diff --git a/changelog.txt b/changelog.txt index 9b4d630f7..4dfa2e2f1 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,9 @@ +2016-08-27 +========== +* Fixed issue with files not being listed on Windows (AlphaFS and AlphaVSS issue) +* Updated B2 API URL +* Added compact command to WebUI + 2016-08-10 ========== * OneDrive for Business now uploads files in chunks to prevent timeout, thanks @ReneHezer From b6c83f49821dfa4df9cb935f0d964537208be19a Mon Sep 17 00:00:00 2001 From: Max Date: Sun, 28 Aug 2016 18:55:14 +0200 Subject: [PATCH 24/36] prevent from including temporary files in verification file --- Duplicati/Library/Main/Operation/FilelistProcessor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 88544f4c4..3c60c6aae 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -150,7 +150,7 @@ namespace Duplicati.Library.Main.Operation public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream) { var s = new Newtonsoft.Json.JsonSerializer(); - s.Serialize(stream, db.GetRemoteVolumes().Cast().ToArray()); + s.Serialize(stream, db.GetRemoteVolumes().Where(x => x.State != RemoteVolumeState.Temporary).Cast().ToArray()); } /// From c6c83b02464aa03e38e50288bc34a8bbd620a6b5 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 30 Aug 2016 10:00:27 +0200 Subject: [PATCH 25/36] Added a HostKeyException to reliably report host key violations --- Duplicati/Library/Utility/HostKeyException.cs | 65 ++++++++++++++++ .../WebServer/RESTMethods/RemoteOperation.cs | 77 +++++++++++-------- 2 files changed, 110 insertions(+), 32 deletions(-) create mode 100644 Duplicati/Library/Utility/HostKeyException.cs diff --git a/Duplicati/Library/Utility/HostKeyException.cs b/Duplicati/Library/Utility/HostKeyException.cs new file mode 100644 index 000000000..9287570b7 --- /dev/null +++ b/Duplicati/Library/Utility/HostKeyException.cs @@ -0,0 +1,65 @@ +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +namespace Duplicati.Library.Utility +{ + /// + /// Class for reporting host-key violations + /// + [Serializable] + public class HostKeyException : Exception + { + /// + /// The key reported by the host, which is not accepted + /// + /// The reported host key. + public string ReportedHostKey { get; set; } + /// + /// The key which was expected + /// + /// The acceptable host key. + public string AcceptedHostKey { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// The error message to report. + /// The reported host key. + /// The acceptable host key. + /// The inner exception. + public HostKeyException(string message, string reportedkey, string acceptedkey, Exception innerexception) + : base(message, innerexception) + { + ReportedHostKey = reportedkey; + AcceptedHostKey = acceptedkey; + } + + /// + /// Initializes a new instance of the class. + /// + /// The error message to report. + /// The reported host key. + /// The acceptable host key. + public HostKeyException(string message, string reportedkey, string acceptedkey) + : base(message) + { + ReportedHostKey = reportedkey; + AcceptedHostKey = acceptedkey; + } + } +} + diff --git a/Duplicati/Server/WebServer/RESTMethods/RemoteOperation.cs b/Duplicati/Server/WebServer/RESTMethods/RemoteOperation.cs index 50abe67b4..97da907f8 100644 --- a/Duplicati/Server/WebServer/RESTMethods/RemoteOperation.cs +++ b/Duplicati/Server/WebServer/RESTMethods/RemoteOperation.cs @@ -70,40 +70,53 @@ namespace Duplicati.Server.WebServer.RESTMethods where n is Library.Interface.IConnectionModule select n).ToArray(); - try - { - var uri = new Library.Utility.Uri(url); - var qp = uri.QueryParameters; + try + { + var uri = new Library.Utility.Uri(url); + var qp = uri.QueryParameters; - var opts = new Dictionary(); - foreach(var k in qp.Keys.Cast()) - opts[k] = qp[k]; + var opts = new Dictionary(); + foreach (var k in qp.Keys.Cast()) + opts[k] = qp[k]; - foreach(var n in modules) - n.Configure(opts); - - using(var b = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(url, new Dictionary())) - b.Test(); - - info.OutputOK(); - } - catch (Duplicati.Library.Interface.FolderMissingException) - { - info.ReportServerError("missing-folder"); - } - catch (Duplicati.Library.Utility.SslCertificateValidator.InvalidCertificateException icex) - { - if (string.IsNullOrWhiteSpace(icex.Certificate)) - info.ReportServerError(icex.Message); - else - info.ReportServerError("incorrect-cert:" + icex.Certificate); - } - finally - { - foreach(var n in modules) - if (n is IDisposable) - ((IDisposable)n).Dispose(); - } + foreach (var n in modules) + n.Configure(opts); + + using (var b = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(url, new Dictionary())) + b.Test(); + + info.OutputOK(); + } + catch (Duplicati.Library.Interface.FolderMissingException) + { + info.ReportServerError("missing-folder"); + } + catch (Duplicati.Library.Utility.SslCertificateValidator.InvalidCertificateException icex) + { + if (string.IsNullOrWhiteSpace(icex.Certificate)) + info.ReportServerError(icex.Message); + else + info.ReportServerError("incorrect-cert:" + icex.Certificate); + } + catch (Duplicati.Library.Utility.HostKeyException hex) + { + if (string.IsNullOrWhiteSpace(hex.ReportedHostKey)) + info.ReportServerError(hex.Message); + else + { + info.ReportServerError(string.Format( + @"incorrect-host-key:""{0}"", accepted-host-key:""{1}""", + hex.ReportedHostKey, + hex.AcceptedHostKey + )); + } + } + finally + { + foreach (var n in modules) + if (n is IDisposable) + ((IDisposable)n).Dispose(); + } } public void GET(string key, RequestInfo info) From f3572892beb7f9c0fc6f8765998786ba5cf20b6f Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 30 Aug 2016 10:00:48 +0200 Subject: [PATCH 26/36] Changed SSH backend to report invalid host keys via the HostKeyException --- .../Library/Backend/SSHv2/SSHv2Backend.cs | 8 +- Duplicati/Library/Backend/SSHv2/Strings.cs | 76 +++++++++---------- .../Utility/Duplicati.Library.Utility.csproj | 1 + 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 4a7cd7543..fb7ab7d5a 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -32,7 +32,7 @@ namespace Duplicati.Library.Backend public const string SSH_KEYFILE_OPTION = "ssh-keyfile"; public const string SSH_KEYFILE_INLINE = "ssh-key"; public const string SSH_FINGERPRINT_OPTION = "ssh-fingerprint"; - public const string SSH_FINGERPRINT_ACCEPT_ANY_OPTION = "shh-accept-any-fingerprints"; + public const string SSH_FINGERPRINT_ACCEPT_ANY_OPTION = "ssh-accept-any-fingerprints"; public const string KEYFILE_URI = "sshkey://"; Dictionary m_options; @@ -255,10 +255,10 @@ namespace Duplicati.Library.Backend string hostFingerprint = e.HostKeyName + " " + e.KeyLength.ToString() + " " + BitConverter.ToString(e.FingerPrint).Replace('-', ':'); if (string.IsNullOrEmpty(m_fingerprint)) - throw new Exception(Strings.SSHv2Backend.FingerprintNotSpecifiedManagedError(hostFingerprint.ToLower())); + throw new Library.Utility.HostKeyException(Strings.SSHv2Backend.FingerprintNotSpecifiedManagedError(hostFingerprint.ToLower(), SSH_FINGERPRINT_OPTION, SSH_FINGERPRINT_ACCEPT_ANY_OPTION), hostFingerprint, m_fingerprint); if (hostFingerprint.ToLower() != m_fingerprint.ToLower()) - throw new Exception(Strings.SSHv2Backend.FingerprintNotMatchManagedError(hostFingerprint.ToLower())); + throw new Library.Utility.HostKeyException(Strings.SSHv2Backend.FingerprintNotMatchManagedError(hostFingerprint.ToLower()), hostFingerprint, m_fingerprint); else e.CanTrust = true; }; @@ -293,7 +293,7 @@ namespace Duplicati.Library.Backend public List List() { - List files = new List(); + var files = new List(); string path = "."; diff --git a/Duplicati/Library/Backend/SSHv2/Strings.cs b/Duplicati/Library/Backend/SSHv2/Strings.cs index 39da27636..5670ca444 100644 --- a/Duplicati/Library/Backend/SSHv2/Strings.cs +++ b/Duplicati/Library/Backend/SSHv2/Strings.cs @@ -1,38 +1,38 @@ -using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.Strings { - internal static class KeyGenerator { - public static string Description { get { return LC.L(@"Module for generating SSH private/public keys"); } } - public static string DisplayName { get { return LC.L(@"SSH Key Generator"); } } - public static string KeyUsernameShort { get { return LC.L(@"Public key username"); } } - public static string KeyUsernameLong { get { return LC.L(@"A username to append to the public key"); } } - public static string KeyTypeShort { get { return LC.L(@"The key type"); } } - public static string KeyTypeLong { get { return LC.L(@"Determines the type of key to generate"); } } - public static string KeyLenShort { get { return LC.L(@"The key length"); } } - public static string KeyLenLong { get { return LC.L(@"The length of the key in bits"); } } - } - internal static class KeyUploader { - public static string Description { get { return LC.L(@"Module for uploading SSH public keys"); } } - public static string DisplayName { get { return LC.L(@"SSH Key Uploader"); } } - public static string UrlShort { get { return LC.L(@"The SSH connection URL"); } } - public static string UrlLong { get { return LC.L(@"The SSH connection URL used to establish the connection"); } } - public static string PubkeyShort { get { return LC.L(@"The SSH public key to append"); } } - public static string PubkeyLong { get { return LC.L(@"The SSH public key must be a valid SSH string, which is appended to the .ssh/authorized_keys file"); } } - } - internal static class SSHv2Backend { - public static string Description { get { return LC.L(@"This backend can read and write data to an SSH based backend, using SFTP. Allowed formats are ""ssh://hostname/folder"" or ""ssh://username:password@hostname/folder""."); } } - public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } } - public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supplies the password used to connect to the server"); } } - public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } } - public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supplies the username used to connect to the server"); } } - public static string DescriptionFingerprintLong { get { return LC.L(@"The server fingerprint used for validation of server identity. Format is eg. ""ssh - rsa 4096 11:22:33:44:55:66:77:88:99:00:11:22:33:44:55:66""."); } } - public static string DescriptionFingerprintShort { get { return LC.L(@"Supplies server fingerprint used for validation of server identity"); } } - public static string DescriptionSshkeyfileLong { get { return LC.L(@"Points to a valid OpenSSH keyfile. If the file is encrypted, the password supplied is used to decrypt the keyfile. If this option is supplied, the password is not used to authenticate. This option only works when using the managed SSH client."); } } - public static string DescriptionSshkeyfileShort { get { return LC.L(@"Uses a SSH private key to authenticate"); } } - public static string DescriptionSshkeyLong(string urlprefix) { return LC.L(@"An url-encoded SSH private key. The private key must be prefixed with {0}. If the file is encrypted, the password supplied is used to decrypt the keyfile. If this option is supplied, the password is not used to authenticate. This option only works when using the managed SSH client.", urlprefix); } - public static string DescriptionSshkeyShort { get { return LC.L(@"Uses a SSH private key to authenticate"); } } - public static string DisplayName { get { return LC.L(@"SFTP (SSH)"); } } - public static string FolderNotFoundManagedError(string foldername, string message) { return LC.L(@"Unable to set folder to {0}, error message: {1}", foldername, message); } - public static string FingerprintNotMatchManagedError(string fingerprint) { return LC.L(@"Validation of server fingerprint failed. Server returned fingerprint ""{0}"". Cause of this message is either not correct configuration or Man-in-the-middle attack!", fingerprint); } - public static string FingerprintNotSpecifiedManagedError(string fingerprint) { return LC.L(@"Please add --ssh-fingerprint=""{0}"" to trust this host. Or --shh-accept-any-fingerprints (NOT SECURE) only for test reasons!", fingerprint); } - } -} +using Duplicati.Library.Localization.Short; +namespace Duplicati.Library.Backend.Strings { + internal static class KeyGenerator { + public static string Description { get { return LC.L(@"Module for generating SSH private/public keys"); } } + public static string DisplayName { get { return LC.L(@"SSH Key Generator"); } } + public static string KeyUsernameShort { get { return LC.L(@"Public key username"); } } + public static string KeyUsernameLong { get { return LC.L(@"A username to append to the public key"); } } + public static string KeyTypeShort { get { return LC.L(@"The key type"); } } + public static string KeyTypeLong { get { return LC.L(@"Determines the type of key to generate"); } } + public static string KeyLenShort { get { return LC.L(@"The key length"); } } + public static string KeyLenLong { get { return LC.L(@"The length of the key in bits"); } } + } + internal static class KeyUploader { + public static string Description { get { return LC.L(@"Module for uploading SSH public keys"); } } + public static string DisplayName { get { return LC.L(@"SSH Key Uploader"); } } + public static string UrlShort { get { return LC.L(@"The SSH connection URL"); } } + public static string UrlLong { get { return LC.L(@"The SSH connection URL used to establish the connection"); } } + public static string PubkeyShort { get { return LC.L(@"The SSH public key to append"); } } + public static string PubkeyLong { get { return LC.L(@"The SSH public key must be a valid SSH string, which is appended to the .ssh/authorized_keys file"); } } + } + internal static class SSHv2Backend { + public static string Description { get { return LC.L(@"This backend can read and write data to an SSH based backend, using SFTP. Allowed formats are ""ssh://hostname/folder"" or ""ssh://username:password@hostname/folder""."); } } + public static string DescriptionAuthPasswordLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } } + public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supplies the password used to connect to the server"); } } + public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } } + public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supplies the username used to connect to the server"); } } + public static string DescriptionFingerprintLong { get { return LC.L(@"The server fingerprint used for validation of server identity. Format is eg. ""rsa 4096 11:22:33:44:55:66:77:88:99:00:11:22:33:44:55:66""."); } } + public static string DescriptionFingerprintShort { get { return LC.L(@"Supplies server fingerprint used for validation of server identity"); } } + public static string DescriptionSshkeyfileLong { get { return LC.L(@"Points to a valid OpenSSH keyfile. If the file is encrypted, the password supplied is used to decrypt the keyfile. If this option is supplied, the password is not used to authenticate. This option only works when using the managed SSH client."); } } + public static string DescriptionSshkeyfileShort { get { return LC.L(@"Uses a SSH private key to authenticate"); } } + public static string DescriptionSshkeyLong(string urlprefix) { return LC.L(@"An url-encoded SSH private key. The private key must be prefixed with {0}. If the file is encrypted, the password supplied is used to decrypt the keyfile. If this option is supplied, the password is not used to authenticate. This option only works when using the managed SSH client.", urlprefix); } + public static string DescriptionSshkeyShort { get { return LC.L(@"Uses a SSH private key to authenticate"); } } + public static string DisplayName { get { return LC.L(@"SFTP (SSH)"); } } + public static string FolderNotFoundManagedError(string foldername, string message) { return LC.L(@"Unable to set folder to {0}, error message: {1}", foldername, message); } + public static string FingerprintNotMatchManagedError(string fingerprint) { return LC.L(@"Validation of server fingerprint failed. Server returned fingerprint ""{0}"". Cause of this message is either not correct configuration or Man-in-the-middle attack!", fingerprint); } + public static string FingerprintNotSpecifiedManagedError(string fingerprint, string hostkeyoption, string allkeysoptions) { return LC.L(@"Please add --{1}=""{0}"" to trust this host. Optionally you can use --{2} (NOT SECURE) for testing!", fingerprint); } + } +} diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index d5a4696bc..c3f9e9fbf 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -67,6 +67,7 @@ + From 3666d0268ab74e156ff49d90ed410809e09e2721 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 30 Aug 2016 11:36:31 +0200 Subject: [PATCH 27/36] Implemented UI check for host keys, to allow easy SSH hostkey protection. --- Duplicati/Library/Backend/SSHv2/Strings.cs | 4 +- .../ngax/scripts/directives/backupEditUri.js | 587 ++++++++++-------- 2 files changed, 320 insertions(+), 271 deletions(-) diff --git a/Duplicati/Library/Backend/SSHv2/Strings.cs b/Duplicati/Library/Backend/SSHv2/Strings.cs index 5670ca444..448af41e7 100644 --- a/Duplicati/Library/Backend/SSHv2/Strings.cs +++ b/Duplicati/Library/Backend/SSHv2/Strings.cs @@ -24,7 +24,7 @@ namespace Duplicati.Library.Backend.Strings { public static string DescriptionAuthPasswordShort { get { return LC.L(@"Supplies the password used to connect to the server"); } } public static string DescriptionAuthUsernameLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } } public static string DescriptionAuthUsernameShort { get { return LC.L(@"Supplies the username used to connect to the server"); } } - public static string DescriptionFingerprintLong { get { return LC.L(@"The server fingerprint used for validation of server identity. Format is eg. ""rsa 4096 11:22:33:44:55:66:77:88:99:00:11:22:33:44:55:66""."); } } + public static string DescriptionFingerprintLong { get { return LC.L(@"The server fingerprint used for validation of server identity. Format is eg. ""ssh-rsa 4096 11:22:33:44:55:66:77:88:99:00:11:22:33:44:55:66""."); } } public static string DescriptionFingerprintShort { get { return LC.L(@"Supplies server fingerprint used for validation of server identity"); } } public static string DescriptionSshkeyfileLong { get { return LC.L(@"Points to a valid OpenSSH keyfile. If the file is encrypted, the password supplied is used to decrypt the keyfile. If this option is supplied, the password is not used to authenticate. This option only works when using the managed SSH client."); } } public static string DescriptionSshkeyfileShort { get { return LC.L(@"Uses a SSH private key to authenticate"); } } @@ -33,6 +33,6 @@ namespace Duplicati.Library.Backend.Strings { public static string DisplayName { get { return LC.L(@"SFTP (SSH)"); } } public static string FolderNotFoundManagedError(string foldername, string message) { return LC.L(@"Unable to set folder to {0}, error message: {1}", foldername, message); } public static string FingerprintNotMatchManagedError(string fingerprint) { return LC.L(@"Validation of server fingerprint failed. Server returned fingerprint ""{0}"". Cause of this message is either not correct configuration or Man-in-the-middle attack!", fingerprint); } - public static string FingerprintNotSpecifiedManagedError(string fingerprint, string hostkeyoption, string allkeysoptions) { return LC.L(@"Please add --{1}=""{0}"" to trust this host. Optionally you can use --{2} (NOT SECURE) for testing!", fingerprint); } + public static string FingerprintNotSpecifiedManagedError(string fingerprint, string hostkeyoption, string allkeysoptions) { return LC.L(@"Please add --{1}=""{0}"" to trust this host. Optionally you can use --{2} (NOT SECURE) for testing!", fingerprint, hostkeyoption, allkeysoptions); } } } diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/backupEditUri.js b/Duplicati/Server/webroot/ngax/scripts/directives/backupEditUri.js index 5644f0e06..3798f3d51 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/backupEditUri.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/backupEditUri.js @@ -2,298 +2,347 @@ backupApp.directive('backupEditUri', function() { return { restrict: 'E', scope: { - uri: '=uri', - hide: '=hide' + uri: '=uri', + hide: '=hide' }, templateUrl: 'templates/edituri.html', controller: function($scope, AppService, AppUtils, SystemInfo, EditUriBackendConfig, DialogService, EditUriBuiltins) { - var scope = $scope; + var scope = $scope; - var builduri = function(callback) { + var builduri = function(callback) { - function validationCompleted() { - if (EditUriBackendConfig.builders[scope.Backend.Key] == null) - callback(EditUriBackendConfig.defaultbuilder(scope)); - else - callback(EditUriBackendConfig.builders[scope.Backend.Key](scope)); - }; + function validationCompleted() { + if (EditUriBackendConfig.builders[scope.Backend.Key] == null) + callback(EditUriBackendConfig.defaultbuilder(scope)); + else + callback(EditUriBackendConfig.builders[scope.Backend.Key](scope)); + }; - if (EditUriBackendConfig.validaters[scope.Backend.Key] == null) - EditUriBackendConfig.defaultvalidater(scope, validationCompleted); - else - EditUriBackendConfig.validaters[scope.Backend.Key](scope, validationCompleted); - } + if (EditUriBackendConfig.validaters[scope.Backend.Key] == null) + EditUriBackendConfig.defaultvalidater(scope, validationCompleted); + else + EditUriBackendConfig.validaters[scope.Backend.Key](scope, validationCompleted); + } - function performConnectionTest(uri) { + function performConnectionTest(uri) { - var hasTriedCreate = false; - var hasTriedCert = false; - var hasTriedMozroots = false; - var dlg = null; + var hasTriedCreate = false; + var hasTriedCert = false; + var hasTriedMozroots = false; + var hasTriedHostkey = false; + var dlg = null; - var testConnection = function() { - scope.Testing = true; - if (dlg != null) - dlg.dismiss(); + var testConnection = function() { + scope.Testing = true; + if (dlg != null) + dlg.dismiss(); - dlg = DialogService.dialog('Testing ...', 'Testing connection ...', [], null, function() { - AppService.post('/remoteoperation/test', uri).then(function() { - scope.Testing = false; - dlg.dismiss(); + dlg = DialogService.dialog('Testing ...', 'Testing connection ...', [], null, function() { + AppService.post('/remoteoperation/test', uri).then(function() { + scope.Testing = false; + dlg.dismiss(); - if (EditUriBackendConfig.testers[scope.Backend.Key] != null) - EditUriBackendConfig.testers[scope.Backend.Key](scope, function() { - DialogService.dialog('Success', 'Connection worked!'); - }); - else - DialogService.dialog('Success', 'Connection worked!'); + if (EditUriBackendConfig.testers[scope.Backend.Key] != null) + EditUriBackendConfig.testers[scope.Backend.Key](scope, function() { + DialogService.dialog('Success', 'Connection worked!'); + }); + else + DialogService.dialog('Success', 'Connection worked!'); - }, handleError); - }); + }, handleError); + }); - }; + }; - var createFolder = function() { - scope.Testing = true; - AppService.post('/remoteoperation/create', uri).then(testConnection, handleError); - }; + var createFolder = function() { + hasTriedCreate = true; + scope.Testing = true; + AppService.post('/remoteoperation/create', uri).then(testConnection, handleError); + }; - var appendApprovedCert = function(hash) - { - for(var n in scope.AdvancedOptions) { - if (scope.AdvancedOptions[n].indexOf('--accept-specified-ssl-hash=') == 0) - { - var certs = scope.AdvancedOptions[n].substr('--accept-specified-ssl-hash='.length).split(','); - for(var i in certs) - if (certs[i] == hash) - return; + var appendApprovedCert = function(hash) + { + hasTriedCert = true; + for(var n in scope.AdvancedOptions) { + if (scope.AdvancedOptions[n].indexOf('--accept-specified-ssl-hash=') == 0) + { + var certs = scope.AdvancedOptions[n].substr('--accept-specified-ssl-hash='.length).split(','); + for(var i in certs) + if (certs[i] == hash) + return; - scope.AdvancedOptions[n] += ',' + hash; - return; - } - } + scope.AdvancedOptions[n] += ',' + hash; + return; + } + } - scope.AdvancedOptions.push('--accept-specified-ssl-hash=' + hash); - } + scope.AdvancedOptions.push('--accept-specified-ssl-hash=' + hash); + } + + var askApproveCert = function(hash) { + DialogService.dialog('Trust server certificate?', 'The server certificate could not be validated.\nDo you want to approve the SSL certificate with the hash: ' + hash + '?', ['No', 'Yes'], function(ix) { + if (ix == 1) { + appendApprovedCert(hash); + builduri(function(res) { + uri = res; + testConnection(); + }); + } + }); + }; + + var hasCertApproved = function(hash) + { + for(var n in scope.AdvancedOptions) { + if (scope.AdvancedOptions[n].indexOf('--accept-specified-ssl-hash=') == 0) + { + var certs = scope.AdvancedOptions[n].substr('--accept-specified-ssl-hash='.length).split(','); + for(var i in certs) + if (certs[i] == hash) + return true; + + break; + } + } + + return false; + } + + var handleError = function(data) { + + scope.Testing = false; + if (dlg != null) + dlg.dismiss(); + + var message = data.statusText; + + if (!hasTriedCreate && message == 'missing-folder') + { + var folder = scope.Folder; + if ((folder || "") == "") + folder = scope.Path; + if ((folder || "") == "") + folder = ''; + + DialogService.dialog('Create folder?', 'The folder ' + folder + ' does not exist\nCreate it now?', ['No', 'Yes'], function(ix) { + if (ix == 1) + createFolder(); + }); + } + else if (!hasTriedCert && message.indexOf('incorrect-cert:') == 0) + { + var hash = message.substr('incorrect-cert:'.length); + if (hasCertApproved(hash)) { + if (data.data != null && data.data.Message != null) + message = data.data.Message; + + DialogService.dialog('Error', 'Failed to connect: ' + message); + return; + } + + if ($scope.SystemInfo.MonoVersion != null && !hasTriedMozroots) { + + hasTriedMozroots = true; + + AppService.post('/webmodule/check-mono-ssl', {'mono-ssl-config': 'List'}).then(function(data) { + if (data.data.Result.count == 0) { + if (confirm('You appear to be running Mono with no SSL certificates loaded.\nDo you want to import the list of trusted certificates from Mozilla?')) + { + scope.Testing = true; + AppService.post('/webmodule/check-mono-ssl', {'mono-ssl-config': 'Install'}).then(function(data) { + scope.Testing = false; + if (data.data.Result.count == 0) { + DialogService.dialog('Import failed', 'Import completed, but no certificates were found after the import'); + } else { + testConnection(); + } + }, function(resp) { + + scope.Testing = false; + message = resp.statusText; + if (data.data != null && data.data.Message != null) + message = data.data.Message; + + DialogService.dialog('Error', 'Failed to import: ' + message); + + }); + } + else + { + askApproveCert(hash); + } + } else { + askApproveCert(hash); + } + + }, AppUtils.connectionError); + + } + else + { + askApproveCert(hash); + } + } + else if (!hasTriedHostkey && message.indexOf('incorrect-host-key:') == 0) + { + var re = /incorrect-host-key\s*:\s*"([^"]*)"(,\s*accepted-host-key\s*:\s*"([^"]*)")?/; + var m = re.exec(message); + var key = null; + var prev = null; + if (m != null) { + key = m[1] || ''; + prev = m[3] || ''; + } + + if ((key || '').trim().length == 0 || key == prev) { + if (data.data != null && data.data.Message != null) + message = data.data.Message; + + DialogService.dialog('Error', 'Failed to connect: ' + message); + } + else + { + var message = ((prev || '').trim().length == 0) ? + ('No certificate was specified previously, please verify with the server administrator that the key is correct: ' + key + ' \n\nDo you want to approve the reported host key?') + : + ('The host key has changed, please check with the server administrator if this is correct, otherwise you could be the victim of a MAN-IN-THE-MIDDLE attack.\n\nDo you want to REPLACE your CURRENT host key "' + prev + '" with the REPORTED host key: ' + key + '?'); + + DialogService.dialog('Trust host certificate?', message, ['No', 'Yes'], function(ix) { + if (ix == 1) { + hasTriedHostkey = true; + for(var n in scope.AdvancedOptions) { + if (scope.AdvancedOptions[n].indexOf('--ssh-fingerprint=') == 0) { + scope.AdvancedOptions.splice(n, 1); + break; + } + } + + scope.AdvancedOptions.push('--ssh-fingerprint=' + key); + builduri(function(res) { + uri = res; + testConnection(); + }); + } + }); + + } + } + else + { + if (data.data != null && data.data.Message != null) + message = data.data.Message; + + DialogService.dialog('Error', 'Failed to connect: ' + message); + } + } + + testConnection(); + }; + + $scope.testConnection = function() { + builduri(performConnectionTest); + }; + + $scope.save = function() { + builduri(function(res) { + scope.uri = res; + scope.hide(); + }); + }; + + $scope.contains_value = AppUtils.contains_value; + + $scope.$watch('Backend', function() { + if (scope.Backend == null) { + $scope.TemplateUrl = null; + $scope.AdvanceOptionList = null; + return; + } + + var opts = angular.copy(scope.Backend.Options); + for(var n in opts) + opts[n].Category = scope.Backend.DisplayName; + + for(var m in SystemInfo.state.ConnectionModules) + { + var t = angular.copy(SystemInfo.state.ConnectionModules[m].Options); + for(var n in t) + t[n].Category = SystemInfo.state.ConnectionModules[m].DisplayName; + opts.push.apply(opts, t); + } + + $scope.AdvanceOptionList = opts; + scope.SupportsSSL = false; + for(var n in scope.Backend.Options) + if (scope.Backend.Options[n].Name == 'use-ssl') + scope.SupportsSSL = true; + + scope.TemplateUrl = EditUriBackendConfig.templates[scope.Backend.Key]; + if (scope.TemplateUrl == null) + scope.TemplateUrl = EditUriBackendConfig.defaulttemplate; + + if (EditUriBackendConfig.loaders[scope.Backend.Key] == null) + return; + + EditUriBackendConfig.loaders[scope.Backend.Key](scope); + }); + + var reparseuri = function() { + scope.Backend = scope.DefaultBackend; + + var parts = AppUtils.decode_uri(scope.uri); + + for(var n in scope.SystemInfo.GroupedBackendModules) { + if (scope.SystemInfo.GroupedBackendModules[n].Key == parts['backend-type']) { + scope.Backend = $scope.SystemInfo.GroupedBackendModules[n]; + break; + } + + if ((scope.SystemInfo.GroupedBackendModules[n].Key + 's') == parts['backend-type']) { + var hasssl = false; + var bk = scope.SystemInfo.GroupedBackendModules[n]; + + for(var o in bk.Options) { + if (bk.Options[o].Name == 'use-ssl') { + hasssl = true; + break; + } + } + + if (hasssl) { + scope.Backend = bk; + parts['--use-ssl'] = true; + break; + } + } + } - var askApproveCert = function(hash) { - DialogService.dialog('Trust server certificate?', 'The server certificate could not be validated.\nDo you want to approve the SSL certificate with the hash: ' + hash + '?', ['No', 'Yes'], function(ix) { - if (ix == 1) { - appendApprovedCert(hash); - testConnection(); - } - }); - }; + scope.Username = parts['--auth-username']; + scope.Password = parts['--auth-password']; + scope.UseSSL = parts['--use-ssl']; + scope.Port = parts['server-port']; + scope.Server = parts['server-name']; + scope.Path = parts['server-path']; - var hasCertApproved = function(hash) - { - for(var n in scope.AdvancedOptions) { - if (scope.AdvancedOptions[n].indexOf('--accept-specified-ssl-hash=') == 0) - { - var certs = scope.AdvancedOptions[n].substr('--accept-specified-ssl-hash='.length).split(','); - for(var i in certs) - if (certs[i] == hash) - return true; + if (scope.Backend != null && scope.uri != null && EditUriBackendConfig.parsers[scope.Backend.Key]) + EditUriBackendConfig.parsers[scope.Backend.Key](scope, parts['backend-type'], parts['server-name'], parts['server-path'], parts['server-port'], parts); - break; - } - } + delete parts['--auth-username']; + delete parts['--auth-password']; + delete parts['--use-ssl']; + scope.AdvancedOptions = AppUtils.serializeAdvancedOptionsToArray(parts); + }; - return false; - } + $scope.SystemInfo = SystemInfo.watch($scope, function() { + for(var n in scope.SystemInfo.GroupedBackendModules) + if (scope.SystemInfo.GroupedBackendModules[n].Key == EditUriBackendConfig.defaultbackend) + scope.DefaultBackend = scope.SystemInfo.GroupedBackendModules[n]; - var handleError = function(data) { - - scope.Testing = false; - if (dlg != null) - dlg.dismiss(); - - var message = data.statusText; - - if (!hasTriedCreate && message == 'missing-folder') - { - var folder = scope.Folder; - if ((folder || "") == "") - folder = scope.Path; - if ((folder || "") == "") - folder = ''; - - DialogService.dialog('Create folder?', 'The folder ' + folder + ' does not exist\nCreate it now?', ['No', 'Yes'], function(ix) { - if (ix == 1) - createFolder(); - }); - } - else if (!hasTriedCert && message.indexOf('incorrect-cert:') == 0) - { - var hash = message.substr('incorrect-cert:'.length); - if (hasCertApproved(hash)) { - if (data.data != null && data.data.Message != null) - message = data.data.Message; - - DialogService.dialog('Error', 'Failed to connect: ' + message); - return; - } - - if ($scope.SystemInfo.MonoVersion != null && !hasTriedMozroots) { - - hasTriedMozroots = true; - - AppService.post('/webmodule/check-mono-ssl', {'mono-ssl-config': 'List'}).then(function(data) { - if (data.data.Result.count == 0) { - if (confirm('You appear to be running Mono with no SSL certificates loaded.\nDo you want to import the list of trusted certificates from Mozilla?')) - { - scope.Testing = true; - AppService.post('/webmodule/check-mono-ssl', {'mono-ssl-config': 'Install'}).then(function(data) { - scope.Testing = false; - if (data.data.Result.count == 0) { - DialogService.dialog('Import failed', 'Import completed, but no certificates were found after the import'); - } else { - testConnection(); - } - }, function(resp) { - - scope.Testing = false; - message = resp.statusText; - if (data.data != null && data.data.Message != null) - message = data.data.Message; - - DialogService.dialog('Error', 'Failed to import: ' + message); - - }); - } - else - { - askApproveCert(hash); - } - } else { - askApproveCert(hash); - } - - }, AppUtils.connectionError); - - } - else - { - askApproveCert(hash); - } - } - else - { - if (data.data != null && data.data.Message != null) - message = data.data.Message; - - DialogService.dialog('Error', 'Failed to connect: ' + message); - } - } - - testConnection(); - }; - - $scope.testConnection = function() { - builduri(performConnectionTest); - }; - - $scope.save = function() { - builduri(function(res) { - scope.uri = res; - scope.hide(); - }); - }; - - $scope.contains_value = AppUtils.contains_value; - - $scope.$watch('Backend', function() { - if (scope.Backend == null) { - $scope.TemplateUrl = null; - $scope.AdvanceOptionList = null; - return; - } - - var opts = angular.copy(scope.Backend.Options); - for(var n in opts) - opts[n].Category = scope.Backend.DisplayName; - - for(var m in SystemInfo.state.ConnectionModules) - { - var t = angular.copy(SystemInfo.state.ConnectionModules[m].Options); - for(var n in t) - t[n].Category = SystemInfo.state.ConnectionModules[m].DisplayName; - opts.push.apply(opts, t); - } - - $scope.AdvanceOptionList = opts; - scope.SupportsSSL = false; - for(var n in scope.Backend.Options) - if (scope.Backend.Options[n].Name == 'use-ssl') - scope.SupportsSSL = true; - - scope.TemplateUrl = EditUriBackendConfig.templates[scope.Backend.Key]; - if (scope.TemplateUrl == null) - scope.TemplateUrl = EditUriBackendConfig.defaulttemplate; - - if (EditUriBackendConfig.loaders[scope.Backend.Key] == null) - return; - - EditUriBackendConfig.loaders[scope.Backend.Key](scope); - }); - - var reparseuri = function() { - scope.Backend = scope.DefaultBackend; - - var parts = AppUtils.decode_uri(scope.uri); - - for(var n in scope.SystemInfo.GroupedBackendModules) { - if (scope.SystemInfo.GroupedBackendModules[n].Key == parts['backend-type']) { - scope.Backend = $scope.SystemInfo.GroupedBackendModules[n]; - break; - } - - if ((scope.SystemInfo.GroupedBackendModules[n].Key + 's') == parts['backend-type']) { - var hasssl = false; - var bk = scope.SystemInfo.GroupedBackendModules[n]; - - for(var o in bk.Options) { - if (bk.Options[o].Name == 'use-ssl') { - hasssl = true; - break; - } - } - - if (hasssl) { - scope.Backend = bk; - parts['--use-ssl'] = true; - break; - } - } - } - - - scope.Username = parts['--auth-username']; - scope.Password = parts['--auth-password']; - scope.UseSSL = parts['--use-ssl']; - scope.Port = parts['server-port']; - scope.Server = parts['server-name']; - scope.Path = parts['server-path']; - - if (scope.Backend != null && scope.uri != null && EditUriBackendConfig.parsers[scope.Backend.Key]) - EditUriBackendConfig.parsers[scope.Backend.Key](scope, parts['backend-type'], parts['server-name'], parts['server-path'], parts['server-port'], parts); - - delete parts['--auth-username']; - delete parts['--auth-password']; - delete parts['--use-ssl']; - scope.AdvancedOptions = AppUtils.serializeAdvancedOptionsToArray(parts); - }; - - $scope.SystemInfo = SystemInfo.watch($scope, function() { - for(var n in scope.SystemInfo.GroupedBackendModules) - if (scope.SystemInfo.GroupedBackendModules[n].Key == EditUriBackendConfig.defaultbackend) - scope.DefaultBackend = scope.SystemInfo.GroupedBackendModules[n]; - - reparseuri(); - }); - $scope.$watch('uri', reparseuri); - } + reparseuri(); + }); + $scope.$watch('uri', reparseuri); + } }; }); \ No newline at end of file From 72e3782b0a60173339e978a8ab3066af7e828f4b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 30 Aug 2016 12:26:57 +0200 Subject: [PATCH 28/36] Added error reporting to detect errors such as the one reported in #1905. --- Duplicati/Library/Main/Operation/BackupHandler.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index b6afc2a74..e8e57240c 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -97,6 +97,12 @@ namespace Duplicati.Library.Main.Operation public bool AttributeFilter(string rootpath, string path, FileAttributes attributes) { + if ((attributes & Library.Utility.Utility.ATTRIBUTE_ERROR) == Library.Utility.Utility.ATTRIBUTE_ERROR) + { + if (m_logWriter != null) + m_logWriter.AddWarning(string.Format("Error reported while accessing file {0}", path), null); + } + try { if (m_snapshot.IsBlockDevice(path)) From f393f6b8c29df72cd2d169a71a81403139a0ec6b Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 30 Aug 2016 12:28:17 +0200 Subject: [PATCH 29/36] Fixed #1905 The AlphaVSS library now returns path from the listfiles/listfolders without the UNC prefix, so the code re-adds this to ensure the mapping from snapshot path to logical path is maintained. --- Duplicati/Library/Snapshots/WindowsSnapshot.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 4b895eb3c..08b1305ce 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -218,9 +218,10 @@ namespace Duplicati.Library.Snapshots tmp = Alphaleonis.Win32.Filesystem.Directory.GetDirectories(spath); } + volumePath = SystemIOWindows.PrefixWithUNC(volumePath); for (int i = 0; i < tmp.Length; i++) - tmp[i] = root + tmp[i].Substring(volumePath.Length); + tmp[i] = root + SystemIOWindows.PrefixWithUNC(tmp[i]).Substring(volumePath.Length); return tmp; } @@ -254,8 +255,10 @@ namespace Duplicati.Library.Snapshots tmp = Alphaleonis.Win32.Filesystem.Directory.GetFiles(spath); } + volumePath = SystemIOWindows.PrefixWithUNC(volumePath); + for (int i = 0; i < tmp.Length; i++) - tmp[i] = root + tmp[i].Substring(volumePath.Length); + tmp[i] = root + SystemIOWindows.PrefixWithUNC(tmp[i]).Substring(volumePath.Length); return tmp; } From f573037aaeaeb24a5e8097739803e70514dabf10 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 30 Aug 2016 14:48:05 +0200 Subject: [PATCH 30/36] Fixed error messages from the pre-build cleanup --- build-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-release.sh b/build-release.sh index 9943619f0..466c84cd0 100644 --- a/build-release.sh +++ b/build-release.sh @@ -99,7 +99,7 @@ ${NUGET} restore Duplicati.sln ${XBUILD} /p:Configuration=Debug "BuildTools/AutoUpdateBuilder/AutoUpdateBuilder.sln" ${XBUILD} /p:Configuration=Release /target:Clean Duplicati.sln -find Duplicati -type d -name Release -exec rm -rf {} \; +find Duplicati -type d -name Release | xargs rm -rf ${XBUILD} /p:Configuration=Release Duplicati.sln BUILD_STATUS=$? From fb259a2e84dfa4716326b40e1a55e80b10253d71 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 30 Aug 2016 15:03:10 +0200 Subject: [PATCH 31/36] Added installation of .config files during the install process --- build-release.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build-release.sh b/build-release.sh index 466c84cd0..a9086f53c 100644 --- a/build-release.sh +++ b/build-release.sh @@ -128,6 +128,10 @@ for FN in "Duplicati/Library/Snapshots/bin/Release/SnapshotQuery.exe" "Duplicati cp "${FN}" "${UPDATE_SOURCE}/alphavss/" done +# Install the assembly redirects for all Duplicati .exe files +find "${UPDATE_SOURCE}" -type f -name Duplicati.*.exe -maxdepth 1 -exec cp Installer/AssemblyRedirects.xml {}.config \; + +# Clean some unwanted build files if [ -e "${UPDATE_SOURCE}/control_dir" ]; then rm -rf "${UPDATE_SOURCE}/control_dir"; fi if [ -e "${UPDATE_SOURCE}/Duplicati-server.sqlite" ]; then rm "${UPDATE_SOURCE}/Duplicati-server.sqlite"; fi if [ -e "${UPDATE_SOURCE}/Duplicati.debug.log" ]; then rm "${UPDATE_SOURCE}/Duplicati.debug.log"; fi From cac13217a664c8846656d90a380d4db4b4a26c94 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 30 Aug 2016 15:03:27 +0200 Subject: [PATCH 32/36] Updated the assembly-redirects file --- Installer/AssemblyRedirects.xml | 40 ++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/Installer/AssemblyRedirects.xml b/Installer/AssemblyRedirects.xml index 6a85f4900..17eb462c3 100644 --- a/Installer/AssemblyRedirects.xml +++ b/Installer/AssemblyRedirects.xml @@ -1,18 +1,32 @@ - - + + - - - - + - - - - + + + + + - - - + + + + + + + + + + + + + + + + + + + \ No newline at end of file From a942c481726b619991f8f0239bcfa9a7891ee71c Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 31 Aug 2016 09:19:24 +0200 Subject: [PATCH 33/36] Fixed accessing metadata through the snapshot, if one exists. This fixes #1911 and #1693 --- Duplicati/Library/Snapshots/WindowsSnapshot.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 08b1305ce..ebfbf01b3 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -393,7 +393,7 @@ namespace Duplicati.Library.Snapshots /// The file or folder to examine public Dictionary GetMetadata(string file) { - return _ioWin.GetMetadata(file); + return _ioWin.GetMetadata(GetSnapshotPath(file)); } /// @@ -410,7 +410,7 @@ namespace Duplicati.Library.Snapshots /// Gets a unique hardlink target ID /// /// The hardlink ID - /// The file or folder to examine + /// The file or folder to examine public string HardlinkTargetID(string path) { return null; From b27aa3fe63ce74f99dffc0e3ec372c590b6f77ec Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 31 Aug 2016 10:05:27 +0200 Subject: [PATCH 34/36] Updated SharpAESCrypt to 1.3.2 targeting .Net 4.5. This fixes #1887 --- .../Library/Encryption/Duplicati.Library.Encryption.csproj | 2 +- Duplicati/Library/Encryption/packages.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Encryption/Duplicati.Library.Encryption.csproj b/Duplicati/Library/Encryption/Duplicati.Library.Encryption.csproj index 85068a3d3..022e12ef5 100644 --- a/Duplicati/Library/Encryption/Duplicati.Library.Encryption.csproj +++ b/Duplicati/Library/Encryption/Duplicati.Library.Encryption.csproj @@ -38,7 +38,7 @@ - ..\..\..\packages\SharpAESCrypt.exe.1.3\lib\net20\SharpAESCrypt.exe + ..\..\..\packages\SharpAESCrypt.exe.1.3.2\lib\net45\SharpAESCrypt.exe diff --git a/Duplicati/Library/Encryption/packages.config b/Duplicati/Library/Encryption/packages.config index 2f8a57881..04c131435 100644 --- a/Duplicati/Library/Encryption/packages.config +++ b/Duplicati/Library/Encryption/packages.config @@ -1,4 +1,4 @@  - + \ No newline at end of file From 5a413b3e9ba12904ddba467efa733296aa5ba2fc Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 31 Aug 2016 11:12:04 +0200 Subject: [PATCH 35/36] Added support for terrabyte sizes in Sizeparser. This fixes #1906 --- Duplicati/Library/Utility/Sizeparser.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Utility/Sizeparser.cs b/Duplicati/Library/Utility/Sizeparser.cs index 0ffc2c20b..74c551e8d 100644 --- a/Duplicati/Library/Utility/Sizeparser.cs +++ b/Duplicati/Library/Utility/Sizeparser.cs @@ -32,7 +32,7 @@ namespace Duplicati.Library.Utility size = size.ToLower().Trim(); - if (size.EndsWith("gb") || size.EndsWith("mb") || size.EndsWith("kb") || size.EndsWith("b")) + if (size.EndsWith("tb") || size.EndsWith("gb") || size.EndsWith("mb") || size.EndsWith("kb") || size.EndsWith("b")) return ParseSize(size); else return ParseSize(size + " " + defaultSuffix); @@ -49,7 +49,12 @@ namespace Duplicati.Library.Utility long factor = 1; - if (size.EndsWith("gb")) + if (size.EndsWith("tb")) + { + factor = 1024L * 1024 * 1024 * 1024; + size = size.Substring(0, size.Length - 2).Trim(); + } + else if (size.EndsWith("gb")) { factor = 1024 * 1024 * 1024; size = size.Substring(0, size.Length - 2).Trim(); From 5d5a8b652c931a64d9896674747e9f5191cb0519 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 31 Aug 2016 11:13:45 +0200 Subject: [PATCH 36/36] Changed the label in the UI for dblock size as many users thought it was a limit on the size of files being backed up. --- Duplicati/Server/webroot/ngax/templates/addoredit.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Server/webroot/ngax/templates/addoredit.html b/Duplicati/Server/webroot/ngax/templates/addoredit.html index 4922ee689..ffb2c8b46 100755 --- a/Duplicati/Server/webroot/ngax/templates/addoredit.html +++ b/Duplicati/Server/webroot/ngax/templates/addoredit.html @@ -218,7 +218,7 @@

General options

- +