From 73b6fe2a017133edbc5d77e47abf5dac828ab17b Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 13:41:57 -0600 Subject: [PATCH 01/36] Add extension methods to the general Utility class for emulating the await keyword in non-async code. Using GetAwaiter().GetResult() is similar to using Task.Result, but rethrows the original exception rather than wrapping it in an (unnecessary) AggregateException. --- Duplicati/Library/Utility/Utility.cs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 8847d36cc..8e840d0aa 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -24,6 +24,7 @@ using System.Linq; using System; using System.Collections.Generic; using System.Text; +using System.Threading.Tasks; namespace Duplicati.Library.Utility { @@ -1572,5 +1573,31 @@ namespace Duplicati.Library.Utility { return string.Join(" ", args.Select(x => WrapCommandLineElement(x, allowEnvExpansion))); } + + /// + /// Utility method that emulates C#'s built in await keyword without requiring the calling method to be async. + /// This method should be preferred over using Task.Result, as it doesn't wrap singular exceptions in AggregateExceptions. + /// (It uses Task.GetAwaiter().GetResult(), which is the same thing that await uses under the covers.) + /// https://stackoverflow.com/questions/17284517/is-task-result-the-same-as-getawaiter-getresult + /// + /// Task to await + public static void Await(this Task task) + { + task.GetAwaiter().GetResult(); + } + + /// + /// Utility method that emulates C#'s built in await keyword without requiring the calling method to be async. + /// This method should be preferred over using Task.Result, as it doesn't wrap singular exceptions in AggregateExceptions. + /// (It uses Task.GetAwaiter().GetResult(), which is the same thing that await uses under the covers.) + /// https://stackoverflow.com/questions/17284517/is-task-result-the-same-as-getawaiter-getresult + /// + /// Result type + /// Task to await + /// Task result + public static T Await(this Task task) + { + return task.GetAwaiter().GetResult(); + } } } From 2b4fb9a65f7d8245c1c02ded4c9cb270eb3f24ea Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 13:44:15 -0600 Subject: [PATCH 02/36] SizeParser can use OrdinalIgnoreCase instead of ToLower(). --- Duplicati/Library/Utility/Sizeparser.cs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Duplicati/Library/Utility/Sizeparser.cs b/Duplicati/Library/Utility/Sizeparser.cs index 46a6d8671..477b201d3 100644 --- a/Duplicati/Library/Utility/Sizeparser.cs +++ b/Duplicati/Library/Utility/Sizeparser.cs @@ -30,9 +30,13 @@ namespace Duplicati.Library.Utility if (string.IsNullOrEmpty(size)) return 0; - size = size.ToLower().Trim(); + size = size.Trim(); - if (size.EndsWith("tb", StringComparison.Ordinal) || size.EndsWith("gb", StringComparison.Ordinal) || size.EndsWith("mb", StringComparison.Ordinal) || size.EndsWith("kb", StringComparison.Ordinal) || size.EndsWith("b", StringComparison.Ordinal)) + if (size.EndsWith("tb", StringComparison.OrdinalIgnoreCase) || + size.EndsWith("gb", StringComparison.OrdinalIgnoreCase) || + size.EndsWith("mb", StringComparison.OrdinalIgnoreCase) || + size.EndsWith("kb", StringComparison.OrdinalIgnoreCase) || + size.EndsWith("b", StringComparison.OrdinalIgnoreCase)) return ParseSize(size); else return ParseSize(size + " " + defaultSuffix); @@ -45,31 +49,31 @@ namespace Duplicati.Library.Utility string origsize = size; - size = size.Trim().ToLower(); + size = size.Trim(); long factor = 1; - if (size.EndsWith("tb", StringComparison.Ordinal)) + if (size.EndsWith("tb", StringComparison.OrdinalIgnoreCase)) { factor = 1024L * 1024 * 1024 * 1024; size = size.Substring(0, size.Length - 2).Trim(); } - else if (size.EndsWith("gb", StringComparison.Ordinal)) + else if (size.EndsWith("gb", StringComparison.OrdinalIgnoreCase)) { factor = 1024 * 1024 * 1024; size = size.Substring(0, size.Length - 2).Trim(); } - else if (size.EndsWith("mb", StringComparison.Ordinal)) + else if (size.EndsWith("mb", StringComparison.OrdinalIgnoreCase)) { factor = 1024 * 1024; size = size.Substring(0, size.Length - 2).Trim(); } - else if (size.EndsWith("kb", StringComparison.Ordinal)) + else if (size.EndsWith("kb", StringComparison.OrdinalIgnoreCase)) { factor = 1024; size = size.Substring(0, size.Length - 2).Trim(); } - else if (size.EndsWith("b", StringComparison.Ordinal)) + else if (size.EndsWith("b", StringComparison.OrdinalIgnoreCase)) size = size.Substring(0, size.Length - 1).Trim(); long r; From 00c1a296daf9be5bb1b7c78b6208e76b21373c3d Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 13:46:49 -0600 Subject: [PATCH 03/36] BackendTester should explicitly call Test() when performing the initial check to see if the main folder exists. Without this, the auto create folders flag was broken for backends which lazily enumerate List() (as they instead throw after the try block is ended) Calling Test() dierctly also better matches the IBackend interface contract. --- Duplicati/CommandLine/BackendTester/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index fad9b76df..19f46fae0 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -164,6 +164,7 @@ namespace Duplicati.CommandLine.BackendTester IEnumerable curlist = null; try { + backend.Test(); curlist = backend.List(); } catch (FolderMissingException fex) From cff8f172fcd8325f9c6f243cc0f8044c877b011c Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 13:48:21 -0600 Subject: [PATCH 04/36] Add support for 'trim-filename-spaces' flag to BackendTester. This flag ensures that any randomly generated file names don't have leading or trailing whitespace, which some backends might not support. --- .../CommandLine/BackendTester/Program.cs | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index 19f46fae0..96459f803 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -211,6 +211,7 @@ namespace Duplicati.CommandLine.BackendTester int max_filename_size = 80; bool disableStreaming = Library.Utility.Utility.ParseBoolOption(options, "disable-streaming-transfers"); bool skipOverwriteTest = Library.Utility.Utility.ParseBoolOption(options, "skip-overwrite-test"); + bool trimFilenameSpaces = Library.Utility.Utility.ParseBoolOption(options, "trim-filename-spaces"); if (options.ContainsKey("number-of-files")) number_of_files = int.Parse(options["number-of-files"]); @@ -233,17 +234,13 @@ namespace Duplicati.CommandLine.BackendTester List files = new List(); for (int i = 0; i < number_of_files; i++) { - - StringBuilder filename = new StringBuilder(); - int filenamelen = rnd.Next(min_filename_size, max_filename_size); - for (int j = 0; j < filenamelen; j++) - filename.Append(allowedChars[rnd.Next(0, allowedChars.Length)]); + string filename = CreateRandomRemoteFileName(min_filename_size, max_filename_size, allowedChars, trimFilenameSpaces, rnd); string localfilename = CreateRandomFile(tf, i, min_file_size, max_file_size, rnd); //Calculate local hash and length using (System.IO.FileStream fs = new System.IO.FileStream(localfilename, System.IO.FileMode.Open, System.IO.FileAccess.Read)) - files.Add(new TempFile(filename.ToString(), localfilename, sha.ComputeHash(fs), fs.Length)); + files.Add(new TempFile(filename, localfilename, sha.ComputeHash(fs), fs.Length)); } byte[] dummyFileHash = null; @@ -411,6 +408,20 @@ namespace Duplicati.CommandLine.BackendTester } } + private static string CreateRandomRemoteFileName(int min_filename_size, int max_filename_size, string allowedChars, bool trimFilenameSpaces, Random rnd) + { + StringBuilder filenameBuilder = new StringBuilder(); + int filenamelen = rnd.Next(min_filename_size, max_filename_size); + for (int j = 0; j < filenamelen; j++) + filenameBuilder.Append(allowedChars[rnd.Next(0, allowedChars.Length)]); + + string filename = filenameBuilder.ToString(); + if (trimFilenameSpaces) + filename = filename.Trim(); + + return filename; + } + private static string CreateRandomFile(Library.Utility.TempFolder tf, int i, int min_file_size, int max_file_size, Random rnd) { Console.Write("Generating file {0}", i); @@ -447,12 +458,12 @@ namespace Duplicati.CommandLine.BackendTester new CommandLineArgument("max-file-size", CommandLineArgument.ArgumentType.Size, "The maximum allowed file size", "File sizes are chosen at random, this value is the upper bound", "50mb"), new CommandLineArgument("min-filename-length", CommandLineArgument.ArgumentType.Integer, "The minimum allowed filename length", "File name lengths are chosen at random, this value is the lower bound", "5"), new CommandLineArgument("max-filename-length", CommandLineArgument.ArgumentType.Integer, "The minimum allowed filename length", "File name lengths are chosen at random, this value is the upper bound", "80"), + new CommandLineArgument("trim-filename-spaces", CommandLineArgument.ArgumentType.Boolean, "Trims whitespace from filenames", "A value that indicates if whitespace should be trimmed from the ends of randomly generated filenames", "false"), new CommandLineArgument("auto-create-folder", CommandLineArgument.ArgumentType.Boolean, "Allows automatic folder creation", "A value that indicates if missing folders are created automatically", "false"), new CommandLineArgument("skip-overwrite-test", CommandLineArgument.ArgumentType.Boolean, "Bypasses the overwrite test", "A value that indicates if dummy files should be uploaded prior to uploading the real files", "false"), new CommandLineArgument("auto-clean", CommandLineArgument.ArgumentType.Boolean, "Removes any files found in target folder", "A value that indicates if all files in the target folder should be deleted before starting the first test", "false"), new CommandLineArgument("force", CommandLineArgument.ArgumentType.Boolean, "Activates file deletion", "A value that indicates if existing files should really be deleted when using auto-clean", "false"), }); - } } } From dba0b3772e7577d2061240c5fa075511b56e49f1 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 13:50:06 -0600 Subject: [PATCH 05/36] Add a test to BackendTester for backends which support renaming. This is done by renaming the second file (or first if only one file is being created) and updating the in memory set of expected files, while keeping track of the original info so it can be reported if it is still present (e.g., if the rename operated as a copy instead). --- .../CommandLine/BackendTester/Program.cs | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index 96459f803..c38132d4b 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -266,6 +266,23 @@ namespace Duplicati.CommandLine.BackendTester for (int i = 0; i < files.Count; i++) Uploadfile(files[i].localfilename, i, files[i].remotefilename, backend, disableStreaming); + TempFile originalRenamedFile = null; + string renamedFileNewName = null; + IRenameEnabledBackend renameEnabledBackend = backend as IRenameEnabledBackend; + if (renameEnabledBackend != null) + { + // Rename the second file in the list, if there are more than one. If not, just do the first one. + int renameIndex = files.Count > 1 ? 1 : 0; + originalRenamedFile = files[renameIndex]; + + renamedFileNewName = CreateRandomRemoteFileName(min_filename_size, max_filename_size, allowedChars, trimFilenameSpaces, rnd); + + Console.WriteLine("Renaming file {0} from {1} to {2}", renameIndex, originalRenamedFile.remotefilename, renamedFileNewName); + + renameEnabledBackend.Rename(originalRenamedFile.remotefilename, renamedFileNewName); + files[renameIndex] = new TempFile(renamedFileNewName, originalRenamedFile.localfilename, originalRenamedFile.hash, originalRenamedFile.length); + } + Console.WriteLine("Verifying file list ..."); curlist = backend.List(); @@ -288,7 +305,14 @@ namespace Duplicati.CommandLine.BackendTester } if (!found) - Console.WriteLine("*** File with name {0} was found on server but not uploaded!", fe.Name); + if (originalRenamedFile != null && renamedFileNewName != null && originalRenamedFile.remotefilename == fe.Name) + { + Console.WriteLine("*** File with name {0} was found on server but was supposed to have been renamed to {1}!", fe.Name, renamedFileNewName); + } + else + { + Console.WriteLine("*** File with name {0} was found on server but not uploaded!", fe.Name); + } } foreach (TempFile tx in files) From 4e4b9a56d6bc43aaf6dace1d96480f5fd7feb885 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 13:53:32 -0600 Subject: [PATCH 06/36] Adds a test to the BackendTester for quota-enabled backends. This test simply loads the quota information and displays it (or displays an error if the backend was unable to get it) --- .../CommandLine/BackendTester/Program.cs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index c38132d4b..9b97f76a6 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -383,6 +383,38 @@ namespace Duplicati.CommandLine.BackendTester } } + + // Test quota retrieval + IQuotaEnabledBackend quotaEnabledBackend = backend as IQuotaEnabledBackend; + if (quotaEnabledBackend != null) + { + Console.WriteLine("Checking quota..."); + IQuotaInfo quota = null; + bool noException; + try + { + quota = quotaEnabledBackend.Quota; + noException = true; + } + catch (Exception ex) + { + Console.WriteLine("*** Checking quota information failed: {0}", ex); + noException = false; + } + + if (noException) + { + if (quota != null) + { + Console.WriteLine("Free Space: {0}", Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); + Console.WriteLine("Total Space: {0}", Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace)); + } + else + { + Console.WriteLine("Unable to retrieve quota information"); + } + } + } } finally { From f9a8b30b2ff5af2670f9c1d5a9fa399133cd6d1f Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 13:56:05 -0600 Subject: [PATCH 07/36] Update JSONWebHelper.PostAndGetJSONData to remove parameters that aren't actually being used (setup and setupreq). These parameters weren't being used in the method body, or being passed in by any callers (it looks like a copy / paste error when a previous method was converted into this one) These are replaced with a new optional parameter (method) which allows the caller to request a custom HTTP method to be used. --- Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs index 3547a8a05..0b82f4c00 100644 --- a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs +++ b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs @@ -163,10 +163,9 @@ namespace Duplicati.Library /// The deserialized JSON data. /// The remote URL /// The data to json-serialize and POST in the request - /// A callback method that can be used to customize the request, e.g. by setting the method, content-type and headers. - /// A callback method that can be used to submit data into the body of the request. + /// Alternate HTTP method to use /// The type of data to return. - public virtual T PostAndGetJSONData(string url, object item, Action setup = null, Action setupreq = null) + public virtual T PostAndGetJSONData(string url, object item, string method = null) { var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(item)); @@ -174,7 +173,7 @@ namespace Duplicati.Library url, req => { - req.Method = "POST"; + req.Method = method ?? "POST"; req.ContentType = "application/json; charset=utf-8"; req.ContentLength = data.Length; }, From 20fc793deed429ea2e29c6f88c6ba86ddcbf93f8 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 14:00:52 -0600 Subject: [PATCH 08/36] Add OAuthHttpMessageHandler helper class to OAuthHelper library. This class implements the abstract HttpMessageHandler class (mostly via the default HttpClientHandler implementation). This abstract class is used by System.Net.Http.HttpClient as the underlying HTTP mechanism, and in this particular case is used to automatically add an authorization header to each request. It also provides a method for marking a request as one that should not be authenticated. --- .../Duplicati.Library.OAuthHelper.csproj | 4 +- .../OAuthHelper/OAuthHttpMessageHandler.cs | 61 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs diff --git a/Duplicati/Library/Backend/OAuthHelper/Duplicati.Library.OAuthHelper.csproj b/Duplicati/Library/Backend/OAuthHelper/Duplicati.Library.OAuthHelper.csproj index f3005f88e..c0515756c 100644 --- a/Duplicati/Library/Backend/OAuthHelper/Duplicati.Library.OAuthHelper.csproj +++ b/Duplicati/Library/Backend/OAuthHelper/Duplicati.Library.OAuthHelper.csproj @@ -1,4 +1,4 @@ - + Debug @@ -34,8 +34,10 @@ ..\..\..\..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll + + diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs new file mode 100644 index 000000000..7bc67a69c --- /dev/null +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs @@ -0,0 +1,61 @@ +// Copyright (C) 2018, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; + +namespace Duplicati.Library +{ + public class OAuthHttpMessageHandler : HttpClientHandler + { + /// + /// Requests which contain a property with this name (in 'request.Properties') will not have the authentication header automatically added. + /// + public const string DISABLE_AUTHENTICATION_PROPERTY = "OAuthHttpMessageHandler_DisableAuthentication"; + + private OAuthHelper m_oauth; + + public OAuthHttpMessageHandler(string authid, string protocolKey) + { + this.m_oauth = new OAuthHelper(authid, protocolKey); + } + + /// + /// Prevents authentication from being applied on the given request + /// + /// Request to not authenticate + /// Request to not authenticate + public HttpRequestMessage PreventAuthentication(HttpRequestMessage request) + { + request.Properties[DISABLE_AUTHENTICATION_PROPERTY] = true; + return request; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (!request.Properties.ContainsKey(DISABLE_AUTHENTICATION_PROPERTY)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", this.m_oauth.AccessToken); + } + + return base.SendAsync(request, cancellationToken); + } + } +} From c542d659972310d2d00f57647e7a8a2690f7af7a Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 14:01:28 -0600 Subject: [PATCH 09/36] Update service agreement and privacy policy link for OneDrive backend. --- Duplicati/Library/Backend/OneDrive/OneDrive.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/OneDrive/OneDrive.cs b/Duplicati/Library/Backend/OneDrive/OneDrive.cs index 4f5a59a98..018e48c35 100644 --- a/Duplicati/Library/Backend/OneDrive/OneDrive.cs +++ b/Duplicati/Library/Backend/OneDrive/OneDrive.cs @@ -9,6 +9,9 @@ namespace Duplicati.Library.Backend { public class OneDrive : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend { + private const string SERVICES_AGREEMENT = "https://www.microsoft.com/en-us/servicesagreement"; + private const string PRIVACY_STATEMENT = "https://privacy.microsoft.com/en-us/privacystatement"; + private const string AUTHID_OPTION = "authid"; private const string WLID_SERVER = "https://apis.live.net/v5.0"; @@ -381,9 +384,9 @@ namespace Duplicati.Library.Backend { get { return Strings.OneDrive.Description( "Microsoft Service Agreement", - "http://explore.live.com/microsoft-service-agreement", + SERVICES_AGREEMENT, "Microsoft Online Privacy Statement", - "http://privacy.microsoft.com/en-us/fullnotice.mspx" + PRIVACY_STATEMENT ); } } From a98c967f70e4df352a5afc226ea1509d24e51b7f Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 14:18:49 -0600 Subject: [PATCH 10/36] Add new backends for OneDrive+OneDrive for Business, SharePoint, and Microsoft Groups based on the Microsoft Graph API. These backends are implemented via the common base class, MicrosoftGraphBackend, with only a few minor tweaks (different id parameters, protocol keys, descriptions, etc.) All three backends support streaming, quota retrieval, and rename. --- .../Duplicati.Library.Backend.OneDrive.csproj | 7 + .../Library/Backend/OneDrive/Exceptions.cs | 92 ++++ .../Backend/OneDrive/MicrosoftGraphBackend.cs | 504 +++++++++++++++++ .../Backend/OneDrive/MicrosoftGraphTypes.cs | 515 ++++++++++++++++++ .../Backend/OneDrive/MicrosoftGroup.cs | 63 +++ .../Library/Backend/OneDrive/OneDriveV2.cs | 65 +++ .../Library/Backend/OneDrive/SharePointV2.cs | 63 +++ Duplicati/Library/Backend/OneDrive/Strings.cs | 38 ++ 8 files changed, 1347 insertions(+) create mode 100644 Duplicati/Library/Backend/OneDrive/Exceptions.cs create mode 100644 Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs create mode 100644 Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs create mode 100644 Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs create mode 100644 Duplicati/Library/Backend/OneDrive/OneDriveV2.cs create mode 100644 Duplicati/Library/Backend/OneDrive/SharePointV2.cs diff --git a/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj b/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj index 03a475406..259004c1d 100644 --- a/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj +++ b/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj @@ -46,10 +46,17 @@ ..\..\..\..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll + + + + + + + diff --git a/Duplicati/Library/Backend/OneDrive/Exceptions.cs b/Duplicati/Library/Backend/OneDrive/Exceptions.cs new file mode 100644 index 000000000..2aa732d17 --- /dev/null +++ b/Duplicati/Library/Backend/OneDrive/Exceptions.cs @@ -0,0 +1,92 @@ +using System; +using System.Net.Http; + +using Duplicati.Library.Utility; + +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.MicrosoftGraph +{ + public class MicrosoftGraphException : Exception + { + public MicrosoftGraphException(HttpResponseMessage response) + : this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage.RequestUri), response) + { + } + + public MicrosoftGraphException(string message, HttpResponseMessage response) + : this(message, response, null) + { + } + + public MicrosoftGraphException(string message, HttpResponseMessage response, Exception innerException) + : base(BuildFullMessage(message, response), innerException) + { + this.Response = response; + } + + public string RequestUrl => this.Response.RequestMessage.RequestUri.ToString(); + public HttpResponseMessage Response { get; private set; } + + protected static string ResponseToString(HttpResponseMessage response) + { + if (response != null) + { + string content = response.Content.ReadAsStringAsync().Await(); + return string.Format("{0}\n{1}\n{2}", response.RequestMessage, response, JsonConvert.SerializeObject(JsonConvert.DeserializeObject(content), Formatting.Indented)); + } + else + { + return null; + } + } + + private static string BuildFullMessage(string message, HttpResponseMessage response) + { + if (response != null) + { + return string.Format("{0}\n{1}", message, ResponseToString(response)); + } + else + { + return message; + } + } + } + + public class DriveItemNotFoundException : MicrosoftGraphException + { + public DriveItemNotFoundException(HttpResponseMessage response) + : base(string.Format("Item at {0} was not found", response?.RequestMessage?.RequestUri?.ToString() ?? ""), response) + { + } + } + + public class UploadSessionException : MicrosoftGraphException + { + public UploadSessionException( + HttpResponseMessage originalResponse, + int fragment, + int fragmentCount, + MicrosoftGraphException fragmentException) + : base( + string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.RequestMessage?.RequestUri?.ToString() ?? ""), + originalResponse, + fragmentException) + { + this.Fragment = fragment; + this.FragmentCount = fragmentCount; + this.InnerException = fragmentException; + } + + public string CreateSessionRequestUrl => this.RequestUrl; + public HttpResponseMessage CreateSessionResponse => this.Response; + + public int Fragment { get; private set; } + public int FragmentCount { get; private set; } + public string FragmentRequestUrl => this.InnerException.RequestUrl; + public HttpResponseMessage FragmentResponse => this.InnerException.Response; + + public new MicrosoftGraphException InnerException { get; private set; } + } +} diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs new file mode 100644 index 000000000..7f8f1ad4f --- /dev/null +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -0,0 +1,504 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; + +using Duplicati.Library.Backend.MicrosoftGraph; +using Duplicati.Library.Interface; +using Duplicati.Library.Utility; + +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend +{ + /// + /// Base class for all backends based on the Microsoft Graph API: + /// https://developer.microsoft.com/en-us/graph/ + /// + /// + /// HttpClient is used instead of OAuthHelper because OAuthHelper internally converts URLs to System.Uri, which throws UriFormatException when the URL contains ':' characters. + /// https://stackoverflow.com/questions/2143856/why-does-colon-in-uri-passed-to-uri-makerelativeuri-cause-an-exception + /// https://social.msdn.microsoft.com/Forums/vstudio/en-US/bf11fc74-975a-4c4d-8335-8e0579d17fdf/uri-containing-colons-incorrectly-throws-uriformatexception?forum=netfxbcl + /// + /// Note that instead of using Task.Result to wait for the results of asynchronous operations, + /// this class uses the Utility.Await() extension method, since it doesn't wrap exceptions in AggregateExceptions. + /// + public abstract class MicrosoftGraphBackend : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend + { + private const string SERVICES_AGREEMENT = "https://www.microsoft.com/en-us/servicesagreement"; + private const string PRIVACY_STATEMENT = "https://privacy.microsoft.com/en-us/privacystatement"; + + private const string BASE_ADDRESS = "https://graph.microsoft.com"; + + private const string AUTHID_OPTION = "authid"; + private const string UPLOAD_SESSION_FRAGMENT_SIZE_OPTION = "fragment-size"; + private const string UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION = "fragment-retry-count"; + private const string UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION = "fragment-retry-delay"; + + private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT = 5; + private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY = 500; + + /// + /// Max size of file that can be uploaded in a single PUT request is 4 MB: + /// https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/api/driveitem_put_content + /// + private const int PUT_MAX_SIZE = 4 * 1000 * 1000; + + /// + /// Max size of each individual upload in an upload session is 60 MiB: + /// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession + /// + private const int UPLOAD_SESSION_FRAGMENT_MAX_SIZE = 60 * 1024 * 1024; + + /// + /// Default fragment size of 10 MiB, as the documentation recommends something in the range of 5-10 MiB, + /// and it still complies with the 320 KiB multiple requirement. + /// + private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE = 10 * 1024 * 1024; + + /// + /// Each fragment in an upload session must be a size that is multiple of this size. + /// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession + /// There is some confusion in the docs as to whether this is actually required, however... + /// + private const int UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE = 320 * 1024; + + private static readonly HttpMethod PatchMethod = new HttpMethod("PATCH"); + + protected delegate string DescriptionTemplateDelegate(string mssadescription, string mssalink, string msopdescription, string msoplink); + + private readonly JsonSerializer m_serializer = new JsonSerializer(); + private readonly OAuthHttpMessageHandler m_authenticator; + private readonly HttpClient m_client; + private readonly string m_path; + private readonly int fragmentSize; + private readonly int fragmentRetryCount; + private readonly int fragmentRetryDelay; // In milliseconds + + protected MicrosoftGraphBackend() { } // Constructor needed for dynamic loading to find it + + protected MicrosoftGraphBackend(string url, Dictionary options) + { + string authid; + options.TryGetValue(AUTHID_OPTION, out authid); + if (string.IsNullOrEmpty(authid)) + throw new UserInformationException(Strings.MicrosoftGraph.MissingAuthId(OAuthHelper.OAUTH_LOGIN_URL(this.ProtocolKey))); + + string fragmentSizeStr; + if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, out fragmentSizeStr) && int.TryParse(fragmentSizeStr, out this.fragmentSize))) + { + this.fragmentSize = UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE; + } + + string fragmentRetryCountStr; + if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, out fragmentRetryCountStr) && int.TryParse(fragmentRetryCountStr, out this.fragmentRetryCount))) + { + this.fragmentRetryCount = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT; + } + + string fragmentRetryDelayStr; + if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, out fragmentRetryDelayStr) && int.TryParse(fragmentRetryDelayStr, out this.fragmentRetryDelay))) + { + this.fragmentRetryDelay = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY; + } + + this.m_authenticator = new OAuthHttpMessageHandler(authid, this.ProtocolKey); + this.m_client = new HttpClient(this.m_authenticator, true); + this.m_client.BaseAddress = new System.Uri(BASE_ADDRESS); + + // Extract out the path to the backup root folder from the given URI + var uri = new Utility.Uri(url); + + this.m_path = NormalizeSlashes(Utility.Uri.UrlDecode(uri.HostAndPath)); + } + + public abstract string ProtocolKey { get; } + + public abstract string DisplayName { get; } + + public string Description + { + get + { + return this.DescriptionTemplate( + "Microsoft Service Agreement", + SERVICES_AGREEMENT, + "Microsoft Online Privacy Statement", + PRIVACY_STATEMENT); + } + } + + public IList SupportedCommands + { + get + { + return new[] + { + new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.MicrosoftGraph.AuthIdShort, Strings.MicrosoftGraph.AuthIdLong(OAuthHelper.OAUTH_LOGIN_URL(this.ProtocolKey))), + new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentSizeShort, Strings.MicrosoftGraph.FragmentSizeLong, Library.Utility.Utility.FormatSizeString(UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE)), + new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryCountShort, Strings.MicrosoftGraph.FragmentRetryCountLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT.ToString()), + new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryDelayShort, Strings.MicrosoftGraph.FragmentRetryDelayLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY.ToString()), + } + .Concat(this.AdditionalSupportedCommands).ToList(); + } + } + + public string DNSName + { + get { return null; } + } + + public IQuotaInfo Quota + { + get + { + Drive driveInfo = this.Get(this.DrivePrefix); + if (driveInfo.Quota != null) + { + return new QuotaInfo(driveInfo.Quota.Total, driveInfo.Quota.Remaining); + } + + return null; + } + } + + /// + /// Override-able fragment indicating the API version to use each query + /// + protected virtual string ApiVersion + { + get { return "/v1.0"; } + } + + /// + /// Normalized fragment (starting with a slash and ending without one) for the path to the drive to be used. + /// For example: "/me/drive" for the default drive for a user. + /// + protected abstract string DrivePath { get; } + + protected abstract DescriptionTemplateDelegate DescriptionTemplate { get; } + + protected virtual IList AdditionalSupportedCommands + { + get + { + return new ICommandLineArgument[0]; + } + } + + private string DrivePrefix + { + get { return this.ApiVersion + this.DrivePath; } + } + + public void CreateFolder() + { + string parentFolder = "root"; + string parentFolderPath = string.Empty; + foreach (string folder in this.m_path.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries)) + { + string nextPath = parentFolderPath + "/" + folder; + DriveItem folderItem; + try + { + folderItem = this.Get(string.Format("{0}/root:{1}", this.DrivePrefix, NormalizeSlashes(nextPath))); + } + catch (DriveItemNotFoundException) + { + DriveItem newFolder = new DriveItem() + { + Name = folder, + Folder = new FolderFacet(), + }; + + folderItem = this.Post(string.Format("{0}/items/{1}/children", this.DrivePrefix, parentFolder), newFolder); + } + + parentFolder = folderItem.Id; + parentFolderPath = nextPath; + } + } + + public IEnumerable List() + { + try + { + return this.Enumerate(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.m_path)) + .Where(item => item.IsFile && !item.IsDeleted) // Exclude non-files and deleted items (not sure if they show up in this listing, but make sure anyway) + .Select(item => + new FileEntry( + item.Name, + item.Size ?? 0, // Files should always have a size, but folders don't need it + item.FileSystemInfo?.LastAccessedDateTime?.UtcDateTime ?? new DateTime(), + item.FileSystemInfo?.LastModifiedDateTime?.UtcDateTime ?? item.LastModifiedDateTime?.UtcDateTime ?? new DateTime())); + } + catch (DriveItemNotFoundException ex) + { + // If there's an 'item not found' exception here, it means the root folder didn't exist. + throw new FolderMissingException(ex); + } + } + + public void Get(string remotename, string filename) + { + using (FileStream fileStream = File.OpenWrite(filename)) + { + this.Get(remotename, fileStream); + } + } + + public void Get(string remotename, Stream stream) + { + var response = this.m_client.GetAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename))).Await(); + this.CheckResponse(response); + using (Stream responseStream = response.Content.ReadAsStreamAsync().Await()) + { + responseStream.CopyTo(stream); + } + } + + public void Rename(string oldname, string newname) + { + this.Patch(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.m_path, NormalizeSlashes(oldname)), new DriveItem() { Name = newname }); + } + + public void Put(string remotename, string filename) + { + using (FileStream fileStream = File.OpenRead(filename)) + { + this.Put(remotename, fileStream); + } + } + + public void Put(string remotename, Stream stream) + { + // PUT only supports up to 4 MB file uploads. There's a separate process for larger files. + if (stream.Length < PUT_MAX_SIZE) + { + StreamContent streamContent = new StreamContent(stream); + streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + var response = this.m_client.PutAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename)), streamContent).Await(); + + // Make sure this response is a valid drive item, though we don't actually use it for anything currently. + var result = this.ParseResponse(response); + } + else + { + // This file is too large to be sent in a single request, so we need to send it in pieces in an upload session: + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession + // The documentation seems somewhat contradictory - it states that uploads must be done sequentially, + // but also states that the nextExpectedRanges value returned may indicate multiple ranges... + // For now, this plays it safe and does a sequential upload. + HttpRequestMessage createSessionRequest = new HttpRequestMessage(HttpMethod.Post, string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename))); + HttpResponseMessage createSessionResponse = this.m_client.SendAsync(createSessionRequest).Await(); + UploadSession uploadSession = this.ParseResponse(createSessionResponse); + + // If the stream's total length is less than the chosen fragment size, then we should make the buffer only as large as the stream. + int fragmentSize = (int)Math.Min(this.fragmentSize, stream.Length); + + byte[] fragmentBuffer = new byte[fragmentSize]; + int read = 0; + for (int offset = 0; offset < stream.Length; offset += read) + { + read = stream.Read(fragmentBuffer, 0, fragmentSize); + + int retryCount = this.fragmentRetryCount; + for (int attempt = 0; attempt < retryCount; attempt++) + { + ByteArrayContent fragmentContent = new ByteArrayContent(fragmentBuffer, 0, read); + fragmentContent.Headers.ContentLength = read; + fragmentContent.Headers.ContentRange = new ContentRangeHeaderValue(offset, offset + read - 1, stream.Length); + + HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Put, uploadSession.UploadUrl); + request.Content = fragmentContent; + + // The uploaded put requests will error if they are authenticated + this.m_authenticator.PreventAuthentication(request); + + HttpResponseMessage response = null; + try + { + response = this.m_client.SendAsync(request).Await(); + } + catch (MicrosoftGraphException ex) + { + // Error handling based on recommendations here: + // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession#best-practices + if (attempt >= retryCount - 1) + { + // We've used up all our retry attempts + throw new UploadSessionException(createSessionResponse, offset / fragmentSize, (int)Math.Ceiling((double)stream.Length / fragmentSize), ex); + } + else if ((int)ex.Response.StatusCode >= 500 && (int)ex.Response.StatusCode < 600) + { + // If a 5xx error code is hit, we should use an exponential backoff strategy before retrying. + // To make things simpler, we just use the current attempt number as the exponential factor. + Thread.Sleep((int)Math.Pow(2, attempt) * this.fragmentRetryDelay); // If this is changed to use tasks, this should be changed to Task.Await() + continue; + } + else if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + // 404 is a special case indicating the upload session no longer exists, so the fragment shouldn't be retried. + // Instead we'll let the caller re-attempt the whole file. + throw new UploadSessionException(createSessionResponse, offset / fragmentSize, (int)Math.Ceiling((double)stream.Length / fragmentSize), ex); + } + else if ((int)ex.Response.StatusCode >= 400 && (int)ex.Response.StatusCode < 500) + { + // If a 4xx error code is hit, we should retry without the backoff attempt + continue; + } + else + { + // Other errors should be rethrown + throw new UploadSessionException(createSessionResponse, offset / fragmentSize, (int)Math.Ceiling((double)stream.Length / fragmentSize), ex); + } + } + + // Note: On the last request, the json result includes the default properties of the item that was uploaded, + // instead of just the upload session results. + var result = this.ParseResponse(response); + + // If we successfully sent this piece, then we can break out of the retry loop + break; + } + } + } + } + + public void Delete(string remotename) + { + var response = this.m_client.DeleteAsync(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename))).Await(); + this.CheckResponse(response); + } + + public void Test() + { + try + { + string rootPath = string.Format("{0}/root:{1}", this.DrivePrefix, this.m_path); + DriveItem rootFolder = this.Get(rootPath); + } + catch (DriveItemNotFoundException ex) + { + // Wrap the existing item not found error in a 'FolderMissingException' + throw new FolderMissingException(ex); + } + } + + public void Dispose() + { + if (this.m_client != null) + { + this.m_client.Dispose(); + } + } + + /// + /// Normalizes the slashes in a url fragment. For example: + /// "" => "" + /// "test" => "/test" + /// "test/" => "/test" + /// "a\b" => "/a/b" + /// + /// Url fragment to normalize + /// Normalized fragment + private static string NormalizeSlashes(string url) + { + url = url.Replace('\\', '/'); + + if (url.Length != 0 && !url.StartsWith("/", StringComparison.Ordinal)) + url = "/" + url; + + if (url.EndsWith("/", StringComparison.Ordinal)) + url = url.Substring(0, url.Length - 1); + + return url; + } + + private T Get(string url) + { + return this.SendRequest(HttpMethod.Get, url); + } + + private T Post(string url, T body) + { + return this.SendRequest(HttpMethod.Post, url, body); + } + + private T Patch(string url, T body) + { + return this.SendRequest(PatchMethod, url, body); + } + + private T SendRequest(HttpMethod method, string url) + { + var request = new HttpRequestMessage(method, url); + return this.SendRequest(request); + } + + private T SendRequest(HttpMethod method, string url, T body) + { + var request = new HttpRequestMessage(method, url); + if (body != null) + { + request.Content = new StringContent(JsonConvert.SerializeObject(body), Encoding.UTF8, "application/json"); + } + + return this.SendRequest(request); + } + + private T SendRequest(HttpRequestMessage request) + { + var response = this.m_client.SendAsync(request).Await(); + return this.ParseResponse(response); + } + + private IEnumerable Enumerate(string url) + { + string nextUrl = url; + while (!string.IsNullOrEmpty(nextUrl)) + { + GraphCollection results = this.Get>(nextUrl); + foreach (T result in results.Value) + { + yield return result; + } + + nextUrl = results.ODataNextLink; + } + } + + private void CheckResponse(HttpResponseMessage response) + { + if (!response.IsSuccessStatusCode) + { + if (response.StatusCode == HttpStatusCode.NotFound) + { + // It looks like this is an 'item not found' exception, so wrap it in a new exception class to make it easier to pick things out. + throw new DriveItemNotFoundException(response); + } + else + { + // Throw a wrapper exception to make it easier for the caller to look at specific status codes, etc. + throw new MicrosoftGraphException(response); + } + } + } + + private T ParseResponse(HttpResponseMessage response) + { + this.CheckResponse(response); + using (Stream responseStream = response.Content.ReadAsStreamAsync().Await()) + using (StreamReader reader = new StreamReader(responseStream)) + using (JsonTextReader jsonReader = new JsonTextReader(reader)) + { + return this.m_serializer.Deserialize(jsonReader); + } + } + } +} diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs new file mode 100644 index 000000000..d65feeaea --- /dev/null +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs @@ -0,0 +1,515 @@ +using System; + +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +/// +/// Types are based on definitions from: +/// https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/onedrive +/// +/// Note that some classes don't have the full set of properties defined, particularly if they don't seem like they are needed. +/// +namespace Duplicati.Library.Backend.MicrosoftGraph +{ + public class Identity + { + [JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)] + public string Id { get; set; } + + [JsonProperty("displayName", NullValueHandling = NullValueHandling.Ignore)] + public string DisplayName { get; set; } + } + + public class IdentitySet + { + /// + /// The optional application associated with this action + /// + [JsonProperty("application", NullValueHandling = NullValueHandling.Ignore)] + public Identity Application { get; set; } + + /// + /// The optional device associated with this action + /// + [JsonProperty("device", NullValueHandling = NullValueHandling.Ignore)] + public Identity Device { get; set; } + + /// + /// The optional user associated with this action + /// + [JsonProperty("user", NullValueHandling = NullValueHandling.Ignore)] + public Identity User { get; set; } + } + + public class BaseItem + { + [JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)] + public string Id { get; set; } + + [JsonProperty("createdBy", NullValueHandling = NullValueHandling.Ignore)] + public IdentitySet CreatedBy { get; set; } + + [JsonProperty("createdDateTime", NullValueHandling = NullValueHandling.Ignore)] + public DateTimeOffset? CreatedDateTime { get; set; } + + [JsonProperty("description", NullValueHandling = NullValueHandling.Ignore)] + public string Description { get; set; } + + [JsonProperty("lastModifiedBy", NullValueHandling = NullValueHandling.Ignore)] + public IdentitySet LastModifiedBy { get; set; } + + [JsonProperty("lastModifiedDateTime", NullValueHandling = NullValueHandling.Ignore)] + public DateTimeOffset? LastModifiedDateTime { get; set; } + + /// + /// Note: OneDrive and OneDrive for Business don't allow the following characters in file names: + /// " * : < > ? / \ | + /// https://support.office.com/en-us/article/Invalid-file-names-and-file-types-in-OneDrive-OneDrive-for-Business-and-SharePoint-64883a5d-228e-48f5-b3d2-eb39e07630fa + /// If appears it also follows the Windows conventions for handling leading and trailing spaces, + /// meaning the ASCII space character is trimmed off of both the front and back of the file name: + /// https://support.microsoft.com/en-us/help/2829981/support-for-whitespace-characters-in-file-and-folder-names-for-windows + /// + [JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)] + public string Name { get; set; } + + [JsonProperty("webUrl", NullValueHandling = NullValueHandling.Ignore)] + public string WebUrl { get; set; } + } + + public enum QuotaState + { + /// + /// The drive has plenty of remaining quota + /// + Normal, + + /// + /// Remaining quota is under 10% + /// + Nearing, + + /// + /// Remaining quota is under 1% + /// + Critical, + + /// + /// No remaining quota - files can only be deleted and no new files can be added + /// + Exceeded, + + // Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums + // without respecting cases. Once we can use that, we can remove these copies of the nicely named enums. + normal = Normal, + nearing = Nearing, + critical = Critical, + exceeded = Exceeded, + } + + public class Quota + { + [JsonProperty("total", NullValueHandling = NullValueHandling.Ignore)] + public long Total { get; set; } + + [JsonProperty("used", NullValueHandling = NullValueHandling.Ignore)] + public long Used { get; set; } + + [JsonProperty("remaining", NullValueHandling = NullValueHandling.Ignore)] + public long Remaining { get; set; } + + [JsonProperty("deleted", NullValueHandling = NullValueHandling.Ignore)] + public long Deleted { get; set; } + + [JsonProperty("state", NullValueHandling = NullValueHandling.Ignore)] + [JsonConverter(typeof(StringEnumConverter))] + public QuotaState State { get; set; } + } + + public enum DriveType + { + /// + /// OneDrive personal drives + /// + Personal, + + /// + /// OneDrive for Business drives + /// + Business, + + /// + /// SharePoint document libraries + /// + DocumentLibrary, + + // Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums + // without respecting cases. Once we can use that, we can remove these copies of the nicely named enums. + personal = Personal, + business = Business, + documentLibrary = DocumentLibrary, + } + + public class Drive : BaseItem + { + [JsonProperty("driveType", NullValueHandling = NullValueHandling.Ignore)] + [JsonConverter(typeof(StringEnumConverter))] + public DriveType DriveType { get; set; } + + [JsonProperty("owner", NullValueHandling = NullValueHandling.Ignore)] + public IdentitySet Owner { get; set; } + + [JsonProperty("quota", NullValueHandling = NullValueHandling.Ignore)] + public Quota Quota { get; set; } + } + + public class HashesType + { + /// + /// SHA1 hash + /// Available only in OneDrive personal + /// + [JsonProperty("sha1Hash", NullValueHandling = NullValueHandling.Ignore)] + public string Sha1Hash { get; set; } + + /// + /// CRC32 hash + /// Available only in OneDrive personal + /// + [JsonProperty("crc32Hash", NullValueHandling = NullValueHandling.Ignore)] + public string Crc32Hash { get; set; } + + /// + /// A proprietary hash of the file that can be used to determine if the file's contents have changed. + /// Available on in OneDrive for Business and SharePoint Server 2016. + /// + [JsonProperty("quickXorHash", NullValueHandling = NullValueHandling.Ignore)] + public string QuickXorHash { get; set; } + } + + public class FileFacet + { + [JsonProperty("hashes", NullValueHandling = NullValueHandling.Ignore)] + public HashesType Hashes { get; set; } + + [JsonProperty("mimeType", NullValueHandling = NullValueHandling.Ignore)] + public string MimeType { get; set; } + } + + public enum SortBy + { + Default = 0, + + /// + /// Sorted by 'name' property + /// + Name, + + /// + /// Sorted by type of item + /// + Type, + + /// + /// Sorted by 'size' property + /// + Size, + + /// + /// Sorted by 'takenDateTime' property of the photos facet, or 'createdDateTime' if that isn't available + /// + TakenOrCreatedDateTime, + + /// + /// Sorted by 'lastModifiedDateTime' property + /// + LastModifiedDateTime, + + /// + /// Sorted by a custom user specified sequence + /// + Sequence, + + // Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums + // without respecting cases. Once we can use that, we can remove these copies of the nicely named enums. + @default = Default, + name = Name, + type = Type, + size = Size, + takenOrCreatedDateTime = TakenOrCreatedDateTime, + lastModifiedDateTime = LastModifiedDateTime, + sequence = Sequence, + } + + public enum SortOrder + { + Ascending, + Descending, + + // Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums + // without respecting cases. Once we can use that, we can remove these copies of the nicely named enums. + ascending = Ascending, + descending = Descending, + } + + public enum ViewType + { + Default = 0, + Icons, + Details, + Thumbnails, + + // Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums + // without respecting cases. Once we can use that, we can remove these copies of the nicely named enums. + @default = Default, + icons = Icons, + details = Details, + thumbnails = Thumbnails, + } + + public class FolderView + { + [JsonProperty("sortBy", NullValueHandling = NullValueHandling.Ignore)] + [JsonConverter(typeof(StringEnumConverter))] + public SortBy? SortBy { get; set; } + + [JsonProperty("sortOrder", NullValueHandling = NullValueHandling.Ignore)] + [JsonConverter(typeof(StringEnumConverter))] + public SortOrder? SortOrder { get; set; } + + [JsonProperty("viewType", NullValueHandling = NullValueHandling.Ignore)] + [JsonConverter(typeof(StringEnumConverter))] + public ViewType? ViewType { get; set; } + } + + public class FolderFacet + { + [JsonProperty("childCount", NullValueHandling = NullValueHandling.Ignore)] + public long? ChildCount { get; set; } + + [JsonProperty("view", NullValueHandling = NullValueHandling.Ignore)] + public FolderView View { get; set; } + } + + /// + /// Note: These results are different from those in the core DriveItem type - + /// they are the times the client reported from the local file system. + /// For example, if a file is created locally on Monday, but uploaded on Tuesday, + /// the DriveItem.CreatedDateTime will be Tuesday, but FileSystemInfoFacet.CreatedDateTime will be Monday. + /// + public class FileSystemInfoFacet + { + [JsonProperty("createdDateTime", NullValueHandling = NullValueHandling.Ignore)] + public DateTimeOffset? CreatedDateTime { get; set; } + + /// + /// This is not available in OneDrive for Business or SharePoint. + /// + [JsonProperty("lastAccessedDateTime", NullValueHandling = NullValueHandling.Ignore)] + public DateTimeOffset? LastAccessedDateTime { get; set; } + + [JsonProperty("lastModifiedDateTime", NullValueHandling = NullValueHandling.Ignore)] + public DateTimeOffset? LastModifiedDateTime { get; set; } + } + + public class PackageFacet + { + /// + /// Package type - currently only oneNote is defined, but others could be used. + /// + [JsonProperty("type", NullValueHandling = NullValueHandling.Ignore)] + public string Type { get; set; } + } + + public class DeletedFacet + { + [JsonProperty("state", NullValueHandling = NullValueHandling.Ignore)] + public string State { get; set; } + } + + public class RemoteItemFacet + { + [JsonProperty("remoteItem", NullValueHandling = NullValueHandling.Ignore)] + public RemoteItemFacet RemoteItem { get; set; } + } + + public class RootFacet + { + } + + public class SpecialFolderFacet + { + /// + /// The unique identifier which can be used under /drive/special/ + /// + [JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)] + public string Name { get; set; } + } + + /// + /// To get an item from one of these references: + /// GET https://graph.microsoft.com/v1.0/drives/{driveId}/items/{id} + /// + public class ItemReference + { + [JsonProperty("driveId", NullValueHandling = NullValueHandling.Ignore)] + public string DriveId { get; set; } + + [JsonProperty("driveType", NullValueHandling = NullValueHandling.Ignore)] + public DriveType DriveType { get; set; } + + [JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)] + public string Id { get; set; } + + [JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)] + public string Name { get; set; } + + [JsonProperty("path", NullValueHandling = NullValueHandling.Ignore)] + public string Path { get; set; } + } + + public enum ConflictBehavior + { + Fail, + Replace, + Rename, + + // Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums + // without respecting cases. Once we can use that, we can remove these copies of the nicely named enums. + fail = Fail, + replace = Replace, + rename = Rename, + } + + public class DriveItem : BaseItem + { + [JsonProperty("parentReference", NullValueHandling = NullValueHandling.Ignore)] + public ItemReference ParentReference { get; set; } + + [JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)] + public long? Size { get; set; } + + [JsonProperty("file", NullValueHandling = NullValueHandling.Ignore)] + public FileFacet File { get; set; } + + [JsonProperty("folder", NullValueHandling = NullValueHandling.Ignore)] + public FolderFacet Folder { get; set; } + + [JsonProperty("fileSystemInfo", NullValueHandling = NullValueHandling.Ignore)] + public FileSystemInfoFacet FileSystemInfo { get; set; } + + [JsonProperty("package", NullValueHandling = NullValueHandling.Ignore)] + public PackageFacet Package { get; set; } + + [JsonProperty("deleted", NullValueHandling = NullValueHandling.Ignore)] + public DeletedFacet Deleted { get; set; } + + [JsonProperty("remoteItem", NullValueHandling = NullValueHandling.Ignore)] + public RemoteItemFacet RemoteItem { get; set; } + + [JsonProperty("root", NullValueHandling = NullValueHandling.Ignore)] + public RootFacet Root { get; set; } + + [JsonProperty("specialFolder", NullValueHandling = NullValueHandling.Ignore)] + public SpecialFolderFacet SpecialFolder { get; set; } + + [JsonProperty("@microsoft.graph.conflictBehavior", NullValueHandling = NullValueHandling.Ignore)] + [JsonConverter(typeof(StringEnumConverter))] + public ConflictBehavior? ConflictBehavior { get; set; } + + [JsonProperty("@microsoft.graph.downloadUrl", NullValueHandling = NullValueHandling.Ignore)] + public string DownloadUrl { get; set; } + + [JsonIgnore] + public bool IsFile + { + get + { + return this.File != null; + } + } + + [JsonIgnore] + public bool IsFolder + { + get + { + return this.Folder != null; + } + } + + [JsonIgnore] + public bool IsPackage + { + get + { + return this.Package != null; + } + } + + [JsonIgnore] + public bool TreatLikeFolder + { + get + { + return this.IsFolder || this.IsPackage; + } + } + + [JsonIgnore] + public bool IsDeleted + { + get + { + return this.Deleted != null; + } + } + + [JsonIgnore] + public bool IsRemoteItem + { + get + { + return this.RemoteItem != null; + } + } + + [JsonIgnore] + public bool IsRoot + { + get + { + return this.Root != null; + } + } + + [JsonIgnore] + public bool IsSpecialFolder + { + get + { + return this.SpecialFolder != null; + } + } + } + + public class GraphCollection + { + [JsonProperty("value", NullValueHandling = NullValueHandling.Ignore)] + public T[] Value { get; set; } + + [JsonProperty("@odata.nextLink", NullValueHandling = NullValueHandling.Ignore)] + public string ODataNextLink { get; set; } + } + + public class UploadSession + { + [JsonProperty("uploadUrl", NullValueHandling = NullValueHandling.Ignore)] + public string UploadUrl { get; set; } + + [JsonProperty("expirationDateTime", NullValueHandling = NullValueHandling.Ignore)] + public DateTimeOffset? ExpirationDateTime { get; set; } + + [JsonProperty("nextExpectedRanges", NullValueHandling = NullValueHandling.Ignore)] + public string[] NextExpectedRanges { get; set; } + } +} diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs new file mode 100644 index 000000000..20916fd8a --- /dev/null +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs @@ -0,0 +1,63 @@ +using System.Collections.Generic; + +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Backend +{ + public class MicrosoftGroup : MicrosoftGraphBackend + { + private const string GROUP_ID_OPTION = "group-id"; + + private readonly string drivePath; + + public MicrosoftGroup() { } // Constructor needed for dynamic loading to find it + + public MicrosoftGroup(string url, Dictionary options) + : base(url, options) + { + string groupId; + if (options.TryGetValue(GROUP_ID_OPTION, out groupId)) + { + this.drivePath = string.Format("/groups/{0}", groupId); + } + else + { + throw new UserInformationException(Strings.MicrosoftGroup.MissingGroupId); + } + } + + public override string ProtocolKey + { + get { return "msgroup"; } + } + + public override string DisplayName + { + get { return Strings.MicrosoftGroup.DisplayName; } + } + + protected override string DrivePath + { + get { return this.drivePath; } + } + + protected override DescriptionTemplateDelegate DescriptionTemplate + { + get + { + return Strings.MicrosoftGroup.Description; + } + } + + protected override IList AdditionalSupportedCommands + { + get + { + return new ICommandLineArgument[] + { + new CommandLineArgument(GROUP_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.MicrosoftGroup.GroupIdShort, Strings.MicrosoftGroup.GroupIdLong), + }; + } + } + } +} diff --git a/Duplicati/Library/Backend/OneDrive/OneDriveV2.cs b/Duplicati/Library/Backend/OneDrive/OneDriveV2.cs new file mode 100644 index 000000000..366617c93 --- /dev/null +++ b/Duplicati/Library/Backend/OneDrive/OneDriveV2.cs @@ -0,0 +1,65 @@ +using System.Collections.Generic; + +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Backend +{ + public class OneDriveV2 : MicrosoftGraphBackend + { + private const string DRIVE_ID_OPTION = "drive-id"; + + private const string DEFAULT_DRIVE_PATH = "/me/drive"; + + private readonly string drivePath; + + public OneDriveV2() { } // Constructor needed for dynamic loading to find it + + public OneDriveV2(string url, Dictionary options) + : base(url, options) + { + string driveId; + if (options.TryGetValue(DRIVE_ID_OPTION, out driveId)) + { + this.drivePath = string.Format("/drives/{0}", driveId); + } + else + { + this.drivePath = DEFAULT_DRIVE_PATH; + } + } + + public override string ProtocolKey + { + get { return "onedrivev2"; } + } + + public override string DisplayName + { + get { return Strings.OneDriveV2.DisplayName; } + } + + protected override string DrivePath + { + get { return this.drivePath; } + } + + protected override DescriptionTemplateDelegate DescriptionTemplate + { + get + { + return Strings.OneDriveV2.Description; + } + } + + protected override IList AdditionalSupportedCommands + { + get + { + return new ICommandLineArgument[] + { + new CommandLineArgument(DRIVE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.OneDriveV2.DriveIdShort, Strings.OneDriveV2.DriveIdLong(DEFAULT_DRIVE_PATH)), + }; + } + } + } +} diff --git a/Duplicati/Library/Backend/OneDrive/SharePointV2.cs b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs new file mode 100644 index 000000000..b2f40aee0 --- /dev/null +++ b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs @@ -0,0 +1,63 @@ +using System.Collections.Generic; + +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Backend +{ + public class SharePointV2 : MicrosoftGraphBackend + { + private const string SITE_ID_OPTION = "site-id"; + + private readonly string drivePath; + + public SharePointV2() { } // Constructor needed for dynamic loading to find it + + public SharePointV2(string url, Dictionary options) + : base(url, options) + { + string siteId; + if (options.TryGetValue(SITE_ID_OPTION, out siteId)) + { + this.drivePath = string.Format("/sites/{0}", siteId); + } + else + { + throw new UserInformationException(Strings.SharePointV2.MissingSiteId); + } + } + + public override string ProtocolKey + { + get { return "sharepoint"; } + } + + public override string DisplayName + { + get { return Strings.SharePointV2.DisplayName; } + } + + protected override string DrivePath + { + get { return this.drivePath; } + } + + protected override DescriptionTemplateDelegate DescriptionTemplate + { + get + { + return Strings.SharePointV2.Description; + } + } + + protected override IList AdditionalSupportedCommands + { + get + { + return new ICommandLineArgument[] + { + new CommandLineArgument(SITE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.SharePointV2.SiteIdShort, Strings.SharePointV2.SiteIdLong), + }; + } + } + } +} diff --git a/Duplicati/Library/Backend/OneDrive/Strings.cs b/Duplicati/Library/Backend/OneDrive/Strings.cs index 64c9718c6..7c564df11 100644 --- a/Duplicati/Library/Backend/OneDrive/Strings.cs +++ b/Duplicati/Library/Backend/OneDrive/Strings.cs @@ -11,7 +11,45 @@ namespace Duplicati.Library.Backend.Strings public static string AuthidShort { get { return LC.L(@"The authorization code"); } } public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); } public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files on Microsoft OneDrive. Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); } + } + internal static class MicrosoftGraph + { + public static string AuthIdShort { get { return OneDrive.AuthidShort; } } + public static string AuthIdLong(string url) { return OneDrive.AuthidLong(url); } + public static string MissingAuthId(string url) { return LC.L(@"No Auth-ID was provided - you can get one from {0}", url); } + public static string FragmentSizeShort { get { return LC.L(@"Fragment size for large uploads"); } } + public static string FragmentSizeLong { get { return LC.L(@"Size of individual fragments which are uploaded separately for large files. It is recommended to be between 5-10 MiB (though a smaller value may work better on a slower or less reliable connection), and to be a multiple of 320 KiB."); } } + public static string FragmentRetryCountShort { get { return LC.L(@"Number of retries for each fragment"); } } + public static string FragmentRetryCountLong { get { return LC.L(@"Number of retry attempts made for each fragment before failing the overall file upload"); } } + public static string FragmentRetryDelayShort { get { return LC.L(@"Millisecond delay between fragment errors"); } } + public static string FragmentRetryDelayLong { get { return LC.L(@"Amount of time (in milliseconds) to wait between failures when uploading fragments"); } } + } + + internal static class OneDriveV2 + { + public static string DisplayName { get { return LC.L(@"Microsoft OneDrive v2"); } } + public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files in Microsoft OneDrive or Microsoft OneDrive for Business via the Microsoft Graph API. Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); } + public static string DriveIdShort { get { return LC.L(@"Optional ID of the drive"); } } + public static string DriveIdLong(string defaultDrive) { return LC.L(@"ID of the drive to store data in. If no drive is specified, the default OneDrive or OneDrive for Business drive will be used via '{0}'.", defaultDrive); } + } + + internal static class SharePointV2 + { + public static string DisplayName { get { return LC.L(@"Microsoft SharePoint v2"); } } + public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files in a Microsoft SharePoint site via the Microsoft Graph API. Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); } + public static string SiteIdShort { get { return LC.L(@"ID of the site"); } } + public static string SiteIdLong { get { return LC.L(@"ID of the site to store data in"); } } + public static string MissingSiteId { get { return LC.L(@"No site ID was provided"); } } + } + + internal static class MicrosoftGroup + { + public static string DisplayName { get { return LC.L(@"Microsoft Group"); } } + public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files in a Microsoft Office 365 Group via the Microsoft Graph API. Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); } + public static string GroupIdShort { get { return LC.L(@"ID of the group"); } } + public static string GroupIdLong { get { return LC.L(@"ID of the group to store data in"); } } + public static string MissingGroupId { get { return LC.L(@"No group ID was provided"); } } } } From f4c61008739ebf5d58adb4dfde3f399f6ce45d7d Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 15:07:46 -0600 Subject: [PATCH 11/36] Add Duplicati + version as the user agent in the MS Graph backends. --- Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index 7f8f1ad4f..503e65fe6 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; +using System.Reflection; using System.Text; using System.Threading; @@ -68,6 +69,8 @@ namespace Duplicati.Library.Backend /// private const int UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE = 320 * 1024; + private static readonly string USER_AGENT_VERSION = Assembly.GetExecutingAssembly().GetName().Version.ToString(); + private static readonly HttpMethod PatchMethod = new HttpMethod("PATCH"); protected delegate string DescriptionTemplateDelegate(string mssadescription, string mssalink, string msopdescription, string msoplink); @@ -110,6 +113,7 @@ namespace Duplicati.Library.Backend this.m_authenticator = new OAuthHttpMessageHandler(authid, this.ProtocolKey); this.m_client = new HttpClient(this.m_authenticator, true); this.m_client.BaseAddress = new System.Uri(BASE_ADDRESS); + this.m_client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("Duplicati", USER_AGENT_VERSION)); // Extract out the path to the backup root folder from the given URI var uri = new Utility.Uri(url); From 94341a640dfa5a1647460922a39de58390fef22d Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 15:08:30 -0600 Subject: [PATCH 12/36] Implement DNSName method in MicrosoftGraphBackend base class. --- .../Backend/OneDrive/MicrosoftGraphBackend.cs | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index 503e65fe6..ce7bb3e2f 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -83,6 +83,8 @@ namespace Duplicati.Library.Backend private readonly int fragmentRetryCount; private readonly int fragmentRetryDelay; // In milliseconds + private string[] dnsNames = null; + protected MicrosoftGraphBackend() { } // Constructor needed for dynamic loading to find it protected MicrosoftGraphBackend(string url, Dictionary options) @@ -152,9 +154,34 @@ namespace Duplicati.Library.Backend } } - public string DNSName + public string[] DNSName { - get { return null; } + get + { + if (this.dnsNames == null) + { + // The DNS names that this instance may need to access include: + // - Core graph API endpoint + // - Upload session endpoint (which seems to be different depending on the drive being accessed - not sure if it can vary for a single drive) + // To get the upload session endpoint, we can start an upload session and then immediately cancel it. + // We pick a random file name (using a guid) to make sure we don't conflict with an existing file + string dnsTestFile = string.Format("DNSNameTest-{0}", Guid.NewGuid()); + UploadSession uploadSession = this.Post(string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.m_path, NormalizeSlashes(dnsTestFile)), null); + + // Cancelling an upload session is done by sending a DELETE to the upload URL + var request = new HttpRequestMessage(HttpMethod.Delete, uploadSession.UploadUrl); + var response = this.m_client.SendAsync(request).Await(); + this.CheckResponse(response); + + this.dnsNames = new[] + { + new System.Uri(BASE_ADDRESS).Host, + new System.Uri(uploadSession.UploadUrl).Host, + }; + } + + return this.dnsNames; + } } public IQuotaInfo Quota From c57d7e75cd5eddf4ef4d9b2a84d290a7854e0a4a Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 15:09:06 -0600 Subject: [PATCH 13/36] Add helpId parameter to UserInformationExceptions in MicrosoftGraphBackends. --- Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs | 2 +- Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs | 2 +- Duplicati/Library/Backend/OneDrive/SharePointV2.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index ce7bb3e2f..ac3d79fa3 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -92,7 +92,7 @@ namespace Duplicati.Library.Backend string authid; options.TryGetValue(AUTHID_OPTION, out authid); if (string.IsNullOrEmpty(authid)) - throw new UserInformationException(Strings.MicrosoftGraph.MissingAuthId(OAuthHelper.OAUTH_LOGIN_URL(this.ProtocolKey))); + throw new UserInformationException(Strings.MicrosoftGraph.MissingAuthId(OAuthHelper.OAUTH_LOGIN_URL(this.ProtocolKey)), "MicrosoftGraphBackendMissingAuthId"); string fragmentSizeStr; if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, out fragmentSizeStr) && int.TryParse(fragmentSizeStr, out this.fragmentSize))) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs index 20916fd8a..60318077c 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs @@ -22,7 +22,7 @@ namespace Duplicati.Library.Backend } else { - throw new UserInformationException(Strings.MicrosoftGroup.MissingGroupId); + throw new UserInformationException(Strings.MicrosoftGroup.MissingGroupId, "MicrosoftGroupMissingGroupId"); } } diff --git a/Duplicati/Library/Backend/OneDrive/SharePointV2.cs b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs index b2f40aee0..6fe94fb42 100644 --- a/Duplicati/Library/Backend/OneDrive/SharePointV2.cs +++ b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs @@ -22,7 +22,7 @@ namespace Duplicati.Library.Backend } else { - throw new UserInformationException(Strings.SharePointV2.MissingSiteId); + throw new UserInformationException(Strings.SharePointV2.MissingSiteId, "SharePointMissingSiteId"); } } From 5646e3de9dd88824fb4c6ffd2c8f12ea6020c0b9 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 15:26:17 -0600 Subject: [PATCH 14/36] Add a test for the DNSName property to the BackendTester and report the DNS names it returns. --- .../CommandLine/BackendTester/Program.cs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index 9b97f76a6..0714e88da 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -415,6 +415,28 @@ namespace Duplicati.CommandLine.BackendTester } } } + + // Test DNSName lookup + Console.WriteLine("Checking DNS names used by this backend..."); + try + { + string[] dnsNames = backend.DNSName; + if (dnsNames != null) + { + foreach (string dnsName in dnsNames) + { + Console.WriteLine(dnsName); + } + } + else + { + Console.WriteLine("No DNS names reported"); + } + } + catch (Exception ex) + { + Console.WriteLine("*** Checking DNSName failed: {0}", ex); + } } finally { From dbb28c5f62496af72834a1d4e97a1de2ade66435 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 15:26:52 -0600 Subject: [PATCH 15/36] Make sure DNS names reported by Microsoft Graph backends are unique. --- Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index ac3d79fa3..7b2768193 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -168,7 +168,7 @@ namespace Duplicati.Library.Backend string dnsTestFile = string.Format("DNSNameTest-{0}", Guid.NewGuid()); UploadSession uploadSession = this.Post(string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.m_path, NormalizeSlashes(dnsTestFile)), null); - // Cancelling an upload session is done by sending a DELETE to the upload URL + // Canceling an upload session is done by sending a DELETE to the upload URL var request = new HttpRequestMessage(HttpMethod.Delete, uploadSession.UploadUrl); var response = this.m_client.SendAsync(request).Await(); this.CheckResponse(response); @@ -177,7 +177,9 @@ namespace Duplicati.Library.Backend { new System.Uri(BASE_ADDRESS).Host, new System.Uri(uploadSession.UploadUrl).Host, - }; + } + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); } return this.dnsNames; From 6c05da925ecf4313daa840a6645b38678cc63f1c Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 20 Mar 2018 15:28:57 -0600 Subject: [PATCH 16/36] Explicitly request the 'replace' conflict resolution behavior when uploading large files. --- .../Backend/OneDrive/MicrosoftGraphBackend.cs | 14 +++++++++++--- .../Backend/OneDrive/MicrosoftGraphTypes.cs | 3 +++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index 7b2768193..d09d0e03a 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -328,6 +328,10 @@ namespace Duplicati.Library.Backend // but also states that the nextExpectedRanges value returned may indicate multiple ranges... // For now, this plays it safe and does a sequential upload. HttpRequestMessage createSessionRequest = new HttpRequestMessage(HttpMethod.Post, string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename))); + + // Indicate that we want to replace any existing content with this new data we're uploading + StringContent createSessionContent = this.PrepareContent(new UploadSession() { Item = new DriveItem() { ConflictBehavior = ConflictBehavior.Replace } }); + HttpResponseMessage createSessionResponse = this.m_client.SendAsync(createSessionRequest).Await(); UploadSession uploadSession = this.ParseResponse(createSessionResponse); @@ -392,8 +396,7 @@ namespace Duplicati.Library.Backend } } - // Note: On the last request, the json result includes the default properties of the item that was uploaded, - // instead of just the upload session results. + // Note: On the last request, the json result includes the default properties of the item that was uploaded var result = this.ParseResponse(response); // If we successfully sent this piece, then we can break out of the retry loop @@ -479,7 +482,7 @@ namespace Duplicati.Library.Backend var request = new HttpRequestMessage(method, url); if (body != null) { - request.Content = new StringContent(JsonConvert.SerializeObject(body), Encoding.UTF8, "application/json"); + request.Content = this.PrepareContent(body); } return this.SendRequest(request); @@ -533,5 +536,10 @@ namespace Duplicati.Library.Backend return this.m_serializer.Deserialize(jsonReader); } } + + private StringContent PrepareContent(T body) + { + return new StringContent(JsonConvert.SerializeObject(body), Encoding.UTF8, "application/json"); + } } } diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs index d65feeaea..3f8575f29 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs @@ -511,5 +511,8 @@ namespace Duplicati.Library.Backend.MicrosoftGraph [JsonProperty("nextExpectedRanges", NullValueHandling = NullValueHandling.Ignore)] public string[] NextExpectedRanges { get; set; } + + [JsonProperty("item", NullValueHandling = NullValueHandling.Ignore)] + public DriveItem Item { get; set; } } } From 55c182b9463f82d9d92c3233676e9f939c1b9cc2 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Wed, 21 Mar 2018 11:16:27 -0600 Subject: [PATCH 17/36] Strip authorization header and access token out of Microsoft Graph exception messages. --- Duplicati/Library/Backend/OneDrive/Exceptions.cs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/OneDrive/Exceptions.cs b/Duplicati/Library/Backend/OneDrive/Exceptions.cs index 2aa732d17..fa8e154ea 100644 --- a/Duplicati/Library/Backend/OneDrive/Exceptions.cs +++ b/Duplicati/Library/Backend/OneDrive/Exceptions.cs @@ -1,5 +1,7 @@ using System; using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading.Tasks; using Duplicati.Library.Utility; @@ -9,6 +11,8 @@ namespace Duplicati.Library.Backend.MicrosoftGraph { public class MicrosoftGraphException : Exception { + private static readonly Regex authorizationHeaderRemover = new Regex(@"Authorization:\s*Bearer\s+[A-Za-z0-9+/=]+", RegexOptions.IgnoreCase); + public MicrosoftGraphException(HttpResponseMessage response) : this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage.RequestUri), response) { @@ -32,8 +36,14 @@ namespace Duplicati.Library.Backend.MicrosoftGraph { if (response != null) { - string content = response.Content.ReadAsStringAsync().Await(); - return string.Format("{0}\n{1}\n{2}", response.RequestMessage, response, JsonConvert.SerializeObject(JsonConvert.DeserializeObject(content), Formatting.Indented)); + // Start to read the content + Task content = response.Content.ReadAsStringAsync(); + + // Since the exception message may be saved / sent in logs, we want to prevent the authorization header from being included. + // it wouldn't be as bad as recording the username/password in logs, since the token will expire, but it doesn't hurt to be safe. + // So we replace anything in the request that looks like the auth header with a safe version. + string requestMessage = authorizationHeaderRemover.Replace(response.RequestMessage.ToString(), "Authorization: Bearer ABC...XYZ"); + return string.Format("{0}\n{1}\n{2}", requestMessage, response, JsonConvert.SerializeObject(JsonConvert.DeserializeObject(content.Await()), Formatting.Indented)); } else { From 97a93a071f2df0916337fc9308c7d8a403b1d63b Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Wed, 21 Mar 2018 11:23:50 -0600 Subject: [PATCH 18/36] Make m_oauth readonly in OAuthHttpMessageHandler. --- .../Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs index 7bc67a69c..affc0e77d 100644 --- a/Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs @@ -30,7 +30,7 @@ namespace Duplicati.Library /// public const string DISABLE_AUTHENTICATION_PROPERTY = "OAuthHttpMessageHandler_DisableAuthentication"; - private OAuthHelper m_oauth; + private readonly OAuthHelper m_oauth; public OAuthHttpMessageHandler(string authid, string protocolKey) { From 438c84f824ba379655dba12fb1da0e95ae6a8290 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Wed, 21 Mar 2018 11:32:41 -0600 Subject: [PATCH 19/36] Ensure custom fragment sizes fit the recommended sizes (under 60 MiB and multiple of 320 KiB) --- .../Library/Backend/OneDrive/MicrosoftGraphBackend.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index d09d0e03a..e0f6d531f 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -95,7 +95,16 @@ namespace Duplicati.Library.Backend throw new UserInformationException(Strings.MicrosoftGraph.MissingAuthId(OAuthHelper.OAUTH_LOGIN_URL(this.ProtocolKey)), "MicrosoftGraphBackendMissingAuthId"); string fragmentSizeStr; - if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, out fragmentSizeStr) && int.TryParse(fragmentSizeStr, out this.fragmentSize))) + if (options.TryGetValue(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, out fragmentSizeStr) && int.TryParse(fragmentSizeStr, out this.fragmentSize)) + { + // Make sure the fragment size is a multiple of the desired multiple size. + // If it isn't, we round down to the nearest multiple below it. + this.fragmentSize = (this.fragmentSize / UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE) * UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE; + + // Make sure the fragment size isn't larger than the maximum + this.fragmentSize = Math.Min(this.fragmentSize, UPLOAD_SESSION_FRAGMENT_MAX_SIZE); + } + else { this.fragmentSize = UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE; } From 077fcc78b12b701fa818b43077812e7429cd0929 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Wed, 21 Mar 2018 17:09:37 -0600 Subject: [PATCH 20/36] Fix LogfilelogfiltersLong help parameter, so that running the main help page doesn't crash. --- Duplicati/Library/Main/Strings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index c3f0562cd..23d317db3 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -248,7 +248,7 @@ namespace Duplicati.Library.Main.Strings public static string LogfileloglevelLong { get { return LC.L(@"Specifies the amount of log information to write into the file specified by --log-file"); } } public static string LogfileloglevelShort { get { return LC.L(@"Log file information level"); } } public static string LogfilelogfiltersShort { get { return LC.L(@"Applies filters to the file log data"); } } - public static string LogfilelogfiltersLong(string delimiter) { return LC.L(@"This option accepts filters that removes or includes messages regardless of their log level. Multiple filters are supported by separating with {0}. Filters are matched against the log tag and assumed to be including, unless they start with '-'. Regular expressions are supported within hard braces. Example: ""+Path*{0}+*Mail*{0}-[.*DNS]"" "); } + public static string LogfilelogfiltersLong(string delimiter) { return LC.L(@"This option accepts filters that removes or includes messages regardless of their log level. Multiple filters are supported by separating with {0}. Filters are matched against the log tag and assumed to be including, unless they start with '-'. Regular expressions are supported within hard braces. Example: ""+Path*{0}+*Mail*{0}-[.*DNS]"" ", delimiter); } public static string ConsoleloglevelLong { get { return LC.L(@"Specifies the amount of log information to write as console output"); } } public static string ConsoleloglevelShort { get { return LC.L(@"Console information level"); } } public static string ConsolelogfiltersShort { get { return LC.L(@"Applies filters to the console log data"); } } From 258799266eaabcd96461cfcf38d4278621570f02 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Thu, 22 Mar 2018 14:40:37 -0600 Subject: [PATCH 21/36] SharePoint and Group drives appear to report quota info that is all zeros. In that case, pretend that no quota info was reported instead of causing incorrect quota warnings and errors. --- .../Library/Backend/OneDrive/MicrosoftGraphBackend.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index e0f6d531f..cc51a91f3 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -202,7 +202,13 @@ namespace Duplicati.Library.Backend Drive driveInfo = this.Get(this.DrivePrefix); if (driveInfo.Quota != null) { - return new QuotaInfo(driveInfo.Quota.Total, driveInfo.Quota.Remaining); + // Some sources (SharePoint for example) seem to return 0 for these values even when the quota isn't exceeded.. + // As a special test, if all the returned values are 0, we pretend that no quota was reported. + // This way we don't send spurious warnings because the quota looks like it is exceeded. + if (driveInfo.Quota.Total != 0 || driveInfo.Quota.Remaining != 0 || driveInfo.Quota.Used != 0) + { + return new QuotaInfo(driveInfo.Quota.Total, driveInfo.Quota.Remaining); + } } return null; From 987cda7e1d5af12b87a2c6acd0f1085c973f669a Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Thu, 22 Mar 2018 14:41:19 -0600 Subject: [PATCH 22/36] When uploading large files, MS graph backends need to parse the response inside the try block for retries to work (since that is where errors are thrown) --- Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index cc51a91f3..7047e0b89 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -376,6 +376,9 @@ namespace Duplicati.Library.Backend try { response = this.m_client.SendAsync(request).Await(); + + // Note: On the last request, the json result includes the default properties of the item that was uploaded + var result = this.ParseResponse(response); } catch (MicrosoftGraphException ex) { @@ -411,9 +414,6 @@ namespace Duplicati.Library.Backend } } - // Note: On the last request, the json result includes the default properties of the item that was uploaded - var result = this.ParseResponse(response); - // If we successfully sent this piece, then we can break out of the retry loop break; } From 5bec41b665e158869cb618c88aad324caed68f2c Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Thu, 22 Mar 2018 15:20:17 -0600 Subject: [PATCH 23/36] Update both SharePoint and Office 365 group backends to support more human-discernable methods of specifying the backend destination. For SharePoint, this means instead of giving the site ID as a parameter (and a relative path as the backup URL), the full URL to the folder in SharePoint (e.g., 'sharepoint://{tenant}.sharepoint.com/{PathToSite}/{DocumentBase}/{subfolder}') can be given, and the site ID will be looked up. (This should match the existing SharePoint backend, including the convention of using '//' to indicate the document root, though I can't explicitly test that since the SharePoint sites I have access to require 2FA, which the old backend doesn't support.) I've tested both explicit ID and site path methods of specifying the destination, and they appear to be working. For Office 365 groups, it now supports looking up a group by its email address. --- .../Backend/OneDrive/MicrosoftGraphBackend.cs | 54 +++++----- .../Backend/OneDrive/MicrosoftGraphTypes.cs | 36 ++++++- .../Backend/OneDrive/MicrosoftGroup.cs | 46 +++++++- .../Library/Backend/OneDrive/SharePointV2.cs | 101 +++++++++++++++++- Duplicati/Library/Backend/OneDrive/Strings.cs | 8 +- 5 files changed, 209 insertions(+), 36 deletions(-) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index 7047e0b89..3af282422 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -127,9 +127,7 @@ namespace Duplicati.Library.Backend this.m_client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("Duplicati", USER_AGENT_VERSION)); // Extract out the path to the backup root folder from the given URI - var uri = new Utility.Uri(url); - - this.m_path = NormalizeSlashes(Utility.Uri.UrlDecode(uri.HostAndPath)); + this.m_path = NormalizeSlashes(this.GetRootPathFromUrl(url)); } public abstract string ProtocolKey { get; } @@ -449,39 +447,25 @@ namespace Duplicati.Library.Backend } } - /// - /// Normalizes the slashes in a url fragment. For example: - /// "" => "" - /// "test" => "/test" - /// "test/" => "/test" - /// "a\b" => "/a/b" - /// - /// Url fragment to normalize - /// Normalized fragment - private static string NormalizeSlashes(string url) + protected virtual string GetRootPathFromUrl(string url) { - url = url.Replace('\\', '/'); + // Extract out the path to the backup root folder from the given URI + var uri = new Utility.Uri(url); - if (url.Length != 0 && !url.StartsWith("/", StringComparison.Ordinal)) - url = "/" + url; - - if (url.EndsWith("/", StringComparison.Ordinal)) - url = url.Substring(0, url.Length - 1); - - return url; + return Utility.Uri.UrlDecode(uri.HostAndPath); } - private T Get(string url) + protected T Get(string url) { return this.SendRequest(HttpMethod.Get, url); } - private T Post(string url, T body) + protected T Post(string url, T body) { return this.SendRequest(HttpMethod.Post, url, body); } - private T Patch(string url, T body) + protected T Patch(string url, T body) { return this.SendRequest(PatchMethod, url, body); } @@ -552,6 +536,28 @@ namespace Duplicati.Library.Backend } } + /// + /// Normalizes the slashes in a url fragment. For example: + /// "" => "" + /// "test" => "/test" + /// "test/" => "/test" + /// "a\b" => "/a/b" + /// + /// Url fragment to normalize + /// Normalized fragment + private static string NormalizeSlashes(string url) + { + url = url.Replace('\\', '/'); + + if (url.Length != 0 && !url.StartsWith("/", StringComparison.Ordinal)) + url = "/" + url; + + if (url.EndsWith("/", StringComparison.Ordinal)) + url = url.Substring(0, url.Length - 1); + + return url; + } + private StringContent PrepareContent(T body) { return new StringContent(JsonConvert.SerializeObject(body), Encoding.UTF8, "application/json"); diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs index 3f8575f29..1e2332ee4 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphTypes.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using Newtonsoft.Json; using Newtonsoft.Json.Converters; @@ -122,7 +123,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph [JsonProperty("state", NullValueHandling = NullValueHandling.Ignore)] [JsonConverter(typeof(StringEnumConverter))] - public QuotaState State { get; set; } + public QuotaState? State { get; set; } } public enum DriveType @@ -515,4 +516,37 @@ namespace Duplicati.Library.Backend.MicrosoftGraph [JsonProperty("item", NullValueHandling = NullValueHandling.Ignore)] public DriveItem Item { get; set; } } + + public class SharePointSite : BaseItem + { + } + + public class Group : BaseItem + { + [JsonProperty("mail", NullValueHandling = NullValueHandling.Ignore)] + public string Mail { get; set; } + + [JsonProperty("proxyAddresses", NullValueHandling = NullValueHandling.Ignore)] + public string[] ProxyAddresses { get; set; } + + [JsonProperty("mailNickname", NullValueHandling = NullValueHandling.Ignore)] + public string MailNickname { get; set; } + + [JsonProperty("groupTypes", NullValueHandling = NullValueHandling.Ignore)] + public string[] GroupTypes { get; set; } + + [JsonIgnore] + public bool IsUnifiedGroup + { + get + { + if (this.GroupTypes != null) + { + return this.GroupTypes.Any(type => string.Equals("Unified", type, StringComparison.OrdinalIgnoreCase)); + } + + return false; + } + } + } } diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs index 60318077c..9b138d452 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs @@ -1,11 +1,14 @@ using System.Collections.Generic; +using System.Linq; +using Duplicati.Library.Backend.MicrosoftGraph; using Duplicati.Library.Interface; namespace Duplicati.Library.Backend { public class MicrosoftGroup : MicrosoftGraphBackend { + private const string GROUP_EMAIL_OPTION = "group-email"; private const string GROUP_ID_OPTION = "group-id"; private readonly string drivePath; @@ -15,15 +18,30 @@ namespace Duplicati.Library.Backend public MicrosoftGroup(string url, Dictionary options) : base(url, options) { - string groupId; - if (options.TryGetValue(GROUP_ID_OPTION, out groupId)) + string groupId = null; + string groupEmail; + if (options.TryGetValue(GROUP_EMAIL_OPTION, out groupEmail)) { - this.drivePath = string.Format("/groups/{0}", groupId); + groupId = this.GetGroupIdFromEmail(groupEmail); } - else + + string groupIdOption; + if (options.TryGetValue(GROUP_ID_OPTION, out groupIdOption)) + { + if (!string.IsNullOrEmpty(groupId) && !string.Equals(groupId, groupIdOption)) + { + throw new UserInformationException(Strings.MicrosoftGroup.ConflictingGroupId(groupIdOption, groupId), "MicrosoftGroupConflictingGroupId"); + } + + groupId = groupIdOption; + } + + if (string.IsNullOrEmpty(groupId)) { throw new UserInformationException(Strings.MicrosoftGroup.MissingGroupId, "MicrosoftGroupMissingGroupId"); } + + this.drivePath = string.Format("/groups/{0}/drive", groupId); } public override string ProtocolKey @@ -59,5 +77,25 @@ namespace Duplicati.Library.Backend }; } } + + private string GetGroupIdFromEmail(string email) + { + // We can get all groups that have the given email as one of their addresses with: + // https://graph.microsoft.com/v1.0/groups?$filter=mail eq '{email}' or proxyAddresses/any(x:x eq 'smtp:{email}') + string request = string.Format("{0}/groups?$filter=mail eq '{1}' or proxyAddresses/any(x:x eq 'smtp:{1}')", this.ApiVersion, email); + GraphCollection groups = this.Get>(request); + if (groups.Value.Length == 0) + { + throw new UserInformationException(Strings.MicrosoftGroup.NoGroupsWithEmail(email), "MicrosoftGroupNoGroupsWithEmail"); + } + else if (groups.Value.Length > 1) + { + throw new UserInformationException(Strings.MicrosoftGroup.MultipleGroupsWithEmail(email), "MicrosoftGroupMultipleGroupsWithEmail"); + } + else + { + return groups.Value.Single().Id; + } + } } } diff --git a/Duplicati/Library/Backend/OneDrive/SharePointV2.cs b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs index 6fe94fb42..662fb1c27 100644 --- a/Duplicati/Library/Backend/OneDrive/SharePointV2.cs +++ b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs @@ -1,5 +1,9 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Duplicati.Library.Backend.MicrosoftGraph; using Duplicati.Library.Interface; namespace Duplicati.Library.Backend @@ -9,21 +13,31 @@ namespace Duplicati.Library.Backend private const string SITE_ID_OPTION = "site-id"; private readonly string drivePath; + private string siteId = null; public SharePointV2() { } // Constructor needed for dynamic loading to find it public SharePointV2(string url, Dictionary options) : base(url, options) { - string siteId; - if (options.TryGetValue(SITE_ID_OPTION, out siteId)) + // Check to see if a site ID was explicitly provided + string siteIdOption; + if (options.TryGetValue(SITE_ID_OPTION, out siteIdOption)) { - this.drivePath = string.Format("/sites/{0}", siteId); + if (!string.IsNullOrEmpty(this.siteId) && !string.Equals(this.siteId, siteIdOption)) + { + throw new UserInformationException(Strings.SharePointV2.ConflictingSiteId(siteIdOption, this.siteId), "SharePointConflictingSiteId"); + } + + this.siteId = siteIdOption; } - else + + if (string.IsNullOrEmpty(this.siteId)) { throw new UserInformationException(Strings.SharePointV2.MissingSiteId, "SharePointMissingSiteId"); } + + this.drivePath = string.Format("/sites/{0}/drive", this.siteId); } public override string ProtocolKey @@ -59,5 +73,82 @@ namespace Duplicati.Library.Backend }; } } + + /// + /// This method takes an input URL, which could be either in the format that the old SharePoint backend accepted or + /// just the host and path (effectively, just the path) that the graph base backend expects, + /// and converts it to just the local path within the drive. + /// + /// At the same time, if the given URL is in the form of a full SharePoint site URL, then it also determines the site-id and stores it. + /// + /// Input URL + /// Path within the drive + protected override string GetRootPathFromUrl(string url) + { + Uri uri = new Uri(url); + SharePointSite site = this.GetSharePointSite(uri); + if (site != null) + { + // Get the web URL of the site's main drive + try + { + Drive drive = this.Get(string.Format("{0}/sites/{1}/drive", this.ApiVersion, site.Id)); + + this.siteId = site.Id; + Uri driveWebUrl = new Uri(drive.WebUrl); + + // Make sure to replace any "//" in the original path with "/", so the substrings line up. + return uri.LocalPath.Replace("//", "/").Substring(driveWebUrl.LocalPath.Length); + } + catch (MicrosoftGraphException) + { + // Couldn't get the drive info, so assume the URL we were given isn't actually a full SharePoint site. + } + } + + return base.GetRootPathFromUrl(url); + } + + private SharePointSite GetSharePointSite(Uri url) + { + UriBuilder uri = new UriBuilder(url); + + // We can get a SharePoint site's info by querying /v1.0/sites/{hostname}:{siteWebPath}. + // Since this full URL likely has the web path as some subpart of it, we check against each subpath to see if that is a site, + // and if it is, we record the site ID. + string requestBase = string.Format("{0}/sites/{1}", this.ApiVersion, uri.Host); + + // Just like the original SharePoint backend, use the "//" as a hint at where the site might be + int siteHint = uri.Path.IndexOf("//"); + if (siteHint >= 0) + { + try + { + string request = string.Format("{0}:/{1}", requestBase, uri.Path.Substring(0, siteHint)); + return this.Get(request); + } + catch (MicrosoftGraphException) + { + // This isn't the right path + } + } + + string[] pathPieces = uri.Path.Split(new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, StringSplitOptions.RemoveEmptyEntries); + for (int i = pathPieces.Length; i > 0; i--) + { + try + { + string request = string.Format("{0}:/{1}", requestBase, string.Join("/", pathPieces.Take(i))); + return this.Get(request); + } + catch (MicrosoftGraphException) + { + // This isn't the right path + } + } + + // Couldn't find the site + return null; + } } } diff --git a/Duplicati/Library/Backend/OneDrive/Strings.cs b/Duplicati/Library/Backend/OneDrive/Strings.cs index 7c564df11..1e9932143 100644 --- a/Duplicati/Library/Backend/OneDrive/Strings.cs +++ b/Duplicati/Library/Backend/OneDrive/Strings.cs @@ -41,15 +41,19 @@ namespace Duplicati.Library.Backend.Strings public static string SiteIdShort { get { return LC.L(@"ID of the site"); } } public static string SiteIdLong { get { return LC.L(@"ID of the site to store data in"); } } public static string MissingSiteId { get { return LC.L(@"No site ID was provided"); } } + public static string ConflictingSiteId(string given, string found) { return LC.L(@"Conflicting site IDs used: given {0} but found {1}", given, found); } } internal static class MicrosoftGroup { - public static string DisplayName { get { return LC.L(@"Microsoft Group"); } } - public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files in a Microsoft Office 365 Group via the Microsoft Graph API. Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); } + public static string DisplayName { get { return LC.L(@"Microsoft Office 365 Group"); } } + public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files in a Microsoft Office 365 Group via the Microsoft Graph API. Allowed formats are ""sharepoint://tenant.sharepoint.com/{{PathToWeb}}//{{Documents}}/subfolder"" (with ""//"" being optionally used to indicate the root document folder), or just ""sharepoint://subfolder"" (in which case you must also explicitly specify the SharePoint site's ID via --site-id). Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); } public static string GroupIdShort { get { return LC.L(@"ID of the group"); } } public static string GroupIdLong { get { return LC.L(@"ID of the group to store data in"); } } public static string MissingGroupId { get { return LC.L(@"No group ID was provided"); } } + public static string NoGroupsWithEmail(string email) { return LC.L(@"No groups were found with the given email address: {0}", email); } + public static string MultipleGroupsWithEmail(string email) { return LC.L(@"Multiple groups were found with the given email address: {0}", email); } + public static string ConflictingGroupId(string given, string found) { return LC.L(@"Conflicting group IDs used: given {0} but found {1}", given, found); } } } From 7491da2add4f6a28cc2087895a1154a56d146cfd Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Thu, 22 Mar 2018 16:14:24 -0600 Subject: [PATCH 24/36] Add entry for --group-email so it isn't treated as an unknown property. --- Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs | 1 + Duplicati/Library/Backend/OneDrive/Strings.cs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs index 9b138d452..809147a0c 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs @@ -74,6 +74,7 @@ namespace Duplicati.Library.Backend return new ICommandLineArgument[] { new CommandLineArgument(GROUP_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.MicrosoftGroup.GroupIdShort, Strings.MicrosoftGroup.GroupIdLong), + new CommandLineArgument(GROUP_EMAIL_OPTION, CommandLineArgument.ArgumentType.String, Strings.MicrosoftGroup.GroupEmailShort, Strings.MicrosoftGroup.GroupEmailLong), }; } } diff --git a/Duplicati/Library/Backend/OneDrive/Strings.cs b/Duplicati/Library/Backend/OneDrive/Strings.cs index 1e9932143..fa8e76e88 100644 --- a/Duplicati/Library/Backend/OneDrive/Strings.cs +++ b/Duplicati/Library/Backend/OneDrive/Strings.cs @@ -50,6 +50,8 @@ namespace Duplicati.Library.Backend.Strings public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files in a Microsoft Office 365 Group via the Microsoft Graph API. Allowed formats are ""sharepoint://tenant.sharepoint.com/{{PathToWeb}}//{{Documents}}/subfolder"" (with ""//"" being optionally used to indicate the root document folder), or just ""sharepoint://subfolder"" (in which case you must also explicitly specify the SharePoint site's ID via --site-id). Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); } public static string GroupIdShort { get { return LC.L(@"ID of the group"); } } public static string GroupIdLong { get { return LC.L(@"ID of the group to store data in"); } } + public static string GroupEmailShort { get { return LC.L(@"Email of the group"); } } + public static string GroupEmailLong { get { return LC.L(@"Email of the group to store data in"); } } public static string MissingGroupId { get { return LC.L(@"No group ID was provided"); } } public static string NoGroupsWithEmail(string email) { return LC.L(@"No groups were found with the given email address: {0}", email); } public static string MultipleGroupsWithEmail(string email) { return LC.L(@"Multiple groups were found with the given email address: {0}", email); } From 448577f5ff786f72dd54e3de71a5ddef10e8e7f4 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Thu, 22 Mar 2018 16:15:09 -0600 Subject: [PATCH 25/36] Handle cases where the SharePoint path is given with an 'https' prefix in the UI. --- Duplicati/Library/Backend/OneDrive/SharePointV2.cs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Duplicati/Library/Backend/OneDrive/SharePointV2.cs b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs index 662fb1c27..712728d63 100644 --- a/Duplicati/Library/Backend/OneDrive/SharePointV2.cs +++ b/Duplicati/Library/Backend/OneDrive/SharePointV2.cs @@ -86,6 +86,16 @@ namespace Duplicati.Library.Backend protected override string GetRootPathFromUrl(string url) { Uri uri = new Uri(url); + + // If the user gave a URL like "https://{tenant}.sharepoint.com/path" in the UI, + // it might appear here as "sharepoint://https://{tenant}.sharepoint.com/path". + // If that's the case, the URIs Host will be https, and we should create a new URI without it. + if (string.Equals(uri.Host, "https", StringComparison.OrdinalIgnoreCase) || string.Equals(uri.Host, "http", StringComparison.OrdinalIgnoreCase)) + { + // LocalPath will already be prefixed by "//", due to the https:// part. + uri = new Uri(string.Format("{0}:{1}", uri.Scheme, uri.LocalPath)); + } + SharePointSite site = this.GetSharePointSite(uri); if (site != null) { From df64c3284021ed37df3eb97ea0e930b6e3714402 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Thu, 22 Mar 2018 16:19:49 -0600 Subject: [PATCH 26/36] Add new MS graph backends to the web UI. The AuthID link won't work until the graph API definitions are added to the Duplicati OAuth handler site, however. --- .../scripts/services/EditUriBackendConfig.js | 7 ++ .../ngax/scripts/services/EditUriBuiltins.js | 87 ++++++++++++++++--- .../ngax/scripts/services/SystemInfo.js | 3 + .../ngax/templates/backends/msgroup.html | 17 ++++ .../ngax/templates/backends/sharepoint.html | 12 +++ 5 files changed, 116 insertions(+), 10 deletions(-) create mode 100644 Duplicati/Server/webroot/ngax/templates/backends/msgroup.html create mode 100644 Duplicati/Server/webroot/ngax/templates/backends/sharepoint.html diff --git a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBackendConfig.js b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBackendConfig.js index 69f30a1d0..c4330ff8c 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBackendConfig.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBackendConfig.js @@ -80,6 +80,13 @@ backupApp.service('EditUriBackendConfig', function(AppService, AppUtils, SystemI return true; }; + this.recommend_field = function (scope, field, label, reason, continuation) { + if ((scope[field] || '').trim().length == 0) + return self.show_warning_dialog(gettextCatalog.getString('You should fill in {{field}}{{reason}}', { field: label || field, reason: reason }), continuation); + else + continuation(); + }; + this.require_server = function(scope) { if ((scope.Server || '').trim().length == 0) return self.show_error_dialog(gettextCatalog.getString('You must fill in the server name or address')); diff --git a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js index c2b07a5ac..668509b1c 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js @@ -17,6 +17,9 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, EditUriBackendConfig.templates['googledrive'] = 'templates/backends/oauth.html'; EditUriBackendConfig.templates['hubic'] = 'templates/backends/oauth.html'; EditUriBackendConfig.templates['onedrive'] = 'templates/backends/oauth.html'; + EditUriBackendConfig.templates['onedrivev2'] = 'templates/backends/oauth.html'; + EditUriBackendConfig.templates['sharepoint'] = 'templates/backends/sharepoint.html'; + EditUriBackendConfig.templates['msgroup'] = 'templates/backends/msgroup.html'; EditUriBackendConfig.templates['amzcd'] = 'templates/backends/oauth.html'; EditUriBackendConfig.templates['openstack'] = 'templates/backends/openstack.html'; EditUriBackendConfig.templates['azure'] = 'templates/backends/azure.html'; @@ -193,6 +196,9 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, EditUriBackendConfig.loaders['googledrive'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.loaders['hubic'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.loaders['onedrive'] = function() { return this['oauth-base'].apply(this, arguments); }; + EditUriBackendConfig.loaders['onedrivev2'] = function() { return this['oauth-base'].apply(this, arguments); }; + EditUriBackendConfig.loaders['sharepoint'] = function() { return this['oauth-base'].apply(this, arguments); }; + EditUriBackendConfig.loaders['msgroup'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.loaders['amzcd'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.loaders['box'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.loaders['dropbox'] = function() { return this['oauth-base'].apply(this, arguments); }; @@ -299,6 +305,8 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, EditUriBackendConfig.parsers['googledrive'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.parsers['hubic'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.parsers['onedrive'] = function() { return this['oauth-base'].apply(this, arguments); }; + EditUriBackendConfig.parsers['onedrivev2'] = function() { return this['oauth-base'].apply(this, arguments); }; + EditUriBackendConfig.parsers['sharepoint'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.parsers['amzcd'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.parsers['box'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.parsers['dropbox'] = function() { return this['oauth-base'].apply(this, arguments); }; @@ -320,6 +328,17 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, EditUriBackendConfig.mergeServerAndPath(scope); }; + EditUriBackendConfig.parsers['msgroup'] = function (scope, module, server, port, path, options) { + + scope.msgroup_group_email = options['--group-email']; + + var nukeopts = ['--group-email']; + for (var x in nukeopts) + delete options[nukeopts[x]]; + + this['oauth-base'].apply(this, arguments); + }; + EditUriBackendConfig.parsers['gcs'] = function(scope, module, server, port, path, options) { scope.gcs_location = scope.gcs_location_custom = options['--gcs-location']; @@ -435,6 +454,9 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, EditUriBackendConfig.builders['googledrive'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.builders['hubic'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.builders['onedrive'] = function() { return this['oauth-base'].apply(this, arguments); }; + EditUriBackendConfig.builders['onedrivev2'] = function() { return this['oauth-base'].apply(this, arguments); }; + EditUriBackendConfig.builders['sharepoint'] = function() { return this['oauth-base'].apply(this, arguments); }; + EditUriBackendConfig.builders['msgroup'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.builders['amzcd'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.builders['box'] = function() { return this['oauth-base'].apply(this, arguments); }; EditUriBackendConfig.builders['dropbox'] = function() { return this['oauth-base'].apply(this, arguments); }; @@ -483,6 +505,26 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, return url; }; + EditUriBackendConfig.builders['msgroup'] = function (scope) { + var opts = { + 'group-email': scope.msgroup_group_email, + 'authid': scope.AuthID, + } + + if ((opts['group-email'] || '') == '') + delete opts['group-email']; + + EditUriBackendConfig.merge_in_advanced_options(scope, opts); + + var url = AppUtils.format('{0}://{1}{2}', + scope.Backend.Key, + scope.Path || '', + AppUtils.encodeDictAsUrl(opts) + ); + + return url; + }; + EditUriBackendConfig.builders['gcs'] = function(scope) { var opts = { 'gcs-location': AppUtils.contains_value(scope.gcs_locations, scope.gcs_location) ? scope.gcs_location : scope.gcs_location_custom, @@ -643,7 +685,7 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, EditUriBackendConfig.validaters['tahoe'] = EditUriBackendConfig.validaters['ssh']; - EditUriBackendConfig.validaters['onedrive'] = function(scope, continuation) { + EditUriBackendConfig.validaters['authid-base'] = function(scope, continuation) { var res = EditUriBackendConfig.require_field(scope, 'AuthID', gettextCatalog.getString('AuthID')); @@ -651,17 +693,48 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, EditUriBackendConfig.recommend_path(scope, continuation); }; - EditUriBackendConfig.validaters['hubic'] = function(scope, continuation) { + EditUriBackendConfig.validaters['googledrive'] = EditUriBackendConfig.validaters['authid-base']; + EditUriBackendConfig.validaters['gcs'] = EditUriBackendConfig.validaters['authid-base']; + EditUriBackendConfig.validaters['amzcd'] = EditUriBackendConfig.validaters['authid-base']; + EditUriBackendConfig.validaters['box'] = EditUriBackendConfig.validaters['authid-base']; + EditUriBackendConfig.validaters['dropbox'] = EditUriBackendConfig.validaters['authid-base']; + EditUriBackendConfig.validaters['onedrive'] = EditUriBackendConfig.validaters['authid-base']; + EditUriBackendConfig.validaters['onedrivev2'] = EditUriBackendConfig.validaters['authid-base']; + EditUriBackendConfig.validaters['sharepoint'] = EditUriBackendConfig.validaters['authid-base']; + + EditUriBackendConfig.validaters['msgroup'] = function (scope, continuation) { + + EditUriBackendConfig.validaters['authid-base'](scope, function () { + var res = + EditUriBackendConfig.recommend_field(scope, 'msgroup_group_email', gettextCatalog.getString('Group email'), gettextCatalog.getString(' unless you are explicitly specifying --group-id'), continuation); + + if (res) + continuation(); + }); + }; + + EditUriBackendConfig.validaters['msgroup'] = function (scope, continuation) { + + EditUriBackendConfig.validaters['authid-base'](scope, function () { + var res = + EditUriBackendConfig.recommend_field(scope, 'msgroup_group_email', gettextCatalog.getString('Group email'), gettextCatalog.getString(' unless you are explicitly specifying --group-id'), continuation); + + if (res) + continuation(); + }); + }; + + EditUriBackendConfig.validaters['hubic'] = function (scope, continuation) { var prefix1 = 'HubiC-DeskBackup_Duplicati/'; var prefix2 = 'default/' - EditUriBackendConfig.validaters['onedrive'](scope, function() { + EditUriBackendConfig.validaters['authid-base'](scope, function () { var p = (scope.Path || '').trim(); if (p.length > 0 && p.indexOf(prefix2) != 0 && p.indexOf(prefix1) != 0) { - DialogService.dialog(gettextCatalog.getString('Adjust path name?'), gettextCatalog.getString('The path should start with "{{prefix1}}" or "{{prefix2}}", otherwise you will not be able to see the files in the HubiC web interface.\n\nDo you want to add the prefix to the path automatically?', {prefix1: prefix1, prefix2: prefix2 }), [gettextCatalog.getString('Cancel'), gettextCatalog.getString('No'), gettextCatalog.getString('Yes')], function(ix) { + DialogService.dialog(gettextCatalog.getString('Adjust path name?'), gettextCatalog.getString('The path should start with "{{prefix1}}" or "{{prefix2}}", otherwise you will not be able to see the files in the HubiC web interface.\n\nDo you want to add the prefix to the path automatically?', { prefix1: prefix1, prefix2: prefix2 }), [gettextCatalog.getString('Cancel'), gettextCatalog.getString('No'), gettextCatalog.getString('Yes')], function (ix) { if (ix == 2) { while (p.indexOf('/') == 0) p = p.substr(1); @@ -678,12 +751,6 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo, }; - EditUriBackendConfig.validaters['googledrive'] = EditUriBackendConfig.validaters['onedrive']; - EditUriBackendConfig.validaters['gcs'] = EditUriBackendConfig.validaters['onedrive']; - EditUriBackendConfig.validaters['amzcd'] = EditUriBackendConfig.validaters['onedrive']; - EditUriBackendConfig.validaters['box'] = EditUriBackendConfig.validaters['onedrive']; - EditUriBackendConfig.validaters['dropbox'] = EditUriBackendConfig.validaters['onedrive']; - EditUriBackendConfig.validaters['azure'] = function(scope, continuation) { var res = EditUriBackendConfig.require_field(scope, 'Username', gettextCatalog.getString('Account name')) && diff --git a/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js b/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js index c0ac108be..8ec89f270 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js @@ -57,6 +57,9 @@ backupApp.service('SystemInfo', function($rootScope, $timeout, $cookies, AppServ 'azure': null, 'googledrive': null, 'onedrive': null, + 'onedrivev2': null, + 'sharepoint': null, + 'msgroup': null, 'cloudfiles': null, 'gcs': null, 'openstack': null, diff --git a/Duplicati/Server/webroot/ngax/templates/backends/msgroup.html b/Duplicati/Server/webroot/ngax/templates/backends/msgroup.html new file mode 100644 index 000000000..8be273d22 --- /dev/null +++ b/Duplicati/Server/webroot/ngax/templates/backends/msgroup.html @@ -0,0 +1,17 @@ +
+ + +
+ +
+ + +
+ +
+ + +
diff --git a/Duplicati/Server/webroot/ngax/templates/backends/sharepoint.html b/Duplicati/Server/webroot/ngax/templates/backends/sharepoint.html new file mode 100644 index 000000000..4749d72f1 --- /dev/null +++ b/Duplicati/Server/webroot/ngax/templates/backends/sharepoint.html @@ -0,0 +1,12 @@ +
+ + +
+ +
+ + +
\ No newline at end of file From 5a4d170165f9d59addf2072f61dbee3b56d20419 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Fri, 23 Mar 2018 15:03:04 -0600 Subject: [PATCH 27/36] Update strings for Microsoft Group backend. --- Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs | 2 +- Duplicati/Library/Backend/OneDrive/Strings.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs index 809147a0c..335e42b87 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGroup.cs @@ -38,7 +38,7 @@ namespace Duplicati.Library.Backend if (string.IsNullOrEmpty(groupId)) { - throw new UserInformationException(Strings.MicrosoftGroup.MissingGroupId, "MicrosoftGroupMissingGroupId"); + throw new UserInformationException(Strings.MicrosoftGroup.MissingGroupIdAndEmailAddress, "MicrosoftGroupMissingGroupIdAndEmailAddress"); } this.drivePath = string.Format("/groups/{0}/drive", groupId); diff --git a/Duplicati/Library/Backend/OneDrive/Strings.cs b/Duplicati/Library/Backend/OneDrive/Strings.cs index fa8e76e88..20b15ade6 100644 --- a/Duplicati/Library/Backend/OneDrive/Strings.cs +++ b/Duplicati/Library/Backend/OneDrive/Strings.cs @@ -50,9 +50,9 @@ namespace Duplicati.Library.Backend.Strings public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files in a Microsoft Office 365 Group via the Microsoft Graph API. Allowed formats are ""sharepoint://tenant.sharepoint.com/{{PathToWeb}}//{{Documents}}/subfolder"" (with ""//"" being optionally used to indicate the root document folder), or just ""sharepoint://subfolder"" (in which case you must also explicitly specify the SharePoint site's ID via --site-id). Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); } public static string GroupIdShort { get { return LC.L(@"ID of the group"); } } public static string GroupIdLong { get { return LC.L(@"ID of the group to store data in"); } } - public static string GroupEmailShort { get { return LC.L(@"Email of the group"); } } - public static string GroupEmailLong { get { return LC.L(@"Email of the group to store data in"); } } - public static string MissingGroupId { get { return LC.L(@"No group ID was provided"); } } + public static string GroupEmailShort { get { return LC.L(@"Email address of the group"); } } + public static string GroupEmailLong { get { return LC.L(@"Email address of the group to store data in"); } } + public static string MissingGroupIdAndEmailAddress { get { return LC.L(@"No group ID or group email address was provided"); } } public static string NoGroupsWithEmail(string email) { return LC.L(@"No groups were found with the given email address: {0}", email); } public static string MultipleGroupsWithEmail(string email) { return LC.L(@"Multiple groups were found with the given email address: {0}", email); } public static string ConflictingGroupId(string given, string found) { return LC.L(@"Conflicting group IDs used: given {0} but found {1}", given, found); } From 407bead9f97b7483a410335fa59ad07b1c57ce24 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Fri, 23 Mar 2018 15:04:23 -0600 Subject: [PATCH 28/36] Change some fragment settings. Default retry delay is now 1000ms, and fragment sizes must be at least the multiple size (320 KiB) --- Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index 3af282422..b26bed0a0 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -42,7 +42,7 @@ namespace Duplicati.Library.Backend private const string UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION = "fragment-retry-delay"; private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT = 5; - private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY = 500; + private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY = 1000; /// /// Max size of file that can be uploaded in a single PUT request is 4 MB: @@ -101,8 +101,8 @@ namespace Duplicati.Library.Backend // If it isn't, we round down to the nearest multiple below it. this.fragmentSize = (this.fragmentSize / UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE) * UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE; - // Make sure the fragment size isn't larger than the maximum - this.fragmentSize = Math.Min(this.fragmentSize, UPLOAD_SESSION_FRAGMENT_MAX_SIZE); + // Make sure the fragment size isn't larger than the maximum, or smaller than the minimum + this.fragmentSize = Math.Max(Math.Min(this.fragmentSize, UPLOAD_SESSION_FRAGMENT_MAX_SIZE), UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE); } else { From f93f9dcc5ade5f418a17628938a93568e9443ae6 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Fri, 23 Mar 2018 15:07:51 -0600 Subject: [PATCH 29/36] Add new utility class, OAuthHttpClient. This class extends HttpClient and uses an OAuthHttpMessageHandler under the covers to automatically authenticate requests. Additionally, it automatically respects the global HttpContextSettings for overall timeout, read/write timeout, and SSL certificate validation. (BufferRequests isn't currently handled, as HttpClient doesn't seem to easily expose that flag, and as of .NET 4.5, seems to not buffer by default.) --- .../Duplicati.Library.OAuthHelper.csproj | 2 + .../Backend/OAuthHelper/OAuthHttpClient.cs | 108 ++++++++++++++++++ .../OAuthHelper/OAuthHttpMessageHandler.cs | 4 +- .../Duplicati.Library.Backend.OneDrive.csproj | 1 + .../Backend/OneDrive/MicrosoftGraphBackend.cs | 16 +-- 5 files changed, 116 insertions(+), 15 deletions(-) create mode 100644 Duplicati/Library/Backend/OAuthHelper/OAuthHttpClient.cs diff --git a/Duplicati/Library/Backend/OAuthHelper/Duplicati.Library.OAuthHelper.csproj b/Duplicati/Library/Backend/OAuthHelper/Duplicati.Library.OAuthHelper.csproj index c0515756c..1423e53ce 100644 --- a/Duplicati/Library/Backend/OAuthHelper/Duplicati.Library.OAuthHelper.csproj +++ b/Duplicati/Library/Backend/OAuthHelper/Duplicati.Library.OAuthHelper.csproj @@ -35,8 +35,10 @@ ..\..\..\..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll + + diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthHttpClient.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthHttpClient.cs new file mode 100644 index 000000000..de319a37e --- /dev/null +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthHttpClient.cs @@ -0,0 +1,108 @@ +// Copyright (C) 2018, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Reflection; +using System.Threading.Tasks; + +using Duplicati.Library.Utility; + +namespace Duplicati.Library +{ + public class OAuthHttpClient : HttpClient + { + private static readonly string USER_AGENT_VERSION = Assembly.GetExecutingAssembly().GetName().Version.ToString(); + + private readonly OAuthHttpMessageHandler m_authenticator; + + public OAuthHttpClient(string authid, string protocolKey) + : this(CreateMessageHandler(authid, protocolKey)) + { + } + + private OAuthHttpClient(OAuthHttpMessageHandler authenticator) + : base(authenticator, true) + { + this.m_authenticator = authenticator; + + // Set the overall timeout + if (HttpContextSettings.OperationTimeout > TimeSpan.Zero) + { + this.Timeout = HttpContextSettings.OperationTimeout; + } + + // We would also set AllowReadStreamBuffering = HttpContextSettings.BufferRequests, except HttpClient doesn't appear to expose this. + // However, starting in .NET 4.5, it looks like HttpClient doesn't buffer by default. + // https://www.strathweb.com/2012/09/dealing-with-large-files-in-asp-net-web-api/ + + // Set the default user agent + this.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("Duplicati", USER_AGENT_VERSION)); + } + + /// + /// Sends an async request with optional authentication. + /// + /// Http request + /// Whether to authenticate the request + /// Http response + public async Task SendAsync(HttpRequestMessage request, bool authenticate) + { + if (!authenticate) + { + this.PreventAuthentication(request); + } + + return await this.SendAsync(request); + } + + /// + /// Prevents authentication from being applied on the given request + /// + /// Request to not authenticate + /// Request to not authenticate + public HttpRequestMessage PreventAuthentication(HttpRequestMessage request) + { + return this.m_authenticator.PreventAuthentication(request); + } + + /// + /// Create a message handler with the global timeout / certificate settings. + /// + /// OAuth Auth-ID + /// Protocol key + /// Http message handler + private static OAuthHttpMessageHandler CreateMessageHandler(string authid, string protocolKey) + { + OAuthHttpMessageHandler handler = new OAuthHttpMessageHandler(authid, protocolKey); + + // Set the read/write timeout + if (HttpContextSettings.OperationTimeout > TimeSpan.Zero) + { + handler.ReadWriteTimeout = (int)HttpContextSettings.ReadWriteTimeout.TotalMilliseconds; + } + + // Set the certificate validator + if (HttpContextSettings.CertificateValidator != null) + { + handler.ServerCertificateValidationCallback = HttpContextSettings.CertificateValidator.ValidateServerCertficate; + } + + return handler; + } + } +} diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs index affc0e77d..00758d58e 100644 --- a/Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs @@ -14,8 +14,6 @@ // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -using System; -using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Threading; @@ -23,7 +21,7 @@ using System.Threading.Tasks; namespace Duplicati.Library { - public class OAuthHttpMessageHandler : HttpClientHandler + public class OAuthHttpMessageHandler : WebRequestHandler { /// /// Requests which contain a property with this name (in 'request.Properties') will not have the authentication header automatically added. diff --git a/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj b/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj index 259004c1d..d2a52d15f 100644 --- a/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj +++ b/Duplicati/Library/Backend/OneDrive/Duplicati.Library.Backend.OneDrive.csproj @@ -47,6 +47,7 @@ ..\..\..\..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll + diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index b26bed0a0..5ca73f142 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Headers; -using System.Reflection; using System.Text; using System.Threading; @@ -69,15 +68,12 @@ namespace Duplicati.Library.Backend /// private const int UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE = 320 * 1024; - private static readonly string USER_AGENT_VERSION = Assembly.GetExecutingAssembly().GetName().Version.ToString(); - private static readonly HttpMethod PatchMethod = new HttpMethod("PATCH"); protected delegate string DescriptionTemplateDelegate(string mssadescription, string mssalink, string msopdescription, string msoplink); private readonly JsonSerializer m_serializer = new JsonSerializer(); - private readonly OAuthHttpMessageHandler m_authenticator; - private readonly HttpClient m_client; + private readonly OAuthHttpClient m_client; private readonly string m_path; private readonly int fragmentSize; private readonly int fragmentRetryCount; @@ -121,10 +117,8 @@ namespace Duplicati.Library.Backend this.fragmentRetryDelay = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY; } - this.m_authenticator = new OAuthHttpMessageHandler(authid, this.ProtocolKey); - this.m_client = new HttpClient(this.m_authenticator, true); + this.m_client = new OAuthHttpClient(authid, this.ProtocolKey); this.m_client.BaseAddress = new System.Uri(BASE_ADDRESS); - this.m_client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("Duplicati", USER_AGENT_VERSION)); // Extract out the path to the backup root folder from the given URI this.m_path = NormalizeSlashes(this.GetRootPathFromUrl(url)); @@ -367,13 +361,11 @@ namespace Duplicati.Library.Backend HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Put, uploadSession.UploadUrl); request.Content = fragmentContent; - // The uploaded put requests will error if they are authenticated - this.m_authenticator.PreventAuthentication(request); - HttpResponseMessage response = null; try { - response = this.m_client.SendAsync(request).Await(); + // The uploaded put requests will error if they are authenticated + response = this.m_client.SendAsync(request, false).Await(); // Note: On the last request, the json result includes the default properties of the item that was uploaded var result = this.ParseResponse(response); From c9e8ee31ab0beda5d92db4a3e8646506b44f2a26 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Fri, 23 Mar 2018 15:10:05 -0600 Subject: [PATCH 30/36] When unwrapping TargetInvocationException, we should still put the inner exception in a new Exception. Otherwise, the callstack in the exception is overwritten, hiding the original call stack where the exception actually occurred. --- Duplicati/Library/DynamicLoader/BackendLoader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/DynamicLoader/BackendLoader.cs b/Duplicati/Library/DynamicLoader/BackendLoader.cs index 4eec1a3bb..a00bf7937 100644 --- a/Duplicati/Library/DynamicLoader/BackendLoader.cs +++ b/Duplicati/Library/DynamicLoader/BackendLoader.cs @@ -98,7 +98,7 @@ namespace Duplicati.Library.DynamicLoader { // Unwrap exceptions for nicer display if (tex.InnerException != null) - throw tex.InnerException; + throw new Exception("Unwrapped TargetInvocationException", tex.InnerException); throw; } From fe71c3fd485e5f69fada8b50c000bace8f70e6e1 Mon Sep 17 00:00:00 2001 From: dbddhkpde <37509447+dbddhkpde@users.noreply.github.com> Date: Sun, 25 Mar 2018 20:15:38 +0200 Subject: [PATCH 31/36] Just try if I can add somethin in my fork --- TEST | 1 + 1 file changed, 1 insertion(+) create mode 100644 TEST diff --git a/TEST b/TEST new file mode 100644 index 000000000..2a02d41ce --- /dev/null +++ b/TEST @@ -0,0 +1 @@ +TEST From 73a46fdf191af375678e6940d696d84bf9abc1d1 Mon Sep 17 00:00:00 2001 From: dbddhkpde <37509447+dbddhkpde@users.noreply.github.com> Date: Sun, 25 Mar 2018 20:16:15 +0200 Subject: [PATCH 32/36] was only a test --- TEST | 1 - 1 file changed, 1 deletion(-) delete mode 100644 TEST diff --git a/TEST b/TEST deleted file mode 100644 index 2a02d41ce..000000000 --- a/TEST +++ /dev/null @@ -1 +0,0 @@ -TEST From 1f094ca0feb090f1dff78f99dfb1b48ce7b6a653 Mon Sep 17 00:00:00 2001 From: dbddhkpde <37509447+dbddhkpde@users.noreply.github.com> Date: Sun, 25 Mar 2018 20:40:51 +0200 Subject: [PATCH 33/36] GUI-Extension images Extend the GUI with a small progress bar within the backup job "details": - add current action - change the icon of the active job (paused, running) - show / hide the GUI-extension (progress bar, filename, action) only if page-event contains a current filename --- Duplicati/Server/webroot/ngax/img/backuppause.png | Bin 0 -> 871 bytes .../Server/webroot/ngax/img/backuprunning.png | Bin 0 -> 887 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 Duplicati/Server/webroot/ngax/img/backuppause.png create mode 100644 Duplicati/Server/webroot/ngax/img/backuprunning.png diff --git a/Duplicati/Server/webroot/ngax/img/backuppause.png b/Duplicati/Server/webroot/ngax/img/backuppause.png new file mode 100644 index 0000000000000000000000000000000000000000..cffabaeb97e5489ef3f075133bb3bbafc901050b GIT binary patch literal 871 zcmeAS@N?(olHy`uVBq!ia0y~yU{C~M4mJh`hVUnc@);Ny7>k44ofy`glX=O&z`&C3 z=qM&&cqjLgrKk2BzztE{-7)?nkdjclZX%9RKL7H`&B6zr*E}hNkP16gMRm z*CiE-aWV5_oV^xJ5nMXql8Btfq2?!(gyLd?yG}Oys2t*+WVvtu)u`pEbN^eWy|?VE zuRRr2z5f5N`L@|>_lkBMeN>-bb}c&b>(ahGQ3ZVO3nV`$+zk(%r^J@P`)*-IMU$@t zPj<~r9ao8fmm2m)b0*e4N*g3{)D@K*4rLc5>1RSTfeyv;V zU!2Lxjv0Qox>DHzYcoz~zus-Rdam%<3lFbwY6phN7xGwr`8PvbZ&7@W;wAahh1_AE z_V47E{oHHtxm~I3p{>b9qb25S!39G16MZGDo5Lb1Qxw-ZeBQQF+4n*tvqW>r2A-o< zOLo6vHA@y*Q+Rlz^m)Pl;^;3Pwg%jy5z=3irdd&iZf>Wb9@lgy}& z48?5O4DHvwtIz)_N&L2>A6Z`NxO+QJ?8Vtbs^%XOb}X$ShJ^gBH? Z&vo1Jf^~_ZB?AKkgQu&X%Q~loCIACug$4is literal 0 HcmV?d00001 diff --git a/Duplicati/Server/webroot/ngax/img/backuprunning.png b/Duplicati/Server/webroot/ngax/img/backuprunning.png new file mode 100644 index 0000000000000000000000000000000000000000..f275eee6ccfb244236e80b3e7dbba8efd97590e8 GIT binary patch literal 887 zcmeAS@N?(olHy`uVBq!ia0y~yU{C~M4mJh`hVUnc@);Ny7>k44ofy`glX=O&z`&C3 z=qM&&cqjLgrKk2BznpE{-7)?nkftck~8I9RKK@IO#})WTeEE4S`)ACwp8n zf?T?KJ6R8%ny4+H_gJezUbJ!$E!C&T~~4 z58dx~6BWF)a=B|V?<&#U%G|#x=hhS_ag|tnJ?Is(w*N&N-`@MrxVj~z;y8E?OWkRj z`p5sB^YK3(Gk(?zR)3J2m0|iwdT*pf>-!CAJab*8*Vt_0313*b`--6Yi=&D@ZZ;9K zHb{P4SJn77qs=tW=G!C7Lxm^BY{kmi^b!;wwagRy#JGm%S5ooC=J^|*z4&N;?pS6` zYw2U>94F;NoR@kISlwxz`kwuZ?xnQ1e7>7lGudZ!*)Xo{uUO;bEN*u|YW@5i^OwAkUOaiRb@hvk&I&X$(CxeZ$*JYL3-H$9AMu6 z?hyT6B~3}^kTgr~!!yQ*m-;q+lU;G5?0Rauyv)VL?;7@L9(J->+qw7$%iaIHQQB+u za)q{pww%BDcGv0V?ptx0{ZaR)urHjiqI2kS)rIdKnG0JTt#zE@7Tt`M Date: Sun, 25 Mar 2018 20:45:36 +0200 Subject: [PATCH 34/36] GUI-Extension less modification progress bar Extend the GUI with a small progress bar within the backup job "details": - add current action - change the icon of the active job (paused, running) - show / hide the GUI-extension (progress bar, filename, action) only if page-event contains a current filename --- Duplicati/Server/webroot/ngax/less/style.less | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/less/style.less b/Duplicati/Server/webroot/ngax/less/style.less index 0e0188c8b..23f31f39b 100755 --- a/Duplicati/Server/webroot/ngax/less/style.less +++ b/Duplicati/Server/webroot/ngax/less/style.less @@ -906,10 +906,19 @@ body { border-top: 0px none; } + .progress-small { + text-align: center; + height: 18px; + background: rgba(164, 209,235, 0.50); + } + + .progress-small-bg { + border: 1px #65b1dd solid; + width: 200px; + } + a { font-size: 30px; - padding-left: 55px; - background: url('../img/backup.png') no-repeat 5px 6px; font-weight: 300; display: inline-block; From a9222e88ae77a7cd6c00dd45bcceff71b61e47d5 Mon Sep 17 00:00:00 2001 From: dbddhkpde <37509447+dbddhkpde@users.noreply.github.com> Date: Sun, 25 Mar 2018 20:49:10 +0200 Subject: [PATCH 35/36] GUI-Extension progress bar, icons and show/hide logic Extend the GUI with a small progress bar within the backup job "details": - add current action - change the icon of the active job (paused, running) - show / hide the GUI-extension (progress bar, filename, action) only if page-event contains a current filename --- .../Server/webroot/ngax/templates/home.html | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Duplicati/Server/webroot/ngax/templates/home.html b/Duplicati/Server/webroot/ngax/templates/home.html index 2875e76e9..da18f9ba3 100644 --- a/Duplicati/Server/webroot/ngax/templates/home.html +++ b/Duplicati/Server/webroot/ngax/templates/home.html @@ -2,7 +2,11 @@
- {{item.Backup.Name}} + + + + + {{item.Backup.Name}} @@ -61,6 +65,20 @@
{{item.Backup.Metadata.TargetSizeString}} / {{$count}} Version
+
+
Current action:
+
{{state.lastPgEvent.Phase}}
+
Progress:
+
+
+
+ {{(1 - (state.lastPgEvent.CurrentFilesize - state.lastPgEvent.CurrentFileoffset) / state.lastPgEvent.CurrentFilesize) * 100 | number:2}}% +
+
+
+
Current file:
+
{{state.lastPgEvent.CurrentFilename}}
+
From 28da943376aac7c9bc61c759f541cef64a8a4528 Mon Sep 17 00:00:00 2001 From: Kenneth Hsu Date: Sat, 24 Mar 2018 20:30:36 -0700 Subject: [PATCH 36/36] Use YAML file to manage AppVeyor build configuration. This was exported from the AppVeyor UI, with a few modifications made to upload the test results and throw an exception in the case of a test failure. Note that this overrides the settings specified in the AppVeyor UI. --- .appveyor.yml | 77 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .appveyor.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..91879d85d --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,77 @@ +version: 1.0.{build} +image: Visual Studio 2017 +configuration: Debug +platform: Any CPU +environment: + UNITTEST_BASEFOLDER: C:\projects\duplicati\testdata + COVERALLS_REPO_TOKEN: + secure: szQsrkP5rvra8L60SomD73/dFvRwot0UuyA566zqzI2qmLOr+MhLN0AyC8BTbhYY +before_build: + - cmd: nuget restore Duplicati.sln +build: + project: Duplicati.sln + verbosity: minimal +test_script: + - ps: | + Start-FileDownload https://s3.amazonaws.com/duplicati-test-file-hosting/DSMCBE.zip + + 7z x DSMCBE.zip -otestdata + mkdir .\testData\data + + nuget install OpenCover -Version 4.6.166 -OutputDirectory packages + nuget install coveralls.net -Version 0.6.0 -OutputDirectory packages + nuget install NUnit.Runners -Version 3.4.0 -OutputDirectory packages + + $testDir = ".\Duplicati\UnitTest\bin\Debug" + $tests = @("$testDir\Duplicati.UnitTest.dll") + $testsPassed = $true + + foreach ($elem in $tests) { + $testResultsFile = "$elem" + ".xml" + + .\packages\OpenCover.4.6.166\tools\OpenCover.Console.exe ` + -register:user ` + -target:.\packages\NUnit.ConsoleRunner.3.4.0\tools\nunit3-console.exe ` + "-targetargs:""$elem"" /framework:net-4.5 /where:cat!=BulkData /result:""$testResultsFile""" ` + "-filter:+[Duplicati.Library.Main]* -[UnitTest]*" ` + -output:opencover.xml ` + -returntargetcode ` + + # Keep track of test failures so that we can throw an exception after + # uploading coverage and test results. + $testsPassed = $testsPassed -and $? + + # Upload test results to AppVeyor. + $wc = New-Object "System.Net.WebClient" + $wc.UploadFile("https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\""$testResultsFile"")) + } + + if ($Env:COVERALLS_REPO_TOKEN) { + $revision = git rev-parse HEAD + $branch = git rev-parse --abbrev-ref HEAD + $commitAuthor = git show --quiet --format="%aN" $revision + $commitEmail = git show --quiet --format="%aE" $revision + $commitMessage = git show --quiet --format="%s" $revision + + .\packages\coveralls.net.0.6.0\tools\csmacnz.Coveralls.exe ` + --opencover -i opencover.xml ` + --repoToken "$Env:COVERALLS_REPO_TOKEN" ` + --commitId "$revision" ` + --commitBranch "$branch" ` + --commitAuthor "$commitAuthor" ` + --commitEmail "$commitEmail" ` + --commitMessage "$commitMessage" ` + --useRelativePaths ` + --basePath .\Duplicati\UnitTest\bin\Debug + } + + if (!$testsPassed) { + throw "Tests failed." + } +notifications: + - provider: Webhook + url: https://webhooks.gitter.im/e/a2c55bac2b5c38838e0f + method: POST + on_build_success: true + on_build_failure: true + on_build_status_changed: true