From 3fd663e923153bee42bde45053cc6a4ccdfe7437 Mon Sep 17 00:00:00 2001 From: "Marcelo C." Date: Thu, 20 Mar 2025 15:23:31 -0300 Subject: [PATCH] Terminal Box.com replacement of deprecated WebClient/OAuth Creating of the new OAuthHelper for HttpClient. Removal of manual assembly of the multipart post. --- Duplicati/Library/Backend/Box/BoxBackend.cs | 403 +++++------------- .../Backend/Box/Model/CreateItemRequest.cs | 9 + .../Backend/Box/Model/ErrorResponse.cs | 13 + .../Library/Backend/Box/Model/FileEntity.cs | 20 + .../Library/Backend/Box/Model/FileList.cs | 11 + .../Library/Backend/Box/Model/FolderList.cs | 9 + .../Library/Backend/Box/Model/IDReference.cs | 8 + .../Backend/Box/Model/ListFolderResponse.cs | 21 + .../Library/Backend/Box/Model/MiniFolder.cs | 11 + .../Library/Backend/Box/Model/MiniUser.cs | 10 + .../Library/Backend/Box/Model/OrderEntry.cs | 9 + .../Backend/Box/Model/ShortListResponse.cs | 8 + .../Library/Backend/Box/Model/UploadEmail.cs | 9 + .../OAuthHelper/JSONWebHelperHttpClient.cs | 74 +++- .../OAuthHelper/OAuthHelperHttpClient.cs | 192 +++++++++ .../OAuthHelper/OAuthServiceResponse.cs | 33 ++ 16 files changed, 534 insertions(+), 306 deletions(-) create mode 100644 Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs create mode 100644 Duplicati/Library/Backend/Box/Model/ErrorResponse.cs create mode 100644 Duplicati/Library/Backend/Box/Model/FileEntity.cs create mode 100644 Duplicati/Library/Backend/Box/Model/FileList.cs create mode 100644 Duplicati/Library/Backend/Box/Model/FolderList.cs create mode 100644 Duplicati/Library/Backend/Box/Model/IDReference.cs create mode 100644 Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs create mode 100644 Duplicati/Library/Backend/Box/Model/MiniFolder.cs create mode 100644 Duplicati/Library/Backend/Box/Model/MiniUser.cs create mode 100644 Duplicati/Library/Backend/Box/Model/OrderEntry.cs create mode 100644 Duplicati/Library/Backend/Box/Model/ShortListResponse.cs create mode 100644 Duplicati/Library/Backend/Box/Model/UploadEmail.cs create mode 100644 Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs create mode 100644 Duplicati/Library/Backend/OAuthHelper/OAuthServiceResponse.cs diff --git a/Duplicati/Library/Backend/Box/BoxBackend.cs b/Duplicati/Library/Backend/Box/BoxBackend.cs index d256255d9..4e49abcc7 100644 --- a/Duplicati/Library/Backend/Box/BoxBackend.cs +++ b/Duplicati/Library/Backend/Box/BoxBackend.cs @@ -19,24 +19,27 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Common.IO; -using Duplicati.Library.Interface; -using Newtonsoft.Json; using System; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Net; +using System.Net.Http; +using System.Net.Http.Json; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using Duplicati.Library.Common.IO; +using Duplicati.Library.Interface; +using Duplicati.Library.Logging; +using Newtonsoft.Json; +using Uri = Duplicati.Library.Utility.Uri; namespace Duplicati.Library.Backend.Box { - // ReSharper disable once ClassNeverInstantiated.Global - // This class is instantiated dynamically in the BackendLoader. - public class BoxBackend : IBackend, IStreamingBackend + public class BoxBackend : IStreamingBackend { - private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + private static readonly string LOGTAG = Log.LogTagFromType(); private const string AUTHID_OPTION = "authid"; private const string REALLY_DELETE_OPTION = "box-delete-from-trash"; @@ -46,14 +49,14 @@ namespace Duplicati.Library.Backend.Box private const int PAGE_SIZE = 200; - private readonly BoxHelper m_oauth; - private readonly string m_path; - private readonly bool m_deleteFromTrash; + private readonly BoxHelper _oAuthHelper; + private readonly string _path; + private readonly bool _deleteFromTrash; - private string m_currentfolder; - private readonly Dictionary m_filecache = new Dictionary(); + private string _currentFolder; + private readonly Dictionary _fileCache = new(); - private class BoxHelper : OAuthHelper + private class BoxHelper : OAuthHelperHttpClient { public BoxHelper(string authid) : base(authid, "box.com") @@ -61,108 +64,88 @@ namespace Duplicati.Library.Backend.Box AutoAuthHeader = true; } - protected override void ParseException(Exception ex) + public override void AttemptParseAndThrowException(Exception ex, HttpResponseMessage responseContext = null) { - Exception newex = null; - try - { - if (ex is WebException exception && exception.Response is HttpWebResponse hs) - { - string rawdata = null; - using (var rs = Library.Utility.AsyncHttpRequest.TrySetTimeout(hs.GetResponseStream())) - using (var sr = new System.IO.StreamReader(rs)) - rawdata = sr.ReadToEnd(); + if (ex is not HttpRequestException || responseContext == null) + return; + + if (ex is HttpRequestException && responseContext is { StatusCode: HttpStatusCode.TooManyRequests }) + throw new TooManyRequestException(responseContext.Headers.RetryAfter); - if (string.IsNullOrWhiteSpace(rawdata)) - return; - - newex = new Exception("Raw message: " + rawdata); - - var msg = JsonConvert.DeserializeObject(rawdata); - newex = new Exception(string.Format("{0} - {1}: {2}", msg.Status, msg.Code, msg.Message)); - - /*if (msg.ContextInfo != null && msg.ContextInfo.Length > 0) - newex = new Exception(string.Format("{0} - {1}: {2}{3}{4}", msg.Status, msg.Code, msg.Message, Environment.NewLine, string.Join("; ", from n in msg.ContextInfo select n.Message))); - */ - } - } - catch (Exception ex2) - { - Library.Logging.Log.WriteWarningMessage(LOGTAG, "BoxErrorParser", ex2, "Failed to parse error from Box"); - } - - if (newex != null) - throw newex; + using var stream = responseContext.Content.ReadAsStream(); + using var reader = new StreamReader(stream); + var rawData = reader.ReadToEnd(); + var errorResponse = JsonConvert.DeserializeObject(rawData); + throw new UserInformationException($"Box.com ErrorResponse: {errorResponse.Status} - {errorResponse.Code}: {errorResponse.Message}", "box.com"); } } - // ReSharper disable once UnusedMember.Global - // This constructor is needed by the BackendLoader. public BoxBackend() { } - - // ReSharper disable once UnusedMember.Global - // This constructor is needed by the BackendLoader. + public BoxBackend(string url, Dictionary options) { - var uri = new Utility.Uri(url); + var uri = new Uri(url); - m_path = Util.AppendDirSeparator(uri.HostAndPath, "/"); + _path = Util.AppendDirSeparator(uri.HostAndPath, "/"); string authid = null; - if (options.ContainsKey(AUTHID_OPTION)) - authid = options[AUTHID_OPTION]; + if (options.TryGetValue(AUTHID_OPTION, out var option)) + authid = option; - m_deleteFromTrash = Library.Utility.Utility.ParseBoolOption(options, REALLY_DELETE_OPTION); + _deleteFromTrash = Utility.Utility.ParseBoolOption(options, REALLY_DELETE_OPTION); - m_oauth = new BoxHelper(authid); + _oAuthHelper = new BoxHelper(authid); } private async Task GetCurrentFolderWithCacheAsync(CancellationToken cancelToken) { - if (m_currentfolder == null) + if (_currentFolder == null) await GetCurrentFolderAsync(false, cancelToken).ConfigureAwait(false); - return m_currentfolder; + return _currentFolder; } private async Task GetCurrentFolderAsync(bool create, CancellationToken cancelToken) { var parentid = "0"; - foreach (var p in m_path.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries)) + foreach (var p in _path.Split(["/"], StringSplitOptions.RemoveEmptyEntries)) { - var el = (MiniFolder)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p).ConfigureAwait(false); + var el = (MiniFolder)await PagedFileListResponse(parentid, true, cancelToken).FirstOrDefaultAsync(x => x.Name == p, cancellationToken: cancelToken).ConfigureAwait(false); if (el == null) { if (!create) throw new FolderMissingException(); - - el = await m_oauth.PostAndGetJSONDataAsync( - string.Format("{0}/folders", BOX_API_URL), - cancelToken, - new CreateItemRequest() { Name = p, Parent = new IDReference() { ID = parentid } } + + el = await _oAuthHelper.PostAndGetJsonDataAsync( + $"{BOX_API_URL}/folders", + new CreateItemRequest + { + Name = p, Parent = new IDReference { ID = parentid } + }, + cancelToken ).ConfigureAwait(false); } parentid = el.ID; } - m_currentfolder = parentid; + _currentFolder = parentid; } - private async Task GetFileIDAsync(string name, CancellationToken cancelToken) + private async Task GetFileIdAsync(string name, CancellationToken cancelToken) { - if (m_filecache.ContainsKey(name)) - return m_filecache[name]; + if (_fileCache.TryGetValue(name, out var async)) + return async; // Make sure we enumerate this, otherwise the m_filecache is empty. var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false); - await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync().ConfigureAwait(false); + await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync(cancellationToken: cancelToken).ConfigureAwait(false); - if (m_filecache.ContainsKey(name)) - return m_filecache[name]; + if (_fileCache.TryGetValue(name, out var idAsync)) + return idAsync; throw new FileMissingException(); } @@ -173,11 +156,11 @@ namespace Duplicati.Library.Backend.Box var done = false; if (!onlyfolders) - m_filecache.Clear(); + _fileCache.Clear(); do { - var resp = await m_oauth.GetJSONDataAsync($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", cancelToken).ConfigureAwait(false); + var resp = await _oAuthHelper.GetJsonDataAsync($"{BOX_API_URL}/folders/{parentid}/items?limit={PAGE_SIZE}&offset={offset}&fields=name,size,modified_at", cancelToken).ConfigureAwait(false); if (resp.Entries == null || resp.Entries.Length == 0) break; @@ -189,13 +172,11 @@ namespace Duplicati.Library.Backend.Box done = true; break; } - else - { - if (!onlyfolders && f.Type == "file") - m_filecache[f.Name] = f.ID; - yield return f; - } + if (!onlyfolders && f.Type == "file") + _fileCache[f.Name] = f.ID; + + yield return f; } offset = offset + PAGE_SIZE; @@ -206,62 +187,59 @@ namespace Duplicati.Library.Backend.Box } while (!done); } - #region IStreamingBackend implementation - - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false); - var createreq = new CreateItemRequest() + var createreq = new CreateItemRequest { Name = remotename, - Parent = new IDReference() + Parent = new IDReference { ID = currentFolder } }; - if (m_filecache.Count == 0) - await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync().ConfigureAwait(false); + if (_fileCache.Count == 0) + await PagedFileListResponse(currentFolder, false, cancelToken).LastOrDefaultAsync(cancelToken).ConfigureAwait(false); - var existing = m_filecache.ContainsKey(remotename); + var existing = _fileCache.ContainsKey(remotename); + + var multipartForm = new MultipartFormDataContent("----DuplicatiFormBoundary" + Guid.NewGuid().ToString("N")); try { string url; - var items = new List(2); if (existing) - url = $"{BOX_UPLOAD_URL}/{m_filecache[remotename]}/content"; + url = $"{BOX_UPLOAD_URL}/{_fileCache[remotename]}/content"; else { url = $"{BOX_UPLOAD_URL}/content"; - items.Add(new MultipartItem(createreq, "attributes")); + multipartForm.Add(JsonContent.Create(createreq),"attributes"); } - - items.Add(new MultipartItem(stream, "file", remotename)); - - var res = (await m_oauth.PostMultipartAndGetJSONDataAsync(url, null, cancelToken, items.ToArray())).Entries.First(); - m_filecache[remotename] = res.ID; + + multipartForm.Add(new StreamContent(stream), "file", remotename); + + var res = (await _oAuthHelper.PostMultipartAndGetJsonDataAsync(url, cancelToken, multipartForm)).Entries.First(); + _fileCache[remotename] = res.ID; } catch { - m_filecache.Clear(); + _fileCache.Clear(); throw; } } - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { - var fileId = await GetFileIDAsync(remotename, cancelToken).ConfigureAwait(false); - using (var resp = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}/content", BOX_API_URL, fileId), cancelToken).ConfigureAwait(false)) - using (var rs = Duplicati.Library.Utility.AsyncHttpRequest.TrySetTimeout(resp.GetResponseStream())) - await Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken).ConfigureAwait(false); + var fileId = await GetFileIdAsync(remotename, cancelToken).ConfigureAwait(false); + using var request = _oAuthHelper.CreateRequest($"{BOX_API_URL}/files/{fileId}/content"); + using var resp = await _oAuthHelper.GetResponseAsync(request, cancelToken).ConfigureAwait(false); + resp.EnsureSuccessStatusCode(); + await using var responseStream = await resp.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false); + await Utility.Utility.CopyStreamAsync(responseStream, stream, cancelToken).ConfigureAwait(false); } - #endregion - - #region IBackend implementation - public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) { var currentFolder = await GetCurrentFolderWithCacheAsync(cancelToken).ConfigureAwait(false); @@ -271,33 +249,38 @@ namespace Duplicati.Library.Backend.Box public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) - await PutAsync(remotename, fs, cancelToken); + await using FileStream fs = File.OpenRead(filename); + await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.Create(filename)) - await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); + await using FileStream fs = File.Create(filename); + await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task DeleteAsync(string remotename, CancellationToken cancelToken) { - var fileid = await GetFileIDAsync(remotename, cancelToken).ConfigureAwait(false); + var fileId = await GetFileIdAsync(remotename, cancelToken).ConfigureAwait(false); try { - using (var r = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}", BOX_API_URL, fileid), cancelToken, null, "DELETE").ConfigureAwait(false)) + using (var request = _oAuthHelper.CreateRequest($"{BOX_API_URL}/files/{fileId}","DELETE")) + using (var r = await _oAuthHelper.GetResponseAsync(request, cancelToken).ConfigureAwait(false)) { } - if (m_deleteFromTrash) - using (var r = await m_oauth.GetResponseAsync(string.Format("{0}/files/{1}/trash", BOX_API_URL, fileid), cancelToken, null, "DELETE").ConfigureAwait(false)) + if (_deleteFromTrash) + { + using var request = _oAuthHelper.CreateRequest($"{BOX_API_URL}/files/{fileId}/trash","DELETE"); + using (var r = await _oAuthHelper + .GetResponseAsync(request, cancelToken).ConfigureAwait(false)) { } + } } catch { - m_filecache.Clear(); + _fileCache.Clear(); throw; } } @@ -310,202 +293,26 @@ namespace Duplicati.Library.Backend.Box return GetCurrentFolderAsync(true, cancellationToken); } - public string DisplayName - { - get - { - return Strings.Box.DisplayName; - } - } + public string DisplayName => Strings.Box.DisplayName; - public string ProtocolKey - { - get - { - return "box"; - } - } + public string ProtocolKey => "box"; - public IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Box.AuthidShort, Strings.Box.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("box.com"))), - new CommandLineArgument(REALLY_DELETE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.Box.ReallydeleteShort, Strings.Box.ReallydeleteLong), - }); - } - } + public IList SupportedCommands => + new List([ + new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Box.AuthidShort, Strings.Box.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("box.com"))), + new CommandLineArgument(REALLY_DELETE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.Box.ReallydeleteShort, Strings.Box.ReallydeleteLong) + ]); - public string Description - { - get - { - return Strings.Box.Description; - } - } + public string Description => Strings.Box.Description; - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string[] { + public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { new System.Uri(BOX_API_URL).Host, new System.Uri(BOX_UPLOAD_URL).Host }); - #endregion - - #region IDisposable implementation - public void Dispose() { } - - #endregion - - private class MiniUser : IDReference - { - [JsonProperty("type")] - public string Type { get; set; } - [JsonProperty("name")] - public string Name { get; set; } - [JsonProperty("login")] - public string Login { get; set; } - } - - private class MiniFolder : IDReference - { - [JsonProperty("type")] - public string Type { get; set; } - [JsonProperty("name")] - public string Name { get; set; } - [JsonProperty("etag")] - public string ETag { get; set; } - [JsonProperty("sequence_id")] - public string SequenceID { get; set; } - } - - private class FileEntity : MiniFolder - { - public FileEntity() { Size = -1; } - - [JsonProperty("sha1")] - public string SHA1 { get; set; } - - [JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)] - public long Size { get; set; } - [JsonProperty("modified_at", NullValueHandling = NullValueHandling.Ignore)] - public DateTime ModifiedAt { get; set; } - } - - private class FolderList - { - [JsonProperty("total_count")] - public long TotalCount { get; set; } - [JsonProperty("entries")] - public MiniFolder[] Entries { get; set; } - } - - private class FileList - { - [JsonProperty("total_count")] - public long TotalCount { get; set; } - [JsonProperty("entries")] - public FileEntity[] Entries { get; set; } - [JsonProperty("offset")] - public long Offset { get; set; } - [JsonProperty("limit")] - public long Limit { get; set; } - } - - private class UploadEmail - { - [JsonProperty("access")] - public string Access { get; set; } - [JsonProperty("email")] - public string Email { get; set; } - } - - private class ListFolderResponse : MiniFolder - { - [JsonProperty("created_at")] - public DateTime CreatedAt { get; set; } - [JsonProperty("modified_at")] - public DateTime ModifiedAt { get; set; } - [JsonProperty("description")] - public string Description { get; set; } - [JsonProperty("size")] - public long Size { get; set; } - - [JsonProperty("path_collection")] - public FolderList PathCollection { get; set; } - - [JsonProperty("created_by")] - public MiniUser CreatedBy { get; set; } - [JsonProperty("modified_by")] - public MiniUser ModifiedBy { get; set; } - [JsonProperty("owned_by")] - public MiniUser OwnedBy { get; set; } - - [JsonProperty("shared_link")] - public MiniUser SharedLink { get; set; } - - [JsonProperty("folder_upload_email")] - public UploadEmail FolderUploadEmail { get; set; } - - [JsonProperty("parent")] - public MiniFolder Parent { get; set; } - - [JsonProperty("item_status")] - public string ItemStatus { get; set; } - - [JsonProperty("item_collection")] - public FileList ItemCollection { get; set; } - - } - - private class OrderEntry - { - [JsonProperty("by")] - public string By { get; set; } - [JsonProperty("direction")] - public string Direction { get; set; } - } - - private class ShortListResponse : FileList - { - [JsonProperty("order")] - public OrderEntry[] Order { get; set; } - - } - - private class IDReference - { - [JsonProperty("id")] - public string ID { get; set; } - } - - private class CreateItemRequest - { - [JsonProperty("name")] - public string Name { get; set; } - [JsonProperty("parent")] - public IDReference Parent { get; set; } - } - - private class ErrorResponse - { - [JsonProperty("type")] - public string Type { get; set; } - [JsonProperty("status")] - public int Status { get; set; } - [JsonProperty("code")] - public string Code { get; set; } - [JsonProperty("help_url")] - public string HelpUrl { get; set; } - [JsonProperty("message")] - public string Message { get; set; } - [JsonProperty("request_id")] - public string RequestId { get; set; } - - } } } diff --git a/Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs b/Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs new file mode 100644 index 000000000..b84ecd32b --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/CreateItemRequest.cs @@ -0,0 +1,9 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class CreateItemRequest +{ + [JsonProperty("name")] public string Name { get; set; } + [JsonProperty("parent")] public IDReference Parent { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/ErrorResponse.cs b/Duplicati/Library/Backend/Box/Model/ErrorResponse.cs new file mode 100644 index 000000000..627780c8f --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/ErrorResponse.cs @@ -0,0 +1,13 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class ErrorResponse +{ + [JsonProperty("type")] public string Type { get; set; } + [JsonProperty("status")] public int Status { get; set; } + [JsonProperty("code")] public string Code { get; set; } + [JsonProperty("help_url")] public string HelpUrl { get; set; } + [JsonProperty("message")] public string Message { get; set; } + [JsonProperty("request_id")] public string RequestId { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/FileEntity.cs b/Duplicati/Library/Backend/Box/Model/FileEntity.cs new file mode 100644 index 000000000..55e222724 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/FileEntity.cs @@ -0,0 +1,20 @@ +using System; +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class FileEntity : MiniFolder +{ + public FileEntity() + { + Size = -1; + } + + [JsonProperty("sha1")] public string SHA1 { get; set; } + + [JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)] + public long Size { get; set; } + + [JsonProperty("modified_at", NullValueHandling = NullValueHandling.Ignore)] + public DateTime ModifiedAt { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/FileList.cs b/Duplicati/Library/Backend/Box/Model/FileList.cs new file mode 100644 index 000000000..999aab376 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/FileList.cs @@ -0,0 +1,11 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class FileList +{ + [JsonProperty("total_count")] public long TotalCount { get; set; } + [JsonProperty("entries")] public FileEntity[] Entries { get; set; } + [JsonProperty("offset")] public long Offset { get; set; } + [JsonProperty("limit")] public long Limit { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/FolderList.cs b/Duplicati/Library/Backend/Box/Model/FolderList.cs new file mode 100644 index 000000000..028188322 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/FolderList.cs @@ -0,0 +1,9 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class FolderList +{ + [JsonProperty("total_count")] public long TotalCount { get; set; } + [JsonProperty("entries")] public MiniFolder[] Entries { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/IDReference.cs b/Duplicati/Library/Backend/Box/Model/IDReference.cs new file mode 100644 index 000000000..29ba6cf14 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/IDReference.cs @@ -0,0 +1,8 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class IDReference +{ + [JsonProperty("id")] public string ID { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs b/Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs new file mode 100644 index 000000000..395bdaaba --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/ListFolderResponse.cs @@ -0,0 +1,21 @@ +using System; +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class ListFolderResponse : MiniFolder +{ + [JsonProperty("created_at")] public DateTime CreatedAt { get; set; } + [JsonProperty("modified_at")] public DateTime ModifiedAt { get; set; } + [JsonProperty("description")] public string Description { get; set; } + [JsonProperty("size")] public long Size { get; set; } + [JsonProperty("path_collection")] public FolderList PathCollection { get; set; } + [JsonProperty("created_by")] public MiniUser CreatedBy { get; set; } + [JsonProperty("modified_by")] public MiniUser ModifiedBy { get; set; } + [JsonProperty("owned_by")] public MiniUser OwnedBy { get; set; } + [JsonProperty("shared_link")] public MiniUser SharedLink { get; set; } + [JsonProperty("folder_upload_email")] public UploadEmail FolderUploadEmail { get; set; } + [JsonProperty("parent")] public MiniFolder Parent { get; set; } + [JsonProperty("item_status")] public string ItemStatus { get; set; } + [JsonProperty("item_collection")] public FileList ItemCollection { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/MiniFolder.cs b/Duplicati/Library/Backend/Box/Model/MiniFolder.cs new file mode 100644 index 000000000..26363b542 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/MiniFolder.cs @@ -0,0 +1,11 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class MiniFolder : IDReference +{ + [JsonProperty("type")] public string Type { get; set; } + [JsonProperty("name")] public string Name { get; set; } + [JsonProperty("etag")] public string ETag { get; set; } + [JsonProperty("sequence_id")] public string SequenceID { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/MiniUser.cs b/Duplicati/Library/Backend/Box/Model/MiniUser.cs new file mode 100644 index 000000000..cc3af66ef --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/MiniUser.cs @@ -0,0 +1,10 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class MiniUser : IDReference +{ + [JsonProperty("type")] public string Type { get; set; } + [JsonProperty("name")] public string Name { get; set; } + [JsonProperty("login")] public string Login { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/OrderEntry.cs b/Duplicati/Library/Backend/Box/Model/OrderEntry.cs new file mode 100644 index 000000000..5a573fb9c --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/OrderEntry.cs @@ -0,0 +1,9 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class OrderEntry +{ + [JsonProperty("by")] public string By { get; set; } + [JsonProperty("direction")] public string Direction { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/ShortListResponse.cs b/Duplicati/Library/Backend/Box/Model/ShortListResponse.cs new file mode 100644 index 000000000..4f8db34fd --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/ShortListResponse.cs @@ -0,0 +1,8 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class ShortListResponse : FileList +{ + [JsonProperty("order")] public OrderEntry[] Order { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/Box/Model/UploadEmail.cs b/Duplicati/Library/Backend/Box/Model/UploadEmail.cs new file mode 100644 index 000000000..de24408d9 --- /dev/null +++ b/Duplicati/Library/Backend/Box/Model/UploadEmail.cs @@ -0,0 +1,9 @@ +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.Box; + +public class UploadEmail +{ + [JsonProperty("access")] public string Access { get; set; } + [JsonProperty("email")] public string Email { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs index cfcc46b76..214a6122d 100644 --- a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs +++ b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs @@ -38,7 +38,7 @@ namespace Duplicati.Library; public class JsonWebHelperHttpClient(HttpClient httpClient) { /// - /// HttpClient reference + /// HttpClient reference for inheritors /// protected readonly HttpClient _httpClient = httpClient; @@ -62,7 +62,36 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) return request; } + + /// + /// Performs a multipart post and parses the response as JSON + /// + /// The parsed JSON item. + /// The url to post to. + /// Token to cancel the operation. + /// The multipart items. + /// The return type parameter. + public virtual async Task PostMultipartAndGetJsonDataAsync(string url, CancellationToken cancellationToken, MultipartContent parts) + { + using var response = await PostMultipartAsync(url, cancellationToken, parts).ConfigureAwait(false); + return ReadJsonResponse(response); + } + + /// + /// Performs a multipart post + /// + /// The response. + /// The url to post to. + /// Token to cancel the operation. + /// The multipart items. + protected virtual async Task PostMultipartAsync(string url, CancellationToken cancellationToken, MultipartContent parts) + { + using var req = CreateRequest(url, "POST"); + req.Content = parts; + return await _httpClient.SendAsync(req, cancellationToken).ConfigureAwait(false); + } + /// /// Execute Get request and return response and deserializes JSON response into the given type /// @@ -72,7 +101,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Destination Type protected virtual async Task GetJsonDataAsync(string url, CancellationToken cancellationToken, Action setup = null) { - var req = CreateRequest(url); + using var req = CreateRequest(url); if (setup != null) setup(req); @@ -103,6 +132,26 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) } ).ConfigureAwait(false); } + + /// + /// Executes a web request and json-deserializes the results as the specified type + /// + /// The deserialized JSON data. + /// The remote URL + /// Token to cancel the operation. + /// The type of data to return. + public virtual async Task GetJsonDataAsync(string url, CancellationToken cancellationToken) + { + + return await GetJsonDataAsync( + url, + cancellationToken, + request => + { + request.Method = HttpMethod.Get; + } + ).ConfigureAwait(false); + } /// /// Reads the JSON response from the server and deserializes it into the given type @@ -113,7 +162,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) protected virtual async Task ReadJsonResponseAsync(HttpRequestMessage req, CancellationToken cancellationToken) { using var resp = await GetResponseAsync(req, cancellationToken).ConfigureAwait(false); - return await ReadJsonResponsAsync(resp, cancellationToken).ConfigureAwait(false); + return await ReadJsonResponseAsync(resp, cancellationToken).ConfigureAwait(false); } /// @@ -151,7 +200,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Type to cast to /// /// Exception when failing to deserialize the JSON to the Type - protected virtual async Task ReadJsonResponsAsync(HttpResponseMessage response, CancellationToken cancellationToken) + protected virtual async Task ReadJsonResponseAsync(HttpResponseMessage response, CancellationToken cancellationToken) { await using var rs = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); await using var ps = new StreamPeekReader(rs); @@ -171,13 +220,22 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) } } + /// + /// Use this method to register an exception handler, + /// which can throw another, more meaningful exception + /// + public virtual async Task AttemptParseAndThrowExceptionAsync(Exception ex, HttpResponseMessage responseContext = null, CancellationToken cancellationToken = default) + { + + } + /// /// Use this method to register an exception handler, /// which can throw another, more meaningful exception /// public virtual void AttemptParseAndThrowException(Exception ex, HttpResponseMessage responseContext = null) { - + } /// @@ -186,13 +244,14 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Request object /// Cancellation Token /// - private async Task GetResponseAsync(HttpRequestMessage req, CancellationToken cancellationToken) + public async Task GetResponseAsync(HttpRequestMessage req, CancellationToken cancellationToken) { HttpResponseMessage response = null; try { response = await _httpClient.SendAsync(req, cancellationToken).ConfigureAwait(false); - + + response.EnsureSuccessStatusCode(); return response; } catch (Exception ex) @@ -277,6 +336,5 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) return source.Read(buffer, offset, count) + br; } - } } \ No newline at end of file diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs new file mode 100644 index 000000000..83bf40618 --- /dev/null +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs @@ -0,0 +1,192 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using Duplicati.Library.Utility; + +namespace Duplicati.Library; + +public class OAuthHelperHttpClient : JsonWebHelperHttpClient +{ + private string _Token; + private string _Authid; + private DateTime _mTokenExpires = DateTime.UtcNow; + private string OAuthLoginUrl { get; } + + /// + /// Timeout for authentication requests + /// + private const int AUTHENTICATION_TIMEOUT_SECONDS = 25; + + private const int MAX_AUTHORIZATION_RETRIES = 5; + + private static string OAUTH_LOGIN_URL(string modulename) + { + var u = new Utility.Uri(OAuthContextSettings.ServerURL); + var addr = u.SetPath("") + .SetQuery((u.Query ?? "") + (string.IsNullOrWhiteSpace(u.Query) ? "" : "&") + "type={0}"); + return string.Format(addr.ToString(), modulename); + } + + /// + /// Set to true to automatically add the Authorization header to requests + /// + public bool AutoAuthHeader { get; set; } + + /// + /// Set to true if the provider does not use refresh tokens, but only access tokens + /// + public bool AccessTokenOnly { get; set; } + + /// + /// If true (the default), when a v1 authid is being used it will be swapped + /// with a v2 authid, when the OAuth service returns one (which it dypically + /// does after a provider token refresh has been performed). Some providers + /// are not compatible with v2 authid, tyically because they generate a new + /// refresh token with every access token refresh and invalidates the old. + /// If the oauth service still returns a v2 authid for such a provider, + /// set this property to false to make Duplicati ignore it. + /// + public bool AutoV2 { get; set; } = true; + + public OAuthHelperHttpClient(string authid, string servicename, HttpClient httpClient = null, string useragent = null) + : base(httpClient ?? HttpClientHelper.CreateClient()) + { + _Authid = authid; + OAuthLoginUrl = OAUTH_LOGIN_URL(servicename); + + if (string.IsNullOrEmpty(authid)) + throw new Interface.UserInformationException( + Strings.OAuthHelper.MissingAuthID(OAuthLoginUrl), "MissingAuthID"); + } + + private HttpRequestMessage CreateRequest(string url, string method, bool noAuthorization) + { + var request = base.CreateRequest(url, method); + if (!noAuthorization && AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url)) request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", AccessToken); + return request; + } + + public override HttpRequestMessage CreateRequest(string url, string method = null) + { + return CreateRequest(url, method, false); + } + + private string AccessToken + { + get + { + if (AccessTokenOnly) + return _Authid; + + if (_Token == null || _mTokenExpires < DateTime.UtcNow) + { + var retries = 0; + + while (true) + { + HttpResponseMessage response = null; + try + { + using var request = base.CreateRequest(OAuthContextSettings.ServerURL); + + request.Headers.TryAddWithoutValidation("X-AuthID", _Authid); + + using var timeoutToken = new CancellationTokenSource(); + timeoutToken.CancelAfter(TimeSpan.FromSeconds(AUTHENTICATION_TIMEOUT_SECONDS)); + + response = _httpClient.Send(request, timeoutToken.Token); + response.EnsureSuccessStatusCode(); + + var res = ReadJsonResponse(response); + + _mTokenExpires = DateTime.UtcNow.AddSeconds(res.expires - 30); + if (AutoV2 && !string.IsNullOrWhiteSpace(res.v2_authid)) + _Authid = res.v2_authid; + return _Token = res.access_token; + } + catch (Exception ex) + { + var clientError = false; + + try + { + // Only retry once on client errors + if (ex is HttpRequestException { StatusCode: not null } exception) + { + var sc = (int)exception.StatusCode; + clientError = sc is >= 400 and <= 499; + } + } + catch + { + // ignored + } + + if (response != null && response.Headers.Contains("X-Reason")) + { + var msg = response.Headers.GetValues("X-Reason").FirstOrDefault(); + + if (string.IsNullOrWhiteSpace(msg)) + msg = response.StatusCode.ToString(); + + //TODO: Test; this was ported from old code but looks fragile. + if (response.StatusCode == HttpStatusCode.ServiceUnavailable) + { + string errorKey = msg == response.StatusCode.ToString() + ? "OAuthOverQuotaError" + : "OAuthLoginError"; + + string errorMessage = errorKey == "OAuthOverQuotaError" + ? Strings.OAuthHelper.OverQuotaError + : Strings.OAuthHelper.AuthorizationFailure(msg, OAuthLoginUrl); + + throw new Interface.UserInformationException(errorMessage, errorKey, + errorKey == "OAuthLoginError" ? ex : null); + } + } + + if (retries >= (clientError ? 1 : MAX_AUTHORIZATION_RETRIES)) + { + AttemptParseAndThrowException(ex, response); + throw; + } + + Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retries))); + retries++; + } + finally + { + response?.Dispose(); + } + + } + } + + return _Token; + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthServiceResponse.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthServiceResponse.cs new file mode 100644 index 000000000..ee721cac9 --- /dev/null +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthServiceResponse.cs @@ -0,0 +1,33 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Library; + +public class OAuthServiceResponse +{ + public string access_token { get; set; } + + [Newtonsoft.Json.JsonProperty(NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public int expires { get; set; } + + [Newtonsoft.Json.JsonProperty(NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public string v2_authid { get; set; } +} \ No newline at end of file