diff --git a/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj b/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj index 198df6dd6..be6989d62 100644 --- a/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj +++ b/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj @@ -4,7 +4,10 @@ net8.0 Library Copyright © 2025 Team Duplicati, MIT license + enable + enable + diff --git a/Duplicati/Library/Backend/GoogleServices/GCSConfig.cs b/Duplicati/Library/Backend/GoogleServices/GCSConfig.cs index 2166afad0..4effd5c35 100644 --- a/Duplicati/Library/Backend/GoogleServices/GCSConfig.cs +++ b/Duplicati/Library/Backend/GoogleServices/GCSConfig.cs @@ -1,29 +1,25 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; using Duplicati.Library.Interface; -using System.Collections.Generic; -using System.Linq; -using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.GoogleServices { @@ -32,7 +28,7 @@ namespace Duplicati.Library.Backend.GoogleServices public class GCSConfig : IWebModule { private const ConfigType DEFAULT_CONFIG_TYPE = ConfigType.Locations; - private static readonly string DEFAULT_CONFIG_TYPE_STR = Enum.GetName(typeof(ConfigType), DEFAULT_CONFIG_TYPE); + private static readonly string DEFAULT_CONFIG_TYPE_STR = DEFAULT_CONFIG_TYPE.ToString(); private const string KEY_CONFIGTYPE = "gcs-config"; public enum ConfigType @@ -46,15 +42,13 @@ namespace Duplicati.Library.Backend.GoogleServices #region IWebModule implementation - public System.Collections.Generic.IDictionary Execute(System.Collections.Generic.IDictionary options) + public IDictionary Execute(IDictionary options) { - string k; - options.TryGetValue(KEY_CONFIGTYPE, out k); + options.TryGetValue(KEY_CONFIGTYPE, out var k); if (string.IsNullOrWhiteSpace(k)) k = DEFAULT_CONFIG_TYPE_STR; - ConfigType ct; - if (!Enum.TryParse(k, true, out ct)) + if (!Enum.TryParse(k, true, out var ct)) ct = DEFAULT_CONFIG_TYPE; switch (ct) @@ -66,23 +60,17 @@ namespace Duplicati.Library.Backend.GoogleServices } } - public string Key { get { return "gcs-getconfig"; } } + public string Key => "gcs-getconfig"; - public string DisplayName { get { return LC.L("Google Cloud Storage configuration module"); } } + public string DisplayName => Strings.GCSConfig.DisplayName; - public string Description { get { return LC.L("Expose Google Cloud Storage configuration as a web module"); } } + public string Description => Strings.GCSConfig.Description; - public System.Collections.Generic.IList SupportedCommands - { - get - { - return new List([ - new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, LC.L("The config to get"), LC.L("Provide different config values"), DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType))) - - ]); - } - } + public IList SupportedCommands => + [ + new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, Strings.GCSConfig.ConfigTypeShort, Strings.GCSConfig.ConfigTypeLong, DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType))) + ]; #endregion } diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs index 30e9db6de..c566115d2 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs @@ -23,14 +23,11 @@ using Duplicati.Library.Backend.GoogleServices; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; -using System; -using System.Collections.Generic; using System.Net; using System.Runtime.CompilerServices; using System.Text; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend.GoogleCloudStorage { @@ -38,7 +35,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage // This class is instantiated dynamically in the BackendLoader. public class GoogleCloudStorage : IBackend, IStreamingBackend, IRenameEnabledBackend, ITimeoutExemptBackend { - private const string AUTHID_OPTION = "authid"; + private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("gcs"); private const string PROJECT_OPTION = "gcs-project"; private const string LOCATION_OPTION = "gcs-location"; @@ -46,17 +43,22 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage private readonly string m_bucket; private readonly string m_prefix; - private readonly string m_project; + private readonly string? m_project; private readonly OAuthHelper m_oauth; - private readonly string m_location; - private readonly string m_storage_class; + private readonly string? m_location; + private readonly string? m_storage_class; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; public GoogleCloudStorage() { + m_bucket = null!; + m_prefix = null!; + m_oauth = null!; + m_timeouts = null!; } - public GoogleCloudStorage(string url, Dictionary options) + public GoogleCloudStorage(string url, Dictionary options) { var uri = new Utility.Uri(url); @@ -67,38 +69,37 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage if (m_prefix.StartsWith("/", StringComparison.Ordinal)) m_prefix = m_prefix.Substring(1); - string authid; - options.TryGetValue(AUTHID_OPTION, out authid); - options.TryGetValue(PROJECT_OPTION, out m_project); - options.TryGetValue(LOCATION_OPTION, out m_location); - options.TryGetValue(STORAGECLASS_OPTION, out m_storage_class); + var authId = AuthIdOptionsHelper.Parse(options) + .RequireCredentials(TOKEN_URL); - if (string.IsNullOrEmpty(authid)) - throw new UserInformationException(Strings.GoogleCloudStorage.MissingAuthID(AUTHID_OPTION), "GoogleCloudStorageMissingAuthID"); + m_timeouts = TimeoutOptionsHelper.Parse(options); + m_project = options.GetValueOrDefault(PROJECT_OPTION); + m_location = options.GetValueOrDefault(LOCATION_OPTION); + m_storage_class = options.GetValueOrDefault(STORAGECLASS_OPTION); - m_oauth = new OAuthHelper(authid, this.ProtocolKey); + m_oauth = new OAuthHelper(authId.AuthId!, this.ProtocolKey); m_oauth.AutoAuthHeader = true; } private class ListBucketResponse { - public string nextPageToken { get; set; } - public BucketResourceItem[] items { get; set; } + public string? nextPageToken { get; set; } + public BucketResourceItem[]? items { get; set; } } private class BucketResourceItem { - public string name { get; set; } + public string? name { get; set; } public DateTime? updated { get; set; } public long? size { get; set; } } private class CreateBucketRequest { - public string name { get; set; } - public string location { get; set; } - public string storageClass { get; set; } + public string? name { get; set; } + public string? location { get; set; } + public string? storageClass { get; set; } } private async Task HandleListExceptions(Func> func) @@ -123,12 +124,15 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage var url = WebApi.GoogleCloudStorage.ListUrl(m_bucket, Utility.Uri.UrlEncode(m_prefix)); while (true) { - var resp = await HandleListExceptions(() => m_oauth.ReadJSONResponseAsync(url, cancelToken)).ConfigureAwait(false); + var resp = await HandleListExceptions(() => + Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => + m_oauth.ReadJSONResponseAsync(url, ct)) + ).ConfigureAwait(false); if (resp.items != null) foreach (var f in resp.items) { - var name = f.name; + var name = f.name ?? ""; if (name.StartsWith(m_prefix, StringComparison.OrdinalIgnoreCase)) name = name.Substring(m_prefix.Length); if (f.size == null) @@ -148,33 +152,33 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) + using (var fs = File.OpenRead(filename)) await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.Create(filename)) + using (var fs = File.Create(filename)) await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public Task DeleteAsync(string remotename, CancellationToken cancelToken) { - var req = m_oauth.CreateRequest(WebApi.GoogleCloudStorage.DeleteUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename))); req.Method = "DELETE"; - return m_oauth.ReadJSONResponseAsync(req, cancelToken); + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_oauth.ReadJSONResponseAsync(req, ct)); } public Task TestAsync(CancellationToken cancelToken) => this.TestListAsync(cancelToken); - public async Task CreateFolderAsync(CancellationToken cancelToken) + public Task CreateFolderAsync(CancellationToken cancelToken) { if (string.IsNullOrEmpty(m_project)) throw new UserInformationException(Strings.GoogleCloudStorage.ProjectIDMissingError(PROJECT_OPTION), "GoogleCloudStorageMissingProjectID"); - var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest + var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest { name = m_bucket, location = m_location, @@ -186,23 +190,19 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage req.ContentLength = data.Length; req.ContentType = "application/json; charset=UTF-8"; - var areq = new AsyncHttpRequest(req); + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct => + { + var areq = new AsyncHttpRequest(req); + using (var rs = areq.GetRequestStream()) + await rs.WriteAsync(data, 0, data.Length, ct); - using (var rs = areq.GetRequestStream()) - await rs.WriteAsync(data, 0, data.Length, cancelToken); - - await m_oauth.ReadJSONResponseAsync(areq, cancelToken).ConfigureAwait(false); + await m_oauth.ReadJSONResponseAsync(areq, ct).ConfigureAwait(false); + }); } - public string DisplayName - { - get { return Strings.GoogleCloudStorage.DisplayName; } - } + public string DisplayName => Strings.GoogleCloudStorage.DisplayName; - public string ProtocolKey - { - get { return "gcs"; } - } + public string ProtocolKey => "gcs"; public IList SupportedCommands { @@ -211,50 +211,49 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage StringBuilder locations = new StringBuilder(); StringBuilder storageClasses = new StringBuilder(); - foreach (KeyValuePair s in WebApi.GoogleCloudStorage.KNOWN_GCS_LOCATIONS) + foreach (var s in WebApi.GoogleCloudStorage.KNOWN_GCS_LOCATIONS) locations.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); - foreach (KeyValuePair s in WebApi.GoogleCloudStorage.KNOWN_GCS_STORAGE_CLASSES) + foreach (var s in WebApi.GoogleCloudStorage.KNOWN_GCS_STORAGE_CLASSES) storageClasses.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); - return new List(new ICommandLineArgument[] { + return [ new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.LocationDescriptionShort, Strings.GoogleCloudStorage.LocationDescriptionLong(locations.ToString())), new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.StorageclassDescriptionShort, Strings.GoogleCloudStorage.StorageclassDescriptionLong(storageClasses.ToString())), - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.GoogleCloudStorage.AuthidShort, Strings.GoogleCloudStorage.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("gcs"))), + .. AuthIdOptionsHelper.GetOptions(TOKEN_URL), new CommandLineArgument(PROJECT_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.ProjectDescriptionShort, Strings.GoogleCloudStorage.ProjectDescriptionLong), - }); + .. TimeoutOptionsHelper.GetOptions(), + ]; } } - public string Description - { - get { return Strings.GoogleCloudStorage.Description; } - } + public string Description => Strings.GoogleCloudStorage.Description; public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(WebApi.GoogleCloudStorage.Hosts()); #endregion - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { var item = new BucketResourceItem { name = m_prefix + remotename }; var url = WebApi.GoogleCloudStorage.PutUrl(m_bucket); - var res = await GoogleCommon.ChunkedUploadWithResumeAsync(m_oauth, item, url, stream, cancelToken).ConfigureAwait(false); + var res = await GoogleCommon.ChunkedUploadWithResumeAsync(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken).ConfigureAwait(false); if (res == null) throw new Exception("Upload succeeded, but no data was returned"); } - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { try { - var url = WebApi.GoogleCloudStorage.GetUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)); + var url = WebApi.GoogleCloudStorage.GetUrl(m_bucket, Utility.Uri.UrlPathEncode(m_prefix + remotename)); var req = m_oauth.CreateRequest(url); var areq = new AsyncHttpRequest(req); - using (var resp = areq.GetResponse()) - using (var rs = areq.GetResponseStream()) - await Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken).ConfigureAwait(false); + using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponse()).ConfigureAwait(false)) + using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false)) + using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false); } catch (WebException wex) { @@ -265,9 +264,9 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage } } - public async Task RenameAsync(string oldname, string newname, CancellationToken cancelToken) + public Task RenameAsync(string oldname, string newname, CancellationToken cancelToken) { - var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem + var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem { name = m_prefix + newname, })); @@ -277,11 +276,14 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage req.ContentLength = data.Length; req.ContentType = "application/json; charset=UTF-8"; - var areq = new AsyncHttpRequest(req); - using (var rs = areq.GetRequestStream()) - await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false); + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct => + { + var areq = new AsyncHttpRequest(req); + using (var rs = areq.GetRequestStream()) + await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false); - await m_oauth.ReadJSONResponseAsync(req, cancelToken).ConfigureAwait(false); + await m_oauth.ReadJSONResponseAsync(req, ct).ConfigureAwait(false); + }); } #region IDisposable implementation diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleCommon.cs b/Duplicati/Library/Backend/GoogleServices/GoogleCommon.cs index bce49da94..05dd67b5f 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleCommon.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleCommon.cs @@ -1,30 +1,27 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Utility; using Newtonsoft.Json; -using System; using System.Net; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend.GoogleServices { @@ -42,7 +39,7 @@ namespace Duplicati.Library.Backend.GoogleServices /// The Oauth instance /// The resumeable uploaduri /// The length of the entire stream - private static long QueryUploadRange(OAuthHelper oauth, string uploaduri, long streamlength, out T response) + private static long QueryUploadRange(OAuthHelper oauth, string uploaduri, long streamlength, out T? response) where T : class { response = null; @@ -51,8 +48,9 @@ namespace Duplicati.Library.Backend.GoogleServices req.ContentLength = 0; req.Headers["Content-Range"] = string.Format("bytes */{0}", streamlength); + // TODO: Apply timeout when this is upgraded to HttpClient var areq = new AsyncHttpRequest(req); - using(var resp = oauth.GetResponseWithoutException(areq)) + using (var resp = oauth.GetResponseWithoutException(areq)) { var code = (int)resp.StatusCode; @@ -62,7 +60,7 @@ namespace Duplicati.Library.Backend.GoogleServices response = oauth.ReadJSONResponse(resp); if (response == null) throw new Exception(string.Format("Upload succeeded, but no data was returned, status code: {0}", code)); - + return streamlength; } @@ -73,9 +71,9 @@ namespace Duplicati.Library.Backend.GoogleServices // https://code.google.com/a/google.com/p/apps-api-issues/issues/detail?id=3884 if (resp.Headers["Range"] == null) - return 0; + return 0; else - return long.Parse(resp.Headers["Range"].Split(new char[] { '-' })[1]) + 1; + return long.Parse(resp.Headers["Range"]!.Split(new char[] { '-' })[1]) + 1; } else throw new WebException(string.Format("Unexpected status code: {0}", code), null, WebExceptionStatus.ServerProtocolViolation, resp); @@ -90,10 +88,12 @@ namespace Duplicati.Library.Backend.GoogleServices /// The data to submit as JSON metadata. /// The URL to register the upload session against. /// The stream with content data to upload. + /// The short request timeout. + /// The read write timeout. /// The HTTP Method. /// The type of data to upload as metadata. /// The type of data returned from the upload. - public static async Task ChunkedUploadWithResumeAsync(OAuthHelper oauth, TRequest requestdata, string url, System.IO.Stream stream, CancellationToken cancelToken, string method = "POST") + public static async Task ChunkedUploadWithResumeAsync(OAuthHelper oauth, TRequest requestdata, string url, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken, string method = "POST") where TRequest : class where TResponse : class { @@ -105,25 +105,30 @@ namespace Duplicati.Library.Backend.GoogleServices if (data != null) req.ContentType = "application/json; charset=UTF-8"; - + req.Headers["X-Upload-Content-Type"] = "application/octet-stream"; req.Headers["X-Upload-Content-Length"] = stream.Length.ToString(); - var areq = new AsyncHttpRequest(req); - if (data != null) - using(var rs = areq.GetRequestStream()) - await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false); - - string uploaduri; - using(var resp = (HttpWebResponse)areq.GetResponse()) + var uploaduri = await Utility.Utility.WithTimeout(shortTimeout, cancelToken, async ct => { - if (resp.StatusCode != HttpStatusCode.OK || string.IsNullOrWhiteSpace(resp.Headers["Location"])) - throw new WebException("Failed to start upload session", null, WebExceptionStatus.UnknownError, resp); + var areq = new AsyncHttpRequest(req); + if (data != null) + using (var rs = areq.GetRequestStream()) + await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false); - uploaduri = resp.Headers["Location"]; - } + using (var resp = (HttpWebResponse)areq.GetResponse()) + { + if (resp.StatusCode != HttpStatusCode.OK || string.IsNullOrWhiteSpace(resp.Headers["Location"])) + throw new WebException("Failed to start upload session", null, WebExceptionStatus.UnknownError, resp); - return await ChunkedUploadAsync(oauth, uploaduri, stream, cancelToken).ConfigureAwait(false); + return resp.Headers["Location"]; + } + }).ConfigureAwait(false); + + if (string.IsNullOrWhiteSpace(uploaduri)) + throw new Exception("Failed to start upload session"); + + return await ChunkedUploadAsync(oauth, uploaduri, stream, shortTimeout, readWriteTimeout, cancelToken).ConfigureAwait(false); } /// @@ -133,8 +138,10 @@ namespace Duplicati.Library.Backend.GoogleServices /// The Oauth instance /// The resumeable uploaduri /// The stream with data to upload. + /// The short request timeout. + /// The read write timeout. /// The type of data in the response. - private static async Task ChunkedUploadAsync(OAuthHelper oauth, string uploaduri, System.IO.Stream stream, CancellationToken cancelToken) + private static async Task ChunkedUploadAsync(OAuthHelper oauth, string uploaduri, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken) where T : class { var queryRange = false; @@ -143,14 +150,13 @@ namespace Duplicati.Library.Backend.GoogleServices var buffer = new byte[Library.Utility.Utility.DEFAULT_BUFFER_SIZE]; // Repeatedly try uploading until all retries are done - while(true) + while (true) { try { if (queryRange) { - T re; - offset = GoogleCommon.QueryUploadRange(oauth, uploaduri, stream.Length, out re); + offset = QueryUploadRange(oauth, uploaduri, stream.Length, out var re); queryRange = false; if (re != null) @@ -172,25 +178,26 @@ namespace Duplicati.Library.Backend.GoogleServices // Upload the remaining data var areq = new AsyncHttpRequest(req); - using(var rs = areq.GetRequestStream()) + using (var rs = areq.GetRequestStream()) + using (var ts = rs.ObserveWriteTimeout(readWriteTimeout)) { var remaining = chunkSize; - while(remaining > 0) + while (remaining > 0) { var n = stream.Read(buffer, 0, (int)Math.Min(remaining, Library.Utility.Utility.DEFAULT_BUFFER_SIZE)); - await rs.WriteAsync(buffer, 0, n, cancelToken); + await ts.WriteAsync(buffer, 0, n, cancelToken); remaining -= n; } } // Check the response - using(var resp = oauth.GetResponseWithoutException(areq)) + using (var resp = await Utility.Utility.WithTimeout(shortTimeout, cancelToken, _ => oauth.GetResponseWithoutException(areq)).ConfigureAwait(false)) { var code = (int)resp.StatusCode; if (code == 308 && resp.Headers["Range"] != null) { - offset = long.Parse(resp.Headers["Range"].Split(new char[] {'-'})[1]) + 1; + offset = long.Parse(resp.Headers["Range"]!.Split(new char[] { '-' })[1]) + 1; retries = 0; } else if (code >= 200 && code <= 299) diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index 96088c5b7..6c210aead 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -23,16 +23,11 @@ using Duplicati.Library.Backend.GoogleServices; using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using Newtonsoft.Json; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; using System.Net; using System.Runtime.CompilerServices; using System.Text; -using System.Threading; -using System.Threading.Tasks; namespace Duplicati.Library.Backend.GoogleDrive { @@ -40,41 +35,51 @@ namespace Duplicati.Library.Backend.GoogleDrive // This class is instantiated dynamically in the BackendLoader. public class GoogleDrive : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend, ITimeoutExemptBackend { - private const string AUTHID_OPTION = "authid"; private const string TEAMDRIVE_ID = "googledrive-teamdrive-id"; private const string FOLDER_MIMETYPE = "application/vnd.google-apps.folder"; + private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("googledrive"); + private readonly string m_path; - private readonly string m_teamDriveID; + private readonly string? m_teamDriveID; private readonly OAuthHelper m_oauth; private readonly Dictionary m_filecache; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; - private string m_currentFolderId; + private string? m_currentFolderId; public GoogleDrive() { + m_path = null!; + m_oauth = null!; + m_filecache = null!; + m_timeouts = null!; } - public GoogleDrive(string url, Dictionary options) + public GoogleDrive(string url, Dictionary options) { var uri = new Utility.Uri(url); m_path = Util.AppendDirSeparator(uri.HostAndPath, "/"); - string authid = null; - if (options.ContainsKey(AUTHID_OPTION)) - authid = options[AUTHID_OPTION]; + var auth = AuthIdOptionsHelper.Parse(options) + .RequireCredentials(TOKEN_URL); - if (options.ContainsKey(TEAMDRIVE_ID)) - m_teamDriveID = options[TEAMDRIVE_ID]; - - m_oauth = new OAuthHelper(authid, this.ProtocolKey) { AutoAuthHeader = true }; + m_timeouts = TimeoutOptionsHelper.Parse(options); + m_teamDriveID = options.GetValueOrDefault(TEAMDRIVE_ID); + m_oauth = new OAuthHelper(auth.AuthId!, this.ProtocolKey) { AutoAuthHeader = true }; m_filecache = new Dictionary(); } private async Task GetFolderIdAsync(string path, bool autocreate, CancellationToken cancelToken) { - var curparent = m_teamDriveID ?? (await GetAboutInfoAsync(cancelToken).ConfigureAwait(false)).rootFolderId; + var curparent = string.IsNullOrWhiteSpace(m_teamDriveID) + ? (await GetAboutInfoAsync(cancelToken).ConfigureAwait(false)).rootFolderId + : m_teamDriveID; + + if (string.IsNullOrEmpty(curparent)) + throw new UserInformationException("Unable to get root folder", "GoogleDriveNoRootFolder"); + var curdisplay = new StringBuilder("/"); foreach (var p in path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)) @@ -97,6 +102,10 @@ namespace Duplicati.Library.Backend.GoogleDrive } curdisplay.Append(p).Append("/"); + + if (string.IsNullOrWhiteSpace(curparent)) + throw new UserInformationException($"Unable to get folder id for {curdisplay}", "GoogleDriveNoFolderId"); + } return curparent; @@ -110,11 +119,9 @@ namespace Duplicati.Library.Backend.GoogleDrive return m_currentFolderId; } - private async Task GetFileEntriesAsync(string remotename, bool throwMissingException, CancellationToken cancelToken) + private async Task GetFileEntriesAsync(string remotename, bool throwMissingException, CancellationToken cancelToken) { - GoogleDriveFolderItem[] entries; - m_filecache.TryGetValue(remotename, out entries); - + m_filecache.TryGetValue(remotename, out var entries); if (entries != null) return entries; @@ -124,7 +131,7 @@ namespace Duplicati.Library.Backend.GoogleDrive if (entries == null || entries.Length == 0) { if (throwMissingException) - throw new FileMissingException(); + throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename)); else return null; } @@ -139,7 +146,7 @@ namespace Duplicati.Library.Backend.GoogleDrive #region IStreamingBackend implementation - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { try { @@ -147,10 +154,9 @@ namespace Duplicati.Library.Backend.GoogleDrive if (m_filecache.Count == 0) await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ } - GoogleDriveFolderItem[] files; - m_filecache.TryGetValue(remotename, out files); + m_filecache.TryGetValue(remotename, out var files); - string fileId = null; + string? fileId = null; if (files != null) { if (files.Length == 1) @@ -174,7 +180,7 @@ namespace Duplicati.Library.Backend.GoogleDrive teamDriveId = m_teamDriveID }; - var res = await GoogleCommon.ChunkedUploadWithResumeAsync(m_oauth, item, url, stream, cancelToken, isUpdate ? "PUT" : "POST").ConfigureAwait(false); + var res = await GoogleCommon.ChunkedUploadWithResumeAsync(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken, isUpdate ? "PUT" : "POST").ConfigureAwait(false); m_filecache[remotename] = [res]; } catch @@ -184,19 +190,22 @@ namespace Duplicati.Library.Backend.GoogleDrive } } - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { // Prevent repeated download url lookups if (m_filecache.Count == 0) await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ } - var fileId = (await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false)).OrderByDescending(x => x.createdDate).First().id; + var fileId = (await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false))?.OrderByDescending(x => x.createdDate).FirstOrDefault()?.id; + if (string.IsNullOrEmpty(fileId)) + throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename)); var req = m_oauth.CreateRequest(WebApi.GoogleDrive.GetUrl(fileId)); var areq = new AsyncHttpRequest(req); - using (var resp = (HttpWebResponse)areq.GetResponse()) - using (var rs = areq.GetResponseStream()) - await Duplicati.Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken); + using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => (HttpWebResponse)areq.GetResponse()).ConfigureAwait(false)) + using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false)) + using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken); } #endregion @@ -216,7 +225,7 @@ namespace Duplicati.Library.Backend.GoogleDrive var currentFolderId = await GetCurrentFolderIdAsync(cancelToken).ConfigureAwait(false); await foreach (var n in ListFolder(currentFolderId, null, null, cancelToken).ConfigureAwait(false)) { - FileEntry fe = null; + FileEntry? fe = null; if (n.fileSize == null) fe = new FileEntry(n.title); @@ -231,10 +240,9 @@ namespace Duplicati.Library.Backend.GoogleDrive if (!fe.IsFolder) { - GoogleDriveFolderItem[] lst; - if (!m_filecache.TryGetValue(fe.Name, out lst)) + if (!m_filecache.TryGetValue(fe.Name, out var lst)) { - m_filecache[fe.Name] = new GoogleDriveFolderItem[] { n }; + m_filecache[fe.Name] = [n]; } else { @@ -262,13 +270,13 @@ namespace Duplicati.Library.Backend.GoogleDrive public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - using (var fs = System.IO.File.OpenRead(filename)) + using (var fs = File.OpenRead(filename)) await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (var fs = System.IO.File.Create(filename)) + using (var fs = File.Create(filename)) await GetAsync(remotename, fs, cancelToken); } @@ -276,13 +284,19 @@ namespace Duplicati.Library.Backend.GoogleDrive { try { - foreach (var fileid in from n in await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false) select n.id) + var entries = await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false); + if (entries == null) + throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename)); + + foreach (var fileid in entries.Select(x => x.id).WhereNotNullOrWhiteSpace()) { var url = WebApi.GoogleDrive.DeleteUrl(Library.Utility.Uri.UrlPathEncode(fileid), m_teamDriveID); - await m_oauth.GetJSONDataAsync(url, cancelToken, x => - { - x.Method = "DELETE"; - }); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_oauth.GetJSONDataAsync(url, cancelToken, x => + { + x.Method = "DELETE"; + }) + ).ConfigureAwait(false); } m_filecache.Remove(remotename); @@ -304,32 +318,19 @@ namespace Duplicati.Library.Backend.GoogleDrive m_currentFolderId = await GetFolderIdAsync(m_path, true, cancelToken).ConfigureAwait(false); } - public string DisplayName - { - get - { - return Strings.GoogleDrive.DisplayName; - } - } + public string DisplayName => Strings.GoogleDrive.DisplayName; - public string ProtocolKey - { - get - { - return "googledrive"; - } - } + public string ProtocolKey => "googledrive"; - public System.Collections.Generic.IList SupportedCommands => new List(new ICommandLineArgument[] { - new CommandLineArgument(AUTHID_OPTION, - CommandLineArgument.ArgumentType.Password, - Strings.GoogleDrive.AuthidShort, - Strings.GoogleDrive.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("googledrive"))), - new CommandLineArgument(TEAMDRIVE_ID, - CommandLineArgument.ArgumentType.String, - Strings.GoogleDrive.TeamDriveIdShort, - Strings.GoogleDrive.TeamDriveIdLong), - }); + public IList SupportedCommands => + [ + .. AuthIdOptionsHelper.GetOptions(TOKEN_URL), + new CommandLineArgument(TEAMDRIVE_ID, + CommandLineArgument.ArgumentType.String, + Strings.GoogleDrive.TeamDriveIdShort, + Strings.GoogleDrive.TeamDriveIdLong), + .. TimeoutOptionsHelper.GetOptions(), + ]; public string Description { @@ -342,7 +343,7 @@ namespace Duplicati.Library.Backend.GoogleDrive #endregion #region IQuotaEnabledBackend implementation - public async Task GetQuotaInfoAsync(CancellationToken cancelToken) + public async Task GetQuotaInfoAsync(CancellationToken cancelToken) { try { @@ -365,9 +366,10 @@ namespace Duplicati.Library.Backend.GoogleDrive try { var files = await GetFileEntriesAsync(oldname, true, cancellationToken).ConfigureAwait(false); + if (files == null) + throw new FileMissingException(Strings.GoogleDrive.FilenotFound(oldname)); if (files.Length > 1) - throw new UserInformationException(string.Format(Strings.GoogleDrive.MultipleEntries(oldname, m_path)), - "GoogleDriveMultipleEntries"); + throw new UserInformationException(Strings.GoogleDrive.MultipleEntries(oldname, m_path), "GoogleDriveMultipleEntries"); using var stream = new MemoryStream(); await GetAsync(oldname, stream, cancellationToken).ConfigureAwait(false); @@ -396,13 +398,13 @@ namespace Duplicati.Library.Backend.GoogleDrive private class GoogleDriveParentReference { - public string id { get; set; } + public string? id { get; set; } } private class GoogleDriveListResponse { - public string nextPageToken { get; set; } - public GoogleDriveFolderItem[] items { get; set; } + public string? nextPageToken { get; set; } + public GoogleDriveFolderItem[]? items { get; set; } } private class GoogleDriveFolderItemLabels @@ -412,41 +414,44 @@ namespace Duplicati.Library.Backend.GoogleDrive private class GoogleDriveFolderItem { - public string id { get; set; } - public string title { get; set; } - public string description { get; set; } - public string mimeType { get; set; } - public GoogleDriveFolderItemLabels labels { get; set; } + public string? id { get; set; } + public string? title { get; set; } + public string? description { get; set; } + public string? mimeType { get; set; } + public GoogleDriveFolderItemLabels? labels { get; set; } public DateTime? createdDate { get; set; } public DateTime? modifiedDate { get; set; } public long? fileSize { get; set; } - public string teamDriveId { get; set; } - public GoogleDriveParentReference[] parents { get; set; } + public string? teamDriveId { get; set; } + public GoogleDriveParentReference[]? parents { get; set; } } private class GoogleDriveAboutResponse { public long? quotaBytesTotal { get; set; } public long? quotaBytesUsed { get; set; } - public string rootFolderId { get; set; } + public string? rootFolderId { get; set; } } - private async IAsyncEnumerable ListFolder(string parentfolder, bool? onlyFolders, string name, [EnumeratorCancellation] CancellationToken cancelToken) + private async IAsyncEnumerable ListFolder(string parentfolder, bool? onlyFolders, string? name, [EnumeratorCancellation] CancellationToken cancelToken) { - var fileQuery = new string[] { + var fileQuery = new string?[] { string.IsNullOrEmpty(name) ? null : string.Format("title = '{0}'", EscapeTitleEntries(name)), onlyFolders == null ? null : string.Format("mimeType {0}= '{1}'", onlyFolders.Value ? "" : "!", FOLDER_MIMETYPE), string.Format("'{0}' in parents", EscapeTitleEntries(parentfolder)), "trashed=false" }; - var encodedFileQuery = Library.Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null))); + var encodedFileQuery = Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null))); var url = WebApi.GoogleDrive.ListUrl(encodedFileQuery, m_teamDriveID); while (true) { - var res = await m_oauth.GetJSONDataAsync(url, cancelToken).ConfigureAwait(false); - foreach (var n in res.items) + var res = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct + => m_oauth.GetJSONDataAsync(url, ct) + ).ConfigureAwait(false); + + foreach (var n in res.items ?? []) yield return n; var token = res.nextPageToken; @@ -458,9 +463,8 @@ namespace Duplicati.Library.Backend.GoogleDrive } private Task GetAboutInfoAsync(CancellationToken cancelToken) - { - return m_oauth.GetJSONDataAsync(WebApi.GoogleDrive.AboutInfoUrl(), cancelToken); - } + => Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_oauth.GetJSONDataAsync(WebApi.GoogleDrive.AboutInfoUrl(), ct)); private Task CreateFolderAsync(string name, string parent, CancellationToken cancelToken) { @@ -470,25 +474,27 @@ namespace Duplicati.Library.Backend.GoogleDrive description = name, mimeType = FOLDER_MIMETYPE, labels = new GoogleDriveFolderItemLabels { hidden = true }, - parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = parent } } + parents = [new GoogleDriveParentReference { id = parent }] }; - var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder)); + var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder)); - return m_oauth.GetJSONDataAsync( - WebApi.GoogleDrive.CreateFolderUrl(m_teamDriveID), - cancelToken, - x => - { - x.Method = "POST"; - x.ContentType = "application/json; charset=UTF-8"; - x.ContentLength = data.Length; + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct + => m_oauth.GetJSONDataAsync( + WebApi.GoogleDrive.CreateFolderUrl(m_teamDriveID), + ct, + x => + { + x.Method = "POST"; + x.ContentType = "application/json; charset=UTF-8"; + x.ContentLength = data.Length; - }, async (req, ct) => - { - using (var rs = req.GetRequestStream()) - await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false); - }); + }, async (req, ct) => + { + using (var rs = req.GetRequestStream()) + await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false); + }) + ); } } } diff --git a/Duplicati/Library/Backend/GoogleServices/Strings.cs b/Duplicati/Library/Backend/GoogleServices/Strings.cs index 35e660397..0d4a899f4 100644 --- a/Duplicati/Library/Backend/GoogleServices/Strings.cs +++ b/Duplicati/Library/Backend/GoogleServices/Strings.cs @@ -1,35 +1,33 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; namespace Duplicati.Library.Backend.Strings { - internal static class GoogleCloudStorage { + internal static class GoogleCloudStorage + { public static string Description { get { return LC.L(@"This backend can read and write data to Google Cloud Storage. Allowed format is ""gcs://bucket/folder""."); } } public static string DisplayName { get { return LC.L(@"Google Cloud Storage"); } } - public static string MissingAuthID(string url) { return LC.L(@"You need an AuthID. You can get it from: {0}", url); } public static string ProjectIDMissingError(string projectoption) { return LC.L(@"You must supply a project ID with --{0} for creating a bucket.", projectoption); } - public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); } - public static string AuthidShort { get { return LC.L(@"The authorization code"); } } public static string LocationDescriptionLong(string regions) { return LC.L(@"This option is only used when creating new buckets. Use this option to change what region the data is stored in. Charges vary with bucket location. Known bucket locations: {0}", regions); } public static string LocationDescriptionShort { get { return LC.L(@"Specify location option for creating a bucket"); } } @@ -40,13 +38,21 @@ namespace Duplicati.Library.Backend.Strings public static string ProjectDescriptionShort { get { return LC.L(@"Specify project for creating a bucket"); } } } - internal static class GoogleDrive { + internal static class GoogleDrive + { public static string Description { get { return LC.L(@"This backend can read and write data to Google Drive. Allowed format is ""googledrive://folder/subfolder""."); } } public static string DisplayName { get { return LC.L(@"Google Drive"); } } public static string MultipleEntries(string folder, string parent) { return LC.L(@"There is more than one item named ""{0}"" in the folder ""{1}"".", folder, parent); } - public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); } - public static string AuthidShort { get { return LC.L(@"The authorization code"); } } + public static string FilenotFound(string name) { return LC.L(@"File not found: {0}", name); } public static string TeamDriveIdLong { get { return LC.L("This option sets the team drive to use. Leaving it empty uses the personal drive."); } } public static string TeamDriveIdShort { get { return LC.L("Team drive ID"); } } } + + internal static class GCSConfig + { + public static string DisplayName => LC.L("Google Cloud Storage configuration module"); + public static string Description => LC.L("Expose Google Cloud Storage configuration as a web module"); + public static string ConfigTypeShort => LC.L("The config to get"); + public static string ConfigTypeLong => LC.L("Provide different config values"); + } } diff --git a/Duplicati/Library/Backend/GoogleServices/WebApi.cs b/Duplicati/Library/Backend/GoogleServices/WebApi.cs index 10d868ac6..5f488a610 100644 --- a/Duplicati/Library/Backend/GoogleServices/WebApi.cs +++ b/Duplicati/Library/Backend/GoogleServices/WebApi.cs @@ -1,25 +1,24 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System.Collections.Generic; using System.Collections.Specialized; using Duplicati.Library.Utility; @@ -28,41 +27,41 @@ namespace Duplicati.Library.Backend.WebApi public static class GoogleCloudStorage { // From: https://cloud.google.com/storage/docs/bucket-locations - public static readonly KeyValuePair[] KNOWN_GCS_LOCATIONS = { - new KeyValuePair("(default)", null), - new KeyValuePair("Europe", "EU"), - new KeyValuePair("United States", "US"), - new KeyValuePair("Asia", "ASIA"), + public static readonly KeyValuePair[] KNOWN_GCS_LOCATIONS = { + new KeyValuePair("(default)", null), + new KeyValuePair("Europe", "EU"), + new KeyValuePair("United States", "US"), + new KeyValuePair("Asia", "ASIA"), //Regional buckets: https://cloud.google.com/storage/docs/regional-buckets - new KeyValuePair("Eastern Asia-Pacific", "ASIA-EAST1"), - new KeyValuePair("Central United States 1", "US-CENTRAL1"), - new KeyValuePair("Central United States 2", "US-CENTRAL2"), - new KeyValuePair("Eastern United States 1", "US-EAST1"), - new KeyValuePair("Eastern United States 2", "US-EAST2"), - new KeyValuePair("Eastern United States 3", "US-EAST3"), - new KeyValuePair("Western United States", "US-WEST1"), + new KeyValuePair("Eastern Asia-Pacific", "ASIA-EAST1"), + new KeyValuePair("Central United States 1", "US-CENTRAL1"), + new KeyValuePair("Central United States 2", "US-CENTRAL2"), + new KeyValuePair("Eastern United States 1", "US-EAST1"), + new KeyValuePair("Eastern United States 2", "US-EAST2"), + new KeyValuePair("Eastern United States 3", "US-EAST3"), + new KeyValuePair("Western United States", "US-WEST1"), }; - public static readonly KeyValuePair[] KNOWN_GCS_STORAGE_CLASSES = { - new KeyValuePair("(default)", null), - new KeyValuePair("Standard", "STANDARD"), - new KeyValuePair("Durable Reduced Availability (DRA)", "DURABLE_REDUCED_AVAILABILITY"), - new KeyValuePair("Nearline", "NEARLINE"), + public static readonly KeyValuePair[] KNOWN_GCS_STORAGE_CLASSES = { + new KeyValuePair("(default)", null), + new KeyValuePair("Standard", "STANDARD"), + new KeyValuePair("Durable Reduced Availability (DRA)", "DURABLE_REDUCED_AVAILABILITY"), + new KeyValuePair("Nearline", "NEARLINE"), }; - public static string[] Hosts() - { - return new [] { new System.Uri(Url.UPLOAD).Host, - new System.Uri(Url.API).Host }; - } + public static string[] Hosts() => + [ + new System.Uri(Url.UPLOAD).Host, + new System.Uri(Url.API).Host + ]; public static string DeleteUrl(string bucketId, string objectId) { var path = BucketObjectPath(bucketId, objectId); - return Uri.UriBuilder(Url.API, path); + return Utility.Uri.UriBuilder(Url.API, path); } public static string CreateFolderUrl(string projectId) @@ -72,20 +71,16 @@ namespace Duplicati.Library.Backend.WebApi { QueryParam.Project, projectId } }; - return Uri.UriBuilder(Url.API, Path.Bucket, queryParams); + return Utility.Uri.UriBuilder(Url.API, Path.Bucket, queryParams); } public static string RenameUrl(string bucketId, string objectId) - { - return Uri.UriBuilder(Url.API, BucketObjectPath(bucketId, objectId)); - } + => Utility.Uri.UriBuilder(Url.API, BucketObjectPath(bucketId, objectId)); public static string ListUrl(string bucketId, string prefix) - { - return ListUrl(bucketId, prefix, null); - } + => ListUrl(bucketId, prefix, null); - public static string ListUrl(string bucketId, string prefix, string token) + public static string ListUrl(string bucketId, string prefix, string? token) { var queryParams = new NameValueCollection { { QueryParam.Prefix, prefix } @@ -96,7 +91,7 @@ namespace Duplicati.Library.Backend.WebApi queryParams.Set(QueryParam.PageToken, token); } - return Uri.UriBuilder(Url.API, BucketObjectPath(bucketId), queryParams); + return Utility.Uri.UriBuilder(Url.API, BucketObjectPath(bucketId), queryParams); } public static string PutUrl(string bucketId) @@ -106,7 +101,7 @@ namespace Duplicati.Library.Backend.WebApi { QueryParam.UploadType, QueryValue.Resumable } }; var path = UrlPath.Create(Path.Bucket).Append(bucketId).Append(Path.Object).ToString(); - return Uri.UriBuilder(Url.UPLOAD, path, queryParams); + return Utility.Uri.UriBuilder(Url.UPLOAD, path, queryParams); } public static string GetUrl(string bucketId, string objectId) @@ -118,7 +113,7 @@ namespace Duplicati.Library.Backend.WebApi }; var path = BucketObjectPath(bucketId, objectId); - return Uri.UriBuilder(Url.API, path, queryParams); + return Utility.Uri.UriBuilder(Url.API, path, queryParams); } private static class Url @@ -148,36 +143,27 @@ namespace Duplicati.Library.Backend.WebApi public const string Object = "o"; } - private static string BucketObjectPath(string bucketId, string objectId = null) - { - return UrlPath.Create(Path.Bucket) - .Append(bucketId) - .Append(Path.Object) - .Append(objectId).ToString(); - } + private static string BucketObjectPath(string bucketId, string? objectId = null) + => UrlPath.Create(Path.Bucket) + .Append(bucketId) + .Append(Path.Object) + .Append(objectId).ToString(); } public static class GoogleDrive { public static string[] Hosts() - { - return new [] { new System.Uri(Url.DRIVE).Host, new System.Uri(Url.UPLOAD).Host }; - - } + => [new System.Uri(Url.DRIVE).Host, new System.Uri(Url.UPLOAD).Host]; public static string GetUrl(string fileId) - { - return FileQueryUrl(fileId, new NameValueCollection{ + => FileQueryUrl(fileId, new NameValueCollection{ { QueryParam.Alt, QueryValue.Media } }); - } - public static string DeleteUrl(string fileId, string teamDriveId) - { - return FileQueryUrl(Uri.UrlPathEncode(fileId), AddTeamDriveParam(teamDriveId)); - } + public static string DeleteUrl(string fileId, string? teamDriveId) + => FileQueryUrl(Utility.Uri.UrlPathEncode(fileId), AddTeamDriveParam(teamDriveId)); - public static string PutUrl(string fileId, bool useTeamDrive) + public static string PutUrl(string? fileId, bool useTeamDrive) { var queryParams = new NameValueCollection { { QueryParam.UploadType, @@ -189,16 +175,14 @@ namespace Duplicati.Library.Backend.WebApi } return !string.IsNullOrWhiteSpace(fileId) ? - FileUploadUrl(Uri.UrlPathEncode(fileId), queryParams) : + FileUploadUrl(Utility.Uri.UrlPathEncode(fileId), queryParams) : FileUploadUrl(queryParams); } - public static string ListUrl(string fileQuery, string teamDriveId) - { - return ListUrl(fileQuery, teamDriveId, null); - } - - public static string ListUrl(string fileQuery, string teamDriveId, string token) + public static string ListUrl(string fileQuery, string? teamDriveId) + => ListUrl(fileQuery, teamDriveId, null); + + public static string ListUrl(string fileQuery, string? teamDriveId, string? token) { var queryParams = new NameValueCollection { @@ -207,7 +191,7 @@ namespace Duplicati.Library.Backend.WebApi }; queryParams.Add(AddTeamDriveParam(teamDriveId)); - + if (token != null) { queryParams.Set(QueryParam.PageToken, token); @@ -216,15 +200,11 @@ namespace Duplicati.Library.Backend.WebApi return FileQueryUrl(queryParams); } - public static string CreateFolderUrl(string teamDriveId) - { - return FileQueryUrl(AddTeamDriveParam(teamDriveId)); - } + public static string CreateFolderUrl(string? teamDriveId) + => FileQueryUrl(AddTeamDriveParam(teamDriveId)); public static string AboutInfoUrl() - { - return Uri.UriBuilder(Url.DRIVE, Path.About); - } + => Utility.Uri.UriBuilder(Url.DRIVE, Path.About); private static class Url { @@ -259,40 +239,30 @@ namespace Duplicati.Library.Backend.WebApi } private static string FileQueryUrl(NameValueCollection values) - { - return Uri.UriBuilder(Url.DRIVE, Path.File, values); - } + => Utility.Uri.UriBuilder(Url.DRIVE, Path.File, values); - private static string FileQueryUrl(string fileId, NameValueCollection values = null) - { - return Uri.UriBuilder(Url.DRIVE, UrlPath.Create(Path.File).Append(fileId).ToString(), - values); - } + private static string FileQueryUrl(string fileId, NameValueCollection? values = null) + => Utility.Uri.UriBuilder(Url.DRIVE, UrlPath.Create(Path.File).Append(fileId).ToString(), values); - private static string FileUploadUrl(string fileId, NameValueCollection values) - { - return Uri.UriBuilder(Url.UPLOAD, UrlPath.Create(Path.File).Append(fileId).ToString(), - values); - } + private static string FileUploadUrl(string fileId, NameValueCollection? values) + => Utility.Uri.UriBuilder(Url.UPLOAD, UrlPath.Create(Path.File).Append(fileId).ToString(), values); private static string FileUploadUrl(NameValueCollection values) - { - return Uri.UriBuilder(Url.UPLOAD, Path.File, values); - } + => Utility.Uri.UriBuilder(Url.UPLOAD, Path.File, values); - private static NameValueCollection AddTeamDriveParam(string teamDriveId) + private static NameValueCollection AddTeamDriveParam(string? teamDriveId) { return teamDriveId != null ? new NameValueCollection { - { WebApi.GoogleDrive.QueryParam.SupportsTeamDrive, - WebApi.GoogleDrive.QueryValue.True }, - { WebApi.GoogleDrive.QueryParam.TeamDriveId, + { QueryParam.SupportsTeamDrive, + QueryValue.True }, + { QueryParam.TeamDriveId, teamDriveId }, - { WebApi.GoogleDrive.QueryParam.IncludeTeamDrive, - WebApi.GoogleDrive.QueryValue.True }, - { WebApi.GoogleDrive.QueryParam.corpora, - WebApi.GoogleDrive.QueryValue.TeamDrive } + { QueryParam.IncludeTeamDrive, + QueryValue.True }, + { QueryParam.corpora, + QueryValue.TeamDrive } - } : new NameValueCollection( ); + } : new NameValueCollection(); } } } \ No newline at end of file diff --git a/Duplicati/Library/Interface/IWebModule.cs b/Duplicati/Library/Interface/IWebModule.cs index c057858aa..a0a316fc2 100644 --- a/Duplicati/Library/Interface/IWebModule.cs +++ b/Duplicati/Library/Interface/IWebModule.cs @@ -1,25 +1,24 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; using System.Collections.Generic; namespace Duplicati.Library.Interface