haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated Google services

Browse Source
This commit is contained in:
Kenneth Skovhede
2025-03-21 12:34:06 +01:00
parent 57c9aab3e4
commit 73bb3d811b
8 changed files with 403 additions and 422 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj
+3
View File
@@ -4,7 +4,10 @@
<TargetFramework>net8.0</TargetFramework>
<OutputType>Library</OutputType>
<Copyright>Copyright © 2025 Team Duplicati, MIT license</Copyright>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
Duplicati/Library/Backend/GoogleServices/GCSConfig.cs
+30 -42
View File
@@ -1,29 +1,25 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using Duplicati.Library.Interface;
using System.Collections.Generic;
using System.Linq;
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.GoogleServices
{
@@ -32,7 +28,7 @@ namespace Duplicati.Library.Backend.GoogleServices
public class GCSConfig : IWebModule
{
private const ConfigType DEFAULT_CONFIG_TYPE = ConfigType.Locations;
private static readonly string DEFAULT_CONFIG_TYPE_STR = Enum.GetName(typeof(ConfigType), DEFAULT_CONFIG_TYPE);
private static readonly string DEFAULT_CONFIG_TYPE_STR = DEFAULT_CONFIG_TYPE.ToString();
private const string KEY_CONFIGTYPE = "gcs-config";
public enum ConfigType
@@ -46,15 +42,13 @@ namespace Duplicati.Library.Backend.GoogleServices
#region IWebModule implementation
public System.Collections.Generic.IDictionary<string, string> Execute(System.Collections.Generic.IDictionary<string, string> options)
public IDictionary<string, string?> Execute(IDictionary<string, string?> options)
{
string k;
options.TryGetValue(KEY_CONFIGTYPE, out k);
options.TryGetValue(KEY_CONFIGTYPE, out var k);
if (string.IsNullOrWhiteSpace(k))
k = DEFAULT_CONFIG_TYPE_STR;
ConfigType ct;
if (!Enum.TryParse<ConfigType>(k, true, out ct))
if (!Enum.TryParse<ConfigType>(k, true, out var ct))
ct = DEFAULT_CONFIG_TYPE;
switch (ct)
@@ -66,23 +60,17 @@ namespace Duplicati.Library.Backend.GoogleServices
}
}
public string Key { get { return "gcs-getconfig"; } }
public string Key => "gcs-getconfig";
public string DisplayName { get { return LC.L("Google Cloud Storage configuration module"); } }
public string DisplayName => Strings.GCSConfig.DisplayName;
public string Description { get { return LC.L("Expose Google Cloud Storage configuration as a web module"); } }
public string Description => Strings.GCSConfig.Description;
public System.Collections.Generic.IList<ICommandLineArgument> SupportedCommands
{
get
{
return new List<ICommandLineArgument>([
new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, LC.L("The config to get"), LC.L("Provide different config values"), DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType)))
]);
}
}
public IList<ICommandLineArgument> SupportedCommands =>
[
new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, Strings.GCSConfig.ConfigTypeShort, Strings.GCSConfig.ConfigTypeLong, DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType)))
];
#endregion
}
Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs
+68 -66
View File
@@ -23,14 +23,11 @@ using Duplicati.Library.Backend.GoogleServices;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Net;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend.GoogleCloudStorage
{
@@ -38,7 +35,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
// This class is instantiated dynamically in the BackendLoader.
public class GoogleCloudStorage : IBackend, IStreamingBackend, IRenameEnabledBackend, ITimeoutExemptBackend
{
private const string AUTHID_OPTION = "authid";
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("gcs");
private const string PROJECT_OPTION = "gcs-project";
private const string LOCATION_OPTION = "gcs-location";
@@ -46,17 +43,22 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
private readonly string m_bucket;
private readonly string m_prefix;
private readonly string m_project;
private readonly string? m_project;
private readonly OAuthHelper m_oauth;
private readonly string m_location;
private readonly string m_storage_class;
private readonly string? m_location;
private readonly string? m_storage_class;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
public GoogleCloudStorage()
{
m_bucket = null!;
m_prefix = null!;
m_oauth = null!;
m_timeouts = null!;
}
public GoogleCloudStorage(string url, Dictionary<string, string> options)
public GoogleCloudStorage(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
@@ -67,38 +69,37 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
if (m_prefix.StartsWith("/", StringComparison.Ordinal))
m_prefix = m_prefix.Substring(1);
string authid;
options.TryGetValue(AUTHID_OPTION, out authid);
options.TryGetValue(PROJECT_OPTION, out m_project);
options.TryGetValue(LOCATION_OPTION, out m_location);
options.TryGetValue(STORAGECLASS_OPTION, out m_storage_class);
var authId = AuthIdOptionsHelper.Parse(options)
.RequireCredentials(TOKEN_URL);
if (string.IsNullOrEmpty(authid))
throw new UserInformationException(Strings.GoogleCloudStorage.MissingAuthID(AUTHID_OPTION), "GoogleCloudStorageMissingAuthID");
m_timeouts = TimeoutOptionsHelper.Parse(options);
m_project = options.GetValueOrDefault(PROJECT_OPTION);
m_location = options.GetValueOrDefault(LOCATION_OPTION);
m_storage_class = options.GetValueOrDefault(STORAGECLASS_OPTION);
m_oauth = new OAuthHelper(authid, this.ProtocolKey);
m_oauth = new OAuthHelper(authId.AuthId!, this.ProtocolKey);
m_oauth.AutoAuthHeader = true;
}
private class ListBucketResponse
{
public string nextPageToken { get; set; }
public BucketResourceItem[] items { get; set; }
public string? nextPageToken { get; set; }
public BucketResourceItem[]? items { get; set; }
}
private class BucketResourceItem
{
public string name { get; set; }
public string? name { get; set; }
public DateTime? updated { get; set; }
public long? size { get; set; }
}
private class CreateBucketRequest
{
public string name { get; set; }
public string location { get; set; }
public string storageClass { get; set; }
public string? name { get; set; }
public string? location { get; set; }
public string? storageClass { get; set; }
}
private async Task<T> HandleListExceptions<T>(Func<Task<T>> func)
@@ -123,12 +124,15 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
var url = WebApi.GoogleCloudStorage.ListUrl(m_bucket, Utility.Uri.UrlEncode(m_prefix));
while (true)
{
var resp = await HandleListExceptions(() => m_oauth.ReadJSONResponseAsync<ListBucketResponse>(url, cancelToken)).ConfigureAwait(false);
var resp = await HandleListExceptions(() =>
Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct =>
m_oauth.ReadJSONResponseAsync<ListBucketResponse>(url, ct))
).ConfigureAwait(false);
if (resp.items != null)
foreach (var f in resp.items)
{
var name = f.name;
var name = f.name ?? "";
if (name.StartsWith(m_prefix, StringComparison.OrdinalIgnoreCase))
name = name.Substring(m_prefix.Length);
if (f.size == null)
@@ -148,33 +152,33 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
using (var fs = File.OpenRead(filename))
await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (System.IO.FileStream fs = System.IO.File.Create(filename))
using (var fs = File.Create(filename))
await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
var req = m_oauth.CreateRequest(WebApi.GoogleCloudStorage.DeleteUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)));
req.Method = "DELETE";
return m_oauth.ReadJSONResponseAsync<object>(req, cancelToken);
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.ReadJSONResponseAsync<object>(req, ct));
}
public Task TestAsync(CancellationToken cancelToken)
=> this.TestListAsync(cancelToken);
public async Task CreateFolderAsync(CancellationToken cancelToken)
public Task CreateFolderAsync(CancellationToken cancelToken)
{
if (string.IsNullOrEmpty(m_project))
throw new UserInformationException(Strings.GoogleCloudStorage.ProjectIDMissingError(PROJECT_OPTION), "GoogleCloudStorageMissingProjectID");
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest
var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest
{
name = m_bucket,
location = m_location,
@@ -186,23 +190,19 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
req.ContentLength = data.Length;
req.ContentType = "application/json; charset=UTF-8";
var areq = new AsyncHttpRequest(req);
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct);
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, cancelToken);
await m_oauth.ReadJSONResponseAsync<BucketResourceItem>(areq, cancelToken).ConfigureAwait(false);
await m_oauth.ReadJSONResponseAsync<BucketResourceItem>(areq, ct).ConfigureAwait(false);
});
}
public string DisplayName
{
get { return Strings.GoogleCloudStorage.DisplayName; }
}
public string DisplayName => Strings.GoogleCloudStorage.DisplayName;
public string ProtocolKey
{
get { return "gcs"; }
}
public string ProtocolKey => "gcs";
public IList<ICommandLineArgument> SupportedCommands
{
@@ -211,50 +211,49 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
StringBuilder locations = new StringBuilder();
StringBuilder storageClasses = new StringBuilder();
foreach (KeyValuePair<string, string> s in WebApi.GoogleCloudStorage.KNOWN_GCS_LOCATIONS)
foreach (var s in WebApi.GoogleCloudStorage.KNOWN_GCS_LOCATIONS)
locations.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
foreach (KeyValuePair<string, string> s in WebApi.GoogleCloudStorage.KNOWN_GCS_STORAGE_CLASSES)
foreach (var s in WebApi.GoogleCloudStorage.KNOWN_GCS_STORAGE_CLASSES)
storageClasses.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
return [
new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.LocationDescriptionShort, Strings.GoogleCloudStorage.LocationDescriptionLong(locations.ToString())),
new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.StorageclassDescriptionShort, Strings.GoogleCloudStorage.StorageclassDescriptionLong(storageClasses.ToString())),
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.GoogleCloudStorage.AuthidShort, Strings.GoogleCloudStorage.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("gcs"))),
.. AuthIdOptionsHelper.GetOptions(TOKEN_URL),
new CommandLineArgument(PROJECT_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.ProjectDescriptionShort, Strings.GoogleCloudStorage.ProjectDescriptionLong),
});
.. TimeoutOptionsHelper.GetOptions(),
];
}
}
public string Description
{
get { return Strings.GoogleCloudStorage.Description; }
}
public string Description => Strings.GoogleCloudStorage.Description;
public Task<string[]> GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(WebApi.GoogleCloudStorage.Hosts());
#endregion
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
var item = new BucketResourceItem { name = m_prefix + remotename };
var url = WebApi.GoogleCloudStorage.PutUrl(m_bucket);
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<BucketResourceItem, BucketResourceItem>(m_oauth, item, url, stream, cancelToken).ConfigureAwait(false);
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<BucketResourceItem, BucketResourceItem>(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken).ConfigureAwait(false);
if (res == null)
throw new Exception("Upload succeeded, but no data was returned");
}
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
try
{
var url = WebApi.GoogleCloudStorage.GetUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename));
var url = WebApi.GoogleCloudStorage.GetUrl(m_bucket, Utility.Uri.UrlPathEncode(m_prefix + remotename));
var req = m_oauth.CreateRequest(url);
var areq = new AsyncHttpRequest(req);
using (var resp = areq.GetResponse())
using (var rs = areq.GetResponseStream())
await Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken).ConfigureAwait(false);
using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponse()).ConfigureAwait(false))
using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false))
using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false);
}
catch (WebException wex)
{
@@ -265,9 +264,9 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
}
}
public async Task RenameAsync(string oldname, string newname, CancellationToken cancelToken)
public Task RenameAsync(string oldname, string newname, CancellationToken cancelToken)
{
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem
var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem
{
name = m_prefix + newname,
}));
@@ -277,11 +276,14 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
req.ContentLength = data.Length;
req.ContentType = "application/json; charset=UTF-8";
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false);
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false);
await m_oauth.ReadJSONResponseAsync<BucketResourceItem>(req, cancelToken).ConfigureAwait(false);
await m_oauth.ReadJSONResponseAsync<BucketResourceItem>(req, ct).ConfigureAwait(false);
});
}
#region IDisposable implementation
Duplicati/Library/Backend/GoogleServices/GoogleCommon.cs
+57 -50
View File
@@ -1,30 +1,27 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Utility;
using Newtonsoft.Json;
using System;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend.GoogleServices
{
@@ -42,7 +39,7 @@ namespace Duplicati.Library.Backend.GoogleServices
/// <param name="oauth">The Oauth instance</param>
/// <param name="uploaduri">The resumeable uploaduri</param>
/// <param name="streamlength">The length of the entire stream</param>
private static long QueryUploadRange<T>(OAuthHelper oauth, string uploaduri, long streamlength, out T response)
private static long QueryUploadRange<T>(OAuthHelper oauth, string uploaduri, long streamlength, out T? response)
where T : class
{
response = null;
@@ -51,8 +48,9 @@ namespace Duplicati.Library.Backend.GoogleServices
req.ContentLength = 0;
req.Headers["Content-Range"] = string.Format("bytes */{0}", streamlength);
// TODO: Apply timeout when this is upgraded to HttpClient
var areq = new AsyncHttpRequest(req);
using(var resp = oauth.GetResponseWithoutException(areq))
using (var resp = oauth.GetResponseWithoutException(areq))
{
var code = (int)resp.StatusCode;
@@ -62,7 +60,7 @@ namespace Duplicati.Library.Backend.GoogleServices
response = oauth.ReadJSONResponse<T>(resp);
if (response == null)
throw new Exception(string.Format("Upload succeeded, but no data was returned, status code: {0}", code));
return streamlength;
}
@@ -73,9 +71,9 @@ namespace Duplicati.Library.Backend.GoogleServices
// https://code.google.com/a/google.com/p/apps-api-issues/issues/detail?id=3884
if (resp.Headers["Range"] == null)
return 0;
return 0;
else
return long.Parse(resp.Headers["Range"].Split(new char[] { '-' })[1]) + 1;
return long.Parse(resp.Headers["Range"]!.Split(new char[] { '-' })[1]) + 1;
}
else
throw new WebException(string.Format("Unexpected status code: {0}", code), null, WebExceptionStatus.ServerProtocolViolation, resp);
@@ -90,10 +88,12 @@ namespace Duplicati.Library.Backend.GoogleServices
/// <param name="requestdata">The data to submit as JSON metadata.</param>
/// <param name="url">The URL to register the upload session against.</param>
/// <param name="stream">The stream with content data to upload.</param>
/// <param name="shortTimeout">The short request timeout.</param>
/// <param name="readWriteTimeout">The read write timeout.</param>
/// <param name="method">The HTTP Method.</param>
/// <typeparam name="TRequest">The type of data to upload as metadata.</typeparam>
/// <typeparam name="TResponse">The type of data returned from the upload.</typeparam>
public static async Task<TResponse> ChunkedUploadWithResumeAsync<TRequest, TResponse>(OAuthHelper oauth, TRequest requestdata, string url, System.IO.Stream stream, CancellationToken cancelToken, string method = "POST")
public static async Task<TResponse> ChunkedUploadWithResumeAsync<TRequest, TResponse>(OAuthHelper oauth, TRequest requestdata, string url, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken, string method = "POST")
where TRequest : class
where TResponse : class
{
@@ -105,25 +105,30 @@ namespace Duplicati.Library.Backend.GoogleServices
if (data != null)
req.ContentType = "application/json; charset=UTF-8";
req.Headers["X-Upload-Content-Type"] = "application/octet-stream";
req.Headers["X-Upload-Content-Length"] = stream.Length.ToString();
var areq = new AsyncHttpRequest(req);
if (data != null)
using(var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false);
string uploaduri;
using(var resp = (HttpWebResponse)areq.GetResponse())
var uploaduri = await Utility.Utility.WithTimeout(shortTimeout, cancelToken, async ct =>
{
if (resp.StatusCode != HttpStatusCode.OK || string.IsNullOrWhiteSpace(resp.Headers["Location"]))
throw new WebException("Failed to start upload session", null, WebExceptionStatus.UnknownError, resp);
var areq = new AsyncHttpRequest(req);
if (data != null)
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false);
uploaduri = resp.Headers["Location"];
}
using (var resp = (HttpWebResponse)areq.GetResponse())
{
if (resp.StatusCode != HttpStatusCode.OK || string.IsNullOrWhiteSpace(resp.Headers["Location"]))
throw new WebException("Failed to start upload session", null, WebExceptionStatus.UnknownError, resp);
return await ChunkedUploadAsync<TResponse>(oauth, uploaduri, stream, cancelToken).ConfigureAwait(false);
return resp.Headers["Location"];
}
}).ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(uploaduri))
throw new Exception("Failed to start upload session");
return await ChunkedUploadAsync<TResponse>(oauth, uploaduri, stream, shortTimeout, readWriteTimeout, cancelToken).ConfigureAwait(false);
}
/// <summary>
@@ -133,8 +138,10 @@ namespace Duplicati.Library.Backend.GoogleServices
/// <param name="oauth">The Oauth instance</param>
/// <param name="uploaduri">The resumeable uploaduri</param>
/// <param name="stream">The stream with data to upload.</param>
/// <param name="shortTimeout">The short request timeout.</param>
/// <param name="readWriteTimeout">The read write timeout.</param>
/// <typeparam name="T">The type of data in the response.</typeparam>
private static async Task<T> ChunkedUploadAsync<T>(OAuthHelper oauth, string uploaduri, System.IO.Stream stream, CancellationToken cancelToken)
private static async Task<T> ChunkedUploadAsync<T>(OAuthHelper oauth, string uploaduri, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken)
where T : class
{
var queryRange = false;
@@ -143,14 +150,13 @@ namespace Duplicati.Library.Backend.GoogleServices
var buffer = new byte[Library.Utility.Utility.DEFAULT_BUFFER_SIZE];
// Repeatedly try uploading until all retries are done
while(true)
while (true)
{
try
{
if (queryRange)
{
T re;
offset = GoogleCommon.QueryUploadRange(oauth, uploaduri, stream.Length, out re);
offset = QueryUploadRange<T>(oauth, uploaduri, stream.Length, out var re);
queryRange = false;
if (re != null)
@@ -172,25 +178,26 @@ namespace Duplicati.Library.Backend.GoogleServices
// Upload the remaining data
var areq = new AsyncHttpRequest(req);
using(var rs = areq.GetRequestStream())
using (var rs = areq.GetRequestStream())
using (var ts = rs.ObserveWriteTimeout(readWriteTimeout))
{
var remaining = chunkSize;
while(remaining > 0)
while (remaining > 0)
{
var n = stream.Read(buffer, 0, (int)Math.Min(remaining, Library.Utility.Utility.DEFAULT_BUFFER_SIZE));
await rs.WriteAsync(buffer, 0, n, cancelToken);
await ts.WriteAsync(buffer, 0, n, cancelToken);
remaining -= n;
}
}
// Check the response
using(var resp = oauth.GetResponseWithoutException(areq))
using (var resp = await Utility.Utility.WithTimeout(shortTimeout, cancelToken, _ => oauth.GetResponseWithoutException(areq)).ConfigureAwait(false))
{
var code = (int)resp.StatusCode;
if (code == 308 && resp.Headers["Range"] != null)
{
offset = long.Parse(resp.Headers["Range"].Split(new char[] {'-'})[1]) + 1;
offset = long.Parse(resp.Headers["Range"]!.Split(new char[] { '-' })[1]) + 1;
retries = 0;
}
else if (code >= 200 && code <= 299)
Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs
+111 -105
View File
@@ -23,16 +23,11 @@ using Duplicati.Library.Backend.GoogleServices;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.CompilerServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library.Backend.GoogleDrive
{
@@ -40,41 +35,51 @@ namespace Duplicati.Library.Backend.GoogleDrive
// This class is instantiated dynamically in the BackendLoader.
public class GoogleDrive : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend, ITimeoutExemptBackend
{
private const string AUTHID_OPTION = "authid";
private const string TEAMDRIVE_ID = "googledrive-teamdrive-id";
private const string FOLDER_MIMETYPE = "application/vnd.google-apps.folder";
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("googledrive");
private readonly string m_path;
private readonly string m_teamDriveID;
private readonly string? m_teamDriveID;
private readonly OAuthHelper m_oauth;
private readonly Dictionary<string, GoogleDriveFolderItem[]> m_filecache;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
private string m_currentFolderId;
private string? m_currentFolderId;
public GoogleDrive()
{
m_path = null!;
m_oauth = null!;
m_filecache = null!;
m_timeouts = null!;
}
public GoogleDrive(string url, Dictionary<string, string> options)
public GoogleDrive(string url, Dictionary<string, string?> options)
{
var uri = new Utility.Uri(url);
m_path = Util.AppendDirSeparator(uri.HostAndPath, "/");
string authid = null;
if (options.ContainsKey(AUTHID_OPTION))
authid = options[AUTHID_OPTION];
var auth = AuthIdOptionsHelper.Parse(options)
.RequireCredentials(TOKEN_URL);
if (options.ContainsKey(TEAMDRIVE_ID))
m_teamDriveID = options[TEAMDRIVE_ID];
m_oauth = new OAuthHelper(authid, this.ProtocolKey) { AutoAuthHeader = true };
m_timeouts = TimeoutOptionsHelper.Parse(options);
m_teamDriveID = options.GetValueOrDefault(TEAMDRIVE_ID);
m_oauth = new OAuthHelper(auth.AuthId!, this.ProtocolKey) { AutoAuthHeader = true };
m_filecache = new Dictionary<string, GoogleDriveFolderItem[]>();
}
private async Task<string> GetFolderIdAsync(string path, bool autocreate, CancellationToken cancelToken)
{
var curparent = m_teamDriveID ?? (await GetAboutInfoAsync(cancelToken).ConfigureAwait(false)).rootFolderId;
var curparent = string.IsNullOrWhiteSpace(m_teamDriveID)
? (await GetAboutInfoAsync(cancelToken).ConfigureAwait(false)).rootFolderId
: m_teamDriveID;
if (string.IsNullOrEmpty(curparent))
throw new UserInformationException("Unable to get root folder", "GoogleDriveNoRootFolder");
var curdisplay = new StringBuilder("/");
foreach (var p in path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries))
@@ -97,6 +102,10 @@ namespace Duplicati.Library.Backend.GoogleDrive
}
curdisplay.Append(p).Append("/");
if (string.IsNullOrWhiteSpace(curparent))
throw new UserInformationException($"Unable to get folder id for {curdisplay}", "GoogleDriveNoFolderId");
}
return curparent;
@@ -110,11 +119,9 @@ namespace Duplicati.Library.Backend.GoogleDrive
return m_currentFolderId;
}
private async Task<GoogleDriveFolderItem[]> GetFileEntriesAsync(string remotename, bool throwMissingException, CancellationToken cancelToken)
private async Task<GoogleDriveFolderItem[]?> GetFileEntriesAsync(string remotename, bool throwMissingException, CancellationToken cancelToken)
{
GoogleDriveFolderItem[] entries;
m_filecache.TryGetValue(remotename, out entries);
m_filecache.TryGetValue(remotename, out var entries);
if (entries != null)
return entries;
@@ -124,7 +131,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
if (entries == null || entries.Length == 0)
{
if (throwMissingException)
throw new FileMissingException();
throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename));
else
return null;
}
@@ -139,7 +146,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
#region IStreamingBackend implementation
public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
try
{
@@ -147,10 +154,9 @@ namespace Duplicati.Library.Backend.GoogleDrive
if (m_filecache.Count == 0)
await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ }
GoogleDriveFolderItem[] files;
m_filecache.TryGetValue(remotename, out files);
m_filecache.TryGetValue(remotename, out var files);
string fileId = null;
string? fileId = null;
if (files != null)
{
if (files.Length == 1)
@@ -174,7 +180,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
teamDriveId = m_teamDriveID
};
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<GoogleDriveFolderItem, GoogleDriveFolderItem>(m_oauth, item, url, stream, cancelToken, isUpdate ? "PUT" : "POST").ConfigureAwait(false);
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<GoogleDriveFolderItem, GoogleDriveFolderItem>(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken, isUpdate ? "PUT" : "POST").ConfigureAwait(false);
m_filecache[remotename] = [res];
}
catch
@@ -184,19 +190,22 @@ namespace Duplicati.Library.Backend.GoogleDrive
}
}
public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken)
public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken)
{
// Prevent repeated download url lookups
if (m_filecache.Count == 0)
await foreach (var file in ListAsync(cancelToken).ConfigureAwait(false)) { /* Enumerate the full listing */ }
var fileId = (await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false)).OrderByDescending(x => x.createdDate).First().id;
var fileId = (await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false))?.OrderByDescending(x => x.createdDate).FirstOrDefault()?.id;
if (string.IsNullOrEmpty(fileId))
throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename));
var req = m_oauth.CreateRequest(WebApi.GoogleDrive.GetUrl(fileId));
var areq = new AsyncHttpRequest(req);
using (var resp = (HttpWebResponse)areq.GetResponse())
using (var rs = areq.GetResponseStream())
await Duplicati.Library.Utility.Utility.CopyStreamAsync(rs, stream, cancelToken);
using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => (HttpWebResponse)areq.GetResponse()).ConfigureAwait(false))
using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false))
using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken);
}
#endregion
@@ -216,7 +225,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
var currentFolderId = await GetCurrentFolderIdAsync(cancelToken).ConfigureAwait(false);
await foreach (var n in ListFolder(currentFolderId, null, null, cancelToken).ConfigureAwait(false))
{
FileEntry fe = null;
FileEntry? fe = null;
if (n.fileSize == null)
fe = new FileEntry(n.title);
@@ -231,10 +240,9 @@ namespace Duplicati.Library.Backend.GoogleDrive
if (!fe.IsFolder)
{
GoogleDriveFolderItem[] lst;
if (!m_filecache.TryGetValue(fe.Name, out lst))
if (!m_filecache.TryGetValue(fe.Name, out var lst))
{
m_filecache[fe.Name] = new GoogleDriveFolderItem[] { n };
m_filecache[fe.Name] = [n];
}
else
{
@@ -262,13 +270,13 @@ namespace Duplicati.Library.Backend.GoogleDrive
public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (var fs = System.IO.File.OpenRead(filename))
using (var fs = File.OpenRead(filename))
await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken)
{
using (var fs = System.IO.File.Create(filename))
using (var fs = File.Create(filename))
await GetAsync(remotename, fs, cancelToken);
}
@@ -276,13 +284,19 @@ namespace Duplicati.Library.Backend.GoogleDrive
{
try
{
foreach (var fileid in from n in await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false) select n.id)
var entries = await GetFileEntriesAsync(remotename, true, cancelToken).ConfigureAwait(false);
if (entries == null)
throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename));
foreach (var fileid in entries.Select(x => x.id).WhereNotNullOrWhiteSpace())
{
var url = WebApi.GoogleDrive.DeleteUrl(Library.Utility.Uri.UrlPathEncode(fileid), m_teamDriveID);
await m_oauth.GetJSONDataAsync<object>(url, cancelToken, x =>
{
x.Method = "DELETE";
});
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<object>(url, cancelToken, x =>
{
x.Method = "DELETE";
})
).ConfigureAwait(false);
}
m_filecache.Remove(remotename);
@@ -304,32 +318,19 @@ namespace Duplicati.Library.Backend.GoogleDrive
m_currentFolderId = await GetFolderIdAsync(m_path, true, cancelToken).ConfigureAwait(false);
}
public string DisplayName
{
get
{
return Strings.GoogleDrive.DisplayName;
}
}
public string DisplayName => Strings.GoogleDrive.DisplayName;
public string ProtocolKey
{
get
{
return "googledrive";
}
}
public string ProtocolKey => "googledrive";
public System.Collections.Generic.IList<ICommandLineArgument> SupportedCommands => new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument(AUTHID_OPTION,
CommandLineArgument.ArgumentType.Password,
Strings.GoogleDrive.AuthidShort,
Strings.GoogleDrive.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("googledrive"))),
new CommandLineArgument(TEAMDRIVE_ID,
CommandLineArgument.ArgumentType.String,
Strings.GoogleDrive.TeamDriveIdShort,
Strings.GoogleDrive.TeamDriveIdLong),
});
public IList<ICommandLineArgument> SupportedCommands =>
[
.. AuthIdOptionsHelper.GetOptions(TOKEN_URL),
new CommandLineArgument(TEAMDRIVE_ID,
CommandLineArgument.ArgumentType.String,
Strings.GoogleDrive.TeamDriveIdShort,
Strings.GoogleDrive.TeamDriveIdLong),
.. TimeoutOptionsHelper.GetOptions(),
];
public string Description
{
@@ -342,7 +343,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
#endregion
#region IQuotaEnabledBackend implementation
public async Task<IQuotaInfo> GetQuotaInfoAsync(CancellationToken cancelToken)
public async Task<IQuotaInfo?> GetQuotaInfoAsync(CancellationToken cancelToken)
{
try
{
@@ -365,9 +366,10 @@ namespace Duplicati.Library.Backend.GoogleDrive
try
{
var files = await GetFileEntriesAsync(oldname, true, cancellationToken).ConfigureAwait(false);
if (files == null)
throw new FileMissingException(Strings.GoogleDrive.FilenotFound(oldname));
if (files.Length > 1)
throw new UserInformationException(string.Format(Strings.GoogleDrive.MultipleEntries(oldname, m_path)),
"GoogleDriveMultipleEntries");
throw new UserInformationException(Strings.GoogleDrive.MultipleEntries(oldname, m_path), "GoogleDriveMultipleEntries");
using var stream = new MemoryStream();
await GetAsync(oldname, stream, cancellationToken).ConfigureAwait(false);
@@ -396,13 +398,13 @@ namespace Duplicati.Library.Backend.GoogleDrive
private class GoogleDriveParentReference
{
public string id { get; set; }
public string? id { get; set; }
}
private class GoogleDriveListResponse
{
public string nextPageToken { get; set; }
public GoogleDriveFolderItem[] items { get; set; }
public string? nextPageToken { get; set; }
public GoogleDriveFolderItem[]? items { get; set; }
}
private class GoogleDriveFolderItemLabels
@@ -412,41 +414,44 @@ namespace Duplicati.Library.Backend.GoogleDrive
private class GoogleDriveFolderItem
{
public string id { get; set; }
public string title { get; set; }
public string description { get; set; }
public string mimeType { get; set; }
public GoogleDriveFolderItemLabels labels { get; set; }
public string? id { get; set; }
public string? title { get; set; }
public string? description { get; set; }
public string? mimeType { get; set; }
public GoogleDriveFolderItemLabels? labels { get; set; }
public DateTime? createdDate { get; set; }
public DateTime? modifiedDate { get; set; }
public long? fileSize { get; set; }
public string teamDriveId { get; set; }
public GoogleDriveParentReference[] parents { get; set; }
public string? teamDriveId { get; set; }
public GoogleDriveParentReference[]? parents { get; set; }
}
private class GoogleDriveAboutResponse
{
public long? quotaBytesTotal { get; set; }
public long? quotaBytesUsed { get; set; }
public string rootFolderId { get; set; }
public string? rootFolderId { get; set; }
}
private async IAsyncEnumerable<GoogleDriveFolderItem> ListFolder(string parentfolder, bool? onlyFolders, string name, [EnumeratorCancellation] CancellationToken cancelToken)
private async IAsyncEnumerable<GoogleDriveFolderItem> ListFolder(string parentfolder, bool? onlyFolders, string? name, [EnumeratorCancellation] CancellationToken cancelToken)
{
var fileQuery = new string[] {
var fileQuery = new string?[] {
string.IsNullOrEmpty(name) ? null : string.Format("title = '{0}'", EscapeTitleEntries(name)),
onlyFolders == null ? null : string.Format("mimeType {0}= '{1}'", onlyFolders.Value ? "" : "!", FOLDER_MIMETYPE),
string.Format("'{0}' in parents", EscapeTitleEntries(parentfolder)),
"trashed=false"
};
var encodedFileQuery = Library.Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null)));
var encodedFileQuery = Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null)));
var url = WebApi.GoogleDrive.ListUrl(encodedFileQuery, m_teamDriveID);
while (true)
{
var res = await m_oauth.GetJSONDataAsync<GoogleDriveListResponse>(url, cancelToken).ConfigureAwait(false);
foreach (var n in res.items)
var res = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<GoogleDriveListResponse>(url, ct)
).ConfigureAwait(false);
foreach (var n in res.items ?? [])
yield return n;
var token = res.nextPageToken;
@@ -458,9 +463,8 @@ namespace Duplicati.Library.Backend.GoogleDrive
}
private Task<GoogleDriveAboutResponse> GetAboutInfoAsync(CancellationToken cancelToken)
{
return m_oauth.GetJSONDataAsync<GoogleDriveAboutResponse>(WebApi.GoogleDrive.AboutInfoUrl(), cancelToken);
}
=> Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<GoogleDriveAboutResponse>(WebApi.GoogleDrive.AboutInfoUrl(), ct));
private Task<GoogleDriveFolderItem> CreateFolderAsync(string name, string parent, CancellationToken cancelToken)
{
@@ -470,25 +474,27 @@ namespace Duplicati.Library.Backend.GoogleDrive
description = name,
mimeType = FOLDER_MIMETYPE,
labels = new GoogleDriveFolderItemLabels { hidden = true },
parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = parent } }
parents = [new GoogleDriveParentReference { id = parent }]
};
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder));
var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder));
return m_oauth.GetJSONDataAsync<GoogleDriveFolderItem>(
WebApi.GoogleDrive.CreateFolderUrl(m_teamDriveID),
cancelToken,
x =>
{
x.Method = "POST";
x.ContentType = "application/json; charset=UTF-8";
x.ContentLength = data.Length;
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<GoogleDriveFolderItem>(
WebApi.GoogleDrive.CreateFolderUrl(m_teamDriveID),
ct,
x =>
{
x.Method = "POST";
x.ContentType = "application/json; charset=UTF-8";
x.ContentLength = data.Length;
}, async (req, ct) =>
{
using (var rs = req.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false);
});
}, async (req, ct) =>
{
using (var rs = req.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false);
})
);
}
}
}
Duplicati/Library/Backend/GoogleServices/Strings.cs
+32 -26
View File
@@ -1,35 +1,33 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
{
internal static class GoogleCloudStorage {
internal static class GoogleCloudStorage
{
public static string Description { get { return LC.L(@"This backend can read and write data to Google Cloud Storage. Allowed format is ""gcs://bucket/folder""."); } }
public static string DisplayName { get { return LC.L(@"Google Cloud Storage"); } }
public static string MissingAuthID(string url) { return LC.L(@"You need an AuthID. You can get it from: {0}", url); }
public static string ProjectIDMissingError(string projectoption) { return LC.L(@"You must supply a project ID with --{0} for creating a bucket.", projectoption); }
public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); }
public static string AuthidShort { get { return LC.L(@"The authorization code"); } }
public static string LocationDescriptionLong(string regions) { return LC.L(@"This option is only used when creating new buckets. Use this option to change what region the data is stored in. Charges vary with bucket location. Known bucket locations:
{0}", regions); }
public static string LocationDescriptionShort { get { return LC.L(@"Specify location option for creating a bucket"); } }
@@ -40,13 +38,21 @@ namespace Duplicati.Library.Backend.Strings
public static string ProjectDescriptionShort { get { return LC.L(@"Specify project for creating a bucket"); } }
}
internal static class GoogleDrive {
internal static class GoogleDrive
{
public static string Description { get { return LC.L(@"This backend can read and write data to Google Drive. Allowed format is ""googledrive://folder/subfolder""."); } }
public static string DisplayName { get { return LC.L(@"Google Drive"); } }
public static string MultipleEntries(string folder, string parent) { return LC.L(@"There is more than one item named ""{0}"" in the folder ""{1}"".", folder, parent); }
public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); }
public static string AuthidShort { get { return LC.L(@"The authorization code"); } }
public static string FilenotFound(string name) { return LC.L(@"File not found: {0}", name); }
public static string TeamDriveIdLong { get { return LC.L("This option sets the team drive to use. Leaving it empty uses the personal drive."); } }
public static string TeamDriveIdShort { get { return LC.L("Team drive ID"); } }
}
internal static class GCSConfig
{
public static string DisplayName => LC.L("Google Cloud Storage configuration module");
public static string Description => LC.L("Expose Google Cloud Storage configuration as a web module");
public static string ConfigTypeShort => LC.L("The config to get");
public static string ConfigTypeLong => LC.L("Provide different config values");
}
}
Duplicati/Library/Backend/GoogleServices/WebApi.cs
+83 -113
View File
@@ -1,25 +1,24 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Collections.Generic;
using System.Collections.Specialized;
using Duplicati.Library.Utility;
@@ -28,41 +27,41 @@ namespace Duplicati.Library.Backend.WebApi
public static class GoogleCloudStorage
{
// From: https://cloud.google.com/storage/docs/bucket-locations
public static readonly KeyValuePair<string, string>[] KNOWN_GCS_LOCATIONS = {
new KeyValuePair<string, string>("(default)", null),
new KeyValuePair<string, string>("Europe", "EU"),
new KeyValuePair<string, string>("United States", "US"),
new KeyValuePair<string, string>("Asia", "ASIA"),
public static readonly KeyValuePair<string, string?>[] KNOWN_GCS_LOCATIONS = {
new KeyValuePair<string, string?>("(default)", null),
new KeyValuePair<string, string?>("Europe", "EU"),
new KeyValuePair<string, string?>("United States", "US"),
new KeyValuePair<string, string?>("Asia", "ASIA"),
//Regional buckets: https://cloud.google.com/storage/docs/regional-buckets
new KeyValuePair<string, string>("Eastern Asia-Pacific", "ASIA-EAST1"),
new KeyValuePair<string, string>("Central United States 1", "US-CENTRAL1"),
new KeyValuePair<string, string>("Central United States 2", "US-CENTRAL2"),
new KeyValuePair<string, string>("Eastern United States 1", "US-EAST1"),
new KeyValuePair<string, string>("Eastern United States 2", "US-EAST2"),
new KeyValuePair<string, string>("Eastern United States 3", "US-EAST3"),
new KeyValuePair<string, string>("Western United States", "US-WEST1"),
new KeyValuePair<string, string?>("Eastern Asia-Pacific", "ASIA-EAST1"),
new KeyValuePair<string, string?>("Central United States 1", "US-CENTRAL1"),
new KeyValuePair<string, string?>("Central United States 2", "US-CENTRAL2"),
new KeyValuePair<string, string?>("Eastern United States 1", "US-EAST1"),
new KeyValuePair<string, string?>("Eastern United States 2", "US-EAST2"),
new KeyValuePair<string, string?>("Eastern United States 3", "US-EAST3"),
new KeyValuePair<string, string?>("Western United States", "US-WEST1"),
};
public static readonly KeyValuePair<string, string>[] KNOWN_GCS_STORAGE_CLASSES = {
new KeyValuePair<string, string>("(default)", null),
new KeyValuePair<string, string>("Standard", "STANDARD"),
new KeyValuePair<string, string>("Durable Reduced Availability (DRA)", "DURABLE_REDUCED_AVAILABILITY"),
new KeyValuePair<string, string>("Nearline", "NEARLINE"),
public static readonly KeyValuePair<string, string?>[] KNOWN_GCS_STORAGE_CLASSES = {
new KeyValuePair<string, string?>("(default)", null),
new KeyValuePair<string, string?>("Standard", "STANDARD"),
new KeyValuePair<string, string?>("Durable Reduced Availability (DRA)", "DURABLE_REDUCED_AVAILABILITY"),
new KeyValuePair<string, string?>("Nearline", "NEARLINE"),
};
public static string[] Hosts()
{
return new [] { new System.Uri(Url.UPLOAD).Host,
new System.Uri(Url.API).Host };
}
public static string[] Hosts() =>
[
new System.Uri(Url.UPLOAD).Host,
new System.Uri(Url.API).Host
];
public static string DeleteUrl(string bucketId, string objectId)
{
var path = BucketObjectPath(bucketId, objectId);
return Uri.UriBuilder(Url.API, path);
return Utility.Uri.UriBuilder(Url.API, path);
}
public static string CreateFolderUrl(string projectId)
@@ -72,20 +71,16 @@ namespace Duplicati.Library.Backend.WebApi
{ QueryParam.Project, projectId }
};
return Uri.UriBuilder(Url.API, Path.Bucket, queryParams);
return Utility.Uri.UriBuilder(Url.API, Path.Bucket, queryParams);
}
public static string RenameUrl(string bucketId, string objectId)
{
return Uri.UriBuilder(Url.API, BucketObjectPath(bucketId, objectId));
}
=> Utility.Uri.UriBuilder(Url.API, BucketObjectPath(bucketId, objectId));
public static string ListUrl(string bucketId, string prefix)
{
return ListUrl(bucketId, prefix, null);
}
=> ListUrl(bucketId, prefix, null);
public static string ListUrl(string bucketId, string prefix, string token)
public static string ListUrl(string bucketId, string prefix, string? token)
{
var queryParams = new NameValueCollection {
{ QueryParam.Prefix, prefix }
@@ -96,7 +91,7 @@ namespace Duplicati.Library.Backend.WebApi
queryParams.Set(QueryParam.PageToken, token);
}
return Uri.UriBuilder(Url.API, BucketObjectPath(bucketId), queryParams);
return Utility.Uri.UriBuilder(Url.API, BucketObjectPath(bucketId), queryParams);
}
public static string PutUrl(string bucketId)
@@ -106,7 +101,7 @@ namespace Duplicati.Library.Backend.WebApi
{ QueryParam.UploadType, QueryValue.Resumable }
};
var path = UrlPath.Create(Path.Bucket).Append(bucketId).Append(Path.Object).ToString();
return Uri.UriBuilder(Url.UPLOAD, path, queryParams);
return Utility.Uri.UriBuilder(Url.UPLOAD, path, queryParams);
}
public static string GetUrl(string bucketId, string objectId)
@@ -118,7 +113,7 @@ namespace Duplicati.Library.Backend.WebApi
};
var path = BucketObjectPath(bucketId, objectId);
return Uri.UriBuilder(Url.API, path, queryParams);
return Utility.Uri.UriBuilder(Url.API, path, queryParams);
}
private static class Url
@@ -148,36 +143,27 @@ namespace Duplicati.Library.Backend.WebApi
public const string Object = "o";
}
private static string BucketObjectPath(string bucketId, string objectId = null)
{
return UrlPath.Create(Path.Bucket)
.Append(bucketId)
.Append(Path.Object)
.Append(objectId).ToString();
}
private static string BucketObjectPath(string bucketId, string? objectId = null)
=> UrlPath.Create(Path.Bucket)
.Append(bucketId)
.Append(Path.Object)
.Append(objectId).ToString();
}
public static class GoogleDrive
{
public static string[] Hosts()
{
return new [] { new System.Uri(Url.DRIVE).Host, new System.Uri(Url.UPLOAD).Host };
}
=> [new System.Uri(Url.DRIVE).Host, new System.Uri(Url.UPLOAD).Host];
public static string GetUrl(string fileId)
{
return FileQueryUrl(fileId, new NameValueCollection{
=> FileQueryUrl(fileId, new NameValueCollection{
{ QueryParam.Alt, QueryValue.Media }
});
}
public static string DeleteUrl(string fileId, string teamDriveId)
{
return FileQueryUrl(Uri.UrlPathEncode(fileId), AddTeamDriveParam(teamDriveId));
}
public static string DeleteUrl(string fileId, string? teamDriveId)
=> FileQueryUrl(Utility.Uri.UrlPathEncode(fileId), AddTeamDriveParam(teamDriveId));
public static string PutUrl(string fileId, bool useTeamDrive)
public static string PutUrl(string? fileId, bool useTeamDrive)
{
var queryParams = new NameValueCollection {
{ QueryParam.UploadType,
@@ -189,16 +175,14 @@ namespace Duplicati.Library.Backend.WebApi
}
return !string.IsNullOrWhiteSpace(fileId) ?
FileUploadUrl(Uri.UrlPathEncode(fileId), queryParams) :
FileUploadUrl(Utility.Uri.UrlPathEncode(fileId), queryParams) :
FileUploadUrl(queryParams);
}
public static string ListUrl(string fileQuery, string teamDriveId)
{
return ListUrl(fileQuery, teamDriveId, null);
}
public static string ListUrl(string fileQuery, string teamDriveId, string token)
public static string ListUrl(string fileQuery, string? teamDriveId)
=> ListUrl(fileQuery, teamDriveId, null);
public static string ListUrl(string fileQuery, string? teamDriveId, string? token)
{
var queryParams = new NameValueCollection
{
@@ -207,7 +191,7 @@ namespace Duplicati.Library.Backend.WebApi
};
queryParams.Add(AddTeamDriveParam(teamDriveId));
if (token != null)
{
queryParams.Set(QueryParam.PageToken, token);
@@ -216,15 +200,11 @@ namespace Duplicati.Library.Backend.WebApi
return FileQueryUrl(queryParams);
}
public static string CreateFolderUrl(string teamDriveId)
{
return FileQueryUrl(AddTeamDriveParam(teamDriveId));
}
public static string CreateFolderUrl(string? teamDriveId)
=> FileQueryUrl(AddTeamDriveParam(teamDriveId));
public static string AboutInfoUrl()
{
return Uri.UriBuilder(Url.DRIVE, Path.About);
}
=> Utility.Uri.UriBuilder(Url.DRIVE, Path.About);
private static class Url
{
@@ -259,40 +239,30 @@ namespace Duplicati.Library.Backend.WebApi
}
private static string FileQueryUrl(NameValueCollection values)
{
return Uri.UriBuilder(Url.DRIVE, Path.File, values);
}
=> Utility.Uri.UriBuilder(Url.DRIVE, Path.File, values);
private static string FileQueryUrl(string fileId, NameValueCollection values = null)
{
return Uri.UriBuilder(Url.DRIVE, UrlPath.Create(Path.File).Append(fileId).ToString(),
values);
}
private static string FileQueryUrl(string fileId, NameValueCollection? values = null)
=> Utility.Uri.UriBuilder(Url.DRIVE, UrlPath.Create(Path.File).Append(fileId).ToString(), values);
private static string FileUploadUrl(string fileId, NameValueCollection values)
{
return Uri.UriBuilder(Url.UPLOAD, UrlPath.Create(Path.File).Append(fileId).ToString(),
values);
}
private static string FileUploadUrl(string fileId, NameValueCollection? values)
=> Utility.Uri.UriBuilder(Url.UPLOAD, UrlPath.Create(Path.File).Append(fileId).ToString(), values);
private static string FileUploadUrl(NameValueCollection values)
{
return Uri.UriBuilder(Url.UPLOAD, Path.File, values);
}
=> Utility.Uri.UriBuilder(Url.UPLOAD, Path.File, values);
private static NameValueCollection AddTeamDriveParam(string teamDriveId)
private static NameValueCollection AddTeamDriveParam(string? teamDriveId)
{
return teamDriveId != null ? new NameValueCollection {
{ WebApi.GoogleDrive.QueryParam.SupportsTeamDrive,
WebApi.GoogleDrive.QueryValue.True },
{ WebApi.GoogleDrive.QueryParam.TeamDriveId,
{ QueryParam.SupportsTeamDrive,
QueryValue.True },
{ QueryParam.TeamDriveId,
teamDriveId },
{ WebApi.GoogleDrive.QueryParam.IncludeTeamDrive,
WebApi.GoogleDrive.QueryValue.True },
{ WebApi.GoogleDrive.QueryParam.corpora,
WebApi.GoogleDrive.QueryValue.TeamDrive }
{ QueryParam.IncludeTeamDrive,
QueryValue.True },
{ QueryParam.corpora,
QueryValue.TeamDrive }
} : new NameValueCollection( );
} : new NameValueCollection();
}
}
}
Duplicati/Library/Interface/IWebModule.cs
+19 -20
View File
@@ -1,25 +1,24 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
namespace Duplicati.Library.Interface