haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6292 from marceloduplicati/feature/openstack-httpmigration

Browse Source 
Refactors OpenStack backend to use HttpClient
This commit is contained in:
Kenneth Skovhede
2025-05-24 13:46:13 +02:00
committed by GitHub
parent c6327c0f1f 06c5b14eda
commit 08f4ef897f
10 changed files with 837 additions and 654 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs
+12 -1
View File
@@ -60,6 +60,17 @@ public class JsonWebHelperHttpClient(HttpClient httpClient)
return Task.FromResult(request);
}
/// <summary>
/// Centralized method to prepare a request with the given URL and method setting useragent
/// </summary>
/// <param name="url">Url</param>
/// <param name="method">Method</param>
public virtual HttpRequestMessage CreateRequest(string url, string? method = null)
{
var request = new HttpRequestMessage(method != null ? HttpMethod.Parse(method) : HttpMethod.Get, url);
request.Headers.Add("User-Agent", UserAgent);
return request;
}
/// <summary>
/// Performs a multipart post and parses the response as JSON
/// </summary>
@@ -156,7 +167,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient)
/// <param name="req">Request object</param>
/// <param name="cancellationToken">Cancellation Token</param>
/// <typeparam name="T">Destination Type</typeparam>
protected virtual async Task<T> ReadJsonResponseAsync<T>(HttpRequestMessage req, CancellationToken cancellationToken)
public virtual async Task<T> ReadJsonResponseAsync<T>(HttpRequestMessage req, CancellationToken cancellationToken)
{
using var resp = await GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
return await ReadJsonResponseAsync<T>(resp, cancellationToken).ConfigureAwait(false);
Duplicati/Library/Backend/OpenStack/Keystone3AuthRequest.cs
+119
View File
@@ -0,0 +1,119 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
namespace Duplicati.Library.Backend.OpenStack;
internal class Keystone3AuthRequest
{
public class AuthContainer
{
public Identity? identity { get; set; }
public Scope? scope { get; set; }
}
public class Identity
{
[JsonProperty(ItemConverterType = typeof(StringEnumConverter))]
public IdentityMethods[] methods { get; set; }
[JsonProperty("password", NullValueHandling = NullValueHandling.Ignore)]
public PasswordBasedRequest? PasswordCredentials { get; set; }
public Identity()
{
methods = new[] { IdentityMethods.password };
}
}
public class Scope
{
public Project? project;
}
public enum IdentityMethods
{
password,
}
public class PasswordBasedRequest
{
public UserCredentials? user { get; set; }
}
public class UserCredentials
{
public Domain domain { get; set; }
public string name { get; set; }
public string password { get; set; }
public UserCredentials()
{
domain = null!;
name = null!;
password = null!;
}
public UserCredentials(Domain domain, string name, string password)
{
this.domain = domain;
this.name = name;
this.password = password;
}
}
public class Domain
{
public string? name { get; set; }
public Domain(string? name)
{
this.name = name;
}
}
public class Project
{
public Domain domain { get; set; }
public string name { get; set; }
public Project(Domain domain, string name)
{
this.domain = domain;
this.name = name;
}
}
public AuthContainer auth { get; set; }
public Keystone3AuthRequest(string? domain_name, string username, string? password, string? project_name)
{
Domain domain = new Domain(domain_name);
auth = new AuthContainer();
auth.identity = new Identity();
auth.identity.PasswordCredentials = new PasswordBasedRequest();
auth.identity.PasswordCredentials.user = new UserCredentials(domain, username, password!);
auth.scope = new Scope();
auth.scope.project = new Project(domain, project_name!);
}
}
Duplicati/Library/Backend/OpenStack/Keystone3AuthResponse.cs
+49
View File
@@ -0,0 +1,49 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.OpenStack;
internal class Keystone3AuthResponse
{
public TokenClass? token { get; set; }
public class EndpointItem
{
// 'interface' is a reserved keyword, so we need this decorator to map it
[JsonProperty(PropertyName = "interface")]
public string? interface_name { get; set; }
public string? region { get; set; }
public string? url { get; set; }
}
public class CatalogItem
{
public EndpointItem[]? endpoints { get; set; }
public string? name { get; set; }
public string? type { get; set; }
}
public class TokenClass
{
public CatalogItem[]? catalog { get; set; }
public DateTime? expires_at { get; set; }
}
}
Duplicati/Library/Backend/OpenStack/OpenStackAuthRequest.cs
+88
View File
@@ -0,0 +1,88 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.OpenStack;
internal class OpenStackAuthRequest
{
public class AuthContainer
{
[JsonProperty("RAX-KSKEY:apiKeyCredentials", NullValueHandling = NullValueHandling.Ignore)]
public ApiKeyBasedRequest? ApiCredentials { get; set; }
[JsonProperty("passwordCredentials", NullValueHandling = NullValueHandling.Ignore)]
public PasswordBasedRequest? PasswordCredentials { get; set; }
[JsonProperty("tenantName", NullValueHandling = NullValueHandling.Ignore)]
public string? TenantName { get; set; }
[JsonProperty("token", NullValueHandling = NullValueHandling.Ignore)]
public TokenBasedRequest? Token { get; set; }
}
public class ApiKeyBasedRequest
{
public string? username { get; set; }
public string? apiKey { get; set; }
}
public class PasswordBasedRequest
{
public string? username { get; set; }
public string? password { get; set; }
public string? tenantName { get; set; }
}
public class TokenBasedRequest
{
public string? id { get; set; }
}
public AuthContainer auth { get; set; }
public OpenStackAuthRequest(string? tenantname, string username, string? password, string? apikey)
{
auth = new AuthContainer
{
TenantName = tenantname
};
if (string.IsNullOrEmpty(apikey))
{
auth.PasswordCredentials = new PasswordBasedRequest
{
username = username,
password = password,
};
}
else
{
auth.ApiCredentials = new ApiKeyBasedRequest
{
username = username,
apiKey = apikey
};
}
}
}
Duplicati/Library/Backend/OpenStack/OpenStackAuthResponse.cs
+54
View File
@@ -0,0 +1,54 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
namespace Duplicati.Library.Backend.OpenStack;
internal class OpenStackAuthResponse
{
public AccessClass? access { get; set; }
public class TokenClass
{
public string? id { get; set; }
public DateTime? expires { get; set; }
}
public class EndpointItem
{
public string? region { get; set; }
public string? tenantId { get; set; }
public string? publicURL { get; set; }
public string? internalURL { get; set; }
}
public class ServiceItem
{
public string? name { get; set; }
public string? type { get; set; }
public EndpointItem[]? endpoints { get; set; }
}
public class AccessClass
{
public TokenClass? token { get; set; }
public ServiceItem[]? serviceCatalog { get; set; }
}
}
Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs
+418 -618
View File
File diff suppressed because it is too large Load Diff
Duplicati/Library/Backend/OpenStack/OpenStackStorageItem.cs
+31
View File
@@ -0,0 +1,31 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
namespace Duplicati.Library.Backend.OpenStack;
internal class OpenStackStorageItem
{
public string? name { get; set; }
public DateTime? last_modified { get; set; }
public long? bytes { get; set; }
public string? content_type { get; set; }
public string? subdir { get; set; }
}
Duplicati/Library/Backend/OpenStack/OpenStackWebHelper.cs
+34
View File
@@ -0,0 +1,34 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Utility;
namespace Duplicati.Library.Backend.OpenStack;
internal class OpenStackWebHelper(OpenStackStorage parent, HttpClient httpClient) : JsonWebHelperHttpClient(httpClient)
{
public override HttpRequestMessage CreateRequest(string url, string? method = null)
{
var request = base.CreateRequest(url, method);
request.Headers.Add("X-Auth-Token", parent.GetAccessToken(CancellationToken.None).Await());
return request;
}
}
Duplicati/Library/Backend/OpenStack/Strings.cs
+29 -32
View File
@@ -21,37 +21,34 @@
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings
namespace Duplicati.Library.Backend.Strings;
internal static class OpenStack
{
internal static class OpenStack
{
public static string Description { get { return LC.L(@"This backend can read and write data to Swift (OpenStack Object Storage). Allowed format is ""openstack://container/folder""."); } }
public static string DisplayName { get { return LC.L(@"OpenStack Simple Storage"); } }
public static string MissingOptionError(string optionname) { return LC.L(@"Missing required option: {0}", optionname); }
public static string PasswordOptionLong(string tenantnameoption) { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD"". If the password is supplied, --{0} must also be set.", tenantnameoption); }
public static string PasswordOptionShort { get { return LC.L(@"Supply the password used to connect to the server"); } }
public static string DomainnameOptionLong { get { return LC.L(@"The domain name of the user used to connect to the server."); } }
public static string DomainnameOptionShort { get { return LC.L(@"Supply the domain used to connect to the server"); } }
public static string UsernameOptionLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string UsernameOptionShort { get { return LC.L(@"Supply the username used to connect to the server"); } }
public static string TenantnameOptionLong { get { return LC.L(@"The Tenant Name is commonly the paying user account name. This option must be supplied when authenticating with a password, but is not required when using an API key."); } }
public static string TenantnameOptionShort { get { return LC.L(@"Supply the Tenant Name used to connect to the server"); } }
public static string ApikeyOptionLong { get { return LC.L(@"The API key can be used to connect without supplying a password and tenant ID with some providers."); } }
public static string ApikeyOptionShort { get { return LC.L(@"Supply the API key used to connect to the server"); } }
public static string AuthuriOptionLong(string providers) { return LC.L(@"The authentication URL is used to authenticate the user and find the storage service. The URL commonly ends with ""/v2.0"" for v2 and ""/v3"" for v3. Known providers are: {0}{1}", System.Environment.NewLine, providers); }
public static string AuthuriOptionShort { get { return LC.L(@"Supply the authentication URL"); } }
public static string VersionOptionLong { get { return LC.L(@"The keystone API version to use. Valid values are 'v2' and 'v3'."); } }
public static string VersionOptionShort { get { return LC.L(@"The keystone API version to use"); } }
public static string RegionOptionLong { get { return LC.L(@"By default, the first reported endpoint will be used for file transfers. To select a specific region, provide the region name. If no such region is supported, the default (first reported) endpoint is used."); } }
public static string RegionOptionShort { get { return LC.L(@"Supply the prefered region for endpoints"); } }
}
internal static class SwiftConfig
{
public static string Description { get { return LC.L(@"Expose OpenStack configuration as a web module"); } }
public static string DisplayName { get { return LC.L(@"OpenStack configuration module"); } }
public static string ConfigTypeOptionLong { get { return LC.L(@"Provide different config values"); } }
public static string ConfigTypeOptionShort { get { return LC.L(@"The config to get"); } }
}
public static string Description => LC.L(@"This backend can read and write data to Swift (OpenStack Object Storage). Allowed format is ""openstack://container/folder"".");
public static string DisplayName => LC.L(@"OpenStack Simple Storage");
public static string MissingOptionError(string optionname) { return LC.L(@"Missing required option: {0}", optionname); }
public static string PasswordOptionLong(string tenantnameoption) { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD"". If the password is supplied, --{0} must also be set.", tenantnameoption); }
public static string PasswordOptionShort => LC.L(@"Supply the password used to connect to the server");
public static string DomainnameOptionLong => LC.L(@"The domain name of the user used to connect to the server.");
public static string DomainnameOptionShort => LC.L(@"Supply the domain used to connect to the server");
public static string UsernameOptionLong => LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME"".");
public static string UsernameOptionShort => LC.L(@"Supply the username used to connect to the server");
public static string TenantnameOptionLong => LC.L(@"The Tenant Name is commonly the paying user account name. This option must be supplied when authenticating with a password, but is not required when using an API key.");
public static string TenantnameOptionShort => LC.L(@"Supply the Tenant Name used to connect to the server");
public static string ApikeyOptionLong => LC.L(@"The API key can be used to connect without supplying a password and tenant ID with some providers.");
public static string ApikeyOptionShort => LC.L(@"Supply the API key used to connect to the server");
public static string AuthuriOptionLong(string providers) { return LC.L(@"The authentication URL is used to authenticate the user and find the storage service. The URL commonly ends with ""/v2.0"" for v2 and ""/v3"" for v3. Known providers are: {0}{1}", Environment.NewLine, providers); }
public static string AuthuriOptionShort => LC.L(@"Supply the authentication URL");
public static string VersionOptionLong => LC.L(@"The keystone API version to use. Valid values are 'v2' and 'v3'.");
public static string VersionOptionShort => LC.L(@"The keystone API version to use");
public static string RegionOptionLong => LC.L(@"By default, the first reported endpoint will be used for file transfers. To select a specific region, provide the region name. If no such region is supported, the default (first reported) endpoint is used.");
public static string RegionOptionShort => LC.L(@"Supply the prefered region for endpoints");
}
internal static class SwiftConfig
{
public static string Description { get { return LC.L(@"Expose OpenStack configuration as a web module"); } }
public static string DisplayName { get { return LC.L(@"OpenStack configuration module"); } }
public static string ConfigTypeOptionLong { get { return LC.L(@"Provide different config values"); } }
public static string ConfigTypeOptionShort { get { return LC.L(@"The config to get"); } }
}
Duplicati/Library/Backend/OpenStack/OpenStackConfig.cs → Duplicati/Library/Backend/OpenStack/SwiftConfig.cs
+3 -3
View File
@@ -29,7 +29,7 @@ namespace Duplicati.Library.Backend.OpenStack
private static readonly string DEFAULT_CONFIG_TYPE_STR = DEFAULT_CONFIG_TYPE.ToString();
private const string KEY_CONFIGTYPE = "openstack-config";
public enum ConfigType
private enum ConfigType
{
Providers,
Versions,
@@ -50,9 +50,9 @@ namespace Duplicati.Library.Backend.OpenStack
switch (ct)
{
case ConfigType.Versions:
return OpenStackStorage.OPENSTACK_VERSIONS.ToDictionary((x) => x.Key, (y) => y.Value);
return OpenStackStorage.OpenstackVersions.ToDictionary((x) => x.Key, (y) => y.Value);
default:
return OpenStackStorage.KNOWN_OPENSTACK_PROVIDERS.ToDictionary((x) => x.Key, (y) => y.Value);
return OpenStackStorage.KnownOpenstackProviders.ToDictionary((x) => x.Key, (y) => y.Value);
}
}