diff --git a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs index 325bcbcb3..3e6e225e8 100644 --- a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs +++ b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelperHttpClient.cs @@ -60,6 +60,17 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) return Task.FromResult(request); } + /// + /// Centralized method to prepare a request with the given URL and method setting useragent + /// + /// Url + /// Method + public virtual HttpRequestMessage CreateRequest(string url, string? method = null) + { + var request = new HttpRequestMessage(method != null ? HttpMethod.Parse(method) : HttpMethod.Get, url); + request.Headers.Add("User-Agent", UserAgent); + return request; + } /// /// Performs a multipart post and parses the response as JSON /// @@ -156,7 +167,7 @@ public class JsonWebHelperHttpClient(HttpClient httpClient) /// Request object /// Cancellation Token /// Destination Type - protected virtual async Task ReadJsonResponseAsync(HttpRequestMessage req, CancellationToken cancellationToken) + public virtual async Task ReadJsonResponseAsync(HttpRequestMessage req, CancellationToken cancellationToken) { using var resp = await GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); return await ReadJsonResponseAsync(resp, cancellationToken).ConfigureAwait(false); diff --git a/Duplicati/Library/Backend/OpenStack/Keystone3AuthRequest.cs b/Duplicati/Library/Backend/OpenStack/Keystone3AuthRequest.cs new file mode 100644 index 000000000..4c55628e3 --- /dev/null +++ b/Duplicati/Library/Backend/OpenStack/Keystone3AuthRequest.cs @@ -0,0 +1,119 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace Duplicati.Library.Backend.OpenStack; +internal class Keystone3AuthRequest +{ + public class AuthContainer + { + public Identity? identity { get; set; } + public Scope? scope { get; set; } + } + + public class Identity + { + [JsonProperty(ItemConverterType = typeof(StringEnumConverter))] + public IdentityMethods[] methods { get; set; } + + [JsonProperty("password", NullValueHandling = NullValueHandling.Ignore)] + public PasswordBasedRequest? PasswordCredentials { get; set; } + + public Identity() + { + methods = new[] { IdentityMethods.password }; + } + } + + public class Scope + { + public Project? project; + } + + public enum IdentityMethods + { + password, + } + + public class PasswordBasedRequest + { + public UserCredentials? user { get; set; } + } + + public class UserCredentials + { + public Domain domain { get; set; } + public string name { get; set; } + public string password { get; set; } + + public UserCredentials() + { + domain = null!; + name = null!; + password = null!; + } + public UserCredentials(Domain domain, string name, string password) + { + this.domain = domain; + this.name = name; + this.password = password; + } + + } + + public class Domain + { + public string? name { get; set; } + + public Domain(string? name) + { + this.name = name; + } + } + + public class Project + { + public Domain domain { get; set; } + public string name { get; set; } + + public Project(Domain domain, string name) + { + this.domain = domain; + this.name = name; + } + } + + public AuthContainer auth { get; set; } + + public Keystone3AuthRequest(string? domain_name, string username, string? password, string? project_name) + { + Domain domain = new Domain(domain_name); + + auth = new AuthContainer(); + auth.identity = new Identity(); + auth.identity.PasswordCredentials = new PasswordBasedRequest(); + auth.identity.PasswordCredentials.user = new UserCredentials(domain, username, password!); + auth.scope = new Scope(); + auth.scope.project = new Project(domain, project_name!); + } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OpenStack/Keystone3AuthResponse.cs b/Duplicati/Library/Backend/OpenStack/Keystone3AuthResponse.cs new file mode 100644 index 000000000..6b0bbac59 --- /dev/null +++ b/Duplicati/Library/Backend/OpenStack/Keystone3AuthResponse.cs @@ -0,0 +1,49 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.OpenStack; +internal class Keystone3AuthResponse +{ + public TokenClass? token { get; set; } + + public class EndpointItem + { + // 'interface' is a reserved keyword, so we need this decorator to map it + [JsonProperty(PropertyName = "interface")] + public string? interface_name { get; set; } + public string? region { get; set; } + public string? url { get; set; } + } + + public class CatalogItem + { + public EndpointItem[]? endpoints { get; set; } + public string? name { get; set; } + public string? type { get; set; } + } + public class TokenClass + { + public CatalogItem[]? catalog { get; set; } + public DateTime? expires_at { get; set; } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackAuthRequest.cs b/Duplicati/Library/Backend/OpenStack/OpenStackAuthRequest.cs new file mode 100644 index 000000000..f1a32a036 --- /dev/null +++ b/Duplicati/Library/Backend/OpenStack/OpenStackAuthRequest.cs @@ -0,0 +1,88 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Newtonsoft.Json; + +namespace Duplicati.Library.Backend.OpenStack; +internal class OpenStackAuthRequest +{ + public class AuthContainer + { + [JsonProperty("RAX-KSKEY:apiKeyCredentials", NullValueHandling = NullValueHandling.Ignore)] + public ApiKeyBasedRequest? ApiCredentials { get; set; } + + [JsonProperty("passwordCredentials", NullValueHandling = NullValueHandling.Ignore)] + public PasswordBasedRequest? PasswordCredentials { get; set; } + + [JsonProperty("tenantName", NullValueHandling = NullValueHandling.Ignore)] + public string? TenantName { get; set; } + + [JsonProperty("token", NullValueHandling = NullValueHandling.Ignore)] + public TokenBasedRequest? Token { get; set; } + + } + + public class ApiKeyBasedRequest + { + public string? username { get; set; } + public string? apiKey { get; set; } + } + + public class PasswordBasedRequest + { + public string? username { get; set; } + public string? password { get; set; } + public string? tenantName { get; set; } + } + + public class TokenBasedRequest + { + public string? id { get; set; } + } + + public AuthContainer auth { get; set; } + + public OpenStackAuthRequest(string? tenantname, string username, string? password, string? apikey) + { + auth = new AuthContainer + { + TenantName = tenantname + }; + + if (string.IsNullOrEmpty(apikey)) + { + auth.PasswordCredentials = new PasswordBasedRequest + { + username = username, + password = password, + }; + } + else + { + auth.ApiCredentials = new ApiKeyBasedRequest + { + username = username, + apiKey = apikey + }; + } + + } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackAuthResponse.cs b/Duplicati/Library/Backend/OpenStack/OpenStackAuthResponse.cs new file mode 100644 index 000000000..7f193f590 --- /dev/null +++ b/Duplicati/Library/Backend/OpenStack/OpenStackAuthResponse.cs @@ -0,0 +1,54 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Library.Backend.OpenStack; +internal class OpenStackAuthResponse +{ + public AccessClass? access { get; set; } + + public class TokenClass + { + public string? id { get; set; } + public DateTime? expires { get; set; } + } + + public class EndpointItem + { + public string? region { get; set; } + public string? tenantId { get; set; } + public string? publicURL { get; set; } + public string? internalURL { get; set; } + } + + public class ServiceItem + { + public string? name { get; set; } + public string? type { get; set; } + public EndpointItem[]? endpoints { get; set; } + } + + public class AccessClass + { + public TokenClass? token { get; set; } + public ServiceItem[]? serviceCatalog { get; set; } + } + +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs index e77c8294b..70f79b58e 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs @@ -19,662 +19,454 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using Duplicati.Library.Interface; -using Duplicati.Library.Utility; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; using System.Net; +using System.Net.Http.Headers; +using System.Reflection; +using System.Runtime.CompilerServices; using System.Text; using Duplicati.Library.Common.IO; -using System.Runtime.CompilerServices; +using Duplicati.Library.Interface; +using Duplicati.Library.Utility; using Duplicati.Library.Utility.Options; -using System.Threading.Tasks; +using Newtonsoft.Json; +using Exception = System.Exception; +using Uri = Duplicati.Library.Utility.Uri; -namespace Duplicati.Library.Backend.OpenStack +namespace Duplicati.Library.Backend.OpenStack; + +public class OpenStackStorage : IStreamingBackend { - public class OpenStackStorage : IBackend, IStreamingBackend + private const string DOMAINNAME_OPTION = "openstack-domain-name"; + private const string TENANTNAME_OPTION = "openstack-tenant-name"; + private const string AUTHURI_OPTION = "openstack-authuri"; + private const string VERSION_OPTION = "openstack-version"; + private const string APIKEY_OPTION = "openstack-apikey"; + private const string REGION_OPTION = "openstack-region"; + + private const int PAGE_LIMIT = 500; + + private readonly string m_container; + private readonly string m_prefix; + + private readonly string? m_domainName; + private readonly string m_username; + private readonly string? m_password; + private readonly string m_authUri; + private readonly string? m_version; + private readonly string? m_tenantName; + private readonly string? m_apikey; + private readonly string? m_region; + private readonly TimeoutOptionsHelper.Timeouts _timeouts; + + /// + /// Lazy cached HttpClient + /// + private readonly Lazy _httpClient = new(HttpClientHelper.CreateClient); + + private string m_simplestorageendpoint; + + private readonly OpenStackWebHelper m_helper; + private OpenStackAuthResponse.TokenClass? m_accessToken; + + public static readonly KeyValuePair[] KnownOpenstackProviders = + [ + new("Rackspace US", "https://identity.api.rackspacecloud.com/v2.0"), + new("Rackspace UK", "https://lon.identity.api.rackspacecloud.com/v2.0"), + new("OVH Cloud Storage", "https://auth.cloud.ovh.net/v3"), + new("Selectel Cloud Storage", "https://auth.selcdn.ru"), + new("Memset Cloud Storage", "https://auth.storage.memset.com"), + new("Infomaniak Swiss Backup cluster 1", "https://swiss-backup.infomaniak.com/identity/v3"), + new("Infomaniak Swiss Backup cluster 2", "https://swiss-backup02.infomaniak.com/identity/v3"), + new("Infomaniak Swiss Backup cluster 3", "https://swiss-backup03.infomaniak.com/identity/v3"), + new("Infomaniak Swiss Backup cluster 4", "https://swiss-backup04.infomaniak.com/identity/v3"), + new("Infomaniak Public Cloud 1", "https://api.pub1.infomaniak.cloud/identity/v3"), + new("Infomaniak Public Cloud 2", "https://api.pub2.infomaniak.cloud/identity/v3"), + new("Catalyst Cloud - nz-hlz-1 (NZ)", "https://api.nz-hlz-1.catalystcloud.io:5000/v3"), + new("Catalyst Cloud - nz-por-1 (NZ)", "https://api.nz-por-1.catalystcloud.io:5000/v3") + ]; + + public static readonly KeyValuePair[] OpenstackVersions = + [ + new("v2.0", "v2"), + new("v3", "v3") + ]; + + public OpenStackStorage() { - private const string DOMAINNAME_OPTION = "openstack-domain-name"; - private const string TENANTNAME_OPTION = "openstack-tenant-name"; - private const string AUTHURI_OPTION = "openstack-authuri"; - private const string VERSION_OPTION = "openstack-version"; - private const string APIKEY_OPTION = "openstack-apikey"; - private const string REGION_OPTION = "openstack-region"; + m_container = null!; + m_prefix = null!; + _timeouts = null!; + m_username = null!; + m_authUri = null!; + m_helper = null!; + m_simplestorageendpoint = null!; + } - private const int PAGE_LIMIT = 500; + public OpenStackStorage(string url, Dictionary options) + { + var uri = new Uri(url); + m_container = uri.Host; + m_prefix = Util.AppendDirSeparator("/" + uri.Path, "/"); + _timeouts = TimeoutOptionsHelper.Parse(options); - private readonly string m_container; - private readonly string m_prefix; + // For OpenStack we do not use a leading slash + if (m_prefix.StartsWith("/", StringComparison.Ordinal)) + m_prefix = m_prefix.Substring(1); - private readonly string? m_domainName; - private readonly string m_username; - private readonly string? m_password; - private readonly string m_authUri; - private readonly string? m_version; - private readonly string? m_tenantName; - private readonly string? m_apikey; - private readonly string? m_region; - private readonly TimeoutOptionsHelper.Timeouts m_timeouts; + var auth = AuthOptionsHelper.Parse(options, uri); - protected string? m_simplestorageendpoint; + options.TryGetValue(DOMAINNAME_OPTION, out m_domainName); + options.TryGetValue(TENANTNAME_OPTION, out m_tenantName); + options.TryGetValue(VERSION_OPTION, out m_version); + options.TryGetValue(APIKEY_OPTION, out m_apikey); + options.TryGetValue(REGION_OPTION, out m_region); - private readonly WebHelper m_helper; - private OpenStackAuthResponse.TokenClass? m_accessToken; + if (!auth.HasUsername) + throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthUsernameOption), "OpenStackMissingUsername"); - public static readonly KeyValuePair[] KNOWN_OPENSTACK_PROVIDERS = { - new KeyValuePair("Rackspace US", "https://identity.api.rackspacecloud.com/v2.0"), - new KeyValuePair("Rackspace UK", "https://lon.identity.api.rackspacecloud.com/v2.0"), - new KeyValuePair("OVH Cloud Storage", "https://auth.cloud.ovh.net/v3"), - new KeyValuePair("Selectel Cloud Storage", "https://auth.selcdn.ru"), - new KeyValuePair("Memset Cloud Storage", "https://auth.storage.memset.com"), - new KeyValuePair("Infomaniak Swiss Backup cluster 1", "https://swiss-backup.infomaniak.com/identity/v3"), - new KeyValuePair("Infomaniak Swiss Backup cluster 2", "https://swiss-backup02.infomaniak.com/identity/v3"), - new KeyValuePair("Infomaniak Swiss Backup cluster 3", "https://swiss-backup03.infomaniak.com/identity/v3"), - new KeyValuePair("Infomaniak Swiss Backup cluster 4", "https://swiss-backup04.infomaniak.com/identity/v3"), - new KeyValuePair("Infomaniak Public Cloud 1", "https://api.pub1.infomaniak.cloud/identity/v3"), - new KeyValuePair("Infomaniak Public Cloud 2", "https://api.pub2.infomaniak.cloud/identity/v3"), - new KeyValuePair("Catalyst Cloud - nz-hlz-1 (NZ)", "https://api.nz-hlz-1.catalystcloud.io:5000/v3"), - new KeyValuePair("Catalyst Cloud - nz-por-1 (NZ)", "https://api.nz-por-1.catalystcloud.io:5000/v3"), - }; + m_username = auth.Username!; + m_password = auth.Password; + var authUri = options.GetValueOrDefault(AUTHURI_OPTION); + if (string.IsNullOrWhiteSpace(authUri)) + throw new UserInformationException(Strings.OpenStack.MissingOptionError(AUTHURI_OPTION), "OpenStackMissingAuthUri"); - public static readonly KeyValuePair[] OPENSTACK_VERSIONS = { - new KeyValuePair("v2.0", "v2"), - new KeyValuePair("v3", "v3"), - }; + m_authUri = authUri; + if (string.IsNullOrWhiteSpace(m_version) && m_authUri.EndsWith("/v3", StringComparison.OrdinalIgnoreCase)) + m_version = "v3"; - - private class Keystone3AuthRequest + switch (m_version) { - public class AuthContainer - { - public Identity? identity { get; set; } - public Scope? scope { get; set; } - } - - public class Identity - { - [JsonProperty(ItemConverterType = typeof(StringEnumConverter))] - public IdentityMethods[] methods { get; set; } - - [JsonProperty("password", NullValueHandling = NullValueHandling.Ignore)] - public PasswordBasedRequest? PasswordCredentials { get; set; } - - public Identity() + case "v3": + if (string.IsNullOrWhiteSpace(m_password)) + throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthPasswordOption), "OpenStackMissingPassword"); + if (string.IsNullOrWhiteSpace(m_domainName)) + throw new UserInformationException(Strings.OpenStack.MissingOptionError(DOMAINNAME_OPTION), "OpenStackMissingDomainName"); + if (string.IsNullOrWhiteSpace(m_tenantName)) + throw new UserInformationException(Strings.OpenStack.MissingOptionError(TENANTNAME_OPTION), "OpenStackMissingTenantName"); + break; + default: + if (string.IsNullOrWhiteSpace(m_apikey)) { - this.methods = new[] { IdentityMethods.password }; - } - } - - public class Scope - { - public Project? project; - } - - public enum IdentityMethods - { - password, - } - - public class PasswordBasedRequest - { - public UserCredentials? user { get; set; } - } - - public class UserCredentials - { - public Domain domain { get; set; } - public string name { get; set; } - public string password { get; set; } - - public UserCredentials() - { - domain = null!; - name = null!; - password = null!; - } - public UserCredentials(Domain domain, string name, string password) - { - this.domain = domain; - this.name = name; - this.password = password; - } - - } - - public class Domain - { - public string? name { get; set; } - - public Domain(string? name) - { - this.name = name; - } - } - - public class Project - { - public Domain domain { get; set; } - public string name { get; set; } - - public Project(Domain domain, string name) - { - this.domain = domain; - this.name = name; - } - } - - public AuthContainer auth { get; set; } - - public Keystone3AuthRequest(string? domain_name, string username, string? password, string? project_name) - { - Domain domain = new Domain(domain_name); - - this.auth = new AuthContainer(); - this.auth.identity = new Identity(); - this.auth.identity.PasswordCredentials = new PasswordBasedRequest(); - this.auth.identity.PasswordCredentials.user = new UserCredentials(domain, username, password!); - this.auth.scope = new Scope(); - this.auth.scope.project = new Project(domain, project_name!); - } - } - - private class OpenStackAuthRequest - { - public class AuthContainer - { - [JsonProperty("RAX-KSKEY:apiKeyCredentials", NullValueHandling = NullValueHandling.Ignore)] - public ApiKeyBasedRequest? ApiCredentials { get; set; } - - [JsonProperty("passwordCredentials", NullValueHandling = NullValueHandling.Ignore)] - public PasswordBasedRequest? PasswordCredentials { get; set; } - - [JsonProperty("tenantName", NullValueHandling = NullValueHandling.Ignore)] - public string? TenantName { get; set; } - - [JsonProperty("token", NullValueHandling = NullValueHandling.Ignore)] - public TokenBasedRequest? Token { get; set; } - - } - - public class ApiKeyBasedRequest - { - public string? username { get; set; } - public string? apiKey { get; set; } - } - - public class PasswordBasedRequest - { - public string? username { get; set; } - public string? password { get; set; } - public string? tenantName { get; set; } - } - - public class TokenBasedRequest - { - public string? id { get; set; } - } - - - public AuthContainer auth { get; set; } - - public OpenStackAuthRequest(string? tenantname, string username, string? password, string? apikey) - { - this.auth = new AuthContainer(); - this.auth.TenantName = tenantname; - - if (string.IsNullOrEmpty(apikey)) - { - this.auth.PasswordCredentials = new PasswordBasedRequest - { - username = username, - password = password, - }; - } - else - { - this.auth.ApiCredentials = new ApiKeyBasedRequest - { - username = username, - apiKey = apikey - }; - } - - } - } - - private class Keystone3AuthResponse - { - public TokenClass? token { get; set; } - - public class EndpointItem - { - // 'interface' is a reserved keyword, so we need this decorator to map it - [JsonProperty(PropertyName = "interface")] - public string? interface_name { get; set; } - public string? region { get; set; } - public string? url { get; set; } - } - - public class CatalogItem - { - public EndpointItem[]? endpoints { get; set; } - public string? name { get; set; } - public string? type { get; set; } - } - public class TokenClass - { - public CatalogItem[]? catalog { get; set; } - public DateTime? expires_at { get; set; } - } - } - - private class OpenStackAuthResponse - { - public AccessClass? access { get; set; } - - public class TokenClass - { - public string? id { get; set; } - public DateTime? expires { get; set; } - } - - public class EndpointItem - { - public string? region { get; set; } - public string? tenantId { get; set; } - public string? publicURL { get; set; } - public string? internalURL { get; set; } - } - - public class ServiceItem - { - public string? name { get; set; } - public string? type { get; set; } - public EndpointItem[]? endpoints { get; set; } - } - - public class AccessClass - { - public TokenClass? token { get; set; } - public ServiceItem[]? serviceCatalog { get; set; } - } - - } - - private class OpenStackStorageItem - { - public string? name { get; set; } - public DateTime? last_modified { get; set; } - public long? bytes { get; set; } - public string? content_type { get; set; } - public string? subdir { get; set; } - } - - private class WebHelper : JSONWebHelper - { - private readonly OpenStackStorage m_parent; - - public WebHelper(OpenStackStorage parent) { m_parent = parent; } - - public override HttpWebRequest CreateRequest(string url, string? method = null) - { - var req = base.CreateRequest(url, method); - req.Headers["X-Auth-Token"] = m_parent.GetAccessToken(CancellationToken.None).Await(); - return req; - } - } - - public OpenStackStorage() - { - m_container = null!; - m_prefix = null!; - m_timeouts = null!; - m_username = null!; - m_authUri = null!; - m_helper = null!; - } - - public OpenStackStorage(string url, Dictionary options) - { - var uri = new Utility.Uri(url); - - m_container = uri.Host; - m_prefix = Util.AppendDirSeparator("/" + uri.Path, "/"); - m_timeouts = TimeoutOptionsHelper.Parse(options); - - // For OpenStack we do not use a leading slash - if (m_prefix.StartsWith("/", StringComparison.Ordinal)) - m_prefix = m_prefix.Substring(1); - - var auth = AuthOptionsHelper.Parse(options, uri); - - options.TryGetValue(DOMAINNAME_OPTION, out m_domainName); - options.TryGetValue(TENANTNAME_OPTION, out m_tenantName); - options.TryGetValue(VERSION_OPTION, out m_version); - options.TryGetValue(APIKEY_OPTION, out m_apikey); - options.TryGetValue(REGION_OPTION, out m_region); - - if (!auth.HasUsername) - throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthUsernameOption), "OpenStackMissingUsername"); - - m_username = auth.Username!; - m_password = auth.Password; - var authUri = options.GetValueOrDefault(AUTHURI_OPTION); - if (string.IsNullOrWhiteSpace(authUri)) - throw new UserInformationException(Strings.OpenStack.MissingOptionError(AUTHURI_OPTION), "OpenStackMissingAuthUri"); - - m_authUri = authUri; - if (string.IsNullOrWhiteSpace(m_version) && m_authUri.EndsWith("/v3", StringComparison.OrdinalIgnoreCase)) - m_version = "v3"; - - switch (m_version) - { - case "v3": if (string.IsNullOrWhiteSpace(m_password)) throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthPasswordOption), "OpenStackMissingPassword"); - if (string.IsNullOrWhiteSpace(m_domainName)) - throw new UserInformationException(Strings.OpenStack.MissingOptionError(DOMAINNAME_OPTION), "OpenStackMissingDomainName"); if (string.IsNullOrWhiteSpace(m_tenantName)) throw new UserInformationException(Strings.OpenStack.MissingOptionError(TENANTNAME_OPTION), "OpenStackMissingTenantName"); - break; - case "v2": - default: - if (string.IsNullOrWhiteSpace(m_apikey)) - { - if (string.IsNullOrWhiteSpace(m_password)) - throw new UserInformationException(Strings.OpenStack.MissingOptionError(AuthOptionsHelper.AuthPasswordOption), "OpenStackMissingPassword"); - if (string.IsNullOrWhiteSpace(m_tenantName)) - throw new UserInformationException(Strings.OpenStack.MissingOptionError(TENANTNAME_OPTION), "OpenStackMissingTenantName"); - } - break; - } - - m_helper = new WebHelper(this); - } - - protected async Task GetAccessToken(CancellationToken cancelToken) - { - if (m_accessToken == null || (m_accessToken.expires.HasValue && (m_accessToken.expires.Value - DateTime.UtcNow).TotalSeconds < 30)) - await GetAuthResponse(cancelToken).ConfigureAwait(false); - - return m_accessToken!.id!; - } - - private static string JoinUrls(string uri, string fragment) - { - fragment = fragment ?? ""; - return uri + (uri.EndsWith("/", StringComparison.Ordinal) ? "" : "/") + (fragment.StartsWith("/", StringComparison.Ordinal) ? fragment.Substring(1) : fragment); - } - private static string JoinUrls(string uri, string fragment1, string fragment2) - { - return JoinUrls(JoinUrls(uri, fragment1), fragment2); - } - - private Task GetAuthResponse(CancellationToken cancellationToken) - { - switch (this.m_version) - { - case "v3": - return GetKeystone3AuthResponse(cancellationToken); - case "v2": - default: - return GetOpenstackAuthResponse(cancellationToken); - } - } - - private async Task GetKeystone3AuthResponse(CancellationToken cancellationToken) - { - var helper = new JSONWebHelper(); - - var req = helper.CreateRequest(JoinUrls(m_authUri, "auth/tokens")); - req.Accept = "application/json"; - req.Method = "POST"; - - var data = Encoding.UTF8.GetBytes( - JsonConvert.SerializeObject( - new Keystone3AuthRequest(m_domainName, m_username, m_password, m_tenantName) - )); - - req.ContentLength = data.Length; - req.ContentType = "application/json; charset=UTF-8"; - - await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, async ct => - { - using (var rs = req.GetRequestStream()) - await rs.WriteAsync(data, 0, data.Length, ct); - }).ConfigureAwait(false); - - - var http_response = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct => req.GetResponse()).ConfigureAwait(false); - - var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct => - { - using (var reader = new StreamReader(http_response.GetResponseStream())) - return JsonConvert.DeserializeObject(reader.ReadToEnd()) - ?? throw new Exception("Failed to parse response"); - }).ConfigureAwait(false); - - if (resp.token == null) - throw new Exception("No token received"); - - var token = http_response.Headers["X-Subject-Token"]; - if (string.IsNullOrWhiteSpace(token)) - throw new Exception("No token received"); - - this.m_accessToken = new OpenStackAuthResponse.TokenClass() - { - id = token, - expires = resp.token.expires_at - }; - - // Grab the endpoint now that we have received it anyway - var fileservice = (resp.token.catalog ?? []).FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)); - if (fileservice == null) - throw new Exception("No object-store service found, is this service supported by the provider?"); - - if (fileservice.endpoints == null || fileservice.endpoints.Length == 0) - throw new Exception("No endpoints found for object-store service"); - - var endpoint = fileservice.endpoints.FirstOrDefault(x => (string.Equals(m_region, x.region) && string.Equals(x.interface_name, "public", StringComparison.OrdinalIgnoreCase))) ?? fileservice.endpoints.First(); - m_simplestorageendpoint = endpoint.url; - - return resp; - } - - private async Task GetOpenstackAuthResponse(CancellationToken cancellationToken) - { - var helper = new JSONWebHelper(); - - var req = helper.CreateRequest(JoinUrls(m_authUri, "tokens")); - req.Accept = "application/json"; - req.Method = "POST"; - - var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancellationToken, ct => helper.ReadJSONResponse( - req, - new OpenStackAuthRequest(m_tenantName, m_username, m_password, m_apikey) - )).ConfigureAwait(false); - - if (resp.access?.token == null) - throw new Exception("No token received"); - - m_accessToken = resp.access.token; - - // Grab the endpoint now that we have received it anyway - var fileservice = (resp.access.serviceCatalog ?? []).FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)); - if (fileservice == null) - throw new Exception("No object-store service found, is this service supported by the provider?"); - - var fileserviceendpoints = fileservice.endpoints ?? []; - var endpoint = fileserviceendpoints.FirstOrDefault(x => string.Equals(m_region, x.region)) - ?? fileserviceendpoints.FirstOrDefault(); - - if (endpoint == null) - throw new Exception("No endpoint found for object-store service"); - - m_simplestorageendpoint = endpoint.publicURL; - - return resp; - } - - protected async Task GetSimpleStorageEndPoint(CancellationToken cancelToken) - { - if (m_simplestorageendpoint == null) - await GetAuthResponse(cancelToken); - - return m_simplestorageendpoint!; - } - - #region IStreamingBackend implementation - public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) - { - var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Utility.Uri.UrlPathEncode(m_prefix + remotename)); - using (var ts = stream.ObserveReadTimeout(m_timeouts.ShortTimeout, false)) - using (await m_helper.GetResponseAsync(url, cancelToken, ts, "PUT").ConfigureAwait(false)) - { } - } - - public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) - { - var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Utility.Uri.UrlPathEncode(m_prefix + remotename)); - - try - { - using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_helper.GetResponse(url)).ConfigureAwait(false)) - using (var rs = AsyncHttpRequest.TrySetTimeout(await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => resp.GetResponseStream()).ConfigureAwait(false))) - using (var ts = rs.ObserveReadTimeout(m_timeouts.ShortTimeout)) - await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false); - } - catch (WebException wex) - { - if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) - throw new FileMissingException(); - else - throw; - } - - } - #endregion - #region IBackend implementation - - private async Task HandleListExceptions(Func> func) - { - try - { - return await func().ConfigureAwait(false); - } - catch (WebException wex) - { - if (wex.Response is HttpWebResponse response && (response.StatusCode == HttpStatusCode.NotFound)) - throw new FolderMissingException(); - else - throw; - } - } - - /// - public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) - { - var plainurl = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT); - if (!string.IsNullOrEmpty(m_prefix)) - plainurl += "&prefix=" + Utility.Uri.UrlEncode(m_prefix); - - var url = plainurl; - - while (true) - { - var req = m_helper.CreateRequest(url); - req.Accept = "application/json"; - - var items = await HandleListExceptions(async () => await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct => m_helper.ReadJSONResponseAsync(req, ct))).ConfigureAwait(false); - foreach (var n in items) - { - var name = n.name; - if (string.IsNullOrWhiteSpace(name)) - continue; - - if (name.StartsWith(m_prefix, StringComparison.Ordinal)) - name = name.Substring(m_prefix.Length); - - if (n.bytes == null) - yield return new FileEntry(name); - else if (n.last_modified == null) - yield return new FileEntry(name, n.bytes.Value); - else - yield return new FileEntry(name, n.bytes.Value, n.last_modified.Value, n.last_modified.Value); } - - if (items.Length != PAGE_LIMIT) - yield break; - - // Prepare next listing entry - url = plainurl + string.Format("&marker={0}", Library.Utility.Uri.UrlEncode(items.Last().name)); - } + break; } - public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) - { - using (FileStream fs = File.OpenRead(filename)) - await PutAsync(remotename, fs, cancelToken); - } + m_helper = new OpenStackWebHelper(this, _httpClient.Value); + } - public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) - { - using (FileStream fs = File.Create(filename)) - await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); - } - public async Task DeleteAsync(string remotename, CancellationToken cancelToken) - { - var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)); - await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_helper.ReadJSONResponseAsync(url, ct, null, "DELETE")).ConfigureAwait(false); - } - public Task TestAsync(CancellationToken cancelToken) - => this.TestReadWritePermissionsAsync(cancelToken); + internal async Task GetAccessToken(CancellationToken cancelToken) + { + if (m_accessToken == null || (m_accessToken.expires.HasValue && (m_accessToken.expires.Value - DateTime.UtcNow).TotalSeconds < 30)) + await GetAuthResponse(cancelToken).ConfigureAwait(false); - public async Task CreateFolderAsync(CancellationToken cancelToken) + return m_accessToken?.id!; + } + + private static string JoinUrls(string uri, string fragment) + { + fragment = fragment ?? ""; + return uri + (uri.EndsWith("/", StringComparison.Ordinal) ? string.Empty : "/") + (fragment.StartsWith("/", StringComparison.Ordinal) ? fragment.Substring(1) : fragment); + } + private static string JoinUrls(string uri, string fragment1, string fragment2) + { + return JoinUrls(JoinUrls(uri, fragment1), fragment2); + } + + private Task GetAuthResponse(CancellationToken cancellationToken) + { + return m_version switch { - var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container); - using (await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_helper.GetResponseAsync(url, cancelToken, null, "PUT")).ConfigureAwait(false)) - { } + "v3" => GetKeystone3AuthResponse(cancellationToken), + _ => GetOpenstackAuthResponse(cancellationToken) + }; + } + + private async Task GetKeystone3AuthResponse(CancellationToken cancellationToken) + { + string result; + using var request = new HttpRequestMessage(HttpMethod.Post, JoinUrls(m_authUri, "auth/tokens")); + request.Headers.UserAgent.ParseAdd( + $"Duplicati CloudStack Client {Assembly.GetExecutingAssembly().GetName().Version?.ToString()}"); + + var authRequestBytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject( + new Keystone3AuthRequest(m_domainName, m_username, m_password, m_tenantName) + )); + + request.Content = new ByteArrayContent(authRequestBytes); + request.Content.Headers.Add("Content-Length", authRequestBytes.Length.ToString()); + request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + + using var response = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, + innerCancellationToken => _httpClient.Value.SendAsync(request, innerCancellationToken)) + .ConfigureAwait(false); + + if (response.StatusCode != HttpStatusCode.OK) + throw new Exception("Failed to fetch region endpoint"); + + await using var s = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + await using var t = s.ObserveReadTimeout(_timeouts.ReadWriteTimeout); + using var reader = new StreamReader(t); + result = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false); + + var parsedResult = JsonConvert.DeserializeObject(result) + ?? throw new Exception("Failed to parse response"); + + if (parsedResult.token == null) + throw new Exception("No token received"); + + var token = response.Headers.GetValues("X-Subject-Token").FirstOrDefault(); //TODO: + if (string.IsNullOrWhiteSpace(token)) + throw new Exception("No token received"); + + m_accessToken = new OpenStackAuthResponse.TokenClass + { + id = token, + expires = parsedResult.token.expires_at + }; + + // Grab the endpoint now that we have received it anyway + var fileService = (parsedResult.token.catalog ?? []).FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)); + if (fileService == null) + throw new Exception("No object-store service found, is this service supported by the provider?"); + + if (fileService.endpoints == null || fileService.endpoints.Length == 0) + throw new Exception("No endpoints found for object-store service"); + + var endpoint = fileService.endpoints.FirstOrDefault(x => string.Equals(m_region, x.region) && string.Equals(x.interface_name, "public", StringComparison.OrdinalIgnoreCase)) ?? fileService.endpoints.First(); + m_simplestorageendpoint = endpoint.url; + + return parsedResult; + } + + private async Task GetOpenstackAuthResponse(CancellationToken cancellationToken) + { + string result; + using var request = new HttpRequestMessage(HttpMethod.Post, JoinUrls(m_authUri, "tokens")); + request.Headers.UserAgent.ParseAdd( + $"Duplicati CloudStack Client {Assembly.GetExecutingAssembly().GetName().Version?.ToString()}"); + + var authRequestBytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject( + new OpenStackAuthRequest(m_tenantName, m_username, m_password, m_apikey) + )); + + request.Content = new ByteArrayContent(authRequestBytes); + request.Content.Headers.Add("Content-Length", authRequestBytes.Length.ToString()); + request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + + using var response = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancellationToken, + innerCancellationToken => _httpClient.Value.SendAsync(request, innerCancellationToken)) + .ConfigureAwait(false); + + if (response.StatusCode != HttpStatusCode.OK) + throw new Exception("Failed to fetch region endpoint"); + + await using var s = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + await using var t = s.ObserveReadTimeout(_timeouts.ReadWriteTimeout); + using var reader = new StreamReader(t); + result = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false); + + var parsedResult = JsonConvert.DeserializeObject(result) + ?? throw new Exception("Failed to parse response"); + + m_accessToken = parsedResult.access?.token ?? throw new Exception("No token received"); + + // Grab the endpoint now that we have received it anyway + var fileservice = (parsedResult.access.serviceCatalog ?? []).FirstOrDefault(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)); + if (fileservice == null) + throw new Exception("No object-store service found, is this service supported by the provider?"); + + var fileserviceendpoints = fileservice.endpoints ?? []; + var endpoint = fileserviceendpoints.FirstOrDefault(x => string.Equals(m_region, x.region)) + ?? fileserviceendpoints.FirstOrDefault(); + + if (endpoint == null) + throw new Exception("No endpoint found for object-store service"); + + m_simplestorageendpoint = endpoint.publicURL; + + return parsedResult; + } + + private async Task GetSimpleStorageEndPoint(CancellationToken cancelToken) + { + if (m_simplestorageendpoint == null) + await GetAuthResponse(cancelToken); + + return m_simplestorageendpoint; + } + + /// + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) + { + var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Uri.UrlPathEncode(m_prefix + remotename)); + using var req = m_helper.CreateRequest(url, "PUT"); + await using var ts = stream.ObserveReadTimeout(_timeouts.ShortTimeout, false); + + req.Content = new StreamContent(ts); + req.Content.Headers.Add("Content-Type", "application/octet-stream"); + req.Content.Headers.Add("Content-Length", stream.Length.ToString()); + using var response = await _httpClient.Value.UploadStream(req, cancelToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + } + + /// + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) + { + var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Uri.UrlPathEncode(m_prefix + remotename)); + + try + { + using var req = m_helper.CreateRequest(url); + using var resp = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => m_helper.GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, ct)).ConfigureAwait(false); + await using var rs = await resp.Content.ReadAsStreamAsync(cancelToken).ConfigureAwait(false); + await using var ts = rs.ObserveReadTimeout(_timeouts.ShortTimeout); + await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false); } - public string DisplayName => Strings.OpenStack.DisplayName; - public string ProtocolKey => "openstack"; - public IList SupportedCommands + catch (HttpRequestException wex) { - get + if (wex.StatusCode == HttpStatusCode.NotFound) + throw new FileMissingException(); + throw; + } + } + + private async Task HandleListExceptions(Func> func) + { + try + { + return await func().ConfigureAwait(false); + } + catch (WebException wex) + { + if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) + throw new FolderMissingException(); + throw; + } + } + + /// + public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) + { + var plainurl = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT); + if (!string.IsNullOrEmpty(m_prefix)) + plainurl += "&prefix=" + Uri.UrlEncode(m_prefix); + + var url = plainurl; + + while (true) + { + using var req = m_helper.CreateRequest(url); + req.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + var items = await HandleListExceptions(async () => await Utility.Utility.WithTimeout(_timeouts.ListTimeout, cancelToken, ct => m_helper.ReadJsonResponseAsync(req, ct))).ConfigureAwait(false); + foreach (var n in items) { - var authuris = new StringBuilder(); - foreach (var s in KNOWN_OPENSTACK_PROVIDERS) - authuris.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); + var name = n.name; + if (string.IsNullOrWhiteSpace(name)) + continue; - return [ - new CommandLineArgument(DOMAINNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.DomainnameOptionShort, Strings.OpenStack.DomainnameOptionLong), - new CommandLineArgument(AuthOptionsHelper.AuthUsernameOption, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong), - new CommandLineArgument(AuthOptionsHelper.AuthPasswordOption, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)), - new CommandLineArgument(TENANTNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.TenantnameOptionShort, Strings.OpenStack.TenantnameOptionLong), - new CommandLineArgument(APIKEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.ApikeyOptionShort, Strings.OpenStack.ApikeyOptionLong), - new CommandLineArgument(AUTHURI_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.AuthuriOptionShort, Strings.OpenStack.AuthuriOptionLong(authuris.ToString())), - new CommandLineArgument(VERSION_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.VersionOptionShort, Strings.OpenStack.VersionOptionLong), - new CommandLineArgument(REGION_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.RegionOptionShort, Strings.OpenStack.RegionOptionLong), - .. TimeoutOptionsHelper.GetOptions(), - ]; + if (name.StartsWith(m_prefix, StringComparison.Ordinal)) + name = name.Substring(m_prefix.Length); + + if (n.bytes == null) + yield return new FileEntry(name); + else if (n.last_modified == null) + yield return new FileEntry(name, n.bytes.Value); + else + yield return new FileEntry(name, n.bytes.Value, n.last_modified.Value, n.last_modified.Value); } - } - public string Description => Strings.OpenStack.Description; - public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { + if (items.Length != PAGE_LIMIT) + yield break; + + // Prepare next listing entry + url = plainurl + $"&marker={Uri.UrlEncode(items.Last().name)}"; + } + } + + /// + public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) + { + await using FileStream fs = File.OpenRead(filename); + await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); + } + + /// + public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) + { + await using FileStream fs = File.Create(filename); + await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); + } + + /// + public async Task DeleteAsync(string remotename, CancellationToken cancelToken) + { + var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container, Uri.UrlPathEncode(m_prefix + remotename)); + using var req = m_helper.CreateRequest(url, "DELETE"); + using var response = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => m_helper.GetResponseAsync(req, HttpCompletionOption.ResponseContentRead, ct)).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + } + + /// + public Task TestAsync(CancellationToken cancelToken) + => this.TestReadWritePermissionsAsync(cancelToken); + + /// + public async Task CreateFolderAsync(CancellationToken cancelToken) + { + var url = JoinUrls(await GetSimpleStorageEndPoint(cancelToken).ConfigureAwait(false), m_container); + using var req = m_helper.CreateRequest(url, "PUT"); + using var response = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, + ct => m_helper.GetResponseAsync(req, HttpCompletionOption.ResponseContentRead, ct)).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + } + + /// + public string DisplayName => Strings.OpenStack.DisplayName; + + /// + public string ProtocolKey => "openstack"; + + /// + public IList SupportedCommands + { + get + { + var authuris = new StringBuilder(); + foreach (var s in KnownOpenstackProviders) + authuris.AppendLine($"{s.Key}: {s.Value}"); + + return [ + new CommandLineArgument(DOMAINNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.DomainnameOptionShort, Strings.OpenStack.DomainnameOptionLong), + new CommandLineArgument(AuthOptionsHelper.AuthUsernameOption, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong), + new CommandLineArgument(AuthOptionsHelper.AuthPasswordOption, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)), + new CommandLineArgument(TENANTNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.TenantnameOptionShort, Strings.OpenStack.TenantnameOptionLong), + new CommandLineArgument(APIKEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.ApikeyOptionShort, Strings.OpenStack.ApikeyOptionLong), + new CommandLineArgument(AUTHURI_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.AuthuriOptionShort, Strings.OpenStack.AuthuriOptionLong(authuris.ToString())), + new CommandLineArgument(VERSION_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.VersionOptionShort, Strings.OpenStack.VersionOptionLong), + new CommandLineArgument(REGION_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.RegionOptionShort, Strings.OpenStack.RegionOptionLong), + .. TimeoutOptionsHelper.GetOptions(), + ]; + } + } + + /// + public string Description => Strings.OpenStack.Description; + + /// + public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new[] { new System.Uri(m_authUri).Host, string.IsNullOrWhiteSpace(m_simplestorageendpoint) ? null : new System.Uri(m_simplestorageendpoint).Host } .WhereNotNullOrWhiteSpace() .ToArray()); - #endregion - #region IDisposable implementation - public void Dispose() - { - } - #endregion + public void Dispose() + { } -} - +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorageItem.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorageItem.cs new file mode 100644 index 000000000..0227eff9d --- /dev/null +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorageItem.cs @@ -0,0 +1,31 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Library.Backend.OpenStack; + +internal class OpenStackStorageItem +{ + public string? name { get; set; } + public DateTime? last_modified { get; set; } + public long? bytes { get; set; } + public string? content_type { get; set; } + public string? subdir { get; set; } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackWebHelper.cs b/Duplicati/Library/Backend/OpenStack/OpenStackWebHelper.cs new file mode 100644 index 000000000..c1219790f --- /dev/null +++ b/Duplicati/Library/Backend/OpenStack/OpenStackWebHelper.cs @@ -0,0 +1,33 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using Duplicati.Library.Utility; + +namespace Duplicati.Library.Backend.OpenStack; +internal class OpenStackWebHelper(OpenStackStorage parent, HttpClient httpClient) : JsonWebHelperHttpClient(httpClient) +{ + public override HttpRequestMessage CreateRequest(string url, string? method = null) + { + var request = base.CreateRequest(url, method); + request.Headers.TryAddWithoutValidation("X-Auth-Token", parent.GetAccessToken(CancellationToken.None).Await()); + return request; + } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OpenStack/Strings.cs b/Duplicati/Library/Backend/OpenStack/Strings.cs index 6abff1fff..c793a2958 100644 --- a/Duplicati/Library/Backend/OpenStack/Strings.cs +++ b/Duplicati/Library/Backend/OpenStack/Strings.cs @@ -21,37 +21,34 @@ using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.Strings +namespace Duplicati.Library.Backend.Strings; +internal static class OpenStack { - internal static class OpenStack - { - public static string Description { get { return LC.L(@"This backend can read and write data to Swift (OpenStack Object Storage). Allowed format is ""openstack://container/folder""."); } } - public static string DisplayName { get { return LC.L(@"OpenStack Simple Storage"); } } - public static string MissingOptionError(string optionname) { return LC.L(@"Missing required option: {0}", optionname); } - public static string PasswordOptionLong(string tenantnameoption) { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD"". If the password is supplied, --{0} must also be set.", tenantnameoption); } - public static string PasswordOptionShort { get { return LC.L(@"Supply the password used to connect to the server"); } } - public static string DomainnameOptionLong { get { return LC.L(@"The domain name of the user used to connect to the server."); } } - public static string DomainnameOptionShort { get { return LC.L(@"Supply the domain used to connect to the server"); } } - public static string UsernameOptionLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } } - public static string UsernameOptionShort { get { return LC.L(@"Supply the username used to connect to the server"); } } - public static string TenantnameOptionLong { get { return LC.L(@"The Tenant Name is commonly the paying user account name. This option must be supplied when authenticating with a password, but is not required when using an API key."); } } - public static string TenantnameOptionShort { get { return LC.L(@"Supply the Tenant Name used to connect to the server"); } } - public static string ApikeyOptionLong { get { return LC.L(@"The API key can be used to connect without supplying a password and tenant ID with some providers."); } } - public static string ApikeyOptionShort { get { return LC.L(@"Supply the API key used to connect to the server"); } } - public static string AuthuriOptionLong(string providers) { return LC.L(@"The authentication URL is used to authenticate the user and find the storage service. The URL commonly ends with ""/v2.0"" for v2 and ""/v3"" for v3. Known providers are: {0}{1}", System.Environment.NewLine, providers); } - public static string AuthuriOptionShort { get { return LC.L(@"Supply the authentication URL"); } } - public static string VersionOptionLong { get { return LC.L(@"The keystone API version to use. Valid values are 'v2' and 'v3'."); } } - public static string VersionOptionShort { get { return LC.L(@"The keystone API version to use"); } } - public static string RegionOptionLong { get { return LC.L(@"By default, the first reported endpoint will be used for file transfers. To select a specific region, provide the region name. If no such region is supported, the default (first reported) endpoint is used."); } } - public static string RegionOptionShort { get { return LC.L(@"Supply the prefered region for endpoints"); } } - } - - internal static class SwiftConfig - { - public static string Description { get { return LC.L(@"Expose OpenStack configuration as a web module"); } } - public static string DisplayName { get { return LC.L(@"OpenStack configuration module"); } } - public static string ConfigTypeOptionLong { get { return LC.L(@"Provide different config values"); } } - public static string ConfigTypeOptionShort { get { return LC.L(@"The config to get"); } } - - } + public static string Description => LC.L(@"This backend can read and write data to Swift (OpenStack Object Storage). Allowed format is ""openstack://container/folder""."); + public static string DisplayName => LC.L(@"OpenStack Simple Storage"); + public static string MissingOptionError(string optionname) { return LC.L(@"Missing required option: {0}", optionname); } + public static string PasswordOptionLong(string tenantnameoption) { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD"". If the password is supplied, --{0} must also be set.", tenantnameoption); } + public static string PasswordOptionShort => LC.L(@"Supply the password used to connect to the server"); + public static string DomainnameOptionLong => LC.L(@"The domain name of the user used to connect to the server."); + public static string DomainnameOptionShort => LC.L(@"Supply the domain used to connect to the server"); + public static string UsernameOptionLong => LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); + public static string UsernameOptionShort => LC.L(@"Supply the username used to connect to the server"); + public static string TenantnameOptionLong => LC.L(@"The Tenant Name is commonly the paying user account name. This option must be supplied when authenticating with a password, but is not required when using an API key."); + public static string TenantnameOptionShort => LC.L(@"Supply the Tenant Name used to connect to the server"); + public static string ApikeyOptionLong => LC.L(@"The API key can be used to connect without supplying a password and tenant ID with some providers."); + public static string ApikeyOptionShort => LC.L(@"Supply the API key used to connect to the server"); + public static string AuthuriOptionLong(string providers) { return LC.L(@"The authentication URL is used to authenticate the user and find the storage service. The URL commonly ends with ""/v2.0"" for v2 and ""/v3"" for v3. Known providers are: {0}{1}", Environment.NewLine, providers); } + public static string AuthuriOptionShort => LC.L(@"Supply the authentication URL"); + public static string VersionOptionLong => LC.L(@"The keystone API version to use. Valid values are 'v2' and 'v3'."); + public static string VersionOptionShort => LC.L(@"The keystone API version to use"); + public static string RegionOptionLong => LC.L(@"By default, the first reported endpoint will be used for file transfers. To select a specific region, provide the region name. If no such region is supported, the default (first reported) endpoint is used."); + public static string RegionOptionShort => LC.L(@"Supply the prefered region for endpoints"); } +internal static class SwiftConfig +{ + public static string Description { get { return LC.L(@"Expose OpenStack configuration as a web module"); } } + public static string DisplayName { get { return LC.L(@"OpenStack configuration module"); } } + public static string ConfigTypeOptionLong { get { return LC.L(@"Provide different config values"); } } + public static string ConfigTypeOptionShort { get { return LC.L(@"The config to get"); } } + +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackConfig.cs b/Duplicati/Library/Backend/OpenStack/SwiftConfig.cs similarity index 89% rename from Duplicati/Library/Backend/OpenStack/OpenStackConfig.cs rename to Duplicati/Library/Backend/OpenStack/SwiftConfig.cs index 280d22cbf..8429badb5 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackConfig.cs +++ b/Duplicati/Library/Backend/OpenStack/SwiftConfig.cs @@ -29,7 +29,7 @@ namespace Duplicati.Library.Backend.OpenStack private static readonly string DEFAULT_CONFIG_TYPE_STR = DEFAULT_CONFIG_TYPE.ToString(); private const string KEY_CONFIGTYPE = "openstack-config"; - public enum ConfigType + private enum ConfigType { Providers, Versions, @@ -50,9 +50,9 @@ namespace Duplicati.Library.Backend.OpenStack switch (ct) { case ConfigType.Versions: - return OpenStackStorage.OPENSTACK_VERSIONS.ToDictionary((x) => x.Key, (y) => y.Value); + return OpenStackStorage.OpenstackVersions.ToDictionary((x) => x.Key, (y) => y.Value); default: - return OpenStackStorage.KNOWN_OPENSTACK_PROVIDERS.ToDictionary((x) => x.Key, (y) => y.Value); + return OpenStackStorage.KnownOpenstackProviders.ToDictionary((x) => x.Key, (y) => y.Value); } }