diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 673fb3528..1dbd8c887 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -21,6 +21,7 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.IO; @@ -168,23 +169,7 @@ namespace Duplicati.Library.Backend { get { - var defaults = new Amazon.S3.AmazonS3Config() - { - // If this is not set, accessing the property will trigger an expensive operation (~30 seconds) - // to get the region endpoint. This stalls the supported commands list. - UseArnRegion = false - }; - - var exts = - typeof(Amazon.S3.AmazonS3Config).GetProperties().Where(x => x.CanRead && x.CanWrite && (x.PropertyType == typeof(string) || x.PropertyType == typeof(bool) || x.PropertyType == typeof(int) || x.PropertyType == typeof(long) || x.PropertyType.IsEnum)) - .Select(x => (ICommandLineArgument)new CommandLineArgument( - "s3-ext-" + x.Name.ToLowerInvariant(), - x.PropertyType == typeof(bool) ? CommandLineArgument.ArgumentType.Boolean : x.PropertyType.IsEnum ? CommandLineArgument.ArgumentType.Enumeration : CommandLineArgument.ArgumentType.String, - x.Name, - string.Format("Extended option {0}", x.Name), - string.Format("{0}", x.GetValue(defaults)), - null, - x.PropertyType.IsEnum ? Enum.GetNames(x.PropertyType) : null)); + var exts = S3AwsClient.GetAwsExtendedOptions(); var normal = new ICommandLineArgument[] { new CommandLineArgument("access_key_secret", CommandLineArgument.ArgumentType.Password, Strings.Idrivee2Backend.KeySecretDescriptionShort, Strings.Idrivee2Backend.KeySecretDescriptionLong, null, new[]{"auth-password"}, null), diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index 023147be9..dbba2f9b5 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -22,6 +22,7 @@ using Amazon.S3; using Amazon.S3.Model; +using Duplicati.Library.Backend.Strings; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using System; @@ -40,6 +41,8 @@ namespace Duplicati.Library.Backend private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private const int ITEM_LIST_LIMIT = 1000; + private const string EXT_OPTION_PREFIX = "s3-ext"; + private readonly string m_locationConstraint; private readonly string m_storageClass; private AmazonS3Client m_client; @@ -54,27 +57,7 @@ namespace Duplicati.Library.Backend cfg.UseHttp = !useSSL; cfg.ServiceURL = (useSSL ? "https://" : "http://") + servername; - foreach (var opt in options.Keys.Where(x => x.StartsWith("s3-ext-", StringComparison.OrdinalIgnoreCase))) - { - var prop = cfg.GetType().GetProperties().FirstOrDefault(x => - string.Equals(x.Name, opt.Substring("s3-ext-".Length), StringComparison.OrdinalIgnoreCase)); - if (prop != null && prop.CanWrite) - { - if (prop.PropertyType == typeof(bool)) - prop.SetValue(cfg, Utility.Utility.ParseBoolOption(options, opt)); - else if (prop.PropertyType.IsEnum) - prop.SetValue(cfg, Enum.Parse(prop.PropertyType, options[opt], true)); - else if (prop.PropertyType == typeof(int)) - prop.SetValue(cfg, int.Parse(options[opt])); - else if (prop.PropertyType == typeof(long)) - prop.SetValue(cfg, long.Parse(options[opt])); - else if (prop.PropertyType == typeof(string)) - prop.SetValue(cfg, options[opt]); - } - - if (prop == null) - Logging.Log.WriteWarningMessage(LOGTAG, "UnsupportedOption", null, "Unsupported option: {0}", opt); - } + CommandLineArgumentMapper.ApplyArguments(cfg, options, EXT_OPTION_PREFIX); m_client = new AmazonS3Client(awsID, awsKey, cfg); @@ -97,7 +80,7 @@ namespace Duplicati.Library.Backend return m_client.PutBucketAsync(request, cancelToken); } - internal static AmazonS3Config GetDefaultAmazonS3Config() + public static AmazonS3Config GetDefaultAmazonS3Config() { return new AmazonS3Config() { @@ -111,6 +94,26 @@ namespace Duplicati.Library.Backend }; } + private static readonly HashSet EXCLUDED_EXTENDED_OPTIONS = new HashSet([ + nameof(AmazonS3Config.USEast1RegionalEndpointValue) + ]); + + /// + /// List of properties that are slow to read the default value from + /// + /// Changes in this list will likely need to be reflected in AWSSecretProvider.cs + private static readonly HashSet SLOW_LOADING_PROPERTIES = new[] { + nameof(AmazonS3Config.RegionEndpoint), + nameof(AmazonS3Config.ServiceURL), + nameof(AmazonS3Config.MaxErrorRetry), + nameof(AmazonS3Config.DefaultConfigurationMode), + nameof(AmazonS3Config.Timeout), + nameof(AmazonS3Config.RetryMode), + }.ToHashSet(); + + public static IEnumerable GetAwsExtendedOptions() + => CommandLineArgumentMapper.MapArguments(GetDefaultAmazonS3Config(), prefix: EXT_OPTION_PREFIX, exclude: EXCLUDED_EXTENDED_OPTIONS, excludeDefaultValue: SLOW_LOADING_PROPERTIES); + public virtual async Task GetFileStreamAsync(string bucketName, string keyName, System.IO.Stream target, CancellationToken cancelToken) { try diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index ec7b5d9ab..e033f82e0 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -21,6 +21,7 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.IO; @@ -322,19 +323,7 @@ namespace Duplicati.Library.Backend foreach (var s in KNOWN_S3_LOCATIONS) locations.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); - var defaults = S3AwsClient.GetDefaultAmazonS3Config(); - - var exts = - typeof(Amazon.S3.AmazonS3Config).GetProperties().Where(x => x.CanRead && x.CanWrite && (x.PropertyType == typeof(string) || x.PropertyType == typeof(bool) || x.PropertyType == typeof(int) || x.PropertyType == typeof(long) || x.PropertyType.IsEnum)) - .Select(x => (ICommandLineArgument)new CommandLineArgument( - "s3-ext-" + x.Name.ToLowerInvariant(), - x.PropertyType == typeof(bool) ? CommandLineArgument.ArgumentType.Boolean : x.PropertyType.IsEnum ? CommandLineArgument.ArgumentType.Enumeration : CommandLineArgument.ArgumentType.String, - x.Name, - string.Format("Extended option {0}", x.Name), - string.Format("{0}", x.GetValue(defaults)), - null, - x.PropertyType.IsEnum ? Enum.GetNames(x.PropertyType) : null)); - + var exts = S3AwsClient.GetAwsExtendedOptions(); var normal = new ICommandLineArgument[] { new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, new string[] {"auth-password"}, null ), diff --git a/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs b/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs index 6cc9bc423..ce195b360 100644 --- a/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs +++ b/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs @@ -43,8 +43,7 @@ namespace Duplicati.Server.Serializable this.Key = backend.ProtocolKey; this.Description = backend.Description; this.DisplayName = backend.DisplayName; - if (backend.SupportedCommands != null) - this.Options = backend.SupportedCommands.ToArray(); + this.Options = backend.SupportedCommands?.ToArray() ?? []; } /// @@ -55,8 +54,7 @@ namespace Duplicati.Server.Serializable this.Key = module.FilenameExtension; this.Description = module.Description; this.DisplayName = module.DisplayName; - if (module.SupportedCommands != null) - this.Options = module.SupportedCommands.ToArray(); + this.Options = module.SupportedCommands?.ToArray() ?? []; } /// @@ -67,8 +65,7 @@ namespace Duplicati.Server.Serializable this.Key = module.FilenameExtension; this.Description = module.Description; this.DisplayName = module.DisplayName; - if (module.SupportedCommands != null) - this.Options = module.SupportedCommands.ToArray(); + this.Options = module.SupportedCommands?.ToArray() ?? []; } /// @@ -79,8 +76,7 @@ namespace Duplicati.Server.Serializable this.Key = module.Key; this.Description = module.Description; this.DisplayName = module.DisplayName; - if (module.SupportedCommands != null) - this.Options = module.SupportedCommands.ToArray(); + this.Options = module.SupportedCommands?.ToArray() ?? []; } /// @@ -91,8 +87,7 @@ namespace Duplicati.Server.Serializable this.Key = module.Key; this.Description = module.Description; this.DisplayName = module.DisplayName; - if (module.SupportedCommands != null) - this.Options = module.SupportedCommands.ToArray(); + this.Options = module.SupportedCommands?.ToArray() ?? []; } /// @@ -103,8 +98,7 @@ namespace Duplicati.Server.Serializable this.Key = module.Key; this.Description = module.Description; this.DisplayName = module.DisplayName; - if (module.SupportedCommands != null) - this.Options = module.SupportedCommands.ToArray(); + this.Options = module.SupportedCommands?.ToArray() ?? []; } /// /// The module key @@ -143,6 +137,7 @@ namespace Duplicati.Server.Serializable { get { + Console.WriteLine("Inside getter"); return (from n in Library.DynamicLoader.BackendLoader.Backends select new DynamicModule(n)) diff --git a/Duplicati/Library/SecretProvider/AWSSecretProvider.cs b/Duplicati/Library/SecretProvider/AWSSecretProvider.cs index 9e65b481d..eae45002c 100644 --- a/Duplicati/Library/SecretProvider/AWSSecretProvider.cs +++ b/Duplicati/Library/SecretProvider/AWSSecretProvider.cs @@ -67,6 +67,7 @@ public class AWSSecretProvider : ISecretProvider /// /// List of properties that slow down the loading of the AWS Secrets Manager client /// + /// Changes in this list will likely need to be reflected in S3AwsClient.cs private static readonly HashSet SlowLoadingProperties = new[] { nameof(AmazonSecretsManagerConfig.RegionEndpoint), nameof(AmazonSecretsManagerConfig.ServiceURL),