From 38a255eb00a3bd7e50ee3efb5aca12c21b209e28 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Fri, 15 Nov 2024 14:02:39 +0100 Subject: [PATCH] Fixed startup delay on Windows. Delay was caused by AWS library performing network queries when reading properties on the configuration object. On top of this, the properties were read twice. Unified reading+application of options between iDrive and AWS S3. --- .../Backend/Idrivee2/Idrivee2Backend.cs | 19 +------- Duplicati/Library/Backend/S3/S3AwsClient.cs | 47 ++++++++++--------- Duplicati/Library/Backend/S3/S3Backend.cs | 15 +----- .../RestAPI/Serializable/ServerSettings.cs | 19 +++----- .../SecretProvider/AWSSecretProvider.cs | 1 + 5 files changed, 37 insertions(+), 64 deletions(-) diff --git a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs index 673fb3528..1dbd8c887 100644 --- a/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs +++ b/Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs @@ -21,6 +21,7 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.IO; @@ -168,23 +169,7 @@ namespace Duplicati.Library.Backend { get { - var defaults = new Amazon.S3.AmazonS3Config() - { - // If this is not set, accessing the property will trigger an expensive operation (~30 seconds) - // to get the region endpoint. This stalls the supported commands list. - UseArnRegion = false - }; - - var exts = - typeof(Amazon.S3.AmazonS3Config).GetProperties().Where(x => x.CanRead && x.CanWrite && (x.PropertyType == typeof(string) || x.PropertyType == typeof(bool) || x.PropertyType == typeof(int) || x.PropertyType == typeof(long) || x.PropertyType.IsEnum)) - .Select(x => (ICommandLineArgument)new CommandLineArgument( - "s3-ext-" + x.Name.ToLowerInvariant(), - x.PropertyType == typeof(bool) ? CommandLineArgument.ArgumentType.Boolean : x.PropertyType.IsEnum ? CommandLineArgument.ArgumentType.Enumeration : CommandLineArgument.ArgumentType.String, - x.Name, - string.Format("Extended option {0}", x.Name), - string.Format("{0}", x.GetValue(defaults)), - null, - x.PropertyType.IsEnum ? Enum.GetNames(x.PropertyType) : null)); + var exts = S3AwsClient.GetAwsExtendedOptions(); var normal = new ICommandLineArgument[] { new CommandLineArgument("access_key_secret", CommandLineArgument.ArgumentType.Password, Strings.Idrivee2Backend.KeySecretDescriptionShort, Strings.Idrivee2Backend.KeySecretDescriptionLong, null, new[]{"auth-password"}, null), diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index 023147be9..dbba2f9b5 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -22,6 +22,7 @@ using Amazon.S3; using Amazon.S3.Model; +using Duplicati.Library.Backend.Strings; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using System; @@ -40,6 +41,8 @@ namespace Duplicati.Library.Backend private static readonly string LOGTAG = Logging.Log.LogTagFromType(); private const int ITEM_LIST_LIMIT = 1000; + private const string EXT_OPTION_PREFIX = "s3-ext"; + private readonly string m_locationConstraint; private readonly string m_storageClass; private AmazonS3Client m_client; @@ -54,27 +57,7 @@ namespace Duplicati.Library.Backend cfg.UseHttp = !useSSL; cfg.ServiceURL = (useSSL ? "https://" : "http://") + servername; - foreach (var opt in options.Keys.Where(x => x.StartsWith("s3-ext-", StringComparison.OrdinalIgnoreCase))) - { - var prop = cfg.GetType().GetProperties().FirstOrDefault(x => - string.Equals(x.Name, opt.Substring("s3-ext-".Length), StringComparison.OrdinalIgnoreCase)); - if (prop != null && prop.CanWrite) - { - if (prop.PropertyType == typeof(bool)) - prop.SetValue(cfg, Utility.Utility.ParseBoolOption(options, opt)); - else if (prop.PropertyType.IsEnum) - prop.SetValue(cfg, Enum.Parse(prop.PropertyType, options[opt], true)); - else if (prop.PropertyType == typeof(int)) - prop.SetValue(cfg, int.Parse(options[opt])); - else if (prop.PropertyType == typeof(long)) - prop.SetValue(cfg, long.Parse(options[opt])); - else if (prop.PropertyType == typeof(string)) - prop.SetValue(cfg, options[opt]); - } - - if (prop == null) - Logging.Log.WriteWarningMessage(LOGTAG, "UnsupportedOption", null, "Unsupported option: {0}", opt); - } + CommandLineArgumentMapper.ApplyArguments(cfg, options, EXT_OPTION_PREFIX); m_client = new AmazonS3Client(awsID, awsKey, cfg); @@ -97,7 +80,7 @@ namespace Duplicati.Library.Backend return m_client.PutBucketAsync(request, cancelToken); } - internal static AmazonS3Config GetDefaultAmazonS3Config() + public static AmazonS3Config GetDefaultAmazonS3Config() { return new AmazonS3Config() { @@ -111,6 +94,26 @@ namespace Duplicati.Library.Backend }; } + private static readonly HashSet EXCLUDED_EXTENDED_OPTIONS = new HashSet([ + nameof(AmazonS3Config.USEast1RegionalEndpointValue) + ]); + + /// + /// List of properties that are slow to read the default value from + /// + /// Changes in this list will likely need to be reflected in AWSSecretProvider.cs + private static readonly HashSet SLOW_LOADING_PROPERTIES = new[] { + nameof(AmazonS3Config.RegionEndpoint), + nameof(AmazonS3Config.ServiceURL), + nameof(AmazonS3Config.MaxErrorRetry), + nameof(AmazonS3Config.DefaultConfigurationMode), + nameof(AmazonS3Config.Timeout), + nameof(AmazonS3Config.RetryMode), + }.ToHashSet(); + + public static IEnumerable GetAwsExtendedOptions() + => CommandLineArgumentMapper.MapArguments(GetDefaultAmazonS3Config(), prefix: EXT_OPTION_PREFIX, exclude: EXCLUDED_EXTENDED_OPTIONS, excludeDefaultValue: SLOW_LOADING_PROPERTIES); + public virtual async Task GetFileStreamAsync(string bucketName, string keyName, System.IO.Stream target, CancellationToken cancelToken) { try diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index ec7b5d9ab..e033f82e0 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -21,6 +21,7 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; +using Duplicati.Library.Utility; using System; using System.Collections.Generic; using System.IO; @@ -322,19 +323,7 @@ namespace Duplicati.Library.Backend foreach (var s in KNOWN_S3_LOCATIONS) locations.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); - var defaults = S3AwsClient.GetDefaultAmazonS3Config(); - - var exts = - typeof(Amazon.S3.AmazonS3Config).GetProperties().Where(x => x.CanRead && x.CanWrite && (x.PropertyType == typeof(string) || x.PropertyType == typeof(bool) || x.PropertyType == typeof(int) || x.PropertyType == typeof(long) || x.PropertyType.IsEnum)) - .Select(x => (ICommandLineArgument)new CommandLineArgument( - "s3-ext-" + x.Name.ToLowerInvariant(), - x.PropertyType == typeof(bool) ? CommandLineArgument.ArgumentType.Boolean : x.PropertyType.IsEnum ? CommandLineArgument.ArgumentType.Enumeration : CommandLineArgument.ArgumentType.String, - x.Name, - string.Format("Extended option {0}", x.Name), - string.Format("{0}", x.GetValue(defaults)), - null, - x.PropertyType.IsEnum ? Enum.GetNames(x.PropertyType) : null)); - + var exts = S3AwsClient.GetAwsExtendedOptions(); var normal = new ICommandLineArgument[] { new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, new string[] {"auth-password"}, null ), diff --git a/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs b/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs index 6cc9bc423..ce195b360 100644 --- a/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs +++ b/Duplicati/Library/RestAPI/Serializable/ServerSettings.cs @@ -43,8 +43,7 @@ namespace Duplicati.Server.Serializable this.Key = backend.ProtocolKey; this.Description = backend.Description; this.DisplayName = backend.DisplayName; - if (backend.SupportedCommands != null) - this.Options = backend.SupportedCommands.ToArray(); + this.Options = backend.SupportedCommands?.ToArray() ?? []; } /// @@ -55,8 +54,7 @@ namespace Duplicati.Server.Serializable this.Key = module.FilenameExtension; this.Description = module.Description; this.DisplayName = module.DisplayName; - if (module.SupportedCommands != null) - this.Options = module.SupportedCommands.ToArray(); + this.Options = module.SupportedCommands?.ToArray() ?? []; } /// @@ -67,8 +65,7 @@ namespace Duplicati.Server.Serializable this.Key = module.FilenameExtension; this.Description = module.Description; this.DisplayName = module.DisplayName; - if (module.SupportedCommands != null) - this.Options = module.SupportedCommands.ToArray(); + this.Options = module.SupportedCommands?.ToArray() ?? []; } /// @@ -79,8 +76,7 @@ namespace Duplicati.Server.Serializable this.Key = module.Key; this.Description = module.Description; this.DisplayName = module.DisplayName; - if (module.SupportedCommands != null) - this.Options = module.SupportedCommands.ToArray(); + this.Options = module.SupportedCommands?.ToArray() ?? []; } /// @@ -91,8 +87,7 @@ namespace Duplicati.Server.Serializable this.Key = module.Key; this.Description = module.Description; this.DisplayName = module.DisplayName; - if (module.SupportedCommands != null) - this.Options = module.SupportedCommands.ToArray(); + this.Options = module.SupportedCommands?.ToArray() ?? []; } /// @@ -103,8 +98,7 @@ namespace Duplicati.Server.Serializable this.Key = module.Key; this.Description = module.Description; this.DisplayName = module.DisplayName; - if (module.SupportedCommands != null) - this.Options = module.SupportedCommands.ToArray(); + this.Options = module.SupportedCommands?.ToArray() ?? []; } /// /// The module key @@ -143,6 +137,7 @@ namespace Duplicati.Server.Serializable { get { + Console.WriteLine("Inside getter"); return (from n in Library.DynamicLoader.BackendLoader.Backends select new DynamicModule(n)) diff --git a/Duplicati/Library/SecretProvider/AWSSecretProvider.cs b/Duplicati/Library/SecretProvider/AWSSecretProvider.cs index 9e65b481d..eae45002c 100644 --- a/Duplicati/Library/SecretProvider/AWSSecretProvider.cs +++ b/Duplicati/Library/SecretProvider/AWSSecretProvider.cs @@ -67,6 +67,7 @@ public class AWSSecretProvider : ISecretProvider /// /// List of properties that slow down the loading of the AWS Secrets Manager client /// + /// Changes in this list will likely need to be reflected in S3AwsClient.cs private static readonly HashSet SlowLoadingProperties = new[] { nameof(AmazonSecretsManagerConfig.RegionEndpoint), nameof(AmazonSecretsManagerConfig.ServiceURL),