haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed startup delay on Windows.

Browse Source 
Delay was caused by AWS library performing network queries when reading properties on the configuration object.
On top of this, the properties were read twice.
Unified reading+application of options between iDrive and AWS S3.
This commit is contained in:
Kenneth Skovhede
2024-11-15 14:04:42 +01:00
parent 4f5be2049a
commit 38a255eb00
5 changed files with 37 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Backend/Idrivee2/Idrivee2Backend.cs
+2 -17
View File
@@ -21,6 +21,7 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System;
using System.Collections.Generic;
using System.IO;
@@ -168,23 +169,7 @@ namespace Duplicati.Library.Backend
{
get
{
var defaults = new Amazon.S3.AmazonS3Config()
{
// If this is not set, accessing the property will trigger an expensive operation (~30 seconds)
// to get the region endpoint. This stalls the supported commands list.
UseArnRegion = false
};
var exts =
typeof(Amazon.S3.AmazonS3Config).GetProperties().Where(x => x.CanRead && x.CanWrite && (x.PropertyType == typeof(string) || x.PropertyType == typeof(bool) || x.PropertyType == typeof(int) || x.PropertyType == typeof(long) || x.PropertyType.IsEnum))
.Select(x => (ICommandLineArgument)new CommandLineArgument(
"s3-ext-" + x.Name.ToLowerInvariant(),
x.PropertyType == typeof(bool) ? CommandLineArgument.ArgumentType.Boolean : x.PropertyType.IsEnum ? CommandLineArgument.ArgumentType.Enumeration : CommandLineArgument.ArgumentType.String,
x.Name,
string.Format("Extended option {0}", x.Name),
string.Format("{0}", x.GetValue(defaults)),
null,
x.PropertyType.IsEnum ? Enum.GetNames(x.PropertyType) : null));
var exts = S3AwsClient.GetAwsExtendedOptions();
var normal = new ICommandLineArgument[] {
new CommandLineArgument("access_key_secret", CommandLineArgument.ArgumentType.Password, Strings.Idrivee2Backend.KeySecretDescriptionShort, Strings.Idrivee2Backend.KeySecretDescriptionLong, null, new[]{"auth-password"}, null),
Duplicati/Library/Backend/S3/S3AwsClient.cs
+25 -22
View File
@@ -22,6 +22,7 @@
using Amazon.S3;
using Amazon.S3.Model;
using Duplicati.Library.Backend.Strings;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System;
@@ -40,6 +41,8 @@ namespace Duplicati.Library.Backend
private static readonly string LOGTAG = Logging.Log.LogTagFromType<S3AwsClient>();
private const int ITEM_LIST_LIMIT = 1000;
private const string EXT_OPTION_PREFIX = "s3-ext";
private readonly string m_locationConstraint;
private readonly string m_storageClass;
private AmazonS3Client m_client;
@@ -54,27 +57,7 @@ namespace Duplicati.Library.Backend
cfg.UseHttp = !useSSL;
cfg.ServiceURL = (useSSL ? "https://" : "http://") + servername;
foreach (var opt in options.Keys.Where(x => x.StartsWith("s3-ext-", StringComparison.OrdinalIgnoreCase)))
{
var prop = cfg.GetType().GetProperties().FirstOrDefault(x =>
string.Equals(x.Name, opt.Substring("s3-ext-".Length), StringComparison.OrdinalIgnoreCase));
if (prop != null && prop.CanWrite)
{
if (prop.PropertyType == typeof(bool))
prop.SetValue(cfg, Utility.Utility.ParseBoolOption(options, opt));
else if (prop.PropertyType.IsEnum)
prop.SetValue(cfg, Enum.Parse(prop.PropertyType, options[opt], true));
else if (prop.PropertyType == typeof(int))
prop.SetValue(cfg, int.Parse(options[opt]));
else if (prop.PropertyType == typeof(long))
prop.SetValue(cfg, long.Parse(options[opt]));
else if (prop.PropertyType == typeof(string))
prop.SetValue(cfg, options[opt]);
}
if (prop == null)
Logging.Log.WriteWarningMessage(LOGTAG, "UnsupportedOption", null, "Unsupported option: {0}", opt);
}
CommandLineArgumentMapper.ApplyArguments(cfg, options, EXT_OPTION_PREFIX);
m_client = new AmazonS3Client(awsID, awsKey, cfg);
@@ -97,7 +80,7 @@ namespace Duplicati.Library.Backend
return m_client.PutBucketAsync(request, cancelToken);
}
internal static AmazonS3Config GetDefaultAmazonS3Config()
public static AmazonS3Config GetDefaultAmazonS3Config()
{
return new AmazonS3Config()
{
@@ -111,6 +94,26 @@ namespace Duplicati.Library.Backend
};
}
private static readonly HashSet<string> EXCLUDED_EXTENDED_OPTIONS = new HashSet<string>([
nameof(AmazonS3Config.USEast1RegionalEndpointValue)
]);
/// <summary>
/// List of properties that are slow to read the default value from
/// </summary>
/// <remarks>Changes in this list will likely need to be reflected in AWSSecretProvider.cs</remarks>
private static readonly HashSet<string> SLOW_LOADING_PROPERTIES = new[] {
nameof(AmazonS3Config.RegionEndpoint),
nameof(AmazonS3Config.ServiceURL),
nameof(AmazonS3Config.MaxErrorRetry),
nameof(AmazonS3Config.DefaultConfigurationMode),
nameof(AmazonS3Config.Timeout),
nameof(AmazonS3Config.RetryMode),
}.ToHashSet();
public static IEnumerable<ICommandLineArgument> GetAwsExtendedOptions()
=> CommandLineArgumentMapper.MapArguments(GetDefaultAmazonS3Config(), prefix: EXT_OPTION_PREFIX, exclude: EXCLUDED_EXTENDED_OPTIONS, excludeDefaultValue: SLOW_LOADING_PROPERTIES);
public virtual async Task GetFileStreamAsync(string bucketName, string keyName, System.IO.Stream target, CancellationToken cancelToken)
{
try
Duplicati/Library/Backend/S3/S3Backend.cs
+2 -13
View File
@@ -21,6 +21,7 @@
using Duplicati.Library.Common.IO;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System;
using System.Collections.Generic;
using System.IO;
@@ -322,19 +323,7 @@ namespace Duplicati.Library.Backend
foreach (var s in KNOWN_S3_LOCATIONS)
locations.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
var defaults = S3AwsClient.GetDefaultAmazonS3Config();
var exts =
typeof(Amazon.S3.AmazonS3Config).GetProperties().Where(x => x.CanRead && x.CanWrite && (x.PropertyType == typeof(string) || x.PropertyType == typeof(bool) || x.PropertyType == typeof(int) || x.PropertyType == typeof(long) || x.PropertyType.IsEnum))
.Select(x => (ICommandLineArgument)new CommandLineArgument(
"s3-ext-" + x.Name.ToLowerInvariant(),
x.PropertyType == typeof(bool) ? CommandLineArgument.ArgumentType.Boolean : x.PropertyType.IsEnum ? CommandLineArgument.ArgumentType.Enumeration : CommandLineArgument.ArgumentType.String,
x.Name,
string.Format("Extended option {0}", x.Name),
string.Format("{0}", x.GetValue(defaults)),
null,
x.PropertyType.IsEnum ? Enum.GetNames(x.PropertyType) : null));
var exts = S3AwsClient.GetAwsExtendedOptions();
var normal = new ICommandLineArgument[] {
new CommandLineArgument("aws-secret-access-key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong,null, new string[] {"auth-password"}, null ),
Duplicati/Library/RestAPI/Serializable/ServerSettings.cs
+7 -12
View File
@@ -43,8 +43,7 @@ namespace Duplicati.Server.Serializable
this.Key = backend.ProtocolKey;
this.Description = backend.Description;
this.DisplayName = backend.DisplayName;
if (backend.SupportedCommands != null)
this.Options = backend.SupportedCommands.ToArray();
this.Options = backend.SupportedCommands?.ToArray() ?? [];
}
/// <summary>
@@ -55,8 +54,7 @@ namespace Duplicati.Server.Serializable
this.Key = module.FilenameExtension;
this.Description = module.Description;
this.DisplayName = module.DisplayName;
if (module.SupportedCommands != null)
this.Options = module.SupportedCommands.ToArray();
this.Options = module.SupportedCommands?.ToArray() ?? [];
}
/// <summary>
@@ -67,8 +65,7 @@ namespace Duplicati.Server.Serializable
this.Key = module.FilenameExtension;
this.Description = module.Description;
this.DisplayName = module.DisplayName;
if (module.SupportedCommands != null)
this.Options = module.SupportedCommands.ToArray();
this.Options = module.SupportedCommands?.ToArray() ?? [];
}
/// <summary>
@@ -79,8 +76,7 @@ namespace Duplicati.Server.Serializable
this.Key = module.Key;
this.Description = module.Description;
this.DisplayName = module.DisplayName;
if (module.SupportedCommands != null)
this.Options = module.SupportedCommands.ToArray();
this.Options = module.SupportedCommands?.ToArray() ?? [];
}
/// <summary>
@@ -91,8 +87,7 @@ namespace Duplicati.Server.Serializable
this.Key = module.Key;
this.Description = module.Description;
this.DisplayName = module.DisplayName;
if (module.SupportedCommands != null)
this.Options = module.SupportedCommands.ToArray();
this.Options = module.SupportedCommands?.ToArray() ?? [];
}
/// <summary>
@@ -103,8 +98,7 @@ namespace Duplicati.Server.Serializable
this.Key = module.Key;
this.Description = module.Description;
this.DisplayName = module.DisplayName;
if (module.SupportedCommands != null)
this.Options = module.SupportedCommands.ToArray();
this.Options = module.SupportedCommands?.ToArray() ?? [];
}
/// <summary>
/// The module key
@@ -143,6 +137,7 @@ namespace Duplicati.Server.Serializable
{
get
{
Console.WriteLine("Inside getter");
return
(from n in Library.DynamicLoader.BackendLoader.Backends
select new DynamicModule(n))
Duplicati/Library/SecretProvider/AWSSecretProvider.cs
+1
View File
@@ -67,6 +67,7 @@ public class AWSSecretProvider : ISecretProvider
/// <summary>
/// List of properties that slow down the loading of the AWS Secrets Manager client
/// </summary>
/// <remarks>Changes in this list will likely need to be reflected in S3AwsClient.cs</remarks>
private static readonly HashSet<string> SlowLoadingProperties = new[] {
nameof(AmazonSecretsManagerConfig.RegionEndpoint),
nameof(AmazonSecretsManagerConfig.ServiceURL),