haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added S3 storage class options to allow the user to specify the InfrequentAccess (IA) storage model.

Browse Source
This commit is contained in:
Kenneth Skovhede
2015-11-18 20:38:27 +01:00
parent a51399145b
commit 5fa7ebaef3
5 changed files with 150 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Backend/S3/S3Backend.cs
+52 -3
View File
@@ -18,6 +18,7 @@
//
#endregion
using System;
using System.Linq;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
@@ -28,6 +29,7 @@ namespace Duplicati.Library.Backend
public class S3 : IBackend, IStreamingBackend, IRenameEnabledBackend
{
public const string RRS_OPTION = "s3-use-rrs";
public const string STORAGECLASS_OPTION = "s3-storage-class";
public const string EU_BUCKETS_OPTION = "s3-european-buckets";
public const string SERVER_NAME = "s3-server-name";
public const string LOCATION_OPTION = "s3-location-constraint";
@@ -65,14 +67,55 @@ namespace Duplicati.Library.Backend
new KeyValuePair<string, string>("ap-southeast-2", "s3-ap-southeast-2.amazonaws.com"),
new KeyValuePair<string, string>("ap-northeast-1", "s3-ap-northeast-1.amazonaws.com"),
new KeyValuePair<string, string>("sa-east-1", "s3-sa-east-1.amazonaws.com"),
};
public static readonly KeyValuePair<string, string>[] KNOWN_S3_STORAGE_CLASSES;
static S3() {
var ns = new List<KeyValuePair<string, string>> {
new KeyValuePair<string, string>("Standard", "STANDARD"),
new KeyValuePair<string, string>("Infrequent Access (IA)", "STANDARD_IA"),
new KeyValuePair<string, string>("Glacier", "GLACIER"),
new KeyValuePair<string, string>("Reduced Redundancy Storage (RRS)", "REDUCED_REDUNDANCY"),
};
try
{
foreach(var sc in ReadStorageClasses())
if (!ns.Select(x => x.Value).Contains(sc.Value))
ns.Add(sc);
}
catch
{
}
KNOWN_S3_STORAGE_CLASSES = ns.ToArray();
}
/// <summary>
/// Fetch storage classes from the API through reflection so we are always updated
/// </summary>
/// <returns>The storage classes.</returns>
private static IEnumerable<KeyValuePair<string, string>> ReadStorageClasses()
{
foreach(var f in typeof(Amazon.S3.S3StorageClass).GetFields(System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Public))
{
if (f.FieldType == typeof(Amazon.S3.S3StorageClass))
{
var name = new Regex("([a-z])([A-Z])").Replace(f.Name, "$1 $2");
var prop = f.GetValue(null) as Amazon.S3.S3StorageClass;
if (prop != null && prop.Value != null)
yield return new KeyValuePair<string, string>(name, prop.Value);
}
}
}
private string m_bucket;
private string m_prefix;
public const string DEFAULT_S3_HOST = "s3.amazonaws.com";
public const string S3_EU_REGION_NAME = "eu-west-1";
public const string S3_RRS_CLASS_NAME = "REDUCED_REDUNDANCY";
private Dictionary<string, string> m_options;
@@ -127,6 +170,11 @@ namespace Duplicati.Library.Backend
if (euBuckets)
locationConstraint = S3_EU_REGION_NAME;
string storageClass;
options.TryGetValue(STORAGECLASS_OPTION, out storageClass);
if (string.IsNullOrWhiteSpace(storageClass) && useRRS)
storageClass = S3_RRS_CLASS_NAME;
string s3host;
options.TryGetValue(SERVER_NAME, out s3host);
if (string.IsNullOrEmpty(s3host))
@@ -194,7 +242,7 @@ namespace Duplicati.Library.Backend
if (m_prefix.Length != 0 && !m_prefix.EndsWith("/"))
m_prefix += "/";
m_wrapper = new S3Wrapper(awsID, awsKey, locationConstraint, host, useRRS, useSSL);
m_wrapper = new S3Wrapper(awsID, awsKey, locationConstraint, host, storageClass, useSSL);
}
public static bool IsValidHostname(string bucketname)
@@ -322,7 +370,8 @@ namespace Duplicati.Library.Backend
new CommandLineArgument("aws_secret_access_key", CommandLineArgument.ArgumentType.Password, Strings.S3Backend.AMZKeyDescriptionShort, Strings.S3Backend.AMZKeyDescriptionLong, null, new string[] {"auth-password"}, null),
new CommandLineArgument("aws_access_key_id", CommandLineArgument.ArgumentType.String, Strings.S3Backend.AMZUserIDDescriptionShort, Strings.S3Backend.AMZUserIDDescriptionLong, null, new string[] {"auth-username"}, null),
new CommandLineArgument(EU_BUCKETS_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.S3EurobucketDescriptionShort, Strings.S3Backend.S3EurobucketDescriptionLong, "false", null, null, Strings.S3Backend.S3EurobucketDeprecationDescription(LOCATION_OPTION, S3_EU_REGION_NAME)),
new CommandLineArgument(RRS_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.S3UseRRSDescriptionShort, Strings.S3Backend.S3UseRRSDescriptionLong, "false"),
new CommandLineArgument(RRS_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.S3UseRRSDescriptionShort, Strings.S3Backend.S3UseRRSDescriptionLong, "false", null, null, Strings.S3Backend.S3RRSDeprecationDescription(STORAGECLASS_OPTION, S3_RRS_CLASS_NAME)),
new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3StorageclassDescriptionShort, Strings.S3Backend.S3StorageclassDescriptionLong),
new CommandLineArgument(SERVER_NAME, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3ServerNameDescriptionShort, Strings.S3Backend.S3ServerNameDescriptionLong(hostnames.ToString()), DEFAULT_S3_HOST),
new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.S3Backend.S3LocationDescriptionShort, Strings.S3Backend.S3LocationDescriptionLong(locations.ToString())),
new CommandLineArgument(SSL_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.S3Backend.DescriptionUseSSLShort, Strings.S3Backend.DescriptionUseSSLLong),
Duplicati/Library/Backend/S3/S3Config.cs
+4 -1
View File
@@ -33,7 +33,8 @@ namespace Duplicati.Library.Backend
{
Providers,
Regions,
RegionHosts
RegionHosts,
StorageClasses
}
public S3Config()
@@ -59,6 +60,8 @@ namespace Duplicati.Library.Backend
return S3.DEFAULT_S3_LOCATION_BASED_HOSTS.ToDictionary((x) => x.Key, (y) => y.Value);
case ConfigType.Regions:
return S3.KNOWN_S3_LOCATIONS.ToDictionary((x) => x.Key, (y) => y.Value);
case ConfigType.StorageClasses:
return S3.KNOWN_S3_STORAGE_CLASSES.ToDictionary((x) => x.Key, (y) => y.Value);
default:
return S3.KNOWN_S3_PROVIDERS.ToDictionary((x) => x.Key, (y) => y.Value);
}
Duplicati/Library/Backend/S3/S3Wrapper.cs
+5 -4
View File
@@ -35,10 +35,10 @@ namespace Duplicati.Library.Backend
private const int ITEM_LIST_LIMIT = 1000;
protected string m_locationConstraint;
protected bool m_useRRS;
protected string m_storageClass;
protected AmazonS3Client m_client;
public S3Wrapper(string awsID, string awsKey, string locationConstraint, string servername, bool useRRS, bool useSSL)
public S3Wrapper(string awsID, string awsKey, string locationConstraint, string servername, string storageClass, bool useSSL)
{
AmazonS3Config cfg = new AmazonS3Config();
@@ -50,7 +50,7 @@ namespace Duplicati.Library.Backend
m_client = new Amazon.S3.AmazonS3Client(awsID, awsKey, cfg);
m_locationConstraint = locationConstraint;
m_useRRS = useRRS;
m_storageClass = storageClass;
}
public void AddBucket(string bucketName)
@@ -98,7 +98,8 @@ namespace Duplicati.Library.Backend
objectAddRequest.BucketName = bucketName;
objectAddRequest.Key = keyName;
objectAddRequest.InputStream = source;
objectAddRequest.StorageClass = m_useRRS ? S3StorageClass.ReducedRedundancy : S3StorageClass.Standard;
if (!string.IsNullOrWhiteSpace(m_storageClass))
objectAddRequest.StorageClass = new S3StorageClass(m_storageClass);
m_client.PutObject(objectAddRequest);
}
Duplicati/Library/Backend/S3/Strings.cs
+40 -36
View File
@@ -1,36 +1,40 @@
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings {
internal static class S3Backend {
public static string AMZKeyDescriptionLong { get { return LC.L(@"The AWS ""Secret Access Key"" can be obtained after logging into your AWS account, this can also be supplied through the ""auth-password"" property"); } }
public static string AMZKeyDescriptionShort { get { return LC.L(@"The AWS ""Secret Access Key"""); } }
public static string AMZUserIDDescriptionLong { get { return LC.L(@"The AWS ""Access Key ID"" can be obtained after logging into your AWS account, this can also be supplied through the ""auth-username"" property"); } }
public static string AMZUserIDDescriptionShort { get { return LC.L(@"The AWS ""Access Key ID"""); } }
public static string DisplayName { get { return LC.L(@"Amazon S3"); } }
public static string AuthPasswordDescriptionLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string AuthPasswordDescriptionShort { get { return LC.L(@"Supplies the password used to connect to the server"); } }
public static string AuthUsernameDescriptionLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string AuthUsernameDescriptionShort { get { return LC.L(@"Supplies the username used to connect to the server"); } }
public static string NoAMZKeyError { get { return LC.L(@"No Amazon S3 secret key given"); } }
public static string NoAMZUserIDError { get { return LC.L(@"No Amazon S3 userID given"); } }
public static string S3EurobucketDescriptionLong { get { return LC.L(@"This flag is only used when creating new buckets. If the flag is set, the bucket is created on a european server. This flag forces the ""s3-use-new-style"" flag. Amazon charges slightly more for european buckets."); } }
public static string S3EurobucketDescriptionShort { get { return LC.L(@"Use a European server"); } }
public static string S3NewStyleDescriptionLong { get { return LC.L(@"Specify this argument to make the S3 backend use subdomains rather than the previous url prefix method. See the Amazon S3 documentation for more details."); } }
public static string S3NewStyleDescriptionShort { get { return LC.L(@"Use subdomain calling style"); } }
public static string UnableToDecodeBucketnameError(string url) { return LC.L(@"Unable to determine the bucket name for host: {0}", url); }
public static string S3UseRRSDescriptionLong { get { return LC.L(@"This flag toggles the use of the special RRS header. Files stored using RRS are more likely to disapear than those stored normally, but also costs less to store. See the full description here: http://aws.amazon.com/about-aws/whats-new/2010/05/19/announcing-amazon-s3-reduced-redundancy-storage/"); } }
public static string S3UseRRSDescriptionShort { get { return LC.L(@"Use Reduced Redundancy Storage"); } }
public static string DeprecatedUrlFormat(string url) { return LC.L(@"You are using a deprected url format, please change it to: {0}", url); }
public static string Description_v2 { get { return LC.L(@"This backend can read and write data to an Amazon S3 compatible server. Allowed formats are: ""s3://bucketname/prefix"""); } }
public static string OptionsAreMutuallyExclusiveError(string option1, string option2) { return LC.L(@"The options --{0} and --{1} are mutually exclusive", option1, option2); }
public static string S3EurobucketDeprecationDescription(string optionname, string optionvalue) { return LC.L(@"Please use --{0}={1} instead", optionname, optionvalue); }
public static string S3LocationDescriptionLong(string regions) { return LC.L(@"This option is only used when creating new buckets. Use this option to change what region the data is stored in. Amazon charges slightly more for non-US buckets. Known bucket locations:
{0}", regions); }
public static string S3LocationDescriptionShort { get { return LC.L(@"Specifies S3 location constraints"); } }
public static string S3ServerNameDescriptionLong(string providers) { return LC.L(@"Companies other than Amazon are now supporting the S3 API, meaning that this backend can read and write data to those providers as well. Use this option to set the hostname. Currently known providers are:
{0}", providers); }
public static string S3ServerNameDescriptionShort { get { return LC.L(@"Specifies an alternate S3 server name"); } }
public static string S3NewStyleDeprecation { get { return LC.L(@"The subdomain calling option does nothing, the library will pick the right calling convention"); } }
public static string DescriptionUseSSLLong { get { return LC.L(@"Use this flag to communicate using Secure Socket Layer (SSL) over http (https). Note that bucket names containing a period has problems with SSL connections."); } }
public static string DescriptionUseSSLShort { get { return LC.L(@"Instructs Duplicati to use an SSL (https) connection"); } }
}
}
using Duplicati.Library.Localization.Short;
namespace Duplicati.Library.Backend.Strings {
internal static class S3Backend {
public static string AMZKeyDescriptionLong { get { return LC.L(@"The AWS ""Secret Access Key"" can be obtained after logging into your AWS account, this can also be supplied through the ""auth-password"" property"); } }
public static string AMZKeyDescriptionShort { get { return LC.L(@"The AWS ""Secret Access Key"""); } }
public static string AMZUserIDDescriptionLong { get { return LC.L(@"The AWS ""Access Key ID"" can be obtained after logging into your AWS account, this can also be supplied through the ""auth-username"" property"); } }
public static string AMZUserIDDescriptionShort { get { return LC.L(@"The AWS ""Access Key ID"""); } }
public static string DisplayName { get { return LC.L(@"Amazon S3"); } }
public static string AuthPasswordDescriptionLong { get { return LC.L(@"The password used to connect to the server. This may also be supplied as the environment variable ""AUTH_PASSWORD""."); } }
public static string AuthPasswordDescriptionShort { get { return LC.L(@"Supplies the password used to connect to the server"); } }
public static string AuthUsernameDescriptionLong { get { return LC.L(@"The username used to connect to the server. This may also be supplied as the environment variable ""AUTH_USERNAME""."); } }
public static string AuthUsernameDescriptionShort { get { return LC.L(@"Supplies the username used to connect to the server"); } }
public static string NoAMZKeyError { get { return LC.L(@"No Amazon S3 secret key given"); } }
public static string NoAMZUserIDError { get { return LC.L(@"No Amazon S3 userID given"); } }
public static string S3EurobucketDescriptionLong { get { return LC.L(@"This flag is only used when creating new buckets. If the flag is set, the bucket is created on a european server. This flag forces the ""s3-use-new-style"" flag. Amazon charges slightly more for european buckets."); } }
public static string S3EurobucketDescriptionShort { get { return LC.L(@"Use a European server"); } }
public static string S3NewStyleDescriptionLong { get { return LC.L(@"Specify this argument to make the S3 backend use subdomains rather than the previous url prefix method. See the Amazon S3 documentation for more details."); } }
public static string S3NewStyleDescriptionShort { get { return LC.L(@"Use subdomain calling style"); } }
public static string UnableToDecodeBucketnameError(string url) { return LC.L(@"Unable to determine the bucket name for host: {0}", url); }
public static string S3UseRRSDescriptionLong { get { return LC.L(@"This flag toggles the use of the special RRS header. Files stored using RRS are more likely to disapear than those stored normally, but also costs less to store. See the full description here: http://aws.amazon.com/about-aws/whats-new/2010/05/19/announcing-amazon-s3-reduced-redundancy-storage/"); } }
public static string S3UseRRSDescriptionShort { get { return LC.L(@"Use Reduced Redundancy Storage"); } }
public static string DeprecatedUrlFormat(string url) { return LC.L(@"You are using a deprected url format, please change it to: {0}", url); }
public static string Description_v2 { get { return LC.L(@"This backend can read and write data to an Amazon S3 compatible server. Allowed formats are: ""s3://bucketname/prefix"""); } }
public static string OptionsAreMutuallyExclusiveError(string option1, string option2) { return LC.L(@"The options --{0} and --{1} are mutually exclusive", option1, option2); }
public static string S3EurobucketDeprecationDescription(string optionname, string optionvalue) { return LC.L(@"Please use --{0}={1} instead", optionname, optionvalue); }
public static string S3LocationDescriptionLong(string regions) { return LC.L(@"This option is only used when creating new buckets. Use this option to change what region the data is stored in. Amazon charges slightly more for non-US buckets. Known bucket locations:
{0}", regions); }
public static string S3LocationDescriptionShort { get { return LC.L(@"Specifies S3 location constraints"); } }
public static string S3ServerNameDescriptionLong(string providers) { return LC.L(@"Companies other than Amazon are now supporting the S3 API, meaning that this backend can read and write data to those providers as well. Use this option to set the hostname. Currently known providers are:
{0}", providers); }
public static string S3ServerNameDescriptionShort { get { return LC.L(@"Specifies an alternate S3 server name"); } }
public static string S3NewStyleDeprecation { get { return LC.L(@"The subdomain calling option does nothing, the library will pick the right calling convention"); } }
public static string DescriptionUseSSLLong { get { return LC.L(@"Use this flag to communicate using Secure Socket Layer (SSL) over http (https). Note that bucket names containing a period has problems with SSL connections."); } }
public static string DescriptionUseSSLShort { get { return LC.L(@"Instructs Duplicati to use an SSL (https) connection"); } }
public static string S3StorageclassDescriptionLong { get { return LC.L(@"Use this option to specify a storage class. If this option is not used, the server will choose a default storage class."); } }
public static string S3StorageclassDescriptionShort { get { return LC.L(@"Specify storage class"); } }
public static string S3RRSDeprecationDescription(string optionname, string optionvalue) { return LC.L(@"Please use --{0}={1} instead", optionname, optionvalue); }
}
}
Duplicati/Server/webroot/greeno/scripts/plugins.js
+49 -3
View File
@@ -237,9 +237,12 @@ $(document).ready(function() {
passwordwatermark: 'AWS Secret Key',
serverdrop_field: null,
regiondrop_field: null,
storageclassdrop_field: null,
bucket_field: null,
known_hosts: null,
known_regions: null,
known_storage_classes: null,
setup_hosts_after_config: function() {
if (this.known_hosts != null && this.serverdrop_field != null)
@@ -276,6 +279,23 @@ $(document).ready(function() {
}
},
setup_storage_classes_after_config: function() {
if (this.known_storage_classes != null && this.storageclassdrop_field != null) {
var buckets = [];
for (var k in this.known_storage_classes)
buckets.push({label: k + ' (' + this.known_storage_classes[k] + ')', value: this.known_storage_classes[k]});
this.storageclassdrop_field.autocomplete({
minLength: 0,
source: buckets,
});
var self = this;
this.storageclassdrop_field.click(function() {
self.storageclassdrop_field.autocomplete('search', '');
});
}
},
setup: function(dlg, div) {
var self = this;
@@ -285,7 +305,7 @@ $(document).ready(function() {
var serverdrop = EDIT_URI.createFieldset({label: 'S3 servername', name: 's3-server', after: $('#server-path'), watermark: 'Click for a list of providers'});
var bucketfield = EDIT_URI.createFieldset({label: 'S3 Bucket name', name: 's3-bucket', after: $('#server-username-and-password'), title: 'Use / to access subfolders in the bucket', watermark: 'Enter bucket name'});
var regiondrop = EDIT_URI.createFieldset({label: 'Bucket create region', name: 's3-region', before: $('#server-options-label'), watermark: 'Click for a list of regions', title: 'Note that region is only used when creating buckets'});
var rrscheck = EDIT_URI.createFieldset({'label': 'Use RRS', name: 's3-rrs', type: 'checkbox', before: $('#server-options-label'), title: 'Reduced Redundancy Storage is cheaper, but less reliable'});
var storageclassdrop = EDIT_URI.createFieldset({label: 'Storage class', name: 's3-storage-class', after: regiondrop.outer, watermark: 'Click for a list of storage classes'});
var signuplink = EDIT_URI.createFieldset({'label': '&nbsp;', href: this.PLUGIN_S3_LINK, type: 'link', before: bucketfield.outer, 'title': 'Click here for the sign up page'});
signuplink.outer.css('margin-bottom', '10px');
@@ -293,6 +313,8 @@ $(document).ready(function() {
this.serverdrop_field = serverdrop.field;
this.regiondrop_field = regiondrop.field;
this.bucket_field = bucketfield.field;
this.storageclassdrop_field = storageclassdrop.field;
if (self.known_hosts == null) {
APP_DATA.callServer({action: 'send-command', command: 's3-getconfig', 's3-config': 'Providers'}, function(data) {
@@ -317,6 +339,19 @@ $(document).ready(function() {
} else {
this.setup_regions_after_config();
}
if (self.known_storage_classes == null) {
APP_DATA.callServer({action: 'send-command', command: 's3-getconfig', 's3-config': 'StorageClasses'}, function(data) {
self.known_storage_classes = data.Result;
self.setup_storage_classes_after_config();
},
function(data, success, message) {
alert('Failed to get S3 config: ' + message);
});
} else {
this.setup_storage_classes_after_config();
}
},
cleanup: function(dlg, div) {
@@ -324,9 +359,20 @@ $(document).ready(function() {
$('#server-path').show();
this.serverdrop_field = null;
this.regiondrop_field = null;
this.storageclassdrop_field = null;
this.bucket_field = null;
},
fill_form: function(el, cfg) {
// Upgrade from checkbox to string
if (APP_UTIL.parseBoolOption(cfg['--s3-use-rrs']) && !cfg['--s3-location-constraint']) {
delete cfg['--s3-use-rrs'];
cfg['--s3-location-constraint'] = 'REDUCED_REDUNDANCY';
}
EDIT_URI.fill_form(el, cfg);
},
validate: function(dlg, values) {
if (!EDIT_URI.validate_input(values, true))
return;
@@ -356,14 +402,14 @@ $(document).ready(function() {
fill_form_map: {
'server-path': 's3-bucket',
'--s3-server-name': 's3-server',
'--s3-use-rrs': 's3-rrs',
'--s3-storage-class': 's3-storage-class',
'--s3-location-constraint': 's3-region'
},
fill_dict_map: {
's3-bucket': 'server-path',
's3-server': '--s3-server-name',
's3-rrs': '--s3-use-rrs',
's3-storage-class': '--s3-storage-class',
's3-region': '--s3-location-constraint'
}
}