From 474e15e97525528c80dee237efda22c9f3d34a7e Mon Sep 17 00:00:00 2001 From: "kenneth.skovhede@gmail.com" Date: Thu, 13 May 2010 18:21:42 +0000 Subject: [PATCH] Fixed issue #127. I am unable to reproduce the error with the "test connection" that you describe. Was it with a particular bucket/folder name? I have also added some logic to ensure that it does not repeatedly ask to prefix the username. git-svn-id: https://duplicati.googlecode.com/svn/trunk@406 59da171f-624f-0410-aa54-27559c288bec --- Duplicati/Library/Backend/S3/S3Backend.cs | 30 ++++++++++--------- Duplicati/Library/Backend/S3/S3UI.cs | 24 ++++++++++++--- Duplicati/Library/Backend/S3/S3Wrapper.cs | 5 ++-- .../Backend/S3/Strings/S3Backend.Designer.cs | 7 +++-- .../Library/Backend/S3/Strings/S3Backend.resx | 5 +++- 5 files changed, 48 insertions(+), 23 deletions(-) diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 79e214e25..7af2c3c58 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -35,6 +35,8 @@ namespace Duplicati.Library.Backend private Affirma.ThreeSharp.CallingFormat m_format; private bool m_euBuckets; + private const string S3_HOST = "s3.amazonaws.com"; + Dictionary m_options; public S3() @@ -86,9 +88,12 @@ namespace Duplicati.Library.Backend m_format = options.ContainsKey("s3-use-new-style") ? Affirma.ThreeSharp.CallingFormat.SUBDOMAIN : Affirma.ThreeSharp.CallingFormat.REGULAR; m_euBuckets = options.ContainsKey("s3-european-buckets"); - if (m_host.ToLower() == "s3.amazonaws.com") + if (m_host.ToLower() == S3_HOST) { - m_bucket = u.PathAndQuery; + m_bucket = System.Web.HttpUtility.UrlDecode(u.PathAndQuery); + + if (m_bucket.StartsWith("/")) + m_bucket = m_bucket.Substring(1); if (m_bucket.Contains("/")) { @@ -99,27 +104,25 @@ namespace Duplicati.Library.Backend } else { - //Vanity style, do a CNAME lookup - if (!m_host.ToLower().EndsWith(".s3.amazonaws.com")) + //If it is vanity style, do a CNAME lookup + if (!m_host.ToLower().EndsWith("." + S3_HOST)) { System.Net.IPHostEntry ent = System.Net.Dns.GetHostEntry(m_host); - List entries = new List(); - entries.AddRange(ent.Aliases); - - foreach (string s in entries) - if (s.ToLower().EndsWith(".s3.amazonaws.com")) + foreach (string s in ent.Aliases) + if (s.EndsWith("." + S3_HOST, StringComparison.InvariantCultureIgnoreCase)) { m_host = s; break; } } - if (m_host.ToLower().EndsWith(".s3.amazonaws.com")) + //Subdomain type lookup + if (m_host.ToLower().EndsWith("." + S3_HOST)) { m_format = Affirma.ThreeSharp.CallingFormat.SUBDOMAIN; - m_bucket = m_host.Substring(0, m_host.Length - ".s3.amazonaws.com".Length); - m_host = "s3.amazonaws.com"; - m_prefix = u.PathAndQuery; + m_bucket = m_host.Substring(0, m_host.Length - ("." + S3_HOST).Length); + m_host = S3_HOST; + m_prefix = System.Web.HttpUtility.UrlDecode(u.PathAndQuery); if (m_prefix.StartsWith("/")) m_prefix = m_prefix.Substring(1); @@ -139,7 +142,6 @@ namespace Duplicati.Library.Backend private List List(bool isTesting) { - try { S3Wrapper con = CreateRequest(); diff --git a/Duplicati/Library/Backend/S3/S3UI.cs b/Duplicati/Library/Backend/S3/S3UI.cs index b140389ed..56ba0e74c 100644 --- a/Duplicati/Library/Backend/S3/S3UI.cs +++ b/Duplicati/Library/Backend/S3/S3UI.cs @@ -38,6 +38,8 @@ namespace Duplicati.Library.Backend private const string PREFIX = "Prefix"; private const string HASTESTED = "UI: HasTested"; private const string HASCREATEDBUCKET = "UI: Has created bucket"; + private const string HASSUGGESTEDPREFIX = "UI: Has suggested prefix"; + private const string HASSUGGESTEDLOWERCASE = "UI: Has suggested lowercase"; private const string S3_PATH = "s3.amazonaws.com"; @@ -46,6 +48,8 @@ namespace Duplicati.Library.Backend private IDictionary m_options; private bool m_hasTested; private bool m_hasCreatedbucket = false; + private bool m_hasSuggestedPrefix = false; + private bool m_hasSuggestedLowerCase = false; public S3UI(IDictionary options) : this() @@ -91,6 +95,8 @@ namespace Duplicati.Library.Backend m_options.Clear(); m_options[HASTESTED] = m_hasTested.ToString(); m_options[HASCREATEDBUCKET] = m_hasCreatedbucket.ToString(); + m_options[HASSUGGESTEDPREFIX] = m_hasSuggestedPrefix.ToString(); + m_options[HASSUGGESTEDLOWERCASE] = m_hasSuggestedLowerCase.ToString(); m_options[ACCESS_ID] = AWS_ID.Text; m_options[ACCESS_KEY] = AWS_KEY.Text; @@ -123,6 +129,12 @@ namespace Duplicati.Library.Backend if (!m_options.ContainsKey(HASCREATEDBUCKET) || !bool.TryParse(m_options[HASCREATEDBUCKET], out m_hasCreatedbucket)) m_hasCreatedbucket = false; + + if (!m_options.ContainsKey(HASSUGGESTEDPREFIX) || !bool.TryParse(m_options[HASSUGGESTEDPREFIX], out m_hasSuggestedPrefix)) + m_hasSuggestedPrefix = false; + + if (!m_options.ContainsKey(HASSUGGESTEDLOWERCASE) || !bool.TryParse(m_options[HASSUGGESTEDLOWERCASE], out m_hasSuggestedLowerCase)) + m_hasSuggestedLowerCase = false; } /// @@ -182,15 +194,15 @@ namespace Duplicati.Library.Backend return false; } - if (!BucketName.Text.ToLower().StartsWith(AWS_ID.Text.ToLower())) + if (!m_hasSuggestedPrefix && !BucketName.Text.ToLower().StartsWith(AWS_ID.Text.ToLower())) { DialogResult res = MessageBox.Show(this, Strings.S3UI.BucketnameNotPrefixedWarning, Application.ProductName, MessageBoxButtons.YesNoCancel, MessageBoxIcon.Warning); if (res == DialogResult.Yes) BucketName.Text = AWS_ID.Text.ToLower() + "-" + BucketName.Text; + else if (res == DialogResult.No) + m_hasSuggestedPrefix = true; else if (res == DialogResult.Cancel) - { return false; - } } string bucketname; @@ -207,7 +219,7 @@ namespace Duplicati.Library.Backend prefix = null; } - if (bucketname.ToLower() != bucketname) + if (!m_hasSuggestedLowerCase && bucketname.ToLower() != bucketname) { string reason = UseEuroBuckets.Checked ? Strings.S3UI.EuroBucketsRequireLowerCaseError : @@ -223,6 +235,8 @@ namespace Duplicati.Library.Backend { return false; } + else if (res == DialogResult.No) + m_hasSuggestedLowerCase = true; } if (checkForBucket) @@ -293,6 +307,8 @@ namespace Duplicati.Library.Backend { m_hasTested = false; m_hasCreatedbucket = false; + m_hasSuggestedPrefix = false; + m_hasSuggestedLowerCase = false; } private void UseEuroBuckets_CheckedChanged(object sender, EventArgs e) diff --git a/Duplicati/Library/Backend/S3/S3Wrapper.cs b/Duplicati/Library/Backend/S3/S3Wrapper.cs index f54455b46..dac538037 100644 --- a/Duplicati/Library/Backend/S3/S3Wrapper.cs +++ b/Duplicati/Library/Backend/S3/S3Wrapper.cs @@ -33,6 +33,7 @@ namespace Duplicati.Library.Backend /// public class S3Wrapper { + private const int ITEM_LIST_LIMIT = 1000; private static Dictionary> RedirectCache = new Dictionary>(); protected const string EU_LOCATION_CONSTRAINT = "EU"; @@ -138,7 +139,7 @@ namespace Duplicati.Library.Backend string redirUrl = GetRedirectUrl(bucketName, null); List files = new List(); - //We truncate after 1000 elements, and then repeat + //We truncate after ITEM_LIST_LIMIT elements, and then repeat while (isTruncated) { //Due to a resource leak in the S3 code, we flush it here @@ -149,7 +150,7 @@ namespace Duplicati.Library.Backend if (!string.IsNullOrEmpty(filename)) listRequest.QueryList.Add("marker", filename); - listRequest.QueryList.Add("max-keys", "1000"); + listRequest.QueryList.Add("max-keys", ITEM_LIST_LIMIT.ToString()); if (!string.IsNullOrEmpty(prefix)) listRequest.QueryList.Add("prefix", prefix); diff --git a/Duplicati/Library/Backend/S3/Strings/S3Backend.Designer.cs b/Duplicati/Library/Backend/S3/Strings/S3Backend.Designer.cs index 49aafdffc..5912a1a23 100644 --- a/Duplicati/Library/Backend/S3/Strings/S3Backend.Designer.cs +++ b/Duplicati/Library/Backend/S3/Strings/S3Backend.Designer.cs @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ // // This code was generated by a tool. -// Runtime Version:2.0.50727.3053 +// Runtime Version:2.0.50727.4927 // // Changes to this file may cause incorrect behavior and will be lost if // the code is regenerated. @@ -97,7 +97,10 @@ namespace Duplicati.Library.Backend.Strings { } /// - /// Looks up a localized string similar to This backend can read and write data to an Amazon S3 based backend. Allowed formats are "s3://bucketname/prefix" or "s3://aws_id:aws_key@bucketname/prefix". Note that if you AWS ID or Key contains either ":", "/" or "@" you cannot use the second form, but must supply them through arguments.. + /// Looks up a localized string similar to This backend can read and write data to an Amazon S3 based backend. Allowed formats are (subdomain type): "s3://bucketname.s3.amazonaws.com/prefix" or "s3://aws_id:aws_key@bucketname.s3.amazonaws.com/prefix". + ///The url suffix model is also accepted: "s3://s3.amazonaws.com/bucketname/prefix". + ///Any hostname not using s3.amazonaws.com is considered a vanity style, and the bucket name is attempted extracted through CNAME lookup. + ///Note that if you AWS ID or Key contains either ":", "/" or "@" you cannot supply t [rest of string was truncated]";. /// internal static string Description { get { diff --git a/Duplicati/Library/Backend/S3/Strings/S3Backend.resx b/Duplicati/Library/Backend/S3/Strings/S3Backend.resx index 9b9abe025..43683c8ce 100644 --- a/Duplicati/Library/Backend/S3/Strings/S3Backend.resx +++ b/Duplicati/Library/Backend/S3/Strings/S3Backend.resx @@ -130,7 +130,10 @@ The AWS "Access Key ID" - This backend can read and write data to an Amazon S3 based backend. Allowed formats are "s3://bucketname/prefix" or "s3://aws_id:aws_key@bucketname/prefix". Note that if you AWS ID or Key contains either ":", "/" or "@" you cannot use the second form, but must supply them through arguments. + This backend can read and write data to an Amazon S3 based backend. Allowed formats are (subdomain type): "s3://bucketname.s3.amazonaws.com/prefix" or "s3://aws_id:aws_key@bucketname.s3.amazonaws.com/prefix". +The url suffix model is also accepted: "s3://s3.amazonaws.com/bucketname/prefix". +Any hostname not using s3.amazonaws.com is considered a vanity style, and the bucket name is attempted extracted through CNAME lookup. +Note that if you AWS ID or Key contains either ":", "/" or "@" you cannot supply them in the url, but must pass them through arguments. Amazon S3 based