Tyler Gill
38883285ee
By changing to IEnumerable, it is possible to iterate only a portion of the list, which is useful when not all entries are needed (e.g., when testing a connection). All existing backends have been updated, and any which were able to be changed to yield return results in a straightforward way now do. Many backends had a try/catch in the List() method. Due to the fact that yield returns can't be placed within a try/catch block, these have been refactored to either scope the try/catch to the parts that (should) be the only places throwing exceptions, so that exceptions are still caught and handled. Note that lazy evaluation may cause some changes in behavior - exceptions that were previously thrown at the point of invokation of List() may now be thrown while it is being enumerated. I believe this will not be problematic though, as the only well-known exception seems to be FolderMissingException, which should be thrown by Test(), but TestList() attempts to enumerate the list to force this exception. Any places that require the legacy behavior can get it by simply converting the lazy enumerable to a List()
433 lines
17 KiB
C#
433 lines
17 KiB
C#
// Copyright (C) 2015, The Duplicati Team
|
|
// http://www.duplicati.com, info@duplicati.com
|
|
//
|
|
// This library is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU Lesser General Public License as
|
|
// published by the Free Software Foundation; either version 2.1 of the
|
|
// License, or (at your option) any later version.
|
|
//
|
|
// This library is distributed in the hope that it will be useful, but
|
|
// WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
// Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public
|
|
// License along with this library; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using Duplicati.Library.Interface;
|
|
using System.Linq;
|
|
using Duplicati.Library.Utility;
|
|
using Newtonsoft.Json;
|
|
using Duplicati.Library.Strings;
|
|
using System.Net;
|
|
using System.Text;
|
|
|
|
namespace Duplicati.Library.Backend.OpenStack
|
|
{
|
|
public class OpenStackStorage : IBackend, IStreamingBackend
|
|
{
|
|
private const string USERNAME_OPTION = "auth-username";
|
|
private const string PASSWORD_OPTION = "auth-password";
|
|
private const string TENANTNAME_OPTION = "openstack-tenant-name";
|
|
private const string AUTHURI_OPTION = "openstack-authuri";
|
|
private const string APIKEY_OPTION = "openstack-apikey";
|
|
private const string REGION_OPTION = "openstack-region";
|
|
|
|
private const int PAGE_LIMIT = 500;
|
|
|
|
|
|
private string m_container;
|
|
private string m_prefix;
|
|
|
|
private string m_username;
|
|
private string m_password;
|
|
private string m_authUri;
|
|
private string m_tenantName;
|
|
private string m_apikey;
|
|
private string m_region;
|
|
|
|
protected string m_simplestorageendpoint;
|
|
|
|
private WebHelper m_helper = null;
|
|
private OpenStackAuthResponse.TokenClass m_accessToken;
|
|
|
|
public static readonly KeyValuePair<string, string>[] KNOWN_OPENSTACK_PROVIDERS = new KeyValuePair<string,string>[] {
|
|
new KeyValuePair<string, string>("Rackspace US", "https://identity.api.rackspacecloud.com/v2.0"),
|
|
new KeyValuePair<string, string>("Rackspace UK", "https://lon.identity.api.rackspacecloud.com/v2.0"),
|
|
new KeyValuePair<string, string>("OVH Cloud Storage", "https://auth.cloud.ovh.net/v2.0"),
|
|
new KeyValuePair<string, string>("Selectel Cloud Storage", "https://auth.selcdn.ru"),
|
|
};
|
|
|
|
private class OpenStackAuthRequest
|
|
{
|
|
public class AuthContainer
|
|
{
|
|
[JsonProperty("RAX-KSKEY:apiKeyCredentials", NullValueHandling = NullValueHandling.Ignore)]
|
|
public ApiKeyBasedRequest ApiCredentials { get; set; }
|
|
|
|
[JsonProperty("passwordCredentials", NullValueHandling = NullValueHandling.Ignore)]
|
|
public PasswordBasedRequest PasswordCredentials { get; set; }
|
|
|
|
[JsonProperty("tenantName", NullValueHandling = NullValueHandling.Ignore)]
|
|
public string TenantName { get; set; }
|
|
|
|
[JsonProperty("token", NullValueHandling = NullValueHandling.Ignore)]
|
|
public TokenBasedRequest Token { get; set; }
|
|
|
|
}
|
|
|
|
public class ApiKeyBasedRequest
|
|
{
|
|
public string username { get; set; }
|
|
public string apiKey { get; set; }
|
|
}
|
|
|
|
public class PasswordBasedRequest
|
|
{
|
|
public string username { get; set; }
|
|
public string password { get; set; }
|
|
public string tenantName { get; set; }
|
|
}
|
|
|
|
public class TokenBasedRequest
|
|
{
|
|
public string id { get; set; }
|
|
}
|
|
|
|
|
|
public AuthContainer auth { get; set; }
|
|
|
|
public OpenStackAuthRequest(string tenantname, string username, string password, string apikey)
|
|
{
|
|
this.auth = new AuthContainer();
|
|
this.auth.TenantName = tenantname;
|
|
|
|
if (string.IsNullOrEmpty(apikey))
|
|
{
|
|
this.auth.PasswordCredentials = new PasswordBasedRequest() {
|
|
username = username,
|
|
password = password,
|
|
};
|
|
}
|
|
else
|
|
{
|
|
this.auth.ApiCredentials = new ApiKeyBasedRequest() {
|
|
username = username,
|
|
apiKey = apikey
|
|
};
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
private class OpenStackAuthResponse
|
|
{
|
|
public AccessClass access { get; set; }
|
|
|
|
public class TokenClass
|
|
{
|
|
public string id { get; set; }
|
|
public DateTime? expires { get; set; }
|
|
}
|
|
|
|
public class EndpointItem
|
|
{
|
|
public string region { get; set; }
|
|
public string tenantId { get; set; }
|
|
public string publicURL { get; set; }
|
|
public string internalURL { get; set; }
|
|
}
|
|
|
|
public class ServiceItem
|
|
{
|
|
public string name { get; set; }
|
|
public string type { get; set; }
|
|
public EndpointItem[] endpoints { get; set; }
|
|
}
|
|
|
|
public class AccessClass
|
|
{
|
|
public TokenClass token { get; set; }
|
|
public ServiceItem[] serviceCatalog { get; set; }
|
|
}
|
|
|
|
}
|
|
|
|
private class OpenStackStorageItem
|
|
{
|
|
public string name { get; set; }
|
|
public DateTime? last_modified { get; set; }
|
|
public long? bytes { get; set; }
|
|
public string content_type { get; set; }
|
|
public string subdir { get; set; }
|
|
}
|
|
|
|
private class WebHelper : JSONWebHelper
|
|
{
|
|
private OpenStackStorage m_parent;
|
|
|
|
public WebHelper(OpenStackStorage parent) { m_parent = parent; }
|
|
|
|
public override HttpWebRequest CreateRequest(string url, string method = null)
|
|
{
|
|
var req = base.CreateRequest(url, method);
|
|
req.Headers["X-Auth-Token"] = m_parent.AccessToken;
|
|
return req;
|
|
}
|
|
}
|
|
|
|
public OpenStackStorage()
|
|
{
|
|
}
|
|
|
|
public OpenStackStorage(string url, Dictionary<string, string> options)
|
|
{
|
|
var uri = new Utility.Uri(url);
|
|
|
|
m_container = uri.Host;
|
|
m_prefix = "/" + uri.Path;
|
|
if (!m_prefix.EndsWith("/"))
|
|
m_prefix += "/";
|
|
|
|
// For OpenStack we do not use a leading slash
|
|
if (m_prefix.StartsWith("/"))
|
|
m_prefix = m_prefix.Substring(1);
|
|
|
|
options.TryGetValue(USERNAME_OPTION, out m_username);
|
|
options.TryGetValue(PASSWORD_OPTION, out m_password);
|
|
options.TryGetValue(TENANTNAME_OPTION, out m_tenantName);
|
|
options.TryGetValue(AUTHURI_OPTION, out m_authUri);
|
|
options.TryGetValue(APIKEY_OPTION, out m_apikey);
|
|
options.TryGetValue(REGION_OPTION, out m_region);
|
|
|
|
if (string.IsNullOrWhiteSpace(m_username))
|
|
throw new UserInformationException(Strings.OpenStack.MissingOptionError(USERNAME_OPTION));
|
|
if (string.IsNullOrWhiteSpace(m_authUri))
|
|
throw new UserInformationException(Strings.OpenStack.MissingOptionError(AUTHURI_OPTION));
|
|
|
|
if (string.IsNullOrWhiteSpace(m_apikey))
|
|
{
|
|
if (string.IsNullOrWhiteSpace(m_password))
|
|
throw new UserInformationException(Strings.OpenStack.MissingOptionError(PASSWORD_OPTION));
|
|
if (string.IsNullOrWhiteSpace(m_tenantName))
|
|
throw new UserInformationException(Strings.OpenStack.MissingOptionError(TENANTNAME_OPTION));
|
|
}
|
|
|
|
m_helper = new WebHelper(this);
|
|
}
|
|
|
|
protected virtual string AccessToken
|
|
{
|
|
get
|
|
{
|
|
if (m_accessToken == null || (m_accessToken.expires.HasValue && (m_accessToken.expires.Value - DateTime.UtcNow).TotalSeconds < 30))
|
|
GetAuthResponse();
|
|
|
|
return m_accessToken.id;
|
|
}
|
|
}
|
|
|
|
private string JoinUrls(string uri, string fragment)
|
|
{
|
|
fragment = fragment ?? "";
|
|
return uri + (uri.EndsWith("/") ? "" : "/") + (fragment.StartsWith("/") ? fragment.Substring(1) : fragment);
|
|
}
|
|
private string JoinUrls(string uri, string fragment1, string fragment2)
|
|
{
|
|
return JoinUrls(JoinUrls(uri, fragment1), fragment2);
|
|
}
|
|
|
|
private OpenStackAuthResponse GetAuthResponse()
|
|
{
|
|
var helper = new JSONWebHelper();
|
|
|
|
var req = helper.CreateRequest(JoinUrls(m_authUri, "tokens"));
|
|
req.Accept = "application/json";
|
|
req.Method = "POST";
|
|
|
|
var resp = helper.ReadJSONResponse<OpenStackAuthResponse>(
|
|
req,
|
|
new OpenStackAuthRequest(m_tenantName, m_username, m_password, m_apikey)
|
|
);
|
|
|
|
m_accessToken = resp.access.token;
|
|
|
|
// Grab the endpoint now that we have received it anyway
|
|
var fileservice = resp.access.serviceCatalog.Where(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
|
|
if (fileservice == null)
|
|
throw new Exception("No object-store service found, is this service supported by the provider?");
|
|
|
|
var endpoint = fileservice.endpoints.Where(x => string.Equals(m_region, x.region)).FirstOrDefault() ?? fileservice.endpoints.First();
|
|
|
|
m_simplestorageendpoint = endpoint.publicURL;
|
|
|
|
return resp;
|
|
}
|
|
|
|
protected virtual string SimpleStorageEndPoint
|
|
{
|
|
get
|
|
{
|
|
if (m_simplestorageendpoint == null)
|
|
GetAuthResponse();
|
|
|
|
return m_simplestorageendpoint;
|
|
}
|
|
}
|
|
|
|
#region IStreamingBackend implementation
|
|
public void Put(string remotename, System.IO.Stream stream)
|
|
{
|
|
var url = JoinUrls(SimpleStorageEndPoint, m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename));
|
|
using(m_helper.GetResponse(url, stream, "PUT"))
|
|
{ }
|
|
}
|
|
public void Get(string remotename, System.IO.Stream stream)
|
|
{
|
|
var url = JoinUrls(SimpleStorageEndPoint, m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename));
|
|
|
|
try
|
|
{
|
|
using(var resp = m_helper.GetResponse(url))
|
|
using(var rs = AsyncHttpRequest.TrySetTimeout(resp.GetResponseStream()))
|
|
Library.Utility.Utility.CopyStream(rs, stream);
|
|
}
|
|
catch(WebException wex)
|
|
{
|
|
if (wex.Response is HttpWebResponse && ((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound)
|
|
throw new FileMissingException();
|
|
else
|
|
throw;
|
|
}
|
|
|
|
}
|
|
#endregion
|
|
#region IBackend implementation
|
|
public IEnumerable<IFileEntry> List()
|
|
{
|
|
var plainurl = JoinUrls(SimpleStorageEndPoint, m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT);
|
|
if (!string.IsNullOrEmpty(m_prefix))
|
|
plainurl += "&prefix=" + Library.Utility.Uri.UrlEncode(m_prefix);
|
|
|
|
try
|
|
{
|
|
return ListWithoutExceptionCatch(plainurl);
|
|
}
|
|
catch(WebException wex)
|
|
{
|
|
if (wex.Response is HttpWebResponse && (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound))
|
|
throw new FolderMissingException();
|
|
else
|
|
throw;
|
|
}
|
|
}
|
|
|
|
private IEnumerable<IFileEntry> ListWithoutExceptionCatch(string plainurl)
|
|
{
|
|
var url = plainurl;
|
|
|
|
while (true)
|
|
{
|
|
var req = m_helper.CreateRequest(url);
|
|
req.Accept = "application/json";
|
|
|
|
var items = m_helper.ReadJSONResponse<OpenStackStorageItem[]>(req);
|
|
foreach (var n in items)
|
|
{
|
|
var name = n.name;
|
|
if (name.StartsWith(m_prefix))
|
|
name = name.Substring(m_prefix.Length);
|
|
|
|
if (n.bytes == null)
|
|
yield return new FileEntry(name);
|
|
else if (n.last_modified == null)
|
|
yield return new FileEntry(name, n.bytes.Value);
|
|
else
|
|
yield return new FileEntry(name, n.bytes.Value, n.last_modified.Value, n.last_modified.Value);
|
|
}
|
|
|
|
if (items.Length != PAGE_LIMIT)
|
|
yield break;
|
|
|
|
// Prepare next listing entry
|
|
url = plainurl + string.Format("&marker={0}", Library.Utility.Uri.UrlEncode(items.Last().name));
|
|
}
|
|
}
|
|
|
|
public void Put(string remotename, string filename)
|
|
{
|
|
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
|
|
Put(remotename, fs);
|
|
}
|
|
|
|
public void Get(string remotename, string filename)
|
|
{
|
|
using (System.IO.FileStream fs = System.IO.File.Create(filename))
|
|
Get(remotename, fs);
|
|
}
|
|
public void Delete(string remotename)
|
|
{
|
|
var url = JoinUrls(SimpleStorageEndPoint, m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename));
|
|
m_helper.ReadJSONResponse<object>(url, null, "DELETE");
|
|
}
|
|
public void Test()
|
|
{
|
|
this.TestList();
|
|
}
|
|
public void CreateFolder()
|
|
{
|
|
var url = JoinUrls(SimpleStorageEndPoint, m_container);
|
|
using(m_helper.GetResponse(url, null, "PUT"))
|
|
{ }
|
|
}
|
|
public string DisplayName
|
|
{
|
|
get
|
|
{
|
|
return Strings.OpenStack.DisplayName;
|
|
}
|
|
}
|
|
public string ProtocolKey
|
|
{
|
|
get
|
|
{
|
|
return "openstack";
|
|
}
|
|
}
|
|
public IList<ICommandLineArgument> SupportedCommands
|
|
{
|
|
get
|
|
{
|
|
var authuris = new StringBuilder();
|
|
foreach(var s in KNOWN_OPENSTACK_PROVIDERS)
|
|
authuris.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
|
|
|
|
return new List<ICommandLineArgument>(new CommandLineArgument[] {
|
|
new CommandLineArgument(USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong),
|
|
new CommandLineArgument(PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)),
|
|
new CommandLineArgument(TENANTNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.TenantnameOptionShort, Strings.OpenStack.TenantnameOptionLong),
|
|
new CommandLineArgument(APIKEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.ApikeyOptionShort, Strings.OpenStack.ApikeyOptionLong),
|
|
new CommandLineArgument(AUTHURI_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.AuthuriOptionShort, Strings.OpenStack.AuthuriOptionLong(authuris.ToString())),
|
|
new CommandLineArgument(REGION_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.RegionOptionShort, Strings.OpenStack.RegionOptionLong),
|
|
});
|
|
}
|
|
}
|
|
public string Description
|
|
{
|
|
get
|
|
{
|
|
return Strings.OpenStack.Description;
|
|
}
|
|
}
|
|
#endregion
|
|
#region IDisposable implementation
|
|
public void Dispose()
|
|
{
|
|
}
|
|
#endregion
|
|
}
|
|
}
|
|
|