Kenneth Skovhede
842fd96543
Added ID's to each log message and each exception to allow later introduction of a Knowledgebase service that explains each error in more detail.
446 lines
17 KiB
C#
446 lines
17 KiB
C#
// Copyright (C) 2015, The Duplicati Team
|
|
// http://www.duplicati.com, info@duplicati.com
|
|
//
|
|
// This library is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU Lesser General Public License as
|
|
// published by the Free Software Foundation; either version 2.1 of the
|
|
// License, or (at your option) any later version.
|
|
//
|
|
// This library is distributed in the hope that it will be useful, but
|
|
// WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
// Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public
|
|
// License along with this library; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using Duplicati.Library.Interface;
|
|
using System.Linq;
|
|
using Duplicati.Library.Utility;
|
|
using Newtonsoft.Json;
|
|
using Duplicati.Library.Strings;
|
|
using System.Net;
|
|
using System.Text;
|
|
|
|
namespace Duplicati.Library.Backend.OpenStack
|
|
{
|
|
public class OpenStackStorage : IBackend, IStreamingBackend
|
|
{
|
|
private const string USERNAME_OPTION = "auth-username";
|
|
private const string PASSWORD_OPTION = "auth-password";
|
|
private const string TENANTNAME_OPTION = "openstack-tenant-name";
|
|
private const string AUTHURI_OPTION = "openstack-authuri";
|
|
private const string APIKEY_OPTION = "openstack-apikey";
|
|
private const string REGION_OPTION = "openstack-region";
|
|
|
|
private const int PAGE_LIMIT = 500;
|
|
|
|
|
|
private string m_container;
|
|
private string m_prefix;
|
|
|
|
private string m_username;
|
|
private string m_password;
|
|
private string m_authUri;
|
|
private string m_tenantName;
|
|
private string m_apikey;
|
|
private string m_region;
|
|
|
|
protected string m_simplestorageendpoint;
|
|
|
|
private WebHelper m_helper = null;
|
|
private OpenStackAuthResponse.TokenClass m_accessToken;
|
|
|
|
public static readonly KeyValuePair<string, string>[] KNOWN_OPENSTACK_PROVIDERS = new KeyValuePair<string,string>[] {
|
|
new KeyValuePair<string, string>("Rackspace US", "https://identity.api.rackspacecloud.com/v2.0"),
|
|
new KeyValuePair<string, string>("Rackspace UK", "https://lon.identity.api.rackspacecloud.com/v2.0"),
|
|
new KeyValuePair<string, string>("OVH Cloud Storage", "https://auth.cloud.ovh.net/v2.0"),
|
|
new KeyValuePair<string, string>("Selectel Cloud Storage", "https://auth.selcdn.ru"),
|
|
};
|
|
|
|
private class OpenStackAuthRequest
|
|
{
|
|
public class AuthContainer
|
|
{
|
|
[JsonProperty("RAX-KSKEY:apiKeyCredentials", NullValueHandling = NullValueHandling.Ignore)]
|
|
public ApiKeyBasedRequest ApiCredentials { get; set; }
|
|
|
|
[JsonProperty("passwordCredentials", NullValueHandling = NullValueHandling.Ignore)]
|
|
public PasswordBasedRequest PasswordCredentials { get; set; }
|
|
|
|
[JsonProperty("tenantName", NullValueHandling = NullValueHandling.Ignore)]
|
|
public string TenantName { get; set; }
|
|
|
|
[JsonProperty("token", NullValueHandling = NullValueHandling.Ignore)]
|
|
public TokenBasedRequest Token { get; set; }
|
|
|
|
}
|
|
|
|
public class ApiKeyBasedRequest
|
|
{
|
|
public string username { get; set; }
|
|
public string apiKey { get; set; }
|
|
}
|
|
|
|
public class PasswordBasedRequest
|
|
{
|
|
public string username { get; set; }
|
|
public string password { get; set; }
|
|
public string tenantName { get; set; }
|
|
}
|
|
|
|
public class TokenBasedRequest
|
|
{
|
|
public string id { get; set; }
|
|
}
|
|
|
|
|
|
public AuthContainer auth { get; set; }
|
|
|
|
public OpenStackAuthRequest(string tenantname, string username, string password, string apikey)
|
|
{
|
|
this.auth = new AuthContainer();
|
|
this.auth.TenantName = tenantname;
|
|
|
|
if (string.IsNullOrEmpty(apikey))
|
|
{
|
|
this.auth.PasswordCredentials = new PasswordBasedRequest() {
|
|
username = username,
|
|
password = password,
|
|
};
|
|
}
|
|
else
|
|
{
|
|
this.auth.ApiCredentials = new ApiKeyBasedRequest() {
|
|
username = username,
|
|
apiKey = apikey
|
|
};
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
private class OpenStackAuthResponse
|
|
{
|
|
public AccessClass access { get; set; }
|
|
|
|
public class TokenClass
|
|
{
|
|
public string id { get; set; }
|
|
public DateTime? expires { get; set; }
|
|
}
|
|
|
|
public class EndpointItem
|
|
{
|
|
public string region { get; set; }
|
|
public string tenantId { get; set; }
|
|
public string publicURL { get; set; }
|
|
public string internalURL { get; set; }
|
|
}
|
|
|
|
public class ServiceItem
|
|
{
|
|
public string name { get; set; }
|
|
public string type { get; set; }
|
|
public EndpointItem[] endpoints { get; set; }
|
|
}
|
|
|
|
public class AccessClass
|
|
{
|
|
public TokenClass token { get; set; }
|
|
public ServiceItem[] serviceCatalog { get; set; }
|
|
}
|
|
|
|
}
|
|
|
|
private class OpenStackStorageItem
|
|
{
|
|
public string name { get; set; }
|
|
public DateTime? last_modified { get; set; }
|
|
public long? bytes { get; set; }
|
|
public string content_type { get; set; }
|
|
public string subdir { get; set; }
|
|
}
|
|
|
|
private class WebHelper : JSONWebHelper
|
|
{
|
|
private OpenStackStorage m_parent;
|
|
|
|
public WebHelper(OpenStackStorage parent) { m_parent = parent; }
|
|
|
|
public override HttpWebRequest CreateRequest(string url, string method = null)
|
|
{
|
|
var req = base.CreateRequest(url, method);
|
|
req.Headers["X-Auth-Token"] = m_parent.AccessToken;
|
|
return req;
|
|
}
|
|
}
|
|
|
|
public OpenStackStorage()
|
|
{
|
|
}
|
|
|
|
public OpenStackStorage(string url, Dictionary<string, string> options)
|
|
{
|
|
var uri = new Utility.Uri(url);
|
|
|
|
m_container = uri.Host;
|
|
m_prefix = "/" + uri.Path;
|
|
if (!m_prefix.EndsWith("/", StringComparison.Ordinal))
|
|
m_prefix += "/";
|
|
|
|
// For OpenStack we do not use a leading slash
|
|
if (m_prefix.StartsWith("/", StringComparison.Ordinal))
|
|
m_prefix = m_prefix.Substring(1);
|
|
|
|
options.TryGetValue(USERNAME_OPTION, out m_username);
|
|
options.TryGetValue(PASSWORD_OPTION, out m_password);
|
|
options.TryGetValue(TENANTNAME_OPTION, out m_tenantName);
|
|
options.TryGetValue(AUTHURI_OPTION, out m_authUri);
|
|
options.TryGetValue(APIKEY_OPTION, out m_apikey);
|
|
options.TryGetValue(REGION_OPTION, out m_region);
|
|
|
|
if (string.IsNullOrWhiteSpace(m_username))
|
|
throw new UserInformationException(Strings.OpenStack.MissingOptionError(USERNAME_OPTION), "OpenStackMissingUsername");
|
|
if (string.IsNullOrWhiteSpace(m_authUri))
|
|
throw new UserInformationException(Strings.OpenStack.MissingOptionError(AUTHURI_OPTION), "OpenStackMissingAuthUri");
|
|
|
|
if (string.IsNullOrWhiteSpace(m_apikey))
|
|
{
|
|
if (string.IsNullOrWhiteSpace(m_password))
|
|
throw new UserInformationException(Strings.OpenStack.MissingOptionError(PASSWORD_OPTION), "OpenStackMissingPassword");
|
|
if (string.IsNullOrWhiteSpace(m_tenantName))
|
|
throw new UserInformationException(Strings.OpenStack.MissingOptionError(TENANTNAME_OPTION), "OpenStackMissingTenantName");
|
|
}
|
|
|
|
m_helper = new WebHelper(this);
|
|
}
|
|
|
|
protected virtual string AccessToken
|
|
{
|
|
get
|
|
{
|
|
if (m_accessToken == null || (m_accessToken.expires.HasValue && (m_accessToken.expires.Value - DateTime.UtcNow).TotalSeconds < 30))
|
|
GetAuthResponse();
|
|
|
|
return m_accessToken.id;
|
|
}
|
|
}
|
|
|
|
private string JoinUrls(string uri, string fragment)
|
|
{
|
|
fragment = fragment ?? "";
|
|
return uri + (uri.EndsWith("/", StringComparison.Ordinal) ? "" : "/") + (fragment.StartsWith("/", StringComparison.Ordinal) ? fragment.Substring(1) : fragment);
|
|
}
|
|
private string JoinUrls(string uri, string fragment1, string fragment2)
|
|
{
|
|
return JoinUrls(JoinUrls(uri, fragment1), fragment2);
|
|
}
|
|
|
|
private OpenStackAuthResponse GetAuthResponse()
|
|
{
|
|
var helper = new JSONWebHelper();
|
|
|
|
var req = helper.CreateRequest(JoinUrls(m_authUri, "tokens"));
|
|
req.Accept = "application/json";
|
|
req.Method = "POST";
|
|
|
|
var resp = helper.ReadJSONResponse<OpenStackAuthResponse>(
|
|
req,
|
|
new OpenStackAuthRequest(m_tenantName, m_username, m_password, m_apikey)
|
|
);
|
|
|
|
m_accessToken = resp.access.token;
|
|
|
|
// Grab the endpoint now that we have received it anyway
|
|
var fileservice = resp.access.serviceCatalog.Where(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)).FirstOrDefault();
|
|
if (fileservice == null)
|
|
throw new Exception("No object-store service found, is this service supported by the provider?");
|
|
|
|
var endpoint = fileservice.endpoints.Where(x => string.Equals(m_region, x.region)).FirstOrDefault() ?? fileservice.endpoints.First();
|
|
|
|
m_simplestorageendpoint = endpoint.publicURL;
|
|
|
|
return resp;
|
|
}
|
|
|
|
protected virtual string SimpleStorageEndPoint
|
|
{
|
|
get
|
|
{
|
|
if (m_simplestorageendpoint == null)
|
|
GetAuthResponse();
|
|
|
|
return m_simplestorageendpoint;
|
|
}
|
|
}
|
|
|
|
#region IStreamingBackend implementation
|
|
public void Put(string remotename, System.IO.Stream stream)
|
|
{
|
|
var url = JoinUrls(SimpleStorageEndPoint, m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename));
|
|
using(m_helper.GetResponse(url, stream, "PUT"))
|
|
{ }
|
|
}
|
|
public void Get(string remotename, System.IO.Stream stream)
|
|
{
|
|
var url = JoinUrls(SimpleStorageEndPoint, m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename));
|
|
|
|
try
|
|
{
|
|
using(var resp = m_helper.GetResponse(url))
|
|
using(var rs = AsyncHttpRequest.TrySetTimeout(resp.GetResponseStream()))
|
|
Library.Utility.Utility.CopyStream(rs, stream);
|
|
}
|
|
catch(WebException wex)
|
|
{
|
|
if (wex.Response is HttpWebResponse && ((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound)
|
|
throw new FileMissingException();
|
|
else
|
|
throw;
|
|
}
|
|
|
|
}
|
|
#endregion
|
|
#region IBackend implementation
|
|
|
|
private T HandleListExceptions<T>(Func<T> func)
|
|
{
|
|
try
|
|
{
|
|
return func();
|
|
}
|
|
catch (WebException wex)
|
|
{
|
|
if (wex.Response is HttpWebResponse && (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound))
|
|
throw new FolderMissingException();
|
|
else
|
|
throw;
|
|
}
|
|
}
|
|
|
|
public IEnumerable<IFileEntry> List()
|
|
{
|
|
var plainurl = JoinUrls(SimpleStorageEndPoint, m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT);
|
|
if (!string.IsNullOrEmpty(m_prefix))
|
|
plainurl += "&prefix=" + Library.Utility.Uri.UrlEncode(m_prefix);
|
|
|
|
var url = plainurl;
|
|
|
|
while (true)
|
|
{
|
|
var req = m_helper.CreateRequest(url);
|
|
req.Accept = "application/json";
|
|
|
|
var items = HandleListExceptions(() => m_helper.ReadJSONResponse<OpenStackStorageItem[]>(req));
|
|
foreach (var n in items)
|
|
{
|
|
var name = n.name;
|
|
if (name.StartsWith(m_prefix, StringComparison.Ordinal))
|
|
name = name.Substring(m_prefix.Length);
|
|
|
|
if (n.bytes == null)
|
|
yield return new FileEntry(name);
|
|
else if (n.last_modified == null)
|
|
yield return new FileEntry(name, n.bytes.Value);
|
|
else
|
|
yield return new FileEntry(name, n.bytes.Value, n.last_modified.Value, n.last_modified.Value);
|
|
}
|
|
|
|
if (items.Length != PAGE_LIMIT)
|
|
yield break;
|
|
|
|
// Prepare next listing entry
|
|
url = plainurl + string.Format("&marker={0}", Library.Utility.Uri.UrlEncode(items.Last().name));
|
|
}
|
|
}
|
|
|
|
public void Put(string remotename, string filename)
|
|
{
|
|
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
|
|
Put(remotename, fs);
|
|
}
|
|
|
|
public void Get(string remotename, string filename)
|
|
{
|
|
using (System.IO.FileStream fs = System.IO.File.Create(filename))
|
|
Get(remotename, fs);
|
|
}
|
|
public void Delete(string remotename)
|
|
{
|
|
var url = JoinUrls(SimpleStorageEndPoint, m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename));
|
|
m_helper.ReadJSONResponse<object>(url, null, "DELETE");
|
|
}
|
|
public void Test()
|
|
{
|
|
this.TestList();
|
|
}
|
|
public void CreateFolder()
|
|
{
|
|
var url = JoinUrls(SimpleStorageEndPoint, m_container);
|
|
using(m_helper.GetResponse(url, null, "PUT"))
|
|
{ }
|
|
}
|
|
public string DisplayName
|
|
{
|
|
get
|
|
{
|
|
return Strings.OpenStack.DisplayName;
|
|
}
|
|
}
|
|
public string ProtocolKey
|
|
{
|
|
get
|
|
{
|
|
return "openstack";
|
|
}
|
|
}
|
|
public IList<ICommandLineArgument> SupportedCommands
|
|
{
|
|
get
|
|
{
|
|
var authuris = new StringBuilder();
|
|
foreach(var s in KNOWN_OPENSTACK_PROVIDERS)
|
|
authuris.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
|
|
|
|
return new List<ICommandLineArgument>(new CommandLineArgument[] {
|
|
new CommandLineArgument(USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong),
|
|
new CommandLineArgument(PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)),
|
|
new CommandLineArgument(TENANTNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.TenantnameOptionShort, Strings.OpenStack.TenantnameOptionLong),
|
|
new CommandLineArgument(APIKEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.ApikeyOptionShort, Strings.OpenStack.ApikeyOptionLong),
|
|
new CommandLineArgument(AUTHURI_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.AuthuriOptionShort, Strings.OpenStack.AuthuriOptionLong(authuris.ToString())),
|
|
new CommandLineArgument(REGION_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.RegionOptionShort, Strings.OpenStack.RegionOptionLong),
|
|
});
|
|
}
|
|
}
|
|
public string Description
|
|
{
|
|
get
|
|
{
|
|
return Strings.OpenStack.Description;
|
|
}
|
|
}
|
|
|
|
public virtual string[] DNSName
|
|
{
|
|
get
|
|
{
|
|
return new string[] {
|
|
new System.Uri(m_authUri).Host,
|
|
string.IsNullOrWhiteSpace(m_simplestorageendpoint) ? null : new System.Uri(m_simplestorageendpoint).Host
|
|
};
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
#region IDisposable implementation
|
|
public void Dispose()
|
|
{
|
|
}
|
|
#endregion
|
|
}
|
|
}
|
|
|