Kenneth Skovhede
842fd96543
Added ID's to each log message and each exception to allow later introduction of a Knowledgebase service that explains each error in more detail.
620 lines
22 KiB
C#
620 lines
22 KiB
C#
// Copyright (C) 2015, The Duplicati Team
|
|
// http://www.duplicati.com, info@duplicati.com
|
|
//
|
|
// This library is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU Lesser General Public License as
|
|
// published by the Free Software Foundation; either version 2.1 of the
|
|
// License, or (at your option) any later version.
|
|
//
|
|
// This library is distributed in the hope that it will be useful, but
|
|
// WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
// Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public
|
|
// License along with this library; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
using System;
|
|
using System.Linq;
|
|
using System.Collections.Generic;
|
|
using Duplicati.Library.Interface;
|
|
using Newtonsoft.Json;
|
|
using System.IO;
|
|
using System.Net;
|
|
|
|
namespace Duplicati.Library.Backend.AmazonCloudDrive
|
|
{
|
|
public class AmzCD : IBackend, IStreamingBackend, IRenameEnabledBackend
|
|
{
|
|
private const string AUTHID_OPTION = "authid";
|
|
private const string LABELS_OPTION = "amzcd-labels";
|
|
private const string DELAY_OPTION = "amzcd-consistency-delay";
|
|
|
|
private const string DEFAULT_LABELS = "duplicati,backup";
|
|
private const string DEFAULT_DELAY = "15s";
|
|
|
|
private const int PAGE_SIZE = 200;
|
|
|
|
private const string CLOUDRIVE_MASTER_URL = "https://drive.amazonaws.com/drive/v1/account/endpoint";
|
|
private const string CACHE_FILE_NAME_TEMPLATE = "dupl-amzcd-endp-{0}-cache.txt";
|
|
private static readonly TimeSpan KEEP_CACHE_FILE_TIME = TimeSpan.FromDays(5);
|
|
|
|
private const string CONTENT_KIND_FOLDER = "FOLDER";
|
|
private const string CONTENT_KIND_FILE = "FILE";
|
|
|
|
private EndpointInfo m_endPointInfo;
|
|
private ResourceModel m_curdir;
|
|
|
|
private string m_path;
|
|
private string[] m_labels;
|
|
|
|
private OAuthHelper m_oauth;
|
|
private Dictionary<string, string> m_filecache;
|
|
private string m_userid;
|
|
private DateTime m_waitUntil;
|
|
private TimeSpan m_delayTimeSpan;
|
|
|
|
private enum RemoteOperation
|
|
{
|
|
First,
|
|
List,
|
|
Get,
|
|
Put,
|
|
Delete,
|
|
Rename
|
|
}
|
|
|
|
private RemoteOperation m_lastOperation = RemoteOperation.First;
|
|
|
|
public AmzCD()
|
|
{
|
|
}
|
|
|
|
public AmzCD(string url, Dictionary<string, string> options)
|
|
{
|
|
var uri = new Utility.Uri(url);
|
|
|
|
m_path = uri.HostAndPath;
|
|
if (!m_path.EndsWith("/", StringComparison.Ordinal))
|
|
m_path += "/";
|
|
|
|
string authid = null;
|
|
if (options.ContainsKey(AUTHID_OPTION))
|
|
authid = options[AUTHID_OPTION];
|
|
|
|
string labels = DEFAULT_LABELS;
|
|
if (options.ContainsKey(LABELS_OPTION))
|
|
labels = options[LABELS_OPTION];
|
|
|
|
string delay = DEFAULT_DELAY;
|
|
if (options.ContainsKey(DELAY_OPTION))
|
|
delay = options[DELAY_OPTION];
|
|
|
|
if (!string.IsNullOrWhiteSpace(labels))
|
|
m_labels = labels.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries);
|
|
|
|
if (string.IsNullOrWhiteSpace(delay))
|
|
m_delayTimeSpan = new TimeSpan(0);
|
|
else
|
|
m_delayTimeSpan = Library.Utility.Timeparser.ParseTimeSpan(delay);
|
|
m_waitUntil = DateTime.Now + m_delayTimeSpan;
|
|
|
|
m_oauth = new OAuthHelper(authid, this.ProtocolKey) { AutoAuthHeader = true };
|
|
m_userid = authid.Split(new string[] {":"}, StringSplitOptions.RemoveEmptyEntries).First();
|
|
}
|
|
|
|
private void EnforceConsistencyDelay(RemoteOperation lastop)
|
|
{
|
|
if (m_lastOperation == RemoteOperation.First)
|
|
m_lastOperation = lastop;
|
|
|
|
if (lastop == m_lastOperation)
|
|
return;
|
|
|
|
m_lastOperation = lastop;
|
|
|
|
var wait = m_waitUntil - DateTime.Now;
|
|
if (wait.Ticks > 0)
|
|
System.Threading.Thread.Sleep(wait);
|
|
}
|
|
|
|
private string CacheFilePath { get { return Path.Combine(Utility.TempFolder.SystemTempPath, string.Format(CACHE_FILE_NAME_TEMPLATE, m_userid)); } }
|
|
|
|
private void RefreshMetadataAndContentUrl()
|
|
{
|
|
try
|
|
{
|
|
if (File.Exists(CacheFilePath) && (DateTime.Now - File.GetLastWriteTime(CacheFilePath)) < KEEP_CACHE_FILE_TIME)
|
|
{
|
|
m_endPointInfo = JsonConvert.DeserializeObject<EndpointInfo>(File.ReadAllText(CacheFilePath));
|
|
return;
|
|
}
|
|
}
|
|
catch
|
|
{
|
|
}
|
|
|
|
m_endPointInfo = m_oauth.GetJSONData<EndpointInfo>(CLOUDRIVE_MASTER_URL);
|
|
|
|
try
|
|
{
|
|
if (m_endPointInfo.CustomerExists && m_endPointInfo.ContentUrl != null && m_endPointInfo.MetadataUrl != null)
|
|
File.WriteAllText(CacheFilePath, JsonConvert.SerializeObject(m_endPointInfo));
|
|
}
|
|
catch
|
|
{
|
|
}
|
|
}
|
|
|
|
public string ContentUrl
|
|
{
|
|
get
|
|
{
|
|
if (m_endPointInfo == null)
|
|
RefreshMetadataAndContentUrl();
|
|
while (m_endPointInfo.ContentUrl.EndsWith("/", StringComparison.Ordinal))
|
|
m_endPointInfo.ContentUrl = m_endPointInfo.ContentUrl.Substring(0, m_endPointInfo.ContentUrl.Length - 1);
|
|
return m_endPointInfo.ContentUrl;
|
|
}
|
|
}
|
|
|
|
public string MetadataUrl
|
|
{
|
|
get
|
|
{
|
|
if (m_endPointInfo == null)
|
|
RefreshMetadataAndContentUrl();
|
|
while (m_endPointInfo.MetadataUrl.EndsWith("/", StringComparison.Ordinal))
|
|
m_endPointInfo.MetadataUrl = m_endPointInfo.MetadataUrl.Substring(0, m_endPointInfo.MetadataUrl.Length - 1);
|
|
return m_endPointInfo.MetadataUrl;
|
|
}
|
|
}
|
|
|
|
private Dictionary<string, string> FileCache
|
|
{
|
|
get
|
|
{
|
|
if (m_filecache == null)
|
|
List();
|
|
|
|
return m_filecache;
|
|
}
|
|
}
|
|
|
|
private ResourceModel GetCurrentDirectory(bool createMissing, string altpath = null)
|
|
{
|
|
var rootResponse = m_oauth.GetJSONData<ListResponse>(string.Format("{0}/nodes?filters=isRoot:true", MetadataUrl));
|
|
var parent = rootResponse.Data.First();
|
|
var curpath = new List<string>();
|
|
foreach(var p in (altpath ?? m_path).Split(new string[] {"/"}, StringSplitOptions.RemoveEmptyEntries))
|
|
{
|
|
var requestUrl = string.Format("{0}/nodes?filters=kind:{1}%20and%20parents:{2}%20and%20name:{3}", MetadataUrl, CONTENT_KIND_FOLDER, Utility.Uri.UrlEncode(parent.ID), Utility.Uri.UrlPathEncode(EscapeFiltersValue(p)));
|
|
var self = m_oauth.GetJSONData<ListResponse>(requestUrl);
|
|
if (self == null || self.Count == 0 || self.Data == null || self.Data.Length == 0)
|
|
{
|
|
if (!createMissing)
|
|
throw new FolderMissingException(string.Format("Unable to find folder {0} in {1}", p, "/" + string.Join("/", curpath)));
|
|
|
|
// Create the folder
|
|
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateItemRequest() {
|
|
Name = p,
|
|
Parents = new string[] { parent.ID },
|
|
Kind = CONTENT_KIND_FOLDER,
|
|
Labels = m_labels
|
|
}));
|
|
|
|
parent = m_oauth.GetJSONData<ResourceModel>(
|
|
string.Format("{0}/nodes", MetadataUrl),
|
|
|
|
// Setup
|
|
req =>
|
|
{
|
|
req.Method = "POST";
|
|
req.ContentLength = data.Length;
|
|
req.ContentType = "application/json";
|
|
},
|
|
|
|
// Post the folder entry
|
|
req =>
|
|
{
|
|
using(var rs = req.GetRequestStream())
|
|
rs.Write(data, 0, data.Length);
|
|
}
|
|
);
|
|
m_waitUntil = DateTime.Now + m_delayTimeSpan;
|
|
}
|
|
else if (self != null && self.Count > 1)
|
|
throw new UserInformationException(Strings.AmzCD.MultipleEntries(p, "/" + string.Join("/", curpath)), "AmzCDMultipleEntries");
|
|
else
|
|
parent = self.Data.First();
|
|
}
|
|
|
|
return parent;
|
|
}
|
|
|
|
private ResourceModel CurrentDirectory
|
|
{
|
|
get
|
|
{
|
|
if (m_curdir == null)
|
|
m_curdir = GetCurrentDirectory(false);
|
|
|
|
return m_curdir;
|
|
}
|
|
}
|
|
|
|
private string GetFileID(string name)
|
|
{
|
|
var recent = m_filecache == null;
|
|
|
|
string id;
|
|
FileCache.TryGetValue(name, out id);
|
|
if (!string.IsNullOrWhiteSpace(id))
|
|
return id;
|
|
|
|
// Reset to make sure it is actually missing
|
|
if (!recent)
|
|
{
|
|
List();
|
|
|
|
FileCache.TryGetValue(name, out id);
|
|
if (!string.IsNullOrWhiteSpace(id))
|
|
return id;
|
|
}
|
|
|
|
throw new FileMissingException();
|
|
}
|
|
|
|
private static System.Text.RegularExpressions.Regex FILTERS_VALUE_ESCAPECHAR = new System.Text.RegularExpressions.Regex(@"[+\-&|!(){}\[\]^'""~*?:\\ ]", System.Text.RegularExpressions.RegexOptions.Compiled);
|
|
|
|
public static string EscapeFiltersValue(string value)
|
|
{
|
|
return FILTERS_VALUE_ESCAPECHAR.Replace(value, (m) => {
|
|
return @"\" + m.Value;
|
|
});
|
|
}
|
|
|
|
#region IStreamingBackend implementation
|
|
public void Put(string remotename, System.IO.Stream stream)
|
|
{
|
|
EnforceConsistencyDelay(RemoteOperation.Put);
|
|
|
|
var overwrite = FileCache.ContainsKey(remotename);
|
|
var fileid = overwrite ? m_filecache[remotename] : null;
|
|
|
|
var url = string.Format(overwrite ? "{0}/nodes/{1}/content?suppress=deduplication" : "{0}/nodes?suppress=deduplication", ContentUrl, fileid);
|
|
|
|
var createreq = new CreateItemRequest() {
|
|
Name = remotename,
|
|
Kind = CONTENT_KIND_FILE,
|
|
Labels = m_labels,
|
|
Parents = new string[] { CurrentDirectory.ID }
|
|
};
|
|
|
|
try
|
|
{
|
|
var item = m_oauth.PostMultipartAndGetJSONData<ResourceModel>(
|
|
url,
|
|
|
|
req =>
|
|
{
|
|
req.Method = overwrite ? "PUT" : "POST";
|
|
},
|
|
|
|
new MultipartItem(createreq, name: "metadata"),
|
|
new MultipartItem(stream, name: "content", filename: remotename)
|
|
|
|
);
|
|
|
|
if (m_filecache != null)
|
|
m_filecache[item.Name] = item.ID;
|
|
}
|
|
catch(Exception ex)
|
|
{
|
|
#if DEBUG
|
|
if (ex is WebException)
|
|
using(var sr = new StreamReader((ex as WebException).Response.GetResponseStream()))
|
|
Console.WriteLine(sr.ReadToEnd());
|
|
#endif
|
|
|
|
m_filecache = null;
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
m_waitUntil = DateTime.Now + m_delayTimeSpan;
|
|
}
|
|
}
|
|
public void Get(string remotename, System.IO.Stream stream)
|
|
{
|
|
EnforceConsistencyDelay(RemoteOperation.Get);
|
|
|
|
using (var resp = m_oauth.GetResponse(string.Format("{0}/nodes/{1}/content", ContentUrl, GetFileID(remotename))))
|
|
using(var rs = Library.Utility.AsyncHttpRequest.TrySetTimeout(resp.GetResponseStream()))
|
|
Utility.Utility.CopyStream(rs, stream);
|
|
}
|
|
#endregion
|
|
|
|
#region IBackend implementation
|
|
public IEnumerable<IFileEntry> List()
|
|
{
|
|
EnforceConsistencyDelay(RemoteOperation.List);
|
|
|
|
var query = string.Format("{0}/nodes?filters=parents:{1}&limit={2}", MetadataUrl, Utility.Uri.UrlEncode(CurrentDirectory.ID), PAGE_SIZE);
|
|
var res = new List<IFileEntry>();
|
|
string nextToken = null;
|
|
m_filecache = null;
|
|
var cache = new Dictionary<string, string>();
|
|
|
|
do
|
|
{
|
|
var lst = m_oauth.GetJSONData<ListResponse>(query + (string.IsNullOrWhiteSpace(nextToken) ? "" : ("&startToken=" + nextToken)));
|
|
if (lst.Data != null)
|
|
{
|
|
foreach(var n in lst.Data)
|
|
{
|
|
|
|
if (string.Equals(CONTENT_KIND_FOLDER, n.Kind, StringComparison.OrdinalIgnoreCase))
|
|
res.Add(new FileEntry(n.Name) { IsFolder = true });
|
|
else if (string.Equals(CONTENT_KIND_FILE, n.Kind, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
cache[n.Name] = n.ID;
|
|
|
|
if (n.ContentProperties == null)
|
|
res.Add(new FileEntry(n.Name) {
|
|
LastAccess = n.LastModified,
|
|
LastModification = n.LastModified
|
|
});
|
|
else
|
|
res.Add(new FileEntry(n.Name, n.ContentProperties.Size, n.LastModified, n.LastModified));
|
|
}
|
|
}
|
|
}
|
|
|
|
nextToken = lst.NextToken;
|
|
|
|
// Contrary to the documentation, nextToken is null when the set is done
|
|
if (lst.Count == 0)
|
|
break;
|
|
|
|
// Docs say to check for empty response ...
|
|
//if (lst.Count < PAGE_SIZE)
|
|
// break;
|
|
} while(nextToken != null);
|
|
|
|
m_filecache = cache;
|
|
return res;
|
|
|
|
}
|
|
public void Put(string remotename, string filename)
|
|
{
|
|
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
|
|
Put(remotename, fs);
|
|
}
|
|
public void Get(string remotename, string filename)
|
|
{
|
|
using (System.IO.FileStream fs = System.IO.File.Create(filename))
|
|
Get(remotename, fs);
|
|
}
|
|
public void Delete(string remotename)
|
|
{
|
|
EnforceConsistencyDelay(RemoteOperation.Delete);
|
|
|
|
try
|
|
{
|
|
using(m_oauth.GetResponse(string.Format("{0}/trash/{1}", MetadataUrl, GetFileID(remotename)), null, "PUT"))
|
|
{
|
|
}
|
|
|
|
m_filecache.Remove(remotename);
|
|
}
|
|
catch
|
|
{
|
|
m_filecache = null;
|
|
}
|
|
m_waitUntil = DateTime.Now + m_delayTimeSpan;
|
|
}
|
|
public void Test()
|
|
{
|
|
this.TestList();
|
|
}
|
|
public void CreateFolder()
|
|
{
|
|
EnforceConsistencyDelay(RemoteOperation.List);
|
|
GetCurrentDirectory(true);
|
|
}
|
|
public string DisplayName
|
|
{
|
|
get
|
|
{
|
|
return Strings.AmzCD.DisplayName;
|
|
}
|
|
}
|
|
|
|
public string ProtocolKey
|
|
{
|
|
get
|
|
{
|
|
return "amzcd";
|
|
}
|
|
}
|
|
|
|
public IList<ICommandLineArgument> SupportedCommands
|
|
{
|
|
get {
|
|
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
|
|
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.AmzCD.AuthidShort, Strings.AmzCD.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("amzcd"))),
|
|
new CommandLineArgument(LABELS_OPTION, CommandLineArgument.ArgumentType.String, Strings.AmzCD.LabelsShort, Strings.AmzCD.LabelsLong, DEFAULT_LABELS),
|
|
new CommandLineArgument(DELAY_OPTION, CommandLineArgument.ArgumentType.Timespan, Strings.AmzCD.DelayShort, Strings.AmzCD.DelayLong, DEFAULT_DELAY),
|
|
});
|
|
}
|
|
}
|
|
|
|
public string Description
|
|
{
|
|
get
|
|
{
|
|
return Strings.AmzCD.Description;
|
|
}
|
|
}
|
|
|
|
public string[] DNSName
|
|
{
|
|
get
|
|
{
|
|
var contentUrl = string.Empty;
|
|
var metdataUrl = string.Empty;
|
|
|
|
if (m_endPointInfo != null)
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(m_endPointInfo.ContentUrl))
|
|
contentUrl = new Uri(m_endPointInfo.ContentUrl).Host;
|
|
if (!string.IsNullOrWhiteSpace(m_endPointInfo.MetadataUrl))
|
|
metdataUrl = new Uri(m_endPointInfo.MetadataUrl).Host;
|
|
}
|
|
|
|
return new string[] {
|
|
new Uri(CLOUDRIVE_MASTER_URL).Host,
|
|
contentUrl,
|
|
metdataUrl
|
|
};
|
|
}
|
|
}
|
|
|
|
#endregion
|
|
#region IDisposable implementation
|
|
public void Dispose()
|
|
{
|
|
}
|
|
#endregion
|
|
|
|
#region IRenameEnabledBackend
|
|
public void Rename(string oldname, string newname)
|
|
{
|
|
EnforceConsistencyDelay(RemoteOperation.Rename);
|
|
|
|
var id = GetFileID(oldname);
|
|
|
|
var data = System.Text.Encoding.UTF8.GetBytes(
|
|
JsonConvert.SerializeObject(new CreateItemRequest() { Name = newname })
|
|
);
|
|
|
|
try
|
|
{
|
|
var resp = m_oauth.GetJSONData<ResourceModel>(
|
|
string.Format("{0}/nodes/{1}", MetadataUrl, id),
|
|
req =>
|
|
{
|
|
req.Method = "PATCH";
|
|
req.ContentType = "application/json";
|
|
req.ContentLength = data.Length;
|
|
},
|
|
|
|
req =>
|
|
{
|
|
using(var rs = req.GetRequestStream())
|
|
rs.Write(data, 0, data.Length);
|
|
}
|
|
);
|
|
|
|
m_filecache.Remove(oldname);
|
|
m_filecache[newname] = resp.ID;
|
|
}
|
|
catch
|
|
{
|
|
m_filecache = null;
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
m_waitUntil = DateTime.Now + m_delayTimeSpan;
|
|
}
|
|
}
|
|
#endregion
|
|
|
|
#region JSON Classes
|
|
private class ListResponse
|
|
{
|
|
[JsonProperty("count")]
|
|
public long Count { get; set; }
|
|
[JsonProperty("nextToken")]
|
|
public string NextToken { get; set; }
|
|
[JsonProperty("data")]
|
|
public ResourceModel[] Data { get; set; }
|
|
}
|
|
|
|
private class ResourceModel
|
|
{
|
|
[JsonProperty("id")]
|
|
public string ID { get; set; }
|
|
[JsonProperty("name")]
|
|
public string Name { get; set; }
|
|
[JsonProperty("kind")]
|
|
public string Kind { get; set; }
|
|
[JsonProperty("version")]
|
|
public long Version { get; set; }
|
|
[JsonProperty("modifiedDate")]
|
|
public DateTime LastModified { get; set; }
|
|
[JsonProperty("createdDate")]
|
|
public DateTime CreatedDate { get; set; }
|
|
[JsonProperty("labels")]
|
|
public string[] Labels { get; set; }
|
|
[JsonProperty("description")]
|
|
public string Description { get; set; }
|
|
[JsonProperty("createdBy")]
|
|
public string CreatedBy { get; set; }
|
|
[JsonProperty("parents")]
|
|
public string[] Parents { get; set; }
|
|
[JsonProperty("status")]
|
|
public string Status { get; set; }
|
|
[JsonProperty("contentProperties")]
|
|
public ContentProperties ContentProperties { get; set; }
|
|
}
|
|
|
|
private class ContentProperties
|
|
{
|
|
[JsonProperty("version")]
|
|
public long Version { get; set; }
|
|
[JsonProperty("md5")]
|
|
public string MD5 { get; set; }
|
|
[JsonProperty("size")]
|
|
public long Size { get; set; }
|
|
|
|
[JsonProperty("contentType")]
|
|
public string ContentType { get; set; }
|
|
[JsonProperty("extension")]
|
|
public string Extension { get; set; }
|
|
[JsonProperty("contentDate")]
|
|
public DateTime ContentDate { get; set; }
|
|
}
|
|
|
|
private class EndpointInfo
|
|
{
|
|
[JsonProperty("customerExists")]
|
|
public bool CustomerExists { get; set; }
|
|
[JsonProperty("contentUrl")]
|
|
public string ContentUrl { get; set; }
|
|
[JsonProperty("metadataUrl")]
|
|
public string MetadataUrl { get; set; }
|
|
}
|
|
|
|
private class CreateItemRequest
|
|
{
|
|
[JsonProperty("name")]
|
|
public string Name { get; set; }
|
|
[JsonProperty("kind")]
|
|
public string Kind { get; set; }
|
|
[JsonProperty("labels", NullValueHandling = NullValueHandling.Ignore)]
|
|
public string[] Labels { get; set; }
|
|
[JsonProperty("properties", NullValueHandling = NullValueHandling.Ignore)]
|
|
public Dictionary<string, string> Properties { get; set; }
|
|
[JsonProperty("parents")]
|
|
public string[] Parents { get; set; }
|
|
}
|
|
#endregion
|
|
|
|
|
|
}
|
|
}
|
|
|