Merge pull request #40 from duplicati/master

upstream
This commit is contained in:
Max
2018-03-31 17:36:58 +02:00
committed by GitHub
28 changed files with 2137 additions and 38 deletions
+77
View File
@@ -0,0 +1,77 @@
version: 1.0.{build}
image: Visual Studio 2017
configuration: Debug
platform: Any CPU
environment:
UNITTEST_BASEFOLDER: C:\projects\duplicati\testdata
COVERALLS_REPO_TOKEN:
secure: szQsrkP5rvra8L60SomD73/dFvRwot0UuyA566zqzI2qmLOr+MhLN0AyC8BTbhYY
before_build:
- cmd: nuget restore Duplicati.sln
build:
project: Duplicati.sln
verbosity: minimal
test_script:
- ps: |
Start-FileDownload https://s3.amazonaws.com/duplicati-test-file-hosting/DSMCBE.zip
7z x DSMCBE.zip -otestdata
mkdir .\testData\data
nuget install OpenCover -Version 4.6.166 -OutputDirectory packages
nuget install coveralls.net -Version 0.6.0 -OutputDirectory packages
nuget install NUnit.Runners -Version 3.4.0 -OutputDirectory packages
$testDir = ".\Duplicati\UnitTest\bin\Debug"
$tests = @("$testDir\Duplicati.UnitTest.dll")
$testsPassed = $true
foreach ($elem in $tests) {
$testResultsFile = "$elem" + ".xml"
.\packages\OpenCover.4.6.166\tools\OpenCover.Console.exe `
-register:user `
-target:.\packages\NUnit.ConsoleRunner.3.4.0\tools\nunit3-console.exe `
"-targetargs:""$elem"" /framework:net-4.5 /where:cat!=BulkData /result:""$testResultsFile""" `
"-filter:+[Duplicati.Library.Main]* -[UnitTest]*" `
-output:opencover.xml `
-returntargetcode `
# Keep track of test failures so that we can throw an exception after
# uploading coverage and test results.
$testsPassed = $testsPassed -and $?
# Upload test results to AppVeyor.
$wc = New-Object "System.Net.WebClient"
$wc.UploadFile("https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path .\""$testResultsFile""))
}
if ($Env:COVERALLS_REPO_TOKEN) {
$revision = git rev-parse HEAD
$branch = git rev-parse --abbrev-ref HEAD
$commitAuthor = git show --quiet --format="%aN" $revision
$commitEmail = git show --quiet --format="%aE" $revision
$commitMessage = git show --quiet --format="%s" $revision
.\packages\coveralls.net.0.6.0\tools\csmacnz.Coveralls.exe `
--opencover -i opencover.xml `
--repoToken "$Env:COVERALLS_REPO_TOKEN" `
--commitId "$revision" `
--commitBranch "$branch" `
--commitAuthor "$commitAuthor" `
--commitEmail "$commitEmail" `
--commitMessage "$commitMessage" `
--useRelativePaths `
--basePath .\Duplicati\UnitTest\bin\Debug
}
if (!$testsPassed) {
throw "Tests failed."
}
notifications:
- provider: Webhook
url: https://webhooks.gitter.im/e/a2c55bac2b5c38838e0f
method: POST
on_build_success: true
on_build_failure: true
on_build_status_changed: true
+98 -8
View File
@@ -164,6 +164,7 @@ namespace Duplicati.CommandLine.BackendTester
IEnumerable<Library.Interface.IFileEntry> curlist = null;
try
{
backend.Test();
curlist = backend.List();
}
catch (FolderMissingException fex)
@@ -210,6 +211,7 @@ namespace Duplicati.CommandLine.BackendTester
int max_filename_size = 80;
bool disableStreaming = Library.Utility.Utility.ParseBoolOption(options, "disable-streaming-transfers");
bool skipOverwriteTest = Library.Utility.Utility.ParseBoolOption(options, "skip-overwrite-test");
bool trimFilenameSpaces = Library.Utility.Utility.ParseBoolOption(options, "trim-filename-spaces");
if (options.ContainsKey("number-of-files"))
number_of_files = int.Parse(options["number-of-files"]);
@@ -232,17 +234,13 @@ namespace Duplicati.CommandLine.BackendTester
List<TempFile> files = new List<TempFile>();
for (int i = 0; i < number_of_files; i++)
{
StringBuilder filename = new StringBuilder();
int filenamelen = rnd.Next(min_filename_size, max_filename_size);
for (int j = 0; j < filenamelen; j++)
filename.Append(allowedChars[rnd.Next(0, allowedChars.Length)]);
string filename = CreateRandomRemoteFileName(min_filename_size, max_filename_size, allowedChars, trimFilenameSpaces, rnd);
string localfilename = CreateRandomFile(tf, i, min_file_size, max_file_size, rnd);
//Calculate local hash and length
using (System.IO.FileStream fs = new System.IO.FileStream(localfilename, System.IO.FileMode.Open, System.IO.FileAccess.Read))
files.Add(new TempFile(filename.ToString(), localfilename, sha.ComputeHash(fs), fs.Length));
files.Add(new TempFile(filename, localfilename, sha.ComputeHash(fs), fs.Length));
}
byte[] dummyFileHash = null;
@@ -268,6 +266,23 @@ namespace Duplicati.CommandLine.BackendTester
for (int i = 0; i < files.Count; i++)
Uploadfile(files[i].localfilename, i, files[i].remotefilename, backend, disableStreaming);
TempFile originalRenamedFile = null;
string renamedFileNewName = null;
IRenameEnabledBackend renameEnabledBackend = backend as IRenameEnabledBackend;
if (renameEnabledBackend != null)
{
// Rename the second file in the list, if there are more than one. If not, just do the first one.
int renameIndex = files.Count > 1 ? 1 : 0;
originalRenamedFile = files[renameIndex];
renamedFileNewName = CreateRandomRemoteFileName(min_filename_size, max_filename_size, allowedChars, trimFilenameSpaces, rnd);
Console.WriteLine("Renaming file {0} from {1} to {2}", renameIndex, originalRenamedFile.remotefilename, renamedFileNewName);
renameEnabledBackend.Rename(originalRenamedFile.remotefilename, renamedFileNewName);
files[renameIndex] = new TempFile(renamedFileNewName, originalRenamedFile.localfilename, originalRenamedFile.hash, originalRenamedFile.length);
}
Console.WriteLine("Verifying file list ...");
curlist = backend.List();
@@ -290,7 +305,14 @@ namespace Duplicati.CommandLine.BackendTester
}
if (!found)
Console.WriteLine("*** File with name {0} was found on server but not uploaded!", fe.Name);
if (originalRenamedFile != null && renamedFileNewName != null && originalRenamedFile.remotefilename == fe.Name)
{
Console.WriteLine("*** File with name {0} was found on server but was supposed to have been renamed to {1}!", fe.Name, renamedFileNewName);
}
else
{
Console.WriteLine("*** File with name {0} was found on server but not uploaded!", fe.Name);
}
}
foreach (TempFile tx in files)
@@ -361,6 +383,60 @@ namespace Duplicati.CommandLine.BackendTester
}
}
// Test quota retrieval
IQuotaEnabledBackend quotaEnabledBackend = backend as IQuotaEnabledBackend;
if (quotaEnabledBackend != null)
{
Console.WriteLine("Checking quota...");
IQuotaInfo quota = null;
bool noException;
try
{
quota = quotaEnabledBackend.Quota;
noException = true;
}
catch (Exception ex)
{
Console.WriteLine("*** Checking quota information failed: {0}", ex);
noException = false;
}
if (noException)
{
if (quota != null)
{
Console.WriteLine("Free Space: {0}", Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace));
Console.WriteLine("Total Space: {0}", Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace));
}
else
{
Console.WriteLine("Unable to retrieve quota information");
}
}
}
// Test DNSName lookup
Console.WriteLine("Checking DNS names used by this backend...");
try
{
string[] dnsNames = backend.DNSName;
if (dnsNames != null)
{
foreach (string dnsName in dnsNames)
{
Console.WriteLine(dnsName);
}
}
else
{
Console.WriteLine("No DNS names reported");
}
}
catch (Exception ex)
{
Console.WriteLine("*** Checking DNSName failed: {0}", ex);
}
}
finally
{
@@ -410,6 +486,20 @@ namespace Duplicati.CommandLine.BackendTester
}
}
private static string CreateRandomRemoteFileName(int min_filename_size, int max_filename_size, string allowedChars, bool trimFilenameSpaces, Random rnd)
{
StringBuilder filenameBuilder = new StringBuilder();
int filenamelen = rnd.Next(min_filename_size, max_filename_size);
for (int j = 0; j < filenamelen; j++)
filenameBuilder.Append(allowedChars[rnd.Next(0, allowedChars.Length)]);
string filename = filenameBuilder.ToString();
if (trimFilenameSpaces)
filename = filename.Trim();
return filename;
}
private static string CreateRandomFile(Library.Utility.TempFolder tf, int i, int min_file_size, int max_file_size, Random rnd)
{
Console.Write("Generating file {0}", i);
@@ -446,12 +536,12 @@ namespace Duplicati.CommandLine.BackendTester
new CommandLineArgument("max-file-size", CommandLineArgument.ArgumentType.Size, "The maximum allowed file size", "File sizes are chosen at random, this value is the upper bound", "50mb"),
new CommandLineArgument("min-filename-length", CommandLineArgument.ArgumentType.Integer, "The minimum allowed filename length", "File name lengths are chosen at random, this value is the lower bound", "5"),
new CommandLineArgument("max-filename-length", CommandLineArgument.ArgumentType.Integer, "The minimum allowed filename length", "File name lengths are chosen at random, this value is the upper bound", "80"),
new CommandLineArgument("trim-filename-spaces", CommandLineArgument.ArgumentType.Boolean, "Trims whitespace from filenames", "A value that indicates if whitespace should be trimmed from the ends of randomly generated filenames", "false"),
new CommandLineArgument("auto-create-folder", CommandLineArgument.ArgumentType.Boolean, "Allows automatic folder creation", "A value that indicates if missing folders are created automatically", "false"),
new CommandLineArgument("skip-overwrite-test", CommandLineArgument.ArgumentType.Boolean, "Bypasses the overwrite test", "A value that indicates if dummy files should be uploaded prior to uploading the real files", "false"),
new CommandLineArgument("auto-clean", CommandLineArgument.ArgumentType.Boolean, "Removes any files found in target folder", "A value that indicates if all files in the target folder should be deleted before starting the first test", "false"),
new CommandLineArgument("force", CommandLineArgument.ArgumentType.Boolean, "Activates file deletion", "A value that indicates if existing files should really be deleted when using auto-clean", "false"),
});
}
}
}
@@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
@@ -34,8 +34,12 @@
<Reference Include="Newtonsoft.Json">
<HintPath>..\..\..\..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.WebRequest" />
</ItemGroup>
<ItemGroup>
<Compile Include="OAuthHttpClient.cs" />
<Compile Include="OAuthHttpMessageHandler.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="OAuthHelper.cs" />
<Compile Include="Strings.cs" />
@@ -163,10 +163,9 @@ namespace Duplicati.Library
/// <returns>The deserialized JSON data.</returns>
/// <param name="url">The remote URL</param>
/// <param name="item">The data to json-serialize and POST in the request</param>
/// <param name="setup">A callback method that can be used to customize the request, e.g. by setting the method, content-type and headers.</param>
/// <param name="setupreq">A callback method that can be used to submit data into the body of the request.</param>
/// <param name="method">Alternate HTTP method to use</param>
/// <typeparam name="T">The type of data to return.</typeparam>
public virtual T PostAndGetJSONData<T>(string url, object item, Action<HttpWebRequest> setup = null, Action<AsyncHttpRequest> setupreq = null)
public virtual T PostAndGetJSONData<T>(string url, object item, string method = null)
{
var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(item));
@@ -174,7 +173,7 @@ namespace Duplicati.Library
url,
req =>
{
req.Method = "POST";
req.Method = method ?? "POST";
req.ContentType = "application/json; charset=utf-8";
req.ContentLength = data.Length;
},
@@ -0,0 +1,108 @@
// Copyright (C) 2018, The Duplicati Team
// http://www.duplicati.com, info@duplicati.com
//
// This library is free software; you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as
// published by the Free Software Foundation; either version 2.1 of the
// License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using System;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Reflection;
using System.Threading.Tasks;
using Duplicati.Library.Utility;
namespace Duplicati.Library
{
public class OAuthHttpClient : HttpClient
{
private static readonly string USER_AGENT_VERSION = Assembly.GetExecutingAssembly().GetName().Version.ToString();
private readonly OAuthHttpMessageHandler m_authenticator;
public OAuthHttpClient(string authid, string protocolKey)
: this(CreateMessageHandler(authid, protocolKey))
{
}
private OAuthHttpClient(OAuthHttpMessageHandler authenticator)
: base(authenticator, true)
{
this.m_authenticator = authenticator;
// Set the overall timeout
if (HttpContextSettings.OperationTimeout > TimeSpan.Zero)
{
this.Timeout = HttpContextSettings.OperationTimeout;
}
// We would also set AllowReadStreamBuffering = HttpContextSettings.BufferRequests, except HttpClient doesn't appear to expose this.
// However, starting in .NET 4.5, it looks like HttpClient doesn't buffer by default.
// https://www.strathweb.com/2012/09/dealing-with-large-files-in-asp-net-web-api/
// Set the default user agent
this.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("Duplicati", USER_AGENT_VERSION));
}
/// <summary>
/// Sends an async request with optional authentication.
/// </summary>
/// <param name="request">Http request</param>
/// <param name="authenticate">Whether to authenticate the request</param>
/// <returns>Http response</returns>
public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, bool authenticate)
{
if (!authenticate)
{
this.PreventAuthentication(request);
}
return await this.SendAsync(request);
}
/// <summary>
/// Prevents authentication from being applied on the given request
/// </summary>
/// <param name="request">Request to not authenticate</param>
/// <returns>Request to not authenticate</returns>
public HttpRequestMessage PreventAuthentication(HttpRequestMessage request)
{
return this.m_authenticator.PreventAuthentication(request);
}
/// <summary>
/// Create a message handler with the global timeout / certificate settings.
/// </summary>
/// <param name="authid">OAuth Auth-ID</param>
/// <param name="protocolKey">Protocol key</param>
/// <returns>Http message handler</returns>
private static OAuthHttpMessageHandler CreateMessageHandler(string authid, string protocolKey)
{
OAuthHttpMessageHandler handler = new OAuthHttpMessageHandler(authid, protocolKey);
// Set the read/write timeout
if (HttpContextSettings.OperationTimeout > TimeSpan.Zero)
{
handler.ReadWriteTimeout = (int)HttpContextSettings.ReadWriteTimeout.TotalMilliseconds;
}
// Set the certificate validator
if (HttpContextSettings.CertificateValidator != null)
{
handler.ServerCertificateValidationCallback = HttpContextSettings.CertificateValidator.ValidateServerCertficate;
}
return handler;
}
}
}
@@ -0,0 +1,59 @@
// Copyright (C) 2018, The Duplicati Team
// http://www.duplicati.com, info@duplicati.com
//
// This library is free software; you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as
// published by the Free Software Foundation; either version 2.1 of the
// License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using System.Net.Http;
using System.Net.Http.Headers;
using System.Threading;
using System.Threading.Tasks;
namespace Duplicati.Library
{
public class OAuthHttpMessageHandler : WebRequestHandler
{
/// <summary>
/// Requests which contain a property with this name (in 'request.Properties') will not have the authentication header automatically added.
/// </summary>
public const string DISABLE_AUTHENTICATION_PROPERTY = "OAuthHttpMessageHandler_DisableAuthentication";
private readonly OAuthHelper m_oauth;
public OAuthHttpMessageHandler(string authid, string protocolKey)
{
this.m_oauth = new OAuthHelper(authid, protocolKey);
}
/// <summary>
/// Prevents authentication from being applied on the given request
/// </summary>
/// <param name="request">Request to not authenticate</param>
/// <returns>Request to not authenticate</returns>
public HttpRequestMessage PreventAuthentication(HttpRequestMessage request)
{
request.Properties[DISABLE_AUTHENTICATION_PROPERTY] = true;
return request;
}
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (!request.Properties.ContainsKey(DISABLE_AUTHENTICATION_PROPERTY))
{
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", this.m_oauth.AccessToken);
}
return base.SendAsync(request, cancellationToken);
}
}
}
@@ -46,10 +46,18 @@
<Reference Include="Newtonsoft.Json">
<HintPath>..\..\..\..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.WebRequest" />
</ItemGroup>
<ItemGroup>
<Compile Include="Exceptions.cs" />
<Compile Include="MicrosoftGraphBackend.cs" />
<Compile Include="MicrosoftGraphTypes.cs" />
<Compile Include="MicrosoftGroup.cs" />
<Compile Include="OneDrive.cs" />
<Compile Include="OneDriveV2.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="SharePointV2.cs" />
<Compile Include="Strings.cs" />
</ItemGroup>
<ItemGroup>
@@ -0,0 +1,102 @@
using System;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Duplicati.Library.Utility;
using Newtonsoft.Json;
namespace Duplicati.Library.Backend.MicrosoftGraph
{
public class MicrosoftGraphException : Exception
{
private static readonly Regex authorizationHeaderRemover = new Regex(@"Authorization:\s*Bearer\s+[A-Za-z0-9+/=]+", RegexOptions.IgnoreCase);
public MicrosoftGraphException(HttpResponseMessage response)
: this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage.RequestUri), response)
{
}
public MicrosoftGraphException(string message, HttpResponseMessage response)
: this(message, response, null)
{
}
public MicrosoftGraphException(string message, HttpResponseMessage response, Exception innerException)
: base(BuildFullMessage(message, response), innerException)
{
this.Response = response;
}
public string RequestUrl => this.Response.RequestMessage.RequestUri.ToString();
public HttpResponseMessage Response { get; private set; }
protected static string ResponseToString(HttpResponseMessage response)
{
if (response != null)
{
// Start to read the content
Task<string> content = response.Content.ReadAsStringAsync();
// Since the exception message may be saved / sent in logs, we want to prevent the authorization header from being included.
// it wouldn't be as bad as recording the username/password in logs, since the token will expire, but it doesn't hurt to be safe.
// So we replace anything in the request that looks like the auth header with a safe version.
string requestMessage = authorizationHeaderRemover.Replace(response.RequestMessage.ToString(), "Authorization: Bearer ABC...XYZ");
return string.Format("{0}\n{1}\n{2}", requestMessage, response, JsonConvert.SerializeObject(JsonConvert.DeserializeObject(content.Await()), Formatting.Indented));
}
else
{
return null;
}
}
private static string BuildFullMessage(string message, HttpResponseMessage response)
{
if (response != null)
{
return string.Format("{0}\n{1}", message, ResponseToString(response));
}
else
{
return message;
}
}
}
public class DriveItemNotFoundException : MicrosoftGraphException
{
public DriveItemNotFoundException(HttpResponseMessage response)
: base(string.Format("Item at {0} was not found", response?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"), response)
{
}
}
public class UploadSessionException : MicrosoftGraphException
{
public UploadSessionException(
HttpResponseMessage originalResponse,
int fragment,
int fragmentCount,
MicrosoftGraphException fragmentException)
: base(
string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"),
originalResponse,
fragmentException)
{
this.Fragment = fragment;
this.FragmentCount = fragmentCount;
this.InnerException = fragmentException;
}
public string CreateSessionRequestUrl => this.RequestUrl;
public HttpResponseMessage CreateSessionResponse => this.Response;
public int Fragment { get; private set; }
public int FragmentCount { get; private set; }
public string FragmentRequestUrl => this.InnerException.RequestUrl;
public HttpResponseMessage FragmentResponse => this.InnerException.Response;
public new MicrosoftGraphException InnerException { get; private set; }
}
}
@@ -0,0 +1,558 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Threading;
using Duplicati.Library.Backend.MicrosoftGraph;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Newtonsoft.Json;
namespace Duplicati.Library.Backend
{
/// <summary>
/// Base class for all backends based on the Microsoft Graph API:
/// https://developer.microsoft.com/en-us/graph/
/// </summary>
/// <remarks>
/// HttpClient is used instead of OAuthHelper because OAuthHelper internally converts URLs to System.Uri, which throws UriFormatException when the URL contains ':' characters.
/// https://stackoverflow.com/questions/2143856/why-does-colon-in-uri-passed-to-uri-makerelativeuri-cause-an-exception
/// https://social.msdn.microsoft.com/Forums/vstudio/en-US/bf11fc74-975a-4c4d-8335-8e0579d17fdf/uri-containing-colons-incorrectly-throws-uriformatexception?forum=netfxbcl
///
/// Note that instead of using Task.Result to wait for the results of asynchronous operations,
/// this class uses the Utility.Await() extension method, since it doesn't wrap exceptions in AggregateExceptions.
/// </remarks>
public abstract class MicrosoftGraphBackend : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend
{
private const string SERVICES_AGREEMENT = "https://www.microsoft.com/en-us/servicesagreement";
private const string PRIVACY_STATEMENT = "https://privacy.microsoft.com/en-us/privacystatement";
private const string BASE_ADDRESS = "https://graph.microsoft.com";
private const string AUTHID_OPTION = "authid";
private const string UPLOAD_SESSION_FRAGMENT_SIZE_OPTION = "fragment-size";
private const string UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION = "fragment-retry-count";
private const string UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION = "fragment-retry-delay";
private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT = 5;
private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY = 1000;
/// <summary>
/// Max size of file that can be uploaded in a single PUT request is 4 MB:
/// https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/api/driveitem_put_content
/// </summary>
private const int PUT_MAX_SIZE = 4 * 1000 * 1000;
/// <summary>
/// Max size of each individual upload in an upload session is 60 MiB:
/// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
/// </summary>
private const int UPLOAD_SESSION_FRAGMENT_MAX_SIZE = 60 * 1024 * 1024;
/// <summary>
/// Default fragment size of 10 MiB, as the documentation recommends something in the range of 5-10 MiB,
/// and it still complies with the 320 KiB multiple requirement.
/// </summary>
private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE = 10 * 1024 * 1024;
/// <summary>
/// Each fragment in an upload session must be a size that is multiple of this size.
/// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
/// There is some confusion in the docs as to whether this is actually required, however...
/// </summary>
private const int UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE = 320 * 1024;
private static readonly HttpMethod PatchMethod = new HttpMethod("PATCH");
protected delegate string DescriptionTemplateDelegate(string mssadescription, string mssalink, string msopdescription, string msoplink);
private readonly JsonSerializer m_serializer = new JsonSerializer();
private readonly OAuthHttpClient m_client;
private readonly string m_path;
private readonly int fragmentSize;
private readonly int fragmentRetryCount;
private readonly int fragmentRetryDelay; // In milliseconds
private string[] dnsNames = null;
protected MicrosoftGraphBackend() { } // Constructor needed for dynamic loading to find it
protected MicrosoftGraphBackend(string url, Dictionary<string, string> options)
{
string authid;
options.TryGetValue(AUTHID_OPTION, out authid);
if (string.IsNullOrEmpty(authid))
throw new UserInformationException(Strings.MicrosoftGraph.MissingAuthId(OAuthHelper.OAUTH_LOGIN_URL(this.ProtocolKey)), "MicrosoftGraphBackendMissingAuthId");
string fragmentSizeStr;
if (options.TryGetValue(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, out fragmentSizeStr) && int.TryParse(fragmentSizeStr, out this.fragmentSize))
{
// Make sure the fragment size is a multiple of the desired multiple size.
// If it isn't, we round down to the nearest multiple below it.
this.fragmentSize = (this.fragmentSize / UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE) * UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE;
// Make sure the fragment size isn't larger than the maximum, or smaller than the minimum
this.fragmentSize = Math.Max(Math.Min(this.fragmentSize, UPLOAD_SESSION_FRAGMENT_MAX_SIZE), UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE);
}
else
{
this.fragmentSize = UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE;
}
string fragmentRetryCountStr;
if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, out fragmentRetryCountStr) && int.TryParse(fragmentRetryCountStr, out this.fragmentRetryCount)))
{
this.fragmentRetryCount = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT;
}
string fragmentRetryDelayStr;
if (!(options.TryGetValue(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, out fragmentRetryDelayStr) && int.TryParse(fragmentRetryDelayStr, out this.fragmentRetryDelay)))
{
this.fragmentRetryDelay = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY;
}
this.m_client = new OAuthHttpClient(authid, this.ProtocolKey);
this.m_client.BaseAddress = new System.Uri(BASE_ADDRESS);
// Extract out the path to the backup root folder from the given URI
this.m_path = NormalizeSlashes(this.GetRootPathFromUrl(url));
}
public abstract string ProtocolKey { get; }
public abstract string DisplayName { get; }
public string Description
{
get
{
return this.DescriptionTemplate(
"Microsoft Service Agreement",
SERVICES_AGREEMENT,
"Microsoft Online Privacy Statement",
PRIVACY_STATEMENT);
}
}
public IList<ICommandLineArgument> SupportedCommands
{
get
{
return new[]
{
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.MicrosoftGraph.AuthIdShort, Strings.MicrosoftGraph.AuthIdLong(OAuthHelper.OAUTH_LOGIN_URL(this.ProtocolKey))),
new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentSizeShort, Strings.MicrosoftGraph.FragmentSizeLong, Library.Utility.Utility.FormatSizeString(UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE)),
new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryCountShort, Strings.MicrosoftGraph.FragmentRetryCountLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT.ToString()),
new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryDelayShort, Strings.MicrosoftGraph.FragmentRetryDelayLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY.ToString()),
}
.Concat(this.AdditionalSupportedCommands).ToList();
}
}
public string[] DNSName
{
get
{
if (this.dnsNames == null)
{
// The DNS names that this instance may need to access include:
// - Core graph API endpoint
// - Upload session endpoint (which seems to be different depending on the drive being accessed - not sure if it can vary for a single drive)
// To get the upload session endpoint, we can start an upload session and then immediately cancel it.
// We pick a random file name (using a guid) to make sure we don't conflict with an existing file
string dnsTestFile = string.Format("DNSNameTest-{0}", Guid.NewGuid());
UploadSession uploadSession = this.Post<UploadSession>(string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.m_path, NormalizeSlashes(dnsTestFile)), null);
// Canceling an upload session is done by sending a DELETE to the upload URL
var request = new HttpRequestMessage(HttpMethod.Delete, uploadSession.UploadUrl);
var response = this.m_client.SendAsync(request).Await();
this.CheckResponse(response);
this.dnsNames = new[]
{
new System.Uri(BASE_ADDRESS).Host,
new System.Uri(uploadSession.UploadUrl).Host,
}
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToArray();
}
return this.dnsNames;
}
}
public IQuotaInfo Quota
{
get
{
Drive driveInfo = this.Get<Drive>(this.DrivePrefix);
if (driveInfo.Quota != null)
{
// Some sources (SharePoint for example) seem to return 0 for these values even when the quota isn't exceeded..
// As a special test, if all the returned values are 0, we pretend that no quota was reported.
// This way we don't send spurious warnings because the quota looks like it is exceeded.
if (driveInfo.Quota.Total != 0 || driveInfo.Quota.Remaining != 0 || driveInfo.Quota.Used != 0)
{
return new QuotaInfo(driveInfo.Quota.Total, driveInfo.Quota.Remaining);
}
}
return null;
}
}
/// <summary>
/// Override-able fragment indicating the API version to use each query
/// </summary>
protected virtual string ApiVersion
{
get { return "/v1.0"; }
}
/// <summary>
/// Normalized fragment (starting with a slash and ending without one) for the path to the drive to be used.
/// For example: "/me/drive" for the default drive for a user.
/// </summary>
protected abstract string DrivePath { get; }
protected abstract DescriptionTemplateDelegate DescriptionTemplate { get; }
protected virtual IList<ICommandLineArgument> AdditionalSupportedCommands
{
get
{
return new ICommandLineArgument[0];
}
}
private string DrivePrefix
{
get { return this.ApiVersion + this.DrivePath; }
}
public void CreateFolder()
{
string parentFolder = "root";
string parentFolderPath = string.Empty;
foreach (string folder in this.m_path.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries))
{
string nextPath = parentFolderPath + "/" + folder;
DriveItem folderItem;
try
{
folderItem = this.Get<DriveItem>(string.Format("{0}/root:{1}", this.DrivePrefix, NormalizeSlashes(nextPath)));
}
catch (DriveItemNotFoundException)
{
DriveItem newFolder = new DriveItem()
{
Name = folder,
Folder = new FolderFacet(),
};
folderItem = this.Post(string.Format("{0}/items/{1}/children", this.DrivePrefix, parentFolder), newFolder);
}
parentFolder = folderItem.Id;
parentFolderPath = nextPath;
}
}
public IEnumerable<IFileEntry> List()
{
try
{
return this.Enumerate<DriveItem>(string.Format("{0}/root:{1}:/children", this.DrivePrefix, this.m_path))
.Where(item => item.IsFile && !item.IsDeleted) // Exclude non-files and deleted items (not sure if they show up in this listing, but make sure anyway)
.Select(item =>
new FileEntry(
item.Name,
item.Size ?? 0, // Files should always have a size, but folders don't need it
item.FileSystemInfo?.LastAccessedDateTime?.UtcDateTime ?? new DateTime(),
item.FileSystemInfo?.LastModifiedDateTime?.UtcDateTime ?? item.LastModifiedDateTime?.UtcDateTime ?? new DateTime()));
}
catch (DriveItemNotFoundException ex)
{
// If there's an 'item not found' exception here, it means the root folder didn't exist.
throw new FolderMissingException(ex);
}
}
public void Get(string remotename, string filename)
{
using (FileStream fileStream = File.OpenWrite(filename))
{
this.Get(remotename, fileStream);
}
}
public void Get(string remotename, Stream stream)
{
var response = this.m_client.GetAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename))).Await();
this.CheckResponse(response);
using (Stream responseStream = response.Content.ReadAsStreamAsync().Await())
{
responseStream.CopyTo(stream);
}
}
public void Rename(string oldname, string newname)
{
this.Patch(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.m_path, NormalizeSlashes(oldname)), new DriveItem() { Name = newname });
}
public void Put(string remotename, string filename)
{
using (FileStream fileStream = File.OpenRead(filename))
{
this.Put(remotename, fileStream);
}
}
public void Put(string remotename, Stream stream)
{
// PUT only supports up to 4 MB file uploads. There's a separate process for larger files.
if (stream.Length < PUT_MAX_SIZE)
{
StreamContent streamContent = new StreamContent(stream);
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
var response = this.m_client.PutAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename)), streamContent).Await();
// Make sure this response is a valid drive item, though we don't actually use it for anything currently.
var result = this.ParseResponse<DriveItem>(response);
}
else
{
// This file is too large to be sent in a single request, so we need to send it in pieces in an upload session:
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession
// The documentation seems somewhat contradictory - it states that uploads must be done sequentially,
// but also states that the nextExpectedRanges value returned may indicate multiple ranges...
// For now, this plays it safe and does a sequential upload.
HttpRequestMessage createSessionRequest = new HttpRequestMessage(HttpMethod.Post, string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename)));
// Indicate that we want to replace any existing content with this new data we're uploading
StringContent createSessionContent = this.PrepareContent(new UploadSession() { Item = new DriveItem() { ConflictBehavior = ConflictBehavior.Replace } });
HttpResponseMessage createSessionResponse = this.m_client.SendAsync(createSessionRequest).Await();
UploadSession uploadSession = this.ParseResponse<UploadSession>(createSessionResponse);
// If the stream's total length is less than the chosen fragment size, then we should make the buffer only as large as the stream.
int fragmentSize = (int)Math.Min(this.fragmentSize, stream.Length);
byte[] fragmentBuffer = new byte[fragmentSize];
int read = 0;
for (int offset = 0; offset < stream.Length; offset += read)
{
read = stream.Read(fragmentBuffer, 0, fragmentSize);
int retryCount = this.fragmentRetryCount;
for (int attempt = 0; attempt < retryCount; attempt++)
{
ByteArrayContent fragmentContent = new ByteArrayContent(fragmentBuffer, 0, read);
fragmentContent.Headers.ContentLength = read;
fragmentContent.Headers.ContentRange = new ContentRangeHeaderValue(offset, offset + read - 1, stream.Length);
HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Put, uploadSession.UploadUrl);
request.Content = fragmentContent;
HttpResponseMessage response = null;
try
{
// The uploaded put requests will error if they are authenticated
response = this.m_client.SendAsync(request, false).Await();
// Note: On the last request, the json result includes the default properties of the item that was uploaded
var result = this.ParseResponse<UploadSession>(response);
}
catch (MicrosoftGraphException ex)
{
// Error handling based on recommendations here:
// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession#best-practices
if (attempt >= retryCount - 1)
{
// We've used up all our retry attempts
throw new UploadSessionException(createSessionResponse, offset / fragmentSize, (int)Math.Ceiling((double)stream.Length / fragmentSize), ex);
}
else if ((int)ex.Response.StatusCode >= 500 && (int)ex.Response.StatusCode < 600)
{
// If a 5xx error code is hit, we should use an exponential backoff strategy before retrying.
// To make things simpler, we just use the current attempt number as the exponential factor.
Thread.Sleep((int)Math.Pow(2, attempt) * this.fragmentRetryDelay); // If this is changed to use tasks, this should be changed to Task.Await()
continue;
}
else if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
// 404 is a special case indicating the upload session no longer exists, so the fragment shouldn't be retried.
// Instead we'll let the caller re-attempt the whole file.
throw new UploadSessionException(createSessionResponse, offset / fragmentSize, (int)Math.Ceiling((double)stream.Length / fragmentSize), ex);
}
else if ((int)ex.Response.StatusCode >= 400 && (int)ex.Response.StatusCode < 500)
{
// If a 4xx error code is hit, we should retry without the backoff attempt
continue;
}
else
{
// Other errors should be rethrown
throw new UploadSessionException(createSessionResponse, offset / fragmentSize, (int)Math.Ceiling((double)stream.Length / fragmentSize), ex);
}
}
// If we successfully sent this piece, then we can break out of the retry loop
break;
}
}
}
}
public void Delete(string remotename)
{
var response = this.m_client.DeleteAsync(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.m_path, NormalizeSlashes(remotename))).Await();
this.CheckResponse(response);
}
public void Test()
{
try
{
string rootPath = string.Format("{0}/root:{1}", this.DrivePrefix, this.m_path);
DriveItem rootFolder = this.Get<DriveItem>(rootPath);
}
catch (DriveItemNotFoundException ex)
{
// Wrap the existing item not found error in a 'FolderMissingException'
throw new FolderMissingException(ex);
}
}
public void Dispose()
{
if (this.m_client != null)
{
this.m_client.Dispose();
}
}
protected virtual string GetRootPathFromUrl(string url)
{
// Extract out the path to the backup root folder from the given URI
var uri = new Utility.Uri(url);
return Utility.Uri.UrlDecode(uri.HostAndPath);
}
protected T Get<T>(string url)
{
return this.SendRequest<T>(HttpMethod.Get, url);
}
protected T Post<T>(string url, T body)
{
return this.SendRequest(HttpMethod.Post, url, body);
}
protected T Patch<T>(string url, T body)
{
return this.SendRequest(PatchMethod, url, body);
}
private T SendRequest<T>(HttpMethod method, string url)
{
var request = new HttpRequestMessage(method, url);
return this.SendRequest<T>(request);
}
private T SendRequest<T>(HttpMethod method, string url, T body)
{
var request = new HttpRequestMessage(method, url);
if (body != null)
{
request.Content = this.PrepareContent(body);
}
return this.SendRequest<T>(request);
}
private T SendRequest<T>(HttpRequestMessage request)
{
var response = this.m_client.SendAsync(request).Await();
return this.ParseResponse<T>(response);
}
private IEnumerable<T> Enumerate<T>(string url)
{
string nextUrl = url;
while (!string.IsNullOrEmpty(nextUrl))
{
GraphCollection<T> results = this.Get<GraphCollection<T>>(nextUrl);
foreach (T result in results.Value)
{
yield return result;
}
nextUrl = results.ODataNextLink;
}
}
private void CheckResponse(HttpResponseMessage response)
{
if (!response.IsSuccessStatusCode)
{
if (response.StatusCode == HttpStatusCode.NotFound)
{
// It looks like this is an 'item not found' exception, so wrap it in a new exception class to make it easier to pick things out.
throw new DriveItemNotFoundException(response);
}
else
{
// Throw a wrapper exception to make it easier for the caller to look at specific status codes, etc.
throw new MicrosoftGraphException(response);
}
}
}
private T ParseResponse<T>(HttpResponseMessage response)
{
this.CheckResponse(response);
using (Stream responseStream = response.Content.ReadAsStreamAsync().Await())
using (StreamReader reader = new StreamReader(responseStream))
using (JsonTextReader jsonReader = new JsonTextReader(reader))
{
return this.m_serializer.Deserialize<T>(jsonReader);
}
}
/// <summary>
/// Normalizes the slashes in a url fragment. For example:
/// "" => ""
/// "test" => "/test"
/// "test/" => "/test"
/// "a\b" => "/a/b"
/// </summary>
/// <param name="url">Url fragment to normalize</param>
/// <returns>Normalized fragment</returns>
private static string NormalizeSlashes(string url)
{
url = url.Replace('\\', '/');
if (url.Length != 0 && !url.StartsWith("/", StringComparison.Ordinal))
url = "/" + url;
if (url.EndsWith("/", StringComparison.Ordinal))
url = url.Substring(0, url.Length - 1);
return url;
}
private StringContent PrepareContent<T>(T body)
{
return new StringContent(JsonConvert.SerializeObject(body), Encoding.UTF8, "application/json");
}
}
}
@@ -0,0 +1,552 @@
using System;
using System.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
/// <summary>
/// Types are based on definitions from:
/// https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/onedrive
///
/// Note that some classes don't have the full set of properties defined, particularly if they don't seem like they are needed.
/// </summary>
namespace Duplicati.Library.Backend.MicrosoftGraph
{
public class Identity
{
[JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)]
public string Id { get; set; }
[JsonProperty("displayName", NullValueHandling = NullValueHandling.Ignore)]
public string DisplayName { get; set; }
}
public class IdentitySet
{
/// <summary>
/// The optional application associated with this action
/// </summary>
[JsonProperty("application", NullValueHandling = NullValueHandling.Ignore)]
public Identity Application { get; set; }
/// <summary>
/// The optional device associated with this action
/// </summary>
[JsonProperty("device", NullValueHandling = NullValueHandling.Ignore)]
public Identity Device { get; set; }
/// <summary>
/// The optional user associated with this action
/// </summary>
[JsonProperty("user", NullValueHandling = NullValueHandling.Ignore)]
public Identity User { get; set; }
}
public class BaseItem
{
[JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)]
public string Id { get; set; }
[JsonProperty("createdBy", NullValueHandling = NullValueHandling.Ignore)]
public IdentitySet CreatedBy { get; set; }
[JsonProperty("createdDateTime", NullValueHandling = NullValueHandling.Ignore)]
public DateTimeOffset? CreatedDateTime { get; set; }
[JsonProperty("description", NullValueHandling = NullValueHandling.Ignore)]
public string Description { get; set; }
[JsonProperty("lastModifiedBy", NullValueHandling = NullValueHandling.Ignore)]
public IdentitySet LastModifiedBy { get; set; }
[JsonProperty("lastModifiedDateTime", NullValueHandling = NullValueHandling.Ignore)]
public DateTimeOffset? LastModifiedDateTime { get; set; }
/// <summary>
/// Note: OneDrive and OneDrive for Business don't allow the following characters in file names:
/// " * : < > ? / \ |
/// https://support.office.com/en-us/article/Invalid-file-names-and-file-types-in-OneDrive-OneDrive-for-Business-and-SharePoint-64883a5d-228e-48f5-b3d2-eb39e07630fa
/// If appears it also follows the Windows conventions for handling leading and trailing spaces,
/// meaning the ASCII space character is trimmed off of both the front and back of the file name:
/// https://support.microsoft.com/en-us/help/2829981/support-for-whitespace-characters-in-file-and-folder-names-for-windows
/// </summary>
[JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)]
public string Name { get; set; }
[JsonProperty("webUrl", NullValueHandling = NullValueHandling.Ignore)]
public string WebUrl { get; set; }
}
public enum QuotaState
{
/// <summary>
/// The drive has plenty of remaining quota
/// </summary>
Normal,
/// <summary>
/// Remaining quota is under 10%
/// </summary>
Nearing,
/// <summary>
/// Remaining quota is under 1%
/// </summary>
Critical,
/// <summary>
/// No remaining quota - files can only be deleted and no new files can be added
/// </summary>
Exceeded,
// Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums
// without respecting cases. Once we can use that, we can remove these copies of the nicely named enums.
normal = Normal,
nearing = Nearing,
critical = Critical,
exceeded = Exceeded,
}
public class Quota
{
[JsonProperty("total", NullValueHandling = NullValueHandling.Ignore)]
public long Total { get; set; }
[JsonProperty("used", NullValueHandling = NullValueHandling.Ignore)]
public long Used { get; set; }
[JsonProperty("remaining", NullValueHandling = NullValueHandling.Ignore)]
public long Remaining { get; set; }
[JsonProperty("deleted", NullValueHandling = NullValueHandling.Ignore)]
public long Deleted { get; set; }
[JsonProperty("state", NullValueHandling = NullValueHandling.Ignore)]
[JsonConverter(typeof(StringEnumConverter))]
public QuotaState? State { get; set; }
}
public enum DriveType
{
/// <summary>
/// OneDrive personal drives
/// </summary>
Personal,
/// <summary>
/// OneDrive for Business drives
/// </summary>
Business,
/// <summary>
/// SharePoint document libraries
/// </summary>
DocumentLibrary,
// Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums
// without respecting cases. Once we can use that, we can remove these copies of the nicely named enums.
personal = Personal,
business = Business,
documentLibrary = DocumentLibrary,
}
public class Drive : BaseItem
{
[JsonProperty("driveType", NullValueHandling = NullValueHandling.Ignore)]
[JsonConverter(typeof(StringEnumConverter))]
public DriveType DriveType { get; set; }
[JsonProperty("owner", NullValueHandling = NullValueHandling.Ignore)]
public IdentitySet Owner { get; set; }
[JsonProperty("quota", NullValueHandling = NullValueHandling.Ignore)]
public Quota Quota { get; set; }
}
public class HashesType
{
/// <summary>
/// SHA1 hash
/// Available only in OneDrive personal
/// </summary>
[JsonProperty("sha1Hash", NullValueHandling = NullValueHandling.Ignore)]
public string Sha1Hash { get; set; }
/// <summary>
/// CRC32 hash
/// Available only in OneDrive personal
/// </summary>
[JsonProperty("crc32Hash", NullValueHandling = NullValueHandling.Ignore)]
public string Crc32Hash { get; set; }
/// <summary>
/// A proprietary hash of the file that can be used to determine if the file's contents have changed.
/// Available on in OneDrive for Business and SharePoint Server 2016.
/// </summary>
[JsonProperty("quickXorHash", NullValueHandling = NullValueHandling.Ignore)]
public string QuickXorHash { get; set; }
}
public class FileFacet
{
[JsonProperty("hashes", NullValueHandling = NullValueHandling.Ignore)]
public HashesType Hashes { get; set; }
[JsonProperty("mimeType", NullValueHandling = NullValueHandling.Ignore)]
public string MimeType { get; set; }
}
public enum SortBy
{
Default = 0,
/// <summary>
/// Sorted by 'name' property
/// </summary>
Name,
/// <summary>
/// Sorted by type of item
/// </summary>
Type,
/// <summary>
/// Sorted by 'size' property
/// </summary>
Size,
/// <summary>
/// Sorted by 'takenDateTime' property of the photos facet, or 'createdDateTime' if that isn't available
/// </summary>
TakenOrCreatedDateTime,
/// <summary>
/// Sorted by 'lastModifiedDateTime' property
/// </summary>
LastModifiedDateTime,
/// <summary>
/// Sorted by a custom user specified sequence
/// </summary>
Sequence,
// Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums
// without respecting cases. Once we can use that, we can remove these copies of the nicely named enums.
@default = Default,
name = Name,
type = Type,
size = Size,
takenOrCreatedDateTime = TakenOrCreatedDateTime,
lastModifiedDateTime = LastModifiedDateTime,
sequence = Sequence,
}
public enum SortOrder
{
Ascending,
Descending,
// Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums
// without respecting cases. Once we can use that, we can remove these copies of the nicely named enums.
ascending = Ascending,
descending = Descending,
}
public enum ViewType
{
Default = 0,
Icons,
Details,
Thumbnails,
// Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums
// without respecting cases. Once we can use that, we can remove these copies of the nicely named enums.
@default = Default,
icons = Icons,
details = Details,
thumbnails = Thumbnails,
}
public class FolderView
{
[JsonProperty("sortBy", NullValueHandling = NullValueHandling.Ignore)]
[JsonConverter(typeof(StringEnumConverter))]
public SortBy? SortBy { get; set; }
[JsonProperty("sortOrder", NullValueHandling = NullValueHandling.Ignore)]
[JsonConverter(typeof(StringEnumConverter))]
public SortOrder? SortOrder { get; set; }
[JsonProperty("viewType", NullValueHandling = NullValueHandling.Ignore)]
[JsonConverter(typeof(StringEnumConverter))]
public ViewType? ViewType { get; set; }
}
public class FolderFacet
{
[JsonProperty("childCount", NullValueHandling = NullValueHandling.Ignore)]
public long? ChildCount { get; set; }
[JsonProperty("view", NullValueHandling = NullValueHandling.Ignore)]
public FolderView View { get; set; }
}
/// <summary>
/// Note: These results are different from those in the core DriveItem type -
/// they are the times the client reported from the local file system.
/// For example, if a file is created locally on Monday, but uploaded on Tuesday,
/// the DriveItem.CreatedDateTime will be Tuesday, but FileSystemInfoFacet.CreatedDateTime will be Monday.
/// </summary>
public class FileSystemInfoFacet
{
[JsonProperty("createdDateTime", NullValueHandling = NullValueHandling.Ignore)]
public DateTimeOffset? CreatedDateTime { get; set; }
/// <summary>
/// This is not available in OneDrive for Business or SharePoint.
/// </summary>
[JsonProperty("lastAccessedDateTime", NullValueHandling = NullValueHandling.Ignore)]
public DateTimeOffset? LastAccessedDateTime { get; set; }
[JsonProperty("lastModifiedDateTime", NullValueHandling = NullValueHandling.Ignore)]
public DateTimeOffset? LastModifiedDateTime { get; set; }
}
public class PackageFacet
{
/// <summary>
/// Package type - currently only oneNote is defined, but others could be used.
/// </summary>
[JsonProperty("type", NullValueHandling = NullValueHandling.Ignore)]
public string Type { get; set; }
}
public class DeletedFacet
{
[JsonProperty("state", NullValueHandling = NullValueHandling.Ignore)]
public string State { get; set; }
}
public class RemoteItemFacet
{
[JsonProperty("remoteItem", NullValueHandling = NullValueHandling.Ignore)]
public RemoteItemFacet RemoteItem { get; set; }
}
public class RootFacet
{
}
public class SpecialFolderFacet
{
/// <summary>
/// The unique identifier which can be used under /drive/special/
/// </summary>
[JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)]
public string Name { get; set; }
}
/// <summary>
/// To get an item from one of these references:
/// GET https://graph.microsoft.com/v1.0/drives/{driveId}/items/{id}
/// </summary>
public class ItemReference
{
[JsonProperty("driveId", NullValueHandling = NullValueHandling.Ignore)]
public string DriveId { get; set; }
[JsonProperty("driveType", NullValueHandling = NullValueHandling.Ignore)]
public DriveType DriveType { get; set; }
[JsonProperty("id", NullValueHandling = NullValueHandling.Ignore)]
public string Id { get; set; }
[JsonProperty("name", NullValueHandling = NullValueHandling.Ignore)]
public string Name { get; set; }
[JsonProperty("path", NullValueHandling = NullValueHandling.Ignore)]
public string Path { get; set; }
}
public enum ConflictBehavior
{
Fail,
Replace,
Rename,
// Newer versions of JSON.NET add StringEnumCaseInsensitiveConverter, which can be used to serialize/deserialize enums
// without respecting cases. Once we can use that, we can remove these copies of the nicely named enums.
fail = Fail,
replace = Replace,
rename = Rename,
}
public class DriveItem : BaseItem
{
[JsonProperty("parentReference", NullValueHandling = NullValueHandling.Ignore)]
public ItemReference ParentReference { get; set; }
[JsonProperty("size", NullValueHandling = NullValueHandling.Ignore)]
public long? Size { get; set; }
[JsonProperty("file", NullValueHandling = NullValueHandling.Ignore)]
public FileFacet File { get; set; }
[JsonProperty("folder", NullValueHandling = NullValueHandling.Ignore)]
public FolderFacet Folder { get; set; }
[JsonProperty("fileSystemInfo", NullValueHandling = NullValueHandling.Ignore)]
public FileSystemInfoFacet FileSystemInfo { get; set; }
[JsonProperty("package", NullValueHandling = NullValueHandling.Ignore)]
public PackageFacet Package { get; set; }
[JsonProperty("deleted", NullValueHandling = NullValueHandling.Ignore)]
public DeletedFacet Deleted { get; set; }
[JsonProperty("remoteItem", NullValueHandling = NullValueHandling.Ignore)]
public RemoteItemFacet RemoteItem { get; set; }
[JsonProperty("root", NullValueHandling = NullValueHandling.Ignore)]
public RootFacet Root { get; set; }
[JsonProperty("specialFolder", NullValueHandling = NullValueHandling.Ignore)]
public SpecialFolderFacet SpecialFolder { get; set; }
[JsonProperty("@microsoft.graph.conflictBehavior", NullValueHandling = NullValueHandling.Ignore)]
[JsonConverter(typeof(StringEnumConverter))]
public ConflictBehavior? ConflictBehavior { get; set; }
[JsonProperty("@microsoft.graph.downloadUrl", NullValueHandling = NullValueHandling.Ignore)]
public string DownloadUrl { get; set; }
[JsonIgnore]
public bool IsFile
{
get
{
return this.File != null;
}
}
[JsonIgnore]
public bool IsFolder
{
get
{
return this.Folder != null;
}
}
[JsonIgnore]
public bool IsPackage
{
get
{
return this.Package != null;
}
}
[JsonIgnore]
public bool TreatLikeFolder
{
get
{
return this.IsFolder || this.IsPackage;
}
}
[JsonIgnore]
public bool IsDeleted
{
get
{
return this.Deleted != null;
}
}
[JsonIgnore]
public bool IsRemoteItem
{
get
{
return this.RemoteItem != null;
}
}
[JsonIgnore]
public bool IsRoot
{
get
{
return this.Root != null;
}
}
[JsonIgnore]
public bool IsSpecialFolder
{
get
{
return this.SpecialFolder != null;
}
}
}
public class GraphCollection<T>
{
[JsonProperty("value", NullValueHandling = NullValueHandling.Ignore)]
public T[] Value { get; set; }
[JsonProperty("@odata.nextLink", NullValueHandling = NullValueHandling.Ignore)]
public string ODataNextLink { get; set; }
}
public class UploadSession
{
[JsonProperty("uploadUrl", NullValueHandling = NullValueHandling.Ignore)]
public string UploadUrl { get; set; }
[JsonProperty("expirationDateTime", NullValueHandling = NullValueHandling.Ignore)]
public DateTimeOffset? ExpirationDateTime { get; set; }
[JsonProperty("nextExpectedRanges", NullValueHandling = NullValueHandling.Ignore)]
public string[] NextExpectedRanges { get; set; }
[JsonProperty("item", NullValueHandling = NullValueHandling.Ignore)]
public DriveItem Item { get; set; }
}
public class SharePointSite : BaseItem
{
}
public class Group : BaseItem
{
[JsonProperty("mail", NullValueHandling = NullValueHandling.Ignore)]
public string Mail { get; set; }
[JsonProperty("proxyAddresses", NullValueHandling = NullValueHandling.Ignore)]
public string[] ProxyAddresses { get; set; }
[JsonProperty("mailNickname", NullValueHandling = NullValueHandling.Ignore)]
public string MailNickname { get; set; }
[JsonProperty("groupTypes", NullValueHandling = NullValueHandling.Ignore)]
public string[] GroupTypes { get; set; }
[JsonIgnore]
public bool IsUnifiedGroup
{
get
{
if (this.GroupTypes != null)
{
return this.GroupTypes.Any(type => string.Equals("Unified", type, StringComparison.OrdinalIgnoreCase));
}
return false;
}
}
}
}
@@ -0,0 +1,102 @@
using System.Collections.Generic;
using System.Linq;
using Duplicati.Library.Backend.MicrosoftGraph;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Backend
{
public class MicrosoftGroup : MicrosoftGraphBackend
{
private const string GROUP_EMAIL_OPTION = "group-email";
private const string GROUP_ID_OPTION = "group-id";
private readonly string drivePath;
public MicrosoftGroup() { } // Constructor needed for dynamic loading to find it
public MicrosoftGroup(string url, Dictionary<string, string> options)
: base(url, options)
{
string groupId = null;
string groupEmail;
if (options.TryGetValue(GROUP_EMAIL_OPTION, out groupEmail))
{
groupId = this.GetGroupIdFromEmail(groupEmail);
}
string groupIdOption;
if (options.TryGetValue(GROUP_ID_OPTION, out groupIdOption))
{
if (!string.IsNullOrEmpty(groupId) && !string.Equals(groupId, groupIdOption))
{
throw new UserInformationException(Strings.MicrosoftGroup.ConflictingGroupId(groupIdOption, groupId), "MicrosoftGroupConflictingGroupId");
}
groupId = groupIdOption;
}
if (string.IsNullOrEmpty(groupId))
{
throw new UserInformationException(Strings.MicrosoftGroup.MissingGroupIdAndEmailAddress, "MicrosoftGroupMissingGroupIdAndEmailAddress");
}
this.drivePath = string.Format("/groups/{0}/drive", groupId);
}
public override string ProtocolKey
{
get { return "msgroup"; }
}
public override string DisplayName
{
get { return Strings.MicrosoftGroup.DisplayName; }
}
protected override string DrivePath
{
get { return this.drivePath; }
}
protected override DescriptionTemplateDelegate DescriptionTemplate
{
get
{
return Strings.MicrosoftGroup.Description;
}
}
protected override IList<ICommandLineArgument> AdditionalSupportedCommands
{
get
{
return new ICommandLineArgument[]
{
new CommandLineArgument(GROUP_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.MicrosoftGroup.GroupIdShort, Strings.MicrosoftGroup.GroupIdLong),
new CommandLineArgument(GROUP_EMAIL_OPTION, CommandLineArgument.ArgumentType.String, Strings.MicrosoftGroup.GroupEmailShort, Strings.MicrosoftGroup.GroupEmailLong),
};
}
}
private string GetGroupIdFromEmail(string email)
{
// We can get all groups that have the given email as one of their addresses with:
// https://graph.microsoft.com/v1.0/groups?$filter=mail eq '{email}' or proxyAddresses/any(x:x eq 'smtp:{email}')
string request = string.Format("{0}/groups?$filter=mail eq '{1}' or proxyAddresses/any(x:x eq 'smtp:{1}')", this.ApiVersion, email);
GraphCollection<Group> groups = this.Get<GraphCollection<Group>>(request);
if (groups.Value.Length == 0)
{
throw new UserInformationException(Strings.MicrosoftGroup.NoGroupsWithEmail(email), "MicrosoftGroupNoGroupsWithEmail");
}
else if (groups.Value.Length > 1)
{
throw new UserInformationException(Strings.MicrosoftGroup.MultipleGroupsWithEmail(email), "MicrosoftGroupMultipleGroupsWithEmail");
}
else
{
return groups.Value.Single().Id;
}
}
}
}
@@ -9,6 +9,9 @@ namespace Duplicati.Library.Backend
{
public class OneDrive : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend
{
private const string SERVICES_AGREEMENT = "https://www.microsoft.com/en-us/servicesagreement";
private const string PRIVACY_STATEMENT = "https://privacy.microsoft.com/en-us/privacystatement";
private const string AUTHID_OPTION = "authid";
private const string WLID_SERVER = "https://apis.live.net/v5.0";
@@ -381,9 +384,9 @@ namespace Duplicati.Library.Backend
{
get { return Strings.OneDrive.Description(
"Microsoft Service Agreement",
"http://explore.live.com/microsoft-service-agreement",
SERVICES_AGREEMENT,
"Microsoft Online Privacy Statement",
"http://privacy.microsoft.com/en-us/fullnotice.mspx"
PRIVACY_STATEMENT
);
}
}
@@ -0,0 +1,65 @@
using System.Collections.Generic;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Backend
{
public class OneDriveV2 : MicrosoftGraphBackend
{
private const string DRIVE_ID_OPTION = "drive-id";
private const string DEFAULT_DRIVE_PATH = "/me/drive";
private readonly string drivePath;
public OneDriveV2() { } // Constructor needed for dynamic loading to find it
public OneDriveV2(string url, Dictionary<string, string> options)
: base(url, options)
{
string driveId;
if (options.TryGetValue(DRIVE_ID_OPTION, out driveId))
{
this.drivePath = string.Format("/drives/{0}", driveId);
}
else
{
this.drivePath = DEFAULT_DRIVE_PATH;
}
}
public override string ProtocolKey
{
get { return "onedrivev2"; }
}
public override string DisplayName
{
get { return Strings.OneDriveV2.DisplayName; }
}
protected override string DrivePath
{
get { return this.drivePath; }
}
protected override DescriptionTemplateDelegate DescriptionTemplate
{
get
{
return Strings.OneDriveV2.Description;
}
}
protected override IList<ICommandLineArgument> AdditionalSupportedCommands
{
get
{
return new ICommandLineArgument[]
{
new CommandLineArgument(DRIVE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.OneDriveV2.DriveIdShort, Strings.OneDriveV2.DriveIdLong(DEFAULT_DRIVE_PATH)),
};
}
}
}
}
@@ -0,0 +1,164 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Duplicati.Library.Backend.MicrosoftGraph;
using Duplicati.Library.Interface;
namespace Duplicati.Library.Backend
{
public class SharePointV2 : MicrosoftGraphBackend
{
private const string SITE_ID_OPTION = "site-id";
private readonly string drivePath;
private string siteId = null;
public SharePointV2() { } // Constructor needed for dynamic loading to find it
public SharePointV2(string url, Dictionary<string, string> options)
: base(url, options)
{
// Check to see if a site ID was explicitly provided
string siteIdOption;
if (options.TryGetValue(SITE_ID_OPTION, out siteIdOption))
{
if (!string.IsNullOrEmpty(this.siteId) && !string.Equals(this.siteId, siteIdOption))
{
throw new UserInformationException(Strings.SharePointV2.ConflictingSiteId(siteIdOption, this.siteId), "SharePointConflictingSiteId");
}
this.siteId = siteIdOption;
}
if (string.IsNullOrEmpty(this.siteId))
{
throw new UserInformationException(Strings.SharePointV2.MissingSiteId, "SharePointMissingSiteId");
}
this.drivePath = string.Format("/sites/{0}/drive", this.siteId);
}
public override string ProtocolKey
{
get { return "sharepoint"; }
}
public override string DisplayName
{
get { return Strings.SharePointV2.DisplayName; }
}
protected override string DrivePath
{
get { return this.drivePath; }
}
protected override DescriptionTemplateDelegate DescriptionTemplate
{
get
{
return Strings.SharePointV2.Description;
}
}
protected override IList<ICommandLineArgument> AdditionalSupportedCommands
{
get
{
return new ICommandLineArgument[]
{
new CommandLineArgument(SITE_ID_OPTION, CommandLineArgument.ArgumentType.String, Strings.SharePointV2.SiteIdShort, Strings.SharePointV2.SiteIdLong),
};
}
}
/// <summary>
/// This method takes an input URL, which could be either in the format that the old SharePoint backend accepted or
/// just the host and path (effectively, just the path) that the graph base backend expects,
/// and converts it to just the local path within the drive.
///
/// At the same time, if the given URL is in the form of a full SharePoint site URL, then it also determines the site-id and stores it.
/// </summary>
/// <param name="url">Input URL</param>
/// <returns>Path within the drive</returns>
protected override string GetRootPathFromUrl(string url)
{
Uri uri = new Uri(url);
// If the user gave a URL like "https://{tenant}.sharepoint.com/path" in the UI,
// it might appear here as "sharepoint://https://{tenant}.sharepoint.com/path".
// If that's the case, the URIs Host will be https, and we should create a new URI without it.
if (string.Equals(uri.Host, "https", StringComparison.OrdinalIgnoreCase) || string.Equals(uri.Host, "http", StringComparison.OrdinalIgnoreCase))
{
// LocalPath will already be prefixed by "//", due to the https:// part.
uri = new Uri(string.Format("{0}:{1}", uri.Scheme, uri.LocalPath));
}
SharePointSite site = this.GetSharePointSite(uri);
if (site != null)
{
// Get the web URL of the site's main drive
try
{
Drive drive = this.Get<Drive>(string.Format("{0}/sites/{1}/drive", this.ApiVersion, site.Id));
this.siteId = site.Id;
Uri driveWebUrl = new Uri(drive.WebUrl);
// Make sure to replace any "//" in the original path with "/", so the substrings line up.
return uri.LocalPath.Replace("//", "/").Substring(driveWebUrl.LocalPath.Length);
}
catch (MicrosoftGraphException)
{
// Couldn't get the drive info, so assume the URL we were given isn't actually a full SharePoint site.
}
}
return base.GetRootPathFromUrl(url);
}
private SharePointSite GetSharePointSite(Uri url)
{
UriBuilder uri = new UriBuilder(url);
// We can get a SharePoint site's info by querying /v1.0/sites/{hostname}:{siteWebPath}.
// Since this full URL likely has the web path as some subpart of it, we check against each subpath to see if that is a site,
// and if it is, we record the site ID.
string requestBase = string.Format("{0}/sites/{1}", this.ApiVersion, uri.Host);
// Just like the original SharePoint backend, use the "//" as a hint at where the site might be
int siteHint = uri.Path.IndexOf("//");
if (siteHint >= 0)
{
try
{
string request = string.Format("{0}:/{1}", requestBase, uri.Path.Substring(0, siteHint));
return this.Get<SharePointSite>(request);
}
catch (MicrosoftGraphException)
{
// This isn't the right path
}
}
string[] pathPieces = uri.Path.Split(new[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }, StringSplitOptions.RemoveEmptyEntries);
for (int i = pathPieces.Length; i > 0; i--)
{
try
{
string request = string.Format("{0}:/{1}", requestBase, string.Join("/", pathPieces.Take(i)));
return this.Get<SharePointSite>(request);
}
catch (MicrosoftGraphException)
{
// This isn't the right path
}
}
// Couldn't find the site
return null;
}
}
}
@@ -11,7 +11,51 @@ namespace Duplicati.Library.Backend.Strings
public static string AuthidShort { get { return LC.L(@"The authorization code"); } }
public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); }
public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files on Microsoft OneDrive. Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); }
}
internal static class MicrosoftGraph
{
public static string AuthIdShort { get { return OneDrive.AuthidShort; } }
public static string AuthIdLong(string url) { return OneDrive.AuthidLong(url); }
public static string MissingAuthId(string url) { return LC.L(@"No Auth-ID was provided - you can get one from {0}", url); }
public static string FragmentSizeShort { get { return LC.L(@"Fragment size for large uploads"); } }
public static string FragmentSizeLong { get { return LC.L(@"Size of individual fragments which are uploaded separately for large files. It is recommended to be between 5-10 MiB (though a smaller value may work better on a slower or less reliable connection), and to be a multiple of 320 KiB."); } }
public static string FragmentRetryCountShort { get { return LC.L(@"Number of retries for each fragment"); } }
public static string FragmentRetryCountLong { get { return LC.L(@"Number of retry attempts made for each fragment before failing the overall file upload"); } }
public static string FragmentRetryDelayShort { get { return LC.L(@"Millisecond delay between fragment errors"); } }
public static string FragmentRetryDelayLong { get { return LC.L(@"Amount of time (in milliseconds) to wait between failures when uploading fragments"); } }
}
internal static class OneDriveV2
{
public static string DisplayName { get { return LC.L(@"Microsoft OneDrive v2"); } }
public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files in Microsoft OneDrive or Microsoft OneDrive for Business via the Microsoft Graph API. Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); }
public static string DriveIdShort { get { return LC.L(@"Optional ID of the drive"); } }
public static string DriveIdLong(string defaultDrive) { return LC.L(@"ID of the drive to store data in. If no drive is specified, the default OneDrive or OneDrive for Business drive will be used via '{0}'.", defaultDrive); }
}
internal static class SharePointV2
{
public static string DisplayName { get { return LC.L(@"Microsoft SharePoint v2"); } }
public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files in a Microsoft SharePoint site via the Microsoft Graph API. Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); }
public static string SiteIdShort { get { return LC.L(@"ID of the site"); } }
public static string SiteIdLong { get { return LC.L(@"ID of the site to store data in"); } }
public static string MissingSiteId { get { return LC.L(@"No site ID was provided"); } }
public static string ConflictingSiteId(string given, string found) { return LC.L(@"Conflicting site IDs used: given {0} but found {1}", given, found); }
}
internal static class MicrosoftGroup
{
public static string DisplayName { get { return LC.L(@"Microsoft Office 365 Group"); } }
public static string Description(string mssadescription, string mssalink, string msopdescription, string msoplink) { return LC.L(@"Stores files in a Microsoft Office 365 Group via the Microsoft Graph API. Allowed formats are ""sharepoint://tenant.sharepoint.com/{{PathToWeb}}//{{Documents}}/subfolder"" (with ""//"" being optionally used to indicate the root document folder), or just ""sharepoint://subfolder"" (in which case you must also explicitly specify the SharePoint site's ID via --site-id). Usage of this backend requires that you agree to the terms in {0} ({1}) and {2} ({3})", mssadescription, mssalink, msopdescription, msoplink); }
public static string GroupIdShort { get { return LC.L(@"ID of the group"); } }
public static string GroupIdLong { get { return LC.L(@"ID of the group to store data in"); } }
public static string GroupEmailShort { get { return LC.L(@"Email address of the group"); } }
public static string GroupEmailLong { get { return LC.L(@"Email address of the group to store data in"); } }
public static string MissingGroupIdAndEmailAddress { get { return LC.L(@"No group ID or group email address was provided"); } }
public static string NoGroupsWithEmail(string email) { return LC.L(@"No groups were found with the given email address: {0}", email); }
public static string MultipleGroupsWithEmail(string email) { return LC.L(@"Multiple groups were found with the given email address: {0}", email); }
public static string ConflictingGroupId(string given, string found) { return LC.L(@"Conflicting group IDs used: given {0} but found {1}", given, found); }
}
}
@@ -98,7 +98,7 @@ namespace Duplicati.Library.DynamicLoader
{
// Unwrap exceptions for nicer display
if (tex.InnerException != null)
throw tex.InnerException;
throw new Exception("Unwrapped TargetInvocationException", tex.InnerException);
throw;
}
+1 -1
View File
@@ -248,7 +248,7 @@ namespace Duplicati.Library.Main.Strings
public static string LogfileloglevelLong { get { return LC.L(@"Specifies the amount of log information to write into the file specified by --log-file"); } }
public static string LogfileloglevelShort { get { return LC.L(@"Log file information level"); } }
public static string LogfilelogfiltersShort { get { return LC.L(@"Applies filters to the file log data"); } }
public static string LogfilelogfiltersLong(string delimiter) { return LC.L(@"This option accepts filters that removes or includes messages regardless of their log level. Multiple filters are supported by separating with {0}. Filters are matched against the log tag and assumed to be including, unless they start with '-'. Regular expressions are supported within hard braces. Example: ""+Path*{0}+*Mail*{0}-[.*DNS]"" "); }
public static string LogfilelogfiltersLong(string delimiter) { return LC.L(@"This option accepts filters that removes or includes messages regardless of their log level. Multiple filters are supported by separating with {0}. Filters are matched against the log tag and assumed to be including, unless they start with '-'. Regular expressions are supported within hard braces. Example: ""+Path*{0}+*Mail*{0}-[.*DNS]"" ", delimiter); }
public static string ConsoleloglevelLong { get { return LC.L(@"Specifies the amount of log information to write as console output"); } }
public static string ConsoleloglevelShort { get { return LC.L(@"Console information level"); } }
public static string ConsolelogfiltersShort { get { return LC.L(@"Applies filters to the console log data"); } }
+12 -8
View File
@@ -30,9 +30,13 @@ namespace Duplicati.Library.Utility
if (string.IsNullOrEmpty(size))
return 0;
size = size.ToLower().Trim();
size = size.Trim();
if (size.EndsWith("tb", StringComparison.Ordinal) || size.EndsWith("gb", StringComparison.Ordinal) || size.EndsWith("mb", StringComparison.Ordinal) || size.EndsWith("kb", StringComparison.Ordinal) || size.EndsWith("b", StringComparison.Ordinal))
if (size.EndsWith("tb", StringComparison.OrdinalIgnoreCase) ||
size.EndsWith("gb", StringComparison.OrdinalIgnoreCase) ||
size.EndsWith("mb", StringComparison.OrdinalIgnoreCase) ||
size.EndsWith("kb", StringComparison.OrdinalIgnoreCase) ||
size.EndsWith("b", StringComparison.OrdinalIgnoreCase))
return ParseSize(size);
else
return ParseSize(size + " " + defaultSuffix);
@@ -45,31 +49,31 @@ namespace Duplicati.Library.Utility
string origsize = size;
size = size.Trim().ToLower();
size = size.Trim();
long factor = 1;
if (size.EndsWith("tb", StringComparison.Ordinal))
if (size.EndsWith("tb", StringComparison.OrdinalIgnoreCase))
{
factor = 1024L * 1024 * 1024 * 1024;
size = size.Substring(0, size.Length - 2).Trim();
}
else if (size.EndsWith("gb", StringComparison.Ordinal))
else if (size.EndsWith("gb", StringComparison.OrdinalIgnoreCase))
{
factor = 1024 * 1024 * 1024;
size = size.Substring(0, size.Length - 2).Trim();
}
else if (size.EndsWith("mb", StringComparison.Ordinal))
else if (size.EndsWith("mb", StringComparison.OrdinalIgnoreCase))
{
factor = 1024 * 1024;
size = size.Substring(0, size.Length - 2).Trim();
}
else if (size.EndsWith("kb", StringComparison.Ordinal))
else if (size.EndsWith("kb", StringComparison.OrdinalIgnoreCase))
{
factor = 1024;
size = size.Substring(0, size.Length - 2).Trim();
}
else if (size.EndsWith("b", StringComparison.Ordinal))
else if (size.EndsWith("b", StringComparison.OrdinalIgnoreCase))
size = size.Substring(0, size.Length - 1).Trim();
long r;
+27
View File
@@ -24,6 +24,7 @@ using System.Linq;
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;
namespace Duplicati.Library.Utility
{
@@ -1572,5 +1573,31 @@ namespace Duplicati.Library.Utility
{
return string.Join(" ", args.Select(x => WrapCommandLineElement(x, allowEnvExpansion)));
}
/// <summary>
/// Utility method that emulates C#'s built in await keyword without requiring the calling method to be async.
/// This method should be preferred over using Task.Result, as it doesn't wrap singular exceptions in AggregateExceptions.
/// (It uses Task.GetAwaiter().GetResult(), which is the same thing that await uses under the covers.)
/// https://stackoverflow.com/questions/17284517/is-task-result-the-same-as-getawaiter-getresult
/// </summary>
/// <param name="task">Task to await</param>
public static void Await(this Task task)
{
task.GetAwaiter().GetResult();
}
/// <summary>
/// Utility method that emulates C#'s built in await keyword without requiring the calling method to be async.
/// This method should be preferred over using Task.Result, as it doesn't wrap singular exceptions in AggregateExceptions.
/// (It uses Task.GetAwaiter().GetResult(), which is the same thing that await uses under the covers.)
/// https://stackoverflow.com/questions/17284517/is-task-result-the-same-as-getawaiter-getresult
/// </summary>
/// <typeparam name="T">Result type</typeparam>
/// <param name="task">Task to await</param>
/// <returns>Task result</returns>
public static T Await<T>(this Task<T> task)
{
return task.GetAwaiter().GetResult();
}
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 871 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 887 B

+11 -2
View File
@@ -906,10 +906,19 @@ body {
border-top: 0px none;
}
.progress-small {
text-align: center;
height: 18px;
background: rgba(164, 209,235, 0.50);
}
.progress-small-bg {
border: 1px #65b1dd solid;
width: 200px;
}
a {
font-size: 30px;
padding-left: 55px;
background: url('../img/backup.png') no-repeat 5px 6px;
font-weight: 300;
display: inline-block;
@@ -80,6 +80,13 @@ backupApp.service('EditUriBackendConfig', function(AppService, AppUtils, SystemI
return true;
};
this.recommend_field = function (scope, field, label, reason, continuation) {
if ((scope[field] || '').trim().length == 0)
return self.show_warning_dialog(gettextCatalog.getString('You should fill in {{field}}{{reason}}', { field: label || field, reason: reason }), continuation);
else
continuation();
};
this.require_server = function(scope) {
if ((scope.Server || '').trim().length == 0)
return self.show_error_dialog(gettextCatalog.getString('You must fill in the server name or address'));
@@ -17,6 +17,9 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo,
EditUriBackendConfig.templates['googledrive'] = 'templates/backends/oauth.html';
EditUriBackendConfig.templates['hubic'] = 'templates/backends/oauth.html';
EditUriBackendConfig.templates['onedrive'] = 'templates/backends/oauth.html';
EditUriBackendConfig.templates['onedrivev2'] = 'templates/backends/oauth.html';
EditUriBackendConfig.templates['sharepoint'] = 'templates/backends/sharepoint.html';
EditUriBackendConfig.templates['msgroup'] = 'templates/backends/msgroup.html';
EditUriBackendConfig.templates['amzcd'] = 'templates/backends/oauth.html';
EditUriBackendConfig.templates['openstack'] = 'templates/backends/openstack.html';
EditUriBackendConfig.templates['azure'] = 'templates/backends/azure.html';
@@ -193,6 +196,9 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo,
EditUriBackendConfig.loaders['googledrive'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.loaders['hubic'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.loaders['onedrive'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.loaders['onedrivev2'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.loaders['sharepoint'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.loaders['msgroup'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.loaders['amzcd'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.loaders['box'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.loaders['dropbox'] = function() { return this['oauth-base'].apply(this, arguments); };
@@ -299,6 +305,8 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo,
EditUriBackendConfig.parsers['googledrive'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.parsers['hubic'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.parsers['onedrive'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.parsers['onedrivev2'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.parsers['sharepoint'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.parsers['amzcd'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.parsers['box'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.parsers['dropbox'] = function() { return this['oauth-base'].apply(this, arguments); };
@@ -320,6 +328,17 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo,
EditUriBackendConfig.mergeServerAndPath(scope);
};
EditUriBackendConfig.parsers['msgroup'] = function (scope, module, server, port, path, options) {
scope.msgroup_group_email = options['--group-email'];
var nukeopts = ['--group-email'];
for (var x in nukeopts)
delete options[nukeopts[x]];
this['oauth-base'].apply(this, arguments);
};
EditUriBackendConfig.parsers['gcs'] = function(scope, module, server, port, path, options) {
scope.gcs_location = scope.gcs_location_custom = options['--gcs-location'];
@@ -435,6 +454,9 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo,
EditUriBackendConfig.builders['googledrive'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.builders['hubic'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.builders['onedrive'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.builders['onedrivev2'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.builders['sharepoint'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.builders['msgroup'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.builders['amzcd'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.builders['box'] = function() { return this['oauth-base'].apply(this, arguments); };
EditUriBackendConfig.builders['dropbox'] = function() { return this['oauth-base'].apply(this, arguments); };
@@ -483,6 +505,26 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo,
return url;
};
EditUriBackendConfig.builders['msgroup'] = function (scope) {
var opts = {
'group-email': scope.msgroup_group_email,
'authid': scope.AuthID,
}
if ((opts['group-email'] || '') == '')
delete opts['group-email'];
EditUriBackendConfig.merge_in_advanced_options(scope, opts);
var url = AppUtils.format('{0}://{1}{2}',
scope.Backend.Key,
scope.Path || '',
AppUtils.encodeDictAsUrl(opts)
);
return url;
};
EditUriBackendConfig.builders['gcs'] = function(scope) {
var opts = {
'gcs-location': AppUtils.contains_value(scope.gcs_locations, scope.gcs_location) ? scope.gcs_location : scope.gcs_location_custom,
@@ -643,7 +685,7 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo,
EditUriBackendConfig.validaters['tahoe'] = EditUriBackendConfig.validaters['ssh'];
EditUriBackendConfig.validaters['onedrive'] = function(scope, continuation) {
EditUriBackendConfig.validaters['authid-base'] = function(scope, continuation) {
var res =
EditUriBackendConfig.require_field(scope, 'AuthID', gettextCatalog.getString('AuthID'));
@@ -651,17 +693,48 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo,
EditUriBackendConfig.recommend_path(scope, continuation);
};
EditUriBackendConfig.validaters['hubic'] = function(scope, continuation) {
EditUriBackendConfig.validaters['googledrive'] = EditUriBackendConfig.validaters['authid-base'];
EditUriBackendConfig.validaters['gcs'] = EditUriBackendConfig.validaters['authid-base'];
EditUriBackendConfig.validaters['amzcd'] = EditUriBackendConfig.validaters['authid-base'];
EditUriBackendConfig.validaters['box'] = EditUriBackendConfig.validaters['authid-base'];
EditUriBackendConfig.validaters['dropbox'] = EditUriBackendConfig.validaters['authid-base'];
EditUriBackendConfig.validaters['onedrive'] = EditUriBackendConfig.validaters['authid-base'];
EditUriBackendConfig.validaters['onedrivev2'] = EditUriBackendConfig.validaters['authid-base'];
EditUriBackendConfig.validaters['sharepoint'] = EditUriBackendConfig.validaters['authid-base'];
EditUriBackendConfig.validaters['msgroup'] = function (scope, continuation) {
EditUriBackendConfig.validaters['authid-base'](scope, function () {
var res =
EditUriBackendConfig.recommend_field(scope, 'msgroup_group_email', gettextCatalog.getString('Group email'), gettextCatalog.getString(' unless you are explicitly specifying --group-id'), continuation);
if (res)
continuation();
});
};
EditUriBackendConfig.validaters['msgroup'] = function (scope, continuation) {
EditUriBackendConfig.validaters['authid-base'](scope, function () {
var res =
EditUriBackendConfig.recommend_field(scope, 'msgroup_group_email', gettextCatalog.getString('Group email'), gettextCatalog.getString(' unless you are explicitly specifying --group-id'), continuation);
if (res)
continuation();
});
};
EditUriBackendConfig.validaters['hubic'] = function (scope, continuation) {
var prefix1 = 'HubiC-DeskBackup_Duplicati/';
var prefix2 = 'default/'
EditUriBackendConfig.validaters['onedrive'](scope, function() {
EditUriBackendConfig.validaters['authid-base'](scope, function () {
var p = (scope.Path || '').trim();
if (p.length > 0 && p.indexOf(prefix2) != 0 && p.indexOf(prefix1) != 0) {
DialogService.dialog(gettextCatalog.getString('Adjust path name?'), gettextCatalog.getString('The path should start with "{{prefix1}}" or "{{prefix2}}", otherwise you will not be able to see the files in the HubiC web interface.\n\nDo you want to add the prefix to the path automatically?', {prefix1: prefix1, prefix2: prefix2 }), [gettextCatalog.getString('Cancel'), gettextCatalog.getString('No'), gettextCatalog.getString('Yes')], function(ix) {
DialogService.dialog(gettextCatalog.getString('Adjust path name?'), gettextCatalog.getString('The path should start with "{{prefix1}}" or "{{prefix2}}", otherwise you will not be able to see the files in the HubiC web interface.\n\nDo you want to add the prefix to the path automatically?', { prefix1: prefix1, prefix2: prefix2 }), [gettextCatalog.getString('Cancel'), gettextCatalog.getString('No'), gettextCatalog.getString('Yes')], function (ix) {
if (ix == 2) {
while (p.indexOf('/') == 0)
p = p.substr(1);
@@ -678,12 +751,6 @@ backupApp.service('EditUriBuiltins', function(AppService, AppUtils, SystemInfo,
};
EditUriBackendConfig.validaters['googledrive'] = EditUriBackendConfig.validaters['onedrive'];
EditUriBackendConfig.validaters['gcs'] = EditUriBackendConfig.validaters['onedrive'];
EditUriBackendConfig.validaters['amzcd'] = EditUriBackendConfig.validaters['onedrive'];
EditUriBackendConfig.validaters['box'] = EditUriBackendConfig.validaters['onedrive'];
EditUriBackendConfig.validaters['dropbox'] = EditUriBackendConfig.validaters['onedrive'];
EditUriBackendConfig.validaters['azure'] = function(scope, continuation) {
var res =
EditUriBackendConfig.require_field(scope, 'Username', gettextCatalog.getString('Account name')) &&
@@ -57,6 +57,9 @@ backupApp.service('SystemInfo', function($rootScope, $timeout, $cookies, AppServ
'azure': null,
'googledrive': null,
'onedrive': null,
'onedrivev2': null,
'sharepoint': null,
'msgroup': null,
'cloudfiles': null,
'gcs': null,
'openstack': null,
@@ -0,0 +1,17 @@
<div class="input text">
<label for="msgroup_group_email" translate>Group email</label>
<input type="text" name="msgroup_group_email" id="msgroup_group_email" ng-model="$parent.msgroup_group_email" placeholder="{{'Enter the email address of the Office 365 group' | translate}}" />
</div>
<div class="input text">
<label for="msgroup_path" translate>Path on server</label>
<input type="text" name="msgroup_path" id="msgroup_path" ng-model="$parent.Path" placeholder="{{'Enter the destination path' | translate}}" />
</div>
<div class="input text">
<label for="msgroup_authid">
<a ng-show="oauth_in_progress" href="{{oauth_start_link}}" target="_blank" translate>AuthID</a>
<a ng-hide="oauth_in_progress" href ng-click="oauth_start_token_creation()" translate>AuthID</a>
</label>
<input type="text" name="msgroup_authid" id="msgroup_authid" ng-model="$parent.AuthID" placeholder="{{'Click the AuthID link to create an AuthID' | translate}}" />
</div>
@@ -0,0 +1,12 @@
<div class="input text">
<label for="sharepoint_path" translate>Destination path</label>
<input type="text" name="sharepoint_path" id="sharepoint_path" ng-model="$parent.Path" placeholder="{{'Enter the full destination path, including the server name, but without https' | translate}}" />
</div>
<div class="input text">
<label for="sharepoint_authid">
<a ng-show="oauth_in_progress" href="{{oauth_start_link}}" target="_blank" translate>AuthID</a>
<a ng-hide="oauth_in_progress" href ng-click="oauth_start_token_creation()" translate>AuthID</a>
</label>
<input type="text" name="sharepoint_authid" id="sharepoint_authid" ng-model="$parent.AuthID" placeholder="{{'Click the AuthID link to create an AuthID' | translate}}" />
</div>
@@ -2,7 +2,11 @@
<div class="tasks" ng-controller="HomeController">
<div class="tasklist">
<div ng-repeat="item in backups" class="task">
<a href ng-click="$parent.TaskMenuID == item.Backup.ID ? $parent.TaskMenuID = null : $parent.TaskMenuID = item.Backup.ID">{{item.Backup.Name}}
<a href ng-click="$parent.TaskMenuID == item.Backup.ID ? $parent.TaskMenuID = null : $parent.TaskMenuID = item.Backup.ID">
<img ng-show="item.Backup.ID != state.activeTask.Item2" src="img/backup.png">
<img ng-show="item.Backup.ID == state.activeTask.Item2 && state.programState == 'Running'" src="img/backuprunning.png">
<img ng-show="item.Backup.ID == state.activeTask.Item2 && state.programState == 'Paused'" src="img/backuppause.png">
{{item.Backup.Name}}
<img ng-show="$parent.TaskMenuID == item.Backup.ID" src="img/arrow_collapse_blue.png">
<img ng-show="$parent.TaskMenuID != item.Backup.ID" src="img/arrow_expand_blue.png">
</a>
@@ -61,6 +65,20 @@
<dd translate translate-n="parseInt(item.Backup.Metadata.BackupListCount)" translate-plural="{{item.Backup.Metadata.TargetSizeString}} / {{$count}} Versions">{{item.Backup.Metadata.TargetSizeString}} / {{$count}} Version</dd>
</div>
</div>
<div ng-hide="item.Backup.ID != state.activeTask.Item2 || state.lastPgEvent.CurrentFilename == null">
<dt translate>Current action:</dt>
<dd>{{state.lastPgEvent.Phase}}</dd>
<dt translate>Progress:</dt>
<dd>
<div class="progress-small-bg">
<div class="progress-small" style="width: {{(1 - (state.lastPgEvent.CurrentFilesize - state.lastPgEvent.CurrentFileoffset) / state.lastPgEvent.CurrentFilesize) * 200}}px">
{{(1 - (state.lastPgEvent.CurrentFilesize - state.lastPgEvent.CurrentFileoffset) / state.lastPgEvent.CurrentFilesize) * 100 | number:2}}%
</div>
</div>
</dd>
<dt translate>Current file:</dt>
<dd>{{state.lastPgEvent.CurrentFilename}}</dd>
</div>
</dl>
</div>
</div>