haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into refactored

Browse Source
This commit is contained in:
verhoek
2018-05-18 08:05:19 +02:00
parent d9fc1cac08 b7b4fbe778
commit 2f41e02ca9
14 changed files with 478 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+1
View File
@@ -55,6 +55,7 @@ jobs:
- env: CATEGORY=Purge
- env: CATEGORY=Serialization
- env: CATEGORY=Utility
- env: CATEGORY=UriUtility
- env: CATEGORY=GUI
addons:
Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs
+1 -1
View File
@@ -134,7 +134,7 @@ namespace Duplicati.GUI.TrayIcon
public virtual IBrowserWindow ShowUrlInWindow(string url)
{
//Fallback is to just show the window in a browser
Duplicati.Library.Utility.UrlUtillity.OpenURL(url, Program.BrowserCommand);
Duplicati.Library.Utility.UrlUtility.OpenURL(url, Program.BrowserCommand);
return null;
}
Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj
+1
View File
@@ -41,6 +41,7 @@
<Compile Include="GoogleCommon.cs" />
<Compile Include="GoogleDrive.cs" />
<Compile Include="GCSConfig.cs" />
<Compile Include="WebApi.cs" />
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
<ItemGroup>
Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs
+17 -16
View File
@@ -18,7 +18,6 @@ using System;
using Duplicati.Library.Interface;
using System.Collections.Generic;
using System.Net;
using System.Web;
using Duplicati.Library.Utility;
using Newtonsoft.Json;
using System.Text;
@@ -97,9 +96,9 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
m_oauth = new OAuthHelper(authid, this.ProtocolKey);
m_oauth.AutoAuthHeader = true;
}
}
private class ListBucketResponse
{
public string kind { get; set; }
@@ -205,7 +204,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
if (string.IsNullOrEmpty(m_project))
throw new UserInformationException(Strings.GoogleCloudStorage.ProjectIDMissingError(PROJECT_OPTION), "GoogleCloudStorageMissingProjectID");
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest() {
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest
{
name = m_bucket,
location = m_location,
storageClass = m_storage_class
@@ -220,7 +220,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
var areq = new AsyncHttpRequest(req);
using(var rs = areq.GetRequestStream())
using (var rs = areq.GetRequestStream())
rs.Write(data, 0, data.Length);
m_oauth.ReadJSONResponse<BucketResourceItem>(areq);
@@ -246,8 +246,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
foreach (KeyValuePair<string, string> s in KNOWN_GCS_LOCATIONS)
locations.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
foreach (KeyValuePair<string, string> s in KNOWN_GCS_STORAGE_CLASSES)
storageClasses.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
storageClasses.AppendLine(string.Format("{0}: {1}", s.Key, s.Value));
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.LocationDescriptionShort, Strings.GoogleCloudStorage.LocationDescriptionLong(locations.ToString())),
new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.StorageclassDescriptionShort, Strings.GoogleCloudStorage.StorageclassDescriptionLong(locations.ToString())),
@@ -276,8 +276,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
var res = GoogleCommon.ChunckedUploadWithResume<BucketResourceItem, BucketResourceItem>(m_oauth, item, url, stream);
if (res == null)
throw new Exception(string.Format("Upload succeeded, but no data was returned"));
throw new Exception("Upload succeeded, but no data was returned");
}
public void Get(string remotename, System.IO.Stream stream)
@@ -288,8 +288,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
var req = m_oauth.CreateRequest(url);
var areq = new AsyncHttpRequest(req);
using(var resp = areq.GetResponse())
using(var rs = areq.GetResponseStream())
using (var resp = areq.GetResponse())
using (var rs = areq.GetResponseStream())
Library.Utility.Utility.CopyStream(rs, stream);
}
catch (WebException wex)
@@ -304,7 +304,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
public void Rename(string oldname, string newname)
{
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem() {
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem
{
name = m_prefix + newname,
}));
@@ -315,7 +316,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
req.ContentType = "application/json; charset=UTF-8";
var areq = new AsyncHttpRequest(req);
using(var rs = areq.GetRequestStream())
using (var rs = areq.GetRequestStream())
rs.Write(data, 0, data.Length);
m_oauth.ReadJSONResponse<BucketResourceItem>(req);
@@ -323,8 +324,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
#region IDisposable implementation
public void Dispose()
{
{
}
#endregion
}
Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs
+125 -85
View File
@@ -15,14 +15,18 @@
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using System.Net;
using Newtonsoft.Json;
using Duplicati.Library.Backend.GoogleServices;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using System.Collections.Generic;
using System.Net;
using Newtonsoft.Json;
using Duplicati.Library.Backend.GoogleServices;
using System.Text;
namespace Duplicati.Library.Backend.GoogleDrive
{
public class GoogleDrive : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend
@@ -31,13 +35,11 @@ namespace Duplicati.Library.Backend.GoogleDrive
private const string DISABLE_TEAMDRIVE_OPTION = "googledrive-disable-teamdrive";
private const string FOLDER_MIMETYPE = "application/vnd.google-apps.folder";
private const string DRIVE_API_UPLOAD_URL = "https://www.googleapis.com/upload/drive/v2";
private const string DRIVE_API_URL = "https://www.googleapis.com/drive/v2";
private readonly string m_path;
private readonly bool m_useTeamDrive = true;
private string m_path;
private bool m_useTeamDrive = true;
private OAuthHelper m_oauth;
private readonly OAuthHelper m_oauth;
private string m_currentFolderId;
private Dictionary<string, GoogleDriveFolderItem[]> m_filecache;
@@ -66,26 +68,30 @@ namespace Duplicati.Library.Backend.GoogleDrive
private string GetFolderId(string path, bool autocreate = false)
{
var curparent = GetAboutInfo().rootFolderId;
var curdisplay = "/";
var curparent = GetAboutInfo().rootFolderId;
var curdisplay = new StringBuilder("/");
foreach(var p in path.Split(new char[] {'/'}, StringSplitOptions.RemoveEmptyEntries))
foreach (var p in path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries))
{
var res = ListFolder(curparent, true, p).ToArray();
if (res.Length == 0)
{
if (!autocreate)
throw new FolderMissingException();
curparent = CreateFolder(p, curparent).id;
}
else if (res.Length > 1)
throw new UserInformationException(Strings.GoogleDrive.MultipleEntries(p, curdisplay), "GoogleDriveMultipleEntries");
else
curparent = res[0].id;
curdisplay += p + "/";
var res = ListFolder(curparent, true, p).ToArray();
if (res.Length == 0)
{
if (!autocreate)
throw new FolderMissingException();
curparent = CreateFolder(p, curparent).id;
}
else if (res.Length > 1)
{
throw new UserInformationException(Strings.GoogleDrive.MultipleEntries(p, curdisplay.ToString()), "GoogleDriveMultipleEntries");
}
else
{
curparent = res[0].id;
}
curdisplay.Append(p).Append("/");
}
return curparent;
@@ -110,17 +116,17 @@ namespace Duplicati.Library.Backend.GoogleDrive
if (entries != null)
return entries;
var list = ListFolder(CurrentFolderId, false, remotename).ToArray();
entries = ListFolder(CurrentFolderId, false, remotename).ToArray();
if (list == null || list.Length == 0)
if (entries == null || entries.Length == 0)
{
if (throwMissingException)
throw new FileMissingException();
else
return null;
}
return m_filecache[remotename] = list;
return m_filecache[remotename] = entries;
}
private static string EscapeTitleEntries(string title)
@@ -141,27 +147,32 @@ namespace Duplicati.Library.Backend.GoogleDrive
GoogleDriveFolderItem[] files;
m_filecache.TryGetValue(remotename, out files);
string fileid = null;
string fileId = null;
if (files != null)
{
if (files.Length == 1)
fileid = files[0].id;
fileId = files[0].id;
else
Delete(remotename);
}
var isUpdate = !string.IsNullOrWhiteSpace(fileid);
var isUpdate = !string.IsNullOrWhiteSpace(fileId);
var values = new NameValueCollection {
{ WebApi.GoogleDrive.QueryParam.UploadType,
WebApi.GoogleDrive.QueryValue.Resumable } };
var url = isUpdate ?
string.Format("{0}/files/{1}?uploadType=resumable", DRIVE_API_UPLOAD_URL, Library.Utility.Uri.UrlPathEncode(fileid)) :
string.Format("{0}/files?uploadType=resumable", DRIVE_API_UPLOAD_URL);
WebApi.GoogleDrive.FileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values) :
WebApi.GoogleDrive.FileUploadUrl(values);
var item = new GoogleDriveFolderItem() {
var item = new GoogleDriveFolderItem
{
title = remotename,
description = remotename,
mimeType = "application/octet-stream",
labels = new GoogleDriveFolderItemLabels { hidden = true },
parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference() { id = CurrentFolderId } }
parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = CurrentFolderId } }
};
var res = GoogleCommon.ChunckedUploadWithResume<GoogleDriveFolderItem, GoogleDriveFolderItem>(m_oauth, item, url, stream, isUpdate ? "PUT" : "POST");
@@ -172,7 +183,6 @@ namespace Duplicati.Library.Backend.GoogleDrive
m_filecache.Clear();
throw;
}
}
public void Get(string remotename, System.IO.Stream stream)
@@ -181,12 +191,15 @@ namespace Duplicati.Library.Backend.GoogleDrive
if (m_filecache.Count == 0)
foreach (var file in List()) { /* Enumerate the full listing */ }
var fileid = GetFileEntries(remotename).OrderByDescending(x => x.createdDate).First().id;
var fileId = GetFileEntries(remotename).OrderByDescending(x => x.createdDate).First().id;
var req = m_oauth.CreateRequest(string.Format("{0}/files/{1}?alt=media{2}", DRIVE_API_URL, fileid, m_useTeamDrive ? "&supportsTeamDrives=true" : string.Empty));
var url = WebApi.GoogleDrive.FileQueryUrl(fileId, new NameValueCollection{
{ WebApi.GoogleDrive.QueryParam.Alt, WebApi.GoogleDrive.QueryValue.Media }
});
var req = m_oauth.CreateRequest(url);
var areq = new AsyncHttpRequest(req);
using(var resp = (HttpWebResponse)areq.GetResponse())
using(var rs = areq.GetResponseStream())
using (var resp = (HttpWebResponse)areq.GetResponse())
using (var rs = areq.GetResponseStream())
Duplicati.Library.Utility.Utility.CopyStream(rs, stream);
}
@@ -247,7 +260,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
}
}
}
public void Put(string remotename, string filename)
{
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
@@ -264,11 +277,11 @@ namespace Duplicati.Library.Backend.GoogleDrive
{
try
{
foreach(var fileid in from n in GetFileEntries(remotename) select n.id)
foreach (var fileid in from n in GetFileEntries(remotename) select n.id)
{
var url = string.Format("{0}/files/{1}{2}", DRIVE_API_URL, Library.Utility.Uri.UrlPathEncode(fileid), m_useTeamDrive ? "?supportsTeamDrives=true" : string.Empty);
m_oauth.GetJSONData<object>(url, x => {
var url = WebApi.GoogleDrive.FileQueryUrl(Library.Utility.Uri.UrlPathEncode(fileid), SupportsTeamDriveParam());
m_oauth.GetJSONData<object>(url, x =>
{
x.Method = "DELETE";
});
}
@@ -310,15 +323,16 @@ namespace Duplicati.Library.Backend.GoogleDrive
}
}
public System.Collections.Generic.IList<ICommandLineArgument> SupportedCommands
{
get {
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.GoogleDrive.AuthidShort, Strings.GoogleDrive.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("googledrive"))),
new CommandLineArgument(DISABLE_TEAMDRIVE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.GoogleDrive.DisableTeamDriveShort, Strings.GoogleDrive.DisableTeamDriveLong),
public System.Collections.Generic.IList<ICommandLineArgument> SupportedCommands => new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument(AUTHID_OPTION,
CommandLineArgument.ArgumentType.Password,
Strings.GoogleDrive.AuthidShort,
Strings.GoogleDrive.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("googledrive"))),
new CommandLineArgument(DISABLE_TEAMDRIVE_OPTION,
CommandLineArgument.ArgumentType.Boolean,
Strings.GoogleDrive.DisableTeamDriveShort,
Strings.GoogleDrive.DisableTeamDriveLong),
});
}
}
public string Description
{
@@ -348,8 +362,8 @@ namespace Duplicati.Library.Backend.GoogleDrive
}
public string[] DNSName
{
get { return new string[] { new System.Uri(DRIVE_API_URL).Host, new System.Uri(DRIVE_API_UPLOAD_URL).Host }; }
{
get { return new string[] { new System.Uri(WebApi.GoogleDrive.Url.DRIVE).Host, new System.Uri(WebApi.GoogleDrive.Url.UPLOAD).Host }; }
}
#endregion
@@ -365,21 +379,22 @@ namespace Duplicati.Library.Backend.GoogleDrive
var newfile = JsonConvert.DeserializeObject<GoogleDriveFolderItem>(JsonConvert.SerializeObject(files[0]));
newfile.title = newname;
newfile.parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference() { id = CurrentFolderId } };
newfile.parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = CurrentFolderId } };
var url = string.Format("{0}/files/{1}", DRIVE_API_UPLOAD_URL, Library.Utility.Uri.UrlPathEncode(files[0].id));
var url = WebApi.GoogleDrive.FileQueryUrl(Library.Utility.Uri.UrlPathEncode(files[0].id));
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(newfile));
var nf = m_oauth.GetJSONData<GoogleDriveFolderItem>(url, x => {
var nf = m_oauth.GetJSONData<GoogleDriveFolderItem>(url, x =>
{
x.Method = "PUT";
x.ContentLength = data.Length;
x.ContentType = "application/json; charset=UTF-8";
}, x => {
using(var rs = x.GetRequestStream())
}, x =>
{
using (var rs = x.GetRequestStream())
rs.Write(data, 0, data.Length);
});
m_filecache[newname] = new GoogleDriveFolderItem[] { nf };
m_filecache.Remove(oldname);
}
@@ -397,9 +412,24 @@ namespace Duplicati.Library.Backend.GoogleDrive
public void Dispose()
{
}
#endregion
private NameValueCollection SupportsTeamDriveParam()
{
return m_useTeamDrive ? new NameValueCollection {
{ WebApi.GoogleDrive.QueryParam.SupportsTeamDrive,
WebApi.GoogleDrive.QueryValue.True }
} : null;
}
#endregion
private NameValueCollection IncludeTeamDriveParam()
{
return m_useTeamDrive ? new NameValueCollection {
{ WebApi.GoogleDrive.QueryParam.IncludeTeamDrive,
WebApi.GoogleDrive.QueryValue.True } } : null;
}
private class GoogleDriveParentReference
{
@@ -470,40 +500,52 @@ namespace Duplicati.Library.Backend.GoogleDrive
private IEnumerable<GoogleDriveFolderItem> ListFolder(string parentfolder, bool? onlyFolders = null, string name = null)
{
var p = new string[] {
var fileQuery = new string[] {
string.IsNullOrEmpty(name) ? null : string.Format("title = '{0}'", EscapeTitleEntries(name)),
onlyFolders == null ? null : string.Format("mimeType {0}= '{1}'", onlyFolders.Value ? "" : "!", FOLDER_MIMETYPE),
string.Format("'{0}' in parents", EscapeTitleEntries(parentfolder))
};
var url = string.Format("{0}/files?q={1}{2}", DRIVE_API_URL, Library.Utility.Uri.UrlEncode(string.Join(" and ", p.Where(x => x != null))), m_useTeamDrive ? "&supportsTeamDrives=true&includeTeamDriveItems=true" : string.Empty);
var token = string.Empty;
var queryParams = new NameValueCollection
{
{WebApi.GoogleDrive.QueryParam.File,
Library.Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null)))},
};
queryParams.Add(SupportsTeamDriveParam());
queryParams.Add(IncludeTeamDriveParam());
do
while (true)
{
var res = m_oauth.GetJSONData<GoogleDriveListResponse>(url + (string.IsNullOrWhiteSpace(token) ? "" : "&pageToken=" + Library.Utility.Uri.UrlEncode(token)));
var url = WebApi.GoogleDrive.FileQueryUrl(queryParams);
var res = m_oauth.GetJSONData<GoogleDriveListResponse>(url);
foreach (var n in res.items)
yield return n;
token = res.nextPageToken;
} while (!string.IsNullOrWhiteSpace(token));
var token = res.nextPageToken;
if (string.IsNullOrWhiteSpace(token))
break;
queryParams.Set(WebApi.GoogleDrive.QueryParam.PageToken, token);
}
}
private GoogleDriveAboutResponse GetAboutInfo()
{
var url = string.Format("{0}/about", DRIVE_API_URL);
{
var url = Library.Utility.Uri.UriBuilder(WebApi.GoogleDrive.Url.DRIVE, WebApi.GoogleDrive.Path.About);
return m_oauth.GetJSONData<GoogleDriveAboutResponse>(url);
}
private GoogleDriveFolderItem CreateFolder(string name, string parent)
{
var url = string.Format("{0}/files{1}", DRIVE_API_URL, m_useTeamDrive ? "?supportsTeamDrives=true" : string.Empty);
var folder = new GoogleDriveFolderItem() {
var url = WebApi.GoogleDrive.FileQueryUrl(SupportsTeamDriveParam());
var folder = new GoogleDriveFolderItem()
{
title = name,
description = name,
mimeType = FOLDER_MIMETYPE,
labels = new GoogleDriveFolderItemLabels { hidden = true },
parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference() { id = parent } }
parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = parent } }
};
var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder));
@@ -514,13 +556,11 @@ namespace Duplicati.Library.Backend.GoogleDrive
x.ContentType = "application/json; charset=UTF-8";
x.ContentLength = data.Length;
}, req => {
using(var rs = req.GetRequestStream())
}, req =>
{
using (var rs = req.GetRequestStream())
rs.Write(data, 0, data.Length);
});
}
}
}
Duplicati/Library/Backend/GoogleServices/WebApi.cs
+72
View File
@@ -0,0 +1,72 @@
// Copyright (C) 2018, The Duplicati Team
// http://www.duplicati.com, info@duplicati.com
//
// This library is free software; you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as
// published by the Free Software Foundation; either version 2.1 of the
// License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using System.Collections.Specialized;
namespace Duplicati.Library.Backend.WebApi
{
static class GoogleDrive
{
public static class Url
{
public const string DRIVE = "https://www.googleapis.com/drive/v2";
public const string UPLOAD = "https://www.googleapis.com/upload/drive/v2";
}
public static class Path
{
public const string File = "files";
public const string About = "about";
}
public static class QueryParam
{
public const string File = "q";
public const string SupportsTeamDrive = "supportsTeamDrives";
public const string IncludeTeamDrive = "includeTeamDriveItems";
public const string PageToken = "pageToken";
public const string UploadType = "uploadType";
public const string Alt = "alt";
}
public static class QueryValue
{
public const string True = "true";
public const string Resumable = "resumable";
public const string Media = "media";
}
public static string FileQueryUrl(NameValueCollection values)
{
return Library.Utility.Uri.UriBuilder(Url.DRIVE, Path.File, values);
}
public static string FileQueryUrl(string fileId, NameValueCollection values = null)
{
return Library.Utility.Uri.UriBuilder(Url.DRIVE, Library.Utility.Uri.ConcatPaths(Path.File, fileId), values);
}
public static string FileUploadUrl(string fileId, NameValueCollection values)
{
return Library.Utility.Uri.UriBuilder(Url.UPLOAD, Library.Utility.Uri.ConcatPaths(Path.File, fileId), values);
}
public static string FileUploadUrl(NameValueCollection values)
{
return Library.Utility.Uri.UriBuilder(Url.UPLOAD, Path.File, values);
}
}
}
Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs
+4 -4
View File
@@ -140,9 +140,9 @@ namespace Duplicati.Library
/// <returns>The deserialized JSON data.</returns>
/// <param name="url">The remote URL</param>
/// <param name="setup">A callback method that can be used to customize the request, e.g. by setting the method, content-type and headers.</param>
/// <param name="setupreq">A callback method that can be used to submit data into the body of the request.</param>
/// <param name="setupbodyreq">A callback method that can be used to submit data into the body of the request.</param>
/// <typeparam name="T">The type of data to return.</typeparam>
public virtual T GetJSONData<T>(string url, Action<HttpWebRequest> setup = null, Action<AsyncHttpRequest> setupreq = null)
public virtual T GetJSONData<T>(string url, Action<HttpWebRequest> setup = null, Action<AsyncHttpRequest> setupbodyreq = null)
{
var req = CreateRequest(url);
@@ -151,8 +151,8 @@ namespace Duplicati.Library
var areq = new AsyncHttpRequest(req);
if (setupreq != null)
setupreq(areq);
if (setupbodyreq != null)
setupbodyreq(areq);
return ReadJSONResponse<T>(areq);
}
Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs
+1
View File
@@ -64,6 +64,7 @@ namespace Duplicati.Library.Backend.OpenStack
new KeyValuePair<string, string>("Rackspace UK", "https://lon.identity.api.rackspacecloud.com/v2.0"),
new KeyValuePair<string, string>("OVH Cloud Storage", "https://auth.cloud.ovh.net/v2.0"),
new KeyValuePair<string, string>("Selectel Cloud Storage", "https://auth.selcdn.ru"),
new KeyValuePair<string, string>("Memset Cloud Storage", "https://auth.storage.memset.com"),
};
public static readonly KeyValuePair<string, string>[] OPENSTACK_VERSIONS = {
Duplicati/Library/Utility/Duplicati.Library.Utility.csproj
+1 -1
View File
@@ -61,7 +61,7 @@
<Compile Include="TempFolder.cs" />
<Compile Include="ThrottledStream.cs" />
<Compile Include="Timeparser.cs" />
<Compile Include="UrlUtillity.cs" />
<Compile Include="UrlUtility.cs" />
<Compile Include="Utility.cs" />
<Compile Include="Win32.cs" />
<Compile Include="WorkerThread.cs" />
Duplicati/Library/Utility/Uri.cs
+180 -86
View File
@@ -17,6 +17,8 @@
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using System;
using System.Collections.Specialized;
using System.Linq;
using System.Text;
using System.Web;
namespace Duplicati.Library.Utility
@@ -62,18 +64,18 @@ namespace Duplicati.Library.Utility
/// <summary>
/// The password, if any
/// </summary>
public readonly string Password;
public readonly string Password;
/// <summary>
/// The original URI.
/// </summary>
public readonly string OriginalUri;
public readonly string OriginalUri;
/// <summary>
/// Cache for the query parameters.
/// </summary>
private NameValueCollection m_queryParams;
private NameValueCollection m_queryParams;
/// <summary>
/// Gets the paramters in the query string
/// </summary>
@@ -88,12 +90,12 @@ namespace Duplicati.Library.Utility
m_queryParams = new NameValueCollection();
else
m_queryParams = ParseQueryString(Query);
}
}
return m_queryParams;
}
}
}
/// <summary>
/// Gets the host and path.
/// </summary>
@@ -109,8 +111,8 @@ namespace Duplicati.Library.Utility
else
return Host + (Path == null ? "" : "/" + Path);
}
}
}
/// <summary>
/// Gets the path and query.
/// </summary>
@@ -119,10 +121,10 @@ namespace Duplicati.Library.Utility
{
get
{
return (Path ?? "") + (Query == null ? "" : "?" + Query);
return (Path ?? "") + (Query == null ? "" : "?" + Query);
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="Duplicati.Library.Utility.Uri"/> struct.
/// </summary>
@@ -130,8 +132,8 @@ namespace Duplicati.Library.Utility
public Uri(string url)
{
if (string.IsNullOrEmpty(url))
throw new ArgumentNullException(nameof(url));
throw new ArgumentNullException(nameof(url));
m_queryParams = null;
this.OriginalUri = url;
@@ -159,8 +161,8 @@ namespace Duplicati.Library.Utility
{
}
throw new ArgumentException(Strings.Uri.UriParseError(url), nameof(url));
}
}
this.Scheme = m.Groups["scheme"].Value;
var h = m.Groups["hostname"].Success ? m.Groups["hostname"].Value : "";
@@ -185,8 +187,8 @@ namespace Duplicati.Library.Utility
this.Port = int.Parse(m.Groups["port"].Value);
else
this.Port = -1;
}
}
/// <summary>
/// Constructs a free-form URI from components
/// </summary>
@@ -208,13 +210,13 @@ namespace Duplicati.Library.Utility
Password = password;
Port = port;
OriginalUri = AsString(scheme, host, path, query, username, password, port);
}
}
/// <summary>
/// Returns a <see cref="System.String"/> that represents the current <see cref="Duplicati.Library.Utility.Uri"/>.
/// </summary>
/// <returns>A <see cref="System.String"/> that represents the current <see cref="Duplicati.Library.Utility.Uri"/>.</returns>
public override string ToString ()
public override string ToString()
{
return AsString(Scheme, Host, Path, Query, Username, Password, Port);
}
@@ -226,8 +228,8 @@ namespace Duplicati.Library.Utility
{
if (string.IsNullOrEmpty(Host))
throw new ArgumentException(Strings.Uri.NoHostname(OriginalUri));
}
}
/// <summary>
/// Constructs an url-like string from components.
/// </summary>
@@ -243,21 +245,21 @@ namespace Duplicati.Library.Utility
{
var s = scheme + "://";
if (!string.IsNullOrEmpty(username) || !string.IsNullOrEmpty(password))
{
{
s += UrlEncode(username ?? "");
s += ":";
s += UrlEncode(password ?? "");
s += "@";
}
}
if (!string.IsNullOrEmpty(host))
{
s += host;
if (port != -1)
s += ":" + port.ToString();
}
}
if (!string.IsNullOrEmpty(path))
{
if (!string.IsNullOrEmpty(host) && !path.StartsWith("/", StringComparison.Ordinal))
@@ -267,9 +269,9 @@ namespace Duplicati.Library.Utility
if (!string.IsNullOrEmpty(query))
s += "?" + query;
return s;
}
return s;
}
/// <summary>
/// Creates a new instance with another scheme
/// </summary>
@@ -278,8 +280,8 @@ namespace Duplicati.Library.Utility
public Uri SetScheme(string scheme)
{
return new Uri(scheme, Host, Path, Query, Username, Password, Port);
}
}
/// <summary>
/// Creates a new instance with another host
/// </summary>
@@ -288,8 +290,8 @@ namespace Duplicati.Library.Utility
public Uri SetHost(string host)
{
return new Uri(Scheme, host, Path, Query, Username, Password, Port);
}
}
/// <summary>
/// Creates a new instance with another path
/// </summary>
@@ -298,8 +300,8 @@ namespace Duplicati.Library.Utility
public Uri SetPath(string path)
{
return new Uri(Scheme, Host, path, Query, Username, Password, Port);
}
}
/// <summary>
/// Creates a new instance with another query
/// </summary>
@@ -319,8 +321,8 @@ namespace Duplicati.Library.Utility
public Uri SetCredentials(string username, string password)
{
return new Uri(Scheme, Host, Path, Query, username, password, Port);
}
}
/// <summary>
/// Creates a new instance with another port
/// </summary>
@@ -329,13 +331,13 @@ namespace Duplicati.Library.Utility
public Uri SetPort(int port)
{
return new Uri(Scheme, Host, Path, Query, Username, Password, port);
}
}
/// <summary>
/// The regular expression that matches %20 type values in a querystring
/// </summary>
private static System.Text.RegularExpressions.Regex RE_ESCAPECHAR = new System.Text.RegularExpressions.Regex(@"[^0-9a-zA-Z\-_]", System.Text.RegularExpressions.RegexOptions.Compiled);
private static System.Text.RegularExpressions.Regex RE_ESCAPECHAR = new System.Text.RegularExpressions.Regex(@"[^0-9a-zA-Z\-_]", System.Text.RegularExpressions.RegexOptions.Compiled);
/// <summary>
/// Encodes a URL, like System.Web.HttpUtility.UrlEncode
/// </summary>
@@ -345,44 +347,45 @@ namespace Duplicati.Library.Utility
public static string UrlPathEncode(string value, System.Text.Encoding encoding = null)
{
return UrlEncode(value, encoding, "%20");
}
}
/// <summary>
/// Encodes a URL, like System.Web.HttpUtility.UrlEncode
/// </summary>
/// <returns>The encoded URL</returns>
/// <param name="value">The URL fragment to encode</param>
/// <param name="encoding">The encoding to use</param>
public static string UrlEncode(string value, System.Text.Encoding encoding = null, string spacevalue = "+")
public static string UrlEncode(string value, System.Text.Encoding encoding = null, string spacevalue = "+")
{
if (value == null)
throw new ArgumentNullException(nameof(value));
throw new ArgumentNullException(nameof(value));
encoding = encoding ?? System.Text.Encoding.UTF8;
var encoder = encoding.GetEncoder();
var inbuf = new char[1];
var inbuf = new char[1];
var outbuf = new byte[4];
return RE_ESCAPECHAR.Replace(value, (m) => {
return RE_ESCAPECHAR.Replace(value, (m) =>
{
if (m.Value == " ")
return spacevalue;
inbuf[0] = m.Value[0];
try
return spacevalue;
inbuf[0] = m.Value[0];
try
{
var len = encoder.GetBytes(inbuf, 0, 1, outbuf, 0, true);
return "%" + BitConverter.ToString(outbuf, 0, len).Replace("-", "%");
}
catch
{
}
//Fallback
}
//Fallback
return m.Value;
});
});
}
/// <summary>
@@ -399,15 +402,16 @@ namespace Duplicati.Library.Utility
public static string UrlDecode(string value, System.Text.Encoding encoding = null)
{
if (value == null)
throw new ArgumentNullException(nameof(value));
encoding = encoding ?? System.Text.Encoding.UTF8;
throw new ArgumentNullException(nameof(value));
encoding = encoding ?? System.Text.Encoding.UTF8;
var decoder = encoding.GetDecoder();
var inbuf = new byte[8];
var outbuf = new char[8];
return RE_NUMBER.Replace(value, (m) => {
return RE_NUMBER.Replace(value, (m) =>
{
if (m.Value == "+")
return " ";
@@ -416,24 +420,24 @@ namespace Duplicati.Library.Utility
var hex = m.Groups["number"].Value;
var bytelen = hex.Length / 2;
Utility.HexStringAsByteArray(hex, inbuf);
var c = decoder.GetChars(inbuf, 0, bytelen, outbuf, 0);
var c = decoder.GetChars(inbuf, 0, bytelen, outbuf, 0);
return new string(outbuf, 0, c);
}
catch
{
}
//Fallback
}
//Fallback
return m.Value;
});
}
});
}
/// <summary>
/// The regular expression that matches a=b type values in a querystring
/// </summary>
private static System.Text.RegularExpressions.Regex RE_URLPARAM = new System.Text.RegularExpressions.Regex(@"(?<key>[^\=\&]+)(\=(?<value>[^\&]*))?", System.Text.RegularExpressions.RegexOptions.Compiled);
private static System.Text.RegularExpressions.Regex RE_URLPARAM = new System.Text.RegularExpressions.Regex(@"(?<key>[^\=\&]+)(\=(?<value>[^\&]*))?", System.Text.RegularExpressions.RegexOptions.Compiled);
/// <summary>
/// Parses the query string.
/// This is a duplicate of the System.Web.HttpUtility.ParseQueryString that does not work well on Mono
@@ -447,14 +451,104 @@ namespace Duplicati.Library.Utility
if (query.StartsWith("?", StringComparison.Ordinal))
query = query.Substring(1);
if (string.IsNullOrEmpty(query))
return new NameValueCollection(StringComparer.OrdinalIgnoreCase);
return new NameValueCollection(StringComparer.OrdinalIgnoreCase);
var result = new NameValueCollection(StringComparer.OrdinalIgnoreCase);
foreach(System.Text.RegularExpressions.Match m in RE_URLPARAM.Matches(query))
result.Add(UrlDecode(m.Groups["key"].Value), UrlDecode(m.Groups["value"].Success ? m.Groups["value"].Value : ""));
return result;
}
foreach (System.Text.RegularExpressions.Match m in RE_URLPARAM.Matches(query))
result.Add(UrlDecode(m.Groups["key"].Value), UrlDecode(m.Groups["value"].Success ? m.Groups["value"].Value : ""));
return result;
}
/// <summary>
/// Build the querystring to be used in a URL
/// </summary>
/// <returns>The generated querystring</returns>
/// <param name="query">A collection of name value pairs to be translated into a query string</param>
/// <param name="delimiter">The delimiter to separate key value pairs in the query string</param>
public static string BuildUriQuery(NameValueCollection query, string delimiter)
{
if (query == null)
throw new ArgumentNullException(nameof(query));
StringBuilder builder = new StringBuilder();
foreach (var key in query.Cast<string>().Where(key => !string.IsNullOrEmpty(query[key])))
{
builder.Append(builder.Length == 0 ? string.Empty : delimiter)
.Append(key)
.Append("=")
.Append(query[key]);
}
return builder.ToString();
}
/// <summary>
/// Build the querystring to be used in a URL
/// </summary>
/// <returns>The generated querystring</returns>
/// <param name="query">A collection of name value pairs to be translated into a query string that is
/// ampsersand delimited.</param>
public static string BuildUriQuery(NameValueCollection query)
{
return BuildUriQuery(query, "&");
}
/// <summary>
/// Builds a URL together using a base URL, a path and a query.
/// </summary>
/// <returns>The built together URL.</returns>
/// <param name="url">Base URL, containing schema, host, port.</param>
/// <param name="path">Base path.</param>
/// <param name="query">A collection of name value pairs to be translated into a query string.</param>
public static string UriBuilder(string url, string path, NameValueCollection query)
{
var builder = new UriBuilder(url)
{
Path = ConcatPaths(ExtractPath(url), path),
Query = query != null ? BuildUriQuery(query) : null
};
return builder.Uri.AbsoluteUri;
}
/// <summary>
/// Concats paths of URIs.
/// </summary>
/// <returns>The concatenated paths.</returns>
/// <param name="path1">Path1.</param>
/// <param name="path2">Path2.</param>
public static string ConcatPaths(string path1, string path2)
{
if (string.IsNullOrEmpty(path2))
{
return path1;
}
return path1.TrimEnd('/') + '/' + path2;
}
/// <summary>
/// Grab path part of a URI.
/// At the moment, simple implementation does not remove fragments.
/// </summary>
/// <returns>The path.</returns>
/// <param name="url">URL.</param>
public static string ExtractPath(string url)
{
return (new Uri(url)).Path;
}
/// <summary>
/// Builds a URL together using a base URL and path.
/// </summary>
/// <returns>The built together URL.</returns>
/// <param name="url">Base URL, containing schema, host, port.</param>
/// <param name="path">Base path.</param>
public static string UriBuilder(string url, string path)
{
return UriBuilder(url, path, null);
}
}
}
Duplicati/Library/Utility/UrlUtillity.cs → Duplicati/Library/Utility/UrlUtility.cs
+1 -1
View File
@@ -23,7 +23,7 @@ using System.Text;
namespace Duplicati.Library.Utility
{
public static class UrlUtillity
public static class UrlUtility
{
/// <summary>
/// The file path to the system browser selected
Duplicati/Library/Utility/Utility.cs
+3 -12
View File
@@ -1,4 +1,3 @@
#region Disclaimer / License
// Copyright (C) 2015, The Duplicati Team
// http://www.duplicati.com, info@duplicati.com
//
@@ -15,21 +14,13 @@
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
using System.Text.RegularExpressions;
using System.Linq;
using System.Threading.Tasks;
#endregion
using System;
using System.Text.RegularExpressions;
using System.Linq;
using System.Collections.Generic;
using System.IO;
using System.Text;
using System.Linq;
using System.Threading.Tasks;
using System.Text;
using System.Text.RegularExpressions;
namespace Duplicati.Library.Utility
{
Duplicati/UnitTest/Duplicati.UnitTest.csproj
+2 -1
View File
@@ -50,6 +50,7 @@
<Compile Include="UtilityTests.cs" />
<Compile Include="FilterTest.cs" />
<Compile Include="ResultFormatSerializerProviderTest.cs" />
<Compile Include="UriUtilityTests.cs" />
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
<ItemGroup>
@@ -173,4 +174,4 @@
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
</Project>
</Project>
Duplicati/UnitTest/UriUtilityTests.cs
+69
View File
@@ -0,0 +1,69 @@
// Copyright (C) 2018, The Duplicati Team
// http://www.duplicati.com, info@duplicati.com
//
// This library is free software; you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as
// published by the Free Software Foundation; either version 2.1 of the
// License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using NUnit.Framework;
using System.Collections.Specialized;
namespace Duplicati.UnitTest
{
public class UriUtilityTests
{
[Test]
[Category("UriUtility")]
public static void TestBuildUriQuery()
{
var query = new NameValueCollection { { "a", "b" } };
var queryUrl = Library.Utility.Uri.BuildUriQuery(query);
Assert.AreEqual("a=b", queryUrl);
query.Add(new NameValueCollection { { "c", "d" } });
queryUrl = Library.Utility.Uri.BuildUriQuery(query);
Assert.AreEqual("a=b&c=d", queryUrl);
}
[Test]
[Category("UriUtility")]
public static void TestUrlBuilder()
{
var baseUrl = "http://localhost";
var path = "files";
var query = new NameValueCollection { { "a", "b" }, { "c", "d" } };
var url = Library.Utility.Uri.UriBuilder(baseUrl, path, query);
Assert.AreEqual(baseUrl + "/" + path + "?a=b&c=d", url);
}
[Test]
[Category("UriUtility")]
public static void TestExtractPath()
{
var url = "http://localhost/a/b";
var path = Library.Utility.Uri.ExtractPath(url);
Assert.AreEqual("a/b", path);
}
[Test]
[Category("UriUtility")]
public static void TestConcatPaths()
{
var path1 = "/a";
var path2 = "b/";
Assert.AreEqual("/a/b/", Library.Utility.Uri.ConcatPaths(path1, path2));
Assert.AreEqual("/a", Library.Utility.Uri.ConcatPaths(path1, null));
Assert.AreEqual("/b/", Library.Utility.Uri.ConcatPaths(string.Empty, path2));
}
}
}