diff --git a/.travis.yml b/.travis.yml index d6580239b..8a18829db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,6 +55,7 @@ jobs: - env: CATEGORY=Purge - env: CATEGORY=Serialization - env: CATEGORY=Utility + - env: CATEGORY=UriUtility - env: CATEGORY=GUI addons: diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs index 4b4788e81..36c5b0a36 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs @@ -134,7 +134,7 @@ namespace Duplicati.GUI.TrayIcon public virtual IBrowserWindow ShowUrlInWindow(string url) { //Fallback is to just show the window in a browser - Duplicati.Library.Utility.UrlUtillity.OpenURL(url, Program.BrowserCommand); + Duplicati.Library.Utility.UrlUtility.OpenURL(url, Program.BrowserCommand); return null; } diff --git a/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj b/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj index 817a57af5..070b4832a 100644 --- a/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj +++ b/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj @@ -41,6 +41,7 @@ + diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs index b426eab5b..b0d336d03 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs @@ -18,7 +18,6 @@ using System; using Duplicati.Library.Interface; using System.Collections.Generic; using System.Net; -using System.Web; using Duplicati.Library.Utility; using Newtonsoft.Json; using System.Text; @@ -97,9 +96,9 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage m_oauth = new OAuthHelper(authid, this.ProtocolKey); m_oauth.AutoAuthHeader = true; - } - - + } + + private class ListBucketResponse { public string kind { get; set; } @@ -205,7 +204,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage if (string.IsNullOrEmpty(m_project)) throw new UserInformationException(Strings.GoogleCloudStorage.ProjectIDMissingError(PROJECT_OPTION), "GoogleCloudStorageMissingProjectID"); - var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest() { + var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest + { name = m_bucket, location = m_location, storageClass = m_storage_class @@ -220,7 +220,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage var areq = new AsyncHttpRequest(req); - using(var rs = areq.GetRequestStream()) + using (var rs = areq.GetRequestStream()) rs.Write(data, 0, data.Length); m_oauth.ReadJSONResponse(areq); @@ -246,8 +246,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage foreach (KeyValuePair s in KNOWN_GCS_LOCATIONS) locations.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); foreach (KeyValuePair s in KNOWN_GCS_STORAGE_CLASSES) - storageClasses.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); - + storageClasses.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); + return new List(new ICommandLineArgument[] { new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.LocationDescriptionShort, Strings.GoogleCloudStorage.LocationDescriptionLong(locations.ToString())), new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.StorageclassDescriptionShort, Strings.GoogleCloudStorage.StorageclassDescriptionLong(locations.ToString())), @@ -276,8 +276,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage var res = GoogleCommon.ChunckedUploadWithResume(m_oauth, item, url, stream); if (res == null) - throw new Exception(string.Format("Upload succeeded, but no data was returned")); - + throw new Exception("Upload succeeded, but no data was returned"); + } public void Get(string remotename, System.IO.Stream stream) @@ -288,8 +288,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage var req = m_oauth.CreateRequest(url); var areq = new AsyncHttpRequest(req); - using(var resp = areq.GetResponse()) - using(var rs = areq.GetResponseStream()) + using (var resp = areq.GetResponse()) + using (var rs = areq.GetResponseStream()) Library.Utility.Utility.CopyStream(rs, stream); } catch (WebException wex) @@ -304,7 +304,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage public void Rename(string oldname, string newname) { - var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem() { + var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem + { name = m_prefix + newname, })); @@ -315,7 +316,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage req.ContentType = "application/json; charset=UTF-8"; var areq = new AsyncHttpRequest(req); - using(var rs = areq.GetRequestStream()) + using (var rs = areq.GetRequestStream()) rs.Write(data, 0, data.Length); m_oauth.ReadJSONResponse(req); @@ -323,8 +324,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage #region IDisposable implementation public void Dispose() - { - + { + } #endregion } diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index fe846c529..fef6f15f9 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -15,14 +15,18 @@ // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; +using System.Collections.Generic; +using System.Collections.Specialized; using System.Linq; +using System.Net; + +using Newtonsoft.Json; + +using Duplicati.Library.Backend.GoogleServices; using Duplicati.Library.Interface; using Duplicati.Library.Utility; -using System.Collections.Generic; -using System.Net; -using Newtonsoft.Json; -using Duplicati.Library.Backend.GoogleServices; - +using System.Text; + namespace Duplicati.Library.Backend.GoogleDrive { public class GoogleDrive : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend @@ -31,13 +35,11 @@ namespace Duplicati.Library.Backend.GoogleDrive private const string DISABLE_TEAMDRIVE_OPTION = "googledrive-disable-teamdrive"; private const string FOLDER_MIMETYPE = "application/vnd.google-apps.folder"; - private const string DRIVE_API_UPLOAD_URL = "https://www.googleapis.com/upload/drive/v2"; - private const string DRIVE_API_URL = "https://www.googleapis.com/drive/v2"; + + private readonly string m_path; + private readonly bool m_useTeamDrive = true; - private string m_path; - private bool m_useTeamDrive = true; - - private OAuthHelper m_oauth; + private readonly OAuthHelper m_oauth; private string m_currentFolderId; private Dictionary m_filecache; @@ -66,26 +68,30 @@ namespace Duplicati.Library.Backend.GoogleDrive private string GetFolderId(string path, bool autocreate = false) { - var curparent = GetAboutInfo().rootFolderId; - var curdisplay = "/"; + var curparent = GetAboutInfo().rootFolderId; + var curdisplay = new StringBuilder("/"); - foreach(var p in path.Split(new char[] {'/'}, StringSplitOptions.RemoveEmptyEntries)) + foreach (var p in path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)) { - var res = ListFolder(curparent, true, p).ToArray(); - - if (res.Length == 0) - { - if (!autocreate) - throw new FolderMissingException(); - - curparent = CreateFolder(p, curparent).id; - } - else if (res.Length > 1) - throw new UserInformationException(Strings.GoogleDrive.MultipleEntries(p, curdisplay), "GoogleDriveMultipleEntries"); - else - curparent = res[0].id; - - curdisplay += p + "/"; + var res = ListFolder(curparent, true, p).ToArray(); + + if (res.Length == 0) + { + if (!autocreate) + throw new FolderMissingException(); + + curparent = CreateFolder(p, curparent).id; + } + else if (res.Length > 1) + { + throw new UserInformationException(Strings.GoogleDrive.MultipleEntries(p, curdisplay.ToString()), "GoogleDriveMultipleEntries"); + } + else + { + curparent = res[0].id; + } + + curdisplay.Append(p).Append("/"); } return curparent; @@ -110,17 +116,17 @@ namespace Duplicati.Library.Backend.GoogleDrive if (entries != null) return entries; - var list = ListFolder(CurrentFolderId, false, remotename).ToArray(); + entries = ListFolder(CurrentFolderId, false, remotename).ToArray(); - if (list == null || list.Length == 0) + if (entries == null || entries.Length == 0) { if (throwMissingException) throw new FileMissingException(); else return null; } - - return m_filecache[remotename] = list; + + return m_filecache[remotename] = entries; } private static string EscapeTitleEntries(string title) @@ -141,27 +147,32 @@ namespace Duplicati.Library.Backend.GoogleDrive GoogleDriveFolderItem[] files; m_filecache.TryGetValue(remotename, out files); - string fileid = null; + string fileId = null; if (files != null) { if (files.Length == 1) - fileid = files[0].id; + fileId = files[0].id; else Delete(remotename); } - var isUpdate = !string.IsNullOrWhiteSpace(fileid); + var isUpdate = !string.IsNullOrWhiteSpace(fileId); + + var values = new NameValueCollection { + { WebApi.GoogleDrive.QueryParam.UploadType, + WebApi.GoogleDrive.QueryValue.Resumable } }; var url = isUpdate ? - string.Format("{0}/files/{1}?uploadType=resumable", DRIVE_API_UPLOAD_URL, Library.Utility.Uri.UrlPathEncode(fileid)) : - string.Format("{0}/files?uploadType=resumable", DRIVE_API_UPLOAD_URL); + WebApi.GoogleDrive.FileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values) : + WebApi.GoogleDrive.FileUploadUrl(values); - var item = new GoogleDriveFolderItem() { + var item = new GoogleDriveFolderItem + { title = remotename, description = remotename, mimeType = "application/octet-stream", labels = new GoogleDriveFolderItemLabels { hidden = true }, - parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference() { id = CurrentFolderId } } + parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = CurrentFolderId } } }; var res = GoogleCommon.ChunckedUploadWithResume(m_oauth, item, url, stream, isUpdate ? "PUT" : "POST"); @@ -172,7 +183,6 @@ namespace Duplicati.Library.Backend.GoogleDrive m_filecache.Clear(); throw; } - } public void Get(string remotename, System.IO.Stream stream) @@ -181,12 +191,15 @@ namespace Duplicati.Library.Backend.GoogleDrive if (m_filecache.Count == 0) foreach (var file in List()) { /* Enumerate the full listing */ } - var fileid = GetFileEntries(remotename).OrderByDescending(x => x.createdDate).First().id; + var fileId = GetFileEntries(remotename).OrderByDescending(x => x.createdDate).First().id; - var req = m_oauth.CreateRequest(string.Format("{0}/files/{1}?alt=media{2}", DRIVE_API_URL, fileid, m_useTeamDrive ? "&supportsTeamDrives=true" : string.Empty)); + var url = WebApi.GoogleDrive.FileQueryUrl(fileId, new NameValueCollection{ + { WebApi.GoogleDrive.QueryParam.Alt, WebApi.GoogleDrive.QueryValue.Media } + }); + var req = m_oauth.CreateRequest(url); var areq = new AsyncHttpRequest(req); - using(var resp = (HttpWebResponse)areq.GetResponse()) - using(var rs = areq.GetResponseStream()) + using (var resp = (HttpWebResponse)areq.GetResponse()) + using (var rs = areq.GetResponseStream()) Duplicati.Library.Utility.Utility.CopyStream(rs, stream); } @@ -247,7 +260,7 @@ namespace Duplicati.Library.Backend.GoogleDrive } } } - + public void Put(string remotename, string filename) { using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) @@ -264,11 +277,11 @@ namespace Duplicati.Library.Backend.GoogleDrive { try { - foreach(var fileid in from n in GetFileEntries(remotename) select n.id) + foreach (var fileid in from n in GetFileEntries(remotename) select n.id) { - var url = string.Format("{0}/files/{1}{2}", DRIVE_API_URL, Library.Utility.Uri.UrlPathEncode(fileid), m_useTeamDrive ? "?supportsTeamDrives=true" : string.Empty); - - m_oauth.GetJSONData(url, x => { + var url = WebApi.GoogleDrive.FileQueryUrl(Library.Utility.Uri.UrlPathEncode(fileid), SupportsTeamDriveParam()); + m_oauth.GetJSONData(url, x => + { x.Method = "DELETE"; }); } @@ -310,15 +323,16 @@ namespace Duplicati.Library.Backend.GoogleDrive } } - public System.Collections.Generic.IList SupportedCommands - { - get { - return new List(new ICommandLineArgument[] { - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.GoogleDrive.AuthidShort, Strings.GoogleDrive.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("googledrive"))), - new CommandLineArgument(DISABLE_TEAMDRIVE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.GoogleDrive.DisableTeamDriveShort, Strings.GoogleDrive.DisableTeamDriveLong), + public System.Collections.Generic.IList SupportedCommands => new List(new ICommandLineArgument[] { + new CommandLineArgument(AUTHID_OPTION, + CommandLineArgument.ArgumentType.Password, + Strings.GoogleDrive.AuthidShort, + Strings.GoogleDrive.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("googledrive"))), + new CommandLineArgument(DISABLE_TEAMDRIVE_OPTION, + CommandLineArgument.ArgumentType.Boolean, + Strings.GoogleDrive.DisableTeamDriveShort, + Strings.GoogleDrive.DisableTeamDriveLong), }); - } - } public string Description { @@ -348,8 +362,8 @@ namespace Duplicati.Library.Backend.GoogleDrive } public string[] DNSName - { - get { return new string[] { new System.Uri(DRIVE_API_URL).Host, new System.Uri(DRIVE_API_UPLOAD_URL).Host }; } + { + get { return new string[] { new System.Uri(WebApi.GoogleDrive.Url.DRIVE).Host, new System.Uri(WebApi.GoogleDrive.Url.UPLOAD).Host }; } } #endregion @@ -365,21 +379,22 @@ namespace Duplicati.Library.Backend.GoogleDrive var newfile = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(files[0])); newfile.title = newname; - newfile.parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference() { id = CurrentFolderId } }; + newfile.parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = CurrentFolderId } }; - var url = string.Format("{0}/files/{1}", DRIVE_API_UPLOAD_URL, Library.Utility.Uri.UrlPathEncode(files[0].id)); + var url = WebApi.GoogleDrive.FileQueryUrl(Library.Utility.Uri.UrlPathEncode(files[0].id)); var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(newfile)); - var nf = m_oauth.GetJSONData(url, x => { + var nf = m_oauth.GetJSONData(url, x => + { x.Method = "PUT"; x.ContentLength = data.Length; x.ContentType = "application/json; charset=UTF-8"; - }, x => { - - using(var rs = x.GetRequestStream()) + }, x => + { + using (var rs = x.GetRequestStream()) rs.Write(data, 0, data.Length); }); - + m_filecache[newname] = new GoogleDriveFolderItem[] { nf }; m_filecache.Remove(oldname); } @@ -397,9 +412,24 @@ namespace Duplicati.Library.Backend.GoogleDrive public void Dispose() { + } + + #endregion + + private NameValueCollection SupportsTeamDriveParam() + { + return m_useTeamDrive ? new NameValueCollection { + { WebApi.GoogleDrive.QueryParam.SupportsTeamDrive, + WebApi.GoogleDrive.QueryValue.True } + } : null; } - #endregion + private NameValueCollection IncludeTeamDriveParam() + { + return m_useTeamDrive ? new NameValueCollection { + { WebApi.GoogleDrive.QueryParam.IncludeTeamDrive, + WebApi.GoogleDrive.QueryValue.True } } : null; + } private class GoogleDriveParentReference { @@ -470,40 +500,52 @@ namespace Duplicati.Library.Backend.GoogleDrive private IEnumerable ListFolder(string parentfolder, bool? onlyFolders = null, string name = null) { - var p = new string[] { + var fileQuery = new string[] { string.IsNullOrEmpty(name) ? null : string.Format("title = '{0}'", EscapeTitleEntries(name)), onlyFolders == null ? null : string.Format("mimeType {0}= '{1}'", onlyFolders.Value ? "" : "!", FOLDER_MIMETYPE), string.Format("'{0}' in parents", EscapeTitleEntries(parentfolder)) }; - var url = string.Format("{0}/files?q={1}{2}", DRIVE_API_URL, Library.Utility.Uri.UrlEncode(string.Join(" and ", p.Where(x => x != null))), m_useTeamDrive ? "&supportsTeamDrives=true&includeTeamDriveItems=true" : string.Empty); - var token = string.Empty; + var queryParams = new NameValueCollection + { + {WebApi.GoogleDrive.QueryParam.File, + Library.Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null)))}, + }; + queryParams.Add(SupportsTeamDriveParam()); + queryParams.Add(IncludeTeamDriveParam()); - do + while (true) { - var res = m_oauth.GetJSONData(url + (string.IsNullOrWhiteSpace(token) ? "" : "&pageToken=" + Library.Utility.Uri.UrlEncode(token))); + var url = WebApi.GoogleDrive.FileQueryUrl(queryParams); + var res = m_oauth.GetJSONData(url); foreach (var n in res.items) yield return n; - token = res.nextPageToken; - } while (!string.IsNullOrWhiteSpace(token)); + var token = res.nextPageToken; + if (string.IsNullOrWhiteSpace(token)) + break; + + queryParams.Set(WebApi.GoogleDrive.QueryParam.PageToken, token); + } } private GoogleDriveAboutResponse GetAboutInfo() - { - var url = string.Format("{0}/about", DRIVE_API_URL); + { + var url = Library.Utility.Uri.UriBuilder(WebApi.GoogleDrive.Url.DRIVE, WebApi.GoogleDrive.Path.About); return m_oauth.GetJSONData(url); } private GoogleDriveFolderItem CreateFolder(string name, string parent) { - var url = string.Format("{0}/files{1}", DRIVE_API_URL, m_useTeamDrive ? "?supportsTeamDrives=true" : string.Empty); - var folder = new GoogleDriveFolderItem() { + var url = WebApi.GoogleDrive.FileQueryUrl(SupportsTeamDriveParam()); + + var folder = new GoogleDriveFolderItem() + { title = name, description = name, mimeType = FOLDER_MIMETYPE, labels = new GoogleDriveFolderItemLabels { hidden = true }, - parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference() { id = parent } } + parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = parent } } }; var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder)); @@ -514,13 +556,11 @@ namespace Duplicati.Library.Backend.GoogleDrive x.ContentType = "application/json; charset=UTF-8"; x.ContentLength = data.Length; - }, req => { - - using(var rs = req.GetRequestStream()) + }, req => + { + using (var rs = req.GetRequestStream()) rs.Write(data, 0, data.Length); - }); - } } } diff --git a/Duplicati/Library/Backend/GoogleServices/WebApi.cs b/Duplicati/Library/Backend/GoogleServices/WebApi.cs new file mode 100644 index 000000000..0c53eeb63 --- /dev/null +++ b/Duplicati/Library/Backend/GoogleServices/WebApi.cs @@ -0,0 +1,72 @@ +// Copyright (C) 2018, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System.Collections.Specialized; + +namespace Duplicati.Library.Backend.WebApi +{ + static class GoogleDrive + { + public static class Url + { + public const string DRIVE = "https://www.googleapis.com/drive/v2"; + public const string UPLOAD = "https://www.googleapis.com/upload/drive/v2"; + } + + public static class Path + { + public const string File = "files"; + public const string About = "about"; + } + + public static class QueryParam + { + public const string File = "q"; + public const string SupportsTeamDrive = "supportsTeamDrives"; + public const string IncludeTeamDrive = "includeTeamDriveItems"; + public const string PageToken = "pageToken"; + public const string UploadType = "uploadType"; + public const string Alt = "alt"; + } + + public static class QueryValue + { + public const string True = "true"; + public const string Resumable = "resumable"; + public const string Media = "media"; + } + + public static string FileQueryUrl(NameValueCollection values) + { + return Library.Utility.Uri.UriBuilder(Url.DRIVE, Path.File, values); + } + + public static string FileQueryUrl(string fileId, NameValueCollection values = null) + { + return Library.Utility.Uri.UriBuilder(Url.DRIVE, Library.Utility.Uri.ConcatPaths(Path.File, fileId), values); + } + + public static string FileUploadUrl(string fileId, NameValueCollection values) + { + return Library.Utility.Uri.UriBuilder(Url.UPLOAD, Library.Utility.Uri.ConcatPaths(Path.File, fileId), values); + } + + public static string FileUploadUrl(NameValueCollection values) + { + return Library.Utility.Uri.UriBuilder(Url.UPLOAD, Path.File, values); + } + } +} \ No newline at end of file diff --git a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs index 0b82f4c00..53be6521a 100644 --- a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs +++ b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs @@ -140,9 +140,9 @@ namespace Duplicati.Library /// The deserialized JSON data. /// The remote URL /// A callback method that can be used to customize the request, e.g. by setting the method, content-type and headers. - /// A callback method that can be used to submit data into the body of the request. + /// A callback method that can be used to submit data into the body of the request. /// The type of data to return. - public virtual T GetJSONData(string url, Action setup = null, Action setupreq = null) + public virtual T GetJSONData(string url, Action setup = null, Action setupbodyreq = null) { var req = CreateRequest(url); @@ -151,8 +151,8 @@ namespace Duplicati.Library var areq = new AsyncHttpRequest(req); - if (setupreq != null) - setupreq(areq); + if (setupbodyreq != null) + setupbodyreq(areq); return ReadJSONResponse(areq); } diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs index e8915a629..952091942 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs @@ -64,6 +64,7 @@ namespace Duplicati.Library.Backend.OpenStack new KeyValuePair("Rackspace UK", "https://lon.identity.api.rackspacecloud.com/v2.0"), new KeyValuePair("OVH Cloud Storage", "https://auth.cloud.ovh.net/v2.0"), new KeyValuePair("Selectel Cloud Storage", "https://auth.selcdn.ru"), + new KeyValuePair("Memset Cloud Storage", "https://auth.storage.memset.com"), }; public static readonly KeyValuePair[] OPENSTACK_VERSIONS = { diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index 2536ca9d1..f3b04d18d 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -61,7 +61,7 @@ - + diff --git a/Duplicati/Library/Utility/Uri.cs b/Duplicati/Library/Utility/Uri.cs index 7e42f025c..cbdaa7a1c 100644 --- a/Duplicati/Library/Utility/Uri.cs +++ b/Duplicati/Library/Utility/Uri.cs @@ -17,6 +17,8 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; using System.Collections.Specialized; +using System.Linq; +using System.Text; using System.Web; namespace Duplicati.Library.Utility @@ -62,18 +64,18 @@ namespace Duplicati.Library.Utility /// /// The password, if any /// - public readonly string Password; - + public readonly string Password; + /// /// The original URI. /// - public readonly string OriginalUri; - + public readonly string OriginalUri; + /// /// Cache for the query parameters. /// - private NameValueCollection m_queryParams; - + private NameValueCollection m_queryParams; + /// /// Gets the paramters in the query string /// @@ -88,12 +90,12 @@ namespace Duplicati.Library.Utility m_queryParams = new NameValueCollection(); else m_queryParams = ParseQueryString(Query); - } - + } + return m_queryParams; } - } - + } + /// /// Gets the host and path. /// @@ -109,8 +111,8 @@ namespace Duplicati.Library.Utility else return Host + (Path == null ? "" : "/" + Path); } - } - + } + /// /// Gets the path and query. /// @@ -119,10 +121,10 @@ namespace Duplicati.Library.Utility { get { - return (Path ?? "") + (Query == null ? "" : "?" + Query); + return (Path ?? "") + (Query == null ? "" : "?" + Query); } - } - + } + /// /// Initializes a new instance of the struct. /// @@ -130,8 +132,8 @@ namespace Duplicati.Library.Utility public Uri(string url) { if (string.IsNullOrEmpty(url)) - throw new ArgumentNullException(nameof(url)); - + throw new ArgumentNullException(nameof(url)); + m_queryParams = null; this.OriginalUri = url; @@ -159,8 +161,8 @@ namespace Duplicati.Library.Utility { } throw new ArgumentException(Strings.Uri.UriParseError(url), nameof(url)); - } - + } + this.Scheme = m.Groups["scheme"].Value; var h = m.Groups["hostname"].Success ? m.Groups["hostname"].Value : ""; @@ -185,8 +187,8 @@ namespace Duplicati.Library.Utility this.Port = int.Parse(m.Groups["port"].Value); else this.Port = -1; - } - + } + /// /// Constructs a free-form URI from components /// @@ -208,13 +210,13 @@ namespace Duplicati.Library.Utility Password = password; Port = port; OriginalUri = AsString(scheme, host, path, query, username, password, port); - } - + } + /// /// Returns a that represents the current . /// /// A that represents the current . - public override string ToString () + public override string ToString() { return AsString(Scheme, Host, Path, Query, Username, Password, Port); } @@ -226,8 +228,8 @@ namespace Duplicati.Library.Utility { if (string.IsNullOrEmpty(Host)) throw new ArgumentException(Strings.Uri.NoHostname(OriginalUri)); - } - + } + /// /// Constructs an url-like string from components. /// @@ -243,21 +245,21 @@ namespace Duplicati.Library.Utility { var s = scheme + "://"; if (!string.IsNullOrEmpty(username) || !string.IsNullOrEmpty(password)) - { - + { + s += UrlEncode(username ?? ""); s += ":"; s += UrlEncode(password ?? ""); s += "@"; - } - + } + if (!string.IsNullOrEmpty(host)) { s += host; if (port != -1) s += ":" + port.ToString(); - } - + } + if (!string.IsNullOrEmpty(path)) { if (!string.IsNullOrEmpty(host) && !path.StartsWith("/", StringComparison.Ordinal)) @@ -267,9 +269,9 @@ namespace Duplicati.Library.Utility if (!string.IsNullOrEmpty(query)) s += "?" + query; - return s; - } - + return s; + } + /// /// Creates a new instance with another scheme /// @@ -278,8 +280,8 @@ namespace Duplicati.Library.Utility public Uri SetScheme(string scheme) { return new Uri(scheme, Host, Path, Query, Username, Password, Port); - } - + } + /// /// Creates a new instance with another host /// @@ -288,8 +290,8 @@ namespace Duplicati.Library.Utility public Uri SetHost(string host) { return new Uri(Scheme, host, Path, Query, Username, Password, Port); - } - + } + /// /// Creates a new instance with another path /// @@ -298,8 +300,8 @@ namespace Duplicati.Library.Utility public Uri SetPath(string path) { return new Uri(Scheme, Host, path, Query, Username, Password, Port); - } - + } + /// /// Creates a new instance with another query /// @@ -319,8 +321,8 @@ namespace Duplicati.Library.Utility public Uri SetCredentials(string username, string password) { return new Uri(Scheme, Host, Path, Query, username, password, Port); - } - + } + /// /// Creates a new instance with another port /// @@ -329,13 +331,13 @@ namespace Duplicati.Library.Utility public Uri SetPort(int port) { return new Uri(Scheme, Host, Path, Query, Username, Password, port); - } - + } + /// /// The regular expression that matches %20 type values in a querystring /// - private static System.Text.RegularExpressions.Regex RE_ESCAPECHAR = new System.Text.RegularExpressions.Regex(@"[^0-9a-zA-Z\-_]", System.Text.RegularExpressions.RegexOptions.Compiled); - + private static System.Text.RegularExpressions.Regex RE_ESCAPECHAR = new System.Text.RegularExpressions.Regex(@"[^0-9a-zA-Z\-_]", System.Text.RegularExpressions.RegexOptions.Compiled); + /// /// Encodes a URL, like System.Web.HttpUtility.UrlEncode /// @@ -345,44 +347,45 @@ namespace Duplicati.Library.Utility public static string UrlPathEncode(string value, System.Text.Encoding encoding = null) { return UrlEncode(value, encoding, "%20"); - } - + } + /// /// Encodes a URL, like System.Web.HttpUtility.UrlEncode /// /// The encoded URL /// The URL fragment to encode /// The encoding to use - public static string UrlEncode(string value, System.Text.Encoding encoding = null, string spacevalue = "+") + public static string UrlEncode(string value, System.Text.Encoding encoding = null, string spacevalue = "+") { if (value == null) - throw new ArgumentNullException(nameof(value)); - + throw new ArgumentNullException(nameof(value)); + encoding = encoding ?? System.Text.Encoding.UTF8; var encoder = encoding.GetEncoder(); - var inbuf = new char[1]; + var inbuf = new char[1]; var outbuf = new byte[4]; - return RE_ESCAPECHAR.Replace(value, (m) => { + return RE_ESCAPECHAR.Replace(value, (m) => + { if (m.Value == " ") - return spacevalue; - - inbuf[0] = m.Value[0]; - - try + return spacevalue; + + inbuf[0] = m.Value[0]; + + try { var len = encoder.GetBytes(inbuf, 0, 1, outbuf, 0, true); return "%" + BitConverter.ToString(outbuf, 0, len).Replace("-", "%"); } catch { - } - - //Fallback + } + + //Fallback return m.Value; - }); - + }); + } /// @@ -399,15 +402,16 @@ namespace Duplicati.Library.Utility public static string UrlDecode(string value, System.Text.Encoding encoding = null) { if (value == null) - throw new ArgumentNullException(nameof(value)); - - encoding = encoding ?? System.Text.Encoding.UTF8; - + throw new ArgumentNullException(nameof(value)); + + encoding = encoding ?? System.Text.Encoding.UTF8; + var decoder = encoding.GetDecoder(); var inbuf = new byte[8]; var outbuf = new char[8]; - return RE_NUMBER.Replace(value, (m) => { + return RE_NUMBER.Replace(value, (m) => + { if (m.Value == "+") return " "; @@ -416,24 +420,24 @@ namespace Duplicati.Library.Utility var hex = m.Groups["number"].Value; var bytelen = hex.Length / 2; Utility.HexStringAsByteArray(hex, inbuf); - var c = decoder.GetChars(inbuf, 0, bytelen, outbuf, 0); + var c = decoder.GetChars(inbuf, 0, bytelen, outbuf, 0); return new string(outbuf, 0, c); } catch { - } - - //Fallback + } + + //Fallback return m.Value; - }); - - } - + }); + + } + /// /// The regular expression that matches a=b type values in a querystring /// - private static System.Text.RegularExpressions.Regex RE_URLPARAM = new System.Text.RegularExpressions.Regex(@"(?[^\=\&]+)(\=(?[^\&]*))?", System.Text.RegularExpressions.RegexOptions.Compiled); - + private static System.Text.RegularExpressions.Regex RE_URLPARAM = new System.Text.RegularExpressions.Regex(@"(?[^\=\&]+)(\=(?[^\&]*))?", System.Text.RegularExpressions.RegexOptions.Compiled); + /// /// Parses the query string. /// This is a duplicate of the System.Web.HttpUtility.ParseQueryString that does not work well on Mono @@ -447,14 +451,104 @@ namespace Duplicati.Library.Utility if (query.StartsWith("?", StringComparison.Ordinal)) query = query.Substring(1); if (string.IsNullOrEmpty(query)) - return new NameValueCollection(StringComparer.OrdinalIgnoreCase); - + return new NameValueCollection(StringComparer.OrdinalIgnoreCase); + var result = new NameValueCollection(StringComparer.OrdinalIgnoreCase); - foreach(System.Text.RegularExpressions.Match m in RE_URLPARAM.Matches(query)) - result.Add(UrlDecode(m.Groups["key"].Value), UrlDecode(m.Groups["value"].Success ? m.Groups["value"].Value : "")); - - return result; - } + foreach (System.Text.RegularExpressions.Match m in RE_URLPARAM.Matches(query)) + result.Add(UrlDecode(m.Groups["key"].Value), UrlDecode(m.Groups["value"].Success ? m.Groups["value"].Value : "")); + + return result; + } + + /// + /// Build the querystring to be used in a URL + /// + /// The generated querystring + /// A collection of name value pairs to be translated into a query string + /// The delimiter to separate key value pairs in the query string + public static string BuildUriQuery(NameValueCollection query, string delimiter) + { + + if (query == null) + throw new ArgumentNullException(nameof(query)); + + StringBuilder builder = new StringBuilder(); + foreach (var key in query.Cast().Where(key => !string.IsNullOrEmpty(query[key]))) + { + builder.Append(builder.Length == 0 ? string.Empty : delimiter) + .Append(key) + .Append("=") + .Append(query[key]); + } + + return builder.ToString(); + } + + /// + /// Build the querystring to be used in a URL + /// + /// The generated querystring + /// A collection of name value pairs to be translated into a query string that is + /// ampsersand delimited. + public static string BuildUriQuery(NameValueCollection query) + { + return BuildUriQuery(query, "&"); + } + + /// + /// Builds a URL together using a base URL, a path and a query. + /// + /// The built together URL. + /// Base URL, containing schema, host, port. + /// Base path. + /// A collection of name value pairs to be translated into a query string. + public static string UriBuilder(string url, string path, NameValueCollection query) + { + var builder = new UriBuilder(url) + { + Path = ConcatPaths(ExtractPath(url), path), + Query = query != null ? BuildUriQuery(query) : null + }; + return builder.Uri.AbsoluteUri; + } + + /// + /// Concats paths of URIs. + /// + /// The concatenated paths. + /// Path1. + /// Path2. + public static string ConcatPaths(string path1, string path2) + { + if (string.IsNullOrEmpty(path2)) + { + return path1; + } + + return path1.TrimEnd('/') + '/' + path2; + } + + /// + /// Grab path part of a URI. + /// At the moment, simple implementation does not remove fragments. + /// + /// The path. + /// URL. + public static string ExtractPath(string url) + { + return (new Uri(url)).Path; + } + + /// + /// Builds a URL together using a base URL and path. + /// + /// The built together URL. + /// Base URL, containing schema, host, port. + /// Base path. + public static string UriBuilder(string url, string path) + { + return UriBuilder(url, path, null); + } } } diff --git a/Duplicati/Library/Utility/UrlUtillity.cs b/Duplicati/Library/Utility/UrlUtility.cs similarity index 97% rename from Duplicati/Library/Utility/UrlUtillity.cs rename to Duplicati/Library/Utility/UrlUtility.cs index 84469e019..8e2c87a36 100644 --- a/Duplicati/Library/Utility/UrlUtillity.cs +++ b/Duplicati/Library/Utility/UrlUtility.cs @@ -23,7 +23,7 @@ using System.Text; namespace Duplicati.Library.Utility { - public static class UrlUtillity + public static class UrlUtility { /// /// The file path to the system browser selected diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 794cd6690..9e6147989 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1,4 +1,3 @@ -#region Disclaimer / License // Copyright (C) 2015, The Duplicati Team // http://www.duplicati.com, info@duplicati.com // @@ -15,21 +14,13 @@ // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// -using System.Text.RegularExpressions; -using System.Linq; -using System.Threading.Tasks; - - -#endregion - using System; -using System.Text.RegularExpressions; -using System.Linq; using System.Collections.Generic; using System.IO; -using System.Text; +using System.Linq; using System.Threading.Tasks; +using System.Text; +using System.Text.RegularExpressions; namespace Duplicati.Library.Utility { diff --git a/Duplicati/UnitTest/Duplicati.UnitTest.csproj b/Duplicati/UnitTest/Duplicati.UnitTest.csproj index da8f73258..fabf1506f 100644 --- a/Duplicati/UnitTest/Duplicati.UnitTest.csproj +++ b/Duplicati/UnitTest/Duplicati.UnitTest.csproj @@ -50,6 +50,7 @@ + @@ -173,4 +174,4 @@ - \ No newline at end of file + diff --git a/Duplicati/UnitTest/UriUtilityTests.cs b/Duplicati/UnitTest/UriUtilityTests.cs new file mode 100644 index 000000000..84d43c8a3 --- /dev/null +++ b/Duplicati/UnitTest/UriUtilityTests.cs @@ -0,0 +1,69 @@ +// Copyright (C) 2018, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +using NUnit.Framework; +using System.Collections.Specialized; + +namespace Duplicati.UnitTest +{ + public class UriUtilityTests + { + [Test] + [Category("UriUtility")] + public static void TestBuildUriQuery() + { + var query = new NameValueCollection { { "a", "b" } }; + var queryUrl = Library.Utility.Uri.BuildUriQuery(query); + Assert.AreEqual("a=b", queryUrl); + query.Add(new NameValueCollection { { "c", "d" } }); + queryUrl = Library.Utility.Uri.BuildUriQuery(query); + Assert.AreEqual("a=b&c=d", queryUrl); + } + + [Test] + [Category("UriUtility")] + public static void TestUrlBuilder() + { + var baseUrl = "http://localhost"; + var path = "files"; + var query = new NameValueCollection { { "a", "b" }, { "c", "d" } }; + var url = Library.Utility.Uri.UriBuilder(baseUrl, path, query); + Assert.AreEqual(baseUrl + "/" + path + "?a=b&c=d", url); + } + + [Test] + [Category("UriUtility")] + public static void TestExtractPath() + { + var url = "http://localhost/a/b"; + var path = Library.Utility.Uri.ExtractPath(url); + Assert.AreEqual("a/b", path); + } + + + [Test] + [Category("UriUtility")] + public static void TestConcatPaths() + { + var path1 = "/a"; + var path2 = "b/"; + Assert.AreEqual("/a/b/", Library.Utility.Uri.ConcatPaths(path1, path2)); + Assert.AreEqual("/a", Library.Utility.Uri.ConcatPaths(path1, null)); + Assert.AreEqual("/b/", Library.Utility.Uri.ConcatPaths(string.Empty, path2)); + } + } +}