From ddba3459bf2df6d22780da00ec86edb711b83263 Mon Sep 17 00:00:00 2001 From: verhoek Date: Mon, 30 Apr 2018 20:50:49 +0200 Subject: [PATCH 01/16] Removed extra local var. --- Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index fe846c529..4b36761bc 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -110,9 +110,9 @@ namespace Duplicati.Library.Backend.GoogleDrive if (entries != null) return entries; - var list = ListFolder(CurrentFolderId, false, remotename).ToArray(); + entries = ListFolder(CurrentFolderId, false, remotename).ToArray(); - if (list == null || list.Length == 0) + if (entries == null || entries.Length == 0) { if (throwMissingException) throw new FileMissingException(); @@ -120,7 +120,7 @@ namespace Duplicati.Library.Backend.GoogleDrive return null; } - return m_filecache[remotename] = list; + return m_filecache[remotename] = entries; } private static string EscapeTitleEntries(string title) From 55e349f4101c39457503d96b171a213abdd46086 Mon Sep 17 00:00:00 2001 From: verhoek Date: Wed, 9 May 2018 18:57:04 +0200 Subject: [PATCH 02/16] Renamed variable to indicate meaning properly. --- Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs index 0b82f4c00..53be6521a 100644 --- a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs +++ b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs @@ -140,9 +140,9 @@ namespace Duplicati.Library /// The deserialized JSON data. /// The remote URL /// A callback method that can be used to customize the request, e.g. by setting the method, content-type and headers. - /// A callback method that can be used to submit data into the body of the request. + /// A callback method that can be used to submit data into the body of the request. /// The type of data to return. - public virtual T GetJSONData(string url, Action setup = null, Action setupreq = null) + public virtual T GetJSONData(string url, Action setup = null, Action setupbodyreq = null) { var req = CreateRequest(url); @@ -151,8 +151,8 @@ namespace Duplicati.Library var areq = new AsyncHttpRequest(req); - if (setupreq != null) - setupreq(areq); + if (setupbodyreq != null) + setupbodyreq(areq); return ReadJSONResponse(areq); } From d303db30d62b3ffb72a12efc9fbf31ddb5124382 Mon Sep 17 00:00:00 2001 From: verhoek Date: Fri, 11 May 2018 18:23:16 +0200 Subject: [PATCH 03/16] Replacing magic strings in google drive rest calls by query key value pairs. --- .../Backend/GoogleServices/GoogleDrive.cs | 83 ++++++++++++++++--- Duplicati/Library/Utility/Uri.cs | 45 +++++++++- Duplicati/UnitTest/Duplicati.UnitTest.csproj | 1 + Duplicati/UnitTest/UrlUtilityTests.cs | 48 +++++++++++ 4 files changed, 164 insertions(+), 13 deletions(-) create mode 100644 Duplicati/UnitTest/UrlUtilityTests.cs diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index 4b36761bc..9d3041cee 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -22,6 +22,7 @@ using System.Collections.Generic; using System.Net; using Newtonsoft.Json; using Duplicati.Library.Backend.GoogleServices; +using System.Collections.Specialized; namespace Duplicati.Library.Backend.GoogleDrive { @@ -141,20 +142,23 @@ namespace Duplicati.Library.Backend.GoogleDrive GoogleDriveFolderItem[] files; m_filecache.TryGetValue(remotename, out files); - string fileid = null; + string fileId = null; if (files != null) { if (files.Length == 1) - fileid = files[0].id; + fileId = files[0].id; else Delete(remotename); } - var isUpdate = !string.IsNullOrWhiteSpace(fileid); + var isUpdate = !string.IsNullOrWhiteSpace(fileId); + + var values = new NameValueCollection { { GoogleWebApiStrings.UploadType, "resumable" } }; + PrepareFileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values); var url = isUpdate ? - string.Format("{0}/files/{1}?uploadType=resumable", DRIVE_API_UPLOAD_URL, Library.Utility.Uri.UrlPathEncode(fileid)) : - string.Format("{0}/files?uploadType=resumable", DRIVE_API_UPLOAD_URL); + PrepareFileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values): + PrepareFileUploadUrl(values); var item = new GoogleDriveFolderItem() { title = remotename, @@ -181,9 +185,12 @@ namespace Duplicati.Library.Backend.GoogleDrive if (m_filecache.Count == 0) foreach (var file in List()) { /* Enumerate the full listing */ } - var fileid = GetFileEntries(remotename).OrderByDescending(x => x.createdDate).First().id; + var fileId = GetFileEntries(remotename).OrderByDescending(x => x.createdDate).First().id; - var req = m_oauth.CreateRequest(string.Format("{0}/files/{1}?alt=media{2}", DRIVE_API_URL, fileid, m_useTeamDrive ? "&supportsTeamDrives=true" : string.Empty)); + var url = PrepareFileQueryUrl(fileId, new NameValueCollection{ + { "alt", "media"} + }); + var req = m_oauth.CreateRequest(url); var areq = new AsyncHttpRequest(req); using(var resp = (HttpWebResponse)areq.GetResponse()) using(var rs = areq.GetResponseStream()) @@ -266,8 +273,7 @@ namespace Duplicati.Library.Backend.GoogleDrive { foreach(var fileid in from n in GetFileEntries(remotename) select n.id) { - var url = string.Format("{0}/files/{1}{2}", DRIVE_API_URL, Library.Utility.Uri.UrlPathEncode(fileid), m_useTeamDrive ? "?supportsTeamDrives=true" : string.Empty); - + var url = PrepareFileQueryUrl(Library.Utility.Uri.UrlPathEncode(fileid), SupportsTeamDriveOption()); m_oauth.GetJSONData(url, x => { x.Method = "DELETE"; }); @@ -367,7 +373,7 @@ namespace Duplicati.Library.Backend.GoogleDrive newfile.title = newname; newfile.parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference() { id = CurrentFolderId } }; - var url = string.Format("{0}/files/{1}", DRIVE_API_UPLOAD_URL, Library.Utility.Uri.UrlPathEncode(files[0].id)); + var url = PrepareFileQueryUrl(Library.Utility.Uri.UrlPathEncode(files[0].id)); var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(newfile)); var nf = m_oauth.GetJSONData(url, x => { @@ -401,6 +407,52 @@ namespace Duplicati.Library.Backend.GoogleDrive #endregion + + private static class GoogleWebApiStrings + { + public static string FilePath { get { return "files"; } } + public static string SupportsTeamDriveOption { get { return "supportsTeamDrives"; } } + public static string IncludeTeamDriveOption { get { return "includeTeamDriveItems"; } } + public static string UploadType { get { return "uploadType"; } } + } + + + private NameValueCollection SupportsTeamDriveOption() + { + return m_useTeamDrive? new NameValueCollection { { GoogleWebApiStrings.SupportsTeamDriveOption, "true"}} : null; + } + + private NameValueCollection IncludeTeamDriveOption() + { + return m_useTeamDrive ? new NameValueCollection { { GoogleWebApiStrings.IncludeTeamDriveOption, "true" } } : null; + } + + private string PrepareFileQueryUrl(NameValueCollection values) + { + return Library.Utility.Uri.UrlBuilder(DRIVE_API_URL, GoogleWebApiStrings.FilePath, values); + } + + private string PrepareFileQueryUrl(string fileId, NameValueCollection values = null) + { + var path = GoogleWebApiStrings.FilePath; + path += "/" + fileId; + + return Library.Utility.Uri.UrlBuilder(DRIVE_API_URL, path, values); + } + + private string PrepareFileUploadUrl(string fileId, NameValueCollection values) + { + var path = GoogleWebApiStrings.FilePath; + path += "/" + fileId; + return Library.Utility.Uri.UrlBuilder(DRIVE_API_UPLOAD_URL, path, values); + } + + private string PrepareFileUploadUrl(NameValueCollection values) + { + var path = GoogleWebApiStrings.FilePath; + return Library.Utility.Uri.UrlBuilder(DRIVE_API_UPLOAD_URL, path, values); + } + private class GoogleDriveParentReference { public string kind { get; set; } @@ -476,7 +528,13 @@ namespace Duplicati.Library.Backend.GoogleDrive string.Format("'{0}' in parents", EscapeTitleEntries(parentfolder)) }; - var url = string.Format("{0}/files?q={1}{2}", DRIVE_API_URL, Library.Utility.Uri.UrlEncode(string.Join(" and ", p.Where(x => x != null))), m_useTeamDrive ? "&supportsTeamDrives=true&includeTeamDriveItems=true" : string.Empty); + var queryParams = new NameValueCollection + { + {"q", Library.Utility.Uri.UrlEncode(string.Join(" and ", p.Where(x => x != null)))}, + }; + queryParams.Add(SupportsTeamDriveOption()); + queryParams.Add(IncludeTeamDriveOption()); + var url = PrepareFileQueryUrl(queryParams); var token = string.Empty; do @@ -497,7 +555,8 @@ namespace Duplicati.Library.Backend.GoogleDrive private GoogleDriveFolderItem CreateFolder(string name, string parent) { - var url = string.Format("{0}/files{1}", DRIVE_API_URL, m_useTeamDrive ? "?supportsTeamDrives=true" : string.Empty); + var url = PrepareFileQueryUrl(SupportsTeamDriveOption()); + var folder = new GoogleDriveFolderItem() { title = name, description = name, diff --git a/Duplicati/Library/Utility/Uri.cs b/Duplicati/Library/Utility/Uri.cs index 7e42f025c..d9c5fa601 100644 --- a/Duplicati/Library/Utility/Uri.cs +++ b/Duplicati/Library/Utility/Uri.cs @@ -17,6 +17,8 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; using System.Collections.Specialized; +using System.Linq; +using System.Text; using System.Web; namespace Duplicati.Library.Utility @@ -454,7 +456,48 @@ namespace Duplicati.Library.Utility result.Add(UrlDecode(m.Groups["key"].Value), UrlDecode(m.Groups["value"].Success ? m.Groups["value"].Value : "")); return result; - } + } + + /// + /// Build the querystring to be used in a URL + /// + /// The generated querystring + /// A collection of name value pairs to be translated into a query string + /// The delimiter to separate key value pairs in the query string + public static string BuildUriQuery(NameValueCollection query, string delimiter = "&") { + + if (query == null) + throw new ArgumentNullException(nameof(query)); + + StringBuilder builder = new StringBuilder(); + foreach (var key in query.Cast().Where(key => !string.IsNullOrEmpty(query[key]))) + { + builder.Append(builder.Length == 0 ? "?" : delimiter) + .Append(key) + .Append("=") + .Append(query[key]); + } + + return builder.ToString(); + } + + /// + /// Builds a URL together using a base URL, a path and a query. + /// + /// The built together URL. + /// Base URL, containing schema, host, port. + /// Base path. + /// A collection of name value pairs to be translated into a query string. + public static string UrlBuilder(string baseUrl, string basePath, NameValueCollection query) + { + var builder = new UriBuilder(baseUrl) + { + Path = basePath, + Query = query != null ? BuildUriQuery(query) : null + }; + return builder.Uri.AbsoluteUri; + } + } } diff --git a/Duplicati/UnitTest/Duplicati.UnitTest.csproj b/Duplicati/UnitTest/Duplicati.UnitTest.csproj index da8f73258..d8faa52d5 100644 --- a/Duplicati/UnitTest/Duplicati.UnitTest.csproj +++ b/Duplicati/UnitTest/Duplicati.UnitTest.csproj @@ -50,6 +50,7 @@ + diff --git a/Duplicati/UnitTest/UrlUtilityTests.cs b/Duplicati/UnitTest/UrlUtilityTests.cs new file mode 100644 index 000000000..4ad5fd17c --- /dev/null +++ b/Duplicati/UnitTest/UrlUtilityTests.cs @@ -0,0 +1,48 @@ +// Copyright (C) 2018, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +using NUnit.Framework; +using System.Collections.Specialized; + +namespace Duplicati.UnitTest +{ + public class UrlUtilityTests + { + [Test] + [Category("UrlUtility")] + public void TestBuildUriQuery() + { + var query = new NameValueCollection { { "a", "b" } }; + var queryUrl = Library.Utility.Uri.BuildUriQuery(query); + Assert.AreEqual("?a=b", queryUrl); + query.Add(new NameValueCollection { { "c", "d" } }); + queryUrl = Library.Utility.Uri.BuildUriQuery(query); + Assert.AreEqual("?a=b&c=d", queryUrl); + } + + [Test] + [Category("UrlUtility")] + public void TestUrlBuilder() + { + var baseUrl = "http://localhost"; + var path = "files"; + var query = new NameValueCollection { { "a", "b" }, { "c", "d" } }; + var url = Library.Utility.Uri.UrlBuilder(baseUrl, path, query); + Assert.AreEqual(baseUrl + "/" + path + "?a=b&c=d", url); + } + } +} From 78a46c3a5c60be265bfbc5521ce95cd38746cab5 Mon Sep 17 00:00:00 2001 From: verhoek Date: Fri, 11 May 2018 19:09:14 +0200 Subject: [PATCH 04/16] Renamed misspelled class urluttility to urlutility and newly added urlutility to uriutility. --- .../Duplicati.GUI.TrayIcon/TrayIconBase.cs | 2 +- .../Backend/GoogleServices/GoogleDrive.cs | 29 ++++++++----------- .../Utility/Duplicati.Library.Utility.csproj | 2 +- Duplicati/Library/Utility/Uri.cs | 2 +- .../Utility/{UrlUtillity.cs => UrlUtility.cs} | 2 +- Duplicati/UnitTest/Duplicati.UnitTest.csproj | 4 +-- ...{UrlUtilityTests.cs => UriUtilityTests.cs} | 4 +-- 7 files changed, 20 insertions(+), 25 deletions(-) rename Duplicati/Library/Utility/{UrlUtillity.cs => UrlUtility.cs} (97%) rename Duplicati/UnitTest/{UrlUtilityTests.cs => UriUtilityTests.cs} (92%) diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs index 4b4788e81..36c5b0a36 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/TrayIconBase.cs @@ -134,7 +134,7 @@ namespace Duplicati.GUI.TrayIcon public virtual IBrowserWindow ShowUrlInWindow(string url) { //Fallback is to just show the window in a browser - Duplicati.Library.Utility.UrlUtillity.OpenURL(url, Program.BrowserCommand); + Duplicati.Library.Utility.UrlUtility.OpenURL(url, Program.BrowserCommand); return null; } diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index 9d3041cee..92c46a0ce 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -153,7 +153,7 @@ namespace Duplicati.Library.Backend.GoogleDrive var isUpdate = !string.IsNullOrWhiteSpace(fileId); - var values = new NameValueCollection { { GoogleWebApiStrings.UploadType, "resumable" } }; + var values = new NameValueCollection { { GoogleDriveApiStrings.UploadTypeOption, "resumable" } }; PrepareFileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values); var url = isUpdate ? @@ -176,7 +176,6 @@ namespace Duplicati.Library.Backend.GoogleDrive m_filecache.Clear(); throw; } - } public void Get(string remotename, System.IO.Stream stream) @@ -381,11 +380,10 @@ namespace Duplicati.Library.Backend.GoogleDrive x.ContentLength = data.Length; x.ContentType = "application/json; charset=UTF-8"; }, x => { - using(var rs = x.GetRequestStream()) rs.Write(data, 0, data.Length); }); - + m_filecache[newname] = new GoogleDriveFolderItem[] { nf }; m_filecache.Remove(oldname); } @@ -407,34 +405,34 @@ namespace Duplicati.Library.Backend.GoogleDrive #endregion - - private static class GoogleWebApiStrings + private static class GoogleDriveApiStrings { public static string FilePath { get { return "files"; } } + public static string AboutPath { get { return "about"; } } public static string SupportsTeamDriveOption { get { return "supportsTeamDrives"; } } public static string IncludeTeamDriveOption { get { return "includeTeamDriveItems"; } } - public static string UploadType { get { return "uploadType"; } } + public static string UploadTypeOption { get { return "uploadType"; } } } private NameValueCollection SupportsTeamDriveOption() { - return m_useTeamDrive? new NameValueCollection { { GoogleWebApiStrings.SupportsTeamDriveOption, "true"}} : null; + return m_useTeamDrive? new NameValueCollection { { GoogleDriveApiStrings.SupportsTeamDriveOption, "true"}} : null; } private NameValueCollection IncludeTeamDriveOption() { - return m_useTeamDrive ? new NameValueCollection { { GoogleWebApiStrings.IncludeTeamDriveOption, "true" } } : null; + return m_useTeamDrive ? new NameValueCollection { { GoogleDriveApiStrings.IncludeTeamDriveOption, "true" } } : null; } private string PrepareFileQueryUrl(NameValueCollection values) { - return Library.Utility.Uri.UrlBuilder(DRIVE_API_URL, GoogleWebApiStrings.FilePath, values); + return Library.Utility.Uri.UrlBuilder(DRIVE_API_URL, GoogleDriveApiStrings.FilePath, values); } private string PrepareFileQueryUrl(string fileId, NameValueCollection values = null) { - var path = GoogleWebApiStrings.FilePath; + var path = GoogleDriveApiStrings.FilePath; path += "/" + fileId; return Library.Utility.Uri.UrlBuilder(DRIVE_API_URL, path, values); @@ -442,14 +440,14 @@ namespace Duplicati.Library.Backend.GoogleDrive private string PrepareFileUploadUrl(string fileId, NameValueCollection values) { - var path = GoogleWebApiStrings.FilePath; + var path = GoogleDriveApiStrings.FilePath; path += "/" + fileId; return Library.Utility.Uri.UrlBuilder(DRIVE_API_UPLOAD_URL, path, values); } private string PrepareFileUploadUrl(NameValueCollection values) { - var path = GoogleWebApiStrings.FilePath; + var path = GoogleDriveApiStrings.FilePath; return Library.Utility.Uri.UrlBuilder(DRIVE_API_UPLOAD_URL, path, values); } @@ -549,7 +547,7 @@ namespace Duplicati.Library.Backend.GoogleDrive private GoogleDriveAboutResponse GetAboutInfo() { - var url = string.Format("{0}/about", DRIVE_API_URL); + var url = Library.Utility.Uri.UrlBuilder(DRIVE_API_URL, GoogleDriveApiStrings.AboutPath); return m_oauth.GetJSONData(url); } @@ -574,12 +572,9 @@ namespace Duplicati.Library.Backend.GoogleDrive x.ContentLength = data.Length; }, req => { - using(var rs = req.GetRequestStream()) rs.Write(data, 0, data.Length); - }); - } } } diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index 77b51de14..fa291b25d 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -60,7 +60,7 @@ - + diff --git a/Duplicati/Library/Utility/Uri.cs b/Duplicati/Library/Utility/Uri.cs index d9c5fa601..b44cac2e0 100644 --- a/Duplicati/Library/Utility/Uri.cs +++ b/Duplicati/Library/Utility/Uri.cs @@ -488,7 +488,7 @@ namespace Duplicati.Library.Utility /// Base URL, containing schema, host, port. /// Base path. /// A collection of name value pairs to be translated into a query string. - public static string UrlBuilder(string baseUrl, string basePath, NameValueCollection query) + public static string UriBuilder(string baseUrl, string basePath, NameValueCollection query = null) { var builder = new UriBuilder(baseUrl) { diff --git a/Duplicati/Library/Utility/UrlUtillity.cs b/Duplicati/Library/Utility/UrlUtility.cs similarity index 97% rename from Duplicati/Library/Utility/UrlUtillity.cs rename to Duplicati/Library/Utility/UrlUtility.cs index 84469e019..8e2c87a36 100644 --- a/Duplicati/Library/Utility/UrlUtillity.cs +++ b/Duplicati/Library/Utility/UrlUtility.cs @@ -23,7 +23,7 @@ using System.Text; namespace Duplicati.Library.Utility { - public static class UrlUtillity + public static class UrlUtility { /// /// The file path to the system browser selected diff --git a/Duplicati/UnitTest/Duplicati.UnitTest.csproj b/Duplicati/UnitTest/Duplicati.UnitTest.csproj index d8faa52d5..fabf1506f 100644 --- a/Duplicati/UnitTest/Duplicati.UnitTest.csproj +++ b/Duplicati/UnitTest/Duplicati.UnitTest.csproj @@ -50,7 +50,7 @@ - + @@ -174,4 +174,4 @@ - \ No newline at end of file + diff --git a/Duplicati/UnitTest/UrlUtilityTests.cs b/Duplicati/UnitTest/UriUtilityTests.cs similarity index 92% rename from Duplicati/UnitTest/UrlUtilityTests.cs rename to Duplicati/UnitTest/UriUtilityTests.cs index 4ad5fd17c..d795afc91 100644 --- a/Duplicati/UnitTest/UrlUtilityTests.cs +++ b/Duplicati/UnitTest/UriUtilityTests.cs @@ -20,7 +20,7 @@ using System.Collections.Specialized; namespace Duplicati.UnitTest { - public class UrlUtilityTests + public class UriUtilityTests { [Test] [Category("UrlUtility")] @@ -41,7 +41,7 @@ namespace Duplicati.UnitTest var baseUrl = "http://localhost"; var path = "files"; var query = new NameValueCollection { { "a", "b" }, { "c", "d" } }; - var url = Library.Utility.Uri.UrlBuilder(baseUrl, path, query); + var url = Library.Utility.Uri.UriBuilder(baseUrl, path, query); Assert.AreEqual(baseUrl + "/" + path + "?a=b&c=d", url); } } From 44b5b0027672ce2fb37eca4974d1865a68d9cd94 Mon Sep 17 00:00:00 2001 From: verhoek Date: Fri, 11 May 2018 19:57:09 +0200 Subject: [PATCH 05/16] Renamed some googledriveapi var names. Modified ListFolder to be more readable. --- .../Backend/GoogleServices/GoogleDrive.cs | 50 ++++++++++--------- Duplicati/Library/Utility/Uri.cs | 1 - Duplicati/UnitTest/UriUtilityTests.cs | 2 +- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index 92c46a0ce..ae3fbb160 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -153,7 +153,7 @@ namespace Duplicati.Library.Backend.GoogleDrive var isUpdate = !string.IsNullOrWhiteSpace(fileId); - var values = new NameValueCollection { { GoogleDriveApiStrings.UploadTypeOption, "resumable" } }; + var values = new NameValueCollection { { GoogleDriveApiStrings.UploadTypeParam, "resumable" } }; PrepareFileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values); var url = isUpdate ? @@ -272,7 +272,7 @@ namespace Duplicati.Library.Backend.GoogleDrive { foreach(var fileid in from n in GetFileEntries(remotename) select n.id) { - var url = PrepareFileQueryUrl(Library.Utility.Uri.UrlPathEncode(fileid), SupportsTeamDriveOption()); + var url = PrepareFileQueryUrl(Library.Utility.Uri.UrlPathEncode(fileid), SupportsTeamDriveParam()); m_oauth.GetJSONData(url, x => { x.Method = "DELETE"; }); @@ -409,25 +409,26 @@ namespace Duplicati.Library.Backend.GoogleDrive { public static string FilePath { get { return "files"; } } public static string AboutPath { get { return "about"; } } - public static string SupportsTeamDriveOption { get { return "supportsTeamDrives"; } } - public static string IncludeTeamDriveOption { get { return "includeTeamDriveItems"; } } - public static string UploadTypeOption { get { return "uploadType"; } } + public static string SupportsTeamDriveParam { get { return "supportsTeamDrives"; } } + public static string IncludeTeamDriveParam { get { return "includeTeamDriveItems"; } } + public static string UploadTypeParam { get { return "uploadType"; } } + public static string pageTokenParam { get { return "uploadType"; } } } - private NameValueCollection SupportsTeamDriveOption() + private NameValueCollection SupportsTeamDriveParam() { - return m_useTeamDrive? new NameValueCollection { { GoogleDriveApiStrings.SupportsTeamDriveOption, "true"}} : null; + return m_useTeamDrive? new NameValueCollection { { GoogleDriveApiStrings.SupportsTeamDriveParam, "true"}} : null; } - private NameValueCollection IncludeTeamDriveOption() + private NameValueCollection IncludeTeamDriveParam() { - return m_useTeamDrive ? new NameValueCollection { { GoogleDriveApiStrings.IncludeTeamDriveOption, "true" } } : null; + return m_useTeamDrive ? new NameValueCollection { { GoogleDriveApiStrings.IncludeTeamDriveParam, "true" } } : null; } private string PrepareFileQueryUrl(NameValueCollection values) { - return Library.Utility.Uri.UrlBuilder(DRIVE_API_URL, GoogleDriveApiStrings.FilePath, values); + return Library.Utility.Uri.UriBuilder(DRIVE_API_URL, GoogleDriveApiStrings.FilePath, values); } private string PrepareFileQueryUrl(string fileId, NameValueCollection values = null) @@ -435,20 +436,20 @@ namespace Duplicati.Library.Backend.GoogleDrive var path = GoogleDriveApiStrings.FilePath; path += "/" + fileId; - return Library.Utility.Uri.UrlBuilder(DRIVE_API_URL, path, values); + return Library.Utility.Uri.UriBuilder(DRIVE_API_URL, path, values); } private string PrepareFileUploadUrl(string fileId, NameValueCollection values) { var path = GoogleDriveApiStrings.FilePath; path += "/" + fileId; - return Library.Utility.Uri.UrlBuilder(DRIVE_API_UPLOAD_URL, path, values); + return Library.Utility.Uri.UriBuilder(DRIVE_API_UPLOAD_URL, path, values); } private string PrepareFileUploadUrl(NameValueCollection values) { var path = GoogleDriveApiStrings.FilePath; - return Library.Utility.Uri.UrlBuilder(DRIVE_API_UPLOAD_URL, path, values); + return Library.Utility.Uri.UriBuilder(DRIVE_API_UPLOAD_URL, path, values); } private class GoogleDriveParentReference @@ -530,30 +531,33 @@ namespace Duplicati.Library.Backend.GoogleDrive { {"q", Library.Utility.Uri.UrlEncode(string.Join(" and ", p.Where(x => x != null)))}, }; - queryParams.Add(SupportsTeamDriveOption()); - queryParams.Add(IncludeTeamDriveOption()); - var url = PrepareFileQueryUrl(queryParams); - var token = string.Empty; + queryParams.Add(SupportsTeamDriveParam()); + queryParams.Add(IncludeTeamDriveParam()); - do + while (true) { - var res = m_oauth.GetJSONData(url + (string.IsNullOrWhiteSpace(token) ? "" : "&pageToken=" + Library.Utility.Uri.UrlEncode(token))); + var url = PrepareFileQueryUrl(queryParams); + var res = m_oauth.GetJSONData(url); foreach (var n in res.items) yield return n; - token = res.nextPageToken; - } while (!string.IsNullOrWhiteSpace(token)); + var token = res.nextPageToken; + if (string.IsNullOrWhiteSpace(token)) + break; + + queryParams.Set(GoogleDriveApiStrings.pageTokenParam, token); + } } private GoogleDriveAboutResponse GetAboutInfo() { - var url = Library.Utility.Uri.UrlBuilder(DRIVE_API_URL, GoogleDriveApiStrings.AboutPath); + var url = Library.Utility.Uri.UriBuilder(DRIVE_API_URL, GoogleDriveApiStrings.AboutPath); return m_oauth.GetJSONData(url); } private GoogleDriveFolderItem CreateFolder(string name, string parent) { - var url = PrepareFileQueryUrl(SupportsTeamDriveOption()); + var url = PrepareFileQueryUrl(SupportsTeamDriveParam()); var folder = new GoogleDriveFolderItem() { title = name, diff --git a/Duplicati/Library/Utility/Uri.cs b/Duplicati/Library/Utility/Uri.cs index b44cac2e0..150552dcb 100644 --- a/Duplicati/Library/Utility/Uri.cs +++ b/Duplicati/Library/Utility/Uri.cs @@ -497,7 +497,6 @@ namespace Duplicati.Library.Utility }; return builder.Uri.AbsoluteUri; } - } } diff --git a/Duplicati/UnitTest/UriUtilityTests.cs b/Duplicati/UnitTest/UriUtilityTests.cs index d795afc91..5c4c30e65 100644 --- a/Duplicati/UnitTest/UriUtilityTests.cs +++ b/Duplicati/UnitTest/UriUtilityTests.cs @@ -43,6 +43,6 @@ namespace Duplicati.UnitTest var query = new NameValueCollection { { "a", "b" }, { "c", "d" } }; var url = Library.Utility.Uri.UriBuilder(baseUrl, path, query); Assert.AreEqual(baseUrl + "/" + path + "?a=b&c=d", url); - } + } } } From 051b030836006dd66cf92728bf603c22edda6c65 Mon Sep 17 00:00:00 2001 From: verhoek Date: Sat, 12 May 2018 12:42:51 +0200 Subject: [PATCH 06/16] Fixed typo regarding pageToken. Integrated other strings in static string class. --- .../Backend/GoogleServices/GoogleDrive.cs | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index ae3fbb160..ba8baf607 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -35,10 +35,10 @@ namespace Duplicati.Library.Backend.GoogleDrive private const string DRIVE_API_UPLOAD_URL = "https://www.googleapis.com/upload/drive/v2"; private const string DRIVE_API_URL = "https://www.googleapis.com/drive/v2"; - private string m_path; + private readonly string m_path; private bool m_useTeamDrive = true; - private OAuthHelper m_oauth; + private readonly OAuthHelper m_oauth; private string m_currentFolderId; private Dictionary m_filecache; @@ -120,7 +120,7 @@ namespace Duplicati.Library.Backend.GoogleDrive else return null; } - + return m_filecache[remotename] = entries; } @@ -153,7 +153,7 @@ namespace Duplicati.Library.Backend.GoogleDrive var isUpdate = !string.IsNullOrWhiteSpace(fileId); - var values = new NameValueCollection { { GoogleDriveApiStrings.UploadTypeParam, "resumable" } }; + var values = new NameValueCollection { { GoogleDriveApiStrings.UploadTypeParam, GoogleDriveApiStrings.UploadTypeResumableValue } }; PrepareFileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values); var url = isUpdate ? @@ -187,7 +187,7 @@ namespace Duplicati.Library.Backend.GoogleDrive var fileId = GetFileEntries(remotename).OrderByDescending(x => x.createdDate).First().id; var url = PrepareFileQueryUrl(fileId, new NameValueCollection{ - { "alt", "media"} + { GoogleDriveApiStrings.AltParam, GoogleDriveApiStrings.AltMediaValue } }); var req = m_oauth.CreateRequest(url); var areq = new AsyncHttpRequest(req); @@ -253,7 +253,7 @@ namespace Duplicati.Library.Backend.GoogleDrive } } } - + public void Put(string remotename, string filename) { using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) @@ -319,8 +319,14 @@ namespace Duplicati.Library.Backend.GoogleDrive { get { return new List(new ICommandLineArgument[] { - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.GoogleDrive.AuthidShort, Strings.GoogleDrive.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("googledrive"))), - new CommandLineArgument(DISABLE_TEAMDRIVE_OPTION, CommandLineArgument.ArgumentType.Boolean, Strings.GoogleDrive.DisableTeamDriveShort, Strings.GoogleDrive.DisableTeamDriveLong), + new CommandLineArgument(AUTHID_OPTION, + CommandLineArgument.ArgumentType.Password, + Strings.GoogleDrive.AuthidShort, + Strings.GoogleDrive.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("googledrive"))), + new CommandLineArgument(DISABLE_TEAMDRIVE_OPTION, + CommandLineArgument.ArgumentType.Boolean, + Strings.GoogleDrive.DisableTeamDriveShort, + Strings.GoogleDrive.DisableTeamDriveLong), }); } } @@ -408,22 +414,28 @@ namespace Duplicati.Library.Backend.GoogleDrive private static class GoogleDriveApiStrings { public static string FilePath { get { return "files"; } } + public static string FileQueryParam { get { return "q"; } } public static string AboutPath { get { return "about"; } } public static string SupportsTeamDriveParam { get { return "supportsTeamDrives"; } } public static string IncludeTeamDriveParam { get { return "includeTeamDriveItems"; } } + public static string True { get { return "true"; } } + public static string False { get { return "false"; } } + public static string PageTokenParam { get { return "pageToken"; } } public static string UploadTypeParam { get { return "uploadType"; } } - public static string pageTokenParam { get { return "uploadType"; } } + public static string UploadTypeResumableValue { get { return "resumable"; } } + public static string AltParam { get { return "alt"; } } + public static string AltMediaValue { get { return "media"; } } } private NameValueCollection SupportsTeamDriveParam() { - return m_useTeamDrive? new NameValueCollection { { GoogleDriveApiStrings.SupportsTeamDriveParam, "true"}} : null; + return m_useTeamDrive? new NameValueCollection { { GoogleDriveApiStrings.SupportsTeamDriveParam, GoogleDriveApiStrings.True }} : null; } private NameValueCollection IncludeTeamDriveParam() { - return m_useTeamDrive ? new NameValueCollection { { GoogleDriveApiStrings.IncludeTeamDriveParam, "true" } } : null; + return m_useTeamDrive ? new NameValueCollection { { GoogleDriveApiStrings.IncludeTeamDriveParam, GoogleDriveApiStrings.True } } : null; } private string PrepareFileQueryUrl(NameValueCollection values) @@ -521,7 +533,7 @@ namespace Duplicati.Library.Backend.GoogleDrive private IEnumerable ListFolder(string parentfolder, bool? onlyFolders = null, string name = null) { - var p = new string[] { + var fileQuery = new string[] { string.IsNullOrEmpty(name) ? null : string.Format("title = '{0}'", EscapeTitleEntries(name)), onlyFolders == null ? null : string.Format("mimeType {0}= '{1}'", onlyFolders.Value ? "" : "!", FOLDER_MIMETYPE), string.Format("'{0}' in parents", EscapeTitleEntries(parentfolder)) @@ -529,7 +541,7 @@ namespace Duplicati.Library.Backend.GoogleDrive var queryParams = new NameValueCollection { - {"q", Library.Utility.Uri.UrlEncode(string.Join(" and ", p.Where(x => x != null)))}, + {GoogleDriveApiStrings.FileQueryParam, Library.Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null)))}, }; queryParams.Add(SupportsTeamDriveParam()); queryParams.Add(IncludeTeamDriveParam()); @@ -545,7 +557,7 @@ namespace Duplicati.Library.Backend.GoogleDrive if (string.IsNullOrWhiteSpace(token)) break; - queryParams.Set(GoogleDriveApiStrings.pageTokenParam, token); + queryParams.Set(GoogleDriveApiStrings.PageTokenParam, token); } } From 007e4c082923a2f64e048a8d527fe2a68468471f Mon Sep 17 00:00:00 2001 From: verhoek Date: Sun, 13 May 2018 12:48:00 +0200 Subject: [PATCH 07/16] Cleaned up imports. --- Duplicati/Library/Utility/Utility.cs | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index bcc3bb611..f37a1961d 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1,4 +1,3 @@ -#region Disclaimer / License // Copyright (C) 2015, The Duplicati Team // http://www.duplicati.com, info@duplicati.com // @@ -15,21 +14,13 @@ // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -// -using System.Text.RegularExpressions; -using System.Linq; -using System.Threading.Tasks; - - -#endregion - using System; -using System.Text.RegularExpressions; -using System.Linq; using System.Collections.Generic; using System.IO; -using System.Text; +using System.Linq; using System.Threading.Tasks; +using System.Text; +using System.Text.RegularExpressions; namespace Duplicati.Library.Utility { From a67248f51f35ce300d66740c5ad40867586311f8 Mon Sep 17 00:00:00 2001 From: verhoek Date: Sun, 13 May 2018 16:22:41 +0200 Subject: [PATCH 08/16] Moved Api strings to separate class similarly to the constant setup in Strings.cs. --- ...cati.Library.Backend.GoogleServices.csproj | 1 + .../Backend/GoogleServices/GoogleDrive.cs | 126 +++++++++--------- .../Library/Backend/GoogleServices/WebApi.cs | 52 ++++++++ 3 files changed, 115 insertions(+), 64 deletions(-) create mode 100644 Duplicati/Library/Backend/GoogleServices/WebApi.cs diff --git a/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj b/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj index 817a57af5..070b4832a 100644 --- a/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj +++ b/Duplicati/Library/Backend/GoogleServices/Duplicati.Library.Backend.GoogleServices.csproj @@ -41,6 +41,7 @@ + diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index ba8baf607..b0cc9d879 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -32,9 +32,9 @@ namespace Duplicati.Library.Backend.GoogleDrive private const string DISABLE_TEAMDRIVE_OPTION = "googledrive-disable-teamdrive"; private const string FOLDER_MIMETYPE = "application/vnd.google-apps.folder"; - private const string DRIVE_API_UPLOAD_URL = "https://www.googleapis.com/upload/drive/v2"; - private const string DRIVE_API_URL = "https://www.googleapis.com/drive/v2"; - + private const string DRIVE_API_UPLOAD_URL = "https://www.googleapis.com/upload/drive/v2"; + // private const string DRIVE_API_URL = "https://www.googleapis.com/drive/v2"; + private readonly string m_path; private bool m_useTeamDrive = true; @@ -70,7 +70,7 @@ namespace Duplicati.Library.Backend.GoogleDrive var curparent = GetAboutInfo().rootFolderId; var curdisplay = "/"; - foreach(var p in path.Split(new char[] {'/'}, StringSplitOptions.RemoveEmptyEntries)) + foreach (var p in path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)) { var res = ListFolder(curparent, true, p).ToArray(); @@ -151,16 +151,19 @@ namespace Duplicati.Library.Backend.GoogleDrive Delete(remotename); } - var isUpdate = !string.IsNullOrWhiteSpace(fileId); - - var values = new NameValueCollection { { GoogleDriveApiStrings.UploadTypeParam, GoogleDriveApiStrings.UploadTypeResumableValue } }; + var isUpdate = !string.IsNullOrWhiteSpace(fileId); + + var values = new NameValueCollection { + { WebApi.GoogleDrive.QueryParam.UploadType, + WebApi.GoogleDrive.QueryValue.Resumable } }; PrepareFileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values); var url = isUpdate ? - PrepareFileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values): + PrepareFileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values) : PrepareFileUploadUrl(values); - var item = new GoogleDriveFolderItem() { + var item = new GoogleDriveFolderItem() + { title = remotename, description = remotename, mimeType = "application/octet-stream", @@ -186,13 +189,13 @@ namespace Duplicati.Library.Backend.GoogleDrive var fileId = GetFileEntries(remotename).OrderByDescending(x => x.createdDate).First().id; - var url = PrepareFileQueryUrl(fileId, new NameValueCollection{ - { GoogleDriveApiStrings.AltParam, GoogleDriveApiStrings.AltMediaValue } + var url = PrepareFileQueryUrl(fileId, new NameValueCollection{ + { WebApi.GoogleDrive.QueryParam.Alt, WebApi.GoogleDrive.QueryValue.Media } }); var req = m_oauth.CreateRequest(url); var areq = new AsyncHttpRequest(req); - using(var resp = (HttpWebResponse)areq.GetResponse()) - using(var rs = areq.GetResponseStream()) + using (var resp = (HttpWebResponse)areq.GetResponse()) + using (var rs = areq.GetResponseStream()) Duplicati.Library.Utility.Utility.CopyStream(rs, stream); } @@ -270,10 +273,11 @@ namespace Duplicati.Library.Backend.GoogleDrive { try { - foreach(var fileid in from n in GetFileEntries(remotename) select n.id) + foreach (var fileid in from n in GetFileEntries(remotename) select n.id) { var url = PrepareFileQueryUrl(Library.Utility.Uri.UrlPathEncode(fileid), SupportsTeamDriveParam()); - m_oauth.GetJSONData(url, x => { + m_oauth.GetJSONData(url, x => + { x.Method = "DELETE"; }); } @@ -317,7 +321,8 @@ namespace Duplicati.Library.Backend.GoogleDrive public System.Collections.Generic.IList SupportedCommands { - get { + get + { return new List(new ICommandLineArgument[] { new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, @@ -359,8 +364,8 @@ namespace Duplicati.Library.Backend.GoogleDrive } public string[] DNSName - { - get { return new string[] { new System.Uri(DRIVE_API_URL).Host, new System.Uri(DRIVE_API_UPLOAD_URL).Host }; } + { + get { return new string[] { new System.Uri(WebApi.GoogleDrive.Url.DRIVE).Host, new System.Uri(WebApi.GoogleDrive.Url.UPLOAD).Host }; } } #endregion @@ -381,12 +386,14 @@ namespace Duplicati.Library.Backend.GoogleDrive var url = PrepareFileQueryUrl(Library.Utility.Uri.UrlPathEncode(files[0].id)); var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(newfile)); - var nf = m_oauth.GetJSONData(url, x => { + var nf = m_oauth.GetJSONData(url, x => + { x.Method = "PUT"; x.ContentLength = data.Length; x.ContentType = "application/json; charset=UTF-8"; - }, x => { - using(var rs = x.GetRequestStream()) + }, x => + { + using (var rs = x.GetRequestStream()) rs.Write(data, 0, data.Length); }); @@ -407,60 +414,48 @@ namespace Duplicati.Library.Backend.GoogleDrive public void Dispose() { - } - + } + #endregion - - private static class GoogleDriveApiStrings - { - public static string FilePath { get { return "files"; } } - public static string FileQueryParam { get { return "q"; } } - public static string AboutPath { get { return "about"; } } - public static string SupportsTeamDriveParam { get { return "supportsTeamDrives"; } } - public static string IncludeTeamDriveParam { get { return "includeTeamDriveItems"; } } - public static string True { get { return "true"; } } - public static string False { get { return "false"; } } - public static string PageTokenParam { get { return "pageToken"; } } - public static string UploadTypeParam { get { return "uploadType"; } } - public static string UploadTypeResumableValue { get { return "resumable"; } } - public static string AltParam { get { return "alt"; } } - public static string AltMediaValue { get { return "media"; } } - } - - + private NameValueCollection SupportsTeamDriveParam() { - return m_useTeamDrive? new NameValueCollection { { GoogleDriveApiStrings.SupportsTeamDriveParam, GoogleDriveApiStrings.True }} : null; + return m_useTeamDrive ? new NameValueCollection { + { WebApi.GoogleDrive.QueryParam.SupportsTeamDrive, + WebApi.GoogleDrive.QueryValue.True } + } : null; } private NameValueCollection IncludeTeamDriveParam() { - return m_useTeamDrive ? new NameValueCollection { { GoogleDriveApiStrings.IncludeTeamDriveParam, GoogleDriveApiStrings.True } } : null; + return m_useTeamDrive ? new NameValueCollection { + { WebApi.GoogleDrive.QueryParam.IncludeTeamDrive, + WebApi.GoogleDrive.QueryValue.True } } : null; } private string PrepareFileQueryUrl(NameValueCollection values) - { - return Library.Utility.Uri.UriBuilder(DRIVE_API_URL, GoogleDriveApiStrings.FilePath, values); + { + return Library.Utility.Uri.UriBuilder(WebApi.GoogleDrive.Url.DRIVE, WebApi.GoogleDrive.Path.File, values); } private string PrepareFileQueryUrl(string fileId, NameValueCollection values = null) - { - var path = GoogleDriveApiStrings.FilePath; - path += "/" + fileId; - - return Library.Utility.Uri.UriBuilder(DRIVE_API_URL, path, values); + { + var path = WebApi.GoogleDrive.Path.File; + path += "/" + fileId; + + return Library.Utility.Uri.UriBuilder(WebApi.GoogleDrive.Url.DRIVE, path, values); } private string PrepareFileUploadUrl(string fileId, NameValueCollection values) - { - var path = GoogleDriveApiStrings.FilePath; + { + var path = WebApi.GoogleDrive.Path.File; path += "/" + fileId; return Library.Utility.Uri.UriBuilder(DRIVE_API_UPLOAD_URL, path, values); } private string PrepareFileUploadUrl(NameValueCollection values) - { - var path = GoogleDriveApiStrings.FilePath; + { + var path = WebApi.GoogleDrive.Path.File; return Library.Utility.Uri.UriBuilder(DRIVE_API_UPLOAD_URL, path, values); } @@ -540,8 +535,9 @@ namespace Duplicati.Library.Backend.GoogleDrive }; var queryParams = new NameValueCollection - { - {GoogleDriveApiStrings.FileQueryParam, Library.Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null)))}, + { + {WebApi.GoogleDrive.QueryParam.File, + Library.Utility.Uri.UrlEncode(string.Join(" and ", fileQuery.Where(x => x != null)))}, }; queryParams.Add(SupportsTeamDriveParam()); queryParams.Add(IncludeTeamDriveParam()); @@ -555,15 +551,15 @@ namespace Duplicati.Library.Backend.GoogleDrive var token = res.nextPageToken; if (string.IsNullOrWhiteSpace(token)) - break; - - queryParams.Set(GoogleDriveApiStrings.PageTokenParam, token); + break; + + queryParams.Set(WebApi.GoogleDrive.QueryParam.PageToken, token); } } private GoogleDriveAboutResponse GetAboutInfo() - { - var url = Library.Utility.Uri.UriBuilder(DRIVE_API_URL, GoogleDriveApiStrings.AboutPath); + { + var url = Library.Utility.Uri.UriBuilder(WebApi.GoogleDrive.Url.DRIVE, WebApi.GoogleDrive.Path.About); return m_oauth.GetJSONData(url); } @@ -571,7 +567,8 @@ namespace Duplicati.Library.Backend.GoogleDrive { var url = PrepareFileQueryUrl(SupportsTeamDriveParam()); - var folder = new GoogleDriveFolderItem() { + var folder = new GoogleDriveFolderItem() + { title = name, description = name, mimeType = FOLDER_MIMETYPE, @@ -587,8 +584,9 @@ namespace Duplicati.Library.Backend.GoogleDrive x.ContentType = "application/json; charset=UTF-8"; x.ContentLength = data.Length; - }, req => { - using(var rs = req.GetRequestStream()) + }, req => + { + using (var rs = req.GetRequestStream()) rs.Write(data, 0, data.Length); }); } diff --git a/Duplicati/Library/Backend/GoogleServices/WebApi.cs b/Duplicati/Library/Backend/GoogleServices/WebApi.cs new file mode 100644 index 000000000..d31a62eea --- /dev/null +++ b/Duplicati/Library/Backend/GoogleServices/WebApi.cs @@ -0,0 +1,52 @@ +// Copyright (C) 2018, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +namespace Duplicati.Library.Backend.WebApi +{ + internal static class GoogleDrive + { + internal static class Url + { + public const string DRIVE = "https://www.googleapis.com/drive/v2"; + public const string UPLOAD = "https://www.googleapis.com/upload/drive/v2"; + } + + internal static class Path + { + public static string File => "files"; + public static string About { get { return "about"; } } + } + + internal static class QueryParam + { + public static string File => "q"; + public static string SupportsTeamDrive { get { return "supportsTeamDrives"; } } + public static string IncludeTeamDrive { get { return "includeTeamDriveItems"; } } + public static string PageToken { get { return "pageToken"; } } + public static string UploadType { get { return "uploadType"; } } + public static string Alt { get { return "alt"; } } + } + + internal static class QueryValue + { + public static string True { get { return "true"; } } + public static string False { get { return "false"; } } + public static string Resumable { get { return "resumable"; } } + public static string Media { get { return "media"; } } + } + + } +} \ No newline at end of file From dae12fee53f20ea125c1b0a6160773d9c3ec5783 Mon Sep 17 00:00:00 2001 From: verhoek Date: Sun, 13 May 2018 16:27:14 +0200 Subject: [PATCH 09/16] Organized imports. --- .../Library/Backend/GoogleServices/GoogleDrive.cs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index b0cc9d879..6d20a033b 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -15,14 +15,16 @@ // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; +using System.Collections.Generic; +using System.Collections.Specialized; using System.Linq; +using System.Net; + +using Newtonsoft.Json; + +using Duplicati.Library.Backend.GoogleServices; using Duplicati.Library.Interface; using Duplicati.Library.Utility; -using System.Collections.Generic; -using System.Net; -using Newtonsoft.Json; -using Duplicati.Library.Backend.GoogleServices; -using System.Collections.Specialized; namespace Duplicati.Library.Backend.GoogleDrive { From 6dd64a553d8df71b81a38903af2def33539b89b8 Mon Sep 17 00:00:00 2001 From: verhoek Date: Sun, 13 May 2018 17:19:09 +0200 Subject: [PATCH 10/16] Fixed code quality issues. --- .../GoogleServices/GoogleCloudStorage.cs | 33 +-- .../Backend/GoogleServices/GoogleDrive.cs | 105 ++++----- .../Library/Backend/GoogleServices/WebApi.cs | 80 ++++--- Duplicati/Library/Utility/Uri.cs | 199 ++++++++++-------- 4 files changed, 217 insertions(+), 200 deletions(-) diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs index b426eab5b..b0d336d03 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs @@ -18,7 +18,6 @@ using System; using Duplicati.Library.Interface; using System.Collections.Generic; using System.Net; -using System.Web; using Duplicati.Library.Utility; using Newtonsoft.Json; using System.Text; @@ -97,9 +96,9 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage m_oauth = new OAuthHelper(authid, this.ProtocolKey); m_oauth.AutoAuthHeader = true; - } - - + } + + private class ListBucketResponse { public string kind { get; set; } @@ -205,7 +204,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage if (string.IsNullOrEmpty(m_project)) throw new UserInformationException(Strings.GoogleCloudStorage.ProjectIDMissingError(PROJECT_OPTION), "GoogleCloudStorageMissingProjectID"); - var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest() { + var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest + { name = m_bucket, location = m_location, storageClass = m_storage_class @@ -220,7 +220,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage var areq = new AsyncHttpRequest(req); - using(var rs = areq.GetRequestStream()) + using (var rs = areq.GetRequestStream()) rs.Write(data, 0, data.Length); m_oauth.ReadJSONResponse(areq); @@ -246,8 +246,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage foreach (KeyValuePair s in KNOWN_GCS_LOCATIONS) locations.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); foreach (KeyValuePair s in KNOWN_GCS_STORAGE_CLASSES) - storageClasses.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); - + storageClasses.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); + return new List(new ICommandLineArgument[] { new CommandLineArgument(LOCATION_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.LocationDescriptionShort, Strings.GoogleCloudStorage.LocationDescriptionLong(locations.ToString())), new CommandLineArgument(STORAGECLASS_OPTION, CommandLineArgument.ArgumentType.String, Strings.GoogleCloudStorage.StorageclassDescriptionShort, Strings.GoogleCloudStorage.StorageclassDescriptionLong(locations.ToString())), @@ -276,8 +276,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage var res = GoogleCommon.ChunckedUploadWithResume(m_oauth, item, url, stream); if (res == null) - throw new Exception(string.Format("Upload succeeded, but no data was returned")); - + throw new Exception("Upload succeeded, but no data was returned"); + } public void Get(string remotename, System.IO.Stream stream) @@ -288,8 +288,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage var req = m_oauth.CreateRequest(url); var areq = new AsyncHttpRequest(req); - using(var resp = areq.GetResponse()) - using(var rs = areq.GetResponseStream()) + using (var resp = areq.GetResponse()) + using (var rs = areq.GetResponseStream()) Library.Utility.Utility.CopyStream(rs, stream); } catch (WebException wex) @@ -304,7 +304,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage public void Rename(string oldname, string newname) { - var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem() { + var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem + { name = m_prefix + newname, })); @@ -315,7 +316,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage req.ContentType = "application/json; charset=UTF-8"; var areq = new AsyncHttpRequest(req); - using(var rs = areq.GetRequestStream()) + using (var rs = areq.GetRequestStream()) rs.Write(data, 0, data.Length); m_oauth.ReadJSONResponse(req); @@ -323,8 +324,8 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage #region IDisposable implementation public void Dispose() - { - + { + } #endregion } diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index 6d20a033b..a5b240da0 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -25,7 +25,8 @@ using Newtonsoft.Json; using Duplicati.Library.Backend.GoogleServices; using Duplicati.Library.Interface; using Duplicati.Library.Utility; - +using System.Text; + namespace Duplicati.Library.Backend.GoogleDrive { public class GoogleDrive : IBackend, IStreamingBackend, IQuotaEnabledBackend, IRenameEnabledBackend @@ -34,11 +35,9 @@ namespace Duplicati.Library.Backend.GoogleDrive private const string DISABLE_TEAMDRIVE_OPTION = "googledrive-disable-teamdrive"; private const string FOLDER_MIMETYPE = "application/vnd.google-apps.folder"; - private const string DRIVE_API_UPLOAD_URL = "https://www.googleapis.com/upload/drive/v2"; - // private const string DRIVE_API_URL = "https://www.googleapis.com/drive/v2"; - private readonly string m_path; - private bool m_useTeamDrive = true; + private readonly string m_path; + private readonly bool m_useTeamDrive = true; private readonly OAuthHelper m_oauth; private string m_currentFolderId; @@ -69,26 +68,30 @@ namespace Duplicati.Library.Backend.GoogleDrive private string GetFolderId(string path, bool autocreate = false) { - var curparent = GetAboutInfo().rootFolderId; - var curdisplay = "/"; + var curparent = GetAboutInfo().rootFolderId; + var curdisplay = new StringBuilder("/"); foreach (var p in path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries)) { - var res = ListFolder(curparent, true, p).ToArray(); - - if (res.Length == 0) - { - if (!autocreate) - throw new FolderMissingException(); - - curparent = CreateFolder(p, curparent).id; - } - else if (res.Length > 1) - throw new UserInformationException(Strings.GoogleDrive.MultipleEntries(p, curdisplay), "GoogleDriveMultipleEntries"); - else - curparent = res[0].id; - - curdisplay += p + "/"; + var res = ListFolder(curparent, true, p).ToArray(); + + if (res.Length == 0) + { + if (!autocreate) + throw new FolderMissingException(); + + curparent = CreateFolder(p, curparent).id; + } + else if (res.Length > 1) + { + throw new UserInformationException(Strings.GoogleDrive.MultipleEntries(p, curdisplay.ToString()), "GoogleDriveMultipleEntries"); + } + else + { + curparent = res[0].id; + } + + curdisplay.Append(p).Append("/"); } return curparent; @@ -158,19 +161,19 @@ namespace Duplicati.Library.Backend.GoogleDrive var values = new NameValueCollection { { WebApi.GoogleDrive.QueryParam.UploadType, WebApi.GoogleDrive.QueryValue.Resumable } }; - PrepareFileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values); + WebApi.GoogleDrive.FileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values); var url = isUpdate ? - PrepareFileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values) : - PrepareFileUploadUrl(values); + WebApi.GoogleDrive.FileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values) : + WebApi.GoogleDrive.FileUploadUrl(values); - var item = new GoogleDriveFolderItem() + var item = new GoogleDriveFolderItem { title = remotename, description = remotename, mimeType = "application/octet-stream", labels = new GoogleDriveFolderItemLabels { hidden = true }, - parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference() { id = CurrentFolderId } } + parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = CurrentFolderId } } }; var res = GoogleCommon.ChunckedUploadWithResume(m_oauth, item, url, stream, isUpdate ? "PUT" : "POST"); @@ -191,7 +194,7 @@ namespace Duplicati.Library.Backend.GoogleDrive var fileId = GetFileEntries(remotename).OrderByDescending(x => x.createdDate).First().id; - var url = PrepareFileQueryUrl(fileId, new NameValueCollection{ + var url = WebApi.GoogleDrive.FileQueryUrl(fileId, new NameValueCollection{ { WebApi.GoogleDrive.QueryParam.Alt, WebApi.GoogleDrive.QueryValue.Media } }); var req = m_oauth.CreateRequest(url); @@ -277,7 +280,7 @@ namespace Duplicati.Library.Backend.GoogleDrive { foreach (var fileid in from n in GetFileEntries(remotename) select n.id) { - var url = PrepareFileQueryUrl(Library.Utility.Uri.UrlPathEncode(fileid), SupportsTeamDriveParam()); + var url = WebApi.GoogleDrive.FileQueryUrl(Library.Utility.Uri.UrlPathEncode(fileid), SupportsTeamDriveParam()); m_oauth.GetJSONData(url, x => { x.Method = "DELETE"; @@ -321,11 +324,7 @@ namespace Duplicati.Library.Backend.GoogleDrive } } - public System.Collections.Generic.IList SupportedCommands - { - get - { - return new List(new ICommandLineArgument[] { + public System.Collections.Generic.IList SupportedCommands => new List(new ICommandLineArgument[] { new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.GoogleDrive.AuthidShort, @@ -335,8 +334,6 @@ namespace Duplicati.Library.Backend.GoogleDrive Strings.GoogleDrive.DisableTeamDriveShort, Strings.GoogleDrive.DisableTeamDriveLong), }); - } - } public string Description { @@ -383,9 +380,9 @@ namespace Duplicati.Library.Backend.GoogleDrive var newfile = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(files[0])); newfile.title = newname; - newfile.parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference() { id = CurrentFolderId } }; + newfile.parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = CurrentFolderId } }; - var url = PrepareFileQueryUrl(Library.Utility.Uri.UrlPathEncode(files[0].id)); + var url = WebApi.GoogleDrive.FileQueryUrl(Library.Utility.Uri.UrlPathEncode(files[0].id)); var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(newfile)); var nf = m_oauth.GetJSONData(url, x => @@ -435,32 +432,6 @@ namespace Duplicati.Library.Backend.GoogleDrive WebApi.GoogleDrive.QueryValue.True } } : null; } - private string PrepareFileQueryUrl(NameValueCollection values) - { - return Library.Utility.Uri.UriBuilder(WebApi.GoogleDrive.Url.DRIVE, WebApi.GoogleDrive.Path.File, values); - } - - private string PrepareFileQueryUrl(string fileId, NameValueCollection values = null) - { - var path = WebApi.GoogleDrive.Path.File; - path += "/" + fileId; - - return Library.Utility.Uri.UriBuilder(WebApi.GoogleDrive.Url.DRIVE, path, values); - } - - private string PrepareFileUploadUrl(string fileId, NameValueCollection values) - { - var path = WebApi.GoogleDrive.Path.File; - path += "/" + fileId; - return Library.Utility.Uri.UriBuilder(DRIVE_API_UPLOAD_URL, path, values); - } - - private string PrepareFileUploadUrl(NameValueCollection values) - { - var path = WebApi.GoogleDrive.Path.File; - return Library.Utility.Uri.UriBuilder(DRIVE_API_UPLOAD_URL, path, values); - } - private class GoogleDriveParentReference { public string kind { get; set; } @@ -546,7 +517,7 @@ namespace Duplicati.Library.Backend.GoogleDrive while (true) { - var url = PrepareFileQueryUrl(queryParams); + var url = WebApi.GoogleDrive.FileQueryUrl(queryParams); var res = m_oauth.GetJSONData(url); foreach (var n in res.items) yield return n; @@ -567,7 +538,7 @@ namespace Duplicati.Library.Backend.GoogleDrive private GoogleDriveFolderItem CreateFolder(string name, string parent) { - var url = PrepareFileQueryUrl(SupportsTeamDriveParam()); + var url = WebApi.GoogleDrive.FileQueryUrl(SupportsTeamDriveParam()); var folder = new GoogleDriveFolderItem() { @@ -575,7 +546,7 @@ namespace Duplicati.Library.Backend.GoogleDrive description = name, mimeType = FOLDER_MIMETYPE, labels = new GoogleDriveFolderItemLabels { hidden = true }, - parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference() { id = parent } } + parents = new GoogleDriveParentReference[] { new GoogleDriveParentReference { id = parent } } }; var data = System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder)); diff --git a/Duplicati/Library/Backend/GoogleServices/WebApi.cs b/Duplicati/Library/Backend/GoogleServices/WebApi.cs index d31a62eea..3d5ee8d5c 100644 --- a/Duplicati/Library/Backend/GoogleServices/WebApi.cs +++ b/Duplicati/Library/Backend/GoogleServices/WebApi.cs @@ -14,39 +14,59 @@ // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System.Collections.Specialized; + namespace Duplicati.Library.Backend.WebApi { - internal static class GoogleDrive - { - internal static class Url - { - public const string DRIVE = "https://www.googleapis.com/drive/v2"; - public const string UPLOAD = "https://www.googleapis.com/upload/drive/v2"; - } + static class GoogleDrive + { + public static class Url + { + public const string DRIVE = "https://www.googleapis.com/drive/v2"; + public const string UPLOAD = "https://www.googleapis.com/upload/drive/v2"; + } - internal static class Path - { - public static string File => "files"; - public static string About { get { return "about"; } } - } + public static class Path + { + public const string File = "files"; + public const string About = "about"; + } - internal static class QueryParam - { - public static string File => "q"; - public static string SupportsTeamDrive { get { return "supportsTeamDrives"; } } - public static string IncludeTeamDrive { get { return "includeTeamDriveItems"; } } - public static string PageToken { get { return "pageToken"; } } - public static string UploadType { get { return "uploadType"; } } - public static string Alt { get { return "alt"; } } - } + public static class QueryParam + { + public const string File = "q"; + public const string SupportsTeamDrive = "supportsTeamDrives"; + public const string IncludeTeamDrive = "includeTeamDriveItems"; + public const string PageToken = "pageToken"; + public const string UploadType = "uploadType"; + public const string Alt = "alt"; + } + + public static class QueryValue + { + public const string True = "true"; + public const string Resumable = "resumable"; + public const string Media = "media"; + } - internal static class QueryValue - { - public static string True { get { return "true"; } } - public static string False { get { return "false"; } } - public static string Resumable { get { return "resumable"; } } - public static string Media { get { return "media"; } } - } - - } + public static string FileQueryUrl(NameValueCollection values) + { + return Library.Utility.Uri.UriBuilder(Url.DRIVE, Path.File, values); + } + + public static string FileQueryUrl(string fileId, NameValueCollection values = null) + { + return Library.Utility.Uri.UriBuilder(Url.DRIVE, Path.File + '/' + fileId + fileId, values); + } + + public static string FileUploadUrl(string fileId, NameValueCollection values) + { + return Library.Utility.Uri.UriBuilder(Url.UPLOAD, Path.File + '/' + fileId + fileId, values); + } + + public static string FileUploadUrl(NameValueCollection values) + { + return Library.Utility.Uri.UriBuilder(Url.UPLOAD, Path.File, values); + } + } } \ No newline at end of file diff --git a/Duplicati/Library/Utility/Uri.cs b/Duplicati/Library/Utility/Uri.cs index 150552dcb..649acf825 100644 --- a/Duplicati/Library/Utility/Uri.cs +++ b/Duplicati/Library/Utility/Uri.cs @@ -64,18 +64,18 @@ namespace Duplicati.Library.Utility /// /// The password, if any /// - public readonly string Password; - + public readonly string Password; + /// /// The original URI. /// - public readonly string OriginalUri; - + public readonly string OriginalUri; + /// /// Cache for the query parameters. /// - private NameValueCollection m_queryParams; - + private NameValueCollection m_queryParams; + /// /// Gets the paramters in the query string /// @@ -90,12 +90,12 @@ namespace Duplicati.Library.Utility m_queryParams = new NameValueCollection(); else m_queryParams = ParseQueryString(Query); - } - + } + return m_queryParams; } - } - + } + /// /// Gets the host and path. /// @@ -111,8 +111,8 @@ namespace Duplicati.Library.Utility else return Host + (Path == null ? "" : "/" + Path); } - } - + } + /// /// Gets the path and query. /// @@ -121,10 +121,10 @@ namespace Duplicati.Library.Utility { get { - return (Path ?? "") + (Query == null ? "" : "?" + Query); + return (Path ?? "") + (Query == null ? "" : "?" + Query); } - } - + } + /// /// Initializes a new instance of the struct. /// @@ -132,8 +132,8 @@ namespace Duplicati.Library.Utility public Uri(string url) { if (string.IsNullOrEmpty(url)) - throw new ArgumentNullException(nameof(url)); - + throw new ArgumentNullException(nameof(url)); + m_queryParams = null; this.OriginalUri = url; @@ -161,8 +161,8 @@ namespace Duplicati.Library.Utility { } throw new ArgumentException(Strings.Uri.UriParseError(url), nameof(url)); - } - + } + this.Scheme = m.Groups["scheme"].Value; var h = m.Groups["hostname"].Success ? m.Groups["hostname"].Value : ""; @@ -187,8 +187,8 @@ namespace Duplicati.Library.Utility this.Port = int.Parse(m.Groups["port"].Value); else this.Port = -1; - } - + } + /// /// Constructs a free-form URI from components /// @@ -210,13 +210,13 @@ namespace Duplicati.Library.Utility Password = password; Port = port; OriginalUri = AsString(scheme, host, path, query, username, password, port); - } - + } + /// /// Returns a that represents the current . /// /// A that represents the current . - public override string ToString () + public override string ToString() { return AsString(Scheme, Host, Path, Query, Username, Password, Port); } @@ -228,8 +228,8 @@ namespace Duplicati.Library.Utility { if (string.IsNullOrEmpty(Host)) throw new ArgumentException(Strings.Uri.NoHostname(OriginalUri)); - } - + } + /// /// Constructs an url-like string from components. /// @@ -245,21 +245,21 @@ namespace Duplicati.Library.Utility { var s = scheme + "://"; if (!string.IsNullOrEmpty(username) || !string.IsNullOrEmpty(password)) - { - + { + s += UrlEncode(username ?? ""); s += ":"; s += UrlEncode(password ?? ""); s += "@"; - } - + } + if (!string.IsNullOrEmpty(host)) { s += host; if (port != -1) s += ":" + port.ToString(); - } - + } + if (!string.IsNullOrEmpty(path)) { if (!string.IsNullOrEmpty(host) && !path.StartsWith("/", StringComparison.Ordinal)) @@ -269,9 +269,9 @@ namespace Duplicati.Library.Utility if (!string.IsNullOrEmpty(query)) s += "?" + query; - return s; - } - + return s; + } + /// /// Creates a new instance with another scheme /// @@ -280,8 +280,8 @@ namespace Duplicati.Library.Utility public Uri SetScheme(string scheme) { return new Uri(scheme, Host, Path, Query, Username, Password, Port); - } - + } + /// /// Creates a new instance with another host /// @@ -290,8 +290,8 @@ namespace Duplicati.Library.Utility public Uri SetHost(string host) { return new Uri(Scheme, host, Path, Query, Username, Password, Port); - } - + } + /// /// Creates a new instance with another path /// @@ -300,8 +300,8 @@ namespace Duplicati.Library.Utility public Uri SetPath(string path) { return new Uri(Scheme, Host, path, Query, Username, Password, Port); - } - + } + /// /// Creates a new instance with another query /// @@ -321,8 +321,8 @@ namespace Duplicati.Library.Utility public Uri SetCredentials(string username, string password) { return new Uri(Scheme, Host, Path, Query, username, password, Port); - } - + } + /// /// Creates a new instance with another port /// @@ -331,13 +331,13 @@ namespace Duplicati.Library.Utility public Uri SetPort(int port) { return new Uri(Scheme, Host, Path, Query, Username, Password, port); - } - + } + /// /// The regular expression that matches %20 type values in a querystring /// - private static System.Text.RegularExpressions.Regex RE_ESCAPECHAR = new System.Text.RegularExpressions.Regex(@"[^0-9a-zA-Z\-_]", System.Text.RegularExpressions.RegexOptions.Compiled); - + private static System.Text.RegularExpressions.Regex RE_ESCAPECHAR = new System.Text.RegularExpressions.Regex(@"[^0-9a-zA-Z\-_]", System.Text.RegularExpressions.RegexOptions.Compiled); + /// /// Encodes a URL, like System.Web.HttpUtility.UrlEncode /// @@ -347,44 +347,45 @@ namespace Duplicati.Library.Utility public static string UrlPathEncode(string value, System.Text.Encoding encoding = null) { return UrlEncode(value, encoding, "%20"); - } - + } + /// /// Encodes a URL, like System.Web.HttpUtility.UrlEncode /// /// The encoded URL /// The URL fragment to encode /// The encoding to use - public static string UrlEncode(string value, System.Text.Encoding encoding = null, string spacevalue = "+") + public static string UrlEncode(string value, System.Text.Encoding encoding = null, string spacevalue = "+") { if (value == null) - throw new ArgumentNullException(nameof(value)); - + throw new ArgumentNullException(nameof(value)); + encoding = encoding ?? System.Text.Encoding.UTF8; var encoder = encoding.GetEncoder(); - var inbuf = new char[1]; + var inbuf = new char[1]; var outbuf = new byte[4]; - return RE_ESCAPECHAR.Replace(value, (m) => { + return RE_ESCAPECHAR.Replace(value, (m) => + { if (m.Value == " ") - return spacevalue; - - inbuf[0] = m.Value[0]; - - try + return spacevalue; + + inbuf[0] = m.Value[0]; + + try { var len = encoder.GetBytes(inbuf, 0, 1, outbuf, 0, true); return "%" + BitConverter.ToString(outbuf, 0, len).Replace("-", "%"); } catch { - } - - //Fallback + } + + //Fallback return m.Value; - }); - + }); + } /// @@ -401,15 +402,16 @@ namespace Duplicati.Library.Utility public static string UrlDecode(string value, System.Text.Encoding encoding = null) { if (value == null) - throw new ArgumentNullException(nameof(value)); - - encoding = encoding ?? System.Text.Encoding.UTF8; - + throw new ArgumentNullException(nameof(value)); + + encoding = encoding ?? System.Text.Encoding.UTF8; + var decoder = encoding.GetDecoder(); var inbuf = new byte[8]; var outbuf = new char[8]; - return RE_NUMBER.Replace(value, (m) => { + return RE_NUMBER.Replace(value, (m) => + { if (m.Value == "+") return " "; @@ -418,24 +420,24 @@ namespace Duplicati.Library.Utility var hex = m.Groups["number"].Value; var bytelen = hex.Length / 2; Utility.HexStringAsByteArray(hex, inbuf); - var c = decoder.GetChars(inbuf, 0, bytelen, outbuf, 0); + var c = decoder.GetChars(inbuf, 0, bytelen, outbuf, 0); return new string(outbuf, 0, c); } catch { - } - - //Fallback + } + + //Fallback return m.Value; - }); - - } - + }); + + } + /// /// The regular expression that matches a=b type values in a querystring /// - private static System.Text.RegularExpressions.Regex RE_URLPARAM = new System.Text.RegularExpressions.Regex(@"(?[^\=\&]+)(\=(?[^\&]*))?", System.Text.RegularExpressions.RegexOptions.Compiled); - + private static System.Text.RegularExpressions.Regex RE_URLPARAM = new System.Text.RegularExpressions.Regex(@"(?[^\=\&]+)(\=(?[^\&]*))?", System.Text.RegularExpressions.RegexOptions.Compiled); + /// /// Parses the query string. /// This is a duplicate of the System.Web.HttpUtility.ParseQueryString that does not work well on Mono @@ -449,13 +451,13 @@ namespace Duplicati.Library.Utility if (query.StartsWith("?", StringComparison.Ordinal)) query = query.Substring(1); if (string.IsNullOrEmpty(query)) - return new NameValueCollection(StringComparer.OrdinalIgnoreCase); - + return new NameValueCollection(StringComparer.OrdinalIgnoreCase); + var result = new NameValueCollection(StringComparer.OrdinalIgnoreCase); - foreach(System.Text.RegularExpressions.Match m in RE_URLPARAM.Matches(query)) - result.Add(UrlDecode(m.Groups["key"].Value), UrlDecode(m.Groups["value"].Success ? m.Groups["value"].Value : "")); - - return result; + foreach (System.Text.RegularExpressions.Match m in RE_URLPARAM.Matches(query)) + result.Add(UrlDecode(m.Groups["key"].Value), UrlDecode(m.Groups["value"].Success ? m.Groups["value"].Value : "")); + + return result; } /// @@ -464,7 +466,8 @@ namespace Duplicati.Library.Utility /// The generated querystring /// A collection of name value pairs to be translated into a query string /// The delimiter to separate key value pairs in the query string - public static string BuildUriQuery(NameValueCollection query, string delimiter = "&") { + public static string BuildUriQuery(NameValueCollection query, string delimiter) + { if (query == null) throw new ArgumentNullException(nameof(query)); @@ -481,6 +484,17 @@ namespace Duplicati.Library.Utility return builder.ToString(); } + /// + /// Build the querystring to be used in a URL + /// + /// The generated querystring + /// A collection of name value pairs to be translated into a query string that is + /// ampsersand delimited. + public static string BuildUriQuery(NameValueCollection query) + { + return BuildUriQuery(query, "&"); + } + /// /// Builds a URL together using a base URL, a path and a query. /// @@ -488,7 +502,7 @@ namespace Duplicati.Library.Utility /// Base URL, containing schema, host, port. /// Base path. /// A collection of name value pairs to be translated into a query string. - public static string UriBuilder(string baseUrl, string basePath, NameValueCollection query = null) + public static string UriBuilder(string baseUrl, string basePath, NameValueCollection query) { var builder = new UriBuilder(baseUrl) { @@ -497,6 +511,17 @@ namespace Duplicati.Library.Utility }; return builder.Uri.AbsoluteUri; } + + /// + /// Builds a URL together using a base URL and path. + /// + /// The built together URL. + /// Base URL, containing schema, host, port. + /// Base path. + public static string UriBuilder(string baseUrl, string basePath) + { + return UriBuilder(baseUrl, basePath, null); + } } } From bbf489c86fe7e53178664b7eab9349e5d4abb16f Mon Sep 17 00:00:00 2001 From: verhoek Date: Mon, 14 May 2018 22:45:55 +0200 Subject: [PATCH 11/16] Corrected appending paths to urls. --- Duplicati/Library/Utility/Uri.cs | 45 +++++++++++++++++++++------ Duplicati/UnitTest/UriUtilityTests.cs | 25 +++++++++++++-- 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/Duplicati/Library/Utility/Uri.cs b/Duplicati/Library/Utility/Uri.cs index 649acf825..440ca14a0 100644 --- a/Duplicati/Library/Utility/Uri.cs +++ b/Duplicati/Library/Utility/Uri.cs @@ -499,28 +499,55 @@ namespace Duplicati.Library.Utility /// Builds a URL together using a base URL, a path and a query. /// /// The built together URL. - /// Base URL, containing schema, host, port. - /// Base path. + /// Base URL, containing schema, host, port. + /// Base path. /// A collection of name value pairs to be translated into a query string. - public static string UriBuilder(string baseUrl, string basePath, NameValueCollection query) + public static string UriBuilder(string url, string path, NameValueCollection query) { - var builder = new UriBuilder(baseUrl) + var builder = new UriBuilder(url) { - Path = basePath, + Path = ConcatPaths(ExtractPath(url), path), Query = query != null ? BuildUriQuery(query) : null }; return builder.Uri.AbsoluteUri; } + /// + /// Concats paths of URIs. + /// + /// The concatenated paths. + /// Path1. + /// Path2. + public static string ConcatPaths(string path1, string path2) + { + if (string.IsNullOrEmpty(path2)) + { + return path1; + } + + return path1.TrimEnd('/') + '/' + path2; + } + + /// + /// Grab path part of a URI. + /// At the moment, simple implementation does not remove fragments. + /// + /// The path. + /// URL. + public static string ExtractPath(string url) + { + return (new Uri(url)).Path; + } + /// /// Builds a URL together using a base URL and path. /// /// The built together URL. - /// Base URL, containing schema, host, port. - /// Base path. - public static string UriBuilder(string baseUrl, string basePath) + /// Base URL, containing schema, host, port. + /// Base path. + public static string UriBuilder(string url, string path) { - return UriBuilder(baseUrl, basePath, null); + return UriBuilder(url, path, null); } } } diff --git a/Duplicati/UnitTest/UriUtilityTests.cs b/Duplicati/UnitTest/UriUtilityTests.cs index 5c4c30e65..c5d5e527d 100644 --- a/Duplicati/UnitTest/UriUtilityTests.cs +++ b/Duplicati/UnitTest/UriUtilityTests.cs @@ -24,7 +24,7 @@ namespace Duplicati.UnitTest { [Test] [Category("UrlUtility")] - public void TestBuildUriQuery() + public static void TestBuildUriQuery() { var query = new NameValueCollection { { "a", "b" } }; var queryUrl = Library.Utility.Uri.BuildUriQuery(query); @@ -36,7 +36,7 @@ namespace Duplicati.UnitTest [Test] [Category("UrlUtility")] - public void TestUrlBuilder() + public static void TestUrlBuilder() { var baseUrl = "http://localhost"; var path = "files"; @@ -44,5 +44,26 @@ namespace Duplicati.UnitTest var url = Library.Utility.Uri.UriBuilder(baseUrl, path, query); Assert.AreEqual(baseUrl + "/" + path + "?a=b&c=d", url); } + + [Test] + [Category("UrlUtility")] + public static void TestExtractPath() + { + var url = "http://localhost/a/b"; + var path = Library.Utility.Uri.ExtractPath(url); + Assert.AreEqual("a/b", path); + } + + + [Test] + [Category("UrlUtility")] + public static void TestConcatPaths() + { + var path1 = "/a"; + var path2 = "b/"; + Assert.AreEqual("/a/b/", Library.Utility.Uri.ConcatPaths(path1, path2)); + Assert.AreEqual("/a", Library.Utility.Uri.ConcatPaths(path1, null)); + Assert.AreEqual("/b/", Library.Utility.Uri.ConcatPaths(string.Empty, path2)); + } } } From 1cde60c640fb4c590e79ed258bd69826abc4a625 Mon Sep 17 00:00:00 2001 From: verhoek Date: Tue, 15 May 2018 20:28:52 +0200 Subject: [PATCH 12/16] Removed misplaced leftover line. --- Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index a5b240da0..fef6f15f9 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -161,7 +161,6 @@ namespace Duplicati.Library.Backend.GoogleDrive var values = new NameValueCollection { { WebApi.GoogleDrive.QueryParam.UploadType, WebApi.GoogleDrive.QueryValue.Resumable } }; - WebApi.GoogleDrive.FileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values); var url = isUpdate ? WebApi.GoogleDrive.FileUploadUrl(Library.Utility.Uri.UrlPathEncode(fileId), values) : From 0cca2797dfd24cce9cfb1682bf2ce8f369ce4bc7 Mon Sep 17 00:00:00 2001 From: verhoek Date: Tue, 15 May 2018 20:29:08 +0200 Subject: [PATCH 13/16] Use concat paths in helper functions of googledrive. --- Duplicati/Library/Backend/GoogleServices/WebApi.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/GoogleServices/WebApi.cs b/Duplicati/Library/Backend/GoogleServices/WebApi.cs index 3d5ee8d5c..0c53eeb63 100644 --- a/Duplicati/Library/Backend/GoogleServices/WebApi.cs +++ b/Duplicati/Library/Backend/GoogleServices/WebApi.cs @@ -56,12 +56,12 @@ namespace Duplicati.Library.Backend.WebApi public static string FileQueryUrl(string fileId, NameValueCollection values = null) { - return Library.Utility.Uri.UriBuilder(Url.DRIVE, Path.File + '/' + fileId + fileId, values); + return Library.Utility.Uri.UriBuilder(Url.DRIVE, Library.Utility.Uri.ConcatPaths(Path.File, fileId), values); } public static string FileUploadUrl(string fileId, NameValueCollection values) { - return Library.Utility.Uri.UriBuilder(Url.UPLOAD, Path.File + '/' + fileId + fileId, values); + return Library.Utility.Uri.UriBuilder(Url.UPLOAD, Library.Utility.Uri.ConcatPaths(Path.File, fileId), values); } public static string FileUploadUrl(NameValueCollection values) From be79384966e7338cf70bf0012d36c7d215f7c107 Mon Sep 17 00:00:00 2001 From: verhoek Date: Tue, 15 May 2018 21:50:14 +0200 Subject: [PATCH 14/16] Removed duplicate ? when building urls with query parameters. --- Duplicati/Library/Utility/Uri.cs | 2 +- Duplicati/UnitTest/UriUtilityTests.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Utility/Uri.cs b/Duplicati/Library/Utility/Uri.cs index 440ca14a0..cbdaa7a1c 100644 --- a/Duplicati/Library/Utility/Uri.cs +++ b/Duplicati/Library/Utility/Uri.cs @@ -475,7 +475,7 @@ namespace Duplicati.Library.Utility StringBuilder builder = new StringBuilder(); foreach (var key in query.Cast().Where(key => !string.IsNullOrEmpty(query[key]))) { - builder.Append(builder.Length == 0 ? "?" : delimiter) + builder.Append(builder.Length == 0 ? string.Empty : delimiter) .Append(key) .Append("=") .Append(query[key]); diff --git a/Duplicati/UnitTest/UriUtilityTests.cs b/Duplicati/UnitTest/UriUtilityTests.cs index c5d5e527d..3a0aff8d5 100644 --- a/Duplicati/UnitTest/UriUtilityTests.cs +++ b/Duplicati/UnitTest/UriUtilityTests.cs @@ -28,10 +28,10 @@ namespace Duplicati.UnitTest { var query = new NameValueCollection { { "a", "b" } }; var queryUrl = Library.Utility.Uri.BuildUriQuery(query); - Assert.AreEqual("?a=b", queryUrl); + Assert.AreEqual("a=b", queryUrl); query.Add(new NameValueCollection { { "c", "d" } }); queryUrl = Library.Utility.Uri.BuildUriQuery(query); - Assert.AreEqual("?a=b&c=d", queryUrl); + Assert.AreEqual("a=b&c=d", queryUrl); } [Test] From 7a8a040e4b470e6c5a07fdbb05373b88486afca3 Mon Sep 17 00:00:00 2001 From: verhoek Date: Wed, 16 May 2018 08:11:17 +0200 Subject: [PATCH 15/16] Renamed unit test category UrlUtility to UriUtility for consistency. Added to travis.yml. --- .travis.yml | 1 + Duplicati/UnitTest/UriUtilityTests.cs | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index bb59178ae..456bc9352 100644 --- a/.travis.yml +++ b/.travis.yml @@ -55,6 +55,7 @@ jobs: - env: CATEGORY=Purge - env: CATEGORY=Serialization - env: CATEGORY=Utility + - env: CATEGORY=UriUtility - env: CATEGORY=GUI addons: diff --git a/Duplicati/UnitTest/UriUtilityTests.cs b/Duplicati/UnitTest/UriUtilityTests.cs index 3a0aff8d5..84d43c8a3 100644 --- a/Duplicati/UnitTest/UriUtilityTests.cs +++ b/Duplicati/UnitTest/UriUtilityTests.cs @@ -23,7 +23,7 @@ namespace Duplicati.UnitTest public class UriUtilityTests { [Test] - [Category("UrlUtility")] + [Category("UriUtility")] public static void TestBuildUriQuery() { var query = new NameValueCollection { { "a", "b" } }; @@ -35,7 +35,7 @@ namespace Duplicati.UnitTest } [Test] - [Category("UrlUtility")] + [Category("UriUtility")] public static void TestUrlBuilder() { var baseUrl = "http://localhost"; @@ -46,7 +46,7 @@ namespace Duplicati.UnitTest } [Test] - [Category("UrlUtility")] + [Category("UriUtility")] public static void TestExtractPath() { var url = "http://localhost/a/b"; @@ -56,7 +56,7 @@ namespace Duplicati.UnitTest [Test] - [Category("UrlUtility")] + [Category("UriUtility")] public static void TestConcatPaths() { var path1 = "/a"; From 54ae844536dd63f6a153f4f143e5fb8fde36a88e Mon Sep 17 00:00:00 2001 From: Tom Whitwell Date: Thu, 17 May 2018 12:03:30 +0100 Subject: [PATCH 16/16] Add Memset's Cloud Storage to Openstack providers --- Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs index e8915a629..952091942 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs @@ -64,6 +64,7 @@ namespace Duplicati.Library.Backend.OpenStack new KeyValuePair("Rackspace UK", "https://lon.identity.api.rackspacecloud.com/v2.0"), new KeyValuePair("OVH Cloud Storage", "https://auth.cloud.ovh.net/v2.0"), new KeyValuePair("Selectel Cloud Storage", "https://auth.selcdn.ru"), + new KeyValuePair("Memset Cloud Storage", "https://auth.storage.memset.com"), }; public static readonly KeyValuePair[] OPENSTACK_VERSIONS = {