haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove OAuthHelper

Browse Source 
This PR removes OAuthHelper.

Initial commit is removal of OAuthHelper in GoogleCommon, GoogleDrive and GCS.
This commit is contained in:
Kenneth Skovhede
2025-05-23 10:40:55 +02:00
parent 8393a28c6b
commit d3b65f1e92
12 changed files with 318 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Backend/Box/BoxBackend.cs
+1 -1
View File
@@ -33,7 +33,7 @@ namespace Duplicati.Library.Backend.Box
{
public class BoxBackend : IStreamingBackend
{
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("box.com");
private static readonly string TOKEN_URL = OAuthHelperHttpClient.OAUTH_LOGIN_URL("box.com");
private const string AUTHID_OPTION = "authid";
private const string REALLY_DELETE_OPTION = "box-delete-from-trash";
Duplicati/Library/Backend/Dropbox/Dropbox.cs
+1 -1
View File
@@ -30,7 +30,7 @@ namespace Duplicati.Library.Backend
// This class is instantiated dynamically in the BackendLoader.
public class Dropbox : IBackend, IStreamingBackend
{
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("dropbox");
private static readonly string TOKEN_URL = OAuthHelperHttpClient.OAUTH_LOGIN_URL("dropbox");
private readonly string m_path;
private readonly DropboxHelper dbx;
Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs
+52 -45
View File
@@ -26,6 +26,7 @@ using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using Newtonsoft.Json;
using System.Net;
using System.Net.Http.Json;
using System.Runtime.CompilerServices;
using System.Text;
@@ -35,7 +36,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
// This class is instantiated dynamically in the BackendLoader.
public class GoogleCloudStorage : IBackend, IStreamingBackend, IRenameEnabledBackend
{
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("gcs");
private static readonly string TOKEN_URL = OAuthHelperHttpClient.OAUTH_LOGIN_URL("gcs");
private const string PROJECT_OPTION = "gcs-project";
private const string LOCATION_OPTION = "gcs-location";
@@ -44,7 +45,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
private readonly string m_bucket;
private readonly string m_prefix;
private readonly string? m_project;
private readonly OAuthHelper m_oauth;
private readonly OAuthHelperHttpClient m_oauth;
private readonly string? m_location;
private readonly string? m_storage_class;
@@ -77,7 +78,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
m_location = options.GetValueOrDefault(LOCATION_OPTION);
m_storage_class = options.GetValueOrDefault(STORAGECLASS_OPTION);
m_oauth = new OAuthHelper(authId.AuthId!, this.ProtocolKey);
m_oauth = new OAuthHelperHttpClient(authId.AuthId!, this.ProtocolKey);
m_oauth.AutoAuthHeader = true;
}
@@ -125,9 +126,17 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
while (true)
{
var resp = await HandleListExceptions(() =>
Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct =>
m_oauth.ReadJSONResponseAsync<ListBucketResponse>(url, ct))
).ConfigureAwait(false);
Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, async ct =>
{
using var req = await m_oauth.CreateRequestAsync(url, HttpMethod.Get, ct).ConfigureAwait(false);
req.Headers.Add("Accept", "application/json");
using var resp = await m_oauth.GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
resp.EnsureSuccessStatusCode();
return await resp.Content.ReadFromJsonAsync<ListBucketResponse>(ct).ConfigureAwait(false)
?? throw new Exception("Failed to parse response");
})).ConfigureAwait(false);
if (resp.items != null)
foreach (var f in resp.items)
@@ -161,43 +170,43 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
using (var fs = File.Create(filename))
await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false);
}
public Task DeleteAsync(string remotename, CancellationToken cancelToken)
public async Task DeleteAsync(string remotename, CancellationToken cancelToken)
{
var req = m_oauth.CreateRequest(WebApi.GoogleCloudStorage.DeleteUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)));
req.Method = "DELETE";
using var req = await m_oauth.CreateRequestAsync(WebApi.GoogleCloudStorage.DeleteUrl(m_bucket, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)), HttpMethod.Delete, cancelToken);
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.ReadJSONResponseAsync<object>(req, ct));
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=>
{
using var resp = m_oauth.GetResponseAsync(req, HttpCompletionOption.ResponseContentRead, ct);
resp.Result.EnsureSuccessStatusCode();
}).ConfigureAwait(false);
}
public Task TestAsync(CancellationToken cancelToken)
=> this.TestReadWritePermissionsAsync(cancelToken);
public Task CreateFolderAsync(CancellationToken cancelToken)
public async Task CreateFolderAsync(CancellationToken cancelToken)
{
if (string.IsNullOrEmpty(m_project))
throw new UserInformationException(Strings.GoogleCloudStorage.ProjectIDMissingError(PROJECT_OPTION), "GoogleCloudStorageMissingProjectID");
var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new CreateBucketRequest
using var req = await m_oauth.CreateRequestAsync(WebApi.GoogleCloudStorage.CreateFolderUrl(m_project), HttpMethod.Post, cancelToken);
req.Content = JsonContent.Create(new CreateBucketRequest
{
name = m_bucket,
location = m_location,
storageClass = m_storage_class
}));
var req = m_oauth.CreateRequest(WebApi.GoogleCloudStorage.CreateFolderUrl(m_project));
req.Method = "POST";
req.ContentLength = data.Length;
req.ContentType = "application/json; charset=UTF-8";
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct);
await m_oauth.ReadJSONResponseAsync<BucketResourceItem>(areq, ct).ConfigureAwait(false);
});
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
using var resp = await m_oauth.GetResponseAsync(req, HttpCompletionOption.ResponseContentRead, ct).ConfigureAwait(false);
resp.EnsureSuccessStatusCode();
var res = await resp.Content.ReadFromJsonAsync<BucketResourceItem>(ct).ConfigureAwait(false);
if (res == null)
throw new Exception("Create folder succeeded, but no data was returned");
}).ConfigureAwait(false);
}
public string DisplayName => Strings.GoogleCloudStorage.DisplayName;
@@ -236,7 +245,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
var item = new BucketResourceItem { name = m_prefix + remotename };
var url = WebApi.GoogleCloudStorage.PutUrl(m_bucket);
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<BucketResourceItem, BucketResourceItem>(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken).ConfigureAwait(false);
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<BucketResourceItem, BucketResourceItem>(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken, HttpMethod.Post).ConfigureAwait(false);
if (res == null)
throw new Exception("Upload succeeded, but no data was returned");
@@ -247,12 +256,11 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
try
{
var url = WebApi.GoogleCloudStorage.GetUrl(m_bucket, Utility.Uri.UrlPathEncode(m_prefix + remotename));
var req = m_oauth.CreateRequest(url);
var areq = new AsyncHttpRequest(req);
using var req = await m_oauth.CreateRequestAsync(url, HttpMethod.Get, cancelToken);
using var resp = await Library.Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_oauth.GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, ct)).ConfigureAwait(false);
using var source = await Library.Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => resp.Content.ReadAsStreamAsync(cancelToken)).ConfigureAwait(false);
using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponse()).ConfigureAwait(false))
using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false))
using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
using (var ts = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken).ConfigureAwait(false);
}
catch (WebException wex)
@@ -264,26 +272,25 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage
}
}
public Task RenameAsync(string oldname, string newname, CancellationToken cancelToken)
public async Task RenameAsync(string oldname, string newname, CancellationToken cancelToken)
{
var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(new BucketResourceItem
{
name = m_prefix + newname,
}));
var req = m_oauth.CreateRequest(WebApi.GoogleCloudStorage.RenameUrl(m_bucket, Utility.Uri.UrlPathEncode(m_prefix + oldname)));
req.Method = "PATCH";
req.ContentLength = data.Length;
req.ContentType = "application/json; charset=UTF-8";
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
var req = await m_oauth.CreateRequestAsync(WebApi.GoogleCloudStorage.RenameUrl(m_bucket, Utility.Uri.UrlPathEncode(m_prefix + oldname)), HttpMethod.Patch, cancelToken).ConfigureAwait(false);
req.Content = JsonContent.Create(new BucketResourceItem
{
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false);
await m_oauth.ReadJSONResponseAsync<BucketResourceItem>(req, ct).ConfigureAwait(false);
name = m_prefix + newname,
});
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, async ct =>
{
using var resp = await m_oauth.GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
resp.EnsureSuccessStatusCode();
await resp.Content.ReadFromJsonAsync<BucketResourceItem>(ct).ConfigureAwait(false);
}).ConfigureAwait(false);
}
#region IDisposable implementation
Duplicati/Library/Backend/GoogleServices/GoogleCommon.cs
+82 -107
View File
@@ -22,6 +22,8 @@
using Duplicati.Library.Utility;
using Newtonsoft.Json;
using System.Net;
using System.Net.Http.Headers;
using System.Net.Http.Json;
namespace Duplicati.Library.Backend.GoogleServices
{
@@ -35,49 +37,54 @@ namespace Duplicati.Library.Backend.GoogleServices
/// <summary>
/// Helper method that queries a resumeable upload uri for progress
/// </summary>
/// <returns>The upload range.</returns>
/// <typeparam name="T">The type of data in the response.</typeparam>
/// <param name="oauth">The Oauth instance</param>
/// <param name="uploaduri">The resumeable uploaduri</param>
/// <param name="streamlength">The length of the entire stream</param>
private static long QueryUploadRange<T>(OAuthHelper oauth, string uploaduri, long streamlength, out T? response)
/// <param name="shortTimeout">The short request timeout.</param>
/// <param name="readWriteTimeout">The read write timeout.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Tuple with the range and the response.</returns>
private static async Task<(long, T? Response)> QueryUploadRange<T>(OAuthHelperHttpClient oauth, string uploaduri, long streamlength, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancellationToken)
where T : class
{
response = null;
var req = oauth.CreateRequest(uploaduri);
req.Method = "PUT";
req.ContentLength = 0;
req.Headers["Content-Range"] = string.Format("bytes */{0}", streamlength);
using var req = await oauth.CreateRequestAsync(uploaduri, HttpMethod.Put, cancellationToken);
req.Content = new ByteArrayContent(Array.Empty<byte>());
req.Headers.TryAddWithoutValidation("Content-Range", $"bytes */{streamlength}");
// TODO: Apply timeout when this is upgraded to HttpClient
var areq = new AsyncHttpRequest(req);
using (var resp = oauth.GetResponseWithoutException(areq))
using var resp = await Library.Utility.Utility.WithTimeout(
shortTimeout,
cancellationToken,
ct => oauth.GetResponseAsync(req, HttpCompletionOption.ResponseContentRead, ct)
).ConfigureAwait(false);
var code = (int)resp.StatusCode;
// If the upload is completed, we get 201 or 200
if (code >= 200 && code <= 299)
{
var code = (int)resp.StatusCode;
// No timeout here, beause the response content is already read
var response = await resp.Content.ReadFromJsonAsync<T>(cancellationToken);
if (response == null)
throw new Exception(string.Format("Upload succeeded, but no data was returned, status code: {0}", code));
// If the upload is completed, we get 201 or 200
if (code >= 200 && code <= 299)
{
response = oauth.ReadJSONResponse<T>(resp);
if (response == null)
throw new Exception(string.Format("Upload succeeded, but no data was returned, status code: {0}", code));
return streamlength;
}
if (code == 308)
{
// A lack of a Range header is undocumented,
// but seems to occur when no data has reached the server:
// https://code.google.com/a/google.com/p/apps-api-issues/issues/detail?id=3884
if (resp.Headers["Range"] == null)
return 0;
else
return long.Parse(resp.Headers["Range"]!.Split(new char[] { '-' })[1]) + 1;
}
else
throw new WebException(string.Format("Unexpected status code: {0}", code), null, WebExceptionStatus.ServerProtocolViolation, resp);
return (streamlength, response);
}
if (code == 308)
{
// A lack of a Range header is undocumented,
// but seems to occur when no data has reached the server:
// https://code.google.com/a/google.com/p/apps-api-issues/issues/detail?id=3884
if (!resp.Content.Headers.TryGetValues("Range", out var rangeValues))
return (0, null);
return (long.Parse(rangeValues.First().Split('-')[1]) + 1, null);
}
else
throw new HttpRequestException(HttpRequestError.HttpProtocolError, string.Format("Unexpected status code: {0}", code), null, resp.StatusCode);
}
/// <summary>
@@ -93,36 +100,28 @@ namespace Duplicati.Library.Backend.GoogleServices
/// <param name="method">The HTTP Method.</param>
/// <typeparam name="TRequest">The type of data to upload as metadata.</typeparam>
/// <typeparam name="TResponse">The type of data returned from the upload.</typeparam>
public static async Task<TResponse> ChunkedUploadWithResumeAsync<TRequest, TResponse>(OAuthHelper oauth, TRequest requestdata, string url, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken, string method = "POST")
public static async Task<TResponse> ChunkedUploadWithResumeAsync<TRequest, TResponse>(OAuthHelperHttpClient oauth, TRequest requestdata, string url, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken, HttpMethod method)
where TRequest : class
where TResponse : class
{
var data = requestdata == null ? null : System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(requestdata));
using var req = await oauth.CreateRequestAsync(url, method, cancelToken);
if (requestdata != null)
req.Content = JsonContent.Create(requestdata);
req.Headers.TryAddWithoutValidation("X-Upload-Content-Type", "application/octet-stream");
req.Headers.TryAddWithoutValidation("X-Upload-Content-Length", stream.Length.ToString());
var req = oauth.CreateRequest(url);
req.Method = method;
req.ContentLength = data == null ? 0 : data.Length;
if (data != null)
req.ContentType = "application/json; charset=UTF-8";
req.Headers["X-Upload-Content-Type"] = "application/octet-stream";
req.Headers["X-Upload-Content-Length"] = stream.Length.ToString();
var uploaduri = await Utility.Utility.WithTimeout(shortTimeout, cancelToken, async ct =>
{
var areq = new AsyncHttpRequest(req);
if (data != null)
using (var rs = areq.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false);
using (var resp = (HttpWebResponse)areq.GetResponse())
using var resp = await oauth.GetResponseAsync(req, HttpCompletionOption.ResponseContentRead, ct).ConfigureAwait(false);
if (resp.StatusCode != HttpStatusCode.OK ||
!resp.Headers.TryGetValues("Location", out var locationValues) ||
string.IsNullOrWhiteSpace(locationValues.FirstOrDefault()))
{
if (resp.StatusCode != HttpStatusCode.OK || string.IsNullOrWhiteSpace(resp.Headers["Location"]))
throw new WebException("Failed to start upload session", null, WebExceptionStatus.UnknownError, resp);
return resp.Headers["Location"];
throw new HttpRequestException(HttpRequestError.Unknown, "Failed to start upload session", null, resp.StatusCode);
}
return locationValues.First();
}).ConfigureAwait(false);
if (string.IsNullOrWhiteSpace(uploaduri))
@@ -141,14 +140,13 @@ namespace Duplicati.Library.Backend.GoogleServices
/// <param name="shortTimeout">The short request timeout.</param>
/// <param name="readWriteTimeout">The read write timeout.</param>
/// <typeparam name="T">The type of data in the response.</typeparam>
private static async Task<T> ChunkedUploadAsync<T>(OAuthHelper oauth, string uploaduri, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken)
private static async Task<T> ChunkedUploadAsync<T>(OAuthHelperHttpClient oauth, string uploaduri, Stream stream, TimeSpan shortTimeout, TimeSpan readWriteTimeout, CancellationToken cancelToken)
where T : class
{
var queryRange = false;
var retries = 0;
var offset = 0L;
var buffer = new byte[Library.Utility.Utility.DEFAULT_BUFFER_SIZE];
HttpWebResponse? resp = null;
// Repeatedly try uploading until all retries are done
while (true)
@@ -157,7 +155,7 @@ namespace Duplicati.Library.Backend.GoogleServices
{
if (queryRange)
{
offset = QueryUploadRange<T>(oauth, uploaduri, stream.Length, out var re);
(offset, var re) = await QueryUploadRange<T>(oauth, uploaduri, stream.Length, shortTimeout, readWriteTimeout, cancelToken).ConfigureAwait(false);
queryRange = false;
if (re != null)
@@ -168,75 +166,55 @@ namespace Duplicati.Library.Backend.GoogleServices
if (stream.Position != offset)
stream.Position = offset;
var req = oauth.CreateRequest(uploaduri);
req.Method = "PUT";
req.ContentType = "application/octet-stream";
using var req = await oauth.CreateRequestAsync(uploaduri, HttpMethod.Put, cancelToken);
var chunkSize = Math.Min(UPLOAD_CHUNK_SIZE, stream.Length - offset);
req.ContentLength = chunkSize;
req.Headers["Content-Range"] = string.Format("bytes {0}-{1}/{2}", offset, offset + chunkSize - 1, stream.Length);
// Upload the remaining data
var areq = new AsyncHttpRequest(req);
using (var rs = areq.GetRequestStream())
using (var ts = rs.ObserveWriteTimeout(readWriteTimeout))
{
var remaining = chunkSize;
while (remaining > 0)
{
var n = stream.Read(buffer, 0, (int)Math.Min(remaining, Library.Utility.Utility.DEFAULT_BUFFER_SIZE));
await ts.WriteAsync(buffer, 0, n, cancelToken);
remaining -= n;
}
}
using var ls = new ReadLimitLengthStream(stream, offset, chunkSize);
using var ts = ls.ObserveWriteTimeout(readWriteTimeout);
req.Content = new StreamContent(ts);
req.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
req.Headers.TryAddWithoutValidation("Content-Range", $"bytes {offset}-{offset + chunkSize - 1}/{stream.Length}");
using var resp = await oauth.GetResponseAsync(req, HttpCompletionOption.ResponseContentRead, cancelToken).ConfigureAwait(false);
// Check the response
resp = await Utility.Utility.WithTimeout(shortTimeout, cancelToken, _ => oauth.GetResponseWithoutException(areq)).ConfigureAwait(false);
var code = (int)resp.StatusCode;
if (code == 308 && resp.Headers["Range"] != null)
if ((int)resp.StatusCode == 308 &&
resp.Headers.TryGetValues("Range", out var rangeValues) &&
!string.IsNullOrWhiteSpace(rangeValues.FirstOrDefault()))
{
offset = long.Parse(resp.Headers["Range"]!.Split(new char[] { '-' })[1]) + 1;
offset = long.Parse(rangeValues.First().Split('-')[1]) + 1;
retries = 0;
}
else if (code >= 200 && code <= 299)
{
offset += chunkSize;
if (offset != stream.Length)
throw new Exception(string.Format("Upload succeeded prematurely. Uploaded: {0}, total size: {1}", offset, stream.Length));
throw new HttpRequestException(HttpRequestError.HttpProtocolError, string.Format("Upload succeeded prematurely. Uploaded: {0}, total size: {1}", offset, stream.Length), null, resp.StatusCode);
//Verify that the response is also valid
var res = oauth.ReadJSONResponse<T>(resp);
if (res == null)
throw new Exception(string.Format("Upload succeeded, but no data was returned, status code: {0}", code));
return res;
//Verify that the response is also valid (no timeout guard, as we already read the content)
return await resp.Content.ReadFromJsonAsync<T>(cancelToken).ConfigureAwait(false)
?? throw new HttpRequestException(HttpRequestError.HttpProtocolError, string.Format("Upload succeeded, but no data was returned, status code: {0}", code), null, resp.StatusCode);
}
else
{
throw new WebException(string.Format("Unexpected status code: {0}", code), null, WebExceptionStatus.ServerProtocolViolation, resp);
throw new HttpRequestException(HttpRequestError.HttpProtocolError, string.Format("Unexpected status code: {0}", code), null, resp.StatusCode);
}
}
catch (Exception ex)
{
var retry = false;
// If we get a 5xx error, or some network issue, we retry
if (ex is WebException exception && exception.Response is HttpWebResponse response)
// Check for HttpRequestException and inspect status code if available
if (ex is HttpRequestException httpEx && httpEx.StatusCode.HasValue)
{
try
{
var code = (int)response.StatusCode;
retry = code >= 500 && code <= 599;
}
catch
{
// Assume this is a transient error
retry = true;
}
var code = (int)httpEx.StatusCode.Value;
retry = code >= 500 && code <= 599;
}
else if (ex is System.Net.Sockets.SocketException || ex is IOException || ex.InnerException is System.Net.Sockets.SocketException || ex.InnerException is System.IO.IOException)
else if (ex is IOException ||
ex is System.Net.Sockets.SocketException ||
ex.InnerException is IOException ||
ex.InnerException is System.Net.Sockets.SocketException)
{
retry = true;
}
@@ -251,12 +229,9 @@ namespace Duplicati.Library.Backend.GoogleServices
queryRange = true;
}
else
{
throw;
}
finally
{
// Wait until the end of the request to dispose of the response, as it may be needed for error handling
resp?.Dispose();
}
}
}
}
Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs
+47 -34
View File
@@ -25,7 +25,7 @@ using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Duplicati.Library.Utility.Options;
using Newtonsoft.Json;
using System.Net;
using System.Net.Http.Json;
using System.Runtime.CompilerServices;
using System.Text;
@@ -38,11 +38,11 @@ namespace Duplicati.Library.Backend.GoogleDrive
private const string TEAMDRIVE_ID = "googledrive-teamdrive-id";
private const string FOLDER_MIMETYPE = "application/vnd.google-apps.folder";
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("googledrive");
private static readonly string TOKEN_URL = OAuthHelperHttpClient.OAUTH_LOGIN_URL("googledrive");
private readonly string m_path;
private readonly string? m_teamDriveID;
private readonly OAuthHelper m_oauth;
private readonly OAuthHelperHttpClient m_oauth;
private readonly Dictionary<string, GoogleDriveFolderItem[]> m_filecache;
private readonly TimeoutOptionsHelper.Timeouts m_timeouts;
@@ -67,7 +67,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
m_timeouts = TimeoutOptionsHelper.Parse(options);
m_teamDriveID = options.GetValueOrDefault(TEAMDRIVE_ID);
m_oauth = new OAuthHelper(auth.AuthId!, this.ProtocolKey) { AutoAuthHeader = true };
m_oauth = new OAuthHelperHttpClient(auth.AuthId!, this.ProtocolKey) { AutoAuthHeader = true };
m_filecache = new Dictionary<string, GoogleDriveFolderItem[]>();
}
@@ -180,7 +180,7 @@ namespace Duplicati.Library.Backend.GoogleDrive
teamDriveId = m_teamDriveID
};
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<GoogleDriveFolderItem, GoogleDriveFolderItem>(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken, isUpdate ? "PUT" : "POST").ConfigureAwait(false);
var res = await GoogleCommon.ChunkedUploadWithResumeAsync<GoogleDriveFolderItem, GoogleDriveFolderItem>(m_oauth, item, url, stream, m_timeouts.ShortTimeout, m_timeouts.ReadWriteTimeout, cancelToken, isUpdate ? HttpMethod.Put : HttpMethod.Post).ConfigureAwait(false);
m_filecache[remotename] = [res];
}
catch
@@ -200,10 +200,11 @@ namespace Duplicati.Library.Backend.GoogleDrive
if (string.IsNullOrEmpty(fileId))
throw new FileMissingException(Strings.GoogleDrive.FilenotFound(remotename));
var req = m_oauth.CreateRequest(WebApi.GoogleDrive.GetUrl(fileId));
var areq = new AsyncHttpRequest(req);
using (var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => (HttpWebResponse)areq.GetResponse()).ConfigureAwait(false))
using (var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false))
var req = await m_oauth.CreateRequestAsync(WebApi.GoogleDrive.GetUrl(fileId), HttpMethod.Get, cancelToken).ConfigureAwait(false);
using var resp = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => m_oauth.GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, ct)).ConfigureAwait(false);
resp.EnsureSuccessStatusCode();
using var rs = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct => resp.Content.ReadAsStreamAsync(ct)).ConfigureAwait(false);
using (var ts = rs.ObserveReadTimeout(m_timeouts.ReadWriteTimeout))
await Utility.Utility.CopyStreamAsync(ts, stream, cancelToken);
}
@@ -291,12 +292,13 @@ namespace Duplicati.Library.Backend.GoogleDrive
foreach (var fileid in entries.Select(x => x.id).WhereNotNullOrWhiteSpace())
{
var url = WebApi.GoogleDrive.DeleteUrl(Library.Utility.Uri.UrlPathEncode(fileid), m_teamDriveID);
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<object>(url, cancelToken, x =>
await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken,
async ct =>
{
x.Method = "DELETE";
})
).ConfigureAwait(false);
using var req = await m_oauth.CreateRequestAsync(url, HttpMethod.Delete, ct).ConfigureAwait(false);
using var resp = await m_oauth.GetResponseAsync(req, HttpCompletionOption.ResponseContentRead, ct).ConfigureAwait(false);
resp.EnsureSuccessStatusCode();
}).ConfigureAwait(false);
}
m_filecache.Remove(remotename);
@@ -447,9 +449,16 @@ namespace Duplicati.Library.Backend.GoogleDrive
while (true)
{
var res = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<GoogleDriveListResponse>(url, ct)
).ConfigureAwait(false);
var res = await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken,
async ct =>
{
using var req = await m_oauth.CreateRequestAsync(url, HttpMethod.Get, ct).ConfigureAwait(false);
using var resp = await m_oauth.GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
resp.EnsureSuccessStatusCode();
return await resp.Content.ReadFromJsonAsync<GoogleDriveListResponse>(ct).ConfigureAwait(false)
?? throw new UserInformationException(Strings.GoogleDrive.ListResponseError, "GoogleDriveListResponseError");
}).ConfigureAwait(false);
foreach (var n in res.items ?? [])
yield return n;
@@ -463,8 +472,16 @@ namespace Duplicati.Library.Backend.GoogleDrive
}
private Task<GoogleDriveAboutResponse> GetAboutInfoAsync(CancellationToken cancelToken)
=> Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<GoogleDriveAboutResponse>(WebApi.GoogleDrive.AboutInfoUrl(), ct));
=> Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken,
async ct =>
{
using var req = await m_oauth.CreateRequestAsync(WebApi.GoogleDrive.AboutInfoUrl(), HttpMethod.Get, ct).ConfigureAwait(false);
using var resp = await m_oauth.GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
resp.EnsureSuccessStatusCode();
return await resp.Content.ReadFromJsonAsync<GoogleDriveAboutResponse>(ct).ConfigureAwait(false)
?? throw new UserInformationException(Strings.GoogleDrive.AboutResponseError, "GoogleDriveAboutResponseError");
});
private Task<GoogleDriveFolderItem> CreateFolderAsync(string name, string parent, CancellationToken cancelToken)
{
@@ -479,22 +496,18 @@ namespace Duplicati.Library.Backend.GoogleDrive
var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(folder));
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, ct
=> m_oauth.GetJSONDataAsync<GoogleDriveFolderItem>(
WebApi.GoogleDrive.CreateFolderUrl(m_teamDriveID),
ct,
x =>
{
x.Method = "POST";
x.ContentType = "application/json; charset=UTF-8";
x.ContentLength = data.Length;
return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken,
async ct =>
{
using var req = await m_oauth.CreateRequestAsync(WebApi.GoogleDrive.CreateFolderUrl(m_teamDriveID), HttpMethod.Post, ct).ConfigureAwait(false);
req.Content = JsonContent.Create(data);
}, async (req, ct) =>
{
using (var rs = req.GetRequestStream())
await rs.WriteAsync(data, 0, data.Length, ct).ConfigureAwait(false);
})
);
using var resp = await m_oauth.GetResponseAsync(req, HttpCompletionOption.ResponseHeadersRead, ct).ConfigureAwait(false);
resp.EnsureSuccessStatusCode();
return await resp.Content.ReadFromJsonAsync<GoogleDriveFolderItem>(ct).ConfigureAwait(false)
?? throw new UserInformationException(Strings.GoogleDrive.CreateFolderResponseError, "GoogleDriveListResponseError");
});
}
}
}
Duplicati/Library/Backend/GoogleServices/Strings.cs
+3
View File
@@ -46,6 +46,9 @@ namespace Duplicati.Library.Backend.Strings
public static string FilenotFound(string name) { return LC.L(@"File not found: {0}", name); }
public static string TeamDriveIdLong { get { return LC.L("This option sets the team drive to use. Leaving it empty uses the personal drive."); } }
public static string TeamDriveIdShort { get { return LC.L("Team drive ID"); } }
public static string ListResponseError { get { return LC.L(@"The list response was not valid."); } }
public static string AboutResponseError { get { return LC.L(@"The about response was not valid."); } }
public static string CreateFolderResponseError { get { return LC.L(@"The create folder response was not valid."); } }
}
internal static class GCSConfig
Duplicati/Library/Backend/OAuthHelper/OAuthContrextSettings.cs
+79
View File
@@ -0,0 +1,79 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using Duplicati.Library.Utility;
#nullable enable
namespace Duplicati.Library;
/// <summary>
/// Class for providing call-context access to http settings
/// </summary>
public static class OAuthContextSettings
{
/// <summary>
/// The URL to use for the OAuth login hosted on GAE.
/// </summary>
public const string DUPLICATI_OAUTH_SERVICE = "https://duplicati-oauth-handler.appspot.com/refresh";
/// <summary>
/// The URL to use for the new hosted OAuth login server.
/// </summary>
public const string DUPLICATI_OAUTH_SERVICE_NEW = "https://oauth-service.duplicati.com/refresh";
/// <summary>
/// The struct wrapping the OAuth settings
/// </summary>
private struct OAuthSettings
{
/// <summary>
/// The server url
/// </summary>
public string ServerURL;
}
/// <summary>
/// Starts the session.
/// </summary>
/// <returns>The session.</returns>
/// <param name="serverurl">The url to use for the server.</param>
public static IDisposable StartSession(string serverurl)
{
return CallContextSettings<OAuthSettings>.StartContext(new OAuthSettings { ServerURL = serverurl });
}
/// <summary>
/// Gets the server URL to use for OAuth.
/// </summary>
public static string ServerURL
{
get
{
var r = CallContextSettings<OAuthSettings>.Settings.ServerURL;
return string.IsNullOrWhiteSpace(r) ? DUPLICATI_OAUTH_SERVICE : r;
}
}
/// <summary>
/// Gets the server URL to use for OAuth, without applying a default.
/// </summary>
public static string? ServerURLRaw => CallContextSettings<OAuthSettings>.Settings.ServerURL;
}
Duplicati/Library/Backend/OAuthHelper/OAuthHelperHttpClient.cs
+22 -8
View File
@@ -48,15 +48,29 @@ public class OAuthHelperHttpClient : JsonWebHelperHttpClient
private const int MAX_AUTHORIZATION_RETRIES = 5;
/// <summary>
/// URL to use for OAuth login
/// Returns the URL to use for obtaining an OAuth token for the given module.
/// </summary>
/// <param name="modulename">The name of the module</param>
/// <returns>The URL to use for OAuth login</returns>
private static string OAUTH_LOGIN_URL(string modulename)
/// <param name="modulename">The name of the module to use.</param>
/// <returns>The URL to use for obtaining an OAuth token.</returns>
public static string OAUTH_LOGIN_URL(string modulename)
{
var u = new Utility.Uri(OAuthContextSettings.ServerURL);
var addr = u.SetPath("")
.SetQuery((u.Query ?? "") + (string.IsNullOrWhiteSpace(u.Query) ? "" : "&") + "type={0}");
var u = new Library.Utility.Uri(OAuthContextSettings.ServerURL);
var addr = u.SetPath("").SetQuery((u.Query ?? "") + (string.IsNullOrWhiteSpace(u.Query) ? "" : "&") + "type={0}");
return string.Format(addr.ToString(), modulename);
}
/// <summary>
/// Returns the URL to use for obtaining an OAuth token for the given module, defaulting to the new server.
/// </summary>
/// <param name="modulename">The name of the module to use.</param>
public static string OAUTH_LOGIN_URL_NEW(string modulename)
{
var baseUrl = OAuthContextSettings.ServerURLRaw;
if (string.IsNullOrWhiteSpace(baseUrl))
baseUrl = OAuthContextSettings.DUPLICATI_OAUTH_SERVICE_NEW;
var u = new Library.Utility.Uri(baseUrl);
var addr = u.SetPath("").SetQuery((u.Query ?? "") + (string.IsNullOrWhiteSpace(u.Query) ? "" : "&") + "type={0}");
return string.Format(addr.ToString(), modulename);
}
@@ -104,7 +118,7 @@ public class OAuthHelperHttpClient : JsonWebHelperHttpClient
public override Task<HttpRequestMessage> CreateRequestAsync(string url, HttpMethod method, CancellationToken cancellationToken)
=> CreateRequestAsync(url, method, false, cancellationToken);
private async Task<string> GetAccessTokenAsync(CancellationToken cancellationToken)
public async Task<string> GetAccessTokenAsync(CancellationToken cancellationToken)
{
if (AccessTokenOnly)
return _Authid;
Duplicati/Library/Backend/OAuthHelper/OAuthHttpMessageHandler.cs
+9 -10
View File
@@ -33,13 +33,15 @@ namespace Duplicati.Library
/// </summary>
public const string DISABLE_AUTHENTICATION_PROPERTY = "OAuthHttpMessageHandler_DisableAuthentication";
private readonly OAuthHelper m_oauth;
private readonly OAuthHelperHttpClient m_oauth;
public OAuthHttpMessageHandler(string authid, string protocolKey)
{
this.m_oauth = new OAuthHelper(authid, protocolKey);
this.m_oauth = new OAuthHelperHttpClient(authid, protocolKey);
}
private static readonly HttpRequestOptionsKey<bool> PreventAuthenticationOption = new HttpRequestOptionsKey<bool>("PreventAuthentication");
/// <summary>
/// Prevents authentication from being applied on the given request
/// </summary>
@@ -47,18 +49,15 @@ namespace Duplicati.Library
/// <returns>Request to not authenticate</returns>
public HttpRequestMessage PreventAuthentication(HttpRequestMessage request)
{
request.Properties[DISABLE_AUTHENTICATION_PROPERTY] = true;
request.Options.Set(PreventAuthenticationOption, true);
return request;
}
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
if (!request.Properties.ContainsKey(DISABLE_AUTHENTICATION_PROPERTY))
{
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", this.m_oauth.AccessToken);
}
return base.SendAsync(request, cancellationToken);
if (!request.Options.TryGetValue(PreventAuthenticationOption, out var preventAuth) || !preventAuth)
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await this.m_oauth.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false));
return await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
}
}
}
Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs
+1 -1
View File
@@ -115,7 +115,7 @@ namespace Duplicati.Library.Backend
private readonly Lazy<string> rootPathFromURL;
private string RootPath => this.rootPathFromURL.Value;
private string TokenUrl => OAuthHelper.OAUTH_LOGIN_URL(ProtocolKey);
private string TokenUrl => OAuthHelperHttpClient.OAUTH_LOGIN_URL(ProtocolKey);
protected MicrosoftGraphBackend()
{
Duplicati/Library/Backend/pCloud/pCloudBackend.cs
+1 -1
View File
@@ -39,7 +39,7 @@ public class pCloudBackend : IStreamingBackend
/// <summary>
/// The URL used to get a new token
/// </summary>
private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL_NEW("pcloud");
private static readonly string TOKEN_URL = OAuthHelperHttpClient.OAUTH_LOGIN_URL_NEW("pcloud");
/// <summary>
/// Implementation of interface property for the backend key
/// </summary>
Duplicati/Library/Modules/Builtin/HttpOptions.cs
+20 -20
View File
@@ -1,22 +1,22 @@
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// Copyright (C) 2025, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
@@ -117,7 +117,7 @@ namespace Duplicati.Library.Modules.Builtin
new Duplicati.Library.Interface.CommandLineArgument(OPTION_DISABLE_NAGLING, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.HttpOptions.DisableNagleShort, Strings.HttpOptions.DisableNagleLong, "false"),
new Duplicati.Library.Interface.CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.HttpOptions.DescriptionAcceptHashShort, Strings.HttpOptions.DescriptionAcceptHashLong2),
new Duplicati.Library.Interface.CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.HttpOptions.DescriptionAcceptAnyCertificateShort, Strings.HttpOptions.DescriptionAcceptAnyCertificateLong),
new Duplicati.Library.Interface.CommandLineArgument(OPTION_OAUTH_URL, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.HttpOptions.OauthurlShort, Strings.HttpOptions.OauthurlLong, OAuthHelper.DUPLICATI_OAUTH_SERVICE),
new Duplicati.Library.Interface.CommandLineArgument(OPTION_OAUTH_URL, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.HttpOptions.OauthurlShort, Strings.HttpOptions.OauthurlLong, OAuthContextSettings.DUPLICATI_OAUTH_SERVICE),
new Duplicati.Library.Interface.CommandLineArgument(OPTION_SSL_VERSIONS, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Flags, Strings.HttpOptions.SslversionsShort, Strings.HttpOptions.SslversionsLong, defaultssl, null, sslnames),
new Duplicati.Library.Interface.CommandLineArgument(OPTION_OPERATION_TIMEOUT, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.HttpOptions.OperationtimeoutShort, Strings.HttpOptions.OperationtimeoutLong),