// Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Utility; using Newtonsoft.Json; using System; using System.Net; using System.Threading; using System.Threading.Tasks; namespace Duplicati.Library.Backend.GoogleServices { internal static class GoogleCommon { /// /// The size of upload chunks /// private const long UPLOAD_CHUNK_SIZE = 1024 * 1024 * 10; /// /// Helper method that queries a resumeable upload uri for progress /// /// The upload range. /// The Oauth instance /// The resumeable uploaduri /// The length of the entire stream private static long QueryUploadRange(OAuthHelper oauth, string uploaduri, long streamlength, out T response) where T : class { response = null; var req = oauth.CreateRequest(uploaduri); req.Method = "PUT"; req.ContentLength = 0; req.Headers["Content-Range"] = string.Format("bytes */{0}", streamlength); var areq = new AsyncHttpRequest(req); using(var resp = oauth.GetResponseWithoutException(areq)) { var code = (int)resp.StatusCode; // If the upload is completed, we get 201 or 200 if (code >= 200 && code <= 299) { response = oauth.ReadJSONResponse(resp); if (response == null) throw new Exception(string.Format("Upload succeeded, but no data was returned, status code: {0}", code)); return streamlength; } if (code == 308) { // A lack of a Range header is undocumented, // but seems to occur when no data has reached the server: // https://code.google.com/a/google.com/p/apps-api-issues/issues/detail?id=3884 if (resp.Headers["Range"] == null) return 0; else return long.Parse(resp.Headers["Range"].Split(new char[] { '-' })[1]) + 1; } else throw new WebException(string.Format("Unexpected status code: {0}", code), null, WebExceptionStatus.ServerProtocolViolation, resp); } } /// /// Uploads the requestdata as JSON and starts a resumeable upload session, then uploads the stream in chunks /// /// Serialized result of the last request. /// The Oauth instance. /// The data to submit as JSON metadata. /// The URL to register the upload session against. /// The stream with content data to upload. /// The HTTP Method. /// The type of data to upload as metadata. /// The type of data returned from the upload. public static async Task ChunkedUploadWithResumeAsync(OAuthHelper oauth, TRequest requestdata, string url, System.IO.Stream stream, CancellationToken cancelToken, string method = "POST") where TRequest : class where TResponse : class { var data = requestdata == null ? null : System.Text.Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(requestdata)); var req = oauth.CreateRequest(url); req.Method = method; req.ContentLength = data == null ? 0 : data.Length; if (data != null) req.ContentType = "application/json; charset=UTF-8"; req.Headers["X-Upload-Content-Type"] = "application/octet-stream"; req.Headers["X-Upload-Content-Length"] = stream.Length.ToString(); var areq = new AsyncHttpRequest(req); if (data != null) using(var rs = areq.GetRequestStream()) await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false); string uploaduri; using(var resp = (HttpWebResponse)areq.GetResponse()) { if (resp.StatusCode != HttpStatusCode.OK || string.IsNullOrWhiteSpace(resp.Headers["Location"])) throw new WebException("Failed to start upload session", null, WebExceptionStatus.UnknownError, resp); uploaduri = resp.Headers["Location"]; } return await ChunkedUploadAsync(oauth, uploaduri, stream, cancelToken).ConfigureAwait(false); } /// /// Helper method that performs a chunked upload, and queries for http status after each chunk /// /// The response item /// The Oauth instance /// The resumeable uploaduri /// The stream with data to upload. /// The type of data in the response. private static async Task ChunkedUploadAsync(OAuthHelper oauth, string uploaduri, System.IO.Stream stream, CancellationToken cancelToken) where T : class { var queryRange = false; var retries = 0; var offset = 0L; var buffer = new byte[Library.Utility.Utility.DEFAULT_BUFFER_SIZE]; // Repeatedly try uploading until all retries are done while(true) { try { if (queryRange) { T re; offset = GoogleCommon.QueryUploadRange(oauth, uploaduri, stream.Length, out re); queryRange = false; if (re != null) return re; } //Seek into the right place if (stream.Position != offset) stream.Position = offset; var req = oauth.CreateRequest(uploaduri); req.Method = "PUT"; req.ContentType = "application/octet-stream"; var chunkSize = Math.Min(UPLOAD_CHUNK_SIZE, stream.Length - offset); req.ContentLength = chunkSize; req.Headers["Content-Range"] = string.Format("bytes {0}-{1}/{2}", offset, offset + chunkSize - 1, stream.Length); // Upload the remaining data var areq = new AsyncHttpRequest(req); using(var rs = areq.GetRequestStream()) { var remaining = chunkSize; while(remaining > 0) { var n = stream.Read(buffer, 0, (int)Math.Min(remaining, Library.Utility.Utility.DEFAULT_BUFFER_SIZE)); await rs.WriteAsync(buffer, 0, n, cancelToken); remaining -= n; } } // Check the response using(var resp = oauth.GetResponseWithoutException(areq)) { var code = (int)resp.StatusCode; if (code == 308 && resp.Headers["Range"] != null) { offset = long.Parse(resp.Headers["Range"].Split(new char[] {'-'})[1]) + 1; retries = 0; } else if (code >= 200 && code <= 299) { offset += chunkSize; if (offset != stream.Length) throw new Exception(string.Format("Upload succeeded prematurely. Uploaded: {0}, total size: {1}", offset, stream.Length)); //Verify that the response is also valid var res = oauth.ReadJSONResponse(resp); if (res == null) throw new Exception(string.Format("Upload succeeded, but no data was returned, status code: {0}", code)); return res; } else { throw new WebException(string.Format("Unexpected status code: {0}", code), null, WebExceptionStatus.ServerProtocolViolation, resp); } } } catch (Exception ex) { var retry = false; // If we get a 5xx error, or some network issue, we retry if (ex is WebException exception && exception.Response is HttpWebResponse response) { var code = (int)response.StatusCode; retry = code >= 500 && code <= 599; } else if (ex is System.Net.Sockets.SocketException || ex is System.IO.IOException || ex.InnerException is System.Net.Sockets.SocketException || ex.InnerException is System.IO.IOException) { retry = true; } // Retry with exponential backoff if (retry && retries < 5) { System.Threading.Thread.Sleep(TimeSpan.FromSeconds(Math.Pow(2, retries))); retries++; // Ask server where we left off queryRange = true; } else throw; } } } } }