diff --git a/Duplicati/Library/Backend/FTP/FTPBackend.cs b/Duplicati/Library/Backend/FTP/FTPBackend.cs index ecb87bae4..35c8f6ca0 100644 --- a/Duplicati/Library/Backend/FTP/FTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/FTPBackend.cs @@ -457,7 +457,7 @@ namespace Duplicati.Library.Backend { var client = await CreateClient(cancelToken).ConfigureAwait(false); var clientRemoteName = PreparePathForClient(remotename); - await using var inputStream = await client.OpenRead(clientRemoteName, token: cancelToken); + await using var inputStream = await Utility.Utility.WithTimeout(_timeouts.ShortTimeout, cancelToken, ct => client.OpenRead(clientRemoteName, token: ct)).ConfigureAwait(false); await using var timeoutStream = inputStream.ObserveReadTimeout(_timeouts.ReadWriteTimeout); await CoreUtility.CopyStreamAsync(inputStream, output, false, cancelToken).ConfigureAwait(false); } diff --git a/Duplicati/Library/Backend/Jottacloud/Duplicati.Library.Backend.Jottacloud.csproj b/Duplicati/Library/Backend/Jottacloud/Duplicati.Library.Backend.Jottacloud.csproj index 0b042b92e..3cba2fed0 100644 --- a/Duplicati/Library/Backend/Jottacloud/Duplicati.Library.Backend.Jottacloud.csproj +++ b/Duplicati/Library/Backend/Jottacloud/Duplicati.Library.Backend.Jottacloud.csproj @@ -5,6 +5,8 @@ Library Duplicati.Library.Backend Copyright © 2025 Team Duplicati, MIT license + enable + enable diff --git a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs index 9a4c25336..ec4e5f30f 100644 --- a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs +++ b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs @@ -22,12 +22,11 @@ using Duplicati.Library.Common.IO; using Duplicati.Library.Interface; using Duplicati.Library.Localization.Short; -using System; -using System.Collections.Generic; +using Duplicati.Library.Utility; +using Duplicati.Library.Utility.Options; using System.Net; using System.Runtime.CompilerServices; using System.Security.Cryptography; -using System.Threading; using System.Threading.Tasks; namespace Duplicati.Library.Backend { @@ -35,7 +34,7 @@ namespace Duplicati.Library.Backend // This class is instantiated dynamically in the BackendLoader. public class Jottacloud : IBackend, IStreamingBackend { - private const string AUTHID_OPTION = "authid"; + private static readonly string TOKEN_URL = OAuthHelper.OAUTH_LOGIN_URL("jottacloud"); private const string JFS_ROOT = "https://jfs.jottacloud.com/jfs"; private const string JFS_ROOT_UPLOAD = "https://up.jottacloud.com/jfs"; // Separate host for uploading files private const string JFS_BUILTIN_DEVICE = "Jotta"; // The built-in device used for the built-in Sync and Archive mount points. @@ -60,6 +59,7 @@ namespace Duplicati.Library.Backend private static readonly string JFS_DEFAULT_THREADS = "4"; private readonly int m_threads; private readonly long m_chunksize; + private readonly TimeoutOptionsHelper.Timeouts m_timeouts; private readonly JottacloudAuthHelper m_oauth; @@ -70,23 +70,33 @@ namespace Duplicati.Library.Backend /// static Jottacloud() { - System.Net.ServicePointManager.DefaultConnectionLimit = 1000; + // TODO: Remove this when HttpClient migration is done + ServicePointManager.DefaultConnectionLimit = 1000; } public Jottacloud() { + m_device = null!; + m_mountPoint = null!; + m_path = null!; + m_url_device = null!; + m_url = null!; + m_url_upload = null!; + m_timeouts = null!; + m_oauth = null!; } - public Jottacloud(string url, Dictionary options) + public Jottacloud(string url, Dictionary options) { // Duplicati back-end url for Jottacloud is in format "jottacloud://folder/subfolder", we transform them to // the Jottacloud REST API (JFS) url format "https://jfs.jottacloud.com/jfs/[username]/[device]/[mountpoint]/[folder]/[subfolder]". // Find out what JFS device to use. - if (options.ContainsKey(JFS_DEVICE_OPTION)) + var device = options.GetValueOrDefault(JFS_DEVICE_OPTION); + if (!string.IsNullOrWhiteSpace(device)) { // Custom device specified. - m_device = options[JFS_DEVICE_OPTION]; + m_device = device; if (string.Equals(m_device, JFS_BUILTIN_DEVICE, StringComparison.OrdinalIgnoreCase)) { m_device_builtin = true; // Device is configured, but value set to the built-in device! @@ -105,10 +115,11 @@ namespace Duplicati.Library.Backend } // Find out what JFS mount point to use on the device. - if (options.ContainsKey(JFS_MOUNT_POINT_OPTION)) + var mountPoint = options.GetValueOrDefault(JFS_MOUNT_POINT_OPTION); + if (!string.IsNullOrWhiteSpace(mountPoint)) { // Custom mount point specified. - m_mountPoint = options[JFS_MOUNT_POINT_OPTION]; + m_mountPoint = mountPoint; // If we are using the built-in device make sure we have picked a mount point that we can use. if (m_device_builtin) @@ -136,10 +147,10 @@ namespace Duplicati.Library.Backend m_mountPoint = JFS_DEFAULT_CUSTOM_MOUNT_POINT; // Set a suitable default mount point for custom (backup) devices. } - string authid = null; - if (options.ContainsKey(AUTHID_OPTION)) - authid = options[AUTHID_OPTION]; - m_oauth = new JottacloudAuthHelper(authid); + var authId = AuthIdOptionsHelper.Parse(options) + .RequireCredentials(TOKEN_URL); + + m_oauth = new JottacloudAuthHelper(authId.AuthId!); // Build URL var u = new Utility.Uri(url); @@ -152,36 +163,32 @@ namespace Duplicati.Library.Backend m_url = m_url_device + "/" + m_mountPoint + "/" + m_path; m_url_upload = JFS_ROOT_UPLOAD + "/" + m_oauth.Username + "/" + m_device + "/" + m_mountPoint + "/" + m_path; // Different hostname, else identical to m_url. - m_threads = int.Parse(options.ContainsKey(JFS_THREADS) ? options[JFS_THREADS] : JFS_DEFAULT_THREADS); + var jfsThreads = options.GetValueOrDefault(JFS_THREADS); + if (string.IsNullOrWhiteSpace(jfsThreads)) + jfsThreads = JFS_DEFAULT_THREADS; - if (!options.TryGetValue(JFS_CHUNKSIZE, out var tmp)) - { - tmp = JFS_DEFAULT_CHUNKSIZE; - } + m_threads = int.Parse(jfsThreads); - var chunksize = Utility.Sizeparser.ParseSize(tmp, "mb"); + var jfsChunksize = options.GetValueOrDefault(JFS_CHUNKSIZE); + if (string.IsNullOrWhiteSpace(jfsChunksize)) + jfsChunksize = JFS_DEFAULT_CHUNKSIZE; + + var chunksize = Sizeparser.ParseSize(jfsChunksize, "mb"); // Chunk size is bound by BinaryReader.ReadBytes(length) where length is an int. if (chunksize > int.MaxValue || chunksize < 1024) - { - throw new ArgumentOutOfRangeException(nameof(chunksize), string.Format("The chunk size cannot be less than {0}, nor larger than {1}", Utility.Utility.FormatSizeString(1024), Utility.Utility.FormatSizeString(int.MaxValue))); - } + throw new ArgumentOutOfRangeException(nameof(chunksize), $"The chunk size cannot be less than {Utility.Utility.FormatSizeString(1024)}, nor larger than {Utility.Utility.FormatSizeString(int.MaxValue)}"); m_chunksize = chunksize; + m_timeouts = TimeoutOptionsHelper.Parse(options); } #region IBackend Members - public string DisplayName - { - get { return Strings.Jottacloud.DisplayName; } - } + public string DisplayName => Strings.Jottacloud.DisplayName; - public string ProtocolKey - { - get { return "jottacloud"; } - } + public string ProtocolKey => "jottacloud"; /// public async IAsyncEnumerable ListAsync([EnumeratorCancellation] CancellationToken cancelToken) @@ -193,14 +200,17 @@ namespace Duplicati.Library.Backend try { // Send request and load XML response. - var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, "", "", false); - var areq = new Utility.AsyncHttpRequest(req); - using (var rs = areq.GetResponseStream()) - doc.Load(rs); + var req = CreateRequest(WebRequestMethods.Http.Get, "", "", false); + var areq = new AsyncHttpRequest(req); + await Utility.Utility.WithTimeout(m_timeouts.ListTimeout, cancelToken, _ => + { + using (var rs = areq.GetResponseStream()) + doc.Load(rs); + }).ConfigureAwait(false); } - catch (System.Net.WebException wex) + catch (WebException wex) { - if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound) + if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) throw new FolderMissingException(wex); throw; } @@ -209,50 +219,61 @@ namespace Duplicati.Library.Backend // We must check for "deleted" attribute, because files/folders which has it is deleted (attribute contains the timestamp of deletion) // so we treat them as non-existent here. var xRoot = doc.DocumentElement; - if (xRoot.Attributes["deleted"] != null) - { + if (xRoot == null) throw new FolderMissingException(); - } - foreach (System.Xml.XmlNode xFolder in xRoot.SelectNodes("folders/folder[not(@deleted)]")) - { - // Subfolders are only listed with name. We can get a timestamp by sending a request for each folder, but that is probably not necessary? - FileEntry fe = new FileEntry(xFolder.Attributes["name"].Value); - fe.IsFolder = true; - yield return fe; - } - foreach (System.Xml.XmlNode xFile in xRoot.SelectNodes("files/file[not(@deleted)]")) - { - var fe = ToFileEntry(xFile); - if (fe != null) + if (xRoot.Attributes["deleted"] != null) + throw new FolderMissingException(); + + var folders = xRoot.SelectNodes("folders/folder[not(@deleted)]"); + if (folders != null) + foreach (System.Xml.XmlNode xFolder in folders) { - yield return fe; + // Subfolders are only listed with name. We can get a timestamp by sending a request for each folder, but that is probably not necessary? + var name = xFolder.Attributes?["name"]?.Value; + if (string.IsNullOrEmpty(name)) + continue; + + yield return new FileEntry(name) { IsFolder = true }; + } + + var files = xRoot.SelectNodes("files/file[not(@deleted)]"); + if (files != null) + foreach (System.Xml.XmlNode xFile in files) + { + var fe = ToFileEntry(xFile); + if (fe != null) + yield return fe; } - } } - public static IFileEntry ToFileEntry(System.Xml.XmlNode xFile) + public static IFileEntry? ToFileEntry(System.Xml.XmlNode? xFile) { - string name = xFile.Attributes["name"].Value; + if (xFile == null) + return null; + + var name = xFile.Attributes?["name"]?.Value; + if (string.IsNullOrWhiteSpace(name)) + return null; + // Normal files have an "currentRevision", which represent the most recent successfully upload // (could also checked that currentRevision/state is "COMPLETED", but should not be necessary). // There might also be a newer "latestRevision" coming from an incomplete or corrupt upload, // but we ignore that here and use the information about the last valid version. - System.Xml.XmlNode xRevision = xFile.SelectSingleNode("currentRevision"); + var xRevision = xFile.SelectSingleNode("currentRevision"); if (xRevision != null) { - System.Xml.XmlNode xState = xRevision.SelectSingleNode("state"); + var xState = xRevision.SelectSingleNode("state"); if (xState != null && xState.InnerText == "COMPLETED") // Think "currentRevision" always is a complete version, but just to be on the safe side.. { - System.Xml.XmlNode xSize = xRevision.SelectSingleNode("size"); + var xSize = xRevision.SelectSingleNode("size"); long size; if (xSize == null || !long.TryParse(xSize.InnerText, out size)) size = -1; - DateTime lastModified; - System.Xml.XmlNode xModified = xRevision.SelectSingleNode("modified"); // There is created, modified and updated time stamps, but not last accessed. - if (xModified == null || !DateTime.TryParseExact(xModified.InnerText, JFS_DATE_FORMAT, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal, out lastModified)) + + var xModified = xRevision.SelectSingleNode("modified"); // There is created, modified and updated time stamps, but not last accessed. + if (xModified == null || !DateTime.TryParseExact(xModified.InnerText, JFS_DATE_FORMAT, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal, out var lastModified)) lastModified = new DateTime(); - FileEntry fe = new FileEntry(name, size, lastModified, lastModified); - return fe; + return new FileEntry(name, size, lastModified, lastModified); } } return null; @@ -261,22 +282,26 @@ namespace Duplicati.Library.Backend /// /// Retrieves info for a single file (used to determine file size for chunking) /// - /// - /// - public IFileEntry Info(string remotename) + /// The remote file name + /// The cancellation token + /// The file entry + public async Task Info(string remotename, CancellationToken cancelToken) { var doc = new System.Xml.XmlDocument(); try { // Send request and load XML response. - var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, remotename, "", false); + var req = CreateRequest(WebRequestMethods.Http.Get, remotename, "", false); var areq = new Utility.AsyncHttpRequest(req); - using (var rs = areq.GetResponseStream()) - doc.Load(rs); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => + { + using (var rs = areq.GetResponseStream()) + doc.Load(rs); + }).ConfigureAwait(false); } - catch (System.Net.WebException wex) + catch (WebException wex) { - if (wex.Response is HttpWebResponse response && response.StatusCode == System.Net.HttpStatusCode.NotFound) + if (wex.Response is HttpWebResponse response && response.StatusCode == HttpStatusCode.NotFound) throw new FileMissingException(wex); throw; } @@ -285,77 +310,74 @@ namespace Duplicati.Library.Backend // We must check for "deleted" attribute, because files/folders which has it is deleted (attribute contains the timestamp of deletion) // so we treat them as non-existent here. var xFile = doc.DocumentElement; - if (xFile.Attributes["deleted"] != null) - { + if (xFile?.Attributes["deleted"] != null) throw new FileMissingException(string.Format("{0}: {1}", LC.L("The requested file does not exist"), remotename)); - } return ToFileEntry(xFile); } public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) + using (var fs = File.OpenRead(filename)) await PutAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public async Task GetAsync(string remotename, string filename, CancellationToken cancelToken) { - using (System.IO.FileStream fs = System.IO.File.Create(filename)) + using (var fs = File.Create(filename)) await GetAsync(remotename, fs, cancelToken).ConfigureAwait(false); } public Task DeleteAsync(string remotename, CancellationToken cancelToken) { - System.Net.HttpWebRequest req = CreateRequest(System.Net.WebRequestMethods.Http.Post, remotename, "rm=true", false); // rm=true means permanent delete, dl=true would be move to trash. - Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req); - using (System.Net.HttpWebResponse resp = (System.Net.HttpWebResponse)areq.GetResponse()) - { } + var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "rm=true", false); // rm=true means permanent delete, dl=true would be move to trash. + var areq = new Utility.AsyncHttpRequest(req); - return Task.CompletedTask; - } - - public IList SupportedCommands - { - get + return Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => { - return new List([ - new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.Jottacloud.AuthidShort, Strings.Jottacloud.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("jottacloud"))), - new CommandLineArgument(JFS_DEVICE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionDeviceShort, Strings.Jottacloud.DescriptionDeviceLong(JFS_MOUNT_POINT_OPTION)), - new CommandLineArgument(JFS_MOUNT_POINT_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionMountPointShort, Strings.Jottacloud.DescriptionMountPointLong(JFS_DEVICE_OPTION)), - new CommandLineArgument(JFS_THREADS, CommandLineArgument.ArgumentType.Integer, Strings.Jottacloud.ThreadsShort, Strings.Jottacloud.ThreadsLong, JFS_DEFAULT_THREADS), - new CommandLineArgument(JFS_CHUNKSIZE, CommandLineArgument.ArgumentType.Size, Strings.Jottacloud.ChunksizeShort, Strings.Jottacloud.ChunksizeLong, JFS_DEFAULT_CHUNKSIZE), - ]); - } + using (var resp = (HttpWebResponse)areq.GetResponse()) + { } + }); } - public string Description - { - get { return Strings.Jottacloud.Description; } - } + public IList SupportedCommands => [ + .. AuthIdOptionsHelper.GetOptions(TOKEN_URL), + new CommandLineArgument(JFS_DEVICE_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionDeviceShort, Strings.Jottacloud.DescriptionDeviceLong(JFS_MOUNT_POINT_OPTION)), + new CommandLineArgument(JFS_MOUNT_POINT_OPTION, CommandLineArgument.ArgumentType.String, Strings.Jottacloud.DescriptionMountPointShort, Strings.Jottacloud.DescriptionMountPointLong(JFS_DEVICE_OPTION)), + new CommandLineArgument(JFS_THREADS, CommandLineArgument.ArgumentType.Integer, Strings.Jottacloud.ThreadsShort, Strings.Jottacloud.ThreadsLong, JFS_DEFAULT_THREADS), + new CommandLineArgument(JFS_CHUNKSIZE, CommandLineArgument.ArgumentType.Size, Strings.Jottacloud.ChunksizeShort, Strings.Jottacloud.ChunksizeLong, JFS_DEFAULT_CHUNKSIZE), + .. TimeoutOptionsHelper.GetOptions(), + ]; + + public string Description => Strings.Jottacloud.Description; public Task TestAsync(CancellationToken cancelToken) => this.TestListAsync(cancelToken); - public Task CreateFolderAsync(CancellationToken cancelToken) + public async Task CreateFolderAsync(CancellationToken cancelToken) { // When using custom (backup) device we must create the device first (if not already exists). if (!m_device_builtin) { - var req = CreateRequest(System.Net.WebRequestMethods.Http.Post, m_url_device, "type=WORKSTATION"); // Hard-coding device type. Must be one of "WORKSTATION", "LAPTOP", "IMAC", "MACBOOK", "IPAD", "ANDROID", "IPHONE" or "WINDOWS_PHONE". - var areq = new Utility.AsyncHttpRequest(req); - using (var resp = (System.Net.HttpWebResponse)areq.GetResponse()) - { } + var req = CreateRequest(WebRequestMethods.Http.Post, m_url_device, "type=WORKSTATION"); // Hard-coding device type. Must be one of "WORKSTATION", "LAPTOP", "IMAC", "MACBOOK", "IPAD", "ANDROID", "IPHONE" or "WINDOWS_PHONE". + var areq = new AsyncHttpRequest(req); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => + { + using (var resp = (HttpWebResponse)areq.GetResponse()) + { } + }).ConfigureAwait(false); } // Create the folder path, and if using custom mount point it will be created as well in the same operation. { - var req = CreateRequest(System.Net.WebRequestMethods.Http.Post, "", "mkDir=true", false); - var areq = new Utility.AsyncHttpRequest(req); - using (var resp = (System.Net.HttpWebResponse)areq.GetResponse()) - { } + var req = CreateRequest(WebRequestMethods.Http.Post, "", "mkDir=true", false); + var areq = new AsyncHttpRequest(req); + await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => + { + using (var resp = (HttpWebResponse)areq.GetResponse()) + { } + }).ConfigureAwait(false); } - return Task.CompletedTask; } #endregion @@ -368,44 +390,49 @@ namespace Duplicati.Library.Backend #endregion - private System.Net.HttpWebRequest CreateRequest(string method, string url, string queryparams) + private HttpWebRequest CreateRequest(string method, string url, string queryparams) { return m_oauth.CreateRequest(url + (string.IsNullOrEmpty(queryparams) || queryparams.Trim().Length == 0 ? "" : "?" + queryparams), method); } - private System.Net.HttpWebRequest CreateRequest(string method, string remotename, string queryparams, bool upload) + private HttpWebRequest CreateRequest(string method, string remotename, string queryparams, bool upload) { var url = (upload ? m_url_upload : m_url) + Library.Utility.Uri.UrlEncode(remotename).Replace("+", "%20"); return CreateRequest(method, url, queryparams); } public Task GetDNSNamesAsync(CancellationToken cancelToken) => Task.FromResult(new string[] { - new Uri(JFS_ROOT).Host, - new Uri(JFS_ROOT_UPLOAD).Host + new System.Uri(JFS_ROOT).Host, + new System.Uri(JFS_ROOT_UPLOAD).Host }); - public async Task GetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task GetAsync(string remotename, Stream stream, CancellationToken cancelToken) { if (m_threads > 1) { - await ParallelGetAsync(remotename, stream, cancelToken).ConfigureAwait(false); + using (var timeoutStream = stream.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout, false)) + await ParallelGetAsync(remotename, stream, cancelToken).ConfigureAwait(false); return; } // Downloading from Jottacloud: Will only succeed if the file has a completed revision, // and if there are multiple versions of the file we will only get the latest completed version, // ignoring any incomplete or corrupt versions. - var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, remotename, "mode=bin", false); - var areq = new Utility.AsyncHttpRequest(req); - using (var s = areq.GetResponseStream()) - await Utility.Utility.CopyStreamAsync(s, stream, true, cancelToken).ConfigureAwait(false); + var req = CreateRequest(WebRequestMethods.Http.Get, remotename, "mode=bin", false); + var areq = new AsyncHttpRequest(req); + using (var s = await Utility.Utility.WithTimeout(m_timeouts.ShortTimeout, cancelToken, _ => areq.GetResponseStream()).ConfigureAwait(false)) + using (var t = s.ObserveReadTimeout(m_timeouts.ReadWriteTimeout)) + await Utility.Utility.CopyStreamAsync(t, stream, true, cancelToken).ConfigureAwait(false); } /// /// Fetches the file in chunks (parallelized) /// - public async Task ParallelGetAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task ParallelGetAsync(string remotename, Stream stream, CancellationToken cancelToken) { - var size = Info(remotename).Size; + var info = await Info(remotename, cancelToken); + if (info == null) + throw new FileMissingException(remotename); + var size = info.Size; var chunks = new Queue>(); // Tuple => Position (from), Position (to) @@ -428,11 +455,11 @@ namespace Duplicati.Library.Backend var item = chunks.Dequeue(); tasks.Enqueue(Task.Run(() => { - var req = CreateRequest(System.Net.WebRequestMethods.Http.Get, remotename, "mode=bin", false); + var req = CreateRequest(WebRequestMethods.Http.Get, remotename, "mode=bin", false); req.AddRange(item.Item1, item.Item2 - 1); - var areq = new Utility.AsyncHttpRequest(req); + var areq = new AsyncHttpRequest(req); using (var s = areq.GetResponseStream()) - using (var reader = new System.IO.BinaryReader(s)) + using (var reader = new BinaryReader(s)) { var length = item.Item2 - item.Item1; return reader.ReadBytes((int)length); @@ -444,7 +471,7 @@ namespace Duplicati.Library.Backend } } - public async Task PutAsync(string remotename, System.IO.Stream stream, CancellationToken cancelToken) + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) { // Some challenges with uploading to Jottacloud: // - Jottacloud supports use of a custom header where we can tell the server the MD5 hash of the file @@ -461,32 +488,34 @@ namespace Duplicati.Library.Backend // request the server to remove the file again and throw an exception. But there is a requirement that // we specify the file size in a custom header. And if the stream is not seek-able we are not able // to use stream.Length, so we are back at square one. - Duplicati.Library.Utility.TempFile tmpFile = null; - var baseStream = stream; - while (baseStream is Duplicati.Library.Utility.OverrideableStream) - baseStream = typeof(Duplicati.Library.Utility.OverrideableStream).GetField("m_basestream", System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic).GetValue(baseStream) as System.IO.Stream; + + // TODO: This unwrapping code is shared with CloudFiles + TempFile? tmpFile = null; + Stream? baseStream = stream; + while (baseStream is OverrideableStream) + baseStream = typeof(OverrideableStream).GetField("m_basestream", System.Reflection.BindingFlags.DeclaredOnly | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)?.GetValue(baseStream) as System.IO.Stream; if (baseStream == null) throw new Exception(string.Format("Unable to unwrap stream from: {0}", stream.GetType())); string md5Hash; if (baseStream.CanSeek) { var originalPosition = baseStream.Position; - using (var md5 = System.Security.Cryptography.MD5.Create()) - md5Hash = Library.Utility.Utility.ByteArrayAsHexString(md5.ComputeHash(baseStream)); + using (var md5 = MD5.Create()) + md5Hash = Utility.Utility.ByteArrayAsHexString(md5.ComputeHash(baseStream)); baseStream.Position = originalPosition; } else { // No seeking possible, use a temp file - tmpFile = new Duplicati.Library.Utility.TempFile(); - using (var os = System.IO.File.OpenWrite(tmpFile)) + tmpFile = new TempFile(); + using (var os = File.OpenWrite(tmpFile)) using (var hasher = MD5.Create()) - using (var md5 = new Utility.HashCalculatingStream(baseStream, hasher)) + using (var md5 = new HashCalculatingStream(baseStream, hasher)) { await Utility.Utility.CopyStreamAsync(md5, os, true, cancelToken).ConfigureAwait(false); md5Hash = md5.GetFinalHashString(); } - stream = System.IO.File.OpenRead(tmpFile); + stream = File.OpenRead(tmpFile); } try { @@ -497,7 +526,7 @@ namespace Duplicati.Library.Backend // HTTP 404 (Not Found) if file does not exists or it exists with a different hash, in which // case we must send a new request to upload the new content. var fileSize = stream.Length; - var req = CreateRequest(System.Net.WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true); + var req = CreateRequest(WebRequestMethods.Http.Post, remotename, "umode=nomultipart", true); req.Headers.Add("JMd5", md5Hash); // Not required, but it will make the server verify the content and mark the file as corrupt if there is a mismatch. req.Headers.Add("JSize", fileSize.ToString()); // Required, and used to mark file as incomplete if we upload something be the total size of the original file! // File time stamp headers: Since we are working with a stream here we do not know the local file's timestamps, @@ -508,14 +537,15 @@ namespace Duplicati.Library.Backend req.ContentLength = fileSize; // Write post data request - var areq = new Utility.AsyncHttpRequest(req); + var areq = new AsyncHttpRequest(req); using (var rs = areq.GetRequestStream()) + using (var ts = rs.ObserveWriteTimeout(m_timeouts.ReadWriteTimeout)) await Utility.Utility.CopyStreamAsync(stream, rs, true, cancelToken).ConfigureAwait(false); // Send request, and check response - using (var resp = (System.Net.HttpWebResponse)areq.GetResponse()) + using (var resp = (HttpWebResponse)areq.GetResponse()) { - if (resp.StatusCode != System.Net.HttpStatusCode.Created) - throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, null, System.Net.WebExceptionStatus.ProtocolError, resp); + if (resp.StatusCode != HttpStatusCode.Created) + throw new WebException(Strings.Jottacloud.FileUploadError, null, WebExceptionStatus.ProtocolError, resp); // Request seems to be successful, but we must verify the response XML content to be sure that the file // was correctly uploaded: The server will verify the JSize header and mark the file as incomplete if @@ -533,7 +563,7 @@ namespace Duplicati.Library.Backend try { doc.Load(rs); } catch (System.Xml.XmlException) { - throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, System.Net.WebExceptionStatus.ProtocolError); + throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); } bool uploadCompletedSuccessfully = false; var xFile = doc["file"]; @@ -548,7 +578,7 @@ namespace Duplicati.Library.Backend } } if (!uploadCompletedSuccessfully) // Report error (and we just let the incomplete/corrupt file revision stay on the server..) - throw new System.Net.WebException(Strings.Jottacloud.FileUploadError, System.Net.WebExceptionStatus.ProtocolError); + throw new WebException(Strings.Jottacloud.FileUploadError, WebExceptionStatus.ProtocolError); } } } diff --git a/Duplicati/Library/Backend/Jottacloud/Strings.cs b/Duplicati/Library/Backend/Jottacloud/Strings.cs index 876f8d0f0..163519c79 100644 --- a/Duplicati/Library/Backend/Jottacloud/Strings.cs +++ b/Duplicati/Library/Backend/Jottacloud/Strings.cs @@ -19,13 +19,12 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using Duplicati.Library.Localization.Short; -namespace Duplicati.Library.Backend.Strings { +namespace Duplicati.Library.Backend.Strings +{ internal static class Jottacloud { public static string Description { get { return LC.L(@"This backend can read and write data to Jottacloud using its REST protocol. Allowed format is ""jottacloud://folder/subfolder""."); } } public static string DisplayName { get { return LC.L(@"Jottacloud"); } } - public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); } - public static string AuthidShort { get { return LC.L(@"The authorization code"); } } public static string NoUsernameError { get { return LC.L(@"No username found"); } } public static string NoPathError { get { return LC.L(@"No path given. Files cannot be uploaded to the root folder"); } } public static string IllegalMountPoint { get { return LC.L(@"Illegal mount point given."); } } diff --git a/Duplicati/Library/Backend/S3/S3AwsClient.cs b/Duplicati/Library/Backend/S3/S3AwsClient.cs index 9b7aa21ff..b044c732d 100644 --- a/Duplicati/Library/Backend/S3/S3AwsClient.cs +++ b/Duplicati/Library/Backend/S3/S3AwsClient.cs @@ -151,7 +151,7 @@ namespace Duplicati.Library.Backend public virtual async Task AddFileStreamAsync(string bucketName, string keyName, Stream source, CancellationToken cancelToken) { - using var ts = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout); + using var ts = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false); var objectAddRequest = new PutObjectRequest { BucketName = bucketName, diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 6db810f3c..1b452ec4d 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -199,7 +199,7 @@ namespace Duplicati.Library.Backend if (!auth.HasUsername) throw new UserInformationException(Strings.S3Backend.NoAMZUserIDError, "S3NoAmzUserID"); - if (auth.HasPassword) + if (!auth.HasPassword) throw new UserInformationException(Strings.S3Backend.NoAMZKeyError, "S3NoAmzKey"); var useSSL = Utility.Utility.ParseBoolOption(options, SSL_OPTION); diff --git a/Duplicati/Library/Backend/S3/S3MinioClient.cs b/Duplicati/Library/Backend/S3/S3MinioClient.cs index 3c1fbb209..086704209 100644 --- a/Duplicati/Library/Backend/S3/S3MinioClient.cs +++ b/Duplicati/Library/Backend/S3/S3MinioClient.cs @@ -204,7 +204,7 @@ namespace Duplicati.Library.Backend try { - using var t = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout); + using var t = source.ObserveReadTimeout(m_timeouts.ReadWriteTimeout, false); await m_client.PutObjectAsync(bucketName, keyName, t,