using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Utility; namespace RemoteSynchronization { public class LightWeightBackendManager(string backendUrl, Dictionary options, int maxRetries = 3, int retryDelay = 1000, bool autoCreateFolders = false, bool retryWithExponentialBackoff = false) : IDisposable { private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(); public IBackend? _backend = null; private bool _anyDownloaded = false; private bool _anyUploaded = false; private readonly string _backendUrl = backendUrl; private int _instantiations = 0; private readonly int _maxRetries = maxRetries; private Dictionary _options = options; private int _retryDelay = retryDelay; private IStreamingBackend? _streamingBackend = null; public Task DeleteAsync(string remotename, CancellationToken token) { return RetryWithDelay( $"Delete {remotename}", () => _streamingBackend.DeleteAsync(remotename, token), null, false, token ); } public string DisplayName { get { Instantiate(); return _streamingBackend.DisplayName; } } public void Dispose() { try { _streamingBackend?.Dispose(); _streamingBackend = null; _backend?.Dispose(); _backend = null; } catch (Exception ex) { Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", ex, "Error during Dispose", null); } } public Task GetAsync(string remotename, Stream stream, CancellationToken token) { return RetryWithDelay( $"Get {remotename}", async () => { await _streamingBackend.GetAsync(remotename, stream, token).ConfigureAwait(false); _anyDownloaded = true; }, stream, true, token ); } private void Instantiate() { if (_backend != null) { // If we already have a backend, we can just return. return; } if (_instantiations < _maxRetries) { _backend = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(_backendUrl, _options); _streamingBackend = _backend as IStreamingBackend ?? throw new InvalidOperationException("Backend does not support streaming operations."); _instantiations++; } else { throw new InvalidOperationException("Maximum number of backend instantiations reached."); } } public IAsyncEnumerable ListAsync(CancellationToken token) { while (true) { Instantiate(); try { return _streamingBackend.ListAsync(token); } catch (Exception ex) { Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", ex, "Error during operation: List", null); Dispose(); // Dispose current backend and streaming backend TryRecoverFromException(ex, token).Await(); } } } public Task PutAsync(string remotename, Stream stream, CancellationToken token) { return RetryWithDelay( $"Put {remotename}", async () => { await _streamingBackend.PutAsync(remotename, stream, token).ConfigureAwait(false); _anyUploaded = true; }, stream, false, token ); } public Task RenameAsync(string oldname, string newname, CancellationToken token) { Instantiate(); return _backend switch { IStreamingBackend sb => RetryWithDelay( $"Rename {oldname} to {newname}", async () => { // Download the file, rename it, and delete the old one using var downloaded = new MemoryStream(); await sb.GetAsync(oldname, downloaded, token).ConfigureAwait(false); await sb.PutAsync(newname, downloaded, token).ConfigureAwait(false); await sb.DeleteAsync(oldname, token).ConfigureAwait(false); _anyUploaded = true; _anyDownloaded = true; }, null, false, token ), IRenameEnabledBackend ireb => RetryWithDelay( $"Rename {oldname} to {newname}", async () => { await ireb.RenameAsync(oldname, newname, token).ConfigureAwait(false); _anyUploaded = true; _anyDownloaded = true; }, null, false, token ), _ => throw new InvalidOperationException("Backend does not support renaming."), }; } private async Task RetryWithDelay(string operationName, Func action, Stream? stream, bool resetStream, CancellationToken token) { while (true) { // This will throw an exception if we've reached the max // number of retries. Instantiate(); try { await action(); return; // Exit the loop if the action succeeds. } catch (Exception ex) { Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", ex, "Error during operation: {0}", operationName); Dispose(); // Dispose current backend and streaming backend. // Reset the stream, as it's in a potentially faulty state. stream?.Seek(0, SeekOrigin.Begin); if (resetStream) stream?.SetLength(0); // Try to see if we can recover from the error. await TryRecoverFromException(ex, token).ConfigureAwait(false); } } } private async Task TryCreateFolder(CancellationToken token) { try { Instantiate(); // Attempt to create the folder await _backend.CreateFolderAsync(token).ConfigureAwait(false); return true; // Folder creation succeeded } catch (Exception ex) { Dispose(); Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", ex, "Failed to create folder", null); return false; // Folder creation failed } } private async Task TryRecoverFromException(Exception ex, CancellationToken token) { // Copied from Duplicati.Library.Main.Backend.BackendManager.Handler. // Refresh DNS name if we fail to connect in order to prevent issues with incorrect DNS entries. var dnsFailure = ExceptionExtensions.FlattenException(ex) .Any(x => (x is System.Net.WebException wex && wex.Status == System.Net.WebExceptionStatus.NameResolutionFailure) || (x is System.Net.Sockets.SocketException sockEx && sockEx.SocketErrorCode == System.Net.Sockets.SocketError.HostNotFound) ); if (dnsFailure) { try { Instantiate(); foreach (var name in await _backend.GetDNSNamesAsync(token).ConfigureAwait(false) ?? []) if (!string.IsNullOrWhiteSpace(name)) System.Net.Dns.GetHostEntry(name); } catch { } } var recovered = false; // Check if this was a folder missing exception and we are allowed to autocreate folders if (!(_anyDownloaded || _anyUploaded) && autoCreateFolders && ExceptionExtensions.FlattenException(ex).Any(x => x is FolderMissingException)) { if (await TryCreateFolder(token).ConfigureAwait(false)) recovered = true; } // Finally, if we did not recover, wait the specified delay before retrying. if (!recovered && _retryDelay > 0) { if (retryWithExponentialBackoff) _retryDelay <<= 1; // Double the delay for exponential backoff await Task.Delay(_retryDelay, token).ConfigureAwait(false); } } } }