using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using Duplicati.Library.Interface; using Duplicati.Library.Utility; namespace RemoteSynchronization { /// /// A lightweight backend manager that handles remote synchronization operations. /// This class is designed to manage backend operations such as Get, Put, Delete, Rename, and List asynchronously. /// It supports retrying operations with a specified delay and can automatically create folders if they do not exist. /// The class implements IDisposable to ensure proper resource management. /// It uses a streaming backend for efficient file operations and handles exceptions gracefully to allow for retries and recovery. /// A retry incurs a new instantiation of the backend. /// It is designed to mimic the behavior of Duplicati.Library.Main.Backend.BackendManager. /// /// The backend URL string. /// A dictionary of options to pass to the backend. /// The maximum number of retries for failed operations. /// The delay between retries, in milliseconds. /// Whether to automatically create folders if they do not exist. /// Whether to use exponential backoff for retries. public class LightWeightBackendManager(string backendUrl, Dictionary options, int maxRetries = 3, int retryDelay = 1000, bool autoCreateFolders = false, bool retryWithExponentialBackoff = false) : IDisposable { private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(); public IBackend? _backend = null; private bool _anyDownloaded = false; private bool _anyUploaded = false; private readonly string _backendUrl = backendUrl; //private int _instantiations = 0; private readonly int _maxRetries = maxRetries; private readonly Dictionary _options = options; private int _retryDelay = retryDelay; private int _currentRetryDelay = retryDelay; private IStreamingBackend? _streamingBackend = null; /// /// Deletes a file from the remote backend. /// /// The name of the remote file to delete. /// A cancellation token to cancel the operation. /// A task representing the asynchronous delete operation. public Task DeleteAsync(string remotename, CancellationToken token) { return RetryWithDelay( $"Delete {remotename}", () => _streamingBackend!.DeleteAsync(remotename, token), null, false, token ); } /// /// Gets the display name of the backend. /// This property initializes the backend if it has not been instantiated yet. /// /// The display name of the backend. public string DisplayName { get { Instantiate(); return _streamingBackend!.DisplayName; } } /// /// Disposes of the backend and streaming backend resources. /// public void Dispose() { try { _streamingBackend?.Dispose(); _streamingBackend = null; _backend?.Dispose(); _backend = null; } catch (Exception ex) { Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", ex, "Error during Dispose", null); } } /// /// Gets a file from the remote backend and writes it to the specified stream. /// This method retries the operation with a delay if it fails. /// /// The name of the remote file to get. /// The stream to write the file to. /// A cancellation token to cancel the operation. /// A task representing the asynchronous get operation. public Task GetAsync(string remotename, Stream stream, CancellationToken token) { return RetryWithDelay( $"Get {remotename}", async () => { await _streamingBackend!.GetAsync(remotename, stream, token).ConfigureAwait(false); _anyDownloaded = true; }, stream, true, token ); } /// /// Instantiates the backend if it has not been instantiated yet. /// If the backend is already instantiated, it simply returns. /// If the maximum number of retries has been reached, it throws an InvalidOperationException. /// This method is called internally to ensure that the backend is ready for operations. /// /// If the maximum number of instantiations has been reached. private void Instantiate() { if (_backend != null) { // If we already have a backend, we can just return. return; } _backend = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(_backendUrl, _options); _streamingBackend = _backend as IStreamingBackend; if (_streamingBackend == null || !_streamingBackend.SupportsStreaming) { _backend.Dispose(); _backend = null; throw new InvalidOperationException("Backend does not support streaming operations."); } } /// /// Lists the files in the remote backend asynchronously. /// /// A cancellation token to cancel the operation. /// A list of the file entries on the remote backend. public async Task> ListAsync(CancellationToken token) { // TODO It would be more graceful if this method returned an // IAsyncEnumerable instead, capturing failures along the way, // Followed by retrying / resuming the listing from where it // crashed. Current "workaround" is to build the entire list before // returning it. List entries = []; await RetryWithDelay("List", async () => { entries = await _streamingBackend!.ListAsync(token).ToListAsync().ConfigureAwait(false); }, null, false, token) .ConfigureAwait(false); return entries; } /// /// Puts a file to the remote backend from the specified stream. /// /// The name of the remote file to put. /// The stream containing the file data to put. /// A cancellation token to cancel the operation. /// A task representing the asynchronous put operation. public Task PutAsync(string remotename, Stream stream, CancellationToken token) { return RetryWithDelay( $"Put {remotename}", async () => { await _streamingBackend!.PutAsync(remotename, stream, token).ConfigureAwait(false); _anyUploaded = true; }, stream, false, token ); } /// /// Renames a file in the remote backend. /// If the backend supports renaming, it uses the RenameAsync method. /// If the backend does not support renaming, it downloads the file, renames it, and deletes the old one. /// /// The current name of the remote file. /// The new name for the remote file. /// A cancellation token to cancel the operation. /// A task representing the asynchronous rename operation. public Task RenameAsync(string oldname, string newname, CancellationToken token) { Instantiate(); return _backend switch { IStreamingBackend sb => RetryWithDelay( $"Rename {oldname} to {newname}", async () => { // Download the file, rename it, and delete the old one using var downloaded = new MemoryStream(); await sb.GetAsync(oldname, downloaded, token).ConfigureAwait(false); await sb.PutAsync(newname, downloaded, token).ConfigureAwait(false); await sb.DeleteAsync(oldname, token).ConfigureAwait(false); _anyUploaded = true; _anyDownloaded = true; }, null, false, token ), IRenameEnabledBackend ireb => RetryWithDelay( $"Rename {oldname} to {newname}", async () => { await ireb.RenameAsync(oldname, newname, token).ConfigureAwait(false); _anyUploaded = true; _anyDownloaded = true; }, null, false, token ), _ => throw new InvalidOperationException("Backend does not support renaming."), }; } /// /// Retries an operation with a delay if it fails. /// This method will instantiate the backend if it has not been instantiated yet. /// If the operation fails, it will log the error, dispose of the current backend and streaming backend, /// reset the stream if specified, and attempt to recover from the exception. /// If recovery is not possible, it will wait for the specified retry delay before retrying the operation. /// The retry delay can be increased exponentially if specified. /// /// The name of the operation being retried, used for logging. /// The action to perform. /// The stream to use for the operation. Used when resetting the stream during recovery. /// Whether to reset the stream if the operation fails. /// A cancellation token to cancel the operation. /// A task representing the asynchronous operation. private async Task RetryWithDelay(string operationName, Func action, Stream? stream, bool resetStream, CancellationToken token) { int instantiations = 0; _currentRetryDelay = _retryDelay; // Reset the current retry delay to the initial value do { // This will throw an exception if we've reached the max // number of retries. Instantiate(); instantiations++; try { await action().ConfigureAwait(false); return; // Exit the loop if the action succeeds. } catch (Exception ex) { Duplicati.Library.Logging.Log.WriteErrorMessage(LOGTAG, "rsync", ex, "Error during operation: {0}", operationName); Dispose(); // Dispose current backend and streaming backend. // Reset the stream, as it's in a potentially faulty state. if (stream != null && stream.CanSeek) { stream.Seek(0, SeekOrigin.Begin); if (resetStream) stream.SetLength(0); } // Try to see if we can recover from the error. await TryRecoverFromException(ex, token).ConfigureAwait(false); } } while (instantiations < _maxRetries); // If we reach here, it means all retries failed. throw new InvalidOperationException($"Operation '{operationName}' failed after {instantiations} attempts."); } /// /// Attempts to create a folder in the remote backend. /// /// A cancellation token to cancel the operation. /// A task that represents the asynchronous operation. The task result is true if the folder was created successfully, false otherwise. private async Task TryCreateFolder(CancellationToken token) { bool created = false; await RetryWithDelay("CreateFolder", async () => { try { await _backend!.CreateFolderAsync(token).ConfigureAwait(false); created = true; // Folder creation succeeded } catch { created = false; // Folder creation failed } }, null, false, token); return created; } /// /// Attempts to recover from an exception that occurred during a backend operation. /// This method checks for specific types of exceptions, such as DNS resolution failures or folder missing exceptions. /// If a DNS failure is detected, it attempts to refresh the DNS name by re-instantiating the backend and resolving DNS names. /// If the exception is a folder missing exception and auto-creation of folders is enabled, it attempts to create the folder. /// If recovery is not possible, it waits for the specified retry delay. /// The retry delay will be doubled if exponential backoff is enabled. /// /// The exception that occurred during the operation. /// A cancellation token to cancel the operation. /// A task representing the asynchronous recovery operation. private async Task TryRecoverFromException(Exception ex, CancellationToken token) { // Copied from Duplicati.Library.Main.Backend.BackendManager.Handler. // Refresh DNS name if we fail to connect in order to prevent issues with incorrect DNS entries. var dnsFailure = ExceptionExtensions.FlattenException(ex) .Any(x => (x is System.Net.WebException wex && wex.Status == System.Net.WebExceptionStatus.NameResolutionFailure) || (x is System.Net.Sockets.SocketException sockEx && sockEx.SocketErrorCode == System.Net.Sockets.SocketError.HostNotFound) ); if (dnsFailure) { try { Instantiate(); foreach (var name in await _backend!.GetDNSNamesAsync(token).ConfigureAwait(false) ?? []) if (!string.IsNullOrWhiteSpace(name)) System.Net.Dns.GetHostEntry(name); } catch { } } var recovered = false; // Check if this was a folder missing exception and we are allowed to autocreate folders if (!(_anyDownloaded || _anyUploaded) && autoCreateFolders && ExceptionExtensions.FlattenException(ex).Any(x => x is FolderMissingException)) { if (await TryCreateFolder(token).ConfigureAwait(false)) recovered = true; } // Finally, if we did not recover, wait the specified delay before retrying. if (!recovered && _retryDelay > 0) { await Task.Delay(_currentRetryDelay, token).ConfigureAwait(false); if (retryWithExponentialBackoff) _currentRetryDelay <<= 1; // Double the delay for exponential backoff } } } }