using System; using System.Collections.Generic; using System.Linq; using Duplicati.Library.Logging; using Duplicati.Library.Main.Database; namespace Duplicati.Library.Main.Backend; #nullable enable partial class BackendManager { /// /// A class to collect operations performed on the remote destination. /// This class is used to log operations so they can later be written to the database. /// The goal is to keep the database state as close as possible to the remote operation state. /// Since the backend communication is done in parallel with the actual operations, /// we store the performed operations in a queue and flush them to the database when the database is available. /// The main operations are responsible for flushing the messages when comitting a transaction. /// If the operation fails, the logged messages here should still be flushed to the database, /// as they have already been performed on the remote destination. /// private class DatabaseCollector { /// /// The log tag for this class /// private static readonly string LOGTAG = Log.LogTagFromType(); /// /// The lock object for the database queue /// private readonly object m_dbqueuelock = new object(); /// /// The queue of database operations /// private List m_dbqueue = []; /// /// Interface for database entries /// private interface IRemoteOperationEntry { } /// /// Logs an operation performed on the remote destination /// /// The performed action /// The file the operation is performed on /// The result of the operation private sealed record RemoteOperationLogEntry(string Action, string File, string? Result) : IRemoteOperationEntry; /// /// Logs a database update after a remote file operation /// /// The remote name of the volume /// The new state of the volume /// The new size of the volume /// The new hash of the volume private sealed record RemoteVolumeUpdate(string Remotename, RemoteVolumeState State, long Size, string? Hash) : IRemoteOperationEntry; /// /// Logs a rename of a remote file /// /// The old name of the file /// The new name of the file private sealed record RenameRemoteVolume(string Oldname, string Newname) : IRemoteOperationEntry; /// /// Logs an operation performed on the remote destination /// /// The action of the operation /// The file of the operation /// The result of the operation public void LogRemoteOperation(string action, string file, string? result) { lock (m_dbqueuelock) m_dbqueue.Add(new RemoteOperationLogEntry(action, file, result)); } /// /// Logs a database update after a remote file operation /// /// The remote name of the volume /// The new state of the volume /// The new size of the volume /// The new hash of the volume public void LogRemoteVolumeUpdated(string remotename, RemoteVolumeState state, long size, string? hash) { lock (m_dbqueuelock) m_dbqueue.Add(new RemoteVolumeUpdate(remotename, state, size, hash)); } /// /// Logs a remote file rename /// /// The old name of the file /// The new name of the file public void LogRemoteVolumeRenamed(string oldname, string newname) { lock (m_dbqueuelock) m_dbqueue.Add(new RenameRemoteVolume(oldname, newname)); } /// /// Drops all pending messages from the queue /// public void ClearPendingMessages() { lock (m_dbqueuelock) m_dbqueue = []; } /// /// Flushes all messages to the database /// /// The database to write pending messages to /// The transaction to use, if any /// Whether any messages were flushed public bool FlushPendingMessages(LocalDatabase db, System.Data.IDbTransaction? transaction) { List entries; lock (m_dbqueuelock) if (m_dbqueue.Count == 0) return false; else { entries = m_dbqueue; m_dbqueue = []; } // Collect removed volumes for final db cleanup. var volsRemoved = new HashSet(); // As we replace the list, we can now freely access the elements without locking foreach (var e in entries) if (e is RemoteOperationLogEntry operation) db.LogRemoteOperation(operation.Action, operation.File, operation.Result, transaction); else if (e is RemoteVolumeUpdate update && update.State == RemoteVolumeState.Deleted) { db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), transaction); volsRemoved.Add(update.Remotename); } else if (e is RemoteVolumeUpdate dbUpdate) db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash, transaction); else if (e is RenameRemoteVolume rename) db.RenameRemoteFile(rename.Oldname, rename.Newname, transaction); else if (e != null) Log.WriteErrorMessage(LOGTAG, "InvalidQueueElement", null, "Queue had element of type: {0}, {1}", e.GetType(), e); // Finally remove volumes from DB. if (volsRemoved.Count > 0) db.RemoveRemoteVolumes(volsRemoved); return true; } /// /// Flushes all messages to the log after stopping the processing /// public void FlushMessagesToLog() { if (m_dbqueue.Count == 0) return; string message; lock (m_dbqueuelock) message = string.Join("\n", m_dbqueue.Select(e => e switch { RemoteOperationLogEntry operation => $"Operation: {operation.Action} File: {operation.File} Result: {operation.Result}", RemoteVolumeUpdate update => $"Update: {update.Remotename} State: {update.State} Size: {update.Size} Hash: {update.Hash}", RenameRemoteVolume rename => $"Rename: {rename.Oldname} -> {rename.Newname}", _ => $"InvalidQueueElement: {e.GetType()} {e}" })); Log.WriteWarningMessage(LOGTAG, "FlushingMessagesToLog", null, message); } } }