diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index cceec10bd..0e3a38c73 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -294,7 +294,14 @@ namespace Duplicati.Library.Main { try { - await backendManager.DeleteAsync(list[i].File.Name, list[i].File.Size, true, result.TaskControl.ProgressToken).ConfigureAwait(false); + if (m_options.Dryrun) + { + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "Would delete file: {0}", list[i].File.Name); + } + else + { + await backendManager.DeleteAsync(list[i].File.Name, list[i].File.Size, true, result.TaskControl.ProgressToken).ConfigureAwait(false); + } } catch (Exception ex) { diff --git a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs index bcee2e525..17f043712 100644 --- a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs @@ -34,6 +34,8 @@ namespace Duplicati.Library.Main.Operation.Backup /// internal static class DataBlockProcessor { + private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(DataBlockProcessor)); + public static Task Run(Channels channels, BackupDatabase database, IBackendManager backendManager, Options options, ITaskReader taskreader) { return AutomationExtensions.RunTask( @@ -142,8 +144,15 @@ namespace Duplicati.Library.Main.Operation.Backup blockvolume = null; indexvolume = null; - await database.CommitTransactionAsync("CommitAddBlockToOutputFlush", true, taskreader.ProgressToken).ConfigureAwait(false); - await backendManager.PutAsync(blockVolumeCopy, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); + if (options.Dryrun) + { + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadFile", "Would upload file: {0}", blockVolumeCopy.RemoteFilename); + } + else + { + await database.CommitTransactionAsync("CommitAddBlockToOutputFlush", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(blockVolumeCopy, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); + } } diff --git a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs index 0b1008d23..b7c85afa6 100644 --- a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs +++ b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs @@ -47,8 +47,16 @@ internal static class RecreateMissingIndexFiles return; await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false); - await database.CommitTransactionAsync("RecreateMissingIndexFile", true, taskreader.ProgressToken).ConfigureAwait(false); - await backendManager.PutAsync(w, null, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); + + if (options.Dryrun) + { + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadFile", "Would upload file: {0}", w.RemoteFilename); + } + else + { + await database.CommitTransactionAsync("RecreateMissingIndexFile", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(w, null, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); + } } } } diff --git a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs index 6a906da16..e5b2bf1f4 100644 --- a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs @@ -172,8 +172,16 @@ namespace Duplicati.Library.Main.Operation.Backup await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, target.BlockVolume.VolumeID, taskreader.ProgressToken).ConfigureAwait(false); } - await database.CommitTransactionAsync("UploadSpillVolume", true, taskreader.ProgressToken).ConfigureAwait(false); - await backendManager.PutAsync(target.BlockVolume, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); + + if (options.Dryrun) + { + Logging.Log.WriteDryrunMessage(Logging.Log.LogTagFromType(typeof(SpillCollectorProcess)), "WouldUploadFile", "Would upload file: {0}", target.BlockVolume.RemoteFilename); + } + else + { + await database.CommitTransactionAsync("UploadSpillVolume", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(target.BlockVolume, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); + } } } } diff --git a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs index 224eeb83b..0c3d80865 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs @@ -67,9 +67,16 @@ internal static class UploadRealFilelist await taskreader.ProgressRendevouz().ConfigureAwait(false); await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false); - await db.CommitTransactionAsync("CommitUpdateRemoteVolume", true, taskreader.ProgressToken).ConfigureAwait(false); - await backendManager.PutAsync(filesetvolume, null, null, false, () => db.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); + if (options.Dryrun) + { + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadFile", "Would upload file: {0}", filesetvolume.RemoteFilename); + } + else + { + await db.CommitTransactionAsync("CommitUpdateRemoteVolume", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(filesetvolume, null, null, false, () => db.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false); + } } } else diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs index 3817f9522..012255090 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs @@ -116,8 +116,16 @@ namespace Duplicati.Library.Main.Operation.Backup // Because it is registered as "Deleting", it will be removed from remote storage by the cleanup process if it exists if (!string.IsNullOrWhiteSpace(lastTempFilelist.Name) && (lastTempFilelist.State == RemoteVolumeState.Uploading || lastTempFilelist.State == RemoteVolumeState.Temporary)) await database.UpdateRemoteVolumeAsync(lastTempFilelist.Name, RemoteVolumeState.Deleting, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false); - await database.CommitTransactionAsync("CommitUpdateFilelistVolume", true, taskreader.ProgressToken).ConfigureAwait(false); - await backendManager.PutAsync(fsw, null, null, true, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken); + + if (options.Dryrun) + { + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadFile", "Would upload file: {0}", fsw.RemoteFilename); + } + else + { + await database.CommitTransactionAsync("CommitUpdateFilelistVolume", true, taskreader.ProgressToken).ConfigureAwait(false); + await backendManager.PutAsync(fsw, null, null, true, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken); + } } catch { diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 51bfb23b6..8d2806c50 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -377,7 +377,7 @@ namespace Duplicati.Library.Main.Operation { if (options.NoBackendverification) { - await FilelistProcessor.VerifyLocalList(backendManager, database, result.TaskControl.ProgressToken).ConfigureAwait(false); + await FilelistProcessor.VerifyLocalList(backendManager, database, options.Dryrun, result.TaskControl.ProgressToken).ConfigureAwait(false); await UpdateStorageStatsFromDatabase(result, database, options, backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false); } else @@ -389,7 +389,15 @@ namespace Duplicati.Library.Main.Operation // The last temporary filelist was emptied, and scheduled for deletion, so we need to delete it try { - await backendManager.DeleteAsync(updatedLastTemp.Name, updatedLastTemp.Size, true, result.TaskControl.ProgressToken).ConfigureAwait(false); + if (options.Dryrun) + { + Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "Would delete file: {0}", updatedLastTemp.Name); + } + else + { + Log.WriteInformationMessage(LOGTAG, "DeletingFile", "Deleting file: {0}", updatedLastTemp.Name); + await backendManager.DeleteAsync(updatedLastTemp.Name, updatedLastTemp.Size, true, result.TaskControl.ProgressToken).ConfigureAwait(false); + } } catch (Exception ex) { diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 25e8fdfef..9dd1e4600 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -67,9 +67,10 @@ namespace Duplicati.Library.Main.Operation /// /// The backend manager instance to use. /// The database to compare with. + /// If true, no changes will be made. /// The cancellation token to use. /// A task that completes when the verification is done. - public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken) + public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, bool dryrun, CancellationToken cancellationToken) { var locallist = database.GetRemoteVolumes(cancellationToken); await foreach (var i in locallist.ConfigureAwait(false)) @@ -84,10 +85,17 @@ namespace Duplicati.Library.Main.Operation case RemoteVolumeState.Temporary: case RemoteVolumeState.Deleting: case RemoteVolumeState.Uploading: - Logging.Log.WriteInformationMessage(LOGTAG, "RemovingStaleFile", "Removing remote file listed as {0}: {1}", i.State, i.Name); try { - await backendManager.DeleteAsync(i.Name, i.Size, true, cancellationToken).ConfigureAwait(false); + if (dryrun) + { + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "Would delete file: {0}", i.Name); + } + else + { + Logging.Log.WriteInformationMessage(LOGTAG, "RemovingStaleFile", "Removing remote file listed as {0}: {1}", i.State, i.Name); + await backendManager.DeleteAsync(i.Name, i.Size, true, cancellationToken).ConfigureAwait(false); + } } catch (Exception ex) { diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 11650c4a1..819e6f1ff 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -98,7 +98,7 @@ namespace Duplicati.Library.Main.Operation .ConfigureAwait(false); if (m_options.NoBackendverification) - await FilelistProcessor.VerifyLocalList(backendManager, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false); + await FilelistProcessor.VerifyLocalList(backendManager, db, m_options.Dryrun, m_result.TaskControl.ProgressToken).ConfigureAwait(false); else await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false); } diff --git a/Duplicati/UnitTest/DryRunTests.cs b/Duplicati/UnitTest/DryRunTests.cs new file mode 100644 index 000000000..8acc7698d --- /dev/null +++ b/Duplicati/UnitTest/DryRunTests.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.Library.Main; +using NUnit.Framework; +using Assert = NUnit.Framework.Legacy.ClassicAssert; + +namespace Duplicati.UnitTest +{ + public class DryRunTests : BasicSetupHelper + { + [Test] + public void TestBackupDryRunDoesNotUpload() + { + var sourceFolder = Path.Combine(DATAFOLDER, "source"); + var backendFolder = Path.Combine(TARGETFOLDER, "backend"); + + Directory.CreateDirectory(sourceFolder); + Directory.CreateDirectory(backendFolder); + + // Create some test files + File.WriteAllText(Path.Combine(sourceFolder, "test.txt"), "test content"); + + var options = new Dictionary(this.TestOptions) + { + ["dry-run"] = "true" + }; + + using (var controller = new Controller($"file://{backendFolder}", options, null)) + controller.Backup(new[] { sourceFolder }); + + // Verify no files were uploaded + var uploadedFiles = Directory.GetFiles(backendFolder); + Assert.AreEqual(0, uploadedFiles.Length, "No files should be uploaded during dry-run"); + } + + [Test] + public void TestDeleteAllRemoteFilesDryRunDoesNotDelete() + { + var sourceFolder = Path.Combine(DATAFOLDER, "source"); + var backendFolder = Path.Combine(TARGETFOLDER, "backend"); + + Directory.CreateDirectory(sourceFolder); + Directory.CreateDirectory(backendFolder); + + // Create some test files and run a backup + File.WriteAllText(Path.Combine(sourceFolder, "test.txt"), "test content"); + + using (var controller = new Controller($"file://{backendFolder}", this.TestOptions, null)) + { + controller.Backup(new[] { sourceFolder }); + } + + // Count files before dry-run delete + var filesBefore = Directory.GetFiles(backendFolder).Length; + Assert.Greater(filesBefore, 0, "Should have files in backend after backup"); + + // Run delete-all-remote-files with dry-run + var options = new Dictionary(this.TestOptions) + { + ["dry-run"] = "true" + }; + + using (var controller = new Controller($"file://{backendFolder}", options, null)) + { + controller.DeleteAllRemoteFiles(); + } + + // Verify files still exist + var filesAfter = Directory.GetFiles(backendFolder).Length; + Assert.AreEqual(filesBefore, filesAfter, "Files should not be deleted during dry-run"); + } + + [Test] + public void TestVerifyLocalListDryRunDoesNotDelete() + { + var sourceFolder = Path.Combine(DATAFOLDER, "source"); + var backendFolder = Path.Combine(TARGETFOLDER, "backend"); + + Directory.CreateDirectory(sourceFolder); + Directory.CreateDirectory(backendFolder); + + // Create some test files and run a backup + File.WriteAllText(Path.Combine(sourceFolder, "test.txt"), "test content"); + + using (var controller = new Controller($"file://{backendFolder}", this.TestOptions, null)) + { + controller.Backup(new[] { sourceFolder }); + } + + var filesBefore = Directory.GetFiles(backendFolder).Length; + Assert.Greater(filesBefore, 0); + + // Manually mark some files as "Uploading" in the database + using (var connection = new Microsoft.Data.Sqlite.SqliteConnection($"Data Source={DBFILE}")) + { + connection.Open(); + using (var command = connection.CreateCommand()) + { + command.CommandText = "UPDATE RemoteVolume SET State = \"Uploading\" WHERE Type = \"Files\""; + command.ExecuteNonQuery(); + } + } + + // Run a backup with dry-run (which triggers VerifyLocalList) + var options = new Dictionary(this.TestOptions) + { + ["dry-run"] = "true" + }; + + using (var controller = new Controller($"file://{backendFolder}", options, null)) + { + controller.Backup(new[] { sourceFolder }); + } + + // Verify that the files were not deleted + var filesAfter = Directory.GetFiles(backendFolder).Length; + Assert.AreEqual(filesBefore, filesAfter, "Files should not be deleted during dry-run"); + } + + [Test] + public void TestInterruptedBackupRecoveryDryRunDoesNotUpload() + { + var sourceFolder = Path.Combine(DATAFOLDER, "source"); + var backendFolder = Path.Combine(TARGETFOLDER, "backend"); + + Directory.CreateDirectory(sourceFolder); + Directory.CreateDirectory(backendFolder); + + // Create some large files to ensure backup takes some time + var data = new byte[10 * 1024 * 1024]; // 10MB + new Random().NextBytes(data); + File.WriteAllBytes(Path.Combine(sourceFolder, "test.bin"), data); + + // Start a backup and interrupt it + using (var controller = new Controller($"file://{backendFolder}", this.TestOptions, null)) + { + var backupTask = Task.Run(() => controller.Backup(new[] { sourceFolder })); + Thread.Sleep(2000); // Wait 2 seconds + controller.Stop(); + + try + { + backupTask.Wait(); + } + catch (AggregateException) + { + // Expected + } + } + + // Now run recovery with dry-run + var options = new Dictionary(this.TestOptions) + { + ["dry-run"] = "true" + }; + + var filesBefore = Directory.GetFiles(backendFolder).Length; + + using (var controller = new Controller($"file://{backendFolder}", options, null)) + { + controller.Backup(new[] { sourceFolder }); + } + + // Verify no new files were uploaded + var filesAfter = Directory.GetFiles(backendFolder).Length; + Assert.AreEqual(filesBefore, filesAfter, "No files should be uploaded during dry-run recovery"); + } + } +}