diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index ed7d4bb3d..1ff1b98c0 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -218,25 +218,9 @@ namespace Duplicati.Library.Main.Database get { using (var cmd = m_connection.CreateCommand()) - using (var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""IsFullBackup"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC")) - { - var isFullBackupEncountered = false; + using(var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC")) while (rd.Read()) - { - var id = rd.GetInt64(0); - var isFullBackup = rd.GetInt32(1); - var timeStamp = ParseFromEpochSeconds(rd.GetInt64(2)).ToLocalTime(); - - if (isFullBackupEncountered && isFullBackup != BackupType.FULL_BACKUP) continue; - - yield return new KeyValuePair(id, timeStamp); - - if (!isFullBackupEncountered && isFullBackup == BackupType.FULL_BACKUP) - { - isFullBackupEncountered = true; - } - } - } + yield return new KeyValuePair(rd.GetInt64(0), ParseFromEpochSeconds(rd.GetInt64(1)).ToLocalTime()); } } diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index 988ca343f..6a1bd37c8 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -363,36 +363,14 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { - using (var rd = - cmd.ExecuteReader( - @"SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ") - ) + using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")) { - // partial backup sets are only included until the first full backup is encountered - var isFullBackupEncountered = false; while (rd.Read()) { var id = rd.GetInt64(0); - - if (!dict.ContainsKey(id)) - { - continue; - } - var backupType = rd.GetInt32(1); var e = dict[id]; - - if (isFullBackupEncountered && backupType != BackupType.FULL_BACKUP) - { - continue; - } - yield return new Fileset(e, backupType, m_filesets[e].Value, -1L, -1L); - - if (!isFullBackupEncountered && backupType == BackupType.FULL_BACKUP) - { - isFullBackupEncountered = true; - } } } } @@ -421,13 +399,8 @@ namespace Duplicati.Library.Main.Database while (rd.Read()) { var id = rd.GetInt64(0); - if (!dict.ContainsKey(id)) - { - continue; - } var isFullBackup = rd.GetInt32(1); var e = dict[id]; - var filecount = rd.ConvertValueToInt64(2, -1L); var filesizes = rd.ConvertValueToInt64(3, -1L); diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs index 7df4cac92..307d48aba 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs @@ -111,6 +111,9 @@ namespace Duplicati.Library.Main.Operation.Backup foreach (var p in options.ControlFiles.Split(new char[] {System.IO.Path.PathSeparator}, StringSplitOptions.RemoveEmptyEntries)) fsw.AddControlFile(p, options.GetCompressionHintFromFilename(p)); + // We declare this to be a partial backup since the synthetic filelist is only created + // when a backup is interrupted. + fsw.CreateFilesetFile(false); var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime); await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID); await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime); diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index e3c56561c..e67f450bd 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -204,11 +204,18 @@ namespace Duplicati.Library.Main.Operation } } - // Remove backups that are older than date specified via option + // Remove backups that are older than date specified via option while ensuring + // that we always have at least one full backup. var keepTime = m_options.KeepTime; if (keepTime.Ticks > 0) { - toDelete.AddRange(sortedAllBackups.SkipWhile(x => x >= keepTime)); + bool haveFullBackup = false; + toDelete.AddRange(sortedAllBackups.SkipWhile(x => + { + bool keepBackup = (x >= keepTime) || !haveFullBackup; + haveFullBackup = haveFullBackup || db.IsFilesetFullBackup(x); + return keepBackup; + })); } // Remove backups via retention policy option @@ -218,37 +225,30 @@ namespace Duplicati.Library.Main.Operation // and remove oldest backups while there are still more backups than should be kept as specified via option var backupsRemaining = sortedAllBackups.Except(toDelete).ToList(); var fullVersionsToKeep = m_options.KeepVersions; - var fullVersionsKeptCount = 0; if (fullVersionsToKeep > 0 && fullVersionsToKeep < backupsRemaining.Count) { - // keep the number of full backups specified in fullVersionsToKeep. - // once the last full backup t okeep is found, also keep the partials immediately after it the full backup. - // add the remainder of full and partial backups to toDelete - bool foundLastFullBackupToKeep = false; - foreach (var backup in backupsRemaining) + int fullVersionsKept = 0; + ISet intermediatePartials = new HashSet(); + + // Enumerate the collection starting from the most recent full backup. + foreach (DateTime backup in backupsRemaining.SkipWhile(x => !db.IsFilesetFullBackup(x))) { - bool isFullBackup; - if (fullVersionsKeptCount < fullVersionsToKeep) + if (fullVersionsKept >= fullVersionsToKeep) { - isFullBackup = db.IsFilesetFullBackup(backup); - // count only a full backup - if (fullVersionsKeptCount < fullVersionsToKeep && isFullBackup) - { - fullVersionsKeptCount++; - } - continue; + // If we have enough full backups, delete all older backups. + toDelete.Add(backup); } - // do not include any partial backup that precedes the last full backup - if (!foundLastFullBackupToKeep) + else if (db.IsFilesetFullBackup(backup)) { - isFullBackup = db.IsFilesetFullBackup(backup); - if (!isFullBackup) - { - continue; - } - foundLastFullBackupToKeep = true; + // We can delete partial backups that are surrounded by full backups. + toDelete.AddRange(intermediatePartials); + intermediatePartials.Clear(); + fullVersionsKept++; + } + else + { + intermediatePartials.Add(backup); } - toDelete.Add(backup); } } diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs index dae2ab929..ae0b86dfb 100644 --- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text; using Duplicati.Library.Interface; using Duplicati.Library.Main.Database; +using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Operation @@ -109,7 +110,7 @@ namespace Duplicati.Library.Main.Operation if (filteredList.Count == 0) throw new UserInformationException("No filesets found on remote target", "EmptyRemoteFolder"); - var numberSeq = CreateResultSequence(filteredList); + var numberSeq = CreateResultSequence(filteredList, backend, m_options); if (filter.Empty) { m_result.SetResult(numberSeq, null); @@ -203,9 +204,26 @@ namespace Duplicati.Library.Main.Operation return filelistFilter(parsedlist).ToList(); } - public static IEnumerable CreateResultSequence(IEnumerable> filteredList) + private static IEnumerable CreateResultSequence(IEnumerable> filteredList, BackendManager backendManager, Options options) { - return (from n in filteredList select (Library.Interface.IListResultFileset)(new ListResultFileset(n.Key, BackupType.PARTIAL_BACKUP, n.Value.Time.ToLocalTime(), -1, -1))).ToArray(); + List list = new List(); + foreach (KeyValuePair entry in filteredList) + { + AsyncDownloader downloader = new AsyncDownloader(new IRemoteVolume[] {new RemoteVolume(entry.Value.File)}, backendManager); + foreach (IAsyncDownloadedFile file in downloader) + { + // We must obtain the partial/full status from the fileset file in the dlist files. + // Without this, the restore dialog will show all versions as full, or all versions + // as partial. While the dlist files are already downloaded elsewhere, doing so again + // here is the most direct way to obtain the partial/full status without a major + // refactoring. Since restoring directly from the backend files should be a relatively + // rare event, we can work on improving the performance later. + VolumeBase.FilesetData filesetData = VolumeReaderBase.GetFilesetData(entry.Value.CompressionModule, file.TempFile, options); + list.Add(new ListResultFileset(entry.Key, filesetData.IsFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP, entry.Value.Time.ToLocalTime(), -1, -1)); + } + } + + return list.ToArray(); } } } diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 0b31a3f7a..31800bb22 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -162,6 +162,7 @@ namespace Duplicati.Library.Main.Operation var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal); var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup); vol.VolumeID = newids.Item1; + vol.CreateFilesetFile(isOriginalFilesetFullBackup); Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize)); diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index a875900a1..b189c79ea 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -23,11 +23,11 @@ namespace Duplicati.Library.Main.Operation m_backendurl = backend; m_options = options; m_result = result; - + if (options.AllowPassphraseChange) throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported, "PassphraseChangeUnsupported"); } - + public void Run(Library.Utility.IFilter filter = null) { if (!System.IO.File.Exists(m_options.Dbpath)) @@ -287,19 +287,24 @@ namespace Duplicati.Library.Main.Operation if (n.Type == RemoteVolumeType.Files) { var filesetId = db.GetFilesetIdFromRemotename(n.Name); - var w = new FilesetVolumeWriter(m_options, DateTime.UtcNow); - newEntry = w; - w.SetRemoteFilename(n.Name); - db.WriteFileset(w, filesetId, null); + // We cannot wrap the FilesetVolumeWriter in a using statement here because a reference to it is + // retained in newEntry. + FilesetVolumeWriter volumeWriter = new FilesetVolumeWriter(m_options, DateTime.UtcNow); + newEntry = volumeWriter; + volumeWriter.SetRemoteFilename(n.Name); - w.Close(); + db.WriteFileset(volumeWriter, filesetId, null); + DateTime filesetTime = db.FilesetTimes.First(x => x.Key == filesetId).Value; + volumeWriter.CreateFilesetFile(db.IsFilesetFullBackup(filesetTime)); + + volumeWriter.Close(); if (m_options.Dryrun) - Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); + Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size)); else { - db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); - backend.Put(w); + db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); + backend.Put(volumeWriter); } } else if (n.Type == RemoteVolumeType.Index) diff --git a/Duplicati/UnitTest/BasicSetupHelper.cs b/Duplicati/UnitTest/BasicSetupHelper.cs index 5c5ad992a..57a5ca604 100644 --- a/Duplicati/UnitTest/BasicSetupHelper.cs +++ b/Duplicati/UnitTest/BasicSetupHelper.cs @@ -147,7 +147,6 @@ namespace Duplicati.UnitTest opts["dbpath"] = DBFILE; opts["blocksize"] = "10kb"; opts["backup-test-samples"] = "0"; - opts["keep-versions"] = "100"; opts["unittest-mode"] = "true"; return opts; diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs index 398cd241d..9c4ee58cf 100644 --- a/Duplicati/UnitTest/DisruptionTests.cs +++ b/Duplicati/UnitTest/DisruptionTests.cs @@ -28,12 +28,292 @@ namespace Duplicati.UnitTest } } + private async Task RunPartialBackup(Controller controller) + { + this.ModifySourceFiles(); + + // ReSharper disable once AccessToDisposedClosure + Task backupTask = Task.Run(() => controller.Backup(new[] {this.DATAFOLDER})); + + // Block for a small amount of time to allow the ITaskControl to be associated + // with the Controller. Otherwise, the call to Stop will simply be a no-op. + Thread.Sleep(1000); + + controller.Stop(true); + await backupTask.ConfigureAwait(false); + } + public override void SetUp() { base.SetUp(); this.ModifySourceFiles(); } + [Test] + [Category("Disruption")] + public async Task KeepTimeRetention() + { + // Choose a dblock size that is small enough so that more than one volume is needed. + Dictionary options = new Dictionary(this.TestOptions) {["dblock-size"] = "10mb"}; + + // First, run two complete backups followed by a partial backup. We will then set the keep-time + // option so that the threshold lies between the first and second backups. + DateTime firstBackupTime; + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + c.Backup(new[] {this.DATAFOLDER}); + firstBackupTime = c.List().Filesets.First().Time; + } + + // Wait before the second backup so that we can more easily define the keep-time threshold + // to lie between the first and second backups. + Thread.Sleep(5000); + DateTime secondBackupTime; + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + this.ModifySourceFiles(); + c.Backup(new[] {this.DATAFOLDER}); + secondBackupTime = c.List().Filesets.First().Time; + } + + // Run a partial backup. + DateTime thirdBackupTime; + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + await this.RunPartialBackup(c).ConfigureAwait(false); + thirdBackupTime = c.List().Filesets.First().Time; + } + + // Set the keep-time option so that the threshold lies between the first and second backups + // and run the delete operation. + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + options["keep-time"] = $"{(int) ((DateTime.Now - firstBackupTime).TotalSeconds - (secondBackupTime - firstBackupTime).TotalSeconds / 2)}s"; + c.Delete(); + + List filesets = c.List().Filesets.ToList(); + Assert.AreEqual(2, filesets.Count); + Assert.AreEqual(secondBackupTime, filesets[1].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup); + Assert.AreEqual(thirdBackupTime, filesets[0].Time); + Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup); + } + + // Run another partial backup. We will then verify that a full backup is retained + // even when all the "recent" backups are partial. + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + await this.RunPartialBackup(c).ConfigureAwait(false); + DateTime fourthBackupTime = c.List().Filesets.First().Time; + + // Set the keep-time option so that the threshold lies after the most recent full backup + // and run the delete operation. + options["keep-time"] = "1s"; + c.Delete(); + + List filesets = c.List().Filesets.ToList(); + Assert.AreEqual(3, filesets.Count); + Assert.AreEqual(secondBackupTime, filesets[2].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup); + Assert.AreEqual(thirdBackupTime, filesets[1].Time); + Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[1].IsFullBackup); + Assert.AreEqual(fourthBackupTime, filesets[0].Time); + Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup); + } + } + + [Test] + [Category("Disruption")] + public async Task KeepVersionsRetention() + { + // Choose a dblock size that is small enough so that more than one volume is needed. + Dictionary options = new Dictionary(this.TestOptions) {["dblock-size"] = "10mb"}; + + // Run a full backup. + DateTime firstBackupTime; + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + c.Backup(new[] {this.DATAFOLDER}); + firstBackupTime = c.List().Filesets.First().Time; + } + + // Run a partial backup. + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + await this.RunPartialBackup(c).ConfigureAwait(false); + } + + // Run a partial backup. + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + await this.RunPartialBackup(c).ConfigureAwait(false); + } + + // Run a full backup. + DateTime fourthBackupTime; + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + this.ModifySourceFiles(); + c.Backup(new[] {this.DATAFOLDER}); + fourthBackupTime = c.List().Filesets.First().Time; + } + + // Run a partial backup. + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + options["keep-versions"] = "2"; + await this.RunPartialBackup(c).ConfigureAwait(false); + DateTime fifthBackupTime = c.List().Filesets.First().Time; + + // Partial backups that are followed by a full backup can be deleted. + List filesets = c.List().Filesets.ToList(); + Assert.AreEqual(3, filesets.Count); + Assert.AreEqual(firstBackupTime, filesets[2].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup); + Assert.AreEqual(fourthBackupTime, filesets[1].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup); + Assert.AreEqual(fifthBackupTime, filesets[0].Time); + Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup); + } + + // Run a full backup. + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + this.ModifySourceFiles(); + c.Backup(new[] {this.DATAFOLDER}); + DateTime sixthBackupTime = c.List().Filesets.First().Time; + + // Since the last backup was full, we can now expect to have just the 2 most recent full backups. + List filesets = c.List().Filesets.ToList(); + Assert.AreEqual(2, filesets.Count); + Assert.AreEqual(fourthBackupTime, filesets[1].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup); + Assert.AreEqual(sixthBackupTime, filesets[0].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup); + } + } + + [Test] + [Category("Disruption")] + public async Task RetentionPolicyRetention() + { + Dictionary options = new Dictionary(this.TestOptions) + { + // Choose a dblock size that is small enough so that more than one volume is needed. + ["dblock-size"] = "10mb", + + // This test assumes that we can perform 3 backups within 1 minute. + ["retention-policy"] = "1m:59s,U:1m", + ["no-backend-verification"] = "true" + }; + + DateTime firstBackupTime; + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + c.Backup(new[] {this.DATAFOLDER}); + firstBackupTime = c.List().Filesets.First().Time; + + List filesets = c.List().Filesets.ToList(); + Assert.AreEqual(1, filesets.Count); + + this.ModifySourceFiles(); + c.Backup(new[] {this.DATAFOLDER}); + DateTime secondBackupTime = c.List().Filesets.First().Time; + + // Since the most recent backup is not considered in the retention logic, the only backup in the first time frame + // is the initial one. As a result, we should have 2 backups. + filesets = c.List().Filesets.ToList(); + Assert.AreEqual(2, filesets.Count); + Assert.AreEqual(firstBackupTime, filesets[1].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup); + Assert.AreEqual(secondBackupTime, filesets[0].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup); + } + + // Wait so that the next backups fall in the next retention interval. + Thread.Sleep(new TimeSpan(0, 0, 1, 0)); + + DateTime thirdBackupTime; + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + await this.RunPartialBackup(c).ConfigureAwait(false); + thirdBackupTime = c.List().Filesets.First().Time; + + // Since the most recent backup is not considered in the retention logic, there are no backups in the first time + // frame. The original 2 backups have now spilled over to the U:1m specification. Since we keep the first + // backup in the interval, we should be left with the first backup, as well as the third partial one. + List filesets = c.List().Filesets.ToList(); + Assert.AreEqual(2, filesets.Count); + Assert.AreEqual(firstBackupTime, filesets[1].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup); + Assert.AreEqual(thirdBackupTime, filesets[0].Time); + Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup); + } + + DateTime fourthBackupTime; + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + this.ModifySourceFiles(); + c.Backup(new[] {this.DATAFOLDER}); + fourthBackupTime = c.List().Filesets.First().Time; + + // Since the most recent backup is not considered in the retention logic, the third backup is the only backup + // in the first time frame. There is no further spillover, so we simply add the fourth backup to the + // collection of retained backups. + List filesets = c.List().Filesets.ToList(); + Assert.AreEqual(3, filesets.Count); + Assert.AreEqual(firstBackupTime, filesets[2].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup); + Assert.AreEqual(thirdBackupTime, filesets[1].Time); + Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[1].IsFullBackup); + Assert.AreEqual(fourthBackupTime, filesets[0].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup); + + this.ModifySourceFiles(); + c.Backup(new[] {this.DATAFOLDER}); + DateTime fifthBackupTime = c.List().Filesets.First().Time; + + // Since the most recent backup is not considered in the retention logic, we now have two backups in the + // first time frame: the third (partial) and fourth (full). Since the first backup in each interval is + // kept, we would typically keep just the third backup. However, since we should not discard a full + // backup in favor of a partial one, we keep the fourth as well. We also still have the initial backup. + filesets = c.List().Filesets.ToList(); + Assert.AreEqual(4, filesets.Count); + Assert.AreEqual(firstBackupTime, filesets[3].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[3].IsFullBackup); + Assert.AreEqual(thirdBackupTime, filesets[2].Time); + Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[2].IsFullBackup); + Assert.AreEqual(fourthBackupTime, filesets[1].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup); + Assert.AreEqual(fifthBackupTime, filesets[0].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup); + } + + // Wait so that the next backups fall in the next retention interval. + Thread.Sleep(new TimeSpan(0, 0, 1, 0)); + + using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) + { + this.ModifySourceFiles(); + c.Backup(new[] {this.DATAFOLDER}); + DateTime sixthBackupTime = c.List().Filesets.First().Time; + + // Since the most recent backup is not considered in the retention logic, we now have three backups in the + // second time frame: the third (partial), fourth (full), and fifth (full). Since we keep up to the first + // full backup in each time frame, we now drop the fifth backup. + List filesets = c.List().Filesets.ToList(); + Assert.AreEqual(4, filesets.Count); + Assert.AreEqual(firstBackupTime, filesets[3].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[3].IsFullBackup); + Assert.AreEqual(thirdBackupTime, filesets[2].Time); + Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[2].IsFullBackup); + Assert.AreEqual(fourthBackupTime, filesets[1].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup); + Assert.AreEqual(sixthBackupTime, filesets[0].Time); + Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup); + } + } + [Test] [Category("Disruption")] public async Task StopAfterCurrentFile() @@ -49,20 +329,12 @@ namespace Duplicati.UnitTest Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup); } - // Modify the source files and interrupt a backup. - this.ModifySourceFiles(); + // Run a partial backup. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { - // ReSharper disable once AccessToDisposedClosure - Task backupTask = Task.Run(() => c.Backup(new[] {this.DATAFOLDER})); - - // Block for a small amount of time to allow the ITaskControl to be associated - // with the Controller. Otherwise, the call to Stop will simply be a no-op. - Thread.Sleep(1000); + await this.RunPartialBackup(c).ConfigureAwait(false); // If we interrupt the backup, the most recent Fileset should be marked as partial. - c.Stop(true); - await backupTask.ConfigureAwait(false); Assert.AreEqual(2, c.List().Filesets.Count()); Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup); Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup); @@ -94,12 +366,13 @@ namespace Duplicati.UnitTest Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup); } - // Run a complete backup. Listing the Filesets should omit the previous partial backup. + // Run a complete backup. Listing the Filesets should include both full and partial backups. using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null)) { c.Backup(new[] {this.DATAFOLDER}); - Assert.AreEqual(2, c.List().Filesets.Count()); - Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup); + Assert.AreEqual(3, c.List().Filesets.Count()); + Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 2).IsFullBackup); + Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup); Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup); }