From d60248e26a1ab763924e98d41994818f763b0a69 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Sun, 17 Nov 2019 09:30:19 -0800
Subject: [PATCH 01/24] Unhide partial filesets from FilesetTimes property
 getter.

This concerns issue #3982.
---
 .../Library/Main/Database/LocalDatabase.cs    | 20 ++-----------------
 1 file changed, 2 insertions(+), 18 deletions(-)

diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs
index ed7d4bb3d..1ff1b98c0 100644
--- a/Duplicati/Library/Main/Database/LocalDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalDatabase.cs
@@ -218,25 +218,9 @@ namespace Duplicati.Library.Main.Database
             get
             {
                 using (var cmd = m_connection.CreateCommand())
-                using (var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""IsFullBackup"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"))
-                {
-                    var isFullBackupEncountered = false;
+                using(var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"))
                     while (rd.Read())
-                    {
-                        var id = rd.GetInt64(0);
-                        var isFullBackup = rd.GetInt32(1);
-                        var timeStamp = ParseFromEpochSeconds(rd.GetInt64(2)).ToLocalTime();
-
-                        if (isFullBackupEncountered && isFullBackup != BackupType.FULL_BACKUP) continue;
-
-                        yield return new KeyValuePair<long, DateTime>(id, timeStamp);
-
-                        if (!isFullBackupEncountered && isFullBackup == BackupType.FULL_BACKUP)
-                        {
-                            isFullBackupEncountered = true;
-                        }
-                    }
-                }
+                        yield return new KeyValuePair<long, DateTime>(rd.GetInt64(0), ParseFromEpochSeconds(rd.GetInt64(1)).ToLocalTime());
             }
         }
 

From c395e77bc2058702258e8dcd04580e5097ac260e Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Sun, 17 Nov 2019 09:35:42 -0800
Subject: [PATCH 02/24] Unhide partial filesets from QuickSets property getter.

This concerns issue #3982.
---
 .../Main/Database/LocalListDatabase.cs        | 24 +------------------
 1 file changed, 1 insertion(+), 23 deletions(-)

diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs
index 988ca343f..20944536a 100644
--- a/Duplicati/Library/Main/Database/LocalListDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs
@@ -363,36 +363,14 @@ namespace Duplicati.Library.Main.Database
 
                     using (var cmd = m_connection.CreateCommand())
                     {
-                        using (var rd =
-                            cmd.ExecuteReader(
-                                @"SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC ")
-                        )
+                        using (var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""ID"", ""IsFullBackup"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC "))
                         {
-                            // partial backup sets are only included until the first full backup is encountered
-                            var isFullBackupEncountered = false;
                             while (rd.Read())
                             {
                                 var id = rd.GetInt64(0);
-
-                                if (!dict.ContainsKey(id))
-                                {
-                                    continue;
-                                }
-
                                 var backupType = rd.GetInt32(1);
                                 var e = dict[id];
-
-                                if (isFullBackupEncountered && backupType != BackupType.FULL_BACKUP)
-                                {
-                                    continue;
-                                }
-
                                 yield return new Fileset(e, backupType, m_filesets[e].Value, -1L, -1L);
-
-                                if (!isFullBackupEncountered && backupType == BackupType.FULL_BACKUP)
-                                {
-                                    isFullBackupEncountered = true;
-                                }
                             }
                         }
                     }

From 4950e7c2254edc39a1b1fcf4b1bdd82a987854b0 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Sun, 17 Nov 2019 09:39:23 -0800
Subject: [PATCH 03/24] Unhide partial filesets from Sets property getter.

This concerns issue #3982.
---
 Duplicati/Library/Main/Database/LocalListDatabase.cs | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs
index 20944536a..6a1bd37c8 100644
--- a/Duplicati/Library/Main/Database/LocalListDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs
@@ -399,13 +399,8 @@ namespace Duplicati.Library.Main.Database
                             while (rd.Read())
                             {
                                 var id = rd.GetInt64(0);
-                                if (!dict.ContainsKey(id))
-                                {
-                                    continue;
-                                }
                                 var isFullBackup = rd.GetInt32(1);
                                 var e = dict[id];
-
                                 var filecount = rd.ConvertValueToInt64(2, -1L);
                                 var filesizes = rd.ConvertValueToInt64(3, -1L);
 

From 1d42c53319a034e906de5eb8379921f151dcf96b Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Mon, 18 Nov 2019 19:34:25 -0800
Subject: [PATCH 04/24] Fix disruption test to expect partial backups in
 fileset listing.

This concerns issue #3982.
---
 Duplicati/UnitTest/DisruptionTests.cs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs
index 398cd241d..7f404ac62 100644
--- a/Duplicati/UnitTest/DisruptionTests.cs
+++ b/Duplicati/UnitTest/DisruptionTests.cs
@@ -94,12 +94,13 @@ namespace Duplicati.UnitTest
                 Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
             }
 
-            // Run a complete backup.  Listing the Filesets should omit the previous partial backup.
+            // Run a complete backup.  Listing the Filesets should include both full and partial backups.
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
                 c.Backup(new[] {this.DATAFOLDER});
-                Assert.AreEqual(2, c.List().Filesets.Count());
-                Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
+                Assert.AreEqual(3, c.List().Filesets.Count());
+                Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 2).IsFullBackup);
+                Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
                 Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
             }
 

From 17d72be7ff1b97ab4c172aa0a8cfbbdc3569c20d Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Mon, 18 Nov 2019 20:10:56 -0800
Subject: [PATCH 05/24] Remove intermediate partial backups in delete handler.

If surrounded by full backups, partial backups are assumed to be no
longer necessary.

This concerns issue #3982.
---
 .../Library/Main/Operation/DeleteHandler.cs   | 39 +++++++++++--------
 1 file changed, 23 insertions(+), 16 deletions(-)

diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs
index e3c56561c..893f5f397 100644
--- a/Duplicati/Library/Main/Operation/DeleteHandler.cs
+++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs
@@ -221,34 +221,41 @@ namespace Duplicati.Library.Main.Operation
             var fullVersionsKeptCount = 0;
             if (fullVersionsToKeep > 0 && fullVersionsToKeep < backupsRemaining.Count)
             {
-                // keep the number of full backups specified in fullVersionsToKeep.
-                // once the last full backup t okeep is found, also keep the partials immediately after it the full backup.
-                // add the remainder of full and partial backups to toDelete
-                bool foundLastFullBackupToKeep = false;
+                ISet<DateTime> intermediatePartials = new HashSet<DateTime>();
+                bool haveFullBackup = false;
+
+                // Keep the number of full backups specified in fullVersionsToKeep.
+                // Remove partial backups that are surrounded by full backups.
+                // Once enough versions are kept, delete all older backups.
                 foreach (var backup in backupsRemaining)
                 {
-                    bool isFullBackup;
+                    bool isFullBackup = db.IsFilesetFullBackup(backup);
+                    if (isFullBackup)
+                    {
+                        if (haveFullBackup)
+                        {
+                            toDelete.AddRange(intermediatePartials);
+                            intermediatePartials.Clear();
+                        }
+                        haveFullBackup = true;
+                    }
+                    else
+                    {
+                        intermediatePartials.Add(backup);
+                    }
+
                     if (fullVersionsKeptCount < fullVersionsToKeep)
                     {
-                        isFullBackup = db.IsFilesetFullBackup(backup);
                         // count only a full backup
                         if (fullVersionsKeptCount < fullVersionsToKeep && isFullBackup)
                         {
                             fullVersionsKeptCount++;
                         }
-                        continue;
                     }
-                    // do not include any partial backup that precedes the last full backup
-                    if (!foundLastFullBackupToKeep)
+                    else
                     {
-                        isFullBackup = db.IsFilesetFullBackup(backup);
-                        if (!isFullBackup)
-                        {
-                            continue;
-                        }
-                        foundLastFullBackupToKeep = true;
+                        toDelete.Add(backup);
                     }
-                    toDelete.Add(backup);
                 }
             }
 

From 8ab64071f006033c64e2f8acc3682230e915e2fe Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Wed, 20 Nov 2019 20:14:15 -0800
Subject: [PATCH 06/24] Display correct backup type when using direct restore.

Previously, all filesets would be displayed as partial.  Now, we obtain
the partial/full type from the fileset file (not the fileset.json) file
in the dlist file.  If this file does not exist, the backup type
defaults fo full.

Note that this requires downloading and uncompressing the dlist file, so
performance of populating the restore list may be impacted
significantly.

This concerns issue #3982.
---
 .../Library/Main/Operation/ListFilesHandler.cs | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs
index dae2ab929..9aeada5ba 100644
--- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs
+++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs
@@ -4,6 +4,7 @@ using System.Linq;
 using System.Text;
 using Duplicati.Library.Interface;
 using Duplicati.Library.Main.Database;
+using Duplicati.Library.Main.Volumes;
 using Duplicati.Library.Utility;
 
 namespace Duplicati.Library.Main.Operation
@@ -109,7 +110,7 @@ namespace Duplicati.Library.Main.Operation
                 if (filteredList.Count == 0)
                     throw new UserInformationException("No filesets found on remote target", "EmptyRemoteFolder");
 
-                var numberSeq = CreateResultSequence(filteredList);
+                var numberSeq = CreateResultSequence(filteredList, backend, m_options);
                 if (filter.Empty)
                 {
                     m_result.SetResult(numberSeq, null);
@@ -203,9 +204,20 @@ namespace Duplicati.Library.Main.Operation
             return filelistFilter(parsedlist).ToList();
         }
 
-        public static IEnumerable<Library.Interface.IListResultFileset> CreateResultSequence(IEnumerable<KeyValuePair<long, Volumes.IParsedVolume>> filteredList)
+        private static IEnumerable<IListResultFileset> CreateResultSequence(IEnumerable<KeyValuePair<long, IParsedVolume>> filteredList, BackendManager backendManager, Options options)
         {
-            return (from n in filteredList select (Library.Interface.IListResultFileset)(new ListResultFileset(n.Key, BackupType.PARTIAL_BACKUP, n.Value.Time.ToLocalTime(), -1, -1))).ToArray();
+            List<IListResultFileset> list = new List<IListResultFileset>();
+            foreach (KeyValuePair<long, IParsedVolume> entry in filteredList)
+            {
+                AsyncDownloader downloader = new AsyncDownloader(new IRemoteVolume[] {new RemoteVolume(entry.Value.File)}, backendManager);
+                foreach (IAsyncDownloadedFile file in downloader)
+                {
+                    VolumeBase.FilesetData filesetData = VolumeReaderBase.GetFilesetData(entry.Value.CompressionModule, file.TempFile, options);
+                    list.Add(new ListResultFileset(entry.Key, filesetData.IsFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP, entry.Value.Time.ToLocalTime(), -1, -1));
+                }
+            }
+
+            return list.ToArray();
         }
     }
 }

From 0cfce6be39df91e09cd6f352c6f000cc2ee14c6c Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Wed, 20 Nov 2019 21:49:45 -0800
Subject: [PATCH 07/24] Include fileset file when repairing missing dlist file.

This concerns issue #3982.
---
 .../Library/Main/Operation/RepairHandler.cs   | 25 +++++++++++--------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs
index a875900a1..b189c79ea 100644
--- a/Duplicati/Library/Main/Operation/RepairHandler.cs
+++ b/Duplicati/Library/Main/Operation/RepairHandler.cs
@@ -23,11 +23,11 @@ namespace Duplicati.Library.Main.Operation
             m_backendurl = backend;
             m_options = options;
             m_result = result;
-            
+
             if (options.AllowPassphraseChange)
                 throw new UserInformationException(Strings.Common.PassphraseChangeUnsupported, "PassphraseChangeUnsupported");
         }
-        
+
         public void Run(Library.Utility.IFilter filter = null)
         {
             if (!System.IO.File.Exists(m_options.Dbpath))
@@ -287,19 +287,24 @@ namespace Duplicati.Library.Main.Operation
                             if (n.Type == RemoteVolumeType.Files)
                             {
                                 var filesetId = db.GetFilesetIdFromRemotename(n.Name);
-                                var w = new FilesetVolumeWriter(m_options, DateTime.UtcNow);
-                                newEntry = w;
-                                w.SetRemoteFilename(n.Name);
 
-                                db.WriteFileset(w, filesetId, null);
+                                // We cannot wrap the FilesetVolumeWriter in a using statement here because a reference to it is
+                                // retained in newEntry.
+                                FilesetVolumeWriter volumeWriter = new FilesetVolumeWriter(m_options, DateTime.UtcNow);
+                                newEntry = volumeWriter;
+                                volumeWriter.SetRemoteFilename(n.Name);
 
-                                w.Close();
+                                db.WriteFileset(volumeWriter, filesetId, null);
+                                DateTime filesetTime = db.FilesetTimes.First(x => x.Key == filesetId).Value;
+                                volumeWriter.CreateFilesetFile(db.IsFilesetFullBackup(filesetTime));
+
+                                volumeWriter.Close();
                                 if (m_options.Dryrun)
-                                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size));
+                                    Logging.Log.WriteDryrunMessage(LOGTAG, "WouldReUploadFileset", "would re-upload fileset {0}, with size {1}, previous size {2}", n.Name, Library.Utility.Utility.FormatSizeString(new System.IO.FileInfo(volumeWriter.LocalFilename).Length), Library.Utility.Utility.FormatSizeString(n.Size));
                                 else
                                 {
-                                    db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
-                                    backend.Put(w);
+                                    db.UpdateRemoteVolume(volumeWriter.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);
+                                    backend.Put(volumeWriter);
                                 }
                             }
                             else if (n.Type == RemoteVolumeType.Index)

From 51b4ecfb28deb549e666ad1cdf51927a72f0abc0 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Thu, 21 Nov 2019 19:08:55 -0800
Subject: [PATCH 08/24] Include fileset file in dlist files after purge
 operation.

This concerns issue #3982.
---
 Duplicati/Library/Main/Operation/PurgeFilesHandler.cs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs
index 0b31a3f7a..31800bb22 100644
--- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs
+++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs
@@ -162,6 +162,7 @@ namespace Duplicati.Library.Main.Operation
                                     var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal);
                                     var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup);
                                     vol.VolumeID = newids.Item1;
+                                    vol.CreateFilesetFile(isOriginalFilesetFullBackup);
 
                                     Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize));
 

From cd08ca919abbe75a5c1e31a4c08b11af39b9cdb3 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Thu, 21 Nov 2019 19:14:48 -0800
Subject: [PATCH 09/24] Include fileset file in synthetic filelist.

We declare this to be a partial backup, with the presumption being that
a synthetic filelist is only created when a backup is interrupted.

This concerns issue #3982.
---
 .../Library/Main/Operation/Backup/UploadSyntheticFilelist.cs     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs
index 7df4cac92..2d773a815 100644
--- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs
+++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs
@@ -111,6 +111,7 @@ namespace Duplicati.Library.Main.Operation.Backup
                                 foreach (var p in options.ControlFiles.Split(new char[] {System.IO.Path.PathSeparator}, StringSplitOptions.RemoveEmptyEntries))
                                     fsw.AddControlFile(p, options.GetCompressionHintFromFilename(p));
 
+                            fsw.CreateFilesetFile(false);
                             var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime);
                             await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID);
                             await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime);

From 861f0ffa1b8e25fb2017319177ba6a9e0e008e7f Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Sat, 23 Nov 2019 11:24:33 -0800
Subject: [PATCH 10/24] Ensure that one full backup is kept when processing
 keep-time.

this concerns issue #3982.
---
 Duplicati/Library/Main/Operation/DeleteHandler.cs | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs
index 893f5f397..e63edf5b6 100644
--- a/Duplicati/Library/Main/Operation/DeleteHandler.cs
+++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs
@@ -204,11 +204,18 @@ namespace Duplicati.Library.Main.Operation
                 }
             }
 
-            // Remove backups that are older than date specified via option
+            // Remove backups that are older than date specified via option while ensuring
+            // that we always have at least one full backup.
             var keepTime = m_options.KeepTime;
             if (keepTime.Ticks > 0)
             {
-                toDelete.AddRange(sortedAllBackups.SkipWhile(x => x >= keepTime));
+                bool haveFullBackup = false;
+                toDelete.AddRange(sortedAllBackups.SkipWhile(x =>
+                {
+                    bool keepBackup = (x >= keepTime) || !haveFullBackup;
+                    haveFullBackup = haveFullBackup || db.IsFilesetFullBackup(x);
+                    return keepBackup;
+                }));
             }
 
             // Remove backups via retention policy option

From fa3ffb75400f4e954e8b1f62ab4bc050ea9d330a Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Thu, 28 Nov 2019 20:16:57 -0800
Subject: [PATCH 11/24] Add comment to explain download of dlist files during
 direct restore.

---
 Duplicati/Library/Main/Operation/ListFilesHandler.cs | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Duplicati/Library/Main/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/Operation/ListFilesHandler.cs
index 9aeada5ba..ae0b86dfb 100644
--- a/Duplicati/Library/Main/Operation/ListFilesHandler.cs
+++ b/Duplicati/Library/Main/Operation/ListFilesHandler.cs
@@ -212,6 +212,12 @@ namespace Duplicati.Library.Main.Operation
                 AsyncDownloader downloader = new AsyncDownloader(new IRemoteVolume[] {new RemoteVolume(entry.Value.File)}, backendManager);
                 foreach (IAsyncDownloadedFile file in downloader)
                 {
+                    // We must obtain the partial/full status from the fileset file in the dlist files.
+                    // Without this, the restore dialog will show all versions as full, or all versions
+                    // as partial.  While the dlist files are already downloaded elsewhere, doing so again
+                    // here is the most direct way to obtain the partial/full status without a major
+                    // refactoring.  Since restoring directly from the backend files should be a relatively
+                    // rare event, we can work on improving the performance later.
                     VolumeBase.FilesetData filesetData = VolumeReaderBase.GetFilesetData(entry.Value.CompressionModule, file.TempFile, options);
                     list.Add(new ListResultFileset(entry.Key, filesetData.IsFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP, entry.Value.Time.ToLocalTime(), -1, -1));
                 }

From 9eedf757cd0cb01ec84098f200c36ba5eae2f0d7 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Thu, 28 Nov 2019 21:42:58 -0800
Subject: [PATCH 12/24] Extract creation of partial backup to helper method.

---
 Duplicati/UnitTest/DisruptionTests.cs | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs
index 7f404ac62..169fb42ca 100644
--- a/Duplicati/UnitTest/DisruptionTests.cs
+++ b/Duplicati/UnitTest/DisruptionTests.cs
@@ -28,6 +28,21 @@ namespace Duplicati.UnitTest
             }
         }
 
+        private async Task RunPartialBackup(Controller controller)
+        {
+            this.ModifySourceFiles();
+
+            // ReSharper disable once AccessToDisposedClosure
+            Task backupTask = Task.Run(() => controller.Backup(new[] {this.DATAFOLDER}));
+
+            // Block for a small amount of time to allow the ITaskControl to be associated
+            // with the Controller.  Otherwise, the call to Stop will simply be a no-op.
+            Thread.Sleep(1000);
+
+            controller.Stop(true);
+            await backupTask.ConfigureAwait(false);
+        }
+
         public override void SetUp()
         {
             base.SetUp();
@@ -49,20 +64,12 @@ namespace Duplicati.UnitTest
                 Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);
             }
 
-            // Modify the source files and interrupt a backup.
-            this.ModifySourceFiles();
+            // Run a partial backup.
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
-                // ReSharper disable once AccessToDisposedClosure
-                Task<IBackupResults> backupTask = Task.Run(() => c.Backup(new[] {this.DATAFOLDER}));
-
-                // Block for a small amount of time to allow the ITaskControl to be associated
-                // with the Controller.  Otherwise, the call to Stop will simply be a no-op.
-                Thread.Sleep(1000);
+                await this.RunPartialBackup(c);
 
                 // If we interrupt the backup, the most recent Fileset should be marked as partial.
-                c.Stop(true);
-                await backupTask.ConfigureAwait(false);
                 Assert.AreEqual(2, c.List().Filesets.Count());
                 Assert.AreEqual(BackupType.FULL_BACKUP, c.List().Filesets.Single(x => x.Version == 1).IsFullBackup);
                 Assert.AreEqual(BackupType.PARTIAL_BACKUP, c.List().Filesets.Single(x => x.Version == 0).IsFullBackup);

From afecf891dab745a7f06e74e9b47687e12a09e3a1 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Thu, 28 Nov 2019 21:06:24 -0800
Subject: [PATCH 13/24] Add test for keep-time retention logic.

We had to remove the keep-versions options since multiple retention
options are not allowed.

This concerns issue #3982.
---
 Duplicati/UnitTest/BasicSetupHelper.cs |  1 -
 Duplicati/UnitTest/DisruptionTests.cs  | 52 ++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/Duplicati/UnitTest/BasicSetupHelper.cs b/Duplicati/UnitTest/BasicSetupHelper.cs
index 5c5ad992a..57a5ca604 100644
--- a/Duplicati/UnitTest/BasicSetupHelper.cs
+++ b/Duplicati/UnitTest/BasicSetupHelper.cs
@@ -147,7 +147,6 @@ namespace Duplicati.UnitTest
                 opts["dbpath"] = DBFILE;
                 opts["blocksize"] = "10kb";
                 opts["backup-test-samples"] = "0";
-                opts["keep-versions"] = "100";
                 opts["unittest-mode"] = "true";
 
                 return opts;
diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs
index 169fb42ca..13c354864 100644
--- a/Duplicati/UnitTest/DisruptionTests.cs
+++ b/Duplicati/UnitTest/DisruptionTests.cs
@@ -49,6 +49,58 @@ namespace Duplicati.UnitTest
             this.ModifySourceFiles();
         }
 
+        [Test]
+        [Category("Disruption")]
+        public async Task KeepTimeRetention()
+        {
+            // Choose a dblock size that is small enough so that more than one volume is needed.
+            Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions) {["dblock-size"] = "10mb"};
+            
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
+                // First, run two complete backups followed by a partial backup.  We will then set the keep-time
+                // option so that the threshold lies between the first and second backups.
+                c.Backup(new[] {this.DATAFOLDER});
+
+                // Wait before the second backup so that we can more easily define the keep-time threshold
+                // to lie between the first and second backups.
+                Thread.Sleep(5000);
+                this.ModifySourceFiles();
+                c.Backup(new[] {this.DATAFOLDER});
+
+                // Run a partial backup.
+                await this.RunPartialBackup(c);
+
+                // Set the keep-time option so that the threshold lies between the first and second backups
+                // and run the delete operation.
+                List<IListResultFileset> filesets = c.List().Filesets.ToList();
+                DateTime firstBackupTime = filesets[filesets.Count - 1].Time;
+                DateTime secondBackupTime = filesets[filesets.Count - 2].Time;
+                options["keep-time"] = $"{(DateTime.Now - firstBackupTime).Seconds - (secondBackupTime - firstBackupTime).Seconds / 2}s";
+                c.Delete();
+
+                filesets = c.List().Filesets.ToList();
+                Assert.AreEqual(2, filesets.Count);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
+
+                // Run another partial backup.  We will then verify that a full backup is retained
+                // even when all the "recent" backups are partial.
+                await this.RunPartialBackup(c);
+
+                // Set the keep-time option so that the threshold lies after the most recent full backup
+                // and run the delete operation.
+                options["keep-time"] = "1s";
+                c.Delete();
+
+                filesets = c.List().Filesets.ToList();
+                Assert.AreEqual(3, filesets.Count);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
+                Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
+            }
+        }
+
         [Test]
         [Category("Disruption")]
         public async Task StopAfterCurrentFile()

From a47175a9457cf3d29bff903cd93ec9b6cbaf0386 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Fri, 29 Nov 2019 08:22:32 -0800
Subject: [PATCH 14/24] Fix bug in keep-versions logic where dangling partials
 were removed.

If the most recent backups were partials (and not followed by a full
backup), they should not be removed.  A partial backup should only be
removed if it is followed by a full backup.

This concerns issue #3982.
---
 Duplicati/Library/Main/Operation/DeleteHandler.cs | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs
index e63edf5b6..fe2690a95 100644
--- a/Duplicati/Library/Main/Operation/DeleteHandler.cs
+++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs
@@ -239,16 +239,17 @@ namespace Duplicati.Library.Main.Operation
                     bool isFullBackup = db.IsFilesetFullBackup(backup);
                     if (isFullBackup)
                     {
-                        if (haveFullBackup)
-                        {
-                            toDelete.AddRange(intermediatePartials);
-                            intermediatePartials.Clear();
-                        }
+                        toDelete.AddRange(intermediatePartials);
+                        intermediatePartials.Clear();
                         haveFullBackup = true;
                     }
                     else
                     {
-                        intermediatePartials.Add(backup);
+                        // Only consider a partial backup for deletion if we have already encountered a full backup.
+                        if (haveFullBackup)
+                        {
+                            intermediatePartials.Add(backup);
+                        }
                     }
 
                     if (fullVersionsKeptCount < fullVersionsToKeep)

From b4bef4bea0ea6f97e6bd3a4a4e2562330894f5f2 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Fri, 29 Nov 2019 08:26:57 -0800
Subject: [PATCH 15/24] Clarify comment.

---
 Duplicati/Library/Main/Operation/DeleteHandler.cs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs
index fe2690a95..957aaeaa9 100644
--- a/Duplicati/Library/Main/Operation/DeleteHandler.cs
+++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs
@@ -232,7 +232,7 @@ namespace Duplicati.Library.Main.Operation
                 bool haveFullBackup = false;
 
                 // Keep the number of full backups specified in fullVersionsToKeep.
-                // Remove partial backups that are surrounded by full backups.
+                // Remove partial backups that are followed by full backups.
                 // Once enough versions are kept, delete all older backups.
                 foreach (var backup in backupsRemaining)
                 {

From 39ba32c38328d9d14b52248da0f1e4fec4b14572 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Fri, 29 Nov 2019 08:30:10 -0800
Subject: [PATCH 16/24] Remove redundant conditional expression.

---
 Duplicati/Library/Main/Operation/DeleteHandler.cs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs
index 957aaeaa9..876d3cb6a 100644
--- a/Duplicati/Library/Main/Operation/DeleteHandler.cs
+++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs
@@ -254,8 +254,7 @@ namespace Duplicati.Library.Main.Operation
 
                     if (fullVersionsKeptCount < fullVersionsToKeep)
                     {
-                        // count only a full backup
-                        if (fullVersionsKeptCount < fullVersionsToKeep && isFullBackup)
+                        if (isFullBackup)
                         {
                             fullVersionsKeptCount++;
                         }

From 8c9512ac2c4bf2170a76324503d005bcca927e91 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Thu, 28 Nov 2019 23:11:33 -0800
Subject: [PATCH 17/24] Add test for keep-versions retention logic.

---
 Duplicati/UnitTest/DisruptionTests.cs | 83 +++++++++++++++++++++++----
 1 file changed, 71 insertions(+), 12 deletions(-)

diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs
index 13c354864..926f63f85 100644
--- a/Duplicati/UnitTest/DisruptionTests.cs
+++ b/Duplicati/UnitTest/DisruptionTests.cs
@@ -55,24 +55,33 @@ namespace Duplicati.UnitTest
         {
             // Choose a dblock size that is small enough so that more than one volume is needed.
             Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions) {["dblock-size"] = "10mb"};
-            
+
+            // First, run two complete backups followed by a partial backup.  We will then set the keep-time
+            // option so that the threshold lies between the first and second backups.
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
-                // First, run two complete backups followed by a partial backup.  We will then set the keep-time
-                // option so that the threshold lies between the first and second backups.
                 c.Backup(new[] {this.DATAFOLDER});
+            }
 
-                // Wait before the second backup so that we can more easily define the keep-time threshold
-                // to lie between the first and second backups.
-                Thread.Sleep(5000);
+            // Wait before the second backup so that we can more easily define the keep-time threshold
+            // to lie between the first and second backups.
+            Thread.Sleep(5000);
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
                 this.ModifySourceFiles();
                 c.Backup(new[] {this.DATAFOLDER});
+            }
 
-                // Run a partial backup.
+            // Run a partial backup.
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
                 await this.RunPartialBackup(c);
+            }
 
-                // Set the keep-time option so that the threshold lies between the first and second backups
-                // and run the delete operation.
+            // Set the keep-time option so that the threshold lies between the first and second backups
+            // and run the delete operation.
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
                 List<IListResultFileset> filesets = c.List().Filesets.ToList();
                 DateTime firstBackupTime = filesets[filesets.Count - 1].Time;
                 DateTime secondBackupTime = filesets[filesets.Count - 2].Time;
@@ -83,9 +92,12 @@ namespace Duplicati.UnitTest
                 Assert.AreEqual(2, filesets.Count);
                 Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
                 Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
+            }
 
-                // Run another partial backup.  We will then verify that a full backup is retained
-                // even when all the "recent" backups are partial.
+            // Run another partial backup.  We will then verify that a full backup is retained
+            // even when all the "recent" backups are partial.
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
                 await this.RunPartialBackup(c);
 
                 // Set the keep-time option so that the threshold lies after the most recent full backup
@@ -93,7 +105,7 @@ namespace Duplicati.UnitTest
                 options["keep-time"] = "1s";
                 c.Delete();
 
-                filesets = c.List().Filesets.ToList();
+                List<IListResultFileset> filesets = c.List().Filesets.ToList();
                 Assert.AreEqual(3, filesets.Count);
                 Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
                 Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[1].IsFullBackup);
@@ -101,6 +113,53 @@ namespace Duplicati.UnitTest
             }
         }
 
+        [Test]
+        [Category("Disruption")]
+        public async Task KeepVersionsRetention()
+        {
+            // Choose a dblock size that is small enough so that more than one volume is needed.
+            Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions) {["dblock-size"] = "10mb"};
+
+            // Run a full backup.
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
+                c.Backup(new[] {this.DATAFOLDER});
+            }
+
+            // Run a partial backup.
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
+                await this.RunPartialBackup(c);
+            }
+
+            // Run a partial backup.
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
+                await this.RunPartialBackup(c);
+            }
+
+            // Run a full backup.
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
+                this.ModifySourceFiles();
+                c.Backup(new[] {this.DATAFOLDER});
+            }
+
+            // Run a partial backup.
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
+                options["keep-versions"] = "2";
+                await this.RunPartialBackup(c);
+
+                // Partial backups that are followed by a full backup can be deleted.
+                List<IListResultFileset> filesets = c.List().Filesets.ToList();
+                Assert.AreEqual(3, filesets.Count);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
+            }
+        }
+
         [Test]
         [Category("Disruption")]
         public async Task StopAfterCurrentFile()

From b0224194de80ce40c2082c3e0405d21f3591dc4d Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Fri, 29 Nov 2019 08:37:49 -0800
Subject: [PATCH 18/24] Allow continutation after await to run in any context.

---
 Duplicati/UnitTest/DisruptionTests.cs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs
index 926f63f85..4bc7d2741 100644
--- a/Duplicati/UnitTest/DisruptionTests.cs
+++ b/Duplicati/UnitTest/DisruptionTests.cs
@@ -75,7 +75,7 @@ namespace Duplicati.UnitTest
             // Run a partial backup.
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
-                await this.RunPartialBackup(c);
+                await this.RunPartialBackup(c).ConfigureAwait(false);
             }
 
             // Set the keep-time option so that the threshold lies between the first and second backups
@@ -98,7 +98,7 @@ namespace Duplicati.UnitTest
             // even when all the "recent" backups are partial.
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
-                await this.RunPartialBackup(c);
+                await this.RunPartialBackup(c).ConfigureAwait(false);
 
                 // Set the keep-time option so that the threshold lies after the most recent full backup
                 // and run the delete operation.
@@ -129,13 +129,13 @@ namespace Duplicati.UnitTest
             // Run a partial backup.
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
-                await this.RunPartialBackup(c);
+                await this.RunPartialBackup(c).ConfigureAwait(false);
             }
 
             // Run a partial backup.
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
-                await this.RunPartialBackup(c);
+                await this.RunPartialBackup(c).ConfigureAwait(false);
             }
 
             // Run a full backup.
@@ -149,7 +149,7 @@ namespace Duplicati.UnitTest
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
                 options["keep-versions"] = "2";
-                await this.RunPartialBackup(c);
+                await this.RunPartialBackup(c).ConfigureAwait(false);
 
                 // Partial backups that are followed by a full backup can be deleted.
                 List<IListResultFileset> filesets = c.List().Filesets.ToList();
@@ -178,7 +178,7 @@ namespace Duplicati.UnitTest
             // Run a partial backup.
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
-                await this.RunPartialBackup(c);
+                await this.RunPartialBackup(c).ConfigureAwait(false);
 
                 // If we interrupt the backup, the most recent Fileset should be marked as partial.
                 Assert.AreEqual(2, c.List().Filesets.Count());

From 07df3d4d5ed4b45fd26b15da97c1c70e96a4d0db Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Fri, 29 Nov 2019 09:07:43 -0800
Subject: [PATCH 19/24] Verify backup times in addition to partial/full status.

---
 Duplicati/UnitTest/DisruptionTests.cs | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs
index 4bc7d2741..981b00cc1 100644
--- a/Duplicati/UnitTest/DisruptionTests.cs
+++ b/Duplicati/UnitTest/DisruptionTests.cs
@@ -58,39 +58,44 @@ namespace Duplicati.UnitTest
 
             // First, run two complete backups followed by a partial backup.  We will then set the keep-time
             // option so that the threshold lies between the first and second backups.
+            DateTime firstBackupTime;
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
                 c.Backup(new[] {this.DATAFOLDER});
+                firstBackupTime = c.List().Filesets.First().Time;
             }
 
             // Wait before the second backup so that we can more easily define the keep-time threshold
             // to lie between the first and second backups.
             Thread.Sleep(5000);
+            DateTime secondBackupTime;
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
                 this.ModifySourceFiles();
                 c.Backup(new[] {this.DATAFOLDER});
+                secondBackupTime = c.List().Filesets.First().Time;
             }
 
             // Run a partial backup.
+            DateTime thirdBackupTime;
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
                 await this.RunPartialBackup(c).ConfigureAwait(false);
+                thirdBackupTime = c.List().Filesets.First().Time;
             }
 
             // Set the keep-time option so that the threshold lies between the first and second backups
             // and run the delete operation.
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
-                List<IListResultFileset> filesets = c.List().Filesets.ToList();
-                DateTime firstBackupTime = filesets[filesets.Count - 1].Time;
-                DateTime secondBackupTime = filesets[filesets.Count - 2].Time;
                 options["keep-time"] = $"{(DateTime.Now - firstBackupTime).Seconds - (secondBackupTime - firstBackupTime).Seconds / 2}s";
                 c.Delete();
 
-                filesets = c.List().Filesets.ToList();
+                List<IListResultFileset> filesets = c.List().Filesets.ToList();
                 Assert.AreEqual(2, filesets.Count);
+                Assert.AreEqual(secondBackupTime, filesets[1].Time);
                 Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(thirdBackupTime, filesets[0].Time);
                 Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
             }
 
@@ -99,6 +104,7 @@ namespace Duplicati.UnitTest
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
                 await this.RunPartialBackup(c).ConfigureAwait(false);
+                DateTime fourthBackupTime = c.List().Filesets.First().Time;
 
                 // Set the keep-time option so that the threshold lies after the most recent full backup
                 // and run the delete operation.
@@ -107,8 +113,11 @@ namespace Duplicati.UnitTest
 
                 List<IListResultFileset> filesets = c.List().Filesets.ToList();
                 Assert.AreEqual(3, filesets.Count);
+                Assert.AreEqual(secondBackupTime, filesets[2].Time);
                 Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
+                Assert.AreEqual(thirdBackupTime, filesets[1].Time);
                 Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(fourthBackupTime, filesets[0].Time);
                 Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
             }
         }
@@ -121,9 +130,11 @@ namespace Duplicati.UnitTest
             Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions) {["dblock-size"] = "10mb"};
 
             // Run a full backup.
+            DateTime firstBackupTime;
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
                 c.Backup(new[] {this.DATAFOLDER});
+                firstBackupTime = c.List().Filesets.First().Time;
             }
 
             // Run a partial backup.
@@ -139,10 +150,12 @@ namespace Duplicati.UnitTest
             }
 
             // Run a full backup.
+            DateTime fourthBackupTime;
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
                 this.ModifySourceFiles();
                 c.Backup(new[] {this.DATAFOLDER});
+                fourthBackupTime = c.List().Filesets.First().Time;
             }
 
             // Run a partial backup.
@@ -150,12 +163,16 @@ namespace Duplicati.UnitTest
             {
                 options["keep-versions"] = "2";
                 await this.RunPartialBackup(c).ConfigureAwait(false);
+                DateTime fifthBackupTime = c.List().Filesets.First().Time;
 
                 // Partial backups that are followed by a full backup can be deleted.
                 List<IListResultFileset> filesets = c.List().Filesets.ToList();
                 Assert.AreEqual(3, filesets.Count);
+                Assert.AreEqual(firstBackupTime, filesets[2].Time);
                 Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
+                Assert.AreEqual(fourthBackupTime, filesets[1].Time);
                 Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(fifthBackupTime, filesets[0].Time);
                 Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
             }
         }

From 68e7844daeb1a4c9cfb1e3ac404d5ae23df34368 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Fri, 29 Nov 2019 14:13:15 -0800
Subject: [PATCH 20/24] Add test for keep-versions logic where last backup was
 full.

This concerns issue #3982.
---
 Duplicati/UnitTest/DisruptionTests.cs | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs
index 981b00cc1..41c02ad02 100644
--- a/Duplicati/UnitTest/DisruptionTests.cs
+++ b/Duplicati/UnitTest/DisruptionTests.cs
@@ -175,6 +175,22 @@ namespace Duplicati.UnitTest
                 Assert.AreEqual(fifthBackupTime, filesets[0].Time);
                 Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
             }
+
+            // Run a full backup.
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
+                this.ModifySourceFiles();
+                c.Backup(new[] {this.DATAFOLDER});
+                DateTime sixthBackupTime = c.List().Filesets.First().Time;
+
+                // Since the last backup was full, we can now expect to have just the 2 most recent full backups.
+                List<IListResultFileset> filesets = c.List().Filesets.ToList();
+                Assert.AreEqual(2, filesets.Count);
+                Assert.AreEqual(fourthBackupTime, filesets[1].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(sixthBackupTime, filesets[0].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
+            }
         }
 
         [Test]

From 20eb785f098dfa5315f270687a6824b4374748ee Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Fri, 29 Nov 2019 19:04:43 -0800
Subject: [PATCH 21/24] Simplify implementation of keep-versions logic.

---
 .../Library/Main/Operation/DeleteHandler.cs   | 38 ++++++-------------
 1 file changed, 12 insertions(+), 26 deletions(-)

diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs
index 876d3cb6a..e67f450bd 100644
--- a/Duplicati/Library/Main/Operation/DeleteHandler.cs
+++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs
@@ -225,43 +225,29 @@ namespace Duplicati.Library.Main.Operation
             // and remove oldest backups while there are still more backups than should be kept as specified via option
             var backupsRemaining = sortedAllBackups.Except(toDelete).ToList();
             var fullVersionsToKeep = m_options.KeepVersions;
-            var fullVersionsKeptCount = 0;
             if (fullVersionsToKeep > 0 && fullVersionsToKeep < backupsRemaining.Count)
             {
+                int fullVersionsKept = 0;
                 ISet<DateTime> intermediatePartials = new HashSet<DateTime>();
-                bool haveFullBackup = false;
 
-                // Keep the number of full backups specified in fullVersionsToKeep.
-                // Remove partial backups that are followed by full backups.
-                // Once enough versions are kept, delete all older backups.
-                foreach (var backup in backupsRemaining)
+                // Enumerate the collection starting from the most recent full backup.
+                foreach (DateTime backup in backupsRemaining.SkipWhile(x => !db.IsFilesetFullBackup(x)))
                 {
-                    bool isFullBackup = db.IsFilesetFullBackup(backup);
-                    if (isFullBackup)
+                    if (fullVersionsKept >= fullVersionsToKeep)
                     {
+                        // If we have enough full backups, delete all older backups.
+                        toDelete.Add(backup);
+                    }
+                    else if (db.IsFilesetFullBackup(backup))
+                    {
+                        // We can delete partial backups that are surrounded by full backups.
                         toDelete.AddRange(intermediatePartials);
                         intermediatePartials.Clear();
-                        haveFullBackup = true;
+                        fullVersionsKept++;
                     }
                     else
                     {
-                        // Only consider a partial backup for deletion if we have already encountered a full backup.
-                        if (haveFullBackup)
-                        {
-                            intermediatePartials.Add(backup);
-                        }
-                    }
-
-                    if (fullVersionsKeptCount < fullVersionsToKeep)
-                    {
-                        if (isFullBackup)
-                        {
-                            fullVersionsKeptCount++;
-                        }
-                    }
-                    else
-                    {
-                        toDelete.Add(backup);
+                        intermediatePartials.Add(backup);
                     }
                 }
             }

From 30fddd4b913afdb7debd5b2ab3d3aea80fb2afd7 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Fri, 29 Nov 2019 16:22:55 -0800
Subject: [PATCH 22/24] Add test for retention-policy logic.

---
 Duplicati/UnitTest/DisruptionTests.cs | 121 ++++++++++++++++++++++++++
 1 file changed, 121 insertions(+)

diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs
index 41c02ad02..2a776ce3d 100644
--- a/Duplicati/UnitTest/DisruptionTests.cs
+++ b/Duplicati/UnitTest/DisruptionTests.cs
@@ -193,6 +193,127 @@ namespace Duplicati.UnitTest
             }
         }
 
+        [Test]
+        [Category("Disruption")]
+        public async Task RetentionPolicyRetention()
+        {
+            Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions)
+            {
+                // Choose a dblock size that is small enough so that more than one volume is needed.
+                ["dblock-size"] = "10mb",
+
+                // This test assumes that we can perform 3 backups within 1 minute.
+                ["retention-policy"] = "1m:59s,U:1m",
+                ["no-backend-verification"] = "true"
+            };
+
+            DateTime firstBackupTime;
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
+                c.Backup(new[] {this.DATAFOLDER});
+                firstBackupTime = c.List().Filesets.First().Time;
+
+                List<IListResultFileset> filesets = c.List().Filesets.ToList();
+                Assert.AreEqual(1, filesets.Count);
+
+                this.ModifySourceFiles();
+                c.Backup(new[] {this.DATAFOLDER});
+                DateTime secondBackupTime = c.List().Filesets.First().Time;
+
+                // Since the most recent backup is not considered in the retention logic, the only backup in the first time frame
+                // is the initial one.  As a result, we should have 2 backups.
+                filesets = c.List().Filesets.ToList();
+                Assert.AreEqual(2, filesets.Count);
+                Assert.AreEqual(firstBackupTime, filesets[1].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(secondBackupTime, filesets[0].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
+            }
+
+            // Wait so that the next backups fall in the next retention interval.
+            Thread.Sleep(new TimeSpan(0, 0, 1, 0));
+
+            DateTime thirdBackupTime;
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
+                await this.RunPartialBackup(c).ConfigureAwait(false);
+                thirdBackupTime = c.List().Filesets.First().Time;
+
+                // Since the most recent backup is not considered in the retention logic, there are no backups in the first time
+                // frame.  The original 2 backups have now spilled over to the U:1m specification.  Since we keep the first
+                // backup in the interval, we should be left with the first backup, as well as the third partial one.
+                List<IListResultFileset> filesets = c.List().Filesets.ToList();
+                Assert.AreEqual(2, filesets.Count);
+                Assert.AreEqual(firstBackupTime, filesets[1].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(thirdBackupTime, filesets[0].Time);
+                Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
+            }
+
+            DateTime fourthBackupTime;
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
+                this.ModifySourceFiles();
+                c.Backup(new[] {this.DATAFOLDER});
+                fourthBackupTime = c.List().Filesets.First().Time;
+
+                // Since the most recent backup is not considered in the retention logic, the third backup is the only backup
+                // in the first time frame.  There is no further spillover, so we simply add the fourth backup to the
+                // collection of retained backups.
+                List<IListResultFileset> filesets = c.List().Filesets.ToList();
+                Assert.AreEqual(3, filesets.Count);
+                Assert.AreEqual(firstBackupTime, filesets[2].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
+                Assert.AreEqual(thirdBackupTime, filesets[1].Time);
+                Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(fourthBackupTime, filesets[0].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
+
+                this.ModifySourceFiles();
+                c.Backup(new[] {this.DATAFOLDER});
+                DateTime fifthBackupTime = c.List().Filesets.First().Time;
+
+                // Since the most recent backup is not considered in the retention logic, we now have two backups in the
+                // first time frame: the third (partial) and fourth (full).  Since the first backup in each interval is
+                // kept, we would typically keep just the third backup.  However, since we should not discard a full
+                // backup in favor of a partial one, we keep the fourth as well.  We also still have the initial backup.
+                filesets = c.List().Filesets.ToList();
+                Assert.AreEqual(4, filesets.Count);
+                Assert.AreEqual(firstBackupTime, filesets[3].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[3].IsFullBackup);
+                Assert.AreEqual(thirdBackupTime, filesets[2].Time);
+                Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[2].IsFullBackup);
+                Assert.AreEqual(fourthBackupTime, filesets[1].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(fifthBackupTime, filesets[0].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
+            }
+
+            // Wait so that the next backups fall in the next retention interval.
+            Thread.Sleep(new TimeSpan(0, 0, 1, 0));
+
+            using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
+            {
+                this.ModifySourceFiles();
+                c.Backup(new[] {this.DATAFOLDER});
+                DateTime sixthBackupTime = c.List().Filesets.First().Time;
+
+                // Since the most recent backup is not considered in the retention logic, we now have three backups in the
+                // second time frame: the third (partial), fourth (full), and fifth (full).  Since we keep up to the first
+                // full backup in each time frame, we now drop the fifth backup.
+                List<IListResultFileset> filesets = c.List().Filesets.ToList();
+                Assert.AreEqual(4, filesets.Count);
+                Assert.AreEqual(firstBackupTime, filesets[3].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[3].IsFullBackup);
+                Assert.AreEqual(thirdBackupTime, filesets[2].Time);
+                Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[2].IsFullBackup);
+                Assert.AreEqual(fourthBackupTime, filesets[1].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
+                Assert.AreEqual(sixthBackupTime, filesets[0].Time);
+                Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
+            }
+        }
+
         [Test]
         [Category("Disruption")]
         public async Task StopAfterCurrentFile()

From 0f30604046b8f4eb8481118e42324128470bdaf2 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Sat, 30 Nov 2019 13:37:20 -0800
Subject: [PATCH 23/24] Add comment clarifying partial fileset in synthetic
 filelist.

---
 .../Library/Main/Operation/Backup/UploadSyntheticFilelist.cs    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs
index 2d773a815..307d48aba 100644
--- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs
+++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs
@@ -111,6 +111,8 @@ namespace Duplicati.Library.Main.Operation.Backup
                                 foreach (var p in options.ControlFiles.Split(new char[] {System.IO.Path.PathSeparator}, StringSplitOptions.RemoveEmptyEntries))
                                     fsw.AddControlFile(p, options.GetCompressionHintFromFilename(p));
 
+                            // We declare this to be a partial backup since the synthetic filelist is only created
+                            // when a backup is interrupted.
                             fsw.CreateFilesetFile(false);
                             var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime);
                             await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID);

From 7746ff10141f7804386092ca6e394fb4445da916 Mon Sep 17 00:00:00 2001
From: Kenneth Hsu <kennethhsu@gmail.com>
Date: Tue, 3 Dec 2019 18:19:23 -0800
Subject: [PATCH 24/24] Fix derivation of seconds from TimeSpan.

Although the TimeSpan should be less than one minute, we should still
reference TotalSeconds to be safe.
---
 Duplicati/UnitTest/DisruptionTests.cs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Duplicati/UnitTest/DisruptionTests.cs b/Duplicati/UnitTest/DisruptionTests.cs
index 2a776ce3d..9c4ee58cf 100644
--- a/Duplicati/UnitTest/DisruptionTests.cs
+++ b/Duplicati/UnitTest/DisruptionTests.cs
@@ -88,7 +88,7 @@ namespace Duplicati.UnitTest
             // and run the delete operation.
             using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
             {
-                options["keep-time"] = $"{(DateTime.Now - firstBackupTime).Seconds - (secondBackupTime - firstBackupTime).Seconds / 2}s";
+                options["keep-time"] = $"{(int) ((DateTime.Now - firstBackupTime).TotalSeconds - (secondBackupTime - firstBackupTime).TotalSeconds / 2)}s";
                 c.Delete();
 
                 List<IListResultFileset> filesets = c.List().Filesets.ToList();