From c27ea2482625f7e9b4807280c18ad969f67c120d Mon Sep 17 00:00:00 2001 From: Kenneth Hsu Date: Wed, 13 Feb 2019 21:53:19 -0800 Subject: [PATCH 1/4] Ensure that FilesetVolumeWriter removes temp file upon disposal. This ensures the removal of the temporary file containing the filelist.json that remained regardless if files needed to be uploaded or not. This concerns issue #3652. --- Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index b63574b6f..a8742a0bf 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -150,6 +150,7 @@ namespace Duplicati.Library.Main.Volumes public override void Dispose() { this.Close(); + base.Dispose(); } public long FileCount { get { return m_filecount; } } From 3fc9276cddbddc7551af8e60c2aba3c468774e4f Mon Sep 17 00:00:00 2001 From: Kenneth Hsu Date: Sat, 16 Feb 2019 14:15:19 -0800 Subject: [PATCH 2/4] Remove temp files when compact operation is not performed. This concerns issue #3652. --- .../Library/Main/Operation/CompactHandler.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index 701119324..bdd83bc7a 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -110,20 +110,19 @@ namespace Duplicati.Library.Main.Operation newvolindex = new IndexVolumeWriter(m_options); newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction); db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, transaction); - newvolindex.StartVolume(newvol.RemoteFilename); } - + long blocksInVolume = 0; long discardedBlocks = 0; long discardedSize = 0; byte[] buffer = new byte[m_options.Blocksize]; var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray(); - + //These are for bookkeeping var uploadedVolumes = new List>(); var deletedVolumes = new List>(); var downloadedVolumes = new List>(); - + //We start by deleting unused volumes to save space before uploading new stuff var fullyDeleteable = (from v in remoteList where report.DeleteableVolumes.Contains(v.Name) @@ -137,10 +136,11 @@ namespace Duplicati.Library.Main.Operation if (report.ShouldCompact) { + newvolindex.StartVolume(newvol.RemoteFilename); var volumesToDownload = (from v in remoteList where report.CompactableVolumes.Contains(v.Name) select (IRemoteVolume)v).ToList(); - + using(var q = db.CreateBlockQueryHelper(transaction)) { foreach (var entry in new AsyncDownloader(volumesToDownload, backend)) @@ -236,13 +236,18 @@ namespace Duplicati.Library.Main.Operation } } } - + else + { + newvolindex.Dispose(); + newvol.Dispose(); + } + deletedVolumes.AddRange(DoDelete(db, backend, deleteableVolumes, ref transaction)); var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value); var deletedSize = deletedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value); var uploadSize = uploadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a,x) => a + x.Value); - + m_result.DeletedFileCount = deletedVolumes.Count; m_result.DownloadedFileCount = downloadedVolumes.Count; m_result.UploadedFileCount = uploadedVolumes.Count; From daf0311bf86957ae5f2faa87352bbeca16d6ca39 Mon Sep 17 00:00:00 2001 From: Kenneth Hsu Date: Sat, 16 Feb 2019 16:46:52 -0800 Subject: [PATCH 3/4] Allow removal of VolumeWriterBase temp files. This reverts a change made in revision 31ac40083a ("Prevent temp files in VolumeWriterBase from being deleted prematurely" in pull request #2637). It's unclear what issue was being worked around by preventing the removal of the temporary files (based on comments in pull request #2637, it appears to be related to tests that failed only on Travis). However, this workaround prevented the removal of temporary files created during compact operations, which caused several users' temporary directories to be filled up with files that should have been removed. This concerns issue #3652. --- Duplicati/Library/Main/Volumes/VolumeWriterBase.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs b/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs index c427a1ff0..ab152bd09 100644 --- a/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs +++ b/Duplicati/Library/Main/Volumes/VolumeWriterBase.cs @@ -52,9 +52,6 @@ namespace Duplicati.Library.Main.Volumes else m_localfile = new Library.Utility.TempFile(); - // TODO(danstahr): This is a hack! Figure out why the file is being disposed of prematurely. - m_localfile.Protected = true; - ResetRemoteFilename(options, timestamp); m_localFileStream = new System.IO.FileStream(m_localfile, FileMode.Create, FileAccess.Write, FileShare.Read); From e82c11b39b89eff4561c5c22b51633f10eab8018 Mon Sep 17 00:00:00 2001 From: Kenneth Hsu Date: Sat, 16 Feb 2019 20:43:05 -0800 Subject: [PATCH 4/4] Avoid possible null reference exception. --- Duplicati/Library/Main/Operation/CompactHandler.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Duplicati/Library/Main/Operation/CompactHandler.cs b/Duplicati/Library/Main/Operation/CompactHandler.cs index bdd83bc7a..1197f4344 100644 --- a/Duplicati/Library/Main/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/Operation/CompactHandler.cs @@ -93,8 +93,8 @@ namespace Duplicati.Library.Main.Operation report.ReportCompactData(); if (report.ShouldReclaim || report.ShouldCompact) - { - // Workaround where we allow a running backendmanager to be used + { + // Workaround where we allow a running backendmanager to be used using(var bk = sharedBackend == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db) : null) { var backend = bk ?? sharedBackend; @@ -136,7 +136,7 @@ namespace Duplicati.Library.Main.Operation if (report.ShouldCompact) { - newvolindex.StartVolume(newvol.RemoteFilename); + newvolindex?.StartVolume(newvol.RemoteFilename); var volumesToDownload = (from v in remoteList where report.CompactableVolumes.Contains(v.Name) select (IRemoteVolume)v).ToList(); @@ -238,7 +238,7 @@ namespace Duplicati.Library.Main.Operation } else { - newvolindex.Dispose(); + newvolindex?.Dispose(); newvol.Dispose(); }