Merge pull request #6066 from duplicati/feature/remote-empty-index-files

Delete empty index files
2025-03-21 08:31:59 +01:00
parent b0d20638f2 3c741cbcfc
commit a326b43b1d
2 changed files with 47 additions and 2 deletions
@@ -586,6 +586,13 @@ ORDER BY
                foreach (var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"", ""Timestamp"", ""IsFullBackup"" FROM ""Fileset"" WHERE ""VolumeID"" NOT IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" NOT IN (?, ?))", RemoteVolumeType.Files.ToString(), RemoteVolumeState.Deleting.ToString(), RemoteVolumeState.Deleted.ToString()))
                    yield return (rd.ConvertValueToInt64(0), ParseFromEpochSeconds(rd.ConvertValueToInt64(1)), rd.ConvertValueToInt64(2) == BackupType.FULL_BACKUP);
        }
+
+        public IEnumerable<IRemoteVolume> EmptyIndexFiles()
+        {
+            using (var cmd = m_connection.CreateCommand())
+                foreach (var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = ? AND ""State"" IN (?, ?, ?) AND ""ID"" NOT IN (SELECT ""IndexVolumeId"" FROM ""IndexBlockLink"")", RemoteVolumeType.Index.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()))
+                    yield return new RemoteVolume(rd.ConvertValueToString(0), rd.ConvertValueToString(1), rd.ConvertValueToInt64(2));
+        }
    }
 }

@@ -144,9 +144,10 @@ namespace Duplicati.Library.Main.Operation

                var missingRemoteFilesets = db.MissingRemoteFilesets().ToList();
                var missingLocalFilesets = db.MissingLocalFilesets().ToList();
+                var emptyIndexFiles = db.EmptyIndexFiles().ToList();

                var progress = 0;
-                var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count;
+                var targetProgess = tp.ExtraVolumes.Count() + tp.MissingVolumes.Count() + tp.VerificationRequiredVolumes.Count() + missingRemoteFilesets.Count + missingLocalFilesets.Count + emptyIndexFiles.Count;

                var mostRecentLocal = db.FilesetTimes.Select(x => x.Value.ToLocalTime()).Append(DateTime.MinValue).Max();
                var mostRecentRemote = tp.ParsedVolumes.Select(x => x.Time.ToLocalTime()).Append(DateTime.MinValue).Max();
@@ -175,7 +176,7 @@ namespace Duplicati.Library.Main.Operation
                    }
                }

-                if (tp.ExtraVolumes.Any() || tp.MissingVolumes.Any() || tp.VerificationRequiredVolumes.Any() || missingRemoteFilesets.Any() || missingLocalFilesets.Any())
+                if (tp.ExtraVolumes.Any() || tp.MissingVolumes.Any() || tp.VerificationRequiredVolumes.Any() || missingRemoteFilesets.Any() || missingLocalFilesets.Any() || emptyIndexFiles.Any())
                {
                    if (tp.VerificationRequiredVolumes.Any())
                    {
@@ -563,6 +564,43 @@ namespace Duplicati.Library.Main.Operation
                                throw;
                        }
                    }
+
+                    foreach (var emptyIndexFile in emptyIndexFiles)
+                    {
+                        try
+                        {
+                            if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
+                            {
+                                await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false);
+                                return;
+                            }
+
+                            progress++;
+                            m_result.OperationProgressUpdater.UpdateProgress((float)progress / targetProgess);
+
+                            if (m_options.Dryrun)
+                                Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteEmptyIndexFile", "would delete empty index file {0}", emptyIndexFile);
+                            else
+                            {
+                                if (emptyIndexFile.Size > 2048)
+                                {
+                                    Logging.Log.WriteWarningMessage(LOGTAG, "LargeEmptyIndexFile", null, "The empty index file {0} is larger than expected, choosing not to delete it", emptyIndexFile);
+                                }
+                                else
+                                {
+                                    Logging.Log.WriteInformationMessage(LOGTAG, "DeletingEmptyIndexFile", "Deleting empty index file {0}", emptyIndexFile);
+                                    await backendManager.DeleteAsync(emptyIndexFile.Name, emptyIndexFile.Size, false, cancellationToken).ConfigureAwait(false);
+                                }
+                            }
+                        }
+                        catch (Exception ex)
+                        {
+                            Logging.Log.WriteErrorMessage(LOGTAG, "CleanupEmptyIndexFileError", ex, "Failed to perform cleanup for empty index file: {0}, message: {1}", emptyIndexFile, ex.Message);
+
+                            if (ex is System.Threading.ThreadAbortException)
+                                throw;
+                        }
+                    }
                }
                else
                {