Kenneth Skovhede
71c4bb4d3d
# Conflicts: # Duplicati/Library/Main/BackendManager.cs # Duplicati/Library/Main/Database/ExtensionMethods.cs # Duplicati/Library/Main/Operation/BackupHandler.cs # Duplicati/Library/Main/Operation/TestFilterHandler.cs # Duplicati/Library/Main/Options.cs Manually ported the `IsSynlink` changes as well as the DNS refresh code
290 lines
14 KiB
C#
290 lines
14 KiB
C#
// Copyright (C) 2013, The Duplicati Team
|
|
|
|
// http://www.duplicati.com, opensource@duplicati.com
|
|
//
|
|
// This library is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU Lesser General Public License as
|
|
// published by the Free Software Foundation; either version 2.1 of the
|
|
// License, or (at your option) any later version.
|
|
//
|
|
// This library is distributed in the hope that it will be useful, but
|
|
// WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
// Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public
|
|
// License along with this library; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
using System;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Collections.Generic;
|
|
using Duplicati.Library.Interface;
|
|
|
|
namespace Duplicati.Library.Main.Operation
|
|
{
|
|
internal class DeleteHandler
|
|
{
|
|
private DeleteResults m_result;
|
|
protected string m_backendurl;
|
|
protected Options m_options;
|
|
|
|
public DeleteHandler(string backend, Options options, DeleteResults result)
|
|
{
|
|
m_backendurl = backend;
|
|
m_options = options;
|
|
m_result = result;
|
|
}
|
|
|
|
public void Run()
|
|
{
|
|
if (!System.IO.File.Exists(m_options.Dbpath))
|
|
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath));
|
|
|
|
using(var db = new Database.LocalDeleteDatabase(m_options.Dbpath, "Delete"))
|
|
{
|
|
m_result.SetDatabase(db);
|
|
Utility.UpdateOptionsFromDb(db, m_options);
|
|
Utility.VerifyParameters(db, m_options);
|
|
|
|
DoRun(db, false, false, null);
|
|
|
|
if (!m_options.Dryrun)
|
|
{
|
|
db.WriteResults();
|
|
using (new Logging.Timer("CommitDelete"))
|
|
db.CommitTransaction("After delete operation", false);
|
|
}
|
|
else
|
|
db.RollbackTransaction(false);
|
|
}
|
|
}
|
|
|
|
public void DoRun(Database.LocalDeleteDatabase db, bool hasVerifiedBacked, bool forceCompact, BackendManager sharedManager)
|
|
{
|
|
// Workaround where we allow a running backendmanager to be used
|
|
using(var bk = sharedManager == null ? new BackendManager(m_backendurl, m_options, m_result.BackendWriter, db, m_result) : null)
|
|
{
|
|
var backend = bk ?? sharedManager;
|
|
|
|
if (!hasVerifiedBacked && !m_options.NoBackendverification)
|
|
FilelistProcessor.VerifyRemoteList(backend, m_options, db, m_result.BackendWriter);
|
|
|
|
var filesetNumbers = db.FilesetTimes.Zip(Enumerable.Range(0, db.FilesetTimes.Count()), (a, b) => new Tuple<long, DateTime>(b, a.Value)).ToList();
|
|
var sets = db.FilesetTimes.Select(x => x.Value).ToArray();
|
|
var toDelete = GetFilesetsToDelete(sets);
|
|
|
|
if (!m_options.AllowFullRemoval && sets.Length == toDelete.Length)
|
|
{
|
|
m_result.AddMessage(string.Format("Preventing removal of last fileset, use --{0} to allow removal ...", "allow-full-removal"));
|
|
toDelete = toDelete.Skip(1).ToArray();
|
|
}
|
|
|
|
if (toDelete != null && toDelete.Length > 0)
|
|
m_result.AddMessage(string.Format("Deleting {0} remote fileset(s) ...", toDelete.Length));
|
|
|
|
var lst = db.DropFilesetsFromTable(toDelete).ToArray();
|
|
foreach(var f in lst)
|
|
db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null);
|
|
|
|
if (!m_options.Dryrun)
|
|
db.CommitTransaction("Removed filelists");
|
|
|
|
foreach(var f in lst)
|
|
{
|
|
if (m_result.TaskControlRendevouz() == TaskControlState.Stop)
|
|
{
|
|
backend.WaitForComplete();
|
|
return;
|
|
}
|
|
|
|
if (!m_options.Dryrun)
|
|
backend.Delete(f.Key, f.Value);
|
|
else
|
|
m_result.AddDryrunMessage(string.Format("Would delete remote fileset: {0}", f.Key));
|
|
}
|
|
|
|
backend.WaitForEmpty();
|
|
|
|
var count = lst.Length;
|
|
if (!m_options.Dryrun)
|
|
{
|
|
if (count == 0)
|
|
m_result.AddMessage("No remote filesets were deleted");
|
|
else
|
|
m_result.AddMessage(string.Format("Deleted {0} remote fileset(s)", count));
|
|
}
|
|
else
|
|
{
|
|
|
|
if (count == 0)
|
|
m_result.AddDryrunMessage("No remote filesets would be deleted");
|
|
else
|
|
m_result.AddDryrunMessage(string.Format("{0} remote fileset(s) would be deleted", count));
|
|
|
|
if (count > 0 && m_options.Dryrun)
|
|
m_result.AddDryrunMessage("Remove --dry-run to actually delete files");
|
|
}
|
|
|
|
if (!m_options.NoAutoCompact && (forceCompact || (toDelete != null && toDelete.Length > 0)))
|
|
{
|
|
m_result.CompactResults = new CompactResults(m_result);
|
|
new CompactHandler(m_backendurl, m_options, (CompactResults)m_result.CompactResults).DoCompact(db, true, sharedManager);
|
|
}
|
|
|
|
m_result.SetResults(
|
|
from n in filesetNumbers
|
|
where toDelete.Contains(n.Item2)
|
|
select n,
|
|
m_options.Dryrun);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gets the filesets selected for deletion
|
|
/// </summary>
|
|
/// <returns>The filesets to delete</returns>
|
|
/// <param name="allBackups">The list of backups that can be deleted</param>
|
|
private DateTime[] GetFilesetsToDelete(DateTime[] allBackups)
|
|
{
|
|
if (allBackups.Length == 0)
|
|
return allBackups;
|
|
|
|
if (allBackups.Distinct().Count() != allBackups.Length)
|
|
throw new Exception(string.Format("List of backup timestamps contains duplicates: {0}", string.Join(", ", allBackups.Select(x => x.ToString()))));
|
|
|
|
List<DateTime> toDelete = new List<DateTime>();
|
|
|
|
// Remove backups explicitely specified via option
|
|
var versions = m_options.Version;
|
|
if (versions != null && versions.Length > 0)
|
|
foreach (var ix in versions.Distinct())
|
|
if (ix >= 0 && ix < allBackups.Length)
|
|
toDelete.Add(allBackups[ix]);
|
|
|
|
// Remove backups that are older than date specified via option
|
|
var keepTime = m_options.KeepTime;
|
|
if (keepTime.Ticks > 0)
|
|
toDelete.AddRange(allBackups.SkipWhile(x => x >= keepTime));
|
|
|
|
// Remove backups via rentention policy option
|
|
toDelete.AddRange(ApplyRetentionPolicy(allBackups));
|
|
|
|
// Check how many backups will be remaining after the previous steps
|
|
// and remove oldest backups while there are still more backups than should be kept as specified via option
|
|
var backupsRemaining = allBackups.Except(toDelete).ToList();
|
|
var keepVersions = m_options.KeepVersions;
|
|
if (keepVersions > 0 && keepVersions < backupsRemaining.Count())
|
|
toDelete.AddRange(backupsRemaining.Skip(keepVersions));
|
|
|
|
var toDeleteDistinct = toDelete.Distinct().OrderByDescending(x => x).AsEnumerable();
|
|
|
|
var removeCount = toDeleteDistinct.Count();
|
|
if (removeCount > allBackups.Length)
|
|
throw new Exception(string.Format("Too many entries {0} vs {1}, lists: {2} vs {3}", removeCount, allBackups.Length, string.Join(", ", toDeleteDistinct.Select(x => x.ToString())), string.Join(", ", allBackups.Select(x => x.ToString()))));
|
|
|
|
return toDeleteDistinct.ToArray();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Deletes backups according to the retention policy configuration.
|
|
/// Backups that are not within any of the specified time frames will will NOT be deleted.
|
|
/// </summary>
|
|
/// <returns>The filesets to delete</returns>
|
|
/// <param name="backups">The list of backups that can be deleted</param>
|
|
private List<DateTime> ApplyRetentionPolicy(DateTime[] backups)
|
|
{
|
|
// Any work to do?
|
|
var retentionPolicyOptionValues = m_options.RetentionPolicy;
|
|
if (retentionPolicyOptionValues.Count == 0 || backups.Length == 0)
|
|
{
|
|
return new List<DateTime>(); // don't delete any backups
|
|
}
|
|
|
|
Logging.Log.WriteMessage("[Retention Policy]: Start checking if backups can be removed", Logging.LogMessageType.Information);
|
|
|
|
// Work with a copy to not modify the enumeration that the caller passed
|
|
List<DateTime> clonedBackupList = new List<DateTime>(backups);
|
|
|
|
// Make sure the backups are in descending order (newest backup in the beginning)
|
|
clonedBackupList = clonedBackupList.OrderByDescending(x => x).ToList();
|
|
|
|
// Most recent backup usually should never get deleted in this process, so exclude it for now,
|
|
// but keep a reference to potentiall delete it when allow-full-removal is set
|
|
var mostRecentBackup = clonedBackupList.ElementAt(0);
|
|
clonedBackupList.RemoveAt(0);
|
|
var deleteMostRecentBackup = m_options.AllowFullRemoval;
|
|
|
|
Logging.Log.WriteMessage(string.Format("[Retention Policy]: Time frames and intervals pairs: {0}",
|
|
string.Join(", ", retentionPolicyOptionValues)), Logging.LogMessageType.Information);
|
|
|
|
Logging.Log.WriteMessage(string.Format("[Retention Policy]: Backups to consider: {0}",
|
|
string.Join(", ", clonedBackupList)), Logging.LogMessageType.Information);
|
|
|
|
// Collect all potential backups in each time frame and thin out according to the specified interval,
|
|
// starting with the oldest backup in that time frame.
|
|
// The order in which the time frames values are checked has to be from the smallest to the largest.
|
|
List<DateTime> backupsToDelete = new List<DateTime>();
|
|
var now = DateTime.Now;
|
|
foreach (var singleRetentionPolicyOptionValue in retentionPolicyOptionValues.OrderBy(x => x.Timeframe))
|
|
{
|
|
// The timeframe in the retention policy option is only a timespan which has to be applied to the current DateTime to get the actual lower bound
|
|
DateTime timeFrame = (singleRetentionPolicyOptionValue.IsUnlimtedTimeframe()) ? DateTime.MinValue : (now - singleRetentionPolicyOptionValue.Timeframe);
|
|
|
|
Logging.Log.WriteMessage(string.Format("[Retention Policy]: Next time frame and interval pair: {0}", singleRetentionPolicyOptionValue.ToString()), Logging.LogMessageType.Profiling);
|
|
|
|
List<DateTime> backupsInTimeFrame = new List<DateTime>();
|
|
while (clonedBackupList.Count > 0 && clonedBackupList[0] >= timeFrame)
|
|
{
|
|
backupsInTimeFrame.Insert(0, clonedBackupList[0]); // Insert at begining to reverse order, which is nessecary for next step
|
|
clonedBackupList.RemoveAt(0); // remove from here to not handle the same backup in two time frames
|
|
}
|
|
|
|
Logging.Log.WriteMessage(string.Format("[Retention Policy]: Backups in this time frame: {0}",
|
|
string.Join(", ", backupsInTimeFrame)), Logging.LogMessageType.Profiling);
|
|
|
|
// Run through backups in this time frame
|
|
DateTime? lastKept = null;
|
|
foreach (DateTime backup in backupsInTimeFrame)
|
|
{
|
|
// Keep this backup if
|
|
// - no backup has yet been added to the time frame (keeps at least the oldest backup in a time frame)
|
|
// - difference between last added backup and this backup is bigger than the specified interval
|
|
if (lastKept == null || singleRetentionPolicyOptionValue.IsKeepAllVersions() || (backup - lastKept.Value) >= singleRetentionPolicyOptionValue.Interval)
|
|
{
|
|
Logging.Log.WriteMessage(string.Format("[Retention Policy]: Keeping backup: {0}", backup), Logging.LogMessageType.Profiling);
|
|
lastKept = backup;
|
|
}
|
|
else
|
|
{
|
|
Logging.Log.WriteMessage(string.Format("[Retention Policy]: Deleting backup: {0}", backup), Logging.LogMessageType.Profiling);
|
|
backupsToDelete.Add(backup);
|
|
}
|
|
}
|
|
|
|
// Check if most recent backup is outside of this time frame (meaning older/smaller)
|
|
deleteMostRecentBackup &= (mostRecentBackup < timeFrame);
|
|
}
|
|
|
|
// Delete all remaining backups
|
|
backupsToDelete.AddRange(clonedBackupList);
|
|
Logging.Log.WriteMessage(string.Format("[Retention Policy]: Backups outside of all time frames and thus getting deleted: {0}",
|
|
string.Join(", ", clonedBackupList)), Logging.LogMessageType.Information);
|
|
|
|
// Delete most recent backup if allow-full-removal is set and the most current backup is outside of any time frame
|
|
if (deleteMostRecentBackup)
|
|
{
|
|
backupsToDelete.Add(mostRecentBackup);
|
|
Logging.Log.WriteMessage(string.Format("[Retention Policy]: Deleting most recent backup: {0}",
|
|
mostRecentBackup), Logging.LogMessageType.Information);
|
|
}
|
|
|
|
Logging.Log.WriteMessage(string.Format("[Retention Policy]: All backups to delete: {0}",
|
|
string.Join(", ", backupsToDelete.OrderByDescending(x => x))), Logging.LogMessageType.Information);
|
|
|
|
return backupsToDelete;
|
|
}
|
|
}
|
|
}
|
|
|