1346 lines
62 KiB
C#
1346 lines
62 KiB
C#
// Copyright (C) 2026, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using Duplicati.Library.Interface;
|
|
using Duplicati.Library.Logging;
|
|
using Duplicati.Library.Main;
|
|
using Duplicati.Library.Main.Database;
|
|
using Duplicati.Library.Main.Volumes;
|
|
using NUnit.Framework;
|
|
using IFileEntry = Duplicati.Library.Interface.IFileEntry;
|
|
using Utility = Duplicati.Library.Utility.Utility;
|
|
using Assert = NUnit.Framework.Legacy.ClassicAssert;
|
|
|
|
namespace Duplicati.UnitTest
|
|
{
|
|
public class DisruptionTests : BasicSetupHelper
|
|
{
|
|
// Files to create in MB.
|
|
private readonly int[] fileSizes = { 10, 20, 30 };
|
|
|
|
private void ModifySourceFiles()
|
|
{
|
|
foreach (int size in this.fileSizes)
|
|
{
|
|
byte[] data = new byte[size * 1024 * 1024];
|
|
Random rng = new Random();
|
|
rng.NextBytes(data);
|
|
File.WriteAllBytes(Path.Combine(this.DATAFOLDER, size + "MB"), data);
|
|
}
|
|
}
|
|
|
|
private async Task<(IBackupResults Result, List<string> ModifiedFiles)> RunPartialBackupAsync(Controller controller)
|
|
{
|
|
this.ModifySourceFiles();
|
|
|
|
var stopped = new TaskCompletionSource<bool>();
|
|
var modifiedFiles = new List<string>();
|
|
controller.OnOperationStarted = r =>
|
|
{
|
|
var pv = r as ITaskControlProvider;
|
|
if (pv == null)
|
|
throw new Exception("Task control provider not found");
|
|
#if DEBUG
|
|
pv.TaskControl.TestMethodCallback = (path) =>
|
|
{
|
|
modifiedFiles.Add(path);
|
|
if (modifiedFiles.Count == 3)
|
|
{
|
|
Thread.Sleep(500);
|
|
controller.StopAsync().GetAwaiter().GetResult();
|
|
stopped.TrySetResult(true);
|
|
}
|
|
};
|
|
#endif
|
|
};
|
|
|
|
// ReSharper disable once AccessToDisposedClosure
|
|
Task<IBackupResults> backupTask = controller.BackupAsync(new[] { this.DATAFOLDER });
|
|
|
|
var t = await Task.WhenAny(backupTask, stopped.Task).ConfigureAwait(false);
|
|
if (t != stopped.Task)
|
|
throw new Exception("Backup task completed before we could stop it");
|
|
|
|
return (await backupTask.ConfigureAwait(false), modifiedFiles);
|
|
}
|
|
|
|
[SetUp]
|
|
public void SetUp()
|
|
{
|
|
this.ModifySourceFiles();
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
public async Task FilesetFilesAsync()
|
|
{
|
|
#if !DEBUG
|
|
Assert.Ignore("This test requires DEBUG to be defined");
|
|
#endif
|
|
// Choose a dblock size that is small enough so that more than one volume is needed.
|
|
Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions)
|
|
{
|
|
["dblock-size"] = "10mb",
|
|
|
|
// This allows us to inspect the dlist files without needing the BackendManager (which is inaccessible here) to decrypt them.
|
|
["no-encryption"] = "true",
|
|
["disable-file-scanner"] = "true",
|
|
["concurrency-fileprocessors"] = "1",
|
|
};
|
|
|
|
// Run a full backup.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER });
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(0, backupResults.Warnings.Count());
|
|
}
|
|
|
|
// Inject some spacing to allow for the purged fileset
|
|
Thread.Sleep(2000);
|
|
|
|
// Run a partial backup.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
(var backupResults, _) = await this.RunPartialBackupAsync(c).ConfigureAwait(false);
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(1, backupResults.Warnings.Count());
|
|
}
|
|
|
|
async Task<(Dictionary<DateTime, int> map, List<string> filelistFiles)> GetBackupTypesFromRemoteFilesAsync(Controller c)
|
|
{
|
|
var map = new Dictionary<DateTime, int>();
|
|
var filelistFiles = new List<string>();
|
|
|
|
var remoteFiles = await c.ListRemoteAsync();
|
|
foreach (var file in remoteFiles.Files)
|
|
{
|
|
IParsedVolume volume = VolumeBase.ParseFilename(file);
|
|
if (volume != null && volume.FileType == RemoteVolumeType.Files)
|
|
{
|
|
string dlistFile = Path.Combine(this.TARGETFOLDER, volume.File.Name);
|
|
filelistFiles.Add(dlistFile);
|
|
VolumeBase.FilesetData filesetData = VolumeReaderBase.GetFilesetData(volume.CompressionModule, dlistFile, new Options(options));
|
|
map[volume.Time] = filesetData.IsFullBackup ? BackupType.FULL_BACKUP : BackupType.PARTIAL_BACKUP;
|
|
}
|
|
}
|
|
|
|
return (map, filelistFiles);
|
|
}
|
|
|
|
// Purge a file and verify that the fileset file exists in the new dlist files.
|
|
List<string> dlistFiles;
|
|
Dictionary<DateTime, int> backupTypeMap;
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.PurgeFilesAsync(new Library.Utility.FilterExpression($"{this.DATAFOLDER}/*{this.fileSizes[0]}*")));
|
|
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(2, filesets.Count);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets.Single(x => x.Version == 1).IsFullBackup);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets.Single(x => x.Version == 0).IsFullBackup);
|
|
|
|
(backupTypeMap, dlistFiles) = await GetBackupTypesFromRemoteFilesAsync(c);
|
|
}
|
|
|
|
var backupTypes = backupTypeMap.OrderByDescending(x => x.Key).Select(x => x.Value).ToArray();
|
|
Assert.AreEqual(2, backupTypes.Length);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, backupTypes[1]);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, backupTypes[0]);
|
|
|
|
// Remove the dlist files.
|
|
foreach (string dlistFile in dlistFiles)
|
|
{
|
|
File.Delete(dlistFile);
|
|
}
|
|
|
|
// Run a repair and verify that the fileset file exists in the new dlist files.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.RepairAsync());
|
|
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(2, filesets.Count);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets.Single(x => x.Version == 1).IsFullBackup);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets.Single(x => x.Version == 0).IsFullBackup);
|
|
|
|
(backupTypeMap, _) = await GetBackupTypesFromRemoteFilesAsync(c);
|
|
}
|
|
|
|
backupTypes = backupTypeMap.OrderByDescending(x => x.Key).Select(x => x.Value).ToArray();
|
|
Assert.AreEqual(2, backupTypes.Length);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, backupTypes[1]);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, backupTypes[0]);
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
public async Task KeepTimeRetentionAsync()
|
|
{
|
|
#if !DEBUG
|
|
Assert.Ignore("This test requires DEBUG to be defined");
|
|
#endif
|
|
|
|
// Choose a dblock size that is small enough so that more than one volume is needed.
|
|
var options = new Dictionary<string, string>(this.TestOptions)
|
|
{
|
|
["dblock-size"] = "10mb",
|
|
["disable-file-scanner"] = "true",
|
|
["concurrency-fileprocessors"] = "1",
|
|
};
|
|
|
|
// First, run two complete backups followed by a partial backup. We will then set the keep-time
|
|
// option so that the threshold lies between the first and second backups.
|
|
DateTime firstBackupTime;
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
firstBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
}
|
|
|
|
// Wait before the second backup so that we can more easily define the keep-time threshold
|
|
// to lie between the first and second backups.
|
|
Thread.Sleep(5000);
|
|
DateTime secondBackupTime;
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
this.ModifySourceFiles();
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
secondBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
}
|
|
|
|
// Run a partial backup.
|
|
DateTime thirdBackupTime;
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
(var backupResults, _) = await this.RunPartialBackupAsync(c).ConfigureAwait(false);
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(1, backupResults.Warnings.Count());
|
|
thirdBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
}
|
|
|
|
// Set the keep-time option so that the threshold lies between the first and second backups
|
|
// and run the delete operation.
|
|
options["keep-time"] = $"{(int)((DateTime.Now - firstBackupTime).TotalSeconds - (secondBackupTime - firstBackupTime).TotalSeconds / 2)}s";
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.DeleteAsync());
|
|
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(2, filesets.Count);
|
|
Assert.AreEqual(secondBackupTime, filesets[1].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
|
|
Assert.AreEqual(thirdBackupTime, filesets[0].Time);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
|
|
}
|
|
|
|
// Run another partial backup. We will then verify that a full backup is retained
|
|
// even when all the "recent" backups are partial.
|
|
DateTime fourthBackupTime;
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
(var backupResults, _) = await this.RunPartialBackupAsync(c).ConfigureAwait(false);
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(1, backupResults.Warnings.Count());
|
|
fourthBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
}
|
|
|
|
// Set the keep-time option so that the threshold lies after the most recent full backup
|
|
// and run the delete operation.
|
|
options["keep-time"] = "1s";
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.DeleteAsync());
|
|
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(3, filesets.Count);
|
|
Assert.AreEqual(secondBackupTime, filesets[2].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
|
|
Assert.AreEqual(thirdBackupTime, filesets[1].Time);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[1].IsFullBackup);
|
|
Assert.AreEqual(fourthBackupTime, filesets[0].Time);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
public async Task KeepVersionsRetentionAsync()
|
|
{
|
|
#if !DEBUG
|
|
Assert.Ignore("This test requires DEBUG to be defined");
|
|
#endif
|
|
|
|
// Choose a dblock size that is small enough so that more than one volume is needed.
|
|
var options = new Dictionary<string, string>(this.TestOptions)
|
|
{
|
|
["dblock-size"] = "10mb",
|
|
["disable-file-scanner"] = "true",
|
|
["concurrency-fileprocessors"] = "1",
|
|
};
|
|
|
|
// Run a full backup.
|
|
DateTime firstBackupTime;
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
firstBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
}
|
|
|
|
// Run a partial backup.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
(var backupResults, _) = await this.RunPartialBackupAsync(c).ConfigureAwait(false);
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(1, backupResults.Warnings.Count());
|
|
}
|
|
|
|
// Run a partial backup.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
(var backupResults, _) = await this.RunPartialBackupAsync(c).ConfigureAwait(false);
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(1, backupResults.Warnings.Count());
|
|
}
|
|
|
|
// Run a full backup.
|
|
DateTime fourthBackupTime;
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
this.ModifySourceFiles();
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
fourthBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
}
|
|
|
|
// Run a partial backup.
|
|
options["keep-versions"] = "2";
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
(var backupResults, _) = await this.RunPartialBackupAsync(c).ConfigureAwait(false);
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(1, backupResults.Warnings.Count());
|
|
DateTime fifthBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
|
|
// Since we stopped the operation, files were not deleted automatically
|
|
await c.DeleteAsync();
|
|
|
|
// Partial backups that are followed by a full backup can be deleted.
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(3, filesets.Count);
|
|
Assert.AreEqual(firstBackupTime, filesets[2].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
|
|
Assert.AreEqual(fourthBackupTime, filesets[1].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
|
|
Assert.AreEqual(fifthBackupTime, filesets[0].Time);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
|
|
}
|
|
|
|
// Run a full backup.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
this.ModifySourceFiles();
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
var sixthBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
|
|
// Since the last backup was full, we can now expect to have just the 2 most recent full backups.
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(2, filesets.Count);
|
|
Assert.AreEqual(fourthBackupTime, filesets[1].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
|
|
Assert.AreEqual(sixthBackupTime, filesets[0].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
public async Task ListWithoutLocalDbAsync()
|
|
{
|
|
#if !DEBUG
|
|
Assert.Ignore("This test requires DEBUG to be defined");
|
|
#endif
|
|
|
|
// Choose a dblock size that is small enough so that more than one volume is needed.
|
|
Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions)
|
|
{
|
|
["dblock-size"] = "10mb",
|
|
["no-local-db"] = "true",
|
|
["disable-file-scanner"] = "true",
|
|
["concurrency-fileprocessors"] = "1",
|
|
};
|
|
|
|
// Run a full backup.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
|
|
// Run a partial backup.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
(var backupResults, _) = await this.RunPartialBackupAsync(c).ConfigureAwait(false);
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(1, backupResults.Warnings.Count());
|
|
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(2, filesets.Count);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
public async Task RetentionPolicyRetentionAsync()
|
|
{
|
|
#if !DEBUG
|
|
Assert.Ignore("This test requires DEBUG to be defined");
|
|
#endif
|
|
|
|
Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions)
|
|
{
|
|
// Choose a dblock size that is small enough so that more than one volume is needed.
|
|
["dblock-size"] = "10mb",
|
|
|
|
// This test assumes that we can perform 3 backups within 1 minute.
|
|
["retention-policy"] = "1m:59s,U:1m",
|
|
["no-backend-verification"] = "true",
|
|
["disable-file-scanner"] = "true",
|
|
["concurrency-fileprocessors"] = "1"
|
|
};
|
|
|
|
DateTime firstBackupTime;
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
firstBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(1, filesets.Count);
|
|
|
|
this.ModifySourceFiles();
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
DateTime secondBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
|
|
// Since the most recent backup is not considered in the retention logic, the only backup in the first time frame
|
|
// is the initial one. As a result, we should have 2 backups.
|
|
filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(2, filesets.Count);
|
|
Assert.AreEqual(firstBackupTime, filesets[1].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
|
|
Assert.AreEqual(secondBackupTime, filesets[0].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
|
|
}
|
|
|
|
// Wait so that the next backups fall in the next retention interval.
|
|
Thread.Sleep(new TimeSpan(0, 0, 1, 0));
|
|
|
|
DateTime thirdBackupTime;
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
(var backupResults, _) = await this.RunPartialBackupAsync(c).ConfigureAwait(false);
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(1, backupResults.Warnings.Count());
|
|
thirdBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
|
|
// Since we stopped the backup, files were not deleted automatically
|
|
await c.DeleteAsync();
|
|
|
|
// Since the most recent backup is not considered in the retention logic, there are no backups in the first time
|
|
// frame. The original 2 backups have now spilled over to the U:1m specification. Since we keep the first
|
|
// backup in the interval, we should be left with the first backup, as well as the third partial one.
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(2, filesets.Count);
|
|
Assert.AreEqual(firstBackupTime, filesets[1].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
|
|
Assert.AreEqual(thirdBackupTime, filesets[0].Time);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[0].IsFullBackup);
|
|
}
|
|
|
|
DateTime fourthBackupTime;
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
this.ModifySourceFiles();
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
fourthBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
|
|
// Since the most recent backup is not considered in the retention logic, the third backup is the only backup
|
|
// in the first time frame. There is no further spillover, so we simply add the fourth backup to the
|
|
// collection of retained backups.
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(3, filesets.Count);
|
|
Assert.AreEqual(firstBackupTime, filesets[2].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[2].IsFullBackup);
|
|
Assert.AreEqual(thirdBackupTime, filesets[1].Time);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[1].IsFullBackup);
|
|
Assert.AreEqual(fourthBackupTime, filesets[0].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
|
|
|
|
this.ModifySourceFiles();
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
DateTime fifthBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
|
|
// Since the most recent backup is not considered in the retention logic, we now have two backups in the
|
|
// first time frame: the third (partial) and fourth (full). Since the first backup in each interval is
|
|
// kept, we would typically keep just the third backup. However, since we should not discard a full
|
|
// backup in favor of a partial one, we keep the fourth as well. We also still have the initial backup.
|
|
filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(4, filesets.Count);
|
|
Assert.AreEqual(firstBackupTime, filesets[3].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[3].IsFullBackup);
|
|
Assert.AreEqual(thirdBackupTime, filesets[2].Time);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[2].IsFullBackup);
|
|
Assert.AreEqual(fourthBackupTime, filesets[1].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
|
|
Assert.AreEqual(fifthBackupTime, filesets[0].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
|
|
}
|
|
|
|
// Wait so that the next backups fall in the next retention interval.
|
|
Thread.Sleep(new TimeSpan(0, 0, 1, 0));
|
|
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
this.ModifySourceFiles();
|
|
var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER });
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(0, backupResults.Warnings.Count());
|
|
var sixthBackupTime = (await c.ListAsync()).Filesets.First().Time;
|
|
|
|
// Since the most recent backup is not considered in the retention logic, we now have three backups in the
|
|
// second time frame: the third (partial), fourth (full), and fifth (full). Since we keep up to the first
|
|
// full backup in each time frame, we now drop the fifth backup.
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(4, filesets.Count);
|
|
Assert.AreEqual(firstBackupTime, filesets[3].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[3].IsFullBackup);
|
|
Assert.AreEqual(thirdBackupTime, filesets[2].Time);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, filesets[2].IsFullBackup);
|
|
Assert.AreEqual(fourthBackupTime, filesets[1].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
|
|
Assert.AreEqual(sixthBackupTime, filesets[0].Time);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
public async Task StopAfterCurrentFileAsync()
|
|
{
|
|
#if !DEBUG
|
|
Assert.Ignore("This test requires DEBUG to be defined");
|
|
#endif
|
|
|
|
Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions)
|
|
{
|
|
// Choose a dblock size that is small enough so that more than one volume is needed.
|
|
["dblock-size"] = "10mb",
|
|
["disable-file-scanner"] = "true",
|
|
["concurrency-fileprocessors"] = "1"
|
|
};
|
|
|
|
// Run a complete backup.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
|
|
Assert.AreEqual(1, (await c.ListAsync()).Filesets.Count());
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, (await c.ListAsync()).Filesets.Single(x => x.Version == 0).IsFullBackup);
|
|
}
|
|
|
|
// Run a partial backup.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
(var backupResults, var modified) = await this.RunPartialBackupAsync(c).ConfigureAwait(false);
|
|
Assert.AreEqual(0, backupResults.Errors.Count());
|
|
Assert.AreEqual(1, backupResults.Warnings.Count());
|
|
if (backupResults.ModifiedFiles == 0)
|
|
throw new Exception($"No files were modified, likely the stop was issued too early, list is: {string.Join(", ", modified)}");
|
|
Assert.That(backupResults.ModifiedFiles, Is.GreaterThan(0), "No files were added, likely the stop was issued too early");
|
|
Assert.That(backupResults.ModifiedFiles, Is.LessThan(fileSizes.Length), "All files were added, likely the stop was issued too late");
|
|
|
|
// If we interrupt the backup, the most recent Fileset should be marked as partial.
|
|
Assert.AreEqual(2, (await c.ListAsync()).Filesets.Count());
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, (await c.ListAsync()).Filesets.Single(x => x.Version == 1).IsFullBackup);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, (await c.ListAsync()).Filesets.Single(x => x.Version == 0).IsFullBackup);
|
|
}
|
|
|
|
// Restore files from the partial backup set.
|
|
Dictionary<string, string> restoreOptions = new Dictionary<string, string>(options) { ["restore-path"] = this.RESTOREFOLDER };
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
|
|
{
|
|
var lastResults = await c.ListAsync("*");
|
|
var partialVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray();
|
|
Assert.GreaterOrEqual(partialVersionFiles.Length, 1);
|
|
await c.RestoreAsync(partialVersionFiles);
|
|
|
|
foreach (string filepath in partialVersionFiles)
|
|
{
|
|
var filename = Path.GetFileName(filepath);
|
|
TestUtils.AssertFilesAreEqual(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), false, filename);
|
|
}
|
|
}
|
|
|
|
// Recreating the database should preserve the backup types.
|
|
File.Delete(this.DBFILE);
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.RepairAsync());
|
|
|
|
Assert.AreEqual(2, (await c.ListAsync()).Filesets.Count());
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, (await c.ListAsync()).Filesets.Single(x => x.Version == 1).IsFullBackup);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, (await c.ListAsync()).Filesets.Single(x => x.Version == 0).IsFullBackup);
|
|
}
|
|
|
|
// Run a complete backup. Listing the Filesets should include both full and partial backups.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
Assert.AreEqual(3, (await c.ListAsync()).Filesets.Count());
|
|
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, (await c.ListAsync()).Filesets.Single(x => x.Version == 2).IsFullBackup);
|
|
Assert.AreEqual(BackupType.PARTIAL_BACKUP, (await c.ListAsync()).Filesets.Single(x => x.Version == 1).IsFullBackup);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, (await c.ListAsync()).Filesets.Single(x => x.Version == 0).IsFullBackup);
|
|
}
|
|
|
|
// Restore files from the full backup set.
|
|
restoreOptions["overwrite"] = "true";
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
|
|
{
|
|
var lastResults = await c.ListAsync("*");
|
|
var fullVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray();
|
|
Assert.AreEqual(this.fileSizes.Length, fullVersionFiles.Length);
|
|
|
|
TestUtils.AssertResults(await c.RestoreAsync(fullVersionFiles));
|
|
|
|
foreach (string filepath in fullVersionFiles)
|
|
{
|
|
string filename = Path.GetFileName(filepath);
|
|
TestUtils.AssertFilesAreEqual(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), false, filename);
|
|
}
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
public async Task StopNowAsync()
|
|
{
|
|
#if !DEBUG
|
|
Assert.Ignore("This test requires DEBUG to be defined");
|
|
#endif
|
|
|
|
// Choose a dblock size that is small enough so that more than one volume is needed.
|
|
Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions)
|
|
{
|
|
["dblock-size"] = "10mb",
|
|
["disable-synthetic-filelist"] = "true",
|
|
["disable-file-scanner"] = "true",
|
|
["concurrency-fileprocessors"] = "1",
|
|
};
|
|
|
|
// Run a complete backup.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(1, filesets.Count);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
|
|
}
|
|
|
|
// Interrupt a backup with "abort".
|
|
this.ModifySourceFiles();
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
var startedTcs = new TaskCompletionSource<bool>();
|
|
c.OnOperationStarted += results =>
|
|
{
|
|
((BackupResults)results).OperationProgressUpdater.PhaseChanged += (phase, previousPhase) =>
|
|
{
|
|
if (phase == OperationPhase.Backup_ProcessingFiles)
|
|
startedTcs.SetResult(true);
|
|
};
|
|
};
|
|
|
|
Task backupTask = c.BackupAsync([DATAFOLDER]);
|
|
|
|
Task completedTask = await Task.WhenAny(startedTcs.Task, Task.Delay(5000));
|
|
if (completedTask == startedTcs.Task)
|
|
{
|
|
await c.AbortAsync();
|
|
try
|
|
{
|
|
await backupTask.ConfigureAwait(false);
|
|
Assert.Fail("Expected OperationCanceledException but none was thrown.");
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
// Success, exception thrown as expected
|
|
}
|
|
}
|
|
else
|
|
{
|
|
Assert.Fail("Operation did not reach ready state within timeout.");
|
|
}
|
|
}
|
|
|
|
// The next backup should proceed without issues.
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
|
{
|
|
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
|
|
|
var filesets = (await c.ListAsync()).Filesets.ToList();
|
|
Assert.AreEqual(2, filesets.Count);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[1].IsFullBackup);
|
|
Assert.AreEqual(BackupType.FULL_BACKUP, filesets[0].IsFullBackup);
|
|
}
|
|
|
|
// Restore from the backup that followed the interruption.
|
|
Dictionary<string, string> restoreOptions = new Dictionary<string, string>(options) { ["restore-path"] = this.RESTOREFOLDER };
|
|
using (var c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
|
|
{
|
|
var lastResults = await c.ListAsync("*");
|
|
var fullVersionFiles = lastResults.Files.Select(x => x.Path).Where(x => !Utility.IsFolder(x, File.GetAttributes)).ToArray();
|
|
Assert.AreEqual(this.fileSizes.Length, fullVersionFiles.Length);
|
|
|
|
TestUtils.AssertResults(await c.RestoreAsync(fullVersionFiles));
|
|
foreach (string filepath in fullVersionFiles)
|
|
{
|
|
string filename = Path.GetFileName(filepath);
|
|
TestUtils.AssertFilesAreEqual(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), false, filename);
|
|
}
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
public async Task TestUploadExceptionWithNoRetriesAsync()
|
|
{
|
|
var testopts = TestOptions;
|
|
testopts["number-of-retries"] = "0";
|
|
testopts["dblock-size"] = "10mb";
|
|
|
|
// Make a base backup
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new string[] { DATAFOLDER }));
|
|
|
|
// Ensure that the target folder only has a single dlist file
|
|
Assert.AreEqual(1, Directory.EnumerateFiles(TARGETFOLDER, "*.dlist.*").Count(), "There should be only one dlist file in the target folder");
|
|
|
|
// Make a new backup that fails uploading a dblock file
|
|
ModifySourceFiles();
|
|
|
|
// Deterministic error backend
|
|
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
|
|
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
|
|
|
|
var hasFailed = false;
|
|
var secondUploadStarted = false;
|
|
var secondUploadCompleted = false;
|
|
|
|
// Fail the compact after the first dblock put is completed
|
|
var uploads = new List<string>();
|
|
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
|
|
{
|
|
if (action.IsGetOperation)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
if (!hasFailed && action == DeterministicErrorBackend.BackendAction.PutBefore)
|
|
{
|
|
// We only fail one upload, but there are no retries
|
|
hasFailed = true;
|
|
|
|
// Make sure we can start a second upload
|
|
Thread.Sleep(1000);
|
|
return true;
|
|
}
|
|
|
|
if (action == DeterministicErrorBackend.BackendAction.PutBefore)
|
|
secondUploadStarted = true;
|
|
if (action == DeterministicErrorBackend.BackendAction.PutAfter)
|
|
secondUploadCompleted = true;
|
|
|
|
return false;
|
|
};
|
|
|
|
using (var c = new Controller(failtarget, testopts, null))
|
|
Assert.ThrowsAsync<DeterministicErrorBackend.DeterministicErrorBackendException>(() => c.BackupAsync(new string[] { DATAFOLDER }));
|
|
|
|
Assert.That(secondUploadStarted, Is.True, "Second upload was not started");
|
|
Assert.That(secondUploadCompleted, Is.True, "Second upload was not started");
|
|
Assert.That(hasFailed, Is.True, "Failed to fail the upload");
|
|
Assert.That(!uploads.Any(x => x.Contains("dlist")), Is.True, "Upload of dlist file was not skipped");
|
|
|
|
// Ensure that the target folder only has a single dlist file
|
|
Assert.AreEqual(1, Directory.EnumerateFiles(TARGETFOLDER, "*.dlist.*").Count(), $"There should be only one dlist file in the target folder: {string.Join(", ", uploads)}");
|
|
|
|
// Create a regular backup
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new string[] { DATAFOLDER }));
|
|
|
|
// Verify that all is in order
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
|
|
try
|
|
{
|
|
TestUtils.AssertResults(await c.TestAsync(long.MaxValue));
|
|
}
|
|
catch (TestUtils.TestVerificationException e)
|
|
{
|
|
using var db = await LocalDatabase.CreateLocalDatabaseAsync(testopts["dbpath"], "test", true, null, CancellationToken.None).ConfigureAwait(false);
|
|
using var cmd = db.Connection.CreateCommand();
|
|
|
|
var sb = new StringBuilder();
|
|
sb.AppendLine(e.Message);
|
|
sb.AppendLine(TestUtils.DumpTable(cmd, "Remotevolume", null));
|
|
sb.AppendLine(TestUtils.DumpTable(cmd, "File", null));
|
|
sb.AppendLine(TestUtils.DumpTable(cmd, "FilesetEntry", null));
|
|
sb.AppendLine(TestUtils.DumpTable(cmd, "Fileset", null));
|
|
|
|
sb.AppendLine("Files in the source folder:");
|
|
foreach (var fe in Directory.EnumerateFileSystemEntries(this.DATAFOLDER))
|
|
sb.AppendLine($"File: {fe}");
|
|
|
|
Assert.Fail(sb.ToString());
|
|
}
|
|
|
|
// Test that we can recreate
|
|
var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
|
|
if (File.Exists(recreatedDatabaseFile))
|
|
File.Delete(recreatedDatabaseFile);
|
|
|
|
testopts["dbpath"] = recreatedDatabaseFile;
|
|
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.RepairAsync());
|
|
|
|
// Check that we have 3 versions
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var listResults = await c.ListAsync();
|
|
TestUtils.AssertResults(listResults);
|
|
Assert.AreEqual(3, listResults.Filesets.Count());
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
[TestCase(true, false)]
|
|
[TestCase(false, false)]
|
|
[TestCase(true, true)]
|
|
[TestCase(false, true)]
|
|
public async Task TestUploadExceptionOnFirstDlistWithRepairAsync(bool before, bool failOnLastDblock)
|
|
{
|
|
var testopts = TestOptions;
|
|
testopts["number-of-retries"] = "0";
|
|
testopts["dblock-size"] = "10mb";
|
|
|
|
// Deterministic error backend
|
|
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
|
|
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
|
|
var operation = before
|
|
? DeterministicErrorBackend.BackendAction.PutBefore
|
|
: DeterministicErrorBackend.BackendAction.PutAfter;
|
|
|
|
// We expect to upload 7 volumes, and fail on the last one
|
|
var count = 7;
|
|
var failed = false;
|
|
|
|
// Fail when uploading the dlist file
|
|
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
|
|
{
|
|
if (failOnLastDblock)
|
|
{
|
|
if (action == operation && remotename.Contains(".dblock."))
|
|
{
|
|
if (Interlocked.Decrement(ref count) == 0)
|
|
{
|
|
// Allow others to progress
|
|
Thread.Sleep(2000);
|
|
failed = true;
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (action == operation && remotename.Contains(".dlist."))
|
|
{
|
|
failed = true;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
};
|
|
|
|
using (var c = new Controller(failtarget, testopts, null))
|
|
Assert.ThrowsAsync<DeterministicErrorBackend.DeterministicErrorBackendException>(() => c.BackupAsync(new string[] { DATAFOLDER }));
|
|
|
|
Assert.That(failed, Is.True, "Failed to fail the upload");
|
|
|
|
// Make spacing for the next backup
|
|
Thread.Sleep(3000);
|
|
|
|
// Issue repair
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.RepairAsync());
|
|
|
|
// Verify that all is in order
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
|
|
TestUtils.AssertResults(await c.TestAsync(long.MaxValue));
|
|
|
|
// If we fail on the dlist after the upload, it is promoted to a regular backup
|
|
var expectedFilesets = before || failOnLastDblock ? 2 : 1;
|
|
|
|
// Make a new backup, should continue as normal
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var backupResults = await c.BackupAsync(new string[] { DATAFOLDER });
|
|
TestUtils.AssertResults(backupResults);
|
|
Assert.AreEqual(expectedFilesets, (await c.ListAsync()).Filesets.Count());
|
|
}
|
|
|
|
// Test that we can recreate
|
|
var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
|
|
if (File.Exists(recreatedDatabaseFile))
|
|
File.Delete(recreatedDatabaseFile);
|
|
|
|
testopts["dbpath"] = recreatedDatabaseFile;
|
|
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.RepairAsync());
|
|
|
|
// Check that we have the correct versions
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var listResults = await c.ListAsync();
|
|
TestUtils.AssertResults(listResults);
|
|
Assert.AreEqual(expectedFilesets, listResults.Filesets.Count());
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
[TestCase(true, false)]
|
|
[TestCase(false, false)]
|
|
[TestCase(true, true)]
|
|
[TestCase(false, true)]
|
|
public async Task TestUploadExceptionOnFirstDlistAsync(bool before, bool failOnLastDblock)
|
|
{
|
|
var testopts = TestOptions;
|
|
testopts["number-of-retries"] = "0";
|
|
testopts["dblock-size"] = "10mb";
|
|
|
|
// Deterministic error backend
|
|
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
|
|
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
|
|
var operation = before
|
|
? DeterministicErrorBackend.BackendAction.PutBefore
|
|
: DeterministicErrorBackend.BackendAction.PutAfter;
|
|
|
|
// We expect to upload 7 volumes, and fail on the last one
|
|
var count = 7;
|
|
var failed = false;
|
|
|
|
// Fail when uploading the dlist file
|
|
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
|
|
{
|
|
if (failOnLastDblock)
|
|
{
|
|
if (action == operation && remotename.Contains(".dblock."))
|
|
{
|
|
if (Interlocked.Decrement(ref count) == 0)
|
|
{
|
|
// Allow others to progress
|
|
Thread.Sleep(2000);
|
|
failed = true;
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (action == operation && remotename.Contains(".dlist."))
|
|
{
|
|
failed = true;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
};
|
|
|
|
using (var c = new Controller(failtarget, testopts, null))
|
|
Assert.ThrowsAsync<DeterministicErrorBackend.DeterministicErrorBackendException>(() => c.BackupAsync(new string[] { DATAFOLDER }));
|
|
|
|
Assert.That(failed, Is.True, "Failed to fail the upload");
|
|
|
|
// Make spacing for the next backup
|
|
Thread.Sleep(3000);
|
|
|
|
// If we fail on the dlist after the upload, it is promoted to a regular backup
|
|
var expectedFilesets = before || failOnLastDblock ? 2 : 1;
|
|
|
|
// Make a new backup, should continue as normal
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var backupResults = await c.BackupAsync(new string[] { DATAFOLDER });
|
|
TestUtils.AssertResults(backupResults);
|
|
Assert.AreEqual(expectedFilesets, (await c.ListAsync()).Filesets.Count());
|
|
}
|
|
|
|
// Verify that all is in order
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
|
|
TestUtils.AssertResults(await c.TestAsync(long.MaxValue));
|
|
|
|
// Test that we can recreate
|
|
var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
|
|
if (File.Exists(recreatedDatabaseFile))
|
|
File.Delete(recreatedDatabaseFile);
|
|
|
|
testopts["dbpath"] = recreatedDatabaseFile;
|
|
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.RepairAsync());
|
|
|
|
// Check that we have 2 versions
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var listResults = await c.ListAsync();
|
|
TestUtils.AssertResults(listResults);
|
|
Assert.AreEqual(expectedFilesets, listResults.Filesets.Count());
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
[TestCase(true)]
|
|
[TestCase(false)]
|
|
public async Task TestRepairWithNoDlistAsync(bool before)
|
|
{
|
|
var testopts = TestOptions;
|
|
testopts["number-of-retries"] = "0";
|
|
testopts["dblock-size"] = "10mb";
|
|
|
|
// Deterministic error backend
|
|
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
|
|
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
|
|
var operation = before
|
|
? DeterministicErrorBackend.BackendAction.PutBefore
|
|
: DeterministicErrorBackend.BackendAction.PutAfter;
|
|
|
|
// Fail when uploading the first dlist file
|
|
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
|
|
{
|
|
if (action == operation && remotename.Contains(".dlist."))
|
|
return true;
|
|
|
|
return false;
|
|
};
|
|
|
|
using (var c = new Controller(failtarget, testopts, null))
|
|
Assert.ThrowsAsync<DeterministicErrorBackend.DeterministicErrorBackendException>(() => c.BackupAsync(new string[] { DATAFOLDER }));
|
|
|
|
// Test that we can repair
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.RepairAsync());
|
|
|
|
// Check that we have 1 version
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var listResults = await c.ListAsync();
|
|
TestUtils.AssertResults(listResults);
|
|
Assert.AreEqual(1, listResults.Filesets.Count());
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
[TestCase(true, true, 3, false)]
|
|
[TestCase(false, false, 3, false)]
|
|
[TestCase(true, false, 3, false)]
|
|
[TestCase(false, true, 3, false)]
|
|
[TestCase(true, true, 3, true)]
|
|
[TestCase(false, true, 3, true)]
|
|
[TestCase(false, false, 3, true)]
|
|
[TestCase(true, false, 3, true)]
|
|
public async Task TestMultiExceptionOnDlistAsync(bool before, bool modifyInBetween, int runs, bool withBase)
|
|
{
|
|
var testopts = TestOptions;
|
|
testopts["number-of-retries"] = "0";
|
|
testopts["dblock-size"] = "10mb";
|
|
// We always have at least 1 backup at the end
|
|
var expectedFilesets = 1;
|
|
// If the files are modified, this will create multiple versions
|
|
if (modifyInBetween)
|
|
{
|
|
// If we prevent the file from being uploaded,
|
|
// it will be a temporary file, and a new synthetic file
|
|
// will be attempted, but that fails as well
|
|
// If it is after, then the upload completes,
|
|
// and it will be promoted to a regular uploaded backup
|
|
expectedFilesets += before ? 1 : runs;
|
|
|
|
// If we modify with a base, then that creates a version as well
|
|
if (withBase)
|
|
expectedFilesets += 1;
|
|
}
|
|
else if (before && !withBase)
|
|
{
|
|
// Because an incomplete fileset is treated as a partial fileset,
|
|
// the final non-interrupted backup will see the syntheticly
|
|
// created fileset as a partial fileset, and create a new non-partial
|
|
// fileset, even though there are no changes between the two
|
|
// This does not happen with a base, because it simply does not
|
|
// attempt to upload the failing filesets
|
|
expectedFilesets += 1;
|
|
}
|
|
|
|
if (withBase)
|
|
{
|
|
// Make a regular backup
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new string[] { DATAFOLDER }));
|
|
}
|
|
|
|
|
|
// Deterministic error backend
|
|
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
|
|
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
|
|
var operation = before
|
|
? DeterministicErrorBackend.BackendAction.PutBefore
|
|
: DeterministicErrorBackend.BackendAction.PutAfter;
|
|
|
|
// Fail when uploading the first dlist file
|
|
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
|
|
{
|
|
if (action == operation && remotename.Contains(".dlist."))
|
|
return true;
|
|
|
|
return false;
|
|
};
|
|
|
|
for (var i = 0; i < runs; i++)
|
|
{
|
|
if (modifyInBetween)
|
|
ModifySourceFiles();
|
|
|
|
using (var c = new Controller(failtarget, testopts, null))
|
|
try
|
|
{
|
|
await c.BackupAsync(new string[] { DATAFOLDER });
|
|
}
|
|
catch (DeterministicErrorBackend.DeterministicErrorBackendException)
|
|
{
|
|
}
|
|
|
|
// Prevent clashes in timestamps
|
|
Thread.Sleep(3000);
|
|
}
|
|
|
|
if (modifyInBetween)
|
|
ModifySourceFiles();
|
|
|
|
// Make a new backup, should continue as normal
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var backupResults = await c.BackupAsync(new string[] { DATAFOLDER });
|
|
TestUtils.AssertResults(backupResults);
|
|
Assert.AreEqual(expectedFilesets, (await c.ListAsync()).Filesets.Count());
|
|
}
|
|
|
|
// Verify that all is in order
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
|
|
TestUtils.AssertResults(await c.TestAsync(long.MaxValue));
|
|
|
|
// Test that we can recreate
|
|
var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
|
|
if (File.Exists(recreatedDatabaseFile))
|
|
File.Delete(recreatedDatabaseFile);
|
|
|
|
testopts["dbpath"] = recreatedDatabaseFile;
|
|
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.RepairAsync());
|
|
|
|
// Check that we have 2 versions
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var listResults = await c.ListAsync();
|
|
TestUtils.AssertResults(listResults);
|
|
Assert.AreEqual(expectedFilesets, listResults.Filesets.Count());
|
|
}
|
|
}
|
|
|
|
private class LogSink : IMessageSink
|
|
{
|
|
public List<string> StartedFiles { get; } = new List<string>();
|
|
public List<string> CompletedFiles { get; } = new List<string>();
|
|
public void BackendEvent(BackendActionType action, BackendEventType type, string path, long size)
|
|
{
|
|
if (action == BackendActionType.Put && type == BackendEventType.Started)
|
|
{
|
|
this.StartedFiles.Add(path);
|
|
}
|
|
if (action == BackendActionType.Put && type == BackendEventType.Completed)
|
|
{
|
|
this.CompletedFiles.Add(path);
|
|
}
|
|
}
|
|
|
|
public void SetBackendProgress(IBackendProgress progress)
|
|
{
|
|
}
|
|
|
|
public void SetOperationProgress(IOperationProgress progress)
|
|
{
|
|
}
|
|
|
|
public void WriteMessage(LogEntry entry)
|
|
{
|
|
}
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
public async Task TestDblockUploadWithOperationCancellationAsync()
|
|
{
|
|
await TestUploadWithOperationCancellationAsync(".dblock.");
|
|
}
|
|
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
public async Task TestDindexUploadWithOperationCancellationAsync()
|
|
{
|
|
await TestUploadWithOperationCancellationAsync(".dindex.");
|
|
}
|
|
|
|
[Test]
|
|
[Category("Disruption")]
|
|
public async Task TestDlistUploadWithOperationCancellationAsync()
|
|
{
|
|
await TestUploadWithOperationCancellationAsync(".dlist.");
|
|
}
|
|
|
|
private async Task TestUploadWithOperationCancellationAsync(string filefragment)
|
|
{
|
|
var testopts = TestOptions;
|
|
testopts["number-of-retries"] = "1";
|
|
testopts["dblock-size"] = "10mb";
|
|
|
|
// Make a base backup
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new string[] { DATAFOLDER }));
|
|
|
|
// Make a new backup that fails uploading a dblock file
|
|
ModifySourceFiles();
|
|
|
|
// Deterministic error backend
|
|
Library.DynamicLoader.BackendLoader.AddBackend(new DeterministicErrorBackend());
|
|
var failtarget = new DeterministicErrorBackend().ProtocolKey + "://" + TARGETFOLDER;
|
|
var uploadCount = 0;
|
|
var failstep = DeterministicErrorBackend.BackendAction.PutBefore;
|
|
|
|
// Fail the upload before
|
|
DeterministicErrorBackend.ErrorGenerator = (DeterministicErrorBackend.BackendAction action, string remotename) =>
|
|
{
|
|
if (action == failstep && remotename.Contains(filefragment))
|
|
{
|
|
// Fail once with a cancellation
|
|
if (Interlocked.Increment(ref uploadCount) == 1)
|
|
throw new OperationCanceledException();
|
|
}
|
|
|
|
return false;
|
|
};
|
|
|
|
using (var c = new Controller(failtarget, testopts, null))
|
|
{
|
|
var sink = new LogSink();
|
|
await c.AppendSinkAsync(sink);
|
|
var res = await c.BackupAsync(new string[] { DATAFOLDER });
|
|
if (uploadCount == 0)
|
|
Assert.Fail("Upload count was not incremented");
|
|
|
|
Assert.AreEqual(2, (await c.ListAsync()).Filesets.Count());
|
|
Assert.AreEqual(2, Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).Count());
|
|
Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist
|
|
Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry
|
|
Assert.AreEqual(sink.CompletedFiles.Count(x => x.Contains(".dblock.")), sink.CompletedFiles.Count(x => x.Contains(".dindex.")));
|
|
Assert.AreEqual(1, res.BackendStatistics.RetryAttempts);
|
|
}
|
|
|
|
uploadCount = 0;
|
|
failstep = DeterministicErrorBackend.BackendAction.PutAfter;
|
|
|
|
ModifySourceFiles();
|
|
using (var c = new Controller(failtarget, testopts, null))
|
|
{
|
|
var sink = new LogSink();
|
|
await c.AppendSinkAsync(sink);
|
|
var res = await c.BackupAsync(new string[] { DATAFOLDER });
|
|
if (uploadCount == 0)
|
|
Assert.Fail("Upload count was not incremented");
|
|
|
|
Assert.AreEqual(3, (await c.ListAsync()).Filesets.Count());
|
|
Assert.AreEqual(3, Directory.GetFiles(TARGETFOLDER, "*.dlist.*", SearchOption.TopDirectoryOnly).Count());
|
|
Assert.AreEqual(7 * 2 + 1, sink.CompletedFiles.Count); // 7 completed (dblock + dindex) + 1 dlist
|
|
Assert.AreEqual(7 * 2 + 2, sink.StartedFiles.Count); // 1 retry
|
|
Assert.AreEqual(sink.CompletedFiles.Count(x => x.Contains(".dblock.")), sink.CompletedFiles.Count(x => x.Contains(".dindex.")));
|
|
Assert.AreEqual(1, res.BackendStatistics.RetryAttempts);
|
|
}
|
|
|
|
ModifySourceFiles();
|
|
// Create a regular backup
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.BackupAsync(new string[] { DATAFOLDER }));
|
|
|
|
// Verify that all is in order
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts.Expand(new { full_remote_verification = true }), null))
|
|
TestUtils.AssertResults(await c.TestAsync(long.MaxValue));
|
|
|
|
// Test that we can recreate
|
|
var recreatedDatabaseFile = Path.Combine(BASEFOLDER, "recreated-database.sqlite");
|
|
if (File.Exists(recreatedDatabaseFile))
|
|
File.Delete(recreatedDatabaseFile);
|
|
|
|
testopts["dbpath"] = recreatedDatabaseFile;
|
|
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
TestUtils.AssertResults(await c.RepairAsync());
|
|
|
|
// Check that we have 3 versions
|
|
using (var c = new Controller("file://" + TARGETFOLDER, testopts, null))
|
|
{
|
|
var listResults = await c.ListAsync();
|
|
TestUtils.AssertResults(listResults);
|
|
Assert.AreEqual(4, listResults.Filesets.Count());
|
|
}
|
|
}
|
|
}
|
|
} |