Kenneth Skovhede
76aae5de7b
Now we only look at the timestamp from the previous fileset, instead of for the last time the file was seen. This should improve lookup times, and thus the running time for backups with many files. This change means that the following sequence of operations will react differently: - backup - delete file - backup - undelete file - backup Prior to this commit, the undeleted file would be seen with the original timestamp and thus can avoid being scanned for changes. After this commit, the file will be treated as a new file, even if it has the previous timestamp and will thus be scanned for changes. An additional switch has been introduced, --check-filetime-only that improves the lookup further by not looking at the metadata or filesize, and thus makes a much lighter call to the database, where fewer tables need to be touched.
317 lines
12 KiB
C#
317 lines
12 KiB
C#
// Copyright (C) 2015, The Duplicati Team
|
|
// http://www.duplicati.com, info@duplicati.com
|
|
//
|
|
// This library is free software; you can redistribute it and/or modify
|
|
// it under the terms of the GNU Lesser General Public License as
|
|
// published by the Free Software Foundation; either version 2.1 of the
|
|
// License, or (at your option) any later version.
|
|
//
|
|
// This library is distributed in the hope that it will be useful, but
|
|
// WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
// Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public
|
|
// License along with this library; if not, write to the Free Software
|
|
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
using System;
|
|
using NUnit.Framework;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Collections.Generic;
|
|
|
|
namespace Duplicati.UnitTest
|
|
{
|
|
[TestFixture]
|
|
public class BorderTests : BasicSetupHelper
|
|
{
|
|
public override void PrepareSourceData()
|
|
{
|
|
base.PrepareSourceData();
|
|
|
|
Directory.CreateDirectory(DATAFOLDER);
|
|
Directory.CreateDirectory(TARGETFOLDER);
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void Run10k()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(1024 * 10);
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void Run10mb()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(1024 * 10, modifyOptions: opts => {
|
|
opts["blocksize"] = "10mb";
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void Run100k()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(1024 * 100);
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void Run12345_1()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(12345);
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void Run12345_2()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(12345, 1024 * 1024 * 10);
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void RunNoMetadata()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(1024 * 10, modifyOptions: opts => {
|
|
opts["skip-metadata"] = "true";
|
|
});
|
|
}
|
|
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void RunMD5()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(1024 * 10, modifyOptions: opts => {
|
|
opts["block-hash-algorithm"] = "MD5";
|
|
opts["file-hash-algorithm"] = "MD5";
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void RunSHA384()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(1024 * 10, modifyOptions: opts => {
|
|
opts["block-hash-algorithm"] = "SHA384";
|
|
opts["file-hash-algorithm"] = "SHA384";
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void RunMixedBlockFile_1()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(1024 * 10, modifyOptions: opts => {
|
|
opts["block-hash-algorithm"] = "MD5";
|
|
opts["file-hash-algorithm"] = "SHA1";
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void RunMixedBlockFile_2()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(1024 * 10, modifyOptions: opts => {
|
|
opts["block-hash-algorithm"] = "MD5";
|
|
opts["file-hash-algorithm"] = "SHA256";
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void RunNoIndexFiles()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(1024 * 10, modifyOptions: opts => {
|
|
opts["index-file-policy"] = "None";
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void RunSlimIndexFiles()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(1024 * 10, modifyOptions: opts => {
|
|
opts["index-file-policy"] = "Lookup";
|
|
});
|
|
}
|
|
|
|
[Test]
|
|
[Category("Border")]
|
|
public void RunQuickTimestamps()
|
|
{
|
|
PrepareSourceData();
|
|
RunCommands(1024 * 10, modifyOptions: opts =>
|
|
{
|
|
opts["check-filetime-only"] = "true";
|
|
});
|
|
}
|
|
public static Dictionary<string, int> WriteTestFilesToFolder(string targetfolder, int blocksize, int basedatasize = 0)
|
|
{
|
|
if (basedatasize <= 0)
|
|
basedatasize = blocksize * 1024;
|
|
|
|
var filenames = new Dictionary<string, int>();
|
|
filenames[""] = basedatasize;
|
|
filenames["-0"] = 0;
|
|
filenames["-1"] = 1;
|
|
|
|
filenames["-p1"] = basedatasize + 1;
|
|
filenames["-p2"] = basedatasize + 2;
|
|
filenames["-p500"] = basedatasize + 500;
|
|
filenames["-m1"] = basedatasize - 1;
|
|
filenames["-m2"] = basedatasize - 2;
|
|
filenames["-m500"] = basedatasize - 500;
|
|
|
|
filenames["-s1"] = blocksize / 4 + 6;
|
|
filenames["-s2"] = blocksize / 10 + 6;
|
|
filenames["-l1"] = blocksize * 4 + 6;
|
|
filenames["-l2"] = blocksize * 10 + 6;
|
|
|
|
filenames["-bm1"] = blocksize - 1;
|
|
filenames["-b"] = blocksize;
|
|
filenames["-bp1"] = blocksize + 1;
|
|
|
|
var data = new byte[filenames.Select(x => x.Value).Max()];
|
|
|
|
foreach (var k in filenames)
|
|
File.WriteAllBytes(Path.Combine(targetfolder, "a" + k.Key), data.Take(k.Value).ToArray());
|
|
|
|
return filenames;
|
|
}
|
|
|
|
private void RunCommands(int blocksize, int basedatasize = 0, Action<Dictionary<string, string>> modifyOptions = null)
|
|
{
|
|
var testopts = TestOptions;
|
|
testopts["verbose"] = "true";
|
|
testopts["blocksize"] = blocksize.ToString() + "b";
|
|
if (modifyOptions != null)
|
|
modifyOptions(testopts);
|
|
|
|
var filenames = WriteTestFilesToFolder(DATAFOLDER, blocksize, basedatasize);
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
|
c.Backup(new string[] { DATAFOLDER });
|
|
|
|
// After the first backup we remove the --blocksize argument as that should be auto-set
|
|
testopts.Remove("blocksize");
|
|
testopts.Remove("block-hash-algorithm");
|
|
testopts.Remove("file-hash-algorithm");
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
|
|
{
|
|
var r = c.List("*");
|
|
//Console.WriteLine("In first backup:");
|
|
//Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
|
|
}
|
|
|
|
var data = new byte[filenames.Select(x => x.Value).Max()];
|
|
new Random().NextBytes(data);
|
|
foreach(var k in filenames)
|
|
File.WriteAllBytes(Path.Combine(DATAFOLDER, "b" + k.Key), data.Take(k.Value).ToArray());
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
|
c.Backup(new string[] { DATAFOLDER });
|
|
|
|
var rn = new Random();
|
|
foreach(var k in filenames)
|
|
{
|
|
rn.NextBytes(data);
|
|
File.WriteAllBytes(Path.Combine(DATAFOLDER, "c" + k.Key), data.Take(k.Value).ToArray());
|
|
}
|
|
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
|
c.Backup(new string[] { DATAFOLDER });
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
|
|
{
|
|
var r = c.List("*");
|
|
//ProgressWriteLine("Newest before deleting:");
|
|
//ProgressWriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
|
|
Assert.AreEqual((filenames.Count * 3) + 1, r.Files.Count());
|
|
}
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0, no_local_db = true }), null))
|
|
{
|
|
var r = c.List("*");
|
|
//ProgressWriteLine("Newest without db:");
|
|
//ProgressWriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
|
|
Assert.AreEqual((filenames.Count * 3) + 1, r.Files.Count());
|
|
}
|
|
|
|
var newdb = Path.Combine(Path.GetDirectoryName(DBFILE), Path.ChangeExtension(Path.GetFileNameWithoutExtension(DBFILE) + "-recreated", Path.GetExtension(DBFILE)));
|
|
if (File.Exists(newdb))
|
|
File.Delete(newdb);
|
|
|
|
testopts["dbpath"] = newdb;
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
|
c.Repair();
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
|
Assert.AreEqual(3, c.List().Filesets.Count());
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 2 }), null))
|
|
{
|
|
var r = c.List("*");
|
|
//ProgressWriteLine("V2 after delete:");
|
|
//ProgressWriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
|
|
Assert.AreEqual((filenames.Count * 1) + 1, r.Files.Count());
|
|
}
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 1 }), null))
|
|
{
|
|
var r = c.List("*");
|
|
//ProgressWriteLine("V1 after delete:");
|
|
//ProgressWriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
|
|
Assert.AreEqual((filenames.Count * 2) + 1, r.Files.Count());
|
|
}
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = 0 }), null))
|
|
{
|
|
var r = c.List("*");
|
|
//ProgressWriteLine("Newest after delete:");
|
|
//ProgressWriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
|
|
Assert.AreEqual((filenames.Count * 3) + 1, r.Files.Count());
|
|
}
|
|
|
|
if (Directory.Exists(RESTOREFOLDER))
|
|
Directory.Delete(RESTOREFOLDER, true);
|
|
Directory.CreateDirectory(RESTOREFOLDER);
|
|
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = RESTOREFOLDER, no_local_blocks = true }), null))
|
|
{
|
|
var r = c.Restore(null);
|
|
Assert.AreEqual(filenames.Count * 3, r.FilesRestored);
|
|
}
|
|
|
|
TestUtils.VerifyDir(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"));
|
|
|
|
using(var tf = new Library.Utility.TempFolder())
|
|
{
|
|
using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { restore_path = (string)tf, no_local_blocks = true }), null))
|
|
{
|
|
var r = c.Restore(new string[] { Path.Combine(DATAFOLDER, "a") + "*" });
|
|
Assert.AreEqual(filenames.Count, r.FilesRestored);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|