From 22582f36fc547a27a8f5b45883cb37f0180fb0bd Mon Sep 17 00:00:00 2001 From: Logan Freijo Date: Mon, 22 Feb 2016 02:18:55 -0500 Subject: [PATCH 1/4] Updated all references to code.google.com/p/duplicati to the new github repo. --- Installer/debian help/debian/control | 2 +- Installer/debian help/duplicati-make-git-snapshot.sh | 2 +- Installer/fedora/duplicati-make-git-snapshot.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Installer/debian help/debian/control b/Installer/debian help/debian/control index 24eca3674..f32dd6330 100644 --- a/Installer/debian help/debian/control +++ b/Installer/debian help/debian/control @@ -5,7 +5,7 @@ Maintainer: Kenneth Skovhede Build-Depends: debhelper (>= 8.0.0) Standards-Version: 3.9.4 Homepage: http://www.duplicati.com -Vcs-Git: https://code.google.com/p/duplicati +Vcs-Git: https://github.com/duplicati/duplicati.git Package: duplicati Architecture: all diff --git a/Installer/debian help/duplicati-make-git-snapshot.sh b/Installer/debian help/duplicati-make-git-snapshot.sh index 80d1c62db..05cd52aee 100644 --- a/Installer/debian help/duplicati-make-git-snapshot.sh +++ b/Installer/debian help/duplicati-make-git-snapshot.sh @@ -21,7 +21,7 @@ echo HEAD ${1:-HEAD} rm -rf $DIRNAME git clone ${REF:+--reference $REF} \ - https://code.google.com/p/duplicati/ $DIRNAME + https://github.com/duplicati/duplicati.git $DIRNAME cd "$DIRNAME" if [ -e "../oem.js" ]; then diff --git a/Installer/fedora/duplicati-make-git-snapshot.sh b/Installer/fedora/duplicati-make-git-snapshot.sh index 35b5f8ae7..b873d2f20 100644 --- a/Installer/fedora/duplicati-make-git-snapshot.sh +++ b/Installer/fedora/duplicati-make-git-snapshot.sh @@ -20,7 +20,7 @@ echo HEAD ${1:-HEAD} rm -rf $DIRNAME git clone ${REF:+--reference $REF} \ - https://code.google.com/p/duplicati/ $DIRNAME + https://github.com/duplicati/duplicati.git $DIRNAME cd "$DIRNAME" if [ -e "../oem.js" ]; then From 39a8f4eb7fe29b831fe9a9f607c1199a15e279d7 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 23 Feb 2016 20:56:24 +0100 Subject: [PATCH 2/4] Fixed an issue with re-creating index files from database in the case where the source file fills the blocklist with consecutive identical blocks. --- Duplicati/Library/Main/Database/LocalDatabase.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 9d9076e83..ec493a6f3 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -961,7 +961,7 @@ namespace Duplicati.Library.Main.Database while (rd.Read()) { var blockhash = rd.GetValue(0).ToString(); - if (blockhash != curHash && curHash != null) + if ((blockhash != curHash && curHash != null) || index + hashsize > buffer.Length) { yield return new Tuple(curHash, buffer, index); curHash = null; @@ -974,7 +974,7 @@ namespace Duplicati.Library.Main.Database index += hashsize; } - if (curHash != null) + if (curHash != null) yield return new Tuple(curHash, buffer, index); From e2c3ba91e6b03bb1bfbec406fd4a4d7b0dc705f1 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 23 Feb 2016 20:56:58 +0100 Subject: [PATCH 3/4] Added consistency check when re-building index files. --- Duplicati/Library/Main/Operation/RepairHandler.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Duplicati/Library/Main/Operation/RepairHandler.cs b/Duplicati/Library/Main/Operation/RepairHandler.cs index 5f0f2f355..039591fd1 100644 --- a/Duplicati/Library/Main/Operation/RepairHandler.cs +++ b/Duplicati/Library/Main/Operation/RepairHandler.cs @@ -239,6 +239,8 @@ namespace Duplicati.Library.Main.Operation var w = new IndexVolumeWriter(m_options); newEntry = w; w.SetRemoteFilename(n.Name); + + var h = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm); foreach(var blockvolume in db.GetBlockVolumesFromIndexName(n.Name)) { @@ -252,7 +254,13 @@ namespace Duplicati.Library.Main.Operation if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full) foreach(var b in db.GetBlocklists(volumeid, m_options.Blocksize, hashsize)) + { + var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); + if (bh != b.Item1) + throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1)); + w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3); + } } w.Close(); From 7e1067e99681c5c5c8d6809656d0e9ab66cbc49c Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 23 Feb 2016 20:57:18 +0100 Subject: [PATCH 4/4] Added extra test for the simple a + b test --- Duplicati/UnitTest/Issue1410.cs | 82 +++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/Duplicati/UnitTest/Issue1410.cs b/Duplicati/UnitTest/Issue1410.cs index 7cc7d6ef5..13f8f38ee 100644 --- a/Duplicati/UnitTest/Issue1410.cs +++ b/Duplicati/UnitTest/Issue1410.cs @@ -14,12 +14,88 @@ // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -using System; using System.IO; using NUnit.Framework; using System.Linq; using System.Collections.Generic; +using System; +using System.IO; +using NUnit.Framework; +using System.Linq; +using System.Collections.Generic; namespace Duplicati.UnitTest { - [TestFixture] public class Issue1410 : BasicSetupHelper - { [TestFixtureSetUp()] public override void PrepareSourceData() { base.PrepareSourceData(); Directory.CreateDirectory(DATAFOLDER); Directory.CreateDirectory(TARGETFOLDER); } [Test] public void RunCommands() { var data = new byte[1024 * 1024 * 10]; //File.WriteAllText(Path.Combine(DATAFOLDER, "a"), "hi"); File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null)) c.Backup(new string[] { DATAFOLDER }); new Random().NextBytes(data); //File.WriteAllText(Path.Combine(DATAFOLDER, "b"), "there"); File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data); using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null)) c.Backup(new string[] { DATAFOLDER }); using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 0 }), null)) { var r = c.List("*"); Console.WriteLine("Newest before deleting:"); Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path))); Assert.AreEqual(3, r.Files.Count()); } using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 0, no_local_db = true }), null)) { var r = c.List("*"); Console.WriteLine("Newest without db:"); Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path))); Assert.AreEqual(3, r.Files.Count()); } File.Delete(DBFILE); using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null)) c.Repair(); using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null)) Assert.AreEqual(c.List().Filesets.Count(), 2); using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 1 }), null)) { var r = c.List("*"); Console.WriteLine("Oldest after delete:"); Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path))); Assert.AreEqual(2, r.Files.Count()); } using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 0 }), null)) { var r = c.List("*"); Console.WriteLine("Newest after delete:"); Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path))); Assert.AreEqual(3, r.Files.Count()); } } + [TestFixture] + public class Issue1410 : BasicSetupHelper + { + [TestFixtureSetUp()] + public override void PrepareSourceData() + { + base.PrepareSourceData(); + + Directory.CreateDirectory(DATAFOLDER); + Directory.CreateDirectory(TARGETFOLDER); + } + + [Test] + public void RunCommands() + { + TestOptions["verbose"] = "true"; + + var data = new byte[1024 * 1024 * 10]; + File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data); + using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null)) + c.Backup(new string[] { DATAFOLDER }); + + using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 0 }), null)) + { + var r = c.List("*"); + Console.WriteLine("In first backup:"); + Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path))); + } + + new Random().NextBytes(data); + File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data); + using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null)) + c.Backup(new string[] { DATAFOLDER }); + + using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 0 }), null)) + { + var r = c.List("*"); + Console.WriteLine("Newest before deleting:"); + Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path))); + Assert.AreEqual(3, r.Files.Count()); + } + + using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 0, no_local_db = true }), null)) + { + var r = c.List("*"); + Console.WriteLine("Newest without db:"); + Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path))); + Assert.AreEqual(3, r.Files.Count()); + } + + + File.Delete(DBFILE); + using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null)) + c.Repair(); + + using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null)) + Assert.AreEqual(c.List().Filesets.Count(), 2); + + using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 1 }), null)) + { + var r = c.List("*"); + Console.WriteLine("Oldest after delete:"); + Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path))); + Assert.AreEqual(2, r.Files.Count()); + } + + using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 0 }), null)) + { + var r = c.List("*"); + Console.WriteLine("Newest after delete:"); + Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path))); + Assert.AreEqual(3, r.Files.Count()); + } + } } }