Merge branch 'master' into concurrent_processing

Duplicati/Library/Main/Database/LocalDatabase.cs

@@ -965,7 +965,7 @@ namespace Duplicati.Library.Main.Database
                     while (rd.Read())
                     {
                         var blockhash = rd.GetValue(0).ToString();
-                        if (blockhash != curHash && curHash != null)
+                        if ((blockhash != curHash && curHash != null) || index + hashsize > buffer.Length)
                         {
                             yield return new Tuple<string, byte[], int>(curHash, buffer, index);
                             curHash = null;
@@ -978,7 +978,7 @@ namespace Duplicati.Library.Main.Database
                         index += hashsize;
                     }
 
-                if (curHash != null)                    
+                if (curHash != null)
                     yield return new Tuple<string, byte[], int>(curHash, buffer, index);
 
 

Duplicati/Library/Main/Operation/RepairHandler.cs

@@ -239,6 +239,8 @@ namespace Duplicati.Library.Main.Operation
                                 var w = new IndexVolumeWriter(m_options);
                                 newEntry = w;
                                 w.SetRemoteFilename(n.Name);
+
+                                var h = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
 								
                                 foreach(var blockvolume in db.GetBlockVolumesFromIndexName(n.Name))
                                 {								
@@ -252,7 +254,13 @@ namespace Duplicati.Library.Main.Operation
                                     
                                     if (m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
                                         foreach(var b in db.GetBlocklists(volumeid, m_options.Blocksize, hashsize))
+                                        {
+                                            var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3));
+                                            if (bh != b.Item1)
+                                                throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1));
+                                            
                                             w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3);
+                                        }
                                 }
 								
                                 w.Close();

Duplicati/UnitTest/Issue1410.cs

@@ -14,12 +14,88 @@
 //  You should have received a copy of the GNU Lesser General Public
 //  License along with this library; if not, write to the Free Software
 //  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-using System;
+using System;
+using System.IO;
+using NUnit.Framework;
+using System.Linq;
+using System.Collections.Generic;
 
 namespace Duplicati.UnitTest
 {
-using System.Collections.Generic;
-
+    [TestFixture]
+    public class Issue1410 : BasicSetupHelper
+    {
+        [TestFixtureSetUp()]
+        public override void PrepareSourceData()
+        {
+            base.PrepareSourceData();
+
+            Directory.CreateDirectory(DATAFOLDER);
+            Directory.CreateDirectory(TARGETFOLDER);
+        }
+
+        [Test]
+        public void RunCommands()
+        {
+            TestOptions["verbose"] = "true";
+
+            var data = new byte[1024 * 1024 * 10];
+            File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
+            using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null))
+                c.Backup(new string[] { DATAFOLDER });
+
+            using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 0 }), null))
+            {
+                var r = c.List("*");
+                Console.WriteLine("In first backup:");
+                Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
+            }
+
+            new Random().NextBytes(data);
+            File.WriteAllBytes(Path.Combine(DATAFOLDER, "b"), data);
+            using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null))
+                c.Backup(new string[] { DATAFOLDER });
+
+            using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 0 }), null))
+            {
+                var r = c.List("*");
+                Console.WriteLine("Newest before deleting:");
+                Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
+                Assert.AreEqual(3, r.Files.Count());
+            }
+
+            using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 0, no_local_db = true }), null))
+            {
+                var r = c.List("*");
+                Console.WriteLine("Newest without db:");
+                Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
+                Assert.AreEqual(3, r.Files.Count());
+            }
+
+
+            File.Delete(DBFILE);
+            using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null))
+                c.Repair();
+
+            using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions, null))
+                Assert.AreEqual(c.List().Filesets.Count(), 2);
+
+            using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 1 }), null))
+            {
+                var r = c.List("*");
+                Console.WriteLine("Oldest after delete:");
+                Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
+                Assert.AreEqual(2, r.Files.Count());
+            }
+                    
+            using(var c = new Library.Main.Controller("file://" + TARGETFOLDER, TestOptions.Expand(new { version = 0 }), null))
+            {
+                var r = c.List("*");
+                Console.WriteLine("Newest after delete:");
+                Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
+                Assert.AreEqual(3, r.Files.Count());
+            }
+        }
     }
 }
 

Installer/debian help/debian/control

@@ -5,7 +5,7 @@ Maintainer: Kenneth Skovhede <kenneth@duplicati.com>
 Build-Depends: debhelper (>= 8.0.0)
 Standards-Version: 3.9.4
 Homepage: http://www.duplicati.com
-Vcs-Git: https://code.google.com/p/duplicati
+Vcs-Git: https://github.com/duplicati/duplicati.git
 
 Package: duplicati
 Architecture: all

Installer/debian help/duplicati-make-git-snapshot.sh

@@ -21,7 +21,7 @@ echo HEAD ${1:-HEAD}
 rm -rf $DIRNAME
 
 git clone ${REF:+--reference $REF} \
-         https://code.google.com/p/duplicati/ $DIRNAME
+         https://github.com/duplicati/duplicati.git $DIRNAME
 
 cd "$DIRNAME"
 if [ -e "../oem.js" ]; then

Installer/fedora/duplicati-make-git-snapshot.sh

@@ -20,7 +20,7 @@ echo HEAD ${1:-HEAD}
 rm -rf $DIRNAME
 
 git clone ${REF:+--reference $REF} \
-         https://code.google.com/p/duplicati/ $DIRNAME
+         https://github.com/duplicati/duplicati.git $DIRNAME
 
 cd "$DIRNAME"
 if [ -e "../oem.js" ]; then