diff --git a/Duplicati/CommandLine/UnitTest.cs b/Duplicati/CommandLine/UnitTest.cs index ec9dcc52e..bf23423ce 100644 --- a/Duplicati/CommandLine/UnitTest.cs +++ b/Duplicati/CommandLine/UnitTest.cs @@ -25,6 +25,7 @@ using System.Collections.Generic; using System.Text; using Duplicati.Library.Logging; using Duplicati.Library.Utility; +using System.Linq; namespace Duplicati.CommandLine { @@ -90,6 +91,11 @@ namespace Duplicati.CommandLine string tempdir = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "tempdir"); + //Filter empty entries, commonly occuring with copy/paste and newlines + folders = (from x in folders + where !string.IsNullOrWhiteSpace(x) + select x).ToArray(); + try { if (System.IO.Directory.Exists(tempdir)) @@ -202,7 +208,7 @@ namespace Duplicati.CommandLine if (options.ContainsKey("fh-dbpath")) { entries = new List(); - foreach (var el in Duplicati.Library.Main.Interface.ParseFhFileList(target, options)) + foreach (var el in Duplicati.Library.Main.Interface.ParseFhFileList(target, options, null)) if (el.FileType == Library.Main.ForestHash.RemoteVolumeType.Files) entries.Add(el.Time.ToLocalTime()); @@ -318,32 +324,35 @@ namespace Duplicati.CommandLine if (partialFolders.ContainsKey("")) partialFolders.Remove(""); + if (partialFolders.ContainsKey(System.IO.Path.DirectorySeparatorChar.ToString())) + partialFolders.Remove(System.IO.Path.DirectorySeparatorChar.ToString()); Dictionary tops = new Dictionary(options); - List filterlist = new List(); - filterlist.AddRange(partialFolders.Keys); - filterlist.AddRange(testfiles); + List filterlist; if (options.ContainsKey("fh-dbpath")) { //We need a minor tweak here because the two models // use releative vs. absolute paths - List t = new List(); - foreach(var s in filterlist) - t.Add(System.IO.Path.Combine(folders[i], s)); + filterlist = (from n in partialFolders.Keys + where !string.IsNullOrWhiteSpace(n) && n != System.IO.Path.DirectorySeparatorChar.ToString() + select Utility.AppendDirSeparator(System.IO.Path.Combine(folders[i], n))) + .Union(testfiles) //Add files with full path + .Union(new string[] { Utility.AppendDirSeparator(folders[i]) }) //Ensure root folder is included + .Distinct().ToList(); - filterlist = t; - if (!filterlist.Contains(Utility.AppendDirSeparator(folders[i]))) - filterlist.Add(Utility.AppendDirSeparator(folders[i])); - - t = new List(); - foreach (var s in testfiles) - t.Add(s.Substring(Utility.AppendDirSeparator(folders[i]).Length)); - testfiles = t; + testfiles = (from n in testfiles select n.Substring(Utility.AppendDirSeparator(folders[i]).Length)).ToList(); } + else + { + filterlist = new List(); + filterlist.AddRange(partialFolders.Keys); + filterlist.AddRange(testfiles); + } + tops["file-to-restore"] = String.Join(System.IO.Path.PathSeparator.ToString(), filterlist.ToArray()); - using (new Timer("Partial restore of " + folders[i])) + /*using (new Timer("Partial restore of " + folders[i])) Log.WriteMessage(Duplicati.Library.Main.Interface.Restore(target, new string[] { ptf }, tops), LogMessageType.Information); Console.WriteLine("Verifying partial restore of: " + folders[i]); @@ -385,7 +394,7 @@ namespace Duplicati.CommandLine Console.WriteLine("Partial restore file differs: " + s); } } - } + }*/ } using (new Timer("Restore of " + folders[i])) diff --git a/Duplicati/Library/Compression/FileArchiveDirectory.cs b/Duplicati/Library/Compression/FileArchiveDirectory.cs index 6889e8347..0d39c858a 100644 --- a/Duplicati/Library/Compression/FileArchiveDirectory.cs +++ b/Duplicati/Library/Compression/FileArchiveDirectory.cs @@ -76,7 +76,7 @@ namespace Duplicati.Library.Compression /// /// An optional prefix that is used to filter the list /// A filtered list of filenames - public IEnumerable> GetFilesWithSize(string prefix) + public IEnumerable> ListFilesWithSize(string prefix) { if (prefix == null) prefix = ""; List> res = new List>(); diff --git a/Duplicati/Library/Compression/FileArchiveZip.cs b/Duplicati/Library/Compression/FileArchiveZip.cs index 21ca5ba8e..94357758f 100644 --- a/Duplicati/Library/Compression/FileArchiveZip.cs +++ b/Duplicati/Library/Compression/FileArchiveZip.cs @@ -230,7 +230,7 @@ namespace Duplicati.Library.Compression /// /// The prefix to match /// A list of files matching the prefix - public IEnumerable> GetFilesWithSize(string prefix) + public IEnumerable> ListFilesWithSize(string prefix) { List> results = new List>(); foreach (IArchiveEntry e in Archive.Entries) diff --git a/Duplicati/Library/Interface/ICompression.cs b/Duplicati/Library/Interface/ICompression.cs index cd0139fe0..6dd78896a 100644 --- a/Duplicati/Library/Interface/ICompression.cs +++ b/Duplicati/Library/Interface/ICompression.cs @@ -71,7 +71,7 @@ namespace Duplicati.Library.Interface /// /// An optional prefix for limiting the files returned /// All files in the archive, matching the prefix, if any - IEnumerable> GetFilesWithSize(string prefix); + IEnumerable> ListFilesWithSize(string prefix); /// /// Returns a stream with data from the given file diff --git a/Duplicati/Library/Main/BackupStatistics.cs b/Duplicati/Library/Main/BackupStatistics.cs index 849d34fc8..319012539 100644 --- a/Duplicati/Library/Main/BackupStatistics.cs +++ b/Duplicati/Library/Main/BackupStatistics.cs @@ -25,8 +25,6 @@ namespace Duplicati.Library.Main { internal class BackupStatistics : CommunicationStatistics { - private DateTime m_beginTime; - private DateTime m_endTime; private long m_deletedFiles; private long m_deletedFolders; private long m_modifiedFiles; @@ -40,6 +38,11 @@ namespace Duplicati.Library.Main private long m_addedFolders; private long m_tooLargeFiles; private long m_filesWithError; + private long m_modifiedFolders; + private long m_modifiedSymlinks; + private long m_addedSymlinks; + private long m_deletedSymlinks; + private bool m_full = false; private bool m_partialBackup = false; private string m_typeReason = null; @@ -47,7 +50,6 @@ namespace Duplicati.Library.Main public BackupStatistics(DuplicatiOperationMode operationMode) : base(operationMode) { - m_beginTime = m_endTime = DateTime.Now; } public bool Full @@ -56,23 +58,6 @@ namespace Duplicati.Library.Main set { m_full = value; } } - public DateTime BeginTime - { - get { return m_beginTime; } - set { m_beginTime = value; } - } - - public DateTime EndTime - { - get { return m_endTime; } - set { m_endTime = value; } - } - - public TimeSpan Duration - { - get { return m_endTime - m_beginTime; } - } - public long DeletedFiles { get { return m_deletedFiles; } @@ -139,6 +124,31 @@ namespace Duplicati.Library.Main set { m_addedFolders = value; } } + public long ModifiedFolders + { + get { return m_modifiedFolders; } + set { m_modifiedFolders = value; } + } + + public long AddedSymlinks + { + get { return m_addedSymlinks; } + set { m_addedSymlinks = value; } + } + + public long ModifiedSymlinks + { + get { return m_modifiedSymlinks; } + set { m_modifiedSymlinks = value; } + } + + public long DeletedSymlinks + { + get { return m_deletedSymlinks; } + set { m_deletedSymlinks = value; } + } + + public long FilesTooLarge { get { return m_tooLargeFiles; } @@ -166,17 +176,20 @@ namespace Duplicati.Library.Main { //TODO: Figure out how to translate this without breaking the output parser StringBuilder sb = new StringBuilder(); + sb.Append(base.ToString()); sb.Append("BackupType : " + (this.Full ? "Full" : "Incremental") + "\r\n"); + if (this.m_typeReason != null) sb.Append("TypeReason : " + this.m_typeReason + "\r\n"); - sb.Append("BeginTime : " + this.BeginTime.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); - sb.Append("EndTime : " + this.EndTime.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); - sb.Append("Duration : " + this.Duration.ToString() + "\r\n"); sb.Append("DeletedFiles : " + this.DeletedFiles.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); sb.Append("DeletedFolders : " + this.DeletedFolders.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); + sb.Append("DeletedSymlinks : " + this.DeletedSymlinks.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); sb.Append("ModifiedFiles : " + this.ModifiedFiles.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); + sb.Append("ModifiedFolders : " + this.ModifiedFolders.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); + sb.Append("ModifiedSymlinks: " + this.ModifiedSymlinks.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); sb.Append("AddedFiles : " + this.AddedFiles.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); sb.Append("AddedFolders : " + this.AddedFolders.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); + sb.Append("AddedSymlinks : " + this.AddedSymlinks.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); sb.Append("ExaminedFiles : " + this.ExaminedFiles.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); sb.Append("OpenedFiles : " + this.OpenedFiles.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); sb.Append("SizeOfModified : " + this.SizeOfModifiedFiles.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); @@ -188,7 +201,6 @@ namespace Duplicati.Library.Main if (m_partialBackup) sb.Append("PartialBackup : true" + "\r\n"); - sb.Append(base.ToString()); return sb.ToString(); } } diff --git a/Duplicati/Library/Main/CommunicationStatistics.cs b/Duplicati/Library/Main/CommunicationStatistics.cs index 60d7d0f71..cf50228fd 100644 --- a/Duplicati/Library/Main/CommunicationStatistics.cs +++ b/Duplicati/Library/Main/CommunicationStatistics.cs @@ -26,6 +26,8 @@ namespace Duplicati.Library.Main public class CommunicationStatistics { private object m_lock = new object(); + private DateTime m_beginTime; + private DateTime m_endTime; private bool m_verboseErrors = false; private long m_numberOfBytesUploaded; private long m_numberOfRemoteCalls; @@ -47,6 +49,7 @@ namespace Duplicati.Library.Main public CommunicationStatistics(DuplicatiOperationMode operationMode) { m_operationMode = operationMode; + m_beginTime = m_endTime = DateTime.Now; } public DuplicatiOperationMode OperationMode { get { return m_operationMode; } } @@ -96,6 +99,23 @@ namespace Duplicati.Library.Main m_numberOfRemoteCalls += value; } + public DateTime BeginTime + { + get { return m_beginTime; } + set { m_beginTime = value; } + } + + public DateTime EndTime + { + get { return m_endTime; } + set { m_endTime = value; } + } + + public TimeSpan Duration + { + get { return m_endTime - m_beginTime; } + } + public void LogError(string errorMessage, Exception ex) { lock (m_lock) @@ -134,12 +154,15 @@ namespace Duplicati.Library.Main { //TODO: Figure out how to translate this without breaking the output parser StringBuilder sb = new StringBuilder(); - sb.Append("Executable : " + Utility.Utility.getEntryAssembly().FullName + "\r\n"); - sb.Append("Library : " + System.Reflection.Assembly.GetExecutingAssembly().FullName + "\r\n"); sb.Append("OperationName : " + this.OperationMode.ToString() + "\r\n"); + sb.Append("BeginTime : " + this.BeginTime.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); + sb.Append("EndTime : " + this.EndTime.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); + sb.Append("Duration : " + this.Duration.ToString() + "\r\n"); sb.Append("BytesUploaded : " + this.NumberOfBytesUploaded.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); sb.Append("BytesDownloaded : " + this.NumberOfBytesDownloaded.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); sb.Append("RemoteCalls : " + this.NumberOfRemoteCalls.ToString(System.Globalization.CultureInfo.InvariantCulture) + "\r\n"); + sb.Append("Executable : " + Utility.Utility.getEntryAssembly().FullName + "\r\n"); + sb.Append("Library : " + System.Reflection.Assembly.GetExecutingAssembly().FullName + "\r\n"); if (m_numberOfErrors > 0) { diff --git a/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql b/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql index 1c592c4c3..dfb8db3fc 100644 --- a/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql +++ b/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql @@ -29,15 +29,15 @@ in a single operation, grouped by an operationID */ -/*TODO: There should be a operation/fileset map -so we can reduce the storage used for repeated -entries of unchanged files */ +CREATE TABLE "OperationFileset" ( + "OperationID" INTEGER NOT NULL, + "FilesetID" INTEGER NOT NULL, + "Scantime" DATETIME NOT NULL +); CREATE TABLE "Fileset" ( "ID" INTEGER PRIMARY KEY, - "OperationID" INTEGER NOT NULL, "Path" TEXT NOT NULL, - "Scantime" DATETIME NOT NULL, "BlocksetID" INTEGER NOT NULL, "MetadataID" INTEGER NOT NULL ); @@ -47,7 +47,7 @@ The blocklist hashes are hashes of fragments of the blocklists */ CREATE TABLE "BlocklistHash" ( - "BlocksetID" INTEGER PRIMARY KEY, + "BlocksetID" INTEGER NOT NULL, "Index" INTEGER NOT NULL, "Hash" TEXT NOT NULL ); diff --git a/Duplicati/Library/Main/ForestHash/Database/ExtensionMethods.cs b/Duplicati/Library/Main/ForestHash/Database/ExtensionMethods.cs index edc488772..06c232c0d 100644 --- a/Duplicati/Library/Main/ForestHash/Database/ExtensionMethods.cs +++ b/Duplicati/Library/Main/ForestHash/Database/ExtensionMethods.cs @@ -30,7 +30,7 @@ namespace Duplicati.Library.Main.ForestHash.Database ((System.Data.IDataParameter)self.Parameters[index]).Value = value; } - public static long ExecuteNonQuery(this System.Data.IDbCommand self, string cmd, params object[] values) + public static int ExecuteNonQuery(this System.Data.IDbCommand self, string cmd, params object[] values) { if (cmd != null) self.CommandText = cmd; @@ -60,18 +60,6 @@ namespace Duplicati.Library.Main.ForestHash.Database return self.ExecuteScalar(); } - public static long GetLastRowID(this System.Data.IDbConnection self, System.Data.IDbTransaction tr = null) - { - using (var c = self.CreateCommand()) - { - if (tr != null) - c.Transaction = tr; - c.CommandText = "SELECT last_insert_rowid()"; - return Convert.ToInt64(c.ExecuteScalar()); - } - - } - public static void DumpSQL(this System.Data.IDbConnection self, string sql, params object[] parameters) { using (var c = self.CreateCommand()) diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs index 01bdb4c82..aac95d332 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs @@ -1,354 +1,585 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace Duplicati.Library.Main.ForestHash.Database -{ - public class LocalBackupDatabase : Localdatabase - { - private readonly System.Data.IDbCommand m_findblockCommand; - private readonly System.Data.IDbCommand m_findblocksetCommand; - private readonly System.Data.IDbCommand m_findfilesetCommand; - private readonly System.Data.IDbCommand m_findmetadatasetCommand; - - private readonly System.Data.IDbCommand m_insertblockCommand; - - private readonly System.Data.IDbCommand m_insertfileCommand; - - private readonly System.Data.IDbCommand m_insertblocksetCommand; - private readonly System.Data.IDbCommand m_insertblocksetentryCommand; - private readonly System.Data.IDbCommand m_insertblocklistHashesCommand; - - private readonly System.Data.IDbCommand m_insertmetadatasetCommand; - - private readonly System.Data.IDbCommand m_selectfileCommand; - private readonly System.Data.IDbCommand m_selectblocklistHashesCommand; - - private readonly System.Data.IDbCommand m_selectfileidsCommand; - private readonly System.Data.IDbCommand m_findremotevolumestateCommand; - private readonly System.Data.IDbCommand m_updateblockCommand; - - private readonly System.Data.IDbCommand m_createremotevolumeCommand; - - public LocalBackupDatabase(string path) - : base(path, "Backup") - { - m_findblockCommand = m_connection.CreateCommand(); - m_insertblockCommand = m_connection.CreateCommand(); - m_insertfileCommand = m_connection.CreateCommand(); - m_insertblocksetCommand = m_connection.CreateCommand(); - m_selectfileCommand = m_connection.CreateCommand(); - m_insertmetadatasetCommand = m_connection.CreateCommand(); - m_findblocksetCommand = m_connection.CreateCommand(); - m_findmetadatasetCommand = m_connection.CreateCommand(); - m_findfilesetCommand = m_connection.CreateCommand(); - m_insertblocksetentryCommand = m_connection.CreateCommand(); - m_selectfileidsCommand = m_connection.CreateCommand(); - m_findremotevolumestateCommand = m_connection.CreateCommand(); - m_updateblockCommand = m_connection.CreateCommand(); - m_createremotevolumeCommand = m_connection.CreateCommand(); - m_insertblocklistHashesCommand = m_connection.CreateCommand(); - m_selectblocklistHashesCommand = m_connection.CreateCommand(); - - m_findblockCommand.CommandText = @"SELECT ""File"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?"; - m_findblockCommand.AddParameters(2); - - m_findblocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Fullhash"" = ? AND ""Length"" = ?"; - m_findblocksetCommand.AddParameters(2); - - m_findmetadatasetCommand.CommandText = @"SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = (SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ?)"; - m_findmetadatasetCommand.AddParameters(2); - - m_findfilesetCommand.CommandText = @"SELECT ""ID"" FROM ""Fileset"" WHERE ""BlocksetID"" = ? AND ""MetadatasetID"" = ?"; - m_findfilesetCommand.AddParameters(2); - - m_insertblockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""File"", ""Size"") VALUES (?, ?, ?)"; - m_insertblockCommand.AddParameters(3); - - m_insertfileCommand.CommandText = @"INSERT INTO ""Fileset"" (""OperationID"", ""Path"", ""Scantime"",""BlocksetID"", ""MetadataID"") VALUES (?, ?, ? ,?, ?)"; - m_insertfileCommand.AddParameter(m_operationid); - m_insertfileCommand.AddParameters(4); - - m_insertblocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?, ?)"; - m_insertblocksetCommand.AddParameters(2); - - m_insertblocksetentryCommand.CommandText = @"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT ? AS A, ? AS B, ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?"; - m_insertblocksetentryCommand.AddParameters(4); - - m_insertblocklistHashesCommand.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?, ?, ?)"; - m_insertblocklistHashesCommand.AddParameters(3); - - m_insertmetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") SELECT ""ID"" FROM ""Blockset"" WHERE ""FullHash"" = ? AND ""Length"" = ?"; - m_insertmetadatasetCommand.AddParameters(2); - - m_selectfileCommand.CommandText = @"SELECT ""A"".""ID"", ""A"".""Scantime"", ""B"".""Length"", ""B"".""FullHash"", ""E"".""FullHash"", ""E"".""Length"", ""B"".""ID"" FROM ""Fileset"" A, ""Blockset"" B, ""Metadataset"" C, ""Operation"" D, ""Blockset"" E WHERE ""A"".""OperationID"" = ""D"".""ID"" AND ""A"".""Path"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"" ORDER BY ""D"".""Timestamp"" DESC "; - m_selectfileCommand.AddParameters(1); - - m_selectblocklistHashesCommand.CommandText = @"SELECT ""Hash"" FROM ""BlocklistHash"" WHERE ""BlocksetID"" = ? ORDER BY ""Index"" ASC "; - m_selectblocklistHashesCommand.AddParameters(1); - - m_selectfileidsCommand.CommandText = @"SELECT ""Path"", ""BlocksetID"", ""MetadataID"" FROM ""Fileset"" WHERE ""ID"" = ?"; - m_selectfileidsCommand.AddParameters(1); - - m_findremotevolumestateCommand.CommandText = @"SELECT ""State"" FROM ""Remotevolume"" WHERE ""Name"" = ?"; - m_findremotevolumestateCommand.AddParameters(1); - - m_updateblockCommand.CommandText = @"UPDATE ""Block"" SET ""File"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? "; - m_updateblockCommand.AddParameters(3); - - m_createremotevolumeCommand.CommandText = @"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"") VALUES (?, ?, ?, ?)"; - m_createremotevolumeCommand.AddParameter(m_operationid); - m_createremotevolumeCommand.AddParameters(3); - } - - /// - /// Adds a block to the local database, returning a value indicating if the value presents a new block - /// - /// The block key - /// The name of the archive that holds the data - /// True if the block should be added to the current output - public bool AddBlock(string key, long size, string archivename) - { - //TODO: Should have some of this in local memory for fast lookup, the DB is a bit slow - var r = m_findblockCommand.ExecuteScalar(null, key, size); - if (r == null || r == DBNull.Value) - { - m_insertblockCommand.SetParameterValue(0, key); - m_insertblockCommand.SetParameterValue(1, archivename); - m_insertblockCommand.SetParameterValue(2, size); - m_insertblockCommand.ExecuteNonQuery(); - return true; - } - else - { - r = m_findremotevolumestateCommand.ExecuteScalar(null, r); - if (r != null && (r.ToString() == RemoteVolumeState.Temporary.ToString() || r.ToString() == RemoteVolumeState.Uploading.ToString() || r.ToString() == RemoteVolumeState.Uploaded.ToString() || r.ToString() == RemoteVolumeState.Verified.ToString())) - { - return false; - } - else - { - m_updateblockCommand.ExecuteNonQuery(null, archivename, key, size); - return true; - } - - } - } - - - /// - /// Adds a blockset to the database, returns a value indicating if the blockset is new - /// - /// The hash of the blockset - /// The size of the blockset - /// The fragmentoffset for the last block - /// The hash of the fragment - /// The list of hashes - /// The id of the blockset, new or old - /// True if the blockset was created, false otherwise - public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, out long blocksetid) - { - object r = m_findblocksetCommand.ExecuteScalar(null, filehash, size); - if (r != null) - { - blocksetid = Convert.ToInt64(r); - //TODO: Make a sweep to see if the blocks match the record? - return false; - } - - using (var tr = m_connection.BeginTransaction()) - { - m_insertblocksetCommand.Transaction = tr; - m_insertblocksetCommand.ExecuteNonQuery(null, size, filehash); - - blocksetid = m_connection.GetLastRowID(tr); - - long ix = 0; - if (blocklistHashes != null) - { - m_insertblocklistHashesCommand.SetParameterValue(0, blocksetid); - m_insertblocklistHashesCommand.Transaction = tr; - foreach (var bh in blocklistHashes) - { - m_insertblocklistHashesCommand.SetParameterValue(1, ix); - m_insertblocklistHashesCommand.SetParameterValue(2, bh); - m_insertblocklistHashesCommand.ExecuteNonQuery(); - } - } - - m_insertblocksetentryCommand.SetParameterValue(0, blocksetid); - m_insertblocksetentryCommand.Transaction = tr; - - - ix = 0; - long remainsize = size; - foreach (var h in hashes) - { - m_insertblocksetentryCommand.SetParameterValue(1, ix); - m_insertblocksetentryCommand.SetParameterValue(2, h); - m_insertblocksetentryCommand.SetParameterValue(3, remainsize < blocksize ? remainsize : blocksize); - var c = m_insertblocksetentryCommand.ExecuteNonQuery(); - if (c != 1) - throw new Exception(string.Format("Unexpected result count: {0}, expected {1}", c, 1)); - ix++; - remainsize -= blocksize; - } - - tr.Commit(); - } - - return true; - - } - - /// - /// Adds a metadata set to the database, and returns a value indicating if the record was new - /// - /// The metadata hash - /// The id of the metadata set - /// True if the set was added to the database, false otherwise - public bool AddMetadataset(string hash, long size, out long metadataid) - { - if (size > 0) - { - object r = m_findmetadatasetCommand.ExecuteScalar(null, hash, size); - if (r != null) - { - metadataid = Convert.ToInt64(r); - return false; - } - - long blocksetid; - AddBlockset(hash, size, (int)size, new string[] { hash }, null, out blocksetid); - - using (var tr = m_connection.BeginTransaction()) - { - m_insertmetadatasetCommand.Transaction = tr; - var c = m_insertmetadatasetCommand.ExecuteNonQuery(null, hash, size); - if (c != 1) - throw new Exception(string.Format("Unexpected result count: {0}, expected: {1}", c, 1)); - - metadataid = m_connection.GetLastRowID(tr); - tr.Commit(); - } - - return true; - } - - metadataid = -2; - return false; - - } - - /// - /// Adds a file record to the database - /// - /// The path to the file - /// The time the file was scanned - /// The ID of the hashkey for the file - /// The ID for the metadata - public void AddFile(string filename, DateTime scantime, long blocksetID, long metadataID) - { - m_insertfileCommand.SetParameterValue(1, filename); - m_insertfileCommand.SetParameterValue(2, scantime); - m_insertfileCommand.SetParameterValue(3, blocksetID); - m_insertfileCommand.SetParameterValue(4, metadataID); - m_insertfileCommand.ExecuteNonQuery(); - } - - /// - /// Gets a file entry from the database - /// - /// The path to the file - /// The id of the file, or -1 - /// The size of the file, or -1 - /// The time the file was last scanned, or the current time - /// The hash of the file - /// The hash of the metadata - /// True if the entry was found, false otherwise - public bool GetFileEntry(string path, out long fileid, out long filesize, out DateTime lastScanned, out string hash, out string metahash, out long metasize, out IList blockHashes) - { - ((System.Data.IDataParameter)m_selectfileCommand.Parameters[0]).Value = path; - using (var rd = m_selectfileCommand.ExecuteReader()) - if (rd.Read()) - { - fileid = Convert.ToInt64(rd.GetValue(0)); - lastScanned = Convert.ToDateTime(rd.GetValue(1)); - filesize = Convert.ToInt64(rd.GetValue(2)); - object h = rd.GetValue(3); - hash = (h == null || h == DBNull.Value) ? null : h.ToString(); - h = rd.GetValue(4); - metahash = (h == null || h == DBNull.Value) ? null : h.ToString(); - metasize = Convert.ToInt64(rd.GetValue(5)); - - m_selectblocklistHashesCommand.SetParameterValue(0, rd.GetValue(6)); - using (var bhrd = m_selectblocklistHashesCommand.ExecuteReader()) - { - blockHashes = new List(); - while (bhrd.Read()) - { - var blh = bhrd.GetValue(0); - if (blh != null && blh != DBNull.Value) - blockHashes.Add(blh.ToString()); - } - - if (blockHashes.Count == 0) - blockHashes = null; - } - - return true; - } - else - { - fileid = -1; - lastScanned = DateTime.UtcNow; - filesize = -1; - hash = null; - metahash = null; - metasize = -1; - blockHashes = null; - return false; - } - - } - - public void AddUnmodifiedFile(long fileid, DateTime scantime) - { - long metadataID; - long blocksetID; - string path; - - m_selectfileidsCommand.SetParameterValue(0, fileid); - using (var rd = m_selectfileidsCommand.ExecuteReader()) - { - path = rd.GetValue(0).ToString(); - blocksetID = Convert.ToInt64(rd.GetValue(1)); - metadataID = Convert.ToInt64(rd.GetValue(2)); - } - - AddFile(path, scantime, blocksetID, metadataID); - } - - public void AddDirectoryEntry(string path, long metadataID, DateTime scantime) - { - AddFile(path, scantime, FOLDER_BLOCKSET_ID, metadataID); - } - - public void RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state) - { - lock (m_lock) - { - m_createremotevolumeCommand.SetParameterValue(1, name); - m_createremotevolumeCommand.SetParameterValue(2, type.ToString()); - m_createremotevolumeCommand.SetParameterValue(3, state.ToString()); - m_createremotevolumeCommand.ExecuteNonQuery(); - } - } - - public void AddSymlinkEntry(string path, long metadataID, DateTime scantime) - { - AddFile(path, scantime, SYMLINK_BLOCKSET_ID, metadataID); - } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; + +namespace Duplicati.Library.Main.ForestHash.Database +{ + public class LocalBackupDatabase : Localdatabase + { + private readonly System.Data.IDbCommand m_findblockCommand; + private readonly System.Data.IDbCommand m_findblocksetCommand; + private readonly System.Data.IDbCommand m_findfilesetCommand; + private readonly System.Data.IDbCommand m_findmetadatasetCommand; + + private readonly System.Data.IDbCommand m_insertblockCommand; + + private readonly System.Data.IDbCommand m_insertfileCommand; + + private readonly System.Data.IDbCommand m_insertblocksetCommand; + private readonly System.Data.IDbCommand m_insertblocksetentryCommand; + private readonly System.Data.IDbCommand m_insertblocklistHashesCommand; + + private readonly System.Data.IDbCommand m_insertmetadatasetCommand; + + //private readonly System.Data.IDbCommand m_selectfileCommand; + private readonly System.Data.IDbCommand m_selectfileSimpleCommand; + private readonly System.Data.IDbCommand m_selectfileHashCommand; + private readonly System.Data.IDbCommand m_selectblocklistHashesCommand; + + private readonly System.Data.IDbCommand m_findremotevolumestateCommand; + private readonly System.Data.IDbCommand m_updateblockCommand; + + private readonly System.Data.IDbCommand m_createremotevolumeCommand; + private readonly System.Data.IDbCommand m_insertfileOperationCommand; + + public LocalBackupDatabase(string path) + : this(CreateConnection(path)) + { + } + + public LocalBackupDatabase(LocalRestoredatabase connection) + : this(connection.Connection) + { + } + + public LocalBackupDatabase(System.Data.IDbConnection connection) + : base(connection, "Backup") + { + m_findblockCommand = m_connection.CreateCommand(); + m_insertblockCommand = m_connection.CreateCommand(); + m_insertfileCommand = m_connection.CreateCommand(); + m_insertblocksetCommand = m_connection.CreateCommand(); + //m_selectfileCommand = m_connection.CreateCommand(); + m_insertmetadatasetCommand = m_connection.CreateCommand(); + m_findblocksetCommand = m_connection.CreateCommand(); + m_findmetadatasetCommand = m_connection.CreateCommand(); + m_findfilesetCommand = m_connection.CreateCommand(); + m_insertblocksetentryCommand = m_connection.CreateCommand(); + m_findremotevolumestateCommand = m_connection.CreateCommand(); + m_updateblockCommand = m_connection.CreateCommand(); + m_createremotevolumeCommand = m_connection.CreateCommand(); + m_insertblocklistHashesCommand = m_connection.CreateCommand(); + m_selectblocklistHashesCommand = m_connection.CreateCommand(); + m_insertfileOperationCommand = m_connection.CreateCommand(); + m_selectfileSimpleCommand = m_connection.CreateCommand(); + m_selectfileHashCommand = m_connection.CreateCommand(); + + m_findblockCommand.CommandText = @"SELECT ""File"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?"; + m_findblockCommand.AddParameters(2); + + m_findblocksetCommand.CommandText = @"SELECT ""ID"" FROM ""Blockset"" WHERE ""Fullhash"" = ? AND ""Length"" = ?"; + m_findblocksetCommand.AddParameters(2); + + m_findmetadatasetCommand.CommandText = @"SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" = (SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ?)"; + m_findmetadatasetCommand.AddParameters(2); + + m_findfilesetCommand.CommandText = @"SELECT ""ID"" FROM ""Fileset"" WHERE ""BlocksetID"" = ? AND ""MetadatasetID"" = ?"; + m_findfilesetCommand.AddParameters(2); + + m_insertblockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""File"", ""Size"") VALUES (?, ?, ?)"; + m_insertblockCommand.AddParameters(3); + + m_insertfileOperationCommand.CommandText = @"INSERT INTO ""OperationFileset"" (""OperationID"", ""FilesetID"", ""Scantime"") VALUES (?, ?, ?)"; + m_insertfileOperationCommand.AddParameter(m_operationid); + m_insertfileOperationCommand.AddParameters(2); + + m_insertfileCommand.CommandText = @"INSERT INTO ""Fileset"" (""Path"",""BlocksetID"", ""MetadataID"") VALUES (?, ? ,?); SELECT last_insert_rowid();"; + m_insertfileCommand.AddParameters(3); + + m_insertblocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?, ?); SELECT last_insert_rowid();"; + m_insertblocksetCommand.AddParameters(2); + + m_insertblocksetentryCommand.CommandText = @"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT ? AS A, ? AS B, ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?"; + m_insertblocksetentryCommand.AddParameters(4); + + m_insertblocklistHashesCommand.CommandText = @"INSERT INTO ""BlocklistHash"" (""BlocksetID"", ""Index"", ""Hash"") VALUES (?, ?, ?)"; + m_insertblocklistHashesCommand.AddParameters(3); + + m_insertmetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();"; + m_insertmetadatasetCommand.AddParameter(); + + m_selectfileSimpleCommand.CommandText = @"SELECT ""OperationFileset"".""FilesetID"", ""OperationFileset"".""Scantime"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""Fileset"".""Path"" = ? ORDER BY ""OperationFileset"".""Scantime"" DESC"; + m_selectfileSimpleCommand.AddParameters(1); + + m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"" INNER JOIN ""Fileset"" ON ""Blockset"".""ID"" = ""Fileset"".""BlocksetID"" WHERE ""Fileset"".""ID"" = ? "; + m_selectfileHashCommand.AddParameters(1); + + //m_selectfileCommand.CommandText = @"SELECT ""A"".""ID"", ""D"".""Scantime"", ""B"".""Length"", ""B"".""FullHash"", ""E"".""FullHash"", ""E"".""Length"", ""B"".""ID"" FROM ""Fileset"" A, ""Blockset"" B, ""Metadataset"" C, ""OperationFileset"" D, ""Blockset"" E, ""Operation"" F WHERE ""A"".""ID"" = ""D"".""FilesetID"" AND ""A"".""Path"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"" AND ""F"".""ID"" = ""D"".""OperationID"" ORDER BY ""F"".""Timestamp"" DESC "; + //m_selectfileCommand.AddParameters(1); + + m_selectblocklistHashesCommand.CommandText = @"SELECT ""Hash"" FROM ""BlocklistHash"" WHERE ""BlocksetID"" = ? ORDER BY ""Index"" ASC "; + m_selectblocklistHashesCommand.AddParameters(1); + + m_findremotevolumestateCommand.CommandText = @"SELECT ""State"" FROM ""Remotevolume"" WHERE ""Name"" = ?"; + m_findremotevolumestateCommand.AddParameters(1); + + m_updateblockCommand.CommandText = @"UPDATE ""Block"" SET ""File"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? "; + m_updateblockCommand.AddParameters(3); + + m_createremotevolumeCommand.CommandText = @"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"") VALUES (?, ?, ?, ?)"; + m_createremotevolumeCommand.AddParameter(m_operationid); + m_createremotevolumeCommand.AddParameters(3); + } + + /// + /// Adds a block to the local database, returning a value indicating if the value presents a new block + /// + /// The block key + /// The name of the archive that holds the data + /// True if the block should be added to the current output + public bool AddBlock(string key, long size, string archivename, System.Data.IDbTransaction transaction = null) + { + //TODO: Should have some of this in local memory for fast lookup, the DB is a bit slow + m_findblockCommand.Transaction = transaction; + var r = m_findblockCommand.ExecuteScalar(null, key, size); + if (r == null || r == DBNull.Value) + { + m_insertblockCommand.Transaction = transaction; + m_insertblockCommand.SetParameterValue(0, key); + m_insertblockCommand.SetParameterValue(1, archivename); + m_insertblockCommand.SetParameterValue(2, size); + m_insertblockCommand.ExecuteNonQuery(); + return true; + } + else + { + m_findremotevolumestateCommand.Transaction = transaction; + r = m_findremotevolumestateCommand.ExecuteScalar(null, r); + if (r != null && (r.ToString() == RemoteVolumeState.Temporary.ToString() || r.ToString() == RemoteVolumeState.Uploading.ToString() || r.ToString() == RemoteVolumeState.Uploaded.ToString() || r.ToString() == RemoteVolumeState.Verified.ToString())) + { + return false; + } + else + { + m_updateblockCommand.Transaction = transaction; + m_updateblockCommand.ExecuteNonQuery(null, archivename, key, size); + return true; + } + + } + } + + + /// + /// Adds a blockset to the database, returns a value indicating if the blockset is new + /// + /// The hash of the blockset + /// The size of the blockset + /// The fragmentoffset for the last block + /// The hash of the fragment + /// The list of hashes + /// The id of the blockset, new or old + /// True if the blockset was created, false otherwise + public bool AddBlockset(string filehash, long size, int blocksize, IEnumerable hashes, IEnumerable blocklistHashes, out long blocksetid, System.Data.IDbTransaction transaction = null) + { + m_findblocksetCommand.Transaction = transaction; + object r = m_findblocksetCommand.ExecuteScalar(null, filehash, size); + if (r != null) + { + blocksetid = Convert.ToInt64(r); + //TODO: Make a sweep to see if the blocks match the record? + return false; + } + + using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + { + m_insertblocksetCommand.Transaction = tr.Parent; + blocksetid = Convert.ToInt64(m_insertblocksetCommand.ExecuteScalar(null, size, filehash)); + + long ix = 0; + if (blocklistHashes != null) + { + m_insertblocklistHashesCommand.SetParameterValue(0, blocksetid); + m_insertblocklistHashesCommand.Transaction = tr.Parent; + foreach (var bh in blocklistHashes) + { + m_insertblocklistHashesCommand.SetParameterValue(1, ix); + m_insertblocklistHashesCommand.SetParameterValue(2, bh); + m_insertblocklistHashesCommand.ExecuteNonQuery(); + ix++; + } + } + + m_insertblocksetentryCommand.SetParameterValue(0, blocksetid); + m_insertblocksetentryCommand.Transaction = tr.Parent; + + + ix = 0; + long remainsize = size; + foreach (var h in hashes) + { + m_insertblocksetentryCommand.SetParameterValue(1, ix); + m_insertblocksetentryCommand.SetParameterValue(2, h); + m_insertblocksetentryCommand.SetParameterValue(3, remainsize < blocksize ? remainsize : blocksize); + var c = m_insertblocksetentryCommand.ExecuteNonQuery(); + if (c != 1) + throw new Exception(string.Format("Unexpected result count: {0}, expected {1}", c, 1)); + ix++; + remainsize -= blocksize; + } + + tr.Commit(); + } + + return true; + + } + + /// + /// Adds a metadata set to the database, and returns a value indicating if the record was new + /// + /// The metadata hash + /// The id of the metadata set + /// True if the set was added to the database, false otherwise + public bool AddMetadataset(string hash, long size, out long metadataid, System.Data.IDbTransaction transaction = null) + { + if (size > 0) + { + m_findmetadatasetCommand.Transaction = transaction; + object r = m_findmetadatasetCommand.ExecuteScalar(null, hash, size); + if (r != null && r != DBNull.Value) + { + metadataid = Convert.ToInt64(r); + return false; + } + + long blocksetid; + AddBlockset(hash, size, (int)size, new string[] { hash }, null, out blocksetid, transaction); + + using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + { + m_insertmetadatasetCommand.Transaction = tr.Parent; + metadataid = Convert.ToInt64(m_insertmetadatasetCommand.ExecuteScalar(null, blocksetid)); + tr.Commit(); + } + + return true; + } + + metadataid = -2; + return false; + + } + + /// + /// Adds a file record to the database + /// + /// The path to the file + /// The time the file was scanned + /// The ID of the hashkey for the file + /// The ID for the metadata + public void AddFile(string filename, DateTime scantime, long blocksetID, long metadataID, System.Data.IDbTransaction transaction = null) + { + m_findfilesetCommand.Transaction = transaction; + m_findfilesetCommand.SetParameterValue(0, blocksetID); + m_findfilesetCommand.SetParameterValue(1, metadataID); + var fileid = m_findfilesetCommand.ExecuteScalar(); + if (fileid == null || fileid == DBNull.Value) + { + using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + { + m_insertfileCommand.Transaction = tr.Parent; + m_insertfileCommand.SetParameterValue(0, filename); + m_insertfileCommand.SetParameterValue(1, blocksetID); + m_insertfileCommand.SetParameterValue(2, metadataID); + fileid = Convert.ToInt64(m_insertfileCommand.ExecuteScalar()); + tr.Commit(); + } + } + + m_insertfileOperationCommand.Transaction = transaction; + m_insertfileOperationCommand.SetParameterValue(1, fileid); + m_insertfileOperationCommand.SetParameterValue(2, scantime); + m_insertfileOperationCommand.ExecuteNonQuery(); + + } + + /// + /// Gets a file entry from the database + /// + /// The path to the file + /// The id of the file, or -1 + /// The size of the file, or -1 + /// The time the file was last scanned, or the current time + /// The hash of the file + /// The hash of the metadata + /// True if the entry was found, false otherwise + /*public bool GetFileEntry(string path, out long fileid, out long filesize, out DateTime lastScanned, out string hash, out string metahash, out long metasize, out IList blockHashes) + { + ((System.Data.IDataParameter)m_selectfileCommand.Parameters[0]).Value = path; + using (var rd = m_selectfileCommand.ExecuteReader()) + if (rd.Read()) + { + fileid = Convert.ToInt64(rd.GetValue(0)); + lastScanned = Convert.ToDateTime(rd.GetValue(1)); + filesize = Convert.ToInt64(rd.GetValue(2)); + object h = rd.GetValue(3); + hash = (h == null || h == DBNull.Value) ? null : h.ToString(); + h = rd.GetValue(4); + metahash = (h == null || h == DBNull.Value) ? null : h.ToString(); + metasize = Convert.ToInt64(rd.GetValue(5)); + + m_selectblocklistHashesCommand.SetParameterValue(0, rd.GetValue(6)); + using (var bhrd = m_selectblocklistHashesCommand.ExecuteReader()) + { + blockHashes = new List(); + while (bhrd.Read()) + { + var blh = bhrd.GetValue(0); + if (blh != null && blh != DBNull.Value) + blockHashes.Add(blh.ToString()); + } + + if (blockHashes.Count == 0) + blockHashes = null; + } + + return true; + } + else + { + fileid = -1; + lastScanned = DateTime.UtcNow; + filesize = -1; + hash = null; + metahash = null; + metasize = -1; + blockHashes = null; + return false; + } + + }*/ + + public void AddUnmodifiedFile(long fileid, DateTime scantime, System.Data.IDbTransaction transaction = null) + { + m_insertfileOperationCommand.Transaction = transaction; + m_insertfileOperationCommand.SetParameterValue(1, fileid); + m_insertfileOperationCommand.SetParameterValue(2, scantime); + m_insertfileOperationCommand.ExecuteNonQuery(); + } + + public void AddDirectoryEntry(string path, long metadataID, DateTime scantime, System.Data.IDbTransaction transaction = null) + { + AddFile(path, scantime, FOLDER_BLOCKSET_ID, metadataID, transaction); + } + + public void RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state) + { + lock (m_lock) + { + m_createremotevolumeCommand.SetParameterValue(1, name); + m_createremotevolumeCommand.SetParameterValue(2, type.ToString()); + m_createremotevolumeCommand.SetParameterValue(3, state.ToString()); + m_createremotevolumeCommand.ExecuteNonQuery(); + } + } + + public void AddSymlinkEntry(string path, long metadataID, DateTime scantime, System.Data.IDbTransaction transaction = null) + { + AddFile(path, scantime, SYMLINK_BLOCKSET_ID, metadataID, transaction); + } + + public long GetFileEntry(string path, out DateTime oldScanned) + { + m_selectfileSimpleCommand.SetParameterValue(0, path); + using(var rd = m_selectfileSimpleCommand.ExecuteReader()) + if (rd.Read()) + { + oldScanned = Convert.ToDateTime(rd.GetValue(1)); + return Convert.ToInt64(rd.GetValue(0)); + } + else + { + oldScanned = DateTime.UtcNow; + return -1; + } + + } + + public string GetFileHash(long fileid) + { + m_selectfileHashCommand.SetParameterValue(0, fileid); + return m_selectfileHashCommand.ExecuteScalar().ToString(); + + } + + private class BlocklistHashEnumerable : IEnumerable + { + private class BlocklistHashEnumerator : IEnumerator + { + private System.Data.IDataReader m_reader; + private BlocklistHashEnumerable m_parent; + private string m_path = null; + private bool m_first = true; + private string m_current = null; + + public BlocklistHashEnumerator(BlocklistHashEnumerable parent, System.Data.IDataReader reader) + { + m_reader = reader; + m_parent = parent; + } + + public string Current { get{ return m_current; } } + + public void Dispose() + { + } + + object System.Collections.IEnumerator.Current { get { return this.Current; } } + + public bool MoveNext() + { + m_first = false; + + if (m_path == null) + { + m_path = m_reader.GetValue(0).ToString(); + m_current = m_reader.GetValue(6).ToString(); + return true; + } + else + { + if (m_current == null) + return false; + + if (!m_reader.Read()) + { + m_current = null; + m_parent.MoreData = false; + return false; + } + + if (m_path != m_reader.GetValue(0).ToString()) + { + m_current = null; + return false; + } + + m_current = m_reader.GetValue(6).ToString(); + return true; + } + } + + public void Reset() + { + if (!m_first) + throw new Exception("Iterator reset not supported"); + + m_first = false; + } + } + + private System.Data.IDataReader m_reader; + + public BlocklistHashEnumerable(System.Data.IDataReader reader) + { + m_reader = reader; + this.MoreData = true; + } + + public bool MoreData { get; protected set; } + + public IEnumerator GetEnumerator() + { + return new BlocklistHashEnumerator(this, m_reader); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + } + + public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume) + { + using (var cmd = m_connection.CreateCommand()) + { + //cmd.CommandText = @"SELECT ""A"".""ID"", ""D"".""Scantime"", ""B"".""Length"", ""B"".""FullHash"", ""E"".""FullHash"", ""E"".""Length"", ""B"".""ID"" FROM ""Fileset"" A, ""Blockset"" B, ""Metadataset"" C, ""OperationFileset"" D, ""Blockset"" E, ""Operation"" F WHERE ""A"".""ID"" = ""D"".""FilesetID"" AND ""F"".""ID"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"" AND ""F"".""ID"" = ""D"".""OperationID"" ORDER BY ""F"".""Timestamp"" DESC "; + cmd.CommandText = @"SELECT ""B"".""BlocksetID"", ""B"".""ID"", ""B"".""Path"", ""D"".""Length"", ""D"".""FullHash"", ""A"".""Scantime"" FROM ""OperationFileset"" A, ""Fileset"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""B"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?) AND ""A"".""OperationID"" = ? "; + cmd.AddParameter(FOLDER_BLOCKSET_ID); + cmd.AddParameter(SYMLINK_BLOCKSET_ID); + cmd.AddParameter(m_operationid); + + using (var rd = cmd.ExecuteReader()) + while(rd.Read()) + { + var blocksetID = Convert.ToInt64(rd.GetValue(0)); + var path = rd.GetValue(2).ToString(); + var metalength = Convert.ToInt64(rd.GetValue(3)); + var metahash = rd.GetValue(4).ToString(); + + if (blocksetID == FOLDER_BLOCKSET_ID) + filesetvolume.AddDirectory(path, metahash, metalength); + else if (blocksetID == SYMLINK_BLOCKSET_ID) + filesetvolume.AddSymlink(path, metahash, metalength); + } + + cmd.CommandText = @"SELECT ""F"".""Path"", ""F"".""Scantime"", ""F"".""Filelength"", ""F"".""Filehash"", ""F"".""Metahash"", ""F"".""Metalength"", ""G"".""Hash"" FROM (SELECT ""A"".""Path"" AS ""Path"", ""D"".""Scantime"" AS ""Scantime"", ""B"".""Length"" AS ""Filelength"", ""B"".""FullHash"" AS ""Filehash"", ""E"".""FullHash"" AS ""Metahash"", ""E"".""Length"" AS ""Metalength"", ""A"".""BlocksetID"" AS ""BlocksetID"" FROM ""Fileset"" A, ""Blockset"" B, ""Metadataset"" C, ""OperationFileset"" D, ""Blockset"" E WHERE ""A"".""ID"" = ""D"".""FilesetID"" AND ""D"".""OperationID"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"") F LEFT OUTER JOIN ""BlocklistHash"" G ON ""G"".""BlocksetID"" = ""F"".""BlocksetID"" ORDER BY ""F"".""Path"", ""G"".""Index"" "; + cmd.Parameters.Clear(); + cmd.AddParameter(m_operationid); + + using (var rd = cmd.ExecuteReader()) + if (rd.Read()) + { + var more = false; + do + { + var path = rd.GetValue(0).ToString(); + var filehash = rd.GetValue(3).ToString(); + var size = Convert.ToInt64(rd.GetValue(2)); + var scantime = Convert.ToDateTime(rd.GetValue(1)); + var metahash = rd.GetValue(4).ToString(); + var metasize = Convert.ToInt64(rd.GetValue(5)); + var blrd = new BlocklistHashEnumerable(rd); + + filesetvolume.AddFile(path, filehash, size, scantime, metahash, metasize, blrd); + more = blrd.MoreData; + + } while (more); + } + } + } + + public void VerifyConsistency() + { + using (var cmd = m_connection.CreateCommand()) + { + var c = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM (SELECT SUM(""Block"".""Size"") AS ""CalcLen"", ""Blockset"".""Length"" AS ""Length"", ""BlocksetEntry"".""BlocksetID"" FROM ""Block"", ""BlocksetEntry"", ""Blockset"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Blockset"".""ID"" = ""BlocksetEntry"".""BlocksetID"" GROUP BY ""BlocksetEntry"".""BlocksetID"") WHERE ""CalcLen"" != ""Length""")); + if (c != 0) + throw new InvalidDataException("Inconsistency detected, not all blocklists were restored correctly"); + } + } + + internal void UpdateChangeStatistics (BackupStatistics m_stat) + { + using (var cmd = m_connection.CreateCommand()) + { + long lastOperationId = -1; + + var lastIdObj = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Timestamp"" < ? AND ""Description"" = ? ORDER BY ""Timestamp"" DESC ", OperationTimestamp, "Backup"); + if (lastIdObj != null && lastIdObj != DBNull.Value) + lastOperationId = Convert.ToInt64(lastIdObj); + + m_stat.AddedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, FOLDER_BLOCKSET_ID, lastOperationId)); + m_stat.AddedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, SYMLINK_BLOCKSET_ID, lastOperationId)); + m_stat.AddedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""BlocksetID"" != ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, lastOperationId)); + + m_stat.DeletedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid)); + m_stat.DeletedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid)); + m_stat.DeletedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""BlocksetID"" != ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid)); + + m_stat.ModifiedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid)); + m_stat.ModifiedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid)); + m_stat.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid)); + + /*m_stat.ModifiedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT * FROM ( (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS A, (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""BlocsetID"" != ? AND ""A"".""MetadataID"" != ""B"".""MetadataID"" )", m_operationid, lastOperationId, FOLDER_BLOCKSET_ID)); + m_stat.ModifiedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT * FROM ( (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS A, (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS B WHERE ""A"".""Path"" = ""B"".""Path"" AND ""A"".""BlocsetID"" != ? AND ""A"".""MetadataID"" != ""B"".""MetadataID"" )", m_operationid, lastOperationId, SYMLINK_BLOCKSET_ID)); + m_stat.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT * FROM ( (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS A, (SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""Fileset"".""ID"" WHERE ""OperationID"" = ?) AS B WHERE ""A"".""Path"" = ""B"".""Path"" AND (""A"".""BlocsetID"" != ""B"".""BlocksetID"" OR ""A"".""MetadataID"" != ""B"".""MetadataID"") )", m_operationid, lastOperationId)); + * */ + } + } + + } +} diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalBlocklistUpdateDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalBlocklistUpdateDatabase.cs index e907467a5..5b5ade4a5 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalBlocklistUpdateDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalBlocklistUpdateDatabase.cs @@ -8,7 +8,8 @@ namespace Duplicati.Library.Main.ForestHash.Database { public partial class LocalBlocklistUpdateDatabase : LocalRestoredatabase { - protected string m_tempblockvolumetable; + protected string m_tempblockvolumetable; + protected string m_temphashtable; public LocalBlocklistUpdateDatabase(string path, long blocksize) : base("Rebuild", path, blocksize) @@ -17,15 +18,14 @@ namespace Duplicati.Library.Main.ForestHash.Database public void FindMissingBlocklistHashes() { - m_tempblockvolumetable = "MissingBlocks-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + m_tempblockvolumetable = "MissingBlocks-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + m_temphashtable = "Hashlist-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); using (var cmd = m_connection.CreateCommand()) { - //cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""Block"".""File"" AS ""File"", ""BlocklistHash"".""Hash"" AS ""Hash"", 0 AS ""Restored"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" AND ""BlocklistHash"".""Hash"" NOT IN ( SELECT DISTINCT ""BlocklistHash"".""Hash"" FROM ""BlocklistHash"", ""Blockset"" WHERE ""BlocklistHash"".""Hash"" = ""Blockset"".""Fullhash"" AND ""Blockset"".""Length"" <= {1} AND ""BlocklistHash"".""Index"" = 0 ) ", m_tempblockvolumetable, m_blocksize); - cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""Block"".""File"" AS ""File"", ""BlocklistHash"".""Hash"" AS ""Hash"", 0 AS ""Restored"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ", m_tempblockvolumetable); - cmd.ExecuteNonQuery(); + cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""Block"".""File"" AS ""File"", ""BlocklistHash"".""Hash"" AS ""Hash"", 0 AS ""Restored"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ", m_tempblockvolumetable)); + cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT DISTINCT ""Blockset"".""ID"", 0, ""Block"".""ID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" < {0} AND ""Blockset"".""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"") ", m_blocksize)); + cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_temphashtable)); - cmd.CommandText = string.Format(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT DISTINCT ""Blockset"".""ID"", 0, ""Block"".""ID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" < {0} AND ""Blockset"".""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"") ", m_blocksize); - cmd.ExecuteNonQuery(); } } @@ -34,11 +34,6 @@ namespace Duplicati.Library.Main.ForestHash.Database return new BlocklistsEnumerable(m_connection, m_tempblockvolumetable, volumename); } - public System.Data.IDbTransaction BeginTransaction() - { - return m_connection.BeginTransaction(); - } - public void UpdateBlocklist(string hash, IEnumerable hashes, long hashsize, System.Data.IDbTransaction transaction) { using (var cmd = m_connection.CreateCommand()) @@ -49,9 +44,25 @@ namespace Duplicati.Library.Main.ForestHash.Database cmd.AddParameter(hash); var c = cmd.ExecuteNonQuery(); if (c != 1) - throw new Exception(string.Format("Blocklist not found {0}", hash)); + throw new Exception(string.Format("Blocklist not found {0}", hash)); + + cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}""", m_temphashtable)); + cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Index"") VALUES (?, ?)", m_temphashtable); + long ix = 0; + foreach (var h in hashes) + { + c = cmd.ExecuteNonQuery(null, h, ix); + if (c != 1) + throw new Exception(string.Format("Insert row error {0}", h)); + ix++; + } + + cmd.CommandText = string.Format(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT ""A"".""BlocksetID"", (""A"".""Index"" * ?) + ""B"".""Index"", ""C"".""ID"" FROM ""BlocklistHash"" A, ""{0}"" B, ""Block"" C WHERE ""A"".""Hash"" = ? AND ""C"".""Hash"" = ""B"".""Hash"" ", m_temphashtable); + c = cmd.ExecuteNonQuery(null, (m_blocksize / hashsize), hash); + if (c == 0 || c % ix != 0) + throw new Exception(string.Format("Wrong number of inserts, got {0} records from {1} hashes!", c, ix)); - cmd.CommandText = @"SELECT ""Index"", ""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Hash"" = ?"; + /*cmd.CommandText = @"SELECT ""Index"", ""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Hash"" = ?"; long index; long id; using (var rd = cmd.ExecuteReader()) @@ -59,8 +70,12 @@ namespace Duplicati.Library.Main.ForestHash.Database if (!rd.Read()) throw new Exception(string.Format("No blocklisthashes for entry \"{0}\"", hash)); + index = Convert.ToInt64(rd.GetValue(0)); - id = Convert.ToInt64(rd.GetValue(1)); + id = Convert.ToInt64(rd.GetValue(1)); + + if (rd.Read()) + throw new Exception("Multiple matches!"); } var ix = index * (m_blocksize / hashsize); @@ -77,7 +92,7 @@ namespace Duplicati.Library.Main.ForestHash.Database throw new Exception(string.Format("Block not found {0}", h)); ix++; - } + }*/ } } @@ -127,7 +142,15 @@ namespace Duplicati.Library.Main.ForestHash.Database cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempblockvolumetable); cmd.ExecuteNonQuery(); } - finally { m_tempblockvolumetable = null; } + finally { m_tempblockvolumetable = null; } + + if (m_temphashtable != null) + try + { + cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_temphashtable); + cmd.ExecuteNonQuery(); + } + finally { m_temphashtable = null; } } } } diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs index 7a3322238..e0afcc603 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs @@ -17,39 +17,51 @@ namespace Duplicati.Library.Main.ForestHash.Database private readonly System.Data.IDbCommand m_removeremotevolumeCommand; private readonly System.Data.IDbCommand m_insertlogCommand; - private readonly System.Data.IDbCommand m_insertremotelogCommand; + private readonly System.Data.IDbCommand m_insertremotelogCommand; public const long FOLDER_BLOCKSET_ID = -100; public const long SYMLINK_BLOCKSET_ID = -200; - public DateTime OperationTimestamp { get; private set; } - + public DateTime OperationTimestamp { get; private set; } + + internal System.Data.IDbConnection Connection { get { return m_connection; } } + + protected static System.Data.IDbConnection CreateConnection(string path) + { + var c = (System.Data.IDbConnection)Activator.CreateInstance(Duplicati.Library.Utility.SQLiteLoader.SQLiteConnectionType); + if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(path))) + System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path)); + + Utility.DatabaseUpgrader.UpgradeDatabase(c, path, typeof(Localdatabase)); + + return c; + } /// /// Creates a new database instance and starts a new operation /// /// The path to the database - /// The name of the operation - public Localdatabase(string path, string operation) + /// The name of the operation + public Localdatabase(string path, string operation) + : this(CreateConnection(path), operation) + { + } + + /// + /// Creates a new database instance and starts a new operation + /// + /// The path to the database + /// The name of the operation + public Localdatabase(System.Data.IDbConnection connection, string operation) { - this.OperationTimestamp = DateTime.UtcNow; - m_connection = (System.Data.IDbConnection)Activator.CreateInstance(Duplicati.Library.Utility.SQLiteLoader.SQLiteConnectionType); - - if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(path))) - System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path)); - - Utility.DatabaseUpgrader.UpgradeDatabase(m_connection, path, typeof(Localdatabase)); + this.OperationTimestamp = DateTime.UtcNow; + m_connection = connection; + + if (m_connection.State != System.Data.ConnectionState.Open) + m_connection.Open(); using (var cmd = m_connection.CreateCommand()) - { - cmd.CommandText = @"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES (?, ?)"; - cmd.AddParameter(operation); - cmd.AddParameter(OperationTimestamp); - - cmd.ExecuteNonQuery(); - cmd.CommandText = "SELECT last_insert_rowid()"; - m_operationid = Convert.ToInt64(cmd.ExecuteScalar()); - } + m_operationid = Convert.ToInt64(cmd.ExecuteScalar( @"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES (?, ?); SELECT last_insert_rowid();", operation, OperationTimestamp)); m_updateremotevolumeCommand = m_connection.CreateCommand(); m_selectremotevolumesCommand = m_connection.CreateCommand(); @@ -79,10 +91,11 @@ namespace Duplicati.Library.Main.ForestHash.Database m_removeremotevolumeCommand.AddParameter(); } - public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash) + public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, System.Data.IDbTransaction transaction = null) { lock (m_lock) - { + { + m_updateremotevolumeCommand.Transaction = transaction; m_updateremotevolumeCommand.SetParameterValue(1, state.ToString()); m_updateremotevolumeCommand.SetParameterValue(2, hash); m_updateremotevolumeCommand.SetParameterValue(3, size); @@ -177,12 +190,12 @@ namespace Duplicati.Library.Main.ForestHash.Database var deletecmd = m_connection.CreateCommand(); deletecmd.Transaction = tr; - var updatedfiles = deletecmd.ExecuteNonQuery(@"UPDATE ""Fileset"" SET ""BlocksetID"" = -1 WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""File"" = ?))", name); - var updatedmetadata = deletecmd.ExecuteNonQuery(@"UPDATE ""Metadataset"" SET ""BlocksetID"" = -1 WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""File"" = ?))", name); - var deletedblocksets = deletecmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""File"" = ?))", name); - var deletedblocksetentries = deletecmd.ExecuteNonQuery(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""File"" = ?))", name); + deletecmd.ExecuteNonQuery(@"UPDATE ""Fileset"" SET ""BlocksetID"" = -1 WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""File"" = ?))", name); + deletecmd.ExecuteNonQuery(@"UPDATE ""Metadataset"" SET ""BlocksetID"" = -1 WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""File"" = ?))", name); + deletecmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""File"" = ?))", name); + deletecmd.ExecuteNonQuery(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""File"" = ?))", name); - var deletedblocks = deletecmd.ExecuteNonQuery(@"DELETE FROM ""Block"" WHERE ""File"" = ?", name); + deletecmd.ExecuteNonQuery(@"DELETE FROM ""Block"" WHERE ""File"" = ?", name); ((System.Data.IDataParameter)m_removeremotevolumeCommand.Parameters[0]).Value = name; m_removeremotevolumeCommand.Transaction = tr; @@ -214,6 +227,54 @@ namespace Duplicati.Library.Main.ForestHash.Database return Convert.ToInt64(r); } + } + + public System.Data.IDbTransaction BeginTransaction() + { + return m_connection.BeginTransaction(); + } + + protected class TemporaryTransactionWrapper : IDisposable + { + private System.Data.IDbTransaction m_parent; + private bool m_isTemporary; + + public TemporaryTransactionWrapper(System.Data.IDbConnection connection, System.Data.IDbTransaction transaction) + { + if (transaction != null) + { + m_parent = transaction; + m_isTemporary = false; + } + else + { + m_parent = connection.BeginTransaction(); + m_isTemporary = true; + } + } + + public System.Data.IDbConnection Connection { get { return m_parent.Connection; } } + public System.Data.IsolationLevel IsolationLevel { get { return m_parent.IsolationLevel; } } + + public void Commit() + { + if (m_isTemporary) + m_parent.Commit(); + } + + public void Rollback() + { + if (m_isTemporary) + m_parent.Rollback(); + } + + public void Dispose() + { + if (m_isTemporary) + m_parent.Dispose(); + } + + public System.Data.IDbTransaction Parent { get { return m_parent; } } } private class LocalFileEntryEnumerable : IEnumerable diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs index 4dfe60198..7ba62668b 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs @@ -1,350 +1,353 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Duplicati.Library.Main.ForestHash.Volumes; - -namespace Duplicati.Library.Main.ForestHash.Database -{ - public partial class LocalRestoredatabase : Localdatabase - { - protected string m_tempfiletable; - protected string m_tempblocktable; - protected long m_blocksize; - - public LocalRestoredatabase(string path, long blocksize) - : this("Restore", path, blocksize) - { - } - - protected LocalRestoredatabase(string operation, string path, long blocksize) - : base(path, operation) - { - //TODO: Should read this from DB? - m_blocksize = blocksize; - } - - public void PrepareRestoreFilelist(DateTime restoretime, string[] p, Utility.FilenameFilter filenameFilter) - { - var guid = Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - - m_tempfiletable = "Fileset-" + guid; - m_tempblocktable = "Blocks-" + guid; - - using (var cmd = m_connection.CreateCommand()) - { - long operationID = GetFilesetID(restoretime); - - cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""Targetpath"" TEXT NULL ) ", m_tempfiletable); - cmd.ExecuteNonQuery(); - - cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL)", m_tempblocktable); - cmd.ExecuteNonQuery(); - - if (filenameFilter == null && (p == null || p.Length == 0)) - { - // Simple case, restore everything - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""Path"", ""BlocksetID"", ""MetadataID"" FROM ""Fileset"" WHERE ""OperationID"" = ? ", m_tempfiletable); - cmd.AddParameter(operationID); - cmd.ExecuteNonQuery(); - } - else if (p != null && p.Length > 0) - { - var filelisttablename = "Filelist-" + guid; - - // Restore files in list - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""Path"", ""BlocksetID"", ""MetadataID"" FROM ""Fileset"" WHERE ""OperationID"" = ? AND ""Path"" = ? ", m_tempfiletable); - cmd.AddParameter(operationID); - cmd.AddParameter(); - - foreach (var s in p) - { - cmd.SetParameterValue(1, s); - var c = cmd.ExecuteNonQuery(); - if (c != 1) - throw new Exception("Failed to insert file into temp table"); - } - } - else - { - // Restore but filter elements based on regexp - cmd.CommandText = string.Format(@"SELECT ""Path"", ""BlocksetID"", ""MetadataID"" FROM ""Fileset"" WHERE ""OperationID"" = ?"); - cmd.AddParameter(operationID); - - object[] values = new object[3]; - using (var cmd2 = m_connection.CreateCommand()) - { - cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?)", m_tempfiletable); - cmd2.AddParameter(); - cmd2.AddParameter(); - cmd2.AddParameter(); - - using (var rd = cmd.ExecuteReader()) - while (rd.Read()) - { - rd.GetValues(values); - if (values[0] != null && filenameFilter.ShouldInclude("", values[0].ToString())) - { - cmd2.SetParameterValue(0, values[0]); - cmd2.SetParameterValue(1, values[0]); - cmd2.SetParameterValue(2, values[0]); - cmd2.ExecuteNonQuery(); - } - } - } - } - } - } - - public string GetLargestPrefix() - { - using (var cmd = m_connection.CreateCommand()) - { - cmd.CommandText = string.Format(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", m_tempfiletable); - var v0 = cmd.ExecuteScalar(); - string maxpath = ""; - if (v0 != null) - maxpath = v0.ToString(); - - cmd.CommandText = string.Format(@"SELECT COUNT(*) FROM ""{0}""", m_tempfiletable); - var filecount = Convert.ToInt64(cmd.ExecuteScalar()); - long foundfiles = -1; - - //TODO: Handle FS case-sensitive? - cmd.CommandText = string.Format(@"SELECT COUNT(*) FROM ""{0}"" WHERE SUBSTR(""Path"", 1, ?) = ?", m_tempfiletable); - cmd.AddParameter(); - cmd.AddParameter(); - - while (filecount != foundfiles && maxpath.Length > 0) - { - var mp = Utility.Utility.AppendDirSeparator(maxpath); - cmd.SetParameterValue(0, mp.Length); - cmd.SetParameterValue(1, mp); - foundfiles = Convert.ToInt64(cmd.ExecuteScalar()); - - if (filecount != foundfiles) - { - var oldlen = maxpath.Length; - maxpath = System.IO.Path.GetDirectoryName(maxpath); - if (maxpath.Length == oldlen) - maxpath = ""; - } - } - - return maxpath == "" ? "" : Utility.Utility.AppendDirSeparator(maxpath); - } - - } - - public void SetTargetPaths(string largest_prefix, string destination) - { - using (var cmd = m_connection.CreateCommand()) - { - destination = Utility.Utility.AppendDirSeparator(destination); - largest_prefix = Utility.Utility.AppendDirSeparator(largest_prefix); - - cmd.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Targetpath"" = ? || SUBSTR(""Path"", ?)", m_tempfiletable); - cmd.AddParameter(destination); - cmd.AddParameter(largest_prefix.Length + 1); - cmd.ExecuteNonQuery(); - } - } - - public void FindMissingBlocks() - { - using (var cmd = m_connection.CreateCommand()) - { - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"") SELECT DISTINCT ""{1}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0 FROM ""{1}"", ""BlocksetEntry"", ""Block"" WHERE ""{1}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ", m_tempblocktable, m_tempfiletable); - var p = cmd.ExecuteNonQuery(); - System.Diagnostics.Trace.WriteLine(string.Format("Blocks to restore: {0}", p)); - } - } - - public interface IExistingFileBlock - { - string Hash { get; } - long Index { get; } - long Size { get; } - } - - public interface IExistingFile - { - string TargetPath { get; } - long Length { get; } - IEnumerable Blocks { get; } - } - - - public interface IBlockSource - { - string Path { get; } - long Size { get; } - long Offset { get; } - } - - public interface IBlockDescriptor - { - string Hash { get; } - long Size { get; } - long Offset { get; } - long Index { get; } - IEnumerable Blocksources { get; } - } - - public interface ILocalBlockSource - { - string TargetPath { get; } - IEnumerable Blocks { get; } - } - - public interface IFileToRestore - { - string Path { get; } - string Hash { get; } - } - - public interface IPatchBlock - { - long Offset { get; } - long Size { get; } - string Key { get; } - } - - public interface IVolumePatch - { - string Path { get; } - IEnumerable Blocks { get; } - } - - public IEnumerable GetExistingFilesWithBlocks() - { - return new ExistingFileEnumerable(m_connection, m_tempfiletable, m_blocksize); - } - - public IEnumerable GetFilesAndSourceBlocks() - { - return new LocalBlockSourceEnumerable(m_connection, m_tempfiletable, m_tempblocktable, m_blocksize); - } - - public IList GetMissingVolumes() - { - using (var cmd = m_connection.CreateCommand()) - { - List result = new List(); - cmd.CommandText = string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"" WHERE ""Name"" IN (SELECT DISTINCT ""Block"".""File"" FROM ""Block"" WHERE ""Block"".""Hash"" IN (SELECT DISTINCT ""{0}"".""Hash"" FROM ""{0}"" WHERE ""{0}"".""Restored"" = 0))", m_tempblocktable); - using (var rd = cmd.ExecuteReader()) - { - object[] r = new object[3]; - while (rd.Read()) - { - rd.GetValues(r); - result.Add(new RemoteVolume( - r[0] == null ? null : r[0].ToString(), - r[1] == null ? null : r[1].ToString(), - r[2] == null ? -1 : Convert.ToInt64(r[2]) - )); - } - } - - return result; - } - } - - public IEnumerable GetFilesWithMissingBlocks(BlockVolumeReader curvolume) - { - return new VolumePatchEnumerable(m_connection, m_tempfiletable, m_tempblocktable, m_blocksize, curvolume); - } - - public IEnumerable GetFilesToRestore() - { - return new FileToRestoreEnumerable(m_connection, m_tempfiletable); - } - - public void DropRestoreTable() - { - using (var cmd = m_connection.CreateCommand()) - { - if (m_tempfiletable != null) - try - { - cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempfiletable); - cmd.ExecuteNonQuery(); - } - finally { m_tempfiletable = null; } - - if (m_tempblocktable != null) - try - { - cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempblocktable); - cmd.ExecuteNonQuery(); - } - finally { m_tempblocktable = null; } - } - } - - public interface IBlockMarker : IDisposable - { - void SetBlockRestored(string targetpath, long index, string hash, long blocksize); - void Commit(); - } - - private class BlockMarker : IBlockMarker - { - private System.Data.IDbCommand m_command; - - public BlockMarker(System.Data.IDbConnection connection, string blocktablename, string filetablename) - { - m_command = connection.CreateCommand(); - m_command.Transaction = connection.BeginTransaction(); - m_command.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""FileID"" = (SELECT ""ID"" FROM ""{1}"" WHERE ""TargetPath"" = ?) AND ""Index"" = ? AND ""Hash"" = ? AND ""Size"" = ? ", blocktablename, filetablename); - m_command.AddParameters(4); - } - - public void SetBlockRestored(string targetpath, long index, string hash, long size) - { - m_command.SetParameterValue(0, targetpath); - m_command.SetParameterValue(1, index); - m_command.SetParameterValue(2, hash); - m_command.SetParameterValue(3, size); - var r = m_command.ExecuteNonQuery(); - if (r != 1) - throw new Exception("Unexpected update result"); - } - - public void Commit() - { - var tr = m_command.Transaction; - m_command.Dispose(); - m_command = null; - tr.Commit(); - tr.Dispose(); - } - - public void Dispose() - { - if (m_command != null) - { - var t = m_command; - m_command = null; - t.Dispose(); - } - } - } - - public IBlockMarker CreateBlockMarker() - { - return new BlockMarker(m_connection, m_tempblocktable, m_tempfiletable); - } - - public override void Dispose() - { - base.Dispose(); - DropRestoreTable(); - } - - public IEnumerable GetTargetFolders() - { - return new FolderListEnumerable(m_connection, m_tempfiletable); - } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Duplicati.Library.Main.ForestHash.Volumes; + +namespace Duplicati.Library.Main.ForestHash.Database +{ + public partial class LocalRestoredatabase : Localdatabase + { + protected string m_tempfiletable; + protected string m_tempblocktable; + protected long m_blocksize; + + public LocalRestoredatabase(string path, long blocksize) + : this("Restore", path, blocksize) + { + } + + protected LocalRestoredatabase(string operation, string path, long blocksize) + : base(path, operation) + { + //TODO: Should read this from DB? + m_blocksize = blocksize; + } + + public void PrepareRestoreFilelist(DateTime restoretime, string[] p, Utility.FilenameFilter filenameFilter) + { + var guid = Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + + m_tempfiletable = "Fileset-" + guid; + m_tempblocktable = "Blocks-" + guid; + + using (var cmd = m_connection.CreateCommand()) + { + long operationID = GetFilesetID(restoretime); + + cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""Targetpath"" TEXT NULL ) ", m_tempfiletable); + cmd.ExecuteNonQuery(); + + cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Restored"" BOOLEAN NOT NULL)", m_tempblocktable); + cmd.ExecuteNonQuery(); + + if (filenameFilter == null && (p == null || p.Length == 0)) + { + // Simple case, restore everything + cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""Fileset"", ""OperationFileset"" WHERE ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" AND ""OperationFileset"".""OperationID"" = ? ", m_tempfiletable); + cmd.AddParameter(operationID); + cmd.ExecuteNonQuery(); + } + else if (p != null && p.Length > 0) + { + // Restore files in list + cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""Fileset"", ""OperationFileset"" WHERE ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" AND ""OperationID"" = ? AND ""Path"" = ? ", m_tempfiletable); + cmd.AddParameter(operationID); + cmd.AddParameter(); + + //TODO: Actually faster to just create a temp table with files, + // and then using a select to create the temporary table + // but there should be some way to report if filenames + // are not found + + foreach (var s in p) + { + cmd.SetParameterValue(1, s); + var c = cmd.ExecuteNonQuery(); + if (c != 1) + throw new Exception("Failed to insert file into temp table"); + } + } + else + { + // Restore but filter elements based on regexp + cmd.CommandText = string.Format(@"SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""Fileset"", ""OperationFileset"" WHERE ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" AND ""OperationID"" = ?"); + cmd.AddParameter(operationID); + + object[] values = new object[3]; + using (var cmd2 = m_connection.CreateCommand()) + { + cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?)", m_tempfiletable); + cmd2.AddParameter(); + cmd2.AddParameter(); + cmd2.AddParameter(); + + using (var rd = cmd.ExecuteReader()) + while (rd.Read()) + { + rd.GetValues(values); + if (values[0] != null && filenameFilter.ShouldInclude("", values[0].ToString())) + { + cmd2.SetParameterValue(0, values[0]); + cmd2.SetParameterValue(1, values[0]); + cmd2.SetParameterValue(2, values[0]); + cmd2.ExecuteNonQuery(); + } + } + } + } + } + } + + public string GetLargestPrefix() + { + using (var cmd = m_connection.CreateCommand()) + { + cmd.CommandText = string.Format(@"SELECT ""Path"" FROM ""{0}"" ORDER BY LENGTH(""Path"") DESC LIMIT 1", m_tempfiletable); + var v0 = cmd.ExecuteScalar(); + string maxpath = ""; + if (v0 != null) + maxpath = v0.ToString(); + + cmd.CommandText = string.Format(@"SELECT COUNT(*) FROM ""{0}""", m_tempfiletable); + var filecount = Convert.ToInt64(cmd.ExecuteScalar()); + long foundfiles = -1; + + //TODO: Handle FS case-sensitive? + cmd.CommandText = string.Format(@"SELECT COUNT(*) FROM ""{0}"" WHERE SUBSTR(""Path"", 1, ?) = ?", m_tempfiletable); + cmd.AddParameter(); + cmd.AddParameter(); + + while (filecount != foundfiles && maxpath.Length > 0) + { + var mp = Utility.Utility.AppendDirSeparator(maxpath); + cmd.SetParameterValue(0, mp.Length); + cmd.SetParameterValue(1, mp); + foundfiles = Convert.ToInt64(cmd.ExecuteScalar()); + + if (filecount != foundfiles) + { + var oldlen = maxpath.Length; + maxpath = System.IO.Path.GetDirectoryName(maxpath); + if (maxpath.Length == oldlen) + maxpath = ""; + } + } + + return maxpath == "" ? "" : Utility.Utility.AppendDirSeparator(maxpath); + } + + } + + public void SetTargetPaths(string largest_prefix, string destination) + { + using (var cmd = m_connection.CreateCommand()) + { + destination = Utility.Utility.AppendDirSeparator(destination); + largest_prefix = Utility.Utility.AppendDirSeparator(largest_prefix); + + cmd.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Targetpath"" = ? || SUBSTR(""Path"", ?)", m_tempfiletable); + cmd.AddParameter(destination); + cmd.AddParameter(largest_prefix.Length + 1); + cmd.ExecuteNonQuery(); + } + } + + public void FindMissingBlocks() + { + using (var cmd = m_connection.CreateCommand()) + { + cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"", ""Restored"") SELECT DISTINCT ""{1}"".""ID"", ""BlocksetEntry"".""Index"", ""Block"".""Hash"", ""Block"".""Size"", 0 FROM ""{1}"", ""BlocksetEntry"", ""Block"" WHERE ""{1}"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ", m_tempblocktable, m_tempfiletable); + var p = cmd.ExecuteNonQuery(); + System.Diagnostics.Trace.WriteLine(string.Format("Blocks to restore: {0}", p)); + } + } + + public interface IExistingFileBlock + { + string Hash { get; } + long Index { get; } + long Size { get; } + } + + public interface IExistingFile + { + string TargetPath { get; } + long Length { get; } + IEnumerable Blocks { get; } + } + + + public interface IBlockSource + { + string Path { get; } + long Size { get; } + long Offset { get; } + } + + public interface IBlockDescriptor + { + string Hash { get; } + long Size { get; } + long Offset { get; } + long Index { get; } + IEnumerable Blocksources { get; } + } + + public interface ILocalBlockSource + { + string TargetPath { get; } + IEnumerable Blocks { get; } + } + + public interface IFileToRestore + { + string Path { get; } + string Hash { get; } + } + + public interface IPatchBlock + { + long Offset { get; } + long Size { get; } + string Key { get; } + } + + public interface IVolumePatch + { + string Path { get; } + IEnumerable Blocks { get; } + } + + public IEnumerable GetExistingFilesWithBlocks() + { + return new ExistingFileEnumerable(m_connection, m_tempfiletable); + } + + public IEnumerable GetFilesAndSourceBlocks() + { + return new LocalBlockSourceEnumerable(m_connection, m_tempfiletable, m_tempblocktable, m_blocksize); + } + + public IList GetMissingVolumes() + { + using (var cmd = m_connection.CreateCommand()) + { + List result = new List(); + cmd.CommandText = string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"" WHERE ""Name"" IN (SELECT DISTINCT ""Block"".""File"" FROM ""Block"" WHERE ""Block"".""Hash"" IN (SELECT DISTINCT ""{0}"".""Hash"" FROM ""{0}"" WHERE ""{0}"".""Restored"" = 0))", m_tempblocktable); + using (var rd = cmd.ExecuteReader()) + { + object[] r = new object[3]; + while (rd.Read()) + { + rd.GetValues(r); + result.Add(new RemoteVolume( + r[0] == null ? null : r[0].ToString(), + r[1] == null ? null : r[1].ToString(), + r[2] == null ? -1 : Convert.ToInt64(r[2]) + )); + } + } + + return result; + } + } + + public IEnumerable GetFilesWithMissingBlocks(BlockVolumeReader curvolume) + { + return new VolumePatchEnumerable(m_connection, m_tempfiletable, m_tempblocktable, m_blocksize, curvolume); + } + + public IEnumerable GetFilesToRestore() + { + return new FileToRestoreEnumerable(m_connection, m_tempfiletable); + } + + public void DropRestoreTable() + { + using (var cmd = m_connection.CreateCommand()) + { + if (m_tempfiletable != null) + try + { + cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempfiletable); + cmd.ExecuteNonQuery(); + } + finally { m_tempfiletable = null; } + + if (m_tempblocktable != null) + try + { + cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempblocktable); + cmd.ExecuteNonQuery(); + } + finally { m_tempblocktable = null; } + } + } + + public interface IBlockMarker : IDisposable + { + void SetBlockRestored(string targetpath, long index, string hash, long blocksize); + void Commit(); + } + + private class BlockMarker : IBlockMarker + { + private System.Data.IDbCommand m_command; + + public BlockMarker(System.Data.IDbConnection connection, string blocktablename, string filetablename) + { + m_command = connection.CreateCommand(); + m_command.Transaction = connection.BeginTransaction(); + m_command.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""FileID"" = (SELECT ""ID"" FROM ""{1}"" WHERE ""TargetPath"" = ?) AND ""Index"" = ? AND ""Hash"" = ? AND ""Size"" = ? ", blocktablename, filetablename); + m_command.AddParameters(4); + } + + public void SetBlockRestored(string targetpath, long index, string hash, long size) + { + m_command.SetParameterValue(0, targetpath); + m_command.SetParameterValue(1, index); + m_command.SetParameterValue(2, hash); + m_command.SetParameterValue(3, size); + var r = m_command.ExecuteNonQuery(); + if (r != 1) + throw new Exception("Unexpected update result"); + } + + public void Commit() + { + var tr = m_command.Transaction; + m_command.Dispose(); + m_command = null; + tr.Commit(); + tr.Dispose(); + } + + public void Dispose() + { + if (m_command != null) + { + var t = m_command; + m_command = null; + t.Dispose(); + } + } + } + + public IBlockMarker CreateBlockMarker() + { + return new BlockMarker(m_connection, m_tempblocktable, m_tempfiletable); + } + + public override void Dispose() + { + base.Dispose(); + DropRestoreTable(); + } + + public IEnumerable GetTargetFolders() + { + return new FolderListEnumerable(m_connection, m_tempfiletable); + } + } +} diff --git a/Duplicati/Library/Main/ForestHash/Database/RestoreIterators.cs b/Duplicati/Library/Main/ForestHash/Database/RestoreIterators.cs index ac4f1145f..4e972754a 100644 --- a/Duplicati/Library/Main/ForestHash/Database/RestoreIterators.cs +++ b/Duplicati/Library/Main/ForestHash/Database/RestoreIterators.cs @@ -175,16 +175,14 @@ namespace Duplicati.Library.Main.ForestHash.Database private System.Data.IDbCommand m_command; private System.Data.IDataReader m_reader; private System.Data.IDbConnection m_connection; - private long m_blocksize; private string m_tablename; private ExistingFile m_file; private string m_current; - public ExistingFileEnumerator(System.Data.IDbConnection con, string tablename, long blocksize) + public ExistingFileEnumerator(System.Data.IDbConnection con, string tablename) { m_connection = con; m_tablename = tablename; - m_blocksize = blocksize; this.Reset(); } @@ -240,18 +238,16 @@ namespace Duplicati.Library.Main.ForestHash.Database private System.Data.IDbConnection m_connection; private string m_tablename; - private long m_blocksize; - public ExistingFileEnumerable(System.Data.IDbConnection connection, string tablename, long blocksize) + public ExistingFileEnumerable(System.Data.IDbConnection connection, string tablename) { m_connection = connection; m_tablename = tablename; - m_blocksize = blocksize; } public IEnumerator GetEnumerator() { - return new ExistingFileEnumerator(m_connection, m_tablename, m_blocksize); + return new ExistingFileEnumerator(m_connection, m_tablename); } System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() @@ -716,7 +712,6 @@ namespace Duplicati.Library.Main.ForestHash.Database private VolumePatch m_patch; private PatchBlock m_block; private string m_currentfile; - private string m_currentkey; private bool m_first; public PatchBlockEnumerator(System.Data.IDataReader reader, VolumePatch patch) @@ -724,7 +719,6 @@ namespace Duplicati.Library.Main.ForestHash.Database m_reader = reader; m_patch = patch; m_currentfile = patch.Path; - m_currentkey = null; this.Reset(); } @@ -753,14 +747,12 @@ namespace Duplicati.Library.Main.ForestHash.Database m_patch.HasMoreData = m_reader.Read(); var more = m_patch.HasMoreData && m_currentfile == m_patch.Path; - m_currentkey = more ? m_block.Key : null; return more; } public void Reset() { m_block = new PatchBlock(m_reader); - m_currentkey = null; m_first = true; } } diff --git a/Duplicati/Library/Main/ForestHash/Enums.cs b/Duplicati/Library/Main/ForestHash/Enums.cs index 09ff01199..1ed900457 100644 --- a/Duplicati/Library/Main/ForestHash/Enums.cs +++ b/Duplicati/Library/Main/ForestHash/Enums.cs @@ -35,7 +35,18 @@ namespace Duplicati.Library.Main.ForestHash /// Indicates that the remote volume has been uploaded, /// but is now missing from the remote location /// - Missing + Missing, + + /// + /// Indicattes that the remote volume should be deleted + /// + Deleting, + + /// + /// Indicates that the remote volume was successfully + /// deleted from the remote location + /// + Deleted } /// diff --git a/Duplicati/Library/Main/ForestHash/FhBackend.cs b/Duplicati/Library/Main/ForestHash/FhBackend.cs index 15d0a63cd..65c0a0211 100644 --- a/Duplicati/Library/Main/ForestHash/FhBackend.cs +++ b/Duplicati/Library/Main/ForestHash/FhBackend.cs @@ -78,6 +78,8 @@ namespace Duplicati.Library.Main.ForestHash var sourcefile = this.LocalFilename + "." + encryption.FilenameExtension; encryption.Encrypt(this.LocalFilename, sourcefile); this.LocalFilename = sourcefile; + this.Hash = null; + this.Size = 0; this.Encrypted = true; } } @@ -109,12 +111,14 @@ namespace Duplicati.Library.Main.ForestHash private Library.Interface.IEncryption m_encryption; private Library.Interface.IBackend m_backend; private string m_backendurl; + private CommunicationStatistics m_stats; - public FhBackend(string backendurl, Options options, Localdatabase database) + public FhBackend(string backendurl, Options options, Localdatabase database, CommunicationStatistics stats) { m_options = options; m_database = database; m_backendurl = backendurl; + m_stats = stats; m_backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions); @@ -163,13 +167,11 @@ namespace Duplicati.Library.Main.ForestHash DoList(item); break; case OperationType.Delete: - throw new MissingMethodException("DELETE"); - //DoDelete(item); - //break; + DoDelete(item); + break; case OperationType.CreateFolder: - throw new MissingMethodException("CREATEFOLDER"); - //DoCreateFolder(item); - //break; + DoCreateFolder(item); + break; case OperationType.Terminate: m_queue.SetCompleted(); break; @@ -182,6 +184,7 @@ namespace Duplicati.Library.Main.ForestHash { retries++; lastException = ex; + m_stats.LogRetryAttempt(string.Format("Operation {0} with file {1} attempt {2} of {3} failed with message: {4}", item.Operation, item.RemoteFilename, retries, m_options.NumberOfRetries, ex.Message), ex); m_database.LogMessage("warning", string.Format("Operation {0} with file {1} attempt {2} of {3} failed with message: {4}", item.Operation, item.RemoteFilename, retries, m_options.NumberOfRetries, ex.Message), ex); try { m_backend.Dispose(); } @@ -224,6 +227,7 @@ namespace Duplicati.Library.Main.ForestHash item.Shadow = null; } + m_stats.AddNumberOfRemoteCalls(1); m_database.LogRemoteOperation("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash })); if (m_backend is Library.Interface.IStreamingBackend) @@ -236,6 +240,7 @@ namespace Duplicati.Library.Main.ForestHash else m_backend.Put(item.RemoteFilename, item.LocalFilename); + m_stats.AddBytesUploaded(item.Size); } private void DoGet(FileEntryItem item) @@ -243,6 +248,7 @@ namespace Duplicati.Library.Main.ForestHash Utility.TempFile tmpfile = null; try { + m_stats.AddNumberOfRemoteCalls(1); tmpfile = new Utility.TempFile(); if (m_backend is Library.Interface.IStreamingBackend) { @@ -253,6 +259,9 @@ namespace Duplicati.Library.Main.ForestHash } else m_backend.Get(item.RemoteFilename, tmpfile); + + m_stats.AddBytesDownloaded(new System.IO.FileInfo(tmpfile).Length); + m_database.LogRemoteOperation("get", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = new System.IO.FileInfo(tmpfile).Length, Hash = FileEntryItem.CalculateSha256Hash(tmpfile) })); if (!m_options.SkipFileHashChecks) { @@ -330,6 +339,46 @@ namespace Duplicati.Library.Main.ForestHash item.Result = r; } + private void DoDelete (FileEntryItem item) + { + m_stats.AddNumberOfRemoteCalls(1); + + string result = null; + try + { + m_backend.Delete(item.RemoteFilename); + } + catch (Exception ex) + { + result = ex.ToString(); + throw; + } + finally + { + m_database.LogRemoteOperation("delete", item.RemoteFilename, result); + } + + m_database.UpdateRemoteVolume(item.RemoteFilename, RemoteVolumeState.Deleted, -1, null); + } + + private void DoCreateFolder (FileEntryItem item) + { + string result = null; + try + { + (m_backend as Library.Interface.IBackend_v2).CreateFolder(); + } + catch (Exception ex) + { + result = ex.ToString(); + throw; + } + finally + { + m_database.LogRemoteOperation("createfolder", item.RemoteFilename, result); + } + } + public void Put(VolumeWriterBase item, ShadowVolumeWriter shadow = null) { if (m_lastException != null) @@ -418,6 +467,27 @@ namespace Duplicati.Library.Main.ForestHash throw m_lastException; } + public void CreateFolder(string remotename) + { + var item = new FileEntryItem(OperationType.CreateFolder, remotename); + if (m_queue.Enqueue(item)) + item.WaitForComplete(); + + if (m_lastException != null) + throw m_lastException; + } + + public void Delete(string remotename, bool synchronous = false) + { + m_database.UpdateRemoteVolume(remotename, RemoteVolumeState.Deleting, -1, null); + var item = new FileEntryItem(OperationType.Delete, remotename); + if (m_queue.Enqueue(item) && synchronous) + item.WaitForComplete(); + + if (m_lastException != null) + throw m_lastException; + } + public void Dispose() { if (m_queue != null && !m_queue.Completed) diff --git a/Duplicati/Library/Main/ForestHash/ForestHash.cs b/Duplicati/Library/Main/ForestHash/ForestHash.cs index 6092e2306..91ddf47bd 100644 --- a/Duplicati/Library/Main/ForestHash/ForestHash.cs +++ b/Duplicati/Library/Main/ForestHash/ForestHash.cs @@ -49,7 +49,7 @@ namespace Duplicati.Library.Main.ForestHash foreach (var s in remotelist) { - if (s.Name.StartsWith(options.BackupPrefix)) + if (s.Name.StartsWith(prefix)) lookup[s.Name] = s; } @@ -183,11 +183,11 @@ namespace Duplicati.Library.Main.ForestHash - internal static IEnumerable ParseFileList(string target, Dictionary options) + internal static IEnumerable ParseFileList(string target, Dictionary options, CommunicationStatistics stat) { var opts = new FhOptions(options); using (var db = new Localdatabase(opts.Fhdbpath, "ParseFileList")) - using (var b = new FhBackend(target, opts, db)) + using (var b = new FhBackend(target, opts, db, stat)) return from n in b.List() let np = Volumes.VolumeBase.ParseFilename(n) diff --git a/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs index 002e924e9..28c4c3ed6 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs @@ -1,404 +1,427 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; -using Duplicati.Library.Main.ForestHash.Database; -using Duplicati.Library.Main.ForestHash.Volumes; - -namespace Duplicati.Library.Main.ForestHash.Operation -{ - internal class BackupHandler : IDisposable - { - private readonly FhOptions m_options; - private readonly BackupStatistics m_stat; - private FhBackend m_backend; - - private readonly byte[] m_blockbuffer; - private readonly byte[] m_blocklistbuffer; - private readonly System.Security.Cryptography.HashAlgorithm m_blockhasher; - private readonly System.Security.Cryptography.HashAlgorithm m_filehasher; - - private LocalBackupDatabase m_database; - private BlockVolumeWriter m_blockvolume; - private FilesetVolumeWriter m_filesetvolume; - private ShadowVolumeWriter m_shadowvolume; - - private Snapshots.ISnapshotService m_snapshot; - private long m_changedfiles; - private long m_changedfolders; - - private string[] m_sources; - - public BackupHandler(string backendurl, FhOptions options, BackupStatistics stat, string[] sources) - { - m_options = options; - m_stat = stat; - m_database = new LocalBackupDatabase(m_options.Fhdbpath); - m_backend = new FhBackend(backendurl, options, m_database); - - m_sources = sources; - m_blockbuffer = new byte[m_options.Fhblocksize]; - m_blocklistbuffer = new byte[m_options.Fhblocksize]; - - m_blockhasher = System.Security.Cryptography.SHA256.Create(); - m_filehasher = System.Security.Cryptography.SHA256.Create(); - - if (!m_blockhasher.CanReuseTransform || !m_filehasher.CanReuseTransform) - throw new Exception(Strings.Foresthash.InvalidCryptoSystem); - } - - private static Snapshots.ISnapshotService GetSnapshot(string[] sourcefolders, Options options, CommunicationStatistics stat) - { - try - { - if (options.SnapShotStrategy != Options.OptimizationStrategy.Off) - return Duplicati.Library.Snapshots.SnapshotUtility.CreateSnapshot(sourcefolders, options.RawOptions); - } - catch (Exception ex) - { - if (options.SnapShotStrategy == Options.OptimizationStrategy.Required) - throw; - else if (options.SnapShotStrategy == Options.OptimizationStrategy.On) - { - stat.LogWarning(string.Format(Strings.RSyncDir.SnapshotFailedError, ex.ToString()), ex); - } - } - - return Utility.Utility.IsClientLinux ? - (Library.Snapshots.ISnapshotService)new Duplicati.Library.Snapshots.NoSnapshotLinux(sourcefolders, options.RawOptions) - : - (Library.Snapshots.ISnapshotService)new Duplicati.Library.Snapshots.NoSnapshotWindows(sourcefolders, options.RawOptions); - } - - - public void Run() - { - ForestHash.VerifyRemoteList(m_backend, m_options, m_database); - - m_blockvolume = new BlockVolumeWriter(m_options); - m_filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp); - m_shadowvolume = new ShadowVolumeWriter(m_options); - - m_database.RegisterRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary); - m_database.RegisterRemoteVolume(m_filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); - m_database.RegisterRemoteVolume(m_shadowvolume.RemoteFilename, RemoteVolumeType.Shadow, RemoteVolumeState.Temporary); - m_shadowvolume.StartVolume(m_blockvolume.RemoteFilename); - - List hashes = new List(10000); - using (m_snapshot = GetSnapshot(m_sources, m_options, m_stat)) - m_snapshot.EnumerateFilesAndFolders(this.HandleFilesystemEntry); - - if (m_changedfiles > 0 || m_changedfolders > 0) - { - if (m_blockvolume.SourceSize > 0) - { - m_backend.Put(m_blockvolume, m_shadowvolume); - } - else - { - m_database.RemoveRemoteVolume(m_blockvolume.RemoteFilename); - m_database.RemoveRemoteVolume(m_shadowvolume.RemoteFilename); - m_shadowvolume.FinishVolume(null, 0); - } - - m_backend.Put(m_filesetvolume); - } - else - { - m_database.LogMessage("info", "removing temp files, as no data needs to be uploaded", null); - m_database.RemoveRemoteVolume(m_blockvolume.RemoteFilename); - m_database.RemoveRemoteVolume(m_filesetvolume.RemoteFilename); - m_database.RemoveRemoteVolume(m_shadowvolume.RemoteFilename); - m_shadowvolume.FinishVolume(null, 0); - } - - m_backend.WaitForComplete(); - } - - private bool HandleFilesystemEntry(string rootpath, string path, System.IO.FileAttributes attributes) - { - if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) - { - if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) - return false; - if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store) - { - Dictionary metadata = null; //snapshot.GetMetadata(path); - if (metadata == null) - metadata = new Dictionary(); - - if (!metadata.ContainsKey("CoreAttributes")) - metadata["CoreAttributes"] = attributes.ToString(); - if (!metadata.ContainsKey("CoreLastWritetime")) - metadata["CoreLastWritetime"] = Utility.Utility.SerializeDateTime(m_snapshot.GetLastWriteTime(path)); - if (!metadata.ContainsKey("CoreSymlinkTarget")) - metadata["CoreSymlinkTarget"] = m_snapshot.GetSymlinkTarget(path); - - var metahash = ForestHash.WrapMetadata(metadata); - m_filesetvolume.AddSymlink(path, metahash.Hash, metahash.Size); - if (AddSymlinkToOutput(path, DateTime.UtcNow, metahash)) - m_changedfolders++; - - //Do not recurse symlinks - return false; - } - } - - if ((attributes & FileAttributes.Directory) == FileAttributes.Directory) - { - Dictionary metadata = null; //snapshot.GetMetadata(path); - if (metadata == null) - metadata = new Dictionary(); - - if (!metadata.ContainsKey("CoreAttributes")) - metadata["CoreAttributes"] = attributes.ToString(); - if (!metadata.ContainsKey("CoreLastWritetime")) - metadata["CoreLastWritetime"] = Utility.Utility.SerializeDateTime(m_snapshot.GetLastWriteTime(path)); - var metahash = ForestHash.WrapMetadata(metadata); - - m_filesetvolume.AddDirectory(path, metahash.Hash, metahash.Size); - if (AddFolderToOutput(path, DateTime.UtcNow, metahash)) - m_changedfolders++; - return true; - } - - string oldHash; - string oldMetahash; - DateTime oldScanned; - long oldId; - long oldSize; - long oldMetasize; - IList oldBlocklistHashes; - bool fileExists = m_database.GetFileEntry(path, out oldId, out oldSize, out oldScanned, out oldHash, out oldMetahash, out oldMetasize, out oldBlocklistHashes); - bool changed = false; - - //Skip symlinks if required - if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint && m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) - return false; - - try - { - DateTime lastModified = m_snapshot.GetLastWriteTime(path); - if (!fileExists || m_options.DisableFiletimeCheck || lastModified > oldScanned) - { - long filesize = 0; - DateTime scantime = DateTime.UtcNow; - - Dictionary metadata = null; //snapshot.GetMetadata(file); - if (metadata == null) - metadata = new Dictionary(); - - if (!metadata.ContainsKey("CoreAttributes")) - metadata["CoreAttributes"] = attributes.ToString(); - if (!metadata.ContainsKey("CoreLastWritetime")) - metadata["CoreLastWritetime"] = Utility.Utility.SerializeDateTime(lastModified); - - var metahashandsize = ForestHash.WrapMetadata(metadata); - var blocklisthashes = new List(); - using (var hashcollector = new HashlistCollector()) - { - using (var fs = new Blockprocessor(m_snapshot.OpenRead(path), m_blockbuffer)) - { - int size; - int blocklistoffset = 0; - - m_filehasher.Initialize(); - - do - { - size = fs.Readblock(); - - m_filehasher.TransformBlock(m_blockbuffer, 0, size, m_blockbuffer, 0); - var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, 0, size); - if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length) - { - var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); - blocklisthashes.Add(blkey); - AddBlockToOutput(blkey, m_blocklistbuffer, blocklistoffset); - blocklistoffset = 0; - } - - Array.Copy(blockkey, 0, m_blocklistbuffer, blocklistoffset, blockkey.Length); - blocklistoffset += blockkey.Length; - - var key = Convert.ToBase64String(blockkey); - AddBlockToOutput(key, m_blockbuffer, size); - hashcollector.Add(key); - filesize += size; - - - } while (size == m_blockbuffer.Length); - - //If all fits in a single block, don't bother with blocklists - if (hashcollector.Count > 1) - { - var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); - blocklisthashes.Add(blkeyfinal); - AddBlockToOutput(blkeyfinal, m_blocklistbuffer, blocklistoffset); - } - } - - - m_filehasher.TransformFinalBlock(m_blockbuffer, 0, 0); - - var filekey = Convert.ToBase64String(m_filehasher.Hash); - if (oldHash != filekey) - { - m_changedfiles++; - AddFileToOutput(path, filesize, scantime, metahashandsize, hashcollector, filekey, blocklisthashes); - changed = true; - } - } - } - - if (!changed) - AddUnmodifiedFile(path, oldId, oldSize, oldScanned, oldHash, oldMetahash, oldMetasize, oldBlocklistHashes); - - } - catch (Exception ex) - { - m_stat.LogWarning(string.Format("Failed to process path: {0}", path), ex); - } - - if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) - { - if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Follow) - return true; - else - return false; - } - - return true; - } - - /// - /// Adds the found file data to the output unless the block already exists - /// - /// The block hash - /// The data matching the hash - private bool AddBlockToOutput(string key, byte[] data, int len) - { - if (m_database.AddBlock(key, len, m_blockvolume.RemoteFilename)) - { - m_blockvolume.AddBlock(key, data, len); - m_shadowvolume.AddBlock(key, len); - if (m_blockvolume.Filesize > m_options.VolumeSize - m_options.Fhblocksize) - { - m_backend.Put(m_blockvolume, m_shadowvolume); - - m_blockvolume = new BlockVolumeWriter(m_options); - m_shadowvolume = new ShadowVolumeWriter(m_options); - m_shadowvolume.StartVolume(m_blockvolume.RemoteFilename); - } - - return true; - } - - return false; - } - - private void AddUnmodifiedFile(string path, long oldId, long size, DateTime scantime, string oldHash, string oldMetahash, long oldMetasize, IList blocklisthashes) - { - m_database.AddUnmodifiedFile(oldId, scantime); - m_filesetvolume.AddFile(path, oldHash, size, scantime, oldMetahash, oldMetasize, blocklisthashes); - } - - - /// - /// Adds a file to the output, - /// - /// The name of the file to record - /// The value of the lastModified timestamp - /// The list of hashes that make up the file - /// The size of the file - /// The offset into a fragment block where the last few bytes are stored - /// A lookup table with various metadata values describing the file - private bool AddFolderToOutput(string filename, DateTime scantime, ForestHash.IMetahash meta) - { - long metadataid; - bool r = false; - - //TODO: If meta.Size > blocksize... - r |= AddBlockToOutput(meta.Hash, meta.Blob, (int)meta.Size); - r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid); - - m_database.AddDirectoryEntry(filename, metadataid, scantime); - return r; - } - - /// - /// Adds a file to the output, - /// - /// The name of the file to record - /// The value of the lastModified timestamp - /// The list of hashes that make up the file - /// The size of the file - /// The offset into a fragment block where the last few bytes are stored - /// A lookup table with various metadata values describing the file - private bool AddSymlinkToOutput(string filename, DateTime scantime, ForestHash.IMetahash meta) - { - long metadataid; - bool r = false; - - //TODO: If meta.Size > blocksize... - r |= AddBlockToOutput(meta.Hash, meta.Blob, (int)meta.Size); - r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid); - - m_database.AddSymlinkEntry(filename, metadataid, scantime); - return r; - } - - /// - /// Adds a file to the output, - /// - /// The name of the file to record - /// The value of the lastModified timestamp - /// The list of hashes that make up the file - /// The size of the file - /// The offset into a fragment block where the last few bytes are stored - /// A lookup table with various metadata values describing the file - private void AddFileToOutput(string filename, long size, DateTime scantime, ForestHash.IMetahash metadata, HashlistCollector hashlist, string filehash, IList blocklisthashes) - { - long metadataid; - long blocksetid; - - //TODO: If metadata.Size > blocksize... - AddBlockToOutput(metadata.Hash, metadata.Blob, (int)metadata.Size); - m_database.AddMetadataset(metadata.Hash, metadata.Size, out metadataid); - - m_database.AddBlockset(filehash, size, m_blockbuffer.Length, hashlist.Hashes, blocklisthashes, out blocksetid); - - m_filesetvolume.AddFile(filename, filehash, size, scantime, metadata.Hash, metadata.Size, blocklisthashes); - m_database.AddFile(filename, scantime, blocksetid, metadataid); - } - - public void Dispose() - { - if (m_backend != null) - { - try { m_backend.Dispose(); } - finally { m_backend = null; } - } - - if (m_blockvolume != null) - { - try { m_blockvolume.Dispose(); } - finally { m_blockvolume = null; } - } - - if (m_filesetvolume != null) - { - try { m_filesetvolume.Dispose(); } - finally { m_filesetvolume = null; } - } - - if (m_shadowvolume != null) - { - try { m_shadowvolume.Dispose(); } - finally { m_shadowvolume = null; } - } - - } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using Duplicati.Library.Main.ForestHash.Database; +using Duplicati.Library.Main.ForestHash.Volumes; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Main.ForestHash.Operation +{ + internal class BackupHandler : IDisposable + { + private readonly FhOptions m_options; + private readonly BackupStatistics m_stat; + private FhBackend m_backend; + + private readonly byte[] m_blockbuffer; + private readonly byte[] m_blocklistbuffer; + private readonly System.Security.Cryptography.HashAlgorithm m_blockhasher; + private readonly System.Security.Cryptography.HashAlgorithm m_filehasher; + + private LocalBackupDatabase m_database; + private System.Data.IDbTransaction m_transaction; + private BlockVolumeWriter m_blockvolume; + private ShadowVolumeWriter m_shadowvolume; + + private Snapshots.ISnapshotService m_snapshot; + private long m_otherchanges; + + private string[] m_sources; + + public BackupHandler(string backendurl, FhOptions options, BackupStatistics stat, string[] sources) + { + m_options = options; + m_stat = stat; + m_database = new LocalBackupDatabase(m_options.Fhdbpath); + m_backend = new FhBackend(backendurl, options, m_database, m_stat); + + m_sources = sources; + m_blockbuffer = new byte[m_options.Fhblocksize]; + m_blocklistbuffer = new byte[m_options.Fhblocksize]; + + m_blockhasher = System.Security.Cryptography.SHA256.Create(); + m_filehasher = System.Security.Cryptography.SHA256.Create(); + + if (!m_blockhasher.CanReuseTransform || !m_filehasher.CanReuseTransform) + throw new Exception(Strings.Foresthash.InvalidCryptoSystem); + } + + private static Snapshots.ISnapshotService GetSnapshot(string[] sourcefolders, Options options, CommunicationStatistics stat) + { + try + { + if (options.SnapShotStrategy != Options.OptimizationStrategy.Off) + return Duplicati.Library.Snapshots.SnapshotUtility.CreateSnapshot(sourcefolders, options.RawOptions); + } + catch (Exception ex) + { + if (options.SnapShotStrategy == Options.OptimizationStrategy.Required) + throw; + else if (options.SnapShotStrategy == Options.OptimizationStrategy.On) + { + stat.LogWarning(string.Format(Strings.RSyncDir.SnapshotFailedError, ex.ToString()), ex); + } + } + + return Utility.Utility.IsClientLinux ? + (Library.Snapshots.ISnapshotService)new Duplicati.Library.Snapshots.NoSnapshotLinux(sourcefolders, options.RawOptions) + : + (Library.Snapshots.ISnapshotService)new Duplicati.Library.Snapshots.NoSnapshotWindows(sourcefolders, options.RawOptions); + } + + + public void Run() + { + ForestHash.VerifyRemoteList(m_backend, m_options, m_database); + + m_blockvolume = new BlockVolumeWriter(m_options); + m_shadowvolume = new ShadowVolumeWriter(m_options); + + m_database.RegisterRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary); + m_database.RegisterRemoteVolume(m_shadowvolume.RemoteFilename, RemoteVolumeType.Shadow, RemoteVolumeState.Temporary); + m_shadowvolume.StartVolume(m_blockvolume.RemoteFilename); + + using (m_transaction = m_database.BeginTransaction()) + { + using (m_snapshot = GetSnapshot(m_sources, m_options, m_stat)) + m_snapshot.EnumerateFilesAndFolders(this.HandleFilesystemEntry); + + m_transaction.Commit(); + } + + m_transaction = null; + + m_database.VerifyConsistency(); + m_database.UpdateChangeStatistics(m_stat); + + if (m_stat.AddedFiles > 0 || m_stat.ModifiedFiles > 0 || m_otherchanges > 0) + { + if (m_blockvolume.SourceSize > 0) + { + m_backend.Put(m_blockvolume, m_shadowvolume); + } + else + { + m_database.RemoveRemoteVolume(m_blockvolume.RemoteFilename); + m_database.RemoveRemoteVolume(m_shadowvolume.RemoteFilename); + m_shadowvolume.FinishVolume(null, 0); + } + + using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp)) + { + m_database.RegisterRemoteVolume(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); + + if (!string.IsNullOrEmpty(m_options.SignatureControlFiles)) + foreach (var p in m_options.SignatureControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries)) + filesetvolume.AddControlFile(p, m_options.GetCompressionHintFromFilename(p)); + + m_database.WriteFileset(filesetvolume); + m_backend.Put(filesetvolume); + } + } + else + { + m_database.LogMessage("info", "removing temp files, as no data needs to be uploaded", null); + m_database.RemoveRemoteVolume(m_blockvolume.RemoteFilename); + m_database.RemoveRemoteVolume(m_shadowvolume.RemoteFilename); + m_shadowvolume.FinishVolume(null, 0); + } + + m_backend.WaitForComplete(); + } + + private bool HandleFilesystemEntry(string rootpath, string path, System.IO.FileAttributes attributes) + { + if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) + { + if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) + return false; + + if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store) + { + Dictionary metadata = null; //snapshot.GetMetadata(path); + if (metadata == null) + metadata = new Dictionary(); + + if (!metadata.ContainsKey("CoreAttributes")) + metadata["CoreAttributes"] = attributes.ToString(); + if (!metadata.ContainsKey("CoreLastWritetime")) + metadata["CoreLastWritetime"] = Utility.Utility.SerializeDateTime(m_snapshot.GetLastWriteTime(path)); + if (!metadata.ContainsKey("CoreSymlinkTarget")) + metadata["CoreSymlinkTarget"] = m_snapshot.GetSymlinkTarget(path); + + var metahash = ForestHash.WrapMetadata(metadata); + //m_filesetvolume.AddSymlink(path, metahash.Hash, metahash.Size); + if (AddSymlinkToOutput(path, DateTime.UtcNow, metahash)) + m_otherchanges++; + + //Do not recurse symlinks + return false; + } + } + + if ((attributes & FileAttributes.Directory) == FileAttributes.Directory) + { + Dictionary metadata = null; //snapshot.GetMetadata(path); + if (metadata == null) + metadata = new Dictionary(); + + if (!metadata.ContainsKey("CoreAttributes")) + metadata["CoreAttributes"] = attributes.ToString(); + if (!metadata.ContainsKey("CoreLastWritetime")) + metadata["CoreLastWritetime"] = Utility.Utility.SerializeDateTime(m_snapshot.GetLastWriteTime(path)); + var metahash = ForestHash.WrapMetadata(metadata); + + //m_filesetvolume.AddDirectory(path, metahash.Hash, metahash.Size); + if (AddFolderToOutput(path, DateTime.UtcNow, metahash)) + m_otherchanges++; + return true; + } + + DateTime oldScanned; + var oldId = m_database.GetFileEntry(path, out oldScanned); + m_stat.ExaminedFiles++; + + bool changed = false; + + //Skip symlinks if required + if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint && m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) + return false; + + try + { + DateTime lastModified = m_snapshot.GetLastWriteTime(path); + if (oldId < 0 || m_options.DisableFiletimeCheck || lastModified > oldScanned) + { + m_stat.OpenedFiles++; + + long filesize = 0; + DateTime scantime = DateTime.UtcNow; + + Dictionary metadata = null; //snapshot.GetMetadata(file); + if (metadata == null) + metadata = new Dictionary(); + + if (!metadata.ContainsKey("CoreAttributes")) + metadata["CoreAttributes"] = attributes.ToString(); + if (!metadata.ContainsKey("CoreLastWritetime")) + metadata["CoreLastWritetime"] = Utility.Utility.SerializeDateTime(lastModified); + + var metahashandsize = ForestHash.WrapMetadata(metadata); + var blocklisthashes = new List(); + var hint = m_options.GetCompressionHintFromFilename(path); + var oldHash = oldId < 0 ? null : m_database.GetFileHash(oldId); + + using (var hashcollector = new HashlistCollector()) + { + using (var fs = new Blockprocessor(m_snapshot.OpenRead(path), m_blockbuffer)) + { + int size; + int blocklistoffset = 0; + + m_filehasher.Initialize(); + + do + { + size = fs.Readblock(); + + m_filehasher.TransformBlock(m_blockbuffer, 0, size, m_blockbuffer, 0); + var blockkey = m_blockhasher.ComputeHash(m_blockbuffer, 0, size); + if (m_blocklistbuffer.Length - blocklistoffset < blockkey.Length) + { + var blkey = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); + blocklisthashes.Add(blkey); + AddBlockToOutput(blkey, m_blocklistbuffer, blocklistoffset, hint); + blocklistoffset = 0; + } + + Array.Copy(blockkey, 0, m_blocklistbuffer, blocklistoffset, blockkey.Length); + blocklistoffset += blockkey.Length; + + var key = Convert.ToBase64String(blockkey); + AddBlockToOutput(key, m_blockbuffer, size, hint); + hashcollector.Add(key); + filesize += size; + + + } while (size == m_blockbuffer.Length); + + //If all fits in a single block, don't bother with blocklists + if (hashcollector.Count > 1) + { + var blkeyfinal = Convert.ToBase64String(m_blockhasher.ComputeHash(m_blocklistbuffer, 0, blocklistoffset)); + blocklisthashes.Add(blkeyfinal); + AddBlockToOutput(blkeyfinal, m_blocklistbuffer, blocklistoffset, CompressionHint.Noncompressible); + } + } + + m_stat.SizeOfExaminedFiles += filesize; + m_filehasher.TransformFinalBlock(m_blockbuffer, 0, 0); + + var filekey = Convert.ToBase64String(m_filehasher.Hash); + if (oldHash != filekey) + { + if (oldId < 0) + { + m_stat.AddedFiles++; + m_stat.SizeOfAddedFiles += filesize; + } + else + { + m_stat.ModifiedFiles++; + m_stat.SizeOfModifiedFiles += filesize; + } + + AddFileToOutput(path, filesize, scantime, metahashandsize, hashcollector, filekey, blocklisthashes); + changed = true; + } + } + } + + if (!changed) + AddUnmodifiedFile(oldId, oldScanned); + + } + catch (Exception ex) + { + m_stat.LogWarning(string.Format("Failed to process path: {0}", path), ex); + m_stat.FilesWithError++; + } + + if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) + { + if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Follow) + return true; + else + return false; + } + + return true; + } + + /// + /// Adds the found file data to the output unless the block already exists + /// + /// The block hash + /// The data matching the hash + private bool AddBlockToOutput(string key, byte[] data, int len, CompressionHint hint) + { + if (m_database.AddBlock(key, len, m_blockvolume.RemoteFilename, m_transaction)) + { + m_blockvolume.AddBlock(key, data, len, hint); + m_shadowvolume.AddBlock(key, len); + if (m_blockvolume.Filesize > m_options.VolumeSize - m_options.Fhblocksize) + { + m_backend.Put(m_blockvolume, m_shadowvolume); + + m_blockvolume = new BlockVolumeWriter(m_options); + m_shadowvolume = new ShadowVolumeWriter(m_options); + m_shadowvolume.StartVolume(m_blockvolume.RemoteFilename); + } + + return true; + } + + return false; + } + + private void AddUnmodifiedFile(long oldId, DateTime scantime) + { + m_database.AddUnmodifiedFile(oldId, scantime, m_transaction); + } + + + /// + /// Adds a file to the output, + /// + /// The name of the file to record + /// The value of the lastModified timestamp + /// The list of hashes that make up the file + /// The size of the file + /// The offset into a fragment block where the last few bytes are stored + /// A lookup table with various metadata values describing the file + private bool AddFolderToOutput(string filename, DateTime scantime, ForestHash.IMetahash meta) + { + long metadataid; + bool r = false; + + //TODO: If meta.Size > blocksize... + r |= AddBlockToOutput(meta.Hash, meta.Blob, (int)meta.Size, CompressionHint.Default); + r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid, m_transaction); + + m_database.AddDirectoryEntry(filename, metadataid, scantime, m_transaction); + return r; + } + + /// + /// Adds a file to the output, + /// + /// The name of the file to record + /// The value of the lastModified timestamp + /// The list of hashes that make up the file + /// The size of the file + /// The offset into a fragment block where the last few bytes are stored + /// A lookup table with various metadata values describing the file + private bool AddSymlinkToOutput(string filename, DateTime scantime, ForestHash.IMetahash meta) + { + long metadataid; + bool r = false; + + //TODO: If meta.Size > blocksize... + r |= AddBlockToOutput(meta.Hash, meta.Blob, (int)meta.Size, CompressionHint.Default); + r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid, m_transaction); + + m_database.AddSymlinkEntry(filename, metadataid, scantime, m_transaction); + return r; + } + + /// + /// Adds a file to the output, + /// + /// The name of the file to record + /// The value of the lastModified timestamp + /// The list of hashes that make up the file + /// The size of the file + /// The offset into a fragment block where the last few bytes are stored + /// A lookup table with various metadata values describing the file + private void AddFileToOutput(string filename, long size, DateTime scantime, ForestHash.IMetahash metadata, HashlistCollector hashlist, string filehash, IList blocklisthashes) + { + long metadataid; + long blocksetid; + + //TODO: If metadata.Size > blocksize... + AddBlockToOutput(metadata.Hash, metadata.Blob, (int)metadata.Size, CompressionHint.Default); + m_database.AddMetadataset(metadata.Hash, metadata.Size, out metadataid, m_transaction); + + m_database.AddBlockset(filehash, size, m_blockbuffer.Length, hashlist.Hashes, blocklisthashes, out blocksetid, m_transaction); + + //m_filesetvolume.AddFile(filename, filehash, size, scantime, metadata.Hash, metadata.Size, blocklisthashes); + m_database.AddFile(filename, scantime, blocksetid, metadataid, m_transaction); + } + + public void Dispose() + { + if (m_backend != null) + { + try { m_backend.Dispose(); } + finally { m_backend = null; } + } + + if (m_blockvolume != null) + { + try { m_blockvolume.Dispose(); } + finally { m_blockvolume = null; } + } + + if (m_shadowvolume != null) + { + try { m_shadowvolume.Dispose(); } + finally { m_shadowvolume = null; } + } + + m_stat.EndTime = DateTime.Now; + } + } +} diff --git a/Duplicati/Library/Main/ForestHash/Operation/CleanupHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/CleanupHandler.cs index 8e3509544..d052c9446 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/CleanupHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/CleanupHandler.cs @@ -9,13 +9,13 @@ namespace Duplicati.Library.Main.ForestHash.Operation { private string m_backendurl; private FhOptions m_options; - private RestoreStatistics m_rs; + private RestoreStatistics m_stat; - public CleanupHandler(string backend, FhOptions options, RestoreStatistics rs) + public CleanupHandler(string backend, FhOptions options, RestoreStatistics stat) { m_backendurl = backend; m_options = options; - m_rs = rs; + m_stat = stat; } public void Run() diff --git a/Duplicati/Library/Main/ForestHash/Operation/DeleteAllButNHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/DeleteAllButNHandler.cs index 8b75b1835..7f068225b 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/DeleteAllButNHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/DeleteAllButNHandler.cs @@ -9,13 +9,13 @@ namespace Duplicati.Library.Main.ForestHash.Operation { private string m_backendurl; private FhOptions m_options; - private RestoreStatistics m_rs; + private RestoreStatistics m_stat; - public DeleteAllButNHandler(string backend, FhOptions options, RestoreStatistics rs) + public DeleteAllButNHandler(string backend, FhOptions options, RestoreStatistics stat) { m_backendurl = backend; m_options = options; - m_rs = rs; + m_stat = stat; } public void Run() diff --git a/Duplicati/Library/Main/ForestHash/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/ListFilesHandler.cs index f3eea16ae..18bccbeae 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/ListFilesHandler.cs @@ -9,13 +9,13 @@ namespace Duplicati.Library.Main.ForestHash.Operation { private string m_backendurl; private FhOptions m_options; - private RestoreStatistics m_rs; + private RestoreStatistics m_stat; - public ListFilesHandler(string backend, FhOptions options, RestoreStatistics rs) + public ListFilesHandler(string backend, FhOptions options, RestoreStatistics stat) { m_backendurl = backend; m_options = options; - m_rs = rs; + m_stat = stat; } public List Run() @@ -36,7 +36,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation // Otherwise, grab info from remote location using (var tmpdb = new Utility.TempFile()) using (var db = new Database.Localdatabase(tmpdb, "ListFiles")) - using (var backend = new FhBackend(m_backendurl, m_options, db)) + using (var backend = new FhBackend(m_backendurl, m_options, db, m_stat)) { var filter = RestoreHandler.FilterFilelist(m_options.RestoreTime); var fileset = filter(from n in backend.List() diff --git a/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs index cf2e2be12..6e7d02511 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs @@ -1,171 +1,173 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Duplicati.Library.Main.ForestHash.Database; -using Duplicati.Library.Main.ForestHash.Volumes; - -namespace Duplicati.Library.Main.ForestHash.Operation -{ - public class RecreateDatabaseHandler : IDisposable - { - private string m_backendurl; - private FhOptions m_options; - private byte[] m_blockbuffer; - private string m_destination; - - public delegate IEnumerable FilterFilelistDelegate(IEnumerable filelist); - public delegate IEnumerable FilenameFilterDelegate(IEnumerable filenamelist); - public delegate void BlockVolumePostProcessor(string volumename, BlockVolumeReader reader); - - public RecreateDatabaseHandler(string backendurl, FhOptions options, string destination) - { - m_options = options; - m_backendurl = backendurl; - - m_destination = destination; - m_blockbuffer = new byte[m_options.Fhblocksize]; - - } - - public void Run(string path, FilterFilelistDelegate filelistfilter = null, FilenameFilterDelegate filenamefilter = null, BlockVolumePostProcessor blockprocessor = null) - { - var hashsize = System.Security.Cryptography.SHA256.Create().HashSize / 8; - - //We build a local database in steps. - using (var restoredb = new LocalBlocklistUpdateDatabase(path, m_options.Fhblocksize)) - using (var backend = new FhBackend(m_backendurl, m_options, restoredb)) - { - //First step is to examine the remote storage to see what - // kind of data we can find - var remotefiles = - (from n in - (from x in backend.List() - select VolumeBase.ParseFilename(x)) - where - n != null - && - n.EncryptionModule == (m_options.NoEncryption ? null : m_options.EncryptionModule) - && - n.Prefix == m_options.BackupPrefix - select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times - - //Then we select the filelist we should work with, - // and create the filelist table to fit - IEnumerable filelists = - from n in remotefiles - where n.FileType == RemoteVolumeType.Files - orderby n.Time descending - select n; - - if (filelistfilter != null) - filelists = filelistfilter(filelists); - - using (var backupdb = new LocalBackupDatabase(path)) - { - foreach (var fl in remotefiles) - backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded); - - var shadowfiles = - from n in remotefiles - where n.FileType == RemoteVolumeType.Shadow - select new RemoteVolume(n.File) as IRemoteVolume; - - //We grab all shadow files, and update the block table - foreach (var sf in new AsyncDownloader(shadowfiles.ToList(), backend)) - { - if (sf.Key.Hash != null && sf.Key.Size > 0) - backupdb.UpdateRemoteVolume(sf.Key.Name, RemoteVolumeState.Verified, sf.Key.Size, sf.Key.Hash); - - using (var svr = new ShadowVolumeReader(RestoreHandler.GetCompressionModule(sf.Key.Name), sf.Value, m_options, hashsize)) - { - //If there are blocklists in the shadow file, update the blocklists - using (var tr = restoredb.BeginTransaction()) - { - foreach (var b in svr.BlockLists) - restoredb.UpdateBlocklist(b.Hash, b.Blocklist, hashsize, tr); - - tr.Commit(); - } - - foreach (var a in svr.Volumes) - { - //Add all block/volume mappings - foreach (var b in a.Blocks) - backupdb.AddBlock(b.Key, b.Value, a.Filename); - - backupdb.UpdateRemoteVolume(a.Filename, RemoteVolumeState.Verified, a.Length, a.Hash); - } - } - } - - //We need this to prepare for the block-lists - var dummylist = new string[0]; - - // Default filter is no filter (pass-through) - if (filenamefilter == null) - filenamefilter = lst => lst; - - //Now record all blocksets and files needed - foreach (var entry in filelists) - using (var filelist = backend.Get(entry.File.Name, entry.File.Size, null)) - using (var filelistreader = new FilesetVolumeReader(RestoreHandler.GetCompressionModule(entry.File.Name), filelist, m_options)) - foreach (var fe in filenamefilter(filelistreader.Files)) - { - if (fe.Type == FilelistEntryType.Folder) - { - long metaid = -1; - if (fe.Metahash != null) - backupdb.AddMetadataset(fe.Metahash, fe.Metasize, out metaid); - backupdb.AddDirectoryEntry(fe.Path, metaid, fe.Time); - - } - else if (fe.Type == FilelistEntryType.File) - { - long metaid = -1; - long blocksetid; - if (fe.Metahash != null) - backupdb.AddMetadataset(fe.Metahash, fe.Metasize, out metaid); - - backupdb.AddBlockset(fe.Hash, fe.Size, m_options.Fhblocksize, dummylist, fe.BlocklistHashes, out blocksetid); - backupdb.AddFile(fe.Path, fe.Time, blocksetid, metaid); - } - } - } - - //We now need some blocklists, so we start by grabbing a - // volume with one of the blocklists - //For each volume we then update the blocklist table - // and then restore the blocks we know - var blockfiles = - from n in remotefiles - where n.FileType == RemoteVolumeType.Blocks - select new RemoteVolume(n.File) as IRemoteVolume; - - restoredb.FindMissingBlocklistHashes(); - - foreach (var sf in new AsyncDownloader(restoredb.GetMissingBlockListVolumes(), backend)) - using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Key.Name), sf.Value, m_options)) - using (var tr = restoredb.BeginTransaction()) - { - foreach (var blocklisthash in restoredb.GetBlockLists(sf.Key.Name)) - restoredb.UpdateBlocklist(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), hashsize, tr); - - tr.Commit(); - - //At this point we can patch files with data from the block volume - if (blockprocessor != null) - blockprocessor(sf.Key.Name, rd); - } - - //All done, we must verify that we have all blocklist fully intact - restoredb.VerifyDatabaseIntegrity(); - - } - } - - public void Dispose() - { - } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Duplicati.Library.Main.ForestHash.Database; +using Duplicati.Library.Main.ForestHash.Volumes; + +namespace Duplicati.Library.Main.ForestHash.Operation +{ + public class RecreateDatabaseHandler : IDisposable + { + private string m_backendurl; + private FhOptions m_options; + private byte[] m_blockbuffer; + private string m_destination; + private CommunicationStatistics m_stat; + + public delegate IEnumerable FilterFilelistDelegate(IEnumerable filelist); + public delegate IEnumerable FilenameFilterDelegate(IEnumerable filenamelist); + public delegate void BlockVolumePostProcessor(string volumename, BlockVolumeReader reader); + + public RecreateDatabaseHandler(string backendurl, FhOptions options, string destination, CommunicationStatistics stat) + { + m_options = options; + m_backendurl = backendurl; + m_stat = stat; + + m_destination = destination; + m_blockbuffer = new byte[m_options.Fhblocksize]; + + } + + public void Run(string path, FilterFilelistDelegate filelistfilter = null, FilenameFilterDelegate filenamefilter = null, BlockVolumePostProcessor blockprocessor = null) + { + var hashsize = System.Security.Cryptography.SHA256.Create().HashSize / 8; + + //We build a local database in steps. + using (var restoredb = new LocalBlocklistUpdateDatabase(path, m_options.Fhblocksize)) + using (var backend = new FhBackend(m_backendurl, m_options, restoredb, m_stat)) + { + //First step is to examine the remote storage to see what + // kind of data we can find + var remotefiles = + (from n in + (from x in backend.List() + select VolumeBase.ParseFilename(x)) + where + n != null + && + n.EncryptionModule == (m_options.NoEncryption ? null : m_options.EncryptionModule) + && + n.Prefix == m_options.BackupPrefix + select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times + + //Then we select the filelist we should work with, + // and create the filelist table to fit + IEnumerable filelists = + from n in remotefiles + where n.FileType == RemoteVolumeType.Files + orderby n.Time descending + select n; + + if (filelistfilter != null) + filelists = filelistfilter(filelists); + + using (var backupdb = new LocalBackupDatabase(restoredb)) + { + foreach (var fl in remotefiles) + backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded); + + var shadowfiles = + from n in remotefiles + where n.FileType == RemoteVolumeType.Shadow + select new RemoteVolume(n.File) as IRemoteVolume; + + //We grab all shadow files, and update the block table + using (var tr = restoredb.BeginTransaction()) + { + foreach (var sf in new AsyncDownloader(shadowfiles.ToList(), backend)) + { + if (sf.Key.Hash != null && sf.Key.Size > 0) + backupdb.UpdateRemoteVolume(sf.Key.Name, RemoteVolumeState.Verified, sf.Key.Size, sf.Key.Hash, tr); + + using (var svr = new ShadowVolumeReader(RestoreHandler.GetCompressionModule(sf.Key.Name), sf.Value, m_options, hashsize)) + { + //If there are blocklists in the shadow file, update the blocklists + foreach (var b in svr.BlockLists) + restoredb.UpdateBlocklist(b.Hash, b.Blocklist, hashsize, tr); + + foreach (var a in svr.Volumes) + { + //Add all block/volume mappings + foreach (var b in a.Blocks) + backupdb.AddBlock(b.Key, b.Value, a.Filename, tr); + + backupdb.UpdateRemoteVolume(a.Filename, RemoteVolumeState.Verified, a.Length, a.Hash, tr); + } + } + } + + tr.Commit(); + } + + //We need this to prepare for the block-lists + var dummylist = new string[0]; + + // Default filter is no filter (pass-through) + if (filenamefilter == null) + filenamefilter = lst => lst; + + //Now record all blocksets and files needed + using (var tr = backupdb.BeginTransaction()) + { + foreach (var entry in filelists) + using (var filelist = backend.Get(entry.File.Name, entry.File.Size, null)) + using (var filelistreader = new FilesetVolumeReader(RestoreHandler.GetCompressionModule(entry.File.Name), filelist, m_options)) + foreach (var fe in filenamefilter(filelistreader.Files)) + { + if (fe.Type == FilelistEntryType.Folder) + { + long metaid = -1; + if (fe.Metahash != null) + backupdb.AddMetadataset(fe.Metahash, fe.Metasize, out metaid, tr); + backupdb.AddDirectoryEntry(fe.Path, metaid, fe.Time, tr); + + } + else if (fe.Type == FilelistEntryType.File) + { + long metaid = -1; + long blocksetid; + if (fe.Metahash != null) + backupdb.AddMetadataset(fe.Metahash, fe.Metasize, out metaid, tr); + + backupdb.AddBlockset(fe.Hash, fe.Size, m_options.Fhblocksize, dummylist, fe.BlocklistHashes, out blocksetid, tr); + backupdb.AddFile(fe.Path, fe.Time, blocksetid, metaid, tr); + } + } + tr.Commit(); + } + } + + //We now need some blocklists, so we start by grabbing a + // volume with one of the blocklists + //For each volume we then update the blocklist table + // and then restore the blocks we know + restoredb.FindMissingBlocklistHashes(); + + foreach (var sf in new AsyncDownloader(restoredb.GetMissingBlockListVolumes(), backend)) + using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Key.Name), sf.Value, m_options)) + using (var tr = restoredb.BeginTransaction()) + { + foreach (var blocklisthash in restoredb.GetBlockLists(sf.Key.Name)) + restoredb.UpdateBlocklist(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), hashsize, tr); + + tr.Commit(); + + //At this point we can patch files with data from the block volume + if (blockprocessor != null) + blockprocessor(sf.Key.Name, rd); + } + + //All done, we must verify that we have all blocklist fully intact + restoredb.VerifyDatabaseIntegrity(); + + } + } + + public void Dispose() + { + m_stat.EndTime = DateTime.Now; + } + } +} diff --git a/Duplicati/Library/Main/ForestHash/Operation/RestoreControlFilesHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/RestoreControlFilesHandler.cs index 5d0b23066..3417b9c1e 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/RestoreControlFilesHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/RestoreControlFilesHandler.cs @@ -23,7 +23,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation { using (var tmpdb = new Utility.TempFile()) using (var db = new Database.Localdatabase(System.IO.File.Exists(m_options.Fhdbpath) ? m_options.Fhdbpath : (string)tmpdb, "RestoreControlFiles")) - using (var backend = new FhBackend(m_backendurl, m_options, db)) + using (var backend = new FhBackend(m_backendurl, m_options, db, m_stat)) { var files = from file in backend.List() let p = Volumes.VolumeBase.ParseFilename(file) diff --git a/Duplicati/Library/Main/ForestHash/Operation/RestoreHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/RestoreHandler.cs index 62f3b994c..5aa7f2959 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/RestoreHandler.cs @@ -101,7 +101,6 @@ namespace Duplicati.Library.Main.ForestHash.Operation // This prevents repeated downloads, except for cases where the blocklists refer blocks // that have been previously handled. A local blockvolume cache can reduce this issue using (var database = new LocalRestoredatabase(tmpdb ?? m_options.Fhdbpath, m_options.Fhblocksize)) - using (var backend = new FhBackend(m_backendurl, m_options, database)) { var hasher = System.Security.Cryptography.SHA256.Create(); if (!hasher.CanReuseTransform) @@ -156,7 +155,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation // TODO: When UpdateMisisngBlocksTable is implemented, the localpatcher can be activated // and this will reduce the need for multiple downloads of the same volume - using (var rdb = new RecreateDatabaseHandler(m_backendurl, m_options, m_destination)) + using (var rdb = new RecreateDatabaseHandler(m_backendurl, m_options, m_destination, m_stat)) rdb.Run(tmpdb, filelistfilter, filenamefilter, /*localpatcher*/ null); } @@ -166,12 +165,12 @@ namespace Duplicati.Library.Main.ForestHash.Operation } - public void Run(string dbpath = null) + public void Run(string dbpath) { //In this case, we check that the remote storage fits with the database. //We can then query the database and find the blocks that we need to do the restore using (var database = new LocalRestoredatabase(dbpath ?? m_options.Fhdbpath, m_options.Fhblocksize)) - using (var backend = new FhBackend(m_backendurl, m_options, database)) + using (var backend = new FhBackend(m_backendurl, m_options, database, m_stat)) { var hasher = System.Security.Cryptography.SHA256.Create(); if (!hasher.CanReuseTransform) @@ -203,6 +202,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation foreach (var restorelist in database.GetFilesWithMissingBlocks(blocks)) { var targetpath = restorelist.Path; + //TODO: Try/Catch using (var file = System.IO.File.Open(targetpath, System.IO.FileMode.OpenOrCreate, System.IO.FileAccess.ReadWrite, System.IO.FileShare.None)) foreach (var targetblock in restorelist.Blocks) { @@ -346,7 +346,8 @@ namespace Duplicati.Library.Main.ForestHash.Operation } public void Dispose() - { + { + m_stat.EndTime = DateTime.Now; } } } diff --git a/Duplicati/Library/Main/ForestHash/Volumes/BlockVolumeReader.cs b/Duplicati/Library/Main/ForestHash/Volumes/BlockVolumeReader.cs index 29c2aac60..b58ea2436 100644 --- a/Duplicati/Library/Main/ForestHash/Volumes/BlockVolumeReader.cs +++ b/Duplicati/Library/Main/ForestHash/Volumes/BlockVolumeReader.cs @@ -21,13 +21,13 @@ namespace Duplicati.Library.Main.ForestHash.Volumes public int ReadBlock(string hash, byte[] m_blockbuffer) { - using (var fs = m_compression.OpenRead(hash.Replace('+', '-').Replace('/', '_'))) + using (var fs = m_compression.OpenRead(Utility.Utility.Base64PlainToBase64Url(hash))) return Utility.Utility.ForceStreamRead(fs, m_blockbuffer, m_blockbuffer.Length); } public IEnumerable ReadBlocklist(string hash, long hashsize) - { - return new BlocklistEnumerable(m_compression, hash.Replace('+', '-').Replace('/', '_'), hashsize); + { + return new BlocklistEnumerable(m_compression, Utility.Utility.Base64PlainToBase64Url(hash), hashsize); } //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, @@ -38,10 +38,10 @@ namespace Duplicati.Library.Main.ForestHash.Volumes get { return - from n in m_compression.GetFilesWithSize(null) + from n in m_compression.ListFilesWithSize(null) let valid = m_base64_urlsafe_detector.Match(n.Key) where valid.Success && valid.Length == n.Key.Length - select new KeyValuePair(n.Key.Replace('-', '+').Replace('_', '/'), n.Value); + select new KeyValuePair(Utility.Utility.Base64UrlToBase64Plain(n.Key), n.Value); } } } diff --git a/Duplicati/Library/Main/ForestHash/Volumes/BlockVolumeWriter.cs b/Duplicati/Library/Main/ForestHash/Volumes/BlockVolumeWriter.cs index ccb58089a..ee1435639 100644 --- a/Duplicati/Library/Main/ForestHash/Volumes/BlockVolumeWriter.cs +++ b/Duplicati/Library/Main/ForestHash/Volumes/BlockVolumeWriter.cs @@ -1,7 +1,8 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; +using System.Text; +using Duplicati.Library.Interface; namespace Duplicati.Library.Main.ForestHash.Volumes { @@ -20,14 +21,13 @@ namespace Duplicati.Library.Main.ForestHash.Volumes { } - public void AddBlock(string hash, byte[] data, int size) + public void AddBlock(string hash, byte[] data, int size, CompressionHint hint) { m_blocks++; m_sourcesize += size; //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, - // to prevent clashes with filename paths where forward slash has special meaning - using (var s = m_compression.CreateFile(hash.Replace('+', '-').Replace('/', '_'), DateTime.UtcNow)) + using (var s = m_compression.CreateFile(Utility.Utility.Base64PlainToBase64Url(hash), hint, DateTime.UtcNow)) s.Write(data, 0, size); } } diff --git a/Duplicati/Library/Main/ForestHash/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/ForestHash/Volumes/FilesetVolumeWriter.cs index 634efe05e..97f63605e 100644 --- a/Duplicati/Library/Main/ForestHash/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/ForestHash/Volumes/FilesetVolumeWriter.cs @@ -3,7 +3,8 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.IO; -using Newtonsoft.Json; +using Newtonsoft.Json; +using Duplicati.Library.Interface; namespace Duplicati.Library.Main.ForestHash.Volumes { @@ -19,22 +20,22 @@ namespace Duplicati.Library.Main.ForestHash.Volumes public FilesetVolumeWriter(FhOptions options, DateTime timestamp) : base(options, timestamp) { - m_streamwriter = new StreamWriter(m_compression.CreateFile(FILELIST, DateTime.UtcNow)); + m_streamwriter = new StreamWriter(m_compression.CreateFile(FILELIST, CompressionHint.Compressible, DateTime.UtcNow)); m_writer = new JsonTextWriter(m_streamwriter); m_writer.WriteStartArray(); - } - - public void AddFile(string name, string filehash, long size, DateTime scantime, string metahash, long metasize, IList blocklisthashes) + } + + public void AddFile(string name, string filehash, long size, DateTime scantime, string metahash, long metasize, IEnumerable blocklisthashes) { AddFileEntry(FilelistEntryType.File, name, filehash, size, scantime, metahash, metasize, blocklisthashes); - } - - public void AddAlternateStream(string name, string filehash, long size, DateTime scantime, string metahash, long metasize, IList blocklisthashes) + } + + public void AddAlternateStream(string name, string filehash, long size, DateTime scantime, string metahash, long metasize, IEnumerable blocklisthashes) { AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, scantime, metahash, metasize, blocklisthashes); } - private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime scantime, string metahash, long metasize, IList blocklisthashes) + private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime scantime, string metahash, long metasize, IEnumerable blocklisthashes) { m_filecount++; m_writer.WriteStartObject(); @@ -56,13 +57,19 @@ namespace Duplicati.Library.Main.ForestHash.Volumes m_writer.WriteValue(metasize); } - if (blocklisthashes != null && blocklisthashes.Count > 0) - { - m_writer.WritePropertyName("blocklists"); - m_writer.WriteStartArray(); - foreach (var s in blocklisthashes) - m_writer.WriteValue(s); - m_writer.WriteEndArray(); + if (blocklisthashes != null) + { + //Slightly akward, but we avoid writing if there are no entries + var en = blocklisthashes.GetEnumerator(); + if (en.MoveNext()) + { + m_writer.WritePropertyName("blocklists"); + m_writer.WriteStartArray(); + m_writer.WriteValue(en.Current); + while(en.MoveNext()) + m_writer.WriteValue(en.Current); + m_writer.WriteEndArray(); + } } m_writer.WriteEndObject(); @@ -104,10 +111,10 @@ namespace Duplicati.Library.Main.ForestHash.Volumes base.Close(); } - public void AddControlFile(string localfile, string filename = null) + public void AddControlFile(string localfile, CompressionHint hint, string filename = null) { filename = filename ?? System.IO.Path.GetFileName(localfile); - using (var t = m_compression.CreateFile(CONTROL_FILES_FOLDER + filename, DateTime.UtcNow)) + using (var t = m_compression.CreateFile(CONTROL_FILES_FOLDER + filename, hint, DateTime.UtcNow)) using (var s = System.IO.File.OpenRead(localfile)) Utility.Utility.CopyStream(s, t); } diff --git a/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeReader.cs b/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeReader.cs index 78ec4eddf..b07ebb5e7 100644 --- a/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeReader.cs +++ b/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeReader.cs @@ -369,7 +369,7 @@ namespace Duplicati.Library.Main.ForestHash.Volumes public void Reset() { - m_files = m_compression.GetFilesWithSize(SHADOW_BLOCKLIST_FOLDER).ToArray(); + m_files = m_compression.ListFilesWithSize(SHADOW_BLOCKLIST_FOLDER).ToArray(); m_index = -1; m_current = null; } diff --git a/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeWriter.cs b/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeWriter.cs index 6b3d75350..36e7fad62 100644 --- a/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeWriter.cs +++ b/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeWriter.cs @@ -3,7 +3,8 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.IO; -using Newtonsoft.Json; +using Newtonsoft.Json; +using Duplicati.Library.Interface; namespace Duplicati.Library.Main.ForestHash.Volumes { @@ -33,7 +34,7 @@ namespace Duplicati.Library.Main.ForestHash.Volumes throw new InvalidOperationException("Previous volume not finished, call FinishVolume before starting a new volume"); m_volumes++; - m_streamwriter = new StreamWriter(m_compression.CreateFile(SHADOW_VOLUME_FOLDER + filename, DateTime.UtcNow)); + m_streamwriter = new StreamWriter(m_compression.CreateFile(SHADOW_VOLUME_FOLDER + filename, CompressionHint.Compressible, DateTime.UtcNow)); m_writer = new JsonTextWriter(m_streamwriter); m_writer.WriteStartObject(); m_writer.WritePropertyName("blocks"); @@ -70,8 +71,7 @@ namespace Duplicati.Library.Main.ForestHash.Volumes { m_blocklists++; //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, - // to prevent clashes with filename paths where forward slash has special meaning - using (var s = m_compression.CreateFile(SHADOW_BLOCKLIST_FOLDER + (hash).Replace('+', '-').Replace('/', '_'), DateTime.UtcNow)) + using (var s = m_compression.CreateFile(SHADOW_BLOCKLIST_FOLDER + Utility.Utility.Base64PlainToBase64Url(hash), CompressionHint.Noncompressible, DateTime.UtcNow)) s.Write(data, 0, size); } diff --git a/Duplicati/Library/Main/ForestHash/Volumes/VolumeWriterBase.cs b/Duplicati/Library/Main/ForestHash/Volumes/VolumeWriterBase.cs index b373c5f6e..8e2916cd9 100644 --- a/Duplicati/Library/Main/ForestHash/Volumes/VolumeWriterBase.cs +++ b/Duplicati/Library/Main/ForestHash/Volumes/VolumeWriterBase.cs @@ -42,7 +42,7 @@ namespace Duplicati.Library.Main.ForestHash.Volumes protected void AddManifestfile() { - using (var sr = new StreamWriter(m_compression.CreateFile(MANIFEST_FILENAME, DateTime.UtcNow), ENCODING)) + using (var sr = new StreamWriter(m_compression.CreateFile(MANIFEST_FILENAME, CompressionHint.Compressible, DateTime.UtcNow), ENCODING)) sr.Write(ManifestData.GetManifestInstance(m_blocksize)); } diff --git a/Duplicati/Library/Main/Interface.cs b/Duplicati/Library/Main/Interface.cs index 591cf20ed..29a1dd722 100644 --- a/Duplicati/Library/Main/Interface.cs +++ b/Duplicati/Library/Main/Interface.cs @@ -3056,9 +3056,10 @@ namespace Duplicati.Library.Main #endregion - public static IEnumerable ParseFhFileList(string target, Dictionary options) + public static IEnumerable ParseFhFileList(string target, Dictionary options, CommunicationStatistics stat) { - return ForestHash.ForestHash.ParseFileList(target, options); + stat = stat ?? new CommunicationStatistics(DuplicatiOperationMode.List); + return ForestHash.ForestHash.ParseFileList(target, options, stat); } public static string CompactBlocks(string target, Dictionary options) diff --git a/Duplicati/Library/Main/RestoreStatistics.cs b/Duplicati/Library/Main/RestoreStatistics.cs index 3f49c7a3b..ce4d16b2b 100644 --- a/Duplicati/Library/Main/RestoreStatistics.cs +++ b/Duplicati/Library/Main/RestoreStatistics.cs @@ -25,8 +25,6 @@ namespace Duplicati.Library.Main { internal class RestoreStatistics : CommunicationStatistics { - private DateTime m_beginTime; - private DateTime m_endTime; private long m_filesRestored; private long m_sizeOfRestoredFiles; private long m_foldersRestored; @@ -37,24 +35,6 @@ namespace Duplicati.Library.Main public RestoreStatistics(DuplicatiOperationMode operationMode) : base(operationMode) { - m_beginTime = m_endTime = DateTime.Now; - } - - public DateTime BeginTime - { - get { return m_beginTime; } - set { m_beginTime = value; } - } - - public DateTime EndTime - { - get { return m_endTime; } - set { m_endTime = value; } - } - - public TimeSpan Duration - { - get { return m_endTime - m_beginTime; } } public long FilesRestored @@ -96,16 +76,13 @@ namespace Duplicati.Library.Main public override string ToString() { StringBuilder sb = new StringBuilder(); - sb.AppendLine("BeginTime : " + this.BeginTime.ToString(System.Globalization.CultureInfo.InvariantCulture)); - sb.AppendLine("EndTime : " + this.EndTime.ToString(System.Globalization.CultureInfo.InvariantCulture)); - sb.AppendLine("Duration : " + this.Duration.ToString()); + sb.Append(base.ToString()); sb.AppendLine("Files restored : " + this.FilesRestored.ToString(System.Globalization.CultureInfo.InvariantCulture)); sb.AppendLine("Files patched : " + this.FilesPatched.ToString(System.Globalization.CultureInfo.InvariantCulture)); sb.AppendLine("Files deleted : " + this.FilesDeleted.ToString(System.Globalization.CultureInfo.InvariantCulture)); sb.AppendLine("Restored size : " + this.SizeOfRestoredFiles.ToString(System.Globalization.CultureInfo.InvariantCulture)); sb.AppendLine("Folders created : " + this.FoldersRestored.ToString(System.Globalization.CultureInfo.InvariantCulture)); sb.AppendLine("Folders deleted : " + this.FoldersDeleted.ToString(System.Globalization.CultureInfo.InvariantCulture)); - sb.Append(base.ToString()); return sb.ToString(); } } diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index 7a0a5515f..c8c9713b8 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -973,5 +973,50 @@ namespace Duplicati.Library.Utility { return System.Reflection.Assembly.GetEntryAssembly() ?? System.Reflection.Assembly.GetExecutingAssembly(); } + + + /// + /// Converts a Base64 encoded string to "base64 for url" + /// See https://en.wikipedia.org/wiki/Base64#URL_applications + /// + /// The base64 encoded string + /// The base64 for url encoded string + public static string Base64PlainToBase64Url(string data) + { + return data.Replace('+', '-').Replace('/', '_'); + } + + /// + /// Converts a "base64 for url" encoded string to a Base64 encoded string. + /// See https://en.wikipedia.org/wiki/Base64#URL_applications + /// + /// The base64 for url encoded string + /// The base64 encoded string + public static string Base64UrlToBase64Plain(string data) + { + return data.Replace('-', '+').Replace('_', '/'); + } + + /// + /// Encodes a byte array into a "base64 for url" encoded string. + /// See https://en.wikipedia.org/wiki/Base64#URL_applications + /// + /// The data to encode + /// The base64 for url encoded string + public static string Base64UrlEncode(byte[] data) + { + return Base64PlainToBase64Url(Convert.ToBase64String(data)); + } + + /// + /// Decodes a "base64 for url" encoded string into the raw byte array. + /// See https://en.wikipedia.org/wiki/Base64#URL_applications + /// + /// The data to decode + /// The raw data + public static byte[] Base64UrlDecode(string data) + { + return Convert.FromBase64String(Base64UrlToBase64Plain(data)); + } } } \ No newline at end of file