diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 3203178e4..861eabfad 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -142,6 +142,49 @@ namespace Duplicati.Library.Main.Database } } + public Tuple GetFilelistWhereClause(DateTime time, long[] versions, IEnumerable> filesetslist = null ) + { + var filesets = (filesetslist ?? this.FilesetTimes).ToArray(); + string query = ""; + var args = new List(); + if (time.Ticks > 0 || (versions != null && versions.Length > 0)) + { + query = " WHERE "; + if (time.Ticks > 0) + { + if (time.Kind == DateTimeKind.Unspecified) + throw new Exception("Invalid DateTime given, must be either local or UTC"); + + query += @" ""Timestamp"" <= ?"; + args.Add(time.ToUniversalTime()); + } + + if (versions != null && versions.Length > 0) + { + var qs =""; + + foreach(var v in versions) + if (v >= 0 && v < filesets.Length) + { + args.Add(filesets[v].Key); + qs += "?,"; + } + + if (qs.Length > 0) + { + qs = qs.Substring(0, qs.Length - 1); + + if (args.Count != 0) + query += " AND "; + + query += @" ""ID"" IN (" + qs + ")"; + } + } + } + + return new Tuple(query, args.ToArray()); + } + public long GetRemoteVolumeID(string file, System.Data.IDbTransaction transaction = null) { m_selectremotevolumeIdCommand.Transaction = transaction; @@ -273,16 +316,18 @@ namespace Duplicati.Library.Main.Database } } - public long GetFilesetID(DateTime restoretime) - { - if (restoretime.Kind == DateTimeKind.Unspecified) - throw new Exception("Invalid DateTime given, must be either local or UTC"); + public long GetFilesetID(DateTime restoretime, long[] versions) + { + if (restoretime.Kind == DateTimeKind.Unspecified) + throw new Exception("Invalid DateTime given, must be either local or UTC"); - using (var cmd = m_connection.CreateCommand()) - { - cmd.CommandText = @"SELECT ""ID"" FROM ""Fileset"" WHERE (strftime(""%s"",?) - strftime(""%s"", ""Timestamp"")) >= 0 ORDER BY ""Timestamp"" DESC"; - cmd.AddParameter(restoretime.ToUniversalTime()); - object r = cmd.ExecuteScalar(); + var tmp = GetFilelistWhereClause(restoretime, versions); + string query = tmp.Item1; + var args = tmp.Item2; + + using(var cmd = m_connection.CreateCommand()) + { + object r = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Fileset"" " + query + @" ORDER BY ""Timestamp"" DESC", args); if (r == null) { cmd.Parameters.Clear(); diff --git a/Duplicati/Library/Main/Database/LocalListDatabase.cs b/Duplicati/Library/Main/Database/LocalListDatabase.cs index f6ccc6aca..2dc9d6269 100644 --- a/Duplicati/Library/Main/Database/LocalListDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListDatabase.cs @@ -53,45 +53,12 @@ namespace Duplicati.Library.Main.Database m_connection = owner.m_connection; m_filesets = owner.FilesetTimes.ToArray(); m_tablename = "Filesets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - string query = ""; - var args = new List(); - if (time.Ticks > 0 || (versions != null && versions.Length > 0)) - { - query = " WHERE "; - if (time.Ticks > 0) - { - if (time.Kind == DateTimeKind.Unspecified) - throw new Exception("Invalid DateTime given, must be either local or UTC"); - - query += @" ""Timestamp"" <= ?"; - args.Add(time.ToUniversalTime()); - } - - if (versions != null && versions.Length > 0) - { - var qs =""; - - foreach(var v in versions) - if (v >= 0 && v < m_filesets.Length) - { - args.Add(m_filesets[v].Key); - qs += "?,"; - } - - if (qs.Length > 0) - { - qs = qs.Substring(0, qs.Length - 1); - - if (args.Count != 0) - query += " AND "; - - query += @" ""ID"" IN (" + qs + ")"; - } - } - } + var tmp = owner.GetFilelistWhereClause(time, versions, m_filesets); + string query = tmp.Item1; + var args = tmp.Item2; using(var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" " + query, m_tablename), args.ToArray()); + cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" " + query, m_tablename), args); } private class Fileversion : IFileversion diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 30cdb627f..3980257e6 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -11,6 +11,9 @@ namespace Duplicati.Library.Main.Database protected string m_tempfiletable; protected string m_tempblocktable; protected long m_blocksize; + protected DateTime m_restoreTime; + + public DateTime RestoreTime { get { return m_restoreTime; } } public LocalRestoreDatabase(string path, long blocksize) : this(new LocalDatabase(path, "Restore"), blocksize) @@ -24,7 +27,7 @@ namespace Duplicati.Library.Main.Database m_blocksize = blocksize; } - public void PrepareRestoreFilelist(DateTime restoretime, Library.Utility.IFilter filter, CommunicationStatistics stat) + public void PrepareRestoreFilelist(DateTime restoretime, long[] versions, Library.Utility.IFilter filter, CommunicationStatistics stat) { var guid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -33,7 +36,8 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { - long filesetId = GetFilesetID(restoretime); + long filesetId = GetFilesetID(restoretime, versions); + m_restoreTime = Convert.ToDateTime(cmd.ExecuteScalar(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = ?", filesetId)); cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""Targetpath"" TEXT NULL ) ", m_tempfiletable); cmd.ExecuteNonQuery(); @@ -188,6 +192,13 @@ namespace Duplicati.Library.Main.Database } } + public void UpdateTargetPath(long ID, string newname) + { + using (var cmd = m_connection.CreateCommand()) + cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""TargetPath"" = ? WHERE ""FileID"" = ?", m_tempfiletable), newname, ID); + + } + public interface IExistingFileBlock { string Hash { get; } @@ -231,6 +242,7 @@ namespace Duplicati.Library.Main.Database { string Path { get; } string Hash { get; } + long ID { get; } } public interface IPatchBlock @@ -334,55 +346,68 @@ namespace Duplicati.Library.Main.Database void SetBlockRestored(long targetfileid, long index, string hash, long blocksize); void Commit(); System.Data.IDbTransaction Transaction { get; } + void SetFileRestored(long id); } private class BlockMarker : IBlockMarker { - private System.Data.IDbCommand m_command; + private System.Data.IDbCommand m_insertblockCommand; + private System.Data.IDbCommand m_insertfileCommand; private string m_updateTable; private string m_blocktablename; - public System.Data.IDbTransaction Transaction { get { return m_command.Transaction; } } + public System.Data.IDbTransaction Transaction { get { return m_insertblockCommand.Transaction; } } public BlockMarker(System.Data.IDbConnection connection, string blocktablename, string filetablename) { - m_command = connection.CreateCommand(); - m_command.Transaction = connection.BeginTransaction(); + m_insertblockCommand = connection.CreateCommand(); + m_insertblockCommand.Transaction = connection.BeginTransaction(); + + m_insertfileCommand = connection.CreateCommand(); + m_insertfileCommand.Transaction = m_insertblockCommand.Transaction; m_blocktablename = blocktablename; m_updateTable = "UpdatedBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - m_command.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)", m_updateTable)); - m_command.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"") VALUES (?, ?, ?, ?) ", m_updateTable); - m_command.AddParameters(4); + m_insertblockCommand.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)", m_updateTable)); + m_insertblockCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"") VALUES (?, ?, ?, ?) ", m_updateTable); + m_insertblockCommand.AddParameters(4); + + m_insertfileCommand.CommandText = string.Format(@"INSERT INTO ""{1}"" (""FileID"", ""Index"", ""Hash"", ""Size"") SELECT ""FileID"", ""Index"", ""Hash"", ""Size"" FROM ""{0}"" WHERE ""{0}"".""FileID"" = ? ", m_blocktablename, m_updateTable); + m_insertfileCommand.AddParameters(1); } public void SetBlockRestored(long targetfileid, long index, string hash, long size) { - m_command.SetParameterValue(0, targetfileid); - m_command.SetParameterValue(1, index); - m_command.SetParameterValue(2, hash); - m_command.SetParameterValue(3, size); - var r = m_command.ExecuteNonQuery(); + m_insertblockCommand.SetParameterValue(0, targetfileid); + m_insertblockCommand.SetParameterValue(1, index); + m_insertblockCommand.SetParameterValue(2, hash); + m_insertblockCommand.SetParameterValue(3, size); + var r = m_insertblockCommand.ExecuteNonQuery(); if (r != 1) throw new Exception("Unexpected insert result"); } + + public void SetFileRestored(long id) + { + m_insertfileCommand.SetParameterValue(0, id); + } public void Commit() { - m_command.Parameters.Clear(); - var rc = m_command.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""ID"" IN (SELECT ""{0}"".""ID"" FROM ""{0}"", ""{1}"" WHERE ""{0}"".""FileID"" = ""{1}"".""FileID"" AND ""{0}"".""Index"" = ""{1}"".""Index"" AND ""{0}"".""Hash"" = ""{1}"".""Hash"" AND ""{0}"".""Size"" = ""{1}"".""Size"" )", m_blocktablename, m_updateTable)); - var nc = Convert.ToInt64(m_command.ExecuteScalar(string.Format(@"SELECT COUNT(*) FROM ""{0}"" ", m_updateTable))); + m_insertblockCommand.Parameters.Clear(); + var rc = m_insertblockCommand.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""ID"" IN (SELECT ""{0}"".""ID"" FROM ""{0}"", ""{1}"" WHERE ""{0}"".""FileID"" = ""{1}"".""FileID"" AND ""{0}"".""Index"" = ""{1}"".""Index"" AND ""{0}"".""Hash"" = ""{1}"".""Hash"" AND ""{0}"".""Size"" = ""{1}"".""Size"" )", m_blocktablename, m_updateTable)); + var nc = Convert.ToInt64(m_insertblockCommand.ExecuteScalar(string.Format(@"SELECT COUNT(*) FROM ""{0}"" ", m_updateTable))); - m_command.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_updateTable)); + m_insertblockCommand.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_updateTable)); m_updateTable = null; if (rc != nc) throw new Exception(string.Format("Inconsistency while marking blocks as updated. Updated blocks: {0}, Registered blocks: {1}", rc, nc)); - var tr = m_command.Transaction; - m_command.Dispose(); - m_command = null; + var tr = m_insertblockCommand.Transaction; + m_insertblockCommand.Dispose(); + m_insertblockCommand = null; tr.Commit(); tr.Dispose(); } @@ -393,17 +418,17 @@ namespace Duplicati.Library.Main.Database { try { - m_command.Parameters.Clear(); - m_command.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_updateTable)); + m_insertblockCommand.Parameters.Clear(); + m_insertblockCommand.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_updateTable)); } catch { } finally { m_updateTable = null; } } - if (m_command != null) + if (m_insertblockCommand != null) { - var t = m_command; - m_command = null; + var t = m_insertblockCommand; + m_insertblockCommand = null; t.Dispose(); } } diff --git a/Duplicati/Library/Main/Operation/RestoreHandler.cs b/Duplicati/Library/Main/Operation/RestoreHandler.cs index 142332c08..249e8c822 100644 --- a/Duplicati/Library/Main/Operation/RestoreHandler.cs +++ b/Duplicati/Library/Main/Operation/RestoreHandler.cs @@ -228,11 +228,17 @@ namespace Duplicati.Library.Main.Operation Utility.VerifyParameters(database, m_options); var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm); + var filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm); if (blockhasher == null) throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm)); if (!blockhasher.CanReuseTransform) throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm)); + if (filehasher == null) + throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FileHashAlgorithm)); + if (!filehasher.CanReuseTransform) + throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.FileHashAlgorithm)); + if (!m_options.NoBackendverification) FilelistProcessor.VerifyRemoteList(backend, m_options, database, m_stat); @@ -241,6 +247,9 @@ namespace Duplicati.Library.Main.Operation //Make the entire output setup CreateDirectoryStructure(database, m_options, m_stat); + + if (!m_options.Overwrite) + UpdateTargetPathsToPreventOverwrite(database, filehasher, m_options, m_stat); //If we are patching an existing target folder, do not touch stuff that is already updated ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, m_stat); @@ -262,12 +271,8 @@ namespace Duplicati.Library.Main.Operation using (var tmpfile = blockvolume.Value) using (var blocks = new BlockVolumeReader(GetCompressionModule(blockvolume.Key.Name), tmpfile, m_options)) PatchWithBlocklist(database, blocks, m_options, m_stat, m_blockbuffer); - - // After all blocks in the files are restored, verify the file hash - var filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm); - if (filehasher == null) - throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FileHashAlgorithm)); + // After all blocks in the files are restored, verify the file hash foreach (var file in database.GetFilesToRestore()) { try @@ -296,6 +301,56 @@ namespace Duplicati.Library.Main.Operation { //TODO: Implement writing metadata } + + private static void UpdateTargetPathsToPreventOverwrite(LocalRestoreDatabase database, System.Security.Cryptography.HashAlgorithm filehasher, Options options, CommunicationStatistics stat) + { + using(var blockmarker = database.CreateBlockMarker()) + { + foreach(var file in database.GetFilesToRestore()) + { + bool rename = true; + try + { + if (System.IO.File.Exists(file.Path)) + { + string key; + using(var fs = System.IO.File.OpenRead(file.Path)) + key = Convert.ToBase64String(filehasher.ComputeHash(fs)); + + if (key == file.Hash) + { + blockmarker.SetFileRestored(file.ID); + rename = false; + } + } + else + { + rename = false; + } + } + catch (Exception ex) + { + stat.LogWarning(string.Format("Failed to read file: \"{0}\", message: {1}", file.Path, ex.Message), ex); + database.LogMessage("Warning", string.Format("Failed to read file: \"{0}\", message: {1}", file.Path, ex.Message), ex, null); + } + + if (rename) + { + //Select a new filename + var ext = System.IO.Path.GetExtension(file.Path); + var newname = System.IO.Path.GetFileNameWithoutExtension(file.Path) + "." + Library.Utility.Utility.SerializeDateTime(database.RestoreTime); + var tr = newname + "." + ext; + var c = 0; + while (System.IO.File.Exists(tr) && c < 1000) + tr = newname + " (" + c.ToString() + ")" + "." + ext; + + database.UpdateTargetPath(file.ID, newname); + } + } + + blockmarker.Commit(); + } + } private static void ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, CommunicationStatistics stat) { @@ -439,7 +494,7 @@ namespace Duplicati.Library.Main.Operation { // Create a temporary table FILES by selecting the files from fileset that matches a specific operation id // Delete all entries from the temp table that are excluded by the filter(s) - database.PrepareRestoreFilelist(options.Time, filter, stat); + database.PrepareRestoreFilelist(options.Time, options.Version, filter, stat); if (!string.IsNullOrEmpty(options.Restorepath)) { diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index b00f7fc2b..9aeb7ff1b 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -386,6 +386,8 @@ namespace Duplicati.Library.Main new CommandLineArgument("quiet-console", CommandLineArgument.ArgumentType.Boolean, Strings.Options.QuietconsoleShort, Strings.Options.QuietconsoleLong, "false"), + new CommandLineArgument("overwrite", CommandLineArgument.ArgumentType.Boolean, Strings.Options.OverwriteShort, Strings.Options.OverwriteLong, "false"), + new CommandLineArgument("dbpath", CommandLineArgument.ArgumentType.Path, Strings.Options.DbpathShort, Strings.Options.DbpathLong), new CommandLineArgument("blocksize", CommandLineArgument.ArgumentType.Size, Strings.Options.BlocksizeShort, Strings.Options.BlocksizeLong, DEFAULT_FH_BLOCKSIZE), new CommandLineArgument("no-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.NometadataShort, Strings.Options.NometadataLong, "false"), @@ -1098,6 +1100,11 @@ namespace Duplicati.Library.Main /// public bool QuietConsole { get { return GetBool("quiet-console"); } } + /// + /// A value indicating restored files overwrite existing ones + /// + public bool Overwrite { get { return GetBool("overwrite"); } } + /// /// Gets the total size in bytes that the backend supports, returns -1 if there is no upper limit /// diff --git a/Duplicati/Library/Main/Strings/Options.Designer.cs b/Duplicati/Library/Main/Strings/Options.Designer.cs index 7799c387f..f753b2e1d 100644 --- a/Duplicati/Library/Main/Strings/Options.Designer.cs +++ b/Duplicati/Library/Main/Strings/Options.Designer.cs @@ -960,6 +960,24 @@ namespace Duplicati.Library.Main.Strings { } } + /// + /// Looks up a localized string similar to Use this option to overwrite target files when restoring, if this option is not set the files will be restored with a timestamp and a number appended.. + /// + internal static string OverwriteLong { + get { + return ResourceManager.GetString("OverwriteLong", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Overwrite files when restoring. + /// + internal static string OverwriteShort { + get { + return ResourceManager.GetString("OverwriteShort", resourceCulture); + } + } + /// /// Looks up a localized string similar to Supply a passphrase that Duplicati will use to encrypt the backup volumes, making them unreadable without the passphrase. This variable can also be supplied through the environment variable PASSPHRASE.. /// diff --git a/Duplicati/Library/Main/Strings/Options.resx b/Duplicati/Library/Main/Strings/Options.resx index cbef5ec33..26b562680 100644 --- a/Duplicati/Library/Main/Strings/Options.resx +++ b/Duplicati/Library/Main/Strings/Options.resx @@ -555,4 +555,10 @@ The file {0} was downloaded and had size {1} but the size was expected to be {2} + + Overwrite files when restoring + + + Use this option to overwrite target files when restoring, if this option is not set the files will be restored with a timestamp and a number appended. + \ No newline at end of file