haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added code to prevent overwriting files when restoring

Browse Source
This commit is contained in:
Kenneth Skovhede
2013-05-20 13:48:44 +02:00
parent 6cbfdec577
commit 3eebabef26
7 changed files with 201 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/LocalDatabase.cs
+54 -9
View File
@@ -142,6 +142,49 @@ namespace Duplicati.Library.Main.Database
}
}
public Tuple<string, object[]> GetFilelistWhereClause(DateTime time, long[] versions, IEnumerable<KeyValuePair<long, DateTime>> filesetslist = null )
{
var filesets = (filesetslist ?? this.FilesetTimes).ToArray();
string query = "";
var args = new List<object>();
if (time.Ticks > 0 || (versions != null && versions.Length > 0))
{
query = " WHERE ";
if (time.Ticks > 0)
{
if (time.Kind == DateTimeKind.Unspecified)
throw new Exception("Invalid DateTime given, must be either local or UTC");
query += @" ""Timestamp"" <= ?";
args.Add(time.ToUniversalTime());
}
if (versions != null && versions.Length > 0)
{
var qs ="";
foreach(var v in versions)
if (v >= 0 && v < filesets.Length)
{
args.Add(filesets[v].Key);
qs += "?,";
}
if (qs.Length > 0)
{
qs = qs.Substring(0, qs.Length - 1);
if (args.Count != 0)
query += " AND ";
query += @" ""ID"" IN (" + qs + ")";
}
}
}
return new Tuple<string, object[]>(query, args.ToArray());
}
public long GetRemoteVolumeID(string file, System.Data.IDbTransaction transaction = null)
{
m_selectremotevolumeIdCommand.Transaction = transaction;
@@ -273,16 +316,18 @@ namespace Duplicati.Library.Main.Database
}
}
public long GetFilesetID(DateTime restoretime)
{
if (restoretime.Kind == DateTimeKind.Unspecified)
throw new Exception("Invalid DateTime given, must be either local or UTC");
public long GetFilesetID(DateTime restoretime, long[] versions)
{
if (restoretime.Kind == DateTimeKind.Unspecified)
throw new Exception("Invalid DateTime given, must be either local or UTC");
using (var cmd = m_connection.CreateCommand())
{
cmd.CommandText = @"SELECT ""ID"" FROM ""Fileset"" WHERE (strftime(""%s"",?) - strftime(""%s"", ""Timestamp"")) >= 0 ORDER BY ""Timestamp"" DESC";
cmd.AddParameter(restoretime.ToUniversalTime());
object r = cmd.ExecuteScalar();
var tmp = GetFilelistWhereClause(restoretime, versions);
string query = tmp.Item1;
var args = tmp.Item2;
using(var cmd = m_connection.CreateCommand())
{
object r = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Fileset"" " + query + @" ORDER BY ""Timestamp"" DESC", args);
if (r == null)
{
cmd.Parameters.Clear();
Duplicati/Library/Main/Database/LocalListDatabase.cs
+4 -37
View File
@@ -53,45 +53,12 @@ namespace Duplicati.Library.Main.Database
m_connection = owner.m_connection;
m_filesets = owner.FilesetTimes.ToArray();
m_tablename = "Filesets-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
string query = "";
var args = new List<object>();
if (time.Ticks > 0 || (versions != null && versions.Length > 0))
{
query = " WHERE ";
if (time.Ticks > 0)
{
if (time.Kind == DateTimeKind.Unspecified)
throw new Exception("Invalid DateTime given, must be either local or UTC");
query += @" ""Timestamp"" <= ?";
args.Add(time.ToUniversalTime());
}
if (versions != null && versions.Length > 0)
{
var qs ="";
foreach(var v in versions)
if (v >= 0 && v < m_filesets.Length)
{
args.Add(m_filesets[v].Key);
qs += "?,";
}
if (qs.Length > 0)
{
qs = qs.Substring(0, qs.Length - 1);
if (args.Count != 0)
query += " AND ";
query += @" ""ID"" IN (" + qs + ")";
}
}
}
var tmp = owner.GetFilelistWhereClause(time, versions, m_filesets);
string query = tmp.Item1;
var args = tmp.Item2;
using(var cmd = m_connection.CreateCommand())
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" " + query, m_tablename), args.ToArray());
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""ID"" AS ""FilesetID"", ""Timestamp"" AS ""Timestamp"" FROM ""Fileset"" " + query, m_tablename), args);
}
private class Fileversion : IFileversion
Duplicati/Library/Main/Database/LocalRestoreDatabase.cs
+51 -26
View File
@@ -11,6 +11,9 @@ namespace Duplicati.Library.Main.Database
protected string m_tempfiletable;
protected string m_tempblocktable;
protected long m_blocksize;
protected DateTime m_restoreTime;
public DateTime RestoreTime { get { return m_restoreTime; } }
public LocalRestoreDatabase(string path, long blocksize)
: this(new LocalDatabase(path, "Restore"), blocksize)
@@ -24,7 +27,7 @@ namespace Duplicati.Library.Main.Database
m_blocksize = blocksize;
}
public void PrepareRestoreFilelist(DateTime restoretime, Library.Utility.IFilter filter, CommunicationStatistics stat)
public void PrepareRestoreFilelist(DateTime restoretime, long[] versions, Library.Utility.IFilter filter, CommunicationStatistics stat)
{
var guid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
@@ -33,7 +36,8 @@ namespace Duplicati.Library.Main.Database
using (var cmd = m_connection.CreateCommand())
{
long filesetId = GetFilesetID(restoretime);
long filesetId = GetFilesetID(restoretime, versions);
m_restoreTime = Convert.ToDateTime(cmd.ExecuteScalar(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = ?", filesetId));
cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""Targetpath"" TEXT NULL ) ", m_tempfiletable);
cmd.ExecuteNonQuery();
@@ -188,6 +192,13 @@ namespace Duplicati.Library.Main.Database
}
}
public void UpdateTargetPath(long ID, string newname)
{
using (var cmd = m_connection.CreateCommand())
cmd.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""TargetPath"" = ? WHERE ""FileID"" = ?", m_tempfiletable), newname, ID);
}
public interface IExistingFileBlock
{
string Hash { get; }
@@ -231,6 +242,7 @@ namespace Duplicati.Library.Main.Database
{
string Path { get; }
string Hash { get; }
long ID { get; }
}
public interface IPatchBlock
@@ -334,55 +346,68 @@ namespace Duplicati.Library.Main.Database
void SetBlockRestored(long targetfileid, long index, string hash, long blocksize);
void Commit();
System.Data.IDbTransaction Transaction { get; }
void SetFileRestored(long id);
}
private class BlockMarker : IBlockMarker
{
private System.Data.IDbCommand m_command;
private System.Data.IDbCommand m_insertblockCommand;
private System.Data.IDbCommand m_insertfileCommand;
private string m_updateTable;
private string m_blocktablename;
public System.Data.IDbTransaction Transaction { get { return m_command.Transaction; } }
public System.Data.IDbTransaction Transaction { get { return m_insertblockCommand.Transaction; } }
public BlockMarker(System.Data.IDbConnection connection, string blocktablename, string filetablename)
{
m_command = connection.CreateCommand();
m_command.Transaction = connection.BeginTransaction();
m_insertblockCommand = connection.CreateCommand();
m_insertblockCommand.Transaction = connection.BeginTransaction();
m_insertfileCommand = connection.CreateCommand();
m_insertfileCommand.Transaction = m_insertblockCommand.Transaction;
m_blocktablename = blocktablename;
m_updateTable = "UpdatedBlocks-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
m_command.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)", m_updateTable));
m_command.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"") VALUES (?, ?, ?, ?) ", m_updateTable);
m_command.AddParameters(4);
m_insertblockCommand.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""FileID"" INTEGER NOT NULL, ""Index"" INTEGER NOT NULL, ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL)", m_updateTable));
m_insertblockCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" (""FileID"", ""Index"", ""Hash"", ""Size"") VALUES (?, ?, ?, ?) ", m_updateTable);
m_insertblockCommand.AddParameters(4);
m_insertfileCommand.CommandText = string.Format(@"INSERT INTO ""{1}"" (""FileID"", ""Index"", ""Hash"", ""Size"") SELECT ""FileID"", ""Index"", ""Hash"", ""Size"" FROM ""{0}"" WHERE ""{0}"".""FileID"" = ? ", m_blocktablename, m_updateTable);
m_insertfileCommand.AddParameters(1);
}
public void SetBlockRestored(long targetfileid, long index, string hash, long size)
{
m_command.SetParameterValue(0, targetfileid);
m_command.SetParameterValue(1, index);
m_command.SetParameterValue(2, hash);
m_command.SetParameterValue(3, size);
var r = m_command.ExecuteNonQuery();
m_insertblockCommand.SetParameterValue(0, targetfileid);
m_insertblockCommand.SetParameterValue(1, index);
m_insertblockCommand.SetParameterValue(2, hash);
m_insertblockCommand.SetParameterValue(3, size);
var r = m_insertblockCommand.ExecuteNonQuery();
if (r != 1)
throw new Exception("Unexpected insert result");
}
public void SetFileRestored(long id)
{
m_insertfileCommand.SetParameterValue(0, id);
}
public void Commit()
{
m_command.Parameters.Clear();
var rc = m_command.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""ID"" IN (SELECT ""{0}"".""ID"" FROM ""{0}"", ""{1}"" WHERE ""{0}"".""FileID"" = ""{1}"".""FileID"" AND ""{0}"".""Index"" = ""{1}"".""Index"" AND ""{0}"".""Hash"" = ""{1}"".""Hash"" AND ""{0}"".""Size"" = ""{1}"".""Size"" )", m_blocktablename, m_updateTable));
var nc = Convert.ToInt64(m_command.ExecuteScalar(string.Format(@"SELECT COUNT(*) FROM ""{0}"" ", m_updateTable)));
m_insertblockCommand.Parameters.Clear();
var rc = m_insertblockCommand.ExecuteNonQuery(string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""ID"" IN (SELECT ""{0}"".""ID"" FROM ""{0}"", ""{1}"" WHERE ""{0}"".""FileID"" = ""{1}"".""FileID"" AND ""{0}"".""Index"" = ""{1}"".""Index"" AND ""{0}"".""Hash"" = ""{1}"".""Hash"" AND ""{0}"".""Size"" = ""{1}"".""Size"" )", m_blocktablename, m_updateTable));
var nc = Convert.ToInt64(m_insertblockCommand.ExecuteScalar(string.Format(@"SELECT COUNT(*) FROM ""{0}"" ", m_updateTable)));
m_command.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_updateTable));
m_insertblockCommand.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_updateTable));
m_updateTable = null;
if (rc != nc)
throw new Exception(string.Format("Inconsistency while marking blocks as updated. Updated blocks: {0}, Registered blocks: {1}", rc, nc));
var tr = m_command.Transaction;
m_command.Dispose();
m_command = null;
var tr = m_insertblockCommand.Transaction;
m_insertblockCommand.Dispose();
m_insertblockCommand = null;
tr.Commit();
tr.Dispose();
}
@@ -393,17 +418,17 @@ namespace Duplicati.Library.Main.Database
{
try
{
m_command.Parameters.Clear();
m_command.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_updateTable));
m_insertblockCommand.Parameters.Clear();
m_insertblockCommand.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", m_updateTable));
}
catch { }
finally { m_updateTable = null; }
}
if (m_command != null)
if (m_insertblockCommand != null)
{
var t = m_command;
m_command = null;
var t = m_insertblockCommand;
m_insertblockCommand = null;
t.Dispose();
}
}
Duplicati/Library/Main/Operation/RestoreHandler.cs
+61 -6
View File
@@ -228,11 +228,17 @@ namespace Duplicati.Library.Main.Operation
Utility.VerifyParameters(database, m_options);
var blockhasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.BlockHashAlgorithm);
var filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);
if (blockhasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.BlockHashAlgorithm));
if (!blockhasher.CanReuseTransform)
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.BlockHashAlgorithm));
if (filehasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FileHashAlgorithm));
if (!filehasher.CanReuseTransform)
throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, m_options.FileHashAlgorithm));
if (!m_options.NoBackendverification)
FilelistProcessor.VerifyRemoteList(backend, m_options, database, m_stat);
@@ -241,6 +247,9 @@ namespace Duplicati.Library.Main.Operation
//Make the entire output setup
CreateDirectoryStructure(database, m_options, m_stat);
if (!m_options.Overwrite)
UpdateTargetPathsToPreventOverwrite(database, filehasher, m_options, m_stat);
//If we are patching an existing target folder, do not touch stuff that is already updated
ScanForExistingTargetBlocks(database, m_blockbuffer, blockhasher, m_stat);
@@ -262,12 +271,8 @@ namespace Duplicati.Library.Main.Operation
using (var tmpfile = blockvolume.Value)
using (var blocks = new BlockVolumeReader(GetCompressionModule(blockvolume.Key.Name), tmpfile, m_options))
PatchWithBlocklist(database, blocks, m_options, m_stat, m_blockbuffer);
// After all blocks in the files are restored, verify the file hash
var filehasher = System.Security.Cryptography.HashAlgorithm.Create(m_options.FileHashAlgorithm);
if (filehasher == null)
throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FileHashAlgorithm));
// After all blocks in the files are restored, verify the file hash
foreach (var file in database.GetFilesToRestore())
{
try
@@ -296,6 +301,56 @@ namespace Duplicati.Library.Main.Operation
{
//TODO: Implement writing metadata
}
private static void UpdateTargetPathsToPreventOverwrite(LocalRestoreDatabase database, System.Security.Cryptography.HashAlgorithm filehasher, Options options, CommunicationStatistics stat)
{
using(var blockmarker = database.CreateBlockMarker())
{
foreach(var file in database.GetFilesToRestore())
{
bool rename = true;
try
{
if (System.IO.File.Exists(file.Path))
{
string key;
using(var fs = System.IO.File.OpenRead(file.Path))
key = Convert.ToBase64String(filehasher.ComputeHash(fs));
if (key == file.Hash)
{
blockmarker.SetFileRestored(file.ID);
rename = false;
}
}
else
{
rename = false;
}
}
catch (Exception ex)
{
stat.LogWarning(string.Format("Failed to read file: \"{0}\", message: {1}", file.Path, ex.Message), ex);
database.LogMessage("Warning", string.Format("Failed to read file: \"{0}\", message: {1}", file.Path, ex.Message), ex, null);
}
if (rename)
{
//Select a new filename
var ext = System.IO.Path.GetExtension(file.Path);
var newname = System.IO.Path.GetFileNameWithoutExtension(file.Path) + "." + Library.Utility.Utility.SerializeDateTime(database.RestoreTime);
var tr = newname + "." + ext;
var c = 0;
while (System.IO.File.Exists(tr) && c < 1000)
tr = newname + " (" + c.ToString() + ")" + "." + ext;
database.UpdateTargetPath(file.ID, newname);
}
}
blockmarker.Commit();
}
}
private static void ScanForExistingSourceBlocksFast(LocalRestoreDatabase database, Options options, byte[] blockbuffer, System.Security.Cryptography.HashAlgorithm hasher, CommunicationStatistics stat)
{
@@ -439,7 +494,7 @@ namespace Duplicati.Library.Main.Operation
{
// Create a temporary table FILES by selecting the files from fileset that matches a specific operation id
// Delete all entries from the temp table that are excluded by the filter(s)
database.PrepareRestoreFilelist(options.Time, filter, stat);
database.PrepareRestoreFilelist(options.Time, options.Version, filter, stat);
if (!string.IsNullOrEmpty(options.Restorepath))
{
Duplicati/Library/Main/Options.cs
+7
View File
@@ -386,6 +386,8 @@ namespace Duplicati.Library.Main
new CommandLineArgument("quiet-console", CommandLineArgument.ArgumentType.Boolean, Strings.Options.QuietconsoleShort, Strings.Options.QuietconsoleLong, "false"),
new CommandLineArgument("overwrite", CommandLineArgument.ArgumentType.Boolean, Strings.Options.OverwriteShort, Strings.Options.OverwriteLong, "false"),
new CommandLineArgument("dbpath", CommandLineArgument.ArgumentType.Path, Strings.Options.DbpathShort, Strings.Options.DbpathLong),
new CommandLineArgument("blocksize", CommandLineArgument.ArgumentType.Size, Strings.Options.BlocksizeShort, Strings.Options.BlocksizeLong, DEFAULT_FH_BLOCKSIZE),
new CommandLineArgument("no-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.NometadataShort, Strings.Options.NometadataLong, "false"),
@@ -1098,6 +1100,11 @@ namespace Duplicati.Library.Main
/// </summary>
public bool QuietConsole { get { return GetBool("quiet-console"); } }
/// <summary>
/// A value indicating restored files overwrite existing ones
/// </summary>
public bool Overwrite { get { return GetBool("overwrite"); } }
/// <summary>
/// Gets the total size in bytes that the backend supports, returns -1 if there is no upper limit
/// </summary>
Duplicati/Library/Main/Strings/Options.Designer.cs
Generated
+18
View File
@@ -960,6 +960,24 @@ namespace Duplicati.Library.Main.Strings {
}
}
/// <summary>
/// Looks up a localized string similar to Use this option to overwrite target files when restoring, if this option is not set the files will be restored with a timestamp and a number appended..
/// </summary>
internal static string OverwriteLong {
get {
return ResourceManager.GetString("OverwriteLong", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to Overwrite files when restoring.
/// </summary>
internal static string OverwriteShort {
get {
return ResourceManager.GetString("OverwriteShort", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to Supply a passphrase that Duplicati will use to encrypt the backup volumes, making them unreadable without the passphrase. This variable can also be supplied through the environment variable PASSPHRASE..
/// </summary>
Duplicati/Library/Main/Strings/Options.resx
+6
View File
@@ -555,4 +555,10 @@
<data name="DownloadedFileSizeError" xml:space="preserve">
<value>The file {0} was downloaded and had size {1} but the size was expected to be {2}</value>
</data>
<data name="OverwriteShort" xml:space="preserve">
<value>Overwrite files when restoring</value>
</data>
<data name="OverwriteLong" xml:space="preserve">
<value>Use this option to overwrite target files when restoring, if this option is not set the files will be restored with a timestamp and a number appended.</value>
</data>
</root>