haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/fix-check-filetime-only'

Browse Source
This commit is contained in:
Kenneth Skovhede
2018-08-12 22:38:22 +02:00
parent 106d3ce532 5507496352
commit 7b840bacb5
9 changed files with 149 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Database/LocalBackupDatabase.cs
+53 -13
View File
@@ -86,10 +86,12 @@ namespace Duplicati.Library.Main.Database
private readonly System.Data.IDbCommand m_findfileCommand;
private readonly System.Data.IDbCommand m_selectfilelastmodifiedCommand;
private readonly System.Data.IDbCommand m_selectfilelastmodifiedWithSizeCommand;
private readonly System.Data.IDbCommand m_selectfileHashCommand;
private readonly System.Data.IDbCommand m_selectblocklistHashesCommand;
private readonly System.Data.IDbCommand m_insertfileOperationCommand;
private readonly System.Data.IDbCommand m_selectfilemetadatahashandsizeCommand;
private PathLookupHelper<PathEntryKeeper> m_pathLookup;
private Dictionary<string, long> m_blockCache;
@@ -123,8 +125,10 @@ namespace Duplicati.Library.Main.Database
m_insertfileOperationCommand = m_connection.CreateCommand();
m_findfileCommand = m_connection.CreateCommand();
m_selectfilelastmodifiedCommand = m_connection.CreateCommand();
m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand();
m_selectfileHashCommand = m_connection.CreateCommand();
m_insertblocksetentryFastCommand = m_connection.CreateCommand();
m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand();
m_findblockCommand.CommandText = @"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?";
m_findblockCommand.AddParameters(2);
@@ -165,6 +169,12 @@ namespace Duplicati.Library.Main.Database
m_selectfilelastmodifiedCommand.CommandText = @"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?";
m_selectfilelastmodifiedCommand.AddParameters(2);
m_selectfilelastmodifiedWithSizeCommand.CommandText = @"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" ";
m_selectfilelastmodifiedWithSizeCommand.AddParameters(2);
m_selectfilemetadatahashandsizeCommand.CommandText = @"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = ? AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" ";
m_selectfilemetadatahashandsizeCommand.AddParameters(1);
//Need a temporary table with path/lastmodified lookups
m_findfileCommand.CommandText =
@" SELECT ""File"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " +
@@ -418,7 +428,7 @@ namespace Duplicati.Library.Main.Database
/// <summary>
/// Gets the metadataset ID from the filehash
/// </summary>
/// <returns><c>true</c>, if metadataset should be recorded, false if it already exists.</returns>
/// <returns><c>true</c>, if metadataset found, false if does not exist.</returns>
/// <param name="filehash">The metadata hash.</param>
/// <param name="size">The size of the metadata.</param>
/// <param name="metadataid">The ID of the metadataset.</param>
@@ -545,23 +555,53 @@ namespace Duplicati.Library.Main.Database
AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction);
}
public long GetFileLastModified(string path, long filesetid, out DateTime oldModified, System.Data.IDbTransaction transaction = null)
public long GetFileLastModified(string path, long filesetid, bool includeLength, out DateTime oldModified, out long length, System.Data.IDbTransaction transaction = null)
{
m_selectfileHashCommand.Transaction = transaction;
m_selectfilelastmodifiedCommand.SetParameterValue(0, path);
m_selectfilelastmodifiedCommand.SetParameterValue(1, filesetid);
using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader(m_logQueries, null))
if (rd.Read())
{
oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
return rd.ConvertValueToInt64(0);
}
if (includeLength)
{
m_selectfilelastmodifiedWithSizeCommand.Transaction = transaction;
m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(0, path);
m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(1, filesetid);
using (var rd = m_selectfilelastmodifiedWithSizeCommand.ExecuteReader(m_logQueries, null))
if (rd.Read())
{
oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
length = rd.ConvertValueToInt64(2);
return rd.ConvertValueToInt64(0);
}
}
else
{
m_selectfilelastmodifiedCommand.Transaction = transaction;
m_selectfilelastmodifiedCommand.SetParameterValue(0, path);
m_selectfilelastmodifiedCommand.SetParameterValue(1, filesetid);
using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader(m_logQueries, null))
if (rd.Read())
{
length = -1;
oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc);
return rd.ConvertValueToInt64(0);
}
}
oldModified = new DateTime(0, DateTimeKind.Utc);
length = -1;
return -1;
}
public long GetFileEntry(string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize)
public Tuple<long, string> GetMetadataHashAndSizeForFile(long fileid, System.Data.IDbTransaction transaction)
{
m_selectfilemetadatahashandsizeCommand.Transaction = transaction;
m_selectfilemetadatahashandsizeCommand.SetParameterValue(0, fileid);
using (var rd = m_selectfilemetadatahashandsizeCommand.ExecuteReader(m_logQueries, null))
if (rd.Read())
return new Tuple<long, string>(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1));
return null;
}
public long GetFileEntry(string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize, System.Data.IDbTransaction transaction)
{
if (m_pathLookup != null)
{
@@ -586,7 +626,7 @@ namespace Duplicati.Library.Main.Database
else
{
m_findfileCommand.SetParameterValue(0, path);
m_findfileCommand.Transaction = transaction;
using(var rd = m_findfileCommand.ExecuteReader(m_logQueries, null))
if (rd.Read())
{
Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs
+9 -5
View File
@@ -94,15 +94,19 @@ namespace Duplicati.Library.Main.Operation.Backup
{
return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified, m_transaction));
}
public Task<Tuple<long, string>> GetMetadataHashAndSizeForFileAsync(long fileid)
{
return RunOnMain(() => m_database.GetMetadataHashAndSizeForFile(fileid, m_transaction));
}
public Task<KeyValuePair<long, DateTime>> GetFileLastModifiedAsync(string path, long lastfilesetid)
public Task<Tuple<long, DateTime, long>> GetFileLastModifiedAsync(string path, long lastfilesetid, bool includeLength)
{
return RunOnMain(() =>
{
DateTime lastModified;
var id = m_database.GetFileLastModified(path, lastfilesetid, out lastModified, m_transaction);
var id = m_database.GetFileLastModified(path, lastfilesetid, includeLength, out var lastModified, out var length, m_transaction);
return new KeyValuePair<long, DateTime>(id, lastModified);
return new Tuple<long, DateTime, long>(id, lastModified, length);
});
}
@@ -114,7 +118,7 @@ namespace Duplicati.Library.Main.Operation.Backup
string oldMetahash;
long oldMetasize;
var id = m_database.GetFileEntry(path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize);
var id = m_database.GetFileEntry(path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize, m_transaction);
return
id < 0 ?
null :
Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs
+1 -1
View File
@@ -68,7 +68,7 @@ namespace Duplicati.Library.Main.Operation.Backup
if (!e.MetadataChanged)
{
var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length);
if (!res.Item1)
if (res.Item1)
return res.Item2;
Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpextedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}, Path: {3}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2, e.Path);
Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs
+59 -15
View File
@@ -49,6 +49,19 @@ namespace Duplicati.Library.Main.Operation.Backup
var EMPTY_METADATA = Utility.WrapMetadata(new Dictionary<string, string>(), options);
var blocksize = options.Blocksize;
// Pre-cache the option variables here to simplify and
// speed up repeated option access below
var SKIPFILESLARGERTHAN = options.SkipFilesLargerThan;
// Zero and max both indicate no size limit
if (SKIPFILESLARGERTHAN == long.MaxValue)
SKIPFILESLARGERTHAN = 0;
var DISABLEFILETIMECHECK = options.DisableFiletimeCheck;
var CHECKFILETIMEONLY = options.CheckFiletimeOnly;
var SKIPMETADATA = options.SkipMetadata;
while (true)
{
var e = await self.Input.ReadAsync();
@@ -60,30 +73,61 @@ namespace Duplicati.Library.Main.Operation.Backup
}
catch(Exception ex)
{
Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed tp read size of file: {0}", e.Path);
Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Path);
}
await stats.AddExaminedFile(filestatsize);
e.MetaHashAndSize = options.StoreMetadata ? Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(e.Path, e.Attributes, options, snapshot), options) : EMPTY_METADATA;
var timestampChanged = e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0;
var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize;
var tooLargeFile = options.SkipFilesLargerThan != long.MaxValue && options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > options.SkipFilesLargerThan;
e.MetadataChanged = !options.CheckFiletimeOnly && !options.SkipMetadata && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash);
if ((e.OldId < 0 || options.DisableFiletimeCheck || timestampChanged || filesizeChanged || e.MetadataChanged) && !tooLargeFile)
// Stop now if the file is too large
var tooLargeFile = SKIPFILESLARGERTHAN != 0 && filestatsize >= 0 && filestatsize > SKIPFILESLARGERTHAN;
if (tooLargeFile)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, e.OldId <= 0, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path);
continue;
}
// Invalid ID indicates a new file
var isNewFile = e.OldId < 0;
// If we disable the filetime check, we always assume that the file has changed
// Otherwise we check that the timestamps are different or if any of them are empty
var timestampChanged = DISABLEFILETIMECHECK || e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0;
// Avoid generating a new matadata blob if timestamp has not changed
// and we only check for timestamp changes
if (CHECKFILETIMEONLY && !timestampChanged && !isNewFile)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
continue;
}
// If we have have disabled the filetime check, we do not have the metadata info
// but we want to know if the metadata is potentially changed
if (!isNewFile && DISABLEFILETIMECHECK)
{
var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId);
if (tp != null)
{
e.OldMetaSize = tp.Item1;
e.OldMetaHash = tp.Item2;
}
}
// Compute current metadata
e.MetaHashAndSize = SKIPMETADATA ? EMPTY_METADATA : Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(e.Path, e.Attributes, options, snapshot), options);
e.MetadataChanged = !SKIPMETADATA && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash);
// Check if the file is new, or something indicates a change
var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize;
if (isNewFile || timestampChanged || filesizeChanged || e.MetadataChanged)
{
Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, isNewFile, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified);
await self.Output.WriteAsync(e);
}
else
{
if (tooLargeFile)
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path);
else
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Path);
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
}
}
Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs
+1 -1
View File
@@ -36,7 +36,7 @@ namespace Duplicati.Library.Main.Operation.Backup
{
Dictionary<string, string> metadata;
if (options.StoreMetadata)
if (!options.SkipMetadata)
{
metadata = snapshot.GetMetadata(path, snapshot.IsSymlink(path, attributes), options.SymlinkPolicy == Options.SymlinkStrategy.Follow);
if (metadata == null)
Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs
+9 -6
View File
@@ -67,6 +67,9 @@ namespace Duplicati.Library.Main.Operation.Backup
{
var emptymetadata = Utility.WrapMetadata(new Dictionary<string, string>(), options);
var CHECKFILETIMEONLY = options.CheckFiletimeOnly;
var DISABLEFILETIMECHECK = options.DisableFiletimeCheck;
while (true)
{
var path = await self.Input.ReadAsync();
@@ -96,16 +99,16 @@ namespace Duplicati.Library.Main.Operation.Backup
{
try
{
if (options.CheckFiletimeOnly || options.DisableFiletimeCheck)
if (CHECKFILETIMEONLY || DISABLEFILETIMECHECK)
{
var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid);
var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid, false);
await self.Output.WriteAsync(new FileEntry() {
OldId = tmp.Key < 0 ? -1 : tmp.Key,
OldId = tmp.Item1,
Path = path,
Attributes = attributes,
LastWrite = lastwrite,
OldModified = tmp.Key < 0 ? new DateTime(0) : tmp.Value,
LastFileSize = -1 ,
OldModified = tmp.Item2,
LastFileSize = tmp.Item3 ,
OldMetaHash = null,
OldMetaSize = -1
});
@@ -182,7 +185,7 @@ namespace Duplicati.Library.Main.Operation.Backup
{
IMetahash metahash;
if (options.StoreMetadata)
if (!options.SkipMetadata)
{
metahash = Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(path, attributes, options, snapshot), options);
}
Duplicati/Library/Main/Options.cs
-18
View File
@@ -511,7 +511,6 @@ namespace Duplicati.Library.Main
new CommandLineArgument("dbpath", CommandLineArgument.ArgumentType.Path, Strings.Options.DbpathShort, Strings.Options.DbpathLong),
new CommandLineArgument("blocksize", CommandLineArgument.ArgumentType.Size, Strings.Options.BlocksizeShort, Strings.Options.BlocksizeLong, DEFAULT_BLOCKSIZE),
new CommandLineArgument("file-read-buffer-size", CommandLineArgument.ArgumentType.Size, Strings.Options.FilereadbuffersizeShort, Strings.Options.FilereadbuffersizeLong, "0kb"),
new CommandLineArgument("store-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.StoremetadataShort, Strings.Options.StoremetadataLong, "true", null, null, Strings.Options.StoremetadataDeprecated),
new CommandLineArgument("skip-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkipmetadataShort, Strings.Options.SkipmetadataLong, "false"),
new CommandLineArgument("restore-permissions", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestorepermissionsShort, Strings.Options.RestorepermissionsLong, "false"),
new CommandLineArgument("skip-restore-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkiprestoreverificationShort, Strings.Options.SkiprestoreverificationLong, "false"),
@@ -1527,23 +1526,6 @@ namespace Duplicati.Library.Main
return (int)t;
}
}
/// <summary>
/// Gets a flag indicating if metadata for files and folders should be ignored
/// </summary>
public bool StoreMetadata
{
get
{
if (m_options.ContainsKey("skip-metadata"))
return !Library.Utility.Utility.ParseBoolOption(m_options, "skip-metadata");
if (m_options.ContainsKey("store-metadata"))
return Library.Utility.Utility.ParseBoolOption(m_options, "store-metadata");
return true;
}
}
/// <summary>
/// Gets a flag indicating if metadata for files and folders should be ignored
Duplicati/Library/Main/Strings.cs
-3
View File
@@ -153,9 +153,6 @@ namespace Duplicati.Library.Main.Strings
public static string DisablefilepathcacheShort { get { return LC.L(@"Reduce memory footprint by disabling in-memory lookups"); } }
public static string UseblockcacheShort { get { return LC.L(@"This option can be used to increase speed in exchange for extra memory use."); } }
public static string UseblockcacheLong { get { return LC.L(@"Store an in-memory block cache"); } }
public static string StoremetadataLong { get { return LC.L(@"Stores metadata, such as file timestamps and attributes. This increases the required storage space as well as the processing time."); } }
public static string StoremetadataShort { get { return LC.L(@"Enables storing file metadata"); } }
public static string StoremetadataDeprecated { get { return LC.L(@"This option is no longer used as metadata is now stored by default"); } }
public static string MetadatahashlookupsizeLong { get { return LC.L(@"A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log."); } }
public static string MetadatahashlookupsizeShort { get { return LC.L(@"Memory used by the metadata hash"); } }
public static string NobackendverificationLong { get { return LC.L(@"If this flag is set, the local database is not compared to the remote filelist on startup. The intended usage for this option is to work correctly in cases where the filelisting is broken or unavailable."); } }
Duplicati/UnitTest/BorderTests.cs
+17
View File
@@ -169,6 +169,18 @@ namespace Duplicati.UnitTest
opts["check-filetime-only"] = "true";
});
}
[Test]
[Category("Border")]
public void RunFullScan()
{
PrepareSourceData();
RunCommands(1024 * 10, modifyOptions: opts =>
{
opts["disable-filetime-check"] = "true";
});
}
public static Dictionary<string, int> WriteTestFilesToFolder(string targetfolder, int blocksize, int basedatasize = 0)
{
if (basedatasize <= 0)
@@ -227,6 +239,11 @@ namespace Duplicati.UnitTest
//Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path)));
}
// Do a "touch" on files to trigger a re-scan, which should do nothing
//foreach (var k in filenames)
//if (File.Exists(Path.Combine(DATAFOLDER, "a" + k.Key)))
//File.SetLastWriteTime(Path.Combine(DATAFOLDER, "a" + k.Key), DateTime.Now.AddSeconds(5));
var data = new byte[filenames.Select(x => x.Value).Max()];
new Random().NextBytes(data);
foreach(var k in filenames)