diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index c463e3c1d..8b601c85e 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -86,10 +86,12 @@ namespace Duplicati.Library.Main.Database private readonly System.Data.IDbCommand m_findfileCommand; private readonly System.Data.IDbCommand m_selectfilelastmodifiedCommand; + private readonly System.Data.IDbCommand m_selectfilelastmodifiedWithSizeCommand; private readonly System.Data.IDbCommand m_selectfileHashCommand; private readonly System.Data.IDbCommand m_selectblocklistHashesCommand; private readonly System.Data.IDbCommand m_insertfileOperationCommand; + private readonly System.Data.IDbCommand m_selectfilemetadatahashandsizeCommand; private PathLookupHelper m_pathLookup; private Dictionary m_blockCache; @@ -123,8 +125,10 @@ namespace Duplicati.Library.Main.Database m_insertfileOperationCommand = m_connection.CreateCommand(); m_findfileCommand = m_connection.CreateCommand(); m_selectfilelastmodifiedCommand = m_connection.CreateCommand(); + m_selectfilelastmodifiedWithSizeCommand = m_connection.CreateCommand(); m_selectfileHashCommand = m_connection.CreateCommand(); m_insertblocksetentryFastCommand = m_connection.CreateCommand(); + m_selectfilemetadatahashandsizeCommand = m_connection.CreateCommand(); m_findblockCommand.CommandText = @"SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?"; m_findblockCommand.AddParameters(2); @@ -165,6 +169,12 @@ namespace Duplicati.Library.Main.Database m_selectfilelastmodifiedCommand.CommandText = @"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?"; m_selectfilelastmodifiedCommand.AddParameters(2); + m_selectfilelastmodifiedWithSizeCommand.CommandText = @"SELECT ""C"".""ID"", ""C"".""LastModified"", ""D"".""Length"" FROM (SELECT ""A"".""ID"", ""B"".""LastModified"", ""A"".""BlocksetID"" FROM (SELECT ""ID"", ""BlocksetID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?) AS ""C"", ""Blockset"" AS ""D"" WHERE ""C"".""BlocksetID"" == ""D"".""ID"" "; + m_selectfilelastmodifiedWithSizeCommand.AddParameters(2); + + m_selectfilemetadatahashandsizeCommand.CommandText = @"SELECT ""Blockset"".""Length"", ""Blockset"".""FullHash"" FROM ""Blockset"", ""Metadataset"", ""File"" WHERE ""File"".""ID"" = ? AND ""Blockset"".""ID"" = ""Metadataset"".""BlocksetID"" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" "; + m_selectfilemetadatahashandsizeCommand.AddParameters(1); + //Need a temporary table with path/lastmodified lookups m_findfileCommand.CommandText = @" SELECT ""File"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " + @@ -418,7 +428,7 @@ namespace Duplicati.Library.Main.Database /// /// Gets the metadataset ID from the filehash /// - /// true, if metadataset should be recorded, false if it already exists. + /// true, if metadataset found, false if does not exist. /// The metadata hash. /// The size of the metadata. /// The ID of the metadataset. @@ -545,23 +555,53 @@ namespace Duplicati.Library.Main.Database AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction); } - public long GetFileLastModified(string path, long filesetid, out DateTime oldModified, System.Data.IDbTransaction transaction = null) + public long GetFileLastModified(string path, long filesetid, bool includeLength, out DateTime oldModified, out long length, System.Data.IDbTransaction transaction = null) { - m_selectfileHashCommand.Transaction = transaction; - m_selectfilelastmodifiedCommand.SetParameterValue(0, path); - m_selectfilelastmodifiedCommand.SetParameterValue(1, filesetid); - using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader(m_logQueries, null)) - if (rd.Read()) - { - oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); - return rd.ConvertValueToInt64(0); - } + if (includeLength) + { + m_selectfilelastmodifiedWithSizeCommand.Transaction = transaction; + m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(0, path); + m_selectfilelastmodifiedWithSizeCommand.SetParameterValue(1, filesetid); + using (var rd = m_selectfilelastmodifiedWithSizeCommand.ExecuteReader(m_logQueries, null)) + if (rd.Read()) + { + oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); + length = rd.ConvertValueToInt64(2); + return rd.ConvertValueToInt64(0); + } + } + else + { + m_selectfilelastmodifiedCommand.Transaction = transaction; + m_selectfilelastmodifiedCommand.SetParameterValue(0, path); + m_selectfilelastmodifiedCommand.SetParameterValue(1, filesetid); + using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader(m_logQueries, null)) + if (rd.Read()) + { + length = -1; + oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); + return rd.ConvertValueToInt64(0); + } + + } oldModified = new DateTime(0, DateTimeKind.Utc); + length = -1; return -1; } - public long GetFileEntry(string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize) + public Tuple GetMetadataHashAndSizeForFile(long fileid, System.Data.IDbTransaction transaction) + { + m_selectfilemetadatahashandsizeCommand.Transaction = transaction; + m_selectfilemetadatahashandsizeCommand.SetParameterValue(0, fileid); + using (var rd = m_selectfilemetadatahashandsizeCommand.ExecuteReader(m_logQueries, null)) + if (rd.Read()) + return new Tuple(rd.ConvertValueToInt64(0), rd.ConvertValueToString(1)); + + return null; + } + + public long GetFileEntry(string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize, System.Data.IDbTransaction transaction) { if (m_pathLookup != null) { @@ -586,7 +626,7 @@ namespace Duplicati.Library.Main.Database else { m_findfileCommand.SetParameterValue(0, path); - + m_findfileCommand.Transaction = transaction; using(var rd = m_findfileCommand.ExecuteReader(m_logQueries, null)) if (rd.Read()) { diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index c080e5eb0..6148998ec 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -94,15 +94,19 @@ namespace Duplicati.Library.Main.Operation.Backup { return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified, m_transaction)); } + + public Task> GetMetadataHashAndSizeForFileAsync(long fileid) + { + return RunOnMain(() => m_database.GetMetadataHashAndSizeForFile(fileid, m_transaction)); + } - public Task> GetFileLastModifiedAsync(string path, long lastfilesetid) + public Task> GetFileLastModifiedAsync(string path, long lastfilesetid, bool includeLength) { return RunOnMain(() => { - DateTime lastModified; - var id = m_database.GetFileLastModified(path, lastfilesetid, out lastModified, m_transaction); + var id = m_database.GetFileLastModified(path, lastfilesetid, includeLength, out var lastModified, out var length, m_transaction); - return new KeyValuePair(id, lastModified); + return new Tuple(id, lastModified, length); }); } @@ -114,7 +118,7 @@ namespace Duplicati.Library.Main.Operation.Backup string oldMetahash; long oldMetasize; - var id = m_database.GetFileEntry(path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize); + var id = m_database.GetFileEntry(path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize, m_transaction); return id < 0 ? null : diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index 18f34b2af..76908c4ae 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -68,7 +68,7 @@ namespace Duplicati.Library.Main.Operation.Backup if (!e.MetadataChanged) { var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length); - if (!res.Item1) + if (res.Item1) return res.Item2; Logging.Log.WriteWarningMessage(FILELOGTAG, "UnexpextedMetadataLookup", null, "Metadata was reported as not changed, but still requires being added?\nHash: {0}, Length: {1}, ID: {2}, Path: {3}", e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, res.Item2, e.Path); diff --git a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs index 2dbf971a0..e93dee92c 100644 --- a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs @@ -49,6 +49,19 @@ namespace Duplicati.Library.Main.Operation.Backup var EMPTY_METADATA = Utility.WrapMetadata(new Dictionary(), options); var blocksize = options.Blocksize; + + // Pre-cache the option variables here to simplify and + // speed up repeated option access below + + var SKIPFILESLARGERTHAN = options.SkipFilesLargerThan; + // Zero and max both indicate no size limit + if (SKIPFILESLARGERTHAN == long.MaxValue) + SKIPFILESLARGERTHAN = 0; + + var DISABLEFILETIMECHECK = options.DisableFiletimeCheck; + var CHECKFILETIMEONLY = options.CheckFiletimeOnly; + var SKIPMETADATA = options.SkipMetadata; + while (true) { var e = await self.Input.ReadAsync(); @@ -60,30 +73,61 @@ namespace Duplicati.Library.Main.Operation.Backup } catch(Exception ex) { - Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed tp read size of file: {0}", e.Path); + Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Path); } await stats.AddExaminedFile(filestatsize); - e.MetaHashAndSize = options.StoreMetadata ? Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(e.Path, e.Attributes, options, snapshot), options) : EMPTY_METADATA; - - var timestampChanged = e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0; - var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize; - var tooLargeFile = options.SkipFilesLargerThan != long.MaxValue && options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > options.SkipFilesLargerThan; - e.MetadataChanged = !options.CheckFiletimeOnly && !options.SkipMetadata && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash); - - if ((e.OldId < 0 || options.DisableFiletimeCheck || timestampChanged || filesizeChanged || e.MetadataChanged) && !tooLargeFile) + // Stop now if the file is too large + var tooLargeFile = SKIPFILESLARGERTHAN != 0 && filestatsize >= 0 && filestatsize > SKIPFILESLARGERTHAN; + if (tooLargeFile) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, e.OldId <= 0, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path); + continue; + } + + // Invalid ID indicates a new file + var isNewFile = e.OldId < 0; + + // If we disable the filetime check, we always assume that the file has changed + // Otherwise we check that the timestamps are different or if any of them are empty + var timestampChanged = DISABLEFILETIMECHECK || e.LastWrite != e.OldModified || e.LastWrite.Ticks == 0 || e.OldModified.Ticks == 0; + + // Avoid generating a new matadata blob if timestamp has not changed + // and we only check for timestamp changes + if (CHECKFILETIMEONLY && !timestampChanged && !isNewFile) + { + Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path); + await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); + continue; + } + + // If we have have disabled the filetime check, we do not have the metadata info + // but we want to know if the metadata is potentially changed + if (!isNewFile && DISABLEFILETIMECHECK) + { + var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId); + if (tp != null) + { + e.OldMetaSize = tp.Item1; + e.OldMetaHash = tp.Item2; + } + } + + // Compute current metadata + e.MetaHashAndSize = SKIPMETADATA ? EMPTY_METADATA : Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(e.Path, e.Attributes, options, snapshot), options); + e.MetadataChanged = !SKIPMETADATA && (e.MetaHashAndSize.Blob.Length != e.OldMetaSize || e.MetaHashAndSize.FileHash != e.OldMetaHash); + + // Check if the file is new, or something indicates a change + var filesizeChanged = filestatsize < 0 || e.LastFileSize < 0 || filestatsize != e.LastFileSize; + if (isNewFile || timestampChanged || filesizeChanged || e.MetadataChanged) + { + Logging.Log.WriteVerboseMessage(FILELOGTAG, "CheckFileForChanges", "Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", e.Path, isNewFile, timestampChanged, filesizeChanged, e.MetadataChanged, e.LastWrite, e.OldModified); await self.Output.WriteAsync(e); } else { - if (tooLargeFile) - Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckTooLarge", "Skipped checking file, because the size exceeds limit {0}", e.Path); - else - Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path); - + Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Path); await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); } } diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs index 4908aec9b..bd982751c 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs @@ -36,7 +36,7 @@ namespace Duplicati.Library.Main.Operation.Backup { Dictionary metadata; - if (options.StoreMetadata) + if (!options.SkipMetadata) { metadata = snapshot.GetMetadata(path, snapshot.IsSymlink(path, attributes), options.SymlinkPolicy == Options.SymlinkStrategy.Follow); if (metadata == null) diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index 3ac0712f2..335a1996c 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -67,6 +67,9 @@ namespace Duplicati.Library.Main.Operation.Backup { var emptymetadata = Utility.WrapMetadata(new Dictionary(), options); + var CHECKFILETIMEONLY = options.CheckFiletimeOnly; + var DISABLEFILETIMECHECK = options.DisableFiletimeCheck; + while (true) { var path = await self.Input.ReadAsync(); @@ -96,16 +99,16 @@ namespace Duplicati.Library.Main.Operation.Backup { try { - if (options.CheckFiletimeOnly || options.DisableFiletimeCheck) + if (CHECKFILETIMEONLY || DISABLEFILETIMECHECK) { - var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid); + var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid, false); await self.Output.WriteAsync(new FileEntry() { - OldId = tmp.Key < 0 ? -1 : tmp.Key, + OldId = tmp.Item1, Path = path, Attributes = attributes, LastWrite = lastwrite, - OldModified = tmp.Key < 0 ? new DateTime(0) : tmp.Value, - LastFileSize = -1 , + OldModified = tmp.Item2, + LastFileSize = tmp.Item3 , OldMetaHash = null, OldMetaSize = -1 }); @@ -182,7 +185,7 @@ namespace Duplicati.Library.Main.Operation.Backup { IMetahash metahash; - if (options.StoreMetadata) + if (!options.SkipMetadata) { metahash = Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(path, attributes, options, snapshot), options); } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 9e437a2c2..626722bd1 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -511,7 +511,6 @@ namespace Duplicati.Library.Main new CommandLineArgument("dbpath", CommandLineArgument.ArgumentType.Path, Strings.Options.DbpathShort, Strings.Options.DbpathLong), new CommandLineArgument("blocksize", CommandLineArgument.ArgumentType.Size, Strings.Options.BlocksizeShort, Strings.Options.BlocksizeLong, DEFAULT_BLOCKSIZE), new CommandLineArgument("file-read-buffer-size", CommandLineArgument.ArgumentType.Size, Strings.Options.FilereadbuffersizeShort, Strings.Options.FilereadbuffersizeLong, "0kb"), - new CommandLineArgument("store-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.StoremetadataShort, Strings.Options.StoremetadataLong, "true", null, null, Strings.Options.StoremetadataDeprecated), new CommandLineArgument("skip-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkipmetadataShort, Strings.Options.SkipmetadataLong, "false"), new CommandLineArgument("restore-permissions", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestorepermissionsShort, Strings.Options.RestorepermissionsLong, "false"), new CommandLineArgument("skip-restore-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkiprestoreverificationShort, Strings.Options.SkiprestoreverificationLong, "false"), @@ -1527,23 +1526,6 @@ namespace Duplicati.Library.Main return (int)t; } } - - /// - /// Gets a flag indicating if metadata for files and folders should be ignored - /// - public bool StoreMetadata - { - get - { - if (m_options.ContainsKey("skip-metadata")) - return !Library.Utility.Utility.ParseBoolOption(m_options, "skip-metadata"); - - if (m_options.ContainsKey("store-metadata")) - return Library.Utility.Utility.ParseBoolOption(m_options, "store-metadata"); - - return true; - } - } /// /// Gets a flag indicating if metadata for files and folders should be ignored diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 131f1e2aa..d0b4b1315 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -153,9 +153,6 @@ namespace Duplicati.Library.Main.Strings public static string DisablefilepathcacheShort { get { return LC.L(@"Reduce memory footprint by disabling in-memory lookups"); } } public static string UseblockcacheShort { get { return LC.L(@"This option can be used to increase speed in exchange for extra memory use."); } } public static string UseblockcacheLong { get { return LC.L(@"Store an in-memory block cache"); } } - public static string StoremetadataLong { get { return LC.L(@"Stores metadata, such as file timestamps and attributes. This increases the required storage space as well as the processing time."); } } - public static string StoremetadataShort { get { return LC.L(@"Enables storing file metadata"); } } - public static string StoremetadataDeprecated { get { return LC.L(@"This option is no longer used as metadata is now stored by default"); } } public static string MetadatahashlookupsizeLong { get { return LC.L(@"A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log."); } } public static string MetadatahashlookupsizeShort { get { return LC.L(@"Memory used by the metadata hash"); } } public static string NobackendverificationLong { get { return LC.L(@"If this flag is set, the local database is not compared to the remote filelist on startup. The intended usage for this option is to work correctly in cases where the filelisting is broken or unavailable."); } } diff --git a/Duplicati/UnitTest/BorderTests.cs b/Duplicati/UnitTest/BorderTests.cs index e3eacad35..913b5137a 100644 --- a/Duplicati/UnitTest/BorderTests.cs +++ b/Duplicati/UnitTest/BorderTests.cs @@ -169,6 +169,18 @@ namespace Duplicati.UnitTest opts["check-filetime-only"] = "true"; }); } + + [Test] + [Category("Border")] + public void RunFullScan() + { + PrepareSourceData(); + RunCommands(1024 * 10, modifyOptions: opts => + { + opts["disable-filetime-check"] = "true"; + }); + } + public static Dictionary WriteTestFilesToFolder(string targetfolder, int blocksize, int basedatasize = 0) { if (basedatasize <= 0) @@ -227,6 +239,11 @@ namespace Duplicati.UnitTest //Console.WriteLine(string.Join(Environment.NewLine, r.Files.Select(x => x.Path))); } + // Do a "touch" on files to trigger a re-scan, which should do nothing + //foreach (var k in filenames) + //if (File.Exists(Path.Combine(DATAFOLDER, "a" + k.Key))) + //File.SetLastWriteTime(Path.Combine(DATAFOLDER, "a" + k.Key), DateTime.Now.AddSeconds(5)); + var data = new byte[filenames.Select(x => x.Value).Max()]; new Random().NextBytes(data); foreach(var k in filenames)