diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index affc3d2dd..cfe6f99ce 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -14,14 +14,18 @@ namespace Duplicati.Library.Main.Database public DateTime Lastmodified; public long FileID; public long Filesize; + public string Metahash; + public long Metasize; private SortedList, long> m_versions; - public PathEntryKeeper(long fileId, DateTime lastmodified, long filesize) + public PathEntryKeeper(long fileId, DateTime lastmodified, long filesize, string metahash, long metasize) { this.FileID = fileId; this.Lastmodified = lastmodified; this.Filesize = filesize; + this.Metahash = metahash; + this.Metasize = metasize; this.m_versions = null; } @@ -151,7 +155,7 @@ namespace Duplicati.Library.Main.Database //Need a temporary table with path/lastmodified lookups m_lastmodifiedLookupTablename = "LastModified-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - m_selectfileSimpleCommand.CommandText = string.Format(@"SELECT ""FileID"", ""Lastmodified"", ""Length"" FROM ""{0}"" WHERE ""BlocksetID"" >= 0 AND ""Path"" = ?", m_lastmodifiedLookupTablename); + m_selectfileSimpleCommand.CommandText = string.Format(@"SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Metahash"", ""Metasize"" FROM ""{0}"" WHERE ""BlocksetID"" >= 0 AND ""Path"" = ?", m_lastmodifiedLookupTablename); m_selectfileSimpleCommand.AddParameters(1); m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""File"" WHERE ""Blockset"".""ID"" = ""File"".""BlocksetID"" AND ""File"".""ID"" = ? "; @@ -181,9 +185,9 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { //Need a temporary table with path/lastmodified lookups - var scantableDefinition = @"SELECT ""A"".""FileID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"" AS ""Lastmodified"", ""File"".""Path"" AS ""Path"", ""Blockset"".""Length"" AS ""Length"" FROM (SELECT ""FilesetEntry"".""FileID"" AS ""FileID"", MAX(""Fileset"".""Timestamp"") AS ""MostRecent"" FROM ""FilesetEntry"", ""Fileset"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" GROUP BY ""FilesetEntry"".""FileID"") A, ""File"", ""Blockset"", ""FilesetEntry"" WHERE ""File"".""ID"" = ""A"".""FileID"" AND ""Blockset"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"""; + var scantableDefinition = @"SELECT ""B"".""ID"" AS ""FileID"", ""D"".""Lastmodified"" AS ""Lastmodified"", ""B"".""Path"" AS ""Path"", ""C"".""Length"" AS ""Length"", ""F"".""Fullhash"" AS ""Metahash"", ""F"".""Length"" AS ""Metasize"" FROM (SELECT ""FilesetEntry"".""FileID"" AS ""FileID"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"", MAX(""Fileset"".""Timestamp"") AS ""MostRecent"" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" GROUP BY ""File"".""Path"") A, ""File"" B, ""Blockset"" C, ""FilesetEntry"" D, ""Metadataset"" E, ""Blockset"" F WHERE ""B"".""ID"" = ""A"".""FileID"" AND ""C"".""ID"" = ""B"".""BlocksetID"" AND ""D"".""FileID"" = ""B"".""ID"" AND ""D"".""FilesetID"" = ""A"".""FilesetID"" AND ""B"".""MetadataID"" = ""E"".""ID"" AND ""F"".""ID"" = ""E"".""BlocksetID"""; cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + scantableDefinition, m_lastmodifiedLookupTablename)); - cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}Index"" ON ""{0}"" (""Path"", ""Lastmodified"", ""Length"", ""FileID"") ", m_lastmodifiedLookupTablename)); + cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}Index"" ON ""{0}"" (""Path"", ""Lastmodified"", ""Length"", ""Metahash"", ""Metasize"", ""FileID"") ", m_lastmodifiedLookupTablename)); if (m_blockHashLookup != null) try @@ -242,14 +246,16 @@ namespace Duplicati.Library.Main.Database if (m_pathLookup != null) using(new Logging.Timer("Build path lastmodified lookup table")) - using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"" FROM ""{0}"" WHERE ""BlocksetID"" >= 0 ", m_lastmodifiedLookupTablename))) + using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ""{0}"" WHERE ""BlocksetID"" >= 0 ", m_lastmodifiedLookupTablename))) while (rd.Read()) { var id = rd.GetInt64(0); var lastmodified = new DateTime(rd.GetInt64(1), DateTimeKind.Utc); var filesize = rd.GetInt64(2); - var path = rd.GetValue(3).ToString(); - m_pathLookup.Insert(path, new PathEntryKeeper(id, lastmodified, filesize)); + var path = rd.GetString(3); + var metahash = rd.GetString(4); + var metasize = rd.GetInt64(5); + m_pathLookup.Insert(path, new PathEntryKeeper(id, lastmodified, filesize, metahash, metasize)); } if (m_pathLookup != null) @@ -267,7 +273,7 @@ namespace Duplicati.Library.Main.Database PathEntryKeeper r; if (!m_pathLookup.TryFind(path, out r)) { - r = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1); + r = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1); r.AddFilesetID(blocksetid, metadataid, filesetid); m_pathLookup.Insert(path, r); } @@ -523,7 +529,7 @@ namespace Duplicati.Library.Main.Database { if (!entryFound) { - entry = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1); + entry = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1); entry.AddFilesetID(blocksetID, metadataID, fileidobj); m_pathLookup.Insert(filename, entry); } @@ -560,7 +566,7 @@ namespace Duplicati.Library.Main.Database AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction); } - public long GetFileEntry(string path, out DateTime oldModified, out long lastFileSize) + public long GetFileEntry(string path, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize) { if (m_pathLookup != null) { @@ -569,12 +575,16 @@ namespace Duplicati.Library.Main.Database { oldModified = tmp.Lastmodified; lastFileSize = tmp.Filesize; + oldMetahash = tmp.Metahash; + oldMetasize = tmp.Metasize; return tmp.FileID; } else { oldModified = new DateTime(0, DateTimeKind.Utc); lastFileSize = -1; + oldMetahash = null; + oldMetasize = -1; return -1; } } @@ -586,12 +596,16 @@ namespace Duplicati.Library.Main.Database { oldModified = new DateTime(rd.GetInt64(1), DateTimeKind.Utc); lastFileSize = rd.GetInt64(2); + oldMetahash = rd.GetString(3); + oldMetasize = rd.GetInt64(4); return rd.GetInt64(0); } else { oldModified = new DateTime(0, DateTimeKind.Utc); lastFileSize = -1; + oldMetahash = null; + oldMetasize = -1; return -1; } } diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index a92447b64..78527f3a4 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -883,32 +883,28 @@ namespace Duplicati.Library.Main.Operation // Last scan time DateTime oldModified; long lastFileSize = -1; - var oldId = m_database.GetFileEntry(path, out oldModified, out lastFileSize); + string oldMetahash; + long oldMetasize; + var oldId = m_database.GetFileEntry(path, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize); long filestatsize = -1; try { filestatsize = m_snapshot.GetFileSize(path); } catch { } + IMetahash metahashandsize = m_options.StoreMetadata ? Utility.WrapMetadata(GenerateMetadata(path, attributes), m_options) : EMPTY_METADATA; + var timestampChanged = lastwrite != oldModified || lastwrite.Ticks == 0 || oldModified.Ticks == 0; var filesizeChanged = filestatsize < 0 || lastFileSize < 0 || filestatsize != lastFileSize; var tooLargeFile = m_options.SkipFilesLargerThan != long.MaxValue && m_options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > m_options.SkipFilesLargerThan; + var metadatachanged = !m_options.SkipMetadata && (metahashandsize.Size != oldMetasize || metahashandsize.Hash != oldMetahash); - if ((oldId < 0 || m_options.DisableFiletimeCheck || timestampChanged || filesizeChanged) && !tooLargeFile) + if ((oldId < 0 || m_options.DisableFiletimeCheck || timestampChanged || filesizeChanged || metadatachanged) && !tooLargeFile) { - m_result.AddVerboseMessage("Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, {4} vs {5}", path, oldId <= 0, timestampChanged, filesizeChanged, lastwrite, oldModified); + m_result.AddVerboseMessage("Checking file for changes {0}, new: {1}, timestamp changed: {2}, size changed: {3}, metadatachanged: {4}, {5} vs {6}", path, oldId <= 0, timestampChanged, filesizeChanged, metadatachanged, lastwrite, oldModified); m_result.OpenedFiles++; long filesize = 0; - IMetahash metahashandsize; - if (m_options.StoreMetadata) - { - metahashandsize = Utility.WrapMetadata(GenerateMetadata(path, attributes), m_options); - } - else - { - metahashandsize = EMPTY_METADATA; - } var hint = m_options.GetCompressionHintFromFilename(path); var oldHash = oldId < 0 ? null : m_database.GetFileHash(oldId); @@ -1005,12 +1001,19 @@ namespace Duplicati.Library.Main.Operation AddFileToOutput(path, filesize, lastwrite, metahashandsize, hashcollector, filekey, blocklisthashes); changed = true; } + else if (metadatachanged) + { + m_result.AddVerboseMessage("File has only metadata changes {0}", path); + AddFileToOutput(path, filesize, lastwrite, metahashandsize, hashcollector, filekey, blocklisthashes); + changed = true; + } else { // When we write the file to output, update the last modified time oldModified = lastwrite; m_result.AddVerboseMessage("File has not changed {0}", path); } + } } else