From 6830eb257df889bd360252906a73faac71253756 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 9 Jul 2013 13:51:08 +0200 Subject: [PATCH] Cleaned up storage of various timestamps to only use second precision --- .../Library/Main/Database/LocalBackupDatabase.cs | 16 ++++++++-------- Duplicati/Library/Main/Database/LocalDatabase.cs | 14 +++++++++++++- .../Library/Main/Database/LocalDeleteDatabase.cs | 2 +- .../Main/Database/LocalRestoreDatabase.cs | 2 +- .../Library/Main/Operation/BackupHandler.cs | 5 +++-- 5 files changed, 26 insertions(+), 13 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 937213533..4988e1088 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -198,7 +198,7 @@ namespace Duplicati.Library.Main.Database while (rd.Read()) { var id = Convert.ToInt64(rd.GetValue(0)); - var scantime = Convert.ToDateTime(rd.GetValue(1)); + var scantime = new DateTime(Convert.ToDateTime(rd.GetValue(1)).Ticks, DateTimeKind.Utc); var path = rd.GetValue(2).ToString(); m_fileScantimeLookup.Add((ulong)path.GetHashCode(), path, new KeyValuePair(id, scantime)); } @@ -557,11 +557,11 @@ namespace Duplicati.Library.Main.Database m_filesetLookup.Add(hashdata, new Tuple(filename, blocksetID, metadataID), Convert.ToInt64(fileidobj)); } } - + m_insertfileOperationCommand.Transaction = transaction; m_insertfileOperationCommand.SetParameterValue(0, m_filesetId); m_insertfileOperationCommand.SetParameterValue(1, fileidobj); - m_insertfileOperationCommand.SetParameterValue(2, scantime); + m_insertfileOperationCommand.SetParameterValue(2, NormalizeDateTime(scantime)); m_insertfileOperationCommand.ExecuteNonQuery(); } @@ -571,7 +571,7 @@ namespace Duplicati.Library.Main.Database m_insertfileOperationCommand.Transaction = transaction; m_insertfileOperationCommand.SetParameterValue(0, m_filesetId); m_insertfileOperationCommand.SetParameterValue(1, fileid); - m_insertfileOperationCommand.SetParameterValue(2, scantime); + m_insertfileOperationCommand.SetParameterValue(2, NormalizeDateTime(scantime)); m_insertfileOperationCommand.ExecuteNonQuery(); } @@ -608,8 +608,8 @@ namespace Duplicati.Library.Main.Database { if (m_fileScantimeLookup != null) m_fileScantimeLookup.FalseNegatives++; - - oldScanned = Convert.ToDateTime(rd.GetValue(1)); + + oldScanned = new DateTime(Convert.ToDateTime(rd.GetValue(1)).Ticks, DateTimeKind.Utc); var id = Convert.ToInt64(rd.GetValue(0)); // We do not add here, because we will never query the same path twice, @@ -687,7 +687,7 @@ namespace Duplicati.Library.Main.Database { long lastFilesetId = -1; - var lastIdObj = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" < ? AND ""ID"" != ? ORDER BY ""Timestamp"" DESC ", timestamp.ToUniversalTime(), filesetid); + var lastIdObj = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" < ? AND ""ID"" != ? ORDER BY ""Timestamp"" DESC ", NormalizeDateTime(timestamp), filesetid); if (lastIdObj != null && lastIdObj != DBNull.Value) lastFilesetId = Convert.ToInt64(lastIdObj); @@ -812,7 +812,7 @@ namespace Duplicati.Library.Main.Database using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { cmd.Transaction = tr.Parent; - m_filesetId = Convert.ToInt64(cmd.ExecuteScalar(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"") VALUES (?, ?, ?); SELECT last_insert_rowid();", m_operationid, timestamp, volumeid)); + m_filesetId = Convert.ToInt64(cmd.ExecuteScalar(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"") VALUES (?, ?, ?); SELECT last_insert_rowid();", m_operationid, NormalizeDateTime(timestamp), volumeid)); tr.Commit(); return m_filesetId; } diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index bcd573104..a7bec8069 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -124,6 +124,18 @@ namespace Duplicati.Library.Main.Database m_result = result; } + /// + /// Normalizes a DateTime instance floor'ed to seconds and in UTC + /// + /// The normalised date time + /// The input time + public static DateTime NormalizeDateTime(DateTime input) + { + var ticks = input.ToUniversalTime().Ticks; + ticks -= ticks % TimeSpan.TicksPerSecond; + return new DateTime(ticks, DateTimeKind.Utc); + } + public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, System.Data.IDbTransaction transaction = null) { m_updateremotevolumeCommand.Transaction = transaction; @@ -166,7 +178,7 @@ namespace Duplicati.Library.Main.Database throw new Exception("Invalid DateTime given, must be either local or UTC"); // Make sure the resolution is the same (i.e. no milliseconds) - time = Library.Utility.Utility.DeserializeDateTime(Library.Utility.Utility.SerializeDateTime(time)).ToUniversalTime(); + time = NormalizeDateTime(time); query += @" ""Timestamp"" <= ?"; args.Add(time); diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index eca6ff417..1734ca16d 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -82,7 +82,7 @@ namespace Duplicati.Library.Main.Database q += ",?"; //First we remove unwanted entries - cmd.ExecuteNonQuery(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" IN (" + q + @")) ", toDelete.Select(x => x.ToUniversalTime()).Cast().ToArray()); + cmd.ExecuteNonQuery(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""Timestamp"" IN (" + q + @")) ", toDelete.Select(x => NormalizeDateTime(x)).Cast().ToArray()); var deleted = cmd.ExecuteNonQuery(@"DELETE FROM ""Fileset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FilesetID"" FROM ""FilesetEntry"") "); if (deleted != toDelete.Length) diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 0c7bbb42c..319639e0a 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -36,7 +36,7 @@ namespace Duplicati.Library.Main.Database using(var cmd = m_connection.CreateCommand()) { - long filesetId = GetFilesetID(restoretime, versions); + long filesetId = GetFilesetID(NormalizeDateTime(restoretime), versions); m_restoreTime = Convert.ToDateTime(cmd.ExecuteScalar(@"SELECT ""Timestamp"" FROM ""Fileset"" WHERE ""ID"" = ?", filesetId)); cmd.Parameters.Clear(); diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index d396c3138..a5da2f8ef 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -475,11 +475,12 @@ namespace Duplicati.Library.Main.Operation m_result.ExaminedFiles++; bool changed = false; + DateTime lastModified = new DateTime(0, DateTimeKind.Utc); try { - DateTime lastModified = m_snapshot.GetLastWriteTime(path); - if (oldId < 0 || m_options.DisableFiletimeCheck || lastModified > oldScanned && (m_options.SkipFilesLargerThan == long.MaxValue || m_snapshot.GetFileSize(path) < m_options.SkipFilesLargerThan)) + lastModified = m_snapshot.GetLastWriteTime(path).ToUniversalTime(); + if (oldId < 0 || m_options.DisableFiletimeCheck || LocalDatabase.NormalizeDateTime(lastModified) >= oldScanned && (m_options.SkipFilesLargerThan == long.MaxValue || m_snapshot.GetFileSize(path) < m_options.SkipFilesLargerThan)) { m_result.AddVerboseMessage("Checking file for changes {0}", path);