diff --git a/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql b/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql index f86ac1ef8..f056e949a 100644 --- a/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql +++ b/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql @@ -25,18 +25,24 @@ CREATE TABLE "Remotevolume" ( ); /* -The fileset is a list of files contained -in a single operation, grouped by an -operationID +The OperationFileset contains an +entry for each file scanned for +a single operation. The scantime +is the time the file was last +scanned */ - CREATE TABLE "OperationFileset" ( "OperationID" INTEGER NOT NULL, - "FilesetID" INTEGER NOT NULL, + "FileEntryID" INTEGER NOT NULL, "Scantime" DATETIME NOT NULL ); -CREATE TABLE "Fileset" ( +/* +The FileEntry contains an ID +for each path and each version +of the data and metadata +*/ +CREATE TABLE "FileEntry" ( "ID" INTEGER PRIMARY KEY, "Path" TEXT NOT NULL, "BlocksetID" INTEGER NOT NULL, @@ -45,7 +51,9 @@ CREATE TABLE "Fileset" ( /* The blocklist hashes are hashes of -fragments of the blocklists +fragments of the blocklists. +They are grouped by the BlocksetID +and ordered by the index */ CREATE TABLE "BlocklistHash" ( "BlocksetID" INTEGER NOT NULL, @@ -53,7 +61,6 @@ CREATE TABLE "BlocklistHash" ( "Hash" TEXT NOT NULL ); - /* The blockset is a list of blocks @@ -73,7 +80,9 @@ CREATE TABLE "Blockset" ( /* The elements of a blocklist, -the hash is the block hash +the hash is the block hash, +they are grouped by the BlocksetID +and ordered by the index */ CREATE TABLE "BlocksetEntry" ( "BlocksetID" INTEGER NOT NULL, @@ -107,7 +116,8 @@ CREATE TABLE "DeletedBlock" ( /* -A metadata set +A metadata set, essentially a placeholder +to easily extend metadatasets with new properties */ CREATE TABLE "Metadataset" ( "ID" INTEGER PRIMARY KEY, @@ -115,7 +125,9 @@ CREATE TABLE "Metadataset" ( ); /* -Operations performed on the backend +Operations performed on the backend, +intended to be used when constructing +an error report or when debugging */ CREATE TABLE "RemoteOperation" ( "ID" INTEGER PRIMARY KEY, @@ -127,7 +139,9 @@ CREATE TABLE "RemoteOperation" ( ); /* -Logged events +Logged events, intended to be used when +constructing an error report or when +debugging */ CREATE TABLE "LogData" ( "ID" INTEGER PRIMARY KEY, @@ -147,7 +161,8 @@ CREATE TABLE "Version" ( ); /* -Settings, such as hash and blocksize +Settings, such as hash and blocksize, +used for verification */ CREATE TABLE "Configuration" ( "Key" TEXT PRIMARY KEY NOT NULL, diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs index 64680c634..54f64ab8c 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs @@ -96,17 +96,17 @@ namespace Duplicati.Library.Main.ForestHash.Database m_findmetadatasetCommand.CommandText = @"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" AND ""C"".""Hash"" = ? AND ""C"".""Size"" = ? LIMIT 1"; m_findmetadatasetCommand.AddParameters(2); - m_findfilesetCommand.CommandText = @"SELECT ""ID"" FROM ""Fileset"" WHERE ""BlocksetID"" = ? AND ""MetadatasetID"" = ? AND ""Path"" = ?"; + m_findfilesetCommand.CommandText = @"SELECT ""ID"" FROM ""FileEntry"" WHERE ""BlocksetID"" = ? AND ""MetadatasetID"" = ? AND ""Path"" = ?"; m_findfilesetCommand.AddParameters(3); m_insertblockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (?, ?, ?)"; m_insertblockCommand.AddParameters(3); - m_insertfileOperationCommand.CommandText = @"INSERT INTO ""OperationFileset"" (""OperationID"", ""FilesetID"", ""Scantime"") VALUES (?, ?, ?)"; + m_insertfileOperationCommand.CommandText = @"INSERT INTO ""OperationFileset"" (""OperationID"", ""FileEntryID"", ""Scantime"") VALUES (?, ?, ?)"; m_insertfileOperationCommand.AddParameter(m_operationid); m_insertfileOperationCommand.AddParameters(2); - m_insertfileCommand.CommandText = @"INSERT INTO ""Fileset"" (""Path"",""BlocksetID"", ""MetadataID"") VALUES (?, ? ,?); SELECT last_insert_rowid();"; + m_insertfileCommand.CommandText = @"INSERT INTO ""FileEntry"" (""Path"",""BlocksetID"", ""MetadataID"") VALUES (?, ? ,?); SELECT last_insert_rowid();"; m_insertfileCommand.AddParameters(3); m_insertblocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?, ?); SELECT last_insert_rowid();"; @@ -124,12 +124,12 @@ namespace Duplicati.Library.Main.ForestHash.Database //Need a temporary table with path/scantime lookups m_scantimelookupTablename = "ScanTime-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); using (var cmd = m_connection.CreateCommand()) - cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT ""OperationFileset"".""FilesetID"" AS ""FilesetID"", MAX(""OperationFileset"".""Scantime"") AS ""Scantime"", ""Fileset"".""Path"" AS ""Path"" FROM ""OperationFileset"" INNER JOIN ""Fileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" GROUP BY ""FilesetID"", ""Path"" ", m_scantimelookupTablename)); + cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT ""OperationFileset"".""FileEntryID"" AS ""FileEntryID"", MAX(""OperationFileset"".""Scantime"") AS ""Scantime"", ""FileEntry"".""Path"" AS ""Path"" FROM ""OperationFileset"" INNER JOIN ""FileEntry"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" GROUP BY ""OperationFileset"".""FileEntryID"", ""FileEntry"".""Path"" ", m_scantimelookupTablename)); - m_selectfileSimpleCommand.CommandText = string.Format(@"SELECT ""FilesetID"", ""Scantime"" FROM ""{0}"" WHERE ""Path"" = ?", m_scantimelookupTablename); + m_selectfileSimpleCommand.CommandText = string.Format(@"SELECT ""FileEntryID"", ""Scantime"" FROM ""{0}"" WHERE ""Path"" = ?", m_scantimelookupTablename); m_selectfileSimpleCommand.AddParameters(1); - m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"" INNER JOIN ""Fileset"" ON ""Blockset"".""ID"" = ""Fileset"".""BlocksetID"" WHERE ""Fileset"".""ID"" = ? "; + m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"" INNER JOIN ""FileEntry"" ON ""Blockset"".""ID"" = ""FileEntry"".""BlocksetID"" WHERE ""FileEntry"".""ID"" = ? "; m_selectfileHashCommand.AddParameters(1); m_selectblocklistHashesCommand.CommandText = @"SELECT ""Hash"" FROM ""BlocklistHash"" WHERE ""BlocksetID"" = ? ORDER BY ""Index"" ASC "; @@ -187,7 +187,7 @@ namespace Duplicati.Library.Main.ForestHash.Database } if (m_fileScantimeLookup != null) - using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FilesetID"", ""Scantime"", ""Path"" FROM ""{0}"" ", m_scantimelookupTablename))) + using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileEntryID"", ""Scantime"", ""Path"" FROM ""{0}"" ", m_scantimelookupTablename))) while (rd.Read()) { var id = Convert.ToInt64(rd.GetValue(0)); @@ -197,7 +197,7 @@ namespace Duplicati.Library.Main.ForestHash.Database } if (m_filesetLookup != null) - using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""Fileset"" "))) + using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""FileEntry"" "))) while (rd.Read()) { var path = rd.GetValue(0).ToString(); @@ -704,8 +704,7 @@ namespace Duplicati.Library.Main.ForestHash.Database { using (var cmd = m_connection.CreateCommand()) { - //cmd.CommandText = @"SELECT ""A"".""ID"", ""D"".""Scantime"", ""B"".""Length"", ""B"".""FullHash"", ""E"".""FullHash"", ""E"".""Length"", ""B"".""ID"" FROM ""Fileset"" A, ""Blockset"" B, ""Metadataset"" C, ""OperationFileset"" D, ""Blockset"" E, ""Operation"" F WHERE ""A"".""ID"" = ""D"".""FilesetID"" AND ""F"".""ID"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"" AND ""F"".""ID"" = ""D"".""OperationID"" ORDER BY ""F"".""Timestamp"" DESC "; - cmd.CommandText = @"SELECT ""B"".""BlocksetID"", ""B"".""ID"", ""B"".""Path"", ""D"".""Length"", ""D"".""FullHash"", ""A"".""Scantime"" FROM ""OperationFileset"" A, ""Fileset"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""B"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?) AND ""A"".""OperationID"" = ? "; + cmd.CommandText = @"SELECT ""B"".""BlocksetID"", ""B"".""ID"", ""B"".""Path"", ""D"".""Length"", ""D"".""FullHash"", ""A"".""Scantime"" FROM ""OperationFileset"" A, ""FileEntry"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""FileEntryID"" = ""B"".""ID"" AND ""B"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?) AND ""A"".""OperationID"" = ? "; cmd.AddParameter(FOLDER_BLOCKSET_ID); cmd.AddParameter(SYMLINK_BLOCKSET_ID); cmd.AddParameter(operationId == null ? m_operationid : operationId.Value); @@ -724,7 +723,7 @@ namespace Duplicati.Library.Main.ForestHash.Database filesetvolume.AddSymlink(path, metahash, metalength); } - cmd.CommandText = @"SELECT ""F"".""Path"", ""F"".""Scantime"", ""F"".""Filelength"", ""F"".""Filehash"", ""F"".""Metahash"", ""F"".""Metalength"", ""G"".""Hash"" FROM (SELECT ""A"".""Path"" AS ""Path"", ""D"".""Scantime"" AS ""Scantime"", ""B"".""Length"" AS ""Filelength"", ""B"".""FullHash"" AS ""Filehash"", ""E"".""FullHash"" AS ""Metahash"", ""E"".""Length"" AS ""Metalength"", ""A"".""BlocksetID"" AS ""BlocksetID"" FROM ""Fileset"" A, ""Blockset"" B, ""Metadataset"" C, ""OperationFileset"" D, ""Blockset"" E WHERE ""A"".""ID"" = ""D"".""FilesetID"" AND ""D"".""OperationID"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"") F LEFT OUTER JOIN ""BlocklistHash"" G ON ""G"".""BlocksetID"" = ""F"".""BlocksetID"" ORDER BY ""F"".""Path"", ""G"".""Index"" "; + cmd.CommandText = @"SELECT ""F"".""Path"", ""F"".""Scantime"", ""F"".""Filelength"", ""F"".""Filehash"", ""F"".""Metahash"", ""F"".""Metalength"", ""G"".""Hash"" FROM (SELECT ""A"".""Path"" AS ""Path"", ""D"".""Scantime"" AS ""Scantime"", ""B"".""Length"" AS ""Filelength"", ""B"".""FullHash"" AS ""Filehash"", ""E"".""FullHash"" AS ""Metahash"", ""E"".""Length"" AS ""Metalength"", ""A"".""BlocksetID"" AS ""BlocksetID"" FROM ""FileEntry"" A, ""Blockset"" B, ""Metadataset"" C, ""OperationFileset"" D, ""Blockset"" E WHERE ""A"".""ID"" = ""D"".""FileEntryID"" AND ""D"".""OperationID"" = ? AND ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""E"".""ID"" = ""C"".""BlocksetID"") F LEFT OUTER JOIN ""BlocklistHash"" G ON ""G"".""BlocksetID"" = ""F"".""BlocksetID"" ORDER BY ""F"".""Path"", ""G"".""Index"" "; cmd.Parameters.Clear(); cmd.AddParameter(m_operationid); @@ -795,28 +794,34 @@ namespace Duplicati.Library.Main.ForestHash.Database base.Dispose(); } + + private long GetPreviousBackupOperationID(System.Data.IDbCommand cmd) + { + long lastOperationId = -1; + + var lastIdObj = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Timestamp"" < ? AND ""Description"" = ? ORDER BY ""Timestamp"" DESC ", OperationTimestamp, "Backup"); + if (lastIdObj != null && lastIdObj != DBNull.Value) + lastOperationId = Convert.ToInt64(lastIdObj); + + return lastOperationId; + } internal void UpdateChangeStatistics (BackupStatistics m_stat) { using (var cmd = m_connection.CreateCommand()) { - long lastOperationId = -1; + var lastOperationId = GetPreviousBackupOperationID(cmd); + m_stat.AddedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, FOLDER_BLOCKSET_ID, lastOperationId)); + m_stat.AddedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, SYMLINK_BLOCKSET_ID, lastOperationId)); + m_stat.AddedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" != ? AND ""FileEntry"".""BlocksetID"" != ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, lastOperationId)); - var lastIdObj = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Timestamp"" < ? AND ""Description"" = ? ORDER BY ""Timestamp"" DESC ", OperationTimestamp, "Backup"); - if (lastIdObj != null && lastIdObj != DBNull.Value) - lastOperationId = Convert.ToInt64(lastIdObj); + m_stat.DeletedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid)); + m_stat.DeletedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid)); + m_stat.DeletedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" != ? AND ""FileEntry"".""BlocksetID"" != ? AND NOT ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid)); - m_stat.AddedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, FOLDER_BLOCKSET_ID, lastOperationId)); - m_stat.AddedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, SYMLINK_BLOCKSET_ID, lastOperationId)); - m_stat.AddedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""BlocksetID"" != ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", m_operationid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, lastOperationId)); - - m_stat.DeletedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid)); - m_stat.DeletedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid)); - m_stat.DeletedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""BlocksetID"" != ? AND NOT ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid)); - - m_stat.ModifiedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid)); - m_stat.ModifiedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" = ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid)); - m_stat.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""BlocksetID"" != ? AND ""Fileset"".""Path"" IN (SELECT ""Path"" FROM ""Fileset"" INNER JOIN ""OperationFileset"" ON ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid)); + m_stat.ModifiedFolders = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, m_operationid)); + m_stat.ModifiedSymlinks = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" = ? AND ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, SYMLINK_BLOCKSET_ID, m_operationid)); + m_stat.ModifiedFiles = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT COUNT(*) FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ? AND ""FileEntry"".""BlocksetID"" != ? AND ""FileEntry"".""BlocksetID"" != ? AND ""FileEntry"".""Path"" IN (SELECT ""Path"" FROM ""FileEntry"" INNER JOIN ""OperationFileset"" ON ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" WHERE ""OperationFileset"".""OperationID"" = ?)", lastOperationId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, m_operationid)); } if (m_blockHashLookup != null && m_fileHashLookup != null && (m_blockHashLookup.FalsePositives > 200 || m_fileHashLookup.FalsePositives > 20)) @@ -878,18 +883,14 @@ namespace Duplicati.Library.Main.ForestHash.Database using(var cmd = m_connection.CreateCommand()) using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) { - long lastOperationId = -1; - - var lastIdObj = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Timestamp"" < ? AND ""Description"" = ? ORDER BY ""Timestamp"" DESC ", OperationTimestamp, "Backup"); - if (lastIdObj != null && lastIdObj != DBNull.Value) - lastOperationId = Convert.ToInt64(lastIdObj); + long lastOperationId = GetPreviousBackupOperationID(cmd); cmd.Transaction = tr.Parent; - cmd.ExecuteNonQuery(@"INSERT INTO ""OperationFileset"" (""OperationID"", ""FilesetID"", ""Scantime"") SELECT ? AS ""OperationID"", ""FilesetID"", ""Scantime"" FROM ""OperationFile"" WHERE ""OperationID"" = ? AND ""FilesetID"" NOT IN (SELECT ""FilesetID"" FROM ""OperationFileset"" WHERE ""OperationID"" = ?) ", m_operationid, lastOperationId, m_operationid); + cmd.ExecuteNonQuery(@"INSERT INTO ""OperationFileset"" (""OperationID"", ""FileEntryID"", ""Scantime"") SELECT ? AS ""OperationID"", ""FileEntryID"", ""Scantime"" FROM ""OperationFile"" WHERE ""OperationID"" = ? AND ""FileEntryID"" NOT IN (SELECT ""FileEntryID"" FROM ""OperationFileset"" WHERE ""OperationID"" = ?) ", m_operationid, lastOperationId, m_operationid); if (deleted != null) { - cmd.CommandText = @"DELETE FROM ""OperationFileset"" WHERE ""OperationID"" = ? AND ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""Path"" = ?) "; + cmd.CommandText = @"DELETE FROM ""OperationFileset"" WHERE ""OperationID"" = ? AND ""FileEntryID"" IN (SELECT ""ID"" FROM ""FileEntry"" WHERE ""Path"" = ?) "; cmd.AddParameter(m_operationid); cmd.AddParameter(); diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalBlocklistUpdateDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalBlocklistUpdateDatabase.cs index b323d5c43..c9103fb55 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalBlocklistUpdateDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalBlocklistUpdateDatabase.cs @@ -1,56 +1,56 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; - -namespace Duplicati.Library.Main.ForestHash.Database -{ - public partial class LocalBlocklistUpdateDatabase : LocalRestoreDatabase - { +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace Duplicati.Library.Main.ForestHash.Database +{ + public partial class LocalBlocklistUpdateDatabase : LocalRestoreDatabase + { protected string m_tempblockvolumetable; - protected string m_temphashtable; - - public LocalBlocklistUpdateDatabase(string path, long blocksize) - : base("Rebuild", path, blocksize) - { - } - - public void FindMissingBlocklistHashes() - { + protected string m_temphashtable; + + public LocalBlocklistUpdateDatabase(string path, long blocksize) + : base("Rebuild", path, blocksize) + { + } + + public void FindMissingBlocklistHashes() + { m_tempblockvolumetable = "MissingBlocks-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); m_temphashtable = "Hashlist-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - using (var cmd = m_connection.CreateCommand()) - { + using (var cmd = m_connection.CreateCommand()) + { cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS SELECT DISTINCT ""Block"".""VolumeID"" AS ""VolumeID"", ""BlocklistHash"".""Hash"" AS ""Hash"", 0 AS ""Restored"" FROM ""Block"", ""BlocklistHash"" WHERE ""Block"".""Hash"" = ""BlocklistHash"".""Hash"" ", m_tempblockvolumetable)); cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT DISTINCT ""Blockset"".""ID"", 0, ""Block"".""ID"" FROM ""Blockset"", ""Block"" WHERE ""Blockset"".""Fullhash"" = ""Block"".""Hash"" AND ""Blockset"".""Length"" < {0} AND ""Blockset"".""ID"" NOT IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"") ", m_blocksize)); cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Hash"" TEXT NOT NULL, ""Index"" INTEGER NOT NULL)", m_temphashtable)); - - } - } - - public IEnumerable GetBlockLists(long volumeid) - { - return new BlocklistsEnumerable(m_connection, m_tempblockvolumetable, volumeid); - } - - public void UpdateBlocklist(string hash, IEnumerable hashes, long hashsize, System.Data.IDbTransaction transaction) - { - using (var cmd = m_connection.CreateCommand()) - { - cmd.Transaction = transaction; - - cmd.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""Restored"" = 0 AND ""Hash"" = ? ", m_tempblockvolumetable); - cmd.AddParameter(hash); - var c = cmd.ExecuteNonQuery(); - if (c != 1) + + } + } + + public IEnumerable GetBlockLists(long volumeid) + { + return new BlocklistsEnumerable(m_connection, m_tempblockvolumetable, volumeid); + } + + public void UpdateBlocklist(string hash, IEnumerable hashes, long hashsize, System.Data.IDbTransaction transaction) + { + using (var cmd = m_connection.CreateCommand()) + { + cmd.Transaction = transaction; + + cmd.CommandText = string.Format(@"UPDATE ""{0}"" SET ""Restored"" = 1 WHERE ""Restored"" = 0 AND ""Hash"" = ? ", m_tempblockvolumetable); + cmd.AddParameter(hash); + var c = cmd.ExecuteNonQuery(); + if (c != 1) throw new Exception(string.Format("Blocklist not found {0}", hash)); cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}""", m_temphashtable)); cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Index"") VALUES (?, ?)", m_temphashtable); long ix = 0; foreach (var h in hashes) - { + { c = cmd.ExecuteNonQuery(null, h, ix); if (c != 1) throw new Exception(string.Format("Insert row error {0}", h)); @@ -58,58 +58,76 @@ namespace Duplicati.Library.Main.ForestHash.Database } cmd.CommandText = string.Format(@"INSERT INTO ""BlocksetEntry"" (""BlocksetID"", ""Index"", ""BlockID"") SELECT ""A"".""BlocksetID"", (""A"".""Index"" * ?) + ""B"".""Index"", ""C"".""ID"" FROM ""BlocklistHash"" A, ""{0}"" B, ""Block"" C WHERE ""A"".""Hash"" = ? AND ""C"".""Hash"" = ""B"".""Hash"" ", m_temphashtable); - c = cmd.ExecuteNonQuery(null, (m_blocksize / hashsize), hash); + c = cmd.ExecuteNonQuery(null, (m_blocksize / hashsize), hash); if (c == 0 || c % ix != 0) - throw new Exception(string.Format("Wrong number of inserts, got {0} records from {1} hashes!", c, ix)); - } - } - - public IList GetMissingBlockListVolumes() - { - using (var cmd = m_connection.CreateCommand()) - { - List result = new List(); - cmd.CommandText = string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""{0}"")", m_tempblockvolumetable); - using (var rd = cmd.ExecuteReader()) - { - object[] r = new object[3]; - while (rd.Read()) - { - rd.GetValues(r); - result.Add(new RemoteVolume( - r[0] == null ? null : r[0].ToString(), - r[1] == null ? null : r[1].ToString(), - r[2] == null ? -1 : Convert.ToInt64(r[2]) - )); - } - } - - return result; - } - } - - public void VerifyDatabaseIntegrity() - { - using (var cmd = m_connection.CreateCommand()) - { - cmd.CommandText = @"SELECT COUNT(*) FROM (SELECT SUM(""Block"".""Size"") AS ""CalcLen"", ""Blockset"".""Length"" AS ""Length"", ""BlocksetEntry"".""BlocksetID"" FROM ""Block"", ""BlocksetEntry"", ""Blockset"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Blockset"".""ID"" = ""BlocksetEntry"".""BlocksetID"" GROUP BY ""BlocksetEntry"".""BlocksetID"") WHERE ""CalcLen"" != ""Length"""; - var c = Convert.ToInt64(cmd.ExecuteScalar()); - if (c != 0) - throw new InvalidDataException("Inconsistency detected, not all blocklists were restored correctly"); - } - } - - public override void Dispose() - { - base.Dispose(); - using (var cmd = m_connection.CreateCommand()) - { - if (m_tempblockvolumetable != null) - try - { - cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempblockvolumetable); - cmd.ExecuteNonQuery(); - } + throw new Exception(string.Format("Wrong number of inserts, got {0} records from {1} hashes!", c, ix)); + } + } + + public IList GetMissingBlockListVolumes() + { + using (var cmd = m_connection.CreateCommand()) + { + List result = new List(); + cmd.CommandText = string.Format(@"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""{0}"")", m_tempblockvolumetable); + using (var rd = cmd.ExecuteReader()) + { + object[] r = new object[3]; + while (rd.Read()) + { + rd.GetValues(r); + result.Add(new RemoteVolume( + r[0] == null ? null : r[0].ToString(), + r[1] == null ? null : r[1].ToString(), + r[2] == null ? -1 : Convert.ToInt64(r[2]) + )); + } + } + + return result; + } + } + + /// + /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created. + /// + /// The ID of the fileset volume to update + /// The timestamp of the operation to create + /// An optional external transaction + public void CreateBackupOperation(long volumeid, DateTime timestamp, System.Data.IDbTransaction transaction = null) + { + using (var cmd = m_connection.CreateCommand()) + using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + { + cmd.Transaction = tr.Parent; + var operationid = cmd.ExecuteNonQuery(@"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES (?, ?); SELECT last_insert_rowid()", "Backup", timestamp); + cmd.ExecuteNonQuery(@"UPDATE ""RemoteVolume"" SET ""OperationID"" = ? WHERE ""ID"" = ?", operationid, volumeid); + tr.Commit(); + } + } + + public void VerifyDatabaseIntegrity() + { + using (var cmd = m_connection.CreateCommand()) + { + cmd.CommandText = @"SELECT COUNT(*) FROM (SELECT SUM(""Block"".""Size"") AS ""CalcLen"", ""Blockset"".""Length"" AS ""Length"", ""BlocksetEntry"".""BlocksetID"" FROM ""Block"", ""BlocksetEntry"", ""Blockset"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Blockset"".""ID"" = ""BlocksetEntry"".""BlocksetID"" GROUP BY ""BlocksetEntry"".""BlocksetID"") WHERE ""CalcLen"" != ""Length"""; + var c = Convert.ToInt64(cmd.ExecuteScalar()); + if (c != 0) + throw new InvalidDataException("Inconsistency detected, not all blocklists were restored correctly"); + } + } + + public override void Dispose() + { + base.Dispose(); + using (var cmd = m_connection.CreateCommand()) + { + if (m_tempblockvolumetable != null) + try + { + cmd.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tempblockvolumetable); + cmd.ExecuteNonQuery(); + } finally { m_tempblockvolumetable = null; } if (m_temphashtable != null) @@ -119,7 +137,7 @@ namespace Duplicati.Library.Main.ForestHash.Database cmd.ExecuteNonQuery(); } finally { m_temphashtable = null; } - } - } - } -} + } + } + } +} diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalCleanupDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalCleanupDatabase.cs index abebe13af..6608409ce 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalCleanupDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalCleanupDatabase.cs @@ -147,7 +147,7 @@ namespace Duplicati.Library.Main.ForestHash.Database public IEnumerable GetSourceFilesWithBlocks(long volumeid, long blocksize) { using(var cmd = m_connection.CreateCommand()) - using(var rd = cmd.ExecuteReader(string.Format(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", ""Fileset"".""Path"", ""BlocksetEntry"".""Index"" * {0} FROM ""Block"", ""BlocksetEntry"", ""Fileset"" WHERE ""Fileset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" = ? ", blocksize), volumeid)) + using(var rd = cmd.ExecuteReader(string.Format(@"SELECT DISTINCT ""Block"".""Hash"", ""Block"".""Size"", ""FileEntry"".""Path"", ""BlocksetEntry"".""Index"" * {0} FROM ""Block"", ""BlocksetEntry"", ""FileEntry"" WHERE ""FileEntry"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""Block"".""ID"" = ""BlocksetEntry"".""BlockID"" AND ""Block"".""VolumeID"" = ? ", blocksize), volumeid)) if (rd.Read()) { var bs = new BlockWithSources(rd); @@ -166,10 +166,10 @@ namespace Duplicati.Library.Main.ForestHash.Database public IEnumerable GetFilesetsUsingBlock(string hash, int size) { - var blocks = @"SELECT DISTINCT ""Fileset"".""ID"" AS ID FROM ""Block"", ""Blockset"", ""BlocksetEntry"", ""Fileset"" WHERE ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Fileset"".""BlocksetID"" = ""Blockset"".""ID"" "; - var blocklists = @"SELECT DISTINCT ""Fileset"".""ID"" AS ID FROM ""Block"", ""Blockset"", ""BlocklistHash"", ""Fileset"" WHERE ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Fileset"".""BlocksetID"" = ""Blockset"".""ID"" "; + var blocks = @"SELECT DISTINCT ""FileEntry"".""ID"" AS ID FROM ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileEntry"" WHERE ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileEntry"".""BlocksetID"" = ""Blockset"".""ID"" "; + var blocklists = @"SELECT DISTINCT ""FileEntry"".""ID"" AS ID FROM ""Block"", ""Blockset"", ""BlocklistHash"", ""FileEntry"" WHERE ""Block"".""Hash"" = ? AND ""Block"".""Size"" = ? AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileEntry"".""BlocksetID"" = ""Blockset"".""ID"" "; - var cmdtxt = @"SELECT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""OperationFileset"" WHERE ""RemoteVolume"".""OperationID"" = ""OperationFileset"".""OperationID"" AND ""RemoteVolume"".""Type"" = ? AND ""OperationFileset"".""FilesetID"" IN (SELECT DISTINCT ""ID"" FROM ( " + blocks + " UNION " + blocklists + " ))"; + var cmdtxt = @"SELECT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""OperationFileset"" WHERE ""RemoteVolume"".""OperationID"" = ""OperationFileset"".""OperationID"" AND ""RemoteVolume"".""Type"" = ? AND ""OperationFileset"".""FileEntryID"" IN (SELECT DISTINCT ""ID"" FROM ( " + blocks + " UNION " + blocklists + " ))"; using(var cmd = m_connection.CreateCommand()) using(var rd = cmd.ExecuteReader(cmdtxt, RemoteVolumeType.Files.ToString(), hash, size, hash, size)) diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs index 4cdd03e69..0bf72d304 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs @@ -210,7 +210,7 @@ namespace Duplicati.Library.Main.ForestHash.Database var deletecmd = m_connection.CreateCommand(); deletecmd.Transaction = tr.Parent; - deletecmd.ExecuteNonQuery(@"UPDATE ""Fileset"" SET ""BlocksetID"" = -1 WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT DISTINCT ID FROM ""RemoteVolume"" WHERE ""Name"" = ?)))", name); + deletecmd.ExecuteNonQuery(@"UPDATE ""FileEntry"" SET ""BlocksetID"" = -1 WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT DISTINCT ID FROM ""RemoteVolume"" WHERE ""Name"" = ?)))", name); deletecmd.ExecuteNonQuery(@"UPDATE ""Metadataset"" SET ""BlocksetID"" = -1 WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT DISTINCT ID FROM ""RemoteVolume"" WHERE ""Name"" = ?)))", name); deletecmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT DISTINCT ID FROM ""RemoteVolume"" WHERE ""Name"" = ?)))", name); deletecmd.ExecuteNonQuery(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN (SELECT DISTINCT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""ID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT DISTINCT ID FROM ""RemoteVolume"" WHERE ""Name"" = ?)))", name); @@ -244,7 +244,7 @@ namespace Duplicati.Library.Main.ForestHash.Database } } - public long GetFilesetID(DateTime restoretime) + public long GetFilesetOperationID(DateTime restoretime) { if (restoretime.Kind == DateTimeKind.Unspecified) throw new Exception("Invalid DateTime given, must be either local or UTC"); @@ -368,10 +368,10 @@ namespace Duplicati.Library.Main.ForestHash.Database } } - public IEnumerable GetFiles(long filesetid) + public IEnumerable GetFiles(long operationID) { using(var cmd = m_connection.CreateCommand()) - using(var rd = cmd.ExecuteReader(@"SELECT ""A"".""Path"", ""B"".""Length"", ""B"".""FullHash"", ""D"".""FullHash"" FROM ""Fileset"" A, ""Blockset"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND ""A"".""OperationID"" = ? ", filesetid)) + using(var rd = cmd.ExecuteReader(@"SELECT ""A"".""Path"", ""B"".""Length"", ""B"".""FullHash"", ""D"".""FullHash"" FROM ""FileEntry"" A, ""Blockset"" B, ""Metadataset"" C, ""Blockset"" D WHERE ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND ""A"".""OperationID"" = ? ", operationID)) while(rd.Read()) yield return new LocalFileEntry(rd); } @@ -421,7 +421,7 @@ namespace Duplicati.Library.Main.ForestHash.Database cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", filetable), s); string first = null; - using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""{0}"".""Path"", ""RemoteVolume"".""Name"", ""Operation"".""Timestamp"" FROM ""{0}"", ""RemoteVolume"", ""Fileset"", ""OperationFileset"" WHERE ""OperationFileset"".""FilesetID"" = ""Fileset"".""ID"" AND ""OperationFileset"".""OperationID"" = ""RemoteVolume"".""OperationID"" AND ""Fileset"".""Path"" = ""{0}"".""Path"" AND ""RemoteVolume"".""Type"" = ? ORDER BY ""Operation"".""Timestamp"" DESC ", filetable), RemoteVolumeType.Files)) + using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""{0}"".""Path"", ""RemoteVolume"".""Name"", ""Operation"".""Timestamp"" FROM ""{0}"", ""RemoteVolume"", ""FileEntry"", ""OperationFileset"" WHERE ""OperationFileset"".""FileEntryID"" = ""FileEntry"".""ID"" AND ""OperationFileset"".""OperationID"" = ""RemoteVolume"".""OperationID"" AND ""FileEntry"".""Path"" = ""{0}"".""Path"" AND ""RemoteVolume"".""Type"" = ? ORDER BY ""Operation"".""Timestamp"" DESC ", filetable), RemoteVolumeType.Files)) while(rd.Read()) { var cur = rd.GetValue(0).ToString(); diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalDeleteDatabase.cs index 27cb4a886..7524eb0c9 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalDeleteDatabase.cs @@ -72,6 +72,16 @@ namespace Duplicati.Library.Main.ForestHash.Database return result; } + + private long GetLastBackupOperationID(System.Data.IDbCommand cmd) + { + long id = -1; + var r = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Description"" = ? ORDER BY ""Timestamp"" DESC LIMIT 1", "Backup"); + if (r != null && r != DBNull.Value) + id = Convert.ToInt64(r); + + return id; + } /// /// Deletes all but n backups from the database and remote storage. @@ -86,11 +96,7 @@ namespace Duplicati.Library.Main.ForestHash.Database cmd.Transaction = transaction; long keepOperationId = -1; if (!allowRemovingLast) - { - var r = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Description"" = ? ORDER BY ""Timestamp"" DESC LIMIT 1", "Backup"); - if (r != null && r != DBNull.Value) - keepOperationId = Convert.ToInt64(keepOperationId); - } + keepOperationId = GetLastBackupOperationID(cmd); //We create a table with the operationIDs that are about to be deleted var tmptablename = "DeletedOperations-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); @@ -117,11 +123,7 @@ namespace Duplicati.Library.Main.ForestHash.Database cmd.Transaction = transaction; long keepOperationId = -1; if (!allowRemovingLast) - { - var r = cmd.ExecuteScalar(@"SELECT ""ID"" FROM ""Operation"" WHERE ""Description"" = ? ORDER BY ""Timestamp"" DESC LIMIT 1", "Backup"); - if (r != null && r != DBNull.Value) - keepOperationId = Convert.ToInt64(keepOperationId); - } + keepOperationId = GetLastBackupOperationID(cmd); var tmptablename = "DeletedOperations-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER NOT NULL)", tmptablename)); @@ -168,9 +170,9 @@ namespace Duplicati.Library.Main.ForestHash.Database cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""OperationFileset"" WHERE ""OperationID"" IN (SELECT ""ID"" FROM ""{0}"") ", tmptablename)); //Then we delete anything that is no longer being referenced - cmd.ExecuteNonQuery(@"DELETE FROM ""Fileset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FilesetID"" FROM ""OperationFileset"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""Fileset"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""Fileset"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") "); + cmd.ExecuteNonQuery(@"DELETE FROM ""FileEntry"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileEntryID"" FROM ""OperationFileset"") "); + cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""FileEntry"") "); + cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""FileEntry"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") "); cmd.ExecuteNonQuery(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") "); //We save the block info for the remote files, before we delete it diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs index 467940ab6..ec89f7229 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalRestoreDatabase.cs @@ -33,7 +33,7 @@ namespace Duplicati.Library.Main.ForestHash.Database using (var cmd = m_connection.CreateCommand()) { - long operationID = GetFilesetID(restoretime); + long operationID = GetFilesetOperationID(restoretime); cmd.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY, ""Path"" TEXT NOT NULL, ""BlocksetID"" INTEGER NOT NULL, ""MetadataID"" INTEGER NOT NULL, ""Targetpath"" TEXT NULL ) ", m_tempfiletable); cmd.ExecuteNonQuery(); @@ -44,7 +44,7 @@ namespace Duplicati.Library.Main.ForestHash.Database if (filenameFilter == null && (p == null || p.Length == 0)) { // Simple case, restore everything - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""Fileset"", ""OperationFileset"" WHERE ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" AND ""OperationFileset"".""OperationID"" = ? ", m_tempfiletable); + cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""FileEntry"".""Path"", ""FileEntry"".""BlocksetID"", ""FileEntry"".""MetadataID"" FROM ""FileEntry"", ""OperationFileset"" WHERE ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" AND ""OperationFileset"".""OperationID"" = ? ", m_tempfiletable); cmd.AddParameter(operationID); cmd.ExecuteNonQuery(); } @@ -65,7 +65,7 @@ namespace Duplicati.Library.Main.ForestHash.Database } //TODO: Handle case-insensitive filename lookup - cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""Fileset"", ""OperationFileset"" WHERE ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" AND ""OperationID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable); + cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""BlocksetID"", ""MetadataID"") SELECT ""FileEntry"".""Path"", ""FileEntry"".""BlocksetID"", ""FileEntry"".""MetadataID"" FROM ""FileEntry"", ""OperationFileset"" WHERE ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" AND ""OperationID"" = ? AND ""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{1}"") ", m_tempfiletable, m_filenamestable); cmd.SetParameterValue(0, operationID); var c = cmd.ExecuteNonQuery(); @@ -93,7 +93,7 @@ namespace Duplicati.Library.Main.ForestHash.Database else { // Restore but filter elements based on regexp - cmd.CommandText = string.Format(@"SELECT ""Fileset"".""Path"", ""Fileset"".""BlocksetID"", ""Fileset"".""MetadataID"" FROM ""Fileset"", ""OperationFileset"" WHERE ""Fileset"".""ID"" = ""OperationFileset"".""FilesetID"" AND ""OperationID"" = ?"); + cmd.CommandText = string.Format(@"SELECT ""FileEntry"".""Path"", ""FileEntry"".""BlocksetID"", ""FileEntry"".""MetadataID"" FROM ""FileEntry"", ""OperationFileset"" WHERE ""FileEntry"".""ID"" = ""OperationFileset"".""FileEntryID"" AND ""OperationID"" = ?"); cmd.AddParameter(operationID); object[] values = new object[3]; diff --git a/Duplicati/Library/Main/ForestHash/Database/RestoreIterators.cs b/Duplicati/Library/Main/ForestHash/Database/RestoreIterators.cs index 284453b9c..f070e1912 100644 --- a/Duplicati/Library/Main/ForestHash/Database/RestoreIterators.cs +++ b/Duplicati/Library/Main/ForestHash/Database/RestoreIterators.cs @@ -1,1188 +1,1188 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using Duplicati.Library.Main.ForestHash.Volumes; - -namespace Duplicati.Library.Main.ForestHash.Database -{ - public partial class LocalRestoreDatabase - { - private class ExistingFileEnumerable : IEnumerable - { - private class ExistingFileEnumerator : IEnumerator, IDisposable - { - private class ExistingFile : IExistingFile - { - private class ExistingFileBlockEnumerable : IEnumerable - { - private class ExistingFileBlockEnumerator : IEnumerator - { - private class ExistingFileBlock : IExistingFileBlock - { - private System.Data.IDataReader m_reader; - - public ExistingFileBlock(System.Data.IDataReader reader) - { - m_reader = reader; - } - - public string Hash - { - get - { - var v = m_reader.GetValue(2); - if (v == null || v == DBNull.Value) - return null; - return v.ToString(); - } - } - - public long Index - { - get - { - var v = m_reader.GetValue(3); - if (v == null || v == DBNull.Value) - return -1; - return Convert.ToInt64(v); - } - } - - public long Size - { - get - { - var v = m_reader.GetValue(4); - if (v == null || v == DBNull.Value) - return -1; - return Convert.ToInt64(v); - } - } - } - - private System.Data.IDataReader m_reader; - private ExistingFile m_file; - private ExistingFileBlock m_block; - private string m_current; - - public ExistingFileBlockEnumerator(System.Data.IDataReader reader, ExistingFile file) - { - m_reader = reader; - m_file = file; - this.Reset(); - } - - public IExistingFileBlock Current - { - get - { - return m_block; - } - } - - public void Dispose() - { - } - - object System.Collections.IEnumerator.Current - { - get { return this.Current; } - } - - public bool MoveNext() - { - if (!m_file.HasMoreData) - return false; - - if (m_current == null) - m_current = m_file.TargetPath; - else - m_file.HasMoreData = m_reader.Read(); - - if (m_file.HasMoreData && m_current != m_file.TargetPath) - return false; - - return m_file.HasMoreData; - } - - public void Reset() - { - m_block = new ExistingFileBlock(m_reader); - m_current = null; - } - } - - private System.Data.IDataReader m_reader; - private ExistingFile m_file; - - public ExistingFileBlockEnumerable(System.Data.IDataReader reader, ExistingFile file) - { - m_reader = reader; - m_file = file; - } - - public IEnumerator GetEnumerator() - { - return new ExistingFileBlockEnumerator(m_reader, m_file); - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - } - - private System.Data.IDataReader m_reader; - public bool HasMoreData { get; set; } - - public ExistingFile(System.Data.IDataReader reader) - { - m_reader = reader; - HasMoreData = m_reader.Read(); - } - - public string TargetPath - { - get - { - var v = m_reader.GetValue(0); - if (v == null || v == DBNull.Value) - return null; - - return v.ToString(); - } - } - - public long Length - { - get - { - var v = m_reader.GetValue(1); - if (v == null || v == DBNull.Value) - return -1; - - return Convert.ToInt64(v); - } - } - - public IEnumerable Blocks - { - get { return new ExistingFileBlockEnumerable(m_reader, this); } - } - } - - private System.Data.IDbCommand m_command; - private System.Data.IDataReader m_reader; - private System.Data.IDbConnection m_connection; - private string m_tablename; - private ExistingFile m_file; - private string m_current; - - public ExistingFileEnumerator(System.Data.IDbConnection con, string tablename) - { - m_connection = con; - m_tablename = tablename; - this.Reset(); - } - - public void Dispose() - { - if (m_reader != null) - { - m_reader.Dispose(); - m_reader = null; - } - - if (m_command != null) - { - m_command.Dispose(); - m_command = null; - } - } - - public IExistingFile Current - { - get { return m_file; } - } - - object System.Collections.IEnumerator.Current - { - get { return m_file; } - } - - public bool MoveNext() - { - if (m_current != null) - { - while (m_file.HasMoreData && m_file.TargetPath == m_current) - m_file.HasMoreData = m_reader.Read(); - } - - m_current = m_file.HasMoreData ? m_file.TargetPath : null; - - return m_file.HasMoreData; - } - - public void Reset() - { - m_file = null; - m_command = m_connection.CreateCommand(); - m_command.CommandText = string.Format(@"SELECT ""{0}"".""TargetPath"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{0}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{0}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{0}"".""TargetPath"", ""BlocksetEntry"".""Index""", m_tablename); - m_reader = m_command.ExecuteReader(); - m_file = new ExistingFile(m_reader); - m_current = null; - - } - } - - private System.Data.IDbConnection m_connection; - private string m_tablename; - - public ExistingFileEnumerable(System.Data.IDbConnection connection, string tablename) - { - m_connection = connection; - m_tablename = tablename; - } - - public IEnumerator GetEnumerator() - { - return new ExistingFileEnumerator(m_connection, m_tablename); - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - } - - - private class LocalBlockSourceEnumerable : IEnumerable - { - private class LocalBlockSourceEnumerator : IEnumerator - { - private class LocalBlockSource : ILocalBlockSource - { - private class BlockDescriptorEnumerable : IEnumerable - { - private class BlockDescriptorEnumerator : IEnumerator - { - private class BlockDescriptor : IBlockDescriptor - { - private class BlockSourceEnumerable : IEnumerable - { - private class BlockSourceEnumerator : IEnumerator - { - private class BlockSource : IBlockSource - { - private System.Data.IDataReader m_reader; - public BlockSource(System.Data.IDataReader reader) - { - m_reader = reader; - } - - public string Path - { - get - { - var v = m_reader.GetValue(5); - if (v == null || v == DBNull.Value) - return null; - return v.ToString(); - } - } - - public long Offset - { - get - { - var v = m_reader.GetValue(6); - if (v == null || v == DBNull.Value) - return -1; - return Convert.ToInt64(v); - } - } - - public long Size - { - get - { - var v = m_reader.GetValue(7); - if (v == null || v == DBNull.Value) - return -1; - return Convert.ToInt64(v); - } - } - } - - private System.Data.IDataReader m_reader; - private LocalBlockSource m_localsource; - private BlockSource m_source; - private BlockDescriptor m_descriptor; - private string m_currenthash; - private string m_currentfile; - - public BlockSourceEnumerator(System.Data.IDataReader reader, LocalBlockSource localsource, BlockDescriptor descriptor) - { - m_reader = reader; - m_localsource = localsource; - m_descriptor = descriptor; - this.Reset(); - } - - public IBlockSource Current - { - get { return m_source; } - } - - public void Dispose() - { - } - - object System.Collections.IEnumerator.Current - { - get { return this.Current; } - } - - public bool MoveNext() - { - if (!m_localsource.HasMoreData) - return false; - - if (m_currenthash != null) - m_localsource.HasMoreData = m_reader.Read(); - - if (!m_localsource.HasMoreData) - return false; - - if (m_currenthash != null && m_currentfile != null) - if (m_currentfile != m_localsource.TargetPath || m_currenthash != m_descriptor.Hash) - return false; - - m_currenthash = m_descriptor.Hash; - m_currentfile = m_localsource.TargetPath; - - return m_localsource.HasMoreData; - } - - public void Reset() - { - m_source = new BlockSource(m_reader); - m_currenthash = null; - m_currentfile = null; - } - } - - private System.Data.IDataReader m_reader; - private LocalBlockSource m_localsource; - private BlockDescriptor m_descriptor; - - public BlockSourceEnumerable(System.Data.IDataReader reader, LocalBlockSource localsource, BlockDescriptor descriptor) - { - m_reader = reader; - m_localsource = localsource; - m_descriptor = descriptor; - } - - public IEnumerator GetEnumerator() - { - return new BlockSourceEnumerator(m_reader, m_localsource, m_descriptor); - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - } - - private System.Data.IDataReader m_reader; - private LocalBlockSource m_localsource; - - public BlockDescriptor(System.Data.IDataReader reader, LocalBlockSource localsource) - { - m_reader = reader; - m_localsource = localsource; - } - - public string Hash - { - get - { - var v = m_reader.GetValue(1); - if (v == null || v == DBNull.Value) - return null; - return v.ToString(); - } - } - - - public long Offset - { - get - { - var v = m_reader.GetValue(2); - if (v == null || v == DBNull.Value) - return -1; - return Convert.ToInt64(v); - } - } - - public long Index - { - get - { - var v = m_reader.GetValue(3); - if (v == null || v == DBNull.Value) - return -1; - return Convert.ToInt64(v); - } - } - - public long Size - { - get - { - var v = m_reader.GetValue(4); - if (v == null || v == DBNull.Value) - return -1; - return Convert.ToInt64(v); - } - } - - public IEnumerable Blocksources - { - get { return new BlockSourceEnumerable(m_reader, m_localsource, this); } - } - } - - private System.Data.IDataReader m_reader; - private LocalBlockSource m_localsource; - private BlockDescriptor m_descriptor; - private string m_currenthash; - private string m_currentfile; - - - public BlockDescriptorEnumerator(System.Data.IDataReader reader, LocalBlockSource localsource) - { - m_reader = reader; - m_localsource = localsource; - this.Reset(); - } - - public IBlockDescriptor Current - { - get { return m_descriptor; } - } - - public void Dispose() - { - } - - object System.Collections.IEnumerator.Current - { - get { return this.Current; } - } - - public bool MoveNext() - { - if (!m_localsource.HasMoreData) - return false; - - if (m_currenthash != null) - { - while (m_localsource.HasMoreData && m_currenthash == m_descriptor.Hash && m_currentfile == m_localsource.TargetPath) - m_localsource.HasMoreData = m_reader.Read(); - - if (m_currentfile != m_localsource.TargetPath) - return false; - } - - m_currenthash = m_localsource.HasMoreData ? m_descriptor.Hash : null; - m_currentfile = m_localsource.HasMoreData ? m_localsource.TargetPath : null; - - return m_localsource.HasMoreData && m_currenthash == m_descriptor.Hash; - - } - - public void Reset() - { - m_descriptor = new BlockDescriptor(m_reader, m_localsource); - m_currenthash = null; - m_currentfile = null; - } - } - - private System.Data.IDataReader m_reader; - private LocalBlockSource m_localsource; - public BlockDescriptorEnumerable(System.Data.IDataReader reader, LocalBlockSource localsource) - { - m_reader = reader; - m_localsource = localsource; - } - - public IEnumerator GetEnumerator() - { - return new BlockDescriptorEnumerator(m_reader, m_localsource); - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - } - - private System.Data.IDataReader m_reader; - public bool HasMoreData { get; set; } - - public LocalBlockSource(System.Data.IDataReader reader) - { - m_reader = reader; - HasMoreData = reader.Read(); - } - - public string TargetPath - { - get - { - var v = m_reader.GetValue(0); - if (v == null || v == DBNull.Value) - return null; - - return v.ToString(); - } - } - - public IEnumerable Blocks - { - get { return new BlockDescriptorEnumerable(m_reader, this); } - } - } - - private System.Data.IDbCommand m_command; - private System.Data.IDataReader m_reader; - private System.Data.IDbConnection m_connection; - private LocalBlockSource m_localsource; - private string m_filetablename; - private string m_blocktablename; - private string m_current; - private long m_blocksize; - - public LocalBlockSourceEnumerator(System.Data.IDbConnection connection, string filetablename, string blocktablename, long blocksize) - { - m_connection = connection; - m_filetablename = filetablename; - m_blocktablename = blocktablename; - m_blocksize = blocksize; - this.Reset(); - } - - public ILocalBlockSource Current - { - get { return m_localsource; } - } - - public void Dispose() - { - if (m_reader != null) - { - m_reader.Dispose(); - m_reader = null; - } - - if (m_command != null) - { - m_command.Dispose(); - m_command = null; - } - } - - object System.Collections.IEnumerator.Current - { - get { throw new NotImplementedException(); } - } - - public bool MoveNext() - { - if (!m_localsource.HasMoreData) - return false; - - if (m_current != null) - { - while (m_localsource.HasMoreData && m_current == m_localsource.TargetPath) - m_localsource.HasMoreData = m_reader.Read(); - } - - m_current = m_localsource.HasMoreData ? m_localsource.TargetPath : null; - - return m_localsource.HasMoreData; - } - - public void Reset() - { - m_command = m_connection.CreateCommand(); - - m_command.CommandText = string.Format(@"SELECT DISTINCT ""A"".""TargetPath"", ""B"".""Hash"", (""B"".""Index"" * {2}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {2}), ""E"".""Size"" FROM ""{0}"" ""A"", ""{1}"" ""B"", ""Fileset"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0 ", m_filetablename, m_blocktablename, m_blocksize); - m_reader = m_command.ExecuteReader(); - m_current = null; - m_localsource = new LocalBlockSource(m_reader); - } - } - - private System.Data.IDbConnection m_connection; - private string m_filetablename; - private string m_blocktablename; - private long m_blocksize; - - public LocalBlockSourceEnumerable(System.Data.IDbConnection connection, string filetablename, string blocktablename, long blocksize) - { - m_connection = connection; - m_filetablename = filetablename; - m_blocktablename = blocktablename; - m_blocksize = blocksize; - } - - public IEnumerator GetEnumerator() - { - return new LocalBlockSourceEnumerator(m_connection, m_filetablename, m_blocktablename, m_blocksize); - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - } - - private class VolumePatchEnumerable : IEnumerable - { - private class VolumePatchEnumerator : IEnumerator - { - private class VolumePatch : IVolumePatch - { - private class PatchBlockEnumerable : IEnumerable - { - private class PatchBlockEnumerator : IEnumerator - { - private class PatchBlock : IPatchBlock - { - private System.Data.IDataReader m_reader; - - public PatchBlock(System.Data.IDataReader reader) - { - m_reader = reader; - } - - public long Offset - { - get - { - var v = m_reader.GetValue(1); - if (v == null || v == DBNull.Value) - return -1; - - return Convert.ToInt64(v); - } - } - - public long Size - { - get - { - var v = m_reader.GetValue(2); - if (v == null || v == DBNull.Value) - return -1; - - return Convert.ToInt64(v); - } - } - - public string Key - { - get - { - var v = m_reader.GetValue(3); - if (v == null || v == DBNull.Value) - return null; - - return v.ToString(); - } - } - } - - private System.Data.IDataReader m_reader; - private VolumePatch m_patch; - private PatchBlock m_block; - private string m_currentfile; - private bool m_first; - - public PatchBlockEnumerator(System.Data.IDataReader reader, VolumePatch patch) - { - m_reader = reader; - m_patch = patch; - m_currentfile = patch.Path; - this.Reset(); - } - - public IPatchBlock Current - { - get { return m_block; } - } - - public void Dispose() - { - } - - object System.Collections.IEnumerator.Current - { - get { return this.Current; } - } - - public bool MoveNext() - { - if (!m_patch.HasMoreData) - return false; - - if (m_first) - m_first = false; - else - m_patch.HasMoreData = m_reader.Read(); - - var more = m_patch.HasMoreData && m_currentfile == m_patch.Path; - return more; - } - - public void Reset() - { - m_block = new PatchBlock(m_reader); - m_first = true; - } - } - - private System.Data.IDataReader m_reader; - private VolumePatch m_patch; - - public PatchBlockEnumerable(System.Data.IDataReader reader, VolumePatch patch) - { - m_reader = reader; - m_patch = patch; - } - - public IEnumerator GetEnumerator() - { - return new PatchBlockEnumerator(m_reader, m_patch); - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - } - - private System.Data.IDataReader m_reader; - public bool HasMoreData { get; set; } - - public VolumePatch(System.Data.IDataReader reader) - { - m_reader = reader; - } - - public string Path - { - get - { - var v = m_reader.GetValue(0); - if (v == null || v == DBNull.Value) - return null; - - return v.ToString(); - } - } - - public IEnumerable Blocks - { - get { return new PatchBlockEnumerable(m_reader, this); } - } - } - - private System.Data.IDbConnection m_connection; - private System.Data.IDataReader m_reader; - private System.Data.IDbCommand m_command; - private string m_filetablename; - private string m_blocktablename; - private BlockVolumeReader m_curvolume; - private VolumePatch m_patch; - private string m_current = null; - private string m_tmptable = null; - private long m_blocksize; - - public VolumePatchEnumerator(System.Data.IDbConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) - { - m_connection = connection; - m_filetablename = filetablename; - m_blocktablename = blocktablename; - m_curvolume = curvolume; - m_blocksize = blocksize; - this.Reset(); - } - - public IVolumePatch Current - { - get { return m_patch; } - } - - public void Dispose() - { - if (m_reader != null) - { - m_reader.Dispose(); - m_reader = null; - } - - if (m_command != null) - { - m_command.Dispose(); - m_command = null; - } - - if (m_tmptable != null) - { - using (var c = m_connection.CreateCommand()) - { - c.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tmptable); - c.ExecuteNonQuery(); - m_tmptable = null; - } - } - } - - object System.Collections.IEnumerator.Current - { - get { return this.Current; } - } - - public bool MoveNext() - { - if (!m_patch.HasMoreData) - return false; - - if (m_current != null) - { - while (m_patch.HasMoreData && m_current == m_patch.Path) - m_patch.HasMoreData = m_reader.Read(); - } - - m_current = m_patch.Path; - - return m_patch.HasMoreData; - } - - public void Reset() - { - this.Dispose(); - using (var c = m_connection.CreateCommand()) - { - var tablename = "VolumeFiles-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - c.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )", tablename); - c.ExecuteNonQuery(); - m_tmptable = tablename; - - c.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Size"") VALUES (?,?)", m_tmptable); - c.AddParameters(2); - foreach (var s in m_curvolume.Blocks) - { - c.SetParameterValue(0, s.Key); - c.SetParameterValue(1, s.Value); - c.ExecuteNonQuery(); - } - } - - m_command = m_connection.CreateCommand(); - m_command.CommandText = string.Format(@"SELECT DISTINCT ""A"".""TargetPath"", (""B"".""Index"" * {3}), ""B"".""Size"", ""C"".""Hash"" FROM ""{0}"" A, ""{1}"" B, ""{2}"" C WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"" AND ""B"".""Restored"" = 0 ORDER BY ""A"".""TargetPath"", ""B"".""Index""", m_filetablename, m_blocktablename, m_tmptable, m_blocksize); - m_reader = m_command.ExecuteReader(); - m_patch = new VolumePatch(m_reader); - m_patch.HasMoreData = m_reader.Read(); - m_current = null; - } - } - - private System.Data.IDbConnection m_connection; - private string m_filetablename; - private string m_blocktablename; - private BlockVolumeReader m_curvolume; - private long m_blocksize; - - public VolumePatchEnumerable(System.Data.IDbConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) - { - m_connection = connection; - m_filetablename = filetablename; - m_blocktablename = blocktablename; - m_blocksize = blocksize; - m_curvolume = curvolume; - } - - public IEnumerator GetEnumerator() - { - return new VolumePatchEnumerator(m_connection, m_filetablename, m_blocktablename, m_blocksize, m_curvolume); - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - } - - private class FileToRestoreEnumerable : IEnumerable - { - private class FileToRestoreEnumerator : IEnumerator - { - private class FileToRestore : IFileToRestore - { - private System.Data.IDataReader m_reader; - public FileToRestore(System.Data.IDataReader reader) - { - m_reader = reader; - } - - public string Path - { - get - { - var v = m_reader.GetValue(0); - if (v == null || v == DBNull.Value) - return null; - return v.ToString(); - } - } - - public string Hash - { - get - { - var v = m_reader.GetValue(1); - if (v == null || v == DBNull.Value) - return null; - return v.ToString(); - } - } - } - - private System.Data.IDbConnection m_connection; - private System.Data.IDataReader m_reader; - private System.Data.IDbCommand m_command; - private string m_tablename; - private FileToRestore m_file; - - public FileToRestoreEnumerator(System.Data.IDbConnection connection, string tablename) - { - m_connection = connection; - m_tablename = tablename; - this.Reset(); - } - - public IFileToRestore Current - { - get { return m_file; } - } - - public void Dispose() - { - if (m_reader != null) - { - m_reader.Dispose(); - m_reader = null; - } - - if (m_command != null) - { - m_command.Dispose(); - m_command = null; - } - } - - object System.Collections.IEnumerator.Current - { - get { return this.Current; } - } - - public bool MoveNext() - { - return m_reader.Read(); - } - - public void Reset() - { - m_command = m_connection.CreateCommand(); - m_command.CommandText = string.Format(@"SELECT ""{0}"".""TargetPath"", ""Blockset"".""FullHash"" FROM ""{0}"",""Blockset"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" ", m_tablename); - m_reader = m_command.ExecuteReader(); - m_file = new FileToRestore(m_reader); - } - } - - private System.Data.IDbConnection m_connection; - private string m_tablename; - - public FileToRestoreEnumerable(System.Data.IDbConnection connection, string tablename) - { - m_connection = connection; - m_tablename = tablename; - } - - public IEnumerator GetEnumerator() - { - return new FileToRestoreEnumerator(m_connection, m_tablename); - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - } - - private class FolderListEnumerable : IEnumerable - { - private class FolderListEnumerator : IEnumerator - { - private System.Data.IDbConnection m_connection; - private System.Data.IDbCommand m_command; - private System.Data.IDataReader m_reader; - private string m_tablename; - - public FolderListEnumerator(System.Data.IDbConnection connection, string tablename) - { - m_connection = connection; - m_tablename = tablename; - this.Reset(); - } - - - public string Current { get; private set; } - - public void Dispose() - { - if (m_reader != null) - try { m_reader.Dispose(); } - finally { m_reader = null; } - - if (m_command != null) - try { m_command.Dispose(); } - finally { m_command = null; } - } - - object System.Collections.IEnumerator.Current { get { return this.Current; } } - - public bool MoveNext() - { - if (!m_reader.Read()) - return false; - - var c = m_reader.GetValue(0); - if (c == null || c == DBNull.Value) - Current = null; - else - Current = c.ToString(); - - return true; - } - - public void Reset() - { - this.Dispose(); - Current = null; - m_command = m_connection.CreateCommand(); - m_command.CommandText = string.Format(@"SELECT ""TargetPath"" FROM ""{0}"" WHERE ""BlocksetID"" == ?", m_tablename); - m_command.AddParameter(FOLDER_BLOCKSET_ID); - m_reader = m_command.ExecuteReader(); - } - } - - private System.Data.IDbConnection m_connection; - private string m_tablename; - public FolderListEnumerable(System.Data.IDbConnection connection, string tablename) - { - m_connection = connection; - m_tablename = tablename; - } - - public IEnumerator GetEnumerator() { return new FolderListEnumerator(m_connection, m_tablename); } - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); } - } - } - - public partial class LocalBlocklistUpdateDatabase - { - private class BlocklistsEnumerable : IEnumerable - { - private class BlocklistEnumerator : IEnumerator - { - private System.Data.IDbConnection m_connection; - private string m_temptable; - private long m_volumeid; - private System.Data.IDataReader m_reader; - private System.Data.IDbCommand m_command; - private string m_current; - - public BlocklistEnumerator(System.Data.IDbConnection connection, string temptable, long volumeid) - { - m_connection = connection; - m_temptable = temptable; - m_volumeid = volumeid; - this.Reset(); - } - - public string Current { get { return m_current; } } - - public void Dispose() - { - if (m_reader != null) - try { m_reader.Dispose(); } - finally { m_reader = null; } - - if (m_command != null) - try { m_command.Dispose(); } - finally { m_command = null; } - } - - object System.Collections.IEnumerator.Current { get { return this.Current; } } - - public bool MoveNext() - { - if (!m_reader.Read()) - { - m_current = null; - return false; - } - - var c = m_reader.GetValue(0); - if (c == null || c == DBNull.Value) - m_current = null; - else - m_current = c.ToString(); - - return true; - } - - public void Reset() - { - this.Dispose(); - m_current = null; - m_command = m_connection.CreateCommand(); - m_command.CommandText = string.Format(@"SELECT ""Hash"" FROM ""{0}"" WHERE ""VolumeID"" = ""{1}"" AND ""Restored"" = 0 ", m_temptable, m_volumeid); - m_reader = m_command.ExecuteReader(); - } - } - - private System.Data.IDbConnection m_connection; - private string m_temptable; - private long m_volumeid; - - public BlocklistsEnumerable(System.Data.IDbConnection connection, string temptable, long volumeid) - { - m_connection = connection; - m_temptable = temptable; - m_volumeid = volumeid; - } - - public IEnumerator GetEnumerator() { return new BlocklistEnumerator(m_connection, m_temptable, m_volumeid); } - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); } - } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Duplicati.Library.Main.ForestHash.Volumes; + +namespace Duplicati.Library.Main.ForestHash.Database +{ + public partial class LocalRestoreDatabase + { + private class ExistingFileEnumerable : IEnumerable + { + private class ExistingFileEnumerator : IEnumerator, IDisposable + { + private class ExistingFile : IExistingFile + { + private class ExistingFileBlockEnumerable : IEnumerable + { + private class ExistingFileBlockEnumerator : IEnumerator + { + private class ExistingFileBlock : IExistingFileBlock + { + private System.Data.IDataReader m_reader; + + public ExistingFileBlock(System.Data.IDataReader reader) + { + m_reader = reader; + } + + public string Hash + { + get + { + var v = m_reader.GetValue(2); + if (v == null || v == DBNull.Value) + return null; + return v.ToString(); + } + } + + public long Index + { + get + { + var v = m_reader.GetValue(3); + if (v == null || v == DBNull.Value) + return -1; + return Convert.ToInt64(v); + } + } + + public long Size + { + get + { + var v = m_reader.GetValue(4); + if (v == null || v == DBNull.Value) + return -1; + return Convert.ToInt64(v); + } + } + } + + private System.Data.IDataReader m_reader; + private ExistingFile m_file; + private ExistingFileBlock m_block; + private string m_current; + + public ExistingFileBlockEnumerator(System.Data.IDataReader reader, ExistingFile file) + { + m_reader = reader; + m_file = file; + this.Reset(); + } + + public IExistingFileBlock Current + { + get + { + return m_block; + } + } + + public void Dispose() + { + } + + object System.Collections.IEnumerator.Current + { + get { return this.Current; } + } + + public bool MoveNext() + { + if (!m_file.HasMoreData) + return false; + + if (m_current == null) + m_current = m_file.TargetPath; + else + m_file.HasMoreData = m_reader.Read(); + + if (m_file.HasMoreData && m_current != m_file.TargetPath) + return false; + + return m_file.HasMoreData; + } + + public void Reset() + { + m_block = new ExistingFileBlock(m_reader); + m_current = null; + } + } + + private System.Data.IDataReader m_reader; + private ExistingFile m_file; + + public ExistingFileBlockEnumerable(System.Data.IDataReader reader, ExistingFile file) + { + m_reader = reader; + m_file = file; + } + + public IEnumerator GetEnumerator() + { + return new ExistingFileBlockEnumerator(m_reader, m_file); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + } + + private System.Data.IDataReader m_reader; + public bool HasMoreData { get; set; } + + public ExistingFile(System.Data.IDataReader reader) + { + m_reader = reader; + HasMoreData = m_reader.Read(); + } + + public string TargetPath + { + get + { + var v = m_reader.GetValue(0); + if (v == null || v == DBNull.Value) + return null; + + return v.ToString(); + } + } + + public long Length + { + get + { + var v = m_reader.GetValue(1); + if (v == null || v == DBNull.Value) + return -1; + + return Convert.ToInt64(v); + } + } + + public IEnumerable Blocks + { + get { return new ExistingFileBlockEnumerable(m_reader, this); } + } + } + + private System.Data.IDbCommand m_command; + private System.Data.IDataReader m_reader; + private System.Data.IDbConnection m_connection; + private string m_tablename; + private ExistingFile m_file; + private string m_current; + + public ExistingFileEnumerator(System.Data.IDbConnection con, string tablename) + { + m_connection = con; + m_tablename = tablename; + this.Reset(); + } + + public void Dispose() + { + if (m_reader != null) + { + m_reader.Dispose(); + m_reader = null; + } + + if (m_command != null) + { + m_command.Dispose(); + m_command = null; + } + } + + public IExistingFile Current + { + get { return m_file; } + } + + object System.Collections.IEnumerator.Current + { + get { return m_file; } + } + + public bool MoveNext() + { + if (m_current != null) + { + while (m_file.HasMoreData && m_file.TargetPath == m_current) + m_file.HasMoreData = m_reader.Read(); + } + + m_current = m_file.HasMoreData ? m_file.TargetPath : null; + + return m_file.HasMoreData; + } + + public void Reset() + { + m_file = null; + m_command = m_connection.CreateCommand(); + m_command.CommandText = string.Format(@"SELECT ""{0}"".""TargetPath"", ""Blockset"".""Length"", ""Block"".""Hash"", ""BlocksetEntry"".""Index"", ""Block"".""Size"" FROM ""{0}"", ""Blockset"", ""BlocksetEntry"", ""Block"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""{0}"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" ORDER BY ""{0}"".""TargetPath"", ""BlocksetEntry"".""Index""", m_tablename); + m_reader = m_command.ExecuteReader(); + m_file = new ExistingFile(m_reader); + m_current = null; + + } + } + + private System.Data.IDbConnection m_connection; + private string m_tablename; + + public ExistingFileEnumerable(System.Data.IDbConnection connection, string tablename) + { + m_connection = connection; + m_tablename = tablename; + } + + public IEnumerator GetEnumerator() + { + return new ExistingFileEnumerator(m_connection, m_tablename); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + } + + + private class LocalBlockSourceEnumerable : IEnumerable + { + private class LocalBlockSourceEnumerator : IEnumerator + { + private class LocalBlockSource : ILocalBlockSource + { + private class BlockDescriptorEnumerable : IEnumerable + { + private class BlockDescriptorEnumerator : IEnumerator + { + private class BlockDescriptor : IBlockDescriptor + { + private class BlockSourceEnumerable : IEnumerable + { + private class BlockSourceEnumerator : IEnumerator + { + private class BlockSource : IBlockSource + { + private System.Data.IDataReader m_reader; + public BlockSource(System.Data.IDataReader reader) + { + m_reader = reader; + } + + public string Path + { + get + { + var v = m_reader.GetValue(5); + if (v == null || v == DBNull.Value) + return null; + return v.ToString(); + } + } + + public long Offset + { + get + { + var v = m_reader.GetValue(6); + if (v == null || v == DBNull.Value) + return -1; + return Convert.ToInt64(v); + } + } + + public long Size + { + get + { + var v = m_reader.GetValue(7); + if (v == null || v == DBNull.Value) + return -1; + return Convert.ToInt64(v); + } + } + } + + private System.Data.IDataReader m_reader; + private LocalBlockSource m_localsource; + private BlockSource m_source; + private BlockDescriptor m_descriptor; + private string m_currenthash; + private string m_currentfile; + + public BlockSourceEnumerator(System.Data.IDataReader reader, LocalBlockSource localsource, BlockDescriptor descriptor) + { + m_reader = reader; + m_localsource = localsource; + m_descriptor = descriptor; + this.Reset(); + } + + public IBlockSource Current + { + get { return m_source; } + } + + public void Dispose() + { + } + + object System.Collections.IEnumerator.Current + { + get { return this.Current; } + } + + public bool MoveNext() + { + if (!m_localsource.HasMoreData) + return false; + + if (m_currenthash != null) + m_localsource.HasMoreData = m_reader.Read(); + + if (!m_localsource.HasMoreData) + return false; + + if (m_currenthash != null && m_currentfile != null) + if (m_currentfile != m_localsource.TargetPath || m_currenthash != m_descriptor.Hash) + return false; + + m_currenthash = m_descriptor.Hash; + m_currentfile = m_localsource.TargetPath; + + return m_localsource.HasMoreData; + } + + public void Reset() + { + m_source = new BlockSource(m_reader); + m_currenthash = null; + m_currentfile = null; + } + } + + private System.Data.IDataReader m_reader; + private LocalBlockSource m_localsource; + private BlockDescriptor m_descriptor; + + public BlockSourceEnumerable(System.Data.IDataReader reader, LocalBlockSource localsource, BlockDescriptor descriptor) + { + m_reader = reader; + m_localsource = localsource; + m_descriptor = descriptor; + } + + public IEnumerator GetEnumerator() + { + return new BlockSourceEnumerator(m_reader, m_localsource, m_descriptor); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + } + + private System.Data.IDataReader m_reader; + private LocalBlockSource m_localsource; + + public BlockDescriptor(System.Data.IDataReader reader, LocalBlockSource localsource) + { + m_reader = reader; + m_localsource = localsource; + } + + public string Hash + { + get + { + var v = m_reader.GetValue(1); + if (v == null || v == DBNull.Value) + return null; + return v.ToString(); + } + } + + + public long Offset + { + get + { + var v = m_reader.GetValue(2); + if (v == null || v == DBNull.Value) + return -1; + return Convert.ToInt64(v); + } + } + + public long Index + { + get + { + var v = m_reader.GetValue(3); + if (v == null || v == DBNull.Value) + return -1; + return Convert.ToInt64(v); + } + } + + public long Size + { + get + { + var v = m_reader.GetValue(4); + if (v == null || v == DBNull.Value) + return -1; + return Convert.ToInt64(v); + } + } + + public IEnumerable Blocksources + { + get { return new BlockSourceEnumerable(m_reader, m_localsource, this); } + } + } + + private System.Data.IDataReader m_reader; + private LocalBlockSource m_localsource; + private BlockDescriptor m_descriptor; + private string m_currenthash; + private string m_currentfile; + + + public BlockDescriptorEnumerator(System.Data.IDataReader reader, LocalBlockSource localsource) + { + m_reader = reader; + m_localsource = localsource; + this.Reset(); + } + + public IBlockDescriptor Current + { + get { return m_descriptor; } + } + + public void Dispose() + { + } + + object System.Collections.IEnumerator.Current + { + get { return this.Current; } + } + + public bool MoveNext() + { + if (!m_localsource.HasMoreData) + return false; + + if (m_currenthash != null) + { + while (m_localsource.HasMoreData && m_currenthash == m_descriptor.Hash && m_currentfile == m_localsource.TargetPath) + m_localsource.HasMoreData = m_reader.Read(); + + if (m_currentfile != m_localsource.TargetPath) + return false; + } + + m_currenthash = m_localsource.HasMoreData ? m_descriptor.Hash : null; + m_currentfile = m_localsource.HasMoreData ? m_localsource.TargetPath : null; + + return m_localsource.HasMoreData && m_currenthash == m_descriptor.Hash; + + } + + public void Reset() + { + m_descriptor = new BlockDescriptor(m_reader, m_localsource); + m_currenthash = null; + m_currentfile = null; + } + } + + private System.Data.IDataReader m_reader; + private LocalBlockSource m_localsource; + public BlockDescriptorEnumerable(System.Data.IDataReader reader, LocalBlockSource localsource) + { + m_reader = reader; + m_localsource = localsource; + } + + public IEnumerator GetEnumerator() + { + return new BlockDescriptorEnumerator(m_reader, m_localsource); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + } + + private System.Data.IDataReader m_reader; + public bool HasMoreData { get; set; } + + public LocalBlockSource(System.Data.IDataReader reader) + { + m_reader = reader; + HasMoreData = reader.Read(); + } + + public string TargetPath + { + get + { + var v = m_reader.GetValue(0); + if (v == null || v == DBNull.Value) + return null; + + return v.ToString(); + } + } + + public IEnumerable Blocks + { + get { return new BlockDescriptorEnumerable(m_reader, this); } + } + } + + private System.Data.IDbCommand m_command; + private System.Data.IDataReader m_reader; + private System.Data.IDbConnection m_connection; + private LocalBlockSource m_localsource; + private string m_filetablename; + private string m_blocktablename; + private string m_current; + private long m_blocksize; + + public LocalBlockSourceEnumerator(System.Data.IDbConnection connection, string filetablename, string blocktablename, long blocksize) + { + m_connection = connection; + m_filetablename = filetablename; + m_blocktablename = blocktablename; + m_blocksize = blocksize; + this.Reset(); + } + + public ILocalBlockSource Current + { + get { return m_localsource; } + } + + public void Dispose() + { + if (m_reader != null) + { + m_reader.Dispose(); + m_reader = null; + } + + if (m_command != null) + { + m_command.Dispose(); + m_command = null; + } + } + + object System.Collections.IEnumerator.Current + { + get { throw new NotImplementedException(); } + } + + public bool MoveNext() + { + if (!m_localsource.HasMoreData) + return false; + + if (m_current != null) + { + while (m_localsource.HasMoreData && m_current == m_localsource.TargetPath) + m_localsource.HasMoreData = m_reader.Read(); + } + + m_current = m_localsource.HasMoreData ? m_localsource.TargetPath : null; + + return m_localsource.HasMoreData; + } + + public void Reset() + { + m_command = m_connection.CreateCommand(); + + m_command.CommandText = string.Format(@"SELECT DISTINCT ""A"".""TargetPath"", ""B"".""Hash"", (""B"".""Index"" * {2}), ""B"".""Index"", ""B"".""Size"", ""C"".""Path"", (""D"".""Index"" * {2}), ""E"".""Size"" FROM ""{0}"" ""A"", ""{1}"" ""B"", ""FileEntry"" ""C"", ""BlocksetEntry"" ""D"", ""Block"" E WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""C"".""BlocksetID"" = ""D"".""BlocksetID"" AND ""D"".""BlockID"" = ""E"".""ID"" AND ""B"".""Hash"" = ""E"".""Hash"" AND ""B"".""Size"" = ""E"".""Size"" AND ""B"".""Restored"" = 0 ", m_filetablename, m_blocktablename, m_blocksize); + m_reader = m_command.ExecuteReader(); + m_current = null; + m_localsource = new LocalBlockSource(m_reader); + } + } + + private System.Data.IDbConnection m_connection; + private string m_filetablename; + private string m_blocktablename; + private long m_blocksize; + + public LocalBlockSourceEnumerable(System.Data.IDbConnection connection, string filetablename, string blocktablename, long blocksize) + { + m_connection = connection; + m_filetablename = filetablename; + m_blocktablename = blocktablename; + m_blocksize = blocksize; + } + + public IEnumerator GetEnumerator() + { + return new LocalBlockSourceEnumerator(m_connection, m_filetablename, m_blocktablename, m_blocksize); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + } + + private class VolumePatchEnumerable : IEnumerable + { + private class VolumePatchEnumerator : IEnumerator + { + private class VolumePatch : IVolumePatch + { + private class PatchBlockEnumerable : IEnumerable + { + private class PatchBlockEnumerator : IEnumerator + { + private class PatchBlock : IPatchBlock + { + private System.Data.IDataReader m_reader; + + public PatchBlock(System.Data.IDataReader reader) + { + m_reader = reader; + } + + public long Offset + { + get + { + var v = m_reader.GetValue(1); + if (v == null || v == DBNull.Value) + return -1; + + return Convert.ToInt64(v); + } + } + + public long Size + { + get + { + var v = m_reader.GetValue(2); + if (v == null || v == DBNull.Value) + return -1; + + return Convert.ToInt64(v); + } + } + + public string Key + { + get + { + var v = m_reader.GetValue(3); + if (v == null || v == DBNull.Value) + return null; + + return v.ToString(); + } + } + } + + private System.Data.IDataReader m_reader; + private VolumePatch m_patch; + private PatchBlock m_block; + private string m_currentfile; + private bool m_first; + + public PatchBlockEnumerator(System.Data.IDataReader reader, VolumePatch patch) + { + m_reader = reader; + m_patch = patch; + m_currentfile = patch.Path; + this.Reset(); + } + + public IPatchBlock Current + { + get { return m_block; } + } + + public void Dispose() + { + } + + object System.Collections.IEnumerator.Current + { + get { return this.Current; } + } + + public bool MoveNext() + { + if (!m_patch.HasMoreData) + return false; + + if (m_first) + m_first = false; + else + m_patch.HasMoreData = m_reader.Read(); + + var more = m_patch.HasMoreData && m_currentfile == m_patch.Path; + return more; + } + + public void Reset() + { + m_block = new PatchBlock(m_reader); + m_first = true; + } + } + + private System.Data.IDataReader m_reader; + private VolumePatch m_patch; + + public PatchBlockEnumerable(System.Data.IDataReader reader, VolumePatch patch) + { + m_reader = reader; + m_patch = patch; + } + + public IEnumerator GetEnumerator() + { + return new PatchBlockEnumerator(m_reader, m_patch); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + } + + private System.Data.IDataReader m_reader; + public bool HasMoreData { get; set; } + + public VolumePatch(System.Data.IDataReader reader) + { + m_reader = reader; + } + + public string Path + { + get + { + var v = m_reader.GetValue(0); + if (v == null || v == DBNull.Value) + return null; + + return v.ToString(); + } + } + + public IEnumerable Blocks + { + get { return new PatchBlockEnumerable(m_reader, this); } + } + } + + private System.Data.IDbConnection m_connection; + private System.Data.IDataReader m_reader; + private System.Data.IDbCommand m_command; + private string m_filetablename; + private string m_blocktablename; + private BlockVolumeReader m_curvolume; + private VolumePatch m_patch; + private string m_current = null; + private string m_tmptable = null; + private long m_blocksize; + + public VolumePatchEnumerator(System.Data.IDbConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) + { + m_connection = connection; + m_filetablename = filetablename; + m_blocktablename = blocktablename; + m_curvolume = curvolume; + m_blocksize = blocksize; + this.Reset(); + } + + public IVolumePatch Current + { + get { return m_patch; } + } + + public void Dispose() + { + if (m_reader != null) + { + m_reader.Dispose(); + m_reader = null; + } + + if (m_command != null) + { + m_command.Dispose(); + m_command = null; + } + + if (m_tmptable != null) + { + using (var c = m_connection.CreateCommand()) + { + c.CommandText = string.Format(@"DROP TABLE ""{0}""", m_tmptable); + c.ExecuteNonQuery(); + m_tmptable = null; + } + } + } + + object System.Collections.IEnumerator.Current + { + get { return this.Current; } + } + + public bool MoveNext() + { + if (!m_patch.HasMoreData) + return false; + + if (m_current != null) + { + while (m_patch.HasMoreData && m_current == m_patch.Path) + m_patch.HasMoreData = m_reader.Read(); + } + + m_current = m_patch.Path; + + return m_patch.HasMoreData; + } + + public void Reset() + { + this.Dispose(); + using (var c = m_connection.CreateCommand()) + { + var tablename = "VolumeFiles-" + Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); + c.CommandText = string.Format(@"CREATE TEMPORARY TABLE ""{0}"" ( ""Hash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL )", tablename); + c.ExecuteNonQuery(); + m_tmptable = tablename; + + c.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Hash"", ""Size"") VALUES (?,?)", m_tmptable); + c.AddParameters(2); + foreach (var s in m_curvolume.Blocks) + { + c.SetParameterValue(0, s.Key); + c.SetParameterValue(1, s.Value); + c.ExecuteNonQuery(); + } + } + + m_command = m_connection.CreateCommand(); + m_command.CommandText = string.Format(@"SELECT DISTINCT ""A"".""TargetPath"", (""B"".""Index"" * {3}), ""B"".""Size"", ""C"".""Hash"" FROM ""{0}"" A, ""{1}"" B, ""{2}"" C WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""Hash"" = ""C"".""Hash"" AND ""B"".""Size"" = ""C"".""Size"" AND ""B"".""Restored"" = 0 ORDER BY ""A"".""TargetPath"", ""B"".""Index""", m_filetablename, m_blocktablename, m_tmptable, m_blocksize); + m_reader = m_command.ExecuteReader(); + m_patch = new VolumePatch(m_reader); + m_patch.HasMoreData = m_reader.Read(); + m_current = null; + } + } + + private System.Data.IDbConnection m_connection; + private string m_filetablename; + private string m_blocktablename; + private BlockVolumeReader m_curvolume; + private long m_blocksize; + + public VolumePatchEnumerable(System.Data.IDbConnection connection, string filetablename, string blocktablename, long blocksize, BlockVolumeReader curvolume) + { + m_connection = connection; + m_filetablename = filetablename; + m_blocktablename = blocktablename; + m_blocksize = blocksize; + m_curvolume = curvolume; + } + + public IEnumerator GetEnumerator() + { + return new VolumePatchEnumerator(m_connection, m_filetablename, m_blocktablename, m_blocksize, m_curvolume); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + } + + private class FileToRestoreEnumerable : IEnumerable + { + private class FileToRestoreEnumerator : IEnumerator + { + private class FileToRestore : IFileToRestore + { + private System.Data.IDataReader m_reader; + public FileToRestore(System.Data.IDataReader reader) + { + m_reader = reader; + } + + public string Path + { + get + { + var v = m_reader.GetValue(0); + if (v == null || v == DBNull.Value) + return null; + return v.ToString(); + } + } + + public string Hash + { + get + { + var v = m_reader.GetValue(1); + if (v == null || v == DBNull.Value) + return null; + return v.ToString(); + } + } + } + + private System.Data.IDbConnection m_connection; + private System.Data.IDataReader m_reader; + private System.Data.IDbCommand m_command; + private string m_tablename; + private FileToRestore m_file; + + public FileToRestoreEnumerator(System.Data.IDbConnection connection, string tablename) + { + m_connection = connection; + m_tablename = tablename; + this.Reset(); + } + + public IFileToRestore Current + { + get { return m_file; } + } + + public void Dispose() + { + if (m_reader != null) + { + m_reader.Dispose(); + m_reader = null; + } + + if (m_command != null) + { + m_command.Dispose(); + m_command = null; + } + } + + object System.Collections.IEnumerator.Current + { + get { return this.Current; } + } + + public bool MoveNext() + { + return m_reader.Read(); + } + + public void Reset() + { + m_command = m_connection.CreateCommand(); + m_command.CommandText = string.Format(@"SELECT ""{0}"".""TargetPath"", ""Blockset"".""FullHash"" FROM ""{0}"",""Blockset"" WHERE ""{0}"".""BlocksetID"" = ""Blockset"".""ID"" ", m_tablename); + m_reader = m_command.ExecuteReader(); + m_file = new FileToRestore(m_reader); + } + } + + private System.Data.IDbConnection m_connection; + private string m_tablename; + + public FileToRestoreEnumerable(System.Data.IDbConnection connection, string tablename) + { + m_connection = connection; + m_tablename = tablename; + } + + public IEnumerator GetEnumerator() + { + return new FileToRestoreEnumerator(m_connection, m_tablename); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + } + + private class FolderListEnumerable : IEnumerable + { + private class FolderListEnumerator : IEnumerator + { + private System.Data.IDbConnection m_connection; + private System.Data.IDbCommand m_command; + private System.Data.IDataReader m_reader; + private string m_tablename; + + public FolderListEnumerator(System.Data.IDbConnection connection, string tablename) + { + m_connection = connection; + m_tablename = tablename; + this.Reset(); + } + + + public string Current { get; private set; } + + public void Dispose() + { + if (m_reader != null) + try { m_reader.Dispose(); } + finally { m_reader = null; } + + if (m_command != null) + try { m_command.Dispose(); } + finally { m_command = null; } + } + + object System.Collections.IEnumerator.Current { get { return this.Current; } } + + public bool MoveNext() + { + if (!m_reader.Read()) + return false; + + var c = m_reader.GetValue(0); + if (c == null || c == DBNull.Value) + Current = null; + else + Current = c.ToString(); + + return true; + } + + public void Reset() + { + this.Dispose(); + Current = null; + m_command = m_connection.CreateCommand(); + m_command.CommandText = string.Format(@"SELECT ""TargetPath"" FROM ""{0}"" WHERE ""BlocksetID"" == ?", m_tablename); + m_command.AddParameter(FOLDER_BLOCKSET_ID); + m_reader = m_command.ExecuteReader(); + } + } + + private System.Data.IDbConnection m_connection; + private string m_tablename; + public FolderListEnumerable(System.Data.IDbConnection connection, string tablename) + { + m_connection = connection; + m_tablename = tablename; + } + + public IEnumerator GetEnumerator() { return new FolderListEnumerator(m_connection, m_tablename); } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); } + } + } + + public partial class LocalBlocklistUpdateDatabase + { + private class BlocklistsEnumerable : IEnumerable + { + private class BlocklistEnumerator : IEnumerator + { + private System.Data.IDbConnection m_connection; + private string m_temptable; + private long m_volumeid; + private System.Data.IDataReader m_reader; + private System.Data.IDbCommand m_command; + private string m_current; + + public BlocklistEnumerator(System.Data.IDbConnection connection, string temptable, long volumeid) + { + m_connection = connection; + m_temptable = temptable; + m_volumeid = volumeid; + this.Reset(); + } + + public string Current { get { return m_current; } } + + public void Dispose() + { + if (m_reader != null) + try { m_reader.Dispose(); } + finally { m_reader = null; } + + if (m_command != null) + try { m_command.Dispose(); } + finally { m_command = null; } + } + + object System.Collections.IEnumerator.Current { get { return this.Current; } } + + public bool MoveNext() + { + if (!m_reader.Read()) + { + m_current = null; + return false; + } + + var c = m_reader.GetValue(0); + if (c == null || c == DBNull.Value) + m_current = null; + else + m_current = c.ToString(); + + return true; + } + + public void Reset() + { + this.Dispose(); + m_current = null; + m_command = m_connection.CreateCommand(); + m_command.CommandText = string.Format(@"SELECT ""Hash"" FROM ""{0}"" WHERE ""VolumeID"" = ""{1}"" AND ""Restored"" = 0 ", m_temptable, m_volumeid); + m_reader = m_command.ExecuteReader(); + } + } + + private System.Data.IDbConnection m_connection; + private string m_temptable; + private long m_volumeid; + + public BlocklistsEnumerable(System.Data.IDbConnection connection, string temptable, long volumeid) + { + m_connection = connection; + m_temptable = temptable; + m_volumeid = volumeid; + } + + public IEnumerator GetEnumerator() { return new BlocklistEnumerator(m_connection, m_temptable, m_volumeid); } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); } + } + } +} diff --git a/Duplicati/Library/Main/ForestHash/Operation/ListFilesHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/ListFilesHandler.cs index 274154a25..744ee0987 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/ListFilesHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/ListFilesHandler.cs @@ -27,9 +27,9 @@ namespace Duplicati.Library.Main.ForestHash.Operation if (System.IO.File.Exists(m_options.Fhdbpath)) using (var db = new Database.LocalDatabase(m_options.Fhdbpath, "ListFiles")) { - var filesetid = db.GetFilesetID(m_options.RestoreTime); + var operationId = db.GetFilesetOperationID(m_options.RestoreTime); return - (from n in db.GetFiles(filesetid) + (from n in db.GetFiles(operationId) select n.Path).ToList(); } diff --git a/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs index b7f66faea..e3349c81e 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs @@ -73,16 +73,23 @@ namespace Duplicati.Library.Main.ForestHash.Operation using (var backupdb = new LocalBackupDatabase(restoredb, m_options)) { foreach (var fl in remotefiles) - backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded); - - var shadowfiles = - from n in remotefiles - where n.FileType == RemoteVolumeType.Shadow - select new RemoteVolume(n.File) as IRemoteVolume; + { + var id = backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded); + + //For filelists, we associate them with a backup operation. + // This enables the delete-older-than and similar operations to work with the restored database + if (fl.FileType == RemoteVolumeType.Files) + restoredb.CreateBackupOperation(id, fl.Time); + } //We grab all shadow files, and update the block table using (var tr = restoredb.BeginTransaction()) { + var shadowfiles = + from n in remotefiles + where n.FileType == RemoteVolumeType.Shadow + select new RemoteVolume(n.File) as IRemoteVolume; + foreach (var sf in new AsyncDownloader(shadowfiles.ToList(), backend)) { if (sf.Key.Hash != null && sf.Key.Size > 0)