diff --git a/Duplicati/Library/Main/Database/Database schema/9. Refactor Paths.sql b/Duplicati/Library/Main/Database/Database schema/9. Refactor Paths.sql new file mode 100644 index 000000000..8dc0c1fa5 --- /dev/null +++ b/Duplicati/Library/Main/Database/Database schema/9. Refactor Paths.sql @@ -0,0 +1,63 @@ +/* +The PathPrefix contains a set +of path prefixes, used to minimize +the space required to store paths +*/ +CREATE TABLE "PathPrefix" ( + "ID" INTEGER PRIMARY KEY, + "Prefix" TEXT NOT NULL +); +CREATE UNIQUE INDEX "PathPrefixPrefix" ON "PathPrefix" ("Prefix"); + + +/* +The FileLookup table contains an ID +for each path and each version +of the data and metadata +*/ +CREATE TABLE "FileLookup" ( + "ID" INTEGER PRIMARY KEY, + "PrefixID" INTEGER NOT NULL, + "Path" TEXT NOT NULL, + "BlocksetID" INTEGER NOT NULL, + "MetadataID" INTEGER NOT NULL +); + +/* Fast path based lookup, single properties are auto-indexed */ +CREATE UNIQUE INDEX "FileLookupPath" ON "FileLookup" ("PrefixID", "Path", "BlocksetID", "MetadataID"); + + +/* Build the prefix table */ +INSERT INTO "PathPrefix" ("Prefix") +SELECT DISTINCT + CASE SUBSTR("Path", LENGTH("Path")) WHEN '/' THEN + rtrim(SUBSTR("Path", 1, LENGTH("Path")-1), replace(replace(SUBSTR("Path", 1, LENGTH("Path")-1), "\", "/"), '/', '')) + ELSE + rtrim("Path", replace(replace("Path", "\", "/"), '/', '')) + END AS "Prefix" +FROM "File"; + +/* Build the path lookup table */ +INSERT INTO "FileLookup" ("Path", "PrefixID", "BlocksetID", "MetadataID") + +SELECT + SUBSTR("Path", LENGTH("ParentFolder") + 1) AS "Path", + "ID" AS "PrefixID", + "BlocksetID", + "MetadataID" +FROM + +(SELECT "Path", "BlocksetID", "MetadataID", + CASE SUBSTR("Path", LENGTH("Path")) WHEN '/' THEN + rtrim(SUBSTR("Path", 1, LENGTH("Path")-1), replace(replace(SUBSTR("Path", 1, LENGTH("Path")-1), "\", "/"), '/', '')) + ELSE + rtrim("Path", replace(replace("Path", "\", "/"), '/', '')) + END AS "ParentFolder" +FROM "File") "A" INNER JOIN "PathPrefix" "B" ON "A"."ParentFolder" = "B"."Prefix"; + +DROP TABLE "File"; + +CREATE VIEW "File" AS SELECT "A"."ID" AS "ID", "B"."Prefix" || "A"."Path" AS "Path", "A"."BlocksetID" AS "BlocksetID", "A"."MetadataID" AS "MetadataID" FROM "FileLookup" "A", "PathPrefix" "B" WHERE "A"."PrefixID" = "B"."ID"; + + +UPDATE "Version" SET "Version" = 9; \ No newline at end of file diff --git a/Duplicati/Library/Main/Database/Database schema/Schema.sql b/Duplicati/Library/Main/Database/Database schema/Schema.sql index 74109b6c0..533ad8918 100644 --- a/Duplicati/Library/Main/Database/Database schema/Schema.sql +++ b/Duplicati/Library/Main/Database/Database schema/Schema.sql @@ -82,21 +82,39 @@ CREATE TABLE "FilesetEntry" ( CREATE INDEX "FilesetentryFileIdIndex" on "FilesetEntry" ("FileID"); +/* +The PathPrefix contains a set +of path prefixes, used to minimize +the space required to store paths +*/ +CREATE TABLE "PathPrefix" ( + "ID" INTEGER PRIMARY KEY, + "Prefix" TEXT NOT NULL +); +CREATE UNIQUE INDEX "PathPrefixPrefix" ON "PathPrefix" ("Prefix"); /* -The FileEntry contains an ID +The FileLookup table contains an ID for each path and each version of the data and metadata */ -CREATE TABLE "File" ( - "ID" INTEGER PRIMARY KEY, - "Path" TEXT NOT NULL, - "BlocksetID" INTEGER NOT NULL, - "MetadataID" INTEGER NOT NULL +CREATE TABLE "FileLookup" ( + "ID" INTEGER PRIMARY KEY, + "PrefixID" INTEGER NOT NULL, + "Path" TEXT NOT NULL, + "BlocksetID" INTEGER NOT NULL, + "MetadataID" INTEGER NOT NULL ); -/* Fast path based lookup */ -CREATE UNIQUE INDEX "FilePath" ON "File" ("Path", "BlocksetID", "MetadataID"); +/* Fast path based lookup, single properties are auto-indexed */ +CREATE UNIQUE INDEX "FileLookupPath" ON "FileLookup" ("PrefixID", "Path", "BlocksetID", "MetadataID"); + +/* +The File view contains an ID +for each path and each version +of the data and metadata +*/ +CREATE VIEW "File" AS SELECT "A"."ID" AS "ID", "B"."Prefix" || "A"."Path" AS "Path", "A"."BlocksetID" AS "BlocksetID", "A"."MetadataID" AS "MetadataID" FROM "FileLookup" "A", "PathPrefix" "B" WHERE "A"."PrefixID" = "B"."ID"; /* The blocklist hashes are hashes of @@ -244,4 +262,4 @@ CREATE TABLE "Configuration" ( "Value" TEXT NOT NULL ); -INSERT INTO "Version" ("Version") VALUES (7); +INSERT INTO "Version" ("Version") VALUES (9); diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs index 51a9726d5..10a0a258f 100644 --- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs @@ -91,7 +91,6 @@ namespace Duplicati.Library.Main.Database private readonly System.Data.IDbCommand m_insertfileOperationCommand; - private PathLookupHelper m_pathLookup; private Dictionary m_blockCache; private long m_filesetId; @@ -131,8 +130,8 @@ namespace Duplicati.Library.Main.Database m_findmetadatasetCommand.CommandText = @"SELECT ""A"".""ID"" FROM ""Metadataset"" A, ""BlocksetEntry"" B, ""Block"" C WHERE ""A"".""BlocksetID"" = ""B"".""BlocksetID"" AND ""B"".""BlockID"" = ""C"".""ID"" AND ""C"".""Hash"" = ? AND ""C"".""Size"" = ?"; m_findmetadatasetCommand.AddParameters(2); - m_findfilesetCommand.CommandText = @"SELECT ""ID"" FROM ""File"" WHERE ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""Path"" = ?"; - m_findfilesetCommand.AddParameters(3); + m_findfilesetCommand.CommandText = @"SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""Path"" = ? AND ""PrefixID"" = ?"; + m_findfilesetCommand.AddParameters(4); m_insertblockCommand.CommandText = @"INSERT INTO ""Block"" (""Hash"", ""VolumeID"", ""Size"") VALUES (?, ?, ?); SELECT last_insert_rowid();"; m_insertblockCommand.AddParameters(3); @@ -140,8 +139,8 @@ namespace Duplicati.Library.Main.Database m_insertfileOperationCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?, ?, ?)"; m_insertfileOperationCommand.AddParameters(3); - m_insertfileCommand.CommandText = @"INSERT INTO ""File"" (""Path"",""BlocksetID"", ""MetadataID"") VALUES (?, ? ,?); SELECT last_insert_rowid();"; - m_insertfileCommand.AddParameters(3); + m_insertfileCommand.CommandText = @"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"",""BlocksetID"", ""MetadataID"") VALUES (?, ?, ? ,?); SELECT last_insert_rowid();"; + m_insertfileCommand.AddParameters(4); m_insertblocksetCommand.CommandText = @"INSERT INTO ""Blockset"" (""Length"", ""FullHash"") VALUES (?, ?); SELECT last_insert_rowid();"; m_insertblocksetCommand.AddParameters(2); @@ -158,22 +157,22 @@ namespace Duplicati.Library.Main.Database m_insertmetadatasetCommand.CommandText = @"INSERT INTO ""Metadataset"" (""BlocksetID"") VALUES (?); SELECT last_insert_rowid();"; m_insertmetadatasetCommand.AddParameter(); - m_selectfilelastmodifiedCommand.CommandText = @"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?"; - m_selectfilelastmodifiedCommand.AddParameters(2); + m_selectfilelastmodifiedCommand.CommandText = @"SELECT ""A"".""ID"", ""B"".""LastModified"" FROM (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ?) ""A"" CROSS JOIN ""FilesetEntry"" ""B"" WHERE ""A"".""ID"" = ""B"".""FileID"" AND ""B"".""FilesetID"" = ?"; + m_selectfilelastmodifiedCommand.AddParameters(3); //Need a temporary table with path/lastmodified lookups - m_findfileCommand.CommandText = - @" SELECT ""File"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " + - @" FROM ""File"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " + - @" WHERE ""File"".""Path"" = ? " + - @" AND ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " + - @" AND ""FileBlockset"".""ID"" = ""File"".""BlocksetID"" " + - @" AND ""Metadataset"".""ID"" = ""File"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " + + m_findfileCommand.CommandText = + @" SELECT ""FileLookup"".""ID"" AS ""FileID"", ""FilesetEntry"".""Lastmodified"", ""FileBlockset"".""Length"", ""MetaBlockset"".""Fullhash"" AS ""Metahash"", ""MetaBlockset"".""Length"" AS ""Metasize"" " + + @" FROM ""FileLookup"", ""FilesetEntry"", ""Fileset"", ""Blockset"" ""FileBlockset"", ""Metadataset"", ""Blockset"" ""MetaBlockset"" " + + @" WHERE ""FileLookup"".""PrefixID"" = ? AND ""FileLookup"".""Path"" = ? " + + @" AND ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" " + + @" AND ""FileBlockset"".""ID"" = ""FileLookup"".""BlocksetID"" " + + @" AND ""Metadataset"".""ID"" = ""FileLookup"".""MetadataID"" AND ""MetaBlockset"".""ID"" = ""Metadataset"".""BlocksetID"" " + @" ORDER BY ""Fileset"".""Timestamp"" DESC " + @" LIMIT 1 "; - m_findfileCommand.AddParameters(1); + m_findfileCommand.AddParameters(2); - m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""File"" WHERE ""Blockset"".""ID"" = ""File"".""BlocksetID"" AND ""File"".""ID"" = ? "; + m_selectfileHashCommand.CommandText = @"SELECT ""Blockset"".""Fullhash"" FROM ""Blockset"", ""FileLookup"" WHERE ""Blockset"".""ID"" = ""FileLookup"".""BlocksetID"" AND ""FileLookup"".""ID"" = ? "; m_selectfileHashCommand.AddParameters(1); m_selectblocklistHashesCommand.CommandText = @"SELECT ""Hash"" FROM ""BlocklistHash"" WHERE ""BlocksetID"" = ? ORDER BY ""Index"" ASC "; @@ -186,75 +185,6 @@ namespace Duplicati.Library.Main.Database /// The option settings public void BuildLookupTable(Options options) { - if (options.UseFilepathCache) - m_pathLookup = new PathLookupHelper(true); - - - //Populate the lookup tables - using (var cmd = m_connection.CreateCommand()) - { - //Need a temporary table with path/lastmodified lookups - var scantableDefinition = - @"SELECT ""A1"".""ID"" AS ""FileID"", ""A1"".""Lastmodified"" AS ""Lastmodified"", ""A1"".""Path"" AS ""Path"", ""C"".""Length"" AS ""Length"", ""F"".""Fullhash"" AS ""Metahash"", ""F"".""Length"" AS ""Metasize"", ""A1"".""BlocksetID"" " + - @" FROM (SELECT ""File"".""ID"", ""File"".""BlocksetID"", ""File"".""MetadataID"", ""FilesetEntry"".""Lastmodified"", ""File"".""Path"", ""Fileset"".""Timestamp"" " + - @" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " + - @" ) ""A1"" LEFT JOIN " + - @" (SELECT ""File"".""Path"", ""Fileset"".""Timestamp"" " + - @" FROM ""FilesetEntry"", ""Fileset"", ""File"" WHERE ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" " + - @" ) ""A2"" ON ""A1"".""Path"" = ""A2"".""Path"" AND ""A1"".""Timestamp"" < ""A2"".""Timestamp"" " + - @" , ""Blockset"" ""C"", ""Metadataset"" ""E"", ""Blockset"" ""F"" " + - @" WHERE ""A2"".""Path"" IS NULL " + - @" AND ""C"".""ID"" = ""A1"".""BlocksetID"" " + - @" AND ""A1"".""MetadataID"" = ""E"".""ID"" " + - @" AND ""F"".""ID"" = ""E"".""BlocksetID"" "; - - if (m_pathLookup != null) - using(new Logging.Timer(LOGTAG, "BuildLastModified", "Build path lastmodified lookup table")) - using (var rd = cmd.ExecuteReader(string.Format(@" SELECT ""FileID"", ""Lastmodified"", ""Length"", ""Path"", ""Metahash"", ""Metasize"" FROM ({0}) WHERE ""BlocksetID"" >= 0 ", scantableDefinition))) - while (rd.Read()) - { - var id = rd.GetInt64(0); - var lastmodified = new DateTime(rd.GetInt64(1), DateTimeKind.Utc); - var filesize = rd.GetInt64(2); - var path = rd.GetString(3); - var metahash = rd.GetString(4); - var metasize = rd.GetInt64(5); - m_pathLookup.Insert(path, new PathEntryKeeper(id, lastmodified, filesize, metahash, metasize)); - } - - if (m_pathLookup != null) - try - { - using(new Logging.Timer(LOGTAG, "BuildPathTable", "Build path lookup table")) - using (var rd = cmd.ExecuteReader(@" SELECT ""Path"", ""BlocksetID"", ""MetadataID"", ""ID"" FROM ""File"" ")) - while (rd.Read()) - { - var path = rd.GetValue(0).ToString(); - var blocksetid = rd.GetInt64(1); - var metadataid = rd.GetInt64(2); - var filesetid = rd.GetInt64(3); - - PathEntryKeeper r; - if (!m_pathLookup.TryFind(path, out r)) - { - r = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1); - r.AddFilesetID(blocksetid, metadataid, filesetid); - m_pathLookup.Insert(path, r); - } - else - r.AddFilesetID(blocksetid, metadataid, filesetid); - } - } - catch (Exception ex) - { - throw new InvalidDataException("Duplicate file entries detected, run repair to fix it", ex); - } - - var tc = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Remotevolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"") AND ""State"" NOT IN (?, ?, ?, ?);", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString()); - if (tc > 0) - throw new InvalidDataException("Detected blocks that are not reachable in the block table"); - } - if (options.UseBlockCache) { string failedhash = null; @@ -435,7 +365,10 @@ namespace Duplicati.Library.Main.Database /// /// Adds a metadata set to the database, and returns a value indicating if the record was new /// - /// The metadata hash + /// The hash of the metadata + /// The size of the metadata + /// The ID of the blockset with the data + /// The transaction to use /// The id of the metadata set /// True if the set was added to the database, false otherwise public bool AddMetadataset(string filehash, long size, long blocksetid, out long metadataid, System.Data.IDbTransaction transaction = null) @@ -453,6 +386,48 @@ namespace Duplicati.Library.Main.Database } } + /// + /// Adds a file record to the database + /// + /// The path prefix ID + /// The path to the file + /// The time the file was modified + /// The ID of the hashkey for the file + /// The ID for the metadata + /// The transaction to use for insertion, or null for no transaction + public void AddFile(long pathprefixid, string filename, DateTime lastmodified, long blocksetID, long metadataID, System.Data.IDbTransaction transaction) + { + var fileidobj = -1L; + + m_findfilesetCommand.Transaction = transaction; + m_findfilesetCommand.SetParameterValue(0, blocksetID); + m_findfilesetCommand.SetParameterValue(1, metadataID); + m_findfilesetCommand.SetParameterValue(2, filename); + m_findfilesetCommand.SetParameterValue(3, pathprefixid); + fileidobj = m_findfilesetCommand.ExecuteScalarInt64(); + + if (fileidobj == -1) + { + using (var tr = new TemporaryTransactionWrapper(m_connection, transaction)) + { + m_insertfileCommand.Transaction = tr.Parent; + m_insertfileCommand.SetParameterValue(0, pathprefixid); + m_insertfileCommand.SetParameterValue(1, filename); + m_insertfileCommand.SetParameterValue(2, blocksetID); + m_insertfileCommand.SetParameterValue(3, metadataID); + + fileidobj = m_insertfileCommand.ExecuteScalarInt64(); + tr.Commit(); + } + } + + m_insertfileOperationCommand.Transaction = transaction; + m_insertfileOperationCommand.SetParameterValue(0, m_filesetId); + m_insertfileOperationCommand.SetParameterValue(1, fileidobj); + m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks); + m_insertfileOperationCommand.ExecuteNonQuery(); + } + /// /// Adds a file record to the database /// @@ -462,61 +437,9 @@ namespace Duplicati.Library.Main.Database /// The ID for the metadata /// The transaction to use for insertion, or null for no transaction public void AddFile(string filename, DateTime lastmodified, long blocksetID, long metadataID, System.Data.IDbTransaction transaction) - { - var fileidobj = -1L; - PathEntryKeeper entry = null; - var entryFound = false; - - if (m_pathLookup != null) - { - if (entryFound = (m_pathLookup.TryFind(filename, out entry) && entry != null)) - { - var fid = entry.GetFilesetID(blocksetID, metadataID); - if (fid >= 0) - fileidobj = fid; - } - } - else - { - m_findfilesetCommand.Transaction = transaction; - m_findfilesetCommand.SetParameterValue(0, blocksetID); - m_findfilesetCommand.SetParameterValue(1, metadataID); - m_findfilesetCommand.SetParameterValue(2, filename); - fileidobj = m_findfilesetCommand.ExecuteScalarInt64(); - } - - if (fileidobj == -1) - { - using(var tr = new TemporaryTransactionWrapper(m_connection, transaction)) - { - m_insertfileCommand.Transaction = tr.Parent; - m_insertfileCommand.SetParameterValue(0, filename); - m_insertfileCommand.SetParameterValue(1, blocksetID); - m_insertfileCommand.SetParameterValue(2, metadataID); - fileidobj = m_insertfileCommand.ExecuteScalarInt64(); - tr.Commit(); - - // We do not need to update this, because we will not ask for the same file twice - if (m_pathLookup != null) - { - if (!entryFound) - { - entry = new PathEntryKeeper(-1, new DateTime(0, DateTimeKind.Utc), -1, null, -1); - entry.AddFilesetID(blocksetID, metadataID, fileidobj); - m_pathLookup.Insert(filename, entry); - } - else - entry.AddFilesetID(blocksetID, metadataID, fileidobj); - } - } - } - - m_insertfileOperationCommand.Transaction = transaction; - m_insertfileOperationCommand.SetParameterValue(0, m_filesetId); - m_insertfileOperationCommand.SetParameterValue(1, fileidobj); - m_insertfileOperationCommand.SetParameterValue(2, lastmodified.ToUniversalTime().Ticks); - m_insertfileOperationCommand.ExecuteNonQuery(); - + { + var split = SplitIntoPrefixAndName(filename); + AddFile(GetOrCreatePathPrefix(split.Key, transaction), split.Value, lastmodified, blocksetID, metadataID, transaction); } public void AddUnmodifiedFile(long fileid, DateTime lastmodified, System.Data.IDbTransaction transaction = null) @@ -538,11 +461,12 @@ namespace Duplicati.Library.Main.Database AddFile(path, lastmodified, SYMLINK_BLOCKSET_ID, metadataID, transaction); } - public long GetFileLastModified(string path, long filesetid, out DateTime oldModified, System.Data.IDbTransaction transaction = null) + public long GetFileLastModified(long prefixid, string path, long filesetid, out DateTime oldModified, System.Data.IDbTransaction transaction = null) { - m_selectfileHashCommand.Transaction = transaction; - m_selectfilelastmodifiedCommand.SetParameterValue(0, path); - m_selectfilelastmodifiedCommand.SetParameterValue(1, filesetid); + m_selectfilelastmodifiedCommand.Transaction = transaction; + m_selectfilelastmodifiedCommand.SetParameterValue(0, prefixid); + m_selectfilelastmodifiedCommand.SetParameterValue(1, path); + m_selectfilelastmodifiedCommand.SetParameterValue(2, filesetid); using (var rd = m_selectfilelastmodifiedCommand.ExecuteReader()) if (rd.Read()) { @@ -554,18 +478,19 @@ namespace Duplicati.Library.Main.Database return -1; } - public long GetFileEntry(string path, long filesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize) + public long GetFileEntry(long prefixid, string path, long lastfilesetid, out DateTime oldModified, out long lastFileSize, out string oldMetahash, out long oldMetasize) { - if (m_pathLookup != null) - { - PathEntryKeeper tmp; - if (m_pathLookup.TryFind(path, out tmp) && tmp != null && tmp.FileID >= 0) + m_findfileCommand.SetParameterValue(0, prefixid); + m_findfileCommand.SetParameterValue(1, path); + + using (var rd = m_findfileCommand.ExecuteReader()) + if (rd.Read()) { - oldModified = tmp.Lastmodified; - lastFileSize = tmp.Filesize; - oldMetahash = tmp.Metahash; - oldMetasize = tmp.Metasize; - return tmp.FileID; + oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); + lastFileSize = rd.GetInt64(2); + oldMetahash = rd.GetString(3); + oldMetasize = rd.GetInt64(4); + return rd.ConvertValueToInt64(0); } else { @@ -575,31 +500,9 @@ namespace Duplicati.Library.Main.Database oldMetasize = -1; return -1; } - } - else - { - m_findfileCommand.SetParameterValue(0, path); - - using(var rd = m_findfileCommand.ExecuteReader()) - if (rd.Read()) - { - oldModified = new DateTime(rd.ConvertValueToInt64(1), DateTimeKind.Utc); - lastFileSize = rd.GetInt64(2); - oldMetahash = rd.GetString(3); - oldMetasize = rd.GetInt64(4); - return rd.ConvertValueToInt64(0); - } - else - { - oldModified = new DateTime(0, DateTimeKind.Utc); - lastFileSize = -1; - oldMetahash = null; - oldMetasize = -1; - return -1; - } - } } + public string GetFileHash(long fileid) { m_selectfileHashCommand.SetParameterValue(0, fileid); @@ -612,8 +515,6 @@ namespace Duplicati.Library.Main.Database public override void Dispose () { - m_pathLookup = null; - base.Dispose(); } @@ -633,8 +534,8 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand()) { var lastFilesetId = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC LIMIT 1"); - var count = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); - var size = cmd.ExecuteScalarInt64(@"SELECT SUM(""Blockset"".""Length"") FROM ""File"", ""FilesetEntry"", ""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); + var count = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FileLookup"" INNER JOIN ""FilesetEntry"" ON ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); + var size = cmd.ExecuteScalarInt64(@"SELECT SUM(""Blockset"".""Length"") FROM ""FileLookup"", ""FilesetEntry"", ""Blockset"" WHERE ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); return new Tuple(count, size); } @@ -644,6 +545,7 @@ namespace Duplicati.Library.Main.Database { using(var cmd = m_connection.CreateCommand(transaction)) { + // TODO: Optimize these queries to not use the "File" view var lastFilesetId = GetPreviousFilesetID(cmd); results.AddedFolders = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, m_filesetId, FOLDER_BLOCKSET_ID, lastFilesetId); results.AddedSymlinks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" = ? AND NOT ""File"".""Path"" IN (SELECT ""Path"" FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ?)", 0, m_filesetId, SYMLINK_BLOCKSET_ID, lastFilesetId); @@ -659,7 +561,7 @@ namespace Duplicati.Library.Main.Database var tmpName2 = "TmpFileList-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); try { - var subqueryFiles = @"SELECT ""File"".""Path"", ""A"".""Fullhash"" AS ""Filehash"", ""B"".""Fullhash"" AS ""Metahash"" FROM ""File"", ""FilesetEntry"", ""Blockset"" A, ""Blockset"" B, ""Metadataset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""A"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""B"".""ID"" "; + var subqueryFiles = @"SELECT ""File"".""Path"" AS ""Path"", ""A"".""Fullhash"" AS ""Filehash"", ""B"".""Fullhash"" AS ""Metahash"" FROM ""File"", ""FilesetEntry"", ""Blockset"" A, ""Blockset"" B, ""Metadataset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""A"".""ID"" = ""File"".""BlocksetID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""B"".""ID"" "; cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName1), lastFilesetId); cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" AS " + subqueryFiles, tmpName2), m_filesetId); diff --git a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs index e77a31ef3..c74dcb061 100644 --- a/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalBugReportDatabase.cs @@ -39,7 +39,9 @@ namespace Duplicati.Library.Main.Database { cmd.Transaction = tr; var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); - + + // TODO: Rewrite this to use PathPrefix + // TODO: Needs to be much faster using(var upcmd = m_connection.CreateCommand()) { @@ -60,8 +62,11 @@ namespace Duplicati.Library.Main.Database cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Message"" = ""ERASED!"" WHERE ""Message"" LIKE ""%/%"" OR ""Message"" LIKE ""%:\%"" "); cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Exception"" = ""ERASED!"" WHERE ""Exception"" LIKE ""%/%"" OR ""Exception"" LIKE ""%:\%"" "); - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""File"" SET ""Path"" = (SELECT ""Obfuscated"" FROM ""{0}"" WHERE ""Path"" = ""RealPath"") ", tablename)); - + + cmd.ExecuteNonQuery(string.Format(@"CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", ""A"".""Obfuscated"" AS ""Path"", ""B"".""BlocksetID"" AS ""BlocksetID"", ""B"".""MetadataID"" AS ""MetadataID"" FROM ""{0}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"") ", tablename)); + cmd.ExecuteNonQuery(@"DROP TABLE ""File"" "); + cmd.ExecuteNonQuery(@"DROP TABLE ""PathPrefix"" "); + cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", tablename)); using(new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport")) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 385ec4aec..a806c6c7a 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -28,6 +28,9 @@ namespace Duplicati.Library.Main.Database private readonly System.Data.IDbCommand m_insertremotelogCommand; private readonly System.Data.IDbCommand m_insertIndexBlockLink; + private readonly System.Data.IDbCommand m_findpathprefixCommand; + private readonly System.Data.IDbCommand m_insertpathprefixCommand; + protected BasicResults m_result; public const long FOLDER_BLOCKSET_ID = -100; @@ -106,6 +109,8 @@ namespace Duplicati.Library.Main.Database m_selectremotevolumeIdCommand = connection.CreateCommand(); m_createremotevolumeCommand = connection.CreateCommand(); m_insertIndexBlockLink = connection.CreateCommand(); + m_findpathprefixCommand = connection.CreateCommand(); + m_insertpathprefixCommand = connection.CreateCommand(); m_insertlogCommand.CommandText = @"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (?, ?, ?, ?, ?)"; m_insertlogCommand.AddParameters(5); @@ -133,6 +138,12 @@ namespace Duplicati.Library.Main.Database m_insertIndexBlockLink.CommandText = @"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)"; m_insertIndexBlockLink.AddParameters(2); + + m_findpathprefixCommand.CommandText = @"SELECT ""ID"" FROM ""PathPrefix"" WHERE ""Prefix"" = ?"; + m_findpathprefixCommand.AddParameter(); + + m_insertpathprefixCommand.CommandText = @"INSERT INTO ""PathPrefix"" (""Prefix"") VALUES (?); SELECT last_insert_rowid(); "; + m_insertpathprefixCommand.AddParameter(); } internal void SetResult(BasicResults result) @@ -418,7 +429,7 @@ namespace Duplicati.Library.Main.Database bsIdsSubQuery = string.Format(@"SELECT ""ID"" FROM ""{0}"" ", blocksetidstable); deletecmd.Parameters.Clear(); - deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""File"" WHERE ""BlocksetID"" IN ({0}) OR ""MetadataID"" IN ({0})", bsIdsSubQuery)); + deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""FileLookup"" WHERE ""BlocksetID"" IN ({0}) OR ""MetadataID"" IN ({0})", bsIdsSubQuery)); deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery)); deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({0})", bsIdsSubQuery)); deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery)); @@ -799,7 +810,7 @@ ON throw new Exception("Detected non-empty blocksets with no associated blocks!"); } - if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0) + if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0) throw new Exception("Detected files associated with non-existing blocksets!"); if (verifyfilelists) @@ -807,9 +818,9 @@ ON using(var cmd2 = m_connection.CreateCommand(transaction)) foreach(var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1))) { - var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS); + var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS); var expandedlist = cmd2.ExecuteScalarInt64(expandedCmd, 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid); - //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); + //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""FileLookup"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID"" AND ""FileLookup"".""BlocksetID"" != ? AND ""FileLookup"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID); var storedlist = cmd2.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?", 0, filesetid); if (expandedlist != storedlist) @@ -1160,6 +1171,8 @@ ORDER BY { using(var cmd = m_connection.CreateCommand()) { + // TODO: Optimize this to not rely on the "File" view, and not instantiate the paths in full + cmd.Transaction = transaction; cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename)); using(var tr = new TemporaryTransactionWrapper(m_connection, transaction)) @@ -1417,5 +1430,71 @@ ORDER BY ); } } + + /// + /// The current index into the path prefix buffer + /// + private int m_pathPrefixIndex = 0; + /// + /// The path prefix lookup list + /// + private readonly KeyValuePair[] m_pathPrefixLookup = new KeyValuePair[5]; + + /// + /// Gets the path prefix ID, optionally creating it in the process. + /// + /// The path prefix ID. + /// The path to get the prefix for. + /// The transaction to use for insertion, or null for no transaction + public long GetOrCreatePathPrefix(string prefix, System.Data.IDbTransaction transaction) + { + // Ring-buffer style lookup + for (var i = 0; i < m_pathPrefixLookup.Length; i++) + { + var ix = (i + m_pathPrefixIndex) % m_pathPrefixLookup.Length; + if (string.Equals(m_pathPrefixLookup[ix].Key, prefix, StringComparison.Ordinal)) + return m_pathPrefixLookup[ix].Value; + } + + m_findpathprefixCommand.Transaction = transaction; + m_findpathprefixCommand.SetParameterValue(0, prefix); + var id = m_findpathprefixCommand.ExecuteScalarInt64(); + if (id < 0) + { + m_insertpathprefixCommand.Transaction = transaction; + m_insertpathprefixCommand.SetParameterValue(0, prefix); + id = m_insertpathprefixCommand.ExecuteScalarInt64(); + } + + m_pathPrefixIndex = (m_pathPrefixIndex + 1) % m_pathPrefixLookup.Length; + m_pathPrefixLookup[m_pathPrefixIndex] = new KeyValuePair(prefix, id); + + return id; + } + + /// + /// The path separators on this system + /// + private static readonly char[] _pathseparators = new char[] { + System.IO.Path.DirectorySeparatorChar, + System.IO.Path.AltDirectorySeparatorChar, + }; + + /// + /// Helper method that splits a path on the last path separator + /// + /// The prefix and name. + /// The path to split. + public static KeyValuePair SplitIntoPrefixAndName(string path) + { + if (path == null || path.Length == 0) + throw new ArgumentException($"Invalid path: {path}", nameof(path)); + + int nLast = path.TrimEnd(_pathseparators).LastIndexOfAny(_pathseparators); + if (nLast >= 0) + return new KeyValuePair(path.Substring(0, nLast + 1), path.Substring(nLast + 1)); + + throw new ArgumentException($"Invalid path: {path}", nameof(path)); + } } } diff --git a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs index fe5ec4287..010a74da6 100644 --- a/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDeleteDatabase.cs @@ -80,9 +80,9 @@ namespace Duplicati.Library.Main.Database //Then we delete anything that is no longer being referenced cmd.ExecuteNonQuery(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Fileset"")"); cmd.ExecuteNonQuery(@"DELETE FROM ""ChangeJournalData"" WHERE ""FilesetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Fileset"")"); - cmd.ExecuteNonQuery(@"DELETE FROM ""File"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""File"") "); - cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""File"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") "); + cmd.ExecuteNonQuery(@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"") "); + cmd.ExecuteNonQuery(@"DELETE FROM ""Metadataset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""MetadataID"" FROM ""FileLookup"") "); + cmd.ExecuteNonQuery(@"DELETE FROM ""Blockset"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""BlocksetID"" FROM ""FileLookup"" UNION SELECT DISTINCT ""BlocksetID"" FROM ""Metadataset"") "); cmd.ExecuteNonQuery(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") "); cmd.ExecuteNonQuery(@"DELETE FROM ""BlocklistHash"" WHERE ""BlocksetID"" NOT IN (SELECT DISTINCT ""ID"" FROM ""Blockset"") "); @@ -129,8 +129,8 @@ namespace Duplicati.Library.Main.Database var tmptablename = "UsageReport-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()); var usedBlocks = @"SELECT SUM(""Block"".""Size"") AS ""ActiveSize"", ""Block"".""VolumeID"" AS ""VolumeID"" FROM ""Block"", ""Remotevolume"" WHERE ""Block"".""VolumeID"" = ""Remotevolume"".""ID"" AND ""Block"".""ID"" NOT IN (SELECT ""Block"".""ID"" FROM ""Block"",""DeletedBlock"" WHERE ""Block"".""Hash"" = ""DeletedBlock"".""Hash"" AND ""Block"".""Size"" = ""DeletedBlock"".""Size"") GROUP BY ""Block"".""VolumeID"" "; - var lastmodifiedFile = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""File"", ""BlocksetEntry"", ""Block"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""File"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" "; - var lastmodifiedMetadata = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""File"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""FilesetEntry"".""FileID"" = ""File"".""ID"" AND ""File"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" "; + var lastmodifiedFile = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""FileLookup"", ""BlocksetEntry"", ""Block"" WHERE ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" "; + var lastmodifiedMetadata = @"SELECT ""Block"".""VolumeID"" AS ""VolumeID"", ""Fileset"".""Timestamp"" AS ""Sorttime"" FROM ""Fileset"", ""FilesetEntry"", ""FileLookup"", ""BlocksetEntry"", ""Block"", ""Metadataset"" WHERE ""FilesetEntry"".""FileID"" = ""FileLookup"".""ID"" AND ""FileLookup"".""MetadataID"" = ""Metadataset"".""ID"" AND ""Metadataset"".""BlocksetID"" = ""BlocksetEntry"".""BlocksetID"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" "; var scantime = @"SELECT ""VolumeID"" AS ""VolumeID"", MIN(""Sorttime"") AS ""Sorttime"" FROM (" + lastmodifiedFile + @" UNION " + lastmodifiedMetadata + @") GROUP BY ""VolumeID"" "; var active = @"SELECT ""A"".""ActiveSize"" AS ""ActiveSize"", 0 AS ""InactiveSize"", ""A"".""VolumeID"" AS ""VolumeID"", CASE WHEN ""B"".""Sorttime"" IS NULL THEN 0 ELSE ""B"".""Sorttime"" END AS ""Sorttime"" FROM (" + usedBlocks + @") A LEFT OUTER JOIN (" + scantime + @") B ON ""B"".""VolumeID"" = ""A"".""VolumeID"" "; diff --git a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs index 2820daf82..e293b7329 100644 --- a/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListAffectedDatabase.cs @@ -63,7 +63,7 @@ namespace Duplicati.Library.Main.Database var sql = string.Format( @"SELECT DISTINCT ""FilesetID"" FROM (" + - @"SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" IN ( SELECT ""ID"" FROM ""File"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN ( SELECT ""ID"" From ""Block"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0})))))" + + @"SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" IN ( SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN ( SELECT ""ID"" From ""Block"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0})))))" + " UNION " + @"SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))" + ")", @@ -137,9 +137,9 @@ namespace Duplicati.Library.Main.Database { var sql = string.Format( @"SELECT DISTINCT ""Name"" FROM ( " + - @" SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))))) " + + @" SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))))) " + @" UNION " + - @" SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""Metadataset"" WHERE ""ID"" IN ( SELECT ""MetadataID"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))))))" + + @" SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""Metadataset"" WHERE ""ID"" IN ( SELECT ""MetadataID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN ({0}))))))))" + @")", string.Join(",", items.Select(x => "?")) ); diff --git a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs index 77755401d..46bd09590 100644 --- a/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalListBrokenFilesDatabase.cs @@ -24,9 +24,9 @@ namespace Duplicati.Library.Main.Database { private const string BROKEN_FILE_IDS = @" SELECT DISTINCT ""ID"" FROM ( - SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""File"" WHERE ""BlocksetID"" != {0} AND ""BlocksetID"" != {1} + SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" WHERE ""BlocksetID"" != {0} AND ""BlocksetID"" != {1} UNION - SELECT ""A"".""ID"" AS ""ID"", ""B"".""BlocksetID"" AS ""BlocksetID"" FROM ""File"" A LEFT JOIN ""Metadataset"" B ON ""A"".""MetadataID"" = ""B"".""ID"" + SELECT ""A"".""ID"" AS ""ID"", ""B"".""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" A LEFT JOIN ""Metadataset"" B ON ""A"".""MetadataID"" = ""B"".""ID"" ) WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN ( diff --git a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs index 7893160bf..a0d71b868 100644 --- a/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalPurgeDatabase.cs @@ -50,7 +50,7 @@ namespace Duplicati.Library.Main.Database internal long CountOrphanFiles(System.Data.IDbTransaction transaction) { using (var cmd = m_connection.CreateCommand(transaction)) - using (var rd = cmd.ExecuteReader(@"SELECT COUNT(*) FROM ""File"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")")) + using (var rd = cmd.ExecuteReader(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")")) if (rd.Read()) return rd.ConvertValueToInt64(0, 0); else @@ -156,7 +156,7 @@ namespace Duplicati.Library.Main.Database using (var cmd = m_connection.CreateCommand(m_transaction)) { RemovedFileCount = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""{0}""", m_tablename), 0); - RemovedFileSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""C"".""Length"") FROM ""{0}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename), 0); + RemovedFileSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""C"".""Length"") FROM ""{0}"" A, ""FileLookup"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ", m_tablename), 0); var filesetcount = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = " + ParentID), 0); if (filesetcount == RemovedFileCount) throw new Duplicati.Library.Interface.UserInformationException(string.Format("Refusing to purge {0} files from fileset with ID {1}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", RemovedFileCount, ParentID), "PurgeWouldRemoveEntireFileset"); diff --git a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs index def09b51a..ba21f8407 100644 --- a/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRecreateDatabase.cs @@ -65,8 +65,6 @@ namespace Duplicati.Library.Main.Database private readonly System.Data.IDbCommand m_insertBlockCommand; private readonly System.Data.IDbCommand m_insertDuplicateBlockCommand; - private readonly PathLookupHelper m_filesetLookup; - private string m_tempblocklist; private string m_tempsmalllist; @@ -152,9 +150,9 @@ namespace Duplicati.Library.Main.Database m_findHashBlockCommand = m_connection.CreateCommand(); m_insertBlockCommand = m_connection.CreateCommand(); m_insertDuplicateBlockCommand = m_connection.CreateCommand(); - - m_insertFileCommand.CommandText = @"INSERT INTO ""File"" (""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?); SELECT last_insert_rowid();"; - m_insertFileCommand.AddParameters(3); + + m_insertFileCommand.CommandText = @"INSERT INTO ""FileLookup"" (""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") VALUES (?,?,?,?); SELECT last_insert_rowid();"; + m_insertFileCommand.AddParameters(4); m_insertFilesetEntryCommand.CommandText = @"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") VALUES (?,?,?)"; m_insertFilesetEntryCommand.AddParameters(3); @@ -183,8 +181,8 @@ namespace Duplicati.Library.Main.Database m_findMetadatasetCommand.CommandText = @"SELECT ""Metadataset"".""ID"" FROM ""Metadataset"",""Blockset"" WHERE ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" AND ""Blockset"".""FullHash"" = ? AND ""Blockset"".""Length"" = ? "; m_findMetadatasetCommand.AddParameters(2); - m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? "; - m_findFilesetCommand.AddParameters(3); + m_findFilesetCommand.CommandText = @"SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? "; + m_findFilesetCommand.AddParameters(4); m_findblocklisthashCommand.CommandText = string.Format(@"SELECT DISTINCT ""BlockListHash"" FROM ""{0}"" WHERE ""BlockListHash"" = ? ", m_tempblocklist); m_findblocklisthashCommand.AddParameters(1); @@ -197,9 +195,6 @@ namespace Duplicati.Library.Main.Database m_insertDuplicateBlockCommand.CommandText = @"INSERT INTO ""DuplicateBlock"" (""BlockID"", ""VolumeID"") VALUES ((SELECT ""ID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?), ?)"; m_insertDuplicateBlockCommand.AddParameters(3); - - if (options.UseFilepathCache) - m_filesetLookup = new PathLookupHelper(); } public void FindMissingBlocklistHashes(long hashsize, long blocksize, System.Data.IDbTransaction transaction) @@ -292,59 +287,40 @@ namespace Duplicati.Library.Main.Database } } - public void AddDirectoryEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction) + public void AddDirectoryEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction) { - AddEntry(FilelistEntryType.Folder, filesetid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction); + AddEntry(FilelistEntryType.Folder, filesetid, pathprefixid, path, time, FOLDER_BLOCKSET_ID, metadataid, transaction); } - public void AddSymlinkEntry(long filesetid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction) + public void AddSymlinkEntry(long filesetid, long pathprefixid, string path, DateTime time, long metadataid, System.Data.IDbTransaction transaction) { - AddEntry(FilelistEntryType.Symlink, filesetid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction); + AddEntry(FilelistEntryType.Symlink, filesetid, pathprefixid, path, time, SYMLINK_BLOCKSET_ID, metadataid, transaction); } - public void AddFileEntry(long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction) + public void AddFileEntry(long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction) { - AddEntry(FilelistEntryType.File , filesetid, path, time, blocksetid, metadataid, transaction); + AddEntry(FilelistEntryType.File, filesetid, pathprefixid, path, time, blocksetid, metadataid, transaction); } - private void AddEntry(FilelistEntryType type, long filesetid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction) + private void AddEntry(FilelistEntryType type, long filesetid, long pathprefixid, string path, DateTime time, long blocksetid, long metadataid, System.Data.IDbTransaction transaction) { var fileid = -1L; - if (m_filesetLookup != null) - { - PathEntryKeeper e; - if (m_filesetLookup.TryFind(path, out e)) - fileid = e.GetFilesetID(blocksetid, metadataid); - } - else - { - m_findFilesetCommand.Transaction = transaction; - m_findFilesetCommand.SetParameterValue(0, path); - m_findFilesetCommand.SetParameterValue(1, blocksetid); - m_findFilesetCommand.SetParameterValue(2, metadataid); - fileid = m_findFilesetCommand.ExecuteScalarInt64(-1); - } - + m_findFilesetCommand.Transaction = transaction; + m_findFilesetCommand.SetParameterValue(0, pathprefixid); + m_findFilesetCommand.SetParameterValue(1, path); + m_findFilesetCommand.SetParameterValue(2, blocksetid); + m_findFilesetCommand.SetParameterValue(3, metadataid); + fileid = m_findFilesetCommand.ExecuteScalarInt64(-1); + if (fileid < 0) { m_insertFileCommand.Transaction = transaction; - m_insertFileCommand.SetParameterValue(0, path); - m_insertFileCommand.SetParameterValue(1, blocksetid); - m_insertFileCommand.SetParameterValue(2, metadataid); + m_insertFileCommand.SetParameterValue(0, pathprefixid); + m_insertFileCommand.SetParameterValue(1, path); + m_insertFileCommand.SetParameterValue(2, blocksetid); + m_insertFileCommand.SetParameterValue(3, metadataid); fileid = m_insertFileCommand.ExecuteScalarInt64(-1); - if (m_filesetLookup != null) - { - PathEntryKeeper e; - if (m_filesetLookup.TryFind(path, out e)) - e.AddFilesetID(blocksetid, metadataid, fileid); - else - { - e = new PathEntryKeeper(); - e.AddFilesetID(blocksetid, metadataid, fileid); - m_filesetLookup.Insert(path, e); - } - } } m_insertFilesetEntryCommand.Transaction = transaction; diff --git a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs index b7fe07dd2..4f704eca8 100644 --- a/Duplicati/Library/Main/Database/LocalRepairDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRepairDatabase.cs @@ -196,8 +196,8 @@ namespace Duplicati.Library.Main.Database public IEnumerable GetFilesetsUsingMissingBlocks() { - var blocks = @"SELECT DISTINCT ""File"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""File"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""File"".""BlocksetID"" = ""Blockset"".""ID"" "; - var blocklists = @"SELECT DISTINCT ""File"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""File"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""File"".""BlocksetID"" = ""Blockset"".""ID"" "; + var blocks = @"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocksetEntry"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""BlocksetEntry"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "; + var blocklists = @"SELECT DISTINCT ""FileLookup"".""ID"" AS ID FROM ""{0}"", ""Block"", ""Blockset"", ""BlocklistHash"", ""FileLookup"" WHERE ""Block"".""Hash"" = ""{0}"".""Hash"" AND ""Block"".""Size"" = ""{0}"".""Size"" AND ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""BlocklistHash"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FileLookup"".""BlocksetID"" = ""Blockset"".""ID"" "; var cmdtxt = @"SELECT DISTINCT ""RemoteVolume"".""Name"", ""RemoteVolume"".""Hash"", ""RemoteVolume"".""Size"" FROM ""RemoteVolume"", ""FilesetEntry"", ""Fileset"" WHERE ""RemoteVolume"".""ID"" = ""Fileset"".""VolumeID"" AND ""Fileset"".""ID"" = ""FilesetEntry"".""FilesetID"" AND ""RemoteVolume"".""Type"" = ? AND ""FilesetEntry"".""FileID"" IN (SELECT DISTINCT ""ID"" FROM ( " + blocks + " UNION " + blocklists + " ))"; @@ -285,9 +285,9 @@ namespace Duplicati.Library.Main.Database c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(0)); } - cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""File"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{0}"") ", tablename)); + cmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT ""ID"" FROM ""{0}"") ", tablename)); cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}-Ix"" ON ""{0}"" (""ID"", ""MetadataID"")", tablename)); - cmd.ExecuteNonQuery(string.Format(@"UPDATE ""File"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{0}"" A WHERE ""A"".""ID"" = ""File"".""ID"") ", tablename)); + cmd.ExecuteNonQuery(string.Format(@"UPDATE ""FileLookup"" SET ""MetadataID"" = (SELECT ""MetadataID"" FROM ""{0}"" A WHERE ""A"".""ID"" = ""FileLookup"".""ID"") ", tablename)); cmd.ExecuteNonQuery(string.Format(@"DROP TABLE ""{0}"" ", tablename)); cmd.CommandText = sql_count; @@ -306,23 +306,23 @@ namespace Duplicati.Library.Main.Database using(var tr = m_connection.BeginTransaction()) using(var cmd = m_connection.CreateCommand(tr)) { - var sql_count = @"SELECT COUNT(*) FROM (SELECT ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""File"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1"; + var sql_count = @"SELECT COUNT(*) FROM (SELECT ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Duplicates"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Duplicates"" > 1"; var x = cmd.ExecuteScalarInt64(sql_count, 0); if (x > 0) { Logging.Log.WriteInformationMessage(LOGTAG, "DuplicateFileEntries", "Found duplicate file entries, repairing"); - var sql = @"SELECT ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( - SELECT MIN(""ID"") AS ""ID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""File"" GROUP BY ""Path"", ""BlocksetID"", ""MetadataID"") + var sql = @"SELECT ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", ""Entries"" FROM ( + SELECT MIN(""ID"") AS ""ID"", ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"", COUNT(*) as ""Entries"" FROM ""FileLookup"" GROUP BY ""PrefixID"", ""Path"", ""BlocksetID"", ""MetadataID"") WHERE ""Entries"" > 1 ORDER BY ""ID"""; using(var c2 = m_connection.CreateCommand(tr)) { - c2.CommandText = @"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)"; - c2.CommandText += @"; DELETE FROM ""File"" WHERE ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?"; + c2.CommandText = @"UPDATE ""FilesetEntry"" SET ""FileID"" = ? WHERE ""FileID"" IN (SELECT ""ID"" FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ?)"; + c2.CommandText += @"; DELETE FROM ""FileLookup"" WHERE ""PrefixID"" = ? AND ""Path"" = ? AND ""BlocksetID"" = ? AND ""MetadataID"" = ? AND ""ID"" != ?"; foreach(var rd in cmd.ExecuteReaderEnumerable(sql)) - c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(0)); + c2.ExecuteNonQuery(null, rd.GetValue(0), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(4), rd.GetValue(1), rd.GetValue(2), rd.GetValue(3), rd.GetValue(4), rd.GetValue(0)); } cmd.CommandText = sql_count; diff --git a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs index 3c657ffff..65ccf242e 100644 --- a/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalRestoreDatabase.cs @@ -213,6 +213,8 @@ namespace Duplicati.Library.Main.Database // better suited to speed up commit on UpdateBlocks cmd.ExecuteNonQuery(string.Format(@"CREATE INDEX ""{0}_FileIdIndexIndex"" ON ""{0}"" (""FileId"", ""Index"")", m_tempblocktable)); + // TODO: Optimize to use the path prefix + if (filter == null || filter.Empty) { // Simple case, restore everything diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index 18ab94a2a..eb3645159 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -197,6 +197,14 @@ + + + + + + + + - - - - PreserveNewest diff --git a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs index c080e5eb0..f8c1e227d 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackupDatabase.cs @@ -95,18 +95,18 @@ namespace Duplicati.Library.Main.Operation.Backup return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified, m_transaction)); } - public Task> GetFileLastModifiedAsync(string path, long lastfilesetid) + public Task> GetFileLastModifiedAsync(long prefixid, string path, long lastfilesetid) { return RunOnMain(() => { DateTime lastModified; - var id = m_database.GetFileLastModified(path, lastfilesetid, out lastModified, m_transaction); + var id = m_database.GetFileLastModified(prefixid, path, lastfilesetid, out lastModified, m_transaction); return new KeyValuePair(id, lastModified); }); } - public Task GetFileEntryAsync(string path, long lastfilesetid) + public Task GetFileEntryAsync(long prefixid, string path, long lastfilesetid) { return RunOnMain(() => { DateTime oldModified; @@ -114,7 +114,7 @@ namespace Duplicati.Library.Main.Operation.Backup string oldMetahash; long oldMetasize; - var id = m_database.GetFileEntry(path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize); + var id = m_database.GetFileEntry(prefixid, path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize); return id < 0 ? null : @@ -132,9 +132,14 @@ namespace Duplicati.Library.Main.Operation.Backup }); } - public Task AddFileAsync(string filename, DateTime lastmodified, long blocksetid, long metadataid) + public Task GetOrCreatePathPrefix(string prefix) { - return RunOnMain(() => m_database.AddFile(filename, lastmodified, blocksetid, metadataid, m_transaction)); + return RunOnMain(() => m_database.GetOrCreatePathPrefix(prefix, m_transaction)); + } + + public Task AddFileAsync(long prefixid, string filename, DateTime lastmodified, long blocksetid, long metadataid) + { + return RunOnMain(() => m_database.AddFile(prefixid, filename, lastmodified, blocksetid, metadataid, m_transaction)); } public Task AddUnmodifiedAsync(long fileid, DateTime lastModified) diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index f0d08d11e..78251f5f5 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -109,12 +109,12 @@ namespace Duplicati.Library.Main.Operation.Backup Logging.Log.WriteVerboseMessage(FILELOGTAG, "WoudlAddChangedFile", "Would add changed file {0}, size {1}", e.Path, Library.Utility.Utility.FormatSizeString(filesize)); } - await database.AddFileAsync(e.Path, e.LastWrite, filestreamdata.Blocksetid, metadataid); + await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid); } else if (e.MetadataChanged) { Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Path); - await database.AddFileAsync(e.Path, e.LastWrite, filestreamdata.Blocksetid, metadataid); + await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid); } else /*if (e.OldId >= 0)*/ { diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index c773a78a9..ad2c070a6 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -38,6 +38,10 @@ namespace Duplicati.Library.Main.Operation.Backup // From input public string Path; + // Split + public long PathPrefixID; + public string Filename; + // From database public long OldId; public DateTime OldModified; @@ -66,6 +70,7 @@ namespace Duplicati.Library.Main.Operation.Backup async self => { var emptymetadata = Utility.WrapMetadata(new Dictionary(), options); + var prevprefix = new KeyValuePair(null, -1); while (true) { @@ -96,12 +101,25 @@ namespace Duplicati.Library.Main.Operation.Backup { try { + var split = Database.LocalDatabase.SplitIntoPrefixAndName(path); + + long prefixid; + if (string.Equals(prevprefix.Key, split.Key, StringComparison.Ordinal)) + prefixid = prevprefix.Value; + else + { + prefixid = await database.GetOrCreatePathPrefix(split.Key); + prevprefix = new KeyValuePair(split.Key, prefixid); + } + if (options.CheckFiletimeOnly || options.DisableFiletimeCheck) { - var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid); + var tmp = await database.GetFileLastModifiedAsync(prefixid, split.Value, lastfilesetid); await self.Output.WriteAsync(new FileEntry() { OldId = tmp.Key < 0 ? -1 : tmp.Key, Path = path, + PathPrefixID = prefixid, + Filename = split.Value, Attributes = attributes, LastWrite = lastwrite, OldModified = tmp.Key < 0 ? new DateTime(0) : tmp.Value, @@ -112,10 +130,12 @@ namespace Duplicati.Library.Main.Operation.Backup } else { - var res = await database.GetFileEntryAsync(path, lastfilesetid); + var res = await database.GetFileEntryAsync(prefixid, split.Value, lastfilesetid); await self.Output.WriteAsync(new FileEntry() { OldId = res == null ? -1 : res.id, Path = path, + PathPrefixID = prefixid, + Filename = split.Value, Attributes = attributes, LastWrite = lastwrite, OldModified = res == null ? new DateTime(0) : res.modified, diff --git a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs index acc463278..ea0ed8621 100644 --- a/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/Operation/RecreateDatabaseHandler.cs @@ -227,11 +227,14 @@ namespace Duplicati.Library.Main.Operation if (expectedmetablocks <= 1) expectedmetablocklisthashes = 0; var metadataid = long.MinValue; + var split = Database.LocalDatabase.SplitIntoPrefixAndName(fe.Path); + var prefixid = restoredb.GetOrCreatePathPrefix(split.Key, tr); + switch (fe.Type) { case FilelistEntryType.Folder: metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr); - restoredb.AddDirectoryEntry(filesetid, fe.Path, fe.Time, metadataid, tr); + restoredb.AddDirectoryEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, tr); break; case FilelistEntryType.File: var expectedblocks = (fe.Size + blocksize - 1) / blocksize; @@ -240,7 +243,7 @@ namespace Duplicati.Library.Main.Operation var blocksetid = restoredb.AddBlockset(fe.Hash, fe.Size, fe.BlocklistHashes, expectedblocklisthashes, tr); metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr); - restoredb.AddFileEntry(filesetid, fe.Path, fe.Time, blocksetid, metadataid, tr); + restoredb.AddFileEntry(filesetid, prefixid, split.Value, fe.Time, blocksetid, metadataid, tr); if (fe.Size <= blocksize) { @@ -255,7 +258,7 @@ namespace Duplicati.Library.Main.Operation break; case FilelistEntryType.Symlink: metadataid = restoredb.AddMetadataset(fe.Metahash, fe.Metasize, fe.MetaBlocklistHashes, expectedmetablocklisthashes, tr); - restoredb.AddSymlinkEntry(filesetid, fe.Path, fe.Time, metadataid, tr); + restoredb.AddSymlinkEntry(filesetid, prefixid, split.Value, fe.Time, metadataid, tr); break; default: Logging.Log.WriteWarningMessage(LOGTAG, "SkippingUnknownFileEntry", null, "Skipping file-entry with unknown type {0}: {1} ", fe.Type, fe.Path); diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 71d998fa6..baf22623d 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -512,7 +512,6 @@ namespace Duplicati.Library.Main new CommandLineArgument("skip-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkipmetadataShort, Strings.Options.SkipmetadataLong, "false"), new CommandLineArgument("restore-permissions", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestorepermissionsShort, Strings.Options.RestorepermissionsLong, "false"), new CommandLineArgument("skip-restore-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkiprestoreverificationShort, Strings.Options.SkiprestoreverificationLong, "false"), - new CommandLineArgument("disable-filepath-cache", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilepathcacheShort, Strings.Options.DisablefilepathcacheLong, "true"), new CommandLineArgument("use-block-cache", CommandLineArgument.ArgumentType.Boolean, Strings.Options.UseblockcacheShort, Strings.Options.UseblockcacheLong, "false"), new CommandLineArgument("changed-files", CommandLineArgument.ArgumentType.Path, Strings.Options.ChangedfilesShort, Strings.Options.ChangedfilesLong), new CommandLineArgument("deleted-files", CommandLineArgument.ArgumentType.Path, Strings.Options.DeletedfilesShort, Strings.Options.DeletedfilesLong("changed-files")), @@ -1578,19 +1577,6 @@ namespace Duplicati.Library.Main get { return Library.Utility.Utility.ParseBoolOption(m_options, "disable-synthetic-filelist"); } } - /// - /// Flag indicating if the filepath cache is disabled - /// - public bool UseFilepathCache - { - get - { - string s; - m_options.TryGetValue("disable-filepath-cache", out s); - return !Library.Utility.Utility.ParseBool(s, true); - } - } - /// /// Flag indicating if the in-memory block cache is used /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 3c8c07849..c10d94813 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -147,8 +147,6 @@ namespace Duplicati.Library.Main.Strings public static string DeletedfilesShort { get { return LC.L(@"List of deleted files"); } } public static string FilehashlookupsizeLong { get { return LC.L(@"A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log."); } } public static string FilehashlookupsizeShort { get { return LC.L(@"Memory used by the file hash"); } } - public static string DisablefilepathcacheLong { get { return LC.L(@"This option can be used to reduce the memory footprint by not keeping paths and modification timestamps in memory"); } } - public static string DisablefilepathcacheShort { get { return LC.L(@"Reduce memory footprint by disabling in-memory lookups"); } } public static string UseblockcacheShort { get { return LC.L(@"This option can be used to increase speed in exchange for extra memory use."); } } public static string UseblockcacheLong { get { return LC.L(@"Store an in-memory block cache"); } } public static string StoremetadataLong { get { return LC.L(@"Stores metadata, such as file timestamps and attributes. This increases the required storage space as well as the processing time."); } }