diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index 5dff70df2..ffc4d4cc3 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -100,8 +100,6 @@ - - @@ -172,6 +170,8 @@ + + diff --git a/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql b/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql index 265c5e56c..bf8851431 100644 --- a/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql +++ b/Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql @@ -25,15 +25,15 @@ CREATE TABLE "Remotevolume" ( ); /* -The shadow-block table contains +The index-block table contains references that explains what block -files a shadow file references. -This is used to remove shadow volumes, +files a index file references. +This is used to remove index volumes, when they no longer reference any block volumes */ -CREATE TABLE "ShadowBlockLink" ( - "ShadowVolumeID" INTEGER NOT NULL, +CREATE TABLE "IndexBlockLink" ( + "IndexVolumeID" INTEGER NOT NULL, "BlockVolumeID" INTEGER NOT NULL ); diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs index b78148f3d..bce9a1a63 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -43,7 +43,7 @@ namespace Duplicati.Library.Main.ForestHash.Database private readonly System.Data.IDbCommand m_updateblockCommand; private readonly System.Data.IDbCommand m_insertfileOperationCommand; - private readonly System.Data.IDbCommand m_insertShadowBlockLink; + private readonly System.Data.IDbCommand m_insertIndexBlockLink; private HashDatabaseProtector m_blockHashLookup; private HashDatabaseProtector m_fileHashLookup; @@ -81,7 +81,7 @@ namespace Duplicati.Library.Main.ForestHash.Database m_selectfileSimpleCommand = m_connection.CreateCommand(); m_selectfileHashCommand = m_connection.CreateCommand(); m_findmetadatasetProbeCommand = m_connection.CreateCommand(); - m_insertShadowBlockLink = m_connection.CreateCommand(); + m_insertIndexBlockLink = m_connection.CreateCommand(); m_findblockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?"; m_findblockCommand.AddParameters(2); @@ -139,8 +139,8 @@ namespace Duplicati.Library.Main.ForestHash.Database m_updateblockCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? "; m_updateblockCommand.AddParameters(3); - m_insertShadowBlockLink.CommandText = @"INSERT INTO ""ShadowBlockLink"" (""ShadowVolumeID"", ""BlockVolumeID"") VALUES (?, ?)"; - m_insertShadowBlockLink.AddParameters(2); + m_insertIndexBlockLink.CommandText = @"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)"; + m_insertIndexBlockLink.AddParameters(2); if (options.FhBlockHashLookupMemory > 0) m_blockHashLookup = new HashDatabaseProtector(HASH_GUESS_SIZE, (ulong)options.FhBlockHashLookupMemory); @@ -793,12 +793,12 @@ namespace Duplicati.Library.Main.ForestHash.Database } } - public void AddShadowBlockLink(long shadowVolumeID, long blockVolumeID, System.Data.IDbTransaction transaction) + public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, System.Data.IDbTransaction transaction) { - m_insertShadowBlockLink.Transaction = transaction; - m_insertShadowBlockLink.SetParameterValue(0, shadowVolumeID); - m_insertShadowBlockLink.SetParameterValue(1, blockVolumeID); - m_insertShadowBlockLink.ExecuteNonQuery(); + m_insertIndexBlockLink.Transaction = transaction; + m_insertIndexBlockLink.SetParameterValue(0, indexVolumeID); + m_insertIndexBlockLink.SetParameterValue(1, blockVolumeID); + m_insertIndexBlockLink.ExecuteNonQuery(); } } diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalCleanupDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalCleanupDatabase.cs index 50d1814b5..6da566f99 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalCleanupDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalCleanupDatabase.cs @@ -153,10 +153,10 @@ namespace Duplicati.Library.Main.ForestHash.Database yield return new RemoteVolume(rd.GetValue(0).ToString(), rd.GetValue(1).ToString(), Convert.ToInt64(rd.GetValue(2))); } - public IEnumerable GetBlockVolumesFromShadowName(string name) + public IEnumerable GetBlockVolumesFromIndexName(string name) { using(var cmd = m_connection.CreateCommand()) - using(var rd = cmd.ExecuteReader(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""ShadowBlockLink"" WHERE ""ShadowVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = ?))", name)) + using(var rd = cmd.ExecuteReader(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = ?))", name)) while (rd.Read()) yield return new RemoteVolume(rd.GetValue(0).ToString(), rd.GetValue(1).ToString(), Convert.ToInt64(rd.GetValue(2))); } diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs index 40c084845..80433087e 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs @@ -226,7 +226,7 @@ namespace Duplicati.Library.Main.ForestHash.Database var volumeid = GetRemoteVolumeID(name, tr.Parent); // If the volume is a block volume, this will update the crosslink table, otherwise nothing will happen - deletecmd.ExecuteNonQuery(@"DELETE FROM ""ShadowBlockLink"" WHERE ""BlockVolumeID"" = ? ", volumeid); + deletecmd.ExecuteNonQuery(@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" = ? ", volumeid); var subQuery = @"(SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" = ?)"; diff --git a/Duplicati/Library/Main/ForestHash/Database/LocalDeleteDatabase.cs b/Duplicati/Library/Main/ForestHash/Database/LocalDeleteDatabase.cs index 4560149ed..ee92e9cb4 100644 --- a/Duplicati/Library/Main/ForestHash/Database/LocalDeleteDatabase.cs +++ b/Duplicati/Library/Main/ForestHash/Database/LocalDeleteDatabase.cs @@ -438,32 +438,32 @@ namespace Duplicati.Library.Main.ForestHash.Database { using(var cmd = m_connection.CreateCommand()) { - // Although the generated shadow volumes are always in pairs, + // Although the generated index volumes are always in pairs, // this code handles many-to-many relations between - // shadow files and block volumes, should this be added later + // index files and block volumes, should this be added later var lookupBlock = new Dictionary>(); - var lookupShadow = new Dictionary>(); + var lookupIndexfiles = new Dictionary>(); cmd.Transaction = transaction; - using(var rd = cmd.ExecuteReader(@"SELECT ""C"".""Name"", ""B"".""Name"", ""B"".""Hash"", ""B"".""Size"" FROM ""ShadowBlockLink"" A, ""RemoteVolume"" B, ""RemoteVolume"" C WHERE ""A"".""ShadowVolumeID"" = ""B"".""ID"" AND ""A"".""BlockVolumeID"" = ""C"".""ID"" ")) + using(var rd = cmd.ExecuteReader(@"SELECT ""C"".""Name"", ""B"".""Name"", ""B"".""Hash"", ""B"".""Size"" FROM ""IndexBlockLink"" A, ""RemoteVolume"" B, ""RemoteVolume"" C WHERE ""A"".""IndexVolumeID"" = ""B"".""ID"" AND ""A"".""BlockVolumeID"" = ""C"".""ID"" ")) while(rd.Read()) { var name = rd.GetValue(0).ToString(); - List shadowList; - if (!lookupBlock.TryGetValue(name, out shadowList)) + List indexfileList; + if (!lookupBlock.TryGetValue(name, out indexfileList)) { - shadowList = new List(); - lookupBlock.Add(name, shadowList); + indexfileList = new List(); + lookupBlock.Add(name, indexfileList); } var v = new RemoteVolume(rd.GetValue(1).ToString(), rd.GetValue(2).ToString(), Convert.ToInt64(rd.GetValue(3))); - shadowList.Add(v); + indexfileList.Add(v); List blockList; - if (!lookupShadow.TryGetValue(v.Name, out blockList)) + if (!lookupIndexfiles.TryGetValue(v.Name, out blockList)) { blockList = new List(); - lookupShadow.Add(v.Name, blockList); + lookupIndexfiles.Add(v.Name, blockList); } blockList.Add(name); } @@ -472,15 +472,15 @@ namespace Duplicati.Library.Main.ForestHash.Database { // Return the input yield return r; - List shadowList; - if (lookupBlock.TryGetValue(r.Name, out shadowList)) - foreach(var sh in shadowList) + List indexfileList; + if (lookupBlock.TryGetValue(r.Name, out indexfileList)) + foreach(var sh in indexfileList) { List backref; - if (lookupShadow.TryGetValue(sh.Name, out backref)) + if (lookupIndexfiles.TryGetValue(sh.Name, out backref)) { //If this is the last reference, - // remove the shadow file as well + // remove the index file as well if (backref.Remove(r.Name) && backref.Count == 0) yield return sh; } diff --git a/Duplicati/Library/Main/ForestHash/Enums.cs b/Duplicati/Library/Main/ForestHash/Enums.cs index 1ed900457..134e3eb5d 100644 --- a/Duplicati/Library/Main/ForestHash/Enums.cs +++ b/Duplicati/Library/Main/ForestHash/Enums.cs @@ -1,94 +1,94 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace Duplicati.Library.Main.ForestHash -{ - /// - /// Describes all states a remote volume can have - /// - public enum RemoteVolumeState - { - /// - /// Indicates that the remote volume is being created - /// - Temporary, - - /// - /// Indicates that the remote volume is being uploaded - /// - Uploading, - - /// - /// Indicates that the remote volume has been uploaded - /// - Uploaded, - - /// - /// Indicates that the remote volume has been uploaded, - /// and seen by a list operation - /// - Verified, - - /// - /// Indicates that the remote volume has been uploaded, - /// but is now missing from the remote location - /// - Missing, - - /// - /// Indicattes that the remote volume should be deleted - /// - Deleting, - - /// - /// Indicates that the remote volume was successfully - /// deleted from the remote location - /// - Deleted - } - - /// - /// Describes the known remote volume types - /// - public enum RemoteVolumeType - { - /// - /// Contains data blocks - /// - Blocks, - /// - /// Contains file lists - /// - Files, - /// - /// Contains redundant lookup information - /// - Shadow - } - - public enum FilelistEntryType - { - /// - /// The actual type of the entry could not be determined - /// - Unknown, - /// - /// The entry is a plain file - /// - File, - /// - /// The entry is a folder - /// - Folder, - /// - /// The entry is an alternate data stream, or resource/data fork - /// - AlternateStream, - /// - /// The entry is a symbolic link - /// - Symlink - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Duplicati.Library.Main.ForestHash +{ + /// + /// Describes all states a remote volume can have + /// + public enum RemoteVolumeState + { + /// + /// Indicates that the remote volume is being created + /// + Temporary, + + /// + /// Indicates that the remote volume is being uploaded + /// + Uploading, + + /// + /// Indicates that the remote volume has been uploaded + /// + Uploaded, + + /// + /// Indicates that the remote volume has been uploaded, + /// and seen by a list operation + /// + Verified, + + /// + /// Indicates that the remote volume has been uploaded, + /// but is now missing from the remote location + /// + Missing, + + /// + /// Indicattes that the remote volume should be deleted + /// + Deleting, + + /// + /// Indicates that the remote volume was successfully + /// deleted from the remote location + /// + Deleted + } + + /// + /// Describes the known remote volume types + /// + public enum RemoteVolumeType + { + /// + /// Contains data blocks + /// + Blocks, + /// + /// Contains file lists + /// + Files, + /// + /// Contains redundant lookup information + /// + Index + } + + public enum FilelistEntryType + { + /// + /// The actual type of the entry could not be determined + /// + Unknown, + /// + /// The entry is a plain file + /// + File, + /// + /// The entry is a folder + /// + Folder, + /// + /// The entry is an alternate data stream, or resource/data fork + /// + AlternateStream, + /// + /// The entry is a symbolic link + /// + Symlink + } +} diff --git a/Duplicati/Library/Main/ForestHash/FhBackend.cs b/Duplicati/Library/Main/ForestHash/FhBackend.cs index 5a3103e28..57296abeb 100644 --- a/Duplicati/Library/Main/ForestHash/FhBackend.cs +++ b/Duplicati/Library/Main/ForestHash/FhBackend.cs @@ -38,21 +38,21 @@ namespace Duplicati.Library.Main.ForestHash public object Result; public string Hash; public long Size; - public ShadowVolumeWriter Shadow; + public IndexVolumeWriter Indexfile; private System.Threading.ManualResetEvent DoneEvent; - public FileEntryItem(OperationType operation, string remotefilename, ShadowVolumeWriter shadow = null) + public FileEntryItem(OperationType operation, string remotefilename, IndexVolumeWriter indexfile = null) { Operation = operation; RemoteFilename = remotefilename; - Shadow = shadow; + Indexfile = indexfile; DoneEvent = new System.Threading.ManualResetEvent(false); } - public FileEntryItem(OperationType operation, string remotefilename, long size, string hash, ShadowVolumeWriter shadow = null) - : this(operation, remotefilename, shadow) + public FileEntryItem(OperationType operation, string remotefilename, long size, string hash, IndexVolumeWriter indexfile = null) + : this(operation, remotefilename, indexfile) { Size = size; Hash = hash; @@ -370,11 +370,11 @@ namespace Duplicati.Library.Main.ForestHash if (item.UpdateHashAndSize(m_options)) m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash); - if (item.Shadow != null) + if (item.Indexfile != null) { - item.Shadow.FinishVolume(item.Hash, item.Size); - item.Shadow.Close(); - item.Shadow = null; + item.Indexfile.FinishVolume(item.Hash, item.Size); + item.Indexfile.Close(); + item.Indexfile = null; } m_stats.AddNumberOfRemoteCalls(1); @@ -558,14 +558,14 @@ namespace Duplicati.Library.Main.ForestHash m_stats.LogMessage("Created target folder"); } - public void Put(VolumeWriterBase item, ShadowVolumeWriter shadow = null) + public void Put(VolumeWriterBase item, IndexVolumeWriter indexfile = null) { if (m_lastException != null) throw m_lastException; item.Close(); m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, -1, null); - var req = new FileEntryItem(OperationType.Put, item.RemoteFilename, shadow); + var req = new FileEntryItem(OperationType.Put, item.RemoteFilename, indexfile); req.LocalFilename = item.LocalFilename; if (m_queue.Enqueue(req) && !m_options.AsynchronousUpload) @@ -574,11 +574,11 @@ namespace Duplicati.Library.Main.ForestHash if (m_lastException != null) throw m_lastException; - if (shadow != null) + if (indexfile != null) { - m_db.LogDbUpdate(shadow.RemoteFilename, RemoteVolumeState.Uploading, -1, null); - var req2 = new FileEntryItem(OperationType.Put, shadow.RemoteFilename); - req2.LocalFilename = shadow.LocalFilename; + m_db.LogDbUpdate(indexfile.RemoteFilename, RemoteVolumeState.Uploading, -1, null); + var req2 = new FileEntryItem(OperationType.Put, indexfile.RemoteFilename); + req2.LocalFilename = indexfile.LocalFilename; if (m_queue.Enqueue(req2) && !m_options.AsynchronousUpload) req2.WaitForComplete(); diff --git a/Duplicati/Library/Main/ForestHash/FhOptions.cs b/Duplicati/Library/Main/ForestHash/FhOptions.cs index 7a0c017b9..132e42363 100644 --- a/Duplicati/Library/Main/ForestHash/FhOptions.cs +++ b/Duplicati/Library/Main/ForestHash/FhOptions.cs @@ -62,7 +62,7 @@ namespace Duplicati.Library.Main.ForestHash new CommandLineArgument("fh-deleted-fileset", CommandLineArgument.ArgumentType.Path, Strings.FhOptions.FhdeletedfilesetShort, string.Format(Strings.FhOptions.FhdeletedfilesetLong, "fh-changed-fileset")), new CommandLineArgument("fh-max-wastesize", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhmaxwastesizeShort, Strings.FhOptions.FhmaxwastesizeLong), - new CommandLineArgument("fh-no-shadowfiles", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnoshadowfilesShort, Strings.FhOptions.FhnoshadowfilesLong, "false"), + new CommandLineArgument("fh-no-indexfiles", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnoindexfilesShort, Strings.FhOptions.FhnoindexfilesLong, "false"), new CommandLineArgument("fh-no-backendverification", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnobackendverificationShort, Strings.FhOptions.FhnobackendverificationLong, "false"), new CommandLineArgument("fh-dry-run", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhdryrunShort, Strings.FhOptions.FhdryrunLong, "false"), @@ -251,11 +251,11 @@ namespace Duplicati.Library.Main.ForestHash } /// - /// Gets a flag indicating if shadow files should be omitted + /// Gets a flag indicating if index files should be omitted /// - public bool FhNoShadowfiles + public bool FhNoIndexfiles { - get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-shadowfiles"); } + get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-indexfiles"); } } /// diff --git a/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs index ba2812bb0..a631101b6 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs @@ -24,7 +24,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation private LocalBackupDatabase m_database; private System.Data.IDbTransaction m_transaction; private BlockVolumeWriter m_blockvolume; - private ShadowVolumeWriter m_shadowvolume; + private IndexVolumeWriter m_indexvolume; private FilesetVolumeWriter m_filesetvolume; private Snapshots.ISnapshotService m_snapshot; @@ -132,10 +132,10 @@ namespace Duplicati.Library.Main.ForestHash.Operation m_blockvolume = new BlockVolumeWriter(m_options); m_blockvolume.VolumeID = m_database.RegisterRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_transaction); - if (!m_options.FhNoShadowfiles) + if (!m_options.FhNoIndexfiles) { - m_shadowvolume = new ShadowVolumeWriter(m_options); - m_shadowvolume.VolumeID = m_database.RegisterRemoteVolume(m_shadowvolume.RemoteFilename, RemoteVolumeType.Shadow, RemoteVolumeState.Temporary, m_transaction); + m_indexvolume = new IndexVolumeWriter(m_options); + m_indexvolume.VolumeID = m_database.RegisterRemoteVolume(m_indexvolume.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_transaction); } if (m_options.FhChangedFilelist != null && m_options.FhChangedFilelist.Length >= 1) @@ -165,29 +165,29 @@ namespace Duplicati.Library.Main.ForestHash.Operation if (m_options.FhDryrun) { m_stat.LogMessage("[Dryrun] Would upload block volume: {0}, size: {1}", m_blockvolume.RemoteFilename, Utility.Utility.FormatSizeString(new FileInfo(m_blockvolume.LocalFilename).Length)); - if (m_shadowvolume != null) + if (m_indexvolume != null) { - UpdateShadowVolume(); - m_shadowvolume.FinishVolume(Utility.Utility.CalculateHash(m_blockvolume.LocalFilename), new FileInfo(m_blockvolume.LocalFilename).Length); - m_stat.LogMessage("[Dryrun] Would upload shadow volume: {0}, size: {1}", m_shadowvolume.RemoteFilename, Utility.Utility.FormatSizeString(new FileInfo(m_shadowvolume.LocalFilename).Length)); + UpdateIndexVolume(); + m_indexvolume.FinishVolume(Utility.Utility.CalculateHash(m_blockvolume.LocalFilename), new FileInfo(m_blockvolume.LocalFilename).Length); + m_stat.LogMessage("[Dryrun] Would upload index volume: {0}, size: {1}", m_indexvolume.RemoteFilename, Utility.Utility.FormatSizeString(new FileInfo(m_indexvolume.LocalFilename).Length)); } } else { m_database.UpdateRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction); - UpdateShadowVolume(); + UpdateIndexVolume(); m_transaction.Commit(); m_transaction = m_database.BeginTransaction(); - m_backend.Put(m_blockvolume, m_shadowvolume); + m_backend.Put(m_blockvolume, m_indexvolume); } } else { m_database.RemoveRemoteVolume(m_blockvolume.RemoteFilename, m_transaction); - if (m_shadowvolume != null) - m_database.RemoveRemoteVolume(m_shadowvolume.RemoteFilename, m_transaction); + if (m_indexvolume != null) + m_database.RemoveRemoteVolume(m_indexvolume.RemoteFilename, m_transaction); } m_database.UpdateChangeStatistics(m_stat); @@ -484,13 +484,13 @@ namespace Duplicati.Library.Main.ForestHash.Operation m_blockvolume.Dispose(); m_blockvolume = null; - if (m_shadowvolume != null) + if (m_indexvolume != null) { - UpdateShadowVolume(); - m_shadowvolume.FinishVolume(Utility.Utility.CalculateHash(m_shadowvolume.LocalFilename), new FileInfo(m_shadowvolume.LocalFilename).Length); - m_stat.LogMessage("[Dryrun] Would upload shadow volume: {0}, size: {1}", m_shadowvolume.RemoteFilename, Utility.Utility.FormatSizeString(new FileInfo(m_shadowvolume.LocalFilename).Length)); - m_shadowvolume.Dispose(); - m_shadowvolume = null; + UpdateIndexVolume(); + m_indexvolume.FinishVolume(Utility.Utility.CalculateHash(m_indexvolume.LocalFilename), new FileInfo(m_indexvolume.LocalFilename).Length); + m_stat.LogMessage("[Dryrun] Would upload index volume: {0}, size: {1}", m_indexvolume.RemoteFilename, Utility.Utility.FormatSizeString(new FileInfo(m_indexvolume.LocalFilename).Length)); + m_indexvolume.Dispose(); + m_indexvolume = null; } } else @@ -498,7 +498,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation //When uploading a new volume, we register the volumes and then flush the transaction // this ensures that the local database and remote storage are as closely related as possible m_database.UpdateRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction); - UpdateShadowVolume(); + UpdateIndexVolume(); m_backend.FlushDbMessages(m_database, m_transaction); m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN); @@ -506,18 +506,18 @@ namespace Duplicati.Library.Main.ForestHash.Operation m_transaction.Commit(); m_transaction = m_database.BeginTransaction(); - m_backend.Put(m_blockvolume, m_shadowvolume); + m_backend.Put(m_blockvolume, m_indexvolume); m_blockvolume = null; - m_shadowvolume = null; + m_indexvolume = null; } m_blockvolume = new BlockVolumeWriter(m_options); m_blockvolume.VolumeID = m_database.RegisterRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_transaction); - if (!m_options.FhNoShadowfiles) + if (!m_options.FhNoIndexfiles) { - m_shadowvolume = new ShadowVolumeWriter(m_options); - m_shadowvolume.VolumeID = m_database.RegisterRemoteVolume(m_shadowvolume.RemoteFilename, RemoteVolumeType.Shadow, RemoteVolumeState.Temporary, m_transaction); + m_indexvolume = new IndexVolumeWriter(m_options); + m_indexvolume.VolumeID = m_database.RegisterRemoteVolume(m_indexvolume.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_transaction); } } @@ -602,17 +602,17 @@ namespace Duplicati.Library.Main.ForestHash.Operation } - private void UpdateShadowVolume() + private void UpdateIndexVolume() { - if (m_shadowvolume != null) + if (m_indexvolume != null) { - m_database.AddShadowBlockLink(m_shadowvolume.VolumeID, m_blockvolume.VolumeID, m_transaction); - m_shadowvolume.StartVolume(m_blockvolume.RemoteFilename); + m_database.AddIndexBlockLink(m_indexvolume.VolumeID, m_blockvolume.VolumeID, m_transaction); + m_indexvolume.StartVolume(m_blockvolume.RemoteFilename); foreach(var b in m_database.GetBlocks(m_blockvolume.VolumeID)) - m_shadowvolume.AddBlock(b.Hash, b.Size); + m_indexvolume.AddBlock(b.Hash, b.Size); - m_database.UpdateRemoteVolume(m_shadowvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction); + m_database.UpdateRemoteVolume(m_indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction); } } @@ -632,11 +632,11 @@ namespace Duplicati.Library.Main.ForestHash.Operation finally { m_blockvolume = null; } } - if (m_shadowvolume != null) + if (m_indexvolume != null) { - try { m_shadowvolume.Dispose(); } - catch (Exception ex) { m_stat.LogError("Failed disposing shadow volume", ex); } - finally { m_shadowvolume = null; } + try { m_indexvolume.Dispose(); } + catch (Exception ex) { m_stat.LogError("Failed disposing index volume", ex); } + finally { m_indexvolume = null; } } m_stat.EndTime = DateTime.Now; diff --git a/Duplicati/Library/Main/ForestHash/Operation/CleanupHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/CleanupHandler.cs index 3ff167796..c516c335b 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/CleanupHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/CleanupHandler.cs @@ -94,12 +94,12 @@ namespace Duplicati.Library.Main.ForestHash.Operation backend.Put(w); } } - else if (n.Type == RemoteVolumeType.Shadow) + else if (n.Type == RemoteVolumeType.Index) { - var w = new ShadowVolumeWriter(m_options); + var w = new IndexVolumeWriter(m_options); w.SetRemoteFilename(n.Name); - foreach(var blockvolume in db.GetBlockVolumesFromShadowName(n.Name)) + foreach(var blockvolume in db.GetBlockVolumesFromIndexName(n.Name)) { w.StartVolume(blockvolume.Name); var volumeid = db.GetRemoteVolumeID(blockvolume.Name); @@ -113,7 +113,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation w.Close(); if (m_options.FhDryrun) - m_stat.LogMessage("[Dryrun] would re-upload shadow file {0}, with size {1}, previous size {2}", n.Name, Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Utility.Utility.FormatSizeString(n.Size)); + m_stat.LogMessage("[Dryrun] would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Utility.Utility.FormatSizeString(n.Size)); else { db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null); diff --git a/Duplicati/Library/Main/ForestHash/Operation/CompactHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/CompactHandler.cs index 2b78f122d..c833667a9 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/CompactHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/CompactHandler.cs @@ -73,12 +73,12 @@ namespace Duplicati.Library.Main.ForestHash.Operation BlockVolumeWriter newvol = new BlockVolumeWriter(m_options); newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction); - ShadowVolumeWriter newvolshadow = null; - if (!m_options.FhNoShadowfiles) + IndexVolumeWriter newvolindex = null; + if (!m_options.FhNoIndexfiles) { - newvolshadow = new ShadowVolumeWriter(m_options); - db.RegisterRemoteVolume(newvolshadow.RemoteFilename, RemoteVolumeType.Shadow, RemoteVolumeState.Temporary, transaction); - newvolshadow.StartVolume(newvol.RemoteFilename); + newvolindex = new IndexVolumeWriter(m_options); + db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction); + newvolindex.StartVolume(newvol.RemoteFilename); } long blocksInVolume = 0; @@ -128,8 +128,8 @@ namespace Duplicati.Library.Main.ForestHash.Operation throw new Exception("Size mismatch problem, {0} vs {1}"); newvol.AddBlock(e.Key, buffer, s, Duplicati.Library.Interface.CompressionHint.Compressible); - if (newvolshadow != null) - newvolshadow.AddBlock(e.Key, e.Value); + if (newvolindex != null) + newvolindex.AddBlock(e.Key, e.Value); db.MoveBlockToNewVolume(e.Key, e.Value, newvol.VolumeID, transaction); blocksInVolume++; @@ -137,11 +137,11 @@ namespace Duplicati.Library.Main.ForestHash.Operation if (newvol.Filesize > m_options.VolumeSize) { uploadedVolumes.Add(new KeyValuePair(newvol.RemoteFilename, new System.IO.FileInfo(newvol.LocalFilename).Length)); - if (newvolshadow != null) - uploadedVolumes.Add(new KeyValuePair(newvolshadow.RemoteFilename, new System.IO.FileInfo(newvolshadow.LocalFilename).Length)); + if (newvolindex != null) + uploadedVolumes.Add(new KeyValuePair(newvolindex.RemoteFilename, new System.IO.FileInfo(newvolindex.LocalFilename).Length)); if (m_options.Force && !m_options.FhDryrun) - backend.Put(newvol, newvolshadow); + backend.Put(newvol, newvolindex); else m_stat.LogMessage("[Dryrun] - Would upload generated blockset of size {0}", Utility.Utility.FormatSizeString(new System.IO.FileInfo(newvol.LocalFilename).Length)); @@ -149,11 +149,11 @@ namespace Duplicati.Library.Main.ForestHash.Operation newvol = new BlockVolumeWriter(m_options); newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction); - if (!m_options.FhNoShadowfiles) + if (!m_options.FhNoIndexfiles) { - newvolshadow = new ShadowVolumeWriter(m_options); - db.RegisterRemoteVolume(newvolshadow.RemoteFilename, RemoteVolumeType.Shadow, RemoteVolumeState.Temporary, transaction); - newvolshadow.StartVolume(newvol.RemoteFilename); + newvolindex = new IndexVolumeWriter(m_options); + db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction); + newvolindex.StartVolume(newvol.RemoteFilename); } blocksInVolume = 0; @@ -176,20 +176,20 @@ namespace Duplicati.Library.Main.ForestHash.Operation if (blocksInVolume > 0) { uploadedVolumes.Add(new KeyValuePair(newvol.RemoteFilename, new System.IO.FileInfo(newvol.LocalFilename).Length)); - if (newvolshadow != null) - uploadedVolumes.Add(new KeyValuePair(newvolshadow.RemoteFilename, new System.IO.FileInfo(newvolshadow.LocalFilename).Length)); + if (newvolindex != null) + uploadedVolumes.Add(new KeyValuePair(newvolindex.RemoteFilename, new System.IO.FileInfo(newvolindex.LocalFilename).Length)); if (m_options.Force && !m_options.FhDryrun) - backend.Put(newvol, newvolshadow); + backend.Put(newvol, newvolindex); else m_stat.LogMessage("[Dryrun] - Would upload generated blockset of size {0}", Utility.Utility.FormatSizeString(new System.IO.FileInfo(newvol.LocalFilename).Length)); } else { db.RemoveRemoteVolume(newvol.RemoteFilename, transaction); - if (newvolshadow != null) + if (newvolindex != null) { - db.RemoveRemoteVolume(newvolshadow.RemoteFilename, transaction); - newvolshadow.FinishVolume(null, 0); + db.RemoveRemoteVolume(newvolindex.RemoteFilename, transaction); + newvolindex.FinishVolume(null, 0); } } } diff --git a/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs b/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs index df90462dd..8f65f617d 100644 --- a/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs +++ b/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs @@ -112,25 +112,25 @@ namespace Duplicati.Library.Main.ForestHash.Operation foreach (var fl in remotefiles) volumeIds[fl.File.Name] = backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded); - //We grab all shadow files, and update the block table + //We grab all index files, and update the block table using (var tr = restoredb.BeginTransaction()) { - var shadowfiles = + var indexfiles = from n in remotefiles - where n.FileType == RemoteVolumeType.Shadow + where n.FileType == RemoteVolumeType.Index select new RemoteVolume(n.File) as IRemoteVolume; - foreach (var sf in new AsyncDownloader(shadowfiles.ToList(), backend)) + foreach (var sf in new AsyncDownloader(indexfiles.ToList(), backend)) using (var tmpfile = sf.Value) { if (sf.Key.Hash != null && sf.Key.Size > 0) backupdb.UpdateRemoteVolume(sf.Key.Name, RemoteVolumeState.Verified, sf.Key.Size, sf.Key.Hash, tr); - using (var svr = new ShadowVolumeReader(RestoreHandler.GetCompressionModule(sf.Key.Name), tmpfile, m_options, hashsize)) + using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Key.Name), tmpfile, m_options, hashsize)) { ForestHash.VerifyParameters(restoredb, m_options); - //If there are blocklists in the shadow file, update the blocklists + //If there are blocklists in the index file, update the blocklists foreach (var b in svr.BlockLists) restoredb.UpdateBlocklist(b.Hash, b.Blocklist, hashsize, tr); @@ -142,7 +142,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation backupdb.AddBlock(b.Key, b.Value, volumeID, tr); backupdb.UpdateRemoteVolume(a.Filename, RemoteVolumeState.Verified, a.Length, a.Hash, tr); - backupdb.AddShadowBlockLink(restoredb.GetRemoteVolumeID(sf.Key.Name), volumeID, tr); + backupdb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Key.Name), volumeID, tr); } } } diff --git a/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeReader.cs b/Duplicati/Library/Main/ForestHash/Volumes/IndexVolumeReader.cs similarity index 81% rename from Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeReader.cs rename to Duplicati/Library/Main/ForestHash/Volumes/IndexVolumeReader.cs index b07ebb5e7..3efdc8249 100644 --- a/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeReader.cs +++ b/Duplicati/Library/Main/ForestHash/Volumes/IndexVolumeReader.cs @@ -1,399 +1,399 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; -using Newtonsoft.Json; -using Duplicati.Library.Interface; - -namespace Duplicati.Library.Main.ForestHash.Volumes -{ - public class ShadowVolumeReader : VolumeReaderBase - { - public interface IShadowBlocklist - { - string Hash { get; } - long Length { get; } - IEnumerable Blocklist { get; } - } - - public interface IShadowBlockVolume - { - string Filename { get; } - long Length { get; } - string Hash { get; } - IEnumerable> Blocks { get; } - } - - private class ShadowBlockVolumeEnumerable : IEnumerable - { - private ICompression m_compression; - - public ShadowBlockVolumeEnumerable(ICompression compression) - { - m_compression = compression; - } - - private class ShadowBlockVolumeEnumerator : IEnumerator - { - private class BlockEnumerable : IEnumerable> - { - private class BlockEnumerator : IEnumerator> - { - private ICompression m_compression; - private string m_filename; - private KeyValuePair? m_current; - private System.IO.StreamReader m_stream; - private JsonReader m_reader; - private bool m_done; - private KeyValuePair? m_volumeProps = null; - - public BlockEnumerator(ICompression compression, string filename) - { - m_compression = compression; - m_filename = filename; - this.Reset(); - } - - public KeyValuePair Current - { - get { return m_current.Value; } - } - - public void Dispose() - { - if (m_reader != null) - this.ReadVolumeProps(); - - if (m_reader != null) - try { m_reader.Close(); } - finally { m_reader = null; } - - if (m_stream != null) - try { m_stream.Dispose(); } - finally { m_stream = null; } - } - - object System.Collections.IEnumerator.Current - { - get { return this.Current; } - } - - public bool MoveNext() - { - if (m_done) - return false; - - if (!m_reader.Read()) - throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading hashes")); - - if (m_reader.TokenType == JsonToken.EndArray) - { - m_done = true; - m_current = null; - return false; - } - - if (m_reader.TokenType != JsonToken.StartObject) - throw new InvalidDataException(string.Format("Invalid JSON, expected StartObject, but got {0}, {1}", m_reader.TokenType, m_reader.Value)); - - var hash = ReadJsonStringProperty(m_reader, "hash"); - var size = ReadJsonInt64Property(m_reader, "size"); - - m_current = new KeyValuePair(hash, size); - - while (m_reader.Read() && m_reader.TokenType != JsonToken.EndObject) - { /* skip */ } - - return true; - } - - public KeyValuePair ReadVolumeProps() - { - if (m_volumeProps != null) - return m_volumeProps.Value; - - while (this.MoveNext()) - { /*skip*/ } - - var hash = ReadJsonStringProperty(m_reader, "volumehash"); - var size = ReadJsonInt64Property(m_reader, "volumesize"); - - return (m_volumeProps = new KeyValuePair(hash, size)).Value; - } - - public void Reset() - { - this.Dispose(); - m_stream = new StreamReader(m_compression.OpenRead(m_filename)); - m_reader = new JsonTextReader(m_stream); - SkipJsonToken(m_reader, JsonToken.StartObject); - var p = SkipJsonToken(m_reader, JsonToken.PropertyName); - if (p == null || p.ToString() != "blocks") - throw new InvalidDataException(string.Format("Invalid JSON, expected property \"blocks\", but got {0}, {1}", m_reader.TokenType, m_reader.Value)); - SkipJsonToken(m_reader, JsonToken.StartArray); - - m_current = null; - m_done = false; - } - } - - private ICompression m_compression; - private string m_filename; - private BlockEnumerator m_enumerator; - - public BlockEnumerable(ICompression compression, string filename) - { - m_compression = compression; - m_filename = filename; - } - - public KeyValuePair ReadVolumeProps() - { - if (m_enumerator == null) - m_enumerator = (BlockEnumerator)this.GetEnumerator(); - - return m_enumerator.ReadVolumeProps(); - } - - public IEnumerator> GetEnumerator() - { - if (m_enumerator != null) - throw new NotSupportedException("Cannot read block stream twice"); - - return m_enumerator = new BlockEnumerator(m_compression, m_filename); - } - - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() - { - return this.GetEnumerator(); - } - } - - private class ShadowBlockVolume : IShadowBlockVolume - { - private ICompression m_compression; - private string m_filename; - private long? m_length; - private string m_hash; - private BlockEnumerable m_blocks; - - public ShadowBlockVolume(ICompression compression, string filename) - { - m_compression = compression; - m_filename = filename; - } - - public string Filename { get { return m_filename.Substring(SHADOW_VOLUME_FOLDER.Length); } } - - private void ReadVolumeProps() - { - if (m_length == null || m_hash == null) - { - if (m_blocks == null) - m_blocks = (BlockEnumerable)this.Blocks; - - var kp = m_blocks.ReadVolumeProps(); - m_length = kp.Value; - m_hash = kp.Key; - } - } - - public long Length - { - get - { - this.ReadVolumeProps(); - return m_length.Value; - } - } - - public string Hash - { - get - { - this.ReadVolumeProps(); - return m_hash; - } - } - - public IEnumerable> Blocks - { - get - { - if (m_blocks != null) - throw new NotSupportedException("Cannot read Blocks twice"); - - return m_blocks = new BlockEnumerable(m_compression, m_filename); - } - } - } - - private ICompression m_compression; - private ShadowBlockVolume m_current; - private string[] m_files; - private long m_index; - - public ShadowBlockVolumeEnumerator(ICompression compression) - { - m_compression = compression; - this.Reset(); - } - - public IShadowBlockVolume Current - { - get { return m_current; } - } - - public void Dispose() - { - } - - object System.Collections.IEnumerator.Current - { - get { return this.Current; } - } - - public bool MoveNext() - { - if (m_index + 1 >= m_files.Length) - return false; - m_index++; - - while (m_index < m_files.Length && ParseFilename(m_files[m_index]) == null) - m_index++; - - m_current = new ShadowBlockVolume(m_compression, m_files[m_index]); - - return true; - } - - public void Reset() - { - m_files = m_compression.ListFiles(SHADOW_VOLUME_FOLDER); - m_index = -1; - m_current = null; - } - } - - public IEnumerator GetEnumerator() { return new ShadowBlockVolumeEnumerator(m_compression); } - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); } - } - - private class ShadowBlocklistEnumerable : IEnumerable - { - private ICompression m_compression; - private long m_hashsize; - - public ShadowBlocklistEnumerable(ICompression compression, long hashsize) - { - m_compression = compression; - m_hashsize = hashsize; - } - - private class ShadowBlocklistEnumerator : IEnumerator - { - private class ShadowBlocklist : IShadowBlocklist - { - private ICompression m_compression; - private string m_filename; - private long m_size; - private long m_hashsize; - - public ShadowBlocklist(ICompression compression, string filename, long size, long hashsize) - { - m_compression = compression; - m_filename = filename; - m_size = size; - m_hashsize = hashsize; - } - - public string Hash - { - //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, - // to prevent clashes with filename paths where forward slash has special meaning - get { return m_filename.Substring(SHADOW_BLOCKLIST_FOLDER.Length).Replace('-', '+').Replace('_', '-'); } - } - - public long Length - { - get { return m_size; } - } - - public IEnumerable Blocklist - { - get { return new BlocklistEnumerable(m_compression, m_filename, m_hashsize); } - } - } - - private ICompression m_compression; - private long m_index; - private KeyValuePair[] m_files; - private ShadowBlocklist m_current; - private long m_hashsize; - - public ShadowBlocklistEnumerator(ICompression compression, long hashsize) - { - m_compression = compression; - m_hashsize = hashsize; - this.Reset(); - } - - public IShadowBlocklist Current { get { return m_current; } } - public void Dispose() { } - object System.Collections.IEnumerator.Current { get { return this.Current; } } - - private readonly System.Text.RegularExpressions.Regex m_base64_urlsafe_detector = new System.Text.RegularExpressions.Regex("[a-zA-Z0-9-_]+={0,2}"); - private bool IsValidBase64Hash(string value, long hashsize) - { - if (value.Length != ((hashsize + 2) / 3) * 4) - return false; - - var m = m_base64_urlsafe_detector.Match(value); - return m.Success && m.Length == value.Length; - } - - public bool MoveNext() - { - if (m_index + 1 >= m_files.Length) - return false; - m_index++; - - while (m_index < m_files.Length && IsValidBase64Hash(m_files[m_index].Key, m_hashsize)) - m_index++; - - m_current = new ShadowBlocklist(m_compression, m_files[m_index].Key, m_files[m_index].Value, m_hashsize); - - return true; - } - - public void Reset() - { - m_files = m_compression.ListFilesWithSize(SHADOW_BLOCKLIST_FOLDER).ToArray(); - m_index = -1; - m_current = null; - } - } - - public IEnumerator GetEnumerator() { return new ShadowBlocklistEnumerator(m_compression, m_hashsize); } - System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); } - } - - private long m_hashsize; - - public ShadowVolumeReader(ICompression compression, FhOptions options, long hashsize) - : base(compression, options) - { - m_hashsize = hashsize; - } - - public ShadowVolumeReader(string compressor, string file, FhOptions options, long hashsize) - : base(compressor, file, options) - { - m_hashsize = hashsize; - } - - public IEnumerable Volumes { get { return new ShadowBlockVolumeEnumerable(m_compression); } } - public IEnumerable BlockLists { get { return new ShadowBlocklistEnumerable(m_compression, m_hashsize); } } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using Newtonsoft.Json; +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Main.ForestHash.Volumes +{ + public class IndexVolumeReader : VolumeReaderBase + { + public interface IIndexBlocklist + { + string Hash { get; } + long Length { get; } + IEnumerable Blocklist { get; } + } + + public interface IIndexBlockVolume + { + string Filename { get; } + long Length { get; } + string Hash { get; } + IEnumerable> Blocks { get; } + } + + private class IndexBlockVolumeEnumerable : IEnumerable + { + private ICompression m_compression; + + public IndexBlockVolumeEnumerable(ICompression compression) + { + m_compression = compression; + } + + private class IndexBlockVolumeEnumerator : IEnumerator + { + private class BlockEnumerable : IEnumerable> + { + private class BlockEnumerator : IEnumerator> + { + private ICompression m_compression; + private string m_filename; + private KeyValuePair? m_current; + private System.IO.StreamReader m_stream; + private JsonReader m_reader; + private bool m_done; + private KeyValuePair? m_volumeProps = null; + + public BlockEnumerator(ICompression compression, string filename) + { + m_compression = compression; + m_filename = filename; + this.Reset(); + } + + public KeyValuePair Current + { + get { return m_current.Value; } + } + + public void Dispose() + { + if (m_reader != null) + this.ReadVolumeProps(); + + if (m_reader != null) + try { m_reader.Close(); } + finally { m_reader = null; } + + if (m_stream != null) + try { m_stream.Dispose(); } + finally { m_stream = null; } + } + + object System.Collections.IEnumerator.Current + { + get { return this.Current; } + } + + public bool MoveNext() + { + if (m_done) + return false; + + if (!m_reader.Read()) + throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading hashes")); + + if (m_reader.TokenType == JsonToken.EndArray) + { + m_done = true; + m_current = null; + return false; + } + + if (m_reader.TokenType != JsonToken.StartObject) + throw new InvalidDataException(string.Format("Invalid JSON, expected StartObject, but got {0}, {1}", m_reader.TokenType, m_reader.Value)); + + var hash = ReadJsonStringProperty(m_reader, "hash"); + var size = ReadJsonInt64Property(m_reader, "size"); + + m_current = new KeyValuePair(hash, size); + + while (m_reader.Read() && m_reader.TokenType != JsonToken.EndObject) + { /* skip */ } + + return true; + } + + public KeyValuePair ReadVolumeProps() + { + if (m_volumeProps != null) + return m_volumeProps.Value; + + while (this.MoveNext()) + { /*skip*/ } + + var hash = ReadJsonStringProperty(m_reader, "volumehash"); + var size = ReadJsonInt64Property(m_reader, "volumesize"); + + return (m_volumeProps = new KeyValuePair(hash, size)).Value; + } + + public void Reset() + { + this.Dispose(); + m_stream = new StreamReader(m_compression.OpenRead(m_filename)); + m_reader = new JsonTextReader(m_stream); + SkipJsonToken(m_reader, JsonToken.StartObject); + var p = SkipJsonToken(m_reader, JsonToken.PropertyName); + if (p == null || p.ToString() != "blocks") + throw new InvalidDataException(string.Format("Invalid JSON, expected property \"blocks\", but got {0}, {1}", m_reader.TokenType, m_reader.Value)); + SkipJsonToken(m_reader, JsonToken.StartArray); + + m_current = null; + m_done = false; + } + } + + private ICompression m_compression; + private string m_filename; + private BlockEnumerator m_enumerator; + + public BlockEnumerable(ICompression compression, string filename) + { + m_compression = compression; + m_filename = filename; + } + + public KeyValuePair ReadVolumeProps() + { + if (m_enumerator == null) + m_enumerator = (BlockEnumerator)this.GetEnumerator(); + + return m_enumerator.ReadVolumeProps(); + } + + public IEnumerator> GetEnumerator() + { + if (m_enumerator != null) + throw new NotSupportedException("Cannot read block stream twice"); + + return m_enumerator = new BlockEnumerator(m_compression, m_filename); + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return this.GetEnumerator(); + } + } + + private class IndexBlockVolume : IIndexBlockVolume + { + private ICompression m_compression; + private string m_filename; + private long? m_length; + private string m_hash; + private BlockEnumerable m_blocks; + + public IndexBlockVolume(ICompression compression, string filename) + { + m_compression = compression; + m_filename = filename; + } + + public string Filename { get { return m_filename.Substring(INDEX_VOLUME_FOLDER.Length); } } + + private void ReadVolumeProps() + { + if (m_length == null || m_hash == null) + { + if (m_blocks == null) + m_blocks = (BlockEnumerable)this.Blocks; + + var kp = m_blocks.ReadVolumeProps(); + m_length = kp.Value; + m_hash = kp.Key; + } + } + + public long Length + { + get + { + this.ReadVolumeProps(); + return m_length.Value; + } + } + + public string Hash + { + get + { + this.ReadVolumeProps(); + return m_hash; + } + } + + public IEnumerable> Blocks + { + get + { + if (m_blocks != null) + throw new NotSupportedException("Cannot read Blocks twice"); + + return m_blocks = new BlockEnumerable(m_compression, m_filename); + } + } + } + + private ICompression m_compression; + private IndexBlockVolume m_current; + private string[] m_files; + private long m_index; + + public IndexBlockVolumeEnumerator(ICompression compression) + { + m_compression = compression; + this.Reset(); + } + + public IIndexBlockVolume Current + { + get { return m_current; } + } + + public void Dispose() + { + } + + object System.Collections.IEnumerator.Current + { + get { return this.Current; } + } + + public bool MoveNext() + { + if (m_index + 1 >= m_files.Length) + return false; + m_index++; + + while (m_index < m_files.Length && ParseFilename(m_files[m_index]) == null) + m_index++; + + m_current = new IndexBlockVolume(m_compression, m_files[m_index]); + + return true; + } + + public void Reset() + { + m_files = m_compression.ListFiles(INDEX_VOLUME_FOLDER); + m_index = -1; + m_current = null; + } + } + + public IEnumerator GetEnumerator() { return new IndexBlockVolumeEnumerator(m_compression); } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); } + } + + private class IndexBlocklistEnumerable : IEnumerable + { + private ICompression m_compression; + private long m_hashsize; + + public IndexBlocklistEnumerable(ICompression compression, long hashsize) + { + m_compression = compression; + m_hashsize = hashsize; + } + + private class IndexBlocklistEnumerator : IEnumerator + { + private class IndexBlocklist : IIndexBlocklist + { + private ICompression m_compression; + private string m_filename; + private long m_size; + private long m_hashsize; + + public IndexBlocklist(ICompression compression, string filename, long size, long hashsize) + { + m_compression = compression; + m_filename = filename; + m_size = size; + m_hashsize = hashsize; + } + + public string Hash + { + //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, + // to prevent clashes with filename paths where forward slash has special meaning + get { return m_filename.Substring(INDEX_BLOCKLIST_FOLDER.Length).Replace('-', '+').Replace('_', '-'); } + } + + public long Length + { + get { return m_size; } + } + + public IEnumerable Blocklist + { + get { return new BlocklistEnumerable(m_compression, m_filename, m_hashsize); } + } + } + + private ICompression m_compression; + private long m_index; + private KeyValuePair[] m_files; + private IndexBlocklist m_current; + private long m_hashsize; + + public IndexBlocklistEnumerator(ICompression compression, long hashsize) + { + m_compression = compression; + m_hashsize = hashsize; + this.Reset(); + } + + public IIndexBlocklist Current { get { return m_current; } } + public void Dispose() { } + object System.Collections.IEnumerator.Current { get { return this.Current; } } + + private readonly System.Text.RegularExpressions.Regex m_base64_urlsafe_detector = new System.Text.RegularExpressions.Regex("[a-zA-Z0-9-_]+={0,2}"); + private bool IsValidBase64Hash(string value, long hashsize) + { + if (value.Length != ((hashsize + 2) / 3) * 4) + return false; + + var m = m_base64_urlsafe_detector.Match(value); + return m.Success && m.Length == value.Length; + } + + public bool MoveNext() + { + if (m_index + 1 >= m_files.Length) + return false; + m_index++; + + while (m_index < m_files.Length && IsValidBase64Hash(m_files[m_index].Key, m_hashsize)) + m_index++; + + m_current = new IndexBlocklist(m_compression, m_files[m_index].Key, m_files[m_index].Value, m_hashsize); + + return true; + } + + public void Reset() + { + m_files = m_compression.ListFilesWithSize(INDEX_BLOCKLIST_FOLDER).ToArray(); + m_index = -1; + m_current = null; + } + } + + public IEnumerator GetEnumerator() { return new IndexBlocklistEnumerator(m_compression, m_hashsize); } + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); } + } + + private long m_hashsize; + + public IndexVolumeReader(ICompression compression, FhOptions options, long hashsize) + : base(compression, options) + { + m_hashsize = hashsize; + } + + public IndexVolumeReader(string compressor, string file, FhOptions options, long hashsize) + : base(compressor, file, options) + { + m_hashsize = hashsize; + } + + public IEnumerable Volumes { get { return new IndexBlockVolumeEnumerable(m_compression); } } + public IEnumerable BlockLists { get { return new IndexBlocklistEnumerable(m_compression, m_hashsize); } } + } +} diff --git a/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeWriter.cs b/Duplicati/Library/Main/ForestHash/Volumes/IndexVolumeWriter.cs similarity index 83% rename from Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeWriter.cs rename to Duplicati/Library/Main/ForestHash/Volumes/IndexVolumeWriter.cs index 36e7fad62..cb72da615 100644 --- a/Duplicati/Library/Main/ForestHash/Volumes/ShadowVolumeWriter.cs +++ b/Duplicati/Library/Main/ForestHash/Volumes/IndexVolumeWriter.cs @@ -1,86 +1,86 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.IO; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; using Newtonsoft.Json; -using Duplicati.Library.Interface; - -namespace Duplicati.Library.Main.ForestHash.Volumes -{ - public class ShadowVolumeWriter : VolumeWriterBase - { - private StreamWriter m_streamwriter = null; - private JsonWriter m_writer = null; - - private long m_volumes = 0; - private long m_blocks = 0; - private long m_blocklists = 0; - - public long VolumeCount { get { return m_volumes; } } - public long BlockCount { get { return m_blocks; } } - public long Blocklists { get { return m_blocklists; } } - - public ShadowVolumeWriter(FhOptions options) - : base(options) - { - } - - public override RemoteVolumeType FileType { get { return RemoteVolumeType.Shadow; } } - - public void StartVolume(string filename) - { - if (m_writer != null || m_streamwriter != null) - throw new InvalidOperationException("Previous volume not finished, call FinishVolume before starting a new volume"); - - m_volumes++; - m_streamwriter = new StreamWriter(m_compression.CreateFile(SHADOW_VOLUME_FOLDER + filename, CompressionHint.Compressible, DateTime.UtcNow)); - m_writer = new JsonTextWriter(m_streamwriter); - m_writer.WriteStartObject(); - m_writer.WritePropertyName("blocks"); - m_writer.WriteStartArray(); - } - - public void AddBlock(string hash, long size) - { - m_writer.WriteStartObject(); - m_writer.WritePropertyName("hash"); - m_writer.WriteValue(hash); - m_writer.WritePropertyName("size"); - m_writer.WriteValue(size); - m_writer.WriteEndObject(); - m_blocks++; - } - - public void FinishVolume(string volumehash, long volumesize) - { - m_writer.WriteEndArray(); - m_writer.WritePropertyName("volumehash"); - m_writer.WriteValue(volumehash); - m_writer.WritePropertyName("volumesize"); - m_writer.WriteValue(volumesize); - m_writer.WriteEndObject(); - - try { m_writer.Close(); } - finally { m_writer = null; } - try { m_streamwriter.Dispose(); } - finally { m_streamwriter = null; } - } - - public void WriteBlocklist(string hash, byte[] data, int size) - { - m_blocklists++; - //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, - using (var s = m_compression.CreateFile(SHADOW_BLOCKLIST_FOLDER + Utility.Utility.Base64PlainToBase64Url(hash), CompressionHint.Noncompressible, DateTime.UtcNow)) - s.Write(data, 0, size); - } - - public override void Dispose() - { - if (m_writer != null || m_streamwriter != null) - throw new InvalidOperationException("Attempted to dispose a shadow volume that was being written"); - base.Dispose(); - } - - } -} +using Duplicati.Library.Interface; + +namespace Duplicati.Library.Main.ForestHash.Volumes +{ + public class IndexVolumeWriter : VolumeWriterBase + { + private StreamWriter m_streamwriter = null; + private JsonWriter m_writer = null; + + private long m_volumes = 0; + private long m_blocks = 0; + private long m_blocklists = 0; + + public long VolumeCount { get { return m_volumes; } } + public long BlockCount { get { return m_blocks; } } + public long Blocklists { get { return m_blocklists; } } + + public IndexVolumeWriter(FhOptions options) + : base(options) + { + } + + public override RemoteVolumeType FileType { get { return RemoteVolumeType.Index; } } + + public void StartVolume(string filename) + { + if (m_writer != null || m_streamwriter != null) + throw new InvalidOperationException("Previous volume not finished, call FinishVolume before starting a new volume"); + + m_volumes++; + m_streamwriter = new StreamWriter(m_compression.CreateFile(INDEX_VOLUME_FOLDER + filename, CompressionHint.Compressible, DateTime.UtcNow)); + m_writer = new JsonTextWriter(m_streamwriter); + m_writer.WriteStartObject(); + m_writer.WritePropertyName("blocks"); + m_writer.WriteStartArray(); + } + + public void AddBlock(string hash, long size) + { + m_writer.WriteStartObject(); + m_writer.WritePropertyName("hash"); + m_writer.WriteValue(hash); + m_writer.WritePropertyName("size"); + m_writer.WriteValue(size); + m_writer.WriteEndObject(); + m_blocks++; + } + + public void FinishVolume(string volumehash, long volumesize) + { + m_writer.WriteEndArray(); + m_writer.WritePropertyName("volumehash"); + m_writer.WriteValue(volumehash); + m_writer.WritePropertyName("volumesize"); + m_writer.WriteValue(volumesize); + m_writer.WriteEndObject(); + + try { m_writer.Close(); } + finally { m_writer = null; } + try { m_streamwriter.Dispose(); } + finally { m_streamwriter = null; } + } + + public void WriteBlocklist(string hash, byte[] data, int size) + { + m_blocklists++; + //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, + using (var s = m_compression.CreateFile(INDEX_BLOCKLIST_FOLDER + Utility.Utility.Base64PlainToBase64Url(hash), CompressionHint.Noncompressible, DateTime.UtcNow)) + s.Write(data, 0, size); + } + + public override void Dispose() + { + if (m_writer != null || m_streamwriter != null) + throw new InvalidOperationException("Attempted to dispose an index volume that was being written"); + base.Dispose(); + } + + } +} diff --git a/Duplicati/Library/Main/ForestHash/Volumes/VolumeBase.cs b/Duplicati/Library/Main/ForestHash/Volumes/VolumeBase.cs index 126c7e74a..d4be594f7 100644 --- a/Duplicati/Library/Main/ForestHash/Volumes/VolumeBase.cs +++ b/Duplicati/Library/Main/ForestHash/Volumes/VolumeBase.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -48,6 +48,7 @@ namespace Duplicati.Library.Main.ForestHash.Volumes throw new InvalidManifestException("FileHash", d.FileHash, filehash); } } + private class ParsedVolume : IParsedVolume { @@ -59,7 +60,25 @@ namespace Duplicati.Library.Main.ForestHash.Volumes public string EncryptionModule { get; private set; } public Library.Interface.IFileEntry File { get; private set; } - private static readonly System.Text.RegularExpressions.Regex FILENAME_REGEXP = new System.Text.RegularExpressions.Regex(@"(?[^\-]+)\-(?(" + string.Join(")|(", Enum.GetNames(typeof(RemoteVolumeType))).ToLowerInvariant() + @"))\-(((?[0-9A-Fa-f]+)|(?