Renamed shadow to index, and reworked the filename format

Duplicati/Library/Main/Duplicati.Library.Main.csproj

@@ -100,8 +100,6 @@
     <Compile Include="ForestHash\Volumes\IFileEntry.cs" />
     <Compile Include="ForestHash\Volumes\InvalidManifestException.cs" />
     <Compile Include="ForestHash\Volumes\IParsedVolume.cs" />
-    <Compile Include="ForestHash\Volumes\ShadowVolumeReader.cs" />
-    <Compile Include="ForestHash\Volumes\ShadowVolumeWriter.cs" />
     <Compile Include="ForestHash\Volumes\VolumeBase.cs" />
     <Compile Include="ForestHash\Volumes\VolumeReaderBase.cs" />
     <Compile Include="ForestHash\Volumes\VolumeWriterBase.cs" />
@@ -172,6 +170,8 @@
     <Compile Include="ForestHash\AsyncDownloader.cs" />
     <Compile Include="ForestHash\Database\LocalBugReportDatabase.cs" />
     <Compile Include="ForestHash\Operation\CreateBugReportHandler.cs" />
+    <Compile Include="ForestHash\Volumes\IndexVolumeWriter.cs" />
+    <Compile Include="ForestHash\Volumes\IndexVolumeReader.cs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Compression\Duplicati.Library.Compression.csproj">

Duplicati/Library/Main/ForestHash/Database/Database schema/Schema.sql

@@ -25,15 +25,15 @@ CREATE TABLE "Remotevolume" (
 );
 
 /*
-The shadow-block table contains
+The index-block table contains
 references that explains what block
-files a shadow file references.
-This is used to remove shadow volumes,
+files a index file references.
+This is used to remove index volumes,
 when they no longer reference any
 block volumes	
 */
-CREATE TABLE "ShadowBlockLink" (
-	"ShadowVolumeID" INTEGER NOT NULL,
+CREATE TABLE "IndexBlockLink" (
+	"IndexVolumeID" INTEGER NOT NULL,
 	"BlockVolumeID" INTEGER NOT NULL
 );
 

Duplicati/Library/Main/ForestHash/Database/LocalBackupDatabase.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -43,7 +43,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
         private readonly System.Data.IDbCommand m_updateblockCommand;
 
         private readonly System.Data.IDbCommand m_insertfileOperationCommand;
-        private readonly System.Data.IDbCommand m_insertShadowBlockLink;
+        private readonly System.Data.IDbCommand m_insertIndexBlockLink;
 		
 		private HashDatabaseProtector<string> m_blockHashLookup;
         private HashDatabaseProtector<string, long> m_fileHashLookup;
@@ -81,7 +81,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
             m_selectfileSimpleCommand = m_connection.CreateCommand();
             m_selectfileHashCommand = m_connection.CreateCommand();
             m_findmetadatasetProbeCommand = m_connection.CreateCommand();
-            m_insertShadowBlockLink = m_connection.CreateCommand();
+            m_insertIndexBlockLink = m_connection.CreateCommand();
 				
 			m_findblockCommand.CommandText = @"SELECT ""VolumeID"" FROM ""Block"" WHERE ""Hash"" = ? AND ""Size"" = ?";
             m_findblockCommand.AddParameters(2);
@@ -139,8 +139,8 @@ namespace Duplicati.Library.Main.ForestHash.Database
             m_updateblockCommand.CommandText = @"UPDATE ""Block"" SET ""VolumeID"" = ? WHERE ""Hash"" = ? AND ""Size"" = ? ";
             m_updateblockCommand.AddParameters(3);
             
-            m_insertShadowBlockLink.CommandText = @"INSERT INTO ""ShadowBlockLink"" (""ShadowVolumeID"", ""BlockVolumeID"") VALUES (?, ?)";
-            m_insertShadowBlockLink.AddParameters(2);
+            m_insertIndexBlockLink.CommandText = @"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)";
+            m_insertIndexBlockLink.AddParameters(2);
             
 			if (options.FhBlockHashLookupMemory > 0)
                 m_blockHashLookup = new HashDatabaseProtector<string>(HASH_GUESS_SIZE, (ulong)options.FhBlockHashLookupMemory);            
@@ -793,12 +793,12 @@ namespace Duplicati.Library.Main.ForestHash.Database
 			}
 		}
 						
-		public void AddShadowBlockLink(long shadowVolumeID, long blockVolumeID, System.Data.IDbTransaction transaction)
+		public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID, System.Data.IDbTransaction transaction)
 		{
-			m_insertShadowBlockLink.Transaction = transaction;
-			m_insertShadowBlockLink.SetParameterValue(0, shadowVolumeID);
-			m_insertShadowBlockLink.SetParameterValue(1, blockVolumeID);
-			m_insertShadowBlockLink.ExecuteNonQuery();
+			m_insertIndexBlockLink.Transaction = transaction;
+			m_insertIndexBlockLink.SetParameterValue(0, indexVolumeID);
+			m_insertIndexBlockLink.SetParameterValue(1, blockVolumeID);
+			m_insertIndexBlockLink.ExecuteNonQuery();
 		}
 		
     }

Duplicati/Library/Main/ForestHash/Database/LocalCleanupDatabase.cs

@@ -153,10 +153,10 @@ namespace Duplicati.Library.Main.ForestHash.Database
 					yield return new RemoteVolume(rd.GetValue(0).ToString(), rd.GetValue(1).ToString(), Convert.ToInt64(rd.GetValue(2)));
 		}
 
-		public IEnumerable<IRemoteVolume> GetBlockVolumesFromShadowName(string name)
+		public IEnumerable<IRemoteVolume> GetBlockVolumesFromIndexName(string name)
 		{
 			using(var cmd = m_connection.CreateCommand())
-			using(var rd = cmd.ExecuteReader(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""ShadowBlockLink"" WHERE ""ShadowVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = ?))", name))
+			using(var rd = cmd.ExecuteReader(@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""ID"" IN (SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" IN (SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" = ?))", name))
 				while (rd.Read())
 					yield return new RemoteVolume(rd.GetValue(0).ToString(), rd.GetValue(1).ToString(), Convert.ToInt64(rd.GetValue(2)));
 		}

Duplicati/Library/Main/ForestHash/Database/LocalDatabase.cs

@@ -226,7 +226,7 @@ namespace Duplicati.Library.Main.ForestHash.Database
             	var volumeid = GetRemoteVolumeID(name, tr.Parent);
                 
 				// If the volume is a block volume, this will update the crosslink table, otherwise nothing will happen
-				deletecmd.ExecuteNonQuery(@"DELETE FROM ""ShadowBlockLink"" WHERE ""BlockVolumeID"" = ? ", volumeid);
+				deletecmd.ExecuteNonQuery(@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" = ? ", volumeid);
 				
 				var subQuery = @"(SELECT DISTINCT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" = ?)";
 

Duplicati/Library/Main/ForestHash/Database/LocalDeleteDatabase.cs

@@ -438,32 +438,32 @@ namespace Duplicati.Library.Main.ForestHash.Database
 		{
 			using(var cmd = m_connection.CreateCommand())
 			{
-				// Although the generated shadow volumes are always in pairs,
+				// Although the generated index volumes are always in pairs,
 				// this code handles many-to-many relations between
-				// shadow files and block volumes, should this be added later
+				// index files and block volumes, should this be added later
 				var lookupBlock = new Dictionary<string, List<IRemoteVolume>>();
-				var lookupShadow = new Dictionary<string, List<string>>();
+				var lookupIndexfiles = new Dictionary<string, List<string>>();
 				
 				cmd.Transaction = transaction;
-					using(var rd = cmd.ExecuteReader(@"SELECT ""C"".""Name"", ""B"".""Name"", ""B"".""Hash"", ""B"".""Size"" FROM ""ShadowBlockLink"" A, ""RemoteVolume"" B, ""RemoteVolume"" C WHERE ""A"".""ShadowVolumeID"" = ""B"".""ID"" AND ""A"".""BlockVolumeID"" = ""C"".""ID"" "))
+					using(var rd = cmd.ExecuteReader(@"SELECT ""C"".""Name"", ""B"".""Name"", ""B"".""Hash"", ""B"".""Size"" FROM ""IndexBlockLink"" A, ""RemoteVolume"" B, ""RemoteVolume"" C WHERE ""A"".""IndexVolumeID"" = ""B"".""ID"" AND ""A"".""BlockVolumeID"" = ""C"".""ID"" "))
 						while(rd.Read())
 						{
 							var name = rd.GetValue(0).ToString();
-							List<IRemoteVolume> shadowList;
-							if (!lookupBlock.TryGetValue(name, out shadowList))
+							List<IRemoteVolume> indexfileList;
+							if (!lookupBlock.TryGetValue(name, out indexfileList))
 							{	
-								shadowList = new List<IRemoteVolume>();
-								lookupBlock.Add(name, shadowList);
+								indexfileList = new List<IRemoteVolume>();
+								lookupBlock.Add(name, indexfileList);
 							}
 							
 							var v = new RemoteVolume(rd.GetValue(1).ToString(), rd.GetValue(2).ToString(), Convert.ToInt64(rd.GetValue(3)));
-							shadowList.Add(v);
+							indexfileList.Add(v);
 
 							List<string> blockList;
-							if (!lookupShadow.TryGetValue(v.Name, out blockList))
+							if (!lookupIndexfiles.TryGetValue(v.Name, out blockList))
 							{	
 								blockList = new List<string>();
-								lookupShadow.Add(v.Name, blockList);
+								lookupIndexfiles.Add(v.Name, blockList);
 							}
 							blockList.Add(name);
 						}
@@ -472,15 +472,15 @@ namespace Duplicati.Library.Main.ForestHash.Database
 				{
 					// Return the input
 					yield return r;
-					List<IRemoteVolume> shadowList;
-					if (lookupBlock.TryGetValue(r.Name, out shadowList))
-						foreach(var sh in shadowList)
+					List<IRemoteVolume> indexfileList;
+					if (lookupBlock.TryGetValue(r.Name, out indexfileList))
+						foreach(var sh in indexfileList)
 						{
 							List<string> backref;
-							if (lookupShadow.TryGetValue(sh.Name, out backref))
+							if (lookupIndexfiles.TryGetValue(sh.Name, out backref))
 							{
 								//If this is the last reference, 
-								// remove the shadow file as well
+								// remove the index file as well
 								if (backref.Remove(r.Name) && backref.Count == 0)
 									yield return sh;
 							}

Duplicati/Library/Main/ForestHash/Enums.cs

@@ -1,94 +1,94 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace Duplicati.Library.Main.ForestHash
-{
-    /// <summary>
-    /// Describes all states a remote volume can have
-    /// </summary>
-    public enum RemoteVolumeState
-    {
-        /// <summary>
-        /// Indicates that the remote volume is being created
-        /// </summary>
-        Temporary,
-
-        /// <summary>
-        /// Indicates that the remote volume is being uploaded
-        /// </summary>
-        Uploading,
-
-        /// <summary>
-        /// Indicates that the remote volume has been uploaded
-        /// </summary>
-        Uploaded,
-
-        /// <summary>
-        /// Indicates that the remote volume has been uploaded,
-        /// and seen by a list operation
-        /// </summary>
-        Verified,
-
-        /// <summary>
-        /// Indicates that the remote volume has been uploaded,
-        /// but is now missing from the remote location
-        /// </summary>
-        Missing,
-
-        /// <summary>
-        /// Indicattes that the remote volume should be deleted
-        /// </summary>
-        Deleting,
-
-        /// <summary>
-        /// Indicates that the remote volume was successfully
-        /// deleted from the remote location
-        /// </summary>
-        Deleted
-    }
-
-    /// <summary>
-    /// Describes the known remote volume types
-    /// </summary>
-    public enum RemoteVolumeType
-    {
-        /// <summary>
-        /// Contains data blocks
-        /// </summary>
-        Blocks,
-        /// <summary>
-        /// Contains file lists
-        /// </summary>
-        Files,
-        /// <summary>
-        /// Contains redundant lookup information
-        /// </summary>
-        Shadow
-    }
-
-    public enum FilelistEntryType
-    {
-        /// <summary>
-        /// The actual type of the entry could not be determined
-        /// </summary>
-        Unknown,
-        /// <summary>
-        /// The entry is a plain file
-        /// </summary>
-        File,
-        /// <summary>
-        /// The entry is a folder
-        /// </summary>
-        Folder,
-        /// <summary>
-        /// The entry is an alternate data stream, or resource/data fork
-        /// </summary>
-        AlternateStream,
-        /// <summary>
-        /// The entry is a symbolic link
-        /// </summary>
-        Symlink
-    }
-}
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Duplicati.Library.Main.ForestHash
+{
+    /// <summary>
+    /// Describes all states a remote volume can have
+    /// </summary>
+    public enum RemoteVolumeState
+    {
+        /// <summary>
+        /// Indicates that the remote volume is being created
+        /// </summary>
+        Temporary,
+
+        /// <summary>
+        /// Indicates that the remote volume is being uploaded
+        /// </summary>
+        Uploading,
+
+        /// <summary>
+        /// Indicates that the remote volume has been uploaded
+        /// </summary>
+        Uploaded,
+
+        /// <summary>
+        /// Indicates that the remote volume has been uploaded,
+        /// and seen by a list operation
+        /// </summary>
+        Verified,
+
+        /// <summary>
+        /// Indicates that the remote volume has been uploaded,
+        /// but is now missing from the remote location
+        /// </summary>
+        Missing,
+
+        /// <summary>
+        /// Indicattes that the remote volume should be deleted
+        /// </summary>
+        Deleting,
+
+        /// <summary>
+        /// Indicates that the remote volume was successfully
+        /// deleted from the remote location
+        /// </summary>
+        Deleted
+    }
+
+    /// <summary>
+    /// Describes the known remote volume types
+    /// </summary>
+    public enum RemoteVolumeType
+    {
+        /// <summary>
+        /// Contains data blocks
+        /// </summary>
+        Blocks,
+        /// <summary>
+        /// Contains file lists
+        /// </summary>
+        Files,
+        /// <summary>
+        /// Contains redundant lookup information
+        /// </summary>
+        Index
+    }
+
+    public enum FilelistEntryType
+    {
+        /// <summary>
+        /// The actual type of the entry could not be determined
+        /// </summary>
+        Unknown,
+        /// <summary>
+        /// The entry is a plain file
+        /// </summary>
+        File,
+        /// <summary>
+        /// The entry is a folder
+        /// </summary>
+        Folder,
+        /// <summary>
+        /// The entry is an alternate data stream, or resource/data fork
+        /// </summary>
+        AlternateStream,
+        /// <summary>
+        /// The entry is a symbolic link
+        /// </summary>
+        Symlink
+    }
+}

Duplicati/Library/Main/ForestHash/FhBackend.cs

@@ -38,21 +38,21 @@ namespace Duplicati.Library.Main.ForestHash
             public object Result;
             public string Hash;
             public long Size;
-            public ShadowVolumeWriter Shadow;
+            public IndexVolumeWriter Indexfile;
 
             private System.Threading.ManualResetEvent DoneEvent;
 
-            public FileEntryItem(OperationType operation, string remotefilename, ShadowVolumeWriter shadow = null)
+            public FileEntryItem(OperationType operation, string remotefilename, IndexVolumeWriter indexfile = null)
             {
                 Operation = operation;
                 RemoteFilename = remotefilename;
-                Shadow = shadow;
+                Indexfile = indexfile;
 
                 DoneEvent = new System.Threading.ManualResetEvent(false);
             }
 
-            public FileEntryItem(OperationType operation, string remotefilename, long size, string hash, ShadowVolumeWriter shadow = null)
-                : this(operation, remotefilename, shadow)
+            public FileEntryItem(OperationType operation, string remotefilename, long size, string hash, IndexVolumeWriter indexfile = null)
+                : this(operation, remotefilename, indexfile)
             {
                 Size = size;
                 Hash = hash;
@@ -370,11 +370,11 @@ namespace Duplicati.Library.Main.ForestHash
             if (item.UpdateHashAndSize(m_options))
             	m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash);
 
-            if (item.Shadow != null)
+            if (item.Indexfile != null)
             {
-                item.Shadow.FinishVolume(item.Hash, item.Size);
-                item.Shadow.Close();
-                item.Shadow = null;
+                item.Indexfile.FinishVolume(item.Hash, item.Size);
+                item.Indexfile.Close();
+                item.Indexfile = null;
             }            
 
             m_stats.AddNumberOfRemoteCalls(1);
@@ -558,14 +558,14 @@ namespace Duplicati.Library.Main.ForestHash
             	m_stats.LogMessage("Created target folder");
         }
 
-        public void Put(VolumeWriterBase item, ShadowVolumeWriter shadow = null)
+        public void Put(VolumeWriterBase item, IndexVolumeWriter indexfile = null)
         {
             if (m_lastException != null)
                 throw m_lastException;
             
             item.Close();
             m_db.LogDbUpdate(item.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
-            var req = new FileEntryItem(OperationType.Put, item.RemoteFilename, shadow);
+            var req = new FileEntryItem(OperationType.Put, item.RemoteFilename, indexfile);
             req.LocalFilename = item.LocalFilename;
 
             if (m_queue.Enqueue(req) && !m_options.AsynchronousUpload)
@@ -574,11 +574,11 @@ namespace Duplicati.Library.Main.ForestHash
             if (m_lastException != null)
                 throw m_lastException;
 
-            if (shadow != null)
+            if (indexfile != null)
             {
-                m_db.LogDbUpdate(shadow.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
-                var req2 = new FileEntryItem(OperationType.Put, shadow.RemoteFilename);
-                req2.LocalFilename = shadow.LocalFilename;
+                m_db.LogDbUpdate(indexfile.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
+                var req2 = new FileEntryItem(OperationType.Put, indexfile.RemoteFilename);
+                req2.LocalFilename = indexfile.LocalFilename;
 
                 if (m_queue.Enqueue(req2) && !m_options.AsynchronousUpload)
                     req2.WaitForComplete();

Duplicati/Library/Main/ForestHash/FhOptions.cs

@@ -62,7 +62,7 @@ namespace Duplicati.Library.Main.ForestHash
                     new CommandLineArgument("fh-deleted-fileset", CommandLineArgument.ArgumentType.Path, Strings.FhOptions.FhdeletedfilesetShort, string.Format(Strings.FhOptions.FhdeletedfilesetLong, "fh-changed-fileset")),
 
                     new CommandLineArgument("fh-max-wastesize", CommandLineArgument.ArgumentType.Size, Strings.FhOptions.FhmaxwastesizeShort, Strings.FhOptions.FhmaxwastesizeLong),
-                    new CommandLineArgument("fh-no-shadowfiles", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnoshadowfilesShort, Strings.FhOptions.FhnoshadowfilesLong, "false"),
+                    new CommandLineArgument("fh-no-indexfiles", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnoindexfilesShort, Strings.FhOptions.FhnoindexfilesLong, "false"),
                     new CommandLineArgument("fh-no-backendverification", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhnobackendverificationShort, Strings.FhOptions.FhnobackendverificationLong, "false"),
                     new CommandLineArgument("fh-dry-run", CommandLineArgument.ArgumentType.Boolean, Strings.FhOptions.FhdryrunShort, Strings.FhOptions.FhdryrunLong, "false"),
 
@@ -251,11 +251,11 @@ namespace Duplicati.Library.Main.ForestHash
         }
         
         /// <summary>
-        /// Gets a flag indicating if shadow files should be omitted
+        /// Gets a flag indicating if index files should be omitted
         /// </summary>
-        public bool FhNoShadowfiles
+        public bool FhNoIndexfiles
         {
-            get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-shadowfiles"); }
+            get { return Utility.Utility.ParseBoolOption(m_options, "fh-no-indexfiles"); }
         }
         
         /// <summary>

Duplicati/Library/Main/ForestHash/Operation/BackupHandler.cs

@@ -24,7 +24,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation
         private LocalBackupDatabase m_database;
         private System.Data.IDbTransaction m_transaction;
         private BlockVolumeWriter m_blockvolume;
-        private ShadowVolumeWriter m_shadowvolume;
+        private IndexVolumeWriter m_indexvolume;
         private FilesetVolumeWriter m_filesetvolume;
 
         private Snapshots.ISnapshotService m_snapshot;
@@ -132,10 +132,10 @@ namespace Duplicati.Library.Main.ForestHash.Operation
 		            m_blockvolume = new BlockVolumeWriter(m_options);
 		            m_blockvolume.VolumeID = m_database.RegisterRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_transaction);
 		            
-		            if (!m_options.FhNoShadowfiles)
+		            if (!m_options.FhNoIndexfiles)
 		            {
-			            m_shadowvolume = new ShadowVolumeWriter(m_options);
-			            m_shadowvolume.VolumeID = m_database.RegisterRemoteVolume(m_shadowvolume.RemoteFilename, RemoteVolumeType.Shadow, RemoteVolumeState.Temporary, m_transaction);
+			            m_indexvolume = new IndexVolumeWriter(m_options);
+			            m_indexvolume.VolumeID = m_database.RegisterRemoteVolume(m_indexvolume.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_transaction);
 		            }
 		                        	
 	                if (m_options.FhChangedFilelist != null && m_options.FhChangedFilelist.Length >= 1)
@@ -165,29 +165,29 @@ namespace Duplicati.Library.Main.ForestHash.Operation
 	                	if (m_options.FhDryrun)
 	                	{
 	                		m_stat.LogMessage("[Dryrun] Would upload block volume: {0}, size: {1}", m_blockvolume.RemoteFilename, Utility.Utility.FormatSizeString(new FileInfo(m_blockvolume.LocalFilename).Length));
-	                		if (m_shadowvolume != null)
+	                		if (m_indexvolume != null)
 	                		{
-			            		UpdateShadowVolume();
-	                			m_shadowvolume.FinishVolume(Utility.Utility.CalculateHash(m_blockvolume.LocalFilename), new FileInfo(m_blockvolume.LocalFilename).Length);
-	                			m_stat.LogMessage("[Dryrun] Would upload shadow volume: {0}, size: {1}", m_shadowvolume.RemoteFilename, Utility.Utility.FormatSizeString(new FileInfo(m_shadowvolume.LocalFilename).Length));
+			            		UpdateIndexVolume();
+	                			m_indexvolume.FinishVolume(Utility.Utility.CalculateHash(m_blockvolume.LocalFilename), new FileInfo(m_blockvolume.LocalFilename).Length);
+	                			m_stat.LogMessage("[Dryrun] Would upload index volume: {0}, size: {1}", m_indexvolume.RemoteFilename, Utility.Utility.FormatSizeString(new FileInfo(m_indexvolume.LocalFilename).Length));
 	                		}
 	                	}
 	                	else
 	                	{
 	                		m_database.UpdateRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
-		            		UpdateShadowVolume();
+		            		UpdateIndexVolume();
 	                		
 	                		m_transaction.Commit();
 	                		m_transaction = m_database.BeginTransaction();
 	                		
-	                    	m_backend.Put(m_blockvolume, m_shadowvolume);
+	                    	m_backend.Put(m_blockvolume, m_indexvolume);
 	                    }
 	                }
 	                else
 	                {
 	                    m_database.RemoveRemoteVolume(m_blockvolume.RemoteFilename, m_transaction);
-	                    if (m_shadowvolume != null)
-		                    m_database.RemoveRemoteVolume(m_shadowvolume.RemoteFilename, m_transaction);
+	                    if (m_indexvolume != null)
+		                    m_database.RemoveRemoteVolume(m_indexvolume.RemoteFilename, m_transaction);
 	                }
 		            
 		            m_database.UpdateChangeStatistics(m_stat);
@@ -484,13 +484,13 @@ namespace Duplicati.Library.Main.ForestHash.Operation
                 		m_blockvolume.Dispose();
                 		m_blockvolume = null;
                 		
-                		if (m_shadowvolume != null)
+                		if (m_indexvolume != null)
                 		{
-		            		UpdateShadowVolume();
-                			m_shadowvolume.FinishVolume(Utility.Utility.CalculateHash(m_shadowvolume.LocalFilename), new FileInfo(m_shadowvolume.LocalFilename).Length);
-                			m_stat.LogMessage("[Dryrun] Would upload shadow volume: {0}, size: {1}", m_shadowvolume.RemoteFilename, Utility.Utility.FormatSizeString(new FileInfo(m_shadowvolume.LocalFilename).Length));
-                			m_shadowvolume.Dispose();
-                			m_shadowvolume = null;
+		            		UpdateIndexVolume();
+                			m_indexvolume.FinishVolume(Utility.Utility.CalculateHash(m_indexvolume.LocalFilename), new FileInfo(m_indexvolume.LocalFilename).Length);
+                			m_stat.LogMessage("[Dryrun] Would upload index volume: {0}, size: {1}", m_indexvolume.RemoteFilename, Utility.Utility.FormatSizeString(new FileInfo(m_indexvolume.LocalFilename).Length));
+                			m_indexvolume.Dispose();
+                			m_indexvolume = null;
                 		}
                 	}
                 	else
@@ -498,7 +498,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation
 	                	//When uploading a new volume, we register the volumes and then flush the transaction
 	                	// this ensures that the local database and remote storage are as closely related as possible
                 		m_database.UpdateRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
-	            		UpdateShadowVolume();
+	            		UpdateIndexVolume();
 	                	
 	                	m_backend.FlushDbMessages(m_database, m_transaction);
         				m_backendLogFlushTimer = DateTime.Now.Add(FLUSH_TIMESPAN);
@@ -506,18 +506,18 @@ namespace Duplicati.Library.Main.ForestHash.Operation
 	                	m_transaction.Commit();
 	                	m_transaction = m_database.BeginTransaction();
 	                	
-	                    m_backend.Put(m_blockvolume, m_shadowvolume);
+	                    m_backend.Put(m_blockvolume, m_indexvolume);
 	                    m_blockvolume = null;
-	                    m_shadowvolume = null;
+	                    m_indexvolume = null;
 	                }
                     
                     m_blockvolume = new BlockVolumeWriter(m_options);
 					m_blockvolume.VolumeID = m_database.RegisterRemoteVolume(m_blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_transaction);
 					
-					if (!m_options.FhNoShadowfiles)
+					if (!m_options.FhNoIndexfiles)
 					{
-	                    m_shadowvolume = new ShadowVolumeWriter(m_options);
-						m_shadowvolume.VolumeID = m_database.RegisterRemoteVolume(m_shadowvolume.RemoteFilename, RemoteVolumeType.Shadow, RemoteVolumeState.Temporary, m_transaction);
+	                    m_indexvolume = new IndexVolumeWriter(m_options);
+						m_indexvolume.VolumeID = m_database.RegisterRemoteVolume(m_indexvolume.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_transaction);
 					}
                 }
 
@@ -602,17 +602,17 @@ namespace Duplicati.Library.Main.ForestHash.Operation
         }
 
 
-        private void UpdateShadowVolume()
+        private void UpdateIndexVolume()
         {
-        	if (m_shadowvolume != null)
+        	if (m_indexvolume != null)
         	{
-	            m_database.AddShadowBlockLink(m_shadowvolume.VolumeID, m_blockvolume.VolumeID, m_transaction);
-	            m_shadowvolume.StartVolume(m_blockvolume.RemoteFilename);
+	            m_database.AddIndexBlockLink(m_indexvolume.VolumeID, m_blockvolume.VolumeID, m_transaction);
+	            m_indexvolume.StartVolume(m_blockvolume.RemoteFilename);
 	            
 	            foreach(var b in m_database.GetBlocks(m_blockvolume.VolumeID))
-	            	m_shadowvolume.AddBlock(b.Hash, b.Size);
+	            	m_indexvolume.AddBlock(b.Hash, b.Size);
 
-    			m_database.UpdateRemoteVolume(m_shadowvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
+    			m_database.UpdateRemoteVolume(m_indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_transaction);
         	}
         }
 
@@ -632,11 +632,11 @@ namespace Duplicati.Library.Main.ForestHash.Operation
                 finally { m_blockvolume = null; }
             }
 
-            if (m_shadowvolume != null)
+            if (m_indexvolume != null)
             {
-                try { m_shadowvolume.Dispose(); }
-                catch (Exception ex) { m_stat.LogError("Failed disposing shadow volume", ex); }
-                finally { m_shadowvolume = null; }
+                try { m_indexvolume.Dispose(); }
+                catch (Exception ex) { m_stat.LogError("Failed disposing index volume", ex); }
+                finally { m_indexvolume = null; }
             }
 
             m_stat.EndTime = DateTime.Now;

Duplicati/Library/Main/ForestHash/Operation/CleanupHandler.cs

@@ -94,12 +94,12 @@ namespace Duplicati.Library.Main.ForestHash.Operation
 									backend.Put(w);
 								}
 							}
-							else if (n.Type == RemoteVolumeType.Shadow)
+							else if (n.Type == RemoteVolumeType.Index)
 							{
-								var w = new ShadowVolumeWriter(m_options);
+								var w = new IndexVolumeWriter(m_options);
 								w.SetRemoteFilename(n.Name);
 								
-								foreach(var blockvolume in db.GetBlockVolumesFromShadowName(n.Name))
+								foreach(var blockvolume in db.GetBlockVolumesFromIndexName(n.Name))
 								{								
 									w.StartVolume(blockvolume.Name);
 									var volumeid = db.GetRemoteVolumeID(blockvolume.Name);
@@ -113,7 +113,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation
 								w.Close();
 								
 								if (m_options.FhDryrun)
-									m_stat.LogMessage("[Dryrun] would re-upload shadow file {0}, with size {1}, previous size {2}", n.Name, Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Utility.Utility.FormatSizeString(n.Size));
+									m_stat.LogMessage("[Dryrun] would re-upload index file {0}, with size {1}, previous size {2}", n.Name, Utility.Utility.FormatSizeString(new System.IO.FileInfo(w.LocalFilename).Length), Utility.Utility.FormatSizeString(n.Size));
 								else
 								{
 									db.UpdateRemoteVolume(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, null);

Duplicati/Library/Main/ForestHash/Operation/CompactHandler.cs

@@ -73,12 +73,12 @@ namespace Duplicati.Library.Main.ForestHash.Operation
 					BlockVolumeWriter newvol = new BlockVolumeWriter(m_options);
 					newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
 	
-					ShadowVolumeWriter newvolshadow = null;
-					if (!m_options.FhNoShadowfiles)
+					IndexVolumeWriter newvolindex = null;
+					if (!m_options.FhNoIndexfiles)
 					{
-						newvolshadow = new ShadowVolumeWriter(m_options);
-						db.RegisterRemoteVolume(newvolshadow.RemoteFilename, RemoteVolumeType.Shadow, RemoteVolumeState.Temporary, transaction);
-						newvolshadow.StartVolume(newvol.RemoteFilename);
+						newvolindex = new IndexVolumeWriter(m_options);
+						db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
+						newvolindex.StartVolume(newvol.RemoteFilename);
 					}
 					
 					long blocksInVolume = 0;
@@ -128,8 +128,8 @@ namespace Duplicati.Library.Main.ForestHash.Operation
 												throw new Exception("Size mismatch problem, {0} vs {1}");
 												
 											newvol.AddBlock(e.Key, buffer, s, Duplicati.Library.Interface.CompressionHint.Compressible);
-											if (newvolshadow != null)
-												newvolshadow.AddBlock(e.Key, e.Value);
+											if (newvolindex != null)
+												newvolindex.AddBlock(e.Key, e.Value);
 												
 											db.MoveBlockToNewVolume(e.Key, e.Value, newvol.VolumeID, transaction);
 											blocksInVolume++;
@@ -137,11 +137,11 @@ namespace Duplicati.Library.Main.ForestHash.Operation
 											if (newvol.Filesize > m_options.VolumeSize)
 											{
 												uploadedVolumes.Add(new KeyValuePair<string, long>(newvol.RemoteFilename, new System.IO.FileInfo(newvol.LocalFilename).Length));
-												if (newvolshadow != null)
-													uploadedVolumes.Add(new KeyValuePair<string, long>(newvolshadow.RemoteFilename, new System.IO.FileInfo(newvolshadow.LocalFilename).Length));
+												if (newvolindex != null)
+													uploadedVolumes.Add(new KeyValuePair<string, long>(newvolindex.RemoteFilename, new System.IO.FileInfo(newvolindex.LocalFilename).Length));
 	
 												if (m_options.Force && !m_options.FhDryrun)
-													backend.Put(newvol, newvolshadow);
+													backend.Put(newvol, newvolindex);
 												else
 													m_stat.LogMessage("[Dryrun] - Would upload generated blockset of size {0}", Utility.Utility.FormatSizeString(new System.IO.FileInfo(newvol.LocalFilename).Length));
 												
@@ -149,11 +149,11 @@ namespace Duplicati.Library.Main.ForestHash.Operation
 												newvol = new BlockVolumeWriter(m_options);
 												newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, transaction);
 				
-												if (!m_options.FhNoShadowfiles)
+												if (!m_options.FhNoIndexfiles)
 												{
-													newvolshadow = new ShadowVolumeWriter(m_options);
-													db.RegisterRemoteVolume(newvolshadow.RemoteFilename, RemoteVolumeType.Shadow, RemoteVolumeState.Temporary, transaction);
-													newvolshadow.StartVolume(newvol.RemoteFilename);
+													newvolindex = new IndexVolumeWriter(m_options);
+													db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, transaction);
+													newvolindex.StartVolume(newvol.RemoteFilename);
 												}
 												
 												blocksInVolume = 0;
@@ -176,20 +176,20 @@ namespace Duplicati.Library.Main.ForestHash.Operation
 							if (blocksInVolume > 0)
 							{
 								uploadedVolumes.Add(new KeyValuePair<string, long>(newvol.RemoteFilename, new System.IO.FileInfo(newvol.LocalFilename).Length));
-								if (newvolshadow != null)
-									uploadedVolumes.Add(new KeyValuePair<string, long>(newvolshadow.RemoteFilename, new System.IO.FileInfo(newvolshadow.LocalFilename).Length));
+								if (newvolindex != null)
+									uploadedVolumes.Add(new KeyValuePair<string, long>(newvolindex.RemoteFilename, new System.IO.FileInfo(newvolindex.LocalFilename).Length));
 								if (m_options.Force && !m_options.FhDryrun)
-									backend.Put(newvol, newvolshadow);
+									backend.Put(newvol, newvolindex);
 								else
 									m_stat.LogMessage("[Dryrun] - Would upload generated blockset of size {0}", Utility.Utility.FormatSizeString(new System.IO.FileInfo(newvol.LocalFilename).Length));
 							}
 							else
 							{
 				                db.RemoveRemoteVolume(newvol.RemoteFilename, transaction);
-			                    if (newvolshadow != null)
+			                    if (newvolindex != null)
 			                    {
-				                    db.RemoveRemoteVolume(newvolshadow.RemoteFilename, transaction);
-				                    newvolshadow.FinishVolume(null, 0);
+				                    db.RemoveRemoteVolume(newvolindex.RemoteFilename, transaction);
+				                    newvolindex.FinishVolume(null, 0);
 			                    }
 							}
 						}

Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs

@@ -112,25 +112,25 @@ namespace Duplicati.Library.Main.ForestHash.Operation
                     foreach (var fl in remotefiles)
                         volumeIds[fl.File.Name] = backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded);
                                        
-                    //We grab all shadow files, and update the block table
+                    //We grab all index files, and update the block table
                     using (var tr = restoredb.BeginTransaction())
                     {
-						var shadowfiles = 
+						var indexfiles = 
 							from n in remotefiles
-							where n.FileType == RemoteVolumeType.Shadow
+							where n.FileType == RemoteVolumeType.Index
 							select new RemoteVolume(n.File) as IRemoteVolume;
 												
-                        foreach (var sf in new AsyncDownloader(shadowfiles.ToList(), backend))
+                        foreach (var sf in new AsyncDownloader(indexfiles.ToList(), backend))
                         using (var tmpfile = sf.Value)
                         {
                             if (sf.Key.Hash != null && sf.Key.Size > 0)
                                 backupdb.UpdateRemoteVolume(sf.Key.Name, RemoteVolumeState.Verified, sf.Key.Size, sf.Key.Hash, tr);
 							
-                            using (var svr = new ShadowVolumeReader(RestoreHandler.GetCompressionModule(sf.Key.Name), tmpfile, m_options, hashsize))
+                            using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Key.Name), tmpfile, m_options, hashsize))
                             {
 					        	ForestHash.VerifyParameters(restoredb, m_options);
 
-                                //If there are blocklists in the shadow file, update the blocklists
+                                //If there are blocklists in the index file, update the blocklists
                                 foreach (var b in svr.BlockLists)
                                     restoredb.UpdateBlocklist(b.Hash, b.Blocklist, hashsize, tr);
 
@@ -142,7 +142,7 @@ namespace Duplicati.Library.Main.ForestHash.Operation
                                         backupdb.AddBlock(b.Key, b.Value, volumeID, tr);
 
                                     backupdb.UpdateRemoteVolume(a.Filename, RemoteVolumeState.Verified, a.Length, a.Hash, tr);
-                                    backupdb.AddShadowBlockLink(restoredb.GetRemoteVolumeID(sf.Key.Name), volumeID, tr);
+                                    backupdb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Key.Name), volumeID, tr);
                                 }
                             }
                         }

Duplicati/Library/Main/ForestHash/Volumes/IndexVolumeReader.cs

@@ -1,399 +1,399 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
-using Newtonsoft.Json;
-using Duplicati.Library.Interface;
-
-namespace Duplicati.Library.Main.ForestHash.Volumes
-{
-    public class ShadowVolumeReader : VolumeReaderBase
-    {
-        public interface IShadowBlocklist
-        {
-            string Hash { get; }
-            long Length { get; }
-            IEnumerable<string> Blocklist { get; }
-        }
-
-        public interface IShadowBlockVolume
-        {
-            string Filename { get; }
-            long Length { get; }
-            string Hash { get; }
-            IEnumerable<KeyValuePair<string, long>> Blocks { get; }
-        }
-
-        private class ShadowBlockVolumeEnumerable : IEnumerable<IShadowBlockVolume>
-        {
-            private ICompression m_compression;
-
-            public ShadowBlockVolumeEnumerable(ICompression compression)
-            {
-                m_compression = compression;
-            }
-
-            private class ShadowBlockVolumeEnumerator : IEnumerator<IShadowBlockVolume>
-            {
-                private class BlockEnumerable : IEnumerable<KeyValuePair<string, long>>
-                {
-                    private class BlockEnumerator : IEnumerator<KeyValuePair<string, long>>
-                    {
-                        private ICompression m_compression;
-                        private string m_filename;
-                        private KeyValuePair<string, long>? m_current;
-                        private System.IO.StreamReader m_stream;
-                        private JsonReader m_reader;
-                        private bool m_done;
-                        private KeyValuePair<string, long>? m_volumeProps = null;
-
-                        public BlockEnumerator(ICompression compression, string filename)
-                        {
-                            m_compression = compression;
-                            m_filename = filename;
-                            this.Reset();
-                        }
-
-                        public KeyValuePair<string, long> Current
-                        {
-                            get { return m_current.Value; }
-                        }
-
-                        public void Dispose()
-                        {
-                            if (m_reader != null)
-                                this.ReadVolumeProps();
-
-                            if (m_reader != null)
-                                try { m_reader.Close(); }
-                                finally { m_reader = null; }
-
-                            if (m_stream != null)
-                                try { m_stream.Dispose(); }
-                                finally { m_stream = null; }
-                        }
-
-                        object System.Collections.IEnumerator.Current
-                        {
-                            get { return this.Current; }
-                        }
-
-                        public bool MoveNext()
-                        {
-                            if (m_done)
-                                return false;
-
-                            if (!m_reader.Read())
-                                throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading hashes"));
-
-                            if (m_reader.TokenType == JsonToken.EndArray)
-                            {
-                                m_done = true;
-                                m_current = null;
-                                return false;
-                            }
-
-                            if (m_reader.TokenType != JsonToken.StartObject)
-                                throw new InvalidDataException(string.Format("Invalid JSON, expected StartObject, but got {0}, {1}", m_reader.TokenType, m_reader.Value));
-
-                            var hash = ReadJsonStringProperty(m_reader, "hash");
-                            var size = ReadJsonInt64Property(m_reader, "size");
-
-                            m_current = new KeyValuePair<string, long>(hash, size);
-
-                            while (m_reader.Read() && m_reader.TokenType != JsonToken.EndObject)
-                            { /* skip */ }
-
-                            return true;
-                        }
-
-                        public KeyValuePair<string, long> ReadVolumeProps()
-                        {
-                            if (m_volumeProps != null)
-                                return m_volumeProps.Value;
-
-                            while (this.MoveNext())
-                            { /*skip*/ }
-
-                            var hash = ReadJsonStringProperty(m_reader, "volumehash");
-                            var size = ReadJsonInt64Property(m_reader, "volumesize");
-
-                            return (m_volumeProps = new KeyValuePair<string, long>(hash, size)).Value;
-                        }
-
-                        public void Reset()
-                        {
-                            this.Dispose();
-                            m_stream = new StreamReader(m_compression.OpenRead(m_filename));
-                            m_reader = new JsonTextReader(m_stream);
-                            SkipJsonToken(m_reader, JsonToken.StartObject);
-                            var p = SkipJsonToken(m_reader, JsonToken.PropertyName);
-                            if (p == null || p.ToString() != "blocks")
-                                throw new InvalidDataException(string.Format("Invalid JSON, expected property \"blocks\", but got {0}, {1}", m_reader.TokenType, m_reader.Value));
-                            SkipJsonToken(m_reader, JsonToken.StartArray);
-
-                            m_current = null;
-                            m_done = false;
-                        }
-                    }
-
-                    private ICompression m_compression;
-                    private string m_filename;
-                    private BlockEnumerator m_enumerator;
-
-                    public BlockEnumerable(ICompression compression, string filename)
-                    {
-                        m_compression = compression;
-                        m_filename = filename;
-                    }
-
-                    public KeyValuePair<string, long> ReadVolumeProps()
-                    {
-                        if (m_enumerator == null)
-                            m_enumerator = (BlockEnumerator)this.GetEnumerator();
-
-                        return m_enumerator.ReadVolumeProps();
-                    }
-
-                    public IEnumerator<KeyValuePair<string, long>> GetEnumerator()
-                    {
-                        if (m_enumerator != null)
-                            throw new NotSupportedException("Cannot read block stream twice");
-
-                        return m_enumerator = new BlockEnumerator(m_compression, m_filename);
-                    }
-
-                    System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
-                    {
-                        return this.GetEnumerator();
-                    }
-                }
-
-                private class ShadowBlockVolume : IShadowBlockVolume
-                {
-                    private ICompression m_compression;
-                    private string m_filename;
-                    private long? m_length;
-                    private string m_hash;
-                    private BlockEnumerable m_blocks;
-
-                    public ShadowBlockVolume(ICompression compression, string filename)
-                    {
-                        m_compression = compression;
-                        m_filename = filename;
-                    }
-
-                    public string Filename { get { return m_filename.Substring(SHADOW_VOLUME_FOLDER.Length); } }
-
-                    private void ReadVolumeProps()
-                    {
-                        if (m_length == null || m_hash == null)
-                        {
-                            if (m_blocks == null)
-                                m_blocks = (BlockEnumerable)this.Blocks;
-
-                            var kp = m_blocks.ReadVolumeProps();
-                            m_length = kp.Value;
-                            m_hash = kp.Key;
-                        }
-                    }
-
-                    public long Length
-                    {
-                        get
-                        {
-                            this.ReadVolumeProps();
-                            return m_length.Value;
-                        }
-                    }
-
-                    public string Hash
-                    {
-                        get
-                        {
-                            this.ReadVolumeProps();
-                            return m_hash;
-                        }
-                    }
-
-                    public IEnumerable<KeyValuePair<string, long>> Blocks
-                    {
-                        get
-                        {
-                            if (m_blocks != null)
-                                throw new NotSupportedException("Cannot read Blocks twice");
-
-                            return m_blocks = new BlockEnumerable(m_compression, m_filename);
-                        }
-                    }
-                }
-
-                private ICompression m_compression;
-                private ShadowBlockVolume m_current;
-                private string[] m_files;
-                private long m_index;
-
-                public ShadowBlockVolumeEnumerator(ICompression compression)
-                {
-                    m_compression = compression;
-                    this.Reset();
-                }
-
-                public IShadowBlockVolume Current
-                {
-                    get { return m_current; }
-                }
-
-                public void Dispose()
-                {
-                }
-
-                object System.Collections.IEnumerator.Current
-                {
-                    get { return this.Current; }
-                }
-
-                public bool MoveNext()
-                {
-                    if (m_index + 1 >= m_files.Length)
-                        return false;
-                    m_index++;
-
-                    while (m_index < m_files.Length && ParseFilename(m_files[m_index]) == null)
-                        m_index++;
-
-                    m_current = new ShadowBlockVolume(m_compression, m_files[m_index]);
-
-                    return true;
-                }
-
-                public void Reset()
-                {
-                    m_files = m_compression.ListFiles(SHADOW_VOLUME_FOLDER);
-                    m_index = -1;
-                    m_current = null;
-                }
-            }
-
-            public IEnumerator<IShadowBlockVolume> GetEnumerator() { return new ShadowBlockVolumeEnumerator(m_compression); }
-            System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); }
-        }
-
-        private class ShadowBlocklistEnumerable : IEnumerable<IShadowBlocklist>
-        {
-            private ICompression m_compression;
-            private long m_hashsize;
-
-            public ShadowBlocklistEnumerable(ICompression compression, long hashsize)
-            {
-                m_compression = compression;
-                m_hashsize = hashsize;
-            }
-
-            private class ShadowBlocklistEnumerator : IEnumerator<IShadowBlocklist>
-            {
-                private class ShadowBlocklist : IShadowBlocklist
-                {
-                    private ICompression m_compression;
-                    private string m_filename;
-                    private long m_size;
-                    private long m_hashsize;
-
-                    public ShadowBlocklist(ICompression compression, string filename, long size, long hashsize)
-                    {
-                        m_compression = compression;
-                        m_filename = filename;
-                        m_size = size;
-                        m_hashsize = hashsize;
-                    }
-
-                    public string Hash
-                    {
-                        //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, 
-                        // to prevent clashes with filename paths where forward slash has special meaning
-                        get { return m_filename.Substring(SHADOW_BLOCKLIST_FOLDER.Length).Replace('-', '+').Replace('_', '-'); }
-                    }
-
-                    public long Length
-                    {
-                        get { return m_size; }
-                    }
-
-                    public IEnumerable<string> Blocklist
-                    {
-                        get { return new BlocklistEnumerable(m_compression, m_filename, m_hashsize); }
-                    }
-                }
-
-                private ICompression m_compression;
-                private long m_index;
-                private KeyValuePair<string, long>[] m_files;
-                private ShadowBlocklist m_current;
-                private long m_hashsize;
-
-                public ShadowBlocklistEnumerator(ICompression compression, long hashsize)
-                {
-                    m_compression = compression;
-                    m_hashsize = hashsize;
-                    this.Reset();
-                }
-
-                public IShadowBlocklist Current { get { return m_current; } }
-                public void Dispose() { }
-                object System.Collections.IEnumerator.Current { get { return this.Current; } }
-
-                private readonly System.Text.RegularExpressions.Regex m_base64_urlsafe_detector = new System.Text.RegularExpressions.Regex("[a-zA-Z0-9-_]+={0,2}");
-                private bool IsValidBase64Hash(string value, long hashsize)
-                {
-                    if (value.Length != ((hashsize + 2) / 3) * 4)
-                        return false;
-
-                    var m = m_base64_urlsafe_detector.Match(value);
-                    return m.Success && m.Length == value.Length;
-                }
-
-                public bool MoveNext()
-                {
-                    if (m_index + 1 >= m_files.Length)
-                        return false;
-                    m_index++;
-
-                    while (m_index < m_files.Length && IsValidBase64Hash(m_files[m_index].Key, m_hashsize))
-                        m_index++;
-
-                    m_current = new ShadowBlocklist(m_compression, m_files[m_index].Key, m_files[m_index].Value, m_hashsize);
-
-                    return true;
-                }
-
-                public void Reset()
-                {
-                    m_files = m_compression.ListFilesWithSize(SHADOW_BLOCKLIST_FOLDER).ToArray();
-                    m_index = -1;
-                    m_current = null;
-                }
-            }
-
-            public IEnumerator<IShadowBlocklist> GetEnumerator() { return new ShadowBlocklistEnumerator(m_compression, m_hashsize); }
-            System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); }
-        }
-
-        private long m_hashsize;
-
-        public ShadowVolumeReader(ICompression compression, FhOptions options, long hashsize)
-            : base(compression, options)
-        {
-            m_hashsize = hashsize;
-        }
-
-        public ShadowVolumeReader(string compressor, string file, FhOptions options, long hashsize)
-            : base(compressor, file, options)
-        {
-            m_hashsize = hashsize;
-        }
-
-        public IEnumerable<IShadowBlockVolume> Volumes { get { return new ShadowBlockVolumeEnumerable(m_compression); } }
-        public IEnumerable<IShadowBlocklist> BlockLists { get { return new ShadowBlocklistEnumerable(m_compression, m_hashsize); } }
-    }
-}
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using Newtonsoft.Json;
+using Duplicati.Library.Interface;
+
+namespace Duplicati.Library.Main.ForestHash.Volumes
+{
+    public class IndexVolumeReader : VolumeReaderBase
+    {
+        public interface IIndexBlocklist
+        {
+            string Hash { get; }
+            long Length { get; }
+            IEnumerable<string> Blocklist { get; }
+        }
+
+        public interface IIndexBlockVolume
+        {
+            string Filename { get; }
+            long Length { get; }
+            string Hash { get; }
+            IEnumerable<KeyValuePair<string, long>> Blocks { get; }
+        }
+
+        private class IndexBlockVolumeEnumerable : IEnumerable<IIndexBlockVolume>
+        {
+            private ICompression m_compression;
+
+            public IndexBlockVolumeEnumerable(ICompression compression)
+            {
+                m_compression = compression;
+            }
+
+            private class IndexBlockVolumeEnumerator : IEnumerator<IIndexBlockVolume>
+            {
+                private class BlockEnumerable : IEnumerable<KeyValuePair<string, long>>
+                {
+                    private class BlockEnumerator : IEnumerator<KeyValuePair<string, long>>
+                    {
+                        private ICompression m_compression;
+                        private string m_filename;
+                        private KeyValuePair<string, long>? m_current;
+                        private System.IO.StreamReader m_stream;
+                        private JsonReader m_reader;
+                        private bool m_done;
+                        private KeyValuePair<string, long>? m_volumeProps = null;
+
+                        public BlockEnumerator(ICompression compression, string filename)
+                        {
+                            m_compression = compression;
+                            m_filename = filename;
+                            this.Reset();
+                        }
+
+                        public KeyValuePair<string, long> Current
+                        {
+                            get { return m_current.Value; }
+                        }
+
+                        public void Dispose()
+                        {
+                            if (m_reader != null)
+                                this.ReadVolumeProps();
+
+                            if (m_reader != null)
+                                try { m_reader.Close(); }
+                                finally { m_reader = null; }
+
+                            if (m_stream != null)
+                                try { m_stream.Dispose(); }
+                                finally { m_stream = null; }
+                        }
+
+                        object System.Collections.IEnumerator.Current
+                        {
+                            get { return this.Current; }
+                        }
+
+                        public bool MoveNext()
+                        {
+                            if (m_done)
+                                return false;
+
+                            if (!m_reader.Read())
+                                throw new InvalidDataException(string.Format("Invalid JSON, EOF found while reading hashes"));
+
+                            if (m_reader.TokenType == JsonToken.EndArray)
+                            {
+                                m_done = true;
+                                m_current = null;
+                                return false;
+                            }
+
+                            if (m_reader.TokenType != JsonToken.StartObject)
+                                throw new InvalidDataException(string.Format("Invalid JSON, expected StartObject, but got {0}, {1}", m_reader.TokenType, m_reader.Value));
+
+                            var hash = ReadJsonStringProperty(m_reader, "hash");
+                            var size = ReadJsonInt64Property(m_reader, "size");
+
+                            m_current = new KeyValuePair<string, long>(hash, size);
+
+                            while (m_reader.Read() && m_reader.TokenType != JsonToken.EndObject)
+                            { /* skip */ }
+
+                            return true;
+                        }
+
+                        public KeyValuePair<string, long> ReadVolumeProps()
+                        {
+                            if (m_volumeProps != null)
+                                return m_volumeProps.Value;
+
+                            while (this.MoveNext())
+                            { /*skip*/ }
+
+                            var hash = ReadJsonStringProperty(m_reader, "volumehash");
+                            var size = ReadJsonInt64Property(m_reader, "volumesize");
+
+                            return (m_volumeProps = new KeyValuePair<string, long>(hash, size)).Value;
+                        }
+
+                        public void Reset()
+                        {
+                            this.Dispose();
+                            m_stream = new StreamReader(m_compression.OpenRead(m_filename));
+                            m_reader = new JsonTextReader(m_stream);
+                            SkipJsonToken(m_reader, JsonToken.StartObject);
+                            var p = SkipJsonToken(m_reader, JsonToken.PropertyName);
+                            if (p == null || p.ToString() != "blocks")
+                                throw new InvalidDataException(string.Format("Invalid JSON, expected property \"blocks\", but got {0}, {1}", m_reader.TokenType, m_reader.Value));
+                            SkipJsonToken(m_reader, JsonToken.StartArray);
+
+                            m_current = null;
+                            m_done = false;
+                        }
+                    }
+
+                    private ICompression m_compression;
+                    private string m_filename;
+                    private BlockEnumerator m_enumerator;
+
+                    public BlockEnumerable(ICompression compression, string filename)
+                    {
+                        m_compression = compression;
+                        m_filename = filename;
+                    }
+
+                    public KeyValuePair<string, long> ReadVolumeProps()
+                    {
+                        if (m_enumerator == null)
+                            m_enumerator = (BlockEnumerator)this.GetEnumerator();
+
+                        return m_enumerator.ReadVolumeProps();
+                    }
+
+                    public IEnumerator<KeyValuePair<string, long>> GetEnumerator()
+                    {
+                        if (m_enumerator != null)
+                            throw new NotSupportedException("Cannot read block stream twice");
+
+                        return m_enumerator = new BlockEnumerator(m_compression, m_filename);
+                    }
+
+                    System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+                    {
+                        return this.GetEnumerator();
+                    }
+                }
+
+                private class IndexBlockVolume : IIndexBlockVolume
+                {
+                    private ICompression m_compression;
+                    private string m_filename;
+                    private long? m_length;
+                    private string m_hash;
+                    private BlockEnumerable m_blocks;
+
+                    public IndexBlockVolume(ICompression compression, string filename)
+                    {
+                        m_compression = compression;
+                        m_filename = filename;
+                    }
+
+                    public string Filename { get { return m_filename.Substring(INDEX_VOLUME_FOLDER.Length); } }
+
+                    private void ReadVolumeProps()
+                    {
+                        if (m_length == null || m_hash == null)
+                        {
+                            if (m_blocks == null)
+                                m_blocks = (BlockEnumerable)this.Blocks;
+
+                            var kp = m_blocks.ReadVolumeProps();
+                            m_length = kp.Value;
+                            m_hash = kp.Key;
+                        }
+                    }
+
+                    public long Length
+                    {
+                        get
+                        {
+                            this.ReadVolumeProps();
+                            return m_length.Value;
+                        }
+                    }
+
+                    public string Hash
+                    {
+                        get
+                        {
+                            this.ReadVolumeProps();
+                            return m_hash;
+                        }
+                    }
+
+                    public IEnumerable<KeyValuePair<string, long>> Blocks
+                    {
+                        get
+                        {
+                            if (m_blocks != null)
+                                throw new NotSupportedException("Cannot read Blocks twice");
+
+                            return m_blocks = new BlockEnumerable(m_compression, m_filename);
+                        }
+                    }
+                }
+
+                private ICompression m_compression;
+                private IndexBlockVolume m_current;
+                private string[] m_files;
+                private long m_index;
+
+                public IndexBlockVolumeEnumerator(ICompression compression)
+                {
+                    m_compression = compression;
+                    this.Reset();
+                }
+
+                public IIndexBlockVolume Current
+                {
+                    get { return m_current; }
+                }
+
+                public void Dispose()
+                {
+                }
+
+                object System.Collections.IEnumerator.Current
+                {
+                    get { return this.Current; }
+                }
+
+                public bool MoveNext()
+                {
+                    if (m_index + 1 >= m_files.Length)
+                        return false;
+                    m_index++;
+
+                    while (m_index < m_files.Length && ParseFilename(m_files[m_index]) == null)
+                        m_index++;
+
+                    m_current = new IndexBlockVolume(m_compression, m_files[m_index]);
+
+                    return true;
+                }
+
+                public void Reset()
+                {
+                    m_files = m_compression.ListFiles(INDEX_VOLUME_FOLDER);
+                    m_index = -1;
+                    m_current = null;
+                }
+            }
+
+            public IEnumerator<IIndexBlockVolume> GetEnumerator() { return new IndexBlockVolumeEnumerator(m_compression); }
+            System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); }
+        }
+
+        private class IndexBlocklistEnumerable : IEnumerable<IIndexBlocklist>
+        {
+            private ICompression m_compression;
+            private long m_hashsize;
+
+            public IndexBlocklistEnumerable(ICompression compression, long hashsize)
+            {
+                m_compression = compression;
+                m_hashsize = hashsize;
+            }
+
+            private class IndexBlocklistEnumerator : IEnumerator<IIndexBlocklist>
+            {
+                private class IndexBlocklist : IIndexBlocklist
+                {
+                    private ICompression m_compression;
+                    private string m_filename;
+                    private long m_size;
+                    private long m_hashsize;
+
+                    public IndexBlocklist(ICompression compression, string filename, long size, long hashsize)
+                    {
+                        m_compression = compression;
+                        m_filename = filename;
+                        m_size = size;
+                        m_hashsize = hashsize;
+                    }
+
+                    public string Hash
+                    {
+                        //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, 
+                        // to prevent clashes with filename paths where forward slash has special meaning
+                        get { return m_filename.Substring(INDEX_BLOCKLIST_FOLDER.Length).Replace('-', '+').Replace('_', '-'); }
+                    }
+
+                    public long Length
+                    {
+                        get { return m_size; }
+                    }
+
+                    public IEnumerable<string> Blocklist
+                    {
+                        get { return new BlocklistEnumerable(m_compression, m_filename, m_hashsize); }
+                    }
+                }
+
+                private ICompression m_compression;
+                private long m_index;
+                private KeyValuePair<string, long>[] m_files;
+                private IndexBlocklist m_current;
+                private long m_hashsize;
+
+                public IndexBlocklistEnumerator(ICompression compression, long hashsize)
+                {
+                    m_compression = compression;
+                    m_hashsize = hashsize;
+                    this.Reset();
+                }
+
+                public IIndexBlocklist Current { get { return m_current; } }
+                public void Dispose() { }
+                object System.Collections.IEnumerator.Current { get { return this.Current; } }
+
+                private readonly System.Text.RegularExpressions.Regex m_base64_urlsafe_detector = new System.Text.RegularExpressions.Regex("[a-zA-Z0-9-_]+={0,2}");
+                private bool IsValidBase64Hash(string value, long hashsize)
+                {
+                    if (value.Length != ((hashsize + 2) / 3) * 4)
+                        return false;
+
+                    var m = m_base64_urlsafe_detector.Match(value);
+                    return m.Success && m.Length == value.Length;
+                }
+
+                public bool MoveNext()
+                {
+                    if (m_index + 1 >= m_files.Length)
+                        return false;
+                    m_index++;
+
+                    while (m_index < m_files.Length && IsValidBase64Hash(m_files[m_index].Key, m_hashsize))
+                        m_index++;
+
+                    m_current = new IndexBlocklist(m_compression, m_files[m_index].Key, m_files[m_index].Value, m_hashsize);
+
+                    return true;
+                }
+
+                public void Reset()
+                {
+                    m_files = m_compression.ListFilesWithSize(INDEX_BLOCKLIST_FOLDER).ToArray();
+                    m_index = -1;
+                    m_current = null;
+                }
+            }
+
+            public IEnumerator<IIndexBlocklist> GetEnumerator() { return new IndexBlocklistEnumerator(m_compression, m_hashsize); }
+            System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { return this.GetEnumerator(); }
+        }
+
+        private long m_hashsize;
+
+        public IndexVolumeReader(ICompression compression, FhOptions options, long hashsize)
+            : base(compression, options)
+        {
+            m_hashsize = hashsize;
+        }
+
+        public IndexVolumeReader(string compressor, string file, FhOptions options, long hashsize)
+            : base(compressor, file, options)
+        {
+            m_hashsize = hashsize;
+        }
+
+        public IEnumerable<IIndexBlockVolume> Volumes { get { return new IndexBlockVolumeEnumerable(m_compression); } }
+        public IEnumerable<IIndexBlocklist> BlockLists { get { return new IndexBlocklistEnumerable(m_compression, m_hashsize); } }
+    }
+}

Duplicati/Library/Main/ForestHash/Volumes/IndexVolumeWriter.cs

@@ -1,86 +1,86 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.IO;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
 using Newtonsoft.Json;
-using Duplicati.Library.Interface;
-
-namespace Duplicati.Library.Main.ForestHash.Volumes
-{
-    public class ShadowVolumeWriter : VolumeWriterBase
-    {
-        private StreamWriter m_streamwriter = null;
-        private JsonWriter m_writer = null;
-
-        private long m_volumes = 0;
-        private long m_blocks = 0;
-        private long m_blocklists = 0;
-
-        public long VolumeCount { get { return m_volumes; } }
-        public long BlockCount { get { return m_blocks; } }
-        public long Blocklists { get { return m_blocklists; } }
-
-        public ShadowVolumeWriter(FhOptions options)
-            : base(options)
-        {
-        }
-
-        public override RemoteVolumeType FileType { get { return RemoteVolumeType.Shadow; } }
-
-        public void StartVolume(string filename)
-        {
-            if (m_writer != null || m_streamwriter != null)
-                throw new InvalidOperationException("Previous volume not finished, call FinishVolume before starting a new volume");
-
-            m_volumes++;
-            m_streamwriter = new StreamWriter(m_compression.CreateFile(SHADOW_VOLUME_FOLDER + filename, CompressionHint.Compressible, DateTime.UtcNow));
-            m_writer = new JsonTextWriter(m_streamwriter);
-            m_writer.WriteStartObject();
-            m_writer.WritePropertyName("blocks");
-            m_writer.WriteStartArray();
-        }
-
-        public void AddBlock(string hash, long size)
-        {
-            m_writer.WriteStartObject();
-            m_writer.WritePropertyName("hash");
-            m_writer.WriteValue(hash);
-            m_writer.WritePropertyName("size");
-            m_writer.WriteValue(size);
-            m_writer.WriteEndObject();
-            m_blocks++;
-        }
-
-        public void FinishVolume(string volumehash, long volumesize)
-        {
-            m_writer.WriteEndArray();
-            m_writer.WritePropertyName("volumehash");
-            m_writer.WriteValue(volumehash);
-            m_writer.WritePropertyName("volumesize");
-            m_writer.WriteValue(volumesize);
-            m_writer.WriteEndObject();
-
-            try { m_writer.Close(); }
-            finally { m_writer = null; }
-            try { m_streamwriter.Dispose(); }
-            finally { m_streamwriter = null; }
-        }
-
-        public void WriteBlocklist(string hash, byte[] data, int size)
-        {
-            m_blocklists++;
-            //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, 
-            using (var s = m_compression.CreateFile(SHADOW_BLOCKLIST_FOLDER + Utility.Utility.Base64PlainToBase64Url(hash), CompressionHint.Noncompressible, DateTime.UtcNow))
-                s.Write(data, 0, size);
-        }
-
-        public override void Dispose()
-        {
-            if (m_writer != null || m_streamwriter != null)
-                throw new InvalidOperationException("Attempted to dispose a shadow volume that was being written");
-            base.Dispose();
-        }
-
-    }
-}
+using Duplicati.Library.Interface;
+
+namespace Duplicati.Library.Main.ForestHash.Volumes
+{
+    public class IndexVolumeWriter : VolumeWriterBase
+    {
+        private StreamWriter m_streamwriter = null;
+        private JsonWriter m_writer = null;
+
+        private long m_volumes = 0;
+        private long m_blocks = 0;
+        private long m_blocklists = 0;
+
+        public long VolumeCount { get { return m_volumes; } }
+        public long BlockCount { get { return m_blocks; } }
+        public long Blocklists { get { return m_blocklists; } }
+
+        public IndexVolumeWriter(FhOptions options)
+            : base(options)
+        {
+        }
+
+        public override RemoteVolumeType FileType { get { return RemoteVolumeType.Index; } }
+
+        public void StartVolume(string filename)
+        {
+            if (m_writer != null || m_streamwriter != null)
+                throw new InvalidOperationException("Previous volume not finished, call FinishVolume before starting a new volume");
+
+            m_volumes++;
+            m_streamwriter = new StreamWriter(m_compression.CreateFile(INDEX_VOLUME_FOLDER + filename, CompressionHint.Compressible, DateTime.UtcNow));
+            m_writer = new JsonTextWriter(m_streamwriter);
+            m_writer.WriteStartObject();
+            m_writer.WritePropertyName("blocks");
+            m_writer.WriteStartArray();
+        }
+
+        public void AddBlock(string hash, long size)
+        {
+            m_writer.WriteStartObject();
+            m_writer.WritePropertyName("hash");
+            m_writer.WriteValue(hash);
+            m_writer.WritePropertyName("size");
+            m_writer.WriteValue(size);
+            m_writer.WriteEndObject();
+            m_blocks++;
+        }
+
+        public void FinishVolume(string volumehash, long volumesize)
+        {
+            m_writer.WriteEndArray();
+            m_writer.WritePropertyName("volumehash");
+            m_writer.WriteValue(volumehash);
+            m_writer.WritePropertyName("volumesize");
+            m_writer.WriteValue(volumesize);
+            m_writer.WriteEndObject();
+
+            try { m_writer.Close(); }
+            finally { m_writer = null; }
+            try { m_streamwriter.Dispose(); }
+            finally { m_streamwriter = null; }
+        }
+
+        public void WriteBlocklist(string hash, byte[] data, int size)
+        {
+            m_blocklists++;
+            //Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications, 
+            using (var s = m_compression.CreateFile(INDEX_BLOCKLIST_FOLDER + Utility.Utility.Base64PlainToBase64Url(hash), CompressionHint.Noncompressible, DateTime.UtcNow))
+                s.Write(data, 0, size);
+        }
+
+        public override void Dispose()
+        {
+            if (m_writer != null || m_streamwriter != null)
+                throw new InvalidOperationException("Attempted to dispose an index volume that was being written");
+            base.Dispose();
+        }
+
+    }
+}

Duplicati/Library/Main/ForestHash/Volumes/VolumeBase.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -48,6 +48,7 @@ namespace Duplicati.Library.Main.ForestHash.Volumes
                     throw new InvalidManifestException("FileHash", d.FileHash, filehash);
             }
         }
+        
 
         private class ParsedVolume : IParsedVolume
         {
@@ -59,7 +60,25 @@ namespace Duplicati.Library.Main.ForestHash.Volumes
             public string EncryptionModule { get; private set; }
             public Library.Interface.IFileEntry File { get; private set; }
 
-            private static readonly System.Text.RegularExpressions.Regex FILENAME_REGEXP = new System.Text.RegularExpressions.Regex(@"(?<prefix>[^\-]+)\-(?<filetype>(" + string.Join(")|(", Enum.GetNames(typeof(RemoteVolumeType))).ToLowerInvariant() + @"))\-(((?<guid>[0-9A-Fa-f]+)|(?<time>\d{8}T\d{6}Z)))\.(?<compression>[^\.]+)(\.(?<encryption>.+))?");
+	        internal static readonly IDictionary<RemoteVolumeType, string> REMOTE_TYPENAME_MAP;
+	        internal static readonly IDictionary<string, RemoteVolumeType> REVERSE_REMOTE_TYPENAME_MAP;
+            private static readonly System.Text.RegularExpressions.Regex FILENAME_REGEXP;
+
+			static ParsedVolume()
+			{
+				var dict = new Dictionary<RemoteVolumeType, string>();
+				dict[RemoteVolumeType.Blocks] = "dblock";
+				dict[RemoteVolumeType.Files] = "dlist";
+				dict[RemoteVolumeType.Index] = "dindex";
+				
+				var reversedict = new Dictionary<string, RemoteVolumeType>(System.StringComparer.InvariantCultureIgnoreCase);
+				foreach(var x in dict)
+					reversedict[x.Value] = x.Key;
+								
+				REMOTE_TYPENAME_MAP = dict;
+				REVERSE_REMOTE_TYPENAME_MAP = reversedict;
+				FILENAME_REGEXP = new System.Text.RegularExpressions.Regex(@"(?<prefix>[^\-]+)\-((?<guid>[0-9A-Fa-f]+)|(backup\-(?<time>\d{8}T\d{6}Z))).(?<filetype>(" + string.Join(")|(", dict.Values) + @"))\.(?<compression>[^\.]+)(\.(?<encryption>.+))?");
+			}
 
             private ParsedVolume() { }
 
@@ -69,10 +88,14 @@ namespace Duplicati.Library.Main.ForestHash.Volumes
                 if (!m.Success || m.Length != filename.Length)
                     return null;
 
+				RemoteVolumeType t;
+				if (!REVERSE_REMOTE_TYPENAME_MAP.TryGetValue(m.Groups["filetype"].Value, out t))
+					return null;
+
                 return new ParsedVolume()
                 {
                     Prefix = m.Groups["prefix"].Value,
-                    FileType = (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), m.Groups["filetype"].Value, true),
+                    FileType = t,
                     Guid = m.Groups["guid"].Success ? m.Groups["guid"].Value : null,
                     Time = m.Groups["time"].Success ? DateTime.ParseExact(m.Groups["time"].Value, "yyyyMMdd'T'HHmmssK", null, System.Globalization.DateTimeStyles.AssumeUniversal).ToUniversalTime() : new DateTime(0, DateTimeKind.Local),
                     CompressionModule = m.Groups["compression"].Value,
@@ -85,10 +108,11 @@ namespace Duplicati.Library.Main.ForestHash.Volumes
 		public static string GenerateFilename(RemoteVolumeType filetype, string prefix, string guid, DateTime timestamp, string compressionmodule, string encryptionmodule)
 		{
 			string volumename;
+
             if (filetype == RemoteVolumeType.Files)
-                volumename = prefix + "-" + (filetype.ToString().ToLowerInvariant()) + "-" + Utility.Utility.SerializeDateTime(timestamp) + "." + compressionmodule;
+                volumename = prefix + "-backup" + "-" + Utility.Utility.SerializeDateTime(timestamp) + "." + (ParsedVolume.REMOTE_TYPENAME_MAP[filetype]) + "." + compressionmodule;
             else
-                volumename = prefix + "-" + (filetype.ToString().ToLowerInvariant()) + "-" + guid + "." + compressionmodule;
+                volumename = prefix + "-" + guid + "." + (ParsedVolume.REMOTE_TYPENAME_MAP[filetype]) + "." + compressionmodule;
 
             if (!string.IsNullOrEmpty(encryptionmodule))
                 volumename += "." + encryptionmodule;
@@ -109,8 +133,8 @@ namespace Duplicati.Library.Main.ForestHash.Volumes
         protected const string MANIFEST_FILENAME = "manifest";
         protected const string FILELIST = "filelist.json";
 
-        protected const string SHADOW_VOLUME_FOLDER = "vol/";
-        protected const string SHADOW_BLOCKLIST_FOLDER = "list/";
+        protected const string INDEX_VOLUME_FOLDER = "vol/";
+        protected const string INDEX_BLOCKLIST_FOLDER = "list/";
 
         protected const string CONTROL_FILES_FOLDER = "extra/";
 

Duplicati/Library/Main/ForestHash/Volumes/VolumeWriterBase.cs

@@ -28,13 +28,23 @@ namespace Duplicati.Library.Main.ForestHash.Volumes
             : this(options, DateTime.UtcNow)
         {
         }
+        
+        private static string GenerateGuid(FhOptions options)
+        {
+        	var s = Guid.NewGuid().ToString("N");
+        	
+        	//We can choose shorter GUIDs here
+        	
+        	return s;
+        	
+        }
 
         public VolumeWriterBase(FhOptions options, DateTime timestamp)
             : base(options)
         {
             m_localfile = new Utility.TempFile();
 
-			m_volumename = GenerateFilename(this.FileType, options.BackupPrefix, Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray()), timestamp, options.CompressionModule, options.NoEncryption ? null : options.EncryptionModule);
+			m_volumename = GenerateFilename(this.FileType, options.BackupPrefix, GenerateGuid(options), timestamp, options.CompressionModule, options.NoEncryption ? null : options.EncryptionModule);
             m_compression = DynamicLoader.CompressionLoader.GetModule(options.CompressionModule, m_localfile, options.RawOptions);
             AddManifestfile();
         }

Duplicati/Library/Main/Strings/FhOptions.Designer.cs

@@ -312,6 +312,24 @@ namespace Duplicati.Library.Main.Strings {
             }
         }
         
+        /// <summary>
+        ///   Looks up a localized string similar to If this flag is set, no index files are uploaded to the remote destination. This reduces the required storage as well as the required transfer, but makes restoring slower as more volumes must be downloaded..
+        /// </summary>
+        internal static string FhnoindexfilesLong {
+            get {
+                return ResourceManager.GetString("FhnoindexfilesLong", resourceCulture);
+            }
+        }
+        
+        /// <summary>
+        ///   Looks up a localized string similar to Do not upload index files.
+        /// </summary>
+        internal static string FhnoindexfilesShort {
+            get {
+                return ResourceManager.GetString("FhnoindexfilesShort", resourceCulture);
+            }
+        }
+        
         /// <summary>
         ///   Looks up a localized string similar to Stores empty metadata, such as file timestamps and attributes. This decreases the required storage space as well as the processing time..
         /// </summary>
@@ -330,24 +348,6 @@ namespace Duplicati.Library.Main.Strings {
             }
         }
         
-        /// <summary>
-        ///   Looks up a localized string similar to If this flag is set, no shadow files are uploaded to the remote destination. This reduces the required storage as well as the required transfer, but makes restoring slower as more volumes must be downloaded..
-        /// </summary>
-        internal static string FhnoshadowfilesLong {
-            get {
-                return ResourceManager.GetString("FhnoshadowfilesLong", resourceCulture);
-            }
-        }
-        
-        /// <summary>
-        ///   Looks up a localized string similar to Do not upload shadow files.
-        /// </summary>
-        internal static string FhnoshadowfilesShort {
-            get {
-                return ResourceManager.GetString("FhnoshadowfilesShort", resourceCulture);
-            }
-        }
-        
         /// <summary>
         ///   Looks up a localized string similar to Enable this option to look into other files on this machine to find existing blocks. This is a fairly slow operation but can limit the size of downloads..
         /// </summary>

Duplicati/Library/Main/Strings/FhOptions.resx

@@ -177,11 +177,11 @@
   <data name="FhnobackendverificationShort" xml:space="preserve">
     <value>Do not query backend at startup</value>
   </data>
-  <data name="FhnoshadowfilesLong" xml:space="preserve">
-    <value>If this flag is set, no shadow files are uploaded to the remote destination. This reduces the required storage as well as the required transfer, but makes restoring slower as more volumes must be downloaded.</value>
+  <data name="FhnoindexfilesLong" xml:space="preserve">
+    <value>If this flag is set, no index files are uploaded to the remote destination. This reduces the required storage as well as the required transfer, but makes restoring slower as more volumes must be downloaded.</value>
   </data>
-  <data name="FhnoshadowfilesShort" xml:space="preserve">
-    <value>Do not upload shadow files</value>
+  <data name="FhnoindexfilesShort" xml:space="preserve">
+    <value>Do not upload index files</value>
   </data>
   <data name="FhmaxwastesizeLong" xml:space="preserve">
     <value>As files are changed, some data stored at the remote destination may not be required. This option controls how much wasted space the destination can contain before being reclaimed. This value cannot be less than the volumesize, the default value is two times the volumesize.</value>