// Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.IO; using System.Threading.Tasks; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Utility; namespace Duplicati.Library.Main.Operation.Common { /// /// A collection class for keeping the temporary data required to build an index file /// internal class TemporaryIndexVolume { /// /// The block hashes /// private readonly Library.Utility.FileBackedStringList blockHashes = new Library.Utility.FileBackedStringList(); /// /// The blocklist hashes /// private readonly Library.Utility.FileBackedStringList blockListHashes = new Library.Utility.FileBackedStringList(); /// /// Cached copy of the blocklist hash size /// private readonly int m_blockhashsize; /// /// Initializes a new instance of the /// class. /// /// The options used in this run. public TemporaryIndexVolume(Options options) { m_blockhashsize = options.BlockhashSize; } /// /// Creates an index volume with the temporary contents /// /// The index volume. /// The name of the block file. /// Hash of the volume /// Size of the volume public async Task CreateVolume(string blockfilename, string blockHash, long blockSize, Options options, DatabaseCommon database) { var w = new IndexVolumeWriter(options); w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); w.StartVolume(blockfilename); foreach (var n in blockHashes) { var args = n.Split(new char[] { ':' }, 2); w.AddBlock(args[1], long.Parse(args[0])); } w.FinishVolume(blockHash, blockSize); var enumerator = blockListHashes.GetEnumerator(); while(enumerator.MoveNext()) { var hash = enumerator.Current; enumerator.MoveNext(); var data = Convert.FromBase64String(enumerator.Current); w.WriteBlocklist(hash, data, 0, data.Length); } w.Close(); return w; } /// /// Copies all entries from this temporary instance to the target /// /// The target volume. /// Only copies the blocklist hashes public void CopyTo(TemporaryIndexVolume target, bool onlyBlocklistHashes) { if (!onlyBlocklistHashes) foreach (var n in blockHashes) target.blockHashes.Add(n); foreach (var n in blockListHashes) target.blockListHashes.Add(n); } /// /// Adds a single block hash to the index /// /// The hash of the block. /// The size of the block. public void AddBlock(string hash, long size) { blockHashes.Add(size.ToString() + ":" + hash); } /// /// Adds a block list hash to the index /// /// The hash of the block to add. /// The size of the block. /// The block contents. public void AddBlockListHash(string hash, long size, byte[] data) { if (size % m_blockhashsize != 0) throw new ArgumentException($"The {nameof(size)} value is {size}, but it must be evenly divisible by the blockhash size ({m_blockhashsize})", nameof(size)); blockListHashes.Add(hash); blockListHashes.Add(Convert.ToBase64String(data, 0, (int)size)); } } /// /// Encapsulates creating an index file from the database contents /// internal static class IndexVolumeCreator { public static async Task CreateIndexVolume(string blockname, Options options, Common.DatabaseCommon database) { using(var h = HashFactory.CreateHasher(options.BlockHashAlgorithm)) { var w = new IndexVolumeWriter(options); w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary); var blockvolume = await database.GetVolumeInfoAsync(blockname); w.StartVolume(blockname); foreach(var b in await database.GetBlocksAsync(blockvolume.ID)) w.AddBlock(b.Hash, b.Size); w.FinishVolume(blockvolume.Hash, blockvolume.Size); if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) foreach(var b in await database.GetBlocklistsAsync(blockvolume.ID, options.Blocksize, options.BlockhashSize)) { var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); if (bh != b.Item1) throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1)); w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3); } w.Close(); // Register that the index file is tracking the block file await database.AddIndexBlockLinkAsync(w.VolumeID, blockvolume.ID); return w; } } /*public static async Task ReCreateIndexVolume(string selfname, Options options, Repair.RepairDatabase database) { using(var h = HashFactory.CreateHasher(options.BlockHashAlgorithm)) { var w = new IndexVolumeWriter(options); w.SetRemoteFilename(selfname); foreach(var blockvolume in await database.GetBlockVolumesFromIndexNameAsync(selfname)) { w.StartVolume(blockvolume.Name); var volumeid = await database.GetRemoteVolumeIDAsync(blockvolume.Name); foreach(var b in await database.GetBlocksAsync(volumeid)) w.AddBlock(b.Hash, b.Size); w.FinishVolume(blockvolume.Hash, blockvolume.Size); if (options.IndexfilePolicy == Options.IndexFileStrategy.Full) foreach(var b in await database.GetBlocklistsAsync(volumeid, options.Blocksize, options.BlockhashSize)) { var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3)); if (bh != b.Item1) throw new Exception(string.Format("Internal consistency check failed, generated index block has wrong hash, {0} vs {1}", bh, b.Item1)); w.WriteBlocklist(b.Item1, b.Item2, 0, b.Item3); } } w.Close(); return w; } }*/ } }