using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.IO; using Duplicati.Library.Interface; using Duplicati.Library.Main.ForestHash.Database; using Newtonsoft.Json; namespace Duplicati.Library.Main.ForestHash { internal class ForestHash { /// /// Runs a backup /// /// The destination url /// The input options /// The status report instance /// The list of folders to back up public static void Backup(string remoteurl, Options options, BackupStatistics stat, string[] sources) { using (var h = new Operation.BackupHandler(remoteurl, new FhOptions(options.RawOptions), stat, sources)) h.Run(); } public static void Restore(string remoteurl, Options options, RestoreStatistics stat, string target) { using (var h = new Operation.RestoreHandler(remoteurl, new FhOptions(options.RawOptions), stat, target)) h.Run(); } /// /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// /// The backend instance to use /// The options used /// The database to compare with public static void VerifyRemoteList(FhBackend backend, Options options, LocalDatabase database, CommunicationStatistics stat) { var tp = RemoteListAnalysis(backend, options, database); long extraCount = 0; long missingCount = 0; foreach(var n in tp.ExtraVolumes) { if (!options.QuietConsole) stat.LogMessage(string.Format("Extra unknown file: {0}", n.File.Name)); stat.LogWarning(string.Format("Extra unknown file: {0}", n.File.Name), null); extraCount++; } foreach(var n in tp.MissingVolumes) { if (!options.QuietConsole) stat.LogMessage(string.Format("Missing file: {0}", n.Name)); stat.LogWarning(string.Format("Missing file: {0}", n.Name), null); missingCount++; } if (extraCount > 0) throw new Exception(string.Format("Found {0} remote files that are not recorded in local storage, please run cleanup", extraCount)); if (missingCount > 0) { if (!tp.BackupPrefixes.Contains(options.BackupPrefix) && tp.BackupPrefixes.Length > 0) throw new Exception(string.Format("Found {0} files that are missing from the remote storage, and no files with the backup prefix {1}, but found the following backup prefixes: {2}", missingCount, options.BackupPrefix, string.Join(", ", tp.BackupPrefixes))); else throw new Exception(string.Format("Found {0} files that are missing from the remote storage, please run cleanup", missingCount)); } } public struct RemoteAnalysisResult { public IEnumerable ParsedVolumes; public IEnumerable ExtraVolumes; public IEnumerable MissingVolumes; public string[] BackupPrefixes { get { return ParsedVolumes.Select(x => x.Prefix).Distinct().ToArray(); } } } /// /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// /// The backend instance to use /// The options used /// The database to compare with public static RemoteAnalysisResult RemoteListAnalysis(FhBackend backend, Options options, LocalDatabase database) { var rawlist = backend.List(); var lookup = new Dictionary(); var remotelist = from n in rawlist let p = Volumes.VolumeBase.ParseFilename(n) where p != null select p; foreach (var s in remotelist) if (s.Prefix == options.BackupPrefix) lookup[s.File.Name] = s; var missing = new List(); var locallist = database.GetRemoteVolumes(); foreach (var i in locallist) { //Ignore those that are deleted if (i.State == RemoteVolumeState.Deleted) continue; if (i.State == RemoteVolumeState.Temporary) { database.LogMessage("info", string.Format("removing file listed as {0}: {1}", i.State, i.Name), null, null); database.RemoveRemoteVolume(i.Name, null); } else { Volumes.IParsedVolume r; if (!lookup.TryGetValue(i.Name, out r)) { if (i.State == RemoteVolumeState.Uploading || i.State == RemoteVolumeState.Deleting || (r != null && r.File.Size != i.Size && r.File.Size >= 0 && i.Size >= 0)) { database.LogMessage("info", string.Format("removing file listed as {0}: {1}", i.State, i.Name), null, null); database.RemoveRemoteVolume(i.Name, null); } else missing.Add(i); } else if (i.State != RemoteVolumeState.Verified) { database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); } lookup.Remove(i.Name); } } return new RemoteAnalysisResult() { ParsedVolumes = remotelist, ExtraVolumes = lookup.Values, MissingVolumes = missing }; } /// /// Interface for describing metadata /// public interface IMetahash { /// /// The base64 encoded hash of the metadata /// string Hash { get; } /// /// The size of the metadata in bytes /// long Size { get; } /// /// The UTF-8 encoded json element with the metadata /// byte[] Blob { get; } /// /// The lookup table of contained metadata values /// Dictionary Values { get; } } /// /// Implementation of the IMetahash interface /// private class Metahash : IMetahash { /// /// The base64 encoded hash /// private readonly string m_hash; /// /// The UTF-8 encoded json element with the metadata /// private readonly byte[] m_blob; /// /// The lookup table with elements /// private readonly Dictionary m_values; public Metahash(Dictionary values, FhOptions options) { m_values = values; var hasher = System.Security.Cryptography.HashAlgorithm.Create(options.FhBlockHashAlgorithm); if (hasher == null) throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, options.FhBlockHashAlgorithm)); if (!hasher.CanReuseTransform) throw new Exception(string.Format(Strings.Foresthash.InvalidCryptoSystem, options.FhBlockHashAlgorithm)); using (var ms = new System.IO.MemoryStream()) using (var w = new StreamWriter(ms, Encoding.UTF8)) { w.Write(JsonConvert.SerializeObject(values)); w.Flush(); m_blob = ms.ToArray(); ms.Position = 0; m_hash = Convert.ToBase64String(hasher.ComputeHash(ms)); } } public string Hash { get { return m_hash; } } public long Size { get { return m_blob.Length; } } public byte[] Blob { get { return m_blob; } } public Dictionary Values { get { return m_values; } } } /// /// Constructs a container for a given metadata dictionary /// /// The metadata values to wrap /// A IMetahash instance public static IMetahash WrapMetadata(Dictionary values, FhOptions options) { return new Metahash(values, options); } internal static void VerifyParameters(LocalDatabase db, FhOptions options) { var newDict = new Dictionary(); newDict.Add("blocksize", options.Fhblocksize.ToString()); newDict.Add("blockhash", options.FhBlockHashAlgorithm); newDict.Add("filehash", options.FhFileHashAlgorithm); var opts = db.GetDbOptions(); var needsUpdate = false; foreach(var k in newDict) if (!opts.ContainsKey(k.Key)) needsUpdate = true; else if (opts[k.Key] != k.Value) throw new Exception(string.Format("Unsupported change of parameter {0} from {1} to {2}", k.Key, opts[k.Key], k.Value)); //Extra sanity check if (db.GetBlocksLargerThan(options.Fhblocksize) > 0) throw new Exception("Unsupported block-size change detected"); if (needsUpdate) db.SetDbOptions(newDict); } internal static IEnumerable ParseFileList(string target, Dictionary options, CommunicationStatistics stat) { var opts = new FhOptions(options); using (var db = new LocalDatabase(opts.Fhdbpath, "ParseFileList")) using (var b = new FhBackend(target, opts, stat, db)) { var res = from n in b.List() let np = Volumes.VolumeBase.ParseFilename(n) where np != null select np; b.WaitForComplete(db, null); return res; } } internal static string CompactBlocks(string target, Dictionary options, CommunicationStatistics stat) { using(var h = new Operation.CompactHandler(target, new FhOptions(options), stat)) return h.Run(); } internal static string RecreateDatabase(string target, Dictionary options, CommunicationStatistics stat) { var opts = new FhOptions(options); using(var h = new Operation.RecreateDatabaseHandler(target, opts, stat)) h.Run(opts.Fhdbpath); return "Recreate Completed"; } public static string DeleteFilesets(string target, string filesets, Dictionary options, CommunicationStatistics stat) { var opts = new FhOptions(options); using(var h = new Operation.DeleteHandler(target, opts, stat)) { h.Filesets = filesets; return h.Run(); } } public static string CreateLogDatabase(string target, Dictionary options, CommunicationStatistics stat) { var opts = new FhOptions(options); using(var h = new Operation.CreateBugReportHandler(target, opts, stat)) h.Run(); return "Create db Completed! Please verify that the database contains no sensitive information before sending!"; } } }