using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.IO; using Duplicati.Library.Interface; using Duplicati.Library.Main.ForestHash.Database; using Newtonsoft.Json; namespace Duplicati.Library.Main.ForestHash { internal class ForestHash { /// /// Runs a backup /// /// The destination url /// The input options /// The status report instance /// The list of folders to back up public static void Backup(string remoteurl, Options options, BackupStatistics stat, string[] sources) { using (var h = new Operation.BackupHandler(remoteurl, new FhOptions(options.RawOptions), stat, sources)) h.Run(); } public static void Restore(string remoteurl, Options options, RestoreStatistics stat, string target) { using (var h = new Operation.RestoreHandler(remoteurl, new FhOptions(options.RawOptions), stat, target)) h.Run(); } /// /// Helper method that verifies uploaded volumes and updates their state in the database. /// Throws an error if there are issues with the remote storage /// /// The backend instance to use /// The options used /// The database to compare with public static void VerifyRemoteList(FhBackend backend, Options options, Localdatabase database) { var remotelist = backend.List(); var lookup = new Dictionary(); var prefix = options.BackupPrefix + "-"; var suffix = "." + options.CompressionModule; if (!options.NoEncryption) suffix += "." + options.EncryptionModule; foreach (var s in remotelist) { if (s.Name.StartsWith(prefix)) lookup[s.Name] = s; } var missing = new List>(); var locallist = database.GetRemoteVolumes(); foreach (var i in locallist) { if (i.State == RemoteVolumeState.Temporary) { database.LogMessage("info", string.Format("removing file listed as {0}: {1}", i.State, i.Name), null); database.RemoveRemoteVolume(i.Name); } else { Library.Interface.IFileEntry r; if (!lookup.TryGetValue(i.Name, out r) || (r.Size != i.Size && r.Size >= 0 && i.Size >= 0)) { if (i.State == RemoteVolumeState.Uploading) { database.LogMessage("info", string.Format("removing file listed as {0}: {1}", i.State, i.Name), null); database.RemoveRemoteVolume(i.Name); } else missing.Add(new KeyValuePair(i, r)); } else if (i.State != RemoteVolumeState.Verified) { database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash); } lookup.Remove(i.Name); } } if (lookup.Count > 0) throw new Exception(string.Format("Found {0} remote files that are not recorded in local storage, please run cleanup", lookup.Count)); if (missing.Count > 0) throw new Exception(string.Format("Found {0} files that are missing from the remote storage, please run cleanup", missing.Count)); } /// /// Interface for describing metadata /// public interface IMetahash { /// /// The base64 encoded hash of the metadata /// string Hash { get; } /// /// The size of the metadata in bytes /// long Size { get; } /// /// The UTF-8 encoded json element with the metadata /// byte[] Blob { get; } /// /// The lookup table of contained metadata values /// Dictionary Values { get; } } /// /// Implementation of the IMetahash interface /// private class Metahash : IMetahash { /// /// The base64 encoded hash /// private readonly string m_hash; /// /// The UTF-8 encoded json element with the metadata /// private readonly byte[] m_blob; /// /// The lookup table with elements /// private readonly Dictionary m_values; public Metahash(Dictionary values) { m_values = values; var sha = System.Security.Cryptography.SHA256.Create(); using (var ms = new System.IO.MemoryStream()) using (var w = new StreamWriter(ms, Encoding.UTF8)) { w.Write(JsonConvert.SerializeObject(values)); w.Flush(); m_blob = ms.ToArray(); ms.Position = 0; m_hash = Convert.ToBase64String(sha.ComputeHash(ms)); } } public string Hash { get { return m_hash; } } public long Size { get { return m_blob.Length; } } public byte[] Blob { get { return m_blob; } } public Dictionary Values { get { return m_values; } } } /// /// Constructs a container for a given metadata dictionary /// /// The metadata values to wrap /// A IMetahash instance public static IMetahash WrapMetadata(Dictionary values) { return new Metahash(values); } internal static IEnumerable ParseFileList(string target, Dictionary options, CommunicationStatistics stat) { var opts = new FhOptions(options); using (var db = new Localdatabase(opts.Fhdbpath, "ParseFileList")) using (var b = new FhBackend(target, opts, db, stat)) return from n in b.List() let np = Volumes.VolumeBase.ParseFilename(n) where np != null select np; } internal static string CompactBlocks(string target, Dictionary options) { throw new NotImplementedException(); } } }