diff --git a/Duplicati.sln b/Duplicati.sln index e98b4f68d..3222ee63e 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -69,6 +69,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.O EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.HubiC", "Duplicati\Library\Backend\HubiC\Duplicati.Library.Backend.HubiC.csproj", "{D60AD540-0E7D-40CE-83AE-D26E01FFE9B8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.RecoveryTool", "Duplicati\CommandLine\RecoveryTool\Duplicati.CommandLine.RecoveryTool.csproj", "{4A010589-76E6-4F05-A5C4-4598D5DF11F8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -130,6 +132,12 @@ Global {33FD1D24-C28F-4C71-933F-98F1586EA76C}.Release|Any CPU.Build.0 = Release|Any CPU {33FD1D24-C28F-4C71-933F-98F1586EA76C}.Test|Any CPU.ActiveCfg = Debug|Any CPU {33FD1D24-C28F-4C71-933F-98F1586EA76C}.Test|Any CPU.Build.0 = Debug|Any CPU + {4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Release|Any CPU.Build.0 = Release|Any CPU + {4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Test|Any CPU.ActiveCfg = Debug|Any CPU + {4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Test|Any CPU.Build.0 = Debug|Any CPU {4D012CB1-4B92-47F4-89B7-BF80A73A2E99}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {4D012CB1-4B92-47F4-89B7-BF80A73A2E99}.Debug|Any CPU.Build.0 = Debug|Any CPU {4D012CB1-4B92-47F4-89B7-BF80A73A2E99}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/Duplicati/CommandLine/Duplicati.CommandLine.csproj b/Duplicati/CommandLine/Duplicati.CommandLine.csproj index 6434b8ae3..2a7347425 100644 --- a/Duplicati/CommandLine/Duplicati.CommandLine.csproj +++ b/Duplicati/CommandLine/Duplicati.CommandLine.csproj @@ -161,6 +161,10 @@ {D60AD540-0E7D-40CE-83AE-D26E01FFE9B8} Duplicati.Library.Backend.HubiC + + {4A010589-76E6-4F05-A5C4-4598D5DF11F8} + Duplicati.CommandLine.RecoveryTool + diff --git a/Duplicati/CommandLine/Program.cs b/Duplicati/CommandLine/Program.cs index 1832c890b..ecf9376cd 100644 --- a/Duplicati/CommandLine/Program.cs +++ b/Duplicati/CommandLine/Program.cs @@ -27,51 +27,6 @@ namespace Duplicati.CommandLine { public class Program { - private class FilterCollector - { - private List m_filters = new List(); - private Library.Utility.IFilter Filter - { - get - { - if (m_filters.Count == 0) - return new Library.Utility.FilterExpression(); - else if (m_filters.Count == 1) - return m_filters[0]; - else - return m_filters.Aggregate((a,b) => Library.Utility.JoinedFilterExpression.Join(a, b)); - } - } - - private Dictionary DoExtractOptions(List args, Func callbackHandler = null) - { - return Library.Utility.CommandLineParser.ExtractOptions(args, (key, value) => { - if (key.Equals("include", StringComparison.InvariantCultureIgnoreCase)) - { - m_filters.Add(new Library.Utility.FilterExpression(value, true)); - return false; - } - else if (key.Equals("exclude", StringComparison.InvariantCultureIgnoreCase)) - { - m_filters.Add(new Library.Utility.FilterExpression(value, false)); - return false; - } - - if (callbackHandler != null) - return callbackHandler(key, value); - - return true; - }); - } - - public static Tuple, Library.Utility.IFilter> ExtractOptions(List args, Func callbackHandler = null) - { - var fc = new FilterCollector(); - var opts = fc.DoExtractOptions(args, callbackHandler); - return new Tuple, Library.Utility.IFilter>(opts, fc.Filter); - } - } - /// /// The main entry point for the application. /// @@ -89,7 +44,7 @@ namespace Duplicati.CommandLine { List cargs = new List(args); - var tmpparsed = FilterCollector.ExtractOptions(cargs); + var tmpparsed = Library.Utility.FilterCollector.ExtractOptions(cargs); var options = tmpparsed.Item1; var filter = tmpparsed.Item2; @@ -300,7 +255,7 @@ namespace Duplicati.CommandLine var newsource = new List(); string newtarget = null; - var tmpparsed = FilterCollector.ExtractOptions(fargs, (key, value) => { + var tmpparsed = Library.Utility.FilterCollector.ExtractOptions(fargs, (key, value) => { if (key.Equals("source", StringComparison.InvariantCultureIgnoreCase)) { newsource.Add(value); diff --git a/Duplicati/CommandLine/RecoveryTool/Download.cs b/Duplicati/CommandLine/RecoveryTool/Download.cs new file mode 100644 index 000000000..e15635224 --- /dev/null +++ b/Duplicati/CommandLine/RecoveryTool/Download.cs @@ -0,0 +1,150 @@ +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.IO; +using System.Collections.Generic; + +namespace Duplicati.CommandLine.RecoveryTool +{ + public static class Download + { + public static int Run(List args, Dictionary options, Library.Utility.IFilter filter) + { + if (args.Count != 3) + { + Console.WriteLine("Invalid argument count ({0} expected 3): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args)); + return 100; + } + + using(var backend = Library.DynamicLoader.BackendLoader.GetBackend(args[1], options)) + { + if (backend == null) + { + Console.WriteLine("Backend not found: {0}{1}Backends supported: {2}", args[1], Environment.NewLine, string.Join(", ", Library.DynamicLoader.BackendLoader.Keys)); + return 100; + } + + var targetfolder = Path.GetFullPath(args[2]); + + if (!Directory.Exists(args[2])) + { + Console.WriteLine("Creating target folder: {0}", targetfolder); + Directory.CreateDirectory(targetfolder); + } + + Console.WriteLine("Listing files on backend: {0} ...", backend.ProtocolKey); + + var lst = backend.List(); + + Console.WriteLine("Found {0} files", lst.Count); + + var i = 0; + var downloaded = 0; + var errors = 0; + var needspass = 0; + string passphrase; + options.TryGetValue("passphrase", out passphrase); + + foreach(var file in lst) + { + try + { + Console.Write("{0}: {1}", i, file.Name); + var p = Duplicati.Library.Main.Volumes.VolumeBase.ParseFilename(file); + if (p == null) + { + Console.WriteLine(" - Not a Duplicati file, ignoring"); + continue; + } + + var local = Path.Combine(targetfolder, file.Name); + if (p.EncryptionModule != null) + { + if (string.IsNullOrWhiteSpace(passphrase)) + { + needspass++; + Console.WriteLine(" - No passphrase supplied, skipping"); + continue; + } + + local = local.Substring(0, local.Length - p.EncryptionModule.Length - 1); + } + + if (p.FileType != Duplicati.Library.Main.RemoteVolumeType.Blocks && p.FileType != Duplicati.Library.Main.RemoteVolumeType.Files) + { + Console.WriteLine(" - Filetype {0}, skipping", p.FileType); + continue; + } + + if (File.Exists(local)) + { + Console.WriteLine(" - Already exists, skipping"); + continue; + } + + Console.Write(" - downloading ({0})...", Library.Utility.Utility.FormatSizeString(file.Size)); + + using(var tf = new Library.Utility.TempFile()) + { + backend.Get(file.Name, tf); + + if (p.EncryptionModule != null) + { + Console.Write(" - decrypting ..."); + using(var m = Library.DynamicLoader.EncryptionLoader.GetModule(p.EncryptionModule, passphrase, options)) + using(var tf2 = new Library.Utility.TempFile()) + { + m.Decrypt(tf, tf2); + File.Copy(tf2, local); + } + } + else + File.Copy(tf, local); + } + + Console.WriteLine(" done!"); + + } + catch(Exception ex) + { + Console.WriteLine(" error: {0}", ex.ToString()); + errors++; + } + + i++; + } + + if (needspass > 0 && downloaded == 0) + { + Console.WriteLine("No files downloaded, try adding --passphrase to decrypt files"); + return 100; + } + + Console.WriteLine("Download complete, of {0} remote files, {1} were downloaded with {2} errors", lst.Count, downloaded, errors); + if (needspass > 0) + Console.WriteLine("Additonally {0} remote files were skipped because of encryption, supply --passphrase to download those"); + + if (errors > 0) + return 200; + else + return 0; + + } + } + } +} + diff --git a/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj b/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj new file mode 100644 index 000000000..218a7e11d --- /dev/null +++ b/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj @@ -0,0 +1,134 @@ + + + + Debug + AnyCPU + 8.0.30703 + 2.0 + {4A010589-76E6-4F05-A5C4-4598D5DF11F8} + Exe + Duplicati.CommandLine.RecoveryTool + Duplicati.CommandLine.RecoveryTool + true + Duplicati.snk + v4.5 + + + true + full + false + bin\Debug + DEBUG; + prompt + 4 + false + + + + + + index /Users/kenneth/testdata/recovery --targetpath=/Users/kenneth/testdata/recovery-restored + + + full + true + bin\Release + prompt + 4 + true + + + + + + + + + + + + + + + + + {7E119745-1F62-43F0-936C-F312A1912C0B} + Duplicati.Library.AutoUpdater + + + {8E4CECFB-0413-4B00-AB93-78D1C3902BD5} + Duplicati.Library.Backend.AzureBlob + + + {1BFAE226-8364-4086-825C-BB83F6F3EE4C} + Duplicati.Library.Backend.CloudFiles + + + {FC9B7611-836F-4127-8B44-A7C31F506807} + Duplicati.Library.Backend.File + + + {F61679A9-E5DE-468A-B5A4-05F92D0143D2} + Duplicati.Library.Backend.FTP + + + {5489181D-950C-44AF-873C-45EB0A3B6BD2} + Duplicati.Library.Backend.GoogleServices + + + {D60AD540-0E7D-40CE-83AE-D26E01FFE9B8} + Duplicati.Library.Backend.HubiC + + + {CCD76347-7DC7-4B42-B7E1-E500E624CAC3} + Duplicati.Library.Backend.OneDrive + + + {D9E4E686-423C-48EC-A392-404E7C00860C} + Duplicati.Library.Backend.OpenStack + + + {C03F6DFD-805A-4BE0-9338-64870ADDB4A2} + Duplicati.Library.Backend.S3 + + + {FF2BF37C-E502-4C98-BEA0-701671DDFA08} + Duplicati.Library.Backend.SSHv2 + + + {C0270709-2A40-43B5-8CF1-69581B9FA2A1} + Duplicati.Library.Backend.TahoeLAFS + + + {BAE27510-8B5D-44B2-B33E-372A98908041} + Duplicati.Library.Backend.WEBDAV + + + {19ECCE09-B5EB-406C-8C57-BAC66997D469} + Duplicati.Library.Compression + + + {0CA86ECF-5BEC-4909-B4F6-110A03B30B92} + Duplicati.Library.DynamicLoader + + + {94484FDB-2EFA-4CF0-9BE6-A561157B4F87} + Duplicati.Library.Encryption + + + {C5899F45-B0FF-483C-9D38-24A9FCAAB237} + Duplicati.Library.Interface + + + {DE3E5D4C-51AB-4E5E-BEE8-E636CEBFBA65} + Duplicati.Library.Utility + + + {10D2D1B7-C664-41D8-9B3A-00040C3D421B} + Duplicati.Library.Main + + + + + + \ No newline at end of file diff --git a/Duplicati/CommandLine/RecoveryTool/Duplicati.snk b/Duplicati/CommandLine/RecoveryTool/Duplicati.snk new file mode 100644 index 000000000..e0c1e2dd8 Binary files /dev/null and b/Duplicati/CommandLine/RecoveryTool/Duplicati.snk differ diff --git a/Duplicati/CommandLine/RecoveryTool/Help.cs b/Duplicati/CommandLine/RecoveryTool/Help.cs new file mode 100644 index 000000000..c6ecd72e9 --- /dev/null +++ b/Duplicati/CommandLine/RecoveryTool/Help.cs @@ -0,0 +1,41 @@ +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.Collections.Generic; + +namespace Duplicati.CommandLine.RecoveryTool +{ + public static class Help + { + private const string RESOURCE_NAME = "help.txt"; + + public static int Run(List args, Dictionary options, Library.Utility.IFilter filter) + { + if (args.Count >= 1 && !string.Equals("help", args[0])) + { + Console.WriteLine("Unsupported command: {0}", args[0]); + Console.WriteLine(); + return 100; + } + + Console.Write(new System.IO.StreamReader(System.Reflection.Assembly.GetExecutingAssembly().GetManifestResourceStream(System.Reflection.Assembly.GetExecutingAssembly().GetName().Name + "." + RESOURCE_NAME)).ReadToEnd()); + + return 0; + } + } +} + diff --git a/Duplicati/CommandLine/RecoveryTool/Index.cs b/Duplicati/CommandLine/RecoveryTool/Index.cs new file mode 100644 index 000000000..6b3eb1540 --- /dev/null +++ b/Duplicati/CommandLine/RecoveryTool/Index.cs @@ -0,0 +1,242 @@ +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; + +namespace Duplicati.CommandLine.RecoveryTool +{ + public static class Index + { + public static int Run(List args, Dictionary options, Library.Utility.IFilter filter) + { + if (args.Count != 2) + { + Console.WriteLine("Invalid argument count ({0} expected 2): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args)); + return 100; + } + + var folder = Path.GetFullPath(args[1]); + + if (!Directory.Exists(folder)) + { + Console.WriteLine("Folder not found: {0}", folder); + return 100; + } + + Directory.SetCurrentDirectory(folder); + + string ixfile; + options.TryGetValue("indexfile", out ixfile); + if (string.IsNullOrWhiteSpace(ixfile)) + ixfile = "index.txt"; + + ixfile = Path.GetFullPath(ixfile); + if (!File.Exists(ixfile)) + { + using(File.Create(ixfile)) + { + } + } + else + { + Console.WriteLine("Sorting existing index file"); + SortFile(ixfile, ixfile); + } + + var filecount = Directory.EnumerateFiles(folder).Count(); + + Console.WriteLine("Processing {0} files", filecount); + + var i = 0; + var errors = 0; + var totalblocks = 0L; + var files = 0; + foreach(var file in Directory.EnumerateFiles(folder)) + { + Console.Write("{0}: {1}", i, file); + + try + { + var p = Duplicati.Library.Main.Volumes.VolumeBase.ParseFilename(file); + if (p == null) + { + Console.WriteLine(" - Not a Duplicati file, ignoring"); + continue; + } + + if (p.FileType != Duplicati.Library.Main.RemoteVolumeType.Blocks) + { + Console.WriteLine(" - Filetype {0}, skipping", p.FileType); + continue; + } + + if (!string.IsNullOrWhiteSpace(p.EncryptionModule)) + { + Console.WriteLine(" - Encrypted, skipping"); + continue; + } + + var filekey = Path.GetFileName(file); + + var blocks = 0; + using(var cp = Library.DynamicLoader.CompressionLoader.GetModule(p.CompressionModule, file, options)) + using(var tf = new Library.Utility.TempFile()) + { + using(var sw = new StreamWriter(tf)) + foreach(var f in cp.ListFiles(null)) + { + sw.WriteLine("{0}, {1}", Library.Utility.Utility.Base64UrlToBase64Plain(f), filekey); + blocks++; + } + + files++; + totalblocks += blocks; + + Console.Write(" {0} hashes found, sorting ...", blocks); + + SortFile(tf, tf); + + Console.WriteLine(" done!"); + + Console.Write("Merging {0} hashes ...", totalblocks); + + MergeFiles(ixfile, tf, ixfile); + + Console.WriteLine(" done!"); + } + } + catch(Exception ex) + { + Console.WriteLine(" error: {0}", ex.ToString()); + errors++; + } + + i++; + } + + Console.WriteLine("Processed {0} files and found {1} hashes", files, totalblocks); + if (errors > 0) + Console.WriteLine("Experienced {0} errors", errors); + + return 0; + } + + public static void SortFile(string filein, string fileout) + { + try + { + // If the file can fit into memory, this is MUCH faster + using(var tfout = new Library.Utility.TempFile()) + { + var data = File.ReadAllLines(filein); + Array.Sort(data, StringComparer.Ordinal); + File.WriteAllLines(tfout, data); + + File.Copy(tfout, fileout, true); + return; + } + } + catch + { + } + + using(var tfin = new Library.Utility.TempFile()) + using(var tfout = new Library.Utility.TempFile()) + { + long swaps; + + File.Copy(filein, tfin, true); + + do + { + swaps = 0L; + + using(var sw = new System.IO.StreamWriter(tfout)) + using(var sr = new System.IO.StreamReader(tfin)) + { + var c1 = sr.ReadLine(); + var c2 = sr.ReadLine(); + + while (c1 != null || c2 != null) + { + if (c1 != null && c1.StartsWith("a")) + Console.Write(""); + var cmp = StringComparer.Ordinal.Compare(c1, c2); + + if (c2 == null || (c1 != null && cmp < 0)) + { + sw.WriteLine(c1); + + c1 = c2; + c2 = sr.ReadLine(); + } + else + { + if (cmp != 0) + sw.WriteLine(c2); + c2 = sr.ReadLine(); + swaps++; + } + } + } + + File.Copy(tfout, tfin, true); + + } while(swaps > 0); + + File.Copy(tfout, fileout, true); + } + } + + private static void MergeFiles(string file1, string file2, string fileout) + { + using(var tf = new Library.Utility.TempFile()) + { + using(var sw = new System.IO.StreamWriter(tf)) + using(var sr1 = new System.IO.StreamReader(file1)) + using(var sr2 = new System.IO.StreamReader(file2)) + { + var c1 = sr1.ReadLine(); + var c2 = sr2.ReadLine(); + + while(c1 != null || c2 != null) + { + var cmp = StringComparer.Ordinal.Compare(c1, c2); + + if (c2 == null || (c1 != null && cmp < 0)) + { + sw.WriteLine(c1); + c1 = sr1.ReadLine(); + } + else + { + if (cmp != 0) + sw.WriteLine(c2); + + c2 = sr2.ReadLine(); + } + } + } + + File.Copy(tf, fileout, true); + } + } + } +} + diff --git a/Duplicati/CommandLine/RecoveryTool/List.cs b/Duplicati/CommandLine/RecoveryTool/List.cs new file mode 100644 index 000000000..e6bd67b80 --- /dev/null +++ b/Duplicati/CommandLine/RecoveryTool/List.cs @@ -0,0 +1,123 @@ +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.Linq; +using System.Collections.Generic; +using System.IO; + +namespace Duplicati.CommandLine.RecoveryTool +{ + public static class List + { + public static int Run(List args, Dictionary options, Library.Utility.IFilter filter) + { + if (args.Count != 2 && args.Count != 3) + { + Console.WriteLine("Invalid argument count ({0} expected 2 or 3): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args)); + return 100; + } + + var folder = Path.GetFullPath(args[1]); + + if (!Directory.Exists(folder)) + { + Console.WriteLine("Folder not found: {0}", folder); + return 100; + } + + Directory.SetCurrentDirectory(folder); + + if (args.Count == 2) + { + var times = ParseListFiles(folder); + foreach(var v in times.Zip(Enumerable.Range(0, times.Length), (a,b) => new KeyValuePair(b, a.Key))) + Console.WriteLine("{0}: {1}", v.Key, v.Value.ToLocalTime()); + } + else + { + var file = SelectListFile(args[2], args[1]); + + var p = Library.Main.Volumes.VolumeBase.ParseFilename(file); + + Library.Main.Volumes.VolumeReaderBase.UpdateOptionsFromManifest(p.CompressionModule, file, new Duplicati.Library.Main.Options(options)); + + foreach(var f in EnumerateFilesInDList(file, filter, options)) + Console.WriteLine("{0} ({1})", f.Path, Library.Utility.Utility.FormatSizeString(f.Size)); + } + + return 0; + } + + public static IEnumerable EnumerateFilesInDList(string file, Duplicati.Library.Utility.IFilter filter, Dictionary options) + { + var p = Library.Main.Volumes.VolumeBase.ParseFilename(file); + using(var cm = Library.DynamicLoader.CompressionLoader.GetModule(p.CompressionModule, file, options)) + using(var filesetreader = new Library.Main.Volumes.FilesetVolumeReader(cm, new Duplicati.Library.Main.Options(options))) + foreach(var f in filesetreader.Files) + { + if (f.Type != Duplicati.Library.Main.FilelistEntryType.File) + continue; + + bool result; + Library.Utility.IFilter evfilter; + bool match = filter.Matches(f.Path, out result, out evfilter); + if (!match || (match && result)) + yield return f; + } + } + + public static string SelectListFile(string time, string folder) + { + if (File.Exists(Path.Combine(folder, time))) + return Path.Combine(folder, time); + + int index; + if (int.TryParse(time, out index)) + { + var items = ParseListFiles(folder); + if (index < 0 || index >= items.Length) + throw new Exception(string.Format("Valid range for version is 0 to {1}", items.Length - 1)); + + return items[index].Value; + } + + var parsedtime = Library.Utility.Timeparser.ParseTimeInterval(time, DateTime.Now, true); + var el = ( + from n in ParseListFiles(folder) + let diff = Math.Abs((n.Key - parsedtime).TotalSeconds) + orderby diff + select n).First(); + + Console.WriteLine("Selected time {0}", el.Key); + + return el.Value; + + } + + public static KeyValuePair[] ParseListFiles(string folder) + { + return ( + from v in Directory.EnumerateFiles(folder) + let p = Library.Main.Volumes.VolumeBase.ParseFilename(v) + where p != null && p.FileType == Duplicati.Library.Main.RemoteVolumeType.Files + orderby p.Time descending + select new KeyValuePair(p.Time, v)).ToArray(); + + } + } +} + diff --git a/Duplicati/CommandLine/RecoveryTool/Program.cs b/Duplicati/CommandLine/RecoveryTool/Program.cs new file mode 100644 index 000000000..b90c08906 --- /dev/null +++ b/Duplicati/CommandLine/RecoveryTool/Program.cs @@ -0,0 +1,76 @@ +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.Linq; +using System.Collections.Generic; + +namespace Duplicati.CommandLine.RecoveryTool +{ + static class Program + { + /// + /// The main entry point for the application. + /// + [STAThread] + public static int Main(string[] args) + { + return Duplicati.Library.AutoUpdater.UpdaterManager.RunFromMostRecent(typeof(Program).GetMethod("RealMain"), args); + } + + private delegate int CommandRunner(List args, Dictionary options, Library.Utility.IFilter filter); + + public static int RealMain(string[] _args) + { + try + { + var args = new List(_args); + var tmpparsed = Library.Utility.FilterCollector.ExtractOptions(args); + var options = tmpparsed.Item1; + var filter = tmpparsed.Item2; + + if (!options.ContainsKey("auth_password") && !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("AUTH_PASSWORD"))) + options["auth_password"] = System.Environment.GetEnvironmentVariable("AUTH_PASSWORD"); + + if (!options.ContainsKey("auth_username") && !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("AUTH_USERNAME"))) + options["auth_username"] = System.Environment.GetEnvironmentVariable("AUTH_USERNAME"); + + if (options.ContainsKey("tempdir") && !string.IsNullOrEmpty(options["tempdir"])) + Library.Utility.TempFolder.SystemTempPath = options["tempdir"]; + + var actions = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + actions["download"] = Download.Run; + actions["index"] = Index.Run; + actions["list"] = List.Run; + actions["restore"] = Restore.Run; + actions["help"] = Help.Run; + + CommandRunner command; + + actions.TryGetValue(args.FirstOrDefault(), out command); + + command = command ?? actions["help"]; + + return command(args, options, filter); + } + catch(Exception ex) + { + Console.WriteLine("Program crashed: {0}{1}", Environment.NewLine, ex.ToString()); + return 200; + } + } + } +} diff --git a/Duplicati/CommandLine/RecoveryTool/Properties/AssemblyInfo.cs b/Duplicati/CommandLine/RecoveryTool/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..54b281749 --- /dev/null +++ b/Duplicati/CommandLine/RecoveryTool/Properties/AssemblyInfo.cs @@ -0,0 +1,43 @@ +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System.Reflection; +using System.Runtime.CompilerServices; + +// Information about this assembly is defined by the following attributes. +// Change them to the values specific to your project. + +[assembly: AssemblyTitle("Duplicati.CommandLine.RecoveryTool")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("")] +[assembly: AssemblyCopyright("kenneth")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}". +// The form "{Major}.{Minor}.*" will automatically update the build and revision, +// and "{Major}.{Minor}.{Build}.*" will update just the revision. + +[assembly: AssemblyVersion("1.0.*")] + +// The following attributes are used to specify the signing key for the assembly, +// if desired. See the Mono documentation for more information about signing. + +//[assembly: AssemblyDelaySign(false)] +//[assembly: AssemblyKeyFile("")] + diff --git a/Duplicati/CommandLine/RecoveryTool/Restore.cs b/Duplicati/CommandLine/RecoveryTool/Restore.cs new file mode 100644 index 000000000..6f7d8b328 --- /dev/null +++ b/Duplicati/CommandLine/RecoveryTool/Restore.cs @@ -0,0 +1,476 @@ +// Copyright (C) 2015, The Duplicati Team +// http://www.duplicati.com, info@duplicati.com +// +// This library is free software; you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as +// published by the Free Software Foundation; either version 2.1 of the +// License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public +// License along with this library; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +using System; +using System.IO; +using System.Linq; +using System.Collections.Generic; + +namespace Duplicati.CommandLine.RecoveryTool +{ + public static class Restore + { + public static int Run(List args, Dictionary options, Library.Utility.IFilter filter) + { + if (args.Count != 2 && args.Count != 3) + { + Console.WriteLine("Invalid argument count ({0} expected 2 or 3): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args)); + return 100; + } + + var folder = Path.GetFullPath(args[1]); + + if (!Directory.Exists(folder)) + { + Console.WriteLine("Folder not found: {0}", folder); + return 100; + } + + Directory.SetCurrentDirectory(folder); + + string targetpath; + options.TryGetValue("targetpath", out targetpath); + + string ixfile; + options.TryGetValue("indexfile", out ixfile); + if (string.IsNullOrWhiteSpace(ixfile)) + ixfile = "index.txt"; + + ixfile = Path.GetFullPath(ixfile); + if (!File.Exists(ixfile)) + { + Console.WriteLine("Index file not found, perhaps you need to run the index command?"); + return 100; + } + + Console.Write("Sorting index file ..."); + Index.SortFile(ixfile, ixfile); + Console.WriteLine(" done!"); + + string filelist; + if (args.Count == 2) + { + var time = List.ParseListFiles(folder).First(); + filelist = time.Value; + + Console.WriteLine("Using set 0 with timestamp {0}", time.Key.ToLocalTime()); + } + else + { + filelist = List.SelectListFile(args[2], args[1]); + } + + Library.Main.Volumes.VolumeReaderBase.UpdateOptionsFromManifest(Path.GetExtension(filelist).Trim('.'), filelist, new Duplicati.Library.Main.Options(options)); + + string blocksize_str; + options.TryGetValue("blocksize", out blocksize_str); + string blockhash_str; + options.TryGetValue("block-hash-algorithm", out blockhash_str); + string filehash_str; + options.TryGetValue("block-hash-algorithm", out filehash_str); + + long blocksize = string.IsNullOrWhiteSpace(blocksize_str) ? 0 : Library.Utility.Sizeparser.ParseSize(blocksize_str); + + if (blocksize <= 0) + { + Console.WriteLine("Invalid blocksize: {0}, try setting --blocksize manually"); + return 100; + } + + var blockhasher = string.IsNullOrWhiteSpace(blockhash_str) ? null : System.Security.Cryptography.HashAlgorithm.Create(blockhash_str); + var filehasher = string.IsNullOrWhiteSpace(filehash_str) ? null : System.Security.Cryptography.HashAlgorithm.Create(filehash_str); + + if (blockhasher == null) + throw new Exception(string.Format("Block hash algorithm not valid: {0}", blockhash_str)); + if (filehasher == null) + throw new Exception(string.Format("File hash algorithm not valid: {0}", filehash_str)); + + var hashesprblock = blocksize / (blockhasher.HashSize / 8); + + using(var mru = new CompressedFileMRUCache(options)) + { + Console.WriteLine("Building lookup table for file hashes"); + var lookup = new HashLookupHelper(ixfile, mru, (int)blocksize, blockhasher.HashSize / 8); + + var filecount = 0L; + string largestprefix = null; + string[] largestprefixparts = null; + + if (!string.IsNullOrWhiteSpace(targetpath)) + Console.WriteLine("Computing restore path"); + + foreach(var f in List.EnumerateFilesInDList(filelist, filter, options)) + { + if (largestprefix == null) + { + largestprefix = f.Path; + largestprefixparts = largestprefix.Split(new char[] { Path.DirectorySeparatorChar }); + } + else if (largestprefix.Length > 1) + { + var parts = f.Path.Split(new char[] { Path.DirectorySeparatorChar }); + + var ni = 0; + for(; ni < Math.Min(parts.Length, largestprefixparts.Length); ni++) + if (!Library.Utility.Utility.ClientFilenameStringComparer.Equals(parts[ni], largestprefixparts[ni])) + break; + + if (ni != largestprefixparts.Length) + { + if (ni == 0) + { + largestprefixparts = new string[0]; + largestprefix = string.Empty; + } + else + { + Array.Resize(ref largestprefixparts, ni - 1); + largestprefix = string.Join(Path.DirectorySeparatorChar.ToString(), largestprefixparts); + } + } + } + filecount++; + } + + Console.WriteLine("Restoring {0} files to {1}", filecount, string.IsNullOrWhiteSpace(targetpath) ? "original position" : targetpath); + + if (Library.Utility.Utility.IsClientLinux || largestprefix.Length > 0) + largestprefix = Library.Utility.Utility.AppendDirSeparator(largestprefix); + + if (!string.IsNullOrEmpty(largestprefix)) + Console.WriteLine("Removing common prefix {0} from files", largestprefix); + + var i = 0L; + var errors = 0L; + foreach(var f in List.EnumerateFilesInDList(filelist, filter, options)) + { + try + { + var targetfile = MapToRestorePath(f.Path, largestprefix, targetpath); + if (!Directory.Exists(Path.GetDirectoryName(targetfile))) + Directory.CreateDirectory(Path.GetDirectoryName(targetfile)); + + Console.Write("{0}: {1} ({2})", i, targetfile, Library.Utility.Utility.FormatSizeString(f.Size)); + + using(var tf = new Library.Utility.TempFile()) + { + using(var sw = File.OpenWrite(tf)) + { + if (f.BlocklistHashes == null) + { + lookup.WriteHash(sw, f.Hash); + } + else + { + var blhi = 0L; + foreach(var blh in f.BlocklistHashes) + { + Console.Write(" {0}", blhi); + var blockhashoffset = blhi * hashesprblock * blocksize; + + try + { + var bi = 0; + foreach(var h in lookup.ReadBlocklistHashes(blh)) + { + try + { + sw.Position = blockhashoffset + (bi * blocksize); + lookup.WriteHash(sw, h); + } + catch(Exception ex) + { + Console.WriteLine("Failed to read hash: {0}{1}{2}", h, Environment.NewLine, ex.ToString()); + } + + bi++; + } + } + catch (Exception ex) + { + Console.WriteLine("Failed to read Blocklist hash: {0}{1}{2}", blh, Environment.NewLine, ex.ToString()); + } + + blhi++; + } + } + } + + string fh; + using(var fs = File.OpenRead(tf)) + fh = Convert.ToBase64String(filehasher.ComputeHash(fs)); + + if (fh == f.Hash) + { + Console.WriteLine(" done!"); + File.Copy(tf, targetfile, true); + } + else + { + Console.Write(" - Restored file hash mismatch"); + if (File.Exists(targetfile)) + Console.WriteLine(" - not overwriting existing file: {0}", targetfile); + else + Console.WriteLine(" - restoring file in damaged condition"); + + + } + } + + } + catch (Exception ex) + { + Console.WriteLine(" error: {0}", ex.ToString()); + errors++; + } + i++; + } + } + + return 0; + } + + private static string MapToRestorePath(string path, string prefixpath, string restorepath) + { + if (string.IsNullOrWhiteSpace(restorepath) || string.IsNullOrWhiteSpace(prefixpath)) + return path; + + return Path.Combine(restorepath, path.Substring(prefixpath.Length)); + } + + + + private class HashLookupHelper : IDisposable + { + private const int LOOKUP_TABLE_SIZE = 2048; + + private CompressedFileMRUCache m_cache; + private int m_hashsize; + private int m_blocksize; + private Stream m_indexfile; + private List m_lookup = new List(); + private List m_offsets = new List(); + private byte[] m_linebuf = new byte[128]; + private byte[] m_newline = Environment.NewLine.Select(x => (byte)x).ToArray(); + + public HashLookupHelper(string indexfile, CompressedFileMRUCache cache, int blocksize, int hashsize) + { + m_cache = cache; + m_blocksize = blocksize; + m_hashsize = hashsize; + m_indexfile = File.OpenRead(indexfile); + + // Build index .... + var hashes = 0L; + string prev = null; + m_indexfile.Position = 0; + foreach(var n in AllFileLines()) + { + if (n.Key != prev) + { + hashes++; + prev = n.Key; + } + } + + Console.WriteLine("Index file has {0} hashes in total", hashes); + + var lookuptablesize = Math.Max(1, Math.Min(LOOKUP_TABLE_SIZE, hashes) - 1); + var lookupincrements = hashes / lookuptablesize; + + Console.WriteLine("Building lookup table with {0} entries, giving increments of {1}", lookuptablesize, lookupincrements); + + prev = null; + var prevoff = 0L; + var hc = 0L; + m_indexfile.Position = 0; + foreach(var n in AllFileLines()) + { + if (n.Key != prev) + { + if ((hc % lookupincrements) == 0) + { + m_lookup.Add(n.Key); + m_offsets.Add(prevoff); + } + + hc++; + prev = n.Key; + prevoff = m_indexfile.Position; + } + } + } + + private string ReadNextLine() + { + var p = m_indexfile.Position; + var max = m_indexfile.Read(m_linebuf, 0, m_linebuf.Length); + if (max == 0) + return null; + + var lfi = 0; + for(int i = 0; i < max; i++) + if (m_linebuf[i] == m_newline[lfi]) + { + if (lfi == m_newline.Length - 1) + { + m_indexfile.Position = p + i + 1; + return System.Text.Encoding.Default.GetString(m_linebuf, 0, i - m_newline.Length + 1); + + } + else + { + lfi++; + } + } + else + { + lfi = 0; + } + + throw new Exception(string.Format("Unexpected long line starting at offset {0}, read {1} bytes without a newline", p, m_linebuf.Length)); + } + + private IEnumerable> AllFileLines() + { + while(true) + { + var str = ReadNextLine(); + if (str == null) + break; + + if (str.Length == 0) + continue; + + var ix = str.IndexOf(", "); + if (ix < 0) + Console.WriteLine("Failed to parse line starting at offset {0} in index file, string: {1}", m_indexfile.Position - str.Length - m_newline.Length, str); + yield return new KeyValuePair(str.Substring(0, ix), str.Substring(ix + 2)); + } + } + + public IEnumerable ReadBlocklistHashes(string hash) + { + var bytes = ReadHash(hash); + for(var i = 0; i < bytes.Length; i += m_hashsize) + yield return Convert.ToBase64String(bytes, i, m_hashsize); + } + + public byte[] ReadHash(string hash) + { + var ix = m_lookup.BinarySearch(hash, StringComparer.Ordinal); + if (ix < 0) + { + ix = ~ix; + if (ix != 0) + ix--; + } + + m_indexfile.Position = m_offsets[ix]; + + foreach(var v in AllFileLines()) + { + if (v.Key == hash) + using(var fs = m_cache.ReadBlock(v.Value, hash)) + { + var buf = new byte[m_blocksize]; + var l = fs.Read(buf, 0, buf.Length); + Array.Resize(ref buf, l); + + return buf; + } + + + if (StringComparer.Ordinal.Compare(hash, v.Key) < 0) + break; + } + + throw new Exception(string.Format("Unable to locate block with hash: {0}", hash)); + } + + public void WriteHash(Stream sw, string hash) + { + var h = ReadHash(hash); + sw.Write(h, 0, h.Length); + } + + + public void Dispose() + { + if (m_indexfile != null) + m_indexfile.Dispose(); + m_indexfile = null; + } + } + + private class CompressedFileMRUCache : IDisposable + { + private Dictionary m_lookup = new Dictionary(); + private List m_mru = new List(); + private Dictionary m_options; + + private const int MAX_OPEN_ARCHIVES = 20; + + public CompressedFileMRUCache(Dictionary options) + { + m_options = options; + } + + public Stream ReadBlock(string filename, string hash) + { + Library.Interface.ICompression cf; + if (!m_lookup.TryGetValue(filename, out cf)) + { + cf = Library.DynamicLoader.CompressionLoader.GetModule(Path.GetExtension(filename).Trim('.'), filename, m_options); + if (cf == null) + throw new Exception(string.Format("Unable to decompress {0}, no such compression module {1}", filename, Path.GetExtension(filename).Trim('.'))); + m_lookup[filename] = cf; + } + + if (m_mru.Count > 0 && m_mru[0] != filename) + { + m_mru.Remove(filename); + m_mru.Insert(0, filename); + } + + while (m_mru.Count > MAX_OPEN_ARCHIVES) + { + var f = m_mru.Last(); + m_lookup[f].Dispose(); + m_lookup.Remove(f); + m_mru.Remove(f); + } + + return cf.OpenRead(Library.Utility.Utility.Base64PlainToBase64Url(hash)); + } + + #region IDisposable implementation + + public void Dispose() + { + foreach(var v in m_lookup.Values) + try { v.Dispose(); } + catch { } + + m_lookup = null; + m_mru = null; + } + + #endregion + } + } +} + diff --git a/Duplicati/CommandLine/RecoveryTool/help.txt b/Duplicati/CommandLine/RecoveryTool/help.txt new file mode 100644 index 000000000..a96cc100b --- /dev/null +++ b/Duplicati/CommandLine/RecoveryTool/help.txt @@ -0,0 +1,74 @@ +Duplicati Recovery Tool +======================= + +This tool performs a recovery of as much data as possible in small steps that must be performed in order. +We recommend that you use Duplicati.CommandLine.exe to do the restore, and rely only on this tool if all else fails. + + +The steps to perform are: +------------------------- + +1: Download + Download files from the remote store and keep them unencrypted on a location available in the local filesystem. + +2: Index + Builds an index file to figure out what data is contained inside the files downloaded + +3: Restore + Restores the files to a destination you choose + + +Optionally you can also run: +---------------------------- + +4: List + Shows what files are available and tests filters + + + +Command descriptions: +===================== + + +Download +-------- + +Usage: +Duplicati.RecoveryTool.exe download [options] + +Downloads all files matching the Duplicati filenames from to , and decrypts them in the process. +The remote url must be one supported by Duplicati. +Use `Duplicati.CommandLine.exe help backends` to see backends and options. + + + +Index +----- +Duplicati.RecoveryTool.exe index [options] + +Examines all files found in localfolder and produces an index.txt file, which is a list of all block hashes found in the files. +The index file can be rather large. It defaults to being stored in but can be specified with --indexfile. +Some files are created in the system temporary folder, use --tempdir to set an alternative temporary folder location. + + + +Restore +------- +Duplicati.RecoveryTool.exe index [version] [options] + +Restores all files to their respective destinations. Use --targetpath to choose another folder where the files are restored into. +Use the filters, --exclude, to perform a partial restore. +Version can be either a number, a filename or a date. If omitted the most recent backup is used. + + + +List +---- +Duplicati.RecoveryTool.exe list [version] [options] + +Lists contents of backups. +Version can be either a number, a filename or a date. +If [version] is omitted a list of backup versions are shown, if [version] is supplied, files from that version are listed. +Use the filters, --exclude, to show a subset of files. + + diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj index ca386af17..6b7cc666a 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj @@ -46,6 +46,7 @@ false + --webservice-webroot=/Users/kenneth/Udvikler/my-new-angular-app bin\Release @@ -276,6 +277,10 @@ {D60AD540-0E7D-40CE-83AE-D26E01FFE9B8} Duplicati.Library.Backend.HubiC + + {4A010589-76E6-4F05-A5C4-4598D5DF11F8} + Duplicati.CommandLine.RecoveryTool + diff --git a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj index 2408d8baf..1b134d619 100644 --- a/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj +++ b/Duplicati/Library/Utility/Duplicati.Library.Utility.csproj @@ -68,6 +68,7 @@ + diff --git a/Duplicati/Server/Duplicati.Server.csproj b/Duplicati/Server/Duplicati.Server.csproj index eb4301658..fb4dae71b 100644 --- a/Duplicati/Server/Duplicati.Server.csproj +++ b/Duplicati/Server/Duplicati.Server.csproj @@ -247,6 +247,10 @@ {D60AD540-0E7D-40CE-83AE-D26E01FFE9B8} Duplicati.Library.Backend.HubiC + + {4A010589-76E6-4F05-A5C4-4598D5DF11F8} + Duplicati.CommandLine.RecoveryTool +