Added a simple restore tool for recovering data from a broken backup.

This fixes #1439.
2015-08-21 21:59:12 +02:00
parent 64bbd93eed
commit 8018545cff
16 changed files with 1383 additions and 47 deletions
@@ -69,6 +69,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.O
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.HubiC", "Duplicati\Library\Backend\HubiC\Duplicati.Library.Backend.HubiC.csproj", "{D60AD540-0E7D-40CE-83AE-D26E01FFE9B8}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.CommandLine.RecoveryTool", "Duplicati\CommandLine\RecoveryTool\Duplicati.CommandLine.RecoveryTool.csproj", "{4A010589-76E6-4F05-A5C4-4598D5DF11F8}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -130,6 +132,12 @@ Global
 		{33FD1D24-C28F-4C71-933F-98F1586EA76C}.Release|Any CPU.Build.0 = Release|Any CPU
 		{33FD1D24-C28F-4C71-933F-98F1586EA76C}.Test|Any CPU.ActiveCfg = Debug|Any CPU
 		{33FD1D24-C28F-4C71-933F-98F1586EA76C}.Test|Any CPU.Build.0 = Debug|Any CPU
+		{4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Release|Any CPU.Build.0 = Release|Any CPU
+		{4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Test|Any CPU.ActiveCfg = Debug|Any CPU
+		{4A010589-76E6-4F05-A5C4-4598D5DF11F8}.Test|Any CPU.Build.0 = Debug|Any CPU
 		{4D012CB1-4B92-47F4-89B7-BF80A73A2E99}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{4D012CB1-4B92-47F4-89B7-BF80A73A2E99}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{4D012CB1-4B92-47F4-89B7-BF80A73A2E99}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -161,6 +161,10 @@
      <Project>{D60AD540-0E7D-40CE-83AE-D26E01FFE9B8}</Project>
      <Name>Duplicati.Library.Backend.HubiC</Name>
    </ProjectReference>
+    <ProjectReference Include="RecoveryTool\Duplicati.CommandLine.RecoveryTool.csproj">
+      <Project>{4A010589-76E6-4F05-A5C4-4598D5DF11F8}</Project>
+      <Name>Duplicati.CommandLine.RecoveryTool</Name>
+    </ProjectReference>
  </ItemGroup>
  <ItemGroup>
    <None Include="Duplicati.snk" />
@@ -27,51 +27,6 @@ namespace Duplicati.CommandLine
 {
    public class Program
    {
-    	private class FilterCollector
-		{
-			private List<Library.Utility.IFilter> m_filters = new List<Library.Utility.IFilter>();
-			private Library.Utility.IFilter Filter 
-            { 
-                get 
-                { 
-                    if (m_filters.Count == 0)
-                        return new Library.Utility.FilterExpression();
-                    else if (m_filters.Count == 1)
-                        return m_filters[0];
-                    else 
-                        return m_filters.Aggregate((a,b) => Library.Utility.JoinedFilterExpression.Join(a, b)); 
-                }
-            }
-			
-			private Dictionary<string, string> DoExtractOptions(List<string> args, Func<string, string, bool> callbackHandler = null)
-			{
-                return Library.Utility.CommandLineParser.ExtractOptions(args, (key, value) => {
-                	if (key.Equals("include", StringComparison.InvariantCultureIgnoreCase))
-                	{
-                		m_filters.Add(new Library.Utility.FilterExpression(value, true));
-                		return false;
-                	}
-                	else if (key.Equals("exclude", StringComparison.InvariantCultureIgnoreCase))
-                	{
-                        m_filters.Add(new Library.Utility.FilterExpression(value, false));
-                		return false;
-                	}
-                	
-                	if (callbackHandler != null)
-                		return callbackHandler(key, value);
-                	
-                	return true;
-                });
-			}
-			
-			public static Tuple<Dictionary<string, string>, Library.Utility.IFilter> ExtractOptions(List<string> args, Func<string, string, bool> callbackHandler = null)
-			{
-				var fc = new FilterCollector();
-				var opts = fc.DoExtractOptions(args, callbackHandler);
-				return new Tuple<Dictionary<string, string>, Library.Utility.IFilter>(opts, fc.Filter);
-			}
-		}
-    
        /// <summary>
        /// The main entry point for the application.
        /// </summary>
@@ -89,7 +44,7 @@ namespace Duplicati.CommandLine
            {
            	List<string> cargs = new List<string>(args);

-				var tmpparsed = FilterCollector.ExtractOptions(cargs);
+                var tmpparsed = Library.Utility.FilterCollector.ExtractOptions(cargs);
 				var options = tmpparsed.Item1;
 				var filter = tmpparsed.Item2;
 				
@@ -300,7 +255,7 @@ namespace Duplicati.CommandLine
                var newsource = new List<string>();
                string newtarget = null;

-				var tmpparsed = FilterCollector.ExtractOptions(fargs, (key, value) => {
+                var tmpparsed = Library.Utility.FilterCollector.ExtractOptions(fargs, (key, value) => {
 					if (key.Equals("source", StringComparison.InvariantCultureIgnoreCase))
 					{
 						newsource.Add(value);
@@ -0,0 +1,150 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.IO;
+using System.Collections.Generic;
+
+namespace Duplicati.CommandLine.RecoveryTool
+{
+    public static class Download
+    {
+        public static int Run(List<string> args, Dictionary<string, string> options, Library.Utility.IFilter filter)
+        {
+            if (args.Count != 3)
+            {
+                Console.WriteLine("Invalid argument count ({0} expected 3): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args));
+                return 100;
+            }
+
+            using(var backend = Library.DynamicLoader.BackendLoader.GetBackend(args[1], options))
+            {
+                if (backend == null)
+                {
+                    Console.WriteLine("Backend not found: {0}{1}Backends supported: {2}", args[1], Environment.NewLine, string.Join(", ", Library.DynamicLoader.BackendLoader.Keys));
+                    return 100;
+                }
+
+                var targetfolder = Path.GetFullPath(args[2]);
+
+                if (!Directory.Exists(args[2]))
+                {
+                    Console.WriteLine("Creating target folder: {0}", targetfolder);
+                    Directory.CreateDirectory(targetfolder);
+                }
+
+                Console.WriteLine("Listing files on backend: {0} ...", backend.ProtocolKey);
+
+                var lst = backend.List();
+
+                Console.WriteLine("Found {0} files", lst.Count);
+
+                var i = 0;
+                var downloaded = 0;
+                var errors = 0;
+                var needspass = 0;
+                string passphrase;
+                options.TryGetValue("passphrase", out passphrase);
+
+                foreach(var file in lst)
+                {
+                    try
+                    {
+                        Console.Write("{0}: {1}", i, file.Name);
+                        var p = Duplicati.Library.Main.Volumes.VolumeBase.ParseFilename(file);
+                        if (p == null)
+                        {
+                            Console.WriteLine(" - Not a Duplicati file, ignoring");
+                            continue;
+                        }
+
+                        var local = Path.Combine(targetfolder, file.Name);
+                        if (p.EncryptionModule != null)
+                        {
+                            if (string.IsNullOrWhiteSpace(passphrase)) 
+                            {
+                                needspass++;
+                                Console.WriteLine(" - No passphrase supplied, skipping");
+                                continue;
+                            }
+
+                            local = local.Substring(0, local.Length - p.EncryptionModule.Length - 1);
+                        }
+
+                        if (p.FileType != Duplicati.Library.Main.RemoteVolumeType.Blocks && p.FileType != Duplicati.Library.Main.RemoteVolumeType.Files)
+                        {
+                            Console.WriteLine(" - Filetype {0}, skipping", p.FileType);
+                            continue;
+                        }
+
+                        if (File.Exists(local))
+                        {
+                            Console.WriteLine(" - Already exists, skipping");
+                            continue;
+                        }
+
+                        Console.Write(" - downloading ({0})...", Library.Utility.Utility.FormatSizeString(file.Size));
+
+                        using(var tf = new Library.Utility.TempFile())
+                        {
+                            backend.Get(file.Name, tf);
+
+                            if (p.EncryptionModule != null)
+                            {
+                                Console.Write(" - decrypting ...");
+                                using(var m = Library.DynamicLoader.EncryptionLoader.GetModule(p.EncryptionModule, passphrase, options))
+                                using(var tf2 = new Library.Utility.TempFile())
+                                {
+                                    m.Decrypt(tf, tf2);
+                                    File.Copy(tf2, local);
+                                }
+                            }
+                            else
+                                File.Copy(tf, local);
+                        }
+
+                        Console.WriteLine(" done!");
+                        
+                    }
+                    catch(Exception ex)
+                    {
+                        Console.WriteLine(" error: {0}", ex.ToString());
+                        errors++;
+                    }
+
+                    i++;
+                }
+
+                if (needspass > 0 && downloaded == 0)
+                {
+                    Console.WriteLine("No files downloaded, try adding --passphrase to decrypt files");
+                    return 100;
+                }
+
+                Console.WriteLine("Download complete, of {0} remote files, {1} were downloaded with {2} errors", lst.Count, downloaded, errors);
+                if (needspass > 0)
+                    Console.WriteLine("Additonally {0} remote files were skipped because of encryption, supply --passphrase to download those");
+
+                if (errors > 0)
+                    return 200;
+                else
+                    return 0;
+                
+            }            
+        }
+    }
+}
+
@@ -0,0 +1,134 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProductVersion>8.0.30703</ProductVersion>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{4A010589-76E6-4F05-A5C4-4598D5DF11F8}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <RootNamespace>Duplicati.CommandLine.RecoveryTool</RootNamespace>
+    <AssemblyName>Duplicati.CommandLine.RecoveryTool</AssemblyName>
+    <SignAssembly>true</SignAssembly>
+    <AssemblyOriginatorKeyFile>Duplicati.snk</AssemblyOriginatorKeyFile>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug</OutputPath>
+    <DefineConstants>DEBUG;</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <ConsolePause>false</ConsolePause>
+    <EnvironmentVariables>
+      <EnvironmentVariables>
+        <Variable name="AUTOUPDATER_Duplicati_SKIP_UPDATE" value="1" />
+      </EnvironmentVariables>
+    </EnvironmentVariables>
+    <Commandlineparameters>index /Users/kenneth/testdata/recovery --targetpath=/Users/kenneth/testdata/recovery-restored</Commandlineparameters>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>full</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release</OutputPath>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <Externalconsole>true</Externalconsole>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Download.cs" />
+    <Compile Include="Help.cs" />
+    <Compile Include="Index.cs" />
+    <Compile Include="List.cs" />
+    <Compile Include="Restore.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
+  <ItemGroup>
+    <ProjectReference Include="..\..\Library\AutoUpdater\Duplicati.Library.AutoUpdater.csproj">
+      <Project>{7E119745-1F62-43F0-936C-F312A1912C0B}</Project>
+      <Name>Duplicati.Library.AutoUpdater</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\AzureBlob\Duplicati.Library.Backend.AzureBlob.csproj">
+      <Project>{8E4CECFB-0413-4B00-AB93-78D1C3902BD5}</Project>
+      <Name>Duplicati.Library.Backend.AzureBlob</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\CloudFiles\Duplicati.Library.Backend.CloudFiles.csproj">
+      <Project>{1BFAE226-8364-4086-825C-BB83F6F3EE4C}</Project>
+      <Name>Duplicati.Library.Backend.CloudFiles</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\File\Duplicati.Library.Backend.File.csproj">
+      <Project>{FC9B7611-836F-4127-8B44-A7C31F506807}</Project>
+      <Name>Duplicati.Library.Backend.File</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\FTP\Duplicati.Library.Backend.FTP.csproj">
+      <Project>{F61679A9-E5DE-468A-B5A4-05F92D0143D2}</Project>
+      <Name>Duplicati.Library.Backend.FTP</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\GoogleServices\Duplicati.Library.Backend.GoogleServices.csproj">
+      <Project>{5489181D-950C-44AF-873C-45EB0A3B6BD2}</Project>
+      <Name>Duplicati.Library.Backend.GoogleServices</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\HubiC\Duplicati.Library.Backend.HubiC.csproj">
+      <Project>{D60AD540-0E7D-40CE-83AE-D26E01FFE9B8}</Project>
+      <Name>Duplicati.Library.Backend.HubiC</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\OneDrive\Duplicati.Library.Backend.OneDrive.csproj">
+      <Project>{CCD76347-7DC7-4B42-B7E1-E500E624CAC3}</Project>
+      <Name>Duplicati.Library.Backend.OneDrive</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\OpenStack\Duplicati.Library.Backend.OpenStack.csproj">
+      <Project>{D9E4E686-423C-48EC-A392-404E7C00860C}</Project>
+      <Name>Duplicati.Library.Backend.OpenStack</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\S3\Duplicati.Library.Backend.S3.csproj">
+      <Project>{C03F6DFD-805A-4BE0-9338-64870ADDB4A2}</Project>
+      <Name>Duplicati.Library.Backend.S3</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\SSHv2\Duplicati.Library.Backend.SSHv2.csproj">
+      <Project>{FF2BF37C-E502-4C98-BEA0-701671DDFA08}</Project>
+      <Name>Duplicati.Library.Backend.SSHv2</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\TahoeLAFS\Duplicati.Library.Backend.TahoeLAFS.csproj">
+      <Project>{C0270709-2A40-43B5-8CF1-69581B9FA2A1}</Project>
+      <Name>Duplicati.Library.Backend.TahoeLAFS</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Backend\WEBDAV\Duplicati.Library.Backend.WEBDAV.csproj">
+      <Project>{BAE27510-8B5D-44B2-B33E-372A98908041}</Project>
+      <Name>Duplicati.Library.Backend.WEBDAV</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Compression\Duplicati.Library.Compression.csproj">
+      <Project>{19ECCE09-B5EB-406C-8C57-BAC66997D469}</Project>
+      <Name>Duplicati.Library.Compression</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\DynamicLoader\Duplicati.Library.DynamicLoader.csproj">
+      <Project>{0CA86ECF-5BEC-4909-B4F6-110A03B30B92}</Project>
+      <Name>Duplicati.Library.DynamicLoader</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Encryption\Duplicati.Library.Encryption.csproj">
+      <Project>{94484FDB-2EFA-4CF0-9BE6-A561157B4F87}</Project>
+      <Name>Duplicati.Library.Encryption</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Interface\Duplicati.Library.Interface.csproj">
+      <Project>{C5899F45-B0FF-483C-9D38-24A9FCAAB237}</Project>
+      <Name>Duplicati.Library.Interface</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Utility\Duplicati.Library.Utility.csproj">
+      <Project>{DE3E5D4C-51AB-4E5E-BEE8-E636CEBFBA65}</Project>
+      <Name>Duplicati.Library.Utility</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\..\Library\Main\Duplicati.Library.Main.csproj">
+      <Project>{10D2D1B7-C664-41D8-9B3A-00040C3D421B}</Project>
+      <Name>Duplicati.Library.Main</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <EmbeddedResource Include="help.txt" />
+  </ItemGroup>
+</Project>
@@ -0,0 +1,41 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Collections.Generic;
+
+namespace Duplicati.CommandLine.RecoveryTool
+{
+    public static class Help
+    {
+        private const string RESOURCE_NAME = "help.txt";
+
+        public static int Run(List<string> args, Dictionary<string, string> options, Library.Utility.IFilter filter)
+        {
+            if (args.Count >= 1 && !string.Equals("help", args[0]))
+            {
+                Console.WriteLine("Unsupported command: {0}", args[0]);
+                Console.WriteLine();
+                return 100;
+            }   
+
+            Console.Write(new System.IO.StreamReader(System.Reflection.Assembly.GetExecutingAssembly().GetManifestResourceStream(System.Reflection.Assembly.GetExecutingAssembly().GetName().Name + "." + RESOURCE_NAME)).ReadToEnd());
+
+            return 0;
+        }
+    }
+}
+
@@ -0,0 +1,242 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.IO;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace Duplicati.CommandLine.RecoveryTool
+{
+    public static class Index
+    {
+        public static int Run(List<string> args, Dictionary<string, string> options, Library.Utility.IFilter filter)
+        {
+            if (args.Count != 2)
+            {
+                Console.WriteLine("Invalid argument count ({0} expected 2): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args));
+                return 100;
+            }
+
+            var folder = Path.GetFullPath(args[1]);
+
+            if (!Directory.Exists(folder))
+            {
+                Console.WriteLine("Folder not found: {0}", folder);
+                return 100;
+            }
+
+            Directory.SetCurrentDirectory(folder);
+
+            string ixfile;
+            options.TryGetValue("indexfile", out ixfile);
+            if (string.IsNullOrWhiteSpace(ixfile))
+                ixfile = "index.txt";
+
+            ixfile = Path.GetFullPath(ixfile);
+            if (!File.Exists(ixfile))
+            {
+                using(File.Create(ixfile))
+                {
+                }
+            }
+            else
+            {
+                Console.WriteLine("Sorting existing index file");
+                SortFile(ixfile, ixfile);
+            }
+
+            var filecount = Directory.EnumerateFiles(folder).Count();
+
+            Console.WriteLine("Processing {0} files", filecount);
+
+            var i = 0;
+            var errors = 0;
+            var totalblocks = 0L;
+            var files = 0;
+            foreach(var file in Directory.EnumerateFiles(folder))
+            {
+                Console.Write("{0}: {1}", i, file);
+
+                try
+                {
+                    var p = Duplicati.Library.Main.Volumes.VolumeBase.ParseFilename(file);
+                    if (p == null)
+                    {
+                        Console.WriteLine(" - Not a Duplicati file, ignoring");
+                        continue;
+                    }
+
+                    if (p.FileType != Duplicati.Library.Main.RemoteVolumeType.Blocks)
+                    {
+                        Console.WriteLine(" - Filetype {0}, skipping", p.FileType);
+                        continue;
+                    }
+
+                    if (!string.IsNullOrWhiteSpace(p.EncryptionModule))
+                    {
+                        Console.WriteLine(" - Encrypted, skipping");
+                        continue;
+                    }
+
+                    var filekey = Path.GetFileName(file);
+
+                    var blocks = 0;
+                    using(var cp = Library.DynamicLoader.CompressionLoader.GetModule(p.CompressionModule, file, options))
+                    using(var tf = new Library.Utility.TempFile())
+                    {
+                        using(var sw = new StreamWriter(tf))
+                            foreach(var f in cp.ListFiles(null))
+                            {
+                                sw.WriteLine("{0}, {1}", Library.Utility.Utility.Base64UrlToBase64Plain(f), filekey);
+                                blocks++;
+                            }
+
+                        files++;
+                        totalblocks += blocks;
+
+                        Console.Write(" {0} hashes found, sorting ...", blocks);
+
+                        SortFile(tf, tf);
+
+                        Console.WriteLine(" done!");
+
+                        Console.Write("Merging {0} hashes ...", totalblocks);
+
+                        MergeFiles(ixfile, tf, ixfile);
+
+                        Console.WriteLine(" done!");
+                    }
+                }
+                catch(Exception ex)
+                {
+                    Console.WriteLine(" error: {0}", ex.ToString());
+                    errors++;
+                }
+
+                i++;
+            }
+
+            Console.WriteLine("Processed {0} files and found {1} hashes", files, totalblocks);
+            if (errors > 0)
+                Console.WriteLine("Experienced {0} errors", errors);
+
+            return 0;
+        }
+
+        public static void SortFile(string filein, string fileout)
+        {
+            try
+            {
+                // If the file can fit into memory, this is MUCH faster
+                using(var tfout = new Library.Utility.TempFile())
+                {
+                    var data = File.ReadAllLines(filein);
+                    Array.Sort(data, StringComparer.Ordinal);
+                    File.WriteAllLines(tfout, data);
+
+                    File.Copy(tfout, fileout, true);
+                    return;
+                }
+            }
+            catch
+            {
+            }
+
+            using(var tfin = new Library.Utility.TempFile())
+            using(var tfout = new Library.Utility.TempFile())
+            {
+                long swaps;
+
+                File.Copy(filein, tfin, true);
+
+                do
+                {
+                    swaps = 0L;
+
+                    using(var sw = new System.IO.StreamWriter(tfout))
+                    using(var sr = new System.IO.StreamReader(tfin))
+                    {
+                        var c1 = sr.ReadLine();
+                        var c2 = sr.ReadLine();
+
+                        while (c1 != null || c2 != null)
+                        {
+                            if (c1 != null && c1.StartsWith("a"))
+                                Console.Write("");
+                            var cmp = StringComparer.Ordinal.Compare(c1, c2);
+
+                            if (c2 == null || (c1 != null && cmp < 0))
+                            {
+                                sw.WriteLine(c1);
+
+                                c1 = c2;
+                                c2 = sr.ReadLine();
+                            }
+                            else
+                            {
+                                if (cmp != 0)
+                                    sw.WriteLine(c2);
+                                c2 = sr.ReadLine();
+                                swaps++;
+                            }
+                        }
+                    }
+
+                    File.Copy(tfout, tfin, true);
+
+                } while(swaps > 0);
+
+                File.Copy(tfout, fileout, true);
+            }
+        }
+
+        private static void MergeFiles(string file1, string file2, string fileout)
+        {
+            using(var tf = new Library.Utility.TempFile())
+            {
+                using(var sw = new System.IO.StreamWriter(tf))
+                using(var sr1 = new System.IO.StreamReader(file1))
+                using(var sr2 = new System.IO.StreamReader(file2))
+                {
+                    var c1 = sr1.ReadLine();
+                    var c2 = sr2.ReadLine();
+
+                    while(c1 != null || c2 != null)
+                    {
+                        var cmp = StringComparer.Ordinal.Compare(c1, c2);
+
+                        if (c2 == null || (c1 != null && cmp < 0))
+                        {
+                            sw.WriteLine(c1);
+                            c1 = sr1.ReadLine();
+                        }
+                        else
+                        {
+                            if (cmp != 0)
+                                sw.WriteLine(c2);
+
+                            c2 = sr2.ReadLine();
+                        }
+                    }
+                }
+
+                File.Copy(tf, fileout, true);
+            }
+        }
+    }
+}
+
@@ -0,0 +1,123 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Linq;
+using System.Collections.Generic;
+using System.IO;
+
+namespace Duplicati.CommandLine.RecoveryTool
+{
+    public static class List
+    {
+        public static int Run(List<string> args, Dictionary<string, string> options, Library.Utility.IFilter filter)
+        {
+            if (args.Count != 2 && args.Count != 3)
+            {
+                Console.WriteLine("Invalid argument count ({0} expected 2 or 3): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args));
+                return 100;
+            }
+
+            var folder = Path.GetFullPath(args[1]);
+
+            if (!Directory.Exists(folder))
+            {
+                Console.WriteLine("Folder not found: {0}", folder);
+                return 100;
+            }
+
+            Directory.SetCurrentDirectory(folder);
+
+            if (args.Count == 2)
+            {
+                var times = ParseListFiles(folder);
+                foreach(var v in times.Zip(Enumerable.Range(0, times.Length), (a,b) => new KeyValuePair<int, DateTime>(b, a.Key)))
+                    Console.WriteLine("{0}: {1}", v.Key, v.Value.ToLocalTime());
+            }
+            else
+            {
+                var file = SelectListFile(args[2], args[1]);
+
+                var p = Library.Main.Volumes.VolumeBase.ParseFilename(file);
+
+                Library.Main.Volumes.VolumeReaderBase.UpdateOptionsFromManifest(p.CompressionModule, file, new Duplicati.Library.Main.Options(options));
+
+                foreach(var f in EnumerateFilesInDList(file, filter, options))
+                    Console.WriteLine("{0} ({1})", f.Path, Library.Utility.Utility.FormatSizeString(f.Size));
+            }
+
+            return 0;
+        }
+
+        public static IEnumerable<Duplicati.Library.Main.Volumes.IFileEntry> EnumerateFilesInDList(string file, Duplicati.Library.Utility.IFilter filter, Dictionary<string, string> options)
+        {
+            var p = Library.Main.Volumes.VolumeBase.ParseFilename(file);
+            using(var cm = Library.DynamicLoader.CompressionLoader.GetModule(p.CompressionModule, file, options))
+            using(var filesetreader = new Library.Main.Volumes.FilesetVolumeReader(cm, new Duplicati.Library.Main.Options(options)))
+                foreach(var f in filesetreader.Files)
+                {
+                    if (f.Type != Duplicati.Library.Main.FilelistEntryType.File)
+                        continue;
+
+                    bool result;
+                    Library.Utility.IFilter evfilter;
+                    bool match = filter.Matches(f.Path, out result, out evfilter);
+                    if (!match || (match && result))
+                        yield return f;
+                }
+        }
+
+        public static string SelectListFile(string time, string folder)
+        {
+            if (File.Exists(Path.Combine(folder, time)))
+                return Path.Combine(folder, time);
+
+            int index;
+            if (int.TryParse(time, out index))
+            {
+                var items = ParseListFiles(folder);
+                if (index < 0 || index >= items.Length)
+                    throw new Exception(string.Format("Valid range for version is 0 to {1}", items.Length - 1));
+
+                return items[index].Value;
+            }
+
+            var parsedtime = Library.Utility.Timeparser.ParseTimeInterval(time, DateTime.Now, true);
+            var el = (
+                from n in ParseListFiles(folder)
+                 let diff = Math.Abs((n.Key - parsedtime).TotalSeconds)
+                 orderby diff
+                 select n).First();
+
+            Console.WriteLine("Selected time {0}", el.Key);
+
+            return el.Value;
+
+        }
+
+        public static KeyValuePair<DateTime, string>[] ParseListFiles(string folder)
+        {
+            return (
+                from v in Directory.EnumerateFiles(folder)
+                let p = Library.Main.Volumes.VolumeBase.ParseFilename(v)
+                where p != null && p.FileType == Duplicati.Library.Main.RemoteVolumeType.Files
+                orderby p.Time descending
+                select new KeyValuePair<DateTime, string>(p.Time, v)).ToArray();
+
+        }
+    }
+}
+
@@ -0,0 +1,76 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.Linq;
+using System.Collections.Generic;
+
+namespace Duplicati.CommandLine.RecoveryTool
+{
+    static class Program
+    {
+        /// <summary>
+        /// The main entry point for the application.
+        /// </summary>
+        [STAThread]
+        public static int Main(string[] args)
+        {
+            return Duplicati.Library.AutoUpdater.UpdaterManager.RunFromMostRecent(typeof(Program).GetMethod("RealMain"), args);
+        }
+
+        private delegate int CommandRunner(List<string> args, Dictionary<string, string> options, Library.Utility.IFilter filter);
+
+        public static int RealMain(string[] _args)
+        {
+            try
+            {
+                var args = new List<string>(_args);
+                var tmpparsed = Library.Utility.FilterCollector.ExtractOptions(args);
+                var options = tmpparsed.Item1;
+                var filter = tmpparsed.Item2;
+
+                if (!options.ContainsKey("auth_password") && !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("AUTH_PASSWORD")))
+                    options["auth_password"] = System.Environment.GetEnvironmentVariable("AUTH_PASSWORD");
+
+                if (!options.ContainsKey("auth_username") && !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("AUTH_USERNAME")))
+                    options["auth_username"] = System.Environment.GetEnvironmentVariable("AUTH_USERNAME");
+
+                if (options.ContainsKey("tempdir") && !string.IsNullOrEmpty(options["tempdir"]))
+                    Library.Utility.TempFolder.SystemTempPath = options["tempdir"];
+
+                var actions = new Dictionary<string, CommandRunner>(StringComparer.InvariantCultureIgnoreCase);
+                actions["download"] = Download.Run;
+                actions["index"] = Index.Run;
+                actions["list"] = List.Run;
+                actions["restore"] = Restore.Run;
+                actions["help"] = Help.Run;
+
+                CommandRunner command;
+
+                actions.TryGetValue(args.FirstOrDefault(), out command);
+
+                command = command ?? actions["help"];
+
+                return command(args, options, filter);
+            }
+            catch(Exception ex)
+            {
+                Console.WriteLine("Program crashed: {0}{1}", Environment.NewLine, ex.ToString());
+                return 200;
+            }
+        }
+    }
+}
@@ -0,0 +1,43 @@
+//  Copyright (C) 2015, The Duplicati Team
+
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System.Reflection;
+using System.Runtime.CompilerServices;
+
+// Information about this assembly is defined by the following attributes.
+// Change them to the values specific to your project.
+
+[assembly: AssemblyTitle("Duplicati.CommandLine.RecoveryTool")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("")]
+[assembly: AssemblyCopyright("kenneth")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// The assembly version has the format "{Major}.{Minor}.{Build}.{Revision}".
+// The form "{Major}.{Minor}.*" will automatically update the build and revision,
+// and "{Major}.{Minor}.{Build}.*" will update just the revision.
+
+[assembly: AssemblyVersion("1.0.*")]
+
+// The following attributes are used to specify the signing key for the assembly,
+// if desired. See the Mono documentation for more information about signing.
+
+//[assembly: AssemblyDelaySign(false)]
+//[assembly: AssemblyKeyFile("")]
@@ -0,0 +1,476 @@
+//  Copyright (C) 2015, The Duplicati Team
+//  http://www.duplicati.com, info@duplicati.com
+//
+//  This library is free software; you can redistribute it and/or modify
+//  it under the terms of the GNU Lesser General Public License as
+//  published by the Free Software Foundation; either version 2.1 of the
+//  License, or (at your option) any later version.
+//
+//  This library is distributed in the hope that it will be useful, but
+//  WITHOUT ANY WARRANTY; without even the implied warranty of
+//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+//  Lesser General Public License for more details.
+//
+//  You should have received a copy of the GNU Lesser General Public
+//  License along with this library; if not, write to the Free Software
+//  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+using System;
+using System.IO;
+using System.Linq;
+using System.Collections.Generic;
+
+namespace Duplicati.CommandLine.RecoveryTool
+{
+    public static class Restore
+    {
+        public static int Run(List<string> args, Dictionary<string, string> options, Library.Utility.IFilter filter)
+        {
+            if (args.Count != 2 && args.Count != 3)
+            {
+                Console.WriteLine("Invalid argument count ({0} expected 2 or 3): {1}{2}", args.Count, Environment.NewLine, string.Join(Environment.NewLine, args));
+                return 100;
+            }
+
+            var folder = Path.GetFullPath(args[1]);
+
+            if (!Directory.Exists(folder))
+            {
+                Console.WriteLine("Folder not found: {0}", folder);
+                return 100;
+            }
+
+            Directory.SetCurrentDirectory(folder);
+
+            string targetpath;
+            options.TryGetValue("targetpath", out targetpath);
+
+            string ixfile;
+            options.TryGetValue("indexfile", out ixfile);
+            if (string.IsNullOrWhiteSpace(ixfile))
+                ixfile = "index.txt";
+
+            ixfile = Path.GetFullPath(ixfile);
+            if (!File.Exists(ixfile))
+            {
+                Console.WriteLine("Index file not found, perhaps you need to run the index command?");
+                return 100;
+            }
+
+            Console.Write("Sorting index file ...");
+            Index.SortFile(ixfile, ixfile);
+            Console.WriteLine(" done!");
+
+            string filelist;
+            if (args.Count == 2)
+            {
+                var time = List.ParseListFiles(folder).First();
+                filelist = time.Value;
+
+                Console.WriteLine("Using set 0 with timestamp {0}", time.Key.ToLocalTime());
+            }
+            else
+            {
+                filelist = List.SelectListFile(args[2], args[1]);
+            }
+
+            Library.Main.Volumes.VolumeReaderBase.UpdateOptionsFromManifest(Path.GetExtension(filelist).Trim('.'), filelist, new Duplicati.Library.Main.Options(options));
+
+            string blocksize_str;
+            options.TryGetValue("blocksize", out blocksize_str);
+            string blockhash_str;
+            options.TryGetValue("block-hash-algorithm", out blockhash_str);
+            string filehash_str;
+            options.TryGetValue("block-hash-algorithm", out filehash_str);
+
+            long blocksize = string.IsNullOrWhiteSpace(blocksize_str) ? 0 : Library.Utility.Sizeparser.ParseSize(blocksize_str);
+
+            if (blocksize <= 0)
+            {
+                Console.WriteLine("Invalid blocksize: {0}, try setting --blocksize manually");
+                return 100;
+            }
+
+            var blockhasher = string.IsNullOrWhiteSpace(blockhash_str) ? null : System.Security.Cryptography.HashAlgorithm.Create(blockhash_str);
+            var filehasher = string.IsNullOrWhiteSpace(filehash_str) ? null : System.Security.Cryptography.HashAlgorithm.Create(filehash_str);
+
+            if (blockhasher == null)
+                throw new Exception(string.Format("Block hash algorithm not valid: {0}", blockhash_str));
+            if (filehasher == null)
+                throw new Exception(string.Format("File hash algorithm not valid: {0}", filehash_str));
+
+            var hashesprblock = blocksize / (blockhasher.HashSize / 8);
+
+            using(var mru = new CompressedFileMRUCache(options))
+            {
+                Console.WriteLine("Building lookup table for file hashes");
+                var lookup = new HashLookupHelper(ixfile, mru, (int)blocksize, blockhasher.HashSize / 8);
+
+                var filecount = 0L;
+                string largestprefix = null;
+                string[] largestprefixparts = null;
+
+                if (!string.IsNullOrWhiteSpace(targetpath))
+                    Console.WriteLine("Computing restore path");
+                
+                foreach(var f in List.EnumerateFilesInDList(filelist, filter, options))
+                {
+                    if (largestprefix == null)
+                    {
+                        largestprefix = f.Path;
+                        largestprefixparts = largestprefix.Split(new char[] { Path.DirectorySeparatorChar });
+                    }
+                    else if (largestprefix.Length > 1)
+                    {
+                        var parts = f.Path.Split(new char[] { Path.DirectorySeparatorChar });
+
+                        var ni = 0;
+                        for(; ni < Math.Min(parts.Length, largestprefixparts.Length); ni++)
+                            if (!Library.Utility.Utility.ClientFilenameStringComparer.Equals(parts[ni], largestprefixparts[ni]))
+                                break;
+
+                        if (ni != largestprefixparts.Length)
+                        {
+                            if (ni == 0)
+                            {
+                                largestprefixparts = new string[0];
+                                largestprefix = string.Empty;
+                            }
+                            else
+                            {
+                                Array.Resize(ref largestprefixparts, ni - 1);
+                                largestprefix = string.Join(Path.DirectorySeparatorChar.ToString(), largestprefixparts);
+                            }
+                        }
+                    }
+                    filecount++;
+                }
+
+                Console.WriteLine("Restoring {0} files to {1}", filecount, string.IsNullOrWhiteSpace(targetpath) ? "original position" : targetpath);
+
+                if (Library.Utility.Utility.IsClientLinux || largestprefix.Length > 0)
+                    largestprefix = Library.Utility.Utility.AppendDirSeparator(largestprefix);
+
+                if (!string.IsNullOrEmpty(largestprefix))
+                    Console.WriteLine("Removing common prefix {0} from files", largestprefix);
+
+                var i = 0L;
+                var errors = 0L;
+                foreach(var f in List.EnumerateFilesInDList(filelist, filter, options))
+                {
+                    try
+                    {
+                        var targetfile = MapToRestorePath(f.Path, largestprefix, targetpath);
+                        if (!Directory.Exists(Path.GetDirectoryName(targetfile)))
+                            Directory.CreateDirectory(Path.GetDirectoryName(targetfile));
+                            
+                        Console.Write("{0}: {1} ({2})", i, targetfile, Library.Utility.Utility.FormatSizeString(f.Size));
+
+                        using(var tf = new Library.Utility.TempFile())
+                        {
+                            using(var sw = File.OpenWrite(tf))
+                            {
+                                if (f.BlocklistHashes == null)
+                                {
+                                    lookup.WriteHash(sw, f.Hash);
+                                }
+                                else
+                                {
+                                    var blhi = 0L;
+                                    foreach(var blh in f.BlocklistHashes)
+                                    {
+                                        Console.Write(" {0}", blhi);
+                                        var blockhashoffset = blhi * hashesprblock * blocksize;
+
+                                        try
+                                        {
+                                            var bi = 0;
+                                            foreach(var h in lookup.ReadBlocklistHashes(blh))
+                                            {
+                                                try
+                                                {
+                                                    sw.Position = blockhashoffset + (bi * blocksize);
+                                                    lookup.WriteHash(sw, h);
+                                                }
+                                                catch(Exception ex)
+                                                {
+                                                    Console.WriteLine("Failed to read hash: {0}{1}{2}", h, Environment.NewLine, ex.ToString());
+                                                }
+
+                                                bi++;
+                                            }
+                                        }
+                                        catch (Exception ex)
+                                        {
+                                            Console.WriteLine("Failed to read Blocklist hash: {0}{1}{2}", blh, Environment.NewLine, ex.ToString());
+                                        }
+
+                                        blhi++;
+                                    }
+                                }
+                            }
+                                
+                            string fh;
+                            using(var fs = File.OpenRead(tf))
+                                fh = Convert.ToBase64String(filehasher.ComputeHash(fs));
+
+                            if (fh == f.Hash)
+                            {
+                                Console.WriteLine(" done!");
+                                File.Copy(tf, targetfile, true);
+                            }
+                            else
+                            {
+                                Console.Write(" - Restored file hash mismatch");
+                                if (File.Exists(targetfile))
+                                    Console.WriteLine(" - not overwriting existing file: {0}", targetfile);
+                                else
+                                    Console.WriteLine(" - restoring file in damaged condition");
+
+
+                            }
+                        }
+
+                    }
+                    catch (Exception ex)
+                    {
+                        Console.WriteLine(" error: {0}", ex.ToString());
+                        errors++;
+                    }
+                    i++;
+                }
+            }
+
+            return 0;
+        }
+
+        private static string MapToRestorePath(string path, string prefixpath, string restorepath)
+        {
+            if (string.IsNullOrWhiteSpace(restorepath) || string.IsNullOrWhiteSpace(prefixpath))
+                return path;
+
+            return Path.Combine(restorepath, path.Substring(prefixpath.Length));
+        }
+
+
+
+        private class HashLookupHelper : IDisposable
+        {
+            private const int LOOKUP_TABLE_SIZE = 2048;
+
+            private CompressedFileMRUCache m_cache;
+            private int m_hashsize;
+            private int m_blocksize;
+            private Stream m_indexfile;
+            private List<string> m_lookup = new List<string>();
+            private List<long> m_offsets = new List<long>();
+            private byte[] m_linebuf = new byte[128];
+            private byte[] m_newline = Environment.NewLine.Select(x => (byte)x).ToArray();
+
+            public HashLookupHelper(string indexfile, CompressedFileMRUCache cache, int blocksize, int hashsize)
+            {
+                m_cache = cache;
+                m_blocksize = blocksize;
+                m_hashsize = hashsize;
+                m_indexfile = File.OpenRead(indexfile);
+
+                // Build index ....
+                var hashes = 0L;
+                string prev = null;
+                m_indexfile.Position = 0;
+                foreach(var n in AllFileLines())
+                {
+                    if (n.Key != prev)
+                    {
+                        hashes++;
+                        prev = n.Key;
+                    }
+                }
+
+                Console.WriteLine("Index file has {0} hashes in total", hashes);
+
+                var lookuptablesize = Math.Max(1, Math.Min(LOOKUP_TABLE_SIZE, hashes) - 1);
+                var lookupincrements = hashes / lookuptablesize;
+
+                Console.WriteLine("Building lookup table with {0} entries, giving increments of {1}", lookuptablesize, lookupincrements);
+
+                prev = null;
+                var prevoff = 0L;
+                var hc = 0L;
+                m_indexfile.Position = 0;
+                foreach(var n in AllFileLines())
+                {
+                    if (n.Key != prev)
+                    {
+                        if ((hc % lookupincrements) == 0)
+                        {
+                            m_lookup.Add(n.Key);
+                            m_offsets.Add(prevoff);
+                        }
+
+                        hc++;
+                        prev = n.Key;
+                        prevoff = m_indexfile.Position;
+                    }
+                }
+            }
+
+            private string ReadNextLine()
+            {
+                var p = m_indexfile.Position;
+                var max = m_indexfile.Read(m_linebuf, 0, m_linebuf.Length);
+                if (max == 0)
+                    return null;
+                
+                var lfi = 0;
+                for(int i = 0; i < max; i++)
+                    if (m_linebuf[i] == m_newline[lfi])
+                    {
+                        if (lfi == m_newline.Length - 1)
+                        {
+                            m_indexfile.Position = p + i + 1;
+                            return System.Text.Encoding.Default.GetString(m_linebuf, 0, i - m_newline.Length + 1);
+
+                        }
+                        else
+                        {
+                            lfi++;
+                        }
+                    }
+                    else
+                    {
+                        lfi = 0;
+                    }
+
+                throw new Exception(string.Format("Unexpected long line starting at offset {0}, read {1} bytes without a newline", p, m_linebuf.Length));                
+            }
+
+            private IEnumerable<KeyValuePair<string, string>> AllFileLines()
+            {
+                while(true)
+                {
+                    var str = ReadNextLine();
+                    if (str == null)
+                        break;
+
+                    if (str.Length == 0)
+                        continue;
+
+                    var ix = str.IndexOf(", ");
+                    if (ix < 0)
+                        Console.WriteLine("Failed to parse line starting at offset {0} in index file, string: {1}", m_indexfile.Position - str.Length - m_newline.Length, str);
+                    yield return new KeyValuePair<string, string>(str.Substring(0, ix), str.Substring(ix + 2));
+                }
+            }
+
+            public IEnumerable<string> ReadBlocklistHashes(string hash)
+            {
+                var bytes = ReadHash(hash);
+                for(var i = 0; i < bytes.Length; i += m_hashsize)
+                    yield return Convert.ToBase64String(bytes, i, m_hashsize);
+            }
+
+            public byte[] ReadHash(string hash)
+            {
+                var ix = m_lookup.BinarySearch(hash, StringComparer.Ordinal);
+                if (ix < 0)
+                {
+                    ix = ~ix;
+                    if (ix != 0)
+                        ix--;
+                }
+
+                m_indexfile.Position = m_offsets[ix];
+
+                foreach(var v in AllFileLines())
+                {
+                    if (v.Key == hash)
+                        using(var fs = m_cache.ReadBlock(v.Value, hash))
+                        {
+                            var buf = new byte[m_blocksize];
+                            var l = fs.Read(buf, 0, buf.Length);
+                            Array.Resize(ref buf, l);
+
+                            return buf;
+                        }
+                    
+                    
+                    if (StringComparer.Ordinal.Compare(hash, v.Key) < 0)
+                        break;
+                }
+
+                throw new Exception(string.Format("Unable to locate block with hash: {0}", hash));
+            }
+
+            public void WriteHash(Stream sw, string hash)
+            {
+                var h = ReadHash(hash);
+                sw.Write(h, 0, h.Length);
+            }
+
+
+            public void Dispose()
+            {
+                if (m_indexfile != null)
+                    m_indexfile.Dispose();
+                m_indexfile = null;
+            }
+        }
+
+        private class CompressedFileMRUCache : IDisposable
+        {
+            private Dictionary<string, Library.Interface.ICompression> m_lookup = new Dictionary<string, Duplicati.Library.Interface.ICompression>();
+            private List<string> m_mru = new List<string>();
+            private Dictionary<string, string> m_options;
+
+            private const int MAX_OPEN_ARCHIVES = 20;
+
+            public CompressedFileMRUCache(Dictionary<string, string> options)
+            {
+                m_options = options;
+            }
+
+            public Stream ReadBlock(string filename, string hash)
+            {
+                Library.Interface.ICompression cf;
+                if (!m_lookup.TryGetValue(filename, out cf))
+                {
+                    cf = Library.DynamicLoader.CompressionLoader.GetModule(Path.GetExtension(filename).Trim('.'), filename, m_options);
+                    if (cf == null)
+                        throw new Exception(string.Format("Unable to decompress {0}, no such compression module {1}", filename, Path.GetExtension(filename).Trim('.')));
+                    m_lookup[filename] = cf;
+                }
+
+                if (m_mru.Count > 0 && m_mru[0] != filename)
+                {
+                    m_mru.Remove(filename);
+                    m_mru.Insert(0, filename);
+                }
+
+                while (m_mru.Count > MAX_OPEN_ARCHIVES)
+                {
+                    var f = m_mru.Last();
+                    m_lookup[f].Dispose();
+                    m_lookup.Remove(f);
+                    m_mru.Remove(f);
+                }
+
+                return cf.OpenRead(Library.Utility.Utility.Base64PlainToBase64Url(hash));
+            }
+
+            #region IDisposable implementation
+
+            public void Dispose()
+            {
+                foreach(var v in m_lookup.Values)
+                    try { v.Dispose(); }
+                    catch { }
+
+                m_lookup = null;
+                m_mru = null;
+            }
+
+            #endregion
+        }
+    }
+}
+
@@ -0,0 +1,74 @@
+Duplicati Recovery Tool
+=======================
+
+This tool performs a recovery of as much data as possible in small steps that must be performed in order.
+We recommend that you use Duplicati.CommandLine.exe to do the restore, and rely only on this tool if all else fails.
+
+
+The steps to perform are:
+-------------------------
+
+1: Download
+  Download files from the remote store and keep them unencrypted on a location available in the local filesystem.
+
+2: Index
+  Builds an index file to figure out what data is contained inside the files downloaded
+
+3: Restore
+  Restores the files to a destination you choose
+
+
+Optionally you can also run:
+----------------------------
+
+4: List
+  Shows what files are available and tests filters
+
+
+
+Command descriptions:
+=====================
+
+
+Download
+--------
+
+Usage:
+Duplicati.RecoveryTool.exe download <remoteurl> <localfolder> [options]
+
+Downloads all files matching the Duplicati filenames from <remoteurl> to <localfolder>, and decrypts them in the process.
+The remote url must be one supported by Duplicati.
+Use `Duplicati.CommandLine.exe help backends` to see backends and options.
+
+
+
+Index
+-----
+Duplicati.RecoveryTool.exe index <localfolder> [options]
+
+Examines all files found in localfolder and produces an index.txt file, which is a list of all block hashes found in the files.
+The index file can be rather large. It defaults to being stored in <localfolder> but can be specified with --indexfile.
+Some files are created in the system temporary folder, use --tempdir to set an alternative temporary folder location.
+
+
+
+Restore
+-------
+Duplicati.RecoveryTool.exe index <localfolder> [version] [options]
+
+Restores all files to their respective destinations. Use --targetpath to choose another folder where the files are restored into.
+Use the filters, --exclude, to perform a partial restore.
+Version can be either a number, a filename or a date. If omitted the most recent backup is used.
+
+
+
+List
+----
+Duplicati.RecoveryTool.exe list <localfolder> [version] [options]
+
+Lists contents of <localfolder> backups.
+Version can be either a number, a filename or a date. 
+If [version] is omitted a list of backup versions are shown, if [version] is supplied, files from that version are listed.
+Use the filters, --exclude, to show a subset of files.
+
+
@@ -46,6 +46,7 @@
      </EnvironmentVariables>
    </EnvironmentVariables>
    <Prefer32Bit>false</Prefer32Bit>
+    <Commandlineparameters>--webservice-webroot=/Users/kenneth/Udvikler/my-new-angular-app</Commandlineparameters>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release|AnyCPU'">
    <OutputPath>bin\Release</OutputPath>
@@ -276,6 +277,10 @@
      <Project>{D60AD540-0E7D-40CE-83AE-D26E01FFE9B8}</Project>
      <Name>Duplicati.Library.Backend.HubiC</Name>
    </ProjectReference>
+    <ProjectReference Include="..\..\CommandLine\RecoveryTool\Duplicati.CommandLine.RecoveryTool.csproj">
+      <Project>{4A010589-76E6-4F05-A5C4-4598D5DF11F8}</Project>
+      <Name>Duplicati.CommandLine.RecoveryTool</Name>
+    </ProjectReference>
  </ItemGroup>
  <ItemGroup>
    <BootstrapperPackage Include=".NETFramework,Version=v4.0">
@@ -68,6 +68,7 @@
    <Compile Include="FileBackedList.cs" />
    <Compile Include="JoinedFilterExpression.cs" />
    <Compile Include="Strings.cs" />
+    <Compile Include="FilterCollector.cs" />
  </ItemGroup>
  <ItemGroup>
    <None Include="app.config" />
@@ -247,6 +247,10 @@
      <Project>{D60AD540-0E7D-40CE-83AE-D26E01FFE9B8}</Project>
      <Name>Duplicati.Library.Backend.HubiC</Name>
    </ProjectReference>
+    <ProjectReference Include="..\CommandLine\RecoveryTool\Duplicati.CommandLine.RecoveryTool.csproj">
+      <Project>{4A010589-76E6-4F05-A5C4-4598D5DF11F8}</Project>
+      <Name>Duplicati.CommandLine.RecoveryTool</Name>
+    </ProjectReference>
  </ItemGroup>
  <ItemGroup>
    <Folder Include="webroot\" />