duplicati/Duplicati/Library/Main/ForestHash/Operation/RecreateDatabaseHandler.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Duplicati.Library.Main.ForestHash.Database;
using Duplicati.Library.Main.ForestHash.Volumes;

namespace Duplicati.Library.Main.ForestHash.Operation
{
    public class RecreateDatabaseHandler : IDisposable
    {
        private string m_backendurl;
        private FhOptions m_options;
        private CommunicationStatistics m_stat;

        public delegate IEnumerable<IParsedVolume> FilterFilelistDelegate(IEnumerable<IParsedVolume> filelist);
        public delegate IEnumerable<IFileEntry> FilenameFilterDelegate(IEnumerable<IFileEntry> filenamelist);
        public delegate void BlockVolumePostProcessor(string volumename, BlockVolumeReader reader);

        public RecreateDatabaseHandler(string backendurl, FhOptions options, CommunicationStatistics stat)
        {
            m_options = options;
            m_backendurl = backendurl;
            m_stat = stat;
        }

		/// <summary>
		/// Run the recreate procedure
		/// </summary>
		/// <param name="path">Path to the database that will be created</param>
		/// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
		/// <param name="filenamefilter">Filters the files in a filelist to prevent downloading unwanted data</param>
		/// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
        public void Run(string path, FilterFilelistDelegate filelistfilter = null, FilenameFilterDelegate filenamefilter = null, BlockVolumePostProcessor blockprocessor = null)
        {
        	if (System.IO.File.Exists(path))
        		throw new Exception(string.Format("Cannot recreate database because file already exists: {0}", path));

			using(var db = new LocalBlocklistUpdateDatabase(path, m_options.Fhblocksize))
        		DoRun(db, filelistfilter, filenamefilter, blockprocessor);
		}
		
		/// <summary>
		/// Run the recreate procedure
		/// </summary>
		/// <param name="path">Path to the database that will be created</param>
		/// <param name="filelistfilter">A filter that can be used to disregard certain remote files, intended to be used to select a certain filelist</param>
		/// <param name="filenamefilter">Filters the files in a filelist to prevent downloading unwanted data</param>
		/// <param name="blockprocessor">A callback hook that can be used to work with downloaded block volumes, intended to be use to recover data blocks while processing blocklists</param>
        internal void DoRun(LocalDatabase dbparent, FilterFilelistDelegate filelistfilter = null, FilenameFilterDelegate filenamefilter = null, BlockVolumePostProcessor blockprocessor = null)
        {
        	var hashalg = System.Security.Cryptography.HashAlgorithm.Create(m_options.FhBlockHashAlgorithm);
			if (hashalg == null)
				throw new Exception(string.Format(Strings.Foresthash.InvalidHashAlgorithm, m_options.FhBlockHashAlgorithm));
            var hashsize = hashalg.HashSize / 8;

            //We build a local database in steps.
            using (var restoredb = new LocalBlocklistUpdateDatabase(dbparent, m_options.Fhblocksize))
            using (var backend = new FhBackend(m_backendurl, m_options, m_stat, restoredb))
            {
            	var volumeIds = new Dictionary<string, long>();

				var rawlist = backend.List();
				
                //First step is to examine the remote storage to see what
                // kind of data we can find
                var remotefiles =
	                    (from x in rawlist
	                    let n = VolumeBase.ParseFilename(x)
	                    where
	                        n != null
	                            &&
	                        n.Prefix == m_options.BackupPrefix
                        select n).ToArray(); //ToArray() ensures that we do not remote-request it multiple times

				if (remotefiles.Length == 0)
				{
					if (rawlist.Count == 0)
						throw new Exception("No files were found at the remote location, perhaps the target url is incorrect?");
					else
					{
						var tmp = 
							(from x in rawlist
	                    		let n = VolumeBase.ParseFilename(x)
	                    	where
	                        	n != null
	                        select n.Prefix).ToArray();
	                    
	                    var types = tmp.Distinct().ToArray();
	                    if (tmp.Length == 0)
							throw new Exception(string.Format("Found {0} files at the remote storage, but none that could be parsed", rawlist.Count));
						else if (types.Length == 1)
							throw new Exception(string.Format("Found {0} parse-able files with the prefix {1}, did you forget to set the backup-prefix?", tmp.Length, types[0]));
						else
							throw new Exception(string.Format("Found {0} parse-able files (of {1} files) with different prefixes: {2}, did you forget to set the backup-prefix?", tmp.Length, rawlist.Count, string.Join(", ", types)));
					}
				}

                //Then we select the filelist we should work with,
                // and create the filelist table to fit
                IEnumerable<IParsedVolume> filelists =
                    from n in remotefiles
                    where n.FileType == RemoteVolumeType.Files
                    orderby n.Time descending
                    select n;

                if (filelistfilter != null)
                    filelists = filelistfilter(filelists).ToArray();

                using (var backupdb = new LocalBackupDatabase(restoredb, m_options))
                {
                    foreach (var fl in remotefiles)
                        volumeIds[fl.File.Name] = backupdb.RegisterRemoteVolume(fl.File.Name, fl.FileType, RemoteVolumeState.Uploaded);
                                       
                    //We grab all index files, and update the block table
                    using (var tr = restoredb.BeginTransaction())
                    {
						var indexfiles = 
							from n in remotefiles
							where n.FileType == RemoteVolumeType.Index
							select new RemoteVolume(n.File) as IRemoteVolume;
												
                        foreach (var sf in new AsyncDownloader(indexfiles.ToList(), backend))
                        using (var tmpfile = sf.Value)
                        {
                            if (sf.Key.Hash != null && sf.Key.Size > 0)
                                backupdb.UpdateRemoteVolume(sf.Key.Name, RemoteVolumeState.Verified, sf.Key.Size, sf.Key.Hash, tr);
							
                            using (var svr = new IndexVolumeReader(RestoreHandler.GetCompressionModule(sf.Key.Name), tmpfile, m_options, hashsize))
                            {
					        	ForestHash.VerifyParameters(restoredb, m_options);

                                //If there are blocklists in the index file, update the blocklists
                                foreach (var b in svr.BlockLists)
                                    restoredb.UpdateBlocklist(b.Hash, b.Blocklist, hashsize, tr);

                                foreach (var a in svr.Volumes)
                                {
									var volumeID = restoredb.GetRemoteVolumeID(a.Filename);
                                    //Add all block/volume mappings
                                    foreach (var b in a.Blocks)
                                        backupdb.AddBlock(b.Key, b.Value, volumeID, tr);

                                    backupdb.UpdateRemoteVolume(a.Filename, RemoteVolumeState.Verified, a.Length, a.Hash, tr);
                                    backupdb.AddIndexBlockLink(restoredb.GetRemoteVolumeID(sf.Key.Name), volumeID, tr);
                                }
                            }
                        }

                        tr.Commit();
                    }

                    //We need this to prepare for the block-lists
                    var dummylist = new string[0];

                    // Default filter is no filter (pass-through)
                    if (filenamefilter == null)
                        filenamefilter = lst => lst;

                    //Now record all blocksets and files needed
                    using (var tr = backupdb.BeginTransaction())
                    {
                    	var filelistWork = (from n in filelists select new RemoteVolume(n.File) as IRemoteVolume).ToList();
                    	foreach (var entry in new AsyncDownloader(filelistWork, backend))
                    	using (var tmpfile = entry.Value)
                        {
                            if (entry.Key.Hash != null && entry.Key.Size > 0)
                                backupdb.UpdateRemoteVolume(entry.Key.Name, RemoteVolumeState.Verified, entry.Key.Size, entry.Key.Hash, tr);

                        	var parsed = VolumeBase.ParseFilename(entry.Key.Name);
		                    // Create timestamped operations based on the file timestamp
                        	backupdb.CreateFileset(volumeIds[entry.Key.Name], parsed.Time, tr);
                            using (var filelistreader = new FilesetVolumeReader(parsed.CompressionModule, tmpfile, m_options))
                                foreach (var fe in filenamefilter(filelistreader.Files))
                                {
                                    if (fe.Type == FilelistEntryType.Folder)
                                    {
                                        long metaid = -1;
                                        if (fe.Metahash != null)
                                            backupdb.AddMetadataset(fe.Metahash, fe.Metasize, out metaid, tr);
                                        backupdb.AddDirectoryEntry(fe.Path, metaid, fe.Time, tr);

                                    }
                                    else if (fe.Type == FilelistEntryType.File)
                                    {
                                        long metaid = -1;
                                        long blocksetid;
                                        if (fe.Metahash != null)
                                            backupdb.AddMetadataset(fe.Metahash, fe.Metasize, out metaid, tr);

                                        backupdb.AddBlockset(fe.Hash, fe.Size, m_options.Fhblocksize, dummylist, fe.BlocklistHashes, out blocksetid, tr);
                                        backupdb.AddFile(fe.Path, fe.Time, blocksetid, metaid, tr);
                                    }
                                }
                        }
                        
                        tr.Commit();
                    }
                }

                //We now need some blocklists, so we start by grabbing a
                // volume with one of the blocklists
                //For each volume we then update the blocklist table
                // and then restore the blocks we know
                restoredb.FindMissingBlocklistHashes();

                foreach (var sf in new AsyncDownloader(restoredb.GetMissingBlockListVolumes(), backend))
                	using (var tmpfile = sf.Value)
                    using (var rd = new BlockVolumeReader(RestoreHandler.GetCompressionModule(sf.Key.Name), tmpfile, m_options))
                    using (var tr = restoredb.BeginTransaction())
                    {
                    	var volumeid = restoredb.GetRemoteVolumeID(sf.Key.Name);
                        foreach (var blocklisthash in restoredb.GetBlockLists(volumeid))
                            restoredb.UpdateBlocklist(blocklisthash, rd.ReadBlocklist(blocklisthash, hashsize), hashsize, tr);

                        tr.Commit();

                        //At this point we can patch files with data from the block volume
                        if (blockprocessor != null)
                            blockprocessor(sf.Key.Name, rd);
                    }

				backend.WaitForComplete(restoredb, null);

                //All done, we must verify that we have all blocklist fully intact
                // if this fails, the db will not be deleted, so it can be used,
                // except to continue a backup
                restoredb.VerifyConsistency(null);

            }
        }

        public void Dispose()
        {
            m_stat.EndTime = DateTime.Now;
        }
    }
}