Files
duplicati/Duplicati/Library/Main/Interface.cs
T
kenneth.skovhede@gmail.com acfcd02a3a Fixed issue #14, and added a more readable output than what was avalible in beta 3.
Added commandline action to purge the signature cache.
Duplicati now removes files from the signature cache when they are deleted from the backend.

git-svn-id: https://duplicati.googlecode.com/svn/trunk@130 59da171f-624f-0410-aa54-27559c288bec
2009-03-23 20:44:25 +00:00

1179 lines
54 KiB
C#

#region Disclaimer / License
// Copyright (C) 2009, Kenneth Skovhede
// http://www.hexad.dk, opensource@hexad.dk
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
#endregion
using System;
using System.Collections.Generic;
using System.Text;
namespace Duplicati.Library.Main
{
public enum DuplicatiOperation
{
Backup,
Restore,
List,
Remove
};
public delegate void OperationProgressEvent(Interface caller, DuplicatiOperation operation, int progress, int subprogress, string message);
public class Interface : IDisposable
{
private string m_backend;
//private Dictionary<string, string> m_options;
private Options m_options;
//The amount of progressbar allocated for reading incremental data
private double m_incrementalFraction = 0.15;
private double m_progress = 0.0;
/// <summary>
/// The filenamestrategy for the signature cache
/// </summary>
FilenameStrategy m_cacheFilenameStrategy = new FilenameStrategy("dpl", "_", true);
public event OperationProgressEvent OperationStarted;
public event OperationProgressEvent OperationCompleted;
public event OperationProgressEvent OperationProgress;
public event OperationProgressEvent OperationError;
public Interface(string backend, Dictionary<string, string> options)
{
m_backend = backend;
m_options = new Options(options);
}
public string Backup(string source)
{
BackupStatistics bs = new BackupStatistics();
SetupCommonOptions();
Backend.IBackend backend = null;
using (new Logging.Timer("Backup from " + source + " to " + m_backend))
{
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Backup, -1, -1, "Loading remote filelist");
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, -1, -1, "Loading remote filelist");
FilenameStrategy fns = new FilenameStrategy(m_options);
bool full = m_options.Full;
backend = Backend.BackendLoader.GetBackend(m_backend, m_options.RawOptions);
if (backend == null)
throw new Exception("Unable to find backend for m_backend: " + m_backend);
backend = new BackendWrapper(bs, backend, m_options);
((BackendWrapper)backend).ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupTransfer_ProgressEvent);
backend = EncryptedBackendWrapper.WrapWithEncryption(backend, m_options);
List<BackupEntry> orphans = m_options.AutoCleanup ? new List<BackupEntry>() : null;
List <BackupEntry > prev = ParseFileList(backend, orphans);
m_progress = 0.0;
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Backup, (int)(m_progress * 100), -1, "Reading incremental data");
if (prev.Count == 0)
full = true;
if (!full)
full = DateTime.Now > m_options.FullIfOlderThan(prev[prev.Count - 1].Time);
List<string> controlfiles = new List<string>();
if (!string.IsNullOrEmpty(m_options.SignatureControlFiles))
controlfiles.AddRange(m_options.SignatureControlFiles.Split(System.IO.Path.PathSeparator));
if (!string.IsNullOrEmpty(m_options.SignatureCachePath) && !System.IO.Directory.Exists(m_options.SignatureCachePath))
System.IO.Directory.CreateDirectory(m_options.SignatureCachePath);
using (Core.TempFolder tempfolder = new Duplicati.Library.Core.TempFolder())
{
List<Core.IFileArchive> patches = new List<Duplicati.Library.Core.IFileArchive>();
if (!full)
{
List<BackupEntry> entries = new List<BackupEntry>();
entries.Add(prev[prev.Count - 1]);
entries.AddRange(prev[prev.Count - 1].Incrementals);
patches = FindPatches(backend, entries, orphans, tempfolder);
}
else
m_incrementalFraction = 0.0;
DateTime backuptime = DateTime.Now;
using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(source, bs, m_options.Filter, patches))
{
if (OperationProgress != null)
{
OperationProgress(this, DuplicatiOperation.Backup, -1, -1, "Building filelist ...");
dir.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupRSyncDir_ProgressEvent);
}
dir.DisableFiletimeCheck = m_options.DisableFiletimeCheck;
using (new Logging.Timer("Initiating multipass"))
dir.InitiateMultiPassDiff(full);
int vol = 0;
long totalsize = 0;
List<string> contenthashes = new List<string>();
List<string> signaturehashes = new List<string>();
bool done = false;
while (!done && totalsize < m_options.MaxSize)
{
using (new Logging.Timer("Multipass " + (vol + 1).ToString()))
{
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, (int)(m_progress * 100), -1, "Creating volume " + (vol + 1).ToString());
//The backendwrapper will remove these
Core.TempFile signaturefile = new Duplicati.Library.Core.TempFile();
Core.TempFile contentfile = new Duplicati.Library.Core.TempFile();
signaturefile.Protected = true;
contentfile.Protected = true;
using (Compression.FileArchiveZip signaturearchive = Duplicati.Library.Compression.FileArchiveZip.CreateArchive(signaturefile))
using (Compression.FileArchiveZip contentarchive = Duplicati.Library.Compression.FileArchiveZip.CreateArchive(contentfile))
{
//Add signature files to archive
foreach (string s in controlfiles)
if (!string.IsNullOrEmpty(s))
using (System.IO.Stream cs = signaturearchive.CreateFile(System.IO.Path.Combine(RSync.RSyncDir.CONTROL_ROOT, System.IO.Path.GetFileName(s))))
using (System.IO.FileStream fs = System.IO.File.OpenRead(s))
Core.Utility.CopyStream(fs, cs);
//Only add control files to the very first volume
controlfiles.Clear();
done = dir.MakeMultiPassDiff(signaturearchive, contentarchive, Math.Min(m_options.VolumeSize, m_options.MaxSize - totalsize));
totalsize += new System.IO.FileInfo(contentfile).Length;
totalsize += signaturearchive.Size;
if (totalsize >= m_options.MaxSize)
dir.FinalizeMultiPass(signaturearchive, contentarchive);
}
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, (int)(m_progress * 100), -1, "Uploading content, volume " + (vol + 1).ToString());
contenthashes.Add(Core.Utility.CalculateHash(contentfile));
using (new Logging.Timer("Writing delta file " + (vol + 1).ToString()))
backend.Put(fns.GenerateFilename(BackupEntry.EntryType.Content, full, backuptime, vol + 1) + ".zip", contentfile);
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, (int)(m_progress * 100), -1, "Uploading signatures, volume " + (vol + 1).ToString());
signaturehashes.Add(Core.Utility.CalculateHash(signaturefile));
using (new Logging.Timer("Writing remote signatures"))
{
//TODO: put this in the backend wrapper, so it can be delayed properly
//For now it works, because the manifest does not contain this entry anyway
//Any problems would only occur when using disabled signature checks
if (!string.IsNullOrEmpty(m_options.SignatureCachePath))
System.IO.File.Copy(signaturefile, System.IO.Path.Combine(m_options.SignatureCachePath, m_cacheFilenameStrategy.GenerateFilename(BackupEntry.EntryType.Signature, full, backuptime, vol + 1)));
backend.Put(fns.GenerateFilename(BackupEntry.EntryType.Signature, full, backuptime, vol + 1) + ".zip", signaturefile);
}
}
//The backend wrapper will remove this
Core.TempFile mf = new Duplicati.Library.Core.TempFile();
mf.Protected = true;
using (new Logging.Timer("Writing manifest"))
{
System.Xml.XmlDocument doc = new System.Xml.XmlDocument();
System.Xml.XmlNode root = doc.AppendChild(doc.CreateElement("Manifest"));
root.Attributes.Append(doc.CreateAttribute("version")).Value = "1";
root.AppendChild(doc.CreateElement("VolumeCount")).InnerText = (vol + 1).ToString();
System.Xml.XmlNode contentroot = root.AppendChild(doc.CreateElement("ContentFiles"));
System.Xml.XmlNode signatureroot = root.AppendChild(doc.CreateElement("SignatureFiles"));
foreach (string s in contenthashes)
contentroot.AppendChild(doc.CreateElement("Hash")).InnerText = s;
foreach (string s in signaturehashes)
signatureroot.AppendChild(doc.CreateElement("Hash")).InnerText = s;
doc.Save(mf);
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, (int)(m_progress * 100), -1, "Uploading manifest, volume " + (vol + 1).ToString());
backend.Put(fns.GenerateFilename(BackupEntry.EntryType.Manifest, full, backuptime) + ".manifest", mf);
}
vol++;
}
}
}
}
finally
{
m_progress = 100.0;
if (backend != null)
backend.Dispose();
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Backup, 100, -1, "Completed");
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, 100, -1, "Completed");
}
}
bs.EndTime = DateTime.Now;
return bs.ToString();
}
private void DeleteSignatureCacheCopy(BackupEntry entry)
{
if (!string.IsNullOrEmpty(m_options.SignatureCachePath))
{
string cacheFilename = m_cacheFilenameStrategy.GenerateFilename(entry.Type, entry.IsFull, entry.Time, entry.VolumeNumber);
if (System.IO.File.Exists(cacheFilename))
try { System.IO.File.Delete(cacheFilename); }
catch { }
}
}
private void DeleteOrphans(Backend.IBackend backend, List<BackupEntry> orphans)
{
foreach (BackupEntry be in orphans)
{
Logging.Log.WriteMessage("Removing leftover file: " + be.Filename, Duplicati.Library.Logging.LogMessageType.Information);
if (m_options.Force)
{
backend.Delete(be.Filename);
DeleteSignatureCacheCopy(be);
}
foreach (BackupEntry bex in be.SignatureFile)
{
Logging.Log.WriteMessage("Removing leftover file: " + bex.Filename, Duplicati.Library.Logging.LogMessageType.Information);
if (m_options.Force)
{
backend.Delete(bex.Filename);
DeleteSignatureCacheCopy(bex);
}
}
foreach (BackupEntry bex in be.ContentVolumes)
{
Logging.Log.WriteMessage("Removing leftover file: " + bex.Filename, Duplicati.Library.Logging.LogMessageType.Information);
if (m_options.Force)
{
backend.Delete(bex.Filename);
DeleteSignatureCacheCopy(bex);
}
}
}
if (!m_options.Force && orphans.Count > 0)
Logging.Log.WriteMessage("No files removed, specify --force to remove files.", Duplicati.Library.Logging.LogMessageType.Information);
}
private void BackupTransfer_ProgressEvent(int progress, string filename)
{
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, (int)(m_progress * 100), progress, filename);
}
private void BackupRSyncDir_ProgressEvent(int progress, string filename)
{
m_progress = ((1.0 - m_incrementalFraction) * (progress / (double)100.0)) + m_incrementalFraction;
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, (int)(m_progress * 100), -1, "Processing: " + filename);
}
public string Restore(string target)
{
SetupCommonOptions();
RestoreStatistics rs = new RestoreStatistics();
Backend.IBackend backend = null;
using (new Logging.Timer("Restore from " + m_backend + " to " + target))
{
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Restore, -1, -1, "Started");
Core.FilenameFilter filter = m_options.Filter;
//Filter is prefered, if both file and filter is specified
if (!m_options.HasFilter && !string.IsNullOrEmpty(m_options.FileToRestore))
{
List<Core.IFilenameFilter> list = new List<Duplicati.Library.Core.IFilenameFilter>();
list.Add(new Core.FilelistFilter(true, m_options.FileToRestore.Split(System.IO.Path.PathSeparator)));
list.Add(new Core.RegularExpressionFilter(false, ".*"));
filter = new Duplicati.Library.Core.FilenameFilter(list);
}
backend = Backend.BackendLoader.GetBackend(m_backend, m_options.RawOptions);
if (backend == null)
throw new Exception("Unable to find backend for target: " + m_backend);
backend = new BackendWrapper(rs, backend, m_options);
BackupEntry bestFit = FindBestMatch(ParseFileList(backend), m_options.RestoreTime);
if (bestFit.EncryptionMode != null)
backend = EncryptedBackendWrapper.WrapWithEncryption(backend, bestFit.EncryptionMode, m_options);
List<BackupEntry> entries = new List<BackupEntry>();
entries.Add(bestFit);
entries.AddRange(bestFit.Incrementals);
int patchno = 0;
using (RSync.RSyncDir sync = new Duplicati.Library.Main.RSync.RSyncDir(target, rs, filter))
{
foreach (BackupEntry be in entries)
{
List<string> contentHashes = null;
if (!m_options.SkipFileHashChecks)
{
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, 0, -1, "Reading manifest file: " + be.Filename);
contentHashes = new List<string>();
using (new Logging.Timer("Get " + be.Filename))
using (Core.TempFile tf = new Duplicati.Library.Core.TempFile())
{
backend.Get(be.Filename, tf);
System.Xml.XmlDocument doc = new System.Xml.XmlDocument();
doc.Load(tf);
foreach (System.Xml.XmlNode n in doc.SelectNodes("Manifest/ContentFiles/Hash"))
contentHashes.Add(n.InnerText);
}
}
foreach (BackupEntry vol in be.ContentVolumes)
{
//Skip nonlisted
if (contentHashes != null && vol.VolumeNumber > contentHashes.Count)
continue; //TODO: Report this
if (vol.CompressionMode != "zip")
throw new Exception("Unexpected compression mode");
using (Core.TempFile patchzip = new Duplicati.Library.Core.TempFile())
{
//TODO: Set better text messages
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, 0, -1, "Patching restore with #" + (patchno + 1).ToString());
using (new Logging.Timer("Get " + vol.Filename))
backend.Get(vol.Filename, patchzip);
if (contentHashes != null)
{
string hash = Core.Utility.CalculateHash(patchzip);
if (hash != contentHashes[vol.VolumeNumber - 1])
throw new Exception("Hash mismatch on file " + vol.Filename + " recorded hash: " + contentHashes[vol.VolumeNumber] + ", actual hash: " + hash);
}
using (new Logging.Timer((patchno == 0 ? "Full restore to: " : "Incremental restore " + patchno.ToString() + " to: ") + target))
sync.Patch(target, new Compression.FileArchiveZip(patchzip));
}
patchno++;
}
}
}
}
finally
{
if (backend != null)
backend.Dispose();
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Restore, 100, -1, "Completed");
}
}
rs.EndTime = DateTime.Now;
//TODO: The RS should have the number of restored files, and the size of those
//but that is a little difficult, because some may be restored, and then removed
return rs.ToString();
}
/// <summary>
/// Restores control files added to a backup.
/// </summary>
/// <param name="target">The folder into which to restore the files</param>
/// <returns>A restore report</returns>
public string RestoreControlFiles(string target)
{
SetupCommonOptions();
RestoreStatistics rs = new RestoreStatistics();
Backend.IBackend backend = null;
using (new Logging.Timer("Restore control files from " + m_backend + " to " + target))
{
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Restore, 0, -1, "Started");
backend = Backend.BackendLoader.GetBackend(m_backend, m_options.RawOptions);
if (backend == null)
throw new Exception("Unable to find backend for target: " + m_backend);
backend = new BackendWrapper(rs, backend, m_options);
List<BackupEntry> attempts = ParseFileList(backend);
List<BackupEntry> flatlist = new List<BackupEntry>();
foreach (BackupEntry be in attempts)
{
flatlist.Add(be);
flatlist.AddRange(be.Incrementals);
}
flatlist.Reverse();
string prefix = Core.Utility.AppendDirSeperator(RSync.RSyncDir.CONTROL_ROOT);
foreach (BackupEntry be in flatlist)
{
Backend.IBackend realbackend = backend;
if (be.EncryptionMode != null)
realbackend = EncryptedBackendWrapper.WrapWithEncryption(backend, be.EncryptionMode, m_options);
if (be.SignatureFile.Count > 0)
using(Core.TempFile z = new Duplicati.Library.Core.TempFile())
{
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, 0, -1, "Reading incremental data from " + be.SignatureFile[0].Filename);
using (new Logging.Timer("Get " + be.SignatureFile[0].Filename))
realbackend.Get(be.SignatureFile[0].Filename, z);
using(Compression.FileArchiveZip fz = new Duplicati.Library.Compression.FileArchiveZip(z))
{
bool any = false;
foreach (string f in fz.ListFiles(prefix))
{
any = true;
using (System.IO.Stream s1 = fz.OpenRead(f))
using (System.IO.Stream s2 = System.IO.File.Create(System.IO.Path.Combine(target, f.Substring(prefix.Length))))
Core.Utility.CopyStream(s1, s2);
}
if (any)
break;
rs.LogError("Failed to find control files in: " + be.SignatureFile[0].Filename);
}
}
}
}
finally
{
if (backend != null)
backend.Dispose();
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Restore, 100, -1, "Completed");
}
}
rs.EndTime = DateTime.Now;
//TODO: The RS should have the number of restored files, and the size of those
//but that is a little difficult, because some may be restored, and then removed
return rs.ToString();
}
public string RemoveAllButNFull()
{
int x = m_options.RemoveAllButNFull;
Backend.IBackend backend = Backend.BackendLoader.GetBackend(m_backend, m_options.RawOptions);
if (backend == null)
throw new Exception("Unable to find backend for target: " + m_backend);
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Remove, 0, -1, "Started");
List<BackupEntry> entries = ParseFileList(backend);
List<BackupEntry> toremove = new List<BackupEntry>();
while (entries.Count > x)
{
BackupEntry be = entries[0];
entries.RemoveAt(0);
be.Incrementals.Reverse();
toremove.AddRange(be.Incrementals);
toremove.Add(be);
}
return RemoveBackupSets(backend, toremove);
}
finally
{
if (backend != null)
backend.Dispose();
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Remove, 100, -1, "Completed");
}
}
public string RemoveOlderThan()
{
StringBuilder sb = new StringBuilder();
DateTime expires = m_options.RemoveOlderThan;
Backend.IBackend backend = Backend.BackendLoader.GetBackend(m_backend, m_options.RawOptions);
if (backend == null)
throw new Exception("Unable to find backend for target: " + m_backend);
try
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.Remove, 0, -1, "Started");
List<BackupEntry> entries = ParseFileList(backend);
List<BackupEntry> toremove = new List<BackupEntry>();
while (entries.Count > 0 && entries[0].Time > expires)
{
BackupEntry be = entries[0];
entries.RemoveAt(0);
bool hasNewer = false;
foreach (BackupEntry bex in be.Incrementals)
if (bex.Time < expires)
{
hasNewer = true;
break;
}
if (hasNewer)
{
List<BackupEntry> t = new List<BackupEntry>(be.Incrementals);
t.Insert(0, be);
for (int i = 0; i < t.Count; i++)
if (t[i].Time > expires)
sb.AppendLine("Not deleting backup at time: " + t[i].Time.ToString(System.Globalization.CultureInfo.InvariantCulture) + ", because later backups depend on it");
break;
}
else
{
be.Incrementals.Reverse();
toremove.AddRange(be.Incrementals);
toremove.Add(be);
}
}
sb.Append(RemoveBackupSets(backend, toremove));
}
finally
{
if (backend != null)
backend.Dispose();
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.Remove, 100, -1, "Completed");
}
return sb.ToString();
}
private string RemoveBackupSets(Backend.IBackend backend, List<BackupEntry> entries)
{
StringBuilder sb = new StringBuilder();
foreach (BackupEntry be in entries)
{
sb.AppendLine("Deleting backup at " + be.Time.ToString(System.Globalization.CultureInfo.InvariantCulture));
if (m_options.Force)
{
//Delete manifest
backend.Delete(be.Filename);
DeleteSignatureCacheCopy(be);
foreach (BackupEntry bex in be.ContentVolumes)
{
backend.Delete(bex.Filename);
DeleteSignatureCacheCopy(bex);
}
foreach (BackupEntry bex in be.SignatureFile)
{
backend.Delete(bex.Filename);
DeleteSignatureCacheCopy(bex);
}
}
}
if (!m_options.Force && entries.Count > 0)
sb.AppendLine("Files are not deleted, use the --force command to actually remove files");
return sb.ToString();
}
public string[] List()
{
SetupCommonOptions();
List<string> res = new List<string>();
Duplicati.Library.Backend.IBackend i = new Duplicati.Library.Backend.BackendLoader(m_backend, m_options.RawOptions);
if (i == null)
throw new Exception("Unable to find backend for target: " + m_backend);
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.List, 0, -1, "Started");
using(i)
foreach (Duplicati.Library.Backend.FileEntry fe in i.List())
res.Add(fe.Name);
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.List, 100, -1, "Completed");
return res.ToArray();
}
public List<BackupEntry> ParseFileList()
{
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.List, 0, -1, "Started");
Backend.IBackend obackend = Backend.BackendLoader.GetBackend(m_backend, m_options.RawOptions);
if (obackend == null)
throw new Exception("Unable to find backend for target: " + m_backend);
using (obackend)
{
List<BackupEntry> data = ParseFileList(obackend);
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.List, 100, -1, "Completed");
return data;
}
}
public string Cleanup()
{
bool anyRemoved = false;
Backend.IBackend backend = Backend.BackendLoader.GetBackend(m_backend, m_options.RawOptions);
if (backend == null)
throw new Exception("Unable to find backend for target: " + m_backend);
using (backend)
{
List<BackupEntry> orphans = new List<BackupEntry>();
List<BackupEntry> sorted = ParseFileList(backend, orphans);
List<BackupEntry> entries = new List<BackupEntry>();
entries.AddRange(sorted);
foreach (BackupEntry be in sorted)
entries.AddRange(be.Incrementals);
DeleteOrphans(backend, orphans);
//Now compare the actual filelist with the manifest
foreach (BackupEntry be in entries)
{
using (new Logging.Timer("Get " + be.Filename))
using (Core.TempFile tf = new Duplicati.Library.Core.TempFile())
{
backend.Get(be.Filename, tf);
System.Xml.XmlDocument doc = new System.Xml.XmlDocument();
doc.Load(tf);
int count = doc.SelectNodes("Manifest/SignatureFiles/Hash").Count;
if (doc.SelectNodes("Manifest/ContentFiles/Hash").Count != count)
throw new Exception("Invalid manifest, wrong count: " + be.Filename);
if (count == 0)
throw new Exception("Invalid manifest, no files: " + be.Filename);
for (int i = count - 1; i < be.SignatureFile.Count; i++)
{
anyRemoved = true;
Logging.Log.WriteMessage("Removing partial file: " + be.SignatureFile[i].Filename, Duplicati.Library.Logging.LogMessageType.Information);
if (m_options.Force)
{
backend.Delete(be.SignatureFile[i].Filename);
DeleteSignatureCacheCopy(be.SignatureFile[i]);
}
}
for (int i = count - 1; i < be.ContentVolumes.Count; i++)
{
anyRemoved = true;
Logging.Log.WriteMessage("Removing partial file: " + be.SignatureFile[i].Filename, Duplicati.Library.Logging.LogMessageType.Information);
if (m_options.Force)
{
backend.Delete(be.ContentVolumes[i].Filename);
DeleteSignatureCacheCopy(be.ContentVolumes[i]);
}
}
}
}
}
if (!m_options.Force && anyRemoved)
Logging.Log.WriteMessage("No files removed, specify --force to remove files.", Duplicati.Library.Logging.LogMessageType.Information);
return ""; //TODO: Write a message here?
}
private List<BackupEntry> ParseFileList(Duplicati.Library.Backend.IBackend backend)
{
return ParseFileList(backend, null);
}
private List<BackupEntry> ParseFileList(Duplicati.Library.Backend.IBackend backend, List<BackupEntry> orphans)
{
SetupCommonOptions();
using (new Logging.Timer("Getting and sorting filelist from " + backend.DisplayName))
{
FilenameStrategy fns = new FilenameStrategy(m_options);
List<BackupEntry> incrementals = new List<BackupEntry>();
List<BackupEntry> fulls = new List<BackupEntry>();
Dictionary<string, List<BackupEntry>> signatures = new Dictionary<string, List<BackupEntry>>();
Dictionary<string, List<BackupEntry>> contents = new Dictionary<string, List<BackupEntry>>();
foreach (Duplicati.Library.Backend.FileEntry fe in backend.List())
{
BackupEntry be = fns.DecodeFilename(fe);
if (be == null)
continue; //Non-duplicati files
if (be.Type == BackupEntry.EntryType.Content)
{
string content = fns.GenerateFilename(BackupEntry.EntryType.Manifest, be.IsFull, be.IsShortName, be.Time) + ".manifest";
if (be.EncryptionMode != null)
content += "." + be.EncryptionMode;
if (!contents.ContainsKey(content))
contents[content] = new List<BackupEntry>();
contents[content].Add(be);
}
else if (be.Type == BackupEntry.EntryType.Signature)
{
string content = fns.GenerateFilename(BackupEntry.EntryType.Manifest, be.IsFull, be.IsShortName, be.Time) + ".manifest";
if (be.EncryptionMode != null)
content += "." + be.EncryptionMode;
if (!signatures.ContainsKey(content))
signatures[content] = new List<BackupEntry>();
signatures[content].Add(be);
}
else if (be.Type != BackupEntry.EntryType.Manifest)
throw new Exception("Invalid entry type");
else if (be.IsFull)
fulls.Add(be);
else
incrementals.Add(be);
}
fulls.Sort(new Sorter());
incrementals.Sort(new Sorter());
foreach (BackupEntry be in fulls)
{
if (contents.ContainsKey(be.Filename))
{
be.ContentVolumes.AddRange(contents[be.Filename]);
contents.Remove(be.Filename);
}
if (signatures.ContainsKey(be.Filename))
{
be.SignatureFile.AddRange(signatures[be.Filename]);
signatures.Remove(be.Filename);
}
}
int index = 0;
foreach (BackupEntry be in incrementals)
{
if (contents.ContainsKey(be.Filename))
{
be.ContentVolumes.AddRange(contents[be.Filename]);
contents.Remove(be.Filename);
}
if (signatures.ContainsKey(be.Filename))
{
be.SignatureFile.AddRange(signatures[be.Filename]);
signatures.Remove(be.Filename);
}
if (index >= fulls.Count || be.Time <= fulls[index].Time)
{
if (orphans == null)
Logging.Log.WriteMessage("Failed to match incremental package to a full: " + be.Filename, Duplicati.Library.Logging.LogMessageType.Warning);
else
orphans.Add(be);
continue;
}
else
{
while (index < fulls.Count - 1 && be.Time > fulls[index + 1].Time)
index++;
fulls[index].Incrementals.Add(be);
}
}
foreach (BackupEntry be in fulls)
{
be.ContentVolumes.Sort(new Sorter());
be.SignatureFile.Sort(new Sorter());
}
foreach (BackupEntry be in incrementals)
{
be.ContentVolumes.Sort(new Sorter());
be.SignatureFile.Sort(new Sorter());
}
if (orphans != null)
{
foreach (List<BackupEntry> lb in contents.Values)
orphans.AddRange(lb);
foreach (List<BackupEntry> lb in signatures.Values)
orphans.AddRange(lb);
}
return fulls;
}
}
public IList<string> ListContent()
{
SetupCommonOptions();
RestoreStatistics rs = new RestoreStatistics();
Duplicati.Library.Backend.IBackend backend = new Duplicati.Library.Backend.BackendLoader(m_backend, m_options.RawOptions);
Core.FilenameFilter filter = m_options.Filter;
if (backend == null)
throw new Exception("Unable to find backend for target: " + m_backend);
backend = new BackendWrapper(rs, backend, m_options);
if (OperationStarted != null)
OperationStarted(this, DuplicatiOperation.List, 0, -1, "Started");
BackupEntry bestFit = FindBestMatch(ParseFileList(backend), m_options.RestoreTime);
if (bestFit.EncryptionMode != null)
backend = EncryptedBackendWrapper.WrapWithEncryption(backend, bestFit.EncryptionMode, m_options);
List<string> res;
using (backend)
using (Core.TempFolder basefolder = new Duplicati.Library.Core.TempFolder())
{
List<BackupEntry> entries = new List<BackupEntry>();
entries.Add(bestFit);
entries.AddRange(bestFit.Incrementals);
List<Core.IFileArchive> patches = FindPatches(backend, entries, null, basefolder);
using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(basefolder, rs, filter, patches))
res = dir.UnmatchedFiles();
}
if (OperationCompleted != null)
OperationCompleted(this, DuplicatiOperation.List, 100, -1, "Completed");
return res;
}
private BackupEntry FindBestMatch(List<BackupEntry> backups, string specifictime)
{
if (string.IsNullOrEmpty(specifictime))
specifictime = "now";
return FindBestMatch(backups, Core.Timeparser.ParseTimeInterval(specifictime, DateTime.Now));
}
private BackupEntry FindBestMatch(List<BackupEntry> backups, DateTime timelimit)
{
if (backups.Count == 0)
throw new Exception("No backups found at remote location");
BackupEntry bestFit = backups[0];
List<BackupEntry> additions = new List<BackupEntry>();
foreach (BackupEntry be in backups)
if (be.Time < timelimit)
{
bestFit = be;
foreach (BackupEntry bex in be.Incrementals)
if (bex.Time <= timelimit)
additions.Add(bex);
}
if (bestFit.SignatureFile.Count == 0 || bestFit.ContentVolumes.Count == 0)
throw new Exception("Unable to parse filenames for the desired volumes");
bestFit.Incrementals = additions;
return bestFit;
}
private void SetupCommonOptions()
{
if (!string.IsNullOrEmpty(m_options.TempDir))
Core.TempFolder.SystemTempPath = m_options.TempDir;
if (!string.IsNullOrEmpty(m_options.ThreadPriority))
System.Threading.Thread.CurrentThread.Priority = Core.Utility.ParsePriority(m_options.ThreadPriority);
}
private List<Core.IFileArchive> FindPatches(Backend.IBackend backend, List<BackupEntry> entries, List<BackupEntry> orphans, string tempfolder)
{
List<Core.IFileArchive> patches = new List<Core.IFileArchive>();
using (new Logging.Timer("Reading incremental data"))
{
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, (int)(m_progress * 100), -1, "Reading incremental data ...");
//Calculate the total number of files to download
//, and verify their order
int incCount = 0;
foreach (BackupEntry be in entries)
{
int volNo = 0;
//Prevent order based bugs
if (entries.IndexOf(be) > 0)
if (entries[entries.IndexOf(be) - 1].Time >= be.Time)
throw new Exception("Bad sorting of backup times detected");
incCount++;
foreach (BackupEntry bes in be.SignatureFile)
{
incCount++;
if (volNo + 1 != bes.VolumeNumber)
throw new Exception("Bad sort order on volumes detected");
volNo++;
}
}
//The incremental part has a fixed cost, and each file has a fixed fraction of that
double unitCost = m_incrementalFraction / incCount;
foreach (BackupEntry be in entries)
{
m_progress += unitCost;
List<string> signatureHashes = null;
if (!m_options.SkipFileHashChecks)
{
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, (int)(m_progress * 100), -1, "Reading manifest file: " + be.Time.ToShortDateString() + " " + be.Time.ToShortTimeString());
signatureHashes = new List<string>();
using (new Logging.Timer("Get " + be.Filename))
using (Core.TempFile tf = new Duplicati.Library.Core.TempFile())
{
backend.Get(be.Filename, tf);
System.Xml.XmlDocument doc = new System.Xml.XmlDocument();
doc.Load(tf);
foreach (System.Xml.XmlNode n in doc.SelectNodes("Manifest/SignatureFiles/Hash"))
signatureHashes.Add(n.InnerText);
}
}
foreach (BackupEntry bes in be.SignatureFile)
{
m_progress += unitCost;
//Skip non-listed incrementals
if (signatureHashes != null && bes.VolumeNumber > signatureHashes.Count)
{
foreach (BackupEntry bec in be.ContentVolumes)
if (bec.VolumeNumber == bes.VolumeNumber)
if (orphans != null) //Auto-cleanup specified
orphans.Add(bes);
else
Logging.Log.WriteMessage("Found a partial file, run cleanup to remove: " + bes.Filename, Duplicati.Library.Logging.LogMessageType.Warning);
if (orphans != null) //Auto-cleanup specified
orphans.Add(bes);
else
Logging.Log.WriteMessage("Found a partial file, run cleanup to remove: " + bes.Filename, Duplicati.Library.Logging.LogMessageType.Warning);
continue;
}
if (OperationProgress != null)
OperationProgress(this, DuplicatiOperation.Backup, (int)(m_progress * 100), -1, "Reading signatures: " + bes.Time.ToShortDateString() + " " + bes.Time.ToShortTimeString() + ", vol " + bes.VolumeNumber.ToString());
string filename = System.IO.Path.Combine(tempfolder, "patch-" + patches.Count.ToString() + ".zip");
if (System.IO.File.Exists(filename))
System.IO.File.Delete(filename);
string cachefilename = null;
if (!string.IsNullOrEmpty(m_options.SignatureCachePath))
cachefilename = System.IO.Path.Combine(m_options.SignatureCachePath, m_cacheFilenameStrategy.GenerateFilename(bes.Type, bes.IsFull, bes.Time, bes.VolumeNumber));
if (cachefilename != null && System.IO.File.Exists(cachefilename))
if (signatureHashes != null && Core.Utility.CalculateHash(cachefilename) == signatureHashes[bes.VolumeNumber - 1])
System.IO.File.Copy(cachefilename, filename); //TODO: Warn on hash mismatch?
if (!System.IO.File.Exists(filename))
using (new Logging.Timer("Get " + bes.Filename))
{
backend.Get(bes.Filename, filename);
string hash = Core.Utility.CalculateHash(filename);
if (signatureHashes != null && hash != signatureHashes[bes.VolumeNumber - 1])
throw new Exception("Hash mismatch on file " + bes.Filename + " recorded hash: " + signatureHashes[bes.VolumeNumber - 1] + ", actual hash: " + hash);
//In case the cache was erased, or corrupt, keep the fresh copy
if (cachefilename != null)
System.IO.File.Copy(filename, cachefilename);
}
patches.Add(new Compression.FileArchiveZip(filename));
}
}
}
if (orphans != null && orphans.Count > 0)
DeleteOrphans(backend, orphans);
return patches;
}
public void PurgeSignatureCache()
{
if (string.IsNullOrEmpty(m_options.SignatureCachePath))
throw new Exception("Signature cache path was not given as an argument");
else
System.IO.Directory.Delete(m_options.SignatureCachePath, true);
}
//Static helpers
public static string[] List(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.List();
}
public static string Backup(string source, string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.Backup(source);
}
public static string Restore(string source, string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(source, options))
return i.Restore(target);
}
public static List<BackupEntry> ParseFileList(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.ParseFileList();
}
public static IList<string> ListContent(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.ListContent();
}
public static string RemoveAllButNFull(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.RemoveAllButNFull();
}
public static string RemoveOlderThan(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.RemoveOlderThan();
}
public static string Cleanup(string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(target, options))
return i.Cleanup();
}
public static string RestoreControlFiles(string source, string target, Dictionary<string, string> options)
{
using (Interface i = new Interface(source, options))
return i.RestoreControlFiles(target);
}
public static void PurgeSignatureCache(Dictionary<string, string> options)
{
Options opt = new Options(options);
if (string.IsNullOrEmpty(opt.SignatureCachePath))
throw new Exception("Signature cache path was not given as an argument");
else
System.IO.Directory.Delete(opt.SignatureCachePath, true);
}
#region IDisposable Members
public void Dispose()
{
}
#endregion
}
}