#region Disclaimer / License
// Copyright (C) 2011, Kenneth Skovhede
// http://www.hexad.dk, opensource@hexad.dk
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
//
#endregion
using System;
using System.Collections.Generic;
using System.Text;
using Duplicati.Library.Utility;
namespace Duplicati.Library.Main
{
///
/// The operations that Duplicati can report
///
public enum DuplicatiOperation
{
///
/// Indicates that the operations is a full or incremental backup
///
Backup,
///
/// Indicates that the operation is a restore type operation
///
Restore,
///
/// Indicates that the operation is a list type operation
///
List,
///
/// Indicates that the operation is a delete type operation
///
Remove
}
///
/// The actual operation that Duplicati supports
///
public enum DuplicatiOperationMode
{
///
/// A backup operation, either full or incremental
///
Backup,
///
/// A full backup
///
BackupFull,
///
/// An incremental backup
///
BackupIncremental,
///
/// A restore operation
///
Restore,
///
/// A restore operation for control files
///
RestoreControlfiles,
///
/// A backup file listing
///
List,
///
/// A list of backup chains found on the backend
///
GetBackupSets,
///
/// A list of the source folders found in a specific backup set
///
ListSourceFolders,
///
/// A list of files found in a specific backup set, only shows content from the single backup set and not the entire chain
///
ListActualSignatureFiles,
///
/// A delete operation performed by looking at the number of existing full backups
///
DeleteAllButNFull,
///
/// A delete operation performed by looking at the number of existing backups
///
DeleteAllButN,
///
/// A delete operation performed by looking at the age of existing backups
///
DeleteOlderThan,
///
/// A cleanup operation that removes orphan files
///
CleanUp,
///
/// A request to create the underlying folder
///
CreateFolder,
///
/// A search for files in signature files
///
FindLastFileVersion,
///
/// Verifies the hashes and backup chain
///
Verify,
///
/// Creates a log of the database suitable for attaching to a bug report
///
CreateLogDb
}
///
/// An enum that describes the level of verification done by the verify command
///
public enum VerificationLevel
{
///
/// Just verify the manifest chain
///
Manifest,
///
/// Verify the manifest chain and all signature files
///
Signature,
///
/// Verify everything, including the content files, requires download of all files
///
Full,
}
///
/// A delegate for reporting progress from within the Duplicati module
///
/// The instance that is running
/// The overall operation type
/// A more specific type of operation
/// The current overall progress of the operation
/// The progress of a transfer
/// A message describing the current operation
/// A message describing the current transfer operation
public delegate void OperationProgressEvent(Interface caller, DuplicatiOperation operation, DuplicatiOperationMode specificoperation, int progress, int subprogress, string message, string submessage);
///
/// A delegate used to report metadata about the current backup, obtained while running a normal operation
///
/// A table with various metadata informations
public delegate void MetadataReportDelegate(IDictionary metadata);
public class Interface : IDisposable
{
///
/// The amount of progressbar allocated for reading incremental data
///
private const double INCREMENAL_COST = 0.10;
///
/// The amount of progressbar allocated for uploading async volumes
///
private const double ASYNC_RESERVED = 0.10;
///
/// The backend url
///
private string m_backend;
///
/// The parsed type-safe version of the commandline options
///
private Options m_options;
///
/// The restult of an operation, used to report via IGenericCallbackModule
///
private object m_result;
///
/// The amount of progressbar allocated for reading incremental data
///
private double m_incrementalFraction = INCREMENAL_COST;
///
/// The amount of progressbar allocated for uploading the remaining volumes in asynchronous mode
///
private double m_asyncReserved = 0.0;
///
/// The current overall progress without taking the reserved amounts into account
///
private double m_progress = 0.0;
///
/// The number of restore patches
///
private int m_restorePatches = 0;
///
/// Cache of the last progress message
///
private string m_lastProgressMessage = "";
///
/// A flag indicating if logging has been set, used to dispose the logging
///
private bool m_hasSetLogging = false;
///
/// A flag toggling if the upload progress is reported,
/// used to prevent showing the progress bar when performing
/// ansynchronous uploads
///
private bool m_allowUploadProgress = true;
///
/// When allowing the upload progress to be reported,
/// there can be a flicker because the upload progresses,
/// before the flag is disabled again. This value
/// is set to DateTime.Now.AddSeconds(1) before
/// entering a potentially blocking operation, which
/// allows the progress to be reported if the call blocks,
/// but prevents flicker from non-blocking calls.
///
private DateTime m_allowUploadProgressAfter = DateTime.Now;
public event OperationProgressEvent OperationStarted;
public event OperationProgressEvent OperationCompleted;
public event OperationProgressEvent OperationProgress;
public event OperationProgressEvent OperationError;
public event MetadataReportDelegate MetadataReport;
///
/// This gets called whenever execution of an operation is started or stopped; it currently handles the AllowSleep option
///
/// Flag indicating execution state
private void OperationRunning(bool isRunning)
{
if (m_options!=null && !m_options.AllowSleep && !Duplicati.Library.Utility.Utility.IsClientLinux)
try
{
Win32.SetThreadExecutionState(Win32.EXECUTION_STATE.ES_CONTINUOUS | (isRunning ? Win32.EXECUTION_STATE.ES_SYSTEM_REQUIRED : 0));
}
catch { } //TODO: Report this somehow
}
///
/// Returns the current overall operation mode based on the actual operation mode
///
/// The overall operation type
private DuplicatiOperation GetOperationType()
{
switch (m_options.MainAction)
{
case DuplicatiOperationMode.Backup:
case DuplicatiOperationMode.BackupFull:
case DuplicatiOperationMode.BackupIncremental:
return DuplicatiOperation.Backup;
case DuplicatiOperationMode.Restore:
case DuplicatiOperationMode.RestoreControlfiles:
return DuplicatiOperation.Restore;
case DuplicatiOperationMode.List:
case DuplicatiOperationMode.GetBackupSets:
case DuplicatiOperationMode.ListSourceFolders:
case DuplicatiOperationMode.ListActualSignatureFiles:
case DuplicatiOperationMode.FindLastFileVersion:
case DuplicatiOperationMode.Verify:
return DuplicatiOperation.List;
case DuplicatiOperationMode.DeleteAllButNFull:
case DuplicatiOperationMode.DeleteOlderThan:
case DuplicatiOperationMode.CleanUp:
return DuplicatiOperation.Remove;
default:
throw new Exception(string.Format(Strings.Interface.UnexpectedOperationTypeError, m_options.MainAction));
}
}
///
/// Constructs a new interface for performing backup and restore operations
///
/// The url for the backend to use
/// All required options
public Interface(string backend, Dictionary options)
{
m_backend = backend;
m_options = new Options(options);
OperationProgress += new OperationProgressEvent(Interface_OperationProgress);
}
///
/// Event handler for the OperationProgres, used to store the last status message
///
private void Interface_OperationProgress(Interface caller, DuplicatiOperation operation, DuplicatiOperationMode specificoperation, int progress, int subprogress, string message, string submessage)
{
m_lastProgressMessage = message;
}
public string Backup(string[] sources)
{
BackupStatistics bs = new BackupStatistics(DuplicatiOperationMode.Backup);
SetupCommonOptions(bs, ref sources);
using (new Logging.Timer("Backup from " + string.Join(";", sources) + " to " + m_backend))
{
var fhopts = new ForestHash.FhOptions(m_options.RawOptions);
if (string.IsNullOrEmpty(fhopts.Fhdbpath))
throw new Exception(string.Format(Strings.Interface.MissingDatabasepathError, "fh-dbpath"));
if (sources == null || sources.Length == 0)
throw new Exception(Strings.Interface.NoSourceFoldersError);
//Make sure they all have the same format and exist
for (int i = 0; i < sources.Length; i++)
{
sources[i] = Utility.Utility.AppendDirSeparator(System.IO.Path.GetFullPath(sources[i]));
if (!System.IO.Directory.Exists(sources[i]))
throw new System.IO.IOException(String.Format(Strings.Interface.SourceFolderIsMissingError, sources[i]));
}
//Sanity check for duplicate folders and multiple inclusions of the same folder
for (int i = 0; i < sources.Length - 1; i++)
{
for (int j = i + 1; j < sources.Length; j++)
if (sources[i].Equals(sources[j], Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase))
throw new Exception(string.Format(Strings.Interface.SourceDirIsIncludedMultipleTimesError, sources[i]));
else if (sources[i].StartsWith(sources[j], Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase))
throw new Exception(string.Format(Strings.Interface.SourceDirsAreRelatedError, sources[i], sources[j]));
}
ForestHash.ForestHash.Backup(m_backend, fhopts, bs, sources);
return bs.ToString();
}
}
public string Restore(string[] target)
{
if (target == null || target.Length != 1)
throw new Exception("Cannot specify more than a single target folder");
var rs = new RestoreStatistics(DuplicatiOperationMode.Restore);
SetupCommonOptions(rs, ref target);
var fhopts = new ForestHash.FhOptions(m_options.RawOptions);
ForestHash.ForestHash.Restore(m_backend, fhopts, rs, target[0]);
return rs.ToString();
}
public string RestoreControlFiles(string target)
{
var rs = new RestoreStatistics(DuplicatiOperationMode.RestoreControlfiles);
SetupCommonOptions(rs);
var fhopts = new ForestHash.FhOptions(m_options.RawOptions);
using (var handler = new ForestHash.Operation.RestoreControlFilesHandler(m_backend, fhopts, rs, target))
handler.Run();
return rs.ToString();
}
public string DeleteAllButN()
{
var rs = new RestoreStatistics(DuplicatiOperationMode.DeleteAllButN);
SetupCommonOptions(rs);
var fhopts = new ForestHash.FhOptions(m_options.RawOptions);
using (var handler = new ForestHash.Operation.DeleteHandler(m_backend, fhopts, rs))
handler.Run();
return rs.ToString();
}
public string Repair()
{
var rs = new RestoreStatistics(DuplicatiOperationMode.CleanUp);
SetupCommonOptions(rs);
var fhopts = new ForestHash.FhOptions(m_options.RawOptions);
using (var handler = new ForestHash.Operation.RepairHandler(m_backend, fhopts, rs))
handler.Run();
return rs.ToString();
}
public List List()
{
var rs = new RestoreStatistics(DuplicatiOperationMode.List);
SetupCommonOptions(rs);
var fhopts = new ForestHash.FhOptions(m_options.RawOptions);
using (var handler = new ForestHash.Operation.ListFilesHandler(m_backend, fhopts, rs))
return handler.Run();
}
private void SetupCommonOptions(CommunicationStatistics stats)
{
string[] tmp = null;
SetupCommonOptions(stats, ref tmp);
}
private void SetupCommonOptions(CommunicationStatistics stats, ref string[] paths)
{
m_options.MainAction = stats.OperationMode;
switch (m_options.MainAction)
{
case DuplicatiOperationMode.Backup:
case DuplicatiOperationMode.BackupFull:
case DuplicatiOperationMode.BackupIncremental:
break;
default:
//It only makes sense to enable auto-creation if we are writing files.
if (!m_options.RawOptions.ContainsKey("disable-autocreate-folder"))
m_options.RawOptions["disable-autocreate-folder"] = "true";
break;
}
//Load all generic modules
m_options.LoadedModules.Clear();
foreach (Library.Interface.IGenericModule m in DynamicLoader.GenericLoader.Modules)
m_options.LoadedModules.Add(new KeyValuePair(Array.IndexOf(m_options.DisableModules, m.Key.ToLower()) < 0 && (m.LoadAsDefault || Array.IndexOf(m_options.EnableModules, m.Key.ToLower()) >= 0), m));
foreach (KeyValuePair mx in m_options.LoadedModules)
if (mx.Key)
{
mx.Value.Configure(m_options.RawOptions);
if (mx.Value is Library.Interface.IGenericCallbackModule)
((Library.Interface.IGenericCallbackModule)mx.Value).OnStart(stats.OperationMode.ToString(), ref m_backend, ref paths);
}
OperationRunning(true);
Library.Logging.Log.LogLevel = m_options.Loglevel;
if (!string.IsNullOrEmpty(m_options.Logfile))
{
m_hasSetLogging = true;
var path = System.IO.Path.GetDirectoryName(System.IO.Path.GetFullPath(m_options.Logfile));
if (!System.IO.Directory.Exists(path))
System.IO.Directory.CreateDirectory(path);
Library.Logging.Log.CurrentLog = new Library.Logging.StreamLog(m_options.Logfile);
}
if (stats != null)
{
stats.VerboseErrors = m_options.DebugOutput;
stats.VerboseRetryErrors = m_options.VerboseRetryErrors;
stats.QuietConsole = m_options.QuietConsole;
}
if (m_options.HasTempDir)
Utility.TempFolder.SystemTempPath = m_options.TempDir;
if (!string.IsNullOrEmpty(m_options.ThreadPriority))
System.Threading.Thread.CurrentThread.Priority = Utility.Utility.ParsePriority(m_options.ThreadPriority);
ValidateOptions(stats);
Library.Logging.Log.WriteMessage(string.Format(Strings.Interface.StartingOperationMessage, m_options.MainAction), Logging.LogMessageType.Information);
}
public List> FindLastFileVersion()
{
var rs = new RestoreStatistics(DuplicatiOperationMode.FindLastFileVersion);
SetupCommonOptions(rs);
var fhopts = new ForestHash.FhOptions(m_options.RawOptions);
using (var handler = new ForestHash.Operation.FindLastFileVersionHandler(m_backend, fhopts, rs))
return handler.Run();
}
///
/// This function will examine all options passed on the commandline, and test for unsupported or deprecated values.
/// Any errors will be logged into the statistics module.
///
/// The commandline options given
/// The backend url
/// The statistics into which warnings are written
private void ValidateOptions(CommunicationStatistics stats)
{
//No point in going through with this if we can't report
if (stats == null)
return;
//Keep a list of all supplied options
Dictionary ropts = new Dictionary(m_options.RawOptions);
//Keep a list of all supported options
Dictionary supportedOptions = new Dictionary();
//There are a few internal options that are not accessible from outside, and thus not listed
foreach (string s in Options.InternalOptions)
supportedOptions[s] = null;
//Figure out what module options are supported in the current setup
List moduleOptions = new List();
Dictionary disabledModuleOptions = new Dictionary();
foreach (KeyValuePair m in m_options.LoadedModules)
if (m.Value.SupportedCommands != null)
if (m.Key)
moduleOptions.AddRange(m.Value.SupportedCommands);
else
{
foreach (Library.Interface.ICommandLineArgument c in m.Value.SupportedCommands)
{
disabledModuleOptions[c.Name] = m.Value.DisplayName + " (" + m.Value.Key + ")";
if (c.Aliases != null)
foreach (string s in c.Aliases)
disabledModuleOptions[s] = disabledModuleOptions[c.Name];
}
}
// Throw url-encoded options into the mix
//TODO: This can hide values if both commandline and url-parameters supply the same key
var ext = new Utility.Uri(m_backend).QueryParameters;
foreach(var k in ext.AllKeys)
ropts[k] = ext[k];
//Now run through all supported options, and look for deprecated options
foreach (IList l in new IList[] {
m_options.SupportedCommands,
DynamicLoader.BackendLoader.GetSupportedCommands(m_backend),
m_options.NoEncryption ? null : DynamicLoader.EncryptionLoader.GetSupportedCommands(m_options.EncryptionModule),
moduleOptions,
DynamicLoader.CompressionLoader.GetSupportedCommands(m_options.CompressionModule) })
{
if (l != null)
foreach (Library.Interface.ICommandLineArgument a in l)
{
if (supportedOptions.ContainsKey(a.Name) && Array.IndexOf(Options.KnownDuplicates, a.Name.ToLower()) < 0)
stats.LogWarning(string.Format(Strings.Interface.DuplicateOptionNameWarning, a.Name), null);
supportedOptions[a.Name] = a;
if (a.Aliases != null)
foreach (string s in a.Aliases)
{
if (supportedOptions.ContainsKey(s) && Array.IndexOf(Options.KnownDuplicates, s.ToLower()) < 0)
stats.LogWarning(string.Format(Strings.Interface.DuplicateOptionNameWarning, s), null);
supportedOptions[s] = a;
}
if (a.Deprecated)
{
List aliases = new List();
aliases.Add(a.Name);
if (a.Aliases != null)
aliases.AddRange(a.Aliases);
foreach (string s in aliases)
if (ropts.ContainsKey(s))
{
string optname = a.Name;
if (a.Name != s)
optname += " (" + s + ")";
stats.LogWarning(string.Format(Strings.Interface.DeprecatedOptionUsedWarning, optname, a.DeprecationMessage), null);
}
}
}
}
//Now look for options that were supplied but not supported
foreach (string s in ropts.Keys)
if (!supportedOptions.ContainsKey(s))
if (disabledModuleOptions.ContainsKey(s))
stats.LogWarning(string.Format(Strings.Interface.UnsupportedOptionDisabledModuleWarning, s, disabledModuleOptions[s]), null);
else
stats.LogWarning(string.Format(Strings.Interface.UnsupportedOptionWarning, s), null);
//Look at the value supplied for each argument and see if is valid according to its type
foreach (string s in ropts.Keys)
{
Library.Interface.ICommandLineArgument arg;
if (supportedOptions.TryGetValue(s, out arg) && arg != null)
{
string validationMessage = ValidateOptionValue(arg, s, ropts[s]);
if (validationMessage != null)
stats.LogWarning(validationMessage, null);
}
}
//TODO: Based on the action, see if all options are relevant
}
#region Static interface
///
/// Checks if the value passed to an option is actually valid.
///
/// The argument being validated
/// The name of the option to validate
/// The value to check
/// Null if no errors are found, an error message otherwise
public static string ValidateOptionValue(Library.Interface.ICommandLineArgument arg, string optionname, string value)
{
if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Enumeration)
{
bool found = false;
foreach (string v in arg.ValidValues ?? new string[0])
if (string.Equals(v, value, StringComparison.CurrentCultureIgnoreCase))
{
found = true;
break;
}
if (!found)
return string.Format(Strings.Interface.UnsupportedEnumerationValue, optionname, value, string.Join(", ", arg.ValidValues ?? new string[0]));
}
else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean)
{
if (!string.IsNullOrEmpty(value) && Utility.Utility.ParseBool(value, true) != Utility.Utility.ParseBool(value, false))
return string.Format(Strings.Interface.UnsupportedBooleanValue, optionname, value);
}
else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Integer)
{
long l;
if (!long.TryParse(value, out l))
return string.Format(Strings.Interface.UnsupportedIntegerValue, optionname, value);
}
else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path)
{
foreach (string p in value.Split(System.IO.Path.DirectorySeparatorChar))
if (p.IndexOfAny(System.IO.Path.GetInvalidPathChars()) >= 0)
return string.Format(Strings.Interface.UnsupportedPathValue, optionname, p);
}
else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Size)
{
try
{
Utility.Sizeparser.ParseSize(value);
}
catch
{
return string.Format(Strings.Interface.UnsupportedSizeValue, optionname, value);
}
}
else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan)
{
try
{
Utility.Timeparser.ParseTimeSpan(value);
}
catch
{
return string.Format(Strings.Interface.UnsupportedTimeValue, optionname, value);
}
}
return null;
}
public static string Backup(string[] source, string target, Dictionary options)
{
using (Interface i = new Interface(target, options))
return i.Backup(source);
}
public static string Restore(string source, string[] target, Dictionary options)
{
using (Interface i = new Interface(source, options))
return i.Restore(target);
}
public static IList List(string target, Dictionary options)
{
using (Interface i = new Interface(target, options))
return i.List();
}
public static string DeleteAllButN(string target, Dictionary options)
{
using (Interface i = new Interface(target, options))
return i.DeleteAllButN();
}
public static string DeleteOlderThan(string target, Dictionary options)
{
using (Interface i = new Interface(target, options))
return i.DeleteAllButN();
}
public static string Repair(string target, Dictionary options)
{
using (Interface i = new Interface(target, options))
return i.Repair();
}
public static string RestoreControlFiles(string source, string target, Dictionary options)
{
using (Interface i = new Interface(source, options))
return i.RestoreControlFiles(target);
}
public static List> FindLastFileVersion(string target, Dictionary options)
{
using (Interface i = new Interface(target, options))
return i.FindLastFileVersion();
}
#endregion
#region IDisposable Members
public void Dispose()
{
if (m_options != null && m_options.LoadedModules != null)
{
foreach (KeyValuePair mx in m_options.LoadedModules)
if (mx.Key && mx.Value is Duplicati.Library.Interface.IGenericCallbackModule)
try { ((Duplicati.Library.Interface.IGenericCallbackModule)mx.Value).OnFinish(m_result); }
catch (Exception ex) { Logging.Log.WriteMessage(string.Format("OnFinish callback {0} failed: {1}", mx.Key, ex.Message), Duplicati.Library.Logging.LogMessageType.Warning, ex); }
foreach (KeyValuePair mx in m_options.LoadedModules)
if (mx.Key && mx.Value is IDisposable)
try { ((IDisposable)mx.Value).Dispose(); }
catch (Exception ex) { Logging.Log.WriteMessage(string.Format("Dispose for {0} failed: {1}", mx.Key, ex.Message), Duplicati.Library.Logging.LogMessageType.Warning, ex); }
m_options.LoadedModules.Clear();
OperationRunning(false);
}
if (m_hasSetLogging && Logging.Log.CurrentLog is Logging.StreamLog)
{
Logging.StreamLog sl = (Logging.StreamLog)Logging.Log.CurrentLog;
Logging.Log.CurrentLog = null;
sl.Dispose();
m_hasSetLogging = false;
}
}
#endregion
public static IEnumerable ParseFhFileList(string target, Dictionary options, CommunicationStatistics stat)
{
stat = stat ?? new CommunicationStatistics(DuplicatiOperationMode.List);
using(var i = new Interface(target, options))
{
i.SetupCommonOptions(stat);
return ForestHash.ForestHash.ParseFileList(target, options, stat);
}
}
public static string CompactBlocks(string target, Dictionary options, CommunicationStatistics stat)
{
stat = stat ?? new CommunicationStatistics(DuplicatiOperationMode.CleanUp);
using(var i = new Interface(target, options))
{
i.SetupCommonOptions(stat);
return ForestHash.ForestHash.CompactBlocks(target, options, stat);
}
}
public static string RecreateDatabase(string target, Dictionary options, CommunicationStatistics stat)
{
stat = stat ?? new CommunicationStatistics(DuplicatiOperationMode.CleanUp);
using(var i = new Interface(target, options))
{
i.SetupCommonOptions(stat);
return ForestHash.ForestHash.RecreateDatabase(target, options, stat);
}
}
public static string DeleteFilesets(string target, string filesets, Dictionary options, CommunicationStatistics stat)
{
stat = stat ?? new CommunicationStatistics(DuplicatiOperationMode.DeleteAllButN);
using(var i = new Interface(target, options))
{
i.SetupCommonOptions(stat);
return ForestHash.ForestHash.DeleteFilesets(target, filesets, options, stat);
}
}
public static string CreateLogDatabase(string target, Dictionary options, CommunicationStatistics stat)
{
stat = stat ?? new CommunicationStatistics(DuplicatiOperationMode.CreateLogDb);
using(var i = new Interface(target, options))
{
i.SetupCommonOptions(stat);
return ForestHash.ForestHash.CreateLogDatabase(target, options, stat);
}
}
}
}