#region Disclaimer / License // Copyright (C) 2011, Kenneth Skovhede // http://www.hexad.dk, opensource@hexad.dk // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public // License as published by the Free Software Foundation; either // version 2.1 of the License, or (at your option) any later version. // // This library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU // Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // #endregion using System; using System.Collections.Generic; using System.Text; using Duplicati.Library.Utility; namespace Duplicati.Library.Main { /// /// The operations that Duplicati can report /// public enum DuplicatiOperation { /// /// Indicates that the operations is a full or incremental backup /// Backup, /// /// Indicates that the operation is a restore type operation /// Restore, /// /// Indicates that the operation is a list type operation /// List, /// /// Indicates that the operation is a delete type operation /// Remove } /// /// The actual operation that Duplicati supports /// public enum DuplicatiOperationMode { /// /// A backup operation, either full or incremental /// Backup, /// /// A full backup /// BackupFull, /// /// An incremental backup /// BackupIncremental, /// /// A restore operation /// Restore, /// /// A restore operation for control files /// RestoreControlfiles, /// /// A backend file listing /// List, /// /// A list of backup chains found on the backend /// GetBackupSets, /// /// A list of files found in a specific backup set, produced by summing through incremental signature files /// ListCurrentFiles, /// /// A list of the source folders found in a specific backup set /// ListSourceFolders, /// /// A list of files found in a specific backup set, only shows content from the single backup set and not the entire chain /// ListActualSignatureFiles, /// /// A delete operation performed by looking at the number of existing full backups /// DeleteAllButNFull, /// /// A delete operation performed by looking at the number of existing backups /// DeleteAllButN, /// /// A delete operation performed by looking at the age of existing backups /// DeleteOlderThan, /// /// A cleanup operation that removes orphan files /// CleanUp, /// /// A request to create the underlying folder /// CreateFolder, /// /// A search for files in signature files /// FindLastFileVersion, /// /// Verifies the hashes and backup chain /// Verify } /// /// An enum that describes the level of verification done by the verify command /// public enum VerificationLevel { /// /// Just verify the manifest chain /// Manifest, /// /// Verify the manifest chain and all signature files /// Signature, /// /// Verify everything, including the content files, requires download of all files /// Full, } /// /// A delegate for reporting progress from within the Duplicati module /// /// The instance that is running /// The overall operation type /// A more specific type of operation /// The current overall progress of the operation /// The progress of a transfer /// A message describing the current operation /// A message describing the current transfer operation public delegate void OperationProgressEvent(Interface caller, DuplicatiOperation operation, DuplicatiOperationMode specificoperation, int progress, int subprogress, string message, string submessage); public class Interface : IDisposable, LiveControl.ILiveControl { /// /// The amount of progressbar allocated for reading incremental data /// private const double INCREMENAL_COST = 0.10; /// /// The amount of progressbar allocated for uploading async volumes /// private const double ASYNC_RESERVED = 0.10; private string m_backend; private Options m_options; /// /// The amount of progressbar allocated for reading incremental data /// private double m_incrementalFraction = INCREMENAL_COST; /// /// The amount of progressbar allocated for uploading the remaining volumes in asynchronous mode /// private double m_asyncReserved = 0.0; /// /// The current overall progress without taking the reserved amounts into account /// private double m_progress = 0.0; /// /// The number of restore patches /// private int m_restorePatches = 0; /// /// Cache of the last progress message /// private string m_lastProgressMessage = ""; /// /// A flag indicating if logging has been set, used to dispose the logging /// private bool m_hasSetLogging = false; /// /// A flag toggling if the upload progress is reported, /// used to prevent showing the progress bar when performing /// ansynchronous uploads /// private bool m_allowUploadProgress = true; /// /// When allowing the upload progress to be reported, /// there can be a flicker because the upload progresses, /// before the flag is disabled again. This value /// is set to DateTime.Now.AddSeconds(1) before /// entering a potentially blocking operation, which /// allows the progress to be reported if the call blocks, /// but prevents flicker from non-blocking calls. /// private DateTime m_allowUploadProgressAfter = DateTime.Now; public event OperationProgressEvent OperationStarted; public event OperationProgressEvent OperationCompleted; public event OperationProgressEvent OperationProgress; public event OperationProgressEvent OperationError; /// /// The live control interface /// private LiveControl.LiveControl m_liveControl; /// /// This gets called whenever execution of an operation is started or stopped; it currently handles the AllowSleep option /// /// Flag indicating execution state private void OperationRunning(bool isRunning) { if (m_options!=null && !m_options.AllowSleep && !Duplicati.Library.Utility.Utility.IsClientLinux) try { Win32.SetThreadExecutionState(Win32.EXECUTION_STATE.ES_CONTINUOUS | (isRunning ? Win32.EXECUTION_STATE.ES_SYSTEM_REQUIRED : 0)); } catch { } //TODO: Report this somehow } /// /// Returns the current overall operation mode based on the actual operation mode /// /// The overall operation type private DuplicatiOperation GetOperationType() { switch (m_options.MainAction) { case DuplicatiOperationMode.Backup: case DuplicatiOperationMode.BackupFull: case DuplicatiOperationMode.BackupIncremental: return DuplicatiOperation.Backup; case DuplicatiOperationMode.Restore: case DuplicatiOperationMode.RestoreControlfiles: return DuplicatiOperation.Restore; case DuplicatiOperationMode.List: case DuplicatiOperationMode.GetBackupSets: case DuplicatiOperationMode.ListCurrentFiles: case DuplicatiOperationMode.ListSourceFolders: case DuplicatiOperationMode.ListActualSignatureFiles: case DuplicatiOperationMode.FindLastFileVersion: case DuplicatiOperationMode.Verify: return DuplicatiOperation.List; case DuplicatiOperationMode.DeleteAllButNFull: case DuplicatiOperationMode.DeleteOlderThan: case DuplicatiOperationMode.CleanUp: return DuplicatiOperation.Remove; default: throw new Exception(string.Format(Strings.Interface.UnexpectedOperationTypeError, m_options.MainAction)); } } /// /// Constructs a new interface for performing backup and restore operations /// /// The url for the backend to use /// All required options public Interface(string backend, Dictionary options) { m_backend = backend; m_options = new Options(options); m_liveControl = new LiveControl.LiveControl(System.Threading.Thread.CurrentThread, m_options); OperationProgress += new OperationProgressEvent(Interface_OperationProgress); } /// /// Event handler for the OperationProgres, used to store the last status message /// private void Interface_OperationProgress(Interface caller, DuplicatiOperation operation, DuplicatiOperationMode specificoperation, int progress, int subprogress, string message, string submessage) { m_lastProgressMessage = message; } /// /// Internal helper to pause and stop when requested /// private void CheckLiveControl() { if (m_liveControl.IsPauseRequested) { OperationRunning(false); try { m_liveControl.PauseIfRequested(); } finally { OperationRunning(true); } } if (m_liveControl.IsStopRequested) throw new LiveControl.ExecutionStoppedException(); } public string Backup(string[] sources) { BackupStatistics bs = new BackupStatistics(DuplicatiOperationMode.Backup); SetupCommonOptions(bs); BackendWrapper backend = null; VerificationFile verification = null; if (m_options.DontReadManifests) throw new Exception(Strings.Interface.ManifestsMustBeReadOnBackups); if (m_options.SkipFileHashChecks) throw new Exception(Strings.Interface.CannotSkipHashChecksOnBackup); if (sources == null || sources.Length == 0) throw new Exception(Strings.Interface.NoSourceFoldersError); //Make sure they all have the same format and exist for (int i = 0; i < sources.Length; i++) { sources[i] = Utility.Utility.AppendDirSeparator(System.IO.Path.GetFullPath(sources[i])); if (!System.IO.Directory.Exists(sources[i])) throw new System.IO.IOException(String.Format(Strings.Interface.SourceFolderIsMissingError, sources[i])); } //Sanity check for duplicate folders and multiple inclusions of the same folder for (int i = 0; i < sources.Length - 1; i++) { for (int j = i + 1; j < sources.Length; j++) if (sources[i].Equals(sources[j], Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase)) throw new Exception(string.Format(Strings.Interface.SourceDirIsIncludedMultipleTimesError, sources[i])); else if (sources[i].StartsWith(sources[j], Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase)) throw new Exception(string.Format(Strings.Interface.SourceDirsAreRelatedError, sources[i], sources[j])); } if (m_options.AsynchronousUpload) { m_asyncReserved = ASYNC_RESERVED; m_allowUploadProgress = false; } //Unused, but triggers errors in the encryption setup here Library.Interface.IEncryption encryptionModule = m_options.NoEncryption ? null : DynamicLoader.EncryptionLoader.GetModule(m_options.EncryptionModule, m_options.Passphrase, m_options.RawOptions); using (new Logging.Timer("Backup from " + string.Join(";", sources) + " to " + m_backend)) { try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusLoadingFilelist, ""); OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusLoadingFilelist, ""); CheckLiveControl(); bool full = m_options.Full; if (full) bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseFlagWasSet, "full")); backend = new BackendWrapper(bs, m_backend, m_options); backend.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupTransfer_ProgressEvent); backend.AsyncItemProcessedEvent += new EventHandler(backend_AsyncItemProcessedEvent); m_progress = 0.0; OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, ""); CheckLiveControl(); List backupsets; if (full) { //This will create the target folder backend.List(false); backupsets = new List(); } else { //This will list all files on the backend and create the target folder backupsets = backend.GetBackupSets(); } if (backupsets.Count == 0) { if (!full) bs.SetTypeReason(Strings.Interface.FullBecauseBackendIsEmpty); full = true; } else { //A prioir backup exists, extract the compression and encryption modules used in the most recent entry string compression = null; string encryption = null; for (int i = backupsets.Count - 1; compression == null && i >= 0; i--) { for (int j = backupsets[i].Incrementals.Count - 1; compression == null && j >= 0; j--) for (int k = backupsets[i].Incrementals[j].Volumes.Count - 1; compression == null && k >= 0; k--) { compression = backupsets[i].Incrementals[j].Volumes[k].Key.Compression; encryption = backupsets[i].Incrementals[j].Volumes[k].Key.EncryptionMode; if (compression != null) break; } for (int k = backupsets[i].Volumes.Count - 1; compression == null && k >= 0; k--) { compression = backupsets[i].Volumes[k].Key.Compression; encryption = backupsets[i].Volumes[k].Key.EncryptionMode; if (compression != null) break; } } if (compression != null) { m_options.SetEncryptionModuleDefault(encryption); m_options.SetCompressionModuleDefault(compression); } } string fullCriteria1 = null; string fullCriteria2 = null; if (!full) { full = DateTime.Now > m_options.FullIfOlderThan(backupsets[backupsets.Count - 1].Time); if (full) bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseLastFullIsFrom, backupsets[backupsets.Count - 1].Time, m_options.FullIfOlderThanValue)); else if (!string.IsNullOrEmpty(m_options.FullIfOlderThanValue)) fullCriteria1 = string.Format(Strings.Interface.IncrementalBecauseLastFullIsFrom, backupsets[backupsets.Count - 1].Time, m_options.FullIfOlderThanValue); } if (!full && m_options.FullIfMoreThanNIncrementals > 0) { full = backupsets[backupsets.Count - 1].Incrementals.Count >= m_options.FullIfMoreThanNIncrementals; if (full) bs.SetTypeReason(string.Format(Strings.Interface.FullBecauseThereAreNIncrementals, backupsets[backupsets.Count - 1].Incrementals.Count, m_options.FullIfMoreThanNIncrementals)); else fullCriteria2 = string.Format(Strings.Interface.IncrementalBecauseThereAreNIncrementals, backupsets[backupsets.Count - 1].Incrementals.Count, m_options.FullIfMoreThanNIncrementals); } bs.Full = full; if (!full) { if (fullCriteria1 == null && fullCriteria2 == null) bs.SetTypeReason(Strings.Interface.IncrementalBecauseNoFlagsWereSet); else if (fullCriteria2 == null) bs.SetTypeReason(fullCriteria1); else if (fullCriteria1 == null) bs.SetTypeReason(fullCriteria2); else bs.SetTypeReason(fullCriteria1 + ". " + fullCriteria2); } List controlfiles = new List(); if (!string.IsNullOrEmpty(m_options.SignatureControlFiles)) controlfiles.AddRange(m_options.SignatureControlFiles.Split(System.IO.Path.PathSeparator)); int vol = 0; long totalsize = 0; Manifestfile manifest = new Manifestfile(); using (Utility.TempFolder tempfolder = new Duplicati.Library.Utility.TempFolder()) { List> patches = new List>(); if (!full) { m_incrementalFraction = INCREMENAL_COST; List entries = new List(); entries.Add(backupsets[backupsets.Count - 1]); entries.AddRange(backupsets[backupsets.Count - 1].Incrementals); //Check before we start the download CheckLiveControl(); VerifyBackupChainWithFiles(backend, entries[entries.Count - 1]); if (m_options.CreateVerificationFile) verification = new VerificationFile(entries, backend.FilenameStrategy); OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, ""); patches = FindPatches(backend, entries, tempfolder, false, bs); //Check before we start the download CheckLiveControl(); Manifestfile latest = GetManifest(backend, backupsets[backupsets.Count - 1]); //Manifest version 1 does not support multiple folders if (latest.Version == 1) latest.SourceDirs = new string[] { sources[0] }; if (latest.SourceDirs.Length != sources.Length) { if (m_options.FullIfSourceFolderChanged) { Logging.Log.WriteMessage("Source folder count changed, issuing full backup", Duplicati.Library.Logging.LogMessageType.Information); if (!full) bs.SetTypeReason(Strings.Interface.FullBecauseSourceFoldersChanged); full = true; } else throw new Exception(string.Format(Strings.Interface.NumberOfSourceFoldersHasChangedError, latest.SourceDirs.Length, sources.Length)); } else { if (!m_options.AllowSourceFolderChange) { foreach (string s1 in latest.SourceDirs) { bool found = false; foreach (string s2 in sources) if (s1.Equals(s2, Utility.Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase)) { found = true; break; } if (!found) { if (m_options.FullIfSourceFolderChanged) { Logging.Log.WriteMessage("Source folders changed, issuing full backup", Duplicati.Library.Logging.LogMessageType.Information); if (!full) bs.SetTypeReason(Strings.Interface.FullBecauseSourceFoldersChanged); full = true; break; //Exit the folder loop } else throw new Exception(string.Format(Strings.Interface.SourceFoldersHasChangedError, s1)); } } manifest.SourceDirs = latest.SourceDirs; } else { manifest.SourceDirs = sources; } } } DateTime backuptime = DateTime.Now; DateTime backupchaintime; if (full) { patches.Clear(); m_incrementalFraction = 0.0; manifest.SourceDirs = sources; if (m_options.CreateVerificationFile) verification = new VerificationFile(new ManifestEntry[0], backend.FilenameStrategy); backupchaintime = backuptime; } else { backupchaintime = patches[0].Key.Time; manifest.PreviousManifestFilename = patches[patches.Count - 1].Key.Filename; manifest.PreviousManifestHash = patches[patches.Count - 1].Key.RemoteHash; } OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusBuildingFilelist, ""); bool completedWithoutChanges; using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(manifest.SourceDirs, bs, m_options.Filter, patches)) { CheckLiveControl(); dir.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupRSyncDir_ProgressEvent); dir.DisableFiletimeCheck = m_options.DisableFiletimeCheck; dir.MaxFileSize = m_options.SkipFilesLargerThan; using (new Logging.Timer("Initiating multipass")) dir.InitiateMultiPassDiff(full, m_options); string tempVolumeFolder = m_options.AsynchronousUpload ? m_options.AsynchronousUploadFolder : (m_options.TempDir ?? Utility.TempFolder.SystemTempPath); bool done = false; while (!done && totalsize < m_options.MaxSize) { using (new Logging.Timer("Multipass " + (vol + 1).ToString())) using (Utility.TempFile signaturefile = new Duplicati.Library.Utility.TempFile(System.IO.Path.Combine(tempVolumeFolder, Guid.NewGuid().ToString()))) using (Utility.TempFile contentfile = new Duplicati.Library.Utility.TempFile(System.IO.Path.Combine(tempVolumeFolder, Guid.NewGuid().ToString()))) { OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusCreatingVolume, vol + 1), ""); CheckLiveControl(); using (Library.Interface.ICompression signaturearchive = DynamicLoader.CompressionLoader.GetModule(m_options.CompressionModule, signaturefile, m_options.RawOptions)) using (Library.Interface.ICompression contentarchive = DynamicLoader.CompressionLoader.GetModule(m_options.CompressionModule, contentfile, m_options.RawOptions)) { //If we are all out, stop now, this may cause incomplete partial files if (m_options.MaxSize - totalsize < (contentarchive.FlushBufferSize + backend.FileSizeOverhead)) break; //Add signature files to archive foreach (string s in controlfiles) if (!string.IsNullOrEmpty(s)) using (System.IO.Stream cs = signaturearchive.CreateFile(System.IO.Path.Combine(RSync.RSyncDir.CONTROL_ROOT, System.IO.Path.GetFileName(s)))) using (System.IO.FileStream fs = System.IO.File.OpenRead(s)) Utility.Utility.CopyStream(fs, cs); //Only add control files to the very first volume controlfiles.Clear(); done = dir.MakeMultiPassDiff(signaturearchive, contentarchive, (Math.Min(m_options.VolumeSize, m_options.MaxSize - totalsize)) - backend.FileSizeOverhead); //TODO: This is not the correct size, we need to account for file size overhead as well totalsize += signaturearchive.Size; totalsize += contentarchive.Size; //TODO: This is not the best way to determine this if (totalsize >= m_options.MaxSize) dir.FinalizeMultiPass(signaturearchive, contentarchive, long.MaxValue); } completedWithoutChanges = done && !dir.AnyChangesFound; if (m_options.UploadUnchangedBackups || full) completedWithoutChanges = false; if (!completedWithoutChanges) { if (m_options.AsynchronousUpload) { m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload; m_allowUploadProgress = true; m_allowUploadProgressAfter = DateTime.Now.AddSeconds(1); } else OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingContentVolume, vol + 1), ""); //Last check before we upload, we do not interrupt transfers CheckLiveControl(); //The backendwrapper will remove these signaturefile.Protected = true; contentfile.Protected = true; ContentEntry ce = new ContentEntry(backuptime, full, vol + 1); SignatureEntry se = new SignatureEntry(backuptime, full, vol + 1); using (new Logging.Timer("Writing delta file " + (vol + 1).ToString())) backend.Put(ce, contentfile); if (!m_options.AsynchronousUpload) OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingSignatureVolume, vol + 1), ""); using (new Logging.Timer("Writing remote signatures")) backend.Put(se, signaturefile); manifest.AddEntries(ce, se); if (verification != null) { verification.AddFile(ce); verification.AddFile(se); } } } if (!completedWithoutChanges) { //The backend wrapper will remove these Utility.TempFile mf = new Duplicati.Library.Utility.TempFile(); using (new Logging.Timer("Writing manifest " + backuptime.ToUniversalTime().ToString("yyyyMMddTHHmmssK"))) { //Alternate primary/secondary ManifestEntry mfe = new ManifestEntry(backuptime, full, manifest.SignatureHashes.Count % 2 != 0); manifest.SelfFilename = backend.GenerateFilename(mfe); manifest.Save(mf); if (!m_options.AsynchronousUpload) OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusUploadingManifestVolume, vol + 1), ""); //Write the file mf.Protected = true; backend.Put(mfe, mf); if (verification != null) verification.UpdateManifest(mfe); } if (verification != null) { using (new Logging.Timer("Writing verification " + backuptime.ToUniversalTime().ToString("yyyyMMddTHHmmssK"))) { Utility.TempFile vt = new Duplicati.Library.Utility.TempFile(); verification.Save(vt); if (!m_options.AsynchronousUpload) OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusUploadingVerificationVolume, ""); vt.Protected = true; backend.Put(new VerificationEntry(backupchaintime), vt); } } if (m_options.AsynchronousUpload) m_allowUploadProgress = false; //The file volume counter vol++; } } } //If we are running asynchronous, we now enter the end-game if (m_options.AsynchronousUpload) { m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload; m_allowUploadProgress = true; m_allowUploadProgressAfter = DateTime.Now; //Before we clear the temp folder, we need to ensure that all volumes are uploaded. //To allow the UI to show some progress while uploading, we perform the remaining // uploads synchronous List> pendingUploads = backend.ExtractPendingUploads(); //Figure out what volume number we are at foreach (KeyValuePair p in pendingUploads) if (p.Key is ManifestEntry) vol--; double unitcost = m_asyncReserved / pendingUploads.Count; //The upload each remaining volume in order foreach (KeyValuePair p in pendingUploads) { string msg; if (p.Key is ManifestEntry) { vol++; msg = string.Format(Strings.Interface.StatusUploadingManifestVolume, vol); } else if (p.Key is SignatureEntry) msg = string.Format(Strings.Interface.StatusUploadingSignatureVolume, ((SignatureEntry)p.Key).Volumenumber); else if (p.Key is ContentEntry) { msg = string.Format(Strings.Interface.StatusUploadingContentVolume, ((ContentEntry)p.Key).Volumenumber); //We allow a stop or pause request here CheckLiveControl(); } else if (p.Key is VerificationEntry) msg = Strings.Interface.StatusUploadingVerificationVolume; else throw new InvalidOperationException(); OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, (int)(m_progress * 100), -1, msg, ""); backend.Put(p.Key, p.Value); m_asyncReserved -= unitcost; m_progress += unitcost; } } } } catch(Exception ex) { //If this is a controlled user-requested stop, wait for the current upload to complete if (backend != null && ex is LiveControl.ExecutionStoppedException) { try { if (m_options.AsynchronousUpload) { m_lastProgressMessage = Strings.Interface.StatusWaitingForUpload; m_allowUploadProgress = true; m_allowUploadProgressAfter = DateTime.Now; //Wait for the current upload to complete and then delete all remaining temporary files foreach (KeyValuePair p in backend.ExtractPendingUploads()) try { if (System.IO.File.Exists(p.Value)) System.IO.File.Delete(p.Value); } catch { } //Better to delete as many as possible rather than choke on a single file } } catch { } //We already have an exception, just go with that } if (backend == null || backend.ManifestUploads == 0) { Logging.Log.WriteMessage(string.Format(Strings.Interface.ErrorRunningBackup, ex.Message), Logging.LogMessageType.Error); throw; //This also activates "finally", unlike in other languages... } Logging.Log.WriteMessage(string.Format(Strings.Interface.PartialUploadMessage, backend.ManifestUploads, ex.Message), Logging.LogMessageType.Warning); bs.LogError(string.Format(Strings.Interface.PartialUploadMessage, backend.ManifestUploads, ex.Message), ex); } finally { m_progress = 100.0; if (backend != null) try { backend.Dispose(); } catch { } if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Backup, bs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } } bs.EndTime = DateTime.Now; return bs.ToString(); } /// /// Event handler that is activated when an asynchronous upload is about to being, /// used to pause uploads if the user has requested a pause /// /// Unused sender argument /// Unused event argument private void backend_AsyncItemProcessedEvent(object sender, EventArgs e) { m_liveControl.PauseIfRequested(); } /// /// Event handler for reporting upload/download progress /// /// The upload/download progress in percent /// The name of the file being transfered private void BackupTransfer_ProgressEvent(int progress, string filename) { if (m_allowUploadProgress && DateTime.Now > m_allowUploadProgressAfter) OperationProgress(this, GetOperationType(), m_options.MainAction, (int)(m_progress * 100), progress, m_lastProgressMessage, filename); } /// /// Event handler for reporting backup progress /// /// The total progress in percent /// The file currently being examined private void BackupRSyncDir_ProgressEvent(int progress, string filename) { m_progress = ((1.0 - m_incrementalFraction) * (progress / (double)100.0)) + m_incrementalFraction; m_progress *= (1.0 - m_asyncReserved); OperationProgress(this, GetOperationType(), m_options.MainAction, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusProcessing, filename), ""); CheckLiveControl(); } /// /// Event handler for reporting restore progress /// /// The total progress in percent /// The file currently being examined private void RestoreRSyncDir_ProgressEvent(int progress, string filename) { double pgPrPatch = ((1.0 - INCREMENAL_COST) / m_restorePatches); int fullProgress = (int)((m_progress * 100) + (pgPrPatch * (progress / 100.0))); OperationProgress(this, GetOperationType(), m_options.MainAction, fullProgress, -1, string.Format(Strings.Interface.StatusProcessing, filename), ""); CheckLiveControl(); } /// /// Validates the manifest chain, starting by evaluating the current manifest and going backwards in the chain /// /// The backend wrapper /// The entry to verify private void VerifyManifestChain(BackendWrapper backend, ManifestEntry entry) { if (m_options.DontReadManifests) throw new InvalidOperationException(Strings.Interface.CannotVerifyChain); while (entry.Previous != null) { Manifestfile parsed = GetManifest(backend, entry); //If this manifest is not version 3, the chain verification stops here if (parsed.Version < 3) return; ManifestEntry previous = entry.Previous; if (entry.Previous.Alternate != null && entry.Previous.Alternate.Filename == parsed.PreviousManifestFilename) previous = entry.Previous.Alternate; if (parsed.PreviousManifestFilename != previous.Filename) throw new System.IO.InvalidDataException(string.Format(Strings.Interface.PreviousManifestFilenameMismatchError, entry.Filename, parsed.PreviousManifestFilename, previous.Filename)); if (!m_options.SkipFileHashChecks) { //Load the Remotehash property GetManifest(backend, previous); if (parsed.PreviousManifestHash != previous.RemoteHash) throw new System.IO.InvalidDataException(string.Format(Strings.Interface.PreviousManifestHashMismatchError, entry.Filename, parsed.PreviousManifestHash, previous.RemoteHash)); } entry = previous; } } /// /// Verifies the backup chain for producing a new backup on top. /// This will check that all files are accounted for in the file list. /// /// The newest entry to check private void VerifyBackupChainWithFiles(BackendWrapper backend, ManifestEntry entry) { VerifyManifestChain(backend, entry); string errorMessage = Environment.NewLine + Strings.Interface.DeleteManifestsSuggestion + Environment.NewLine + Environment.NewLine; while (entry != null) { Manifestfile parsed = GetManifest(backend, entry); errorMessage += entry.Filename + Environment.NewLine; if (entry.Volumes.Count != parsed.SignatureHashes.Count || entry.Volumes.Count != parsed.ContentHashes.Count) { //If we have an extra set, the connection could have died right before the manifest was uploaded if (parsed.SignatureHashes.Count == parsed.ContentHashes.Count && entry.Volumes.Count - 1 == parsed.ContentHashes.Count) { backend.AddOrphan(entry.Volumes[entry.Volumes.Count - 1].Value); backend.AddOrphan(entry.Volumes[entry.Volumes.Count - 1].Key); entry.Volumes.RemoveAt(entry.Volumes.Count - 1); } else { throw new Exception( string.Format(Strings.Interface.ManifestAndFileCountMismatchError, entry.Filename, parsed.SignatureHashes.Count, entry.Volumes.Count) + errorMessage ); } } for(int i = 0; i < entry.Volumes.Count; i++) { if (entry.Volumes[i].Key.Filesize > 0 && parsed.SignatureHashes[i].Size > 0 && entry.Volumes[i].Key.Filesize != parsed.SignatureHashes[i].Size) throw new Exception( string.Format(Strings.Interface.FileSizeMismatchError, entry.Volumes[i].Key.Filename, entry.Volumes[i].Key.Filesize, parsed.SignatureHashes[i].Size) + errorMessage ); if (entry.Volumes[i].Value.Filesize >= 0 && parsed.ContentHashes[i].Size >= 0 && entry.Volumes[i].Value.Filesize != parsed.ContentHashes[i].Size) throw new Exception( string.Format(Strings.Interface.FileSizeMismatchError, entry.Volumes[i].Value.Filename, entry.Volumes[i].Value.Filesize, parsed.ContentHashes[i].Size) + errorMessage ); if (!string.IsNullOrEmpty(parsed.SignatureHashes[i].Name) && !parsed.SignatureHashes[i].Name.Equals(entry.Volumes[i].Key.Fileentry.Name, StringComparison.InvariantCultureIgnoreCase)) throw new Exception( string.Format(Strings.Interface.FilenameMismatchError, parsed.SignatureHashes[i].Name, entry.Volumes[i].Key.Fileentry.Name) + errorMessage ); if (!string.IsNullOrEmpty(parsed.ContentHashes[i].Name) && !parsed.ContentHashes[i].Name.Equals(entry.Volumes[i].Value.Fileentry.Name, StringComparison.InvariantCultureIgnoreCase)) throw new Exception( string.Format(Strings.Interface.FilenameMismatchError, parsed.ContentHashes[i].Name, entry.Volumes[i].Value.Fileentry.Name) + errorMessage ); } entry = entry.Previous; } } /// /// Will attempt to read the manifest file, optionally reverting to the secondary manifest if reading one fails. /// /// The backendwrapper to read from /// The manifest to read /// The parsed manifest private Manifestfile GetManifest(BackendWrapper backend, ManifestEntry entry) { if (m_options.DontReadManifests) { Manifestfile mf = new Manifestfile(); mf.SignatureHashes = null; mf.ContentHashes = null; return mf; } if (entry.ParsedManifest != null) return entry.ParsedManifest; else if (entry.Alternate != null && entry.Alternate.ParsedManifest != null) return entry.Alternate.ParsedManifest; if (OperationProgress != null && backend.Statistics != null) OperationProgress(this, GetOperationType(), backend.Statistics.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusReadingManifest, entry.Time.ToShortDateString() + " " + entry.Time.ToShortTimeString()), ""); bool tryAlternateManifest = false; //This method has some very special logic to ensure correct handling of errors //The assumption is that it is possible to determine if the error occurred due to a // transfer problem or a corrupt file. If the former happens, the operation should // be retried, and thus an exception is thrown. If the latter, the file should // be ignored and the backup file should be used. // //We detect a parsing error, either directly or indirectly through CryptographicException, // and assume that a parsing error is an indication of a broken file. //All other errors are assumed to be transfer problems, and throws exceptions. // //This holds as long as the backend always throws an exception if a partial file // was downloaded. The FTP backend may not honor this, and some webservers // may ommit the "Content-Length" header, which will cause problems. //There is a guard agains partial downloads in BackendWrapper.GetInternal() using (new Logging.Timer("Get " + entry.Filename)) using (Utility.TempFile tf = new Duplicati.Library.Utility.TempFile()) { try { backend.Get(entry, null, tf, null); //We now have the file decrypted, if the next step fails, // its a broken xml or invalid content tryAlternateManifest = true; Manifestfile mf = new Manifestfile(tf, m_options.SkipFileHashChecks); if (string.IsNullOrEmpty(mf.SelfFilename)) mf.SelfFilename = entry.Filename; if (mf.ContentHashes != null && entry.Alternate != null) { //Special case, the manifest has not recorded all volumes, // we must see if the alternate manifest has more volumes if (entry.Volumes.Count > mf.ContentHashes.Count) { //Do not try the alternate, we just did tryAlternateManifest = false; Logging.Log.WriteMessage(string.Format(Strings.Interface.ReadingSecondaryManifestLogMessage, entry.Alternate.Filename), Duplicati.Library.Logging.LogMessageType.Information); Manifestfile amf = null; //Read the alternate file and try to differentiate between a defect file or a partial one bool defectFile = false; try { System.IO.File.Delete(tf); backend.Get(entry.Alternate, null, tf, null); } catch (System.Security.Cryptography.CryptographicException cex) { //We assume that CryptoException means partial file Logging.Log.WriteMessage(string.Format(Strings.Interface.SecondaryManifestReadErrorLogMessage, entry.Alternate.Filename, cex), Duplicati.Library.Logging.LogMessageType.Warning); defectFile = true; } if (!defectFile) { try { amf = new Manifestfile(tf, m_options.SkipFileHashChecks); } catch (Exception ex) { //Parsing error means partial file Logging.Log.WriteMessage(string.Format(Strings.Interface.SecondaryManifestReadErrorLogMessage, entry.Alternate.Filename, ex), Duplicati.Library.Logging.LogMessageType.Warning); defectFile = true; } } //If the alternate manifest is correct, assign it so we have a copy if (!defectFile && amf != null) { if (string.IsNullOrEmpty(amf.SelfFilename)) amf.SelfFilename = entry.Alternate.Filename; //If the alternate manifest has more files than the primary, we use that one if (amf.ContentHashes != null && amf.ContentHashes.Count > mf.ContentHashes.Count) { entry.Alternate.ParsedManifest = amf; if (m_options.SkipFileHashChecks) { mf.SignatureHashes = null; mf.ContentHashes = null; } return amf; } } } } if (m_options.SkipFileHashChecks) { mf.SignatureHashes = null; mf.ContentHashes = null; } entry.ParsedManifest = mf; return mf; } catch (Exception ex) { //Only try secondary if the parsing/decrypting fails, not if the transfer fails if (entry.Alternate != null && (ex is System.Security.Cryptography.CryptographicException || tryAlternateManifest)) { //TODO: If it is a version error, there is no need to read the alternate version Logging.Log.WriteMessage(string.Format(Strings.Interface.PrimaryManifestReadErrorLogMessage, entry.Filename, ex.Message), Duplicati.Library.Logging.LogMessageType.Warning); try { Logging.Log.WriteMessage(string.Format(Strings.Interface.ReadingSecondaryManifestLogMessage, entry.Alternate.Filename), Duplicati.Library.Logging.LogMessageType.Information); return GetManifest(backend, entry.Alternate); } catch (Exception ex2) { Logging.Log.WriteMessage(string.Format(Strings.Interface.SecondaryManifestReadErrorLogMessage, entry.Alternate.Filename, ex2.Message), Duplicati.Library.Logging.LogMessageType.Warning); } } //Report the original error throw; } } } public string Restore(string[] target) { RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.Restore); SetupCommonOptions(rs); m_progress = 0; BackendWrapper backend = null; m_restorePatches = 0; using (new Logging.Timer("Restore from " + m_backend + " to " + string.Join(System.IO.Path.PathSeparator.ToString(), target))) { try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Restore, rs.OperationMode, -1, -1, Strings.Interface.StatusStarted, ""); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, -1, -1, Strings.Interface.StatusStarted, ""); Utility.FilenameFilter filter = m_options.Filter; //Filter is prefered, if both file and filter is specified if (!m_options.HasFilter && !string.IsNullOrEmpty(m_options.FileToRestore)) { List list = new List(); list.Add(new Utility.FilelistFilter(true, m_options.FileToRestore.Split(System.IO.Path.PathSeparator))); list.Add(new Utility.RegularExpressionFilter(false, ".*")); filter = new Duplicati.Library.Utility.FilenameFilter(list); } backend = new BackendWrapper(rs, m_backend, m_options); backend.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(BackupTransfer_ProgressEvent); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, ""); ManifestEntry bestFit = backend.GetBackupSet(m_options.RestoreTime); //We will need all the manifests downloaded anyway if (!m_options.DontReadManifests) { if (bestFit.Incrementals.Count > 0) VerifyManifestChain(backend, bestFit.Incrementals[bestFit.Incrementals.Count - 1]); else VerifyManifestChain(backend, bestFit); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementals, ""); } m_progress = INCREMENAL_COST; List entries = new List(); entries.Add(bestFit); entries.AddRange(bestFit.Incrementals); int patchno = 0; foreach (ManifestEntry be in entries) m_restorePatches += be.Volumes.Count; Manifestfile rootManifest = GetManifest(backend, bestFit); int sourceDirCount = (rootManifest.SourceDirs == null || rootManifest.SourceDirs.Length == 0) ? 1 : rootManifest.SourceDirs.Length; //After reading the first manifest, we know the source folder count if ((rootManifest.SourceDirs == null || rootManifest.SourceDirs.Length == 0) && target.Length > 1) { //V1 support rs.LogWarning(string.Format(Strings.Interface.TooManyTargetFoldersWarning, 1, target.Length), null); Array.Resize(ref target, 1); } else if (target.Length > sourceDirCount) { //If we get too many, we can just cut them off rs.LogWarning(string.Format(Strings.Interface.TooManyTargetFoldersWarning, sourceDirCount, target.Length), null); Array.Resize(ref target, rootManifest.SourceDirs.Length); } else if (target.Length != 1 && target.Length < sourceDirCount) { //If we get too few, we have to bail throw new Exception(string.Format(Strings.Interface.TooFewTargetFoldersError, sourceDirCount, target.Length)); } else if (target.Length == 1 && sourceDirCount > 1) { //If there is just one target folder, we automatically compose target subfolders string[] newtargets = new string[rootManifest.SourceDirs.Length]; List suggestions = new List(); for (int i = 0; i < rootManifest.SourceDirs.Length; i++) { string s = rootManifest.SourceDirs[i]; //HACK: We use a leading / in the path name to detect source OS // all paths are absolute, so this detects all unix like systems string dirSepChar = s.StartsWith("/") ? "/" : "\\"; if (s.EndsWith(dirSepChar)) s = s.Substring(0, s.Length - 1); int lix = s.LastIndexOf(dirSepChar); if (lix < 0 || lix + 1 >= s.Length) s = i.ToString(); else s = s.Substring(lix + 1); foreach (char c in System.IO.Path.GetInvalidFileNameChars()) s = s.Replace(c, '_'); suggestions.Add(s); } Dictionary duplicates = new Dictionary(Library.Utility.Utility.ClientFilenameStringComparer); for (int i = 0; i < suggestions.Count; i++) if (duplicates.ContainsKey(suggestions[i])) duplicates[suggestions[i]]++; else duplicates[suggestions[i]] = 1; for (int i = 0; i < newtargets.Length; i++) { string suffix = duplicates[suggestions[i]] > 1 ? i.ToString() : suggestions[i]; newtargets[i] = System.IO.Path.Combine(target[0], suffix); } target = newtargets; } //Make sure all targets exist foreach(string s in target) if (!System.IO.Directory.Exists(s)) System.IO.Directory.CreateDirectory(s); using (RSync.RSyncDir sync = new Duplicati.Library.Main.RSync.RSyncDir(target, rs, filter)) { sync.ProgressEvent += new Duplicati.Library.Main.RSync.RSyncDir.ProgressEventDelegate(RestoreRSyncDir_ProgressEvent); foreach (ManifestEntry be in entries) { m_progress = ((1.0 - INCREMENAL_COST) * (patchno / (double)m_restorePatches)) + INCREMENAL_COST; CheckLiveControl(); Manifestfile manifest = be == bestFit ? rootManifest : GetManifest(backend, be); CheckLiveControl(); foreach (KeyValuePair vol in be.Volumes) { ContentEntry contentVol = vol.Value; SignatureEntry signatureVol = vol.Key; m_progress = ((1.0 - INCREMENAL_COST) * (patchno / (double)m_restorePatches)) + INCREMENAL_COST; //Skip nonlisted if (manifest.ContentHashes != null && contentVol.Volumenumber > manifest.ContentHashes.Count) { Logging.Log.WriteMessage(string.Format(Strings.Interface.SkippedContentVolumeLogMessage, contentVol.Volumenumber), Duplicati.Library.Logging.LogMessageType.Warning); rs.LogWarning(string.Format(Strings.Interface.SkippedContentVolumeLogMessage, contentVol.Volumenumber), null); patchno++; continue; } using (Utility.TempFile patchzip = new Duplicati.Library.Utility.TempFile()) { OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusPatching, patchno + 1), ""); CheckLiveControl(); if (m_options.HasFilter || !string.IsNullOrEmpty(m_options.FileToRestore)) { bool hasFiles = false; using (Utility.TempFile sigFile = new Duplicati.Library.Utility.TempFile()) { OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusDownloadingSignatureVolume, patchno + 1), ""); try { using (new Logging.Timer("Get " + signatureVol.Filename)) backend.Get(signatureVol, manifest, sigFile, manifest.SignatureHashes == null ? null : manifest.SignatureHashes[signatureVol.Volumenumber - 1]); } catch (BackendWrapper.HashMismathcException hme) { hasFiles = true; rs.LogError(string.Format(Strings.Interface.FileHashFailure, hme.Message), hme); } if (!hasFiles) using (Library.Interface.ICompression patch = DynamicLoader.CompressionLoader.GetModule(signatureVol.Compression, sigFile, m_options.RawOptions)) { foreach(KeyValuePair k in sync.ListPatchFiles(patch)) if (filter.ShouldInclude("", System.IO.Path.DirectorySeparatorChar.ToString() + k.Value)) { //TODO: Perhaps a bit much to download the content archive // if the file is only marked for deletion? hasFiles = true; break; } } } if (!hasFiles) { //Avoid downloading the content file patchno++; continue; } } OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusDownloadingContentVolume, patchno + 1), ""); using (new Logging.Timer("Get " + contentVol.Filename)) backend.Get(contentVol, manifest, patchzip, manifest.ContentHashes == null ? null : manifest.ContentHashes[contentVol.Volumenumber - 1]); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusPatching, patchno + 1), ""); using (new Logging.Timer((patchno == 0 ? "Full restore to: " : "Incremental restore " + patchno.ToString() + " to: ") + string.Join(System.IO.Path.PathSeparator.ToString(), target))) using (Library.Interface.ICompression patch = DynamicLoader.CompressionLoader.GetModule(contentVol.Compression, patchzip, m_options.RawOptions)) sync.Patch(target, patch); } patchno++; } //Make sure there are no partial files, as partial files are not allowed to span backup sets sync.FinalizeRestore(); } } } finally { if (backend != null) backend.Dispose(); if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Restore, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } } rs.EndTime = DateTime.Now; return rs.ToString(); } /// /// Restores control files added to a backup. /// /// The folder into which to restore the files /// A restore report public string RestoreControlFiles(string target) { RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.RestoreControlfiles); SetupCommonOptions(rs); BackendWrapper backend = null; using (new Logging.Timer("Restore control files from " + m_backend + " to " + target)) { try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Restore, rs.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); backend = new BackendWrapper(rs, m_backend, m_options); List attempts = backend.GetBackupSets(); List flatlist = new List(); foreach (ManifestEntry be in attempts) { flatlist.Add(be); flatlist.AddRange(be.Incrementals); } flatlist.Reverse(); string prefix = Utility.Utility.AppendDirSeparator(RSync.RSyncDir.CONTROL_ROOT); foreach (ManifestEntry be in flatlist) { if (be.Volumes.Count > 0) using(Utility.TempFile z = new Duplicati.Library.Utility.TempFile()) { OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, 0, -1, string.Format(Strings.Interface.StatusReadingIncrementalFile, be.Volumes[0].Key.Filename), ""); Manifestfile mf = GetManifest(backend, be); OperationProgress(this, DuplicatiOperation.Restore, rs.OperationMode, 0, -1, string.Format(Strings.Interface.StatusReadingIncrementalFile, be.Volumes[0].Key.Filename), ""); using (new Logging.Timer("Get " + be.Volumes[0].Key.Filename)) backend.Get(be.Volumes[0].Key, mf, z, mf.SignatureHashes == null ? null : mf.SignatureHashes[0]); using(Library.Interface.ICompression fz = DynamicLoader.CompressionLoader.GetModule(be.Volumes[0].Key.Compression, z, m_options.RawOptions)) { bool any = false; foreach (string f in fz.ListFiles(prefix)) { any = true; using (System.IO.Stream s1 = fz.OpenRead(f)) using (System.IO.Stream s2 = System.IO.File.Create(System.IO.Path.Combine(target, f.Substring(prefix.Length)))) Utility.Utility.CopyStream(s1, s2); } if (any) break; rs.LogError(string.Format(Strings.Interface.FailedToFindControlFilesMessage, be.Volumes[0].Key.Filename), null); } } } } finally { if (backend != null) backend.Dispose(); if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Restore, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } } rs.EndTime = DateTime.Now; return rs.ToString(); } public string DeleteAllButN() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.DeleteAllButN); SetupCommonOptions(stats); int x = Math.Max(0, m_options.DeleteAllButNFull); StringBuilder sb = new StringBuilder(); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Remove, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); List flatlist = new List(); List entries = backend.GetBackupSets(); //Get all backups as a flat list foreach (ManifestEntry me in entries) { flatlist.Add(me); flatlist.AddRange(me.Incrementals); } //Now remove all but those requested List toremove = new List(); while (flatlist.Count > x) { toremove.Add(flatlist[0]); flatlist.RemoveAt(0); } //If there are still chains left, make sure we do not end up with a partial chain if (!m_options.AllowFullRemoval || flatlist.Count != 0) { //Go back until we have a full chain while (toremove.Count > 0 && (flatlist.Count == 0 || !flatlist[0].IsFull)) { sb.AppendLine(string.Format(Strings.Interface.NotDeletingBackupSetMessage, toremove[toremove.Count - 1].Time)); flatlist.Insert(0, toremove[toremove.Count - 1]); toremove.RemoveAt(toremove.Count - 1); } } if (toremove.Count > 0 && !m_options.AllowFullRemoval && (flatlist.Count == 0 || !flatlist[0].IsFull)) throw new Exception(Strings.Interface.InternalDeleteCountError); sb.Append(RemoveBackupSets(backend, toremove)); } finally { if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Remove, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } return sb.ToString(); } public string DeleteAllButNFull() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.DeleteAllButNFull); SetupCommonOptions(stats); int x = Math.Max(0, m_options.DeleteAllButNFull); StringBuilder sb = new StringBuilder(); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Remove, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); List entries = backend.GetBackupSets(); List toremove = new List(); while (entries.Count > x) { if (entries.Count == 1 && !m_options.AllowFullRemoval) { sb.AppendLine(string.Format(Strings.Interface.NotDeletingLastFullMessage, entries[0].Time)); break; } ManifestEntry be = entries[0]; entries.RemoveAt(0); be.Incrementals.Reverse(); toremove.AddRange(be.Incrementals); toremove.Add(be); } if (entries.Count == 0 && toremove.Count > 0 && !m_options.AllowFullRemoval) throw new Exception(Strings.Interface.InternalDeleteCountError); sb.Append(RemoveBackupSets(backend, toremove)); } finally { if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Remove, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } return sb.ToString(); } public string DeleteOlderThan() { StringBuilder sb = new StringBuilder(); CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.DeleteOlderThan); SetupCommonOptions(stats); DateTime expires = m_options.RemoveOlderThan; using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) try { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.Remove, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); List entries = backend.GetBackupSets(); List toremove = new List(); while (entries.Count > 0 && entries[0].Time <= expires) { if (entries.Count == 1 && !m_options.AllowFullRemoval) { sb.AppendLine(string.Format(Strings.Interface.NotDeletingLastFullMessage, entries[0].Time)); break; } ManifestEntry be = entries[0]; entries.RemoveAt(0); bool hasNewer = false; foreach (ManifestEntry bex in be.Incrementals) if (bex.Time >= expires) { hasNewer = true; break; } if (hasNewer) { List t = new List(be.Incrementals); t.Insert(0, be); for (int i = 0; i < t.Count; i++) if (t[i].Time <= expires) sb.AppendLine(string.Format(Strings.Interface.NotDeletingBackupSetMessage, t[i].Time.ToString(System.Globalization.CultureInfo.InvariantCulture))); break; } else { be.Incrementals.Reverse(); toremove.AddRange(be.Incrementals); toremove.Add(be); } } if (entries.Count == 0 && toremove.Count > 0 && !m_options.AllowFullRemoval) throw new Exception(Strings.Interface.InternalDeleteCountError); sb.Append(RemoveBackupSets(backend, toremove)); } finally { if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.Remove, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); } return sb.ToString(); } private string RemoveBackupSets(BackendWrapper backend, List entries) { StringBuilder sb = new StringBuilder(); sb.Append(backend.FinishDeleteTransaction(false)); if (entries.Count > 0) { System.Xml.XmlDocument doc = new System.Xml.XmlDocument(); System.Xml.XmlNode root = doc.AppendChild(doc.CreateElement("files")); root.Attributes.Append(doc.CreateAttribute("version")).Value = "1"; foreach (ManifestEntry me in entries) { if (me.Alternate != null) root.AppendChild(doc.CreateElement("file")).InnerText = me.Alternate.Filename; root.AppendChild(doc.CreateElement("file")).InnerText = me.Filename; if (me.Verification != null) root.AppendChild(doc.CreateElement("file")).InnerText = me.Verification.Filename; foreach (KeyValuePair kx in me.Volumes) { root.AppendChild(doc.CreateElement("file")).InnerText = kx.Key.Filename; root.AppendChild(doc.CreateElement("file")).InnerText = kx.Value.Filename; } } if (m_options.Force) { using (TempFile tf = new TempFile()) { doc.Save(tf); tf.Protected = true; backend.WriteDeleteTransactionFile(tf); } } foreach (ManifestEntry me in entries) { sb.AppendLine(string.Format(Strings.Interface.DeletingBackupSetMessage, me.Time.ToString(System.Globalization.CultureInfo.InvariantCulture))); if (m_options.Force) { //Delete manifest if (me.Alternate != null) backend.Delete(me.Alternate); backend.Delete(me); if (me.Verification != null) backend.Delete(me.Verification); foreach (KeyValuePair kx in me.Volumes) { backend.Delete(kx.Key); backend.Delete(kx.Value); } } } if (m_options.Force) backend.RemoveDeleteTransactionFile(); if (!m_options.Force && entries.Count > 0) sb.AppendLine(Strings.Interface.FilesAreNotForceDeletedMessage); } return sb.ToString(); } public string[] List() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.List); SetupCommonOptions(stats); List res = new List(); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) { if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.List, stats.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); foreach (Duplicati.Library.Interface.IFileEntry fe in backend.List(false)) res.Add(fe.Name); if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.List, stats.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); return res.ToArray(); } } public string Cleanup() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.CleanUp); SetupCommonOptions(stats); bool anyRemoved = false; StringBuilder sb = new StringBuilder(); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) { List sorted = backend.GetBackupSets(); List entries = new List(); entries.AddRange(sorted); foreach (ManifestEntry be in sorted) entries.AddRange(be.Incrementals); string cleanup = backend.DeleteOrphans(false); if (!string.IsNullOrEmpty(cleanup)) sb.AppendLine(cleanup); if (m_options.SkipFileHashChecks) throw new Exception(Strings.Interface.CannotCleanWithoutHashesError); if (m_options.DontReadManifests) throw new Exception(Strings.Interface.CannotCleanWithoutHashesError); //We need the manifests anyway, so we verify the chain if (entries.Count > 0) VerifyManifestChain(backend, entries[0]); //Now compare the actual filelist with the manifest foreach (ManifestEntry be in entries) { Manifestfile manifest = GetManifest(backend, be); int count = manifest.ContentHashes.Count; for (int i = count - 1; i < be.Volumes.Count; i++) { anyRemoved = true; string sigmsg = string.Format(Strings.Interface.RemovingPartialFilesMessage, be.Volumes[i].Key.Filename); string cntmsg = string.Format(Strings.Interface.RemovingPartialFilesMessage, be.Volumes[i].Value.Filename); Logging.Log.WriteMessage(sigmsg, Duplicati.Library.Logging.LogMessageType.Information); Logging.Log.WriteMessage(cntmsg, Duplicati.Library.Logging.LogMessageType.Information); sb.AppendLine(sigmsg); sb.AppendLine(cntmsg); if (m_options.Force) { backend.Delete(be.Volumes[i].Key); backend.Delete(be.Volumes[i].Value); } } } } if (!m_options.Force && anyRemoved) { Logging.Log.WriteMessage(Strings.Interface.FilesAreNotForceDeletedMessage, Duplicati.Library.Logging.LogMessageType.Information); sb.AppendLine(Strings.Interface.FilesAreNotForceDeletedMessage); } return sb.ToString(); //TODO: Write a message here? } public IList ListCurrentFiles() { RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.ListCurrentFiles); SetupCommonOptions(rs); Utility.FilenameFilter filter = m_options.Filter; DateTime timelimit = m_options.RestoreTime; if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.List, rs.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); List res; using (BackendWrapper backend = new BackendWrapper(rs, m_backend, m_options)) using (Utility.TempFolder basefolder = new Duplicati.Library.Utility.TempFolder()) { ManifestEntry bestFit = backend.GetBackupSet(timelimit); List entries = new List(); entries.Add(bestFit); entries.AddRange(bestFit.Incrementals); List> patches = FindPatches(backend, entries, basefolder, false, rs); using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(new string[] { basefolder }, rs, filter, patches)) res = dir.UnmatchedFiles(); } if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.List, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); return res; } public string[] ListSourceFolders() { RestoreStatistics rs = new RestoreStatistics(DuplicatiOperationMode.ListSourceFolders); SetupCommonOptions(rs); if (m_options.DontReadManifests) throw new Exception(Strings.Interface.ManifestsMustBeRead); DateTime timelimit = m_options.RestoreTime; if (OperationStarted != null) OperationStarted(this, DuplicatiOperation.List, rs.OperationMode, 0, -1, Strings.Interface.StatusStarted, ""); string[] res; using (BackendWrapper backend = new BackendWrapper(rs, m_backend, m_options)) using (Utility.TempFile mfile = new Duplicati.Library.Utility.TempFile()) { ManifestEntry bestFit = backend.GetBackupSet(timelimit); backend.Get(bestFit, null, mfile, null); res = new Manifestfile(mfile, m_options.SkipFileHashChecks).SourceDirs; } if (OperationCompleted != null) OperationCompleted(this, DuplicatiOperation.List, rs.OperationMode, 100, -1, Strings.Interface.StatusCompleted, ""); return res; } private void SetupCommonOptions(CommunicationStatistics stats) { m_options.MainAction = stats.OperationMode; switch (m_options.MainAction) { case DuplicatiOperationMode.Backup: case DuplicatiOperationMode.BackupFull: case DuplicatiOperationMode.BackupIncremental: break; default: //It only makes sense to enable auto-creation if we are writing files. if (!m_options.RawOptions.ContainsKey("disable-autocreate-folder")) m_options.RawOptions["disable-autocreate-folder"] = "true"; break; } Library.Logging.Log.LogLevel = m_options.Loglevel; OperationRunning(true); if (!string.IsNullOrEmpty(m_options.Logfile)) { m_hasSetLogging = true; Library.Logging.Log.CurrentLog = new Library.Logging.StreamLog(m_options.Logfile); } if (stats != null) { stats.VerboseErrors = m_options.DebugOutput; stats.VerboseRetryErrors = m_options.VerboseRetryErrors; } if (!string.IsNullOrEmpty(m_options.TempDir)) Utility.TempFolder.SystemTempPath = m_options.TempDir; if (!string.IsNullOrEmpty(m_options.ThreadPriority)) System.Threading.Thread.CurrentThread.Priority = Utility.Utility.ParsePriority(m_options.ThreadPriority); //Load all generic modules m_options.LoadedModules.Clear(); foreach (Library.Interface.IGenericModule m in DynamicLoader.GenericLoader.Modules) m_options.LoadedModules.Add(new KeyValuePair(Array.IndexOf(m_options.DisableModules, m.Key.ToLower()) < 0 && (m.LoadAsDefault || Array.IndexOf(m_options.EnableModules, m.Key.ToLower()) >= 0), m)); ValidateOptions(stats); foreach (KeyValuePair mx in m_options.LoadedModules) if (mx.Key) mx.Value.Configure(m_options.RawOptions); Library.Logging.Log.WriteMessage(string.Format(Strings.Interface.StartingOperationMessage, m_options.MainAction), Logging.LogMessageType.Information); } /// /// Downloads all required signature files from the backend. /// /// The backend to read from /// The flattened list of manifests /// The tempfolder set for this operation /// True to ignore files with failed hash signature /// A list of file archives private List> FindPatches(BackendWrapper backend, List entries, string tempfolder, bool allowHashFail, CommunicationStatistics stat) { List> patches = new List>(); using (new Logging.Timer("Reading incremental data")) { OperationProgress(this, GetOperationType(), stat.OperationMode, (int)(m_progress * 100), -1, Strings.Interface.StatusReadingIncrementalData, ""); //Calculate the total number of files to download //, and verify their order int incCount = 0; foreach (ManifestEntry be in entries) { int volNo = 0; //Prevent order based bugs if (entries.IndexOf(be) > 0) if (entries[entries.IndexOf(be) - 1].Time >= be.Time) throw new Exception(Strings.Interface.BadSortingDetectedError); incCount++; foreach (KeyValuePair bes in be.Volumes) { incCount++; if (volNo + 1 != bes.Key.Volumenumber || bes.Key.Volumenumber != bes.Value.Volumenumber) throw new Exception(Strings.Interface.BadVolumeSortOrder); volNo++; } } //The incremental part has a fixed cost, and each file has a fixed fraction of that double unitCost = m_incrementalFraction / incCount; //Ensure that the manifest chain has not been tampered with // since we will read all manifest files anyway, there is no harm in doing it here if (!m_options.DontReadManifests && entries.Count > 0) VerifyManifestChain(backend, entries[entries.Count - 1]); foreach (ManifestEntry be in entries) { m_progress += unitCost; Manifestfile manifest = GetManifest(backend, be); foreach (KeyValuePair bes in be.Volumes) { m_progress += unitCost; //Skip non-listed incrementals if (manifest.SignatureHashes != null && bes.Key.Volumenumber > manifest.SignatureHashes.Count) { backend.AddOrphan(bes.Key); backend.AddOrphan(bes.Value); continue; } OperationProgress(this, GetOperationType(), stat.OperationMode, (int)(m_progress * 100), -1, string.Format(Strings.Interface.StatusReadingSignatureFile, be.Time.ToShortDateString() + " " + be.Time.ToShortTimeString(), bes.Key.Volumenumber), ""); string filename = System.IO.Path.Combine(tempfolder, "patch-" + patches.Count.ToString() + ".zip"); //Check just before we download stuff CheckLiveControl(); try { using (new Logging.Timer("Get " + bes.Key.Filename)) backend.Get(bes.Key, manifest, filename, manifest.SignatureHashes == null ? null : manifest.SignatureHashes[bes.Key.Volumenumber - 1]); } catch (BackendWrapper.HashMismathcException hme) { if (allowHashFail) { if (stat != null) stat.LogError(string.Format(Strings.Interface.FileHashFailure, hme.Message), hme); continue; } else throw; } patches.Add(new KeyValuePair(be, DynamicLoader.CompressionLoader.GetModule(bes.Key.Compression, filename, m_options.RawOptions))); } } } backend.DeleteOrphans(true); return patches; } public void PurgeSignatureCache() { if (string.IsNullOrEmpty(m_options.SignatureCachePath)) throw new Exception(Strings.Interface.SignatureCachePathMissingError); else RemoveSignatureFiles(m_options.SignatureCachePath); } public List GetBackupSets() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.GetBackupSets); SetupCommonOptions(stats); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) return backend.GetBackupSets(); } public List> ListActualSignatureFiles() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.ListActualSignatureFiles); SetupCommonOptions(stats); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) { ManifestEntry bestFit = backend.GetBackupSet(m_options.RestoreTime); if (bestFit.Incrementals.Count > 0) //Get the most recent incremental bestFit = bestFit.Incrementals[bestFit.Incrementals.Count - 1]; using (Utility.TempFolder folder = new Duplicati.Library.Utility.TempFolder()) { List patches = new List(); foreach (KeyValuePair entry in FindPatches(backend, new List(new ManifestEntry[] { bestFit }), folder, false, stats)) patches.Add(entry.Value); using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(new string[] { folder }, stats, null)) return dir.ListPatchFiles(patches); } } } private void CreateFolder() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.CreateFolder); SetupCommonOptions(stats); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) backend.CreateFolder(); } public List> VerifyBackupChain() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.Verify); SetupCommonOptions(stats); List> results = new List>(); if (m_options.DontReadManifests) throw new InvalidOperationException(Strings.Interface.ManifestsMustBeRead); if (m_options.SkipFileHashChecks) throw new InvalidOperationException(Strings.Interface.CannotVerifyWithoutHashes); if (!string.IsNullOrEmpty(m_options.SignatureCachePath)) { stats.LogWarning(Strings.Interface.DisablingSignatureCacheForVerification, null); m_options.SignatureCachePath = null; } using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) { //Find the spot in the chain where we start ManifestEntry bestFit = backend.GetBackupSet(m_options.RestoreTime); //Get the list of manifests to validate List entries = new List(); entries.Add(bestFit); entries.AddRange(bestFit.Incrementals); entries.Reverse(); foreach (ManifestEntry me in entries) { Manifestfile mf = null; try { mf = GetManifest(backend, me); VerifyBackupChainWithFiles(backend, me); if (mf.SignatureHashes.Count != me.Volumes.Count) results.Add(new KeyValuePair(me, new Exception(string.Format(Strings.Interface.ManifestAndFileCountMismatchError, mf.SelfFilename, mf.SignatureHashes.Count, me.Volumes.Count)))); else results.Add(new KeyValuePair(me, null)); } catch (Exception ex) { results.Add(new KeyValuePair(me, ex)); } if (mf != null) { int volumes = Math.Min(mf.SignatureHashes.Count, me.Volumes.Count); for(int i = 0; i (me.Volumes[i].Key, null)); } catch (Exception ex) { results.Add(new KeyValuePair(me.Volumes[i].Key, ex)); } } if (m_options.Verificationlevel == VerificationLevel.Full) { try { using(Utility.TempFile tf = new Duplicati.Library.Utility.TempFile()) backend.Get(me.Volumes[i].Value, mf, tf, mf.ContentHashes[i]); results.Add(new KeyValuePair(me.Volumes[i].Value, null)); } catch (Exception ex) { results.Add(new KeyValuePair(me.Volumes[i].Value, ex)); } } } } } //Re-generate verification file if (m_options.CreateVerificationFile) { //Stop any async operations if (m_options.AsynchronousUpload) backend.ExtractPendingUploads(); VerificationFile vf = new VerificationFile(entries, new FilenameStrategy(m_options)); using (Utility.TempFile tf = new Duplicati.Library.Utility.TempFile()) { vf.Save(tf); tf.Protected = true; backend.Put(new VerificationEntry(entries[entries.Count - 1].Time), tf); } } } return results; } /// /// Reads through a backup and finds the last backup entry that has a specific file /// /// public List> FindLastFileVersion() { CommunicationStatistics stats = new CommunicationStatistics(DuplicatiOperationMode.FindLastFileVersion); SetupCommonOptions(stats); if (m_options.DontReadManifests) throw new Exception(Strings.Interface.ManifestsMustBeRead); if (string.IsNullOrEmpty(m_options.FileToRestore)) throw new Exception(Strings.Interface.NoFilesGivenError); string[] filesToFind = m_options.FileToRestore.Split(System.IO.Path.PathSeparator); KeyValuePair[] results = new KeyValuePair[filesToFind.Length]; for (int i = 0; i < results.Length; i++) results[i] = new KeyValuePair(filesToFind[i], new DateTime(0)); using (BackendWrapper backend = new BackendWrapper(stats, m_backend, m_options)) { //Extract the full backup set list List fulls = backend.GetBackupSets(); //Flatten the list List workList = new List(); //The list is oldest first, this function work newest first fulls.Reverse(); foreach (ManifestEntry f in fulls) { f.Incrementals.Reverse(); workList.AddRange(f.Incrementals); workList.Add(f); } bool warned_manifest_v1 = false; foreach (ManifestEntry mf in workList) { List signatureHashes = null; Manifestfile mfi; using(Utility.TempFile tf = new Duplicati.Library.Utility.TempFile()) { backend.Get(mf, null, tf, null); mfi = new Manifestfile(tf, m_options.SkipFileHashChecks); if (!m_options.SkipFileHashChecks) signatureHashes = mfi.SignatureHashes; } //If there are no volumes, don't stop here bool any_unmatched = true; if (stats != null && !warned_manifest_v1 && (mfi.SourceDirs == null || mfi.SourceDirs.Length == 0)) { warned_manifest_v1 = true; stats.LogWarning(Strings.Interface.ManifestVersionRequiresRelativeNamesWarning, null); } foreach(KeyValuePair e in mf.Volumes) using (Utility.TempFile tf = new Duplicati.Library.Utility.TempFile()) { //Skip non-approved signature files if (signatureHashes != null && e.Key.Volumenumber > signatureHashes.Count) { stats.LogWarning(string.Format(Strings.Interface.SkippedUnlistedSignatureFileWarning, e.Key.Filename), null); continue; } backend.Get(e.Key, mfi, tf, signatureHashes == null ? null : signatureHashes[e.Key.Volumenumber - 1]); any_unmatched = false; RSync.RSyncDir.ContainsFile(mfi, filesToFind, DynamicLoader.CompressionLoader.GetModule(e.Key.Compression, tf, m_options.RawOptions)); for (int i = 0; i < filesToFind.Length; i++) { if (results[i].Value.Ticks == 0 && string.IsNullOrEmpty(filesToFind[i])) results[i] = new KeyValuePair(results[i].Key, mf.Time); else any_unmatched = true; } if (!any_unmatched) break; } if (!any_unmatched) break; } return new List>(results); } } /// /// This function will examine all options passed on the commandline, and test for unsupported or deprecated values. /// Any errors will be logged into the statistics module. /// /// The commandline options given /// The backend url /// The statistics into which warnings are written private void ValidateOptions(CommunicationStatistics stats) { //No point in going through with this if we can't report if (stats == null) return; //Keep a list of all supplied options Dictionary ropts = m_options.RawOptions; //Keep a list of all supported options Dictionary supportedOptions = new Dictionary(); //There are a few internal options that are not accessible from outside, and thus not listed foreach (string s in Options.InternalOptions) supportedOptions[s] = null; //Figure out what module options are supported in the current setup List moduleOptions = new List(); Dictionary disabledModuleOptions = new Dictionary(); foreach (KeyValuePair m in m_options.LoadedModules) if (m.Value.SupportedCommands != null) if (m.Key) moduleOptions.AddRange(m.Value.SupportedCommands); else { foreach (Library.Interface.ICommandLineArgument c in m.Value.SupportedCommands) { disabledModuleOptions[c.Name] = m.Value.DisplayName + " (" + m.Value.Key + ")"; if (c.Aliases != null) foreach (string s in c.Aliases) disabledModuleOptions[s] = disabledModuleOptions[c.Name]; } } //Now run through all supported options, and look for deprecated options foreach (IList l in new IList[] { m_options.SupportedCommands, DynamicLoader.BackendLoader.GetSupportedCommands(m_backend), m_options.NoEncryption ? null : DynamicLoader.EncryptionLoader.GetSupportedCommands(m_options.EncryptionModule), moduleOptions, DynamicLoader.CompressionLoader.GetSupportedCommands(m_options.CompressionModule) }) { if (l != null) foreach (Library.Interface.ICommandLineArgument a in l) { if (supportedOptions.ContainsKey(a.Name) && Array.IndexOf(Options.KnownDuplicates, a.Name.ToLower()) < 0) stats.LogWarning(string.Format(Strings.Interface.DuplicateOptionNameWarning, a.Name), null); supportedOptions[a.Name] = a; if (a.Aliases != null) foreach (string s in a.Aliases) { if (supportedOptions.ContainsKey(s) && Array.IndexOf(Options.KnownDuplicates, s.ToLower()) < 0) stats.LogWarning(string.Format(Strings.Interface.DuplicateOptionNameWarning, s), null); supportedOptions[s] = a; } if (a.Deprecated) { List aliases = new List(); aliases.Add(a.Name); if (a.Aliases != null) aliases.AddRange(a.Aliases); foreach (string s in aliases) if (ropts.ContainsKey(s)) { string optname = a.Name; if (a.Name != s) optname += " (" + s + ")"; stats.LogWarning(string.Format(Strings.Interface.DeprecatedOptionUsedWarning, optname, a.DeprecationMessage), null); } } } } //Now look for options that were supplied but not supported foreach (string s in ropts.Keys) if (!supportedOptions.ContainsKey(s)) if (disabledModuleOptions.ContainsKey(s)) stats.LogWarning(string.Format(Strings.Interface.UnsupportedOptionDisabledModuleWarning, s, disabledModuleOptions[s]), null); else stats.LogWarning(string.Format(Strings.Interface.UnsupportedOptionWarning, s), null); //Look at the value supplied for each argument and see if is valid according to its type foreach (string s in ropts.Keys) { Library.Interface.ICommandLineArgument arg; if (supportedOptions.TryGetValue(s, out arg) && arg != null) { string validationMessage = ValidateOptionValue(arg, s, ropts[s]); if (validationMessage != null) stats.LogWarning(validationMessage, null); } } //TODO: Based on the action, see if all options are relevant } #region Static interface /// /// Checks if the value passed to an option is actually valid. /// /// The argument being validated /// The name of the option to validate /// The value to check /// Null if no errors are found, an error message otherwise public static string ValidateOptionValue(Library.Interface.ICommandLineArgument arg, string optionname, string value) { if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Enumeration) { bool found = false; foreach (string v in arg.ValidValues ?? new string[0]) if (string.Equals(v, value, StringComparison.CurrentCultureIgnoreCase)) { found = true; break; } if (!found) return string.Format(Strings.Interface.UnsupportedEnumerationValue, optionname, value, string.Join(", ", arg.ValidValues ?? new string[0])); } else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean) { if (!string.IsNullOrEmpty(value) && Utility.Utility.ParseBool(value, true) != Utility.Utility.ParseBool(value, false)) return string.Format(Strings.Interface.UnsupportedBooleanValue, optionname, value); } else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Integer) { long l; if (!long.TryParse(value, out l)) return string.Format(Strings.Interface.UnsupportedIntegerValue, optionname, value); } else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path) { foreach (string p in value.Split(System.IO.Path.DirectorySeparatorChar)) if (p.IndexOfAny(System.IO.Path.GetInvalidPathChars()) >= 0) return string.Format(Strings.Interface.UnsupportedPathValue, optionname, p); } else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Size) { try { Utility.Sizeparser.ParseSize(value); } catch { return string.Format(Strings.Interface.UnsupportedSizeValue, optionname, value); } } else if (arg.Type == Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan) { try { Utility.Timeparser.ParseTimeSpan(value); } catch { return string.Format(Strings.Interface.UnsupportedTimeValue, optionname, value); } } return null; } public static void RemoveSignatureFiles(string folder) { FilenameStrategy cachenames = BackendWrapper.CreateCacheFilenameStrategy(); foreach (string s in Utility.Utility.EnumerateFiles(folder)) { BackupEntryBase e = cachenames.ParseFilename(new Duplicati.Library.Interface.FileEntry(System.IO.Path.GetFileName(s))); if (e is SignatureEntry) { try { System.IO.File.Delete(s); } catch {} } } } //Static helpers public static string[] List(string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.List(); } public static string Backup(string[] source, string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.Backup(source); } public static string Restore(string source, string[] target, Dictionary options) { using (Interface i = new Interface(source, options)) return i.Restore(target); } /// /// Returns a list of full backup manifests, with a list of volumes and an incremental chain /// /// The backend to read the data from /// A list of options /// A list of full backup manifests public static List ParseFileList(string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.GetBackupSets(); } public static IList ListCurrentFiles(string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.ListCurrentFiles(); } public static string[] ListSourceFolders(string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.ListSourceFolders(); } public static string DeleteAllButNFull(string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.DeleteAllButNFull(); } public static string DeleteAllButN(string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.DeleteAllButN(); } public static string DeleteOlderThan(string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.DeleteOlderThan(); } public static string Cleanup(string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.Cleanup(); } public static string RestoreControlFiles(string source, string target, Dictionary options) { using (Interface i = new Interface(source, options)) return i.RestoreControlFiles(target); } public static void PurgeSignatureCache(Dictionary options) { Options opt = new Options(options); if (string.IsNullOrEmpty(opt.SignatureCachePath)) throw new Exception(Strings.Interface.SignatureCachePathMissingError); else System.IO.Directory.Delete(opt.SignatureCachePath, true); } public static List> ListActualSignatureFiles(string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.ListActualSignatureFiles(); } public static void CreateFolder(string target, Dictionary options) { using (Interface i = new Interface(target, options)) i.CreateFolder(); } public static List> FindLastFileVersion(string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.FindLastFileVersion(); } public static List> VerifyBackup(string target, Dictionary options) { using (Interface i = new Interface(target, options)) return i.VerifyBackupChain(); } #endregion #region IDisposable Members public void Dispose() { if (m_options != null && m_options.LoadedModules != null) { foreach (KeyValuePair mx in m_options.LoadedModules) if (mx.Key && mx.Value is IDisposable) ((IDisposable)mx.Value).Dispose(); m_options.LoadedModules.Clear(); OperationRunning(false); } if (m_hasSetLogging && Logging.Log.CurrentLog is Logging.StreamLog) { Logging.StreamLog sl = (Logging.StreamLog)Logging.Log.CurrentLog; Logging.Log.CurrentLog = null; sl.Dispose(); m_hasSetLogging = false; } } #endregion #region ILiveControl Members public void Pause() { m_liveControl.Pause(); } public void Resume() { m_liveControl.Resume(); } public void Stop() { m_liveControl.Stop(); } public void Terminate() { m_liveControl.Terminate(); } public bool IsStopRequested { get { return m_liveControl.IsStopRequested; } } public bool IsPauseRequested { get { return m_liveControl.IsPauseRequested; } } public void SetUploadLimit(string limit) { m_liveControl.SetUploadLimit(limit); } public void SetDownloadLimit(string limit) { m_liveControl.SetDownloadLimit(limit); } public void SetThreadPriority(System.Threading.ThreadPriority priority) { m_liveControl.SetThreadPriority(priority); } public void UnsetThreadPriority() { m_liveControl.UnsetThreadPriority(); } #endregion } }