From b60451a7ace0cb7009680c528357a6c51b4805b7 Mon Sep 17 00:00:00 2001 From: wjan Date: Fri, 26 Jan 2018 20:56:19 +0100 Subject: [PATCH 01/64] adds python script to decrypt and encrypt with different encryption or decryption procedures --- Tools/Commandline/ReEncrypt/ReEncrypt.py | 174 +++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 Tools/Commandline/ReEncrypt/ReEncrypt.py diff --git a/Tools/Commandline/ReEncrypt/ReEncrypt.py b/Tools/Commandline/ReEncrypt/ReEncrypt.py new file mode 100644 index 000000000..93ba5aebf --- /dev/null +++ b/Tools/Commandline/ReEncrypt/ReEncrypt.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 + +# by Ben Fisher, https://github.com/downpoured +# a Python script to restore files from Duplicati +# similar to Duplicati.RecoveryTool, but with no dependencies on Mono/.NET +# uses streaming apis to restore a large number of files and use limited RAM. +# supports backups using AES encryption (.aes) or No Encryption (.zip), +# if data uses GPG/other encryption, decrypt files to .zip before running this tool. + +import os +import sys +import io +import json +import sqlite3 +import zipfile +import codecs +import getpass +#import fnmatch +import base64 +import hashlib +import pyAesCrypt +from collections import OrderedDict +from tempfile import mkstemp, mkdtemp, TemporaryFile, TemporaryDirectory, NamedTemporaryFile +from pprint import pprint +from subprocess import Popen, PIPE, run +#import subprocess +#import pexpect +import gnupg +import shutil + + + +## +def mainRestore(options): + # locate dlist + dlists = [name for name in os.listdir(options['orig']['path']) if name.endswith(".dlist.%s" %(options['orig']['extension']))] + + # loop over all dlists; they only need to be enencrypted, and encrypted. They have no relation to the dindex and dblock files. + with NamedTemporaryFile() as temp_file: + for dlist_enc in dlists: + decrypt(options['orig'],os.path.join(options['orig']['path'],dlist_enc),options['orig']['passwd'],temp_file.name) + encrypt(options['new'],temp_file.name,options['new']['passwd'], os.path.join(options['new']['path'],change_ext(dlist_enc,options['orig']['extension'],options['new']['extension']))) + + # locate dlist + dindex = [name for name in os.listdir(options['orig']['path']) if name.endswith(".dindex.%s" %(options['orig']['extension']))] + + with NamedTemporaryFile() as temp_dindex, NamedTemporaryFile() as temp_dindex_reenc, TemporaryDirectory() as temp_path_zip: + for dindex_enc in dindex: + decrypt(options['orig'],os.path.join(options['orig']['path'],dindex_enc),options['orig']['passwd'],temp_dindex.name) + + unzip(temp_dindex,temp_path_zip) + + vol_path = os.path.join(temp_path_zip,'vol') + + for dblock in os.listdir(vol_path): + data = [] + with open(os.path.join(vol_path, dblock)) as data_file: + data = json.load(data_file, object_pairs_hook=OrderedDict) + + expected_hash = data['volumehash'].encode('utf8') + expected_volumesize = data['volumesize'] + + if (options['verify_hash']): + actual_hash = computeHash(os.path.join(options['orig']['path'],dblock)) + actual_volumesize=os.stat(os.path.join(options['orig']['path'],dblock)).st_size + print('dblock: %s expected_hash: %s calc_hash: %s exact: %s' % (dblock,expected_hash.decode('utf8'),actual_hash.decode('utf8'),expected_hash==actual_hash)) + + with NamedTemporaryFile(delete=False) as temp_dblock: + decrypt(options['orig'],os.path.join(options['orig']['path'],dblock),options['orig']['passwd'],temp_dblock.name) + encrypt(options['new'],temp_dblock.name,options['new']['passwd'], os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))) + new_hash = computeHash(os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))) + + data['volumehash'] = new_hash.decode('utf8') + data['volumesize'] = os.stat(os.path.join(options['new']['path'],change_ext(dblock,options['orig']['extension'],options['new']['extension']))).st_size + print('dblock: %s old_hash: %s new_hash: %s' % (dblock,expected_hash.decode('utf8'),data['volumehash'])) + #print(data['volumehash']) + + with open(os.path.join(vol_path,dblock),'w') as data_file: + json.dump(data, data_file) + + os.rename(os.path.join(vol_path, dblock), os.path.join(vol_path, change_ext(dblock,options['orig']['extension'],options['new']['extension']))) + + make_zipfile(temp_dindex_reenc.name,temp_path_zip) + encrypt(options['new'],temp_dindex_reenc.name, options['new']['passwd'],os.path.join(options['new']['path'],change_ext(dindex_enc,options['orig']['extension'],options['new']['extension']))) + +def change_ext(filename, ext_old, ext_new): + return filename.replace(ext_old, ext_new) + +def decrypt(options, encrypted, passw, decrypted): + if options['encryption']=='aes': + bufferSize = 64 * 1024 + pyAesCrypt.decryptFile(encrypted, decrypted, passw, bufferSize) + if options['encryption']: + gpg = gnupg.GPG() + with open(encrypted, 'rb') as f: + status = gpg.decrypt_file(f, output=decrypted,passphrase=passw) + if options['encryption']: + shutil.copy(encrypted,decrypted) + + +def encrypt(options,decrypted, passw, encrypted): + if options['encryption']: + bufferSize = 64 * 1024 + pyAesCrypt.encryptFile(decrypted, encrypted, passw, bufferSize) + if options['encryption']: + gpg = gnupg.GPG() + with open(decrypted, 'rb') as f: + status = gpg.encrypt_file(f, recipients=options['recipients'], output=encrypted, armor=False) + if options['encryption']: + shutil.copy(decrypted,encrypted) + #print('ok: %s' % status.ok) + #print('status: %s' % status.status) + #print('stderr: %s' % status.stderr) + +def unzip(archive, path): + with zipfile.ZipFile(archive.name) as zf: + zf.extractall(path) + +def make_zipfile(output_filename, source_dir): + relroot=source_dir + with zipfile.ZipFile(output_filename, "w", zipfile.ZIP_DEFLATED) as zip: + for root, dirs, files in os.walk(source_dir): + # add directory (needed for empty dirs) + zip.write(root, os.path.relpath(root, relroot)) + for file in files: + filename = os.path.join(root, file) + if os.path.isfile(filename): # regular files only + arcname = os.path.join(os.path.relpath(root, relroot), file) + zip.write(filename, arcname) + + +def rezip(temp_path_z, path): + zf = zipfile.ZipFile(path, "w") + for dirname, subdirs, files in os.walk(temp_path_z): + zf.write(dirname) + for filename in files: + zf.write(os.path.join(dirname, filename)) + zf.close() + +def zipdir(path,ziph): + for root, dirs, files in os.walk(path): + for file in files: + ziph.write(os.path.join(root, file)) + +def computeHash(path): + buffersize=64 * 1024 + hasher = hashlib.sha256() + with open(path, 'rb') as f: + while True: + buffer = f.read(buffersize) + if not buffer: + break + hasher.update(buffer) + return base64.b64encode(hasher.digest()) + +def main(): + options = {} + options['orig'] = {} + options['new'] = {} + options['verify_hash'] = False + options['orig']['encryption'] = 'aes' + options['orig']['extension'] = 'zip.aes' + options['orig']['passwd'] = '123456' + options['orig']['path'] = '/mnt/c/Duplicati/test_aes' + options['new']['encryption'] = 'gpg' + options['new']['extension'] = 'zip.gpg' + options['new']['passwd'] = '' + options['new']['path'] = '/mnt/c/Duplicati/test_de4' + options['new']['recipients'] = ['user@host.com'] + mainRestore(options) + print('Complete.') + +if __name__ == '__main__': + main() From dab0e1102bccc2582001d4945b2c0a3478706597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rad=C3=BCnz?= Date: Tue, 30 Jan 2018 23:46:36 +0100 Subject: [PATCH 02/64] Fixed issue 2994 --- Duplicati/Library/Utility/Utility.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index d31dacdf2..8847d36cc 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -586,19 +586,19 @@ namespace Duplicati.Library.Utility /// /// The size to format /// A human readable string representing the size - public static string FormatSizeString(long size) + public static string FormatSizeString(double size) { - long sizeAbs = Math.Abs(size); // Allow formatting of negative sizes + double sizeAbs = Math.Abs(size); // Allow formatting of negative sizes if (sizeAbs >= 1024 * 1024 * 1024 * 1024L) - return Strings.Utility.FormatStringTB((double)size / (1024 * 1024 * 1024 * 1024L)); + return Strings.Utility.FormatStringTB(size / (1024 * 1024 * 1024 * 1024L)); else if (sizeAbs >= 1024 * 1024 * 1024) - return Strings.Utility.FormatStringGB((double)size / (1024 * 1024 * 1024)); + return Strings.Utility.FormatStringGB(size / (1024 * 1024 * 1024)); else if (sizeAbs >= 1024 * 1024) - return Strings.Utility.FormatStringMB((double)size / (1024 * 1024)); + return Strings.Utility.FormatStringMB(size / (1024 * 1024)); else if (sizeAbs >= 1024) - return Strings.Utility.FormatStringKB((double)size / 1024); + return Strings.Utility.FormatStringKB(size / 1024); else - return Strings.Utility.FormatStringB(size); + return Strings.Utility.FormatStringB((long) size); // safe to cast because lower than 1024 and thus well within range of long } public static System.Threading.ThreadPriority ParsePriority(string value) From f6242e37745aa93a72a6f8a40a80bd7685e6cb07 Mon Sep 17 00:00:00 2001 From: Pectojin Date: Sat, 3 Feb 2018 04:15:24 +0100 Subject: [PATCH 03/64] Added an overloaded AddTask method that allows tasks to skip the queue and updated the api call backup//run to skip the queue to correspond with the expectation of the 'Run now' button in the UI --- Duplicati/Library/Utility/WorkerThread.cs | 34 +++++++++++++++++++ .../Server/WebServer/RESTMethods/Backup.cs | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Utility/WorkerThread.cs b/Duplicati/Library/Utility/WorkerThread.cs index 850ced0d9..ee4bd9b2e 100644 --- a/Duplicati/Library/Utility/WorkerThread.cs +++ b/Duplicati/Library/Utility/WorkerThread.cs @@ -162,6 +162,40 @@ namespace Duplicati.Library.Utility WorkQueueChanged(this); } + /// + /// An overloaded AddTask method that allows a task to skip to the front of a queue + /// It does this by creating a new queue, adding the new task first, and then adding + /// all the old tasks to the new queue. It's cleaner to use a linked list, + /// but the performance difference is negligible on such a small queue. + /// + /// Task. + /// If set to true skip queue. + public void AddTask(Tx task, bool skipQueue) + { + if (!skipQueue) { + // Fall back to default AddTask method + AddTask(task); + return; + } + + lock (m_lock) + { + Queue newQueue = new Queue(); + newQueue.Enqueue(task); + while (m_tasks.Count > 0) + { + Tx n = m_tasks.Dequeue(); + newQueue.Enqueue(n); + } + m_tasks = newQueue; + m_event.Set(); + } + + if (WorkQueueChanged != null) + WorkQueueChanged(this); + } + + /// /// Removes a task from the queue, does not remove the task if it is currently running /// diff --git a/Duplicati/Server/WebServer/RESTMethods/Backup.cs b/Duplicati/Server/WebServer/RESTMethods/Backup.cs index 9015c7753..485cba0f8 100644 --- a/Duplicati/Server/WebServer/RESTMethods/Backup.cs +++ b/Duplicati/Server/WebServer/RESTMethods/Backup.cs @@ -293,7 +293,7 @@ namespace Duplicati.Server.WebServer.RESTMethods } else { - Program.WorkThread.AddTask(Runner.CreateTask(DuplicatiOperation.Backup, backup)); + Program.WorkThread.AddTask(Runner.CreateTask(DuplicatiOperation.Backup, backup), true); Program.StatusEventNotifyer.SignalNewEvent(); } From cc053ff4b0274b2fa77f90d2bd530ce636ccee43 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 6 Feb 2018 12:28:54 -0700 Subject: [PATCH 04/64] Normalize whitespace --- Duplicati/Library/Main/Operation/BackupHandler.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 1bf05ab9b..0123c2b82 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -825,12 +825,12 @@ namespace Duplicati.Library.Main.Operation backend.WaitForEmpty(m_database, m_transaction); if (m_result.TaskControlRendevouz() != TaskControlState.Stop) - CompactIfRequired(backend, lastVolumeSize); - - using (new Logging.Timer("Async backend wait")) - backend.WaitForComplete(m_database, m_transaction); - - if (m_options.UploadVerificationFile) + CompactIfRequired(backend, lastVolumeSize); + + using (new Logging.Timer("Async backend wait")) + backend.WaitForComplete(m_database, m_transaction); + + if (m_options.UploadVerificationFile) { m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); FilelistProcessor.UploadVerificationFile(backend.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction); From 08e0708a9534464a7238d61b242acbffa189257d Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Tue, 6 Feb 2018 13:24:49 -0700 Subject: [PATCH 05/64] Change Duplicati's handling of reparse points, so that not all reparse points are treated as symlinks. This is inspired largely by the new (/ returning) feature of OneDrive in Windows 10 Fall Creator's Update, which downloads files on-demand. A side effect of that change is that the OneDrive folder (and subfolders of it) are marked as reparse points, even though they are not technically standard symlinks. With this change, a new extension method IsSymlink is added for ISnapshotService and ISystemIO, which checks both the file attributes (for reparse point) and the symlink target path (if null, the path is not treated as a symlink). All places that previously checked only the file attributes have been updated to use either this method (or at least the same logic, in the case of the core BackupHandler file check). One side effect of this change is that '--symlink-policy=store' no longer ignores empty symlinks - they are now treated as regular files. If there are empty symlinks, they will now be backed up as if they were regular files, but I don't know what the conditions are that create symlinks like that, so this might not effect anything in practice. --- .../Library/Main/Operation/BackupHandler.cs | 60 +++++++++++-------- .../Main/Operation/TestFilterHandler.cs | 3 +- Duplicati/Library/Snapshots/ISystemIO.cs | 1 + Duplicati/Library/Snapshots/LinuxSnapshot.cs | 2 +- .../Library/Snapshots/NoSnapshotLinux.cs | 2 +- .../Library/Snapshots/NoSnapshotWindows.cs | 6 +- .../Library/Snapshots/SnapshotUtility.cs | 55 +++++++++++++++++ Duplicati/Library/Snapshots/SystemIOLinux.cs | 5 ++ .../Library/Snapshots/SystemIOWindows.cs | 28 +++++++++ .../Library/Snapshots/WindowsSnapshot.cs | 30 ++++------ .../WebServer/RESTMethods/Filesystem.cs | 3 +- 11 files changed, 143 insertions(+), 52 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 0123c2b82..78c02a64e 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -6,6 +6,7 @@ using System.IO; using Duplicati.Library.Main.Database; using Duplicati.Library.Main.Volumes; using Duplicati.Library.Interface; +using Duplicati.Library.Snapshots; namespace Duplicati.Library.Main.Operation { @@ -187,7 +188,7 @@ namespace Duplicati.Library.Main.Operation m_logWriter.AddVerboseMessage("Including path due to filter: {0} => {1}", path, match.ToString()); } - var isSymlink = (attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint; + var isSymlink = m_snapshot.IsSymlink(path, attributes); if (isSymlink && m_symlinkPolicy == Options.SymlinkStrategy.Ignore) { if (m_logWriter != null) @@ -281,8 +282,8 @@ namespace Duplicati.Library.Main.Operation var fa = FileAttributes.Normal; try { fa = snapshot.GetAttributes(path); } catch { } - - if (followSymlinks && ((fa & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint)) + + if (followSymlinks && snapshot.IsSymlink(path, fa)) continue; else if ((fa & FileAttributes.Directory) == FileAttributes.Directory) continue; @@ -970,32 +971,39 @@ namespace Duplicati.Library.Main.Operation if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) { - if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) + // Not all reparse points are symlinks. + // For example, on Windows 10 Fall Creator's Update, the OneDrive folder (and all subfolders) + // are reparse points, which allows the folder to hook into the OneDrive service and download things on-demand. + // If we can't find a symlink target for the current path, we won't treat it as a symlink. + string symlinkTarget = snapshot.GetSymlinkTarget(path); + if (!string.IsNullOrWhiteSpace(symlinkTarget)) { - m_result.AddVerboseMessage("Ignoring symlink {0}", path); - return false; - } - - if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store) - { - Dictionary metadata = GenerateMetadata(snapshot, path, attributes); - - if (!metadata.ContainsKey("CoreSymlinkTarget")) + if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) { - var p = snapshot.GetSymlinkTarget(path); - - if (string.IsNullOrWhiteSpace(p)) - m_result.AddVerboseMessage("Ignoring empty symlink {0}", path); - else - metadata["CoreSymlinkTarget"] = p; + m_result.AddVerboseMessage("Ignoring symlink {0}", path); + return false; } - - var metahash = Utility.WrapMetadata(metadata, m_options); - AddSymlinkToOutput(backend, path, DateTime.UtcNow, metahash); - - m_result.AddVerboseMessage("Stored symlink {0}", path); - //Do not recurse symlinks - return false; + + if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store) + { + Dictionary metadata = GenerateMetadata(snapshot, path, attributes); + + if (!metadata.ContainsKey("CoreSymlinkTarget")) + { + metadata["CoreSymlinkTarget"] = symlinkTarget; + } + + var metahash = Utility.WrapMetadata(metadata, m_options); + AddSymlinkToOutput(backend, path, DateTime.UtcNow, metahash); + + m_result.AddVerboseMessage("Stored symlink {0}", path); + //Do not recurse symlinks + return false; + } + } + else + { + m_result.AddVerboseMessage("Treating empty symlink as regular path {0}", path); } } diff --git a/Duplicati/Library/Main/Operation/TestFilterHandler.cs b/Duplicati/Library/Main/Operation/TestFilterHandler.cs index 366a2ff4b..7b4b4775b 100644 --- a/Duplicati/Library/Main/Operation/TestFilterHandler.cs +++ b/Duplicati/Library/Main/Operation/TestFilterHandler.cs @@ -17,6 +17,7 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; using System.IO; +using Duplicati.Library.Snapshots; namespace Duplicati.Library.Main.Operation { @@ -44,7 +45,7 @@ namespace Duplicati.Library.Main.Operation try { fa = snapshot.GetAttributes(path); } catch { } - if (storeSymlinks && ((fa & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint)) + if (storeSymlinks && snapshot.IsSymlink(path, fa)) { m_result.AddVerboseMessage("Storing symlink: {0}", path); } diff --git a/Duplicati/Library/Snapshots/ISystemIO.cs b/Duplicati/Library/Snapshots/ISystemIO.cs index cf3fef902..1e4d01bd1 100644 --- a/Duplicati/Library/Snapshots/ISystemIO.cs +++ b/Duplicati/Library/Snapshots/ISystemIO.cs @@ -50,6 +50,7 @@ namespace Duplicati.Library.Snapshots FileAttributes GetFileAttributes(string path); void SetFileAttributes(string path, FileAttributes attributes); void CreateSymlink(string symlinkfile, string target, bool asDir); + string GetSymlinkTarget(string path); string PathGetDirectoryName(string path); string PathGetFileName(string path); string PathGetExtension(string path); diff --git a/Duplicati/Library/Snapshots/LinuxSnapshot.cs b/Duplicati/Library/Snapshots/LinuxSnapshot.cs index 714ea00ef..f0723bee9 100644 --- a/Duplicati/Library/Snapshots/LinuxSnapshot.cs +++ b/Duplicati/Library/Snapshots/LinuxSnapshot.cs @@ -455,7 +455,7 @@ namespace Duplicati.Library.Snapshots public string GetSymlinkTarget(string file) { var local = ConvertToSnapshotPath(FindSnapShotByLocalPath(file), file); - return UnixSupport.File.GetSymlinkTarget(NoSnapshot.NormalizePath(local)); + return _sysIO.GetSymlinkTarget(local); } /// diff --git a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs index b4e8dc79a..3aa6f8343 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotLinux.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotLinux.cs @@ -45,7 +45,7 @@ namespace Duplicati.Library.Snapshots /// The symlink target public override string GetSymlinkTarget(string file) { - return UnixSupport.File.GetSymlinkTarget(NormalizePath(file)); + return _sysIO.GetSymlinkTarget(file); } /// diff --git a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs index 3aa2fa3e1..d4a692ee8 100644 --- a/Duplicati/Library/Snapshots/NoSnapshotWindows.cs +++ b/Duplicati/Library/Snapshots/NoSnapshotWindows.cs @@ -46,11 +46,7 @@ namespace Duplicati.Library.Snapshots /// The symlink target public override string GetSymlinkTarget(string file) { - try { return File.GetLinkTargetInfo(SystemIOWindows.PrefixWithUNC(file)).PrintName; } - catch (NotAReparsePointException) { } - catch (UnrecognizedReparsePointException) { } - - return null; + return m_sysIO.GetSymlinkTarget(file); } /// diff --git a/Duplicati/Library/Snapshots/SnapshotUtility.cs b/Duplicati/Library/Snapshots/SnapshotUtility.cs index 9d02515be..4875b8239 100644 --- a/Duplicati/Library/Snapshots/SnapshotUtility.cs +++ b/Duplicati/Library/Snapshots/SnapshotUtility.cs @@ -19,6 +19,7 @@ #endregion using System; using System.Collections.Generic; +using System.IO; using System.Text; namespace Duplicati.Library.Snapshots @@ -71,6 +72,60 @@ namespace Duplicati.Library.Snapshots return new WindowsSnapshot(folders, options); } + /// + /// Extension method for ISnapshotService which determines whether the given path is a symlink. + /// + /// ISnapshotService implementation + /// File or folder path + /// Whether the path is a symlink + public static bool IsSymlink(this ISnapshotService snapshot, string path) + { + return snapshot.IsSymlink(path, snapshot.GetAttributes(path)); + } + + /// + /// Extension method for ISnapshotService which determines whether the given path is a symlink. + /// + /// ISnapshotService implementation + /// File or folder path + /// File attributes + /// Whether the path is a symlink + public static bool IsSymlink(this ISnapshotService snapshot, string path, FileAttributes attributes) + { + // Not all reparse points are symlinks. + // For example, on Windows 10 Fall Creator's Update, the OneDrive folder (and all subfolders) + // are reparse points, which allows the folder to hook into the OneDrive service and download things on-demand. + // If we can't find a symlink target for the current path, we won't treat it as a symlink. + return (attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint && !string.IsNullOrEmpty(snapshot.GetSymlinkTarget(path)); + } + + /// + /// Extension method for ISystemIO which determines whether the given path is a symlink. + /// + /// ISystemIO implementation + /// File or folder path + /// Whether the path is a symlink + public static bool IsSymlink(this ISystemIO systemIO, string path) + { + return systemIO.IsSymlink(path, systemIO.GetFileAttributes(path)); + } + + /// + /// Extension method for ISystemIO which determines whether the given path is a symlink. + /// + /// ISystemIO implementation + /// File or folder path + /// File attributes + /// Whether the path is a symlink + public static bool IsSymlink(this ISystemIO systemIO, string path, FileAttributes attributes) + { + // Not all reparse points are symlinks. + // For example, on Windows 10 Fall Creator's Update, the OneDrive folder (and all subfolders) + // are reparse points, which allows the folder to hook into the OneDrive service and download things on-demand. + // If we can't find a symlink target for the current path, we won't treat it as a symlink. + return (attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint && !string.IsNullOrEmpty(systemIO.GetSymlinkTarget(path)); + } + /// /// Gets an interface for System.IO, which wraps all operations in a platform consistent manner. /// diff --git a/Duplicati/Library/Snapshots/SystemIOLinux.cs b/Duplicati/Library/Snapshots/SystemIOLinux.cs index f97462c66..56ea72b4b 100644 --- a/Duplicati/Library/Snapshots/SystemIOLinux.cs +++ b/Duplicati/Library/Snapshots/SystemIOLinux.cs @@ -104,6 +104,11 @@ namespace Duplicati.Library.Snapshots { UnixSupport.File.CreateSymlink(symlinkfile, target); } + + public string GetSymlinkTarget(string path) + { + return UnixSupport.File.GetSymlinkTarget(NoSnapshot.NormalizePath(path)); + } public string PathGetDirectoryName(string path) { diff --git a/Duplicati/Library/Snapshots/SystemIOWindows.cs b/Duplicati/Library/Snapshots/SystemIOWindows.cs index aba8c26de..01b079b0c 100644 --- a/Duplicati/Library/Snapshots/SystemIOWindows.cs +++ b/Duplicati/Library/Snapshots/SystemIOWindows.cs @@ -21,6 +21,8 @@ using System.Collections.Generic; using System.Security.AccessControl; using System.IO; +using AlphaFS = Alphaleonis.Win32.Filesystem; + namespace Duplicati.Library.Snapshots { @@ -261,6 +263,32 @@ namespace Duplicati.Library.Snapshots throw new System.IO.IOException(string.Format("Unable to create symlink, check account permissions: {0}", symlinkfile)); } + /// + /// Returns the symlink target if the entry is a symlink, and null otherwise + /// + /// The file or folder to examine + /// The symlink target + public string GetSymlinkTarget(string file) + { + try + { + try + { + return AlphaFS.File.GetLinkTargetInfo(file).PrintName; + } + catch (PathTooLongException) { } + + return AlphaFS.File.GetLinkTargetInfo(SystemIOWindows.PrefixWithUNC(file)).PrintName; + } + catch (AlphaFS.NotAReparsePointException) { } + catch (AlphaFS.UnrecognizedReparsePointException) { } + + // This path looks like it isn't actually a symlink + // (Note that some reparse points aren't actually symlinks - + // things like the OneDrive folder in the Windows 10 Fall Creator's Update for example) + return null; + } + public IEnumerable EnumerateFileSystemEntries(string path) { if (!IsPathTooLong(path)) diff --git a/Duplicati/Library/Snapshots/WindowsSnapshot.cs b/Duplicati/Library/Snapshots/WindowsSnapshot.cs index 14104c9c5..075784ce9 100644 --- a/Duplicati/Library/Snapshots/WindowsSnapshot.cs +++ b/Duplicati/Library/Snapshots/WindowsSnapshot.cs @@ -25,6 +25,8 @@ using System.Collections.Generic; using System.IO; using Alphaleonis.Win32.Vss; +using AlphaFS = Alphaleonis.Win32.Filesystem; + namespace Duplicati.Library.Snapshots { /// @@ -129,7 +131,7 @@ namespace Duplicati.Library.Snapshots m_volumes = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (string s in m_sourcepaths) { - string drive = Alphaleonis.Win32.Filesystem.Path.GetPathRoot(s); + string drive = AlphaFS.Path.GetPathRoot(s); if (!m_volumes.ContainsKey(drive)) { if (!m_backup.IsVolumeSupported(drive)) @@ -198,14 +200,14 @@ namespace Duplicati.Library.Snapshots /// A list of non-shadow paths private string[] ListFolders(string folder) { - string root = Utility.Utility.AppendDirSeparator(Alphaleonis.Win32.Filesystem.Path.GetPathRoot(folder)); + string root = Utility.Utility.AppendDirSeparator(AlphaFS.Path.GetPathRoot(folder)); string volumePath = Utility.Utility.AppendDirSeparator(GetSnapshotPath(root)); string[] tmp = null; string spath = GetSnapshotPath(folder); if (SystemIOWindows.IsPathTooLong(spath)) - try { tmp = Alphaleonis.Win32.Filesystem.Directory.GetDirectories(spath); } + try { tmp = AlphaFS.Directory.GetDirectories(spath); } catch (PathTooLongException) { } catch (DirectoryNotFoundException) { } else @@ -216,7 +218,7 @@ namespace Duplicati.Library.Snapshots { spath = SystemIOWindows.PrefixWithUNC(spath); volumePath = SystemIOWindows.PrefixWithUNC(volumePath); - tmp = Alphaleonis.Win32.Filesystem.Directory.GetDirectories(spath); + tmp = AlphaFS.Directory.GetDirectories(spath); } volumePath = SystemIOWindows.PrefixWithUNC(volumePath); @@ -235,14 +237,14 @@ namespace Duplicati.Library.Snapshots /// A list of non-shadow paths private string[] ListFiles(string folder) { - string root = Utility.Utility.AppendDirSeparator(Alphaleonis.Win32.Filesystem.Path.GetPathRoot(folder)); + string root = Utility.Utility.AppendDirSeparator(AlphaFS.Path.GetPathRoot(folder)); string volumePath = Utility.Utility.AppendDirSeparator(GetSnapshotPath(root)); string[] tmp = null; string spath = GetSnapshotPath(folder); if (SystemIOWindows.IsPathTooLong(spath)) - try { tmp = Alphaleonis.Win32.Filesystem.Directory.GetFiles(spath); } + try { tmp = AlphaFS.Directory.GetFiles(spath); } catch (PathTooLongException) { } catch (DirectoryNotFoundException) { } else @@ -253,7 +255,7 @@ namespace Duplicati.Library.Snapshots { spath = SystemIOWindows.PrefixWithUNC(spath); volumePath = SystemIOWindows.PrefixWithUNC(volumePath); - tmp = Alphaleonis.Win32.Filesystem.Directory.GetFiles(spath); + tmp = AlphaFS.Directory.GetFiles(spath); } volumePath = SystemIOWindows.PrefixWithUNC(volumePath); @@ -273,7 +275,7 @@ namespace Duplicati.Library.Snapshots if (!Path.IsPathRooted(localPath)) throw new InvalidOperationException(); - string root = Alphaleonis.Win32.Filesystem.Path.GetPathRoot(localPath); + string root = AlphaFS.Path.GetPathRoot(localPath); string volumePath; if (!m_volumeMap.TryGetValue(root, out volumePath)) @@ -318,7 +320,7 @@ namespace Duplicati.Library.Snapshots } catch (PathTooLongException) { } - return Alphaleonis.Win32.Filesystem.File.GetLastWriteTimeUtc(SystemIOWindows.PrefixWithUNC(spath)); + return AlphaFS.File.GetLastWriteTimeUtc(SystemIOWindows.PrefixWithUNC(spath)); } /// @@ -336,7 +338,7 @@ namespace Duplicati.Library.Snapshots } catch (PathTooLongException) { } - return Alphaleonis.Win32.Filesystem.File.GetCreationTimeUtc(SystemIOWindows.PrefixWithUNC(spath)); + return AlphaFS.File.GetCreationTimeUtc(SystemIOWindows.PrefixWithUNC(spath)); } /// @@ -377,13 +379,7 @@ namespace Duplicati.Library.Snapshots public string GetSymlinkTarget(string file) { string spath = GetSnapshotPath(file); - try - { - return Alphaleonis.Win32.Filesystem.File.GetLinkTargetInfo(spath).PrintName; - } - catch (PathTooLongException) { } - - return Alphaleonis.Win32.Filesystem.File.GetLinkTargetInfo(SystemIOWindows.PrefixWithUNC(spath)).PrintName; + return _ioWin.GetSymlinkTarget(spath); } /// diff --git a/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs b/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs index f93f72013..a6553a527 100644 --- a/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs +++ b/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs @@ -18,6 +18,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.IO; +using Duplicati.Library.Snapshots; namespace Duplicati.Server.WebServer.RESTMethods { @@ -215,7 +216,7 @@ namespace Duplicati.Server.WebServer.RESTMethods try { var attr = systemIO.GetFileAttributes(s); - var isSymlink = (attr & FileAttributes.ReparsePoint) != 0; + var isSymlink = systemIO.IsSymlink(s, attr); var isFolder = (attr & FileAttributes.Directory) != 0; var isFile = !isFolder; var isHidden = (attr & FileAttributes.Hidden) != 0; From 62351d4b4824642983cc01d8676f8b8e7d14b92b Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Wed, 7 Feb 2018 10:35:16 -0700 Subject: [PATCH 06/64] Write standard output even when the --verbose flag is specified. Some information about the backup diagnostics doesn't seem to get logged when using the --verbose flag. So this changes it to log the verbose stuff first, and then also log the regular backup diagnostics. --- Duplicati/CommandLine/Commands.cs | 49 +++++++++++++++---------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/Duplicati/CommandLine/Commands.cs b/Duplicati/CommandLine/Commands.cs index 2957d29b8..1601dac01 100644 --- a/Duplicati/CommandLine/Commands.cs +++ b/Duplicati/CommandLine/Commands.cs @@ -588,37 +588,36 @@ namespace Duplicati.CommandLine if (output.VerboseOutput) { Library.Utility.Utility.PrintSerializeObject(result, outwriter); + outwriter.WriteLine(); } - else + + var parsedStats = result.BackendStatistics as Duplicati.Library.Interface.IParsedBackendStatistics; + output.MessageEvent(string.Format(" Duration of backup: {0:hh\\:mm\\:ss}", result.Duration)); + if (parsedStats != null) { - var parsedStats = result.BackendStatistics as Duplicati.Library.Interface.IParsedBackendStatistics; - output.MessageEvent(string.Format(" Duration of backup: {0:hh\\:mm\\:ss}", result.Duration)); - if (parsedStats != null) + if (parsedStats.KnownFileCount > 0) { - if (parsedStats.KnownFileCount > 0) - { - output.MessageEvent(string.Format(" Remote files: {0}", parsedStats.KnownFileCount)); - output.MessageEvent(string.Format(" Remote size: {0}", Library.Utility.Utility.FormatSizeString(parsedStats.KnownFileSize))); - } - - if (parsedStats.TotalQuotaSpace >= 0) - { - output.MessageEvent(string.Format(" Total remote quota: {0}", Library.Utility.Utility.FormatSizeString(parsedStats.TotalQuotaSpace))); - } - - if (parsedStats.FreeQuotaSpace >= 0) - { - output.MessageEvent(string.Format(" Available remote quota: {0}", Library.Utility.Utility.FormatSizeString(parsedStats.FreeQuotaSpace))); - } + output.MessageEvent(string.Format(" Remote files: {0}", parsedStats.KnownFileCount)); + output.MessageEvent(string.Format(" Remote size: {0}", Library.Utility.Utility.FormatSizeString(parsedStats.KnownFileSize))); } - - output.MessageEvent(string.Format(" Files added: {0}", result.AddedFiles)); - output.MessageEvent(string.Format(" Files deleted: {0}", result.DeletedFiles)); - output.MessageEvent(string.Format(" Files changed: {0}", result.ModifiedFiles)); - output.MessageEvent(string.Format(" Data uploaded: {0}", Library.Utility.Utility.FormatSizeString(result.BackendStatistics.BytesUploaded))); - output.MessageEvent(string.Format(" Data downloaded: {0}", Library.Utility.Utility.FormatSizeString(result.BackendStatistics.BytesDownloaded))); + if (parsedStats.TotalQuotaSpace >= 0) + { + output.MessageEvent(string.Format(" Total remote quota: {0}", Library.Utility.Utility.FormatSizeString(parsedStats.TotalQuotaSpace))); + } + + if (parsedStats.FreeQuotaSpace >= 0) + { + output.MessageEvent(string.Format(" Available remote quota: {0}", Library.Utility.Utility.FormatSizeString(parsedStats.FreeQuotaSpace))); + } } + + output.MessageEvent(string.Format(" Files added: {0}", result.AddedFiles)); + output.MessageEvent(string.Format(" Files deleted: {0}", result.DeletedFiles)); + output.MessageEvent(string.Format(" Files changed: {0}", result.ModifiedFiles)); + + output.MessageEvent(string.Format(" Data uploaded: {0}", Library.Utility.Utility.FormatSizeString(result.BackendStatistics.BytesUploaded))); + output.MessageEvent(string.Format(" Data downloaded: {0}", Library.Utility.Utility.FormatSizeString(result.BackendStatistics.BytesDownloaded))); if (result.ExaminedFiles == 0 && (filter != null || !filter.Empty)) output.MessageEvent("No files were processed. If this was not intentional you may want to use the \"test-filters\" command"); From 0a1e98712a99c981149b6f397cfd55bddfa9d0a9 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Wed, 7 Feb 2018 11:23:52 -0700 Subject: [PATCH 07/64] Adds warnings and errors if the reported available quota either reaches a configurable percentage of the backup size, or when it is completely exhausted. This is configurable via the --quota-warning-threshold command line parameter. --- .../Main/Operation/FilelistProcessor.cs | 24 +++++++++++++- Duplicati/Library/Main/Options.cs | 31 ++++++++++++++++++- Duplicati/Library/Main/ResultClasses.cs | 6 ++++ Duplicati/Library/Main/Strings.cs | 2 ++ 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 042bd6e15..3a8d8e09b 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -210,7 +210,8 @@ namespace Duplicati.Library.Main.Operation select n).ToList(); log.KnownFileCount = remotelist.Count; - log.KnownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum(); + long knownFileSize = remotelist.Select(x => Math.Max(0, x.File.Size)).Sum(); + log.KnownFileSize = knownFileSize; log.UnknownFileCount = unknownlist.Count; log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum(); log.BackupListCount = filesets.Count; @@ -225,6 +226,27 @@ namespace Duplicati.Library.Main.Operation { log.TotalQuotaSpace = quota.TotalQuotaSpace; log.FreeQuotaSpace = quota.FreeQuotaSpace; + + // Check to see if there should be a warning or error about the quota + // Since this processor may be called multiple times during a backup + // (both at the start and end, for example), the log keeps track of + // whether a quota error or warning has been sent already. + // Note that an error can still be sent later even if a warning was sent earlier. + if (!log.ReportedQuotaError && quota.FreeQuotaSpace == 0) + { + log.ReportedQuotaError = true; + log.AddError(string.Format("Backend quota has been exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)), null); + } + else if (!log.ReportedQuotaWarning && !log.ReportedQuotaError && quota.FreeQuotaSpace >= 0) // Negative value means the backend didn't return the quota info + { + // Warnings are sent if the available free space is less than the given percentage of the total backup size. + double warningThreshold = options.QuotaWarningThreshold / (double)100; + if (quota.FreeQuotaSpace < warningThreshold * knownFileSize) + { + log.ReportedQuotaWarning = true; + log.AddWarning(string.Format("Backend quota is close to being exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)), null); + } + } } } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 2ca2c7af9..c917ca3e2 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -74,6 +74,11 @@ namespace Duplicati.Library.Main /// private const string DEFAULT_LOG_RETENTION = "30D"; + /// + /// The default threshold for warning about coming close to quota + /// + private const int DEFAULT_QUOTA_WARNING_THRESHOLD = 10; + /// /// An enumeration that describes the supported strategies for an optimization /// @@ -460,8 +465,9 @@ namespace Duplicati.Library.Main new CommandLineArgument("list-verify-uploads", CommandLineArgument.ArgumentType.Boolean, Strings.Options.ListverifyuploadsShort, Strings.Options.ListverifyuploadsShort, "false"), new CommandLineArgument("allow-sleep", CommandLineArgument.ArgumentType.Boolean, Strings.Options.AllowsleepShort, Strings.Options.AllowsleepLong, "false"), new CommandLineArgument("no-connection-reuse", CommandLineArgument.ArgumentType.Boolean, Strings.Options.NoconnectionreuseShort, Strings.Options.NoconnectionreuseLong, "false"), - + new CommandLineArgument("quota-size", CommandLineArgument.ArgumentType.Size, Strings.Options.QuotasizeShort, Strings.Options.QuotasizeLong), + new CommandLineArgument("quota-warning-threshold", CommandLineArgument.ArgumentType.Integer, Strings.Options.QuotaWarningThresholdShort, Strings.Options.QuotaWarningThresholdLong, DEFAULT_QUOTA_WARNING_THRESHOLD.ToString()), new CommandLineArgument("default-filters", CommandLineArgument.ArgumentType.String, Strings.Options.DefaultFiltersShort, Strings.Options.DefaultFiltersLong(DefaultFilterSet.Windows.ToString(), DefaultFilterSet.OSX.ToString(), DefaultFilterSet.Linux.ToString(), DefaultFilterSet.All.ToString()), string.Empty, new[] { "default-filter" }), @@ -1303,6 +1309,29 @@ namespace Duplicati.Library.Main } } + /// + /// Gets the threshold at which a quota warning should be generated. + /// + /// + /// This is treated as a percentage, where a warning is given when the amount of free space is less than this percentage of the backup size. + /// + public int QuotaWarningThreshold + { + get + { + string tmp; + m_options.TryGetValue("quota-warning-threshold", out tmp); + if (string.IsNullOrEmpty(tmp)) + { + return DEFAULT_QUOTA_WARNING_THRESHOLD; + } + else + { + return int.Parse(tmp); + } + } + } + /// /// Gets the display name of the backup /// diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 78c4733ec..eb3ea0cbb 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -49,6 +49,9 @@ namespace Duplicati.Library.Main long FreeQuotaSpace { set; } long AssignedQuotaSpace { set; } + bool ReportedQuotaError { get; set; } + bool ReportedQuotaWarning { get; set; } + /// /// The backend sends this event when performing an action /// @@ -126,6 +129,9 @@ namespace Duplicati.Library.Main public long FreeQuotaSpace { get; set; } public long AssignedQuotaSpace { get; set; } + public bool ReportedQuotaError { get; set; } + public bool ReportedQuotaWarning { get; set; } + public override OperationMode MainOperation { get { return m_parent.MainOperation; } } public void SendEvent(BackendActionType action, BackendEventType type, string path, long size) diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 7f774ea91..7e4e509a8 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -119,6 +119,8 @@ namespace Duplicati.Library.Main.Strings public static string UploadUnchangedBackupsShort { get { return LC.L(@"Upload empty backup files"); } } public static string QuotasizeLong { get { return LC.L(@"This value can be used to set a known upper limit on the amount of space a backend has. If the backend reports the size itself, this value is ignored"); } } public static string QuotasizeShort { get { return LC.L(@"A reported maximum storage"); } } + public static string QuotaWarningThresholdLong { get { return LC.L(@"Sets a threshold for when to warn about the backend quota being nearly exceeded. It is given as a percentage, and a warning is generated if the amount of available quota is less that this percentage of the total backup size. If the backend does not report the quota information, this value will be ignored"); } } + public static string QuotaWarningThresholdShort { get { return LC.L(@"Threshold for warning about low quota"); } } public static string DefaultFiltersLong(string windows, string osx, string linux, string all) { return LC.L(@"Exclude files that match the given filter sets. Which default filter sets should be used. Valid sets are ""{0}"", ""{1}"", ""{2}"", and ""{3}"". If this parameter is set with no value, the set for the current operating system will be used.", windows, osx, linux, all); } public static string DefaultFiltersShort { get { return LC.L(@"Default filter sets"); } } public static string SymlinkpolicyShort { get { return LC.L(@"Symlink handling"); } } From 1241a7747dc6a91bd05dd6809ce7595269701ed3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 7 Feb 2018 21:56:02 +0100 Subject: [PATCH 08/64] Added cache expiration headers to static and dynamic responses from the server. This should help with #2699 --- Duplicati/Server/WebServer/BodyWriter.cs | 4 ++- .../Server/WebServer/IndexHtmlHandler.cs | 1 + Duplicati/Server/WebServer/Server.cs | 26 ++++++++++++++++--- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Duplicati/Server/WebServer/BodyWriter.cs b/Duplicati/Server/WebServer/BodyWriter.cs index 9757cb5b4..6868a1673 100644 --- a/Duplicati/Server/WebServer/BodyWriter.cs +++ b/Duplicati/Server/WebServer/BodyWriter.cs @@ -38,10 +38,12 @@ namespace Duplicati.Server.WebServer } public BodyWriter(HttpServer.IHttpResponse resp, string jsonp) - : base(resp.Body, resp.Encoding) + : base(resp.Body, resp.Encoding) { m_resp = resp; m_jsonp = jsonp; + if (!m_resp.HeadersSent) + m_resp.AddHeader("Cache-Control", "no-cache, no-store, must-revalidate, max-age=0"); } protected override void Dispose (bool disposing) diff --git a/Duplicati/Server/WebServer/IndexHtmlHandler.cs b/Duplicati/Server/WebServer/IndexHtmlHandler.cs index bc03c4c09..73c7dac14 100644 --- a/Duplicati/Server/WebServer/IndexHtmlHandler.cs +++ b/Duplicati/Server/WebServer/IndexHtmlHandler.cs @@ -49,6 +49,7 @@ namespace Duplicati.Server.WebServer response.Status = System.Net.HttpStatusCode.OK; response.Reason = "OK"; response.ContentType = "text/html; charset=utf-8"; + response.AddHeader("Cache-Control", "no-cache, no-store, must-revalidate, max-age=0"); using (var fs = System.IO.File.OpenRead(System.IO.File.Exists(html) ? html : htm)) { diff --git a/Duplicati/Server/WebServer/Server.cs b/Duplicati/Server/WebServer/Server.cs index 7b4016879..a7bf6e01b 100644 --- a/Duplicati/Server/WebServer/Server.cs +++ b/Duplicati/Server/WebServer/Server.cs @@ -271,26 +271,26 @@ namespace Duplicati.Server.WebServer if (install_webroot != webroot && System.IO.Directory.Exists(System.IO.Path.Combine(install_webroot, "customized"))) { - var customized_files = new FileModule("/customized/", System.IO.Path.Combine(install_webroot, "customized")); + var customized_files = new CacheControlFileHandler("/customized/", System.IO.Path.Combine(install_webroot, "customized")); AddMimeTypes(customized_files); server.Add(customized_files); } if (install_webroot != webroot && System.IO.Directory.Exists(System.IO.Path.Combine(install_webroot, "oem"))) { - var oem_files = new FileModule("/oem/", System.IO.Path.Combine(install_webroot, "oem")); + var oem_files = new CacheControlFileHandler("/oem/", System.IO.Path.Combine(install_webroot, "oem")); AddMimeTypes(oem_files); server.Add(oem_files); } if (install_webroot != webroot && System.IO.Directory.Exists(System.IO.Path.Combine(install_webroot, "package"))) { - var proxy_files = new FileModule("/proxy/", System.IO.Path.Combine(install_webroot, "package")); + var proxy_files = new CacheControlFileHandler("/proxy/", System.IO.Path.Combine(install_webroot, "package")); AddMimeTypes(proxy_files); server.Add(proxy_files); } - var fh = new FileModule("/", webroot, true); + var fh = new CacheControlFileHandler("/", webroot, true); AddMimeTypes(fh); server.Add(fh); @@ -310,5 +310,23 @@ namespace Duplicati.Server.WebServer return false; } } + + private class CacheControlFileHandler : FileModule + { + public CacheControlFileHandler(string baseUri, string basePath, bool useLastModifiedHeader = false) + : base(baseUri, basePath, useLastModifiedHeader) + { + + } + + public override bool Process(HttpServer.IHttpRequest request, HttpServer.IHttpResponse response, HttpServer.Sessions.IHttpSession session) + { + if (!this.CanHandle(request.Uri)) + return false; + + response.AddHeader("Cache-Control", "max-age=" + (60 * 60 * 24)); + return base.Process(request, response, session); + } + } } } From 58146af6b74a9c99645c9d135429c2cb6b763ccc Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 7 Feb 2018 23:35:13 +0100 Subject: [PATCH 09/64] Changed XSRF tokens to only use url-safe chars, as the webserver and browsers fight a bit with it. --- Duplicati/Server/WebServer/AuthenticationHandler.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Duplicati/Server/WebServer/AuthenticationHandler.cs b/Duplicati/Server/WebServer/AuthenticationHandler.cs index 908d6ae6f..1ed058607 100644 --- a/Duplicati/Server/WebServer/AuthenticationHandler.cs +++ b/Duplicati/Server/WebServer/AuthenticationHandler.cs @@ -51,7 +51,7 @@ namespace Duplicati.Server.WebServer { string xsrftoken = request.Headers[XSRF_HEADER_NAME] ?? ""; - if (string.IsNullOrWhiteSpace(xsrftoken)) + if (!string.IsNullOrWhiteSpace(xsrftoken)) xsrftoken = Duplicati.Library.Utility.Uri.UrlDecode(xsrftoken); if (string.IsNullOrWhiteSpace(xsrftoken)) @@ -77,7 +77,8 @@ namespace Duplicati.Server.WebServer var buf = new byte[32]; var expires = DateTime.UtcNow.AddMinutes(XSRF_TIMEOUT_MINUTES); m_prng.GetBytes(buf); - var token = Convert.ToBase64String(buf); + // Don't use non-safe chars, as the current webserver has some "trouble" parsing it + var token = BitConverter.ToString(buf).Replace("-", string.Empty); //Convert.ToBase64String(buf); m_activexsrf.AddOrUpdate(token, key => expires, (key, existingExpires) => { From 9a41cf286712fe025ca07bdc7406eeb672c272d8 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 7 Feb 2018 23:37:55 +0100 Subject: [PATCH 10/64] Fixed not allowing the browser to cache the index.html page. This should fix #2699, #2635 --- Duplicati/Server/WebServer/Server.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Duplicati/Server/WebServer/Server.cs b/Duplicati/Server/WebServer/Server.cs index a7bf6e01b..ead1e596a 100644 --- a/Duplicati/Server/WebServer/Server.cs +++ b/Duplicati/Server/WebServer/Server.cs @@ -324,7 +324,10 @@ namespace Duplicati.Server.WebServer if (!this.CanHandle(request.Uri)) return false; - response.AddHeader("Cache-Control", "max-age=" + (60 * 60 * 24)); + if (request.Uri.AbsolutePath.EndsWith("index.html", StringComparison.Ordinal) || request.Uri.AbsolutePath.EndsWith("index.htm", StringComparison.Ordinal)) + response.AddHeader("Cache-Control", "no-cache, no-store, must-revalidate, max-age=0"); + else + response.AddHeader("Cache-Control", "max-age=" + (60 * 60 * 24)); return base.Process(request, response, session); } } From d16c0f24ecbd2e2d2233196345e36ee3f202b451 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 8 Feb 2018 09:03:51 +0100 Subject: [PATCH 11/64] Removed some unnecessary changes, turned out it was a simple cache issue. This fixes #2699 This fixes #2635 --- Duplicati/Server/WebServer/AuthenticationHandler.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Duplicati/Server/WebServer/AuthenticationHandler.cs b/Duplicati/Server/WebServer/AuthenticationHandler.cs index 1ed058607..0744807b8 100644 --- a/Duplicati/Server/WebServer/AuthenticationHandler.cs +++ b/Duplicati/Server/WebServer/AuthenticationHandler.cs @@ -51,9 +51,6 @@ namespace Duplicati.Server.WebServer { string xsrftoken = request.Headers[XSRF_HEADER_NAME] ?? ""; - if (!string.IsNullOrWhiteSpace(xsrftoken)) - xsrftoken = Duplicati.Library.Utility.Uri.UrlDecode(xsrftoken); - if (string.IsNullOrWhiteSpace(xsrftoken)) { var xsrfq = request.Form[XSRF_HEADER_NAME] ?? request.Form[Duplicati.Library.Utility.Uri.UrlEncode(XSRF_HEADER_NAME)]; @@ -77,8 +74,7 @@ namespace Duplicati.Server.WebServer var buf = new byte[32]; var expires = DateTime.UtcNow.AddMinutes(XSRF_TIMEOUT_MINUTES); m_prng.GetBytes(buf); - // Don't use non-safe chars, as the current webserver has some "trouble" parsing it - var token = BitConverter.ToString(buf).Replace("-", string.Empty); //Convert.ToBase64String(buf); + var token = Convert.ToBase64String(buf); m_activexsrf.AddOrUpdate(token, key => expires, (key, existingExpires) => { From db7c3802390805538e8d3744a119f67cfb7a3c42 Mon Sep 17 00:00:00 2001 From: Pectojin Date: Thu, 8 Feb 2018 20:53:40 +0100 Subject: [PATCH 12/64] added the blueray container .m2ts to the no_compression list --- Duplicati/Library/Main/default_compressed_extensions.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/Duplicati/Library/Main/default_compressed_extensions.txt b/Duplicati/Library/Main/default_compressed_extensions.txt index 77e22fd88..7b189bf7a 100644 --- a/Duplicati/Library/Main/default_compressed_extensions.txt +++ b/Duplicati/Library/Main/default_compressed_extensions.txt @@ -82,6 +82,7 @@ .webm #WebM Video File .wmv #Windows Media Video File .wtv #Windows Recorded TV Show +.m2ts # Blu-ray Disc MPEG-2 Transport Stream # Compressed image files From 065ec87ccc38e4cc7a1e5e097e74a4de265b5b7b Mon Sep 17 00:00:00 2001 From: Pectojin Date: Thu, 8 Feb 2018 21:45:11 +0100 Subject: [PATCH 13/64] made the side menu fixed so it follows as you scroll --- Duplicati/Server/webroot/ngax/less/style.less | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Server/webroot/ngax/less/style.less b/Duplicati/Server/webroot/ngax/less/style.less index 198b6cff0..8fff83afb 100755 --- a/Duplicati/Server/webroot/ngax/less/style.less +++ b/Duplicati/Server/webroot/ngax/less/style.less @@ -677,6 +677,7 @@ body { width: 260px; padding-left: 40px; float: left; + position: fixed; > ul { > li { @@ -883,7 +884,7 @@ body { .content { float: left; - padding-left: 50px; + padding-left: 350px; padding-bottom: 50px; max-width: 700px; From d8bd9abe152ff82f27d4d2d912a6efa83ae89697 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 8 Feb 2018 21:45:36 +0100 Subject: [PATCH 14/64] Fixed a problem with the box.com backend not being able to find files it already uploaded. This fixes #2557 This might also fix #2349 --- Duplicati/Library/Backend/Box/BoxBackend.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Backend/Box/BoxBackend.cs b/Duplicati/Library/Backend/Box/BoxBackend.cs index 02c6d2cca..b20b259c7 100644 --- a/Duplicati/Library/Backend/Box/BoxBackend.cs +++ b/Duplicati/Library/Backend/Box/BoxBackend.cs @@ -145,7 +145,8 @@ namespace Duplicati.Library.Backend.Box if (m_filecache != null && m_filecache.ContainsKey(name)) return m_filecache[name]; - PagedFileListResponse(CurrentFolder, false); + // Make sure we enumerate this, otherwise the m_filecache is not assigned + PagedFileListResponse(CurrentFolder, false).LastOrDefault(); if (m_filecache != null && m_filecache.ContainsKey(name)) return m_filecache[name]; From 8190eb8d9c3e9f5e9a81203b96d9603cc470db92 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Fri, 9 Feb 2018 11:29:00 -0700 Subject: [PATCH 15/64] When running a backup via the GUI, filters are loaded from the database instead of from the options, which means that the default filters aren't taken into account. This change updates the ApplyFilter method used by the Server's Runner method to search for the default filters option, and append those filters to the list if they are found. This should support both global and per-backup settings for default filters, as it checks the options after the two are merged. --- Duplicati/Server/Runner.cs | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/Duplicati/Server/Runner.cs b/Duplicati/Server/Runner.cs index 45fc863c7..b19339477 100644 --- a/Duplicati/Server/Runner.cs +++ b/Duplicati/Server/Runner.cs @@ -512,7 +512,7 @@ namespace Duplicati.Server } } - using(tempfolder) + using (tempfolder) using(var controller = new Duplicati.Library.Main.Controller(backup.TargetURL, options, sink)) { try @@ -536,7 +536,14 @@ namespace Duplicati.Server { case DuplicatiOperation.Backup: { - var filter = ApplyFilter(backup, data.Operation, GetCommonFilter(backup, data.Operation)); + IEnumerable defaultFilters = null; + string defaultFiltersValue; + if (options.TryGetValue("default-filters", out defaultFiltersValue) || options.TryGetValue("default-filter", out defaultFiltersValue)) + { + defaultFilters = Library.Utility.DefaultFilters.GetFilters(Library.Utility.Utility.ExpandEnvironmentVariables(defaultFiltersValue ?? string.Empty)); + } + + var filter = ApplyFilter(backup, data.Operation, GetCommonFilter(backup, data.Operation), defaultFilters); var sources = (from n in backup.Sources let p = SpecialFolders.ExpandEnvironmentVariables(n) @@ -848,7 +855,7 @@ namespace Duplicati.Server return options; } - private static Duplicati.Library.Utility.IFilter ApplyFilter(Duplicati.Server.Serialization.Interface.IBackup backup, DuplicatiOperation mode, Duplicati.Library.Utility.IFilter filter) + private static Library.Utility.IFilter ApplyFilter(Serialization.Interface.IBackup backup, DuplicatiOperation mode, Library.Utility.IFilter filter, IEnumerable defaultFilters) { var f2 = backup.Filters; if (f2 != null && f2.Length > 0) @@ -860,15 +867,20 @@ namespace Duplicati.Server ? SpecialFolders.ExpandEnvironmentVariablesRegexp(n.Expression) : SpecialFolders.ExpandEnvironmentVariables(n.Expression) orderby n.Order - select (Duplicati.Library.Utility.IFilter)(new Duplicati.Library.Utility.FilterExpression(exp, n.Include))) - .Aggregate((a, b) => Duplicati.Library.Utility.FilterExpression.Combine(a, b)); + select (Library.Utility.IFilter)(new Library.Utility.FilterExpression(exp, n.Include))) + .Aggregate((a, b) => Library.Utility.FilterExpression.Combine(a, b)); - return Duplicati.Library.Utility.FilterExpression.Combine(filter, nf); + filter = Library.Utility.FilterExpression.Combine(filter, nf); } - else - return filter; + + if (defaultFilters != null && defaultFilters.Any()) + { + filter = Library.Utility.FilterExpression.Combine(filter, defaultFilters.Aggregate((a, b) => Library.Utility.FilterExpression.Combine(a, b))); + } + + return filter; } - + private static Dictionary GetCommonOptions(Duplicati.Server.Serialization.Interface.IBackup backup, DuplicatiOperation mode) { return From 7bc9a81312fb995335d0d837d6fcef636795048e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rad=C3=BCnz?= Date: Sun, 11 Feb 2018 10:07:37 +0100 Subject: [PATCH 16/64] Delete backups outside of any retention policy time frame --- Duplicati/Library/Main/Operation/DeleteHandler.cs | 12 ++++++------ Duplicati/Library/Main/Strings.cs | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index e6e7325e0..6fde2de22 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -240,8 +240,6 @@ namespace Duplicati.Library.Main.Operation // Collect all potential backups in each time frame and thin out according to the specified interval, // starting with the oldest backup in that time frame. // The order in which the time frames values are checked has to be from the smallest to the largest. - // If backups are not within any time frame, they will NOT be deleted here. - // The --keep-time and --keep-versions switched should be used to ultimately delete backups that are too old List backupsToDelete = new List(); var now = DateTime.Now; foreach (var singleRetentionPolicyOptionValue in retentionPolicyOptionValue.OrderBy(x => x.Key)) @@ -277,16 +275,18 @@ namespace Duplicati.Library.Main.Operation } else { - Logging.Log.WriteMessage(string.Format("[Retention Policy]: Marking backup for deletion: {0}", backup), Logging.LogMessageType.Profiling); + Logging.Log.WriteMessage(string.Format("[Retention Policy]: Deleting backup: {0}", backup), Logging.LogMessageType.Profiling); backupsToDelete.Add(backup); } } } - Logging.Log.WriteMessage(string.Format("[Retention Policy]: Backups outside of all time frames and thus not checked: {0}", - string.Join(", ", clonedBackupList)), Logging.LogMessageType.Profiling); + // Delete all remaining backups + backupsToDelete.AddRange(clonedBackupList); + Logging.Log.WriteMessage(string.Format("[Retention Policy]: Backups outside of all time frames and thus getting deleted: {0}", + string.Join(", ", clonedBackupList)), Logging.LogMessageType.Information); - Logging.Log.WriteMessage(string.Format("[Retention Policy]: Backups to delete: {0}", + Logging.Log.WriteMessage(string.Format("[Retention Policy]: All backups to delete: {0}", string.Join(", ", backupsToDelete.OrderByDescending(x => x))), Logging.LogMessageType.Information); return backupsToDelete; diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 7e4e509a8..eecba8c0b 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -183,7 +183,7 @@ namespace Duplicati.Library.Main.Strings public static string KeeptimeShort { get { return LC.L(@"Keep all versions within a timespan"); } } public static string KeeptimeLong { get { return LC.L(@"Use this option to set the timespan in which backups are kept."); } } public static string RetentionPolicyShort { get { return LC.L(@"Reduce number of versions by deleting old intermediate backups"); } } - public static string RetentionPolicyLong { get { return LC.L(@"Use this option to reduce the number of versions that are kept with increasing version age by deleting most of the old backups. The expected format is a comma seperated list of collon sperated time frame and interval pairs. For example the value ""7D:0s,3M:1D,10Y:2M"" means ""For 7 day keep all backups, for 3 months keep one backup per day and for 10 years one backup every 2nd month"); } } + public static string RetentionPolicyLong { get { return LC.L(@"Use this option to reduce the number of versions that are kept with increasing version age by deleting most of the old backups. The expected format is a comma separated list of colon separated time frame and interval pairs. For example the value ""7D:0s,3M:1D,10Y:2M"" means ""For 7 day keep all backups, for 3 months keep one backup every day, for 10 years one backup every 2nd month and delete every backup older than this."""); } } public static string AllowmissingsourceShort { get { return LC.L(@"Ignore missing source elements"); } } public static string AllowmissingsourceLong { get { return LC.L(@"Use this option to continue even if some source entries are missing."); } } public static string OverwriteShort { get { return LC.L(@"Overwrite files when restoring"); } } From f8cbc911527b0fb11c87557058508eca8471df73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rad=C3=BCnz?= Date: Sun, 11 Feb 2018 10:13:34 +0100 Subject: [PATCH 17/64] Disallow using multiple rentention options (keep-time, keep-versions, rentention-policy) --- Duplicati/Library/Main/Controller.cs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index bcfb8d342..5997ab7c6 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -886,9 +886,31 @@ namespace Duplicati.Library.Main /// The log instance private void ValidateOptions(ILogWriter log) { - if (m_options.KeepTime.Ticks > 0 && m_options.KeepVersions > 0) - throw new Interface.UserInformationException(string.Format("Setting both --{0} and --{1} is not permitted", "keep-versions", "keep-time")); + // Check if only one of the retention options is set + var selectedRetentionOptions = new List(); + if (m_options.KeepTime.Ticks > 0) + { + selectedRetentionOptions.Add("keep-time"); + } + + if (m_options.KeepVersions > 0) + { + selectedRetentionOptions.Add("keep-versions"); + } + + if (m_options.RetentionPolicy.Count() > 0) + { + selectedRetentionOptions.Add("retention-policy"); + } + + if (selectedRetentionOptions.Count() > 1) + { + throw new Interface.UserInformationException(string.Format("Setting multiple retention options ({0}) is not permitted", + String.Join(", ", selectedRetentionOptions.Select(x => "--" + x)))); + } + + // Check Prefix if (!string.IsNullOrWhiteSpace(m_options.Prefix) && m_options.Prefix.Contains("-")) throw new Interface.UserInformationException("The prefix cannot contain hyphens (-)"); From d249d23bdd3f05aae2d8ff6c718e5e0d37e8cf78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rad=C3=BCnz?= Date: Sun, 11 Feb 2018 15:15:06 +0100 Subject: [PATCH 18/64] Comply with option allow-full-removal when applying retention policy --- .../Library/Main/Operation/DeleteHandler.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/DeleteHandler.cs b/Duplicati/Library/Main/Operation/DeleteHandler.cs index 6fde2de22..84539b4f6 100644 --- a/Duplicati/Library/Main/Operation/DeleteHandler.cs +++ b/Duplicati/Library/Main/Operation/DeleteHandler.cs @@ -228,8 +228,11 @@ namespace Duplicati.Library.Main.Operation // Make sure the backups are in descending order (newest backup in the beginning) clonedBackupList = clonedBackupList.OrderByDescending(x => x).ToList(); - // Most current backup should never get deleted in this process, so exclude it + // Most recent backup usually should never get deleted in this process, so exclude it for now, + // but keep a reference to potentiall delete it when allow-full-removal is set + var mostRecentBackup = clonedBackupList.ElementAt(0); clonedBackupList.RemoveAt(0); + var deleteMostRecentBackup = m_options.AllowFullRemoval; Logging.Log.WriteMessage(string.Format("[Retention Policy]: Time frames and intervals pairs: {0}", string.Join(", ", retentionPolicyOptionValue.Select(x => x.Key + " / " + x.Value))), Logging.LogMessageType.Information); @@ -279,6 +282,9 @@ namespace Duplicati.Library.Main.Operation backupsToDelete.Add(backup); } } + + // Check if most recent backup is outside of this time frame (meaning older/smaller) + deleteMostRecentBackup &= (mostRecentBackup < timeFrame); } // Delete all remaining backups @@ -286,6 +292,14 @@ namespace Duplicati.Library.Main.Operation Logging.Log.WriteMessage(string.Format("[Retention Policy]: Backups outside of all time frames and thus getting deleted: {0}", string.Join(", ", clonedBackupList)), Logging.LogMessageType.Information); + // Delete most recent backup if allow-full-removal is set and the most current backup is outside of any time frame + if (deleteMostRecentBackup) + { + backupsToDelete.Add(mostRecentBackup); + Logging.Log.WriteMessage(string.Format("[Retention Policy]: Deleting most recent backup: {0}", + mostRecentBackup), Logging.LogMessageType.Information); + } + Logging.Log.WriteMessage(string.Format("[Retention Policy]: All backups to delete: {0}", string.Join(", ", backupsToDelete.OrderByDescending(x => x))), Logging.LogMessageType.Information); From 5e1a260483a8e6cc97f8996340bce9df7463320f Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 11 Feb 2018 17:51:55 +0100 Subject: [PATCH 19/64] Added limits to the number of backlog items the usage reporter will handle. Older or excessive items are now deleted. --- .../Library/UsageReporter/EventProcessor.cs | 69 +++++++++++++------ 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/Duplicati/Library/UsageReporter/EventProcessor.cs b/Duplicati/Library/UsageReporter/EventProcessor.cs index 998f91019..f0e75414c 100644 --- a/Duplicati/Library/UsageReporter/EventProcessor.cs +++ b/Duplicati/Library/UsageReporter/EventProcessor.cs @@ -37,6 +37,16 @@ namespace Duplicati.Library.UsageReporter /// private const int MAX_QUEUE_SIZE = 500; + /// + /// The maximum number of backlog files to consider + /// + private const int MAX_BACKLOG = 6; + + /// + /// The maximum time a backlog file is considered valid + /// + private static readonly TimeSpan MAX_BACKLOG_AGE = TimeSpan.FromDays(30); + /// /// The time to wait before sending event /// @@ -85,14 +95,7 @@ namespace Duplicati.Library.UsageReporter return; } - foreach (var f in GetAbandonedFiles(null)) - { - // Check if we should exit - if (await self.Input.IsRetiredAsync) - return; - - await self.Output.WriteAsync(f); - } + await ProcessAbandonedFiles(self.Output, self.Input, null); var rs = new ReportSet(); var tf = GetTempFilename(instanceid); @@ -131,13 +134,7 @@ namespace Duplicati.Library.UsageReporter self.Output.WriteNoWait(tf); rs = new ReportSet(); - foreach (var f in GetAbandonedFiles(tf)) - { - if (await self.Input.IsRetiredAsync) - return; - - self.Output.WriteNoWait(f); - } + await ProcessAbandonedFiles(self.Output, self.Input, null); tf = nextFilename; } @@ -148,18 +145,48 @@ namespace Duplicati.Library.UsageReporter return new Tuple>(task, channel); } + + /// + /// Transmits abandoned files to the server, and enforces discard rules + /// + /// An awaitable task. + /// The target channel. + /// The source channel, used to check for stopping conditions + /// The current file, which should not be uploaded. + private static async Task ProcessAbandonedFiles(IWriteChannel target, IReadChannel source, string current) + { + var abandonLeft = MAX_BACKLOG; + var abandonCutOff = long.Parse(DateTime.UtcNow.Add(-MAX_BACKLOG_AGE).ToString("yyyyMMddHHmmss")); + foreach (var f in GetAbandonedFiles(current)) + { + if (await source.IsRetiredAsync) + return; + + if (abandonLeft > 0 && f.Value > abandonCutOff) + { + abandonLeft--; + target.WriteNoWait(f.Key); + } + else + { + try { File.Delete(f.Key); } + catch { } + } + } + + } + /// /// Gets a list of abandoned files, meaning files that appear to be Duplicati files. /// These should have been uploaded, but if they are found they are somehow left-over. /// - /// The abandoned files. + /// The abandoned files, value is the timestamp, key is the filename. /// The current file, which is excluded from the results. - private static IEnumerable GetAbandonedFiles(string current) + private static IEnumerable> GetAbandonedFiles(string current) { - return - from n in GetAbandonedMatches(current) - orderby n.Value - select n.Key; + return + GetAbandonedMatches(current) + .OrderByDescending(x => x.Value); } /// From be7cde4e6e8cfc3b036cb30dd5a4ed4199f7bab6 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Thu, 8 Feb 2018 21:42:05 +0100 Subject: [PATCH 20/64] Fixed the problem where the UI reports a warning but "got 0 warning(s)" --- Duplicati/Server/Runner.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Duplicati/Server/Runner.cs b/Duplicati/Server/Runner.cs index 45fc863c7..8800670fe 100644 --- a/Duplicati/Server/Runner.cs +++ b/Duplicati/Server/Runner.cs @@ -747,7 +747,10 @@ namespace Duplicati.Server "Warning" : string.Format("Warning while running {0}", backup.Name), r.FilesWithError > 0 ? string.Format("Errors affected {0} file(s) ", r.FilesWithError) : - string.Format("Got {0} warning(s) ", r.Warnings.Count()) + (r.Errors.Any() ? + string.Format("Got {0} error(s)", r.Errors.Count()) : + string.Format("Got {0} warning(s)", r.Warnings.Count()) + ) , null, backup.ID, From 7e466e11cf896868e824811f1eeb10039bb7dff3 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 11 Feb 2018 19:54:05 +0100 Subject: [PATCH 21/64] Fixed two cases where commandline arguments were not correctly escaped when being passed to another process. This should also solve the issues with #3001 This fixes #2961 --- Duplicati/Service/Duplicati.Service.csproj | 4 ++++ Duplicati/Service/Runner.cs | 5 +++-- Duplicati/WindowsService/Program.cs | 2 +- Duplicati/WindowsService/WindowsService.csproj | 4 ++++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Duplicati/Service/Duplicati.Service.csproj b/Duplicati/Service/Duplicati.Service.csproj index 53f6d2773..d4721401d 100644 --- a/Duplicati/Service/Duplicati.Service.csproj +++ b/Duplicati/Service/Duplicati.Service.csproj @@ -50,6 +50,10 @@ {7E119745-1F62-43F0-936C-F312A1912C0B} Duplicati.Library.AutoUpdater + + {DE3E5D4C-51AB-4E5E-BEE8-E636CEBFBA65} + Duplicati.Library.Utility + diff --git a/Duplicati/Service/Runner.cs b/Duplicati/Service/Runner.cs index c88a2f595..ae4300815 100644 --- a/Duplicati/Service/Runner.cs +++ b/Duplicati/Service/Runner.cs @@ -16,6 +16,7 @@ // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; +using System.Linq; namespace Duplicati.Service { @@ -57,7 +58,7 @@ namespace Duplicati.Service var exec = System.IO.Path.Combine(path, "Duplicati.Server.exe"); var cmdargs = "--ping-pong-keepalive=true"; if (m_cmdargs != null && m_cmdargs.Length > 0) - cmdargs = cmdargs + " " + string.Join(" ", m_cmdargs); + cmdargs = Duplicati.Library.Utility.Utility.WrapAsCommandLine(new string[] { cmdargs }.Concat(m_cmdargs)); var firstRun = true; var startAttempts = 0; @@ -92,8 +93,8 @@ namespace Duplicati.Service { PingProcess(); m_onStartedAction(); - firstRun = false; } + firstRun = false; while (!m_process.HasExited) { diff --git a/Duplicati/WindowsService/Program.cs b/Duplicati/WindowsService/Program.cs index 9decd6f4e..a8ce33292 100644 --- a/Duplicati/WindowsService/Program.cs +++ b/Duplicati/WindowsService/Program.cs @@ -47,7 +47,7 @@ namespace Duplicati.WindowsService else if (install || uninstall) { // Remove the install and uninstall flags if they are present - var commandline = string.Join(" ", args.Where(x => !(string.Equals("install", x, StringComparison.OrdinalIgnoreCase) || string.Equals("uninstall", x, StringComparison.OrdinalIgnoreCase)))); + var commandline = Library.Utility.Utility.WrapAsCommandLine(args.Where(x => !(string.Equals("install", x, StringComparison.OrdinalIgnoreCase) || string.Equals("uninstall", x, StringComparison.OrdinalIgnoreCase)))); var selfexec = Assembly.GetExecutingAssembly().Location; diff --git a/Duplicati/WindowsService/WindowsService.csproj b/Duplicati/WindowsService/WindowsService.csproj index 984460417..b5cad7eb6 100644 --- a/Duplicati/WindowsService/WindowsService.csproj +++ b/Duplicati/WindowsService/WindowsService.csproj @@ -79,6 +79,10 @@ {7E119745-1F62-43F0-936C-F312A1912C0B} Duplicati.Library.AutoUpdater + + {DE3E5D4C-51AB-4E5E-BEE8-E636CEBFBA65} + Duplicati.Library.Utility +