From 9dbdbb1b2fffe9fde686c3af86d85cf001d3d9a2 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sat, 24 Jun 2023 17:08:03 +0200 Subject: [PATCH] Show files excluded via 'Exclude' section in source picker. Add more metadata to the file list returned by server, so that the source file picker can display filters based on file attributes. Correctly display 'X' in file list if a file is excluded by attributes and not included as a source (#4751). Fix directory separator issue when loading the page and prevent opening folders in source file list. --- Duplicati/Server/Serializable/TreeNode.cs | 12 ++ .../WebServer/RESTMethods/Filesystem.cs | 27 +++- .../controllers/EditBackupController.js | 18 ++- .../scripts/directives/sourceFolderPicker.js | 125 ++++++++++++++---- .../webroot/ngax/scripts/services/AppUtils.js | 17 ++- .../webroot/ngax/templates/addoredit.html | 4 +- 6 files changed, 163 insertions(+), 40 deletions(-) diff --git a/Duplicati/Server/Serializable/TreeNode.cs b/Duplicati/Server/Serializable/TreeNode.cs index 39c49d05e..cf8d3c8e8 100644 --- a/Duplicati/Server/Serializable/TreeNode.cs +++ b/Duplicati/Server/Serializable/TreeNode.cs @@ -43,9 +43,21 @@ namespace Duplicati.Server.Serializable /// public bool hidden { get; set; } /// + /// True if the element has the system file attribute + /// + public bool systemFile { get; set; } + /// + /// True if the element is marked as temporary + /// + public bool temporary { get; set; } + /// /// True if the element is a symlink /// public bool symlink { get; set; } + /// + /// Size of the file. -1 if directory or inaccessible + /// + public long fileSize { get; set; } /// /// Constructs a new TreeNode diff --git a/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs b/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs index 3c2b25aed..5c26570e5 100644 --- a/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs +++ b/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs @@ -19,9 +19,9 @@ using System.Collections.Generic; using System.Linq; using System.IO; using Duplicati.Library.Snapshots; -using Duplicati.Library.Common.IO; -using Duplicati.Library.Common; - +using Duplicati.Library.Common.IO; +using Duplicati.Library.Common; + namespace Duplicati.Server.WebServer.RESTMethods { public class Filesystem : IRESTMethodGET, IRESTMethodPOST, IRESTMethodDocumented @@ -130,7 +130,8 @@ namespace Duplicati.Server.WebServer.RESTMethods if (specialtoken != null) { - res = res.Select(x => { + res = res.Select(x => + { x.resolvedpath = x.id; x.id = specialtoken + x.id.Substring(specialpath.Length); return x; @@ -208,6 +209,13 @@ namespace Duplicati.Server.WebServer.RESTMethods return false; }; + Func getFileSize = (p) => + { + try { return new FileInfo(p).Length; } + catch { } + return -1; + }; + foreach (var s in SystemIO.IO_OS.EnumerateFileSystemEntries(entrypath) // Group directories first .OrderByDescending(f => SystemIO.IO_OS.GetFileAttributes(f) & FileAttributes.Directory) @@ -222,6 +230,9 @@ namespace Duplicati.Server.WebServer.RESTMethods var isFolder = (attr & FileAttributes.Directory) != 0; var isFile = !isFolder; var isHidden = (attr & FileAttributes.Hidden) != 0; + bool isSystem = (attr & FileAttributes.System) != 0; + bool isTemporary = (attr & FileAttributes.Temporary) != 0; + long fileSize = -1; var accessible = isFile || canAccess(s); var isLeaf = isFile || !accessible || isEmptyFolder(s); @@ -233,12 +244,20 @@ namespace Duplicati.Server.WebServer.RESTMethods if (!showHidden && isHidden) continue; + if (isFile) + { + fileSize = getFileSize(s); + } + tn = new Serializable.TreeNode() { id = rawid, text = SystemIO.IO_OS.PathGetFileName(s), hidden = isHidden, symlink = isSymlink, + temporary = isTemporary, + systemFile = isSystem, + fileSize = fileSize, iconCls = isFolder ? (accessible ? (isSymlink ? "x-tree-icon-symlink" : "x-tree-icon-parent") : "x-tree-icon-locked") : "x-tree-icon-leaf", leaf = isLeaf }; diff --git a/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js b/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js index 32b61c0ef..0d14a25f5 100644 --- a/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js +++ b/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js @@ -14,6 +14,7 @@ backupApp.controller('EditBackupController', function ($rootScope, $scope, $rout $scope.ExcludeAttributes = []; $scope.ExcludeLargeFiles = false; + $scope.ExcludeFileSize = null; $scope.fileAttributes = [ {'name': gettextCatalog.getString('Hidden files'), 'value': 'hidden'}, @@ -676,10 +677,19 @@ backupApp.controller('EditBackupController', function ($rootScope, $scope, $rout $scope.$watch("Options['--compression-module']", reloadOptionsList); $scope.$watch("Backup.TargetURL", reloadOptionsList); $scope.$on('systeminfochanged', reloadOptionsList); - $scope.$watch('ExcludeLargeFiles', function() { - if ($scope.Options != null && $scope.Options['--skip-files-larger-than'] == null) - $scope.Options['--skip-files-larger-than'] = '100MB'; - }); + $scope.$watch('ExcludeLargeFiles', function(enabled) { + if ($scope.Options != null) { + if ($scope.Options['--skip-files-larger-than'] == null) { + $scope.Options['--skip-files-larger-than'] = '100MB'; + } + $scope.ExcludeFileSize = enabled ? AppUtils.parseSizeString($scope.Options['--skip-files-larger-than']) : null; + } + }, true); + $scope.$watch("Options['--skip-files-larger-than']", function (value) { + if ($scope.ExcludeLargeFiles) { + $scope.ExcludeFileSize = AppUtils.parseSizeString(value); + } + }, true); $scope.$watch("Schedule.AllowedDays", checkAllowedDaysConfig, true); if ($routeParams.backupid == null) { diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js index 0436aec68..ddfb50afa 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js @@ -5,11 +5,13 @@ backupApp.directive('sourceFolderPicker', function() { scope: { ngSources: '=', ngFilters: '=', - ngShowHidden: '=' + ngShowHidden: '=', + ngExcludeAttributes: '=', + ngExcludeSize: '=' }, templateUrl: 'templates/sourcefolderpicker.html', - controller: function($scope, $timeout, SystemInfo, AppService, AppUtils, gettextCatalog, $anchorScroll) { + controller: function($scope, $timeout, SystemInfo, AppService, AppUtils, DialogService, gettextCatalog, $anchorScroll) { var scope = $scope; scope.systeminfo = SystemInfo.watch($scope); @@ -25,11 +27,11 @@ backupApp.directive('sourceFolderPicker', function() { var filterList = null; var displayMap = {}; - var dirsep = scope.systeminfo.DirectorySeparator || '/'; + scope.dirsep = null; function compareablePath(path) { if (path.substr(0, 1) == '%' && path.substr(path.length - 1, 1) == '%') - path += dirsep; + path += scope.dirsep; return scope.systeminfo.CaseSensitiveFilesystem ? path : path.toLowerCase(); } @@ -66,7 +68,7 @@ backupApp.directive('sourceFolderPicker', function() { n.iconCls = 'x-tree-icon-mssql'; else if (defunctmap[cp]) n.iconCls = 'x-tree-icon-broken'; - else if (cp.substr(cp.length - 1, 1) != dirsep) + else if (cp.substr(cp.length - 1, 1) != scope.dirsep) n.iconCls = 'x-tree-icon-leaf'; setEntryType(n); @@ -118,20 +120,18 @@ backupApp.directive('sourceFolderPicker', function() { } function buildidlookup(sources, map) { - var dirsep = scope.systeminfo.DirectorySeparator || '/'; - map = map || {}; for(var n in sources) { - var parts = compareablePath(n).split(dirsep); + var parts = compareablePath(n).split(scope.dirsep); var p = []; for(var pi in parts) { p.push(parts[pi]); - var r = p.join(dirsep); + var r = p.join(scope.dirsep); var l = r.substr(r.length - 1, 1); - if (l != dirsep) - r += dirsep; + if (l != scope.dirsep) + r += scope.dirsep; map[r] = true; } } @@ -139,6 +139,64 @@ backupApp.directive('sourceFolderPicker', function() { return map; } + function nodeExcludedByAttributes(n) { + // Check ExcludeAttributes + if (scope.ngExcludeAttributes != null && scope.ngExcludeAttributes.length > 0) { + if (scope.ngExcludeAttributes.indexOf('hidden') != -1 && n.hidden) { + return true; + } + if (scope.ngExcludeAttributes.indexOf('system') != -1 && n.systemFile) { + return true; + } + if (scope.ngExcludeAttributes.indexOf('temporary') != -1 && n.temporary) { + return true; + } + } + return false + } + + function nodeExcludedBySize(n) { + if(scope.ngExcludeSize != null) { + return n.fileSize > scope.ngExcludeSize; + } + } + + function shouldIncludeNode(n, checkSize) { + if (checkSize === undefined) { + checkSize = true; + } + + // ExcludeSize overrides source paths + if (checkSize && nodeExcludedBySize(n)) { + return false; + } + + // Check if explicitly included in sources + if (sourcemap[compareablePath(n.id)]) { + return true; + } + + // Result is true if included, false if excluded, null if none match + var result = null; + + // Check filter expression + if (filterList == null) + result = excludemap[compareablePath(n.id)] ? false : null; + else { + result = AppUtils.evalFilter(n.id, filterList, null); + } + + if (result !== false && nodeExcludedByAttributes(n)) { + result = false; + } + + if (result === null) { + // Include by default + result = true; + } + return result; + } + function updateIncludeFlags(root, parentFlag) { if (root != null) root = {children: [root], include: parentFlag}; @@ -147,13 +205,10 @@ backupApp.directive('sourceFolderPicker', function() { if (n.root) return null; - if (sourcemap[compareablePath(n.id)]) + if (sourcemap[compareablePath(n.id)] && !nodeExcludedBySize(n)) n.include = '+'; else if (p != null && p.include == '+') { - if (filterList == null) - n.include = excludemap[compareablePath(n.id)] ? '-' : '+'; - else - n.include = AppUtils.evalFilter(n.id, filterList) ? '+' : '-'; + n.include = shouldIncludeNode(n) ? '+' : '-'; } else if (p != null && p.include == '-') n.include = '-'; @@ -176,15 +231,15 @@ backupApp.directive('sourceFolderPicker', function() { var anySpecials = false; - for(var i = 0; i < (scope.ngFilters || []).length; i++) { - var f = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], dirsep); + for (var i = 0; i < (scope.ngFilters || []).length; i++) { + var f = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], scope.dirsep); if (f != null) { - if (f[1].indexOf('?') != -1 || f[1].indexOf('*') != -1) + if (f[0].indexOf('+') == 0 || f[1].indexOf('?') != -1 || f[1].indexOf('*') != -1) anySpecials = true; else if (f[0] == '-path') excludemap[compareablePath(f[1])] = true; else if (f[0] == '-folder') - excludemap[compareablePath(f[1] + dirsep)] = true; + excludemap[compareablePath(f[1] + scope.dirsep)] = true; else anySpecials = true; } @@ -192,6 +247,14 @@ backupApp.directive('sourceFolderPicker', function() { if (anySpecials) filterList = AppUtils.filterListToRegexps(scope.ngFilters, scope.systeminfo.CaseSensitiveFilesystem); + } + + function syncTreeWithLists() { + if (scope.ngSources == null || sourceNodeChildren == null || scope.dirsep == null) + return; + + sourcemap = {}; + updateFilterList(); sourceNodeChildren.length = 0; @@ -236,7 +299,7 @@ backupApp.directive('sourceFolderPicker', function() { var p = scope.ngSources[i]; if (p.substr(0, 1) == '%' && p.substr(p.length - 1, 1) == '%') - p += dirsep; + p += scope.dirsep; AppService.post('/filesystem/validate', {path: p}).then(function(data) { defunctmap[compareablePath(data.config.data.path)] = false; @@ -259,6 +322,14 @@ backupApp.directive('sourceFolderPicker', function() { $scope.$watch('ngSources', syncTreeWithLists, true); $scope.$watch('ngFilters', syncTreeWithLists, true); + $scope.$watch('ngExcludeAttributes', syncTreeWithLists, true); + $scope.$watch('ngExcludeSize', syncTreeWithLists, true); + $scope.$watch('systeminfo.DirectorySeparator', function (val, oldVal) { + if (val != null) { + scope.dirsep = val; + syncTreeWithLists(); + } + }, true); function findParent(id) { var r = {}; @@ -282,10 +353,8 @@ backupApp.directive('sourceFolderPicker', function() { } $scope.toggleCheck = function(node) { - dirsep = scope.systeminfo.DirectorySeparator || '/'; - var c = compareablePath(node.id); - var c_is_dir = c.substr(c.length - 1, 1) == dirsep; + var c_is_dir = c.substr(c.length - 1, 1) == scope.dirsep; if (node.include == null || node.include == ' ') { @@ -296,7 +365,7 @@ backupApp.directive('sourceFolderPicker', function() { if (s == c) return; - if (s.substr(s.length - 1, 1) == dirsep && c.indexOf(s) == 0) { + if (s.substr(s.length - 1, 1) == scope.dirsep && c.indexOf(s) == 0) { return; } else if (s.indexOf(c) == 0) { scope.ngSources.splice(i, 1); @@ -311,11 +380,11 @@ backupApp.directive('sourceFolderPicker', function() { removePathFromArray(scope.ngSources, node.id); for(var i = scope.ngFilters.length - 1; i >= 0; i--) { - var n = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], dirsep); + var n = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], scope.dirsep); if (n != null) { if (c_is_dir) { if (n[0] == '-path' || n[0] == '-folder') { - if (compareablePath(n[1] + (n[0] == '-folder' ? dirsep : '')).indexOf(c) == 0) + if (compareablePath(n[1] + (n[0] == '-folder' ? scope.dirsep : '')).indexOf(c) == 0) scope.ngFilters.splice(i, 1); } } else { @@ -340,7 +409,7 @@ backupApp.directive('sourceFolderPicker', function() { node.expanded = !node.expanded; self = this; - if (node.root || node.iconCls == 'x-tree-icon-leaf' || node.iconCls == 'x-tree-icon-locked' + if (node.root || node.leaf || node.iconCls == 'x-tree-icon-leaf' || node.iconCls == 'x-tree-icon-locked' || node.iconCls == 'x-tree-icon-hyperv' || node.iconCls == 'x-tree-icon-hypervmachine' || node.iconCls == 'x-tree-icon-mssql' || node.iconCls == 'x-tree-icon-mssqldb') return; diff --git a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js index ed76c72a1..47bd07a29 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js @@ -228,14 +228,27 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer }; - this.splitSizeString = function(val) { + this.splitSizeString = function (val) { var m = (/(\d*)(\w*)/mg).exec(val); var mul = null; if (!m) return [parseInt(val), null]; else return [parseInt(m[1]), m[2]]; - } + }; + + this.parseSizeString = function (val) { + if (val == null) { + return null; + } + var split = this.splitSizeString(val); + var formatSizes = ['B', 'KB', 'MB', 'GB', 'TB']; + var idx = formatSizes.indexOf((split[1]||'').toUpperCase()); + if (idx == -1) { + idx = 0; + } + return split[0] * Math.pow(1024, idx); + }; this.toDisplayDateAndTime = function(dt) { diff --git a/Duplicati/Server/webroot/ngax/templates/addoredit.html b/Duplicati/Server/webroot/ngax/templates/addoredit.html index 084c602f0..cdaa981c3 100644 --- a/Duplicati/Server/webroot/ngax/templates/addoredit.html +++ b/Duplicati/Server/webroot/ngax/templates/addoredit.html @@ -143,7 +143,7 @@ {{'Show hidden folders' | translate}}
- +
@@ -237,7 +237,7 @@ -