From 708715e7058dc01dbb8f963288e19db98ba1ca45 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Fri, 23 Jun 2023 13:30:46 +0200 Subject: [PATCH 001/238] Fix force-locale option for CLI and GUI. - Localization culture is set from multiple threads on the server, so use current local context for every new call instead of global context - Change locale before executing help commands if flag is present - When force-locale is set, also change locale for log messages Closes #4176 --- Duplicati/CommandLine/Commands.cs | 34 +++---- .../Localization/LocalizationService.cs | 11 ++- Duplicati/Library/Localization/Short.cs | 14 ++- Duplicati/Library/Main/Controller.cs | 54 ++--------- .../Main/Duplicati.Library.Main.csproj | 1 + Duplicati/Library/Main/LocaleChange.cs | 90 +++++++++++++++++++ 6 files changed, 126 insertions(+), 78 deletions(-) create mode 100644 Duplicati/Library/Main/LocaleChange.cs diff --git a/Duplicati/CommandLine/Commands.cs b/Duplicati/CommandLine/Commands.cs index 74523762c..541a5b576 100644 --- a/Duplicati/CommandLine/Commands.cs +++ b/Duplicati/CommandLine/Commands.cs @@ -129,13 +129,15 @@ namespace Duplicati.CommandLine public static int Examples(TextWriter outwriter, Action setup, List args, Dictionary options, Library.Utility.IFilter filter) { - Duplicati.CommandLine.Help.PrintUsage(outwriter, "example", options); + using (new Duplicati.Library.Main.LocaleChange(options)) + Duplicati.CommandLine.Help.PrintUsage(outwriter, "example", options); return 0; } public static int Help(TextWriter outwriter, Action setup, List args, Dictionary options, Library.Utility.IFilter filter) { - Duplicati.CommandLine.Help.PrintUsage(outwriter, args.Count > 1? args[1] : "help", options); + using (new Duplicati.Library.Main.LocaleChange(options)) + Duplicati.CommandLine.Help.PrintUsage(outwriter, args.Count > 1 ? args[1] : "help", options); return 0; } @@ -170,8 +172,8 @@ namespace Duplicati.CommandLine options["no-encryption"] = "true"; } - using(var console = new ConsoleOutput(outwriter, options)) - using(var i = new Library.Main.Controller(backend, options, console)) + using (var console = new ConsoleOutput(outwriter, options)) + using (var i = new Library.Main.Controller(backend, options, console)) { setup(i); i.ListAffected(args, res => @@ -254,7 +256,7 @@ namespace Duplicati.CommandLine if (!containsSeparators && arg.StartsWith("@", StringComparison.Ordinal)) { // Convert to Regexp filter and prefix with ".*/" - return $"[.*{Utility.ConvertLiteralToRegExp(Util.DirectorySeparatorString + arg.Substring(1))}]"; + return $"[.*{Library.Utility.Utility.ConvertLiteralToRegExp(Util.DirectorySeparatorString + arg.Substring(1))}]"; } else if (!containsSeparators && !containsWildcards && !arg.StartsWith("[", StringComparison.Ordinal)) { @@ -285,7 +287,7 @@ namespace Duplicati.CommandLine if (endsWithSeparator && arg.StartsWith("@", StringComparison.Ordinal)) { // Convert to Regexp filter and suffix with ".*" - return $"[{Utility.ConvertLiteralToRegExp(arg.Substring(1))}.*]"; + return $"[{Library.Utility.Utility.ConvertLiteralToRegExp(arg.Substring(1))}.*]"; } else if (endsWithSeparator && !containsWildcards && !arg.StartsWith("[", StringComparison.Ordinal)) { @@ -319,7 +321,7 @@ namespace Duplicati.CommandLine filter = new Duplicati.Library.Utility.FilterExpression(); using (var console = new ConsoleOutput(outwriter, options)) - using(var i = new Library.Main.Controller(args[0], options, console)) + using (var i = new Library.Main.Controller(args[0], options, console)) { setup(i); var backend = args[0]; @@ -407,7 +409,7 @@ namespace Duplicati.CommandLine { outwriter.WriteLine("Listing filesets:"); - foreach(var e in res.Filesets) + foreach (var e in res.Filesets) { if (e.FileCount >= 0) outwriter.WriteLine("{0}\t: {1} ({2} files, {3})", e.Version, e.Time, e.FileCount, Library.Utility.Utility.FormatSizeString(e.FileSizes)); @@ -439,10 +441,10 @@ namespace Duplicati.CommandLine else { outwriter.WriteLine("Listing files and versions:"); - foreach(var e in res.Files) + foreach (var e in res.Files) { outwriter.WriteLine(e.Path); - foreach(var nx in res.Filesets.Zip(e.Sizes, (a, b) => new { Index = a.Version, Time = a.Time, Size = b } )) + foreach (var nx in res.Filesets.Zip(e.Sizes, (a, b) => new { Index = a.Version, Time = a.Time, Size = b })) outwriter.WriteLine("{0}\t: {1} {2}", nx.Index, nx.Time, nx.Size < 0 ? " - " : Library.Utility.Utility.FormatSizeString(nx.Size)); outwriter.WriteLine(); @@ -466,7 +468,7 @@ namespace Duplicati.CommandLine } using (var console = new ConsoleOutput(outwriter, options)) - using(var i = new Library.Main.Controller(args[0], options, console)) + using (var i = new Library.Main.Controller(args[0], options, console)) { setup(i); args.RemoveAt(0); @@ -483,7 +485,7 @@ namespace Duplicati.CommandLine else outwriter.WriteLine(Strings.Program.DeletedBackups); - foreach(var f in res.DeletedSets) + foreach (var f in res.DeletedSets) outwriter.WriteLine("{0}: {1}", f.Item1, f.Item2); } } @@ -520,7 +522,7 @@ namespace Duplicati.CommandLine args = SuffixArgsWithAsterisk(PrefixArgsWithAsterisk(args)).ToList(); - using(var output = new ConsoleOutput(outwriter, options)) + using (var output = new ConsoleOutput(outwriter, options)) using (var i = new Library.Main.Controller(backend, options, output)) { output.MessageEvent(string.Format("Restore started at {0}", DateTime.Now)); @@ -721,7 +723,7 @@ namespace Duplicati.CommandLine if (args.Count != 1) return PrintWrongNumberOfArguments(outwriter, args, 1); - using(var console = new ConsoleOutput(outwriter, options)) + using (var console = new ConsoleOutput(outwriter, options)) using (var i = new Library.Main.Controller(args[0], options, console)) { setup(i); @@ -770,7 +772,7 @@ namespace Duplicati.CommandLine } else { - foreach(var n in result.Verifications) + foreach (var n in result.Verifications) { var changecount = n.Value.Count(); if (changecount == 0) @@ -1005,7 +1007,7 @@ namespace Duplicati.CommandLine } using (var console = new ConsoleOutput(outwriter, options)) - using(var i = new Library.Main.Controller("dummy://", options, console)) + using (var i = new Library.Main.Controller("dummy://", options, console)) { setup(i); var result = i.TestFilter(args.ToArray(), filter); diff --git a/Duplicati/Library/Localization/LocalizationService.cs b/Duplicati/Library/Localization/LocalizationService.cs index 20af7bb89..a8008b623 100644 --- a/Duplicati/Library/Localization/LocalizationService.cs +++ b/Duplicati/Library/Localization/LocalizationService.cs @@ -54,7 +54,6 @@ namespace Duplicati.Library.Localization if (ci == null) return null; - LC.setCulture(ci); return new LocalizationContext(ci); } @@ -89,15 +88,15 @@ namespace Duplicati.Library.Localization /// /// Gets a localization provider with the current language /// - public static ILocalizationService Current - { - get + public static ILocalizationService Current + { + get { var lc = System.Runtime.Remoting.Messaging.CallContext.LogicalGetData(LOGICAL_CONTEXT_KEY) as string; if (!string.IsNullOrWhiteSpace(lc)) return Get(new CultureInfo(lc)); - return Get(CultureInfo.CurrentCulture); - } + return Get(CultureInfo.CurrentCulture); + } } /// diff --git a/Duplicati/Library/Localization/Short.cs b/Duplicati/Library/Localization/Short.cs index 5cf29da58..2c04b506d 100644 --- a/Duplicati/Library/Localization/Short.cs +++ b/Duplicati/Library/Localization/Short.cs @@ -16,6 +16,7 @@ // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; +using System.Threading; namespace Duplicati.Library.Localization.Short { @@ -27,15 +28,10 @@ namespace Duplicati.Library.Localization.Short /// /// The instance for translation /// - private static ILocalizationService LS = LocalizationService.Current; - - /// - /// Sets the culture - /// - /// CultureInfo - public static void setCulture(System.Globalization.CultureInfo ci) + private static ILocalizationService LS { - LS = LocalizationService.Get(ci); + // Get up-to-date service (may be changed by temporary contexts) + get => LocalizationService.Current; } /// @@ -93,7 +89,7 @@ namespace Duplicati.Library.Localization.Short public static string L(string message, params object[] args) { return LS.Localize(message, args); - } + } } } diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 5b319fab3..48316a116 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -64,19 +64,9 @@ namespace Duplicati.Library.Main private System.Threading.ThreadPriority? m_resetPriority; /// - /// The localization culture to reset to + /// If not null, active locale change that needs to be reset /// - private System.Globalization.CultureInfo m_resetLocale; - - /// - /// The localization UI culture to reset to - /// - private System.Globalization.CultureInfo m_resetLocaleUI; - - /// - /// True if the locale should be reset - /// - private bool m_doResetLocale; + private LocaleChange m_localeChange = null; /// /// The multi-controller log target @@ -477,26 +467,6 @@ namespace Duplicati.Library.Main } } - /// - /// Attempts to get the locale, but delays linking to the calls as they are missing in some environments - /// - [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] - private static void DoGetLocale(out System.Globalization.CultureInfo locale, out System.Globalization.CultureInfo uiLocale) - { - locale = System.Globalization.CultureInfo.DefaultThreadCurrentCulture; - uiLocale = System.Globalization.CultureInfo.DefaultThreadCurrentUICulture; - } - - /// - /// Attempts to set the locale, but delays linking to the calls as they are missing in some environments - /// - [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] - private static void DoSetLocale(System.Globalization.CultureInfo locale, System.Globalization.CultureInfo uiLocale) - { - System.Globalization.CultureInfo.DefaultThreadCurrentCulture = locale; - System.Globalization.CultureInfo.DefaultThreadCurrentUICulture = uiLocale; - } - private void OnOperationComplete(object result) { if (m_options != null && m_options.LoadedModules != null) @@ -520,14 +490,10 @@ namespace Duplicati.Library.Main m_resetPriority = null; } - if (m_doResetLocale) + if(m_localeChange != null) { - // Wrap the call to avoid loading issues for the setLocale method - DoSetLocale(m_resetLocale, m_resetLocaleUI); - - m_doResetLocale = false; - m_resetLocale = null; - m_resetLocaleUI = null; + m_localeChange.Dispose(); + m_localeChange = null; } if (m_logTarget != null) @@ -635,17 +601,11 @@ namespace Duplicati.Library.Main { try { - var locale = m_options.ForcedLocale; - DoGetLocale(out m_resetLocale, out m_resetLocaleUI); - m_doResetLocale = true; - // Wrap the call to avoid loading issues for the setLocale method - DoSetLocale(locale, locale); + m_localeChange = new LocaleChange(m_options.ForcedLocale); } - catch (Exception ex) // or only: MissingMethodException + catch (Exception ex) { Library.Logging.Log.WriteWarningMessage(LOGTAG, "LocaleChangeError", ex, Strings.Controller.FailedForceLocaleError(ex.Message)); - m_doResetLocale = false; - m_resetLocale = m_resetLocaleUI = null; } } diff --git a/Duplicati/Library/Main/Duplicati.Library.Main.csproj b/Duplicati/Library/Main/Duplicati.Library.Main.csproj index 278813455..8db76b9e9 100644 --- a/Duplicati/Library/Main/Duplicati.Library.Main.csproj +++ b/Duplicati/Library/Main/Duplicati.Library.Main.csproj @@ -142,6 +142,7 @@ + diff --git a/Duplicati/Library/Main/LocaleChange.cs b/Duplicati/Library/Main/LocaleChange.cs new file mode 100644 index 000000000..5d3952b30 --- /dev/null +++ b/Duplicati/Library/Main/LocaleChange.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Duplicati.Library.Main +{ + public class LocaleChange : IDisposable + { + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + + private bool m_doResetLocale = false; + private System.Globalization.CultureInfo m_resetLocale = null; + private System.Globalization.CultureInfo m_resetLocaleUI = null; + + private IDisposable m_localizationContext = null; + + public LocaleChange(System.Globalization.CultureInfo newLocale) + { + if (newLocale == null) + { + return; + } + try + { + DoGetLocale(out m_resetLocale, out m_resetLocaleUI); + m_doResetLocale = true; + // Wrap the call to avoid loading issues for the setLocale method + DoSetLocale(newLocale, newLocale); + m_localizationContext = Localization.LocalizationService.TemporaryContext(newLocale); + } + catch (MissingMethodException ex) + { + m_doResetLocale = false; + m_resetLocale = m_resetLocaleUI = null; + + Library.Logging.Log.WriteWarningMessage(LOGTAG, "LocaleChangeError", ex, Strings.Controller.FailedForceLocaleError(ex.Message)); + } + } + + public LocaleChange(Options options) + : this(options.HasForcedLocale ? options.ForcedLocale : null) + { + } + public LocaleChange(Dictionary options) + : this(new Options(options)) + { + } + + public void Dispose() + { + if (m_doResetLocale) + { + // Wrap the call to avoid loading issues for the setLocale method + DoSetLocale(m_resetLocale, m_resetLocaleUI); + + m_doResetLocale = false; + m_resetLocale = null; + m_resetLocaleUI = null; + } + if(m_localizationContext != null) + { + m_localizationContext.Dispose(); + m_localizationContext = null; + } + } + + /// + /// Attempts to get the locale, but delays linking to the calls as they are missing in some environments + /// + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static void DoGetLocale(out System.Globalization.CultureInfo locale, out System.Globalization.CultureInfo uiLocale) + { + locale = System.Globalization.CultureInfo.DefaultThreadCurrentCulture; + uiLocale = System.Globalization.CultureInfo.DefaultThreadCurrentUICulture; + } + + /// + /// Attempts to set the locale, but delays linking to the calls as they are missing in some environments + /// + [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] + private static void DoSetLocale(System.Globalization.CultureInfo locale, System.Globalization.CultureInfo uiLocale) + { + System.Globalization.CultureInfo.DefaultThreadCurrentCulture = locale; + System.Globalization.CultureInfo.DefaultThreadCurrentUICulture = uiLocale; + } + + } +} From b6643fa47322b4391c478dbbee4cbd2d9bb2220a Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Fri, 23 Jun 2023 14:22:36 +0200 Subject: [PATCH 002/238] Add documentation comments for LocaleChange. --- Duplicati/Library/Main/LocaleChange.cs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/Duplicati/Library/Main/LocaleChange.cs b/Duplicati/Library/Main/LocaleChange.cs index 5d3952b30..402d240e1 100644 --- a/Duplicati/Library/Main/LocaleChange.cs +++ b/Duplicati/Library/Main/LocaleChange.cs @@ -6,6 +6,10 @@ using System.Threading.Tasks; namespace Duplicati.Library.Main { + /// + /// Changes the current locale for all threads and for log messages in the current context. + /// When disposed the changes are reset. + /// public class LocaleChange : IDisposable { private static readonly string LOGTAG = Logging.Log.LogTagFromType(); @@ -16,6 +20,9 @@ namespace Duplicati.Library.Main private IDisposable m_localizationContext = null; + /// + /// Change locale if newLocale is not null + /// public LocaleChange(System.Globalization.CultureInfo newLocale) { if (newLocale == null) @@ -39,10 +46,24 @@ namespace Duplicati.Library.Main } } + /// + /// Change locale if ForcedLocale is specified in options + /// + /// Command line options + /// + /// Thrown if specified locale was not found. + /// public LocaleChange(Options options) : this(options.HasForcedLocale ? options.ForcedLocale : null) { } + /// + /// Change locale if 'force-locale' is specified in options dictionary + /// + /// Command line options + /// + /// Thrown if specified locale was not found. + /// public LocaleChange(Dictionary options) : this(new Options(options)) { From b9d4be60ca9b916a12529a8a8a74b490718ef3f2 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Fri, 23 Jun 2023 16:34:43 +0200 Subject: [PATCH 003/238] Do not set advanced options with empty string to true. There are advanced options such as force-local that expect an empty string in some cases. While saving, these were replaced with 'true' even though there was an equals sign. Now only options without an equals sign are set to true. Closes #4757 --- Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js index 0817690d2..ba90444a3 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js @@ -280,8 +280,6 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer if (eqpos > 0) { key = line.substr(0, eqpos).trim(); value = line.substr(eqpos + 1).trim(); - if (value == '') - value = true; } if (validateCallback) From 0f5b4b97070271935a0d8bc8ce8dfed069d2959a Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Mon, 10 Jul 2023 16:44:24 +0200 Subject: [PATCH 004/238] Fix restore of empty files with recovery tool. Empty files do not have data blocks stored, so don't try to get a block for empty files. --- Duplicati/CommandLine/RecoveryTool/Restore.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Duplicati/CommandLine/RecoveryTool/Restore.cs b/Duplicati/CommandLine/RecoveryTool/Restore.cs index b2fc5282e..e7e6e4462 100644 --- a/Duplicati/CommandLine/RecoveryTool/Restore.cs +++ b/Duplicati/CommandLine/RecoveryTool/Restore.cs @@ -171,7 +171,8 @@ namespace Duplicati.CommandLine.RecoveryTool { if (f.BlocklistHashes == null) { - lookup.WriteHash(sw, f.Hash); + if(f.Size > 0) + lookup.WriteHash(sw, f.Hash); } else { From 6b5623651c87137af5946ef420731e101a94f194 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Tue, 11 Jul 2023 22:32:35 +0200 Subject: [PATCH 005/238] Fix recovery tool restore paths on different OS. On Linux, paths were created with \ instead of / separators if backup was originally from Windows (#4884). Also fix bug that created one unnecessary level of directories in the restore path. --- Duplicati/CommandLine/RecoveryTool/Restore.cs | 111 +++++++++++------- 1 file changed, 70 insertions(+), 41 deletions(-) diff --git a/Duplicati/CommandLine/RecoveryTool/Restore.cs b/Duplicati/CommandLine/RecoveryTool/Restore.cs index e7e6e4462..c95b7aa2f 100644 --- a/Duplicati/CommandLine/RecoveryTool/Restore.cs +++ b/Duplicati/CommandLine/RecoveryTool/Restore.cs @@ -104,53 +104,19 @@ namespace Duplicati.CommandLine.RecoveryTool using (var mru = new CompressedFileMRUCache(options)) { Console.WriteLine("Building lookup table for file hashes"); - using (HashLookupHelper lookup = new HashLookupHelper(ixfile, mru, (int) blocksize, blockhasher.HashSize / 8)) + // Source OS can have different directory separator + string sourceDirsep = null; + using (HashLookupHelper lookup = new HashLookupHelper(ixfile, mru, (int)blocksize, blockhasher.HashSize / 8)) { var filecount = 0L; - string largestprefix = null; - string[] largestprefixparts = null; if (!string.IsNullOrWhiteSpace(targetpath)) Console.WriteLine("Computing restore path"); - foreach (var f in List.EnumerateFilesInDList(filelist, filter, options)) - { - if (largestprefix == null) - { - largestprefix = f.Path; - largestprefixparts = largestprefix.Split(new char[] { Path.DirectorySeparatorChar }); - } - else if (largestprefix.Length > 1) - { - var parts = f.Path.Split(new char[] { Path.DirectorySeparatorChar }); - - var ni = 0; - for (; ni < Math.Min(parts.Length, largestprefixparts.Length); ni++) - if (!Library.Utility.Utility.ClientFilenameStringComparer.Equals(parts[ni], largestprefixparts[ni])) - break; - - if (ni != largestprefixparts.Length) - { - if (ni == 0) - { - largestprefixparts = new string[0]; - largestprefix = string.Empty; - } - else - { - Array.Resize(ref largestprefixparts, ni - 1); - largestprefix = string.Join(Util.DirectorySeparatorString, largestprefixparts); - } - } - } - filecount++; - } + string largestprefix = GetLargestPrefix(from f in List.EnumerateFilesInDList(filelist, filter, options) select f.Path, out sourceDirsep, out filecount); Console.WriteLine("Restoring {0} files to {1}", filecount, string.IsNullOrWhiteSpace(targetpath) ? "original position" : targetpath); - if (Platform.IsClientPosix || largestprefix.Length > 0) - largestprefix = Util.AppendDirSeparator(largestprefix); - if (!string.IsNullOrEmpty(largestprefix)) Console.WriteLine("Removing common prefix {0} from files", largestprefix); @@ -159,7 +125,7 @@ namespace Duplicati.CommandLine.RecoveryTool { try { - var targetfile = MapToRestorePath(f.Path, largestprefix, targetpath); + var targetfile = MapToRestorePath(f.Path, largestprefix, targetpath, sourceDirsep); if (!systemIO.DirectoryExists(systemIO.PathGetDirectoryName(targetfile))) systemIO.DirectoryCreate(systemIO.PathGetDirectoryName(targetfile)); @@ -171,7 +137,7 @@ namespace Duplicati.CommandLine.RecoveryTool { if (f.BlocklistHashes == null) { - if(f.Size > 0) + if (f.Size > 0) lookup.WriteHash(sw, f.Hash); } else @@ -244,8 +210,71 @@ namespace Duplicati.CommandLine.RecoveryTool return 0; } - private static string MapToRestorePath(string path, string prefixpath, string restorepath) + public static string GetLargestPrefix(IEnumerable filePaths, out string sourceDirsep, out long filecount) { + // Get dir separator like in LocalRestoreDatabase.GetLargestPrefix(): + string largestprefix = filePaths.OrderByDescending(p => p.Length).FirstOrDefault() ?? string.Empty; + sourceDirsep = Util.GuessDirSeparator(largestprefix); + + + string[] dirsepSplit = new string[] { sourceDirsep }; + string[] largestprefixparts = largestprefix.Split(dirsepSplit, StringSplitOptions.None); + + // Because only files are in the list, need to remove filename from prefix + // Otherwise, in case of a single file the prefix is not a directory + if (largestprefixparts.Length > 0) + { + Array.Resize(ref largestprefixparts, largestprefixparts.Length - 1); + } + largestprefix = string.Join(sourceDirsep, largestprefixparts); + + + + filecount = 0; + foreach (var path in filePaths) + { + if (largestprefix.Length > 1) + { + // Unix paths starting with / have an empty string in paths[0] + // Completely empty strings should not combine with that, so should have no parts at all + var parts = path.Length == 0 ? new string[0] : path.Split(dirsepSplit, StringSplitOptions.None); + + var ni = 0; + for (; ni < Math.Min(parts.Length, largestprefixparts.Length); ni++) + if (!Library.Utility.Utility.ClientFilenameStringComparer.Equals(parts[ni], largestprefixparts[ni])) + break; + + if (ni != largestprefixparts.Length) + { + if (ni == 0) + { + largestprefixparts = new string[0]; + largestprefix = string.Empty; + } + else + { + // Only the first ni parts match + Array.Resize(ref largestprefixparts, ni); + largestprefix = string.Join(sourceDirsep, largestprefixparts); + } + } + } + filecount++; + } + return largestprefixparts.Length == 0 ? "" : Util.AppendDirSeparator(largestprefix, sourceDirsep); + } + + public static string MapToRestorePath(string path, string prefixpath, string restorepath, string sourceDirsep) + { + if (sourceDirsep != null && sourceDirsep != Util.DirectorySeparatorString && sourceDirsep != Util.AltDirectorySeparatorString) + { + // Replace directory separator in source and prefix path + path = path.Replace(sourceDirsep, Util.DirectorySeparatorString); + if (!string.IsNullOrWhiteSpace(prefixpath)) + { + prefixpath = prefixpath.Replace(sourceDirsep, Util.DirectorySeparatorString); + } + } if (string.IsNullOrWhiteSpace(restorepath)) return path; From cff1049347d569dd30866f375ed8188a6e02719e Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Tue, 11 Jul 2023 22:35:14 +0200 Subject: [PATCH 006/238] Add unit tests for recovery tool restore paths. Test that paths are generated correctly from a different source OS. Also test that empty files are restored by recovery tool. --- Duplicati/UnitTest/RecoveryToolTests.cs | 170 ++++++++++++++++++++++-- 1 file changed, 160 insertions(+), 10 deletions(-) diff --git a/Duplicati/UnitTest/RecoveryToolTests.cs b/Duplicati/UnitTest/RecoveryToolTests.cs index b81d7b25a..a1f524ad3 100644 --- a/Duplicati/UnitTest/RecoveryToolTests.cs +++ b/Duplicati/UnitTest/RecoveryToolTests.cs @@ -33,6 +33,160 @@ namespace Duplicati.UnitTest base.TearDown(); } + [Test] + [Category("RecoveryTool")] + public void RecoveryRestorePrefix() + { + string sourceDirsep; + long fileCount; + string largestprefix; + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { }, out sourceDirsep, out fileCount); + Assert.AreEqual("", largestprefix); + // sourceDirsep unspecified + Assert.AreEqual(0, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { "" }, out sourceDirsep, out fileCount); + Assert.AreEqual("", largestprefix); + // sourceDirsep unspecified + Assert.AreEqual(1, fileCount); + + // Windows paths + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("C:\\Users\\User\\Pictures\\", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(1, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png", + "C:\\Users\\User\\Pictures\\b.jpg", + "C:\\Users\\User\\Pictures\\c.txt" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("C:\\Users\\User\\Pictures\\", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(3, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png", + "C:\\Users\\User\\Pictures\\b.jpg", + "C:\\Users\\User2\\Pictures\\b.jpg" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("C:\\Users\\", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(3, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png", + "C:\\Users\\User\\Pictures\\b.jpg", + "C:\\Data\\a.png" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("C:\\", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(3, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png", + "D:\\Users\\User\\Pictures\\a.jpg" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(2, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "C:\\Users\\User\\Pictures\\a.png", + "" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("", largestprefix); + Assert.AreEqual("\\", sourceDirsep); + Assert.AreEqual(2, fileCount); + + // Unix paths + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "/home/user/pictures/a.png" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("/home/user/pictures/", largestprefix); + Assert.AreEqual("/", sourceDirsep); + Assert.AreEqual(1, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "/home/user/pictures/a.png", + "/home/user/pictures/b.jpg", + "/home/user/pictures/c.txt" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("/home/user/pictures/", largestprefix); + Assert.AreEqual("/", sourceDirsep); + Assert.AreEqual(3, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "/home/user/pictures/a.png", + "/home/user/pictures/b.jpg", + "/home/user2/pictures/c.txt" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("/home/", largestprefix); + Assert.AreEqual("/", sourceDirsep); + Assert.AreEqual(3, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "/home/user/pictures/a.png", + "/media/data/a.png" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("/", largestprefix); + Assert.AreEqual("/", sourceDirsep); + Assert.AreEqual(2, fileCount); + + largestprefix = CommandLine.RecoveryTool.Restore.GetLargestPrefix(new string[] { + "/home/user/pictures/a.png", + "" + }, out sourceDirsep, out fileCount); + Assert.AreEqual("", largestprefix); + Assert.AreEqual("/", sourceDirsep); + Assert.AreEqual(2, fileCount); + } + + [Test] + [Category("RecoveryTool")] + public void RecoveryRestorePathMap() + { + // Windows source paths + string restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("C:\\Users\\User\\Pictures\\a.png", "C:\\Users\\User\\Pictures\\", "restore", "\\"); + // Replace the alt directory separator with normal directory separator to compare paths, + // otherwise the path returned by Path.Combine does not match + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("C:\\Users\\User\\Pictures\\a.png", "C:\\Users\\", "restore", "\\"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "User", "Pictures", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("C:\\Users\\User\\Pictures\\a.png", "C:\\Users\\", "restore" + Path.DirectorySeparatorChar, "\\"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "User", "Pictures", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("C:\\Users\\User\\Pictures\\a.png", "", "restore", "\\"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "C", "Users", "User", "Pictures", "a.png"), restorePath); + // This is an absolute restore path on Windows and a relative path with backslashes on Unix + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("C:\\Users\\User\\Pictures\\a.png", "", "C:\\Users\\User\\restore", "\\"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("C:\\Users\\User\\restore", "C", "Users", "User", "Pictures", "a.png"), restorePath); + + // Unix source paths + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("/home/user/pictures/a.png", "/home/user/pictures/", "restore", "/"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("/home/user/pictures/a.png", "/home/user/pictures/", "restore" + Path.DirectorySeparatorChar, "/"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("/home/user/pictures/a.png", "/home/", "restore", "/"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "user", "pictures", "a.png"), restorePath); + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("/home/user/pictures/a.png", "/", "restore", "/"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("restore", "home", "user", "pictures", "a.png"), restorePath); + // This is an absolute restore path on Windows and a relative path with backslashes on Unix + restorePath = CommandLine.RecoveryTool.Restore.MapToRestorePath("/home/user/pictures/a.png", "/", "C:\\Users\\User\\restore", "/"); + restorePath = restorePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar); + Assert.AreEqual(Path.Combine("C:\\Users\\User\\restore", "home", "user", "pictures", "a.png"), restorePath); + } + [Test] [Category("RecoveryTool")] [TestCase("false")] @@ -40,7 +194,7 @@ namespace Duplicati.UnitTest public void Recover(string buildIndexWithFiles) { // Files to create in MB. - int[] fileSizes = {10, 20, 30}; + int[] fileSizes = { 0, 10, 20, 30 }; foreach (int size in fileSizes) { byte[] data = new byte[size * 1024 * 1024]; @@ -65,7 +219,7 @@ namespace Duplicati.UnitTest string backendURL = "file://" + this.TARGETFOLDER; using (Controller c = new Controller(backendURL, options, null)) { - IBackupResults backupResults = c.Backup(new[] {this.DATAFOLDER}); + IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER }); Assert.AreEqual(0, backupResults.Errors.Count()); Assert.AreEqual(0, backupResults.Warnings.Count()); } @@ -73,24 +227,20 @@ namespace Duplicati.UnitTest // Download the backend files. string downloadFolder = Path.Combine(this.RESTOREFOLDER, "downloadedFiles"); Directory.CreateDirectory(downloadFolder); - int status = CommandLine.RecoveryTool.Program.RealMain(new[] {"download", $"{backendURL}", $"{downloadFolder}", $"--passphrase={options["passphrase"]}"}); + int status = CommandLine.RecoveryTool.Program.RealMain(new[] { "download", $"{backendURL}", $"{downloadFolder}", $"--passphrase={options["passphrase"]}" }); Assert.AreEqual(0, status); // Create the index. - status = CommandLine.RecoveryTool.Program.RealMain(new[] {"index", $"{downloadFolder}", $"--build-index-with-files={buildIndexWithFiles}"}); + status = CommandLine.RecoveryTool.Program.RealMain(new[] { "index", $"{downloadFolder}", $"--build-index-with-files={buildIndexWithFiles}" }); Assert.AreEqual(0, status); // Restore to a different folder. string restoreFolder = Path.Combine(this.RESTOREFOLDER, "restoredFiles"); Directory.CreateDirectory(restoreFolder); - status = CommandLine.RecoveryTool.Program.RealMain(new[] {"restore", $"{downloadFolder}", $"--targetpath={restoreFolder}"}); + status = CommandLine.RecoveryTool.Program.RealMain(new[] { "restore", $"{downloadFolder}", $"--targetpath={restoreFolder}" }); Assert.AreEqual(0, status); - // Since this.DATAFOLDER is a folder, Path.GetFileName will return the name of the - // last folder in the path. - string baseFolder = Path.GetFileName(this.DATAFOLDER); - - TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, Path.Combine(restoreFolder, baseFolder), false, "Verifying restore using RecoveryTool."); + TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, restoreFolder, false, "Verifying restore using RecoveryTool."); } } } \ No newline at end of file From 8af1dbb6fd084c04628b722e5b5f7d21db92a732 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sun, 16 Jul 2023 19:16:11 +0200 Subject: [PATCH 007/238] Add --quota-disable option to disable reported backend quota. If set, the backend is treated as if it does not report a quota at all. No warnings and errors are created and no quota values are set in the backup log. This option is intended to be used if a backend reports a wrong quota, so it needs to be disabled. --- .../Library/Main/Operation/BackupHandler.cs | 2 +- .../Main/Operation/FilelistProcessor.cs | 2 +- Duplicati/Library/Main/Options.cs | 120 ++++++++++-------- Duplicati/Library/Main/Strings.cs | 2 + 4 files changed, 70 insertions(+), 56 deletions(-) diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index ef12b94e8..004e99c37 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -329,7 +329,7 @@ namespace Duplicati.Library.Main.Operation // TODO: If we have a BackendManager, we should query through that using (var backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions)) { - if (backend is IQuotaEnabledBackend enabledBackend) + if (backend is IQuotaEnabledBackend enabledBackend && !m_options.QuotaDisable) { Library.Interface.IQuotaInfo quota = enabledBackend.Quota; if (quota != null) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 0652020d8..5df8770df 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -234,7 +234,7 @@ namespace Duplicati.Library.Main.Operation // TODO: We should query through the backendmanager using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions)) - if (bk is IQuotaEnabledBackend enabledBackend) + if (bk is IQuotaEnabledBackend enabledBackend && !options.QuotaDisable) { Library.Interface.IQuotaInfo quota = enabledBackend.Quota; if (quota != null) diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 1c638ddc4..75a5447f3 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -34,7 +34,7 @@ namespace Duplicati.Library.Main { private const string DEFAULT_BLOCK_HASH_ALGORITHM = "SHA256"; private const string DEFAULT_FILE_HASH_ALGORITHM = "SHA256"; - + /// /// The default block size /// @@ -44,17 +44,17 @@ namespace Duplicati.Library.Main /// The default threshold value /// private const long DEFAULT_THRESHOLD = 25; - + /// /// The default value for maximum number of small files /// private const long DEFAULT_SMALL_FILE_MAX_COUNT = 20; - + /// /// Default size of volumes /// private const string DEFAULT_VOLUME_SIZE = "50mb"; - + /// /// Default value for keep-versions /// @@ -74,7 +74,7 @@ namespace Duplicati.Library.Main /// The default number of hasher instances /// private readonly int DEFAULT_BLOCK_HASHERS = Math.Max(1, Environment.ProcessorCount / 2); - + /// /// The default threshold for warning about coming close to quota /// @@ -133,7 +133,7 @@ namespace Duplicati.Library.Main /// Process only the first hardlink /// First, - + /// /// Process all hardlinks /// @@ -144,7 +144,7 @@ namespace Duplicati.Library.Main /// None } - + /// /// The possible settings for index file usage /// @@ -154,17 +154,17 @@ namespace Duplicati.Library.Main /// Disables usage of index files /// None, - + /// /// Stores only block lookup information in the index files /// Lookup, - + /// /// Stores both block lookup and block lists in the index files /// Full - + } private static string[] GetSupportedHashes() @@ -182,7 +182,7 @@ namespace Duplicati.Library.Main { } } - + return r.ToArray(); } @@ -286,13 +286,13 @@ namespace Duplicati.Library.Main new CommandLineArgument("asynchronous-upload-limit", CommandLineArgument.ArgumentType.Integer, Strings.Options.AsynchronousuploadlimitShort, Strings.Options.AsynchronousuploadlimitLong, "4"), new CommandLineArgument("asynchronous-concurrent-upload-limit", CommandLineArgument.ArgumentType.Integer, Strings.Options.AsynchronousconcurrentuploadlimitShort, Strings.Options.AsynchronousconcurrentuploadlimitLong, "4"), new CommandLineArgument("asynchronous-upload-folder", CommandLineArgument.ArgumentType.Path, Strings.Options.AsynchronousuploadfolderShort, Strings.Options.AsynchronousuploadfolderLong, System.IO.Path.GetTempPath()), - + new CommandLineArgument("disable-streaming-transfers", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisableStreamingShort, Strings.Options.DisableStreamingLong, "false"), new CommandLineArgument("throttle-upload", CommandLineArgument.ArgumentType.Size, Strings.Options.ThrottleuploadShort, Strings.Options.ThrottleuploadLong, "0kb"), new CommandLineArgument("throttle-download", CommandLineArgument.ArgumentType.Size, Strings.Options.ThrottledownloadShort, Strings.Options.ThrottledownloadLong, "0kb"), new CommandLineArgument("skip-files-larger-than", CommandLineArgument.ArgumentType.Size, Strings.Options.SkipfileslargerthanShort, Strings.Options.SkipfileslargerthanLong), - + new CommandLineArgument("upload-unchanged-backups", CommandLineArgument.ArgumentType.Boolean, Strings.Options.UploadUnchangedBackupsShort, Strings.Options.UploadUnchangedBackupsLong, "false"), new CommandLineArgument("snapshot-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SnapshotpolicyShort, Strings.Options.SnapshotpolicyLong, "off", null, Enum.GetNames(typeof(OptimizationStrategy))), @@ -327,6 +327,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("quota-size", CommandLineArgument.ArgumentType.Size, Strings.Options.QuotasizeShort, Strings.Options.QuotasizeLong), new CommandLineArgument("quota-warning-threshold", CommandLineArgument.ArgumentType.Integer, Strings.Options.QuotaWarningThresholdShort, Strings.Options.QuotaWarningThresholdLong, DEFAULT_QUOTA_WARNING_THRESHOLD.ToString()), + new CommandLineArgument("quota-disable", CommandLineArgument.ArgumentType.Boolean, Strings.Options.QuotaDisableShort, Strings.Options.QuotaDisableLong("quota-size"), "false"), new CommandLineArgument("symlink-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.SymlinkpolicyShort, Strings.Options.SymlinkpolicyLong("store", "ignore", "follow"), Enum.GetName(typeof(SymlinkStrategy), SymlinkStrategy.Store), null, Enum.GetNames(typeof(SymlinkStrategy))), new CommandLineArgument("hardlink-policy", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.HardlinkpolicyShort, Strings.Options.HardlinkpolicyLong("first", "all", "none"), Enum.GetName(typeof(HardlinkStrategy), HardlinkStrategy.All), null, Enum.GetNames(typeof(HardlinkStrategy))), @@ -390,7 +391,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("concurrency-max-threads", CommandLineArgument.ArgumentType.Integer, Strings.Options.ConcurrencymaxthreadsShort, Strings.Options.ConcurrencymaxthreadsLong, "0"), new CommandLineArgument("concurrency-block-hashers", CommandLineArgument.ArgumentType.Integer, Strings.Options.ConcurrencyblockhashersShort, Strings.Options.ConcurrencyblockhashersLong, DEFAULT_BLOCK_HASHERS.ToString()), new CommandLineArgument("concurrency-compressors", CommandLineArgument.ArgumentType.Integer, Strings.Options.ConcurrencycompressorsShort, Strings.Options.ConcurrencycompressorsLong, DEFAULT_COMPRESSORS.ToString()), - + new CommandLineArgument("auto-vacuum", CommandLineArgument.ArgumentType.Boolean, Strings.Options.AutoVacuumShort, Strings.Options.AutoVacuumLong, "false"), new CommandLineArgument("disable-file-scanner", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilescannerShort, Strings.Options.DisablefilescannerLong, "false"), new CommandLineArgument("disable-filelist-consistency-checks", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilelistconsistencychecksShort, Strings.Options.DisablefilelistconsistencychecksLong, "false"), @@ -412,7 +413,7 @@ namespace Duplicati.Library.Main /// /// Gets or sets the current main action of the instance /// - public OperationMode MainAction + public OperationMode MainAction { get { return (OperationMode)Enum.Parse(typeof(OperationMode), m_options["main-action"]); } set { m_options["main-action"] = value.ToString(); } @@ -499,7 +500,7 @@ namespace Duplicati.Library.Main return Library.Utility.Timeparser.ParseTimeInterval(m_options["time"], DateTime.Now); } } - + /// /// Gets the versions the restore or list operation is limited to /// @@ -648,7 +649,7 @@ namespace Duplicati.Library.Main m_options.TryGetValue("prefix", out v); if (!string.IsNullOrEmpty(v)) return v; - + return "duplicati"; } } @@ -664,7 +665,7 @@ namespace Duplicati.Library.Main m_options.TryGetValue("keep-versions", out v); if (string.IsNullOrEmpty(v)) return DEFAULT_KEEP_VERSIONS; - + return Math.Max(0, int.Parse(v)); } } @@ -678,7 +679,7 @@ namespace Duplicati.Library.Main { string v; m_options.TryGetValue("keep-time", out v); - + if (string.IsNullOrEmpty(v)) return new DateTime(0); @@ -696,12 +697,14 @@ namespace Duplicati.Library.Main /// public List RetentionPolicy { - get { + get + { var retentionPolicyConfig = new List(); string v; m_options.TryGetValue("retention-policy", out v); - if (string.IsNullOrEmpty(v)) { + if (string.IsNullOrEmpty(v)) + { return retentionPolicyConfig; } @@ -1061,7 +1064,7 @@ namespace Duplicati.Library.Main return value; } } - + /// /// Gets the logfile filename /// @@ -1233,6 +1236,15 @@ namespace Duplicati.Library.Main } } + /// + /// Gets a flag indicating that backup quota reported by the backend should be ignored + /// + /// This is necessary because in some cases the backend might report a wrong quota (especially with some Linux mounts). + public bool QuotaDisable + { + get { return Library.Utility.Utility.ParseBoolOption(m_options, "quota-disable"); } + } + /// /// Gets the display name of the backup /// @@ -1289,15 +1301,15 @@ namespace Duplicati.Library.Main long blocksize = Library.Utility.Sizeparser.ParseSize(tmp, "kb"); if (blocksize > int.MaxValue || blocksize < 1024) throw new ArgumentOutOfRangeException(nameof(blocksize), string.Format("The blocksize cannot be less than {0}, nor larger than {1}", 1024, int.MaxValue)); - + return (int)blocksize; } } - - /// + + /// /// Cache for the block hash size value, to avoid creating new hash instances just to get the size /// - private KeyValuePair m_cachedBlockHashSize; + private KeyValuePair m_cachedBlockHashSize; /// /// Gets the size of the blockhash in bytes. @@ -1307,10 +1319,10 @@ namespace Duplicati.Library.Main { get { - if (m_cachedBlockHashSize.Key != BlockHashAlgorithm) - m_cachedBlockHashSize = new KeyValuePair(BlockHashAlgorithm, Duplicati.Library.Utility.HashAlgorithmHelper.Create(BlockHashAlgorithm).HashSize / 8); - - return m_cachedBlockHashSize.Value; + if (m_cachedBlockHashSize.Key != BlockHashAlgorithm) + m_cachedBlockHashSize = new KeyValuePair(BlockHashAlgorithm, Duplicati.Library.Utility.HashAlgorithmHelper.Create(BlockHashAlgorithm).HashSize / 8); + + return m_cachedBlockHashSize.Value; } } @@ -1374,8 +1386,8 @@ namespace Duplicati.Library.Main return Library.Utility.Utility.ParseBoolOption(m_options, "use-block-cache"); } } - - + + /// /// Gets the compact threshold /// @@ -1407,7 +1419,7 @@ namespace Duplicati.Library.Main return Library.Utility.Sizeparser.ParseSize(v, "mb"); } } - + /// /// Gets the maximum number of small volumes /// @@ -1423,7 +1435,7 @@ namespace Duplicati.Library.Main return Convert.ToInt64(v); } } - + /// /// List of files to check for changes /// @@ -1483,26 +1495,26 @@ namespace Duplicati.Library.Main return v; } } - + /// /// Gets the index file usage method /// public IndexFileStrategy IndexfilePolicy { - get - { + get + { string strategy; if (!m_options.TryGetValue("index-file-policy", out strategy)) strategy = ""; - + IndexFileStrategy res; if (!Enum.TryParse(strategy, true, out res)) res = IndexFileStrategy.Full; - + return res; } } - + /// /// Gets a flag indicating if the check for files on the remote storage should be omitted /// @@ -1510,7 +1522,7 @@ namespace Duplicati.Library.Main { get { return Library.Utility.Utility.ParseBoolOption(m_options, "no-backend-verification"); } } - + /// /// Gets the percentage of samples to test during a backup operation /// @@ -1548,17 +1560,17 @@ namespace Duplicati.Library.Main /// public long BackupTestSampleCount { - get - { + get + { string s; m_options.TryGetValue("backup-test-samples", out s); if (string.IsNullOrEmpty(s)) return 1; - + return long.Parse(s); } } - + /// /// Gets a flag indicating if compacting should not be done automatically /// @@ -1588,21 +1600,21 @@ namespace Duplicati.Library.Main { get { return Library.Utility.Utility.ParseBoolOption(m_options, "allow-missing-source"); } } - + /// /// Gets a value indicating if a verification file should be uploaded after changing the remote store /// public bool UploadVerificationFile { - get { return Library.Utility.Utility.ParseBoolOption(m_options, "upload-verification-file"); } + get { return Library.Utility.Utility.ParseBoolOption(m_options, "upload-verification-file"); } } - + /// /// Gets a value indicating if a passphrase change is allowed /// public bool AllowPassphraseChange { - get { return Library.Utility.Utility.ParseBoolOption(m_options, "allow-passphrase-change"); } + get { return Library.Utility.Utility.ParseBoolOption(m_options, "allow-passphrase-change"); } } /// @@ -1610,12 +1622,12 @@ namespace Duplicati.Library.Main /// public bool Dryrun { - get + get { if (m_options.ContainsKey("dry-run")) - return Library.Utility.Utility.ParseBoolOption(m_options, "dry-run"); + return Library.Utility.Utility.ParseBoolOption(m_options, "dry-run"); else - return Library.Utility.Utility.ParseBoolOption(m_options, "dryrun"); + return Library.Utility.Utility.ParseBoolOption(m_options, "dryrun"); } } @@ -1626,7 +1638,7 @@ namespace Duplicati.Library.Main { get { return Library.Utility.Utility.ParseBoolOption(m_options, "full-remote-verification"); } } - + /// /// The block hash algorithm to use /// @@ -1673,7 +1685,7 @@ namespace Duplicati.Library.Main /// true if no local blocks; otherwise, false. public bool NoLocalBlocks { - get { return Library.Utility.Utility.ParseBoolOption(m_options, "no-local-blocks"); } + get { return Library.Utility.Utility.ParseBoolOption(m_options, "no-local-blocks"); } } /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 9ba71054f..51979e3d2 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -127,6 +127,8 @@ namespace Duplicati.Library.Main.Strings public static string QuotasizeShort { get { return LC.L(@"A reported maximum storage"); } } public static string QuotaWarningThresholdLong { get { return LC.L(@"Sets a threshold for when to warn about the backend quota being nearly exceeded. It is given as a percentage, and a warning is generated if the amount of available quota is less than this percentage of the total backup size. If the backend does not report the quota information, this value will be ignored"); } } public static string QuotaWarningThresholdShort { get { return LC.L(@"Threshold for warning about low quota"); } } + public static string QuotaDisableLong(string optionname) { return LC.L(@"Disable the quota reported by the backend. --{0} can still be used to set a manual quota", optionname); } + public static string QuotaDisableShort { get { return LC.L(@"Disable backend quota"); } } public static string SymlinkpolicyShort { get { return LC.L(@"Symlink handling"); } } public static string SymlinkpolicyLong(string store, string ignore, string follow) { return LC.L(@"Use this option to handle symlinks differently. The ""{0}"" option will simply record a symlink with its name and destination, and a restore will recreate the symlink as a link. Use the option ""{1}"" to ignore all symlinks and not store any information about them. The option ""{2}"" will cause the symlinked target to be backed up and restored as a normal file with the symlink name. Early versions of Duplicati did not support this option and bevhaved as if ""{2}"" was specified.", store, ignore, follow); } public static string HardlinkpolicyShort { get { return LC.L(@"Hardlink handling"); } } From 0754a5cd72bf6040234e14430b2eafeb46ffd0d0 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sun, 16 Jul 2023 19:24:37 +0200 Subject: [PATCH 008/238] Ignore File backend quota if total size is zero. Mono sometimes reports the total size of unknown file systems as zero. Because it should be obvious when the size is actually zero, ignore the quota in this case. --- Duplicati/Library/Backend/File/FileBackend.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index a2b005458..79c410e26 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -234,7 +234,7 @@ namespace Duplicati.Library.Backend } else { - systemIO.FileCopy(sourceFilePath, targetFilePath, true); + systemIO.FileCopy(sourceFilePath, targetFilePath, true); if (m_verifyDestinationLength) VerifyMatchingSize(targetFilePath, sourceFilePath); } @@ -347,7 +347,12 @@ namespace Duplicati.Library.Backend System.IO.DriveInfo driveInfo = this.GetDrive(); if (driveInfo != null) { - return new QuotaInfo(driveInfo.TotalSize, driveInfo.AvailableFreeSpace); + // Check that the total space is above 0, because Mono sometimes reports 0 for unknown file systems + // If the drive actually has a total size of 0, this should be obvious immediately due to write errors + if (driveInfo.TotalSize > 0) + { + return new QuotaInfo(driveInfo.TotalSize, driveInfo.AvailableFreeSpace); + } } if (Platform.IsClientWindows) From aa365f7f34ec7104249d656c036c602e4098ac07 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sun, 16 Jul 2023 19:49:54 +0200 Subject: [PATCH 009/238] Update --quota-size option to assign a backup size. The assigned quota size is checked in addition to the reported backend quota (if available). It can be used to limit the backup size on the remote. Warnings are created according to the same quota warning threshold. Previously, this option had no function. --- .../Main/Operation/FilelistProcessor.cs | 95 ++++++++++++------- Duplicati/Library/Main/Options.cs | 2 +- Duplicati/Library/Main/Strings.cs | 4 +- 3 files changed, 63 insertions(+), 38 deletions(-) diff --git a/Duplicati/Library/Main/Operation/FilelistProcessor.cs b/Duplicati/Library/Main/Operation/FilelistProcessor.cs index 5df8770df..f2d02ea14 100644 --- a/Duplicati/Library/Main/Operation/FilelistProcessor.cs +++ b/Duplicati/Library/Main/Operation/FilelistProcessor.cs @@ -232,42 +232,9 @@ namespace Duplicati.Library.Main.Operation log.BackupListCount = database.FilesetTimes.Count(); log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime(); - // TODO: We should query through the backendmanager - using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions)) - if (bk is IQuotaEnabledBackend enabledBackend && !options.QuotaDisable) - { - Library.Interface.IQuotaInfo quota = enabledBackend.Quota; - if (quota != null) - { - log.TotalQuotaSpace = quota.TotalQuotaSpace; - log.FreeQuotaSpace = quota.FreeQuotaSpace; + CheckQuota(backend, options, log, knownFileSize); - // Check to see if there should be a warning or error about the quota - // Since this processor may be called multiple times during a backup - // (both at the start and end, for example), the log keeps track of - // whether a quota error or warning has been sent already. - // Note that an error can still be sent later even if a warning was sent earlier. - if (!log.ReportedQuotaError && quota.FreeQuotaSpace == 0) - { - log.ReportedQuotaError = true; - Logging.Log.WriteErrorMessage(LOGTAG, "BackendQuotaExceeded", null, "Backend quota has been exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); - } - else if (!log.ReportedQuotaWarning && !log.ReportedQuotaError && quota.FreeQuotaSpace >= 0) // Negative value means the backend didn't return the quota info - { - // Warnings are sent if the available free space is less than the given percentage of the total backup size. - double warningThreshold = options.QuotaWarningThreshold / (double)100; - if (quota.FreeQuotaSpace < warningThreshold * knownFileSize) - { - log.ReportedQuotaWarning = true; - Logging.Log.WriteWarningMessage(LOGTAG, "BackendQuotaNear", null, "Backend quota is close to being exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); - } - } - } - } - - log.AssignedQuotaSpace = options.QuotaSize; - - foreach(var s in remotelist) + foreach (var s in remotelist) lookup[s.File.Name] = s; var missing = new List(); @@ -401,5 +368,63 @@ namespace Duplicati.Library.Main.Operation VerificationRequiredVolumes = missingHash.Select(x => x.Item2) }; } + + private static void CheckQuota(BackendManager backend, Options options, IBackendWriter log, long knownFileSize) + { + // TODO: We should query through the backendmanager + using (var bk = DynamicLoader.BackendLoader.GetBackend(backend.BackendUrl, options.RawOptions)) + if (bk is IQuotaEnabledBackend enabledBackend && !options.QuotaDisable) + { + Library.Interface.IQuotaInfo quota = enabledBackend.Quota; + if (quota != null) + { + log.TotalQuotaSpace = quota.TotalQuotaSpace; + log.FreeQuotaSpace = quota.FreeQuotaSpace; + + // Check to see if there should be a warning or error about the quota + // Since this processor may be called multiple times during a backup + // (both at the start and end, for example), the log keeps track of + // whether a quota error or warning has been sent already. + // Note that an error can still be sent later even if a warning was sent earlier. + if (!log.ReportedQuotaError && quota.FreeQuotaSpace == 0) + { + log.ReportedQuotaError = true; + Logging.Log.WriteErrorMessage(LOGTAG, "BackendQuotaExceeded", null, "Backend quota has been exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); + } + else if (!log.ReportedQuotaWarning && !log.ReportedQuotaError && quota.FreeQuotaSpace >= 0) // Negative value means the backend didn't return the quota info + { + // Warnings are sent if the available free space is less than the given percentage of the total backup size. + double warningThreshold = options.QuotaWarningThreshold / (double)100; + if (quota.FreeQuotaSpace < warningThreshold * knownFileSize) + { + log.ReportedQuotaWarning = true; + Logging.Log.WriteWarningMessage(LOGTAG, "BackendQuotaNear", null, "Backend quota is close to being exceeded: Using {0} of {1} ({2} available)", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(quota.TotalQuotaSpace), Library.Utility.Utility.FormatSizeString(quota.FreeQuotaSpace)); + } + } + } + } + + log.AssignedQuotaSpace = options.QuotaSize; + if (log.AssignedQuotaSpace != -1) + { + // Check assigned quota + if (!log.ReportedQuotaError && knownFileSize > log.AssignedQuotaSpace) + { + log.ReportedQuotaError = true; + Logging.Log.WriteErrorMessage(LOGTAG, "AssignedQuotaExceeded", null, "Assigned quota has been exceeded: Using {0} of {1}", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(log.AssignedQuotaSpace)); + } + else if (!log.ReportedQuotaWarning && !log.ReportedQuotaError) + { + // Warnings are sent if the available free space is less than the given percentage of the total backup size. + double warningThreshold = options.QuotaWarningThreshold / (double)100; + long freeSpace = log.AssignedQuotaSpace - knownFileSize; + if (freeSpace < warningThreshold * knownFileSize) + { + log.ReportedQuotaWarning = true; + Logging.Log.WriteWarningMessage(LOGTAG, "AssignedQuotaNear", null, "Assigned quota is close to being exceeded: Using {0} of {1}", Library.Utility.Utility.FormatSizeString(knownFileSize), Library.Utility.Utility.FormatSizeString(log.AssignedQuotaSpace)); + } + } + } + } } } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 75a5447f3..9639448c5 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -1200,7 +1200,7 @@ namespace Duplicati.Library.Main public bool Overwrite { get { return GetBool("overwrite"); } } /// - /// Gets the total size in bytes that the backend supports, returns -1 if there is no upper limit + /// Gets the total size in bytes that the backup should use, returns -1 if there is no upper limit /// public long QuotaSize { diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 51979e3d2..2eb0db080 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -123,8 +123,8 @@ namespace Duplicati.Library.Main.Strings public static string DebugretryerrorsShort { get { return LC.L(@"Show error messages when a retry is performed"); } } public static string UploadUnchangedBackupsLong { get { return LC.L(@"If no files have changed, Duplicati will not upload a backup set. If the backup data is used to verify that a backup was executed, this option will make Duplicati upload a backupset even if it is empty"); } } public static string UploadUnchangedBackupsShort { get { return LC.L(@"Upload empty backup files"); } } - public static string QuotasizeLong { get { return LC.L(@"This value can be used to set a known upper limit on the amount of space a backend has. If the backend reports the size itself, this value is ignored"); } } - public static string QuotasizeShort { get { return LC.L(@"A reported maximum storage"); } } + public static string QuotasizeLong { get { return LC.L(@"Set a limit to the amount of storage used on the backend (by this backup). This is in addition to the full backend quota, if available. Note: Backups will continue past the quota. This only creates warnings and error messages."); } } + public static string QuotasizeShort { get { return LC.L(@"Limit storage use"); } } public static string QuotaWarningThresholdLong { get { return LC.L(@"Sets a threshold for when to warn about the backend quota being nearly exceeded. It is given as a percentage, and a warning is generated if the amount of available quota is less than this percentage of the total backup size. If the backend does not report the quota information, this value will be ignored"); } } public static string QuotaWarningThresholdShort { get { return LC.L(@"Threshold for warning about low quota"); } } public static string QuotaDisableLong(string optionname) { return LC.L(@"Disable the quota reported by the backend. --{0} can still be used to set a manual quota", optionname); } From c9e9b0eeb7f1fb757a75c6769169f2432a8dc2df Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sat, 24 Jun 2023 15:00:14 +0200 Subject: [PATCH 010/238] Scroll to source data path on double click. --- .../scripts/directives/sourceFolderPicker.js | 77 +++++++++++++++++-- .../ngax/templates/sourcefolderpicker.html | 4 +- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js index afa88466c..0436aec68 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js @@ -1,7 +1,7 @@ backupApp.directive('sourceFolderPicker', function() { return { restrict: 'E', - require: ['ngSources', 'ngFilters'], + require: ['ngSources', 'ngFilters', '$anchorScroll'], scope: { ngSources: '=', ngFilters: '=', @@ -9,13 +9,14 @@ backupApp.directive('sourceFolderPicker', function() { }, templateUrl: 'templates/sourcefolderpicker.html', - controller: function($scope, $timeout, SystemInfo, AppService, AppUtils, gettextCatalog) { + controller: function($scope, $timeout, SystemInfo, AppService, AppUtils, gettextCatalog, $anchorScroll) { var scope = $scope; scope.systeminfo = SystemInfo.watch($scope); var sourceNodeChildren = null; - $scope.treedata = { }; + $scope.treedata = {}; + $scope.expandedPath = null; var sourcemap = {}; var excludemap = {}; @@ -331,8 +332,13 @@ backupApp.directive('sourceFolderPicker', function() { } }; + function shouldExpand(path, expandedPath) { + return expandedPath.indexOf(path) == 0 && path.length < expandedPath.length; + } + $scope.toggleExpanded = function(node) { node.expanded = !node.expanded; + self = this; if (node.root || node.iconCls == 'x-tree-icon-leaf' || node.iconCls == 'x-tree-icon-locked' || node.iconCls == 'x-tree-icon-hyperv' || node.iconCls == 'x-tree-icon-hypervmachine' @@ -347,8 +353,19 @@ backupApp.directive('sourceFolderPicker', function() { node.loading = false; if (node.children != null) - for(var i in node.children) - setEntryType(node.children[i]); + for (var i in node.children) { + var child = node.children[i]; + setEntryType(child); + if (self.expandedPath != null) { + var childPath = compareablePath(child.id); + if (shouldExpand(childPath, self.expandedPath)) { + self.toggleExpanded(child); + } else if (childPath == self.expandedPath) { + self.expandedPath = null; + self.scrollId = child.id; + } + } + } updateIncludeFlags(node, node.include); @@ -360,6 +377,49 @@ backupApp.directive('sourceFolderPicker', function() { } }; + $scope.expandPath = function(path) { + cPath = compareablePath(path); + this.expandedPath = cPath; + traversenodes(function (n, p) { + if (n.root) { + return null; + } + var nodePath = compareablePath(n.id); + if (nodePath == cPath && !n.other) { + // Scroll to node + scope.scrollId = n.id; + // Cancel traverse + return false; + } + + if (shouldExpand(nodePath, cPath)) { + if (!p.expanded) { + // Handle root nodes + scope.toggleExpanded(p); + } + if (!n.expanded) { + scope.toggleExpanded(n); + } + // Continue traverse + return null; + } else { + // Do not continue this subtree + return true; + } + }, this.treedata); + } + + $scope.$watch('scrollId', function (scrollId, oldVal, scope) { + // Scroll to node + if (scrollId != null) { + scope.scrollId = null; + // Need to wait until all nodes are processed + $timeout(function () { + $anchorScroll('node-' + scrollId); + }, 100); + } + }); + $scope.toggleSelected = function(node) { if (scope.selectednode != null) scope.selectednode.selected = false; @@ -368,6 +428,13 @@ backupApp.directive('sourceFolderPicker', function() { scope.selectednode.selected = true; }; + $scope.doubleClick = function (node) { + if (sourceNodeChildren.indexOf(node) != -1) { + // Open folder in file picker + scope.expandPath(node.id); + } + }; + scope.treedata.children = []; AppService.post('/filesystem?onlyfolders=false&showhidden=true', {path: '/'}).then(function(data) { diff --git a/Duplicati/Server/webroot/ngax/templates/sourcefolderpicker.html b/Duplicati/Server/webroot/ngax/templates/sourcefolderpicker.html index 5ea095042..f5926b663 100644 --- a/Duplicati/Server/webroot/ngax/templates/sourcefolderpicker.html +++ b/Duplicati/Server/webroot/ngax/templates/sourcefolderpicker.html @@ -1,11 +1,11 @@
    -
  • +
  • - {{ node.text }} + {{ node.text }}
      From 9dbdbb1b2fffe9fde686c3af86d85cf001d3d9a2 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sat, 24 Jun 2023 17:08:03 +0200 Subject: [PATCH 011/238] Show files excluded via 'Exclude' section in source picker. Add more metadata to the file list returned by server, so that the source file picker can display filters based on file attributes. Correctly display 'X' in file list if a file is excluded by attributes and not included as a source (#4751). Fix directory separator issue when loading the page and prevent opening folders in source file list. --- Duplicati/Server/Serializable/TreeNode.cs | 12 ++ .../WebServer/RESTMethods/Filesystem.cs | 27 +++- .../controllers/EditBackupController.js | 18 ++- .../scripts/directives/sourceFolderPicker.js | 125 ++++++++++++++---- .../webroot/ngax/scripts/services/AppUtils.js | 17 ++- .../webroot/ngax/templates/addoredit.html | 4 +- 6 files changed, 163 insertions(+), 40 deletions(-) diff --git a/Duplicati/Server/Serializable/TreeNode.cs b/Duplicati/Server/Serializable/TreeNode.cs index 39c49d05e..cf8d3c8e8 100644 --- a/Duplicati/Server/Serializable/TreeNode.cs +++ b/Duplicati/Server/Serializable/TreeNode.cs @@ -43,9 +43,21 @@ namespace Duplicati.Server.Serializable ///
public bool hidden { get; set; } /// + /// True if the element has the system file attribute + /// + public bool systemFile { get; set; } + /// + /// True if the element is marked as temporary + /// + public bool temporary { get; set; } + /// /// True if the element is a symlink /// public bool symlink { get; set; } + /// + /// Size of the file. -1 if directory or inaccessible + /// + public long fileSize { get; set; } /// /// Constructs a new TreeNode diff --git a/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs b/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs index 3c2b25aed..5c26570e5 100644 --- a/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs +++ b/Duplicati/Server/WebServer/RESTMethods/Filesystem.cs @@ -19,9 +19,9 @@ using System.Collections.Generic; using System.Linq; using System.IO; using Duplicati.Library.Snapshots; -using Duplicati.Library.Common.IO; -using Duplicati.Library.Common; - +using Duplicati.Library.Common.IO; +using Duplicati.Library.Common; + namespace Duplicati.Server.WebServer.RESTMethods { public class Filesystem : IRESTMethodGET, IRESTMethodPOST, IRESTMethodDocumented @@ -130,7 +130,8 @@ namespace Duplicati.Server.WebServer.RESTMethods if (specialtoken != null) { - res = res.Select(x => { + res = res.Select(x => + { x.resolvedpath = x.id; x.id = specialtoken + x.id.Substring(specialpath.Length); return x; @@ -208,6 +209,13 @@ namespace Duplicati.Server.WebServer.RESTMethods return false; }; + Func getFileSize = (p) => + { + try { return new FileInfo(p).Length; } + catch { } + return -1; + }; + foreach (var s in SystemIO.IO_OS.EnumerateFileSystemEntries(entrypath) // Group directories first .OrderByDescending(f => SystemIO.IO_OS.GetFileAttributes(f) & FileAttributes.Directory) @@ -222,6 +230,9 @@ namespace Duplicati.Server.WebServer.RESTMethods var isFolder = (attr & FileAttributes.Directory) != 0; var isFile = !isFolder; var isHidden = (attr & FileAttributes.Hidden) != 0; + bool isSystem = (attr & FileAttributes.System) != 0; + bool isTemporary = (attr & FileAttributes.Temporary) != 0; + long fileSize = -1; var accessible = isFile || canAccess(s); var isLeaf = isFile || !accessible || isEmptyFolder(s); @@ -233,12 +244,20 @@ namespace Duplicati.Server.WebServer.RESTMethods if (!showHidden && isHidden) continue; + if (isFile) + { + fileSize = getFileSize(s); + } + tn = new Serializable.TreeNode() { id = rawid, text = SystemIO.IO_OS.PathGetFileName(s), hidden = isHidden, symlink = isSymlink, + temporary = isTemporary, + systemFile = isSystem, + fileSize = fileSize, iconCls = isFolder ? (accessible ? (isSymlink ? "x-tree-icon-symlink" : "x-tree-icon-parent") : "x-tree-icon-locked") : "x-tree-icon-leaf", leaf = isLeaf }; diff --git a/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js b/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js index 32b61c0ef..0d14a25f5 100644 --- a/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js +++ b/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js @@ -14,6 +14,7 @@ backupApp.controller('EditBackupController', function ($rootScope, $scope, $rout $scope.ExcludeAttributes = []; $scope.ExcludeLargeFiles = false; + $scope.ExcludeFileSize = null; $scope.fileAttributes = [ {'name': gettextCatalog.getString('Hidden files'), 'value': 'hidden'}, @@ -676,10 +677,19 @@ backupApp.controller('EditBackupController', function ($rootScope, $scope, $rout $scope.$watch("Options['--compression-module']", reloadOptionsList); $scope.$watch("Backup.TargetURL", reloadOptionsList); $scope.$on('systeminfochanged', reloadOptionsList); - $scope.$watch('ExcludeLargeFiles', function() { - if ($scope.Options != null && $scope.Options['--skip-files-larger-than'] == null) - $scope.Options['--skip-files-larger-than'] = '100MB'; - }); + $scope.$watch('ExcludeLargeFiles', function(enabled) { + if ($scope.Options != null) { + if ($scope.Options['--skip-files-larger-than'] == null) { + $scope.Options['--skip-files-larger-than'] = '100MB'; + } + $scope.ExcludeFileSize = enabled ? AppUtils.parseSizeString($scope.Options['--skip-files-larger-than']) : null; + } + }, true); + $scope.$watch("Options['--skip-files-larger-than']", function (value) { + if ($scope.ExcludeLargeFiles) { + $scope.ExcludeFileSize = AppUtils.parseSizeString(value); + } + }, true); $scope.$watch("Schedule.AllowedDays", checkAllowedDaysConfig, true); if ($routeParams.backupid == null) { diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js index 0436aec68..ddfb50afa 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js @@ -5,11 +5,13 @@ backupApp.directive('sourceFolderPicker', function() { scope: { ngSources: '=', ngFilters: '=', - ngShowHidden: '=' + ngShowHidden: '=', + ngExcludeAttributes: '=', + ngExcludeSize: '=' }, templateUrl: 'templates/sourcefolderpicker.html', - controller: function($scope, $timeout, SystemInfo, AppService, AppUtils, gettextCatalog, $anchorScroll) { + controller: function($scope, $timeout, SystemInfo, AppService, AppUtils, DialogService, gettextCatalog, $anchorScroll) { var scope = $scope; scope.systeminfo = SystemInfo.watch($scope); @@ -25,11 +27,11 @@ backupApp.directive('sourceFolderPicker', function() { var filterList = null; var displayMap = {}; - var dirsep = scope.systeminfo.DirectorySeparator || '/'; + scope.dirsep = null; function compareablePath(path) { if (path.substr(0, 1) == '%' && path.substr(path.length - 1, 1) == '%') - path += dirsep; + path += scope.dirsep; return scope.systeminfo.CaseSensitiveFilesystem ? path : path.toLowerCase(); } @@ -66,7 +68,7 @@ backupApp.directive('sourceFolderPicker', function() { n.iconCls = 'x-tree-icon-mssql'; else if (defunctmap[cp]) n.iconCls = 'x-tree-icon-broken'; - else if (cp.substr(cp.length - 1, 1) != dirsep) + else if (cp.substr(cp.length - 1, 1) != scope.dirsep) n.iconCls = 'x-tree-icon-leaf'; setEntryType(n); @@ -118,20 +120,18 @@ backupApp.directive('sourceFolderPicker', function() { } function buildidlookup(sources, map) { - var dirsep = scope.systeminfo.DirectorySeparator || '/'; - map = map || {}; for(var n in sources) { - var parts = compareablePath(n).split(dirsep); + var parts = compareablePath(n).split(scope.dirsep); var p = []; for(var pi in parts) { p.push(parts[pi]); - var r = p.join(dirsep); + var r = p.join(scope.dirsep); var l = r.substr(r.length - 1, 1); - if (l != dirsep) - r += dirsep; + if (l != scope.dirsep) + r += scope.dirsep; map[r] = true; } } @@ -139,6 +139,64 @@ backupApp.directive('sourceFolderPicker', function() { return map; } + function nodeExcludedByAttributes(n) { + // Check ExcludeAttributes + if (scope.ngExcludeAttributes != null && scope.ngExcludeAttributes.length > 0) { + if (scope.ngExcludeAttributes.indexOf('hidden') != -1 && n.hidden) { + return true; + } + if (scope.ngExcludeAttributes.indexOf('system') != -1 && n.systemFile) { + return true; + } + if (scope.ngExcludeAttributes.indexOf('temporary') != -1 && n.temporary) { + return true; + } + } + return false + } + + function nodeExcludedBySize(n) { + if(scope.ngExcludeSize != null) { + return n.fileSize > scope.ngExcludeSize; + } + } + + function shouldIncludeNode(n, checkSize) { + if (checkSize === undefined) { + checkSize = true; + } + + // ExcludeSize overrides source paths + if (checkSize && nodeExcludedBySize(n)) { + return false; + } + + // Check if explicitly included in sources + if (sourcemap[compareablePath(n.id)]) { + return true; + } + + // Result is true if included, false if excluded, null if none match + var result = null; + + // Check filter expression + if (filterList == null) + result = excludemap[compareablePath(n.id)] ? false : null; + else { + result = AppUtils.evalFilter(n.id, filterList, null); + } + + if (result !== false && nodeExcludedByAttributes(n)) { + result = false; + } + + if (result === null) { + // Include by default + result = true; + } + return result; + } + function updateIncludeFlags(root, parentFlag) { if (root != null) root = {children: [root], include: parentFlag}; @@ -147,13 +205,10 @@ backupApp.directive('sourceFolderPicker', function() { if (n.root) return null; - if (sourcemap[compareablePath(n.id)]) + if (sourcemap[compareablePath(n.id)] && !nodeExcludedBySize(n)) n.include = '+'; else if (p != null && p.include == '+') { - if (filterList == null) - n.include = excludemap[compareablePath(n.id)] ? '-' : '+'; - else - n.include = AppUtils.evalFilter(n.id, filterList) ? '+' : '-'; + n.include = shouldIncludeNode(n) ? '+' : '-'; } else if (p != null && p.include == '-') n.include = '-'; @@ -176,15 +231,15 @@ backupApp.directive('sourceFolderPicker', function() { var anySpecials = false; - for(var i = 0; i < (scope.ngFilters || []).length; i++) { - var f = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], dirsep); + for (var i = 0; i < (scope.ngFilters || []).length; i++) { + var f = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], scope.dirsep); if (f != null) { - if (f[1].indexOf('?') != -1 || f[1].indexOf('*') != -1) + if (f[0].indexOf('+') == 0 || f[1].indexOf('?') != -1 || f[1].indexOf('*') != -1) anySpecials = true; else if (f[0] == '-path') excludemap[compareablePath(f[1])] = true; else if (f[0] == '-folder') - excludemap[compareablePath(f[1] + dirsep)] = true; + excludemap[compareablePath(f[1] + scope.dirsep)] = true; else anySpecials = true; } @@ -192,6 +247,14 @@ backupApp.directive('sourceFolderPicker', function() { if (anySpecials) filterList = AppUtils.filterListToRegexps(scope.ngFilters, scope.systeminfo.CaseSensitiveFilesystem); + } + + function syncTreeWithLists() { + if (scope.ngSources == null || sourceNodeChildren == null || scope.dirsep == null) + return; + + sourcemap = {}; + updateFilterList(); sourceNodeChildren.length = 0; @@ -236,7 +299,7 @@ backupApp.directive('sourceFolderPicker', function() { var p = scope.ngSources[i]; if (p.substr(0, 1) == '%' && p.substr(p.length - 1, 1) == '%') - p += dirsep; + p += scope.dirsep; AppService.post('/filesystem/validate', {path: p}).then(function(data) { defunctmap[compareablePath(data.config.data.path)] = false; @@ -259,6 +322,14 @@ backupApp.directive('sourceFolderPicker', function() { $scope.$watch('ngSources', syncTreeWithLists, true); $scope.$watch('ngFilters', syncTreeWithLists, true); + $scope.$watch('ngExcludeAttributes', syncTreeWithLists, true); + $scope.$watch('ngExcludeSize', syncTreeWithLists, true); + $scope.$watch('systeminfo.DirectorySeparator', function (val, oldVal) { + if (val != null) { + scope.dirsep = val; + syncTreeWithLists(); + } + }, true); function findParent(id) { var r = {}; @@ -282,10 +353,8 @@ backupApp.directive('sourceFolderPicker', function() { } $scope.toggleCheck = function(node) { - dirsep = scope.systeminfo.DirectorySeparator || '/'; - var c = compareablePath(node.id); - var c_is_dir = c.substr(c.length - 1, 1) == dirsep; + var c_is_dir = c.substr(c.length - 1, 1) == scope.dirsep; if (node.include == null || node.include == ' ') { @@ -296,7 +365,7 @@ backupApp.directive('sourceFolderPicker', function() { if (s == c) return; - if (s.substr(s.length - 1, 1) == dirsep && c.indexOf(s) == 0) { + if (s.substr(s.length - 1, 1) == scope.dirsep && c.indexOf(s) == 0) { return; } else if (s.indexOf(c) == 0) { scope.ngSources.splice(i, 1); @@ -311,11 +380,11 @@ backupApp.directive('sourceFolderPicker', function() { removePathFromArray(scope.ngSources, node.id); for(var i = scope.ngFilters.length - 1; i >= 0; i--) { - var n = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], dirsep); + var n = AppUtils.splitFilterIntoTypeAndBody(scope.ngFilters[i], scope.dirsep); if (n != null) { if (c_is_dir) { if (n[0] == '-path' || n[0] == '-folder') { - if (compareablePath(n[1] + (n[0] == '-folder' ? dirsep : '')).indexOf(c) == 0) + if (compareablePath(n[1] + (n[0] == '-folder' ? scope.dirsep : '')).indexOf(c) == 0) scope.ngFilters.splice(i, 1); } } else { @@ -340,7 +409,7 @@ backupApp.directive('sourceFolderPicker', function() { node.expanded = !node.expanded; self = this; - if (node.root || node.iconCls == 'x-tree-icon-leaf' || node.iconCls == 'x-tree-icon-locked' + if (node.root || node.leaf || node.iconCls == 'x-tree-icon-leaf' || node.iconCls == 'x-tree-icon-locked' || node.iconCls == 'x-tree-icon-hyperv' || node.iconCls == 'x-tree-icon-hypervmachine' || node.iconCls == 'x-tree-icon-mssql' || node.iconCls == 'x-tree-icon-mssqldb') return; diff --git a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js index ed76c72a1..47bd07a29 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js @@ -228,14 +228,27 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer }; - this.splitSizeString = function(val) { + this.splitSizeString = function (val) { var m = (/(\d*)(\w*)/mg).exec(val); var mul = null; if (!m) return [parseInt(val), null]; else return [parseInt(m[1]), m[2]]; - } + }; + + this.parseSizeString = function (val) { + if (val == null) { + return null; + } + var split = this.splitSizeString(val); + var formatSizes = ['B', 'KB', 'MB', 'GB', 'TB']; + var idx = formatSizes.indexOf((split[1]||'').toUpperCase()); + if (idx == -1) { + idx = 0; + } + return split[0] * Math.pow(1024, idx); + }; this.toDisplayDateAndTime = function(dt) { diff --git a/Duplicati/Server/webroot/ngax/templates/addoredit.html b/Duplicati/Server/webroot/ngax/templates/addoredit.html index 084c602f0..cdaa981c3 100644 --- a/Duplicati/Server/webroot/ngax/templates/addoredit.html +++ b/Duplicati/Server/webroot/ngax/templates/addoredit.html @@ -143,7 +143,7 @@ {{'Show hidden folders' | translate}}
- +
@@ -237,7 +237,7 @@ -
From 5acf4dbaad72bc1dbf36ef0817e2dc1c0e947204 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sat, 24 Jun 2023 17:50:21 +0200 Subject: [PATCH 012/238] Improve file in-/excluding logic in source file picker. Add filters to the beginning instead of the end of the list, so they are not overridden by more general ones. If a file is excluded, add an include filter to include the file. --- .../scripts/directives/sourceFolderPicker.js | 29 +++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js index ddfb50afa..effac4940 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js @@ -219,16 +219,9 @@ backupApp.directive('sourceFolderPicker', function() { }, root); } - function syncTreeWithLists() { - if (scope.ngSources == null || sourceNodeChildren == null) - return; - - dirsep = scope.systeminfo.DirectorySeparator || '/'; - - sourcemap = {}; + function updateFilterList() { excludemap = {}; filterList = null; - var anySpecials = false; for (var i = 0; i < (scope.ngFilters || []).length; i++) { @@ -394,10 +387,28 @@ backupApp.directive('sourceFolderPicker', function() { } } } else { - scope.ngFilters.push("-" + node.id); + removePathFromArray(scope.ngFilters, '+' + node.id); + updateFilterList(); + if (shouldIncludeNode(node, false)) { + // No explicit include filter, add exclude filter to start of list + scope.ngFilters.unshift("-" + node.id); + } } } else if (node.include == '-') { removePathFromArray(scope.ngFilters, '-' + node.id); + updateFilterList(); + if (nodeExcludedByAttributes(node) + && indexOfPathInArray(scope.ngSources, node.id) == -1) { + // Node is excluded by attributes, have to add as source to override + scope.ngSources.push(node.id); + } else if (!shouldIncludeNode(node, false)) { + // No explicit exclude filter, add include filter to start of list + scope.ngFilters.unshift('+' + node.id); + } + if (nodeExcludedBySize(node)) { + DialogService.dialog(gettextCatalog.getString('Cannot include "{{text}}"', node), + gettextCatalog.getString('The file size is {{size}}, larger than the maximum specified size. If the file size decreases, it will be included in future backups.', { size: AppUtils.formatSizeString(node.fileSize) })); + } } }; From 74db90924ef0180a22ea7c8d79cd3b96599223f9 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sat, 24 Jun 2023 21:49:32 +0200 Subject: [PATCH 013/238] Fix processing files repeatedly if present in multiple sources. This can happen if a source is added to force including a system folder. --- .../Library/Main/Operation/Backup/FileEnumerationProcess.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index e816fbbab..993776d25 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -105,7 +105,7 @@ namespace Duplicati.Library.Main.Operation.Backup worklist = snapshot.EnumerateFilesAndFolders(sources, attributeFilter, (rootpath, errorpath, ex) => { Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "FileAccessError", ex, "Error reported while accessing file: {0}", errorpath); - }); + }).Distinct(Library.Utility.Utility.IsFSCaseSensitive ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase); } if (token.IsCancellationRequested) From a1349b1af9e5a566a8c97998ae28cdeb817d10e9 Mon Sep 17 00:00:00 2001 From: Jojo-1000 <33495614+Jojo-1000@users.noreply.github.com> Date: Sun, 25 Jun 2023 00:13:05 +0200 Subject: [PATCH 014/238] Show filter group rules in source file picker. Add REST handler /systeminfo/filtergroups that returns the filter strings for all filter groups on the system. Use these filter strings to apply filter groups to the source file picker. --- Duplicati/Library/Utility/FilterGroups.cs | 17 +++++++ .../WebServer/RESTMethods/SystemInfo.cs | 29 ++++++++--- .../scripts/directives/sourceFolderPicker.js | 3 ++ .../webroot/ngax/scripts/services/AppUtils.js | 49 ++++++++++++++++--- 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/Duplicati/Library/Utility/FilterGroups.cs b/Duplicati/Library/Utility/FilterGroups.cs index e59255453..fdfd85fdb 100644 --- a/Duplicati/Library/Utility/FilterGroups.cs +++ b/Duplicati/Library/Utility/FilterGroups.cs @@ -249,6 +249,23 @@ namespace Duplicati.Library.Utility .Distinct(Utility.ClientFilenameStringComparer); } + public static Dictionary> GetFilterStringMap() + { + // Flag messes with ToString(), so have to do this manually + Dictionary> filterMap = new Dictionary>() + { + { nameof(FilterGroup.None), GetFilterStrings(FilterGroup.None).ToList() }, + { nameof(FilterGroup.SystemFiles), GetFilterStrings(FilterGroup.SystemFiles).ToList() }, + { nameof(FilterGroup.OperatingSystem), GetFilterStrings(FilterGroup.OperatingSystem).ToList() }, + { nameof(FilterGroup.CacheFiles), GetFilterStrings(FilterGroup.CacheFiles).ToList() }, + { nameof(FilterGroup.TemporaryFiles), GetFilterStrings(FilterGroup.TemporaryFiles).ToList() }, + { nameof(FilterGroup.Applications), GetFilterStrings(FilterGroup.Applications).ToList() }, + { nameof(FilterGroup.DefaultExcludes), GetFilterStrings(FilterGroup.DefaultExcludes).ToList() }, + { nameof(FilterGroup.DefaultIncludes), GetFilterStrings(FilterGroup.DefaultIncludes).ToList() } + }; + return filterMap; + } + /// /// Filters all items that have a prefix /// diff --git a/Duplicati/Server/WebServer/RESTMethods/SystemInfo.cs b/Duplicati/Server/WebServer/RESTMethods/SystemInfo.cs index 3932e895a..bb322288a 100644 --- a/Duplicati/Server/WebServer/RESTMethods/SystemInfo.cs +++ b/Duplicati/Server/WebServer/RESTMethods/SystemInfo.cs @@ -26,7 +26,7 @@ namespace Duplicati.Server.WebServer.RESTMethods { public string Description { get { return "Gets various system properties"; } } public IEnumerable> Types - { + { get { return new KeyValuePair[] { @@ -37,9 +37,19 @@ namespace Duplicati.Server.WebServer.RESTMethods public void GET(string key, RequestInfo info) { - info.BodyWriter.OutputOK(SystemData(info)); + if (string.IsNullOrWhiteSpace(key)) + { + info.BodyWriter.OutputOK(SystemData(info)); + } + else if (key.Equals("filtergroups", StringComparison.OrdinalIgnoreCase)) + { + info.BodyWriter.OutputOK(FilterGroups()); + } + else + { + info.OutputError(code: System.Net.HttpStatusCode.NotFound, reason: "Not found"); + } } - public void POST(string key, RequestInfo info) { switch ((key ?? "").ToLowerInvariant()) @@ -110,17 +120,22 @@ namespace Duplicati.Server.WebServer.RESTMethods EnglishName = browserlanguage.EnglishName, DisplayName = browserlanguage.NativeName }, - SupportedLocales = + SupportedLocales = Library.Localization.LocalizationService.SupportedCultures - .Select(x => new { - Code = x, + .Select(x => new { + Code = x, EnglishName = new System.Globalization.CultureInfo(x).EnglishName, DisplayName = new System.Globalization.CultureInfo(x).NativeName - } + } ), BrowserLocaleSupported = Library.Localization.LocalizationService.isCultureSupported(browserlanguage) }; } + + private static object FilterGroups() + { + return new { FilterGroups = Library.Utility.FilterGroups.GetFilterStringMap() }; + } } } diff --git a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js index effac4940..52d405a84 100644 --- a/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js +++ b/Duplicati/Server/webroot/ngax/scripts/directives/sourceFolderPicker.js @@ -517,6 +517,9 @@ backupApp.directive('sourceFolderPicker', function() { scope.treedata.children = []; + // Load filter groups + AppUtils.loadFilterGroups(); + AppService.post('/filesystem?onlyfolders=false&showhidden=true', {path: '/'}).then(function(data) { var usernode = { diff --git a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js index 47bd07a29..0f9681758 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/AppUtils.js @@ -1,4 +1,4 @@ -backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogService, gettextCatalog) { +backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, AppService, DialogService, gettextCatalog) { var apputils = this; @@ -214,6 +214,17 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer $rootScope.$broadcast('apputillookupschanged'); }; + apputils.filterGroupMap = null; + apputils.loadFilterGroups = function (reload) { + if (reload || apputils.filterGroupMap === null) { + AppService.get('/systeminfo/filtergroups').then(function (data) { + apputils.filterGroupMap = angular.copy(data.data.FilterGroups); + + $rootScope.$broadcast('apputillookupschanged'); + }, apputils.connectionError); + } + } + reloadTexts(); $rootScope.$on('gettextLanguageChanged', reloadTexts); @@ -644,6 +655,34 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer return pre + body + suf; }; + this.filterToRegexpStr = function (filter) { + var firstChar = filter.substr(0, 1); + var lastChar = filter.substr(filter.length - 1, 1); + var isFilterGroup = firstChar == '{' && lastChar == '}'; + if (isFilterGroup && this.filterGroupMap !== null) { + // Replace filter groups with filter strings + var filterGroups = filter.substr(1,filter.length - 2).split(','); + var filterStrings = []; + for (var i = 0; i < filterGroups.length; ++i) { + filterStrings = filterStrings.concat(this.filterGroupMap[filterGroups[i].trim()] || []); + } + + filter = filterStrings.map(function (s) { + return '(?:' + apputils.filterToRegexpStr(s) + ')' + }).join('|'); + + } else { + var rx = firstChar == '[' && lastChar == ']'; + if (rx) + filter = filter.substr(1, filter.length - 2); + else { + filter = this.globToRegexp(filter); + } + } + + return filter; + }; + this.filterListToRegexps = function(filters, caseSensitive) { var res = []; @@ -655,14 +694,10 @@ backupApp.service('AppUtils', function($rootScope, $timeout, $cookies, DialogSer var flag = f.substr(0, 1); var filter = f.substr(1); - var rx = filter.substr(0, 1) == '[' && filter.substr(filter.length - 1, 1) == ']'; - if (rx) - filter = filter.substr(1, filter.length - 2); - else - filter = this.globToRegexp(filter); try { - res.push([flag == '+', new RegExp(filter, caseSensitive ? 'g' : 'gi')]); + var regexp = this.filterToRegexpStr(filter); + res.push([flag == '+', new RegExp(regexp, caseSensitive ? 'g' : 'gi')]); } catch (e) { } } From 32e852a392ee094a3f680e7a213000d8b6bc4b00 Mon Sep 17 00:00:00 2001 From: natan Date: Fri, 15 Mar 2024 16:51:01 +0100 Subject: [PATCH 015/238] WIP --- .../LicenseUpdater/LicenseUpdater.csproj | 22 +-- .../Abstractions/IScheduler.cs | 45 +++++ .../Abstractions/IWorkerThreadsManager.cs | 15 ++ .../Database/Connection.cs | 14 +- .../Database/ServerSettings.cs | 6 +- .../Duplicati.Library.RestAPI.csproj | 7 + Duplicati.Library.RestAPI/FIXMEGlobal.cs | 38 ++-- Duplicati.Library.RestAPI/LiveControls.cs | 18 +- .../NotificationUpdateService.cs | 50 ++++++ .../RESTMethods/Backups.cs | 2 +- .../RESTMethods/RequestInfo.cs | 4 +- .../RESTMethods/ServerState.cs | 7 +- Duplicati.Library.RestAPI/Runner.cs | 4 +- Duplicati.Library.RestAPI/Scheduler.cs | 135 ++++++++------ .../Serializable/ServerStatus.cs | 5 +- Duplicati.Library.RestAPI/UpdatePollThread.cs | 16 +- .../WorkerThreadsManager.cs | 34 ++++ .../GUI/Duplicati.GUI.TrayIcon/Program.cs | 2 +- .../AutoUpdater/IUpdateManagerAccessor.cs | 6 + .../AutoUpdater/UpdateManagerAccessor.cs | 6 + Duplicati/Library/Common/ILiveControls.cs | 6 + .../Utility/Abstractions/IBoolParser.cs | 6 + Duplicati/Library/Utility/BoolParser.cs | 11 ++ Duplicati/Library/Utility/WorkerThread.cs | 42 ++--- .../Serializer.cs | 70 ++++---- Duplicati/Server/Program.cs | 133 ++++++-------- Duplicati/UnitTest/ImportExportTests.cs | 4 +- .../WebserverCore/Abstractions/FileEntry.cs | 10 ++ .../Abstractions/ISettingsService.cs | 6 + .../Abstractions/IStatusService.cs | 8 + .../Abstractions/IUpdateService.cs | 6 + .../WebserverCore/Abstractions/IV1Endpoint.cs | 6 + .../Abstractions/ServerSettings.cs | 29 +++ .../WebserverCore/Abstractions/UpdateInfo.cs | 17 ++ .../WebserverCore/ApplicationPartsLogger.cs | 32 ++++ .../Configuration/OptionConfiguration.cs | 14 ++ .../WebserverCore/Database/Entities/Option.cs | 9 + .../WebserverCore/Database/MainDbContext.cs | 14 ++ Duplicati/WebserverCore/Dto/ServerStatus.cs | 24 +++ .../Duplicati.WebserverCore.csproj | 5 + Duplicati/WebserverCore/DuplicatiWebserver.cs | 167 ++++++++++-------- .../Endpoints/ServerStateIv1Endpoint.cs | 11 ++ .../ServiceCollectionsExtensions.cs | 41 +++++ .../Extensions/WebApplicationExtensions.cs | 37 ++++ .../WebserverCore/LegacyHttpRequestShim.cs | 12 +- .../WebserverCore/LegacyHttpResponseShim.cs | 7 +- .../WebserverCore/LegacyHttpSessionShim.cs | 8 +- .../Middlewares/StaticFilesMiddleware.cs | 28 +++ .../Services/Settings/SettingsService.cs | 110 ++++++++++++ .../WebserverCore/Services/StatusService.cs | 92 ++++++++++ .../WebserverCore/Services/UpdateService.cs | 29 +++ .../Services/UpdateServiceLogger.cs | 11 ++ .../WebserverCore/TemporaryIoCAccessor.cs | 6 + .../appsettings.Development.json | 8 + Duplicati/WebserverCore/appsettings.json | 28 +++ 55 files changed, 1144 insertions(+), 339 deletions(-) create mode 100644 Duplicati.Library.RestAPI/Abstractions/IScheduler.cs create mode 100644 Duplicati.Library.RestAPI/Abstractions/IWorkerThreadsManager.cs create mode 100644 Duplicati.Library.RestAPI/NotificationUpdateService.cs create mode 100644 Duplicati.Library.RestAPI/WorkerThreadsManager.cs create mode 100644 Duplicati/Library/AutoUpdater/IUpdateManagerAccessor.cs create mode 100644 Duplicati/Library/AutoUpdater/UpdateManagerAccessor.cs create mode 100644 Duplicati/Library/Common/ILiveControls.cs create mode 100644 Duplicati/Library/Utility/Abstractions/IBoolParser.cs create mode 100644 Duplicati/Library/Utility/BoolParser.cs create mode 100644 Duplicati/WebserverCore/Abstractions/FileEntry.cs create mode 100644 Duplicati/WebserverCore/Abstractions/ISettingsService.cs create mode 100644 Duplicati/WebserverCore/Abstractions/IStatusService.cs create mode 100644 Duplicati/WebserverCore/Abstractions/IUpdateService.cs create mode 100644 Duplicati/WebserverCore/Abstractions/IV1Endpoint.cs create mode 100644 Duplicati/WebserverCore/Abstractions/ServerSettings.cs create mode 100644 Duplicati/WebserverCore/Abstractions/UpdateInfo.cs create mode 100644 Duplicati/WebserverCore/ApplicationPartsLogger.cs create mode 100644 Duplicati/WebserverCore/Database/Configuration/OptionConfiguration.cs create mode 100644 Duplicati/WebserverCore/Database/Entities/Option.cs create mode 100644 Duplicati/WebserverCore/Database/MainDbContext.cs create mode 100644 Duplicati/WebserverCore/Dto/ServerStatus.cs create mode 100644 Duplicati/WebserverCore/Endpoints/ServerStateIv1Endpoint.cs create mode 100644 Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs create mode 100644 Duplicati/WebserverCore/Extensions/WebApplicationExtensions.cs create mode 100644 Duplicati/WebserverCore/Middlewares/StaticFilesMiddleware.cs create mode 100644 Duplicati/WebserverCore/Services/Settings/SettingsService.cs create mode 100644 Duplicati/WebserverCore/Services/StatusService.cs create mode 100644 Duplicati/WebserverCore/Services/UpdateService.cs create mode 100644 Duplicati/WebserverCore/Services/UpdateServiceLogger.cs create mode 100644 Duplicati/WebserverCore/TemporaryIoCAccessor.cs create mode 100644 Duplicati/WebserverCore/appsettings.Development.json create mode 100644 Duplicati/WebserverCore/appsettings.json diff --git a/BuildTools/LicenseUpdater/LicenseUpdater.csproj b/BuildTools/LicenseUpdater/LicenseUpdater.csproj index a44450dc4..551de1203 100755 --- a/BuildTools/LicenseUpdater/LicenseUpdater.csproj +++ b/BuildTools/LicenseUpdater/LicenseUpdater.csproj @@ -1,11 +1,11 @@ - - - - Exe - net8.0 - license_upgrader - enable - enable - - - + + + + Exe + net8.0 + license_upgrader + enable + enable + + + diff --git a/Duplicati.Library.RestAPI/Abstractions/IScheduler.cs b/Duplicati.Library.RestAPI/Abstractions/IScheduler.cs new file mode 100644 index 000000000..0d7016e78 --- /dev/null +++ b/Duplicati.Library.RestAPI/Abstractions/IScheduler.cs @@ -0,0 +1,45 @@ +using System; +using System.Collections.Generic; +using Duplicati.Library.Utility; +using Duplicati.Server; +using Duplicati.Server.Serialization.Interface; + +namespace Duplicati.WebserverCore.Abstractions; + +public interface IScheduler +{ + /// + /// Initializes scheduler + /// + /// The worker thread + void Init(WorkerThread worker); + + IList> GetSchedulerQueueIds(); + + /// + /// Terminates the thread. Any items still in queue will be removed + /// + /// True if the call should block until the thread has exited, false otherwise + void Terminate(bool wait); + + /// + /// An event that is raised when the schedule changes + /// + event EventHandler NewSchedule; + + /// + /// A snapshot copy of the current schedule list + /// + List> Schedule { get; } + + /// + /// A snapshot copy of the current worker queue, that is items that are scheduled, but waiting for execution + /// + List WorkerQueue { get; } + + /// + /// Forces the scheduler to re-evaluate the order. + /// Call this method if something changes + /// + void Reschedule(); +} \ No newline at end of file diff --git a/Duplicati.Library.RestAPI/Abstractions/IWorkerThreadsManager.cs b/Duplicati.Library.RestAPI/Abstractions/IWorkerThreadsManager.cs new file mode 100644 index 000000000..6e4aa3b15 --- /dev/null +++ b/Duplicati.Library.RestAPI/Abstractions/IWorkerThreadsManager.cs @@ -0,0 +1,15 @@ +#nullable enable +using System; +using Duplicati.Library.Utility; +using Duplicati.Server; + +namespace Duplicati.Library.RestAPI.Abstractions; + +public interface IWorkerThreadsManager +{ + void Spawn(Action item); + + Tuple? CurrentTask { get; } + WorkerThread? WorkerThread { get; } + void UpdateThrottleSpeeds(); +} \ No newline at end of file diff --git a/Duplicati.Library.RestAPI/Database/Connection.cs b/Duplicati.Library.RestAPI/Database/Connection.cs index 4fd7b1f90..fb35401d7 100644 --- a/Duplicati.Library.RestAPI/Database/Connection.cs +++ b/Duplicati.Library.RestAPI/Database/Connection.cs @@ -496,7 +496,7 @@ namespace Duplicati.Server.Database } } - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } @@ -598,7 +598,7 @@ namespace Duplicati.Server.Database } tr.Commit(); - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } } @@ -611,7 +611,7 @@ namespace Duplicati.Server.Database { AddOrUpdateSchedule(item, tr); tr.Commit(); - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } } @@ -674,7 +674,7 @@ namespace Duplicati.Server.Database } } - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } @@ -694,7 +694,7 @@ namespace Duplicati.Server.Database lock(m_lock) DeleteFromDb("Schedule", ID); - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } @@ -782,7 +782,7 @@ namespace Duplicati.Server.Database FIXMEGlobal.DataConnection.ApplicationSettings.UnackedWarning = notifications.Any(x => x.ID != id && x.Type == Duplicati.Server.Serialization.NotificationType.Warning); } - FIXMEGlobal.IncrementLastNotificationUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastNotificationUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); return true; @@ -822,7 +822,7 @@ namespace Duplicati.Server.Database FIXMEGlobal.DataConnection.ApplicationSettings.UnackedWarning = true; } - FIXMEGlobal.IncrementLastNotificationUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastNotificationUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } diff --git a/Duplicati.Library.RestAPI/Database/ServerSettings.cs b/Duplicati.Library.RestAPI/Database/ServerSettings.cs index 92b02bcfa..7f6fe2194 100644 --- a/Duplicati.Library.RestAPI/Database/ServerSettings.cs +++ b/Duplicati.Library.RestAPI/Database/ServerSettings.cs @@ -27,7 +27,7 @@ namespace Duplicati.Server.Database { public class ServerSettings { - private static class CONST + public static class CONST { public const string STARTUP_DELAY = "startup-delay"; public const string DOWNLOAD_SPEED_LIMIT = "max-download-speed"; @@ -113,13 +113,13 @@ namespace Duplicati.Server.Database Value = n.Value }, Database.Connection.SERVER_SETTINGS_ID); - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); // In case the usage reporter is enabled or disabled, refresh now FIXMEGlobal.StartOrStopUsageReporter(); // If throttle options were changed, update now - FIXMEGlobal.UpdateThrottleSpeeds(); + FIXMEGlobal.WorkerThreadsManager.UpdateThrottleSpeeds(); } public string StartupDelayDuration diff --git a/Duplicati.Library.RestAPI/Duplicati.Library.RestAPI.csproj b/Duplicati.Library.RestAPI/Duplicati.Library.RestAPI.csproj index e10690aa5..3e01d4bd5 100644 --- a/Duplicati.Library.RestAPI/Duplicati.Library.RestAPI.csproj +++ b/Duplicati.Library.RestAPI/Duplicati.Library.RestAPI.csproj @@ -17,6 +17,9 @@ + + ..\Executables\net8\Duplicati.Server\bin\Debug\net8.0\Duplicati.WebserverCore.dll + ..\thirdparty\HttpServer\HttpServer.dll @@ -32,4 +35,8 @@ + + + + diff --git a/Duplicati.Library.RestAPI/FIXMEGlobal.cs b/Duplicati.Library.RestAPI/FIXMEGlobal.cs index 2a8fc3148..3cb688507 100644 --- a/Duplicati.Library.RestAPI/FIXMEGlobal.cs +++ b/Duplicati.Library.RestAPI/FIXMEGlobal.cs @@ -2,6 +2,11 @@ using Duplicati.Server; using System; using System.Collections.Generic; +using Duplicati.Library.IO; +using Duplicati.Library.RestAPI.Abstractions; +using Duplicati.Library.Utility; +using Duplicati.WebserverCore.Abstractions; +using Microsoft.Extensions.DependencyInjection; namespace Duplicati.Library.RestAPI { @@ -11,7 +16,8 @@ namespace Duplicati.Library.RestAPI */ public static class FIXMEGlobal { - + public static IServiceProvider Provider { get; set; } + /// /// This is the only access to the database /// @@ -20,7 +26,7 @@ namespace Duplicati.Library.RestAPI /// /// The controller interface for pause/resume and throttle options /// - public static LiveControls LiveControl; + public static LiveControls LiveControl => Provider.GetRequiredService(); /// /// A delegate method for creating a copy of the current progress state @@ -30,24 +36,24 @@ namespace Duplicati.Library.RestAPI /// /// The status event signaler, used to control long polling of status updates /// - public static readonly EventPollNotify StatusEventNotifyer = new EventPollNotify(); + public static EventPollNotify StatusEventNotifyer => Provider.GetRequiredService(); + + /// + /// For keeping and incrementing last last events Ids of db save and last notification + /// + public static INotificationUpdateService NotificationUpdateService => Provider.GetRequiredService(); /// /// This is the working thread /// - public static Duplicati.Library.Utility.WorkerThread WorkThread; + public static WorkerThread WorkThread => + Provider.GetRequiredService().WorkerThread; - public static Func PeekLastDataUpdateID; - public static Func PeekLastNotificationUpdateID; - - public static Action IncrementLastDataUpdateID; - - public static Action IncrementLastNotificationUpdateID; + public static IWorkerThreadsManager WorkerThreadsManager => + Provider.GetRequiredService(); public static Action StartOrStopUsageReporter; - public static Action UpdateThrottleSpeeds; - /// /// Gets the folder where Duplicati data is stored /// @@ -56,19 +62,19 @@ namespace Duplicati.Library.RestAPI /// /// This is the scheduling thread /// - public static Scheduler Scheduler; + public static IScheduler Scheduler => Provider.GetRequiredService(); /// /// The log redirect handler /// public static readonly LogWriteHandler LogHandler = new LogWriteHandler(); - public static Func, Server.Database.Connection> GetDatabaseConnection; + public static Func, Server.Database.Connection> GetDatabaseConnection; /// /// The update poll thread. /// - public static UpdatePollThread UpdatePoller; + public static UpdatePollThread UpdatePoller => Provider.GetRequiredService(); /// @@ -93,5 +99,7 @@ namespace Duplicati.Library.RestAPI /// This is the lock to be used before manipulating the shared resources /// public static readonly object MainLock = new object(); + + private static LiveControls _liveControl; } } diff --git a/Duplicati.Library.RestAPI/LiveControls.cs b/Duplicati.Library.RestAPI/LiveControls.cs index 1a51636a3..c3d0d24d0 100644 --- a/Duplicati.Library.RestAPI/LiveControls.cs +++ b/Duplicati.Library.RestAPI/LiveControls.cs @@ -18,9 +18,8 @@ // #endregion using System; -using System.Collections.Generic; -using System.Text; using Duplicati.Library.Common; +using Duplicati.Library.IO; using Duplicati.Library.RestAPI; namespace Duplicati.Server @@ -29,7 +28,7 @@ namespace Duplicati.Server /// This class keeps track of the users modifications regarding /// throttling and pause/resume /// - public class LiveControls + public class LiveControls : ILiveControls { /// /// The tag used for logging @@ -86,6 +85,8 @@ namespace Duplicati.Server /// public LiveControlState State { get { return m_state; } } + public bool IsPaused => State == LiveControlState.Paused; + /// /// The internal variable that tracks the the priority /// @@ -160,7 +161,7 @@ namespace Duplicati.Server /// /// The timer that is activated after a pause period. /// - private readonly System.Threading.Timer m_waitTimer; + private System.Threading.Timer m_waitTimer; /// /// The time that the current pause is expected to expire @@ -170,7 +171,14 @@ namespace Duplicati.Server /// /// Constructs a new instance of the LiveControl /// - public LiveControls(Database.ServerSettings settings) + public LiveControls() + { + } + + /// + /// Constructs a new instance of the LiveControl + /// + public void Init(Database.ServerSettings settings) { m_state = LiveControlState.Running; m_waitTimer = new System.Threading.Timer(m_waitTimer_Tick, this, System.Threading.Timeout.Infinite, System.Threading.Timeout.Infinite); diff --git a/Duplicati.Library.RestAPI/NotificationUpdateService.cs b/Duplicati.Library.RestAPI/NotificationUpdateService.cs new file mode 100644 index 000000000..2bbec0ce3 --- /dev/null +++ b/Duplicati.Library.RestAPI/NotificationUpdateService.cs @@ -0,0 +1,50 @@ +namespace Duplicati.Library.RestAPI; + +public interface INotificationUpdateService +{ + /// + /// An event ID that increases whenever the database is updated + /// + long LastDataUpdateId { get; } + + /// + /// An event ID that increases whenever a notification is updated + /// + long LastNotificationUpdateId { get; } + + void IncrementLastDataUpdateId(); + void IncrementLastNotificationUpdateId(); +} + +public class NotificationUpdateService : INotificationUpdateService +{ + /// + /// An event ID that increases whenever the database is updated + /// + public long LastDataUpdateId { get; private set; } = 0; + + private readonly object _lastDataUpdateIdLock = new(); + + /// + /// An event ID that increases whenever a notification is updated + /// + public long LastNotificationUpdateId { get; private set; } = 0; + + private readonly object _lastNotificationUpdateIdLock = new(); + + public void IncrementLastDataUpdateId() + { + lock (_lastDataUpdateIdLock) + { + LastDataUpdateId++; + } + } + + public void IncrementLastNotificationUpdateId() + { + lock (_lastNotificationUpdateIdLock) + { + LastNotificationUpdateId++; + } + } +} \ No newline at end of file diff --git a/Duplicati.Library.RestAPI/RESTMethods/Backups.cs b/Duplicati.Library.RestAPI/RESTMethods/Backups.cs index bae68fa57..2c06f365d 100644 --- a/Duplicati.Library.RestAPI/RESTMethods/Backups.cs +++ b/Duplicati.Library.RestAPI/RESTMethods/Backups.cs @@ -132,7 +132,7 @@ namespace Duplicati.Server.WebServer.RESTMethods Serializable.ImportExportStructure importedStructure = Backups.LoadConfiguration(configurationFile, importMetadata, getPassword); // This will create the Duplicati-server.sqlite database file if it doesn't exist. - using (Duplicati.Server.Database.Connection connection = FIXMEGlobal.GetDatabaseConnection(advancedOptions)) + using (Duplicati.Server.Database.Connection connection = FIXMEGlobal.GetDatabaseConnection(null, advancedOptions)) { if (connection.Backups.Any(x => x.Name.Equals(importedStructure.Backup.Name, StringComparison.OrdinalIgnoreCase))) { diff --git a/Duplicati.Library.RestAPI/RESTMethods/RequestInfo.cs b/Duplicati.Library.RestAPI/RESTMethods/RequestInfo.cs index 2069b6cf9..f0c511ef4 100644 --- a/Duplicati.Library.RestAPI/RESTMethods/RequestInfo.cs +++ b/Duplicati.Library.RestAPI/RESTMethods/RequestInfo.cs @@ -24,8 +24,8 @@ namespace Duplicati.Server.WebServer.RESTMethods { public class RequestInfo : IDisposable { - public HttpServer.IHttpRequest Request { get; private set; } - public HttpServer.IHttpResponse Response { get; private set; } + public HttpServer.IHttpRequest Request { get; } + public HttpServer.IHttpResponse Response { get; } public HttpServer.Sessions.IHttpSession Session { get; private set; } public BodyWriter BodyWriter { get; private set; } public RequestInfo(HttpServer.IHttpRequest request, HttpServer.IHttpResponse response, HttpServer.Sessions.IHttpSession session) diff --git a/Duplicati.Library.RestAPI/RESTMethods/ServerState.cs b/Duplicati.Library.RestAPI/RESTMethods/ServerState.cs index bd1fa914a..70c382b1d 100644 --- a/Duplicati.Library.RestAPI/RESTMethods/ServerState.cs +++ b/Duplicati.Library.RestAPI/RESTMethods/ServerState.cs @@ -24,7 +24,7 @@ using System.Collections.Generic; namespace Duplicati.Server.WebServer.RESTMethods { - public class ServerState : IRESTMethodGET, IRESTMethodPOST, IRESTMethodDocumented + public class ServerState : IRESTMethodPOST, IRESTMethodDocumented { public void GET(string key, RequestInfo info) { @@ -32,16 +32,17 @@ namespace Duplicati.Server.WebServer.RESTMethods long id = 0; long.TryParse(key, out id); + var serverStatus = new Serializable.ServerStatus(); if (info.LongPollCheck(FIXMEGlobal.StatusEventNotifyer, ref id, out isError)) { //Make sure we do not report a higher number than the eventnotifier says - var st = new Serializable.ServerStatus(); + var st = serverStatus; st.LastEventID = id; info.OutputOK(st); } else if (!isError) { - info.OutputOK(new Serializable.ServerStatus()); + info.OutputOK(serverStatus); } } diff --git a/Duplicati.Library.RestAPI/Runner.cs b/Duplicati.Library.RestAPI/Runner.cs index 1dcdc267c..76554701b 100644 --- a/Duplicati.Library.RestAPI/Runner.cs +++ b/Duplicati.Library.RestAPI/Runner.cs @@ -694,7 +694,7 @@ namespace Duplicati.Server if (ex is UserInformationException exception) messageid = exception.HelpID; - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.DataConnection.RegisterNotification( NotificationType.Error, backup.IsTemporary ? @@ -876,7 +876,7 @@ namespace Duplicati.Server if (!backup.IsTemporary) FIXMEGlobal.DataConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null); - FIXMEGlobal.IncrementLastDataUpdateID(); + FIXMEGlobal.NotificationUpdateService.IncrementLastDataUpdateId(); FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } diff --git a/Duplicati.Library.RestAPI/Scheduler.cs b/Duplicati.Library.RestAPI/Scheduler.cs index 81a8cdf41..7544bb7c2 100644 --- a/Duplicati.Library.RestAPI/Scheduler.cs +++ b/Duplicati.Library.RestAPI/Scheduler.cs @@ -1,4 +1,5 @@ #region Disclaimer / License + // Copyright (C) 2015, The Duplicati Team // http://www.duplicati.com, info@duplicati.com // @@ -18,8 +19,8 @@ // using Duplicati.Server.Serialization.Interface; - #endregion + using System; using System.Collections.Generic; using System.Text; @@ -27,32 +28,37 @@ using System.Linq; using System.Threading; using Duplicati.Library.Utility; using Duplicati.Library.RestAPI; +using Duplicati.WebserverCore.Abstractions; namespace Duplicati.Server { /// /// This class handles scheduled runs of backups /// - public class Scheduler + public class Scheduler : IScheduler { private static readonly string LOGTAG = Duplicati.Library.Logging.Log.LogTagFromType(); /// /// The thread that runs the scheduler /// - private readonly Thread m_thread; + private Thread m_thread; + /// /// A termination flag /// private volatile bool m_terminate; + /// /// The worker thread that is invoked to do work /// - private readonly WorkerThread m_worker; + private WorkerThread m_worker; + /// /// The wait event /// - private readonly AutoResetEvent m_event; + private AutoResetEvent m_event; + /// /// The data synchronization lock /// @@ -67,17 +73,24 @@ namespace Duplicati.Server /// The currently scheduled items /// private KeyValuePair[] m_schedule; - + /// /// List of update tasks, used to set the timestamp on the schedule once completed /// - private readonly Dictionary> m_updateTasks; + private Dictionary> m_updateTasks; /// /// Constructs a new scheduler /// + public Scheduler() + { + } + + /// + /// Initializes scheduler + /// /// The worker thread - public Scheduler(WorkerThread worker) + public void Init(WorkerThread worker) { m_thread = new Thread(new ThreadStart(Runner)); m_worker = worker; @@ -92,6 +105,13 @@ namespace Duplicati.Server m_thread.Start(); } + public IList> GetSchedulerQueueIds() + { + return (from n in WorkerQueue + where n.Backup != null + select new Tuple(n.TaskID, n.Backup.ID)).ToList(); + } + /// /// Forces the scheduler to re-evaluate the order. /// Call this method if something changes @@ -104,13 +124,13 @@ namespace Duplicati.Server /// /// A snapshot copy of the current schedule list /// - public List> Schedule - { - get + public List> Schedule + { + get { lock (m_lock) return m_schedule.ToList(); - } + } } /// @@ -118,10 +138,7 @@ namespace Duplicati.Server /// public List WorkerQueue { - get - { - return (from t in m_worker.CurrentTasks where t != null select t).ToList(); - } + get { return (from t in m_worker.CurrentTasks where t != null select t).ToList(); } } /// @@ -135,8 +152,13 @@ namespace Duplicati.Server if (wait) { - try { m_thread.Join(); } - catch { } + try + { + m_thread.Join(); + } + catch + { + } } } @@ -148,7 +170,8 @@ namespace Duplicati.Server /// The repetition interval /// The days the backup is allowed to run /// The next valid date, or throws an exception if no such date can be found - public static DateTime GetNextValidTime(DateTime basetime, DateTime firstdate, string repetition, DayOfWeek[] allowedDays) + public static DateTime GetNextValidTime(DateTime basetime, DateTime firstdate, string repetition, + DayOfWeek[] allowedDays) { var res = basetime; @@ -195,26 +218,25 @@ namespace Duplicati.Server throw new Exception(Strings.Scheduler.InvalidTimeSetupError(basetime, repetition, sb.ToString())); } - + return res; } - + private void OnCompleted(WorkerThread worker, Runner.IRunnerData task) { Tuple t = null; - lock(m_lock) + lock (m_lock) { if (task != null && m_updateTasks.TryGetValue(task, out t)) m_updateTasks.Remove(task); } - + if (t != null) { t.Item1.Time = t.Item2; t.Item1.LastRun = t.Item3; FIXMEGlobal.DataConnection.AddOrUpdateSchedule(t.Item1); } - } private void OnStartingWork(WorkerThread worker, Runner.IRunnerData task) @@ -223,8 +245,8 @@ namespace Duplicati.Server { return; } - - lock(m_lock) + + lock (m_lock) { if (m_updateTasks.TryGetValue(task, out Tuple scheduleInfo)) { @@ -245,10 +267,10 @@ namespace Duplicati.Server { //TODO: As this is executed repeatedly we should cache it // to avoid frequent db lookups - + //Determine schedule list var lst = FIXMEGlobal.DataConnection.Schedules; - foreach(var sc in lst) + foreach (var sc in lst) { if (!string.IsNullOrEmpty(sc.Repeat)) { @@ -267,7 +289,7 @@ namespace Duplicati.Server { start = startkey.Value; } - + try { // Recover from timedrift issues by overriding the dates if the last run date is in the future. @@ -276,11 +298,13 @@ namespace Duplicati.Server start = DateTime.UtcNow; last = DateTime.UtcNow; } + start = GetNextValidTime(start, last, sc.Repeat, sc.AllowedDays); } catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", ex); + FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", + ex); } //If time is exceeded, run it now @@ -288,15 +312,17 @@ namespace Duplicati.Server { var jobsToRun = new List(); //TODO: Cache this to avoid frequent lookups - foreach(var id in FIXMEGlobal.DataConnection.GetBackupIDsForTags(sc.Tags).Distinct().Select(x => x.ToString())) + foreach (var id in FIXMEGlobal.DataConnection.GetBackupIDsForTags(sc.Tags).Distinct() + .Select(x => x.ToString())) { //See if it is already queued var tmplst = from n in m_worker.CurrentTasks - where n.Operation == Duplicati.Server.Serialization.DuplicatiOperation.Backup - select n.Backup; + where n.Operation == Duplicati.Server.Serialization.DuplicatiOperation.Backup + select n.Backup; var tastTemp = m_worker.CurrentTask; - if (tastTemp != null && tastTemp.Operation == Duplicati.Server.Serialization.DuplicatiOperation.Backup) - tmplst = tmplst.Union(new [] { tastTemp.Backup }); + if (tastTemp != null && tastTemp.Operation == + Duplicati.Server.Serialization.DuplicatiOperation.Backup) + tmplst = tmplst.Union(new[] { tastTemp.Backup }); //If it is not already in queue, put it there if (!tmplst.Any(x => x.ID == id)) @@ -306,13 +332,18 @@ namespace Duplicati.Server { Dictionary options = Duplicati.Server.Runner.GetCommonOptions(); Duplicati.Server.Runner.ApplyOptions(entry, options); - if ((new Duplicati.Library.Main.Options(options)).DisableOnBattery && (Duplicati.Library.Utility.Power.PowerSupply.GetSource() == Duplicati.Library.Utility.Power.PowerSupply.Source.Battery)) + if ((new Duplicati.Library.Main.Options(options)).DisableOnBattery && + (Duplicati.Library.Utility.Power.PowerSupply.GetSource() == + Duplicati.Library.Utility.Power.PowerSupply.Source.Battery)) { - Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, "BackupDisabledOnBattery", "Scheduled backup disabled while on battery power."); + Duplicati.Library.Logging.Log.WriteInformationMessage(LOGTAG, + "BackupDisabledOnBattery", + "Scheduled backup disabled while on battery power."); } else { - jobsToRun.Add(Server.Runner.CreateTask(Duplicati.Server.Serialization.DuplicatiOperation.Backup, entry)); + jobsToRun.Add(Server.Runner.CreateTask( + Duplicati.Server.Serialization.DuplicatiOperation.Backup, entry)); } } } @@ -321,27 +352,32 @@ namespace Duplicati.Server // Calculate next time, by finding the first entry later than now try { - start = GetNextValidTime(start, new DateTime(Math.Max(DateTime.UtcNow.AddSeconds(1).Ticks, start.AddSeconds(1).Ticks), DateTimeKind.Utc), sc.Repeat, sc.AllowedDays); + start = GetNextValidTime(start, + new DateTime( + Math.Max(DateTime.UtcNow.AddSeconds(1).Ticks, start.AddSeconds(1).Ticks), + DateTimeKind.Utc), sc.Repeat, sc.AllowedDays); } - catch(Exception ex) + catch (Exception ex) { - FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), "Scheduler failed to find next date", ex); + FIXMEGlobal.DataConnection.LogError(sc.ID.ToString(), + "Scheduler failed to find next date", ex); continue; } - + Server.Runner.IRunnerData lastJob = jobsToRun.LastOrDefault(); if (lastJob != null) { lock (m_lock) { // The actual last run time will be updated when the StartingWork event is raised. - m_updateTasks[lastJob] = new Tuple(sc, start, DateTime.UtcNow); + m_updateTasks[lastJob] = + new Tuple(sc, start, DateTime.UtcNow); } } foreach (var job in jobsToRun) m_worker.AddTask(job); - + if (start < DateTime.UtcNow) { //TODO: Report this somehow @@ -349,20 +385,20 @@ namespace Duplicati.Server } } - scheduled[sc.ID] = new KeyValuePair(scticks, start); + scheduled[sc.ID] = new KeyValuePair(scticks, start); } } var existing = lst.ToDictionary(x => x.ID); //Sort them, lock as we assign the m_schedule variable - lock(m_lock) + lock (m_lock) m_schedule = (from n in scheduled where existing.ContainsKey(n.Key) orderby n.Value.Value select new KeyValuePair(n.Value.Value, existing[n.Key])).ToArray(); // Remove unused entries - foreach(var c in (from n in scheduled where !existing.ContainsKey(n.Key) select n.Key).ToArray()) + foreach (var c in (from n in scheduled where !existing.ContainsKey(n.Key) select n.Key).ToArray()) scheduled.Remove(c); //Raise event if needed @@ -406,8 +442,7 @@ namespace Duplicati.Server if (allowedDays == null || allowedDays.Length == 0) return true; else - return Array.IndexOf(allowedDays, localTime.DayOfWeek) >= 0; + return Array.IndexOf(allowedDays, localTime.DayOfWeek) >= 0; } - } -} +} \ No newline at end of file diff --git a/Duplicati.Library.RestAPI/Serializable/ServerStatus.cs b/Duplicati.Library.RestAPI/Serializable/ServerStatus.cs index c77ae6027..6f0843018 100644 --- a/Duplicati.Library.RestAPI/Serializable/ServerStatus.cs +++ b/Duplicati.Library.RestAPI/Serializable/ServerStatus.cs @@ -139,10 +139,9 @@ namespace Duplicati.Server.Serializable set { m_lastEventID = value; } } - public long LastDataUpdateID { get { return FIXMEGlobal.PeekLastDataUpdateID(); } } - - public long LastNotificationUpdateID { get { return FIXMEGlobal.PeekLastNotificationUpdateID(); } } + public long LastDataUpdateID => FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; + public long LastNotificationUpdateID => FIXMEGlobal.NotificationUpdateService.LastNotificationUpdateId; } } diff --git a/Duplicati.Library.RestAPI/UpdatePollThread.cs b/Duplicati.Library.RestAPI/UpdatePollThread.cs index c82bd8d2d..869f992d0 100644 --- a/Duplicati.Library.RestAPI/UpdatePollThread.cs +++ b/Duplicati.Library.RestAPI/UpdatePollThread.cs @@ -28,12 +28,12 @@ namespace Duplicati.Server /// public class UpdatePollThread { - private readonly Thread m_thread; + private Thread m_thread; private volatile bool m_terminated = false; private volatile bool m_download = false; private volatile bool m_forceCheck = false; private readonly object m_lock = new object(); - private readonly AutoResetEvent m_waitSignal; + private AutoResetEvent m_waitSignal; private double m_downloadProgress; public bool IsUpdateRequested { get; private set; } = false; @@ -51,14 +51,16 @@ namespace Duplicati.Server FIXMEGlobal.StatusEventNotifyer.SignalNewEvent(); } } - - public UpdatePollThread() + + public void Init() { m_waitSignal = new AutoResetEvent(false); ThreadState = UpdatePollerStates.Waiting; - m_thread = new Thread(Run); - m_thread.IsBackground = true; - m_thread.Name = "UpdatePollThread"; + m_thread = new Thread(Run) + { + IsBackground = true, + Name = "UpdatePollThread" + }; m_thread.Start(); } diff --git a/Duplicati.Library.RestAPI/WorkerThreadsManager.cs b/Duplicati.Library.RestAPI/WorkerThreadsManager.cs new file mode 100644 index 000000000..79caca695 --- /dev/null +++ b/Duplicati.Library.RestAPI/WorkerThreadsManager.cs @@ -0,0 +1,34 @@ +#nullable enable +using System; +using Duplicati.Library.IO; +using Duplicati.Library.RestAPI.Abstractions; +using Duplicati.Library.Utility; +using Duplicati.Server; +using Duplicati.WebserverCore.Abstractions; + +namespace Duplicati.Library.RestAPI; + +public class WorkerThreadsManager(ILiveControls liveControls, IScheduler scheduler) : IWorkerThreadsManager +{ + public WorkerThread? WorkerThread { get; private set; } + + public void Spawn(Action item) + { + WorkerThread = new WorkerThread(item, liveControls.IsPaused); + scheduler.Init(WorkerThread); + } + + public Tuple? CurrentTask + { + get + { + var t = WorkerThread?.CurrentTask; + return t == null ? null : new Tuple(t.TaskID, t.Backup.ID); + } + } + + public void UpdateThrottleSpeeds() + { + WorkerThread?.CurrentTask?.UpdateThrottleSpeed(); + } +} \ No newline at end of file diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs index 857abfeba..db69443ad 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Program.cs @@ -140,7 +140,7 @@ namespace Duplicati.GUI.TrayIcon } else if (Library.Utility.Utility.ParseBoolOption(options, READCONFIGFROMDB_OPTION)) { - databaseConnection = Server.Program.GetDatabaseConnection(options); + databaseConnection = Server.Program.GetDatabaseConnection(null, options); if (databaseConnection != null) { diff --git a/Duplicati/Library/AutoUpdater/IUpdateManagerAccessor.cs b/Duplicati/Library/AutoUpdater/IUpdateManagerAccessor.cs new file mode 100644 index 000000000..26763743a --- /dev/null +++ b/Duplicati/Library/AutoUpdater/IUpdateManagerAccessor.cs @@ -0,0 +1,6 @@ +namespace Duplicati.Library.AutoUpdater; + +public interface IUpdateManagerAccessor +{ + bool HasUpdateInstalled { get; } +} \ No newline at end of file diff --git a/Duplicati/Library/AutoUpdater/UpdateManagerAccessor.cs b/Duplicati/Library/AutoUpdater/UpdateManagerAccessor.cs new file mode 100644 index 000000000..cc1cae43c --- /dev/null +++ b/Duplicati/Library/AutoUpdater/UpdateManagerAccessor.cs @@ -0,0 +1,6 @@ +namespace Duplicati.Library.AutoUpdater; + +public class UpdateManagerAccessor : IUpdateManagerAccessor +{ + public bool HasUpdateInstalled => UpdaterManager.HasUpdateInstalled; +} diff --git a/Duplicati/Library/Common/ILiveControls.cs b/Duplicati/Library/Common/ILiveControls.cs new file mode 100644 index 000000000..cef107b11 --- /dev/null +++ b/Duplicati/Library/Common/ILiveControls.cs @@ -0,0 +1,6 @@ +namespace Duplicati.Library.IO; + +public interface ILiveControls +{ + bool IsPaused { get; } +} \ No newline at end of file diff --git a/Duplicati/Library/Utility/Abstractions/IBoolParser.cs b/Duplicati/Library/Utility/Abstractions/IBoolParser.cs new file mode 100644 index 000000000..782b80a76 --- /dev/null +++ b/Duplicati/Library/Utility/Abstractions/IBoolParser.cs @@ -0,0 +1,6 @@ +namespace Duplicati.Library.Utility.Abstractions; + +public interface IBoolParser +{ + bool ParseBool(string value, bool @default = false); +} \ No newline at end of file diff --git a/Duplicati/Library/Utility/BoolParser.cs b/Duplicati/Library/Utility/BoolParser.cs new file mode 100644 index 000000000..1f4f0d757 --- /dev/null +++ b/Duplicati/Library/Utility/BoolParser.cs @@ -0,0 +1,11 @@ +using Duplicati.Library.Utility.Abstractions; + +namespace Duplicati.Library.Utility; + +public class BoolParser : IBoolParser +{ + public bool ParseBool(string value, bool @default = false) + { + return Utility.ParseBool(value, @default); + } +} \ No newline at end of file diff --git a/Duplicati/Library/Utility/WorkerThread.cs b/Duplicati/Library/Utility/WorkerThread.cs index dfdd99f2a..267b40e89 100644 --- a/Duplicati/Library/Utility/WorkerThread.cs +++ b/Duplicati/Library/Utility/WorkerThread.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2024, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; @@ -391,4 +391,4 @@ namespace Duplicati.Library.Utility return true; } } -} +} \ No newline at end of file diff --git a/Duplicati/Server/Duplicati.Server.Serialization/Serializer.cs b/Duplicati/Server/Duplicati.Server.Serialization/Serializer.cs index 14c410ee7..79ab5492d 100644 --- a/Duplicati/Server/Duplicati.Server.Serialization/Serializer.cs +++ b/Duplicati/Server/Duplicati.Server.Serialization/Serializer.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2024, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.IO; @@ -31,26 +31,28 @@ namespace Duplicati.Server.Serialization { public class Serializer { - protected static readonly JsonSerializerSettings m_jsonSettings; + public static JsonSerializerSettings JsonSettings { get; } protected static readonly Formatting m_jsonFormatting = Formatting.Indented; static Serializer() { - m_jsonSettings = new JsonSerializerSettings(); - m_jsonSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; - m_jsonSettings.Converters = new JsonConverter[] { - new DayOfWeekConcerter(), - new StringEnumConverter(), - new SerializableStatusCreator(), - new SettingsCreator(), - new FilterCreator(), - new NotificationCreator(), - }.ToList(); + JsonSettings = new JsonSerializerSettings + { + ReferenceLoopHandling = ReferenceLoopHandling.Ignore, + Converters = new JsonConverter[] { + new DayOfWeekConcerter(), + new StringEnumConverter(), + new SerializableStatusCreator(), + new SettingsCreator(), + new FilterCreator(), + new NotificationCreator(), + }.ToList() + }; } public static void SerializeJson(System.IO.TextWriter sw, object o, bool preventDispose = false) { - Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(m_jsonSettings); + Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(JsonSettings); var jsonWriter = new JsonTextWriter(sw); using (preventDispose ? null : jsonWriter) { @@ -63,7 +65,7 @@ namespace Duplicati.Server.Serialization public static async Task SerializeJsonAsync(System.IO.TextWriter tw, object o, bool preventDispose = false) { - Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(m_jsonSettings); + Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(JsonSettings); StringBuilder sb = new StringBuilder(); StringWriter sw = new StringWriter(sb); var jsonWriter = new JsonTextWriter(sw); @@ -80,7 +82,7 @@ namespace Duplicati.Server.Serialization public static T Deserialize(System.IO.TextReader sr) { - Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(m_jsonSettings); + Newtonsoft.Json.JsonSerializer jsonSerializer = Newtonsoft.Json.JsonSerializer.Create(JsonSettings); using (var jsonReader = new JsonTextReader(sr)) { jsonReader.Culture = System.Globalization.CultureInfo.InvariantCulture; diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 82dcf0db8..28b2ef77b 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -24,8 +24,13 @@ using System.Globalization; using System.Linq; using Duplicati.Library.Common; using Duplicati.Library.Common.IO; +using Duplicati.Library.IO; using Duplicati.Library.RestAPI; using Duplicati.WebserverCore; +using Duplicati.WebserverCore.Abstractions; +using Duplicati.WebserverCore.Database; +using Microsoft.Extensions.DependencyInjection; +using Sharp.Xmpp.Extensions.Dataforms; namespace Duplicati.Server { @@ -78,12 +83,7 @@ namespace Duplicati.Server /// /// This is the scheduling thread /// - public static Scheduler Scheduler { get => FIXMEGlobal.Scheduler; set => FIXMEGlobal.Scheduler = value; } - - /// - /// This is the working thread - /// - public static Duplicati.Library.Utility.WorkerThread WorkThread { get => FIXMEGlobal.WorkThread; set => FIXMEGlobal.WorkThread = value; } + public static IScheduler Scheduler { get => FIXMEGlobal.Scheduler; } /// /// List of completed task results @@ -108,7 +108,7 @@ namespace Duplicati.Server /// /// The controller interface for pause/resume and throttle options /// - public static LiveControls LiveControl { get => FIXMEGlobal.LiveControl; set => FIXMEGlobal.LiveControl = value; } + public static LiveControls LiveControl { get => DuplicatiWebserver.Provider.GetRequiredService() ; } /// /// The application exit event @@ -120,15 +120,23 @@ namespace Duplicati.Server /// private static WebServer.Server WebServer; + /// + /// Duplicati webserver instance + /// + public static DuplicatiWebserver DuplicatiWebserver { get; set; } + /// /// Callback to shutdown the modern webserver /// - private static Action ShutdownModernWebserver; + private static void ShutdownModernWebserver() + { + DuplicatiWebserver.Stop().GetAwaiter().GetResult(); + } /// /// The update poll thread. /// - public static UpdatePollThread UpdatePoller { get => FIXMEGlobal.UpdatePoller; set => FIXMEGlobal.UpdatePoller = value; } + public static UpdatePollThread UpdatePoller => FIXMEGlobal.UpdatePoller; /// /// An event that is set once the server is ready to respond to requests @@ -138,23 +146,13 @@ namespace Duplicati.Server /// /// The status event signaler, used to control long polling of status updates /// - public static EventPollNotify StatusEventNotifyer { get => FIXMEGlobal.StatusEventNotifyer; } + public static EventPollNotify StatusEventNotifyer => FIXMEGlobal.Provider.GetRequiredService(); /// /// A delegate method for creating a copy of the current progress state /// public static Func GenerateProgressState { get => FIXMEGlobal.GenerateProgressState; set => FIXMEGlobal.GenerateProgressState = value; } - /// - /// An event ID that increases whenever the database is updated - /// - public static long LastDataUpdateID = 0; - - /// - /// An event ID that increases whenever a notification is updated - /// - public static long LastNotificationUpdateID = 0; - /// /// The log redirect handler /// @@ -193,25 +191,10 @@ namespace Duplicati.Server set { DataConnection.ApplicationSettings.ServerPortChanged = value; } } - public static void IncrementLastDataUpdateID() - { - System.Threading.Interlocked.Increment(ref Program.LastDataUpdateID); - } - - public static void IncrementLastNotificationUpdateID() - { - System.Threading.Interlocked.Increment(ref Program.LastNotificationUpdateID); - } - static Program() { - FIXMEGlobal.IncrementLastDataUpdateID = Program.IncrementLastDataUpdateID; - FIXMEGlobal.PeekLastDataUpdateID = () => Program.LastDataUpdateID; - FIXMEGlobal.IncrementLastNotificationUpdateID = Program.IncrementLastNotificationUpdateID; - FIXMEGlobal.PeekLastNotificationUpdateID = () => Program.LastNotificationUpdateID; FIXMEGlobal.GetDatabaseConnection = Program.GetDatabaseConnection; FIXMEGlobal.StartOrStopUsageReporter = Program.StartOrStopUsageReporter; - FIXMEGlobal.UpdateThrottleSpeeds = Program.UpdateThrottleSpeeds; } /// @@ -220,6 +203,9 @@ namespace Duplicati.Server [STAThread] public static int Main(string[] args) { + // var methodInfo = typeof(TemporaryIoCAccessor).Assembly.EntryPoint; + // var program = Activator.CreateInstance(methodInfo!.DeclaringType!); + // methodInfo.Invoke(program, [Array.Empty()]); return Duplicati.Library.AutoUpdater.UpdaterManager.RunFromMostRecent(typeof(Program).GetMethod("RealMain"), args, Duplicati.Library.AutoUpdater.AutoUpdateStrategy.Never); } @@ -278,8 +264,10 @@ namespace Duplicati.Server try { - - DataConnection = GetDatabaseConnection(commandlineOptions); + DuplicatiWebserver = new DuplicatiWebserver(); + DuplicatiWebserver.InitWebServer(); + FIXMEGlobal.Provider = DuplicatiWebserver.Provider; + DataConnection = GetDatabaseConnection(DuplicatiWebserver, commandlineOptions); if (!DataConnection.ApplicationSettings.FixedInvalidBackupId) DataConnection.FixInvalidBackupId(); @@ -292,12 +280,12 @@ namespace Duplicati.Server ApplicationExitEvent = new System.Threading.ManualResetEvent(false); - Library.AutoUpdater.UpdaterManager.OnError += (Exception obj) => + Library.AutoUpdater.UpdaterManager.OnError += obj => { DataConnection.LogError(null, "Error in updater", obj); }; - UpdatePoller = new UpdatePollThread(); + UpdatePoller.Init(); SetPurgeTempFilesTimer(commandlineOptions); @@ -305,7 +293,7 @@ namespace Duplicati.Server SetWorkerThread(); - StartWebServer(commandlineOptions); + StartWebServer(DuplicatiWebserver, commandlineOptions); if (Library.Utility.Utility.ParseBoolOption(commandlineOptions, "ping-pong-keepalive")) { @@ -342,7 +330,7 @@ namespace Duplicati.Server ShutdownModernWebserver(); UpdatePoller?.Terminate(); Scheduler?.Terminate(true); - WorkThread?.Terminate(true); + FIXMEGlobal.WorkThread?.Terminate(true); ApplicationInstance?.Dispose(); PurgeTempFilesTimer?.Dispose(); @@ -360,37 +348,33 @@ namespace Duplicati.Server return 0; } - private static void StartWebServer(Dictionary commandlineOptions) + private static void StartWebServer(DuplicatiWebserver webserver, Dictionary commandlineOptions) { WebServer = new WebServer.Server(commandlineOptions); ServerPortChanged |= WebServer.Port != DataConnection.ApplicationSettings.LastWebserverPort; DataConnection.ApplicationSettings.LastWebserverPort = WebServer.Port; - - var server = new DuplicatiWebserver(); - ShutdownModernWebserver = server.Foo(); + + webserver.Start().GetAwaiter().GetResult(); } private static void SetWorkerThread() { - WorkThread = new Duplicati.Library.Utility.WorkerThread((x) => { Runner.Run(x, true); }, - LiveControl.State == LiveControls.LiveControlState.Paused); - Scheduler = new Scheduler(WorkThread); - - WorkThread.StartingWork += (worker, task) => { SignalNewEvent(null, null); }; - WorkThread.CompletedWork += (worker, task) => { SignalNewEvent(null, null); }; - WorkThread.WorkQueueChanged += (worker) => { SignalNewEvent(null, null); }; - Scheduler.NewSchedule += new EventHandler(SignalNewEvent); - WorkThread.OnError += (worker, task, exception) => + FIXMEGlobal.WorkerThreadsManager.Spawn(x => { Runner.Run(x, true); }); + FIXMEGlobal.WorkThread.StartingWork += (worker, task) => { SignalNewEvent(null, null); }; + FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => { SignalNewEvent(null, null); }; + FIXMEGlobal.WorkThread.WorkQueueChanged += (worker) => { SignalNewEvent(null, null); }; + FIXMEGlobal.Scheduler.NewSchedule += new EventHandler(SignalNewEvent); + FIXMEGlobal.WorkThread.OnError += (worker, task, exception) => { Program.DataConnection.LogError(task?.BackupID, "Error in worker", exception); }; - var lastScheduleId = LastDataUpdateID; + var lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; Program.StatusEventNotifyer.NewEvent += (sender, e) => { - if (lastScheduleId == LastDataUpdateID) return; - lastScheduleId = LastDataUpdateID; + if (lastScheduleId == FIXMEGlobal.NotificationUpdateService.LastDataUpdateId) return; + lastScheduleId = FIXMEGlobal.NotificationUpdateService.LastDataUpdateId; Program.Scheduler.Reschedule(); }; @@ -413,13 +397,13 @@ namespace Duplicati.Server } } - Program.WorkThread.CompletedWork += (worker, task) => { RegisterTaskResult(task.TaskID, null); }; - Program.WorkThread.OnError += (worker, task, exception) => { RegisterTaskResult(task.TaskID, exception); }; + FIXMEGlobal.WorkThread.CompletedWork += (worker, task) => { RegisterTaskResult(task.TaskID, null); }; + FIXMEGlobal.WorkThread.OnError += (worker, task, exception) => { RegisterTaskResult(task.TaskID, exception); }; } private static void SetLiveControls() { - LiveControl = new LiveControls(DataConnection.ApplicationSettings); + LiveControl.Init(DataConnection.ApplicationSettings); LiveControl.StateChanged += LiveControl_StateChanged; LiveControl.ThreadPriorityChanged += LiveControl_ThreadPriorityChanged; LiveControl.ThrottleSpeedChanged += LiveControl_ThrottleSpeedChanged; @@ -589,7 +573,7 @@ namespace Duplicati.Server throw new Exception("Server invoked with --help"); } - public static Database.Connection GetDatabaseConnection(Dictionary commandlineOptions) + public static Database.Connection GetDatabaseConnection(object webserver, Dictionary commandlineOptions) { var dbPassword = Environment.GetEnvironmentVariable(DB_KEY_ENV_NAME); @@ -726,22 +710,11 @@ namespace Duplicati.Server Library.UsageReporter.Reporter.SetReportLevel(reportLevel, disableUsageReporter); } - public static void UpdateThrottleSpeeds() - { - if (Program.WorkThread == null) - return; - - var cur = Program.WorkThread.CurrentTask; - if (cur != null) - cur.UpdateThrottleSpeed(); - } - private static void SignalNewEvent(object sender, EventArgs e) { StatusEventNotifyer.SignalNewEvent(); } - /// /// Handles a change in the LiveControl and updates the Runner /// @@ -765,29 +738,33 @@ namespace Duplicati.Server /// /// This event handler updates the trayicon menu with the current state of the runner. /// - static void LiveControl_StateChanged(object sender, EventArgs e) + /// + private static void LiveControl_StateChanged(object sender, EventArgs e) { + var worker = FIXMEGlobal.WorkThread; switch (LiveControl.State) { case LiveControls.LiveControlState.Paused: { - WorkThread.Pause(); - var t = WorkThread.CurrentTask; + worker.Pause(); + var t = worker.CurrentTask; t?.Pause(); break; } case LiveControls.LiveControlState.Running: { - WorkThread.Resume(); - var t = WorkThread.CurrentTask; + worker.Resume(); + var t = worker.CurrentTask; t?.Resume(); break; } + default: + throw new InvalidOperationException($"State of {nameof(LiveControl)} was not recognized!"); } StatusEventNotifyer.SignalNewEvent(); } - + /// /// Simple method for tracking if the server has crashed /// diff --git a/Duplicati/UnitTest/ImportExportTests.cs b/Duplicati/UnitTest/ImportExportTests.cs index d976ee7c8..0761b5fa2 100644 --- a/Duplicati/UnitTest/ImportExportTests.cs +++ b/Duplicati/UnitTest/ImportExportTests.cs @@ -105,7 +105,7 @@ namespace Duplicati.UnitTest } byte[] jsonByteArray; - using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions)) + using (Program.DataConnection = Program.GetDatabaseConnection(null, advancedOptions)) { jsonByteArray = Server.WebServer.RESTMethods.Backup.ExportToJSON(backup, null); } @@ -127,7 +127,7 @@ namespace Duplicati.UnitTest { Dictionary metadata = new Dictionary {{"SourceFilesCount", "1"}}; Dictionary advancedOptions = new Dictionary {{"server-datafolder", this.serverDatafolder}}; - using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions)) + using (Program.DataConnection = Program.GetDatabaseConnection(null, advancedOptions)) { // Unencrypted file, don't import metadata. string unencryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName()); diff --git a/Duplicati/WebserverCore/Abstractions/FileEntry.cs b/Duplicati/WebserverCore/Abstractions/FileEntry.cs new file mode 100644 index 000000000..320ff6842 --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/FileEntry.cs @@ -0,0 +1,10 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public class FileEntry +{ + public string Path { get; set; } = ""; + public string MD5 { get; set; } = ""; + public string SHA256 { get; set; } = ""; + public DateTime? LastWriteTime { get; set; } + public bool Ignore { get; set; } +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/ISettingsService.cs b/Duplicati/WebserverCore/Abstractions/ISettingsService.cs new file mode 100644 index 000000000..06b07e0a2 --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/ISettingsService.cs @@ -0,0 +1,6 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public interface ISettingsService +{ + ServerSettings GetSettings(); +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/IStatusService.cs b/Duplicati/WebserverCore/Abstractions/IStatusService.cs new file mode 100644 index 000000000..0bbe7c564 --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/IStatusService.cs @@ -0,0 +1,8 @@ +using Duplicati.WebserverCore.Dto; + +namespace Duplicati.WebserverCore.Abstractions; + +public interface IStatusService +{ + ServerStatusDto GetStatus(); +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/IUpdateService.cs b/Duplicati/WebserverCore/Abstractions/IUpdateService.cs new file mode 100644 index 000000000..d19d9486a --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/IUpdateService.cs @@ -0,0 +1,6 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public interface IUpdateService +{ + UpdateInfo? GetUpdateInfo(); +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/IV1Endpoint.cs b/Duplicati/WebserverCore/Abstractions/IV1Endpoint.cs new file mode 100644 index 000000000..057f5d4dc --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/IV1Endpoint.cs @@ -0,0 +1,6 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public interface IV1Endpoint +{ + public static abstract void Map(RouteGroupBuilder group); +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/ServerSettings.cs b/Duplicati/WebserverCore/Abstractions/ServerSettings.cs new file mode 100644 index 000000000..647788d98 --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/ServerSettings.cs @@ -0,0 +1,29 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public class ServerSettings +{ + public string StartupDelay { get; set; } = ""; + public string DownloadSpeedLimit { get; set; }= ""; + public string UploadSpeedLimit { get; set; }= ""; + public string ThreadPriority { get; set; }= ""; + public string LastWebserverPort { get; set; }= ""; + public string IsFirstRun { get; set; }= ""; + public string ServerPortChanged { get; set; }= ""; + public string ServerPassphrase { get; set; }= ""; + public string ServerPassphraseSalt { get; set; }= ""; + public string ServerPassphraseTrayIcon { get; set; }= ""; + public string ServerPassphraseTrayIconHash { get; set; }= ""; + public string UpdateCheckLast { get; set; }= ""; + public string UpdateCheckInterval { get; set; }= ""; + public string UpdateCheckNewVersion { get; set; }= ""; + public bool UnackedError { get; set; } + public bool UnackedWarning { get; set; } + public string ServerListenInterface { get; set; }= ""; + public string ServerSslCertificate { get; set; }= ""; + public string HasFixedInvalidBackupId { get; set; }= ""; + public string UpdateChannel { get; set; }= ""; + public string UsageReporterLevel { get; set; }= ""; + public string HasAskedForPasswordProtection { get; set; }= ""; + public string DisableTrayIconLogin { get; set; }= ""; + public string ServerAllowedHostnames { get; set; }= ""; +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Abstractions/UpdateInfo.cs b/Duplicati/WebserverCore/Abstractions/UpdateInfo.cs new file mode 100644 index 000000000..b3554bf95 --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/UpdateInfo.cs @@ -0,0 +1,17 @@ +namespace Duplicati.WebserverCore.Abstractions; + +public class UpdateInfo +{ + public string Displayname { get; set; } = ""; + public string Version { get; set; } = ""; + public DateTime? ReleaseTime { get; set; } + public string ReleaseType { get; set; } = ""; + public string UpdateSeverity { get; set; } = ""; + public string ChangeInfo { get; set; } = ""; + public long CompressedSize { get; set; } + public long UncompressedSize { get; set; } + public string SHA256 { get; set; } = ""; + public string MD5 { get; set; } = ""; + public string[] RemoteURLS { get; set; } = []; + public FileEntry[] Files { get; set; } = []; +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/ApplicationPartsLogger.cs b/Duplicati/WebserverCore/ApplicationPartsLogger.cs new file mode 100644 index 000000000..7cd5f64c8 --- /dev/null +++ b/Duplicati/WebserverCore/ApplicationPartsLogger.cs @@ -0,0 +1,32 @@ +using Microsoft.AspNetCore.Mvc.ApplicationParts; +using Microsoft.AspNetCore.Mvc.Controllers; + +namespace Duplicati.WebserverCore; + +//Useful for debugging ASP.net magically loading controllers +public class ApplicationPartsLogger(ILogger logger, ApplicationPartManager partManager) + : IHostedService +{ + public Task StartAsync(CancellationToken cancellationToken) + { + // Get the names of all the application parts. This is the short assembly name for AssemblyParts + var applicationParts = partManager.ApplicationParts.Select(x => x.Name); + + // Create a controller feature, and populate it from the application parts + var controllerFeature = new ControllerFeature(); + partManager.PopulateFeature(controllerFeature); + + // Get the names of all of the controllers + var controllers = controllerFeature.Controllers.Select(x => x.Name); + + // Log the application parts and controllers + logger.LogInformation( + "Found the following application parts: '{ApplicationParts}' with the following controllers: '{Controllers}'", + string.Join(", ", applicationParts), string.Join(", ", controllers)); + + return Task.CompletedTask; + } + + // Required by the interface + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} \ No newline at end of file diff --git a/Duplicati/WebserverCore/Database/Configuration/OptionConfiguration.cs b/Duplicati/WebserverCore/Database/Configuration/OptionConfiguration.cs new file mode 100644 index 000000000..23a5deb77 --- /dev/null +++ b/Duplicati/WebserverCore/Database/Configuration/OptionConfiguration.cs @@ -0,0 +1,14 @@ +using Duplicati.WebserverCore.Database.Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace Duplicati.WebserverCore.Database.Configuration; + +public class OptionConfiguration : IEntityTypeConfiguration