From ef631a1199366d47aef5b26f7c53aa91eb7481e9 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 20 May 2025 14:26:10 +0200 Subject: [PATCH] Added feature to suppress warnings This PR adds the option to suppress warnings by their log id. Warning IDs that are supplied to `--suppress-warnings` will be converted to information messages before being logged. If a specific warning is disabled, such as `CompressionReadErrorFallback`, this will then no longer count as a warning for the job, and the log file will see the warning as an information message. This PR also adds two simpler filter options that makes it possible to filter log messages by supplying the log IDs. This can already be achieved with log filters, but the ID filter is a bit simpler to apply, as you only need to know the ID. Finally, this PR also adds common logging for errors in the categories: - Permission denied, id = `PermissionDenied` - File locked, id = `FileLocked` - Path not found, id = `PathNotFound` - Path too long, id = `PathTooLong` These new log ids makes it simpler to ignore warnings about locked or inaccesible files. --- Duplicati/Agent/Program.cs | 2 +- Duplicati/Library/Main/Controller.cs | 2 +- .../Library/Main/ControllerMultiLogTarget.cs | 78 +++-- .../Operation/Backup/FileBlockProcessor.cs | 4 +- .../Backup/FileEnumerationProcess.cs | 15 +- .../Operation/Backup/FilePreFilterProcess.cs | 2 +- .../Operation/Backup/LogExceptionHelper.cs | 57 ++++ .../Operation/Backup/MetadataGenerator.cs | 6 +- .../Operation/Backup/MetadataPreProcess.cs | 7 +- .../Operation/Backup/StreamBlockSplitter.cs | 42 ++- Duplicati/Library/Main/Options.cs | 35 ++- Duplicati/Library/Main/Strings.cs | 6 + Duplicati/Library/RestAPI/LogWriteHandler.cs | 2 +- .../Library/Utility/EnumerableExtensions.cs | 2 +- .../Library/Utility/ExceptionExtensions.cs | 279 ++++++++++++++++++ Duplicati/Library/Utility/Utility.cs | 39 --- Tools/RemoteSynchronization/Program.cs | 2 +- 17 files changed, 451 insertions(+), 129 deletions(-) create mode 100644 Duplicati/Library/Main/Operation/Backup/LogExceptionHelper.cs create mode 100644 Duplicati/Library/Utility/ExceptionExtensions.cs diff --git a/Duplicati/Agent/Program.cs b/Duplicati/Agent/Program.cs index d5d0351c5..157621d3e 100644 --- a/Duplicati/Agent/Program.cs +++ b/Duplicati/Agent/Program.cs @@ -275,7 +275,7 @@ public static class Program using var cts = new CancellationTokenSource(); - var target = new ControllerMultiLogTarget(new ConsoleLogDestination(), LogMessageType.Information, null); + var target = new ControllerMultiLogTarget(new ConsoleLogDestination(), LogMessageType.Information, null, null); using (Log.StartScope(target)) { if (OperatingSystem.IsWindows() && !string.IsNullOrWhiteSpace(agentConfig.WindowsEventLog)) diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 3e242612d..873f51858 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -478,7 +478,7 @@ namespace Duplicati.Library.Main { OnOperationStarted?.Invoke(result); var resultSetter = result as ISetCommonOptions; - using (var logTarget = new ControllerMultiLogTarget(result, Logging.LogMessageType.Information, null)) + using (var logTarget = new ControllerMultiLogTarget(result, Logging.LogMessageType.Information, null, m_options.SuppressWarningsFilter)) using (Logging.Log.StartScope(logTarget, null)) { logTarget.AddTarget(m_messageSink, m_options.ConsoleLoglevel, m_options.ConsoleLogFilter); diff --git a/Duplicati/Library/Main/ControllerMultiLogTarget.cs b/Duplicati/Library/Main/ControllerMultiLogTarget.cs index fe8ddc464..67484b3f0 100644 --- a/Duplicati/Library/Main/ControllerMultiLogTarget.cs +++ b/Duplicati/Library/Main/ControllerMultiLogTarget.cs @@ -1,24 +1,26 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. +#nullable enable + using System; using System.Collections.Generic; using System.Linq; @@ -29,12 +31,25 @@ namespace Duplicati.Library.Main /// /// The log target handler for the controller, which sends to multiple log targets /// - public class ControllerMultiLogTarget : Logging.ILogDestination, IDisposable + public class ControllerMultiLogTarget : ILogDestination, IDisposable { + /// + /// The target entry for the log target + /// + /// The log target. + /// The minimum log level to consider + /// The log filter. + private sealed record TargetEntry(ILogDestination Target, LogMessageType Level, Library.Utility.IFilter Filter); + /// /// The list of log targets to handle /// - private readonly List> m_targets = new List>(); + private readonly List m_targets = new List(); + + /// + /// The filter used to suppress warnings + /// + private readonly HashSet? m_supressWarningFilter; /// /// Initializes a new instance of the class. @@ -42,8 +57,9 @@ namespace Duplicati.Library.Main /// The log target. /// The minimum log level to consider /// The log filter. - public ControllerMultiLogTarget(ILogDestination target, Logging.LogMessageType loglevel, Library.Utility.IFilter filter) + public ControllerMultiLogTarget(ILogDestination target, Logging.LogMessageType loglevel, Library.Utility.IFilter? filter, HashSet? supressWarningFilter) { + m_supressWarningFilter = supressWarningFilter; AddTarget(target, loglevel, filter); } @@ -53,12 +69,12 @@ namespace Duplicati.Library.Main /// The log target. /// The minimum log level to consider /// The log filter. - public void AddTarget(ILogDestination target, LogMessageType loglevel, Library.Utility.IFilter filter) + public void AddTarget(ILogDestination target, LogMessageType loglevel, Library.Utility.IFilter? filter) { if (target == null) return; - m_targets.Add(new Tuple(target, loglevel, filter ?? new Library.Utility.FilterExpression())); + m_targets.Add(new TargetEntry(target, loglevel, filter ?? new Library.Utility.FilterExpression())); } /// @@ -73,7 +89,7 @@ namespace Duplicati.Library.Main public void Dispose() { foreach (var m in m_targets) - (m.Item1 as IDisposable)?.Dispose(); + (m.Target as IDisposable)?.Dispose(); m_targets.Clear(); } @@ -81,7 +97,7 @@ namespace Duplicati.Library.Main /// Gets the minimum log level of all the targets /// public LogMessageType MinimumLevel - => m_targets.Select(x => x.Item2).DefaultIfEmpty(LogMessageType.Error).Min(); + => m_targets.Select(x => x.Level).DefaultIfEmpty(LogMessageType.Error).Min(); /// /// Writes the message to all the destinations. @@ -89,9 +105,13 @@ namespace Duplicati.Library.Main /// Entry. public void WriteMessage(LogEntry entry) { + // If the message is a warning, check if we should suppress it + if (m_supressWarningFilter != null && entry.Level == LogMessageType.Warning && m_supressWarningFilter.Contains(entry.Id)) + entry = new LogEntry(entry.Message, entry.Arguments, LogMessageType.Information, entry.Tag, entry.Id, entry.Exception); + foreach (var e in m_targets) { - var found = e.Item3.Matches(entry.FilterTag, out var result, out var match); + var found = e.Filter.Matches(entry.FilterTag, out var result, out var _); // If there is a filter match, use that if (found) @@ -102,12 +122,12 @@ namespace Duplicati.Library.Main else { // Otherwise, filter by log-level - if (entry.Level < e.Item2) + if (entry.Level < e.Level) continue; } // If we get here, write the message - e.Item1.WriteMessage(entry); + e.Target.WriteMessage(entry); } } } diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index 02a983aac..b404232f0 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -142,8 +142,8 @@ namespace Duplicati.Library.Main.Operation.Backup { if (ex.IsRetiredException()) return; - else - Logging.Log.WriteWarningMessage(FILELOGTAG, "PathProcessingFailed", ex, "Failed to process path: {0}", e.Entry.Path); + + LogExceptionHelper.LogCommonWarning(ex, FILELOGTAG, "FileProcessingFailed", e.Entry.Path); } } }); diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index 848610186..1f655d423 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -96,7 +96,6 @@ namespace Duplicati.Library.Main.Operation.Backup if (ignorenames != null && ignorenames.Length == 0) ignorenames = null; - // Shared filter function with bound variables ValueTask FilterEntry(ISourceProviderEntry entry) => SourceFileEntryFilter(entry, blacklistPaths, hardlinkPolicy, symlinkPolicy, hardlinkmap, fileAttributeFilter, enumeratefilter, ignorenames, mixinqueue, token); @@ -303,7 +302,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorEnumerate", ex, "Failed to enumerate path: {0}", e.Path); + LogExceptionHelper.LogCommonWarning(ex, FILTER_LOGTAG, "PathProcessingErrorEnumerate", e.Path, "Failed to enumerate path: {0}"); } } } @@ -369,7 +368,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorBlockDevice", ex, "Failed to process path: {0}", entry.Path); + LogExceptionHelper.LogCommonWarning(ex, FILTER_LOGTAG, "PathProcessingErrorBlockDevice", entry.Path); return false; } @@ -384,7 +383,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorCharacterDevice", ex, "Failed to process path: {0}", entry.Path); + LogExceptionHelper.LogCommonWarning(ex, FILTER_LOGTAG, "PathProcessingErrorCharacterDevice", entry.Path); return false; } @@ -444,7 +443,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorHardLink", ex, "Failed to process path: {0}", entry.Path); + LogExceptionHelper.LogCommonWarning(ex, FILTER_LOGTAG, "PathProcessingErrorHardLink", entry.Path); return false; } } @@ -465,7 +464,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorIgnoreFile", ex, "Failed to process path: {0}", entry.Path); + LogExceptionHelper.LogCommonWarning(ex, FILTER_LOGTAG, "PathProcessingErrorIgnoreFile", entry.Path); } } @@ -480,7 +479,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILTER_LOGTAG, "PathProcessingErrorAttributes", ex, "Failed to process path, using default attributes: {0}", entry.Path); + LogExceptionHelper.LogCommonWarning(ex, FILTER_LOGTAG, "PathProcessingErrorAttributes", entry.Path, "Failed to process path, using default attributes: {0}"); } // If we exclude files based on attributes, filter that @@ -511,7 +510,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteExplicitMessage(FILTER_LOGTAG, "SymlinkTargetReadError", ex, "Failed to read symlink target for path: {0}", entry.Path); + LogExceptionHelper.LogCommonWarning(ex, FILTER_LOGTAG, "SymlinkTargetReadError", entry.Path, "Failed to read symlink target for path: {0}"); } if (symlinkTarget != null) diff --git a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs index 1574bff60..744606614 100644 --- a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs @@ -77,7 +77,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteExplicitMessage(FILELOGTAG, "FailedToReadSize", ex, "Failed to read size of file: {0}", e.Entry.Path); + LogExceptionHelper.LogCommonWarning(ex, FILELOGTAG, "FailedToReadSize", e.Entry.Path, "Failed to read size on \"{0}\""); } await stats.AddExaminedFile(filestatsize); diff --git a/Duplicati/Library/Main/Operation/Backup/LogExceptionHelper.cs b/Duplicati/Library/Main/Operation/Backup/LogExceptionHelper.cs new file mode 100644 index 000000000..c7125042c --- /dev/null +++ b/Duplicati/Library/Main/Operation/Backup/LogExceptionHelper.cs @@ -0,0 +1,57 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using Duplicati.Library.Logging; +using Duplicati.Library.Utility; + +namespace Duplicati.Library.Main.Operation.Backup; + +/// +/// Helper class to log path exceptions in a common way +/// +public static class LogExceptionHelper +{ + /// + /// Logs a path warning message, using exception detection for the appropriate message + /// + /// The exception to log + /// The log tag to use for the message + /// The id to use for the message + /// The path to use for the message + /// The message to log + public static void LogCommonWarning(Exception? ex, string logtag, string id, string path, string message = "Failed to process path: {0}") + { + if (ex.IsPermissionDeniedException()) + Log.WriteWarningMessage(logtag, "PermissionDenied", ex, "Excluding path due to permission denied: {0}", path); + else if (ex.IsFileLockedException()) + Log.WriteWarningMessage(logtag, "FileLocked", ex, "Excluding path due to file locked: {0}", path); + else if (ex.IsPathNotFoundException()) + Log.WriteWarningMessage(logtag, "PathNotFound", ex, "Excluding path due to path not found: {0}", path); + else if (ex.IsPathTooLongException()) + Log.WriteWarningMessage(logtag, "PathTooLong", ex, "Excluding path due to path too long: {0}", path); + else + Log.WriteWarningMessage(logtag, id, ex, message, path); + + } +} \ No newline at end of file diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs index 6be52d47c..daaad0441 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataGenerator.cs @@ -55,7 +55,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path); + LogExceptionHelper.LogCommonWarning(ex, METALOGTAG, "TimestampReadFailed", entry.Path, "Failed to read timestamp on \"{0}\""); } } @@ -67,7 +67,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(METALOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path); + LogExceptionHelper.LogCommonWarning(ex, METALOGTAG, "TimestampReadFailed", entry.Path, "Failed to read timestamp on \"{0}\""); } } } @@ -80,7 +80,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(METALOGTAG, "MetadataProcessFailed", ex, "Failed to process metadata for \"{0}\", storing empty metadata", entry.Path); + LogExceptionHelper.LogCommonWarning(ex, METALOGTAG, "MetadataProcessFailed", entry.Path, "Failed to process metadata on \"{0}\", storing empty metadata"); return new Dictionary(); } } diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index 2ae10faa1..defdee0bb 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -98,7 +98,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILELOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", entry.Path); + LogExceptionHelper.LogCommonWarning(ex, FILELOGTAG, "TimestampReadFailed", entry.Path, "Failed to read timestamp on \"{0}\""); } try @@ -107,7 +107,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", entry.Path, ex.Message); + LogExceptionHelper.LogCommonWarning(ex, FILELOGTAG, "AttributeReadFailed", entry.Path, "Failed to read attributes on \"{0}\""); } // If we only have metadata, stop here @@ -166,8 +166,7 @@ namespace Duplicati.Library.Main.Operation.Backup if (ex.IsRetiredException()) continue; - Logging.Log.WriteWarningMessage(FILELOGTAG, "ProcessingMetadataFailed", ex, - "Failed to process entry, path: {0}", entry.Path); + LogExceptionHelper.LogCommonWarning(ex, FILELOGTAG, "ProcessingMetadataFailed", entry.Path); } } } diff --git a/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs b/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs index 449ee9cfb..6d63dfd91 100644 --- a/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs +++ b/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs @@ -1,22 +1,22 @@ -// Copyright (C) 2025, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -25,10 +25,8 @@ using Duplicati.Library.Main.Operation.Common; using System.Threading.Tasks; using System.Collections.Generic; using Duplicati.Library.Utility; -using System.Linq; using Duplicati.Library.Interface; using System.IO; -using System.Security.Cryptography; namespace Duplicati.Library.Main.Operation.Backup { @@ -87,7 +85,7 @@ namespace Duplicati.Library.Main.Operation.Backup } catch (Exception ex) { - Logging.Log.WriteWarningMessage(FILELOGTAG, "FileLengthFailure", ex, "Failed to read file length for file {0}", e.Path); + LogExceptionHelper.LogCommonWarning(ex, FILELOGTAG, "FileLengthFailure", e.Path, "Failed to read file length for \"{0}\""); } if (e.IsMetadata && fslen > maxmetadatasize) diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 18312efaa..9cd02b6c9 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -375,11 +375,14 @@ namespace Duplicati.Library.Main new CommandLineArgument("log-file", CommandLineArgument.ArgumentType.Path, Strings.Options.LogfileShort, Strings.Options.LogfileLong), new CommandLineArgument("log-file-log-level", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.LogfileloglevelShort, Strings.Options.LogfileloglevelLong, "Warning", null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType))), new CommandLineArgument("log-file-log-filter", CommandLineArgument.ArgumentType.String, Strings.Options.LogfilelogfiltersShort, Strings.Options.LogfilelogfiltersLong(System.IO.Path.PathSeparator.ToString()), null), + new CommandLineArgument("log-file-log-ignore", CommandLineArgument.ArgumentType.String, Strings.Options.LogfilelogignoreShort, Strings.Options.LogfilelogignoreLong(System.IO.Path.PathSeparator.ToString()), null), new CommandLineArgument("console-log-level", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.ConsoleloglevelShort, Strings.Options.ConsoleloglevelLong, "Warning", null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType))), new CommandLineArgument("console-log-filter", CommandLineArgument.ArgumentType.String, Strings.Options.ConsolelogfiltersShort, Strings.Options.ConsolelogfiltersLong(System.IO.Path.PathSeparator.ToString()), null), + new CommandLineArgument("console-log-ignore", CommandLineArgument.ArgumentType.String, Strings.Options.ConsolelogignoreShort, Strings.Options.ConsolelogignoreLong(System.IO.Path.PathSeparator.ToString()), null), new CommandLineArgument("log-level", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.LoglevelShort, Strings.Options.LoglevelLong, "Warning", null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType)), Strings.Options.LogLevelDeprecated("log-file-log-level", "console-log-level")), + new CommandLineArgument("suppress-warnings", CommandLineArgument.ArgumentType.String, Strings.Options.SuppresswarningsShort, Strings.Options.SuppresswarningsLong), new CommandLineArgument("profile-all-database-queries", CommandLineArgument.ArgumentType.Boolean, Strings.Options.ProfilealldatabasequeriesShort, Strings.Options.ProfilealldatabasequeriesLong, "false"), @@ -1213,31 +1216,31 @@ namespace Duplicati.Library.Main } } + /// + /// Gets the filter used to suppress warning messages. + /// + public HashSet? SuppressWarningsFilter + => m_options.GetValueOrDefault("suppress-warnings")?.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)?.ToHashSet(StringComparer.OrdinalIgnoreCase); + /// /// Gets the filter used for log-file messages. /// /// The log file filter. - public IFilter LogFileLogFilter - { - get - { - m_options.TryGetValue("log-file-log-filter", out var value); - return Library.Utility.FilterExpression.ParseLogFilter(value); - } - } + public IFilter? LogFileLogFilter + => FilterExpression.Combine( + new FilterExpression(m_options.GetValueOrDefault("log-file-log-ignore")?.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)?.Select(x => $"*-{x}"), false), + FilterExpression.ParseLogFilter(m_options.GetValueOrDefault("log-file-log-filter")) + ); /// /// Gets the filter used for console messages. /// /// The log file filter. - public IFilter ConsoleLogFilter - { - get - { - m_options.TryGetValue("console-log-filter", out var value); - return Library.Utility.FilterExpression.ParseLogFilter(value); - } - } + public IFilter? ConsoleLogFilter + => FilterExpression.Combine( + new FilterExpression(m_options.GetValueOrDefault("console-log-ignore")?.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)?.Select(x => $"*-{x}"), false), + FilterExpression.ParseLogFilter(m_options.GetValueOrDefault("console-log-filter")) + ); /// /// Gets the console log detail level diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 6db36589e..7895b20d9 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -123,6 +123,8 @@ namespace Duplicati.Library.Main.Strings public static string LoglevelLong { get { return LC.L(@"Specify the amount of log information to write into the file specified by the option --{0}.", "log-file"); } } public static string LoglevelShort { get { return LC.L(@"Log information level"); } } public static string LogLevelDeprecated(string option1, string option2) { return LC.L("Use the options --{0} and --{1} instead.", option1, option2); } + public static string SuppresswarningsLong { get { return LC.L(@"Suppress warnings and log them as information instead. Use this if you need to silence specific warnings. This option accepts a comma separated list of warning IDs."); } } + public static string SuppresswarningsShort { get { return LC.L(@"Suppress specific warnings"); } } public static string DisableautocreatefolderLong { get { return LC.L(@"If Duplicati detects that the target folder is missing, it will create it automatically. Activate this option to prevent automatic folder creation."); } } public static string DisableautocreatefolderShort { get { return LC.L(@"Disable automatic folder creation"); } } public static string VssexcludewritersLong { get { return LC.L(@"Use this option to exclude faulty writers from a snapshot. This is equivalent to the -wx flag of the vshadow.exe tool, except that it only accepts writer class GUIDs, and not component names or instance GUIDs. Multiple GUIDs must be separated with a semicolon, and most forms of GUIDs are allowed, including with and without curly braces."); } } @@ -289,10 +291,14 @@ namespace Duplicati.Library.Main.Strings public static string LogfileloglevelShort { get { return LC.L(@"Log file information level"); } } public static string LogfilelogfiltersLong(string delimiter) { return LC.L(@"This option accepts filters that removes or includes messages regardless of their log level. Multiple filters are supported by separating with {0}. Filters are matched against the log tag and assumed to be including, unless they start with '-'. Regular expressions are supported within hard braces. Example: ""+Path*{0}+*Mail*{0}-[.*DNS]"" ", delimiter); } public static string LogfilelogfiltersShort { get { return LC.L(@"Apply filters to the file log data"); } } + public static string LogfilelogignoreLong(string option) { return LC.L(@"This is a simplified version of the --{0} option. It will ignore all log messages that have an ID in the list. The list is a comma separated list of log message ids.", option); } + public static string LogfilelogignoreShort { get { return LC.L(@"Ignore log messages with the specified IDs"); } } public static string ConsoleloglevelLong { get { return LC.L(@"Specify the amount of log information to output to the console."); } } public static string ConsoleloglevelShort { get { return LC.L(@"Console information level"); } } public static string ConsolelogfiltersLong(string delimiter) { return LogfilelogfiltersLong(delimiter); } public static string ConsolelogfiltersShort { get { return LC.L(@"Apply filters to the console log data"); } } + public static string ConsolelogignoreLong(string option) { return LogfilelogignoreLong(option); } + public static string ConsolelogignoreShort { get { return LC.L(@"Ignore log messages with the specified IDs"); } } public static string UsebackgroundiopriorityLong { get { return LC.L("This option instructs the operating system to set the current process to use the lowest IO priority level, which can make operations run slower but will interfere less with other operations running at the same time."); } } public static string UsebackgroundiopriorityShort { get { return LC.L("Set the process to use low IO priority"); } } diff --git a/Duplicati/Library/RestAPI/LogWriteHandler.cs b/Duplicati/Library/RestAPI/LogWriteHandler.cs index 6b0613d67..26e3614d9 100644 --- a/Duplicati/Library/RestAPI/LogWriteHandler.cs +++ b/Duplicati/Library/RestAPI/LogWriteHandler.cs @@ -116,7 +116,7 @@ namespace Duplicati.Server private RingBuffer m_buffer; - private readonly ControllerMultiLogTarget m_target = new ControllerMultiLogTarget(null, LogMessageType.Warning, null); + private readonly ControllerMultiLogTarget m_target = new ControllerMultiLogTarget(null, LogMessageType.Warning, null, null); private LogMessageType m_logLevel; public LogWriteHandler() diff --git a/Duplicati/Library/Utility/EnumerableExtensions.cs b/Duplicati/Library/Utility/EnumerableExtensions.cs index 9aca73d90..9a8e6ed06 100644 --- a/Duplicati/Library/Utility/EnumerableExtensions.cs +++ b/Duplicati/Library/Utility/EnumerableExtensions.cs @@ -47,7 +47,7 @@ public static class EnumerableExtensions /// A sequence that contains only the non-null elements from the input sequence. public static IEnumerable WhereNotNull(this IEnumerable source) where T : struct => source.Where(x => x != null) - .Select(x => x.Value); + .Select(x => x!.Value); /// /// Filters a sequence of values to exclude null or whitespace elements. diff --git a/Duplicati/Library/Utility/ExceptionExtensions.cs b/Duplicati/Library/Utility/ExceptionExtensions.cs new file mode 100644 index 000000000..40c807a81 --- /dev/null +++ b/Duplicati/Library/Utility/ExceptionExtensions.cs @@ -0,0 +1,279 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Duplicati.Library.Utility; + +/// +/// Extension methods for exceptions +/// +public static class ExceptionExtensions +{ + /// + /// Flattens an exception and its inner exceptions + /// + /// The exception to flatten + /// An enumerable of exceptions + public static IEnumerable FlattenException(Exception? ex) + { + if (ex == null) + yield break; + + yield return ex; + + if (ex is AggregateException aex) + foreach (var iex in aex.Flatten().InnerExceptions) + foreach (var iex2 in FlattenException(iex)) + yield return iex2; + + foreach (var iex in FlattenException(ex.InnerException)) + yield return iex; + } + + /// + /// Checks if an exception is a stop, cancel or timeout exception + /// + /// The operation to check + /// true if the exception is a stop or cancel exception, false otherwise + public static bool IsAbortOrCancelException(this Exception ex) + => ex is OperationCanceledException || ex is ThreadAbortException || ex is TaskCanceledException || ex is TimeoutException; + + /// + /// Checks if an exception is a stop exception + /// + /// The operation to check + /// true if the exception is a stop exception, false otherwise + public static bool IsAbortException(this Exception ex) + => ex is OperationCanceledException || ex is ThreadAbortException; + + /// + /// List of Windows error codes that indicate a permission denied error + /// + private static readonly int[] WindowsPermissionDeniedCodes = { + 5, // ERROR_ACCESS_DENIED + 19, // ERROR_WRITE_PROTECT + 65, // ERROR_NETWORK_ACCESS_DENIED + 82, // ERROR_CANNOT_MAKE + 1314, // ERROR_PRIVILEGE_NOT_HELD + }; + + /// + /// List of Windows error codes that indicate a file lock error + /// + private static readonly int[] WindowsFileLockCodes = { + 32, // ERROR_SHARING_VIOLATION + 33 // ERROR_LOCK_VIOLATION + }; + + /// + /// List of Windows error codes that indicate a path not found error + /// + private static readonly int[] WindowsPathNotFoundCodes = { + 2, // ERROR_FILE_NOT_FOUND + 3, // ERROR_PATH_NOT_FOUND + 21, // ERROR_NOT_READY (e.g., missing drive) + 267 // ERROR_DIRECTORY + }; + + /// + /// List of Windows error codes that indicate a path too long error + /// + private static readonly int[] WindowsPathTooLongCodes = { + 206, // ERROR_FILENAME_EXCED_RANGE + 3 // ERROR_PATH_NOT_FOUND (used when long path causes resolution failure) + }; + + /// + /// List of Posix error codes that indicate a permission denied error + /// + private static readonly int[] PosixPermissionErrnos = { + 1, // EPERM + 13, // EACCES + 30, // EROFS - Read-only file system + }; + + /// + /// List of Posix error codes that indicate a file lock error + /// + private static readonly int[] PosixFileLockErrnos = { + // Linux file lock errors are not always distinguishable + 11 // EAGAIN (on NFS), or temporary unavailable (used for lock contention in some cases) + }; + + /// + /// List of Posix error codes that indicate a path not found error + /// + private static readonly int[] PosixPathNotFoundErrnos = { + 2, // ENOENT - No such file or directory + 20 // ENOTDIR - A path component is not a directory + }; + + /// + /// List of Posix error codes that indicate a path too long error + /// + private static readonly int[] PosixPathTooLongErrnos = { + 36, // ENAMETOOLONG - File name too long + 63 // ENAMETOOLONG (on some macOS versions) + }; + + + /// + /// Checks if an exception is a permission denied error + /// + /// The operation to check + /// true if the exception is a permission denied error, false otherwise + public static bool IsPermissionDeniedException(this Exception? ex) + { + if (ex is UnauthorizedAccessException) + return true; + + if (ex is Win32Exception win32Ex && OperatingSystem.IsWindows()) + return WindowsPermissionDeniedCodes.Contains(win32Ex.NativeErrorCode); + + if (ex is IOException ioEx) + { + if (ioEx.HResult == unchecked((int)0x80070005)) // E_ACCESSDENIED + return true; + + if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS()) + { + if (ioEx.InnerException is Win32Exception innerWin32 && + PosixPermissionErrnos.Contains(innerWin32.NativeErrorCode)) + return true; + + var errno = ioEx.HResult & 0xFFFF; + return PosixPermissionErrnos.Contains(errno); + } + } + + return false; + } + + /// + /// Checks if an exception is a file lock error + /// + /// The operation to check + /// true if the exception is a file lock error, false otherwise + public static bool IsFileLockedException(this Exception? ex) + { + if (ex is IOException ioEx) + { + if (OperatingSystem.IsWindows()) + { + // Check inner Win32Exception + if (ioEx.InnerException is Win32Exception innerWin32 && + WindowsFileLockCodes.Contains(innerWin32.NativeErrorCode)) + return true; + + // Fallback to HRESULT + var winCode = ioEx.HResult & 0xFFFF; + return WindowsFileLockCodes.Contains(winCode); + } + + if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS()) + { + if (ioEx.InnerException is Win32Exception innerWin32 && + PosixFileLockErrnos.Contains(innerWin32.NativeErrorCode)) + return true; + + var errno = ioEx.HResult & 0xFFFF; + return PosixFileLockErrnos.Contains(errno); + } + } + + return false; + } + + /// + /// Checks if an exception is a path not found error + /// + /// The operation to check + /// true if the exception is a path not found error, false otherwise + public static bool IsPathNotFoundException(this Exception? ex) + { + if (ex is FileNotFoundException || ex is DirectoryNotFoundException) + return true; + + if (ex is Win32Exception win32Ex && OperatingSystem.IsWindows()) + return WindowsPathNotFoundCodes.Contains(win32Ex.NativeErrorCode); + + if (ex is IOException ioEx) + { + var code = ioEx.HResult & 0xFFFF; + + if (OperatingSystem.IsWindows()) + return WindowsPathNotFoundCodes.Contains(code); + + if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS()) + { + if (ioEx.InnerException is Win32Exception innerWin32 && + PosixPathNotFoundErrnos.Contains(innerWin32.NativeErrorCode)) + return true; + + return PosixPathNotFoundErrnos.Contains(code); + } + } + + return false; + } + + /// + /// Checks if an exception is a path too long error + /// + /// The operation to check + /// true if the exception is a path too long error, false otherwise + public static bool IsPathTooLongException(this Exception? ex) + { + if (ex is PathTooLongException) + return true; + + if (ex is Win32Exception win32Ex && OperatingSystem.IsWindows()) + return WindowsPathTooLongCodes.Contains(win32Ex.NativeErrorCode); + + if (ex is IOException ioEx) + { + var code = ioEx.HResult & 0xFFFF; + + if (OperatingSystem.IsWindows()) + return WindowsPathTooLongCodes.Contains(code); + + if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS()) + { + if (ioEx.InnerException is Win32Exception innerWin32 && + PosixPathTooLongErrnos.Contains(innerWin32.NativeErrorCode)) + return true; + + return PosixPathTooLongErrnos.Contains(code); + } + } + + return false; + } +} \ No newline at end of file diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index d29f548ab..1df49958a 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1569,26 +1569,7 @@ namespace Duplicati.Library.Utility public static string IpVersionCompatibleLoopback => HasIPv4Loopback ? IPAddress.Loopback.ToString() : $"[{IPAddress.IPv6Loopback.ToString()}]"; - /// - /// Flattens an exception and its inner exceptions - /// - /// The exception to flatten - /// An enumerable of exceptions - public static IEnumerable FlattenException(Exception? ex) - { - if (ex == null) - yield break; - yield return ex; - - if (ex is AggregateException aex) - foreach (var iex in aex.Flatten().InnerExceptions) - foreach (var iex2 in FlattenException(iex)) - yield return iex2; - - foreach (var iex in FlattenException(ex.InnerException)) - yield return iex; - } /// /// Guesses the URL scheme and returns it @@ -1639,26 +1620,6 @@ namespace Duplicati.Library.Utility return sanitizedUrl; } - /// - /// Checks if an exception is a stop, cancel or timeout exception - /// - /// The operation to check - /// true if the exception is a stop or cancel exception, false otherwise - public static bool IsAbortOrCancelException(this Exception ex) - { - return ex is OperationCanceledException || ex is ThreadAbortException || ex is TaskCanceledException || ex is TimeoutException; - } - - /// - /// Checks if an exception is a stop exception - /// - /// The operation to check - /// true if the exception is a stop exception, false otherwise - public static bool IsAbortException(this Exception ex) - { - return ex is OperationCanceledException || ex is ThreadAbortException; - } - /// /// Formats the string using the invariant culture /// diff --git a/Tools/RemoteSynchronization/Program.cs b/Tools/RemoteSynchronization/Program.cs index 91c8ae793..aa2406c2e 100644 --- a/Tools/RemoteSynchronization/Program.cs +++ b/Tools/RemoteSynchronization/Program.cs @@ -151,7 +151,7 @@ destination will be verified before being overwritten (if they seemingly match). log_level_enum = log_level_parsed ? log_level_enum : Duplicati.Library.Logging.LogMessageType.Information; using var console_sink = new Duplicati.CommandLine.ConsoleOutput(Console.Out, global_options); - using var multi_sink = new Duplicati.Library.Main.ControllerMultiLogTarget(console_sink, log_level_enum, null); + using var multi_sink = new Duplicati.Library.Main.ControllerMultiLogTarget(console_sink, log_level_enum, null, null); // Parse the log file // The log file sink doesn't have to be disposed, as the multi_sink will take care of it