Add option to log HTTP and socket requests

This adds the options `--log-http-requests` and `--log-socket-data` which can be activated to capture diagnostics about HTTP requests and socket operations.

Duplicati/Library/Main/Controller.cs

@@ -486,6 +486,9 @@ namespace Duplicati.Library.Main
             {
                 logTarget.AddTarget(m_messageSink, m_options.ConsoleLoglevel, m_options.ConsoleLogFilter);
                 result.MessageSink = m_messageSink;
+                using var httpListener = m_options.LogHttpRequests || m_options.LogSocketData >= 0
+                    ? new NetworkTrafficLogger(m_options.LogHttpRequests, m_options.LogSocketData)
+                    : null;
 
                 try
                 {

Duplicati/Library/Main/NetworkTrafficLogger.cs

@@ -0,0 +1,148 @@
+// Copyright (C) 2025, The Duplicati Team
+// https://duplicati.com, hello@duplicati.com
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a 
+// copy of this software and associated documentation files (the "Software"), 
+// to deal in the Software without restriction, including without limitation 
+// the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+// and/or sell copies of the Software, and to permit persons to whom the 
+// Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in 
+// all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
+// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
+// DEALINGS IN THE SOFTWARE.
+
+#nullable enable
+
+using System;
+using System.Diagnostics.Tracing;
+using System.Text;
+
+namespace Duplicati.Library.Main;
+
+/// <summary>
+/// Listener for HTTP diagnostics events
+/// This listener captures events from System.Net.Http and System.Net.Sockets
+/// and forwards them to the Duplicati logging system.
+/// </summary>
+/// <param name="enableHttpLogging">If true, enables logging of HTTP events.</param>
+/// <param name="socketDataBytes">The number of bytes of socket data to log, or -1 to disable logging of socket data.</param>
+internal sealed class NetworkTrafficLogger(bool enableHttpLogging, int socketDataBytes) : EventListener
+{
+    /// <summary>
+    /// Log tag for HTTP events
+    /// </summary>
+    private static readonly string LogTagHttp = Logging.Log.LogTagFromType<NetworkTrafficLogger>() + ".Http";
+    /// <summary>
+    /// Log tag for socket events
+    /// </summary>
+    private static readonly string LogTagSocket = Logging.Log.LogTagFromType<NetworkTrafficLogger>() + ".Socket";
+
+    /// <inheritdoc/>
+    protected override void OnEventSourceCreated(EventSource eventSource)
+    {
+        if ((enableHttpLogging && eventSource.Name == "System.Net.Http") || (socketDataBytes >= 0 && eventSource.Name == "System.Net.Sockets"))
+            EnableEvents(eventSource, EventLevel.Verbose, EventKeywords.All);
+    }
+
+    /// <inheritdoc/>
+    protected override void OnEventWritten(EventWrittenEventArgs eventData)
+    {
+        if (enableHttpLogging && eventData.EventSource.Name == "System.Net.Http")
+        {
+            Logging.Log.WriteVerboseMessage(LogTagHttp, eventData.EventName, FormatPayload(eventData));
+        }
+        else if (socketDataBytes >= 0 && eventData.EventSource.Name == "System.Net.Sockets")
+        {
+            Logging.Log.WriteVerboseMessage(LogTagSocket, eventData.EventName, FormatSocketPayload(eventData));
+        }
+    }
+
+    /// <summary>
+    /// Formats the payload of an HTTP event for logging.
+    /// </summary>
+    /// <param name="eventData">The event data containing the payload.</param>
+    /// <returns>A formatted string representation of the payload.</returns>
+    private static string FormatPayload(EventWrittenEventArgs eventData)
+    {
+        var sb = new StringBuilder();
+        sb.AppendLine($"[{eventData.EventSource.Name}] Event: {eventData.EventName}");
+
+        if (eventData.Payload != null && eventData.Payload.Count > 0)
+        {
+            for (int i = 0; i < eventData.Payload.Count; i++)
+            {
+                var name = eventData.PayloadNames?[i] ?? $"Field{i}";
+                var value = FormatValue(eventData.Payload[i]);
+                sb.AppendLine($"  {name,-20}: {value}");
+            }
+        }
+
+        return sb.ToString();
+    }
+
+    /// <summary>
+    /// Formats the payload of a socket event for logging.
+    /// </summary>
+    /// <param name="eventData">The event data containing the payload.</param>
+    /// <returns>A formatted string representation of the socket payload.</returns>
+    private string FormatSocketPayload(EventWrittenEventArgs eventData)
+    {
+        var sb = new StringBuilder();
+        sb.AppendLine($"[{eventData.EventSource.Name}] Event: {eventData.EventName}");
+
+        if (eventData.Payload != null && eventData.Payload.Count > 0)
+        {
+            for (int i = 0; i < eventData.Payload.Count; i++)
+            {
+                var name = eventData.PayloadNames?[i] ?? $"Field{i}";
+                var val = eventData.Payload[i];
+
+                if (val is byte[] bytes)
+                {
+                    int len = socketDataBytes == 0 ? 0 : Math.Min(bytes.Length, socketDataBytes);
+                    if (len > 0)
+                    {
+                        string hex = BitConverter.ToString(bytes, 0, len);
+                        string ascii = Encoding.ASCII.GetString(bytes, 0, len).Replace('\0', '.');
+                        sb.AppendLine($"  {name,-20}: {hex} (ASCII: {ascii})");
+                    }
+                    else
+                    {
+                        sb.AppendLine($"  {name,-20}: [Byte array suppressed]");
+                    }
+                }
+                else
+                {
+                    sb.AppendLine($"  {name,-20}: {FormatValue(val)}");
+                }
+            }
+        }
+
+        return sb.ToString();
+    }
+
+    /// <summary>
+    /// Formats a value for logging.
+    /// </summary>
+    /// <param name="val">The value to format.</param>
+    /// <returns>A formatted string representation of the value.</returns>
+    private static string FormatValue(object? val)
+    {
+        if (val == null)
+            return "(null)";
+        var type = val.GetType();
+
+        if (type.IsEnum)
+            return $"{val} ({Convert.ToInt32(val)})";
+
+        return val.ToString() ?? string.Empty;
+    }
+}

Duplicati/Library/Main/Options.cs

@@ -443,6 +443,9 @@ namespace Duplicati.Library.Main
             new CommandLineArgument("log-level", CommandLineArgument.ArgumentType.Enumeration, Strings.Options.LoglevelShort, Strings.Options.LoglevelLong, "Warning", null, Enum.GetNames(typeof(Duplicati.Library.Logging.LogMessageType)), Strings.Options.LogLevelDeprecated("log-file-log-level", "console-log-level")),
             new CommandLineArgument("suppress-warnings", CommandLineArgument.ArgumentType.String, Strings.Options.SuppresswarningsShort, Strings.Options.SuppresswarningsLong),
 
+            new CommandLineArgument("log-http-requests", CommandLineArgument.ArgumentType.Boolean, Strings.Options.LoghttprequestsShort, Strings.Options.LoghttprequestsLong, "false"),
+            new CommandLineArgument("log-socket-data", CommandLineArgument.ArgumentType.Integer, Strings.Options.LogsocketdataShort, Strings.Options.LogsocketdataLong, "-1"),
+
             new CommandLineArgument("profile-all-database-queries", CommandLineArgument.ArgumentType.Boolean, Strings.Options.ProfilealldatabasequeriesShort, Strings.Options.ProfilealldatabasequeriesLong, "false"),
 
             new CommandLineArgument("list-verify-uploads", CommandLineArgument.ArgumentType.Boolean, Strings.Options.ListverifyuploadsShort, Strings.Options.ListverifyuploadsLong, "false"),
@@ -1041,6 +1044,16 @@ namespace Duplicati.Library.Main
         public HashSet<string>? SuppressWarningsFilter
             => m_options.GetValueOrDefault("suppress-warnings")?.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)?.ToHashSet(StringComparer.OrdinalIgnoreCase);
 
+        /// <summary>
+        /// A value indicating if HTTP requests should be logged
+        /// </summary>
+        public bool LogHttpRequests => GetBool("log-http-requests");
+
+        /// <summary>
+        /// Gets the number of bytes of socket data to include in logs, -1 disables logging
+        /// </summary>
+        public int LogSocketData => GetInt("log-socket-data", -1);
+
         /// <summary>
         /// Gets the filter used for log-file messages.
         /// </summary>

Duplicati/Library/Main/Strings.cs

@@ -133,6 +133,10 @@ namespace Duplicati.Library.Main.Strings
         public static string LogLevelDeprecated(string option1, string option2) { return LC.L("Use the options --{0} and --{1} instead.", option1, option2); }
         public static string SuppresswarningsLong { get { return LC.L(@"Suppress warnings and log them as information instead. Use this if you need to silence specific warnings. This option accepts a comma separated list of warning IDs."); } }
         public static string SuppresswarningsShort { get { return LC.L(@"Suppress specific warnings"); } }
+        public static string LoghttprequestsLong { get { return LC.L(@"Enable logging of HTTP request diagnostics. Messages are logged at the {0} level, so make sure the log outputs at that level.", Logging.LogMessageType.Verbose); } }
+        public static string LoghttprequestsShort { get { return LC.L(@"Log HTTP requests"); } }
+        public static string LogsocketdataLong { get { return LC.L(@"Enable logging of socket data. A value of 0 logs only event messages, a positive value includes that many bytes of data. Use -1 to disable. Messages are logged at the {0} level, so make sure the log outputs at that level.", Logging.LogMessageType.Verbose); } }
+        public static string LogsocketdataShort { get { return LC.L(@"Log socket data"); } }
         public static string DisableautocreatefolderLong { get { return LC.L(@"If Duplicati detects that the target folder is missing, it will create it automatically. Activate this option to prevent automatic folder creation."); } }
         public static string DisableautocreatefolderShort { get { return LC.L(@"Disable automatic folder creation"); } }
         public static string VssexcludewritersLong { get { return LC.L(@"Use this option to exclude faulty writers from a snapshot. This is equivalent to the -wx flag of the vshadow.exe tool, except that it only accepts writer class GUIDs, and not component names or instance GUIDs. Multiple GUIDs must be separated with a semicolon, and most forms of GUIDs are allowed, including with and without curly braces."); } }