// Copyright (C) 2026, The Duplicati Team // https://duplicati.com, hello@duplicati.com // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; using Duplicati.Library.AutoUpdater; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using Uri = System.Uri; namespace Duplicati.Library.Modules.Builtin { /// /// A reporting module that periodically posts the current operation status to a /// remote URL as JSON. While an operation is running the engine pushes progress /// snapshots at a fixed cadence (see ); /// this module posts an update at most once every --http-report-status-interval /// (default 30 seconds), including the latest progress snapshot, the counts of /// backend events and log entries observed so far, and the last few log lines. /// public class HttpReportStatus : IReportModule, IGenericCallbackModule, IDisposable { /// /// The tag used for logging /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); #region Option names /// /// Option used to specify the remote URL to post status reports to. /// private const string OPTION_URL = "http-report-status-url"; /// /// Option used to specify the interval between status reports. /// private const string OPTION_INTERVAL = "http-report-status-interval"; /// /// Option used to specify the maximum number of recent log lines included in each report. /// private const string OPTION_MAX_LOG_LINES = "http-report-status-max-log-lines"; /// /// Option used to accept any SSL certificate. /// private const string OPTION_ACCEPT_ANY_CERTIFICATE = "http-report-status-accept-any-ssl-certificate"; /// /// Option used to accept a specific SSL certificate hash. /// private const string OPTION_ACCEPT_SPECIFIED_CERTIFICATE = "http-report-status-accept-specified-ssl-hash"; /// /// Option used to ignore certificate revocation check failures. /// private const string OPTION_IGNORE_REVOCATION_FAILURE = "http-report-status-ignore-revocation-failure"; /// /// The module-specific option used to disable path redaction in the buffered log /// lines. Mirrors the global allow-paths-in-log-messages option used by the /// reporting helpers. /// private const string OPTION_ALLOW_PATHS_IN_LOG_MESSAGES = "http-report-status-allow-paths-in-log-messages"; /// /// The global option used to disable path redaction in log messages, mirrored /// from Options.cs / ReportHelper. The module-specific option takes /// precedence, falling back to this global setting when not set. /// private const string OPTION_GLOBAL_ALLOW_PATHS_IN_LOG_MESSAGES = "allow-paths-in-log-messages"; #endregion #region Defaults /// /// The default interval between status reports. /// private const string DEFAULT_INTERVAL = "30s"; /// /// The default maximum number of recent log lines to include. /// private const int DEFAULT_MAX_LOG_LINES = 20; #endregion /// /// The module key, used to activate or deactivate the module on the commandline. /// public string Key => "httpreportstatus"; /// /// A localized string describing the module with a friendly name. /// public string DisplayName => Strings.HttpReportStatus.DisplayName; /// /// A localized description of the module. /// public string Description => Strings.HttpReportStatus.Description; /// /// The module is loaded, but inactive unless a url has been set /// public bool LoadAsDefault => true; /// /// Gets a list of supported commandline arguments. /// public IList SupportedCommands => [ new CommandLineArgument(OPTION_URL, CommandLineArgument.ArgumentType.String, Strings.HttpReportStatus.UrlShort, Strings.HttpReportStatus.UrlLong), new CommandLineArgument(OPTION_INTERVAL, CommandLineArgument.ArgumentType.Timespan, Strings.HttpReportStatus.IntervalShort, Strings.HttpReportStatus.IntervalLong, DEFAULT_INTERVAL), new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.HttpReportStatus.MaxLogLinesShort, Strings.HttpReportStatus.MaxLogLinesLong, DEFAULT_MAX_LOG_LINES.ToString()), new CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, CommandLineArgument.ArgumentType.Boolean, Strings.HttpReportStatus.AcceptAnyCertificateShort, Strings.HttpReportStatus.AcceptAnyCertificateLong), new CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, CommandLineArgument.ArgumentType.String, Strings.HttpReportStatus.AcceptSpecifiedCertificateShort, Strings.HttpReportStatus.AcceptSpecifiedCertificateLong), new CommandLineArgument(OPTION_IGNORE_REVOCATION_FAILURE, CommandLineArgument.ArgumentType.Boolean, Strings.HttpReportStatus.IgnoreRevocationFailureShort, Strings.HttpReportStatus.IgnoreRevocationFailureLong, "false"), new CommandLineArgument(OPTION_ALLOW_PATHS_IN_LOG_MESSAGES, CommandLineArgument.ArgumentType.Boolean, Strings.HttpReportStatus.AllowPathsInLogMessagesShort, Strings.HttpReportStatus.AllowPathsInLogMessagesLong, "false"), ]; /// /// The remote URLs to post status reports to. /// private string[] m_urls; /// /// The HTTP handler created during , owned by /// and disposed with it when the module is disposed. /// private HttpClientHandler m_httpHandler; /// /// The HTTP client created during , reused for every post /// and disposed when the module is disposed. /// private HttpClient m_httpClient; /// /// The interval between status reports. /// private TimeSpan m_interval; /// /// The maximum number of recent log lines to include in each report. /// private int m_maxLogLines; /// /// True if paths are allowed in the buffered log lines (i.e. not redacted). /// private bool m_allowPathsInLogMessages; /// /// A read-only copy of the commandline options, used to look up optional overrides /// for the report metadata (e.g. machine-id, backup-id), mirroring /// . /// private IReadOnlyDictionary m_options; /// /// The operation name reported in . /// private string m_operationName; /// /// The remote backend URL, captured in and used to compute /// the backup id and destination type, mirroring . /// private string m_remoteUrl; /// /// The time the operation started, in UTC. /// private DateTime m_operationStarted; /// /// The number of backend events observed so far. /// private int m_backendEvents; /// /// The number of log entries observed so far. /// private int m_logEntries; /// /// The most recent progress snapshot observed. /// private ReportProgressSnapshot m_latestSnapshot; /// /// A ring buffer of the most recent log lines observed. /// private readonly LinkedList m_recentLogLines = new(); /// /// The time the next status report may be sent. /// private DateTime m_nextReportTime; /// /// The lazily-computed environment metadata. The metadata only depends on the /// options and the remote URL (both fixed before an operation starts), so it is /// computed once per operation and reused across every report. /// private Lazy m_metadata; /// /// Lock protecting the mutable counters and buffers. /// private readonly object m_lock = new(); /// /// JSON serializer options that emit property names in camelCase. /// private static readonly JsonSerializerOptions SerializerOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, }; /// /// Configures the module from the commandline options. The module is only /// active when --http-report-status-url is set. /// /// A set of commandline options passed to Duplicati. public void Configure(IDictionary commandlineOptions) { commandlineOptions.TryGetValue(OPTION_URL, out var url); if (string.IsNullOrWhiteSpace(url)) { // Without a URL there is nothing to report to; leave the module inactive. m_urls = []; } else { m_urls = url .Split([',', ';'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) .Distinct() .ToArray(); } if (!IsActive) return; // Keep a read-only copy of the options so BuildMetadata can honor optional // overrides (machine-id, backup-id, backup-name, machine-name) like ReportHelper. m_options = commandlineOptions.AsReadOnly(); m_interval = Utility.Utility.ParseTimespanOption(commandlineOptions.AsReadOnly(), OPTION_INTERVAL, DEFAULT_INTERVAL); if (m_interval <= TimeSpan.Zero) m_interval = TimeSpan.FromSeconds(30); m_maxLogLines = Utility.Utility.ParseIntOption(commandlineOptions.AsReadOnly(), OPTION_MAX_LOG_LINES, DEFAULT_MAX_LOG_LINES); if (m_maxLogLines < 0) m_maxLogLines = DEFAULT_MAX_LOG_LINES; var acceptAnyCertificate = Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_ACCEPT_ANY_CERTIFICATE); var acceptSpecificCertificates = commandlineOptions.ContainsKey(OPTION_ACCEPT_SPECIFIED_CERTIFICATE) ? commandlineOptions[OPTION_ACCEPT_SPECIFIED_CERTIFICATE].Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries) : null; var ignoreRevocationFailure = Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_IGNORE_REVOCATION_FAILURE); m_httpHandler = new HttpClientHandler(); HttpClientHelper.ConfigureHandlerCertificateValidator(m_httpHandler, acceptAnyCertificate, acceptSpecificCertificates, ignoreRevocationFailure); m_httpClient = new HttpClient(m_httpHandler); // Paths in log lines are redacted by default. The module-specific option takes // precedence and falls back to the global allow-paths-in-log-messages setting, // mirroring the behavior of the other reporting modules (see ReportHelper). if (commandlineOptions.TryGetValue(OPTION_ALLOW_PATHS_IN_LOG_MESSAGES, out var allowPathsModule) && bool.TryParse(allowPathsModule, out var parsedModule)) m_allowPathsInLogMessages = parsedModule; else m_allowPathsInLogMessages = Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_GLOBAL_ALLOW_PATHS_IN_LOG_MESSAGES); } /// /// Captures the operation name, remote backend URL and local paths. The remote URL /// is used to compute the backup id and destination type included in each report, /// mirroring . /// /// The full name of the operation. /// The remote backend url. /// The local path, if required. public void OnStart(string operationname, ref string remoteurl, ref string[] localpath) { m_operationName = operationname; m_remoteUrl = remoteurl; } /// /// No-op; completion is reported via instead. /// /// The result object. /// The exception that stopped the operation, or null. public void OnFinish(IBasicResults result, Exception exception) { // Completion is handled by the IReportModule lifecycle; nothing to do here. } /// public Task OnOperationStartedAsync(string operationName, IBasicResults result, CancellationToken cancellationToken) { if (!IsActive) return Task.CompletedTask; lock (m_lock) { m_operationName = operationName ?? "Operation"; m_operationStarted = DateTime.UtcNow; m_backendEvents = 0; m_logEntries = 0; m_latestSnapshot = null; m_recentLogLines.Clear(); m_nextReportTime = DateTime.UtcNow; m_metadata = new Lazy(() => BuildMetadata(), isThreadSafe: true); } return PostAsync(BuildReport("Started", null), cancellationToken); } /// public async Task OnOperationCompletedAsync(IBasicResults result, Exception exception, CancellationToken cancellationToken) { if (!IsActive) return; // Refresh the snapshot with the final result so the completed report is accurate. lock (m_lock) { m_latestSnapshot = new ReportProgressSnapshot( "Complete", 1f, 0, 0, 0, 0, false, null, 0, 0, Array.Empty()); } var report = BuildReport(exception == null ? "Completed" : "Failed", exception?.Message); await PostAsync(report, cancellationToken).ConfigureAwait(false); } /// public Task OnBackendEventAsync(ReportBackendEvent evt, CancellationToken cancellationToken) { if (!IsActive) return Task.CompletedTask; lock (m_lock) m_backendEvents++; return Task.CompletedTask; } /// public Task OnLogEntryAsync(ReportLogEntry entry, CancellationToken cancellationToken) { if (!IsActive || entry == null) return Task.CompletedTask; lock (m_lock) { m_logEntries++; // Redact paths in the buffered log line unless the user has explicitly // allowed paths in log messages, mirroring the reporting helpers. var line = m_allowPathsInLogMessages ? entry.Message : SensitiveDataFilter.RedactPaths(entry.Message); m_recentLogLines.AddLast(line); while (m_recentLogLines.Count > m_maxLogLines) m_recentLogLines.RemoveFirst(); } return Task.CompletedTask; } /// public Task OnProgressTickAsync(ReportProgressSnapshot snapshot, CancellationToken cancellationToken) { if (!IsActive) return Task.CompletedTask; bool shouldReport; lock (m_lock) { m_latestSnapshot = snapshot; shouldReport = DateTime.UtcNow >= m_nextReportTime; if (shouldReport) m_nextReportTime = DateTime.UtcNow + m_interval; } if (!shouldReport) return Task.CompletedTask; return PostAsync(BuildReport("Progress", null), cancellationToken); } /// /// Gets a value indicating whether the module is configured with a target URL. /// /// /// The module is only active when a target URL has been configured, /// so that no per-event interception happens unless there is somewhere to /// report to. public bool IsActive => m_urls != null && m_urls.Length > 0; /// /// Computes the environment metadata included in each report, mirroring the /// template keys resolved by . The /// remote URL captured in drives the backup id, the /// destination type and the destination host suffix. /// /// The metadata for the current report. private ReportMetadata BuildMetadata() => new ReportMetadata { DuplicatiVersion = UpdaterManager.SelfVersion.Version, MachineId = OptionOrDefault("machine-id", DataFolderManager.MachineID), BackupId = OptionOrDefault("backup-id", Utility.Utility.CalculateBackupId(m_remoteUrl)), BackupName = OptionOrDefault("backup-name", System.IO.Path.GetFileNameWithoutExtension(Utility.Utility.getEntryAssembly().Location)), MachineName = OptionOrDefault("machine-name", DataFolderManager.MachineName), DestinationType = Utility.Utility.GuessScheme(m_remoteUrl) ?? "file", DestinationHostSuffix = Utility.Utility.GuessHostSuffixSafe(m_remoteUrl), InstallationType = UpdaterManager.PackageTypeId, OperatingSystem = UpdaterManager.OperatingSystemName, OperatingSystemDetailed = OSInfoHelper.PlatformString, }; /// /// Returns the configured value for the given option key, or the supplied default /// when the option is absent or empty. Used to honor user-supplied overrides for /// the report metadata, mirroring . /// /// The option key to look up. /// The default value to use when the option is not set. /// The option value, or the default. private string OptionOrDefault(string key, string defaultValue) { if (m_options != null && m_options.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)) return value; return defaultValue; } /// /// Builds the JSON-serializable status report from the current state. /// /// The status label for this report (e.g. Started, Progress, Completed). /// An error message to include, or null. /// A status report ready to be serialized and posted. private StatusReport BuildReport(string status, string errorMessage) { lock (m_lock) { return new StatusReport { Operation = m_operationName, Status = status, StartedUtc = m_operationStarted, ReportedUtc = DateTime.UtcNow, ErrorMessage = errorMessage, IsCompleted = status is "Completed" or "Failed", BackendEvents = m_backendEvents, LogEntries = m_logEntries, Progress = m_latestSnapshot == null ? null : new ProgressSnapshot(m_latestSnapshot), RecentLogLines = [.. m_recentLogLines], // Metadata is lazily computed once per operation (see OnOperationStartedAsync) // since it only depends on the options and remote URL. Metadata = m_metadata?.Value, }; } } /// /// Posts the given status report to the configured URL as JSON. Failures are /// logged but never propagated to the caller. /// /// The report to post. /// A token to monitor for cancellation requests. /// A task that represents the asynchronous operation. private async Task PostAsync(StatusReport report, CancellationToken cancellationToken) { if (report == null) return; try { var json = JsonSerializer.Serialize(report, typeof(StatusReport), SerializerOptions); foreach (var url in m_urls) await SendAsync(url, json, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "HttpReportStatusSendError", ex, "Failed to post status report to {0}: {1}", string.Join(", ", m_urls), ex.Message); } } /// /// Sends the JSON body to the configured URL. This is virtual so tests can /// intercept the request without performing real network I/O. /// /// The url to POST to. /// The JSON body to send. /// A token to monitor for cancellation requests. /// A task that represents the asynchronous operation. protected virtual async Task SendAsync(string url, string json, CancellationToken cancellationToken) { using var content = new StringContent(json, Encoding.UTF8, "application/json"); using var response = await m_httpClient.PostAsync(new Uri(url), content, cancellationToken).ConfigureAwait(false); response.EnsureSuccessStatusCode(); } /// /// Disposes the handler created during . /// public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } /// /// Disposes managed and unmanaged resources. /// /// true when disposing managed resources. protected virtual void Dispose(bool disposing) { if (!disposing) return; try { m_httpClient?.Dispose(); } catch { } m_httpClient = null; m_httpHandler = null; } /// /// The JSON status report posted to the remote URL. /// public sealed class StatusReport { /// The operation name. public string Operation { get; set; } /// The status label for this report. public string Status { get; set; } /// The time the operation started, in UTC. public DateTime StartedUtc { get; set; } /// The time this report was generated, in UTC. public DateTime ReportedUtc { get; set; } /// An error message, if the operation failed. public string ErrorMessage { get; set; } /// true once the operation has finished running (Completed/Failed). public bool IsCompleted { get; set; } /// The number of backend events observed so far. public int BackendEvents { get; set; } /// The number of log entries observed so far. public int LogEntries { get; set; } /// The latest progress snapshot, or null. public ProgressSnapshot Progress { get; set; } /// The most recent log lines observed. public List RecentLogLines { get; set; } /// Environment metadata included in every report. public ReportMetadata Metadata { get; set; } } /// /// Environment metadata included in every status report, mirroring the template /// keys resolved by . /// public sealed class ReportMetadata { /// The running Duplicati version. public string DuplicatiVersion { get; set; } /// The stable machine id. public string MachineId { get; set; } /// A stable id derived from the backup's remote URL. public string BackupId { get; set; } /// The backup name (entry assembly name). public string BackupName { get; set; } /// The machine name. public string MachineName { get; set; } /// The destination type (the remote URL scheme). public string DestinationType { get; set; } /// A safe destination host suffix (known public cloud only), or null. public string DestinationHostSuffix { get; set; } /// The installation type (e.g. package type id). public string InstallationType { get; set; } /// The operating system name (e.g. Windows, Linux, MacOS). public string OperatingSystem { get; set; } /// A detailed operating system platform string. public string OperatingSystemDetailed { get; set; } } /// /// The progress portion of the status report. /// public sealed class ProgressSnapshot { /// Parameterless constructor for JSON deserialization. public ProgressSnapshot() { } /// Creates a progress snapshot from the interface record. public ProgressSnapshot(ReportProgressSnapshot snapshot) { Phase = snapshot.Phase; Progress = snapshot.Progress; FilesProcessed = snapshot.FilesProcessed; FileSizeProcessed = snapshot.FileSizeProcessed; FileCount = snapshot.FileCount; FileSize = snapshot.FileSize; CountingFiles = snapshot.CountingFiles; CurrentFilename = snapshot.CurrentFilename; CurrentFileOffset = snapshot.CurrentFileOffset; ActiveTransfers = snapshot.ActiveTransfers?.Length ?? 0; } /// The current operation phase name. public string Phase { get; set; } /// The overall progress, in the range [0, 1]. public float Progress { get; set; } /// The number of files processed so far. public long FilesProcessed { get; set; } /// The number of bytes processed so far. public long FileSizeProcessed { get; set; } /// The total number of files. public long FileCount { get; set; } /// The total number of bytes. public long FileSize { get; set; } /// True if the file count and size are not yet final. public bool CountingFiles { get; set; } /// The file currently being processed, or null. public string CurrentFilename { get; set; } /// The byte offset reached in the file currently being processed. public long CurrentFileOffset { get; set; } /// The number of active backend transfers. public int ActiveTransfers { get; set; } } } }