diff --git a/Duplicati/Library/Modules/Builtin/HttpReportStatus.cs b/Duplicati/Library/Modules/Builtin/HttpReportStatus.cs index c3381dcb7..5e9c415cd 100644 --- a/Duplicati/Library/Modules/Builtin/HttpReportStatus.cs +++ b/Duplicati/Library/Modules/Builtin/HttpReportStatus.cs @@ -27,6 +27,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; +using Duplicati.Library.AutoUpdater; using Duplicati.Library.Interface; using Duplicati.Library.Utility; using Uri = System.Uri; @@ -41,7 +42,7 @@ namespace Duplicati.Library.Modules.Builtin /// (default 30 seconds), including the latest progress snapshot, the counts of /// backend events and log entries observed so far, and the last few log lines. /// - public class HttpReportStatus : IReportModule, IDisposable + public class HttpReportStatus : IReportModule, IGenericCallbackModule, IDisposable { /// /// The tag used for logging @@ -170,11 +171,24 @@ namespace Duplicati.Library.Modules.Builtin /// private bool m_allowPathsInLogMessages; + /// + /// A read-only copy of the commandline options, used to look up optional overrides + /// for the report metadata (e.g. machine-id, backup-id), mirroring + /// . + /// + private IReadOnlyDictionary m_options; + /// /// The operation name reported in . /// private string m_operationName; + /// + /// The remote backend URL, captured in and used to compute + /// the backup id and destination type, mirroring . + /// + private string m_remoteUrl; + /// /// The time the operation started, in UTC. /// @@ -205,6 +219,13 @@ namespace Duplicati.Library.Modules.Builtin /// private DateTime m_nextReportTime; + /// + /// The lazily-computed environment metadata. The metadata only depends on the + /// options and the remote URL (both fixed before an operation starts), so it is + /// computed once per operation and reused across every report. + /// + private Lazy m_metadata; + /// /// Lock protecting the mutable counters and buffers. /// @@ -234,6 +255,10 @@ namespace Duplicati.Library.Modules.Builtin return; } + // Keep a read-only copy of the options so BuildMetadata can honor optional + // overrides (machine-id, backup-id, backup-name, machine-name) like ReportHelper. + m_options = commandlineOptions.AsReadOnly(); + m_interval = Utility.Utility.ParseTimespanOption(commandlineOptions.AsReadOnly(), OPTION_INTERVAL, DEFAULT_INTERVAL); if (m_interval <= TimeSpan.Zero) m_interval = TimeSpan.FromSeconds(30); @@ -260,6 +285,30 @@ namespace Duplicati.Library.Modules.Builtin m_allowPathsInLogMessages = Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_GLOBAL_ALLOW_PATHS_IN_LOG_MESSAGES); } + /// + /// Captures the operation name, remote backend URL and local paths. The remote URL + /// is used to compute the backup id and destination type included in each report, + /// mirroring . + /// + /// The full name of the operation. + /// The remote backend url. + /// The local path, if required. + public void OnStart(string operationname, ref string remoteurl, ref string[] localpath) + { + m_operationName = operationname; + m_remoteUrl = remoteurl; + } + + /// + /// No-op; completion is reported via instead. + /// + /// The result object. + /// The exception that stopped the operation, or null. + public void OnFinish(IBasicResults result, Exception exception) + { + // Completion is handled by the IReportModule lifecycle; nothing to do here. + } + /// public Task OnOperationStartedAsync(string operationName, IBasicResults result, CancellationToken cancellationToken) { @@ -275,6 +324,7 @@ namespace Duplicati.Library.Modules.Builtin m_latestSnapshot = null; m_recentLogLines.Clear(); m_nextReportTime = DateTime.UtcNow; + m_metadata = new Lazy(() => BuildMetadata(), isThreadSafe: true); } return PostAsync(BuildReport("Started", null), cancellationToken); @@ -363,6 +413,43 @@ namespace Duplicati.Library.Modules.Builtin /// report to. public bool IsActive => !string.IsNullOrWhiteSpace(m_url); + /// + /// Computes the environment metadata included in each report, mirroring the + /// template keys resolved by . The + /// remote URL captured in drives the backup id, the + /// destination type and the destination host suffix. + /// + /// The metadata for the current report. + private ReportMetadata BuildMetadata() + => new ReportMetadata + { + DuplicatiVersion = UpdaterManager.SelfVersion.Version, + MachineId = OptionOrDefault("machine-id", DataFolderManager.MachineID), + BackupId = OptionOrDefault("backup-id", Utility.Utility.CalculateBackupId(m_remoteUrl)), + BackupName = OptionOrDefault("backup-name", System.IO.Path.GetFileNameWithoutExtension(Utility.Utility.getEntryAssembly().Location)), + MachineName = OptionOrDefault("machine-name", DataFolderManager.MachineName), + DestinationType = Utility.Utility.GuessScheme(m_remoteUrl) ?? "file", + DestinationHostSuffix = Utility.Utility.GuessHostSuffixSafe(m_remoteUrl), + InstallationType = UpdaterManager.PackageTypeId, + OperatingSystem = UpdaterManager.OperatingSystemName, + OperatingSystemDetailed = OSInfoHelper.PlatformString, + }; + + /// + /// Returns the configured value for the given option key, or the supplied default + /// when the option is absent or empty. Used to honor user-supplied overrides for + /// the report metadata, mirroring . + /// + /// The option key to look up. + /// The default value to use when the option is not set. + /// The option value, or the default. + private string OptionOrDefault(string key, string defaultValue) + { + if (m_options != null && m_options.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)) + return value; + return defaultValue; + } + /// /// Builds the JSON-serializable status report from the current state. /// @@ -380,10 +467,14 @@ namespace Duplicati.Library.Modules.Builtin StartedUtc = m_operationStarted, ReportedUtc = DateTime.UtcNow, ErrorMessage = errorMessage, + IsCompleted = status is "Completed" or "Failed", BackendEvents = m_backendEvents, LogEntries = m_logEntries, Progress = m_latestSnapshot == null ? null : new ProgressSnapshot(m_latestSnapshot), RecentLogLines = [.. m_recentLogLines], + // Metadata is lazily computed once per operation (see OnOperationStartedAsync) + // since it only depends on the options and remote URL. + Metadata = m_metadata?.Value, }; } } @@ -469,6 +560,9 @@ namespace Duplicati.Library.Modules.Builtin /// An error message, if the operation failed. public string ErrorMessage { get; set; } + /// true once the operation has finished running (Completed/Failed). + public bool IsCompleted { get; set; } + /// The number of backend events observed so far. public int BackendEvents { get; set; } @@ -480,6 +574,46 @@ namespace Duplicati.Library.Modules.Builtin /// The most recent log lines observed. public List RecentLogLines { get; set; } + + /// Environment metadata included in every report. + public ReportMetadata Metadata { get; set; } + } + + /// + /// Environment metadata included in every status report, mirroring the template + /// keys resolved by . + /// + public sealed class ReportMetadata + { + /// The running Duplicati version. + public string DuplicatiVersion { get; set; } + + /// The stable machine id. + public string MachineId { get; set; } + + /// A stable id derived from the backup's remote URL. + public string BackupId { get; set; } + + /// The backup name (entry assembly name). + public string BackupName { get; set; } + + /// The machine name. + public string MachineName { get; set; } + + /// The destination type (the remote URL scheme). + public string DestinationType { get; set; } + + /// A safe destination host suffix (known public cloud only), or null. + public string DestinationHostSuffix { get; set; } + + /// The installation type (e.g. package type id). + public string InstallationType { get; set; } + + /// The operating system name (e.g. Windows, Linux, MacOS). + public string OperatingSystem { get; set; } + + /// A detailed operating system platform string. + public string OperatingSystemDetailed { get; set; } } /// diff --git a/Duplicati/Library/Modules/Builtin/ReportHelper.cs b/Duplicati/Library/Modules/Builtin/ReportHelper.cs index d2a5fde35..d0e70e35b 100644 --- a/Duplicati/Library/Modules/Builtin/ReportHelper.cs +++ b/Duplicati/Library/Modules/Builtin/ReportHelper.cs @@ -41,11 +41,6 @@ namespace Duplicati.Library.Modules.Builtin /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - /// - /// The salt used for calculating a backup Id from the remote URL - /// - private const string SALT = "DUPL"; - /// /// Name of the option used to specify subject /// @@ -448,7 +443,7 @@ namespace Duplicati.Library.Modules.Builtin case MACHINE_ID: return DataFolderManager.MachineID; case BACKUP_ID: - return Utility.Utility.ByteArrayAsHexString(Utility.Utility.RepeatedHashWithSalt(m_remoteurl, SALT)); + return Utility.Utility.CalculateBackupId(m_remoteurl); case BACKUP_NAME: return System.IO.Path.GetFileNameWithoutExtension(Utility.Utility.getEntryAssembly().Location); case MACHINE_NAME: @@ -464,7 +459,7 @@ namespace Duplicati.Library.Modules.Builtin var ix = m_remoteurl?.IndexOf("://", StringComparison.OrdinalIgnoreCase) ?? -1; return Utility.Utility.GuessScheme(m_remoteurl) ?? "file"; case DESTINATION_HOST_SUFFIX: - return GuessHostSuffixSafe(m_remoteurl); + return Utility.Utility.GuessHostSuffixSafe(m_remoteurl); case UPDATE_CHANNEL: return UpdaterManager.CurrentChannel.ToString(); default: @@ -646,87 +641,5 @@ namespace Duplicati.Library.Modules.Builtin } } - - /// - /// Regex used to extract the host from a URL - /// - private static readonly Regex hostSuffixRegex = new Regex(@"^(?:[a-zA-Z][a-zA-Z0-9+\-.]*://)?([^/:]+)", RegexOptions.Compiled); - - /// - /// List of host suffixes that are considered public cloud and safe to report - /// - private static readonly IEnumerable safeHostSuffixes = new HashSet([ - ".amazonaws.com", - ".impossibleapi.net", - ".scw.cloud", - ".hosteurope.de", - ".dunkel.de", - ".dreamhost.com", - ".dincloud.com", - ".polisystems.ch", - ".softlayer.net", - ".storadera.com", - ".wasabisys.com", - ".infomaniak.com", - ".infomaniak.cloud", - ".sakurastorage.jp", - ".lyvecloud.seagate.com", - ".digitaloceanspaces.com", - ".backblazeb2.com", - ".cloudian.com", - ".min.io", - ".linodeobjects.com", - ".bunnycdn.com", - ".azure.com", - ".googleapis.com", - ".ibm.com", - ".oracle.com", - ".cloudflare.com", - ".alibaba.com", - ".huawei.com", - ".tencent.com", - ".baidu.com", - ".jd.com", - ".ucloud.cn", - ".qiniu.com", - ".aliyuncs.com", - ".tcloud.com", - ".tencentcloudapi.com", - ".rabata.io", - ".internxt.com", - ".cloudflarestorage.com", - ".storage.selcloud.ru", - ".srvstorage.uz", - ".srvstorage.kz" - ], StringComparer.OrdinalIgnoreCase); - - /// - /// Attempts to guess a safe host suffix from a URL. - /// This attempts to avoid reporting private or sensitive hostnames, and only returns known public cloud suffixes. - /// - /// The URL to analyze. - /// The host suffix, or null if not found or not safe to report. - private static string GuessHostSuffixSafe(string url) - { - var scheme = Utility.Utility.GuessScheme(url); - - // For now, only report S3 host suffixes - if (!string.Equals(scheme, "s3", StringComparison.OrdinalIgnoreCase)) - return null; - - try - { - var hostname = hostSuffixRegex.Match(url).Groups[1].Value; - if (string.IsNullOrEmpty(hostname)) - return null; - - // Return the first matching safe suffix - return safeHostSuffixes.FirstOrDefault(suffix => hostname.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) ?? null; - } - catch - { - return null; - } - } } } diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index e4add3d27..846781dea 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1385,6 +1385,27 @@ namespace Duplicati.Library.Utility } } + /// + /// The salt used for calculating a backup Id from the remote URL. + /// Shared between the reporting modules so the same id is computed everywhere. + /// + public const string BACKUP_ID_SALT = "DUPL"; + + /// + /// Calculates a stable, opaque backup id from the backup's remote URL. The id is + /// a hex string produced by repeatedly hashing the URL with a fixed salt, so the + /// same remote URL always yields the same id without exposing the URL itself. + /// + /// The remote backend url to compute the id for. + /// The backup id as a hex string, or an empty string if the url is null/empty. + public static string CalculateBackupId(string? remoteUrl) + { + if (string.IsNullOrWhiteSpace(remoteUrl)) + return string.Empty; + + return ByteArrayAsHexString(RepeatedHashWithSalt(remoteUrl, BACKUP_ID_SALT)); + } + /// /// Gets the drive letter from the given volume guid. /// This method cannot be inlined since the System.Management types are not implemented in Mono @@ -1669,6 +1690,92 @@ namespace Duplicati.Library.Utility return url[..idx]; } + /// + /// Regex used to extract the host from a URL. + /// + private static readonly Regex hostSuffixRegex = new Regex(@"^(?:[a-zA-Z][a-zA-Z0-9+\-.]*://)?([^/:]+)", RegexOptions.Compiled); + + /// + /// List of host suffixes that are considered public cloud and safe to report. + /// + private static readonly IReadOnlySet safeHostSuffixes = new HashSet([ + ".amazonaws.com", + ".impossibleapi.net", + ".scw.cloud", + ".hosteurope.de", + ".dunkel.de", + ".dreamhost.com", + ".dincloud.com", + ".polisystems.ch", + ".softlayer.net", + ".storadera.com", + ".wasabisys.com", + ".infomaniak.com", + ".infomaniak.cloud", + ".sakurastorage.jp", + ".lyvecloud.seagate.com", + ".digitaloceanspaces.com", + ".backblazeb2.com", + ".cloudian.com", + ".min.io", + ".linodeobjects.com", + ".bunnycdn.com", + ".azure.com", + ".googleapis.com", + ".ibm.com", + ".oracle.com", + ".cloudflare.com", + ".alibaba.com", + ".huawei.com", + ".tencent.com", + ".baidu.com", + ".jd.com", + ".ucloud.cn", + ".qiniu.com", + ".aliyuncs.com", + ".tcloud.com", + ".tencentcloudapi.com", + ".rabata.io", + ".internxt.com", + ".cloudflarestorage.com", + ".storage.selcloud.ru", + ".srvstorage.uz", + ".srvstorage.kz" + ], StringComparer.OrdinalIgnoreCase); + + /// + /// Attempts to guess a safe host suffix from a URL. + /// This attempts to avoid reporting private or sensitive hostnames, and only returns + /// known public cloud suffixes. + /// + /// The URL to analyze. + /// The host suffix, or null if not found or not safe to report. + public static string? GuessHostSuffixSafe(string? url) + { + if (string.IsNullOrWhiteSpace(url)) + return null; + + var scheme = GuessScheme(url); + + // For now, only report S3 host suffixes + if (!string.Equals(scheme, "s3", StringComparison.OrdinalIgnoreCase)) + return null; + + try + { + var hostname = hostSuffixRegex.Match(url).Groups[1].Value; + if (string.IsNullOrEmpty(hostname)) + return null; + + // Return the first matching safe suffix + return safeHostSuffixes.FirstOrDefault(suffix => hostname.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)); + } + catch + { + return null; + } + } + /// /// Returns a url that is safe to display, by removing any credentials /// diff --git a/Duplicati/UnitTest/HttpReportStatusTests.cs b/Duplicati/UnitTest/HttpReportStatusTests.cs index bd66b4077..d3ace22b1 100644 --- a/Duplicati/UnitTest/HttpReportStatusTests.cs +++ b/Duplicati/UnitTest/HttpReportStatusTests.cs @@ -130,12 +130,26 @@ namespace Duplicati.UnitTest // The completed report carries a final progress snapshot at 100%. Assert.IsNotNull(module.Reports[1].Progress); Assert.AreEqual(1f, module.Reports[1].Progress!.Progress); + // The Started report is not completed, the Completed report is. + Assert.IsFalse(module.Reports[0].IsCompleted, "Started report should not be completed"); + Assert.IsTrue(module.Reports[1].IsCompleted, "Completed report should be marked completed"); // The JSON is serialized with camelCase property names. Assert.That(module.PostedJsons[0], Does.Contain("\"operation\""), "JSON keys should be camelCase"); Assert.That(module.PostedJsons[0], Does.Contain("\"startedUtc\"")); Assert.That(module.PostedJsons[0], Does.Not.Contain("\"StartedUtc\"")); } + [Test] + public async Task FailedReportIsCompletedAsync() + { + using var module = CreateConfigured(); + await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None); + await module.OnOperationCompletedAsync(null!, new InvalidOperationException("boom"), CancellationToken.None); + + Assert.AreEqual("Failed", module.Reports[1].Status); + Assert.IsTrue(module.Reports[1].IsCompleted, "Failed report should be marked completed"); + } + [Test] public async Task CompletedCarriesFailureMessageAsync() { @@ -237,6 +251,83 @@ namespace Duplicati.UnitTest Assert.IsTrue(module.LoadAsDefault, "The module should load by default"); } + [Test] + public async Task ReportIncludesEnvironmentMetadataAsync() + { + using var module = CreateConfigured(); + + // The remote URL is captured via the IGenericCallbackModule.OnStart hook, mirroring + // ReportHelper. Drive it to populate the backup id and destination type. + var remoteUrl = "s3://mybucket/path"; + var localPaths = new[] { "/data/source" }; + module.OnStart("Backup", ref remoteUrl, ref localPaths); + await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None); + + var report = module.Reports[0]; + Assert.IsNotNull(report.Metadata, "Report should include environment metadata"); + var metadata = report.Metadata!; + Assert.IsFalse(string.IsNullOrEmpty(metadata.DuplicatiVersion), "DuplicatiVersion should be populated"); + Assert.IsFalse(string.IsNullOrEmpty(metadata.MachineId), "MachineId should be populated"); + Assert.IsFalse(string.IsNullOrEmpty(metadata.BackupId), "BackupId should be populated from the remote URL"); + Assert.IsFalse(string.IsNullOrEmpty(metadata.BackupName), "BackupName should be populated"); + Assert.IsFalse(string.IsNullOrEmpty(metadata.MachineName), "MachineName should be populated"); + Assert.AreEqual("s3", metadata.DestinationType, "DestinationType should be the remote URL scheme"); + Assert.IsFalse(string.IsNullOrEmpty(metadata.OperatingSystem), "OperatingSystem should be populated"); + Assert.IsFalse(string.IsNullOrEmpty(metadata.OperatingSystemDetailed), "OperatingSystemDetailed should be populated"); + Assert.IsFalse(string.IsNullOrEmpty(metadata.InstallationType), "InstallationType should be populated"); + + // The backup id is a stable hex string derived from the remote URL. + Assert.That(metadata.BackupId, Does.Match("^[0-9a-fA-F]+$")); + + // An s3 URL against a known public-cloud host yields a safe host suffix; an + // unknown host yields null. The helper is shared with ReportHelper. + Assert.IsNull(metadata.DestinationHostSuffix, "DestinationHostSuffix should be null for an unknown host"); + } + + [Test] + public async Task DestinationHostSuffixKnownCloudAsync() + { + using var module = CreateConfigured(); + + // An s3 URL against a recognized public-cloud host suffix yields that suffix. + var remoteUrl = "s3://mybucket.s3.amazonaws.com/path"; + var localPaths = new[] { "/data/source" }; + module.OnStart("Backup", ref remoteUrl, ref localPaths); + await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None); + + var metadata = module.Reports[0].Metadata!; + Assert.AreEqual(".amazonaws.com", metadata.DestinationHostSuffix, + "DestinationHostSuffix should be the matched safe cloud suffix"); + } + + [Test] + public async Task MetadataHonorsOptionOverridesAsync() + { + // MachineId, BackupId, BackupName and MachineName honor explicit option values + // before falling back to the computed defaults, mirroring ReportHelper. + var module = new CapturingHttpReportStatus(); + module.Configure(new Dictionary + { + ["http-report-status-url"] = "http://localhost/example", + ["http-report-status-interval"] = "1s", + ["machine-id"] = "custom-machine-id", + ["backup-id"] = "custom-backup-id", + ["backup-name"] = "custom-backup-name", + ["machine-name"] = "custom-machine-name", + }); + + var remoteUrl = "s3://mybucket/path"; + var localPaths = new[] { "/data/source" }; + module.OnStart("Backup", ref remoteUrl, ref localPaths); + await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None); + + var metadata = module.Reports[0].Metadata!; + Assert.AreEqual("custom-machine-id", metadata.MachineId, "MachineId should use the option override"); + Assert.AreEqual("custom-backup-id", metadata.BackupId, "BackupId should use the option override"); + Assert.AreEqual("custom-backup-name", metadata.BackupName, "BackupName should use the option override"); + Assert.AreEqual("custom-machine-name", metadata.MachineName, "MachineName should use the option override"); + } + [Test] public async Task PathsInLogLinesAreRedactedByDefaultAsync() {