Added metadata to activity report

This commit is contained in:
Kenneth Skovhede
2026-07-05 17:02:48 +02:00
parent 4faaa182f4
commit db36f5bbae
4 changed files with 335 additions and 90 deletions
@@ -27,6 +27,7 @@ using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.AutoUpdater;
using Duplicati.Library.Interface;
using Duplicati.Library.Utility;
using Uri = System.Uri;
@@ -41,7 +42,7 @@ namespace Duplicati.Library.Modules.Builtin
/// (default 30 seconds), including the latest progress snapshot, the counts of
/// backend events and log entries observed so far, and the last few log lines.
/// </summary>
public class HttpReportStatus : IReportModule, IDisposable
public class HttpReportStatus : IReportModule, IGenericCallbackModule, IDisposable
{
/// <summary>
/// The tag used for logging
@@ -170,11 +171,24 @@ namespace Duplicati.Library.Modules.Builtin
/// </summary>
private bool m_allowPathsInLogMessages;
/// <summary>
/// A read-only copy of the commandline options, used to look up optional overrides
/// for the report metadata (e.g. <c>machine-id</c>, <c>backup-id</c>), mirroring
/// <see cref="ReportHelper"/>.
/// </summary>
private IReadOnlyDictionary<string, string> m_options;
/// <summary>
/// The operation name reported in <see cref="OnOperationStartedAsync"/>.
/// </summary>
private string m_operationName;
/// <summary>
/// The remote backend URL, captured in <see cref="OnStart"/> and used to compute
/// the backup id and destination type, mirroring <see cref="ReportHelper"/>.
/// </summary>
private string m_remoteUrl;
/// <summary>
/// The time the operation started, in UTC.
/// </summary>
@@ -205,6 +219,13 @@ namespace Duplicati.Library.Modules.Builtin
/// </summary>
private DateTime m_nextReportTime;
/// <summary>
/// The lazily-computed environment metadata. The metadata only depends on the
/// options and the remote URL (both fixed before an operation starts), so it is
/// computed once per operation and reused across every report.
/// </summary>
private Lazy<ReportMetadata> m_metadata;
/// <summary>
/// Lock protecting the mutable counters and buffers.
/// </summary>
@@ -234,6 +255,10 @@ namespace Duplicati.Library.Modules.Builtin
return;
}
// Keep a read-only copy of the options so BuildMetadata can honor optional
// overrides (machine-id, backup-id, backup-name, machine-name) like ReportHelper.
m_options = commandlineOptions.AsReadOnly();
m_interval = Utility.Utility.ParseTimespanOption(commandlineOptions.AsReadOnly(), OPTION_INTERVAL, DEFAULT_INTERVAL);
if (m_interval <= TimeSpan.Zero)
m_interval = TimeSpan.FromSeconds(30);
@@ -260,6 +285,30 @@ namespace Duplicati.Library.Modules.Builtin
m_allowPathsInLogMessages = Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_GLOBAL_ALLOW_PATHS_IN_LOG_MESSAGES);
}
/// <summary>
/// Captures the operation name, remote backend URL and local paths. The remote URL
/// is used to compute the backup id and destination type included in each report,
/// mirroring <see cref="ReportHelper.OnStart"/>.
/// </summary>
/// <param name="operationname">The full name of the operation.</param>
/// <param name="remoteurl">The remote backend url.</param>
/// <param name="localpath">The local path, if required.</param>
public void OnStart(string operationname, ref string remoteurl, ref string[] localpath)
{
m_operationName = operationname;
m_remoteUrl = remoteurl;
}
/// <summary>
/// No-op; completion is reported via <see cref="OnOperationCompletedAsync"/> instead.
/// </summary>
/// <param name="result">The result object.</param>
/// <param name="exception">The exception that stopped the operation, or null.</param>
public void OnFinish(IBasicResults result, Exception exception)
{
// Completion is handled by the IReportModule lifecycle; nothing to do here.
}
/// <inheritdoc />
public Task OnOperationStartedAsync(string operationName, IBasicResults result, CancellationToken cancellationToken)
{
@@ -275,6 +324,7 @@ namespace Duplicati.Library.Modules.Builtin
m_latestSnapshot = null;
m_recentLogLines.Clear();
m_nextReportTime = DateTime.UtcNow;
m_metadata = new Lazy<ReportMetadata>(() => BuildMetadata(), isThreadSafe: true);
}
return PostAsync(BuildReport("Started", null), cancellationToken);
@@ -363,6 +413,43 @@ namespace Duplicati.Library.Modules.Builtin
/// report to.</remarks>
public bool IsActive => !string.IsNullOrWhiteSpace(m_url);
/// <summary>
/// Computes the environment metadata included in each report, mirroring the
/// template keys resolved by <see cref="ReportHelper.GetDefaultValue"/>. The
/// remote URL captured in <see cref="OnStart"/> drives the backup id, the
/// destination type and the destination host suffix.
/// </summary>
/// <returns>The metadata for the current report.</returns>
private ReportMetadata BuildMetadata()
=> new ReportMetadata
{
DuplicatiVersion = UpdaterManager.SelfVersion.Version,
MachineId = OptionOrDefault("machine-id", DataFolderManager.MachineID),
BackupId = OptionOrDefault("backup-id", Utility.Utility.CalculateBackupId(m_remoteUrl)),
BackupName = OptionOrDefault("backup-name", System.IO.Path.GetFileNameWithoutExtension(Utility.Utility.getEntryAssembly().Location)),
MachineName = OptionOrDefault("machine-name", DataFolderManager.MachineName),
DestinationType = Utility.Utility.GuessScheme(m_remoteUrl) ?? "file",
DestinationHostSuffix = Utility.Utility.GuessHostSuffixSafe(m_remoteUrl),
InstallationType = UpdaterManager.PackageTypeId,
OperatingSystem = UpdaterManager.OperatingSystemName,
OperatingSystemDetailed = OSInfoHelper.PlatformString,
};
/// <summary>
/// Returns the configured value for the given option key, or the supplied default
/// when the option is absent or empty. Used to honor user-supplied overrides for
/// the report metadata, mirroring <see cref="ReportHelper"/>.
/// </summary>
/// <param name="key">The option key to look up.</param>
/// <param name="defaultValue">The default value to use when the option is not set.</param>
/// <returns>The option value, or the default.</returns>
private string OptionOrDefault(string key, string defaultValue)
{
if (m_options != null && m_options.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
return value;
return defaultValue;
}
/// <summary>
/// Builds the JSON-serializable status report from the current state.
/// </summary>
@@ -380,10 +467,14 @@ namespace Duplicati.Library.Modules.Builtin
StartedUtc = m_operationStarted,
ReportedUtc = DateTime.UtcNow,
ErrorMessage = errorMessage,
IsCompleted = status is "Completed" or "Failed",
BackendEvents = m_backendEvents,
LogEntries = m_logEntries,
Progress = m_latestSnapshot == null ? null : new ProgressSnapshot(m_latestSnapshot),
RecentLogLines = [.. m_recentLogLines],
// Metadata is lazily computed once per operation (see OnOperationStartedAsync)
// since it only depends on the options and remote URL.
Metadata = m_metadata?.Value,
};
}
}
@@ -469,6 +560,9 @@ namespace Duplicati.Library.Modules.Builtin
/// <summary>An error message, if the operation failed.</summary>
public string ErrorMessage { get; set; }
/// <summary><c>true</c> once the operation has finished running (Completed/Failed).</summary>
public bool IsCompleted { get; set; }
/// <summary>The number of backend events observed so far.</summary>
public int BackendEvents { get; set; }
@@ -480,6 +574,46 @@ namespace Duplicati.Library.Modules.Builtin
/// <summary>The most recent log lines observed.</summary>
public List<string> RecentLogLines { get; set; }
/// <summary>Environment metadata included in every report.</summary>
public ReportMetadata Metadata { get; set; }
}
/// <summary>
/// Environment metadata included in every status report, mirroring the template
/// keys resolved by <see cref="ReportHelper.GetDefaultValue"/>.
/// </summary>
public sealed class ReportMetadata
{
/// <summary>The running Duplicati version.</summary>
public string DuplicatiVersion { get; set; }
/// <summary>The stable machine id.</summary>
public string MachineId { get; set; }
/// <summary>A stable id derived from the backup's remote URL.</summary>
public string BackupId { get; set; }
/// <summary>The backup name (entry assembly name).</summary>
public string BackupName { get; set; }
/// <summary>The machine name.</summary>
public string MachineName { get; set; }
/// <summary>The destination type (the remote URL scheme).</summary>
public string DestinationType { get; set; }
/// <summary>A safe destination host suffix (known public cloud only), or null.</summary>
public string DestinationHostSuffix { get; set; }
/// <summary>The installation type (e.g. package type id).</summary>
public string InstallationType { get; set; }
/// <summary>The operating system name (e.g. <c>Windows</c>, <c>Linux</c>, <c>MacOS</c>).</summary>
public string OperatingSystem { get; set; }
/// <summary>A detailed operating system platform string.</summary>
public string OperatingSystemDetailed { get; set; }
}
/// <summary>
@@ -41,11 +41,6 @@ namespace Duplicati.Library.Modules.Builtin
/// </summary>
private static readonly string LOGTAG = Logging.Log.LogTagFromType<ReportHelper>();
/// <summary>
/// The salt used for calculating a backup Id from the remote URL
/// </summary>
private const string SALT = "DUPL";
/// <summary>
/// Name of the option used to specify subject
/// </summary>
@@ -448,7 +443,7 @@ namespace Duplicati.Library.Modules.Builtin
case MACHINE_ID:
return DataFolderManager.MachineID;
case BACKUP_ID:
return Utility.Utility.ByteArrayAsHexString(Utility.Utility.RepeatedHashWithSalt(m_remoteurl, SALT));
return Utility.Utility.CalculateBackupId(m_remoteurl);
case BACKUP_NAME:
return System.IO.Path.GetFileNameWithoutExtension(Utility.Utility.getEntryAssembly().Location);
case MACHINE_NAME:
@@ -464,7 +459,7 @@ namespace Duplicati.Library.Modules.Builtin
var ix = m_remoteurl?.IndexOf("://", StringComparison.OrdinalIgnoreCase) ?? -1;
return Utility.Utility.GuessScheme(m_remoteurl) ?? "file";
case DESTINATION_HOST_SUFFIX:
return GuessHostSuffixSafe(m_remoteurl);
return Utility.Utility.GuessHostSuffixSafe(m_remoteurl);
case UPDATE_CHANNEL:
return UpdaterManager.CurrentChannel.ToString();
default:
@@ -646,87 +641,5 @@ namespace Duplicati.Library.Modules.Builtin
}
}
/// <summary>
/// Regex used to extract the host from a URL
/// </summary>
private static readonly Regex hostSuffixRegex = new Regex(@"^(?:[a-zA-Z][a-zA-Z0-9+\-.]*://)?([^/:]+)", RegexOptions.Compiled);
/// <summary>
/// List of host suffixes that are considered public cloud and safe to report
/// </summary>
private static readonly IEnumerable<string> safeHostSuffixes = new HashSet<string>([
".amazonaws.com",
".impossibleapi.net",
".scw.cloud",
".hosteurope.de",
".dunkel.de",
".dreamhost.com",
".dincloud.com",
".polisystems.ch",
".softlayer.net",
".storadera.com",
".wasabisys.com",
".infomaniak.com",
".infomaniak.cloud",
".sakurastorage.jp",
".lyvecloud.seagate.com",
".digitaloceanspaces.com",
".backblazeb2.com",
".cloudian.com",
".min.io",
".linodeobjects.com",
".bunnycdn.com",
".azure.com",
".googleapis.com",
".ibm.com",
".oracle.com",
".cloudflare.com",
".alibaba.com",
".huawei.com",
".tencent.com",
".baidu.com",
".jd.com",
".ucloud.cn",
".qiniu.com",
".aliyuncs.com",
".tcloud.com",
".tencentcloudapi.com",
".rabata.io",
".internxt.com",
".cloudflarestorage.com",
".storage.selcloud.ru",
".srvstorage.uz",
".srvstorage.kz"
], StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Attempts to guess a safe host suffix from a URL.
/// This attempts to avoid reporting private or sensitive hostnames, and only returns known public cloud suffixes.
/// </summary>
/// <param name="url">The URL to analyze.</param>
/// <returns>The host suffix, or null if not found or not safe to report.</returns>
private static string GuessHostSuffixSafe(string url)
{
var scheme = Utility.Utility.GuessScheme(url);
// For now, only report S3 host suffixes
if (!string.Equals(scheme, "s3", StringComparison.OrdinalIgnoreCase))
return null;
try
{
var hostname = hostSuffixRegex.Match(url).Groups[1].Value;
if (string.IsNullOrEmpty(hostname))
return null;
// Return the first matching safe suffix
return safeHostSuffixes.FirstOrDefault(suffix => hostname.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) ?? null;
}
catch
{
return null;
}
}
}
}
+107
View File
@@ -1385,6 +1385,27 @@ namespace Duplicati.Library.Utility
}
}
/// <summary>
/// The salt used for calculating a backup Id from the remote URL.
/// Shared between the reporting modules so the same id is computed everywhere.
/// </summary>
public const string BACKUP_ID_SALT = "DUPL";
/// <summary>
/// Calculates a stable, opaque backup id from the backup's remote URL. The id is
/// a hex string produced by repeatedly hashing the URL with a fixed salt, so the
/// same remote URL always yields the same id without exposing the URL itself.
/// </summary>
/// <param name="remoteUrl">The remote backend url to compute the id for.</param>
/// <returns>The backup id as a hex string, or an empty string if the url is null/empty.</returns>
public static string CalculateBackupId(string? remoteUrl)
{
if (string.IsNullOrWhiteSpace(remoteUrl))
return string.Empty;
return ByteArrayAsHexString(RepeatedHashWithSalt(remoteUrl, BACKUP_ID_SALT));
}
/// <summary>
/// Gets the drive letter from the given volume guid.
/// This method cannot be inlined since the System.Management types are not implemented in Mono
@@ -1669,6 +1690,92 @@ namespace Duplicati.Library.Utility
return url[..idx];
}
/// <summary>
/// Regex used to extract the host from a URL.
/// </summary>
private static readonly Regex hostSuffixRegex = new Regex(@"^(?:[a-zA-Z][a-zA-Z0-9+\-.]*://)?([^/:]+)", RegexOptions.Compiled);
/// <summary>
/// List of host suffixes that are considered public cloud and safe to report.
/// </summary>
private static readonly IReadOnlySet<string> safeHostSuffixes = new HashSet<string>([
".amazonaws.com",
".impossibleapi.net",
".scw.cloud",
".hosteurope.de",
".dunkel.de",
".dreamhost.com",
".dincloud.com",
".polisystems.ch",
".softlayer.net",
".storadera.com",
".wasabisys.com",
".infomaniak.com",
".infomaniak.cloud",
".sakurastorage.jp",
".lyvecloud.seagate.com",
".digitaloceanspaces.com",
".backblazeb2.com",
".cloudian.com",
".min.io",
".linodeobjects.com",
".bunnycdn.com",
".azure.com",
".googleapis.com",
".ibm.com",
".oracle.com",
".cloudflare.com",
".alibaba.com",
".huawei.com",
".tencent.com",
".baidu.com",
".jd.com",
".ucloud.cn",
".qiniu.com",
".aliyuncs.com",
".tcloud.com",
".tencentcloudapi.com",
".rabata.io",
".internxt.com",
".cloudflarestorage.com",
".storage.selcloud.ru",
".srvstorage.uz",
".srvstorage.kz"
], StringComparer.OrdinalIgnoreCase);
/// <summary>
/// Attempts to guess a safe host suffix from a URL.
/// This attempts to avoid reporting private or sensitive hostnames, and only returns
/// known public cloud suffixes.
/// </summary>
/// <param name="url">The URL to analyze.</param>
/// <returns>The host suffix, or null if not found or not safe to report.</returns>
public static string? GuessHostSuffixSafe(string? url)
{
if (string.IsNullOrWhiteSpace(url))
return null;
var scheme = GuessScheme(url);
// For now, only report S3 host suffixes
if (!string.Equals(scheme, "s3", StringComparison.OrdinalIgnoreCase))
return null;
try
{
var hostname = hostSuffixRegex.Match(url).Groups[1].Value;
if (string.IsNullOrEmpty(hostname))
return null;
// Return the first matching safe suffix
return safeHostSuffixes.FirstOrDefault(suffix => hostname.EndsWith(suffix, StringComparison.OrdinalIgnoreCase));
}
catch
{
return null;
}
}
/// <summary>
/// Returns a url that is safe to display, by removing any credentials
/// </summary>
@@ -130,12 +130,26 @@ namespace Duplicati.UnitTest
// The completed report carries a final progress snapshot at 100%.
Assert.IsNotNull(module.Reports[1].Progress);
Assert.AreEqual(1f, module.Reports[1].Progress!.Progress);
// The Started report is not completed, the Completed report is.
Assert.IsFalse(module.Reports[0].IsCompleted, "Started report should not be completed");
Assert.IsTrue(module.Reports[1].IsCompleted, "Completed report should be marked completed");
// The JSON is serialized with camelCase property names.
Assert.That(module.PostedJsons[0], Does.Contain("\"operation\""), "JSON keys should be camelCase");
Assert.That(module.PostedJsons[0], Does.Contain("\"startedUtc\""));
Assert.That(module.PostedJsons[0], Does.Not.Contain("\"StartedUtc\""));
}
[Test]
public async Task FailedReportIsCompletedAsync()
{
using var module = CreateConfigured();
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
await module.OnOperationCompletedAsync(null!, new InvalidOperationException("boom"), CancellationToken.None);
Assert.AreEqual("Failed", module.Reports[1].Status);
Assert.IsTrue(module.Reports[1].IsCompleted, "Failed report should be marked completed");
}
[Test]
public async Task CompletedCarriesFailureMessageAsync()
{
@@ -237,6 +251,83 @@ namespace Duplicati.UnitTest
Assert.IsTrue(module.LoadAsDefault, "The module should load by default");
}
[Test]
public async Task ReportIncludesEnvironmentMetadataAsync()
{
using var module = CreateConfigured();
// The remote URL is captured via the IGenericCallbackModule.OnStart hook, mirroring
// ReportHelper. Drive it to populate the backup id and destination type.
var remoteUrl = "s3://mybucket/path";
var localPaths = new[] { "/data/source" };
module.OnStart("Backup", ref remoteUrl, ref localPaths);
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
var report = module.Reports[0];
Assert.IsNotNull(report.Metadata, "Report should include environment metadata");
var metadata = report.Metadata!;
Assert.IsFalse(string.IsNullOrEmpty(metadata.DuplicatiVersion), "DuplicatiVersion should be populated");
Assert.IsFalse(string.IsNullOrEmpty(metadata.MachineId), "MachineId should be populated");
Assert.IsFalse(string.IsNullOrEmpty(metadata.BackupId), "BackupId should be populated from the remote URL");
Assert.IsFalse(string.IsNullOrEmpty(metadata.BackupName), "BackupName should be populated");
Assert.IsFalse(string.IsNullOrEmpty(metadata.MachineName), "MachineName should be populated");
Assert.AreEqual("s3", metadata.DestinationType, "DestinationType should be the remote URL scheme");
Assert.IsFalse(string.IsNullOrEmpty(metadata.OperatingSystem), "OperatingSystem should be populated");
Assert.IsFalse(string.IsNullOrEmpty(metadata.OperatingSystemDetailed), "OperatingSystemDetailed should be populated");
Assert.IsFalse(string.IsNullOrEmpty(metadata.InstallationType), "InstallationType should be populated");
// The backup id is a stable hex string derived from the remote URL.
Assert.That(metadata.BackupId, Does.Match("^[0-9a-fA-F]+$"));
// An s3 URL against a known public-cloud host yields a safe host suffix; an
// unknown host yields null. The helper is shared with ReportHelper.
Assert.IsNull(metadata.DestinationHostSuffix, "DestinationHostSuffix should be null for an unknown host");
}
[Test]
public async Task DestinationHostSuffixKnownCloudAsync()
{
using var module = CreateConfigured();
// An s3 URL against a recognized public-cloud host suffix yields that suffix.
var remoteUrl = "s3://mybucket.s3.amazonaws.com/path";
var localPaths = new[] { "/data/source" };
module.OnStart("Backup", ref remoteUrl, ref localPaths);
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
var metadata = module.Reports[0].Metadata!;
Assert.AreEqual(".amazonaws.com", metadata.DestinationHostSuffix,
"DestinationHostSuffix should be the matched safe cloud suffix");
}
[Test]
public async Task MetadataHonorsOptionOverridesAsync()
{
// MachineId, BackupId, BackupName and MachineName honor explicit option values
// before falling back to the computed defaults, mirroring ReportHelper.
var module = new CapturingHttpReportStatus();
module.Configure(new Dictionary<string, string>
{
["http-report-status-url"] = "http://localhost/example",
["http-report-status-interval"] = "1s",
["machine-id"] = "custom-machine-id",
["backup-id"] = "custom-backup-id",
["backup-name"] = "custom-backup-name",
["machine-name"] = "custom-machine-name",
});
var remoteUrl = "s3://mybucket/path";
var localPaths = new[] { "/data/source" };
module.OnStart("Backup", ref remoteUrl, ref localPaths);
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
var metadata = module.Reports[0].Metadata!;
Assert.AreEqual("custom-machine-id", metadata.MachineId, "MachineId should use the option override");
Assert.AreEqual("custom-backup-id", metadata.BackupId, "BackupId should use the option override");
Assert.AreEqual("custom-backup-name", metadata.BackupName, "BackupName should use the option override");
Assert.AreEqual("custom-machine-name", metadata.MachineName, "MachineName should use the option override");
}
[Test]
public async Task PathsInLogLinesAreRedactedByDefaultAsync()
{