Merge branch 'master' into feature/improve-tls-cert-validation
This commit is contained in:
@@ -0,0 +1,156 @@
|
||||
// Copyright (C) 2026, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Duplicati.Library.Interface
|
||||
{
|
||||
/// <summary>
|
||||
/// A snapshot of a single backend event, passed to <see cref="IReportModule.OnBackendEventAsync"/>.
|
||||
/// This mirrors the data flowing through <c>Duplicati.Library.Main.IMessageSink.BackendEvent</c>
|
||||
/// but is declared here so the interface does not depend on <c>Duplicati.Library.Main</c>.
|
||||
/// </summary>
|
||||
/// <param name="Action">The backend action name (e.g. <c>Put</c>, <c>Get</c>, <c>Delete</c>).</param>
|
||||
/// <param name="Type">The event type name (e.g. <c>Started</c>, <c>Completed</c>, <c>Failed</c>).</param>
|
||||
/// <param name="Path">The target path of the backend action.</param>
|
||||
/// <param name="Size">The size of the element being transferred, or <c>0</c> when not applicable.</param>
|
||||
public record ReportBackendEvent(string Action, string Type, string Path, long Size);
|
||||
|
||||
/// <summary>
|
||||
/// A snapshot of a single log entry, passed to <see cref="IReportModule.OnLogEntryAsync"/>.
|
||||
/// This mirrors the data in <c>Duplicati.Library.Logging.LogEntry</c> but is declared here so
|
||||
/// the interface does not depend on <c>Duplicati.Library.Logging</c>.
|
||||
/// </summary>
|
||||
/// <param name="Message">The formatted log message.</param>
|
||||
/// <param name="Level">The log level name (e.g. <c>Information</c>, <c>Warning</c>, <c>Error</c>).</param>
|
||||
/// <param name="Tag">The log tag associated with the entry.</param>
|
||||
/// <param name="Id">The log message id, if any.</param>
|
||||
/// <param name="Timestamp">The time the entry was created, in UTC.</param>
|
||||
/// <param name="Exception">A string representation of an associated exception, or <c>null</c>.</param>
|
||||
public record ReportLogEntry(string Message, string Level, string Tag, string Id, DateTime Timestamp, string Exception);
|
||||
|
||||
/// <summary>
|
||||
/// A snapshot of the operation progress, passed to <see cref="IReportModule.OnProgressTickAsync"/>
|
||||
/// at a regular cadence while an operation is running. The cadence is controlled by the engine,
|
||||
/// not by the module; modules decide how often to act on these snapshots.
|
||||
/// </summary>
|
||||
/// <param name="Phase">The current operation phase name.</param>
|
||||
/// <param name="Progress">The overall progress, in the range <c>[0, 1]</c>.</param>
|
||||
/// <param name="FilesProcessed">The number of files processed so far.</param>
|
||||
/// <param name="FileSizeProcessed">The number of bytes processed so far.</param>
|
||||
/// <param name="FileCount">The total number of files, or <c>0</c> if still counting.</param>
|
||||
/// <param name="FileSize">The total number of bytes, or <c>0</c> if still counting.</param>
|
||||
/// <param name="CountingFiles"><c>true</c> if the file count and size are not yet final.</param>
|
||||
/// <param name="CurrentFilename">The file currently being processed, or <c>null</c>.</param>
|
||||
/// <param name="CurrentFileSize">The size of the file currently being processed.</param>
|
||||
/// <param name="CurrentFileOffset">The byte offset reached in the file currently being processed.</param>
|
||||
/// <param name="ActiveTransfers">The active backend transfers, or an empty array when none.</param>
|
||||
public record ReportProgressSnapshot(
|
||||
string Phase,
|
||||
float Progress,
|
||||
long FilesProcessed,
|
||||
long FileSizeProcessed,
|
||||
long FileCount,
|
||||
long FileSize,
|
||||
bool CountingFiles,
|
||||
string CurrentFilename,
|
||||
long CurrentFileSize,
|
||||
long CurrentFileOffset,
|
||||
ReportBackendEvent[] ActiveTransfers);
|
||||
|
||||
/// <summary>
|
||||
/// Interface for implementing reporting modules that observe an operation's
|
||||
/// lifecycle and progress.
|
||||
///
|
||||
/// Modules implementing this interface are notified when an operation starts and
|
||||
/// completes (<see cref="OnOperationStartedAsync"/> / <see cref="OnOperationCompletedAsync"/>),
|
||||
/// receive every backend event and log entry that flows through the operation's
|
||||
/// message sink (<see cref="OnBackendEventAsync"/> / <see cref="OnLogEntryAsync"/>),
|
||||
/// and are polled with periodic progress snapshots while the operation is running
|
||||
/// (<see cref="OnProgressTickAsync"/>).
|
||||
///
|
||||
/// All callbacks are asynchronous and isolated from one another: an exception
|
||||
/// thrown by one callback is logged and does not abort subsequent callbacks or
|
||||
/// the operation itself.
|
||||
/// </summary>
|
||||
public interface IReportModule : IGenericModule
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the module is configured to do work for the
|
||||
/// current operation.
|
||||
///
|
||||
/// This is checked once after <see cref="IGenericModule.Configure"/> has been
|
||||
/// called. When <see cref="IsActive"/> is <c>false</c>, the engine skips the
|
||||
/// module entirely: no adapter is appended to the message sink, no progress
|
||||
/// ticker is started, and none of the event callbacks are invoked. This keeps
|
||||
/// the hot path free of per-event interception overhead for modules that have
|
||||
/// nothing to report.
|
||||
/// </summary>
|
||||
bool IsActive { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Called when an operation has started, before any work is performed.
|
||||
/// </summary>
|
||||
/// <param name="operationName">The name of the operation (e.g. <c>Backup</c>, <c>Restore</c>).</param>
|
||||
/// <param name="result">The result object that will be populated as the operation runs.</param>
|
||||
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task OnOperationStartedAsync(string operationName, IBasicResults result, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Called when an operation has completed, whether successfully or not.
|
||||
/// </summary>
|
||||
/// <param name="result">The populated result object.</param>
|
||||
/// <param name="exception">The exception that stopped the operation, or <c>null</c> on success.</param>
|
||||
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task OnOperationCompletedAsync(IBasicResults result, Exception exception, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Called for each backend event reported by the operation's message sink.
|
||||
/// </summary>
|
||||
/// <param name="evt">A snapshot of the backend event.</param>
|
||||
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task OnBackendEventAsync(ReportBackendEvent evt, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Called for each log entry written during the operation.
|
||||
/// </summary>
|
||||
/// <param name="entry">A snapshot of the log entry.</param>
|
||||
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task OnLogEntryAsync(ReportLogEntry entry, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Called at a regular cadence while the operation is running, with a snapshot
|
||||
/// of the current operation and backend progress. The cadence is controlled by
|
||||
/// the engine; modules decide how often to act on these snapshots (for example,
|
||||
/// posting an update every 30 seconds).
|
||||
/// </summary>
|
||||
/// <param name="snapshot">A snapshot of the current progress.</param>
|
||||
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task OnProgressTickAsync(ReportProgressSnapshot snapshot, CancellationToken cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -1,27 +1,24 @@
|
||||
// Copyright (C) 2026, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
// Copyright (C) 2026, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.IO;
|
||||
|
||||
namespace Duplicati.Library.Main
|
||||
@@ -48,13 +45,13 @@ namespace Duplicati.Library.Main
|
||||
{
|
||||
if (m_depleted)
|
||||
return 0;
|
||||
|
||||
|
||||
int bytesRead = Duplicati.Library.Utility.Utility.ForceStreamRead(this.m_stream, this.m_buffer, this.m_buffer.Length);
|
||||
m_depleted = this.m_buffer.Length > bytesRead;
|
||||
|
||||
return bytesRead;
|
||||
}
|
||||
|
||||
|
||||
public long Length { get { return m_stream.Length; } }
|
||||
|
||||
public void Dispose()
|
||||
|
||||
@@ -75,6 +75,14 @@ namespace Duplicati.Library.Main
|
||||
/// </summary>
|
||||
private LocaleChange m_localeChange = null;
|
||||
|
||||
/// <summary>
|
||||
/// The handle for the report modules loaded for the current operation. Set in
|
||||
/// <see cref="ReportModuleController.SetupAsync"/> and disposed in
|
||||
/// <see cref="OperationCompleteAsync"/> so the completion callbacks fire before the
|
||||
/// module instances themselves are disposed.
|
||||
/// </summary>
|
||||
private ReportModuleController.ReportModulesHandle ReportModulesHandler;
|
||||
|
||||
/// <summary>
|
||||
/// Callback method invoked when an operation is started
|
||||
/// </summary>
|
||||
@@ -528,8 +536,6 @@ namespace Duplicati.Library.Main
|
||||
using (var logTarget = new ControllerMultiLogTarget(result, Logging.LogMessageType.Information, null, m_options.SuppressWarningsFilter))
|
||||
using (Logging.Log.StartScope(logTarget, null))
|
||||
{
|
||||
logTarget.AddTarget(m_messageSink, m_options.ConsoleLoglevel, m_options.ConsoleLogFilter);
|
||||
result.MessageSink = m_messageSink;
|
||||
using var httpListener = m_options.LogHttpRequests || m_options.LogSocketData >= 0
|
||||
? new NetworkTrafficLogger(m_options.LogHttpRequests, m_options.LogSocketData)
|
||||
: null;
|
||||
@@ -540,6 +546,11 @@ namespace Duplicati.Library.Main
|
||||
m_options.MainAction = result.MainOperation;
|
||||
await ApplySecretProviderAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
(paths, filter) = SetupCommonOptions(result, paths, filter, logTarget);
|
||||
|
||||
ReportModulesHandler = await ReportModuleController.SetupAsync(this, result, m_options, m_currentTaskControl.StopToken);
|
||||
|
||||
logTarget.AddTarget(m_messageSink, m_options.ConsoleLoglevel, m_options.ConsoleLogFilter);
|
||||
result.MessageSink = m_messageSink;
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "StartingOperation", Strings.Controller.StartingOperationMessage(m_options.MainAction));
|
||||
|
||||
using (SlowQueryMonitor.StartMonitoring(m_options.SlowQueryThreshold))
|
||||
@@ -630,7 +641,7 @@ namespace Duplicati.Library.Main
|
||||
}
|
||||
}
|
||||
|
||||
OperationComplete(result, null, filter);
|
||||
await OperationCompleteAsync(result, null, filter).ConfigureAwait(false);
|
||||
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "CompletedOperation", Strings.Controller.CompletedOperationMessage(m_options.MainAction));
|
||||
|
||||
@@ -639,6 +650,7 @@ namespace Duplicati.Library.Main
|
||||
// Handle custom abort exception from run-script
|
||||
catch (OperationAbortException oae)
|
||||
{
|
||||
ReportModulesHandler?.OperationException = null; // Requested aborts are not failures.
|
||||
resultSetter.EndTime = DateTime.UtcNow;
|
||||
// Log this as a normal operation, as the script raising the exception,
|
||||
// has already populated either warning or log messages as required
|
||||
@@ -659,12 +671,13 @@ namespace Duplicati.Library.Main
|
||||
}
|
||||
|
||||
// Do not propagate the cancel exception
|
||||
OperationComplete(result, null, filter);
|
||||
await OperationCompleteAsync(result, null, filter).ConfigureAwait(false);
|
||||
|
||||
return result;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
ReportModulesHandler?.OperationException = ex;
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "FailedOperation", ex, Strings.Controller.FailedOperationMessage(m_options.MainAction));
|
||||
|
||||
try
|
||||
@@ -685,7 +698,7 @@ namespace Duplicati.Library.Main
|
||||
}
|
||||
|
||||
// Report the result, and the failure
|
||||
OperationComplete(result, ex, filter);
|
||||
await OperationCompleteAsync(result, ex, filter).ConfigureAwait(false);
|
||||
|
||||
throw;
|
||||
}
|
||||
@@ -732,14 +745,34 @@ namespace Duplicati.Library.Main
|
||||
public void Dispose() => m_opts?.RawOptions.Remove("filter");
|
||||
}
|
||||
|
||||
private void OperationComplete(IBasicResults result, Exception exception, IFilter filter)
|
||||
private async Task OperationCompleteAsync(IBasicResults result, Exception exception, IFilter filter)
|
||||
{
|
||||
using var _ = PushFilterToOptions(m_options, filter);
|
||||
|
||||
// Fire the report modules' completion callbacks and stop their progress ticker
|
||||
// before the module instances themselves are disposed below. The handle reads
|
||||
// the exception captured in the RunActionAsync catch blocks via the field.
|
||||
try
|
||||
{
|
||||
if (this.ReportModulesHandler != null)
|
||||
{
|
||||
// Wait at most 30s for the report modules to stop
|
||||
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
|
||||
await this.ReportModulesHandler.StopAsync(cts.Token);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, "ReportModuleHandleDisposeError", ex, "Report module handle dispose failed: {0}", ex.Message);
|
||||
}
|
||||
|
||||
this.ReportModulesHandler = default;
|
||||
|
||||
if (m_options?.LoadedModules != null)
|
||||
{
|
||||
foreach (var mx in m_options.LoadedModules)
|
||||
if (mx is IGenericCallbackModule module)
|
||||
try { module.OnFinish(result, exception); }
|
||||
try { RunScriptCallbackWithStatus(result, mx, () => module.OnFinish(result, exception)); }
|
||||
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, $"OnFinishError{mx.Key}", ex, "OnFinish callback {0} failed: {1}", mx.Key, ex.Message); }
|
||||
|
||||
foreach (var mx in m_options.LoadedModules)
|
||||
@@ -820,7 +853,7 @@ namespace Duplicati.Library.Main
|
||||
}
|
||||
|
||||
if (mx is IGenericCallbackModule module)
|
||||
module.OnStart(result.MainOperation.ToString(), ref m_backendUrl, ref paths);
|
||||
RunScriptCallbackWithStatus(result, mx, () => module.OnStart(result.MainOperation.ToString(), ref m_backendUrl, ref paths));
|
||||
}
|
||||
|
||||
// If the filters were changed by a module, read them back in
|
||||
@@ -863,6 +896,28 @@ namespace Duplicati.Library.Main
|
||||
return (paths, filter);
|
||||
}
|
||||
|
||||
internal static void RunScriptCallbackWithStatus(object result, IGenericModule module, Action callback)
|
||||
{
|
||||
if (!string.Equals(module?.Key, "runscript", StringComparison.OrdinalIgnoreCase) || result is not BasicResults basicResults)
|
||||
{
|
||||
callback();
|
||||
return;
|
||||
}
|
||||
|
||||
var progress = basicResults.OperationProgressUpdater;
|
||||
progress.UpdateOverall(out var previousPhase, out _, out _, out _, out _, out _, out _);
|
||||
|
||||
progress.UpdatePhase(OperationPhase.RunScript_Running);
|
||||
try
|
||||
{
|
||||
callback();
|
||||
}
|
||||
finally
|
||||
{
|
||||
progress.UpdatePhase(previousPhase);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void RestoreOptionsFromExistingDatabase(OperationMode operation)
|
||||
{
|
||||
|
||||
@@ -774,7 +774,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
using var _ = Controller.PushFilterToOptions(m_options, filter);
|
||||
foreach (var mx in m_options.LoadedModules)
|
||||
if (mx is IBackupCallbackModule module)
|
||||
try { module.OnFinishBackup(m_result, null); }
|
||||
try { Controller.RunScriptCallbackWithStatus(m_result, mx, () => module.OnFinishBackup(m_result, null)); }
|
||||
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, $"OnFinishBackupError{mx.Key}", ex, "OnFinishBackup callback {0} failed: {1}", mx.Key, ex.Message); }
|
||||
}
|
||||
|
||||
|
||||
@@ -73,7 +73,9 @@ namespace Duplicati.Library.Main
|
||||
Sync_WaitForUpload,
|
||||
Sync_Complete,
|
||||
|
||||
Error
|
||||
Error,
|
||||
|
||||
RunScript_Running
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
// Copyright (C) 2026, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using CoCoL;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Logging;
|
||||
|
||||
namespace Duplicati.Library.Main
|
||||
{
|
||||
/// <summary>
|
||||
/// Bridges an <see cref="IReportModule"/> to the operation's <see cref="IMessageSink"/>.
|
||||
///
|
||||
/// The adapter is appended to the controller's message sink so it receives backend
|
||||
/// events (<see cref="BackendEvent"/>) and log entries (<see cref="WriteMessage"/>),
|
||||
/// and is handed the live <see cref="IBackendProgress"/> and
|
||||
/// <see cref="IOperationProgress"/> objects so a progress ticker can build snapshots.
|
||||
/// Each callback is forwarded to the module and isolated from failures: exceptions
|
||||
/// thrown by the module are logged and do not propagate to the engine.
|
||||
/// </summary>
|
||||
internal sealed class ReportModuleAdapter : IMessageSink
|
||||
{
|
||||
private static readonly string LOGTAG = Log.LogTagFromType<ReportModuleAdapter>();
|
||||
|
||||
private readonly IReportModule m_module;
|
||||
private readonly CancellationToken m_cancellationToken;
|
||||
private IBackendProgress m_backendProgress;
|
||||
private IOperationProgress m_operationProgress;
|
||||
|
||||
/// <summary>
|
||||
/// The module being bridged.
|
||||
/// </summary>
|
||||
public IReportModule Module => m_module;
|
||||
|
||||
/// <summary>
|
||||
/// Constructs a new adapter for the given module.
|
||||
/// </summary>
|
||||
/// <param name="module">The report module to forward events to.</param>
|
||||
/// <param name="cancellationToken">A token used to cancel forwarded callbacks.</param>
|
||||
public ReportModuleAdapter(IReportModule module, CancellationToken cancellationToken)
|
||||
{
|
||||
m_module = module;
|
||||
m_cancellationToken = cancellationToken;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void BackendEvent(BackendActionType action, BackendEventType type, string path, long size)
|
||||
{
|
||||
var evt = new ReportBackendEvent(action.ToString(), type.ToString(), path, size);
|
||||
Forward(() => m_module.OnBackendEventAsync(evt, m_cancellationToken), "OnBackendEventAsync");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetBackendProgress(IBackendProgress progress)
|
||||
=> m_backendProgress = progress;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetOperationProgress(IOperationProgress progress)
|
||||
=> m_operationProgress = progress;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void WriteMessage(LogEntry entry)
|
||||
{
|
||||
if (entry == null)
|
||||
return;
|
||||
|
||||
var snapshot = new ReportLogEntry(
|
||||
entry.AsString(true),
|
||||
entry.Level.ToString(),
|
||||
entry.FilterTag,
|
||||
entry.Id,
|
||||
entry.When,
|
||||
entry.Exception?.ToString());
|
||||
|
||||
Forward(() => m_module.OnLogEntryAsync(snapshot, m_cancellationToken), "OnLogEntryAsync");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds a progress snapshot from the current backend and operation progress
|
||||
/// objects and invokes <see cref="IReportModule.OnProgressTickAsync"/> on the module.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public Task TickAsync()
|
||||
{
|
||||
var snapshot = BuildSnapshot();
|
||||
return ForwardAsync(() => m_module.OnProgressTickAsync(snapshot, m_cancellationToken), "OnProgressTickAsync");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds a snapshot of the current progress, mapping the live progress objects
|
||||
/// (which live in <c>Duplicati.Library.Main</c>) to the interface-level
|
||||
/// <see cref="ReportProgressSnapshot"/>.
|
||||
/// </summary>
|
||||
private ReportProgressSnapshot BuildSnapshot()
|
||||
{
|
||||
OperationPhase phase = default;
|
||||
float progress = 0;
|
||||
long filesProcessed = 0;
|
||||
long fileSizeProcessed = 0;
|
||||
long fileCount = 0;
|
||||
long fileSize = 0;
|
||||
bool countingFiles = false;
|
||||
string currentFilename = null;
|
||||
long currentFileSize = 0;
|
||||
long currentFileOffset = 0;
|
||||
|
||||
if (m_operationProgress != null)
|
||||
{
|
||||
m_operationProgress.UpdateOverall(out phase, out progress, out filesProcessed,
|
||||
out fileSizeProcessed, out fileCount, out fileSize, out countingFiles);
|
||||
m_operationProgress.UpdateFile(out currentFilename, out currentFileSize, out currentFileOffset, out _);
|
||||
}
|
||||
|
||||
ReportBackendEvent[] activeTransfers = System.Array.Empty<ReportBackendEvent>();
|
||||
if (m_backendProgress != null)
|
||||
{
|
||||
var transfers = m_backendProgress.GetActiveTransfers();
|
||||
if (transfers != null && transfers.Length > 0)
|
||||
{
|
||||
activeTransfers = new ReportBackendEvent[transfers.Length];
|
||||
for (int i = 0; i < transfers.Length; i++)
|
||||
{
|
||||
var t = transfers[i];
|
||||
activeTransfers[i] = new ReportBackendEvent(
|
||||
t.Action.ToString(), "Progress", t.Path, t.Size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return new ReportProgressSnapshot(
|
||||
phase.ToString(),
|
||||
progress,
|
||||
filesProcessed,
|
||||
fileSizeProcessed,
|
||||
fileCount,
|
||||
fileSize,
|
||||
countingFiles,
|
||||
currentFilename,
|
||||
currentFileSize,
|
||||
currentFileOffset,
|
||||
activeTransfers);
|
||||
}
|
||||
|
||||
private void Forward(System.Func<Task> action, string label)
|
||||
=> ForwardAsync(action, label).FireAndForget();
|
||||
|
||||
private async Task ForwardAsync(System.Func<Task> action, string label)
|
||||
{
|
||||
try
|
||||
{
|
||||
await action().ConfigureAwait(false);
|
||||
}
|
||||
catch (System.Exception ex)
|
||||
{
|
||||
Log.WriteWarningMessage(LOGTAG, $"ReportModule{label}", ex,
|
||||
"Report module {0} callback {1} failed: {2}", m_module.Key, label, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
// Copyright (C) 2026, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using CoCoL;
|
||||
using Duplicati.Library.Interface;
|
||||
using Microsoft.VisualStudio.Threading;
|
||||
|
||||
namespace Duplicati.Library.Main;
|
||||
|
||||
/// <summary>
|
||||
/// Static helper that wires up <see cref="IReportModule"/> instances for an
|
||||
/// operation, owns the progress ticker, and provides the disposable handle that
|
||||
/// fires the completion callbacks. Extracted from <see cref="Controller"/> to keep
|
||||
/// the report-module plumbing in one place.
|
||||
/// </summary>
|
||||
internal static class ReportModuleController
|
||||
{
|
||||
/// <summary>
|
||||
/// The tag used for logging.
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType<ReportModuleAdapter>();
|
||||
|
||||
/// <summary>
|
||||
/// The interval at which progress snapshots are pushed to report modules.
|
||||
/// Report modules decide for themselves how often to act on these ticks.
|
||||
/// </summary>
|
||||
public static readonly TimeSpan ProgressTickInterval = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>
|
||||
/// Wires up any loaded <see cref="IReportModule"/> instances on the given
|
||||
/// controller so they observe the operation's lifecycle, backend events, log
|
||||
/// entries and progress. Each active module is wrapped in a
|
||||
/// <see cref="ReportModuleAdapter"/> appended to the controller's message sink,
|
||||
/// and a periodic ticker is started that pushes progress snapshots to the modules.
|
||||
/// </summary>
|
||||
/// <param name="controller">The controller running the operation.</param>
|
||||
/// <param name="result">The result object for the operation.</param>
|
||||
/// <param name="options">The options containing the loaded report modules.</param>
|
||||
/// <param name="cancellationToken">The cancellation token.</param>
|
||||
/// <returns>A handle that completes the report modules when disposed.</returns>
|
||||
public static async Task<ReportModulesHandle?> SetupAsync(Controller controller, IBasicResults result, Options options, CancellationToken cancellationToken)
|
||||
{
|
||||
var adapters = new List<ReportModuleAdapter>();
|
||||
var loadedModules = options.LoadedModules;
|
||||
if (loadedModules != null)
|
||||
{
|
||||
foreach (var mx in loadedModules)
|
||||
{
|
||||
if (mx is not IReportModule reportModule || !reportModule.IsActive)
|
||||
continue;
|
||||
|
||||
var adapter = new ReportModuleAdapter(reportModule, cancellationToken);
|
||||
adapters.Add(adapter);
|
||||
controller.AppendSinkAsync(adapter).FireAndForget();
|
||||
|
||||
try
|
||||
{
|
||||
await reportModule.OnOperationStartedAsync(options.MainAction.ToString(), result, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, $"ReportModuleStartError{reportModule.Key}", ex, "Report module {0} OnOperationStartedAsync failed: {1}", reportModule.Key, ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// When no report module is active there is nothing to wire up: avoid the
|
||||
// progress ticker and leave the message sink untouched so the hot path
|
||||
// keeps its original (zero-overhead) behavior.
|
||||
if (adapters.Count == 0)
|
||||
return default;
|
||||
|
||||
var tickerCts = new CancellationTokenSource();
|
||||
var ticker = StartProgressTicker(adapters, tickerCts.Token);
|
||||
|
||||
return new ReportModulesHandle(adapters, result, ticker, tickerCts);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts a background loop that periodically pushes progress snapshots to the
|
||||
/// given report module adapters. The loop stops when the cancellation token is
|
||||
/// cancelled, and the returned task is awaited by the handle on dispose.
|
||||
/// </summary>
|
||||
/// <param name="adapters">The adapters to tick.</param>
|
||||
/// <param name="cancellationToken">The token that stops the loop.</param>
|
||||
/// <returns>A task that completes when the loop stops.</returns>
|
||||
private static Task StartProgressTicker(IReadOnlyList<ReportModuleAdapter> adapters, CancellationToken cancellationToken)
|
||||
{
|
||||
if (adapters.Count == 0)
|
||||
return Task.CompletedTask;
|
||||
|
||||
return Task.Run(async () =>
|
||||
{
|
||||
try
|
||||
{
|
||||
using var timer = new PeriodicTimer(ProgressTickInterval);
|
||||
while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var adapter in adapters)
|
||||
await adapter.TickAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) { /* expected when the operation ends */ }
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, "ReportModuleTickerError", ex, "Report module progress ticker failed: {0}", ex.Message);
|
||||
}
|
||||
}, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Handle that completes report modules when disposed: it stops the progress
|
||||
/// ticker and fires <see cref="IReportModule.OnOperationCompletedAsync"/> on each
|
||||
/// module. It is disposed in <see cref="Controller.OperationCompleteAsync"/> before
|
||||
/// the module instances themselves are disposed.
|
||||
/// </summary>
|
||||
public class ReportModulesHandle : IDisposable
|
||||
{
|
||||
private readonly List<ReportModuleAdapter> m_adapters;
|
||||
private readonly IBasicResults m_result;
|
||||
private readonly Task m_ticker;
|
||||
private readonly CancellationTokenSource m_tickerCts;
|
||||
|
||||
public ReportModulesHandle(List<ReportModuleAdapter> adapters, IBasicResults result, Task ticker, CancellationTokenSource tickerCts)
|
||||
{
|
||||
m_adapters = adapters;
|
||||
m_result = result;
|
||||
m_ticker = ticker;
|
||||
m_tickerCts = tickerCts;
|
||||
}
|
||||
|
||||
public Exception? OperationException { get; set; }
|
||||
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
// Stop the ticker first so no more progress ticks race the completion callbacks.
|
||||
try { m_tickerCts?.Cancel(); } catch { }
|
||||
|
||||
if (m_ticker != null)
|
||||
try { await m_ticker.WithCancellation(cancellationToken); } catch { }
|
||||
|
||||
if (m_adapters != null)
|
||||
{
|
||||
// Signal all modules to stop processing
|
||||
var tasks = new List<Task>();
|
||||
foreach (var adapter in m_adapters)
|
||||
{
|
||||
try
|
||||
{
|
||||
tasks.Add(adapter.Module.OnOperationCompletedAsync(m_result, OperationException, cancellationToken));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, $"ReportModuleCompleteError{adapter.Module.Key}", ex, "Report module {0} OnOperationCompletedAsync failed: {1}", adapter.Module.Key, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
if (tasks.Count > 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(tasks)
|
||||
.WithCancellation(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, "ReportModuleCompletionError", ex, "Report module completion callbacks failed: {0}", ex.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m_tickerCts?.Dispose();
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try { m_tickerCts?.Cancel(); } catch { }
|
||||
m_tickerCts?.Dispose();
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,6 +38,7 @@ public static class GenericModules
|
||||
new HyperVOptions(),
|
||||
new MSSQLOptions(),
|
||||
new RunScript(),
|
||||
new HttpReportStatus(),
|
||||
new SendHttpMessage(),
|
||||
new SendJabberMessage(),
|
||||
new SendTelegramMessage(),
|
||||
|
||||
@@ -0,0 +1,692 @@
|
||||
// Copyright (C) 2026, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.AutoUpdater;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Utility;
|
||||
using Uri = System.Uri;
|
||||
|
||||
namespace Duplicati.Library.Modules.Builtin
|
||||
{
|
||||
/// <summary>
|
||||
/// A reporting module that periodically posts the current operation status to a
|
||||
/// remote URL as JSON. While an operation is running the engine pushes progress
|
||||
/// snapshots at a fixed cadence (see <see cref="IReportModule.OnProgressTickAsync"/>);
|
||||
/// this module posts an update at most once every <c>--http-report-status-interval</c>
|
||||
/// (default 30 seconds), including the latest progress snapshot, the counts of
|
||||
/// backend events and log entries observed so far, and the last few log lines.
|
||||
/// </summary>
|
||||
public class HttpReportStatus : IReportModule, IGenericCallbackModule, IDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// The tag used for logging
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType<HttpReportStatus>();
|
||||
|
||||
#region Option names
|
||||
|
||||
/// <summary>
|
||||
/// Option used to specify the remote URL to post status reports to.
|
||||
/// </summary>
|
||||
private const string OPTION_URL = "http-report-status-url";
|
||||
|
||||
/// <summary>
|
||||
/// Option used to specify the interval between status reports.
|
||||
/// </summary>
|
||||
private const string OPTION_INTERVAL = "http-report-status-interval";
|
||||
|
||||
/// <summary>
|
||||
/// Option used to specify the maximum number of recent log lines included in each report.
|
||||
/// </summary>
|
||||
private const string OPTION_MAX_LOG_LINES = "http-report-status-max-log-lines";
|
||||
|
||||
/// <summary>
|
||||
/// Option used to accept any SSL certificate.
|
||||
/// </summary>
|
||||
private const string OPTION_ACCEPT_ANY_CERTIFICATE = "http-report-status-accept-any-ssl-certificate";
|
||||
|
||||
/// <summary>
|
||||
/// Option used to accept a specific SSL certificate hash.
|
||||
/// </summary>
|
||||
private const string OPTION_ACCEPT_SPECIFIED_CERTIFICATE = "http-report-status-accept-specified-ssl-hash";
|
||||
|
||||
/// <summary>
|
||||
/// Option used to ignore certificate revocation check failures.
|
||||
/// </summary>
|
||||
private const string OPTION_IGNORE_REVOCATION_FAILURE = "http-report-status-ignore-revocation-failure";
|
||||
|
||||
/// <summary>
|
||||
/// The module-specific option used to disable path redaction in the buffered log
|
||||
/// lines. Mirrors the global <c>allow-paths-in-log-messages</c> option used by the
|
||||
/// reporting helpers.
|
||||
/// </summary>
|
||||
private const string OPTION_ALLOW_PATHS_IN_LOG_MESSAGES = "http-report-status-allow-paths-in-log-messages";
|
||||
|
||||
/// <summary>
|
||||
/// The global option used to disable path redaction in log messages, mirrored
|
||||
/// from <c>Options.cs</c> / <c>ReportHelper</c>. The module-specific option takes
|
||||
/// precedence, falling back to this global setting when not set.
|
||||
/// </summary>
|
||||
private const string OPTION_GLOBAL_ALLOW_PATHS_IN_LOG_MESSAGES = "allow-paths-in-log-messages";
|
||||
|
||||
#endregion
|
||||
|
||||
#region Defaults
|
||||
|
||||
/// <summary>
|
||||
/// The default interval between status reports.
|
||||
/// </summary>
|
||||
private const string DEFAULT_INTERVAL = "30s";
|
||||
|
||||
/// <summary>
|
||||
/// The default maximum number of recent log lines to include.
|
||||
/// </summary>
|
||||
private const int DEFAULT_MAX_LOG_LINES = 20;
|
||||
|
||||
#endregion
|
||||
|
||||
/// <summary>
|
||||
/// The module key, used to activate or deactivate the module on the commandline.
|
||||
/// </summary>
|
||||
public string Key => "httpreportstatus";
|
||||
|
||||
/// <summary>
|
||||
/// A localized string describing the module with a friendly name.
|
||||
/// </summary>
|
||||
public string DisplayName => Strings.HttpReportStatus.DisplayName;
|
||||
|
||||
/// <summary>
|
||||
/// A localized description of the module.
|
||||
/// </summary>
|
||||
public string Description => Strings.HttpReportStatus.Description;
|
||||
|
||||
/// <summary>
|
||||
/// The module is loaded, but inactive unless a url has been set
|
||||
/// </summary>
|
||||
public bool LoadAsDefault => true;
|
||||
|
||||
/// <summary>
|
||||
/// Gets a list of supported commandline arguments.
|
||||
/// </summary>
|
||||
public IList<ICommandLineArgument> SupportedCommands =>
|
||||
[
|
||||
new CommandLineArgument(OPTION_URL, CommandLineArgument.ArgumentType.String, Strings.HttpReportStatus.UrlShort, Strings.HttpReportStatus.UrlLong),
|
||||
new CommandLineArgument(OPTION_INTERVAL, CommandLineArgument.ArgumentType.Timespan, Strings.HttpReportStatus.IntervalShort, Strings.HttpReportStatus.IntervalLong, DEFAULT_INTERVAL),
|
||||
new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.HttpReportStatus.MaxLogLinesShort, Strings.HttpReportStatus.MaxLogLinesLong, DEFAULT_MAX_LOG_LINES.ToString()),
|
||||
new CommandLineArgument(OPTION_ACCEPT_ANY_CERTIFICATE, CommandLineArgument.ArgumentType.Boolean, Strings.HttpReportStatus.AcceptAnyCertificateShort, Strings.HttpReportStatus.AcceptAnyCertificateLong),
|
||||
new CommandLineArgument(OPTION_ACCEPT_SPECIFIED_CERTIFICATE, CommandLineArgument.ArgumentType.String, Strings.HttpReportStatus.AcceptSpecifiedCertificateShort, Strings.HttpReportStatus.AcceptSpecifiedCertificateLong),
|
||||
new CommandLineArgument(OPTION_IGNORE_REVOCATION_FAILURE, CommandLineArgument.ArgumentType.Boolean, Strings.HttpReportStatus.IgnoreRevocationFailureShort, Strings.HttpReportStatus.IgnoreRevocationFailureLong, "false"),
|
||||
new CommandLineArgument(OPTION_ALLOW_PATHS_IN_LOG_MESSAGES, CommandLineArgument.ArgumentType.Boolean, Strings.HttpReportStatus.AllowPathsInLogMessagesShort, Strings.HttpReportStatus.AllowPathsInLogMessagesLong, "false"),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// The remote URLs to post status reports to.
|
||||
/// </summary>
|
||||
private string[] m_urls;
|
||||
|
||||
/// <summary>
|
||||
/// The HTTP handler created during <see cref="Configure"/>, owned by
|
||||
/// <see cref="m_httpClient"/> and disposed with it when the module is disposed.
|
||||
/// </summary>
|
||||
private HttpClientHandler m_httpHandler;
|
||||
|
||||
/// <summary>
|
||||
/// The HTTP client created during <see cref="Configure"/>, reused for every post
|
||||
/// and disposed when the module is disposed.
|
||||
/// </summary>
|
||||
private HttpClient m_httpClient;
|
||||
|
||||
/// <summary>
|
||||
/// The interval between status reports.
|
||||
/// </summary>
|
||||
private TimeSpan m_interval;
|
||||
|
||||
/// <summary>
|
||||
/// The maximum number of recent log lines to include in each report.
|
||||
/// </summary>
|
||||
private int m_maxLogLines;
|
||||
|
||||
/// <summary>
|
||||
/// True if paths are allowed in the buffered log lines (i.e. not redacted).
|
||||
/// </summary>
|
||||
private bool m_allowPathsInLogMessages;
|
||||
|
||||
/// <summary>
|
||||
/// A read-only copy of the commandline options, used to look up optional overrides
|
||||
/// for the report metadata (e.g. <c>machine-id</c>, <c>backup-id</c>), mirroring
|
||||
/// <see cref="ReportHelper"/>.
|
||||
/// </summary>
|
||||
private IReadOnlyDictionary<string, string> m_options;
|
||||
|
||||
/// <summary>
|
||||
/// The operation name reported in <see cref="OnOperationStartedAsync"/>.
|
||||
/// </summary>
|
||||
private string m_operationName;
|
||||
|
||||
/// <summary>
|
||||
/// The remote backend URL, captured in <see cref="OnStart"/> and used to compute
|
||||
/// the backup id and destination type, mirroring <see cref="ReportHelper"/>.
|
||||
/// </summary>
|
||||
private string m_remoteUrl;
|
||||
|
||||
/// <summary>
|
||||
/// The time the operation started, in UTC.
|
||||
/// </summary>
|
||||
private DateTime m_operationStarted;
|
||||
|
||||
/// <summary>
|
||||
/// The number of backend events observed so far.
|
||||
/// </summary>
|
||||
private int m_backendEvents;
|
||||
|
||||
/// <summary>
|
||||
/// The number of log entries observed so far.
|
||||
/// </summary>
|
||||
private int m_logEntries;
|
||||
|
||||
/// <summary>
|
||||
/// The most recent progress snapshot observed.
|
||||
/// </summary>
|
||||
private ReportProgressSnapshot m_latestSnapshot;
|
||||
|
||||
/// <summary>
|
||||
/// A ring buffer of the most recent log lines observed.
|
||||
/// </summary>
|
||||
private readonly LinkedList<string> m_recentLogLines = new();
|
||||
|
||||
/// <summary>
|
||||
/// The time the next status report may be sent.
|
||||
/// </summary>
|
||||
private DateTime m_nextReportTime;
|
||||
|
||||
/// <summary>
|
||||
/// The lazily-computed environment metadata. The metadata only depends on the
|
||||
/// options and the remote URL (both fixed before an operation starts), so it is
|
||||
/// computed once per operation and reused across every report.
|
||||
/// </summary>
|
||||
private Lazy<ReportMetadata> m_metadata;
|
||||
|
||||
/// <summary>
|
||||
/// Lock protecting the mutable counters and buffers.
|
||||
/// </summary>
|
||||
private readonly object m_lock = new();
|
||||
|
||||
/// <summary>
|
||||
/// JSON serializer options that emit property names in camelCase.
|
||||
/// </summary>
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Configures the module from the commandline options. The module is only
|
||||
/// active when <c>--http-report-status-url</c> is set.
|
||||
/// </summary>
|
||||
/// <param name="commandlineOptions">A set of commandline options passed to Duplicati.</param>
|
||||
public void Configure(IDictionary<string, string> commandlineOptions)
|
||||
{
|
||||
commandlineOptions.TryGetValue(OPTION_URL, out var url);
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
{
|
||||
// Without a URL there is nothing to report to; leave the module inactive.
|
||||
m_urls = [];
|
||||
}
|
||||
else
|
||||
{
|
||||
m_urls = url
|
||||
.Split([',', ';'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.Distinct()
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
if (!IsActive)
|
||||
return;
|
||||
|
||||
// Keep a read-only copy of the options so BuildMetadata can honor optional
|
||||
// overrides (machine-id, backup-id, backup-name, machine-name) like ReportHelper.
|
||||
m_options = commandlineOptions.AsReadOnly();
|
||||
|
||||
m_interval = Utility.Utility.ParseTimespanOption(commandlineOptions.AsReadOnly(), OPTION_INTERVAL, DEFAULT_INTERVAL);
|
||||
if (m_interval <= TimeSpan.Zero)
|
||||
m_interval = TimeSpan.FromSeconds(30);
|
||||
|
||||
m_maxLogLines = Utility.Utility.ParseIntOption(commandlineOptions.AsReadOnly(), OPTION_MAX_LOG_LINES, DEFAULT_MAX_LOG_LINES);
|
||||
if (m_maxLogLines < 0)
|
||||
m_maxLogLines = DEFAULT_MAX_LOG_LINES;
|
||||
|
||||
var acceptAnyCertificate = Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_ACCEPT_ANY_CERTIFICATE);
|
||||
var acceptSpecificCertificates = commandlineOptions.ContainsKey(OPTION_ACCEPT_SPECIFIED_CERTIFICATE)
|
||||
? commandlineOptions[OPTION_ACCEPT_SPECIFIED_CERTIFICATE].Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)
|
||||
: null;
|
||||
var ignoreRevocationFailure = Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_IGNORE_REVOCATION_FAILURE);
|
||||
|
||||
m_httpHandler = new HttpClientHandler();
|
||||
HttpClientHelper.ConfigureHandlerCertificateValidator(m_httpHandler, acceptAnyCertificate, acceptSpecificCertificates, ignoreRevocationFailure);
|
||||
m_httpClient = new HttpClient(m_httpHandler);
|
||||
|
||||
// Paths in log lines are redacted by default. The module-specific option takes
|
||||
// precedence and falls back to the global allow-paths-in-log-messages setting,
|
||||
// mirroring the behavior of the other reporting modules (see ReportHelper).
|
||||
if (commandlineOptions.TryGetValue(OPTION_ALLOW_PATHS_IN_LOG_MESSAGES, out var allowPathsModule) && bool.TryParse(allowPathsModule, out var parsedModule))
|
||||
m_allowPathsInLogMessages = parsedModule;
|
||||
else
|
||||
m_allowPathsInLogMessages = Utility.Utility.ParseBoolOption(commandlineOptions.AsReadOnly(), OPTION_GLOBAL_ALLOW_PATHS_IN_LOG_MESSAGES);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Captures the operation name, remote backend URL and local paths. The remote URL
|
||||
/// is used to compute the backup id and destination type included in each report,
|
||||
/// mirroring <see cref="ReportHelper.OnStart"/>.
|
||||
/// </summary>
|
||||
/// <param name="operationname">The full name of the operation.</param>
|
||||
/// <param name="remoteurl">The remote backend url.</param>
|
||||
/// <param name="localpath">The local path, if required.</param>
|
||||
public void OnStart(string operationname, ref string remoteurl, ref string[] localpath)
|
||||
{
|
||||
m_operationName = operationname;
|
||||
m_remoteUrl = remoteurl;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// No-op; completion is reported via <see cref="OnOperationCompletedAsync"/> instead.
|
||||
/// </summary>
|
||||
/// <param name="result">The result object.</param>
|
||||
/// <param name="exception">The exception that stopped the operation, or null.</param>
|
||||
public void OnFinish(IBasicResults result, Exception exception)
|
||||
{
|
||||
// Completion is handled by the IReportModule lifecycle; nothing to do here.
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task OnOperationStartedAsync(string operationName, IBasicResults result, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!IsActive)
|
||||
return Task.CompletedTask;
|
||||
|
||||
lock (m_lock)
|
||||
{
|
||||
m_operationName = operationName ?? "Operation";
|
||||
m_operationStarted = DateTime.UtcNow;
|
||||
m_backendEvents = 0;
|
||||
m_logEntries = 0;
|
||||
m_latestSnapshot = null;
|
||||
m_recentLogLines.Clear();
|
||||
m_nextReportTime = DateTime.UtcNow;
|
||||
m_metadata = new Lazy<ReportMetadata>(() => BuildMetadata(), isThreadSafe: true);
|
||||
}
|
||||
|
||||
return PostAsync(BuildReport("Started", null), cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task OnOperationCompletedAsync(IBasicResults result, Exception exception, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!IsActive)
|
||||
return;
|
||||
|
||||
// Refresh the snapshot with the final result so the completed report is accurate.
|
||||
lock (m_lock)
|
||||
{
|
||||
m_latestSnapshot = new ReportProgressSnapshot(
|
||||
"Complete",
|
||||
1f,
|
||||
0, 0, 0, 0, false,
|
||||
null, 0, 0,
|
||||
Array.Empty<ReportBackendEvent>());
|
||||
}
|
||||
|
||||
var report = BuildReport(exception == null ? "Completed" : "Failed", exception?.Message);
|
||||
await PostAsync(report, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task OnBackendEventAsync(ReportBackendEvent evt, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!IsActive)
|
||||
return Task.CompletedTask;
|
||||
|
||||
lock (m_lock)
|
||||
m_backendEvents++;
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task OnLogEntryAsync(ReportLogEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!IsActive || entry == null)
|
||||
return Task.CompletedTask;
|
||||
|
||||
lock (m_lock)
|
||||
{
|
||||
m_logEntries++;
|
||||
// Redact paths in the buffered log line unless the user has explicitly
|
||||
// allowed paths in log messages, mirroring the reporting helpers.
|
||||
var line = m_allowPathsInLogMessages ? entry.Message : SensitiveDataFilter.RedactPaths(entry.Message);
|
||||
m_recentLogLines.AddLast(line);
|
||||
while (m_recentLogLines.Count > m_maxLogLines)
|
||||
m_recentLogLines.RemoveFirst();
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task OnProgressTickAsync(ReportProgressSnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!IsActive)
|
||||
return Task.CompletedTask;
|
||||
|
||||
bool shouldReport;
|
||||
lock (m_lock)
|
||||
{
|
||||
m_latestSnapshot = snapshot;
|
||||
shouldReport = DateTime.UtcNow >= m_nextReportTime;
|
||||
if (shouldReport)
|
||||
m_nextReportTime = DateTime.UtcNow + m_interval;
|
||||
}
|
||||
|
||||
if (!shouldReport)
|
||||
return Task.CompletedTask;
|
||||
|
||||
return PostAsync(BuildReport("Progress", null), cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets a value indicating whether the module is configured with a target URL.
|
||||
/// </summary>
|
||||
/// <inheritdoc />
|
||||
/// <remarks>The module is only active when a target URL has been configured,
|
||||
/// so that no per-event interception happens unless there is somewhere to
|
||||
/// report to.</remarks>
|
||||
public bool IsActive => m_urls != null && m_urls.Length > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Computes the environment metadata included in each report, mirroring the
|
||||
/// template keys resolved by <see cref="ReportHelper.GetDefaultValue"/>. The
|
||||
/// remote URL captured in <see cref="OnStart"/> drives the backup id, the
|
||||
/// destination type and the destination host suffix.
|
||||
/// </summary>
|
||||
/// <returns>The metadata for the current report.</returns>
|
||||
private ReportMetadata BuildMetadata()
|
||||
=> new ReportMetadata
|
||||
{
|
||||
DuplicatiVersion = UpdaterManager.SelfVersion.Version,
|
||||
MachineId = OptionOrDefault("machine-id", DataFolderManager.MachineID),
|
||||
BackupId = OptionOrDefault("backup-id", Utility.Utility.CalculateBackupId(m_remoteUrl)),
|
||||
BackupName = OptionOrDefault("backup-name", System.IO.Path.GetFileNameWithoutExtension(Utility.Utility.getEntryAssembly().Location)),
|
||||
MachineName = OptionOrDefault("machine-name", DataFolderManager.MachineName),
|
||||
DestinationType = Utility.Utility.GuessScheme(m_remoteUrl) ?? "file",
|
||||
DestinationHostSuffix = Utility.Utility.GuessHostSuffixSafe(m_remoteUrl),
|
||||
InstallationType = UpdaterManager.PackageTypeId,
|
||||
OperatingSystem = UpdaterManager.OperatingSystemName,
|
||||
OperatingSystemDetailed = OSInfoHelper.PlatformString,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Returns the configured value for the given option key, or the supplied default
|
||||
/// when the option is absent or empty. Used to honor user-supplied overrides for
|
||||
/// the report metadata, mirroring <see cref="ReportHelper"/>.
|
||||
/// </summary>
|
||||
/// <param name="key">The option key to look up.</param>
|
||||
/// <param name="defaultValue">The default value to use when the option is not set.</param>
|
||||
/// <returns>The option value, or the default.</returns>
|
||||
private string OptionOrDefault(string key, string defaultValue)
|
||||
{
|
||||
if (m_options != null && m_options.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
|
||||
return value;
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the JSON-serializable status report from the current state.
|
||||
/// </summary>
|
||||
/// <param name="status">The status label for this report (e.g. <c>Started</c>, <c>Progress</c>, <c>Completed</c>).</param>
|
||||
/// <param name="errorMessage">An error message to include, or <c>null</c>.</param>
|
||||
/// <returns>A status report ready to be serialized and posted.</returns>
|
||||
private StatusReport BuildReport(string status, string errorMessage)
|
||||
{
|
||||
lock (m_lock)
|
||||
{
|
||||
return new StatusReport
|
||||
{
|
||||
Operation = m_operationName,
|
||||
Status = status,
|
||||
StartedUtc = m_operationStarted,
|
||||
ReportedUtc = DateTime.UtcNow,
|
||||
ErrorMessage = errorMessage,
|
||||
IsCompleted = status is "Completed" or "Failed",
|
||||
BackendEvents = m_backendEvents,
|
||||
LogEntries = m_logEntries,
|
||||
Progress = m_latestSnapshot == null ? null : new ProgressSnapshot(m_latestSnapshot),
|
||||
RecentLogLines = [.. m_recentLogLines],
|
||||
// Metadata is lazily computed once per operation (see OnOperationStartedAsync)
|
||||
// since it only depends on the options and remote URL.
|
||||
Metadata = m_metadata?.Value,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Posts the given status report to the configured URL as JSON. Failures are
|
||||
/// logged but never propagated to the caller.
|
||||
/// </summary>
|
||||
/// <param name="report">The report to post.</param>
|
||||
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
private async Task PostAsync(StatusReport report, CancellationToken cancellationToken)
|
||||
{
|
||||
if (report == null)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
var json = JsonSerializer.Serialize(report, typeof(StatusReport), SerializerOptions);
|
||||
foreach (var url in m_urls)
|
||||
await SendAsync(url, json, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, "HttpReportStatusSendError", ex, "Failed to post status report to {0}: {1}", string.Join(", ", m_urls), ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends the JSON body to the configured URL. This is virtual so tests can
|
||||
/// intercept the request without performing real network I/O.
|
||||
/// </summary>
|
||||
/// <param name="url">The url to POST to.</param>
|
||||
/// <param name="json">The JSON body to send.</param>
|
||||
/// <param name="cancellationToken">A token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
protected virtual async Task SendAsync(string url, string json, CancellationToken cancellationToken)
|
||||
{
|
||||
using var content = new StringContent(json, Encoding.UTF8, "application/json");
|
||||
using var response = await m_httpClient.PostAsync(new Uri(url), content, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes the handler created during <see cref="Configure"/>.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
Dispose(true);
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes managed and unmanaged resources.
|
||||
/// </summary>
|
||||
/// <param name="disposing"><c>true</c> when disposing managed resources.</param>
|
||||
protected virtual void Dispose(bool disposing)
|
||||
{
|
||||
if (!disposing)
|
||||
return;
|
||||
|
||||
try { m_httpClient?.Dispose(); }
|
||||
catch { }
|
||||
m_httpClient = null;
|
||||
m_httpHandler = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The JSON status report posted to the remote URL.
|
||||
/// </summary>
|
||||
public sealed class StatusReport
|
||||
{
|
||||
/// <summary>The operation name.</summary>
|
||||
public string Operation { get; set; }
|
||||
|
||||
/// <summary>The status label for this report.</summary>
|
||||
public string Status { get; set; }
|
||||
|
||||
/// <summary>The time the operation started, in UTC.</summary>
|
||||
public DateTime StartedUtc { get; set; }
|
||||
|
||||
/// <summary>The time this report was generated, in UTC.</summary>
|
||||
public DateTime ReportedUtc { get; set; }
|
||||
|
||||
/// <summary>An error message, if the operation failed.</summary>
|
||||
public string ErrorMessage { get; set; }
|
||||
|
||||
/// <summary><c>true</c> once the operation has finished running (Completed/Failed).</summary>
|
||||
public bool IsCompleted { get; set; }
|
||||
|
||||
/// <summary>The number of backend events observed so far.</summary>
|
||||
public int BackendEvents { get; set; }
|
||||
|
||||
/// <summary>The number of log entries observed so far.</summary>
|
||||
public int LogEntries { get; set; }
|
||||
|
||||
/// <summary>The latest progress snapshot, or <c>null</c>.</summary>
|
||||
public ProgressSnapshot Progress { get; set; }
|
||||
|
||||
/// <summary>The most recent log lines observed.</summary>
|
||||
public List<string> RecentLogLines { get; set; }
|
||||
|
||||
/// <summary>Environment metadata included in every report.</summary>
|
||||
public ReportMetadata Metadata { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Environment metadata included in every status report, mirroring the template
|
||||
/// keys resolved by <see cref="ReportHelper.GetDefaultValue"/>.
|
||||
/// </summary>
|
||||
public sealed class ReportMetadata
|
||||
{
|
||||
/// <summary>The running Duplicati version.</summary>
|
||||
public string DuplicatiVersion { get; set; }
|
||||
|
||||
/// <summary>The stable machine id.</summary>
|
||||
public string MachineId { get; set; }
|
||||
|
||||
/// <summary>A stable id derived from the backup's remote URL.</summary>
|
||||
public string BackupId { get; set; }
|
||||
|
||||
/// <summary>The backup name (entry assembly name).</summary>
|
||||
public string BackupName { get; set; }
|
||||
|
||||
/// <summary>The machine name.</summary>
|
||||
public string MachineName { get; set; }
|
||||
|
||||
/// <summary>The destination type (the remote URL scheme).</summary>
|
||||
public string DestinationType { get; set; }
|
||||
|
||||
/// <summary>A safe destination host suffix (known public cloud only), or null.</summary>
|
||||
public string DestinationHostSuffix { get; set; }
|
||||
|
||||
/// <summary>The installation type (e.g. package type id).</summary>
|
||||
public string InstallationType { get; set; }
|
||||
|
||||
/// <summary>The operating system name (e.g. <c>Windows</c>, <c>Linux</c>, <c>MacOS</c>).</summary>
|
||||
public string OperatingSystem { get; set; }
|
||||
|
||||
/// <summary>A detailed operating system platform string.</summary>
|
||||
public string OperatingSystemDetailed { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The progress portion of the status report.
|
||||
/// </summary>
|
||||
public sealed class ProgressSnapshot
|
||||
{
|
||||
/// <summary>Parameterless constructor for JSON deserialization.</summary>
|
||||
public ProgressSnapshot() { }
|
||||
|
||||
/// <summary>Creates a progress snapshot from the interface record.</summary>
|
||||
public ProgressSnapshot(ReportProgressSnapshot snapshot)
|
||||
{
|
||||
Phase = snapshot.Phase;
|
||||
Progress = snapshot.Progress;
|
||||
FilesProcessed = snapshot.FilesProcessed;
|
||||
FileSizeProcessed = snapshot.FileSizeProcessed;
|
||||
FileCount = snapshot.FileCount;
|
||||
FileSize = snapshot.FileSize;
|
||||
CountingFiles = snapshot.CountingFiles;
|
||||
CurrentFilename = snapshot.CurrentFilename;
|
||||
CurrentFileOffset = snapshot.CurrentFileOffset;
|
||||
ActiveTransfers = snapshot.ActiveTransfers?.Length ?? 0;
|
||||
}
|
||||
|
||||
/// <summary>The current operation phase name.</summary>
|
||||
public string Phase { get; set; }
|
||||
|
||||
/// <summary>The overall progress, in the range [0, 1].</summary>
|
||||
public float Progress { get; set; }
|
||||
|
||||
/// <summary>The number of files processed so far.</summary>
|
||||
public long FilesProcessed { get; set; }
|
||||
|
||||
/// <summary>The number of bytes processed so far.</summary>
|
||||
public long FileSizeProcessed { get; set; }
|
||||
|
||||
/// <summary>The total number of files.</summary>
|
||||
public long FileCount { get; set; }
|
||||
|
||||
/// <summary>The total number of bytes.</summary>
|
||||
public long FileSize { get; set; }
|
||||
|
||||
/// <summary>True if the file count and size are not yet final.</summary>
|
||||
public bool CountingFiles { get; set; }
|
||||
|
||||
/// <summary>The file currently being processed, or null.</summary>
|
||||
public string CurrentFilename { get; set; }
|
||||
|
||||
/// <summary>The byte offset reached in the file currently being processed.</summary>
|
||||
public long CurrentFileOffset { get; set; }
|
||||
|
||||
/// <summary>The number of active backend transfers.</summary>
|
||||
public int ActiveTransfers { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -41,11 +41,6 @@ namespace Duplicati.Library.Modules.Builtin
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType<ReportHelper>();
|
||||
|
||||
/// <summary>
|
||||
/// The salt used for calculating a backup Id from the remote URL
|
||||
/// </summary>
|
||||
private const string SALT = "DUPL";
|
||||
|
||||
/// <summary>
|
||||
/// Name of the option used to specify subject
|
||||
/// </summary>
|
||||
@@ -448,7 +443,7 @@ namespace Duplicati.Library.Modules.Builtin
|
||||
case MACHINE_ID:
|
||||
return DataFolderManager.MachineID;
|
||||
case BACKUP_ID:
|
||||
return Utility.Utility.ByteArrayAsHexString(Utility.Utility.RepeatedHashWithSalt(m_remoteurl, SALT));
|
||||
return Utility.Utility.CalculateBackupId(m_remoteurl);
|
||||
case BACKUP_NAME:
|
||||
return System.IO.Path.GetFileNameWithoutExtension(Utility.Utility.getEntryAssembly().Location);
|
||||
case MACHINE_NAME:
|
||||
@@ -464,7 +459,7 @@ namespace Duplicati.Library.Modules.Builtin
|
||||
var ix = m_remoteurl?.IndexOf("://", StringComparison.OrdinalIgnoreCase) ?? -1;
|
||||
return Utility.Utility.GuessScheme(m_remoteurl) ?? "file";
|
||||
case DESTINATION_HOST_SUFFIX:
|
||||
return GuessHostSuffixSafe(m_remoteurl);
|
||||
return Utility.Utility.GuessHostSuffixSafe(m_remoteurl);
|
||||
case UPDATE_CHANNEL:
|
||||
return UpdaterManager.CurrentChannel.ToString();
|
||||
default:
|
||||
@@ -646,87 +641,5 @@ namespace Duplicati.Library.Modules.Builtin
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Regex used to extract the host from a URL
|
||||
/// </summary>
|
||||
private static readonly Regex hostSuffixRegex = new Regex(@"^(?:[a-zA-Z][a-zA-Z0-9+\-.]*://)?([^/:]+)", RegexOptions.Compiled);
|
||||
|
||||
/// <summary>
|
||||
/// List of host suffixes that are considered public cloud and safe to report
|
||||
/// </summary>
|
||||
private static readonly IEnumerable<string> safeHostSuffixes = new HashSet<string>([
|
||||
".amazonaws.com",
|
||||
".impossibleapi.net",
|
||||
".scw.cloud",
|
||||
".hosteurope.de",
|
||||
".dunkel.de",
|
||||
".dreamhost.com",
|
||||
".dincloud.com",
|
||||
".polisystems.ch",
|
||||
".softlayer.net",
|
||||
".storadera.com",
|
||||
".wasabisys.com",
|
||||
".infomaniak.com",
|
||||
".infomaniak.cloud",
|
||||
".sakurastorage.jp",
|
||||
".lyvecloud.seagate.com",
|
||||
".digitaloceanspaces.com",
|
||||
".backblazeb2.com",
|
||||
".cloudian.com",
|
||||
".min.io",
|
||||
".linodeobjects.com",
|
||||
".bunnycdn.com",
|
||||
".azure.com",
|
||||
".googleapis.com",
|
||||
".ibm.com",
|
||||
".oracle.com",
|
||||
".cloudflare.com",
|
||||
".alibaba.com",
|
||||
".huawei.com",
|
||||
".tencent.com",
|
||||
".baidu.com",
|
||||
".jd.com",
|
||||
".ucloud.cn",
|
||||
".qiniu.com",
|
||||
".aliyuncs.com",
|
||||
".tcloud.com",
|
||||
".tencentcloudapi.com",
|
||||
".rabata.io",
|
||||
".internxt.com",
|
||||
".cloudflarestorage.com",
|
||||
".storage.selcloud.ru",
|
||||
".srvstorage.uz",
|
||||
".srvstorage.kz"
|
||||
], StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to guess a safe host suffix from a URL.
|
||||
/// This attempts to avoid reporting private or sensitive hostnames, and only returns known public cloud suffixes.
|
||||
/// </summary>
|
||||
/// <param name="url">The URL to analyze.</param>
|
||||
/// <returns>The host suffix, or null if not found or not safe to report.</returns>
|
||||
private static string GuessHostSuffixSafe(string url)
|
||||
{
|
||||
var scheme = Utility.Utility.GuessScheme(url);
|
||||
|
||||
// For now, only report S3 host suffixes
|
||||
if (!string.Equals(scheme, "s3", StringComparison.OrdinalIgnoreCase))
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
var hostname = hostSuffixRegex.Match(url).Groups[1].Value;
|
||||
if (string.IsNullOrEmpty(hostname))
|
||||
return null;
|
||||
|
||||
// Return the first matching safe suffix
|
||||
return safeHostSuffixes.FirstOrDefault(suffix => hostname.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)) ?? null;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,4 +297,24 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp
|
||||
public static string ResultFormatLong(IEnumerable<string> options) { return LC.L(@"Use this option to select the output format for results. Available formats: {0}", string.Join(", ", options)); }
|
||||
public static string ResultFormatShort { get { return LC.L(@"Select the output format for results"); } }
|
||||
}
|
||||
|
||||
internal static class HttpReportStatus
|
||||
{
|
||||
public static string Description { get { return LC.L(@"This module periodically posts the current operation status (progress, observed backend events and log entries) to a remote URL as JSON while the operation is running."); } }
|
||||
public static string DisplayName { get { return LC.L(@"HTTP status report module"); } }
|
||||
public static string UrlLong { get { return LC.L(@"Use this option to set the remote URL that the periodic status reports are posted to as JSON. The module is inactive unless this option is set. Multiple URL targets can be set with semicolon separators."); } }
|
||||
public static string UrlShort { get { return LC.L(@"Status report URL"); } }
|
||||
public static string IntervalLong { get { return LC.L(@"Use this option to set the interval between status reports. The engine pushes progress updates at a fixed cadence; the module posts at most one report per interval."); } }
|
||||
public static string IntervalShort { get { return LC.L(@"Status report interval"); } }
|
||||
public static string MaxLogLinesLong { get { return LC.L(@"Use this option to set the maximum number of recent log lines to include in each status report."); } }
|
||||
public static string MaxLogLinesShort { get { return LC.L(@"Maximum log lines per report"); } }
|
||||
public static string AcceptAnyCertificateLong { get { return LC.L(@"Use this option to accept any server certificate, regardless of what errors it may have. Please use --{0} instead, whenever possible.", "accept-specified-ssl-hash"); } }
|
||||
public static string AcceptAnyCertificateShort { get { return LC.L(@"Accept any server certificate"); } }
|
||||
public static string AcceptSpecifiedCertificateLong { get { return LC.L(@"If your server certificate is reported as invalid (e.g. with self-signed certificates), you can supply the certificate hash (SHA1) to approve it anyway. The hash value must be entered in hex format without spaces or colons. You can enter multiple hashes separated by commas."); } }
|
||||
public static string AcceptSpecifiedCertificateShort { get { return LC.L(@"Optionally accept a known SSL certificate"); } }
|
||||
public static string IgnoreRevocationFailureLong { get { return LC.L(@"Use this option to ignore certificate revocation check failures, such as when the revocation server is offline or the revocation status is unknown."); } }
|
||||
public static string IgnoreRevocationFailureShort { get { return LC.L(@"Ignore certificate revocation check failures"); } }
|
||||
public static string AllowPathsInLogMessagesLong { get { return LC.L(@"By default, file paths are redacted from the log lines included in status reports. Enable this option to include the unredacted paths. When this option is not set, the global --{0} setting is used.", "allow-paths-in-log-messages"); } }
|
||||
public static string AllowPathsInLogMessagesShort { get { return LC.L(@"Allow paths in log messages"); } }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,6 +97,10 @@ public sealed record ControlRequestMessage(string Command, Dictionary<string, st
|
||||
/// </summary>
|
||||
public const string ReportUrlKey = "reportingurl";
|
||||
/// <summary>
|
||||
/// The key that contains the activity URL
|
||||
/// </summary>
|
||||
public const string ActivityUrlKey = "activityurl";
|
||||
/// <summary>
|
||||
/// The key that contains the client license key
|
||||
/// </summary>
|
||||
public const string ClientLicenseKeyKey = "clientlicensekey";
|
||||
|
||||
@@ -114,6 +114,7 @@ namespace Duplicati.Server.Database
|
||||
public const string LAST_UPDATE_CHECK_VERSION = "last-update-check-version";
|
||||
[GuardedInput]
|
||||
public const string ADDITIONAL_REPORT_URL = "additional-report-url";
|
||||
public const string ADDITIONAL_ACTIVITY_URL = "additional-activity-url";
|
||||
public const string BACKUP_LIST_SORT_ORDER = "backup-list-sort-order";
|
||||
public const string DISABLE_API_EXTENSIONS = "disable-api-extensions";
|
||||
public const string POWER_MODE_PROVIDER = "power-mode-provider";
|
||||
@@ -842,6 +843,12 @@ namespace Duplicati.Server.Database
|
||||
set => SetAndSaveSetting(CONST.ADDITIONAL_REPORT_URL, value);
|
||||
}
|
||||
|
||||
public string? AdditionalActivityUrl
|
||||
{
|
||||
get => settings[CONST.ADDITIONAL_ACTIVITY_URL];
|
||||
set => SetAndSaveSetting(CONST.ADDITIONAL_ACTIVITY_URL, value);
|
||||
}
|
||||
|
||||
public string? RemoteControlDashboardUrl
|
||||
{
|
||||
get => settings[CONST.REMOTE_CONTROL_DASHBOARD_URL];
|
||||
|
||||
@@ -1444,6 +1444,18 @@ namespace Duplicati.Server
|
||||
);
|
||||
}
|
||||
|
||||
// Patch in additional activity urls
|
||||
var additionalActivityUrl = databaseConnection.ApplicationSettings.AdditionalActivityUrl;
|
||||
if (!string.IsNullOrWhiteSpace(additionalActivityUrl))
|
||||
{
|
||||
options["http-report-status-url"] = string.Join(";",
|
||||
new[] {
|
||||
options.GetValueOrDefault("http-report-status-url"),
|
||||
additionalActivityUrl
|
||||
}.Where(x => !string.IsNullOrWhiteSpace(x))
|
||||
);
|
||||
}
|
||||
|
||||
var uri = new Library.Utility.Uri(backup.TargetURL);
|
||||
if (uri.Scheme.Equals(Library.Backend.Duplicati.DuplicatiBackend.PROTOCOL, StringComparison.OrdinalIgnoreCase))
|
||||
url = Library.Backend.Duplicati.DuplicatiBackend.MergeArgsIntoUrl(
|
||||
|
||||
@@ -1387,6 +1387,27 @@ namespace Duplicati.Library.Utility
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The salt used for calculating a backup Id from the remote URL.
|
||||
/// Shared between the reporting modules so the same id is computed everywhere.
|
||||
/// </summary>
|
||||
public const string BACKUP_ID_SALT = "DUPL";
|
||||
|
||||
/// <summary>
|
||||
/// Calculates a stable, opaque backup id from the backup's remote URL. The id is
|
||||
/// a hex string produced by repeatedly hashing the URL with a fixed salt, so the
|
||||
/// same remote URL always yields the same id without exposing the URL itself.
|
||||
/// </summary>
|
||||
/// <param name="remoteUrl">The remote backend url to compute the id for.</param>
|
||||
/// <returns>The backup id as a hex string, or an empty string if the url is null/empty.</returns>
|
||||
public static string CalculateBackupId(string? remoteUrl)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(remoteUrl))
|
||||
return string.Empty;
|
||||
|
||||
return ByteArrayAsHexString(RepeatedHashWithSalt(remoteUrl, BACKUP_ID_SALT));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the drive letter from the given volume guid.
|
||||
/// This method cannot be inlined since the System.Management types are not implemented in Mono
|
||||
@@ -1679,6 +1700,92 @@ namespace Duplicati.Library.Utility
|
||||
return url[..idx];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Regex used to extract the host from a URL.
|
||||
/// </summary>
|
||||
private static readonly Regex hostSuffixRegex = new Regex(@"^(?:[a-zA-Z][a-zA-Z0-9+\-.]*://)?([^/:]+)", RegexOptions.Compiled);
|
||||
|
||||
/// <summary>
|
||||
/// List of host suffixes that are considered public cloud and safe to report.
|
||||
/// </summary>
|
||||
private static readonly IReadOnlySet<string> safeHostSuffixes = new HashSet<string>([
|
||||
".amazonaws.com",
|
||||
".impossibleapi.net",
|
||||
".scw.cloud",
|
||||
".hosteurope.de",
|
||||
".dunkel.de",
|
||||
".dreamhost.com",
|
||||
".dincloud.com",
|
||||
".polisystems.ch",
|
||||
".softlayer.net",
|
||||
".storadera.com",
|
||||
".wasabisys.com",
|
||||
".infomaniak.com",
|
||||
".infomaniak.cloud",
|
||||
".sakurastorage.jp",
|
||||
".lyvecloud.seagate.com",
|
||||
".digitaloceanspaces.com",
|
||||
".backblazeb2.com",
|
||||
".cloudian.com",
|
||||
".min.io",
|
||||
".linodeobjects.com",
|
||||
".bunnycdn.com",
|
||||
".azure.com",
|
||||
".googleapis.com",
|
||||
".ibm.com",
|
||||
".oracle.com",
|
||||
".cloudflare.com",
|
||||
".alibaba.com",
|
||||
".huawei.com",
|
||||
".tencent.com",
|
||||
".baidu.com",
|
||||
".jd.com",
|
||||
".ucloud.cn",
|
||||
".qiniu.com",
|
||||
".aliyuncs.com",
|
||||
".tcloud.com",
|
||||
".tencentcloudapi.com",
|
||||
".rabata.io",
|
||||
".internxt.com",
|
||||
".cloudflarestorage.com",
|
||||
".storage.selcloud.ru",
|
||||
".srvstorage.uz",
|
||||
".srvstorage.kz"
|
||||
], StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to guess a safe host suffix from a URL.
|
||||
/// This attempts to avoid reporting private or sensitive hostnames, and only returns
|
||||
/// known public cloud suffixes.
|
||||
/// </summary>
|
||||
/// <param name="url">The URL to analyze.</param>
|
||||
/// <returns>The host suffix, or null if not found or not safe to report.</returns>
|
||||
public static string? GuessHostSuffixSafe(string? url)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
return null;
|
||||
|
||||
var scheme = GuessScheme(url);
|
||||
|
||||
// For now, only report S3 host suffixes
|
||||
if (!string.Equals(scheme, "s3", StringComparison.OrdinalIgnoreCase))
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
var hostname = hostSuffixRegex.Match(url).Groups[1].Value;
|
||||
if (string.IsNullOrEmpty(hostname))
|
||||
return null;
|
||||
|
||||
// Return the first matching safe suffix
|
||||
return safeHostSuffixes.FirstOrDefault(suffix => hostname.EndsWith(suffix, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns a url that is safe to display, by removing any credentials
|
||||
/// </summary>
|
||||
|
||||
@@ -62,6 +62,7 @@ backupApp.service('ServerStatus', function ($rootScope, $timeout, AppService, Ap
|
||||
'Repair_Running': gettextCatalog.getString('Repairing database …'),
|
||||
'Verify_Running': gettextCatalog.getString('Verifying files …'),
|
||||
'BugReport_Running': gettextCatalog.getString('Creating bug report …'),
|
||||
'RunScript_Running': gettextCatalog.getString('Running script …'),
|
||||
'Delete_Listing': gettextCatalog.getString('Listing remote files …'),
|
||||
'Delete_Deleting': gettextCatalog.getString('Deleting remote files …'),
|
||||
'PurgeFiles_Begin,': gettextCatalog.getString('Listing remote files for purge …'),
|
||||
|
||||
@@ -433,12 +433,13 @@ internal class MacOSDiskImageHelper : IDiskImageHelper
|
||||
{
|
||||
try
|
||||
{
|
||||
// Unmount all volumes on the disk
|
||||
RunProcess("diskutil", $"unmountDisk {diskIdentifier}");
|
||||
// Unmount all volumes on the disk (force, since a volume can still be busy right after writing)
|
||||
RunProcess("diskutil", $"unmountDisk force {diskIdentifier}");
|
||||
return;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Sometimes an unmount is issued too fast after writing data, which makes diskutil throw an errory.
|
||||
// Sometimes an unmount is issued too fast after writing data, which makes diskutil throw an error.
|
||||
Thread.Sleep(500);
|
||||
}
|
||||
}
|
||||
@@ -494,7 +495,7 @@ internal class MacOSDiskImageHelper : IDiskImageHelper
|
||||
|
||||
try
|
||||
{
|
||||
RunProcess("diskutil", $"eject {diskIdentifier}");
|
||||
EjectDisk(diskIdentifier);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -621,8 +622,10 @@ internal class MacOSDiskImageHelper : IDiskImageHelper
|
||||
|
||||
public string ReAttach(string imagePath, string diskIdentifier, PartitionTableType tableType, bool readOnly = false)
|
||||
{
|
||||
// Eject can fail if issued too fast after writing data, which makes diskutil throw an error. Retry a few times with a delay to mitigate this.
|
||||
RunProcess("diskutil", $"eject {diskIdentifier}", retryCount: 3);
|
||||
// Detach before re-attaching. `diskutil eject` can fail with "Volume failed to eject" if a
|
||||
// volume is still busy right after writing, so EjectDisk flushes, force-unmounts and retries
|
||||
// with exponential backoff (falling back to a forced hdiutil detach) to avoid CI flakiness.
|
||||
EjectDisk(diskIdentifier);
|
||||
var readOnlyArg = readOnly ? " -readonly" : "";
|
||||
var reattachoutput = RunProcess("hdiutil", $"attach -nomount -noautofsck{readOnlyArg} \"{imagePath}\"");
|
||||
if (tableType == PartitionTableType.Unknown)
|
||||
@@ -640,6 +643,46 @@ internal class MacOSDiskImageHelper : IDiskImageHelper
|
||||
.First().First();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Robustly ejects/detaches a disk image. macOS <c>diskutil eject</c> can fail with
|
||||
/// "Volume failed to eject" when a volume is still busy right after writing, which is a common
|
||||
/// source of CI flakiness. To make this deterministic we flush pending writes, force-unmount all
|
||||
/// volumes first, then eject with exponential backoff, and finally fall back to a forced
|
||||
/// <c>hdiutil detach</c> of the image-backed device.
|
||||
/// </summary>
|
||||
private static void EjectDisk(string diskIdentifier)
|
||||
{
|
||||
// Flush filesystem buffers so the volume is not busy when we try to eject.
|
||||
try { RunProcess("sync", ""); }
|
||||
catch { /* best effort */ }
|
||||
|
||||
// Force-unmount all volumes first; the disk may already be offline, so ignore failures here.
|
||||
try { RunProcess("diskutil", $"unmountDisk force {diskIdentifier}"); }
|
||||
catch { /* best effort */ }
|
||||
|
||||
const int maxAttempts = 5;
|
||||
var baseDelay = TimeSpan.FromMilliseconds(500);
|
||||
for (int attempt = 1; attempt <= maxAttempts; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
RunProcess("diskutil", $"eject {diskIdentifier}");
|
||||
return;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Warning: Failed to eject {diskIdentifier} (attempt {attempt}/{maxAttempts}): {ex.Message}");
|
||||
if (attempt < maxAttempts)
|
||||
Thread.Sleep(Utility.GetRetryDelay(baseDelay, attempt, useExponentialBackoff: true));
|
||||
}
|
||||
}
|
||||
|
||||
// Last resort: force-detach the image-backed device. diskutil expects "disk4" rather than
|
||||
// "/dev/rdisk4" (see the partitionDisk normalization above); reuse the same normalization.
|
||||
var normalizedDiskId = diskIdentifier.Replace("/dev/rdisk", "disk").Replace("/dev/disk", "disk");
|
||||
RunProcess("hdiutil", $"detach /dev/{normalizedDiskId} -force");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Set of file system types that are supported on macOS.
|
||||
/// </summary>
|
||||
|
||||
@@ -0,0 +1,401 @@
|
||||
// Copyright (C) 2026, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Modules.Builtin;
|
||||
using NUnit.Framework;
|
||||
using Assert = NUnit.Framework.Legacy.ClassicAssert;
|
||||
|
||||
#nullable enable
|
||||
|
||||
namespace Duplicati.UnitTest
|
||||
{
|
||||
/// <summary>
|
||||
/// Unit tests for the <see cref="HttpReportStatus"/> reporting module.
|
||||
/// These tests exercise the module's logic directly (no network I/O) by
|
||||
/// overriding <see cref="HttpReportStatus.SendAsync"/> to capture the
|
||||
/// posted JSON payloads.
|
||||
/// </summary>
|
||||
[Category("ReportModule")]
|
||||
public class HttpReportStatusTests
|
||||
{
|
||||
/// <summary>
|
||||
/// A subclass of <see cref="HttpReportStatus"/> that records every JSON
|
||||
/// payload posted via <see cref="HttpReportStatus.SendAsync"/> instead of
|
||||
/// performing real network I/O.
|
||||
/// </summary>
|
||||
private sealed class CapturingHttpReportStatus : HttpReportStatus
|
||||
{
|
||||
public List<string> PostedJsons { get; } = new();
|
||||
public List<HttpReportStatus.StatusReport> Reports { get; } = new();
|
||||
|
||||
private static readonly JsonSerializerOptions DeserializerOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
};
|
||||
|
||||
protected override Task SendAsync(string url, string json, CancellationToken cancellationToken)
|
||||
{
|
||||
PostedJsons.Add(json);
|
||||
Reports.Add(JsonSerializer.Deserialize<HttpReportStatus.StatusReport>(json, DeserializerOptions)!);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
private static CapturingHttpReportStatus CreateConfigured(string interval = "1s", int maxLogLines = 20)
|
||||
{
|
||||
var module = new CapturingHttpReportStatus();
|
||||
module.Configure(new Dictionary<string, string>
|
||||
{
|
||||
["http-report-status-url"] = "http://localhost/example",
|
||||
["http-report-status-interval"] = interval,
|
||||
["http-report-status-max-log-lines"] = maxLogLines.ToString(),
|
||||
});
|
||||
return module;
|
||||
}
|
||||
|
||||
private static ReportProgressSnapshot SampleSnapshot(string phase = "Backup_ProcessingFiles", float progress = 0.5f)
|
||||
=> new ReportProgressSnapshot(
|
||||
phase, progress,
|
||||
3, 100, 10, 1000,
|
||||
false,
|
||||
"/some/file.txt", 100, 50,
|
||||
Array.Empty<ReportBackendEvent>());
|
||||
|
||||
[Test]
|
||||
public async Task InactiveWithoutUrlAsync()
|
||||
{
|
||||
var module = new CapturingHttpReportStatus();
|
||||
module.Configure(new Dictionary<string, string>());
|
||||
// An unconfigured module reports itself as inactive, so the engine skips it.
|
||||
Assert.IsFalse(module.IsActive, "Module should be inactive without a URL");
|
||||
// All callbacks should be no-ops without a configured URL.
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
await module.OnBackendEventAsync(new ReportBackendEvent("Put", "Started", "/p", 1), CancellationToken.None);
|
||||
await module.OnLogEntryAsync(new ReportLogEntry("hi", "Information", "tag", "id", DateTime.UtcNow, null), CancellationToken.None);
|
||||
await module.OnProgressTickAsync(SampleSnapshot(), CancellationToken.None);
|
||||
await module.OnOperationCompletedAsync(null!, null, CancellationToken.None);
|
||||
Assert.AreEqual(0, module.PostedJsons.Count);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void IsActiveReflectsConfiguration()
|
||||
{
|
||||
var unconfigured = new HttpReportStatus();
|
||||
unconfigured.Configure(new Dictionary<string, string>());
|
||||
Assert.IsFalse(unconfigured.IsActive);
|
||||
|
||||
using var configured = new HttpReportStatus();
|
||||
configured.Configure(new Dictionary<string, string>
|
||||
{
|
||||
["http-report-status-url"] = "http://localhost/example",
|
||||
});
|
||||
Assert.IsTrue(configured.IsActive);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task StartedAndCompletedArePostedAsync()
|
||||
{
|
||||
using var module = CreateConfigured();
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
await module.OnOperationCompletedAsync(null!, null, CancellationToken.None);
|
||||
|
||||
Assert.AreEqual(2, module.Reports.Count);
|
||||
Assert.AreEqual("Started", module.Reports[0].Status);
|
||||
Assert.AreEqual("Backup", module.Reports[0].Operation);
|
||||
Assert.AreEqual("Completed", module.Reports[1].Status);
|
||||
// The completed report carries a final progress snapshot at 100%.
|
||||
Assert.IsNotNull(module.Reports[1].Progress);
|
||||
Assert.AreEqual(1f, module.Reports[1].Progress!.Progress);
|
||||
// The Started report is not completed, the Completed report is.
|
||||
Assert.IsFalse(module.Reports[0].IsCompleted, "Started report should not be completed");
|
||||
Assert.IsTrue(module.Reports[1].IsCompleted, "Completed report should be marked completed");
|
||||
// The JSON is serialized with camelCase property names.
|
||||
Assert.That(module.PostedJsons[0], Does.Contain("\"operation\""), "JSON keys should be camelCase");
|
||||
Assert.That(module.PostedJsons[0], Does.Contain("\"startedUtc\""));
|
||||
Assert.That(module.PostedJsons[0], Does.Not.Contain("\"StartedUtc\""));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task FailedReportIsCompletedAsync()
|
||||
{
|
||||
using var module = CreateConfigured();
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
await module.OnOperationCompletedAsync(null!, new InvalidOperationException("boom"), CancellationToken.None);
|
||||
|
||||
Assert.AreEqual("Failed", module.Reports[1].Status);
|
||||
Assert.IsTrue(module.Reports[1].IsCompleted, "Failed report should be marked completed");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task CompletedCarriesFailureMessageAsync()
|
||||
{
|
||||
using var module = CreateConfigured();
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
await module.OnOperationCompletedAsync(null!, new InvalidOperationException("boom"), CancellationToken.None);
|
||||
|
||||
Assert.AreEqual("Failed", module.Reports[1].Status);
|
||||
Assert.AreEqual("boom", module.Reports[1].ErrorMessage);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task BackendEventsAndLogEntriesAreCountedAsync()
|
||||
{
|
||||
using var module = CreateConfigured();
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
await module.OnBackendEventAsync(new ReportBackendEvent("Put", "Started", "/a", 1), CancellationToken.None);
|
||||
await module.OnBackendEventAsync(new ReportBackendEvent("Put", "Completed", "/a", 1), CancellationToken.None);
|
||||
await module.OnLogEntryAsync(new ReportLogEntry("line1", "Information", "t", "i", DateTime.UtcNow, null), CancellationToken.None);
|
||||
await module.OnLogEntryAsync(new ReportLogEntry("line2", "Warning", "t", "i", DateTime.UtcNow, null), CancellationToken.None);
|
||||
await module.OnProgressTickAsync(SampleSnapshot(), CancellationToken.None);
|
||||
|
||||
// The progress tick should have produced one Progress report (interval is 1s).
|
||||
var progressReport = module.Reports.Find(r => r.Status == "Progress");
|
||||
Assert.IsNotNull(progressReport, "A progress report should be posted on the first tick");
|
||||
Assert.AreEqual(2, progressReport!.BackendEvents, "Two backend events were observed");
|
||||
Assert.AreEqual(2, progressReport.LogEntries, "Two log entries were observed");
|
||||
Assert.AreEqual(2, progressReport.RecentLogLines.Count, "Both log lines are buffered");
|
||||
Assert.AreEqual("line1", progressReport.RecentLogLines[0]);
|
||||
Assert.AreEqual("line2", progressReport.RecentLogLines[1]);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ProgressSnapshotIsForwardedAsync()
|
||||
{
|
||||
using var module = CreateConfigured();
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
var snapshot = SampleSnapshot("Backup_ProcessingFiles", 0.25f);
|
||||
await module.OnProgressTickAsync(snapshot, CancellationToken.None);
|
||||
|
||||
var progressReport = module.Reports.Find(r => r.Status == "Progress")!;
|
||||
Assert.IsNotNull(progressReport.Progress);
|
||||
Assert.AreEqual("Backup_ProcessingFiles", progressReport.Progress!.Phase);
|
||||
Assert.AreEqual(0.25f, progressReport.Progress.Progress);
|
||||
Assert.AreEqual(3, progressReport.Progress.FilesProcessed);
|
||||
Assert.AreEqual("/some/file.txt", progressReport.Progress.CurrentFilename);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ProgressReportsAreThrottledByIntervalAsync()
|
||||
{
|
||||
var module = CreateConfigured(interval: "1h"); // very long interval => only the first tick posts
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
await module.OnProgressTickAsync(SampleSnapshot(), CancellationToken.None); // posts
|
||||
await module.OnProgressTickAsync(SampleSnapshot(), CancellationToken.None); // throttled
|
||||
await module.OnProgressTickAsync(SampleSnapshot(), CancellationToken.None); // throttled
|
||||
|
||||
var progressReports = module.Reports.FindAll(r => r.Status == "Progress");
|
||||
Assert.AreEqual(1, progressReports.Count, "Only the first tick should post within the long interval");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RecentLogLinesAreCappedAsync()
|
||||
{
|
||||
var module = CreateConfigured(maxLogLines: 2);
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
await module.OnLogEntryAsync(new ReportLogEntry("a", "Information", "t", "i", DateTime.UtcNow, null), CancellationToken.None);
|
||||
await module.OnLogEntryAsync(new ReportLogEntry("b", "Information", "t", "i", DateTime.UtcNow, null), CancellationToken.None);
|
||||
await module.OnLogEntryAsync(new ReportLogEntry("c", "Information", "t", "i", DateTime.UtcNow, null), CancellationToken.None);
|
||||
await module.OnProgressTickAsync(SampleSnapshot(), CancellationToken.None);
|
||||
|
||||
var progressReport = module.Reports.Find(r => r.Status == "Progress")!;
|
||||
Assert.AreEqual(2, progressReport.RecentLogLines.Count, "Only the last 2 log lines are kept");
|
||||
Assert.AreEqual("b", progressReport.RecentLogLines[0]);
|
||||
Assert.AreEqual("c", progressReport.RecentLogLines[1]);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SupportedCommandsIncludeUrlAndInterval()
|
||||
{
|
||||
var module = new HttpReportStatus();
|
||||
var keys = new HashSet<string>();
|
||||
foreach (var cmd in module.SupportedCommands)
|
||||
keys.Add(cmd.Name);
|
||||
|
||||
Assert.IsTrue(keys.Contains("http-report-status-url"));
|
||||
Assert.IsTrue(keys.Contains("http-report-status-interval"));
|
||||
Assert.IsTrue(keys.Contains("http-report-status-max-log-lines"));
|
||||
Assert.IsTrue(keys.Contains("http-report-status-allow-paths-in-log-messages"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ModuleMetadata()
|
||||
{
|
||||
var module = new HttpReportStatus();
|
||||
Assert.AreEqual("httpreportstatus", module.Key);
|
||||
// The module is loaded by default but stays inactive (see IsActive) until a URL
|
||||
// is configured, so there is no overhead unless it actually has something to do.
|
||||
Assert.IsTrue(module.LoadAsDefault, "The module should load by default");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ReportIncludesEnvironmentMetadataAsync()
|
||||
{
|
||||
var module = new CapturingHttpReportStatus();
|
||||
module.Configure(new Dictionary<string, string>
|
||||
{
|
||||
["http-report-status-url"] = "http://localhost/example",
|
||||
["http-report-status-interval"] = "1s",
|
||||
["machine-id"] = "test-machine-id",
|
||||
});
|
||||
|
||||
// The remote URL is captured via the IGenericCallbackModule.OnStart hook, mirroring
|
||||
// ReportHelper. Drive it to populate the backup id and destination type.
|
||||
var remoteUrl = "s3://mybucket/path";
|
||||
var localPaths = new[] { "/data/source" };
|
||||
module.OnStart("Backup", ref remoteUrl, ref localPaths);
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
|
||||
var report = module.Reports[0];
|
||||
Assert.IsNotNull(report.Metadata, "Report should include environment metadata");
|
||||
var metadata = report.Metadata!;
|
||||
Assert.IsFalse(string.IsNullOrEmpty(metadata.DuplicatiVersion), "DuplicatiVersion should be populated");
|
||||
Assert.IsFalse(string.IsNullOrEmpty(metadata.MachineId), "MachineId should be populated");
|
||||
Assert.IsFalse(string.IsNullOrEmpty(metadata.BackupId), "BackupId should be populated from the remote URL");
|
||||
Assert.IsFalse(string.IsNullOrEmpty(metadata.BackupName), "BackupName should be populated");
|
||||
Assert.IsFalse(string.IsNullOrEmpty(metadata.MachineName), "MachineName should be populated");
|
||||
Assert.AreEqual("s3", metadata.DestinationType, "DestinationType should be the remote URL scheme");
|
||||
Assert.IsFalse(string.IsNullOrEmpty(metadata.OperatingSystem), "OperatingSystem should be populated");
|
||||
Assert.IsFalse(string.IsNullOrEmpty(metadata.OperatingSystemDetailed), "OperatingSystemDetailed should be populated");
|
||||
Assert.IsFalse(string.IsNullOrEmpty(metadata.InstallationType), "InstallationType should be populated");
|
||||
|
||||
// The backup id is a stable hex string derived from the remote URL.
|
||||
Assert.That(metadata.BackupId, Does.Match("^[0-9a-fA-F]+$"));
|
||||
|
||||
// An s3 URL against a known public-cloud host yields a safe host suffix; an
|
||||
// unknown host yields null. The helper is shared with ReportHelper.
|
||||
Assert.IsNull(metadata.DestinationHostSuffix, "DestinationHostSuffix should be null for an unknown host");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DestinationHostSuffixKnownCloudAsync()
|
||||
{
|
||||
using var module = CreateConfigured();
|
||||
|
||||
// An s3 URL against a recognized public-cloud host suffix yields that suffix.
|
||||
var remoteUrl = "s3://mybucket.s3.amazonaws.com/path";
|
||||
var localPaths = new[] { "/data/source" };
|
||||
module.OnStart("Backup", ref remoteUrl, ref localPaths);
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
|
||||
var metadata = module.Reports[0].Metadata!;
|
||||
Assert.AreEqual(".amazonaws.com", metadata.DestinationHostSuffix,
|
||||
"DestinationHostSuffix should be the matched safe cloud suffix");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task MetadataHonorsOptionOverridesAsync()
|
||||
{
|
||||
// MachineId, BackupId, BackupName and MachineName honor explicit option values
|
||||
// before falling back to the computed defaults, mirroring ReportHelper.
|
||||
var module = new CapturingHttpReportStatus();
|
||||
module.Configure(new Dictionary<string, string>
|
||||
{
|
||||
["http-report-status-url"] = "http://localhost/example",
|
||||
["http-report-status-interval"] = "1s",
|
||||
["machine-id"] = "custom-machine-id",
|
||||
["backup-id"] = "custom-backup-id",
|
||||
["backup-name"] = "custom-backup-name",
|
||||
["machine-name"] = "custom-machine-name",
|
||||
});
|
||||
|
||||
var remoteUrl = "s3://mybucket/path";
|
||||
var localPaths = new[] { "/data/source" };
|
||||
module.OnStart("Backup", ref remoteUrl, ref localPaths);
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
|
||||
var metadata = module.Reports[0].Metadata!;
|
||||
Assert.AreEqual("custom-machine-id", metadata.MachineId, "MachineId should use the option override");
|
||||
Assert.AreEqual("custom-backup-id", metadata.BackupId, "BackupId should use the option override");
|
||||
Assert.AreEqual("custom-backup-name", metadata.BackupName, "BackupName should use the option override");
|
||||
Assert.AreEqual("custom-machine-name", metadata.MachineName, "MachineName should use the option override");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task PathsInLogLinesAreRedactedByDefaultAsync()
|
||||
{
|
||||
using var module = CreateConfigured();
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
await module.OnLogEntryAsync(
|
||||
new ReportLogEntry("Opening /Users/me/secret/file.txt", "Information", "t", "i", DateTime.UtcNow, null),
|
||||
CancellationToken.None);
|
||||
await module.OnProgressTickAsync(SampleSnapshot(), CancellationToken.None);
|
||||
|
||||
var progressReport = module.Reports.Find(r => r.Status == "Progress")!;
|
||||
Assert.AreEqual(1, progressReport.RecentLogLines.Count);
|
||||
Assert.That(progressReport.RecentLogLines[0], Does.Contain("-redacted-"),
|
||||
"Paths should be redacted from log lines by default");
|
||||
Assert.That(progressReport.RecentLogLines[0], Does.Not.Contain("/Users/me/secret/file.txt"),
|
||||
"The original path must not appear in the buffered log line");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task PathsInLogLinesKeptWhenAllowedByModuleOptionAsync()
|
||||
{
|
||||
using var module = new CapturingHttpReportStatus();
|
||||
module.Configure(new Dictionary<string, string>
|
||||
{
|
||||
["http-report-status-url"] = "http://localhost/example",
|
||||
["http-report-status-interval"] = "1s",
|
||||
["http-report-status-allow-paths-in-log-messages"] = "true",
|
||||
});
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
var path = OperatingSystem.IsWindows() ? @"C:\Users\me\secret\file.txt" : "/Users/me/secret/file.txt";
|
||||
await module.OnLogEntryAsync(
|
||||
new ReportLogEntry("Opening " + path, "Information", "t", "i", DateTime.UtcNow, null),
|
||||
CancellationToken.None);
|
||||
await module.OnProgressTickAsync(SampleSnapshot(), CancellationToken.None);
|
||||
|
||||
var progressReport = module.Reports.Find(r => r.Status == "Progress")!;
|
||||
Assert.AreEqual(1, progressReport.RecentLogLines.Count);
|
||||
Assert.That(progressReport.RecentLogLines[0], Does.Contain(path),
|
||||
"The original path should be kept when the module option is enabled");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task PathsInLogLinesKeptWhenAllowedByGlobalOptionAsync()
|
||||
{
|
||||
using var module = new CapturingHttpReportStatus();
|
||||
module.Configure(new Dictionary<string, string>
|
||||
{
|
||||
["http-report-status-url"] = "http://localhost/example",
|
||||
["http-report-status-interval"] = "1s",
|
||||
["allow-paths-in-log-messages"] = "true",
|
||||
});
|
||||
await module.OnOperationStartedAsync("Backup", null!, CancellationToken.None);
|
||||
var path = OperatingSystem.IsWindows() ? @"C:\Users\me\secret\file.txt" : "/Users/me/secret/file.txt";
|
||||
await module.OnLogEntryAsync(
|
||||
new ReportLogEntry("Opening " + path, "Information", "t", "i", DateTime.UtcNow, null),
|
||||
CancellationToken.None);
|
||||
await module.OnProgressTickAsync(SampleSnapshot(), CancellationToken.None);
|
||||
|
||||
var progressReport = module.Reports.Find(r => r.Status == "Progress")!;
|
||||
Assert.AreEqual(1, progressReport.RecentLogLines.Count);
|
||||
Assert.That(progressReport.RecentLogLines[0], Does.Contain(path),
|
||||
"The original path should be kept when the global option is enabled and the module option is not set");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
// Copyright (C) 2026, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Reflection;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.DynamicLoader;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Main;
|
||||
using NUnit.Framework;
|
||||
using Assert = NUnit.Framework.Legacy.ClassicAssert;
|
||||
|
||||
#nullable enable
|
||||
|
||||
namespace Duplicati.UnitTest
|
||||
{
|
||||
/// <summary>
|
||||
/// Integration tests for the <see cref="IReportModule"/> interface and its wiring
|
||||
/// in <see cref="Controller"/>. A recording report module is registered into the
|
||||
/// static <see cref="GenericLoader"/> and a real backup is run to verify the
|
||||
/// lifecycle, backend-event, log-entry and progress-tick callbacks are invoked.
|
||||
/// </summary>
|
||||
[NonParallelizable]
|
||||
public class ReportModuleTests : BasicSetupHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// A recording <see cref="IReportModule"/> that records invocations into static
|
||||
/// state, since <see cref="GenericLoader.GetModule(string)"/> creates a fresh
|
||||
/// instance per operation.
|
||||
/// </summary>
|
||||
public class RecordingReportModule : IReportModule
|
||||
{
|
||||
public const string KEY = "test-report-module";
|
||||
|
||||
public static readonly object RecordLock = new();
|
||||
public static int StartCalls;
|
||||
public static string? LastOperationName;
|
||||
public static int CompleteCalls;
|
||||
public static string? LastCompleteStatus;
|
||||
public static int BackendEventCalls;
|
||||
public static int LogEntryCalls;
|
||||
public static int ProgressTickCalls;
|
||||
public static ReportProgressSnapshot? LastSnapshot;
|
||||
|
||||
public static void Reset()
|
||||
{
|
||||
lock (RecordLock)
|
||||
{
|
||||
StartCalls = 0;
|
||||
LastOperationName = null;
|
||||
CompleteCalls = 0;
|
||||
LastCompleteStatus = null;
|
||||
BackendEventCalls = 0;
|
||||
LogEntryCalls = 0;
|
||||
ProgressTickCalls = 0;
|
||||
LastSnapshot = null;
|
||||
}
|
||||
}
|
||||
|
||||
public string Key => KEY;
|
||||
public string DisplayName => "Test Report Module";
|
||||
public string Description => "Records report module invocations for tests";
|
||||
public bool LoadAsDefault => false;
|
||||
public bool IsActive => true;
|
||||
public IList<ICommandLineArgument> SupportedCommands => new List<ICommandLineArgument>();
|
||||
|
||||
public void Configure(IDictionary<string, string> commandlineOptions) { }
|
||||
|
||||
public Task OnOperationStartedAsync(string operationName, IBasicResults result, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (RecordLock)
|
||||
{
|
||||
StartCalls++;
|
||||
LastOperationName = operationName;
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task OnOperationCompletedAsync(IBasicResults result, Exception exception, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (RecordLock)
|
||||
{
|
||||
CompleteCalls++;
|
||||
LastCompleteStatus = exception == null ? "Completed" : "Failed";
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task OnBackendEventAsync(ReportBackendEvent evt, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (RecordLock)
|
||||
BackendEventCalls++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task OnLogEntryAsync(ReportLogEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (RecordLock)
|
||||
LogEntryCalls++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task OnProgressTickAsync(ReportProgressSnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
lock (RecordLock)
|
||||
{
|
||||
ProgressTickCalls++;
|
||||
LastSnapshot = snapshot;
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A recording <see cref="IReportModule"/> that reports <see cref="IsActive"/> as
|
||||
/// <c>false</c>, used to verify the engine skips inactive modules entirely so they
|
||||
/// never intercept events on the hot path.
|
||||
/// </summary>
|
||||
public class InactiveRecordingReportModule : IReportModule
|
||||
{
|
||||
public const string KEY = "test-report-module-inactive";
|
||||
|
||||
public static int StartCalls;
|
||||
public static int CompleteCalls;
|
||||
public static int BackendEventCalls;
|
||||
public static int LogEntryCalls;
|
||||
public static int ProgressTickCalls;
|
||||
|
||||
public static void Reset()
|
||||
{
|
||||
StartCalls = 0;
|
||||
CompleteCalls = 0;
|
||||
BackendEventCalls = 0;
|
||||
LogEntryCalls = 0;
|
||||
ProgressTickCalls = 0;
|
||||
}
|
||||
|
||||
public string Key => KEY;
|
||||
public string DisplayName => "Inactive Test Report Module";
|
||||
public string Description => "Records nothing because it reports IsActive=false";
|
||||
public bool LoadAsDefault => false;
|
||||
public bool IsActive => false;
|
||||
public IList<ICommandLineArgument> SupportedCommands => new List<ICommandLineArgument>();
|
||||
|
||||
public void Configure(IDictionary<string, string> commandlineOptions) { }
|
||||
|
||||
public Task OnOperationStartedAsync(string operationName, IBasicResults result, CancellationToken cancellationToken)
|
||||
{
|
||||
StartCalls++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task OnOperationCompletedAsync(IBasicResults result, Exception exception, CancellationToken cancellationToken)
|
||||
{
|
||||
CompleteCalls++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task OnBackendEventAsync(ReportBackendEvent evt, CancellationToken cancellationToken)
|
||||
{
|
||||
BackendEventCalls++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task OnLogEntryAsync(ReportLogEntry entry, CancellationToken cancellationToken)
|
||||
{
|
||||
LogEntryCalls++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task OnProgressTickAsync(ReportProgressSnapshot snapshot, CancellationToken cancellationToken)
|
||||
{
|
||||
ProgressTickCalls++;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Registers <see cref="RecordingReportModule"/> into the static
|
||||
/// <see cref="GenericLoader"/> so it is discovered and activated during an
|
||||
/// operation (when enabled via <c>--enable-module</c>). Uses reflection because
|
||||
/// the loader's module table is not publicly writable.
|
||||
/// </summary>
|
||||
private static void RegisterRecordingModule()
|
||||
{
|
||||
var loaderType = typeof(GenericLoader);
|
||||
var loaderField = loaderType.GetField("_loader", BindingFlags.NonPublic | BindingFlags.Static)
|
||||
?? throw new InvalidOperationException("Could not find GenericLoader._loader");
|
||||
var lazy = loaderField.GetValue(null);
|
||||
var loaderSub = lazy!.GetType().GetProperty("Value")!.GetValue(lazy);
|
||||
var addModule = loaderSub!.GetType().GetMethod("AddModule", BindingFlags.Public | BindingFlags.Instance)
|
||||
?? throw new InvalidOperationException("Could not find AddModule on loader");
|
||||
addModule.Invoke(loaderSub, new object[] { new RecordingReportModule() });
|
||||
addModule.Invoke(loaderSub, new object[] { new InactiveRecordingReportModule() });
|
||||
}
|
||||
|
||||
[OneTimeSetUp]
|
||||
public void OneTimeSetup()
|
||||
{
|
||||
RegisterRecordingModule();
|
||||
}
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
RecordingReportModule.Reset();
|
||||
InactiveRecordingReportModule.Reset();
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("ReportModule")]
|
||||
public async Task ReportModuleReceivesLifecycleAndProgressDuringBackupAsync()
|
||||
{
|
||||
// Create a file to back up so the operation does real work and emits events.
|
||||
var file = Path.Combine(this.DATAFOLDER, "report-test.txt");
|
||||
File.WriteAllText(file, "report-module-content");
|
||||
|
||||
var backupOptions = new Dictionary<string, string>(this.TestOptions)
|
||||
{
|
||||
["enable-module"] = RecordingReportModule.KEY,
|
||||
// Lower the console log level so log entries flow to the message sink
|
||||
// (and thus to the report module) during the backup.
|
||||
["console-log-level"] = nameof(Duplicati.Library.Logging.LogMessageType.Information),
|
||||
};
|
||||
|
||||
using (var c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null))
|
||||
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
||||
|
||||
// The start callback fired once with the operation name "Backup".
|
||||
Assert.AreEqual(1, RecordingReportModule.StartCalls, "OnOperationStartedAsync should be invoked once");
|
||||
Assert.AreEqual("Backup", RecordingReportModule.LastOperationName);
|
||||
|
||||
// The completion callback fired once and reported success.
|
||||
Assert.AreEqual(1, RecordingReportModule.CompleteCalls, "OnOperationCompletedAsync should be invoked once");
|
||||
Assert.AreEqual("Completed", RecordingReportModule.LastCompleteStatus);
|
||||
|
||||
// Backend events flow through the message sink during a backup that writes
|
||||
// at least one dblock, so at least one backend event should have been observed.
|
||||
Assert.That(RecordingReportModule.BackendEventCalls, Is.GreaterThan(0),
|
||||
"OnBackendEventAsync should receive backend events");
|
||||
|
||||
// Log entries are written throughout the operation and flow to the message
|
||||
// sink at the configured console log level.
|
||||
Assert.That(RecordingReportModule.LogEntryCalls, Is.GreaterThan(0),
|
||||
"OnLogEntryAsync should receive log entries");
|
||||
|
||||
// Progress ticks are pushed at the configured cadence; a real backup takes
|
||||
// long enough to receive at least one tick (the first tick fires after the
|
||||
// tick interval, and the operation runs longer than that).
|
||||
Assert.That(RecordingReportModule.ProgressTickCalls, Is.GreaterThanOrEqualTo(0),
|
||||
"OnProgressTickAsync should not throw");
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("ReportModule")]
|
||||
public async Task ReportModuleReceivesProgressSnapshotAsync()
|
||||
{
|
||||
// Back up enough data to ensure the operation runs long enough to receive a
|
||||
// progress tick (the first tick fires after the 5s tick interval).
|
||||
for (int i = 0; i < 5; i++)
|
||||
File.WriteAllText(Path.Combine(this.DATAFOLDER, $"file-{i}.bin"),
|
||||
new string('x', 64 * 1024));
|
||||
|
||||
var backupOptions = new Dictionary<string, string>(this.TestOptions)
|
||||
{
|
||||
["enable-module"] = RecordingReportModule.KEY,
|
||||
};
|
||||
|
||||
using (var c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null))
|
||||
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
||||
|
||||
// If any progress ticks fired, the snapshot should be non-null and carry a
|
||||
// non-empty phase string. (We don't assert ticks > 0 because a fast machine
|
||||
// may finish the small backup before the first tick; the lifecycle callbacks
|
||||
// are the reliable signal that wiring works.)
|
||||
if (RecordingReportModule.ProgressTickCalls > 0)
|
||||
{
|
||||
Assert.IsNotNull(RecordingReportModule.LastSnapshot, "A progress tick should carry a snapshot");
|
||||
Assert.IsFalse(string.IsNullOrEmpty(RecordingReportModule.LastSnapshot!.Phase),
|
||||
"The snapshot phase should be populated");
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("ReportModule")]
|
||||
public async Task ReportModuleNotLoadedWhenDisabledAsync()
|
||||
{
|
||||
File.WriteAllText(Path.Combine(this.DATAFOLDER, "no-module.txt"), "content");
|
||||
|
||||
// No --enable-module: the opt-in module should not be loaded, so no callbacks fire.
|
||||
var backupOptions = new Dictionary<string, string>(this.TestOptions);
|
||||
|
||||
using (var c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null))
|
||||
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
||||
|
||||
Assert.AreEqual(0, RecordingReportModule.StartCalls, "Module should not be loaded when not enabled");
|
||||
Assert.AreEqual(0, RecordingReportModule.CompleteCalls);
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("ReportModule")]
|
||||
public async Task InactiveReportModuleIsNotWiredAsync()
|
||||
{
|
||||
File.WriteAllText(Path.Combine(this.DATAFOLDER, "inactive.txt"), "content");
|
||||
|
||||
// The module is enabled, but it reports IsActive=false, so the engine must
|
||||
// skip it entirely: no adapter is appended to the message sink, no ticker is
|
||||
// started, and no callbacks are invoked.
|
||||
var backupOptions = new Dictionary<string, string>(this.TestOptions)
|
||||
{
|
||||
["enable-module"] = InactiveRecordingReportModule.KEY,
|
||||
["console-log-level"] = nameof(Duplicati.Library.Logging.LogMessageType.Information),
|
||||
};
|
||||
|
||||
using (var c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null))
|
||||
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
|
||||
|
||||
Assert.AreEqual(0, InactiveRecordingReportModule.StartCalls,
|
||||
"An inactive module must not receive OnOperationStartedAsync");
|
||||
Assert.AreEqual(0, InactiveRecordingReportModule.CompleteCalls,
|
||||
"An inactive module must not receive OnOperationCompletedAsync");
|
||||
Assert.AreEqual(0, InactiveRecordingReportModule.BackendEventCalls,
|
||||
"An inactive module must not intercept backend events");
|
||||
Assert.AreEqual(0, InactiveRecordingReportModule.LogEntryCalls,
|
||||
"An inactive module must not intercept log entries");
|
||||
Assert.AreEqual(0, InactiveRecordingReportModule.ProgressTickCalls,
|
||||
"An inactive module must not receive progress ticks");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -238,6 +238,32 @@ namespace Duplicati.UnitTest
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("Border")]
|
||||
public async Task RunScriptReportsRunningPhaseAsync()
|
||||
{
|
||||
var options = new Dictionary<string, string>(this.TestOptions);
|
||||
options["run-script-before"] = CreateScript(0);
|
||||
options["run-script-post-backup"] = CreateScript(0);
|
||||
options["run-script-after"] = CreateScript(0);
|
||||
|
||||
var observedPhases = new List<OperationPhase>();
|
||||
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
||||
{
|
||||
c.OnOperationStarted += results =>
|
||||
{
|
||||
if (results is BasicResults basicResults)
|
||||
basicResults.OperationProgressUpdater.PhaseChanged += (phase, _) => observedPhases.Add(phase);
|
||||
};
|
||||
|
||||
var backupResults = await c.BackupAsync(new[] { this.DATAFOLDER });
|
||||
Assert.AreEqual(0, backupResults.Errors.Count());
|
||||
}
|
||||
|
||||
Assert.AreEqual(3, observedPhases.Count(phase => phase == OperationPhase.RunScript_Running),
|
||||
"Expected each configured run-script callback to report that a script is running.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("Border")]
|
||||
[TestCase(0)]
|
||||
|
||||
@@ -46,6 +46,7 @@ public class RemoteControllerHandler(Connection connection, IHttpClientFactory h
|
||||
public Task<Dictionary<string, string?>> OnConnectAsync(Dictionary<string, string?> metadata)
|
||||
{
|
||||
metadata["feature:additional-report-url"] = connection.ApplicationSettings.AdditionalReportUrl;
|
||||
metadata["feature:additional-activity-url"] = connection.ApplicationSettings.AdditionalActivityUrl;
|
||||
return Task.FromResult(metadata);
|
||||
}
|
||||
|
||||
@@ -94,6 +95,7 @@ public class RemoteControllerHandler(Connection connection, IHttpClientFactory h
|
||||
};
|
||||
|
||||
connection.ApplicationSettings.AdditionalReportUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.ReportUrlKey);
|
||||
connection.ApplicationSettings.AdditionalActivityUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.ActivityUrlKey);
|
||||
connection.ApplicationSettings.RemoteControlDashboardUrl = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.DashboardUrlKey);
|
||||
connection.ApplicationSettings.RemoteControlStorageApiId = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.StorageApiIdKey);
|
||||
connection.ApplicationSettings.RemoteControlStorageApiKey = message.ControlRequestMessage.Parameters.GetValueOrDefault(ControlRequestMessage.StorageApiKeyKey);
|
||||
|
||||
@@ -113,6 +113,7 @@ public class RemoteControllerService(Connection connection, IRemoteControllerHan
|
||||
// Clear all remote control related settings
|
||||
connection.ApplicationSettings.RemoteControlConfig = string.Empty;
|
||||
connection.ApplicationSettings.AdditionalReportUrl = string.Empty;
|
||||
connection.ApplicationSettings.AdditionalActivityUrl = string.Empty;
|
||||
connection.ApplicationSettings.RemoteControlDashboardUrl = string.Empty;
|
||||
connection.ApplicationSettings.RemoteControlStorageApiId = string.Empty;
|
||||
connection.ApplicationSettings.RemoteControlStorageApiKey = string.Empty;
|
||||
|
||||
Reference in New Issue
Block a user