// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System;
using System.Collections.Generic;
using System.Linq;
using Duplicati.Library.Logging;
namespace Duplicati.Library.Main
{
///
/// Interface for receiving messages from the Duplicati operations
///
public interface IMessageSink : Logging.ILogDestination
{
///
/// Handles an event from the backend
///
/// The backend action.
/// The event type.
/// The target path.
/// The size of the element.
void BackendEvent(BackendActionType action, BackendEventType type, string path, long size);
///
/// Sets the backend progress update object
///
/// The backend progress update object
void SetBackendProgress(IBackendProgress progress);
///
/// Sets the operation progress update object
///
/// The operation progress update object
void SetOperationProgress(IOperationProgress progress);
}
///
/// Helper class to allow setting multiple message sinks on a single controller
///
public class MultiMessageSink : IMessageSink
{
///
/// The sinks in this instance
///
private IMessageSink[] m_sinks;
///
/// Initializes a new instance of the class.
///
/// The sinks to use.
public MultiMessageSink(params IMessageSink[] sinks)
{
m_sinks = (sinks ?? new IMessageSink[0]).Where(x => x != null).ToArray();
}
///
/// Appends a new sink to the list
///
/// The sink to append.
public void Append(IMessageSink sink)
{
if (sink == null)
return;
var na = new IMessageSink[m_sinks.Length + 1];
Array.Copy(m_sinks, na, m_sinks.Length);
na[na.Length - 1] = sink;
m_sinks = na;
}
public void SetBackendProgress(IBackendProgress progress)
{
foreach (var s in m_sinks)
s.SetBackendProgress(progress);
}
public void SetOperationProgress(IOperationProgress progress)
{
foreach (var s in m_sinks)
s.SetOperationProgress(progress);
}
public void BackendEvent(BackendActionType action, BackendEventType type, string path, long size)
{
foreach (var s in m_sinks)
s.BackendEvent(action, type, path, size);
}
public void WriteMessage(LogEntry entry)
{
foreach (var s in m_sinks)
s.WriteMessage(entry);
}
}
///
/// State of a single backend action.
///
/// The type of action being performed.
/// The path being operated on.
/// The size of the file being transferred.
/// The current number of transferred bytes.
/// The transfer speed in bytes per second, -1 for unknown.
/// A value indicating if the backend action is blocking operation progress.
public record BackendActionProgress(
BackendActionType Action,
string Path,
long Size,
long Progress,
long BytesPerSecond,
bool IsBlocking);
///
/// Backend progress update object.
/// The engine updates these statistics very often,
/// so an event based system would take up too many resources.
/// Instead, this interface allows the client to poll
/// for updates as often as desired.
///
public interface IBackendProgress
{
///
/// Returns a snapshot of the current backend progress.
///
/// A list of backend action progress items.
BackendActionProgress[] GetActiveTransfers();
}
///
/// Interface for updating the backend progress
///
internal interface IBackendProgressUpdater
{
///
/// Register the start of a new action
///
/// The action that is starting
/// The path being operated on
/// The size of the file being transferred
void StartAction(BackendActionType action, string path, long size);
///
/// Ends an active transfer action
///
/// The action that is ending
/// The path being operated on
void EndAction(BackendActionType action, string path);
///
/// Updates the current progress
///
/// The path being operated on
/// The current number of transferred bytes
void UpdateProgress(string path, long progress);
///
/// Sets a flag indicating if the backend operation is blocking progress
///
/// If set to true the backend is blocking.
void SetBlocking(bool isBlocking);
}
///
/// Combined interface for the backend progress updater and the backend progress item
///
internal interface IBackendProgressUpdaterAndReporter : IBackendProgressUpdater, IBackendProgress
{
}
///
/// Backend progress updater instance
///
internal class BackendProgressUpdater : IBackendProgressUpdaterAndReporter
{
///
/// Lock object to provide snapshot-like access to the data
///
private readonly object m_lock = new object();
///
/// A value indicating when the last blocking was done
///
private DateTime m_blockingSince;
///
/// Information about an active transfer
///
/// The filename of the transfer
/// When the transfer started
/// The type of the transfer
/// The size of the transfer
private sealed record TransferInfo(string Filename, DateTime Started, BackendActionType Type, long Size);
///
/// The number of progress events to keep for each transfer
///
private const int MaxProgressEvents = 30;
///
/// If the backend manager is blocking progress, wait this long before considering it as blocking progress
///
private static readonly TimeSpan BlockingWaitTime = TimeSpan.FromSeconds(5);
///
/// A single recorded progress event
///
private struct ProgressEvent
{
///
/// The time in seconds since the epoch when the event was recorded
///
public long When;
///
/// The number of bytes transferred at the time of the event
///
public long Progress;
}
///
/// The information for the active transfers
///
private Dictionary m_activeTransferInfo = new();
///
/// The active transfer progress
///
///
/// The list is sorted by the time of the event, so the last item is always the most recent progress.
///
private Dictionary> m_activeTransferProgress = new();
///
public void StartAction(BackendActionType action, string path, long size)
{
lock (m_lock)
m_activeTransferInfo[path] = new TransferInfo(path, DateTime.UtcNow, action, size);
}
///
public void UpdateProgress(string path, long progress)
{
lock (m_lock)
{
var ts = (long)(DateTime.UtcNow - DateTime.UnixEpoch).TotalSeconds;
if (!m_activeTransferProgress.TryGetValue(path, out var pg))
m_activeTransferProgress[path] = pg = new List(MaxProgressEvents);
if (pg.Count == 0 || pg.Last().When != ts)
pg.Add(new ProgressEvent { When = ts, Progress = progress });
else
pg[^1] = new ProgressEvent { When = ts, Progress = progress };
// Each bucket is 1 second, so we keep the last MaxProgressEvents seconds of progress
var cutoff = ts - MaxProgressEvents;
// Remove old progress events
while (pg.Count > MaxProgressEvents || (pg.Count > 0 && pg[0].When < cutoff))
{
if (pg.Count > 0)
pg.RemoveAt(0);
else
break; // No more progress events to remove
}
}
}
///
public void EndAction(BackendActionType action, string path)
{
lock (m_lock)
{
m_activeTransferProgress.Remove(path);
m_activeTransferInfo.Remove(path);
}
}
///
public BackendActionProgress[] GetActiveTransfers()
{
lock (m_lock)
{
return m_activeTransferInfo.Values.OrderBy(x => x.Started).Select(x =>
{
var pg = m_activeTransferProgress.GetValueOrDefault(x.Filename) ?? [];
var speed = -1L;
if (pg.Count > 1)
{
var start = pg.FirstOrDefault();
var end = pg.LastOrDefault();
speed = (end.Progress - start.Progress) / (end.When - start.When);
}
return new BackendActionProgress(
x.Type,
x.Filename,
x.Size,
pg.Count > 0 ? pg.Last().Progress : 0, // Use the last recorded progress or 0 if no progress
speed,
m_blockingSince > DateTime.MinValue && (DateTime.UtcNow - m_blockingSince) > BlockingWaitTime
);
}).ToArray();
}
}
///
/// Sets a flag indicating if the backend operation is blocking progress
///
/// If set to true the backend is blocking.
public void SetBlocking(bool isBlocking)
{
lock (m_lock)
m_blockingSince = isBlocking ? DateTime.UtcNow : new DateTime(0);
}
}
public delegate void PhaseChangedDelegate(OperationPhase phase, OperationPhase previousPhase);
///
/// Operation progress update object.
/// The engine updates these statistics very often,
/// so an event based system would take up too many resources.
/// Instead, this interface allows the client to poll
/// for updates as often as desired.
///
public interface IOperationProgress
{
///
/// Update the phase, progress, filesprocessed, filesizeprocessed, filecount, filesize and countingfiles.
///
/// Phase.
/// Progress.
/// Filesprocessed.
/// Filesizeprocessed.
/// Filecount.
/// Filesize.
/// True if the filecount and filesize is incomplete, false otherwise
void UpdateOverall(out OperationPhase phase, out float progress, out long filesprocessed, out long filesizeprocessed, out long filecount, out long filesize, out bool countingfiles);
///
/// Update the filename, filesize, and fileoffset.
///
/// Filename.
/// Filesize.
/// Fileoffset.
void UpdateFile(out string filename, out long filesize, out long fileoffset, out bool filecomplete);
///
/// Gets the current remote sync destination index (1-based) and total count.
/// Returns (0, 0) when not in a remote sync phase.
///
/// The 1-based index of the current destination being synced.
/// The total number of destinations to sync.
void UpdateRemoteSyncDestination(out int destinationIndex, out int destinationCount);
///
/// Gets the estimated time remaining for the current operation, based on the
/// total file size versus the processed file size. The throughput signal is
/// smoothed with an EWMA (exponentially weighted moving average) to reduce
/// jitter from short-term fluctuations. Returns null when no estimate
/// is available (e.g. the total size is unknown or no data has been processed yet).
///
TimeSpan? EstimatedTimeToCompletion { get; }
///
/// Occurs when the phase has changed
///
event PhaseChangedDelegate PhaseChanged;
}
///
/// Interface for updating the operation progress
///
internal interface IOperationProgressUpdater
{
void UpdatePhase(OperationPhase phase);
void UpdateProgress(float progress);
void StartFile(string filename, long size);
void UpdateFileProgress(long offset);
void UpdatefileCount(long filecount, long filesize, bool done);
void UpdatefilesProcessed(long count, long size);
///
/// Sets the current remote sync destination index and total count.
///
/// The 1-based index of the current destination being synced.
/// The total number of destinations to sync.
void UpdateRemoteSyncDestination(int destinationIndex, int destinationCount);
}
internal interface IOperationProgressUpdaterAndReporter : IOperationProgressUpdater, IOperationProgress
{
}
internal class OperationProgressUpdater : IOperationProgressUpdaterAndReporter
{
private readonly object m_lock = new object();
private OperationPhase m_phase;
private float m_progress;
private string m_curfilename;
private long m_curfilesize;
private long m_curfileoffset;
private bool m_curfilecomplete;
private long m_filesprocessed;
private long m_filesizeprocessed;
private long m_filecount;
private long m_filesize;
private bool m_countingFiles;
private int m_remoteSyncDestinationIndex;
private int m_remoteSyncDestinationCount;
///
/// The time the current phase started, used as the basis for throughput calculations.
///
private DateTime m_phaseStarted;
///
/// The EWMA-smoothed throughput in bytes per second. null until the first sample.
///
private double? m_smoothedBytesPerSecond;
///
/// The most recent estimated time to completion, or null when no estimate is available.
///
private TimeSpan? m_estimatedTimeToCompletion;
///
/// The EWMA smoothing factor (alpha). Higher values weight recent samples more heavily.
/// A value around 0.2 gives a moderately smooth signal that still tracks real changes.
///
private const double EwmaAlpha = 0.2;
///
/// The minimum throughput, in bytes per second, used to compute an estimate.
/// Below this the remaining time would be unreasonably large and not meaningful.
///
private const double MinimumBytesPerSecondForEstimate = 1024;
public event PhaseChangedDelegate PhaseChanged;
public void UpdatePhase(OperationPhase phase)
{
OperationPhase prev_phase;
lock (m_lock)
{
prev_phase = m_phase;
m_phase = phase;
m_curfilename = null;
m_curfilesize = 0;
m_curfileoffset = 0;
m_curfilecomplete = false;
// A new phase has a different work profile, so any throughput and
// estimate accumulated so far no longer apply. Reset the EWMA state.
m_phaseStarted = DateTime.UtcNow;
m_smoothedBytesPerSecond = null;
m_estimatedTimeToCompletion = null;
}
if (prev_phase != phase && PhaseChanged != null)
PhaseChanged(phase, prev_phase);
}
public void UpdateProgress(float progress)
{
lock (m_lock)
m_progress = progress;
}
public void StartFile(string filename, long size)
{
lock (m_lock)
{
m_curfilename = filename;
m_curfilesize = size;
m_curfileoffset = 0;
m_curfilecomplete = false;
}
}
public void UpdateFileProgress(long offset)
{
lock (m_lock)
m_curfileoffset = offset;
}
public void UpdatefileCount(long filecount, long filesize, bool done)
{
lock (m_lock)
{
m_filecount = filecount;
m_filesize = filesize;
m_countingFiles = !done;
}
}
public void UpdatefilesProcessed(long count, long size)
{
lock (m_lock)
{
var now = DateTime.UtcNow;
// If no phase start has been recorded yet (e.g. progress reported
// before UpdatePhase), anchor the EWMA window to the first sample.
if (m_phaseStarted == DateTime.MinValue)
m_phaseStarted = now;
var elapsedSeconds = (now - m_phaseStarted).TotalSeconds;
if (elapsedSeconds > 0)
{
// Instantaneous throughput over the whole phase so far.
var instantBytesPerSecond = size / elapsedSeconds;
if (!m_smoothedBytesPerSecond.HasValue)
m_smoothedBytesPerSecond = instantBytesPerSecond;
else
m_smoothedBytesPerSecond =
(EwmaAlpha * instantBytesPerSecond) +
((1.0 - EwmaAlpha) * m_smoothedBytesPerSecond.Value);
}
m_filesprocessed = count;
m_filesizeprocessed = size;
m_curfilecomplete = true;
m_estimatedTimeToCompletion = ComputeEstimatedTimeToCompletion();
}
}
///
/// Computes the estimated time to completion from the total size, the processed
/// size and the EWMA-smoothed throughput. Returns null when no meaningful
/// estimate can be produced.
///
private TimeSpan? ComputeEstimatedTimeToCompletion()
{
var rate = m_smoothedBytesPerSecond;
if (!rate.HasValue || rate.Value < MinimumBytesPerSecondForEstimate)
return null;
// While the file count is still being tallied the total size is incomplete,
// so any remaining-time estimate would be misleading.
if (m_countingFiles)
return null;
var remaining = m_filesize - m_filesizeprocessed;
if (remaining <= 0)
return TimeSpan.Zero;
return TimeSpan.FromSeconds(remaining / rate.Value);
}
public void UpdateRemoteSyncDestination(int destinationIndex, int destinationCount)
{
lock (m_lock)
{
m_remoteSyncDestinationIndex = destinationIndex;
m_remoteSyncDestinationCount = destinationCount;
}
}
///
void IOperationProgress.UpdateRemoteSyncDestination(out int destinationIndex, out int destinationCount)
{
lock (m_lock)
{
destinationIndex = m_remoteSyncDestinationIndex;
destinationCount = m_remoteSyncDestinationCount;
}
}
///
/// Update the phase, progress, filesprocessed, filesizeprocessed, filecount, filesize and countingfiles.
///
/// Phase.
/// Progress.
/// Filesprocessed.
/// Filesizeprocessed.
/// Filecount.
/// Filesize.
/// True if the filecount and filesize is incomplete, false otherwise
public void UpdateOverall(out OperationPhase phase, out float progress, out long filesprocessed, out long filesizeprocessed, out long filecount, out long filesize, out bool countingfiles)
{
lock (m_lock)
{
phase = m_phase;
filesize = m_filesize;
progress = m_progress;
filesprocessed = m_filesprocessed;
filesizeprocessed = m_filesizeprocessed;
filecount = m_filecount;
countingfiles = m_countingFiles;
}
}
///
/// Gets the EWMA-smoothed estimated time to completion, or null when no
/// estimate is available. The estimate is recomputed whenever processed-file
/// statistics are updated, based on the total size versus the processed size.
///
public TimeSpan? EstimatedTimeToCompletion
{
get
{
lock (m_lock)
return m_estimatedTimeToCompletion;
}
}
///
/// Update the filename, filesize, and fileoffset.
///
/// Filename.
/// Filesize.
/// Fileoffset.
public void UpdateFile(out string filename, out long filesize, out long fileoffset, out bool filecomplete)
{
lock (m_lock)
{
filename = m_curfilename;
filesize = m_curfilesize;
fileoffset = m_curfileoffset;
filecomplete = m_curfilecomplete;
}
}
}
}