Kenneth Skovhede
a8327b40aa
The SelfOnly mode was removed as was the "removeAdditionalSources" logic. Instead, the new SelfLimited mode always removes sensitive information, such that by default we do not store any sensitive information, not even with encryption enabled. This is required to avoid accidentially leaking secrets found in remote sources, or the target url, in case the leak happens somewhere else than on the remote destination. Additionally, this resolves an issue where remote source information could leak via the DisplayNames property.
1396 lines
64 KiB
C#
1396 lines
64 KiB
C#
// Copyright (C) 2026, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
|
|
#nullable enable
|
|
|
|
using System;
|
|
using System.Linq;
|
|
using System.Collections.Generic;
|
|
using Duplicati.Library.Interface;
|
|
using Duplicati.Server.Serialization;
|
|
using Duplicati.Library.RestAPI;
|
|
using Duplicati.Library.Utility;
|
|
using System.Threading.Tasks;
|
|
using Duplicati.Server.Database;
|
|
using Duplicati.Server.Serialization.Interface;
|
|
using Duplicati.WebserverCore.Abstractions;
|
|
using System.Threading;
|
|
using System.Text.Json;
|
|
using System.Globalization;
|
|
using CoCoL;
|
|
|
|
namespace Duplicati.Server
|
|
{
|
|
public static class Runner
|
|
{
|
|
public const string TaskSetupFilename = "task-setup.json";
|
|
|
|
public interface IRunnerData : Serialization.Interface.IQueuedTask
|
|
{
|
|
Serialization.Interface.IBackup? Backup { get; }
|
|
IDictionary<string, string?>? ExtraOptions { get; }
|
|
string[]? FilterStrings { get; }
|
|
string[]? ExtraArguments { get; }
|
|
int PageSize { get; }
|
|
int PageOffset { get; }
|
|
bool ReturnExtendedData { get; }
|
|
bool CaseSensitiveSearch { get; }
|
|
bool SearchMetadata { get; }
|
|
void SetController(Library.Main.IController? controller);
|
|
}
|
|
|
|
private class RunnerData : IRunnerData
|
|
{
|
|
private static long RunnerTaskID = 1;
|
|
|
|
public Func<Task>? OnStarting { get; set; }
|
|
public Func<Exception?, Task>? OnFinished { get; set; }
|
|
|
|
/// <summary>
|
|
/// Callback to be executed when the task is finished.
|
|
/// Only supported for the delete operation for now.
|
|
/// </summary>
|
|
internal Action<IRunnerData>? AfterTaskFinished { get; set; }
|
|
|
|
public DuplicatiOperation Operation { get; internal set; }
|
|
public Serialization.Interface.IBackup? Backup { get; internal set; }
|
|
public IDictionary<string, string?>? ExtraOptions { get; internal set; }
|
|
public string[]? FilterStrings { get; internal set; }
|
|
|
|
public string? BackupID { get { return Backup?.ID; } }
|
|
public long TaskID { get { return m_taskID; } }
|
|
|
|
public string[]? ExtraArguments { get; internal set; }
|
|
public int PageSize { get; internal set; } = 0;
|
|
public int PageOffset { get; internal set; } = 0;
|
|
public bool ReturnExtendedData { get; internal set; } = false;
|
|
public bool CaseSensitiveSearch { get; internal set; } = false;
|
|
public bool SearchMetadata { get; internal set; } = false;
|
|
|
|
public DateTime? TaskStarted { get; set; }
|
|
public DateTime? TaskFinished { get; set; }
|
|
|
|
internal Library.Main.IController? Controller { get; set; }
|
|
|
|
public void SetController(Library.Main.IController? controller)
|
|
{
|
|
Controller = controller;
|
|
}
|
|
|
|
public async Task StopAsync()
|
|
{
|
|
if (Controller != null)
|
|
await Controller.StopAsync().ConfigureAwait(false);
|
|
}
|
|
|
|
public async Task AbortAsync()
|
|
{
|
|
if (Controller != null)
|
|
await Controller.AbortAsync().ConfigureAwait(false);
|
|
}
|
|
|
|
public async Task PauseAsync(bool alsoTransfers)
|
|
{
|
|
if (Controller != null)
|
|
await Controller.PauseAsync(alsoTransfers).ConfigureAwait(false);
|
|
}
|
|
|
|
public async Task ResumeAsync()
|
|
{
|
|
if (Controller != null)
|
|
await Controller.ResumeAsync().ConfigureAwait(false);
|
|
}
|
|
|
|
public long OriginalUploadSpeed { get; set; }
|
|
public long OriginalDownloadSpeed { get; set; }
|
|
|
|
public async Task UpdateThrottleSpeedsAsync(string? uploadSpeed, string? downloadSpeed)
|
|
{
|
|
var controller = this.Controller;
|
|
if (controller == null)
|
|
return;
|
|
|
|
var job_upload_throttle = this.OriginalUploadSpeed <= 0 ? long.MaxValue : this.OriginalUploadSpeed;
|
|
var job_download_throttle = this.OriginalDownloadSpeed <= 0 ? long.MaxValue : this.OriginalDownloadSpeed;
|
|
|
|
var server_upload_throttle = long.MaxValue;
|
|
var server_download_throttle = long.MaxValue;
|
|
|
|
try
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(uploadSpeed))
|
|
server_upload_throttle = Sizeparser.ParseSize(uploadSpeed, "kb");
|
|
}
|
|
catch { }
|
|
|
|
try
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(downloadSpeed))
|
|
server_download_throttle = Sizeparser.ParseSize(downloadSpeed, "kb");
|
|
}
|
|
catch { }
|
|
|
|
var upload_throttle = Math.Min(job_upload_throttle, server_upload_throttle);
|
|
var download_throttle = Math.Min(job_download_throttle, server_download_throttle);
|
|
|
|
if (upload_throttle <= 0 || upload_throttle == long.MaxValue)
|
|
upload_throttle = 0;
|
|
|
|
if (download_throttle <= 0 || download_throttle == long.MaxValue)
|
|
download_throttle = 0;
|
|
|
|
await controller.SetThrottleSpeedsAsync(upload_throttle, download_throttle).ConfigureAwait(false);
|
|
}
|
|
|
|
private readonly long m_taskID;
|
|
|
|
public RunnerData()
|
|
{
|
|
m_taskID = System.Threading.Interlocked.Increment(ref RunnerTaskID);
|
|
}
|
|
}
|
|
|
|
private class CustomRunnerTask : RunnerData
|
|
{
|
|
public readonly Action<Library.Main.IMessageSink> Run;
|
|
|
|
public CustomRunnerTask(Action<Library.Main.IMessageSink> runner)
|
|
: base()
|
|
{
|
|
if (runner == null)
|
|
throw new ArgumentNullException(nameof(runner));
|
|
Run = runner;
|
|
Operation = DuplicatiOperation.CustomRunner;
|
|
Backup = new Database.Backup();
|
|
}
|
|
}
|
|
|
|
public static IRunnerData CreateCustomTask(Action<Library.Main.IMessageSink> runner)
|
|
{
|
|
return new CustomRunnerTask(runner);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Parses the raw option value into a <see cref="StoreTaskConfigMode"/>.
|
|
/// For backwards compatibility, boolean true is treated as <see cref="StoreTaskConfigMode.Self"/>,
|
|
/// boolean false as <see cref="StoreTaskConfigMode.None"/>, and null/whitespace as <see cref="StoreTaskConfigMode.Auto"/>.
|
|
/// </summary>
|
|
/// <param name="rawValue">The raw option value.</param>
|
|
/// <returns>The parsed mode.</returns>
|
|
public static StoreTaskConfigMode ParseStoreTaskConfigMode(string? rawValue)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(rawValue))
|
|
return StoreTaskConfigMode.Auto;
|
|
|
|
// Handle boolean values for backwards compatibility, use a lambda capture to see if we hit the default case
|
|
var usedDefault = false;
|
|
var parsedBool = Utility.ParseBool(rawValue, () => { usedDefault = true; return false; });
|
|
if (!usedDefault)
|
|
return parsedBool ? StoreTaskConfigMode.Self : StoreTaskConfigMode.None;
|
|
|
|
return Utility.ParseEnum(rawValue, StoreTaskConfigMode.Auto);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Determines whether the backup is configured to use encryption.
|
|
/// </summary>
|
|
/// <param name="options">The backup options dictionary.</param>
|
|
/// <returns>True if encryption is enabled; otherwise, false.</returns>
|
|
private static bool IsBackupEncryptionEnabled(Dictionary<string, string?> options)
|
|
{
|
|
// In principle, this check should be enough
|
|
if (Utility.ParseBoolOption(options, "no-encryption"))
|
|
return false;
|
|
|
|
// But since we explicitly set the encryption module, we also check that this is set
|
|
if (string.IsNullOrWhiteSpace(options.GetValueOrDefault("encryption-module")))
|
|
return false;
|
|
|
|
// Also, we need a passphrase
|
|
if (string.IsNullOrWhiteSpace(options.GetValueOrDefault("passphrase")))
|
|
return false;
|
|
|
|
return true;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Prepares a backup for export, optionally removing sensitive information.
|
|
/// </summary>
|
|
/// <param name="databaseConnection">The database connection.</param>
|
|
/// <param name="backup">The backup to export.</param>
|
|
/// <param name="removeSecrets">Whether to remove secrets in the export.</param>
|
|
/// <returns>The import/export structure.</returns>
|
|
private static Serializable.ImportExportStructure PrepareBackupForExport(Connection databaseConnection, IBackup backup, bool removeSecrets)
|
|
{
|
|
var prepped = ((Backup)backup).Clone();
|
|
if (removeSecrets)
|
|
prepped.RemoveSensitiveInformation();
|
|
|
|
return databaseConnection.PrepareBackupForExport(prepped);
|
|
}
|
|
|
|
public static IRunnerData CreateTask(DuplicatiOperation operation, IBackup backup, IDictionary<string, string?>? extraOptions = null, string[]? filterStrings = null, string[]? extraArguments = null, int pageSize = 0, int pageOffset = 0, bool returnExtended = false, bool caseSensitiveSearch = false, bool includeMetadata = false)
|
|
{
|
|
return new RunnerData()
|
|
{
|
|
Operation = operation,
|
|
Backup = backup,
|
|
ExtraOptions = extraOptions ?? new Dictionary<string, string?>(),
|
|
FilterStrings = filterStrings,
|
|
ExtraArguments = extraArguments,
|
|
PageSize = pageSize,
|
|
PageOffset = pageOffset,
|
|
ReturnExtendedData = returnExtended,
|
|
CaseSensitiveSearch = caseSensitiveSearch,
|
|
SearchMetadata = includeMetadata
|
|
};
|
|
}
|
|
|
|
public static IRunnerData CreateDeleteTask(IBackup backup, IDictionary<string, string?> extraOptions, Action<IRunnerData>? afterTaskFinished = null)
|
|
{
|
|
return new RunnerData()
|
|
{
|
|
Operation = DuplicatiOperation.Delete,
|
|
Backup = backup,
|
|
ExtraOptions = extraOptions,
|
|
AfterTaskFinished = afterTaskFinished
|
|
};
|
|
}
|
|
|
|
public static IRunnerData CreateListTask(IBackup backup, string[]? filters, bool onlyPrefix, bool allVersions, bool folderContents, DateTime time)
|
|
{
|
|
var dict = new Dictionary<string, string?>();
|
|
if (onlyPrefix)
|
|
dict["list-prefix-only"] = "true";
|
|
if (allVersions)
|
|
dict["all-versions"] = "true";
|
|
if (time.Ticks > 0)
|
|
dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime());
|
|
if (folderContents)
|
|
dict["list-folder-contents"] = "true";
|
|
|
|
return CreateTask(
|
|
DuplicatiOperation.List,
|
|
backup,
|
|
dict,
|
|
filters);
|
|
}
|
|
|
|
public static IRunnerData CreateListFilesetsTask(IBackup backup, Dictionary<string, string?>? extraOptions = null)
|
|
{
|
|
return CreateTask(
|
|
DuplicatiOperation.ListFilesets,
|
|
backup,
|
|
extraOptions ?? new Dictionary<string, string?>());
|
|
}
|
|
|
|
public static IRunnerData CreateListFolderContents(IBackup backup, string[]? folders, DateTime time, int pageSize, int pageOffset, bool returnExtended)
|
|
{
|
|
var dict = new Dictionary<string, string?>();
|
|
if (time.Ticks > 0)
|
|
dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime());
|
|
|
|
return CreateTask(
|
|
DuplicatiOperation.ListFolderContents,
|
|
backup,
|
|
dict,
|
|
extraArguments: folders,
|
|
pageSize: pageSize,
|
|
pageOffset: pageOffset,
|
|
returnExtended: returnExtended);
|
|
}
|
|
|
|
public static IRunnerData ListFileVersionsTask(IBackup backup, string[]? filepaths, int pageSize, int pageOffset)
|
|
{
|
|
var dict = new Dictionary<string, string?>();
|
|
return CreateTask(
|
|
DuplicatiOperation.ListFileVersions,
|
|
backup,
|
|
dict,
|
|
extraArguments: filepaths,
|
|
pageSize: pageSize,
|
|
pageOffset: pageOffset);
|
|
}
|
|
|
|
public static IRunnerData CreateSearchEntriesTask(IBackup backup, string[]? filters, bool caseSensitive, string[]? folders, DateTime time, long[]? version, int pageSize, int pageOffset, bool returnExtendedData, bool searchMetadata)
|
|
{
|
|
var dict = new Dictionary<string, string?>();
|
|
if (time.Ticks > 0)
|
|
dict["time"] = Utility.SerializeDateTime(time.ToUniversalTime());
|
|
if (version != null)
|
|
dict["version"] = string.Join(",", version.Select(v => v.ToString(CultureInfo.InvariantCulture)));
|
|
|
|
return CreateTask(
|
|
DuplicatiOperation.SearchEntries,
|
|
backup,
|
|
dict,
|
|
filters,
|
|
extraArguments: folders,
|
|
pageSize: pageSize,
|
|
pageOffset: pageOffset,
|
|
returnExtended: returnExtendedData,
|
|
caseSensitiveSearch: caseSensitive,
|
|
includeMetadata: searchMetadata);
|
|
}
|
|
|
|
|
|
public static IRunnerData CreateRestoreTask(IBackup backup, string[]? filters,
|
|
DateTime time, string? restoreTarget, bool overwrite, bool restore_permissions,
|
|
bool skip_metadata, string? passphrase)
|
|
{
|
|
var dict = new Dictionary<string, string?>
|
|
{
|
|
["time"] = Utility.SerializeDateTime(time.ToUniversalTime()),
|
|
["overwrite"] = overwrite ? bool.TrueString : bool.FalseString,
|
|
["restore-permissions"] = restore_permissions ? bool.TrueString : bool.FalseString,
|
|
["skip-metadata"] = skip_metadata ? bool.TrueString : bool.FalseString,
|
|
["allow-passphrase-change"] = bool.TrueString
|
|
};
|
|
if (!string.IsNullOrWhiteSpace(restoreTarget))
|
|
dict["restore-path"] = SpecialFolders.ExpandEnvironmentVariables(restoreTarget);
|
|
if (!(passphrase is null))
|
|
dict["passphrase"] = passphrase;
|
|
|
|
return CreateTask(
|
|
DuplicatiOperation.Restore,
|
|
backup,
|
|
dict,
|
|
filters);
|
|
}
|
|
|
|
public static IRunnerData CreateRestoreControlFilesTask(IBackup backup, string restorePath, string[] files)
|
|
{
|
|
var dict = new Dictionary<string, string?>
|
|
{
|
|
["restore-path"] = restorePath
|
|
};
|
|
|
|
return CreateTask(
|
|
DuplicatiOperation.RestoreControlFiles,
|
|
backup,
|
|
dict,
|
|
files);
|
|
}
|
|
private class MessageSink : Library.Main.IMessageSink
|
|
{
|
|
private class ProgressState : IProgressEventData
|
|
{
|
|
private readonly string? m_backupID;
|
|
private readonly long m_taskID;
|
|
|
|
internal Library.Main.BackendActionType m_backendAction;
|
|
internal string? m_backendPath;
|
|
internal long m_backendFileSize;
|
|
internal long m_backendFileProgress;
|
|
internal long m_backendSpeed;
|
|
internal bool m_backendIsBlocking;
|
|
|
|
internal ActiveTransfer[] m_activeTransfers = [];
|
|
|
|
internal string? m_currentFilename;
|
|
internal long m_currentFilesize;
|
|
internal long m_currentFileoffset;
|
|
internal bool m_currentFilecomplete;
|
|
|
|
internal Library.Main.OperationPhase m_phase;
|
|
internal float m_overallProgress;
|
|
internal long m_processedFileCount;
|
|
internal long m_processedFileSize;
|
|
internal long m_totalFileCount;
|
|
internal long m_totalFileSize;
|
|
internal bool m_stillCounting;
|
|
|
|
internal int m_remoteSyncDestinationIndex;
|
|
internal int m_remoteSyncDestinationCount;
|
|
|
|
public ProgressState(long taskId, string? backupId)
|
|
{
|
|
m_backupID = backupId;
|
|
m_taskID = taskId;
|
|
}
|
|
|
|
internal ProgressState Clone()
|
|
{
|
|
var res = (ProgressState)this.MemberwiseClone();
|
|
res.m_activeTransfers = [.. m_activeTransfers];
|
|
return res;
|
|
}
|
|
|
|
#region IProgressEventData implementation
|
|
public string? BackupID { get { return m_backupID; } }
|
|
public long TaskID { get { return m_taskID; } }
|
|
public string BackendAction { get { return m_backendAction.ToString(); } }
|
|
public string? BackendPath { get { return m_backendPath; } }
|
|
public long BackendFileSize { get { return m_backendFileSize; } }
|
|
public long BackendFileProgress { get { return m_backendFileProgress; } }
|
|
public long BackendSpeed { get { return m_backendSpeed; } }
|
|
public bool BackendIsBlocking { get { return m_backendIsBlocking; } }
|
|
public string? CurrentFilename { get { return m_currentFilename; } }
|
|
public long CurrentFilesize { get { return m_currentFilesize; } }
|
|
public long CurrentFileoffset { get { return m_currentFileoffset; } }
|
|
public bool CurrentFilecomplete { get { return m_currentFilecomplete; } }
|
|
public string Phase { get { return m_phase.ToString(); } }
|
|
public float OverallProgress { get { return m_overallProgress; } }
|
|
public long ProcessedFileCount { get { return m_processedFileCount; } }
|
|
public long ProcessedFileSize { get { return m_processedFileSize; } }
|
|
public long TotalFileCount { get { return m_totalFileCount; } }
|
|
public long TotalFileSize { get { return m_totalFileSize; } }
|
|
public bool StillCounting { get { return m_stillCounting; } }
|
|
public ActiveTransfer[] ActiveTransfers => m_activeTransfers;
|
|
public int RemoteSyncDestinationIndex { get { return m_remoteSyncDestinationIndex; } }
|
|
public int RemoteSyncDestinationCount { get { return m_remoteSyncDestinationCount; } }
|
|
|
|
#endregion
|
|
}
|
|
|
|
private readonly ProgressState m_state;
|
|
private Library.Main.IBackendProgress? m_backendProgress;
|
|
private Library.Main.IOperationProgress? m_operationProgress;
|
|
private readonly object m_lock = new object();
|
|
internal TaskCompletionSource<Library.Main.OperationPhase> m_phaseChangeTcs = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
|
|
|
public MessageSink(long taskId, string? backupId)
|
|
{
|
|
m_state = new ProgressState(taskId, backupId);
|
|
}
|
|
|
|
public Serialization.Interface.IProgressEventData Copy()
|
|
{
|
|
lock (m_lock)
|
|
{
|
|
if (m_backendProgress != null)
|
|
{
|
|
var transfers = m_backendProgress.GetActiveTransfers();
|
|
m_state.m_activeTransfers = transfers
|
|
.Select(st => new ActiveTransfer(
|
|
st.Action.ToString(),
|
|
st.Path,
|
|
st.Size,
|
|
st.Progress,
|
|
st.BytesPerSecond,
|
|
st.IsBlocking
|
|
)).ToArray();
|
|
|
|
if (transfers.Any())
|
|
{
|
|
var st = transfers.First();
|
|
m_state.m_backendAction = st.Action;
|
|
m_state.m_backendPath = st.Path;
|
|
m_state.m_backendFileSize = st.Size;
|
|
m_state.m_backendFileProgress = st.Progress;
|
|
m_state.m_backendSpeed = st.BytesPerSecond;
|
|
m_state.m_backendIsBlocking = st.IsBlocking;
|
|
}
|
|
else
|
|
{
|
|
m_state.m_backendAction = Library.Main.BackendActionType.Get;
|
|
m_state.m_backendPath = null;
|
|
m_state.m_backendFileSize = 0;
|
|
m_state.m_backendFileProgress = 0;
|
|
m_state.m_backendSpeed = -1;
|
|
m_state.m_backendIsBlocking = false;
|
|
}
|
|
}
|
|
if (m_operationProgress != null)
|
|
{
|
|
m_operationProgress.UpdateFile(out m_state.m_currentFilename, out m_state.m_currentFilesize, out m_state.m_currentFileoffset, out m_state.m_currentFilecomplete);
|
|
m_operationProgress.UpdateOverall(out m_state.m_phase, out m_state.m_overallProgress, out m_state.m_processedFileCount, out m_state.m_processedFileSize, out m_state.m_totalFileCount, out m_state.m_totalFileSize, out m_state.m_stillCounting);
|
|
m_operationProgress.UpdateRemoteSyncDestination(out m_state.m_remoteSyncDestinationIndex, out m_state.m_remoteSyncDestinationCount);
|
|
}
|
|
|
|
return m_state.Clone();
|
|
}
|
|
}
|
|
|
|
public Task PhaseChangedAsync { get { return m_phaseChangeTcs.Task; } }
|
|
|
|
#region IMessageSink implementation
|
|
public void BackendEvent(Duplicati.Library.Main.BackendActionType action, Duplicati.Library.Main.BackendEventType type, string path, long size)
|
|
{
|
|
lock (m_lock)
|
|
{
|
|
if (path == m_state.m_currentFilename && type != Duplicati.Library.Main.BackendEventType.Started && type != Duplicati.Library.Main.BackendEventType.Progress)
|
|
{
|
|
m_state.m_backendFileSize = 0;
|
|
m_state.m_backendFileProgress = 0;
|
|
m_state.m_backendSpeed = 0;
|
|
}
|
|
}
|
|
}
|
|
|
|
public void SetBackendProgress(Library.Main.IBackendProgress progress)
|
|
{
|
|
lock (m_lock)
|
|
m_backendProgress = progress;
|
|
}
|
|
|
|
public void SetOperationProgress(Library.Main.IOperationProgress progress)
|
|
{
|
|
lock (m_lock)
|
|
{
|
|
if (m_operationProgress != null)
|
|
m_operationProgress.PhaseChanged -= OperationProgress_PhaseChanged;
|
|
m_operationProgress = progress;
|
|
|
|
if (m_operationProgress != null)
|
|
m_operationProgress.PhaseChanged += OperationProgress_PhaseChanged;
|
|
}
|
|
}
|
|
|
|
private void OperationProgress_PhaseChanged(Library.Main.OperationPhase phase, Library.Main.OperationPhase previousPhase)
|
|
{
|
|
var prev = Interlocked.Exchange(ref m_phaseChangeTcs, new TaskCompletionSource<Library.Main.OperationPhase>(TaskCreationOptions.RunContinuationsAsynchronously));
|
|
prev.TrySetResult(phase);
|
|
}
|
|
|
|
public void WriteMessage(Library.Logging.LogEntry entry)
|
|
{
|
|
// Do nothing. Implementation needed for ILogDestination interface.
|
|
}
|
|
#endregion
|
|
}
|
|
public static string GetCommandLine(Connection databaseConnection, IRunnerData data)
|
|
{
|
|
var backup = data.Backup;
|
|
if (backup == null)
|
|
throw new ArgumentNullException(nameof(backup));
|
|
|
|
var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection), out var url);
|
|
if (data.ExtraOptions != null)
|
|
foreach (var k in data.ExtraOptions)
|
|
options[k.Key] = k.Value;
|
|
|
|
var cf = databaseConnection.Filters;
|
|
var bf = backup.Filters;
|
|
|
|
var sources =
|
|
(from n in backup.Sources
|
|
let p = SpecialFolders.ExpandEnvironmentVariables(n)
|
|
where !string.IsNullOrWhiteSpace(p)
|
|
select p).ToArray();
|
|
|
|
var exe = System.IO.Path.Combine(
|
|
Library.AutoUpdater.UpdaterManager.INSTALLATIONDIR,
|
|
Library.AutoUpdater.PackageHelper.GetExecutableName(Library.AutoUpdater.PackageHelper.NamedExecutable.CommandLine)
|
|
);
|
|
|
|
var cmd = new System.Text.StringBuilder();
|
|
cmd.Append(Utility.WrapAsCommandLine([exe, "backup", url], false));
|
|
|
|
cmd.Append(" ");
|
|
cmd.Append(Utility.WrapAsCommandLine(sources, true));
|
|
|
|
// TODO: We should check each option to see if it is a path, and allow expansion on that
|
|
foreach (var opt in options)
|
|
cmd.AppendFormat(" --{0}={1}", opt.Key, Utility.WrapCommandLineElement(opt.Value, false));
|
|
|
|
if (cf != null)
|
|
foreach (var f in cf)
|
|
cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Utility.WrapCommandLineElement(f.Expression, true));
|
|
|
|
if (bf != null)
|
|
foreach (var f in bf)
|
|
cmd.AppendFormat(" --{0}={1}", f.Include ? "include" : "exclude", Utility.WrapCommandLineElement(f.Expression, true));
|
|
|
|
return cmd.ToString();
|
|
}
|
|
|
|
public static string[] GetCommandLineParts(Connection databaseConnection, IRunnerData data)
|
|
{
|
|
var backup = data.Backup;
|
|
if (backup == null)
|
|
throw new ArgumentNullException(nameof(backup));
|
|
|
|
var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection), out var url);
|
|
if (data.ExtraOptions != null)
|
|
foreach (var k in data.ExtraOptions)
|
|
options[k.Key] = k.Value;
|
|
|
|
var cf = databaseConnection.Filters;
|
|
var bf = backup.Filters;
|
|
|
|
var sources =
|
|
(from n in backup.Sources
|
|
let p = SpecialFolders.ExpandEnvironmentVariables(n)
|
|
where !string.IsNullOrWhiteSpace(p)
|
|
select p).ToArray();
|
|
|
|
var parts = new List<string>
|
|
{
|
|
url
|
|
};
|
|
parts.AddRange(sources);
|
|
|
|
foreach (var opt in options)
|
|
parts.Add(string.Format("--{0}={1}", opt.Key, opt.Value));
|
|
|
|
if (cf != null)
|
|
foreach (var f in cf)
|
|
parts.Add(string.Format("--{0}={1}", f.Include ? "include" : "exclude", f.Expression));
|
|
|
|
if (bf != null)
|
|
foreach (var f in bf)
|
|
parts.Add(string.Format("--{0}={1}", f.Include ? "include" : "exclude", f.Expression));
|
|
|
|
return parts.ToArray();
|
|
}
|
|
|
|
public static Task<IBasicResults?> RunAsync(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IApplicationSettings applicationSettings, IQueuedTask data, bool fromQueue)
|
|
{
|
|
if (data is IRunnerData runnerData)
|
|
return RunInternalAsync(databaseConnection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, runnerData, fromQueue);
|
|
|
|
throw new ArgumentException("Invalid task type", nameof(data));
|
|
}
|
|
|
|
private static async Task<IBasicResults?> RunInternalAsync(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, IApplicationSettings applicationSettings, IRunnerData data, bool fromQueue)
|
|
{
|
|
data.TaskStarted = DateTime.Now;
|
|
if (data is CustomRunnerTask task)
|
|
{
|
|
try
|
|
{
|
|
var sink = new MessageSink(task.TaskID, null);
|
|
progressStateProviderService.GenerateProgressState = sink.Copy;
|
|
eventPollNotify.SignalNewEvent();
|
|
eventPollNotify.SignalProgressUpdate(sink.Copy);
|
|
|
|
// Attach a log scope that tags all messages to relay the TaskID and BackupID
|
|
using var _ = Library.Logging.Log.StartScope(log =>
|
|
{
|
|
log[ILogWriteHandler.LiveLogEntry.LOG_EXTRA_TASKID] = data.TaskID.ToString();
|
|
});
|
|
|
|
// Keep emitting progress updates during the operation
|
|
using var cts = new CancellationTokenSource();
|
|
Task.Run(async () =>
|
|
{
|
|
while (!cts.IsCancellationRequested)
|
|
{
|
|
await Task.WhenAny(sink.PhaseChangedAsync, Task.Delay(1000, cts.Token));
|
|
if (!cts.IsCancellationRequested)
|
|
eventPollNotify.SignalProgressUpdate(sink.Copy);
|
|
}
|
|
}).FireAndForget();
|
|
|
|
task.Run(sink);
|
|
eventPollNotify.SignalProgressUpdate(sink.Copy);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
databaseConnection.LogError(string.Empty, "Failed while executing custom task", ex);
|
|
}
|
|
finally
|
|
{
|
|
eventPollNotify.SignalProgressUpdate(null);
|
|
data.TaskFinished = DateTime.Now;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
var backup = data.Backup;
|
|
if (backup == null)
|
|
throw new ArgumentNullException(nameof(backup));
|
|
|
|
backup.Metadata ??= new Dictionary<string, string>();
|
|
TempFolder? tempfolder = null;
|
|
|
|
try
|
|
{
|
|
var sink = new MessageSink(data.TaskID, backup.ID);
|
|
using var cts = new CancellationTokenSource();
|
|
// Non-queue are "wrong" tasks that are running directly and often cause
|
|
// timeouts. They should be removed, but for now we keep them,
|
|
// but do not report progress updates.
|
|
if (fromQueue)
|
|
{
|
|
progressStateProviderService.GenerateProgressState = () => sink.Copy();
|
|
eventPollNotify.SignalNewEvent();
|
|
eventPollNotify.SignalProgressUpdate(sink.Copy);
|
|
|
|
// Keep emitting progress updates during the operation
|
|
Task.Run(async () =>
|
|
{
|
|
while (!cts.IsCancellationRequested)
|
|
{
|
|
await Task.WhenAny(sink.PhaseChangedAsync, Task.Delay(1000, cts.Token));
|
|
if (!cts.IsCancellationRequested)
|
|
eventPollNotify.SignalProgressUpdate(sink.Copy);
|
|
}
|
|
}).FireAndForget();
|
|
}
|
|
|
|
var options = ApplyOptions(databaseConnection, backup, GetCommonOptions(databaseConnection), out var url);
|
|
if (data.ExtraOptions != null)
|
|
foreach (var k in data.ExtraOptions)
|
|
options[k.Key] = k.Value;
|
|
|
|
// Map AdditionalTargetURLs to remote-sync-json-config option for backup operations
|
|
if (data.Operation == DuplicatiOperation.Backup)
|
|
ApplyAdditionalTargetUrls(backup, options);
|
|
|
|
// Pack in the system or task config for easy restore
|
|
if (data.Operation == DuplicatiOperation.Backup)
|
|
tempfolder = StoreTaskConfigAndGetTempFolder(databaseConnection, data, options);
|
|
|
|
var useOutOfProcess = databaseConnection.ApplicationSettings.UseOutOfProcessController;
|
|
|
|
// Attach a log scope that tags all messages to relay the TaskID and BackupID
|
|
using (Library.Logging.Log.StartScope(log =>
|
|
{
|
|
log[ILogWriteHandler.LiveLogEntry.LOG_EXTRA_TASKID] = data.TaskID.ToString();
|
|
log[ILogWriteHandler.LiveLogEntry.LOG_EXTRA_BACKUPID] = data.BackupID;
|
|
}))
|
|
using (tempfolder)
|
|
using (var controller = await CreateControllerAsync(url, options, sink, useOutOfProcess).ConfigureAwait(false))
|
|
{
|
|
try
|
|
{
|
|
if (options.ContainsKey("throttle-upload"))
|
|
((RunnerData)data).OriginalUploadSpeed = Duplicati.Library.Utility.Sizeparser.ParseSize(options["throttle-upload"], "kb");
|
|
}
|
|
catch { }
|
|
|
|
try
|
|
{
|
|
if (options.ContainsKey("throttle-download"))
|
|
((RunnerData)data).OriginalDownloadSpeed = Duplicati.Library.Utility.Sizeparser.ParseSize(options["throttle-download"], "kb");
|
|
}
|
|
catch { }
|
|
|
|
((RunnerData)data).Controller = controller;
|
|
var appSettings = databaseConnection.ApplicationSettings;
|
|
await data.UpdateThrottleSpeedsAsync(appSettings.UploadSpeedLimit, appSettings.DownloadSpeedLimit).ConfigureAwait(false);
|
|
|
|
// Pass on the provider, will be replaced if configured in the backup
|
|
await controller.SetSecretProviderAsync(applicationSettings.SecretProvider).ConfigureAwait(false);
|
|
|
|
if (backup.Metadata.ContainsKey("LastCompactFinished"))
|
|
await controller.SetLastCompactAsync(Utility.DeserializeDateTime(backup.Metadata["LastCompactFinished"])).ConfigureAwait(false);
|
|
|
|
if (backup.Metadata.ContainsKey("LastVacuumFinished"))
|
|
await controller.SetLastVacuumAsync(Utility.DeserializeDateTime(backup.Metadata["LastVacuumFinished"])).ConfigureAwait(false);
|
|
|
|
switch (data.Operation)
|
|
{
|
|
case DuplicatiOperation.Backup:
|
|
{
|
|
var filter = ApplyFilter(backup, GetCommonFilter(databaseConnection));
|
|
var sources = backup.Sources
|
|
.Select(n => SpecialFolders.ExpandEnvironmentVariables(n))
|
|
.WhereNotNullOrWhiteSpace()
|
|
.ToArray();
|
|
|
|
var r = await controller.BackupAsync(sources, filter).ConfigureAwait(false);
|
|
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
|
|
return r;
|
|
}
|
|
case DuplicatiOperation.List:
|
|
{
|
|
var r = await controller.ListAsync(data.FilterStrings, null).ConfigureAwait(false);
|
|
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
|
|
return r;
|
|
}
|
|
case DuplicatiOperation.Repair:
|
|
{
|
|
var r = await controller.RepairAsync(data.FilterStrings == null ? null : new Library.Utility.FilterExpression(data.FilterStrings)).ConfigureAwait(false);
|
|
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
|
|
return r;
|
|
}
|
|
case DuplicatiOperation.RepairUpdate:
|
|
{
|
|
var r = await controller.UpdateDatabaseWithVersionsAsync().ConfigureAwait(false);
|
|
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
|
|
return r;
|
|
}
|
|
case DuplicatiOperation.Remove:
|
|
{
|
|
var r = await controller.DeleteAsync().ConfigureAwait(false);
|
|
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
|
|
return r;
|
|
}
|
|
case DuplicatiOperation.Restore:
|
|
{
|
|
var r = await controller.RestoreAsync(data.FilterStrings).ConfigureAwait(false);
|
|
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
|
|
return r;
|
|
}
|
|
case DuplicatiOperation.RestoreControlFiles:
|
|
{
|
|
var r = await controller.RestoreControlFilesAsync(data.FilterStrings).ConfigureAwait(false);
|
|
return r;
|
|
}
|
|
case DuplicatiOperation.Verify:
|
|
{
|
|
var r = await controller.TestAsync().ConfigureAwait(false);
|
|
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
|
|
return r;
|
|
}
|
|
case DuplicatiOperation.Compact:
|
|
{
|
|
var r = await controller.CompactAsync().ConfigureAwait(false);
|
|
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
|
|
return r;
|
|
}
|
|
case DuplicatiOperation.CreateReport:
|
|
{
|
|
using (var tf = new Library.Utility.TempFile())
|
|
{
|
|
var r = await controller.CreateLogDatabaseAsync(tf).ConfigureAwait(false);
|
|
var tempid = databaseConnection.RegisterTempFile("create-bug-report", r.TargetPath, DateTime.Now.AddDays(3));
|
|
|
|
if (string.Equals(tf, r.TargetPath, Utility.ClientFilenameStringComparison))
|
|
tf.Protected = true;
|
|
|
|
databaseConnection.RegisterNotification(
|
|
NotificationType.Information,
|
|
"Bugreport ready",
|
|
"Bugreport is ready for download",
|
|
null,
|
|
null,
|
|
"bug-report:created:" + tempid,
|
|
null,
|
|
"BugreportCreatedReady",
|
|
"",
|
|
(n, a) => n
|
|
);
|
|
|
|
return r;
|
|
}
|
|
}
|
|
|
|
case DuplicatiOperation.ListRemote:
|
|
{
|
|
var r = await controller.ListRemoteAsync().ConfigureAwait(false);
|
|
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
|
|
return r;
|
|
}
|
|
|
|
case DuplicatiOperation.Delete:
|
|
{
|
|
if (data.ExtraOptions != null)
|
|
{
|
|
if (Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-remote-files"))
|
|
await controller.DeleteAllRemoteFilesAsync().ConfigureAwait(false);
|
|
|
|
if (Utility.ParseBoolOption(data.ExtraOptions.AsReadOnly(), "delete-local-db"))
|
|
{
|
|
options.TryGetValue("dbpath", out var dbpath);
|
|
|
|
if (!string.IsNullOrWhiteSpace(dbpath) && System.IO.File.Exists(dbpath))
|
|
System.IO.File.Delete(dbpath);
|
|
}
|
|
}
|
|
databaseConnection.DeleteBackup(backup);
|
|
(data as RunnerData)?.AfterTaskFinished?.Invoke(data);
|
|
return null;
|
|
}
|
|
case DuplicatiOperation.Vacuum:
|
|
{
|
|
var r = await controller.VacuumAsync().ConfigureAwait(false);
|
|
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
|
|
return r;
|
|
}
|
|
|
|
case DuplicatiOperation.ListFilesets:
|
|
{
|
|
var r = await controller.ListFilesetsAsync().ConfigureAwait(false);
|
|
UpdateMetadataBase(databaseConnection, eventPollNotify, notificationUpdateService, backup, r);
|
|
return r;
|
|
}
|
|
case DuplicatiOperation.ListFolderContents:
|
|
{
|
|
return await controller.ListFolderAsync(data.ExtraArguments, data.PageOffset * data.PageSize, data.PageSize, data.ReturnExtendedData).ConfigureAwait(false);
|
|
}
|
|
|
|
case DuplicatiOperation.ListFileVersions:
|
|
{
|
|
return await controller.ListFileVersionsAsync(data.ExtraArguments, data.PageOffset * data.PageSize, data.PageSize).ConfigureAwait(false);
|
|
}
|
|
case DuplicatiOperation.SearchEntries:
|
|
{
|
|
var parsedfilter = new FilterExpression(data.FilterStrings);
|
|
return await controller.SearchEntriesAsync(data.ExtraArguments, parsedfilter, data.CaseSensitiveSearch, data.PageOffset * data.PageSize, data.PageSize, data.ReturnExtendedData, data.SearchMetadata).ConfigureAwait(false);
|
|
}
|
|
default:
|
|
//TODO: Log this
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
databaseConnection.LogError(data.Backup?.ID, string.Format("Failed while executing {0} \"{1}\" (id: {2})", data.Operation, data.Backup?.Name, data.Backup?.ID), ex);
|
|
if (data.Backup != null)
|
|
UpdateMetadataError(databaseConnection, notificationUpdateService, data.Backup, ex);
|
|
Library.UsageReporter.Reporter.Report(ex);
|
|
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
data.SetController(null);
|
|
data.TaskFinished = DateTime.Now;
|
|
eventPollNotify.SignalProgressUpdate(progressStateProviderService?.GenerateProgressState);
|
|
}
|
|
}
|
|
|
|
private static async Task<Library.Main.IController> CreateControllerAsync(string url, Dictionary<string, string?> options, MessageSink sink, bool useOutOfProcess)
|
|
{
|
|
if (useOutOfProcess)
|
|
return await Library.Main.IPC.ControllerRpcProxy.CreateProxyAsync(url, options, sink);
|
|
return new Library.Main.Controller(url, options, sink);
|
|
}
|
|
|
|
private static TempFolder? StoreTaskConfigAndGetTempFolder(Connection databaseConnection, IRunnerData data, Dictionary<string, string?> options)
|
|
{
|
|
if (data.Backup == null)
|
|
throw new ArgumentNullException(nameof(data.Backup));
|
|
|
|
var mode = ParseStoreTaskConfigMode(options.GetValueOrDefault("store-task-config"));
|
|
var encryptionEnabled = IsBackupEncryptionEnabled(options);
|
|
var effectiveMode = mode.ResolvedTaskConfigMode(encryptionEnabled);
|
|
|
|
options.Remove("store-task-config");
|
|
|
|
if (effectiveMode == null)
|
|
return null;
|
|
|
|
var tempfolder = new TempFolder();
|
|
var temppath = System.IO.Path.Combine(tempfolder, TaskSetupFilename);
|
|
using (var tempfile = Library.Utility.TempFile.WrapExistingFile(temppath))
|
|
{
|
|
IEnumerable<Serializable.ImportExportStructure>? taskdata = null;
|
|
if (effectiveMode.IncludeAllTasks)
|
|
{
|
|
taskdata = databaseConnection.Backups
|
|
.Where(x => !x.IsTemporary)
|
|
.Select(x => PrepareBackupForExport(databaseConnection, databaseConnection.GetBackup(x.ID)!, effectiveMode.RemoveSecrets));
|
|
}
|
|
else
|
|
{
|
|
taskdata = [PrepareBackupForExport(databaseConnection, data.Backup, effectiveMode.RemoveSecrets)];
|
|
}
|
|
|
|
using (var fs = System.IO.File.OpenWrite(tempfile))
|
|
using (var sw = new System.IO.StreamWriter(fs, System.Text.Encoding.UTF8))
|
|
Serializer.SerializeJson(sw, taskdata, true);
|
|
|
|
options.TryGetValue("control-files", out var controlfiles);
|
|
|
|
// Append this to any other control files
|
|
options["control-files"] = string.IsNullOrWhiteSpace(controlfiles)
|
|
? controlfiles = tempfile
|
|
: controlfiles += System.IO.Path.PathSeparator + tempfile;
|
|
|
|
// Don't delete the file now, leave it for when the folder is deleted
|
|
tempfile.Protected = true;
|
|
}
|
|
return tempfolder;
|
|
}
|
|
|
|
private static void UpdateMetadataError(Connection databaseConnection, INotificationUpdateService notificationUpdateService, IBackup backup, Exception ex)
|
|
{
|
|
backup.Metadata["LastErrorDate"] = Utility.SerializeDateTime(DateTime.UtcNow);
|
|
backup.Metadata["LastErrorMessage"] = ex.Message;
|
|
|
|
if (!backup.IsTemporary)
|
|
databaseConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null);
|
|
|
|
string? messageid = null;
|
|
if (ex is UserInformationException exception)
|
|
messageid = exception.HelpID;
|
|
|
|
notificationUpdateService.IncrementLastDataUpdateId();
|
|
databaseConnection.RegisterNotification(
|
|
NotificationType.Error,
|
|
backup.IsTemporary ?
|
|
"Error" : string.Format("Error while running {0}", backup.Name),
|
|
ex.Message,
|
|
ex,
|
|
backup.ID,
|
|
"backup:show-log",
|
|
null,
|
|
messageid,
|
|
null,
|
|
(n, a) =>
|
|
{
|
|
return a.FirstOrDefault(x => x.BackupID == backup.ID) ?? n;
|
|
}
|
|
);
|
|
}
|
|
|
|
private static void UpdateMetadataLastCompact(IBackup backup, ICompactResults r)
|
|
{
|
|
if (r != null)
|
|
{
|
|
backup.Metadata["LastCompactDuration"] = r.Duration.ToString();
|
|
backup.Metadata["LastCompactStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime());
|
|
backup.Metadata["LastCompactFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime());
|
|
}
|
|
}
|
|
|
|
private static void UpdateMetadataLastVacuum(IBackup backup, IVacuumResults r)
|
|
{
|
|
if (r != null)
|
|
{
|
|
backup.Metadata["LastVacuumDuration"] = r.Duration.ToString();
|
|
backup.Metadata["LastVacuumStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime());
|
|
backup.Metadata["LastVacuumFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime());
|
|
}
|
|
}
|
|
|
|
private static void UpdateMetadataStatistics(IBackup backup, IParsedBackendStatistics r)
|
|
{
|
|
if (r != null)
|
|
{
|
|
backup.Metadata["LastBackupDate"] = Utility.SerializeDateTime(r.LastBackupDate.ToUniversalTime());
|
|
backup.Metadata["BackupListCount"] = r.BackupListCount.ToString();
|
|
backup.Metadata["TotalQuotaSpace"] = r.TotalQuotaSpace.ToString();
|
|
backup.Metadata["FreeQuotaSpace"] = r.FreeQuotaSpace.ToString();
|
|
backup.Metadata["AssignedQuotaSpace"] = r.AssignedQuotaSpace.ToString();
|
|
|
|
backup.Metadata["TargetFilesSize"] = r.KnownFileSize.ToString();
|
|
backup.Metadata["TargetFilesCount"] = r.KnownFileCount.ToString();
|
|
backup.Metadata["TargetFilesetsCount"] = r.KnownFilesets.ToString();
|
|
backup.Metadata["TargetSizeString"] = Utility.FormatSizeString(r.KnownFileSize);
|
|
}
|
|
}
|
|
|
|
private static void UpdateMetadataBase(Connection databaseConnection, EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IBackup backup, IBasicResults result)
|
|
{
|
|
if (result is IRestoreResults r1)
|
|
{
|
|
backup.Metadata["LastRestoreDuration"] = r1.Duration.ToString();
|
|
backup.Metadata["LastRestoreStarted"] = Utility.SerializeDateTime(result.BeginTime.ToUniversalTime());
|
|
backup.Metadata["LastRestoreFinished"] = Utility.SerializeDateTime(result.EndTime.ToUniversalTime());
|
|
}
|
|
|
|
if (result is IParsedBackendStatistics r2 && !result.Interrupted)
|
|
{
|
|
UpdateMetadataStatistics(backup, r2);
|
|
}
|
|
|
|
if (result is IBackendStatsticsReporter r3 && !result.Interrupted)
|
|
{
|
|
if (r3.BackendStatistics is IParsedBackendStatistics statistics)
|
|
UpdateMetadataStatistics(backup, statistics);
|
|
}
|
|
|
|
if (result is ICompactResults r4 && !result.Interrupted)
|
|
{
|
|
UpdateMetadataLastCompact(backup, r4);
|
|
|
|
if (r4.VacuumResults != null)
|
|
UpdateMetadataLastVacuum(backup, r4.VacuumResults);
|
|
}
|
|
|
|
if (result is IVacuumResults r5 && !result.Interrupted)
|
|
{
|
|
UpdateMetadataLastVacuum(backup, r5);
|
|
}
|
|
|
|
if (result is IBackupResults r)
|
|
{
|
|
if (!result.Interrupted)
|
|
{
|
|
backup.Metadata["SourceFilesSize"] = r.SizeOfExaminedFiles.ToString();
|
|
backup.Metadata["SourceFilesCount"] = r.ExaminedFiles.ToString();
|
|
backup.Metadata["SourceSizeString"] = Duplicati.Library.Utility.Utility.FormatSizeString(r.SizeOfExaminedFiles);
|
|
backup.Metadata["LastBackupStarted"] = Utility.SerializeDateTime(r.BeginTime.ToUniversalTime());
|
|
backup.Metadata["LastBackupFinished"] = Utility.SerializeDateTime(r.EndTime.ToUniversalTime());
|
|
backup.Metadata["LastBackupDuration"] = r.Duration.ToString();
|
|
|
|
if (r.CompactResults != null)
|
|
UpdateMetadataLastCompact(backup, r.CompactResults);
|
|
|
|
if (r.VacuumResults != null)
|
|
UpdateMetadataLastVacuum(backup, r.VacuumResults);
|
|
}
|
|
|
|
if (r.FilesWithError > 0 || r.Warnings.Any() || r.Errors.Any())
|
|
{
|
|
string message;
|
|
string titleType;
|
|
if (r.FilesWithError > 0)
|
|
{
|
|
message = $"Errors affected {r.FilesWithError} file(s).";
|
|
titleType = "Error";
|
|
}
|
|
else if (r.Errors.Any())
|
|
{
|
|
message = r.Errors.Count() == 1 ? r.Errors.Single() : $"Encountered {r.Errors.Count()} errors.";
|
|
titleType = "Error";
|
|
}
|
|
else
|
|
{
|
|
message = r.Warnings.Count() == 1 ? r.Warnings.Single() : $"Encountered {r.Warnings.Count()} warnings.";
|
|
titleType = "Warning";
|
|
}
|
|
|
|
databaseConnection.RegisterNotification(
|
|
r.FilesWithError == 0 && !r.Errors.Any() ? NotificationType.Warning : NotificationType.Error,
|
|
backup.IsTemporary ? "Warning" : $"{titleType} while running {backup.Name}",
|
|
message,
|
|
null,
|
|
backup.ID,
|
|
"backup:show-log",
|
|
null,
|
|
null,
|
|
null,
|
|
(n, a) => n
|
|
);
|
|
}
|
|
else
|
|
{
|
|
var notificationIds = databaseConnection.GetNotifications()
|
|
.Where(n => n.BackupID == backup.ID)
|
|
.Select(x => x.ID)
|
|
.ToList();
|
|
|
|
foreach (var id in notificationIds)
|
|
try
|
|
{
|
|
databaseConnection.DismissNotification(id);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
databaseConnection.LogError(backup.ID, "Failed to dismiss notification", ex);
|
|
}
|
|
}
|
|
}
|
|
else if (result.ParsedResult != ParsedResultType.Success)
|
|
{
|
|
var type = result.ParsedResult == ParsedResultType.Warning
|
|
? NotificationType.Warning
|
|
: NotificationType.Error;
|
|
|
|
var title = result.ParsedResult == ParsedResultType.Warning
|
|
? (backup.IsTemporary ?
|
|
"Warning" : string.Format("Warning while running {0}", backup.Name))
|
|
: (backup.IsTemporary ?
|
|
"Error" : string.Format("Error while running {0}", backup.Name));
|
|
|
|
var message = result.ParsedResult == ParsedResultType.Warning
|
|
? string.Format("Got {0} warning(s)", result.Warnings.Count())
|
|
: string.Format("Got {0} error(s)", result.Errors.Count());
|
|
|
|
// If there is only one error or warning, show the message
|
|
if (result.ParsedResult == ParsedResultType.Warning && result.Warnings.Count() == 1)
|
|
message = $"Warning: {result.Warnings.Single()}";
|
|
else if (result.ParsedResult == ParsedResultType.Error && result.Errors.Count() == 1)
|
|
message = $"Error: {result.Errors.Single()}";
|
|
|
|
databaseConnection.RegisterNotification(
|
|
type,
|
|
title,
|
|
message,
|
|
null,
|
|
backup.ID,
|
|
"backup:show-log",
|
|
null,
|
|
null,
|
|
"backup:show-log",
|
|
(n, a) => n
|
|
);
|
|
}
|
|
|
|
if (!backup.IsTemporary)
|
|
databaseConnection.SetMetadata(backup.Metadata, long.Parse(backup.ID), null);
|
|
|
|
notificationUpdateService.IncrementLastDataUpdateId();
|
|
eventPollNotify.SignalNewEvent();
|
|
if (!backup.IsTemporary)
|
|
eventPollNotify.SignalBackupListUpdate();
|
|
}
|
|
|
|
private static bool TestIfOptionApplies()
|
|
{
|
|
//TODO: Implement to avoid warnings
|
|
return true;
|
|
}
|
|
|
|
private static void DisableModule(string module, Dictionary<string, string?> options)
|
|
{
|
|
if (options.TryGetValue("enable-module", out var enabledModules))
|
|
{
|
|
var emods = (enabledModules ?? "").Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
|
|
options["enable-module"] = string.Join(",", emods.Where(x => module.Equals(x, StringComparison.OrdinalIgnoreCase)));
|
|
}
|
|
|
|
options.TryGetValue("disable-module", out var disabledModules);
|
|
var mods = (disabledModules ?? "").Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
|
|
options["disable-module"] = string.Join(",", mods.Union(new string[] { module }).Distinct(StringComparer.OrdinalIgnoreCase));
|
|
}
|
|
|
|
/// <summary>
|
|
/// Maps the backup's AdditionalTargetURLs to the remote-sync-json-config option
|
|
/// so the RemoteSynchronizationModule can pick them up.
|
|
/// </summary>
|
|
/// <param name="backup">The backup containing additional target URLs.</param>
|
|
/// <param name="options">The options dictionary to update.</param>
|
|
private static void ApplyAdditionalTargetUrls(IBackup backup, Dictionary<string, string?> options)
|
|
{
|
|
if (backup.AdditionalTargetURLs == null)
|
|
return;
|
|
|
|
var additionalTargets = backup.AdditionalTargetURLs.ToList();
|
|
if (additionalTargets.Count == 0)
|
|
return;
|
|
|
|
var destinations = new List<Dictionary<string, object?>>();
|
|
foreach (var target in additionalTargets)
|
|
{
|
|
var dest = new Dictionary<string, object?>
|
|
{
|
|
["url"] = target.TargetUrl,
|
|
["mode"] = target.Mode
|
|
};
|
|
|
|
if (!string.IsNullOrWhiteSpace(target.Interval))
|
|
dest["interval"] = target.Interval;
|
|
|
|
// Add options from the Options dictionary
|
|
if (target.Options != null)
|
|
{
|
|
foreach (var opt in target.Options)
|
|
{
|
|
// Skip null values and already-handled properties
|
|
if (opt.Value == null)
|
|
continue;
|
|
if (opt.Key.Equals("url", StringComparison.OrdinalIgnoreCase))
|
|
continue;
|
|
if (opt.Key.Equals("mode", StringComparison.OrdinalIgnoreCase))
|
|
continue;
|
|
if (opt.Key.Equals("interval", StringComparison.OrdinalIgnoreCase))
|
|
continue;
|
|
|
|
dest[opt.Key] = opt.Value;
|
|
}
|
|
}
|
|
|
|
destinations.Add(dest);
|
|
}
|
|
|
|
var config = new Dictionary<string, object>
|
|
{
|
|
["sync-on-warnings"] = true,
|
|
["destinations"] = destinations
|
|
};
|
|
|
|
var jsonOptions = new JsonSerializerOptions
|
|
{
|
|
PropertyNamingPolicy = JsonNamingPolicy.KebabCaseLower
|
|
};
|
|
options["remote-sync-json-config"] = JsonSerializer.Serialize(config, jsonOptions);
|
|
}
|
|
|
|
internal static Dictionary<string, string?> ApplyOptions(Connection databaseConnection, Serialization.Interface.IBackup backup, Dictionary<string, string?> options, out string url)
|
|
{
|
|
url = backup.TargetURL;
|
|
options["backup-name"] = backup.Name;
|
|
options["dbpath"] = backup.DBPath;
|
|
options["backup-id"] = $"DB-{backup.ID}";
|
|
|
|
// Apply normal options
|
|
foreach (var o in backup.Settings)
|
|
if (!o.Name.StartsWith("--", StringComparison.Ordinal) && TestIfOptionApplies())
|
|
options[o.Name] = o.Value;
|
|
|
|
// Apply override options
|
|
foreach (var o in backup.Settings)
|
|
if (o.Name.StartsWith("--", StringComparison.Ordinal) && TestIfOptionApplies())
|
|
options[o.Name.Substring(2)] = o.Value;
|
|
|
|
// The server hangs if the module is enabled as there is no console attached
|
|
DisableModule("console-password-input", options);
|
|
|
|
// Patch in additional report urls
|
|
var additionalReportUrl = databaseConnection.ApplicationSettings.AdditionalReportUrl;
|
|
if (!string.IsNullOrWhiteSpace(additionalReportUrl))
|
|
{
|
|
options["send-http-json-urls"] = string.Join(";",
|
|
new[] {
|
|
options.GetValueOrDefault("send-http-json-urls"),
|
|
additionalReportUrl
|
|
}.Where(x => !string.IsNullOrWhiteSpace(x))
|
|
);
|
|
}
|
|
|
|
var uri = new Library.Utility.Uri(backup.TargetURL);
|
|
if (uri.Scheme.Equals(Library.Backend.Duplicati.DuplicatiBackend.PROTOCOL, StringComparison.OrdinalIgnoreCase))
|
|
url = Library.Backend.Duplicati.DuplicatiBackend.MergeArgsIntoUrl(
|
|
url,
|
|
databaseConnection.ApplicationSettings.RemoteControlStorageApiId,
|
|
databaseConnection.ApplicationSettings.RemoteControlStorageApiKey,
|
|
databaseConnection.ApplicationSettings.RemoteControlStorageEndpointUrl
|
|
);
|
|
|
|
return options;
|
|
}
|
|
|
|
private static Library.Utility.IFilter? ApplyFilter(IBackup backup, Library.Utility.IFilter? filter)
|
|
{
|
|
var f2 = backup.Filters;
|
|
if (f2 != null && f2.Length > 0)
|
|
{
|
|
var nf =
|
|
(from n in f2
|
|
let exp =
|
|
n.Expression.StartsWith("[", StringComparison.Ordinal) && n.Expression.EndsWith("]", StringComparison.Ordinal)
|
|
? SpecialFolders.ExpandEnvironmentVariablesRegexp(n.Expression)
|
|
: SpecialFolders.ExpandEnvironmentVariables(n.Expression)
|
|
orderby n.Order
|
|
select (Library.Utility.IFilter)new FilterExpression(exp, n.Include))
|
|
.Aggregate((a, b) => FilterExpression.Combine(a, b));
|
|
|
|
filter = FilterExpression.Combine(filter, nf);
|
|
}
|
|
|
|
return filter;
|
|
}
|
|
|
|
public static Dictionary<string, string?> GetCommonOptions(Connection databaseConnection)
|
|
{
|
|
return
|
|
(from n in databaseConnection.Settings
|
|
where TestIfOptionApplies()
|
|
select n).ToDictionary(k => k.Name.StartsWith("--", StringComparison.Ordinal) ? k.Name.Substring(2) : k.Name, k => (string?)k.Value);
|
|
}
|
|
|
|
private static Library.Utility.IFilter? GetCommonFilter(Connection databaseConnection)
|
|
{
|
|
var filters = databaseConnection.Filters;
|
|
if (filters == null || filters.Length == 0)
|
|
return null;
|
|
|
|
return
|
|
(from n in filters
|
|
orderby n.Order
|
|
let exp = Environment.ExpandEnvironmentVariables(n.Expression)
|
|
select (Library.Utility.IFilter)new FilterExpression(exp, n.Include))
|
|
.Aggregate((a, b) => FilterExpression.Combine(a, b));
|
|
}
|
|
}
|
|
}
|
|
|