Kenneth Skovhede
4faaa182f4
This PR adds a live-reporting module that sends the current progress of backups to a user specified url. The intention is that this can be used for dashboards that want to show the current progress for backups. By default, the module is not configured and has no impact.
1406 lines
71 KiB
C#
1406 lines
71 KiB
C#
// Copyright (C) 2026, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Threading;
|
|
using Duplicati.Library.Utility;
|
|
using Duplicati.Library.Common.IO;
|
|
using Duplicati.Library.Interface;
|
|
using Duplicati.Library.Main.Database;
|
|
using System.Threading.Tasks;
|
|
using Duplicati.Library.Main.Operation.Common;
|
|
using System.IO;
|
|
using Duplicati.Library.Backends;
|
|
using Duplicati.Library.Main.Database.Local;
|
|
|
|
namespace Duplicati.Library.Main
|
|
{
|
|
public class Controller : IDisposable, IController
|
|
{
|
|
/// <summary>
|
|
/// The tag used for logging
|
|
/// </summary>
|
|
private static readonly string LOGTAG = Logging.Log.LogTagFromType<Controller>();
|
|
/// <summary>
|
|
/// The backend url
|
|
/// </summary>
|
|
private string m_backendUrl;
|
|
/// <summary>
|
|
/// The parsed type-safe version of the commandline options
|
|
/// </summary>
|
|
private readonly Options m_options;
|
|
/// <summary>
|
|
/// The optional secret provider, if none provided in options
|
|
/// </summary>
|
|
public ISecretProvider SecretProvider { get; private set; }
|
|
/// <summary>
|
|
/// The destination for all output messages during execution
|
|
/// </summary>
|
|
private IMessageSink m_messageSink;
|
|
|
|
// TODO: The m_current* approach here prevents us from having multiple concurrent operations
|
|
|
|
/// <summary>
|
|
/// The current executing task
|
|
/// </summary>
|
|
private ITaskControl m_currentTaskControl = null;
|
|
|
|
/// <summary>
|
|
/// The current backend manager
|
|
/// </summary>
|
|
private IBackendManager m_currentBackendManager = null;
|
|
|
|
/// <summary>
|
|
/// If not null, active locale change that needs to be reset
|
|
/// </summary>
|
|
private LocaleChange m_localeChange = null;
|
|
|
|
/// <summary>
|
|
/// The handle for the report modules loaded for the current operation. Set in
|
|
/// <see cref="ReportModuleController.Setup"/> and disposed in
|
|
/// <see cref="OperationComplete"/> so the completion callbacks fire before the
|
|
/// module instances themselves are disposed.
|
|
/// </summary>
|
|
private ReportModuleController.ReportModulesHandle ReportModulesHandler;
|
|
|
|
/// <summary>
|
|
/// Callback method invoked when an operation is started
|
|
/// </summary>
|
|
public Action<IBasicResults> OnOperationStarted { get; set; }
|
|
|
|
/// <summary>
|
|
/// Callback method invoked when an operation is completed
|
|
/// </summary>
|
|
public Action<IBasicResults, Exception> OnOperationCompleted { get; set; }
|
|
|
|
/// <summary>
|
|
/// Constructs a new interface for performing backup and restore operations
|
|
/// </summary>
|
|
/// <param name="backendUrl">The url for the backend to use</param>
|
|
/// <param name="options">All required options</param>
|
|
public Controller(string backendUrl, Dictionary<string, string> options, IMessageSink messageSink)
|
|
{
|
|
m_backendUrl = backendUrl;
|
|
m_options = new Options(options);
|
|
m_messageSink = messageSink;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task AppendSinkAsync(IMessageSink sink)
|
|
{
|
|
if (this.m_messageSink is MultiMessageSink messageSink)
|
|
messageSink.Append(sink);
|
|
else
|
|
m_messageSink = new MultiMessageSink(m_messageSink, sink);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task SetSecretProviderAsync(ISecretProvider secretProvider)
|
|
{
|
|
SecretProvider = secretProvider;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IBackupResults> BackupAsync(string[] inputsources, IFilter inputFilter = null)
|
|
{
|
|
UsageReporter.Reporter.Report("USE_BACKEND", new Library.Utility.Uri(m_backendUrl).Scheme);
|
|
UsageReporter.Reporter.Report("USE_COMPRESSION", m_options.CompressionModule);
|
|
UsageReporter.Reporter.Report("USE_ENCRYPTION", m_options.EncryptionModule);
|
|
|
|
// Warn about experimental tar-based compression modules
|
|
if (m_options.CompressionModule.Equals("tzstd", StringComparison.OrdinalIgnoreCase) ||
|
|
m_options.CompressionModule.Equals("tgz", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "ExperimentalCompressionModule", null, $"The compression module '{m_options.CompressionModule}' is experimental and for testing only.");
|
|
}
|
|
|
|
CheckAutoCompactInterval();
|
|
CheckAutoVacuumInterval();
|
|
|
|
return await RunActionAsync(new BackupResults(), inputsources, inputFilter, false, static async config =>
|
|
{
|
|
using (var h = new Operation.BackupHandler(config.Options, config.Result))
|
|
await h.RunAsync(ExpandInputSources(config.Paths, config.Filter, config.Options), config.BackendManager, config.Filter)
|
|
.ConfigureAwait(false);
|
|
|
|
UsageReporter.Reporter.Report("BACKUP_FILECOUNT", config.Result.ExaminedFiles);
|
|
UsageReporter.Reporter.Report("BACKUP_FILESIZE", config.Result.SizeOfExaminedFiles);
|
|
UsageReporter.Reporter.Report("BACKUP_DURATION", (long)config.Result.Duration.TotalSeconds);
|
|
|
|
using (var h = new Operation.RemoteSynchronizationHandler(config.BackendUrl, config.Options, config.Result))
|
|
await h.RunAsync()
|
|
.ConfigureAwait(false);
|
|
}).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IRestoreResults> RestoreAsync(string[] paths, IFilter inputFilter = null)
|
|
{
|
|
return await RunActionAsync(new RestoreResults(), paths, inputFilter, false, static async config =>
|
|
{
|
|
using var restoreDestination =
|
|
(config.Options.Restorepath ?? "").StartsWith("@")
|
|
|
|
// Remote destination
|
|
? await DynamicLoader.RestoreDestinationProviderLoader.GetRestoreDestinationProvider(
|
|
config.Options.Restorepath.Substring(1),
|
|
config.Options.RawOptions,
|
|
config.Result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false)
|
|
|
|
// Local destination
|
|
: new SourceProvider.FileRestoreDestinationProvider(config.Options.Restorepath ?? "", config.Options.AllowRestoreOutsideTargetDirectory);
|
|
|
|
if (restoreDestination == null)
|
|
throw new UserInformationException($"Could not find restore destination for path: {config.Options.Restorepath}", "InvalidRestoreDestination");
|
|
|
|
await new Operation.RestoreHandler(config.Options, config.Result)
|
|
.RunAsync(config.Paths, config.BackendManager, config.Filter, restoreDestination)
|
|
.ConfigureAwait(false);
|
|
|
|
await restoreDestination.Finalize((pg) =>
|
|
{
|
|
config.Result.OperationProgressUpdater.UpdateProgress((float)pg);
|
|
}, config.Result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
config.Result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete);
|
|
config.Result.EndTime = DateTime.UtcNow;
|
|
|
|
UsageReporter.Reporter.Report("RESTORE_FILECOUNT", config.Result.RestoredFiles);
|
|
UsageReporter.Reporter.Report("RESTORE_FILESIZE", config.Result.SizeOfRestoredFiles);
|
|
UsageReporter.Reporter.Report("RESTORE_DURATION", (long)config.Result.Duration.TotalSeconds);
|
|
}).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IRestoreControlFilesResults> RestoreControlFilesAsync(IEnumerable<string> files = null, IFilter inputFilter = null)
|
|
{
|
|
return await RunActionAsync(new RestoreControlFilesResults(), files?.ToArray(), inputFilter, false, static config =>
|
|
new Operation.RestoreControlFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.Paths, config.BackendManager, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IDeleteResults> DeleteAsync()
|
|
{
|
|
return await RunActionAsync(new DeleteResults(), null, null, false, static config =>
|
|
new Operation.DeleteHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IRepairResults> RepairAsync(IFilter inputFilter = null)
|
|
{
|
|
return await RunActionAsync(new RepairResults(), null, inputFilter, false, static config =>
|
|
new Operation.RepairHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListFilesetResults> ListFilesetsAsync()
|
|
=> await RunActionAsync(new ListFilesetResults(), null, null, false, static config =>
|
|
Operation.ListFilesetsHandler.RunAsync(config.Options, config.Result, config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListFolderResults> ListFolderAsync(string[] folders, long offset, long limit, bool extendedData)
|
|
=> await RunActionAsync(new ListFolderResults(), folders, null, new { offset, limit, extendedData }, static config =>
|
|
Operation.ListFolderHandler.RunAsync(config.Options, config.Result, config.Paths, config.Context.offset, config.Context.limit, config.Context.extendedData)
|
|
).ConfigureAwait(false);
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListFileVersionsResults> ListFileVersionsAsync(string[] files, long offset, long limit)
|
|
=> await RunActionAsync(new ListFileVersionsResults(), files, null, new { offset, limit }, static config =>
|
|
Operation.ListFileVersionsHandler.RunAsync(config.Options, config.Result, config.Paths, config.Context.offset, config.Context.limit)
|
|
).ConfigureAwait(false);
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ISearchFilesResults> SearchEntriesAsync(string[] pathprefixes, IFilter inputFilter, bool caseSensitive, long offset, long limit, bool returnExtendedData, bool searchMetadata)
|
|
=> await RunActionAsync(new SearchFilesResults(), pathprefixes, inputFilter, new { caseSensitive, offset, limit, returnExtendedData, searchMetadata }, static config =>
|
|
Operation.SearchEntriesHandler.RunAsync(config.Options, config.Result, config.Paths, config.Filter, config.Context.caseSensitive, config.Context.offset, config.Context.limit, config.Context.returnExtendedData, config.Context.searchMetadata)
|
|
).ConfigureAwait(false);
|
|
|
|
/// <inheritdoc />
|
|
public Task<IListResults> ListAsync()
|
|
=> ListAsync(null, null);
|
|
|
|
/// <inheritdoc />
|
|
public Task<IListResults> ListAsync(string filterstring)
|
|
=> ListAsync(filterstring == null ? null : new string[] { filterstring }, null);
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListResults> ListAsync(IEnumerable<string> filterstrings, IFilter inputFilter)
|
|
{
|
|
return await RunActionAsync(new ListResults(), filterstrings?.ToArray(), inputFilter, false, static config =>
|
|
new Operation.ListFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Paths, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListResults> ListControlFilesAsync(IEnumerable<string> filterstrings, IFilter inputFilter)
|
|
{
|
|
return await RunActionAsync(new ListResults(), filterstrings?.ToArray(), inputFilter, false, static config =>
|
|
new Operation.ListControlFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Paths, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListRemoteResults> ListRemoteAsync()
|
|
{
|
|
return await RunActionAsync(new ListRemoteResults(), null, null, false, static async config =>
|
|
{
|
|
using var tf = File.Exists(config.Options.Dbpath) ? null : new TempFile();
|
|
await using var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? config.Options.Dbpath, "list-remote", true, null, config.Result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
config.Result.SetResult(await config.BackendManager.ListAsync(null, CancellationToken.None).ConfigureAwait(false));
|
|
}).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListRemoteResults> DeleteAllRemoteFilesAsync()
|
|
{
|
|
return await RunActionAsync(new ListRemoteResults(), null, null, false, static async config =>
|
|
{
|
|
config.Result.OperationProgressUpdater.UpdatePhase(OperationPhase.Delete_Listing);
|
|
{
|
|
// Only delete files that match the expected pattern and prefix
|
|
var list = (await config.BackendManager.ListAsync(null, config.Result.TaskControl.ProgressToken).ConfigureAwait(false))
|
|
.Select(x => Volumes.VolumeBase.ParseFilename(x))
|
|
.Where(x => x != null)
|
|
.Where(x => x.Prefix == config.Options.Prefix)
|
|
.ToList();
|
|
|
|
// If the local database is available, we will use it to avoid deleting unrelated files
|
|
// from the backend. Otherwise, we may accidentally delete non-Duplicati files, or
|
|
// files from a different Duplicati configuration that points to the same backend location
|
|
// and uses the same prefix (see issues #2678, #3845, and #4244).
|
|
if (File.Exists(config.Options.Dbpath))
|
|
{
|
|
await using LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(config.Options.Dbpath, "list-remote", true, null, config.Result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
var dbRemoteVolumes = db.GetRemoteVolumesAsync(config.Result.TaskControl.ProgressToken);
|
|
var dbRemoteFiles = await dbRemoteVolumes
|
|
.Select(x => x.Name)
|
|
.ToHashSetAsync(cancellationToken: config.Result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
list = list.Where(x =>
|
|
dbRemoteFiles.Contains(x.File.Name)
|
|
).ToList();
|
|
}
|
|
|
|
config.Result.OperationProgressUpdater.UpdatePhase(OperationPhase.Delete_Deleting);
|
|
config.Result.OperationProgressUpdater.UpdateProgress(0);
|
|
for (var i = 0; i < list.Count; i++)
|
|
{
|
|
try
|
|
{
|
|
if (config.Options.Dryrun)
|
|
{
|
|
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "Would delete file: {0}", list[i].File.Name);
|
|
}
|
|
else
|
|
{
|
|
await config.BackendManager.DeleteAsync(list[i].File.Name, list[i].File.Size, true, config.Result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "DeleteFilesetError", ex, "Failed to delete remote file: {0}", list[i].File.Name);
|
|
}
|
|
config.Result.OperationProgressUpdater.UpdateProgress((float)i / list.Count);
|
|
}
|
|
config.Result.OperationProgressUpdater.UpdateProgress(1);
|
|
}
|
|
}).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ICompactResults> CompactAsync()
|
|
{
|
|
CheckAutoVacuumInterval();
|
|
|
|
return await RunActionAsync(new CompactResults(), null, null, false, static config =>
|
|
new Operation.CompactHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ISetLockResults> SetLocksAsync()
|
|
{
|
|
return await RunActionAsync(new SetLockResults(), null, null, false, static config =>
|
|
new Operation.SetLocksHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IReadLockInfoResults> ReadLockInfoAsync()
|
|
{
|
|
return await RunActionAsync(new ReadLockInfoResults(), null, null, false, static config =>
|
|
new Operation.ReadLockInfoHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IRecreateDatabaseResults> UpdateDatabaseWithVersionsAsync(IFilter inputFilter = null)
|
|
{
|
|
return await RunActionAsync(new RecreateDatabaseResults(), null, inputFilter, false, static async config =>
|
|
{
|
|
var filelistfilter = Operation.RestoreHandler.GetNumberedFilelistFilterDelegate(config.Options.Time, config.Options.Version, singleTimeMatch: true);
|
|
using (var h = new Operation.RecreateDatabaseHandler(config.Options, config.Result))
|
|
await h.RunUpdateAsync(config.BackendManager, config.Filter, filelistfilter, null)
|
|
.ConfigureAwait(false);
|
|
}).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ICreateLogDatabaseResults> CreateLogDatabaseAsync(string targetpath)
|
|
{
|
|
return await RunActionAsync(new CreateLogDatabaseResults(), [targetpath], null, false, static async config =>
|
|
await new Operation.CreateBugReportHandler(config.Paths[0], config.Options, config.Result).RunAsync()
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListChangesResults> ListChangesAsync(string baseVersion, string targetVersion, IEnumerable<string> filterstrings = null, IFilter inputFilter = null, Action<IListChangesResults, IEnumerable<Tuple<ListChangesChangeType, ListChangesElementType, string>>> callback = null)
|
|
{
|
|
|
|
return await RunActionAsync(new ListChangesResults(), [baseVersion, targetVersion], inputFilter, new { filterstrings, callback }, async static config =>
|
|
await new Operation.ListChangesHandler(config.Options, config.Result)
|
|
.RunAsync(config.Paths[0], config.Paths[1], config.BackendManager, config.Context.filterstrings, config.Filter, config.Context.callback)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListAffectedResults> ListAffectedAsync(List<string> args, Action<IListAffectedResults> callback = null)
|
|
{
|
|
return await RunActionAsync(new ListAffectedResults(), args?.ToArray(), null, new { callback }, static config =>
|
|
new Operation.ListAffected(config.Options, config.Result)
|
|
.RunAsync(config.Paths, config.Context.callback)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ITestResults> TestAsync(long samples = 1)
|
|
{
|
|
if (!m_options.RawOptions.ContainsKey("full-remote-verification"))
|
|
m_options.RawOptions["full-remote-verification"] = "true";
|
|
|
|
return await RunActionAsync(new TestResults(), null, null, new { samples }, static config =>
|
|
new Operation.TestHandler(config.Options, config.Result)
|
|
.RunAsync(config.Context.samples, config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ITestFilterResults> TestFilterAsync(string[] paths, IFilter inputFilter = null)
|
|
{
|
|
m_options.RawOptions["dry-run"] = "true";
|
|
m_options.RawOptions["dbpath"] = "INVALID!";
|
|
|
|
// Redirect all messages from the filter to the message sink
|
|
var filtertag = Logging.Log.LogTagFromType(typeof(Operation.Backup.FileEnumerationProcess));
|
|
using (Logging.Log.StartScope(m_messageSink.WriteMessage, x => x.FilterTag.Contains(filtertag)))
|
|
{
|
|
return await RunActionAsync(new TestFilterResults(), paths, inputFilter, false, static config =>
|
|
new Operation.TestFilterHandler(config.Options, config.Result)
|
|
.RunAsync(ExpandInputSources(config.Paths, config.Filter, config.Options), config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ISystemInfoResults> SystemInfoAsync()
|
|
{
|
|
return await RunActionAsync(new SystemInfoResults(), null, null, false, static config =>
|
|
Operation.SystemInfoHandler.RunAsync(config.Result)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IPurgeFilesResults> PurgeFilesAsync(IFilter inputFilter)
|
|
{
|
|
return await RunActionAsync(new PurgeFilesResults(), null, inputFilter, false, static config =>
|
|
new Operation.PurgeFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListBrokenFilesResults> ListBrokenFilesAsync(IFilter inputFilter, Func<long, DateTime, long, string, long, bool> callbackhandler = null)
|
|
{
|
|
return await RunActionAsync(new ListBrokenFilesResults(), null, inputFilter, new { callbackhandler }, static config =>
|
|
new Operation.ListBrokenFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Filter, config.Context.callbackhandler)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IPurgeBrokenFilesResults> PurgeBrokenFilesAsync(IFilter inputFilter)
|
|
{
|
|
return await RunActionAsync(new PurgeBrokenFilesResults(), null, inputFilter, false, static config =>
|
|
new Operation.PurgeBrokenFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ISendMailResults> SendMailAsync()
|
|
{
|
|
m_options.RawOptions["send-mail-level"] = "all";
|
|
m_options.RawOptions["send-mail-any-operation"] = "true";
|
|
const string sendMailModuleKey = "sendmail";
|
|
m_options.RawOptions.TryGetValue("send-mail-to", out string targetmail);
|
|
if (string.IsNullOrWhiteSpace(targetmail))
|
|
throw new Exception(string.Format("No email specified, please use --{0}", "send-mail-to"));
|
|
|
|
m_options.RawOptions["disable-module"] = string.Join(
|
|
",",
|
|
DynamicLoader.GenericLoader.Modules
|
|
.Where(m =>
|
|
!string.Equals(m.Key, sendMailModuleKey, StringComparison.OrdinalIgnoreCase)
|
|
)
|
|
.Select(x => x.Key)
|
|
);
|
|
|
|
/// Forward all messages from the email module to the message sink
|
|
var filtertag = Logging.Log.LogTagFromType(DynamicLoader.GenericLoader.GetModule(sendMailModuleKey).GetType());
|
|
using (Logging.Log.StartScope(m_messageSink.WriteMessage, x => x.FilterTag.Contains(filtertag)))
|
|
{
|
|
return await RunActionAsync(new SendMailResults(), null, null, false, static config =>
|
|
Task.Run(() =>
|
|
{
|
|
config.Result.Lines = [];
|
|
Thread.Sleep(5);
|
|
})
|
|
).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
public async Task<ISyncResults> SyncAsync(string[] sourcePaths, IFilter filter)
|
|
{
|
|
return await RunActionAsync(new SyncResults(), sourcePaths, filter, false, config =>
|
|
new Operation.Sync.SyncHandler(config.Paths, config.Options, config.Result, config.BackendUrl)
|
|
.RunAsync(config.BackendManager, config.Filter)
|
|
);
|
|
}
|
|
|
|
public async Task<IVacuumResults> VacuumAsync()
|
|
{
|
|
return await RunActionAsync(new VacuumResults(), null, null, false, static config =>
|
|
new Operation.VacuumHandler(config.Options, config.Result)
|
|
.RunAsync()
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
private sealed record RunActionData<TRes, TCon>(
|
|
TRes Result,
|
|
TCon Context,
|
|
string BackendUrl,
|
|
string[] Paths,
|
|
Options Options,
|
|
IFilter Filter,
|
|
IBackendManager BackendManager);
|
|
|
|
private async Task<TRes> RunActionAsync<TRes, TCon>(TRes result, string[] paths, IFilter filter, TCon context, Func<RunActionData<TRes, TCon>, Task> method)
|
|
where TRes : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider
|
|
{
|
|
OnOperationStarted?.Invoke(result);
|
|
var resultSetter = result as ISetCommonOptions;
|
|
using (var logTarget = new ControllerMultiLogTarget(result, Logging.LogMessageType.Information, null, m_options.SuppressWarningsFilter))
|
|
using (Logging.Log.StartScope(logTarget, null))
|
|
{
|
|
using var httpListener = m_options.LogHttpRequests || m_options.LogSocketData >= 0
|
|
? new NetworkTrafficLogger(m_options.LogHttpRequests, m_options.LogSocketData)
|
|
: null;
|
|
|
|
try
|
|
{
|
|
m_currentTaskControl = result.TaskControl;
|
|
m_options.MainAction = result.MainOperation;
|
|
await ApplySecretProviderAsync(CancellationToken.None).ConfigureAwait(false);
|
|
(paths, filter) = SetupCommonOptions(result, paths, filter, logTarget);
|
|
|
|
ReportModulesHandler = ReportModuleController.Setup(this, result, m_options);
|
|
|
|
logTarget.AddTarget(m_messageSink, m_options.ConsoleLoglevel, m_options.ConsoleLogFilter);
|
|
result.MessageSink = m_messageSink;
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "StartingOperation", Strings.Controller.StartingOperationMessage(m_options.MainAction));
|
|
|
|
using (SlowQueryMonitor.StartMonitoring(m_options.SlowQueryThreshold))
|
|
using (new ProcessController(m_options))
|
|
using (new Logging.Timer(LOGTAG, string.Format("Run{0}", result.MainOperation), string.Format("Running {0}", result.MainOperation)))
|
|
using (new CoCoL.IsolatedChannelScope())
|
|
using (m_options.ConcurrencyMaxThreads <= 0 ? null : new CoCoL.CappedThreadedThreadPool(m_options.ConcurrencyMaxThreads))
|
|
using (var backend = new Backend.BackendManager(m_backendUrl, m_options, result.BackendWriter, result.TaskControl))
|
|
{
|
|
try
|
|
{
|
|
backend.UpdateThrottleValues(m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond);
|
|
m_currentBackendManager = backend;
|
|
var config = new RunActionData<TRes, TCon>(result, context, m_backendUrl, paths, m_options, filter, backend);
|
|
await method(config).ConfigureAwait(false);
|
|
}
|
|
finally
|
|
{
|
|
m_currentBackendManager = null;
|
|
|
|
// TODO: Should also have a single shared database connection for all operations
|
|
// The transactions should be managed inside the connection, and not passed around
|
|
|
|
// This would allow us to pass the database instance to the backend manager
|
|
// And safeguard against remote operations not being logged in the database
|
|
|
|
// This would also allow us to control the unclean shutdown flag,
|
|
// by toggling this on start and completion of transfers in the manager,
|
|
// instead of relying on the operations to correctly toggle the flag
|
|
//
|
|
// The sync operation owns its own database and records its own audit log.
|
|
if (File.Exists(m_options.Dbpath) && !m_options.NoLocalDb && m_options.MainAction != OperationMode.Sync)
|
|
{
|
|
try
|
|
{
|
|
await using var db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, null, CancellationToken.None).ConfigureAwait(false);
|
|
await backend.StopRunnerAndFlushMessagesAsync(db).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FailedFlushBackendMessages", ex, "Failed to flush backend messages to database: {0}", ex.Message);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
backend.StopRunnerAndDiscardMessages();
|
|
}
|
|
}
|
|
}
|
|
|
|
if (resultSetter.EndTime.Ticks == 0)
|
|
resultSetter.EndTime = DateTime.UtcNow;
|
|
resultSetter.Interrupted = false;
|
|
|
|
// The post-operation result/log writes target the backup LocalDatabase schema
|
|
// (Operation, LogData, DeletedVolume tables). The sync database has its own
|
|
// schema and does not store these, so skip the backup-specific cleanup for
|
|
// sync operations instead of running incompatible SQL against the sync DB.
|
|
if (File.Exists(m_options.Dbpath) && !m_options.Dryrun && m_options.MainAction != OperationMode.Sync)
|
|
{
|
|
try
|
|
{
|
|
await using var db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, null, CancellationToken.None).ConfigureAwait(false);
|
|
await db.WriteResultsAndCommitAsync(result, CancellationToken.None).ConfigureAwait(false);
|
|
await db.PurgeLogDataAsync(m_options.LogRetention, CancellationToken.None).ConfigureAwait(false);
|
|
await db.PurgeDeletedVolumesAsync(DateTime.UtcNow, CancellationToken.None).ConfigureAwait(false);
|
|
|
|
// Vacuum is done AFTER the results are written to the database
|
|
// This means that the information about the vacuum is not stored in the database,
|
|
// but will be reported in the log output and messages sent with any of the reporting modules
|
|
if (m_options.AutoVacuum && result is IResultsWithVacuum vacuumResults && result is BasicResults basicResults)
|
|
{
|
|
try
|
|
{
|
|
vacuumResults.VacuumResults = new VacuumResults(basicResults);
|
|
await new Operation.VacuumHandler(m_options, (VacuumResults)vacuumResults.VacuumResults)
|
|
.RunAsync().ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToVacuum", ex, "Failed to vacuum database");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FailedWriteOperation", ex, "Failed to write operation results to database: {0}", ex.Message);
|
|
}
|
|
}
|
|
|
|
OperationComplete(result, null, filter);
|
|
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "CompletedOperation", Strings.Controller.CompletedOperationMessage(m_options.MainAction));
|
|
|
|
return result;
|
|
}
|
|
// Handle custom abort exception from run-script
|
|
catch (OperationAbortException oae)
|
|
{
|
|
ReportModulesHandler?.OperationException = null; // Requested aborts are not failures.
|
|
resultSetter.EndTime = DateTime.UtcNow;
|
|
// Log this as a normal operation, as the script raising the exception,
|
|
// has already populated either warning or log messages as required
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "AbortOperation", "Aborting operation by request, requested result: {0}", oae.AbortReason);
|
|
|
|
resultSetter.Interrupted = true;
|
|
try
|
|
{
|
|
// No operation was started in database, so write logs to new operation.
|
|
// Sync uses its own database schema
|
|
if (File.Exists(m_options.Dbpath) && !m_options.Dryrun && m_options.MainAction != OperationMode.Sync)
|
|
await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, null, CancellationToken.None).ConfigureAwait(false))
|
|
await db.WriteResultsAndCommitAsync(result, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception we)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FailedWriteOperation", we, we.Message);
|
|
}
|
|
|
|
// Do not propagate the cancel exception
|
|
OperationComplete(result, null, filter);
|
|
|
|
return result;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
ReportModulesHandler?.OperationException = ex;
|
|
Logging.Log.WriteErrorMessage(LOGTAG, "FailedOperation", ex, Strings.Controller.FailedOperationMessage(m_options.MainAction));
|
|
|
|
try
|
|
{
|
|
if (result is BasicResults basicResults)
|
|
basicResults.OperationProgressUpdater.UpdatePhase(OperationPhase.Error);
|
|
|
|
resultSetter.Fatal = true;
|
|
// Write logs to previous operation if database exists.
|
|
// Sync uses its own database schema
|
|
if (File.Exists(m_options.Dbpath) && !m_options.Dryrun && m_options.MainAction != OperationMode.Sync)
|
|
await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, null, CancellationToken.None).ConfigureAwait(false))
|
|
await db.WriteResultsAndCommitAsync(result, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception we)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FailedWriteOperation", we, we.Message);
|
|
}
|
|
|
|
// Report the result, and the failure
|
|
OperationComplete(result, ex, filter);
|
|
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
m_currentTaskControl = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Temporarily pushes the filter into the raw options so modules can read it.
|
|
/// Returns a disposable that removes the key when disposed.
|
|
/// </summary>
|
|
/// <param name="options">The options to push the filter into</param>
|
|
/// <param name="filter">The filter to push</param>
|
|
/// <returns>A disposable that removes the key when disposed</returns>
|
|
internal static IDisposable PushFilterToOptions(Options options, IFilter filter)
|
|
=> PushFilterToOptions(options, filter, out var _);
|
|
/// <summary>
|
|
/// Temporarily pushes the filter into the raw options so modules can read it.
|
|
/// Returns a disposable that removes the key when disposed.
|
|
/// </summary>
|
|
/// <param name="options">The options to push the filter into</param>
|
|
/// <param name="filter">The filter to push</param>
|
|
/// <returns>A disposable that removes the key when disposed</returns>
|
|
internal static IDisposable PushFilterToOptions(Options options, IFilter filter, out string filterString)
|
|
{
|
|
filterString = filter != null && !filter.Empty
|
|
? string.Join(Path.PathSeparator.ToString(), FilterExpression.Serialize(filter))
|
|
: null;
|
|
|
|
options.RawOptions["filter"] = filterString;
|
|
return new FilterOptionScope(options);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Helper to remove the filter option on dispose
|
|
/// </summary>
|
|
private sealed class FilterOptionScope : IDisposable
|
|
{
|
|
private readonly Options m_opts;
|
|
public FilterOptionScope(Options opts) => m_opts = opts;
|
|
public void Dispose() => m_opts?.RawOptions.Remove("filter");
|
|
}
|
|
|
|
private void OperationComplete(IBasicResults result, Exception exception, IFilter filter)
|
|
{
|
|
using var _ = PushFilterToOptions(m_options, filter);
|
|
|
|
// Fire the report modules' completion callbacks and stop their progress ticker
|
|
// before the module instances themselves are disposed below. The handle reads
|
|
// the exception captured in the RunActionAsync catch blocks via the field.
|
|
try { this.ReportModulesHandler?.Dispose(); }
|
|
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "ReportModuleHandleDisposeError", ex, "Report module handle dispose failed: {0}", ex.Message); }
|
|
this.ReportModulesHandler = default;
|
|
|
|
if (m_options?.LoadedModules != null)
|
|
{
|
|
foreach (var mx in m_options.LoadedModules)
|
|
if (mx is IGenericCallbackModule module)
|
|
try { module.OnFinish(result, exception); }
|
|
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, $"OnFinishError{mx.Key}", ex, "OnFinish callback {0} failed: {1}", mx.Key, ex.Message); }
|
|
|
|
foreach (var mx in m_options.LoadedModules)
|
|
if (mx is IDisposable disposable)
|
|
try { disposable.Dispose(); }
|
|
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, $"DisposeError{mx.Key}", ex, "Dispose for {0} failed: {1}", mx.Key, ex.Message); }
|
|
|
|
m_options.ClearLoadedModules();
|
|
}
|
|
|
|
if (m_localeChange != null)
|
|
{
|
|
m_localeChange.Dispose();
|
|
m_localeChange = null;
|
|
}
|
|
|
|
OnOperationCompleted?.Invoke(result, exception);
|
|
}
|
|
|
|
private (string[] paths, IFilter filter) SetupCommonOptions(ISetCommonOptions result, string[] paths, IFilter filter, ControllerMultiLogTarget logTarget)
|
|
{
|
|
m_options.MainAction = result.MainOperation;
|
|
|
|
switch (m_options.MainAction)
|
|
{
|
|
case OperationMode.Backup:
|
|
break;
|
|
|
|
default:
|
|
//It only makes sense to enable auto-creation if we are writing files.
|
|
if (!m_options.RawOptions.ContainsKey("disable-autocreate-folder"))
|
|
m_options.RawOptions["disable-autocreate-folder"] = "true";
|
|
break;
|
|
}
|
|
|
|
if (string.IsNullOrEmpty(m_options.Dbpath))
|
|
m_options.Dbpath = CLIDatabaseLocator.GetDatabasePathForCLI(m_backendUrl, m_options, true, false, m_options.MainAction == OperationMode.Sync);
|
|
|
|
RestoreOptionsFromExistingDatabase(result.MainOperation);
|
|
|
|
//Load generic modules
|
|
m_options.ClearLoadedModules();
|
|
|
|
foreach (var m in DynamicLoader.GenericLoader.Modules)
|
|
{
|
|
var active = !m_options.DisableModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase) && (m.LoadAsDefault || m_options.EnableModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase));
|
|
if (!active)
|
|
continue;
|
|
|
|
m_options.AddLoadedModule(DynamicLoader.GenericLoader.GetModule(m.Key));
|
|
}
|
|
|
|
// Store the URL connection options separately, as these should only be visible to modules implementing IConnectionModule
|
|
var conopts = new Dictionary<string, string>(m_options.RawOptions);
|
|
var qp = new Library.Utility.Uri(m_backendUrl).QueryParameters;
|
|
foreach (var k in qp.Keys)
|
|
conopts[(string)k] = qp[(string)k];
|
|
|
|
// Make the filter read-n-write able in the generic modules
|
|
using (PushFilterToOptions(m_options, filter, out var pristinefilter))
|
|
{
|
|
foreach (var mx in m_options.LoadedModules)
|
|
{
|
|
if (mx is IConnectionModule)
|
|
mx.Configure(conopts);
|
|
else
|
|
mx.Configure(m_options.RawOptions);
|
|
|
|
if (mx is IGenericSourceModule sourcemodule)
|
|
{
|
|
if (sourcemodule.ContainFilesForBackup(paths))
|
|
{
|
|
var sourceoptions = sourcemodule.ParseSourcePaths(ref paths, ref pristinefilter, m_options.RawOptions);
|
|
|
|
foreach (var sourceoption in sourceoptions)
|
|
m_options.RawOptions[sourceoption.Key] = sourceoption.Value;
|
|
}
|
|
}
|
|
|
|
if (mx is IGenericCallbackModule module)
|
|
module.OnStart(result.MainOperation.ToString(), ref m_backendUrl, ref paths);
|
|
}
|
|
|
|
// If the filters were changed by a module, read them back in
|
|
if (pristinefilter != m_options.RawOptions["filter"])
|
|
filter = FilterExpression.Deserialize(m_options.RawOptions["filter"].Split(new string[] { Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries));
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(m_options.Logfile))
|
|
{
|
|
var path = Path.GetDirectoryName(Path.GetFullPath(m_options.Logfile));
|
|
if (!Directory.Exists(path))
|
|
Directory.CreateDirectory(path);
|
|
|
|
logTarget.AddTarget(
|
|
new Library.Logging.StreamLogDestination(m_options.Logfile),
|
|
m_options.LogFileLoglevel,
|
|
m_options.LogFileLogFilter
|
|
);
|
|
}
|
|
|
|
if (m_options.HasTempDir)
|
|
{
|
|
Library.Utility.TempFolder.SystemTempPath = m_options.TempDir;
|
|
}
|
|
|
|
if (m_options.HasForcedLocale)
|
|
{
|
|
try
|
|
{
|
|
m_localeChange = new LocaleChange(m_options.ForcedLocale);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Library.Logging.Log.WriteWarningMessage(LOGTAG, "LocaleChangeError", ex, Strings.Controller.FailedForceLocaleError(ex.Message));
|
|
}
|
|
}
|
|
|
|
ValidateOptions();
|
|
|
|
return (paths, filter);
|
|
}
|
|
|
|
|
|
private void RestoreOptionsFromExistingDatabase(OperationMode operation)
|
|
{
|
|
// The sync database (LocalSyncDatabase) has its own schema, distinct from the
|
|
// backup LocalDatabase, and does not store options to restore. Opening it as a
|
|
// backup database would run the backup schema upgrade against an incompatible
|
|
// schema and throw. Sync reads its options directly from the command-line dict.
|
|
if (operation == OperationMode.Sync)
|
|
return;
|
|
|
|
if (m_options.NoLocalDb || string.IsNullOrWhiteSpace(m_options.Dbpath) || !File.Exists(m_options.Dbpath))
|
|
return;
|
|
|
|
using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, operation.ToString(), true, null, CancellationToken.None).Await();
|
|
Utility.UpdateOptionsFromDbAsync(db, m_options, CancellationToken.None).Await();
|
|
}
|
|
|
|
private async Task ApplySecretProviderAsync(CancellationToken cancellationToken)
|
|
{
|
|
var args = new[] { new Library.Utility.Uri(m_backendUrl) };
|
|
await SecretProviderHelper.ApplySecretProviderAsync([], args, m_options.RawOptions, TempFolder.SystemTempPath, SecretProvider, cancellationToken).ConfigureAwait(false);
|
|
// Write back the backend argument, if it was modified by the secret provider
|
|
m_backendUrl = args[0].ToString();
|
|
}
|
|
|
|
/// <summary>
|
|
/// This function will examine all options passed on the commandline, and test for unsupported or deprecated values.
|
|
/// Any errors will be logged into the statistics module.
|
|
/// </summary>
|
|
private void ValidateOptions()
|
|
{
|
|
// Check if only one of the retention options is set
|
|
var selectedRetentionOptions = new List<String>();
|
|
|
|
if (m_options.KeepTime.Ticks > 0)
|
|
{
|
|
selectedRetentionOptions.Add("keep-time");
|
|
}
|
|
|
|
if (m_options.KeepVersions > 0)
|
|
{
|
|
selectedRetentionOptions.Add("keep-versions");
|
|
}
|
|
|
|
if (m_options.RetentionPolicy.Any())
|
|
{
|
|
selectedRetentionOptions.Add("retention-policy");
|
|
}
|
|
|
|
if (selectedRetentionOptions.Count() > 1)
|
|
{
|
|
throw new UserInformationException(string.Format("Setting multiple retention options ({0}) is not permitted",
|
|
string.Join(", ", selectedRetentionOptions.Select(x => "--" + x))), "MultipleRetentionOptionsNotSupported");
|
|
}
|
|
|
|
// Check Prefix
|
|
if (!string.IsNullOrWhiteSpace(m_options.Prefix) && m_options.Prefix.Contains("-"))
|
|
throw new UserInformationException("The prefix cannot contain hyphens (-)", "PrefixCannotContainHyphens");
|
|
|
|
if (m_options.VolumeSize < m_options.Blocksize * 2)
|
|
throw new UserInformationException("The volume size must be at least twice the block size", "VolumeSizeTooSmall");
|
|
|
|
//Check validity of retention-policy option value
|
|
try
|
|
{
|
|
foreach (var configEntry in m_options.RetentionPolicy)
|
|
{
|
|
if (!configEntry.IsKeepAllVersions() && !configEntry.IsUnlimtedTimeframe() &&
|
|
configEntry.Interval >= configEntry.Timeframe)
|
|
{
|
|
throw new Interface.UserInformationException("An interval cannot be bigger than the timeframe it is in", "IntervalCannotBeBiggerThanTimeFrame");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception e) // simply reading the option value might also result in an exception due to incorrect formatting
|
|
{
|
|
throw new Interface.UserInformationException(string.Format("An error occoured while processing the value of --{0}", "retention-policy"), "RetentionPolicyParseError", e);
|
|
}
|
|
|
|
//Keep a list of all supplied options
|
|
var ropts = new Dictionary<string, string>(m_options.RawOptions);
|
|
|
|
//Keep a list of all supported options
|
|
var supportedOptions = new Dictionary<string, ICommandLineArgument>();
|
|
|
|
//There are a few internal options that are not accessible from outside, and thus not listed
|
|
foreach (var s in Options.InternalOptions)
|
|
supportedOptions[s] = null;
|
|
|
|
//Figure out what module options are supported in the current setup
|
|
var moduleOptions = new List<ICommandLineArgument>();
|
|
var disabledModuleOptions = new Dictionary<string, string>();
|
|
var loadedModuleKeys = m_options.LoadedModules.Select(k => k.Key).ToHashSet();
|
|
|
|
foreach (var m in DynamicLoader.GenericLoader.Modules)
|
|
{
|
|
if (m.SupportedCommands == null)
|
|
continue;
|
|
|
|
if (loadedModuleKeys.Contains(m.Key))
|
|
moduleOptions.AddRange(m.SupportedCommands);
|
|
else
|
|
foreach (var c in m.SupportedCommands)
|
|
{
|
|
disabledModuleOptions[c.Name] = m.DisplayName + " (" + m.Key + ")";
|
|
|
|
if (c.Aliases != null)
|
|
foreach (string s in c.Aliases)
|
|
disabledModuleOptions[s] = disabledModuleOptions[c.Name];
|
|
}
|
|
}
|
|
|
|
// Throw url-encoded options into the mix
|
|
//TODO: This can hide values if both commandline and url-parameters supply the same key
|
|
var ext = new Library.Utility.Uri(m_backendUrl).QueryParameters;
|
|
foreach (var k in ext.AllKeys)
|
|
ropts[k] = ext[k];
|
|
|
|
//Now run through all supported options, and look for deprecated options
|
|
foreach (var l in new IEnumerable<ICommandLineArgument>[] {
|
|
m_options.SupportedCommands,
|
|
DynamicLoader.BackendLoader.GetSupportedCommands(m_backendUrl),
|
|
m_options.NoEncryption ? null : DynamicLoader.EncryptionLoader.GetSupportedCommands(m_options.EncryptionModule),
|
|
moduleOptions,
|
|
DynamicLoader.CompressionLoader.GetSupportedCommands(m_options.CompressionModule) })
|
|
{
|
|
if (l != null)
|
|
foreach (ICommandLineArgument a in l)
|
|
{
|
|
if (supportedOptions.ContainsKey(a.Name) && !Options.KnownDuplicates.Contains(a.Name))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "DuplicateOption", null, Strings.Controller.DuplicateOptionNameWarning(a.Name));
|
|
|
|
supportedOptions[a.Name] = a;
|
|
|
|
if (a.Aliases != null)
|
|
foreach (string s in a.Aliases)
|
|
{
|
|
if (supportedOptions.ContainsKey(s) && !Options.KnownDuplicates.Contains(s))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "DuplicateOption", null, Strings.Controller.DuplicateOptionNameWarning(s));
|
|
|
|
supportedOptions[s] = a;
|
|
}
|
|
|
|
if (a.Deprecated)
|
|
{
|
|
List<string> aliases = new List<string>();
|
|
aliases.Add(a.Name);
|
|
if (a.Aliases != null)
|
|
aliases.AddRange(a.Aliases);
|
|
|
|
foreach (string s in aliases)
|
|
if (ropts.ContainsKey(s))
|
|
{
|
|
string optname = a.Name;
|
|
if (a.Name != s)
|
|
optname += " (" + s + ")";
|
|
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "DeprecatedOption", null, Strings.Controller.DeprecatedOptionUsedWarning(optname, a.DeprecationMessage), null);
|
|
}
|
|
|
|
}
|
|
|
|
if (a.Type == CommandLineArgument.ArgumentType.Boolean && a.DefaultValue == "true" && !a.Deprecated)
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "DefaultBooleanOption", null, Strings.Controller.WrongDefaultBooleanOptionWarning(a.Name), null);
|
|
}
|
|
}
|
|
|
|
//Now look for options that were supplied but not supported
|
|
foreach (var s in ropts.Keys)
|
|
if (!supportedOptions.ContainsKey(s))
|
|
if (disabledModuleOptions.ContainsKey(s))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "UnsupportedDisabledModule", null, Strings.Controller.UnsupportedOptionDisabledModuleWarning(s, disabledModuleOptions[s]), null);
|
|
else
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "UnsupportedOption", null, Strings.Controller.UnsupportedOptionWarning(s), null);
|
|
|
|
//Look at the value supplied for each argument and see if is valid according to its type
|
|
foreach (var s in ropts.Keys)
|
|
{
|
|
if (supportedOptions.TryGetValue(s, out var arg) && arg != null)
|
|
{
|
|
var validationMessage = CommandLineArgumentValidator.ValidateOptionValue(arg, s, ropts[s]);
|
|
if (!string.IsNullOrWhiteSpace(validationMessage))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "OptionValidationError", null, validationMessage);
|
|
}
|
|
}
|
|
|
|
var scheme = Library.Utility.Utility.GuessScheme(m_backendUrl);
|
|
if (DynamicLoader.BackendLoader.GetSupportedCommands(m_backendUrl) == null && !string.Equals(m_backendUrl, "dummy://", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
// If the backend is HTTP or HTTPS, give a custom error message
|
|
if (string.Equals(scheme, "http", StringComparison.OrdinalIgnoreCase) || string.Equals(scheme, "https", StringComparison.OrdinalIgnoreCase))
|
|
throw new UserInformationException(Strings.Controller.BackendNotSupportedHttpError, "BackendNotSupportedHttp");
|
|
|
|
throw new UserInformationException(Strings.Controller.BackendNotSupportedError(scheme, string.Join(", ", DynamicLoader.BackendLoader.Keys)), "BackendNotSupported");
|
|
}
|
|
|
|
//Inform the user about unmaintained backends
|
|
if (BackendModules.DeprecatedBackendModules.Contains(scheme))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "BackendDeprecated", null, $"The backend {scheme} is deprecated and should be migrated away from.");
|
|
|
|
//Warn about experimental diskimage filesystem parsing
|
|
if (m_options.RawOptions.ContainsKey("diskimage-filesystem-parsed"))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "ExperimentalDiskImageFilesystemParsed", null, "The option 'diskimage-filesystem-parsed' is experimental and should be used with caution.");
|
|
|
|
//TODO: Based on the action, see if all options are relevant
|
|
}
|
|
|
|
/// <summary>
|
|
/// Helper method that expands the users chosen source input paths,
|
|
/// and removes duplicate paths
|
|
/// </summary>
|
|
/// <param name="inputsources">The input sources.</param>
|
|
/// <param name="filter">The filter.</param>
|
|
/// <param name="options">The options.</param>
|
|
/// <returns>The expanded and filtered sources.</returns>
|
|
private static string[] ExpandInputSources(string[] inputsources, IFilter filter, Options options)
|
|
{
|
|
if (inputsources == null || inputsources.Length == 0)
|
|
throw new UserInformationException(Strings.Controller.NoSourceFoldersError, "NoSourceFolders");
|
|
|
|
var sources = new List<string>(inputsources.Length);
|
|
|
|
DriveInfo[] drives = null;
|
|
|
|
//Make sure they all have the same format and exist
|
|
foreach (var inputsource in inputsources)
|
|
{
|
|
List<string> expandedSources = new List<string>();
|
|
|
|
if (OperatingSystem.IsWindows() && (inputsource.StartsWith("*:", StringComparison.Ordinal) || inputsource.StartsWith("?:", StringComparison.Ordinal)))
|
|
{
|
|
// *: drive paths are only supported on Windows clients
|
|
// Lazily load the drive info
|
|
drives = drives ?? DriveInfo.GetDrives();
|
|
|
|
// Replace the drive letter with each available drive
|
|
string sourcePath = inputsource.Substring(1);
|
|
foreach (DriveInfo drive in drives)
|
|
{
|
|
string expandedSource = drive.Name[0] + sourcePath;
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "AddingSourcePathFromWildcard", @"Adding source path ""{0}"" due to wildcard source path ""{1}""", expandedSource, inputsource);
|
|
expandedSources.Add(expandedSource);
|
|
}
|
|
}
|
|
else if (OperatingSystem.IsWindows() && inputsource.StartsWith(@"\\?\Volume{", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
// In order to specify a drive by it's volume name, adopt the volume guid path syntax:
|
|
// \\?\Volume{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}
|
|
// The volume guid can be found using the 'mountvol' commandline tool.
|
|
// However, instead of using this path with Windows APIs directory, it is adapted here to a standard path.
|
|
Guid volumeGuid;
|
|
if (Guid.TryParse(inputsource.Substring(@"\\?\Volume{".Length, @"XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX".Length), out volumeGuid))
|
|
{
|
|
string driveLetter = Library.Utility.Utility.GetDriveLetterFromVolumeGuid(volumeGuid);
|
|
if (!string.IsNullOrEmpty(driveLetter))
|
|
{
|
|
string expandedSource = driveLetter + inputsource.Substring(@"\\?\Volume{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}".Length);
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "AddingSourceFromGuid", @"Adding source path ""{0}"" in place of volume guid source path ""{1}""", expandedSource, inputsource);
|
|
expandedSources.Add(expandedSource);
|
|
}
|
|
else
|
|
{
|
|
// Couldn't find a drive where this volume is mounted: abort if requested,
|
|
// otherwise warn (unless missing sources are silently allowed)
|
|
if (options.AbortIfSourceMissing)
|
|
throw new UserInformationException(Strings.Controller.SourceVolumeNameNotFoundError(inputsource, volumeGuid), "MissingSourceFolder");
|
|
else if (!options.AllowMissingSource)
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "SourceVolumeNotFound", null, Strings.Controller.SourceVolumeNameNotFoundError(inputsource, volumeGuid));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// Couldn't parse/find this volume: abort if requested, otherwise warn
|
|
// (unless missing sources are silently allowed)
|
|
if (options.AbortIfSourceMissing)
|
|
throw new UserInformationException(Strings.Controller.SourceVolumeNameInvalidError(inputsource), "SourceVolumeNameInvalid");
|
|
else if (!options.AllowMissingSource)
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "SourceVolumeInvalid", null, Strings.Controller.SourceVolumeNameInvalidError(inputsource));
|
|
}
|
|
}
|
|
else
|
|
{
|
|
expandedSources.Add(inputsource);
|
|
}
|
|
|
|
var foundAnyPaths = false;
|
|
var unauthorized = false;
|
|
foreach (var expandedSource in expandedSources)
|
|
{
|
|
string source;
|
|
// Check if this is a mounted path, avoid logging the source if something fails here
|
|
if (expandedSource.StartsWith("@"))
|
|
{
|
|
// NOTE: If the remote source fails to load,
|
|
// this will be an enumeration warning, but will result
|
|
// in the backup being recorded without files from the source
|
|
// Eventually, this could lead to retention deletion,
|
|
// causing the last backup with the data from the source to be deleted
|
|
|
|
// TODO: We do not honor the PreventEmptySource option here,
|
|
// as we do not know if the source is empty or not
|
|
foundAnyPaths = true;
|
|
sources.Add(expandedSource);
|
|
continue;
|
|
}
|
|
|
|
try
|
|
{
|
|
// TODO: This expands "C:" to CWD, but not C:\
|
|
source = Path.GetFullPath(expandedSource);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Note that we use the original source (with the *) in the error
|
|
throw new UserInformationException(Strings.Controller.InvalidPathError(expandedSource, ex.Message), "InputSourceInvalid", ex);
|
|
}
|
|
|
|
var fi = new FileInfo(source);
|
|
var di = new DirectoryInfo(source);
|
|
if (fi.Exists || di.Exists)
|
|
{
|
|
foundAnyPaths = true;
|
|
|
|
if (!fi.Exists)
|
|
{
|
|
// If the directory exists, but is empty, and we are not allowed to have empty sources, throw an error
|
|
try
|
|
{
|
|
if (options.PreventEmptySource && di.Exists && !di.EnumerateFileSystemInfos().Any())
|
|
throw new UserInformationException(Strings.Controller.SourceFolderEmptyError(inputsource), "SourceFolderEmpty");
|
|
}
|
|
catch (UnauthorizedAccessException ex)
|
|
{
|
|
throw new UserInformationException(Strings.Controller.SourceUnauthorizedError(inputsource), "SourceUnauthorized", ex);
|
|
}
|
|
|
|
source = Util.AppendDirSeparator(source);
|
|
}
|
|
|
|
sources.Add(source);
|
|
}
|
|
else
|
|
{
|
|
try
|
|
{
|
|
// Try to get attributes. Returns -1 if source doesn't exist, otherwise throws an exception.
|
|
// In this case, it is irrelevant to use fileinfo or directoryinfo to retrieve attributes.
|
|
var unused = fi.Attributes;
|
|
}
|
|
catch (UnauthorizedAccessException ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "AddingSourceFolder",
|
|
ex, @"Insufficient permissions to read ""{0}"", skipping", expandedSource);
|
|
unauthorized = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If no paths were found, handle according to options
|
|
if (!foundAnyPaths)
|
|
{
|
|
if (options.AbortIfSourceMissing)
|
|
{
|
|
if (unauthorized)
|
|
throw new UserInformationException(Strings.Controller.SourceUnauthorizedError(inputsource), "SourceUnauthorized");
|
|
|
|
throw new UserInformationException(Strings.Controller.SourceIsMissingError(inputsource), "SourceIsMissing");
|
|
}
|
|
else if (!options.AllowMissingSource)
|
|
{
|
|
if (unauthorized)
|
|
throw new UserInformationException(Strings.Controller.SourceUnauthorizedError(inputsource), "SourceUnauthorized");
|
|
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "SourceIsMissing", null, Strings.Controller.SourceIsMissingWarning(inputsource));
|
|
}
|
|
}
|
|
}
|
|
|
|
//Sanity check for duplicate files/folders
|
|
ISet<string> pathDuplicates;
|
|
sources = Library.Utility.Utility.GetUniqueItems(sources, Library.Utility.Utility.ClientFilenameStringComparer, out pathDuplicates).ToList();
|
|
|
|
foreach (var pathDuplicate in pathDuplicates)
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "RemoveDuplicateSource", "Removing duplicate source: {0}", pathDuplicate);
|
|
|
|
//Sanity check for multiple inclusions of the same folder
|
|
for (int i = 0; i < sources.Count; i++)
|
|
for (int j = 0; j < sources.Count; j++)
|
|
if (i != j && sources[i].StartsWith(sources[j], Library.Utility.Utility.ClientFilenameStringComparison) && sources[i].EndsWith(Util.DirectorySeparatorString, Library.Utility.Utility.ClientFilenameStringComparison))
|
|
{
|
|
if (filter != null)
|
|
{
|
|
bool excludes;
|
|
|
|
FilterExpression.AnalyzeFilters(filter, out _, out excludes);
|
|
|
|
// If there are no excludes, there is no need to keep the folder as a filter
|
|
if (excludes)
|
|
{
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingSubfolderSource", "Removing source \"{0}\" because it is a subfolder of \"{1}\", and using it as an include filter", sources[i], sources[j]);
|
|
filter = JoinedFilterExpression.Join(new FilterExpression(sources[i]), filter);
|
|
}
|
|
else
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingSubfolderSource", "Removing source \"{0}\" because it is a subfolder or subfile of \"{1}\"", sources[i], sources[j]);
|
|
}
|
|
else
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingSubfolderSource", "Removing source \"{0}\" because it is a subfolder or subfile of \"{1}\"", sources[i], sources[j]);
|
|
|
|
sources.RemoveAt(i);
|
|
i--;
|
|
break;
|
|
}
|
|
|
|
// If there is nothing to do, don't run the backup
|
|
if (sources.Count == 0)
|
|
throw new UserInformationException(Strings.Controller.NoSourcesError, "NoSources");
|
|
|
|
return sources.ToArray();
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task PauseAsync(bool alsoTransfers)
|
|
{
|
|
var ct = m_currentTaskControl;
|
|
if (ct != null)
|
|
ct.Pause(alsoTransfers);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task ResumeAsync()
|
|
{
|
|
var ct = m_currentTaskControl;
|
|
if (ct != null)
|
|
ct.Resume();
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task StopAsync()
|
|
{
|
|
var ct = m_currentTaskControl;
|
|
if (ct == null)
|
|
return Task.CompletedTask;
|
|
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "CancellationRequested", "Cancellation Requested");
|
|
ct.Stop();
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task AbortAsync()
|
|
{
|
|
m_currentTaskControl?.Terminate();
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task SetThrottleSpeedsAsync(long maxUploadPrSecond, long maxDownloadPrSecond)
|
|
{
|
|
// Set these values in the options, in case the backend is not up yet
|
|
m_options.MaxUploadPrSecond = maxUploadPrSecond;
|
|
m_options.MaxDownloadPrSecond = maxDownloadPrSecond;
|
|
m_currentBackendManager?.UpdateThrottleValues(maxUploadPrSecond, maxDownloadPrSecond);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The time of the last compact operation
|
|
/// </summary>
|
|
private DateTime m_lastCompact;
|
|
|
|
/// <summary>
|
|
/// The time of the last vacuum operation
|
|
/// </summary>
|
|
private DateTime m_lastVacuum;
|
|
|
|
/// <inheritdoc />
|
|
public Task SetLastCompactAsync(DateTime lastCompact)
|
|
{
|
|
m_lastCompact = lastCompact;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task SetLastVacuumAsync(DateTime lastVacuum)
|
|
{
|
|
m_lastVacuum = lastVacuum;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
private void CheckAutoCompactInterval()
|
|
{
|
|
if (!m_options.NoAutoCompact && (m_lastCompact > DateTime.MinValue) && (m_lastCompact.Add(m_options.AutoCompactInterval) > DateTime.Now))
|
|
{
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Skipping auto compaction until {0}", m_lastCompact.Add(m_options.AutoCompactInterval));
|
|
m_options.RawOptions["no-auto-compact"] = "true";
|
|
}
|
|
}
|
|
|
|
private void CheckAutoVacuumInterval()
|
|
{
|
|
if (m_options.AutoVacuum && (m_lastVacuum > DateTime.MinValue) && (m_lastVacuum.Add(m_options.AutoVacuumInterval) > DateTime.Now))
|
|
{
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "VacuumResults", "Skipping auto vacuum until {0}", m_lastVacuum.Add(m_options.AutoVacuumInterval));
|
|
m_options.RawOptions["auto-vacuum"] = "false";
|
|
}
|
|
}
|
|
|
|
#region IDisposable Members
|
|
|
|
/// <inheritdoc />
|
|
public void Dispose()
|
|
{
|
|
}
|
|
|
|
#endregion
|
|
}
|
|
}
|