Kenneth Skovhede
8663f55e8b
This PR fixes a bug that caused backup searching to be case-sensitive. The PR also exposes the case sensitive-flag in the API endpoint so the UI can toggle case-sensitive searching.
1342 lines
67 KiB
C#
1342 lines
67 KiB
C#
// Copyright (C) 2026, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Threading;
|
|
using Duplicati.Library.Utility;
|
|
using Duplicati.Library.Common.IO;
|
|
using Duplicati.Library.Interface;
|
|
using Duplicati.Library.Main.Database;
|
|
using System.Threading.Tasks;
|
|
using Duplicati.Library.Main.Operation.Common;
|
|
using System.IO;
|
|
using Duplicati.Library.Backends;
|
|
|
|
namespace Duplicati.Library.Main
|
|
{
|
|
public class Controller : IDisposable, IController
|
|
{
|
|
/// <summary>
|
|
/// The tag used for logging
|
|
/// </summary>
|
|
private static readonly string LOGTAG = Logging.Log.LogTagFromType<Controller>();
|
|
/// <summary>
|
|
/// The backend url
|
|
/// </summary>
|
|
private string m_backendUrl;
|
|
/// <summary>
|
|
/// The parsed type-safe version of the commandline options
|
|
/// </summary>
|
|
private readonly Options m_options;
|
|
/// <summary>
|
|
/// The optional secret provider, if none provided in options
|
|
/// </summary>
|
|
public ISecretProvider SecretProvider { get; private set; }
|
|
/// <summary>
|
|
/// The destination for all output messages during execution
|
|
/// </summary>
|
|
private IMessageSink m_messageSink;
|
|
|
|
// TODO: The m_current* approach here prevents us from having multiple concurrent operations
|
|
|
|
/// <summary>
|
|
/// The current executing task
|
|
/// </summary>
|
|
private ITaskControl m_currentTaskControl = null;
|
|
|
|
/// <summary>
|
|
/// The current backend manager
|
|
/// </summary>
|
|
private IBackendManager m_currentBackendManager = null;
|
|
|
|
/// <summary>
|
|
/// If not null, active locale change that needs to be reset
|
|
/// </summary>
|
|
private LocaleChange m_localeChange = null;
|
|
|
|
/// <summary>
|
|
/// Callback method invoked when an operation is started
|
|
/// </summary>
|
|
public Action<IBasicResults> OnOperationStarted { get; set; }
|
|
|
|
/// <summary>
|
|
/// Callback method invoked when an operation is completed
|
|
/// </summary>
|
|
public Action<IBasicResults, Exception> OnOperationCompleted { get; set; }
|
|
|
|
/// <summary>
|
|
/// Constructs a new interface for performing backup and restore operations
|
|
/// </summary>
|
|
/// <param name="backendUrl">The url for the backend to use</param>
|
|
/// <param name="options">All required options</param>
|
|
public Controller(string backendUrl, Dictionary<string, string> options, IMessageSink messageSink)
|
|
{
|
|
m_backendUrl = backendUrl;
|
|
m_options = new Options(options);
|
|
m_messageSink = messageSink;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task AppendSinkAsync(IMessageSink sink)
|
|
{
|
|
if (this.m_messageSink is MultiMessageSink messageSink)
|
|
messageSink.Append(sink);
|
|
else
|
|
m_messageSink = new MultiMessageSink(m_messageSink, sink);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task SetSecretProviderAsync(ISecretProvider secretProvider)
|
|
{
|
|
SecretProvider = secretProvider;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IBackupResults> BackupAsync(string[] inputsources, IFilter inputFilter = null)
|
|
{
|
|
UsageReporter.Reporter.Report("USE_BACKEND", new Library.Utility.Uri(m_backendUrl).Scheme);
|
|
UsageReporter.Reporter.Report("USE_COMPRESSION", m_options.CompressionModule);
|
|
UsageReporter.Reporter.Report("USE_ENCRYPTION", m_options.EncryptionModule);
|
|
|
|
// Warn about experimental tar-based compression modules
|
|
if (m_options.CompressionModule.Equals("tzstd", StringComparison.OrdinalIgnoreCase) ||
|
|
m_options.CompressionModule.Equals("tgz", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "ExperimentalCompressionModule", null, $"The compression module '{m_options.CompressionModule}' is experimental and for testing only.");
|
|
}
|
|
|
|
CheckAutoCompactInterval();
|
|
CheckAutoVacuumInterval();
|
|
|
|
return await RunActionAsync(new BackupResults(), inputsources, inputFilter, false, static async config =>
|
|
{
|
|
using (var h = new Operation.BackupHandler(config.Options, config.Result))
|
|
await h.RunAsync(ExpandInputSources(config.Paths, config.Filter, config.Options), config.BackendManager, config.Filter)
|
|
.ConfigureAwait(false);
|
|
|
|
UsageReporter.Reporter.Report("BACKUP_FILECOUNT", config.Result.ExaminedFiles);
|
|
UsageReporter.Reporter.Report("BACKUP_FILESIZE", config.Result.SizeOfExaminedFiles);
|
|
UsageReporter.Reporter.Report("BACKUP_DURATION", (long)config.Result.Duration.TotalSeconds);
|
|
|
|
using (var h = new Operation.RemoteSynchronizationHandler(config.BackendUrl, config.Options, config.Result))
|
|
await h.RunAsync()
|
|
.ConfigureAwait(false);
|
|
}).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IRestoreResults> RestoreAsync(string[] paths, IFilter inputFilter = null)
|
|
{
|
|
return await RunActionAsync(new RestoreResults(), paths, inputFilter, false, static async config =>
|
|
{
|
|
using var restoreDestination =
|
|
(config.Options.Restorepath ?? "").StartsWith("@")
|
|
|
|
// Remote destination
|
|
? await DynamicLoader.RestoreDestinationProviderLoader.GetRestoreDestinationProvider(
|
|
config.Options.Restorepath.Substring(1),
|
|
config.Options.RawOptions,
|
|
config.Result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false)
|
|
|
|
// Local destination
|
|
: new SourceProvider.FileRestoreDestinationProvider(config.Options.Restorepath ?? "", config.Options.AllowRestoreOutsideTargetDirectory);
|
|
|
|
if (restoreDestination == null)
|
|
throw new UserInformationException($"Could not find restore destination for path: {config.Options.Restorepath}", "InvalidRestoreDestination");
|
|
|
|
await new Operation.RestoreHandler(config.Options, config.Result)
|
|
.RunAsync(config.Paths, config.BackendManager, config.Filter, restoreDestination)
|
|
.ConfigureAwait(false);
|
|
|
|
await restoreDestination.Finalize((pg) =>
|
|
{
|
|
config.Result.OperationProgressUpdater.UpdateProgress((float)pg);
|
|
}, config.Result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
config.Result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_Complete);
|
|
config.Result.EndTime = DateTime.UtcNow;
|
|
|
|
UsageReporter.Reporter.Report("RESTORE_FILECOUNT", config.Result.RestoredFiles);
|
|
UsageReporter.Reporter.Report("RESTORE_FILESIZE", config.Result.SizeOfRestoredFiles);
|
|
UsageReporter.Reporter.Report("RESTORE_DURATION", (long)config.Result.Duration.TotalSeconds);
|
|
}).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IRestoreControlFilesResults> RestoreControlFilesAsync(IEnumerable<string> files = null, IFilter inputFilter = null)
|
|
{
|
|
return await RunActionAsync(new RestoreControlFilesResults(), files?.ToArray(), inputFilter, false, static config =>
|
|
new Operation.RestoreControlFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.Paths, config.BackendManager, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IDeleteResults> DeleteAsync()
|
|
{
|
|
return await RunActionAsync(new DeleteResults(), null, null, false, static config =>
|
|
new Operation.DeleteHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IRepairResults> RepairAsync(IFilter inputFilter = null)
|
|
{
|
|
return await RunActionAsync(new RepairResults(), null, inputFilter, false, static config =>
|
|
new Operation.RepairHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListFilesetResults> ListFilesetsAsync()
|
|
=> await RunActionAsync(new ListFilesetResults(), null, null, false, static config =>
|
|
Operation.ListFilesetsHandler.RunAsync(config.Options, config.Result, config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListFolderResults> ListFolderAsync(string[] folders, long offset, long limit, bool extendedData)
|
|
=> await RunActionAsync(new ListFolderResults(), folders, null, new { offset, limit, extendedData }, static config =>
|
|
Operation.ListFolderHandler.RunAsync(config.Options, config.Result, config.Paths, config.Context.offset, config.Context.limit, config.Context.extendedData)
|
|
).ConfigureAwait(false);
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListFileVersionsResults> ListFileVersionsAsync(string[] files, long offset, long limit)
|
|
=> await RunActionAsync(new ListFileVersionsResults(), files, null, new { offset, limit }, static config =>
|
|
Operation.ListFileVersionsHandler.RunAsync(config.Options, config.Result, config.Paths, config.Context.offset, config.Context.limit)
|
|
).ConfigureAwait(false);
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ISearchFilesResults> SearchEntriesAsync(string[] pathprefixes, IFilter inputFilter, bool caseSensitive, long offset, long limit, bool extendedData)
|
|
=> await RunActionAsync(new SearchFilesResults(), pathprefixes, inputFilter, new { caseSensitive, offset, limit, extendedData }, static config =>
|
|
Operation.SearchEntriesHandler.RunAsync(config.Options, config.Result, config.Paths, config.Filter, config.Context.caseSensitive, config.Context.offset, config.Context.limit, config.Context.extendedData)
|
|
).ConfigureAwait(false);
|
|
|
|
/// <inheritdoc />
|
|
public Task<IListResults> ListAsync()
|
|
=> ListAsync(null, null);
|
|
|
|
/// <inheritdoc />
|
|
public Task<IListResults> ListAsync(string filterstring)
|
|
=> ListAsync(filterstring == null ? null : new string[] { filterstring }, null);
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListResults> ListAsync(IEnumerable<string> filterstrings, IFilter inputFilter)
|
|
{
|
|
return await RunActionAsync(new ListResults(), filterstrings?.ToArray(), inputFilter, false, static config =>
|
|
new Operation.ListFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Paths, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListResults> ListControlFilesAsync(IEnumerable<string> filterstrings, IFilter inputFilter)
|
|
{
|
|
return await RunActionAsync(new ListResults(), filterstrings?.ToArray(), inputFilter, false, static config =>
|
|
new Operation.ListControlFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Paths, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListRemoteResults> ListRemoteAsync()
|
|
{
|
|
return await RunActionAsync(new ListRemoteResults(), null, null, false, static async config =>
|
|
{
|
|
using var tf = File.Exists(config.Options.Dbpath) ? null : new TempFile();
|
|
await using var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? config.Options.Dbpath, "list-remote", true, null, config.Result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
config.Result.SetResult(await config.BackendManager.ListAsync(CancellationToken.None).ConfigureAwait(false));
|
|
}).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListRemoteResults> DeleteAllRemoteFilesAsync()
|
|
{
|
|
return await RunActionAsync(new ListRemoteResults(), null, null, false, static async config =>
|
|
{
|
|
config.Result.OperationProgressUpdater.UpdatePhase(OperationPhase.Delete_Listing);
|
|
{
|
|
// Only delete files that match the expected pattern and prefix
|
|
var list = (await config.BackendManager.ListAsync(config.Result.TaskControl.ProgressToken).ConfigureAwait(false))
|
|
.Select(x => Volumes.VolumeBase.ParseFilename(x))
|
|
.Where(x => x != null)
|
|
.Where(x => x.Prefix == config.Options.Prefix)
|
|
.ToList();
|
|
|
|
// If the local database is available, we will use it to avoid deleting unrelated files
|
|
// from the backend. Otherwise, we may accidentally delete non-Duplicati files, or
|
|
// files from a different Duplicati configuration that points to the same backend location
|
|
// and uses the same prefix (see issues #2678, #3845, and #4244).
|
|
if (File.Exists(config.Options.Dbpath))
|
|
{
|
|
await using LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(config.Options.Dbpath, "list-remote", true, null, config.Result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
var dbRemoteVolumes = db.GetRemoteVolumesAsync(config.Result.TaskControl.ProgressToken);
|
|
var dbRemoteFiles = await dbRemoteVolumes
|
|
.Select(x => x.Name)
|
|
.ToHashSetAsync(cancellationToken: config.Result.TaskControl.ProgressToken)
|
|
.ConfigureAwait(false);
|
|
list = list.Where(x =>
|
|
dbRemoteFiles.Contains(x.File.Name)
|
|
).ToList();
|
|
}
|
|
|
|
config.Result.OperationProgressUpdater.UpdatePhase(OperationPhase.Delete_Deleting);
|
|
config.Result.OperationProgressUpdater.UpdateProgress(0);
|
|
for (var i = 0; i < list.Count; i++)
|
|
{
|
|
try
|
|
{
|
|
if (config.Options.Dryrun)
|
|
{
|
|
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteFile", "Would delete file: {0}", list[i].File.Name);
|
|
}
|
|
else
|
|
{
|
|
await config.BackendManager.DeleteAsync(list[i].File.Name, list[i].File.Size, true, config.Result.TaskControl.ProgressToken).ConfigureAwait(false);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "DeleteFilesetError", ex, "Failed to delete remote file: {0}", list[i].File.Name);
|
|
}
|
|
config.Result.OperationProgressUpdater.UpdateProgress((float)i / list.Count);
|
|
}
|
|
config.Result.OperationProgressUpdater.UpdateProgress(1);
|
|
}
|
|
}).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ICompactResults> CompactAsync()
|
|
{
|
|
CheckAutoVacuumInterval();
|
|
|
|
return await RunActionAsync(new CompactResults(), null, null, false, static config =>
|
|
new Operation.CompactHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ISetLockResults> SetLocksAsync()
|
|
{
|
|
return await RunActionAsync(new SetLockResults(), null, null, false, static config =>
|
|
new Operation.SetLocksHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IReadLockInfoResults> ReadLockInfoAsync()
|
|
{
|
|
return await RunActionAsync(new ReadLockInfoResults(), null, null, false, static config =>
|
|
new Operation.ReadLockInfoHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IRecreateDatabaseResults> UpdateDatabaseWithVersionsAsync(IFilter inputFilter = null)
|
|
{
|
|
return await RunActionAsync(new RecreateDatabaseResults(), null, inputFilter, false, static async config =>
|
|
{
|
|
var filelistfilter = Operation.RestoreHandler.GetNumberedFilelistFilterDelegate(config.Options.Time, config.Options.Version, singleTimeMatch: true);
|
|
using (var h = new Operation.RecreateDatabaseHandler(config.Options, config.Result))
|
|
await h.RunUpdateAsync(config.BackendManager, config.Filter, filelistfilter, null)
|
|
.ConfigureAwait(false);
|
|
}).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ICreateLogDatabaseResults> CreateLogDatabaseAsync(string targetpath)
|
|
{
|
|
return await RunActionAsync(new CreateLogDatabaseResults(), [targetpath], null, false, static async config =>
|
|
await new Operation.CreateBugReportHandler(config.Paths[0], config.Options, config.Result).RunAsync()
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListChangesResults> ListChangesAsync(string baseVersion, string targetVersion, IEnumerable<string> filterstrings = null, IFilter inputFilter = null, Action<IListChangesResults, IEnumerable<Tuple<ListChangesChangeType, ListChangesElementType, string>>> callback = null)
|
|
{
|
|
|
|
return await RunActionAsync(new ListChangesResults(), [baseVersion, targetVersion], inputFilter, new { filterstrings, callback }, async static config =>
|
|
await new Operation.ListChangesHandler(config.Options, config.Result)
|
|
.RunAsync(config.Paths[0], config.Paths[1], config.BackendManager, config.Context.filterstrings, config.Filter, config.Context.callback)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListAffectedResults> ListAffectedAsync(List<string> args, Action<IListAffectedResults> callback = null)
|
|
{
|
|
return await RunActionAsync(new ListAffectedResults(), args?.ToArray(), null, new { callback }, static config =>
|
|
new Operation.ListAffected(config.Options, config.Result)
|
|
.RunAsync(config.Paths, config.Context.callback)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ITestResults> TestAsync(long samples = 1)
|
|
{
|
|
if (!m_options.RawOptions.ContainsKey("full-remote-verification"))
|
|
m_options.RawOptions["full-remote-verification"] = "true";
|
|
|
|
return await RunActionAsync(new TestResults(), null, null, new { samples }, static config =>
|
|
new Operation.TestHandler(config.Options, config.Result)
|
|
.RunAsync(config.Context.samples, config.BackendManager)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ITestFilterResults> TestFilterAsync(string[] paths, IFilter inputFilter = null)
|
|
{
|
|
m_options.RawOptions["dry-run"] = "true";
|
|
m_options.RawOptions["dbpath"] = "INVALID!";
|
|
|
|
// Redirect all messages from the filter to the message sink
|
|
var filtertag = Logging.Log.LogTagFromType(typeof(Operation.Backup.FileEnumerationProcess));
|
|
using (Logging.Log.StartScope(m_messageSink.WriteMessage, x => x.FilterTag.Contains(filtertag)))
|
|
{
|
|
return await RunActionAsync(new TestFilterResults(), paths, inputFilter, false, static config =>
|
|
new Operation.TestFilterHandler(config.Options, config.Result)
|
|
.RunAsync(ExpandInputSources(config.Paths, config.Filter, config.Options), config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ISystemInfoResults> SystemInfoAsync()
|
|
{
|
|
return await RunActionAsync(new SystemInfoResults(), null, null, false, static config =>
|
|
Operation.SystemInfoHandler.RunAsync(config.Result)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IPurgeFilesResults> PurgeFilesAsync(IFilter inputFilter)
|
|
{
|
|
return await RunActionAsync(new PurgeFilesResults(), null, inputFilter, false, static config =>
|
|
new Operation.PurgeFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IListBrokenFilesResults> ListBrokenFilesAsync(IFilter inputFilter, Func<long, DateTime, long, string, long, bool> callbackhandler = null)
|
|
{
|
|
return await RunActionAsync(new ListBrokenFilesResults(), null, inputFilter, new { callbackhandler }, static config =>
|
|
new Operation.ListBrokenFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Filter, config.Context.callbackhandler)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IPurgeBrokenFilesResults> PurgeBrokenFilesAsync(IFilter inputFilter)
|
|
{
|
|
return await RunActionAsync(new PurgeBrokenFilesResults(), null, inputFilter, false, static config =>
|
|
new Operation.PurgeBrokenFilesHandler(config.Options, config.Result)
|
|
.RunAsync(config.BackendManager, config.Filter)
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<ISendMailResults> SendMailAsync()
|
|
{
|
|
m_options.RawOptions["send-mail-level"] = "all";
|
|
m_options.RawOptions["send-mail-any-operation"] = "true";
|
|
const string sendMailModuleKey = "sendmail";
|
|
m_options.RawOptions.TryGetValue("send-mail-to", out string targetmail);
|
|
if (string.IsNullOrWhiteSpace(targetmail))
|
|
throw new Exception(string.Format("No email specified, please use --{0}", "send-mail-to"));
|
|
|
|
m_options.RawOptions["disable-module"] = string.Join(
|
|
",",
|
|
DynamicLoader.GenericLoader.Modules
|
|
.Where(m =>
|
|
!string.Equals(m.Key, sendMailModuleKey, StringComparison.OrdinalIgnoreCase)
|
|
)
|
|
.Select(x => x.Key)
|
|
);
|
|
|
|
/// Forward all messages from the email module to the message sink
|
|
var filtertag = Logging.Log.LogTagFromType(DynamicLoader.GenericLoader.GetModule(sendMailModuleKey).GetType());
|
|
using (Logging.Log.StartScope(m_messageSink.WriteMessage, x => x.FilterTag.Contains(filtertag)))
|
|
{
|
|
return await RunActionAsync(new SendMailResults(), null, null, false, static config =>
|
|
Task.Run(() =>
|
|
{
|
|
config.Result.Lines = [];
|
|
Thread.Sleep(5);
|
|
})
|
|
).ConfigureAwait(false);
|
|
}
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public async Task<IVacuumResults> VacuumAsync()
|
|
{
|
|
return await RunActionAsync(new VacuumResults(), null, null, false, static config =>
|
|
new Operation.VacuumHandler(config.Options, config.Result)
|
|
.RunAsync()
|
|
).ConfigureAwait(false);
|
|
}
|
|
|
|
private sealed record RunActionData<TRes, TCon>(
|
|
TRes Result,
|
|
TCon Context,
|
|
string BackendUrl,
|
|
string[] Paths,
|
|
Options Options,
|
|
IFilter Filter,
|
|
IBackendManager BackendManager);
|
|
|
|
private async Task<TRes> RunActionAsync<TRes, TCon>(TRes result, string[] paths, IFilter filter, TCon context, Func<RunActionData<TRes, TCon>, Task> method)
|
|
where TRes : ISetCommonOptions, ITaskControlProvider, Logging.ILogDestination, IBasicResults, IBackendWriterProvider
|
|
{
|
|
OnOperationStarted?.Invoke(result);
|
|
var resultSetter = result as ISetCommonOptions;
|
|
using (var logTarget = new ControllerMultiLogTarget(result, Logging.LogMessageType.Information, null, m_options.SuppressWarningsFilter))
|
|
using (Logging.Log.StartScope(logTarget, null))
|
|
{
|
|
logTarget.AddTarget(m_messageSink, m_options.ConsoleLoglevel, m_options.ConsoleLogFilter);
|
|
result.MessageSink = m_messageSink;
|
|
using var httpListener = m_options.LogHttpRequests || m_options.LogSocketData >= 0
|
|
? new NetworkTrafficLogger(m_options.LogHttpRequests, m_options.LogSocketData)
|
|
: null;
|
|
|
|
try
|
|
{
|
|
m_currentTaskControl = result.TaskControl;
|
|
m_options.MainAction = result.MainOperation;
|
|
await ApplySecretProviderAsync(CancellationToken.None).ConfigureAwait(false);
|
|
(paths, filter) = SetupCommonOptions(result, paths, filter, logTarget);
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "StartingOperation", Strings.Controller.StartingOperationMessage(m_options.MainAction));
|
|
|
|
using (SlowQueryMonitor.StartMonitoring(m_options.SlowQueryThreshold))
|
|
using (new ProcessController(m_options))
|
|
using (new Logging.Timer(LOGTAG, string.Format("Run{0}", result.MainOperation), string.Format("Running {0}", result.MainOperation)))
|
|
using (new CoCoL.IsolatedChannelScope())
|
|
using (m_options.ConcurrencyMaxThreads <= 0 ? null : new CoCoL.CappedThreadedThreadPool(m_options.ConcurrencyMaxThreads))
|
|
using (var backend = new Backend.BackendManager(m_backendUrl, m_options, result.BackendWriter, result.TaskControl))
|
|
{
|
|
try
|
|
{
|
|
backend.UpdateThrottleValues(m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond);
|
|
m_currentBackendManager = backend;
|
|
var config = new RunActionData<TRes, TCon>(result, context, m_backendUrl, paths, m_options, filter, backend);
|
|
await method(config).ConfigureAwait(false);
|
|
}
|
|
finally
|
|
{
|
|
m_currentBackendManager = null;
|
|
|
|
// TODO: Should also have a single shared database connection for all operations
|
|
// The transactions should be managed inside the connection, and not passed around
|
|
|
|
// This would allow us to pass the database instance to the backend manager
|
|
// And safeguard against remote operations not being logged in the database
|
|
|
|
// This would also allow us to control the unclean shutdown flag,
|
|
// by toggling this on start and completion of transfers in the manager,
|
|
// instead of relying on the operations to correctly toggle the flag
|
|
if (File.Exists(m_options.Dbpath) && !m_options.NoLocalDb)
|
|
{
|
|
try
|
|
{
|
|
await using var db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, null, CancellationToken.None).ConfigureAwait(false);
|
|
await backend.StopRunnerAndFlushMessagesAsync(db).ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FailedFlushBackendMessages", ex, "Failed to flush backend messages to database: {0}", ex.Message);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
backend.StopRunnerAndDiscardMessages();
|
|
}
|
|
}
|
|
}
|
|
|
|
if (resultSetter.EndTime.Ticks == 0)
|
|
resultSetter.EndTime = DateTime.UtcNow;
|
|
resultSetter.Interrupted = false;
|
|
|
|
if (File.Exists(m_options.Dbpath) && !m_options.Dryrun)
|
|
{
|
|
try
|
|
{
|
|
await using var db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, null, CancellationToken.None).ConfigureAwait(false);
|
|
await db.WriteResultsAndCommitAsync(result, CancellationToken.None).ConfigureAwait(false);
|
|
await db.PurgeLogDataAsync(m_options.LogRetention, CancellationToken.None).ConfigureAwait(false);
|
|
await db.PurgeDeletedVolumesAsync(DateTime.UtcNow, CancellationToken.None).ConfigureAwait(false);
|
|
|
|
// Vacuum is done AFTER the results are written to the database
|
|
// This means that the information about the vacuum is not stored in the database,
|
|
// but will be reported in the log output and messages sent with any of the reporting modules
|
|
if (m_options.AutoVacuum && result is IResultsWithVacuum vacuumResults && result is BasicResults basicResults)
|
|
{
|
|
try
|
|
{
|
|
vacuumResults.VacuumResults = new VacuumResults(basicResults);
|
|
await new Operation.VacuumHandler(m_options, (VacuumResults)vacuumResults.VacuumResults)
|
|
.RunAsync().ConfigureAwait(false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToVacuum", ex, "Failed to vacuum database");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FailedWriteOperation", ex, "Failed to write operation results to database: {0}", ex.Message);
|
|
}
|
|
}
|
|
|
|
OperationComplete(result, null, filter);
|
|
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "CompletedOperation", Strings.Controller.CompletedOperationMessage(m_options.MainAction));
|
|
|
|
return result;
|
|
}
|
|
// Handle custom abort exception from run-script
|
|
catch (OperationAbortException oae)
|
|
{
|
|
resultSetter.EndTime = DateTime.UtcNow;
|
|
// Log this as a normal operation, as the script raising the exception,
|
|
// has already populated either warning or log messages as required
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "AbortOperation", "Aborting operation by request, requested result: {0}", oae.AbortReason);
|
|
|
|
resultSetter.Interrupted = true;
|
|
try
|
|
{
|
|
// No operation was started in database, so write logs to new operation
|
|
if (File.Exists(m_options.Dbpath) && !m_options.Dryrun)
|
|
await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, null, CancellationToken.None).ConfigureAwait(false))
|
|
await db.WriteResultsAndCommitAsync(result, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception we)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FailedWriteOperation", we, we.Message);
|
|
}
|
|
|
|
// Do not propagate the cancel exception
|
|
OperationComplete(result, null, filter);
|
|
|
|
return result;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Logging.Log.WriteErrorMessage(LOGTAG, "FailedOperation", ex, Strings.Controller.FailedOperationMessage(m_options.MainAction));
|
|
|
|
try
|
|
{
|
|
if (result is BasicResults basicResults)
|
|
basicResults.OperationProgressUpdater.UpdatePhase(OperationPhase.Error);
|
|
|
|
resultSetter.Fatal = true;
|
|
// Write logs to previous operation if database exists
|
|
if (File.Exists(m_options.Dbpath) && !m_options.Dryrun)
|
|
await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, null, CancellationToken.None).ConfigureAwait(false))
|
|
await db.WriteResultsAndCommitAsync(result, CancellationToken.None).ConfigureAwait(false);
|
|
}
|
|
catch (Exception we)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FailedWriteOperation", we, we.Message);
|
|
}
|
|
|
|
// Report the result, and the failure
|
|
OperationComplete(result, ex, filter);
|
|
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
m_currentTaskControl = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Temporarily pushes the filter into the raw options so modules can read it.
|
|
/// Returns a disposable that removes the key when disposed.
|
|
/// </summary>
|
|
/// <param name="options">The options to push the filter into</param>
|
|
/// <param name="filter">The filter to push</param>
|
|
/// <returns>A disposable that removes the key when disposed</returns>
|
|
internal static IDisposable PushFilterToOptions(Options options, IFilter filter)
|
|
=> PushFilterToOptions(options, filter, out var _);
|
|
/// <summary>
|
|
/// Temporarily pushes the filter into the raw options so modules can read it.
|
|
/// Returns a disposable that removes the key when disposed.
|
|
/// </summary>
|
|
/// <param name="options">The options to push the filter into</param>
|
|
/// <param name="filter">The filter to push</param>
|
|
/// <returns>A disposable that removes the key when disposed</returns>
|
|
internal static IDisposable PushFilterToOptions(Options options, IFilter filter, out string filterString)
|
|
{
|
|
filterString = filter != null && !filter.Empty
|
|
? string.Join(Path.PathSeparator.ToString(), FilterExpression.Serialize(filter))
|
|
: null;
|
|
|
|
options.RawOptions["filter"] = filterString;
|
|
return new FilterOptionScope(options);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Helper to remove the filter option on dispose
|
|
/// </summary>
|
|
private sealed class FilterOptionScope : IDisposable
|
|
{
|
|
private readonly Options m_opts;
|
|
public FilterOptionScope(Options opts) => m_opts = opts;
|
|
public void Dispose() => m_opts?.RawOptions.Remove("filter");
|
|
}
|
|
|
|
private void OperationComplete(IBasicResults result, Exception exception, IFilter filter)
|
|
{
|
|
using var _ = PushFilterToOptions(m_options, filter);
|
|
if (m_options?.LoadedModules != null)
|
|
{
|
|
foreach (var mx in m_options.LoadedModules)
|
|
if (mx is IGenericCallbackModule module)
|
|
try { module.OnFinish(result, exception); }
|
|
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, $"OnFinishError{mx.Key}", ex, "OnFinish callback {0} failed: {1}", mx.Key, ex.Message); }
|
|
|
|
foreach (var mx in m_options.LoadedModules)
|
|
if (mx is IDisposable disposable)
|
|
try { disposable.Dispose(); }
|
|
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, $"DisposeError{mx.Key}", ex, "Dispose for {0} failed: {1}", mx.Key, ex.Message); }
|
|
|
|
m_options.ClearLoadedModules();
|
|
}
|
|
|
|
if (m_localeChange != null)
|
|
{
|
|
m_localeChange.Dispose();
|
|
m_localeChange = null;
|
|
}
|
|
|
|
OnOperationCompleted?.Invoke(result, exception);
|
|
}
|
|
|
|
private (string[] paths, IFilter filter) SetupCommonOptions(ISetCommonOptions result, string[] paths, IFilter filter, ControllerMultiLogTarget logTarget)
|
|
{
|
|
m_options.MainAction = result.MainOperation;
|
|
|
|
switch (m_options.MainAction)
|
|
{
|
|
case OperationMode.Backup:
|
|
break;
|
|
|
|
default:
|
|
//It only makes sense to enable auto-creation if we are writing files.
|
|
if (!m_options.RawOptions.ContainsKey("disable-autocreate-folder"))
|
|
m_options.RawOptions["disable-autocreate-folder"] = "true";
|
|
break;
|
|
}
|
|
|
|
if (string.IsNullOrEmpty(m_options.Dbpath))
|
|
m_options.Dbpath = CLIDatabaseLocator.GetDatabasePathForCLI(m_backendUrl, m_options);
|
|
|
|
RestoreOptionsFromExistingDatabase(result.MainOperation);
|
|
|
|
//Load generic modules
|
|
m_options.ClearLoadedModules();
|
|
|
|
foreach (var m in DynamicLoader.GenericLoader.Modules)
|
|
{
|
|
var active = !m_options.DisableModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase) && (m.LoadAsDefault || m_options.EnableModules.Contains(m.Key, StringComparer.OrdinalIgnoreCase));
|
|
if (!active)
|
|
continue;
|
|
|
|
m_options.AddLoadedModule(DynamicLoader.GenericLoader.GetModule(m.Key));
|
|
}
|
|
|
|
// Store the URL connection options separately, as these should only be visible to modules implementing IConnectionModule
|
|
var conopts = new Dictionary<string, string>(m_options.RawOptions);
|
|
var qp = new Library.Utility.Uri(m_backendUrl).QueryParameters;
|
|
foreach (var k in qp.Keys)
|
|
conopts[(string)k] = qp[(string)k];
|
|
|
|
// Make the filter read-n-write able in the generic modules
|
|
using (PushFilterToOptions(m_options, filter, out var pristinefilter))
|
|
{
|
|
foreach (var mx in m_options.LoadedModules)
|
|
{
|
|
if (mx is IConnectionModule)
|
|
mx.Configure(conopts);
|
|
else
|
|
mx.Configure(m_options.RawOptions);
|
|
|
|
if (mx is IGenericSourceModule sourcemodule)
|
|
{
|
|
if (sourcemodule.ContainFilesForBackup(paths))
|
|
{
|
|
var sourceoptions = sourcemodule.ParseSourcePaths(ref paths, ref pristinefilter, m_options.RawOptions);
|
|
|
|
foreach (var sourceoption in sourceoptions)
|
|
m_options.RawOptions[sourceoption.Key] = sourceoption.Value;
|
|
}
|
|
}
|
|
|
|
if (mx is IGenericCallbackModule module)
|
|
module.OnStart(result.MainOperation.ToString(), ref m_backendUrl, ref paths);
|
|
}
|
|
|
|
// If the filters were changed by a module, read them back in
|
|
if (pristinefilter != m_options.RawOptions["filter"])
|
|
filter = FilterExpression.Deserialize(m_options.RawOptions["filter"].Split(new string[] { Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries));
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(m_options.Logfile))
|
|
{
|
|
var path = Path.GetDirectoryName(Path.GetFullPath(m_options.Logfile));
|
|
if (!Directory.Exists(path))
|
|
Directory.CreateDirectory(path);
|
|
|
|
logTarget.AddTarget(
|
|
new Library.Logging.StreamLogDestination(m_options.Logfile),
|
|
m_options.LogFileLoglevel,
|
|
m_options.LogFileLogFilter
|
|
);
|
|
}
|
|
|
|
if (m_options.HasTempDir)
|
|
{
|
|
Library.Utility.TempFolder.SystemTempPath = m_options.TempDir;
|
|
}
|
|
|
|
if (m_options.HasForcedLocale)
|
|
{
|
|
try
|
|
{
|
|
m_localeChange = new LocaleChange(m_options.ForcedLocale);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Library.Logging.Log.WriteWarningMessage(LOGTAG, "LocaleChangeError", ex, Strings.Controller.FailedForceLocaleError(ex.Message));
|
|
}
|
|
}
|
|
|
|
ValidateOptions();
|
|
|
|
return (paths, filter);
|
|
}
|
|
|
|
|
|
private void RestoreOptionsFromExistingDatabase(OperationMode operation)
|
|
{
|
|
if (m_options.NoLocalDb || string.IsNullOrWhiteSpace(m_options.Dbpath) || !File.Exists(m_options.Dbpath))
|
|
return;
|
|
|
|
using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, operation.ToString(), true, null, CancellationToken.None).Await();
|
|
Utility.UpdateOptionsFromDbAsync(db, m_options, CancellationToken.None).Await();
|
|
}
|
|
|
|
private async Task ApplySecretProviderAsync(CancellationToken cancellationToken)
|
|
{
|
|
var args = new[] { new Library.Utility.Uri(m_backendUrl) };
|
|
await SecretProviderHelper.ApplySecretProviderAsync([], args, m_options.RawOptions, TempFolder.SystemTempPath, SecretProvider, cancellationToken).ConfigureAwait(false);
|
|
// Write back the backend argument, if it was modified by the secret provider
|
|
m_backendUrl = args[0].ToString();
|
|
}
|
|
|
|
/// <summary>
|
|
/// This function will examine all options passed on the commandline, and test for unsupported or deprecated values.
|
|
/// Any errors will be logged into the statistics module.
|
|
/// </summary>
|
|
private void ValidateOptions()
|
|
{
|
|
// Check if only one of the retention options is set
|
|
var selectedRetentionOptions = new List<String>();
|
|
|
|
if (m_options.KeepTime.Ticks > 0)
|
|
{
|
|
selectedRetentionOptions.Add("keep-time");
|
|
}
|
|
|
|
if (m_options.KeepVersions > 0)
|
|
{
|
|
selectedRetentionOptions.Add("keep-versions");
|
|
}
|
|
|
|
if (m_options.RetentionPolicy.Any())
|
|
{
|
|
selectedRetentionOptions.Add("retention-policy");
|
|
}
|
|
|
|
if (selectedRetentionOptions.Count() > 1)
|
|
{
|
|
throw new UserInformationException(string.Format("Setting multiple retention options ({0}) is not permitted",
|
|
string.Join(", ", selectedRetentionOptions.Select(x => "--" + x))), "MultipleRetentionOptionsNotSupported");
|
|
}
|
|
|
|
// Check Prefix
|
|
if (!string.IsNullOrWhiteSpace(m_options.Prefix) && m_options.Prefix.Contains("-"))
|
|
throw new UserInformationException("The prefix cannot contain hyphens (-)", "PrefixCannotContainHyphens");
|
|
|
|
if (m_options.VolumeSize < m_options.Blocksize * 2)
|
|
throw new UserInformationException("The volume size must be at least twice the block size", "VolumeSizeTooSmall");
|
|
|
|
//Check validity of retention-policy option value
|
|
try
|
|
{
|
|
foreach (var configEntry in m_options.RetentionPolicy)
|
|
{
|
|
if (!configEntry.IsKeepAllVersions() && !configEntry.IsUnlimtedTimeframe() &&
|
|
configEntry.Interval >= configEntry.Timeframe)
|
|
{
|
|
throw new Interface.UserInformationException("An interval cannot be bigger than the timeframe it is in", "IntervalCannotBeBiggerThanTimeFrame");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception e) // simply reading the option value might also result in an exception due to incorrect formatting
|
|
{
|
|
throw new Interface.UserInformationException(string.Format("An error occoured while processing the value of --{0}", "retention-policy"), "RetentionPolicyParseError", e);
|
|
}
|
|
|
|
//Keep a list of all supplied options
|
|
var ropts = new Dictionary<string, string>(m_options.RawOptions);
|
|
|
|
//Keep a list of all supported options
|
|
var supportedOptions = new Dictionary<string, ICommandLineArgument>();
|
|
|
|
//There are a few internal options that are not accessible from outside, and thus not listed
|
|
foreach (var s in Options.InternalOptions)
|
|
supportedOptions[s] = null;
|
|
|
|
//Figure out what module options are supported in the current setup
|
|
var moduleOptions = new List<ICommandLineArgument>();
|
|
var disabledModuleOptions = new Dictionary<string, string>();
|
|
var loadedModuleKeys = m_options.LoadedModules.Select(k => k.Key).ToHashSet();
|
|
|
|
foreach (var m in DynamicLoader.GenericLoader.Modules)
|
|
{
|
|
if (m.SupportedCommands == null)
|
|
continue;
|
|
|
|
if (loadedModuleKeys.Contains(m.Key))
|
|
moduleOptions.AddRange(m.SupportedCommands);
|
|
else
|
|
foreach (var c in m.SupportedCommands)
|
|
{
|
|
disabledModuleOptions[c.Name] = m.DisplayName + " (" + m.Key + ")";
|
|
|
|
if (c.Aliases != null)
|
|
foreach (string s in c.Aliases)
|
|
disabledModuleOptions[s] = disabledModuleOptions[c.Name];
|
|
}
|
|
}
|
|
|
|
// Throw url-encoded options into the mix
|
|
//TODO: This can hide values if both commandline and url-parameters supply the same key
|
|
var ext = new Library.Utility.Uri(m_backendUrl).QueryParameters;
|
|
foreach (var k in ext.AllKeys)
|
|
ropts[k] = ext[k];
|
|
|
|
//Now run through all supported options, and look for deprecated options
|
|
foreach (var l in new IEnumerable<ICommandLineArgument>[] {
|
|
m_options.SupportedCommands,
|
|
DynamicLoader.BackendLoader.GetSupportedCommands(m_backendUrl),
|
|
m_options.NoEncryption ? null : DynamicLoader.EncryptionLoader.GetSupportedCommands(m_options.EncryptionModule),
|
|
moduleOptions,
|
|
DynamicLoader.CompressionLoader.GetSupportedCommands(m_options.CompressionModule) })
|
|
{
|
|
if (l != null)
|
|
foreach (ICommandLineArgument a in l)
|
|
{
|
|
if (supportedOptions.ContainsKey(a.Name) && !Options.KnownDuplicates.Contains(a.Name))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "DuplicateOption", null, Strings.Controller.DuplicateOptionNameWarning(a.Name));
|
|
|
|
supportedOptions[a.Name] = a;
|
|
|
|
if (a.Aliases != null)
|
|
foreach (string s in a.Aliases)
|
|
{
|
|
if (supportedOptions.ContainsKey(s) && !Options.KnownDuplicates.Contains(s))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "DuplicateOption", null, Strings.Controller.DuplicateOptionNameWarning(s));
|
|
|
|
supportedOptions[s] = a;
|
|
}
|
|
|
|
if (a.Deprecated)
|
|
{
|
|
List<string> aliases = new List<string>();
|
|
aliases.Add(a.Name);
|
|
if (a.Aliases != null)
|
|
aliases.AddRange(a.Aliases);
|
|
|
|
foreach (string s in aliases)
|
|
if (ropts.ContainsKey(s))
|
|
{
|
|
string optname = a.Name;
|
|
if (a.Name != s)
|
|
optname += " (" + s + ")";
|
|
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "DeprecatedOption", null, Strings.Controller.DeprecatedOptionUsedWarning(optname, a.DeprecationMessage), null);
|
|
}
|
|
|
|
}
|
|
|
|
if (a.Type == CommandLineArgument.ArgumentType.Boolean && a.DefaultValue == "true" && !a.Deprecated)
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "DefaultBooleanOption", null, Strings.Controller.WrongDefaultBooleanOptionWarning(a.Name), null);
|
|
}
|
|
}
|
|
|
|
//Now look for options that were supplied but not supported
|
|
foreach (var s in ropts.Keys)
|
|
if (!supportedOptions.ContainsKey(s))
|
|
if (disabledModuleOptions.ContainsKey(s))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "UnsupportedDisabledModule", null, Strings.Controller.UnsupportedOptionDisabledModuleWarning(s, disabledModuleOptions[s]), null);
|
|
else
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "UnsupportedOption", null, Strings.Controller.UnsupportedOptionWarning(s), null);
|
|
|
|
//Look at the value supplied for each argument and see if is valid according to its type
|
|
foreach (var s in ropts.Keys)
|
|
{
|
|
if (supportedOptions.TryGetValue(s, out var arg) && arg != null)
|
|
{
|
|
var validationMessage = CommandLineArgumentValidator.ValidateOptionValue(arg, s, ropts[s]);
|
|
if (!string.IsNullOrWhiteSpace(validationMessage))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "OptionValidationError", null, validationMessage);
|
|
}
|
|
}
|
|
|
|
var scheme = Library.Utility.Utility.GuessScheme(m_backendUrl);
|
|
if (DynamicLoader.BackendLoader.GetSupportedCommands(m_backendUrl) == null && !string.Equals(m_backendUrl, "dummy://", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
// If the backend is HTTP or HTTPS, give a custom error message
|
|
if (string.Equals(scheme, "http", StringComparison.OrdinalIgnoreCase) || string.Equals(scheme, "https", StringComparison.OrdinalIgnoreCase))
|
|
throw new UserInformationException(Strings.Controller.BackendNotSupportedHttpError, "BackendNotSupportedHttp");
|
|
|
|
throw new UserInformationException(Strings.Controller.BackendNotSupportedError(scheme, string.Join(", ", DynamicLoader.BackendLoader.Keys)), "BackendNotSupported");
|
|
}
|
|
|
|
//Inform the user about unmaintained backends
|
|
if (BackendModules.DeprecatedBackendModules.Contains(scheme))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "BackendDeprecated", null, $"The backend {scheme} is deprecated and should be migrated away from.");
|
|
|
|
//Warn about experimental diskimage filesystem parsing
|
|
if (m_options.RawOptions.ContainsKey("diskimage-filesystem-parsed"))
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "ExperimentalDiskImageFilesystemParsed", null, "The option 'diskimage-filesystem-parsed' is experimental and should be used with caution.");
|
|
|
|
//TODO: Based on the action, see if all options are relevant
|
|
}
|
|
|
|
/// <summary>
|
|
/// Helper method that expands the users chosen source input paths,
|
|
/// and removes duplicate paths
|
|
/// </summary>
|
|
/// <param name="inputsources">The input sources.</param>
|
|
/// <param name="filter">The filter.</param>
|
|
/// <param name="options">The options.</param>
|
|
/// <returns>The expanded and filtered sources.</returns>
|
|
private static string[] ExpandInputSources(string[] inputsources, IFilter filter, Options options)
|
|
{
|
|
if (inputsources == null || inputsources.Length == 0)
|
|
throw new UserInformationException(Strings.Controller.NoSourceFoldersError, "NoSourceFolders");
|
|
|
|
var sources = new List<string>(inputsources.Length);
|
|
|
|
DriveInfo[] drives = null;
|
|
|
|
//Make sure they all have the same format and exist
|
|
foreach (var inputsource in inputsources)
|
|
{
|
|
List<string> expandedSources = new List<string>();
|
|
|
|
if (OperatingSystem.IsWindows() && (inputsource.StartsWith("*:", StringComparison.Ordinal) || inputsource.StartsWith("?:", StringComparison.Ordinal)))
|
|
{
|
|
// *: drive paths are only supported on Windows clients
|
|
// Lazily load the drive info
|
|
drives = drives ?? DriveInfo.GetDrives();
|
|
|
|
// Replace the drive letter with each available drive
|
|
string sourcePath = inputsource.Substring(1);
|
|
foreach (DriveInfo drive in drives)
|
|
{
|
|
string expandedSource = drive.Name[0] + sourcePath;
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "AddingSourcePathFromWildcard", @"Adding source path ""{0}"" due to wildcard source path ""{1}""", expandedSource, inputsource);
|
|
expandedSources.Add(expandedSource);
|
|
}
|
|
}
|
|
else if (OperatingSystem.IsWindows() && inputsource.StartsWith(@"\\?\Volume{", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
// In order to specify a drive by it's volume name, adopt the volume guid path syntax:
|
|
// \\?\Volume{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}
|
|
// The volume guid can be found using the 'mountvol' commandline tool.
|
|
// However, instead of using this path with Windows APIs directory, it is adapted here to a standard path.
|
|
Guid volumeGuid;
|
|
if (Guid.TryParse(inputsource.Substring(@"\\?\Volume{".Length, @"XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX".Length), out volumeGuid))
|
|
{
|
|
string driveLetter = Library.Utility.Utility.GetDriveLetterFromVolumeGuid(volumeGuid);
|
|
if (!string.IsNullOrEmpty(driveLetter))
|
|
{
|
|
string expandedSource = driveLetter + inputsource.Substring(@"\\?\Volume{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}".Length);
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "AddingSourceFromGuid", @"Adding source path ""{0}"" in place of volume guid source path ""{1}""", expandedSource, inputsource);
|
|
expandedSources.Add(expandedSource);
|
|
}
|
|
else
|
|
{
|
|
// If we aren't allow to have missing sources, throw an exception indicating we couldn't find a drive where this volume is mounted
|
|
if (!options.AllowMissingSource)
|
|
throw new UserInformationException(Strings.Controller.SourceVolumeNameNotFoundError(inputsource, volumeGuid), "MissingSourceFolder");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// If we aren't allow to have missing sources, throw an exception indicating we couldn't find this volume
|
|
if (!options.AllowMissingSource)
|
|
throw new UserInformationException(Strings.Controller.SourceVolumeNameInvalidError(inputsource), "SourceVolumeNameInvalid");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
expandedSources.Add(inputsource);
|
|
}
|
|
|
|
var foundAnyPaths = false;
|
|
var unauthorized = false;
|
|
foreach (var expandedSource in expandedSources)
|
|
{
|
|
string source;
|
|
try
|
|
{
|
|
// Check if this is a mounted path
|
|
if (expandedSource.StartsWith("@"))
|
|
{
|
|
// TODO: If the remote source fails to load,
|
|
// this will be an enumeration warning, but will result
|
|
// in the backup being recorded without files from the source
|
|
// Eventually, this could lead to retention deletion,
|
|
// causing the last backup with the data from the source to be deleted
|
|
|
|
// TODO: We do not honor the PreventEmptySource option here,
|
|
// as we do not know if the source is empty or not
|
|
foundAnyPaths = true;
|
|
sources.Add(expandedSource);
|
|
continue;
|
|
}
|
|
|
|
// TODO: This expands "C:" to CWD, but not C:\
|
|
source = Path.GetFullPath(expandedSource);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Note that we use the original source (with the *) in the error
|
|
throw new UserInformationException(Strings.Controller.InvalidPathError(expandedSource, ex.Message), "InputSourceInvalid", ex);
|
|
}
|
|
|
|
var fi = new FileInfo(source);
|
|
var di = new DirectoryInfo(source);
|
|
if (fi.Exists || di.Exists)
|
|
{
|
|
foundAnyPaths = true;
|
|
|
|
if (!fi.Exists)
|
|
{
|
|
// If the directory exists, but is empty, and we are not allowed to have empty sources, throw an error
|
|
try
|
|
{
|
|
if (options.PreventEmptySource && di.Exists && !di.EnumerateFileSystemInfos().Any())
|
|
throw new UserInformationException(Strings.Controller.SourceFolderEmptyError(inputsource), "SourceFolderEmpty");
|
|
}
|
|
catch (UnauthorizedAccessException ex)
|
|
{
|
|
throw new UserInformationException(Strings.Controller.SourceUnauthorizedError(inputsource), "SourceUnauthorized", ex);
|
|
}
|
|
|
|
source = Util.AppendDirSeparator(source);
|
|
}
|
|
|
|
sources.Add(source);
|
|
}
|
|
else
|
|
{
|
|
try
|
|
{
|
|
// Try to get attributes. Returns -1 if source doesn't exist, otherwise throws an exception.
|
|
// In this case, it is irrelevant to use fileinfo or directoryinfo to retrieve attributes.
|
|
var unused = fi.Attributes;
|
|
}
|
|
catch (UnauthorizedAccessException ex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "AddingSourceFolder",
|
|
ex, @"Insufficient permissions to read ""{0}"", skipping", expandedSource);
|
|
unauthorized = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// If no paths were found, and we aren't allowed to have missing sources, throw an error
|
|
if (!foundAnyPaths && !options.AllowMissingSource)
|
|
{
|
|
if (unauthorized)
|
|
throw new UserInformationException(Strings.Controller.SourceUnauthorizedError(inputsource), "SourceUnauthorized");
|
|
|
|
throw new UserInformationException(Strings.Controller.SourceIsMissingError(inputsource), "SourceIsMissing");
|
|
}
|
|
}
|
|
|
|
//Sanity check for duplicate files/folders
|
|
ISet<string> pathDuplicates;
|
|
sources = Library.Utility.Utility.GetUniqueItems(sources, Library.Utility.Utility.ClientFilenameStringComparer, out pathDuplicates).ToList();
|
|
|
|
foreach (var pathDuplicate in pathDuplicates)
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "RemoveDuplicateSource", "Removing duplicate source: {0}", pathDuplicate);
|
|
|
|
//Sanity check for multiple inclusions of the same folder
|
|
for (int i = 0; i < sources.Count; i++)
|
|
for (int j = 0; j < sources.Count; j++)
|
|
if (i != j && sources[i].StartsWith(sources[j], Library.Utility.Utility.ClientFilenameStringComparison) && sources[i].EndsWith(Util.DirectorySeparatorString, Library.Utility.Utility.ClientFilenameStringComparison))
|
|
{
|
|
if (filter != null)
|
|
{
|
|
bool excludes;
|
|
|
|
FilterExpression.AnalyzeFilters(filter, out _, out excludes);
|
|
|
|
// If there are no excludes, there is no need to keep the folder as a filter
|
|
if (excludes)
|
|
{
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingSubfolderSource", "Removing source \"{0}\" because it is a subfolder of \"{1}\", and using it as an include filter", sources[i], sources[j]);
|
|
filter = JoinedFilterExpression.Join(new FilterExpression(sources[i]), filter);
|
|
}
|
|
else
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingSubfolderSource", "Removing source \"{0}\" because it is a subfolder or subfile of \"{1}\"", sources[i], sources[j]);
|
|
}
|
|
else
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingSubfolderSource", "Removing source \"{0}\" because it is a subfolder or subfile of \"{1}\"", sources[i], sources[j]);
|
|
|
|
sources.RemoveAt(i);
|
|
i--;
|
|
break;
|
|
}
|
|
|
|
return sources.ToArray();
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task PauseAsync(bool alsoTransfers)
|
|
{
|
|
var ct = m_currentTaskControl;
|
|
if (ct != null)
|
|
ct.Pause(alsoTransfers);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task ResumeAsync()
|
|
{
|
|
var ct = m_currentTaskControl;
|
|
if (ct != null)
|
|
ct.Resume();
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task StopAsync()
|
|
{
|
|
var ct = m_currentTaskControl;
|
|
if (ct == null)
|
|
return Task.CompletedTask;
|
|
|
|
Logging.Log.WriteVerboseMessage(LOGTAG, "CancellationRequested", "Cancellation Requested");
|
|
ct.Stop();
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task AbortAsync()
|
|
{
|
|
m_currentTaskControl?.Terminate();
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task SetThrottleSpeedsAsync(long maxUploadPrSecond, long maxDownloadPrSecond)
|
|
{
|
|
// Set these values in the options, in case the backend is not up yet
|
|
m_options.MaxUploadPrSecond = maxUploadPrSecond;
|
|
m_options.MaxDownloadPrSecond = maxDownloadPrSecond;
|
|
m_currentBackendManager?.UpdateThrottleValues(maxUploadPrSecond, maxDownloadPrSecond);
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>
|
|
/// The time of the last compact operation
|
|
/// </summary>
|
|
private DateTime m_lastCompact;
|
|
|
|
/// <summary>
|
|
/// The time of the last vacuum operation
|
|
/// </summary>
|
|
private DateTime m_lastVacuum;
|
|
|
|
/// <inheritdoc />
|
|
public Task SetLastCompactAsync(DateTime lastCompact)
|
|
{
|
|
m_lastCompact = lastCompact;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
/// <inheritdoc />
|
|
public Task SetLastVacuumAsync(DateTime lastVacuum)
|
|
{
|
|
m_lastVacuum = lastVacuum;
|
|
return Task.CompletedTask;
|
|
}
|
|
|
|
private void CheckAutoCompactInterval()
|
|
{
|
|
if (!m_options.NoAutoCompact && (m_lastCompact > DateTime.MinValue) && (m_lastCompact.Add(m_options.AutoCompactInterval) > DateTime.Now))
|
|
{
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "CompactResults", "Skipping auto compaction until {0}", m_lastCompact.Add(m_options.AutoCompactInterval));
|
|
m_options.RawOptions["no-auto-compact"] = "true";
|
|
}
|
|
}
|
|
|
|
private void CheckAutoVacuumInterval()
|
|
{
|
|
if (m_options.AutoVacuum && (m_lastVacuum > DateTime.MinValue) && (m_lastVacuum.Add(m_options.AutoVacuumInterval) > DateTime.Now))
|
|
{
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "VacuumResults", "Skipping auto vacuum until {0}", m_lastVacuum.Add(m_options.AutoVacuumInterval));
|
|
m_options.RawOptions["auto-vacuum"] = "false";
|
|
}
|
|
}
|
|
|
|
#region IDisposable Members
|
|
|
|
/// <inheritdoc />
|
|
public void Dispose()
|
|
{
|
|
}
|
|
|
|
#endregion
|
|
}
|
|
}
|