Kenneth Skovhede
586c08522e
Still missing the List and DNSHostnames. The AsyncRequest class is also preventing many backends from fully upgrading.
516 lines
22 KiB
C#
516 lines
22 KiB
C#
// Copyright (C) 2024, The Duplicati Team
|
|
// https://duplicati.com, hello@duplicati.com
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining a
|
|
// copy of this software and associated documentation files (the "Software"),
|
|
// to deal in the Software without restriction, including without limitation
|
|
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
// and/or sell copies of the Software, and to permit persons to whom the
|
|
// Software is furnished to do so, subject to the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be included in
|
|
// all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
|
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
// DEALINGS IN THE SOFTWARE.
|
|
|
|
using CoCoL;
|
|
using Duplicati.Library.Interface;
|
|
using Duplicati.Library.Main.Operation.Common;
|
|
using Duplicati.Library.Main.Volumes;
|
|
using Duplicati.Library.Utility;
|
|
using Newtonsoft.Json;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Runtime.ExceptionServices;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using static Duplicati.Library.Main.Operation.Common.BackendHandler;
|
|
|
|
namespace Duplicati.Library.Main.Operation.Backup
|
|
{
|
|
internal interface IUploadRequest
|
|
{
|
|
}
|
|
|
|
internal class FlushRequest : IUploadRequest
|
|
{
|
|
public Task<long> LastWriteSizeAsync { get { return m_tcs.Task; } }
|
|
private readonly TaskCompletionSource<long> m_tcs = new TaskCompletionSource<long>();
|
|
|
|
public void SetFlushed(long size)
|
|
{
|
|
m_tcs.TrySetResult(size);
|
|
}
|
|
|
|
public void TrySetCanceled()
|
|
{
|
|
m_tcs.TrySetCanceled();
|
|
}
|
|
}
|
|
|
|
internal class IndexVolumeUploadRequest : IUploadRequest
|
|
{
|
|
public IndexVolumeWriter IndexVolume { get; private set; }
|
|
|
|
public IndexVolumeUploadRequest(IndexVolumeWriter indexVolume)
|
|
{
|
|
IndexVolume = indexVolume;
|
|
}
|
|
}
|
|
|
|
internal class FilesetUploadRequest : IUploadRequest
|
|
{
|
|
public FilesetVolumeWriter Fileset { get; private set; }
|
|
|
|
public FilesetUploadRequest(FilesetVolumeWriter fileset)
|
|
{
|
|
Fileset = fileset;
|
|
}
|
|
}
|
|
|
|
internal class VolumeUploadRequest : IUploadRequest
|
|
{
|
|
public FileEntryItem BlockEntry { get; }
|
|
public BlockVolumeWriter BlockVolume { get; }
|
|
public TemporaryIndexVolume IndexVolume { get; }
|
|
public Options Options { get; }
|
|
public BackupDatabase Database { get; }
|
|
|
|
public VolumeUploadRequest(BlockVolumeWriter blockVolume, FileEntryItem blockEntry, TemporaryIndexVolume indexVolume, Options options, BackupDatabase database)
|
|
{
|
|
BlockVolume = blockVolume;
|
|
BlockEntry = blockEntry;
|
|
IndexVolume = indexVolume;
|
|
Options = options;
|
|
Database = database;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// This class encapsulates all requests to the backend
|
|
/// and ensures that the <code>AsynchronousUploadLimit</code> is honored
|
|
/// </summary>
|
|
internal class BackendUploader
|
|
{
|
|
private static readonly string LOGTAG = Logging.Log.LogTagFromType<BackendUploader>();
|
|
private readonly Func<IBackend> m_backendFactory;
|
|
private CancellationTokenSource m_cancelTokenSource;
|
|
private readonly DatabaseCommon m_database;
|
|
private long m_initialUploadThrottleSpeed;
|
|
private int m_lastThrottleCheckTime;
|
|
private long m_lastUploadThrottleSpeed;
|
|
private int m_maxConcurrentUploads;
|
|
private readonly Options m_options;
|
|
private readonly FileProgressThrottler m_progressUpdater;
|
|
private readonly StatsCollector m_stats;
|
|
private readonly ITaskReader m_taskReader;
|
|
|
|
public BackendUploader(Func<IBackend> backendFactory, Options options, DatabaseCommon database, ITaskReader taskReader, StatsCollector stats)
|
|
{
|
|
m_backendFactory = backendFactory;
|
|
m_options = options;
|
|
m_taskReader = taskReader;
|
|
m_stats = stats;
|
|
m_database = database;
|
|
m_progressUpdater = new FileProgressThrottler(stats, options.MaxUploadPrSecond);
|
|
}
|
|
|
|
public Task Run()
|
|
{
|
|
return AutomationExtensions.RunTask(new
|
|
{
|
|
Input = Channels.BackendRequest.ForRead,
|
|
},
|
|
|
|
async self =>
|
|
{
|
|
var workers = new List<Worker>();
|
|
m_maxConcurrentUploads = m_options.AsynchronousConcurrentUploadLimit <= 0 ? int.MaxValue : m_options.AsynchronousConcurrentUploadLimit;
|
|
m_initialUploadThrottleSpeed = m_options.AsynchronousConcurrentUploadLimit <= 0 ? int.MaxValue : m_options.MaxUploadPrSecond / m_maxConcurrentUploads;
|
|
var lastSize = -1L;
|
|
var uploadsInProgress = 0;
|
|
m_cancelTokenSource = new CancellationTokenSource();
|
|
m_progressUpdater.Run(m_cancelTokenSource.Token);
|
|
var filesetsToUpload = new List<FilesetUploadRequest>();
|
|
|
|
try
|
|
{
|
|
while (!await self.Input.IsRetiredAsync && await m_taskReader.ProgressAsync)
|
|
{
|
|
var req = await self.Input.ReadAsync();
|
|
|
|
if (!await m_taskReader.ProgressAsync)
|
|
break;
|
|
|
|
Worker worker = GetWorker(workers);
|
|
|
|
if (req is VolumeUploadRequest volumeUpload)
|
|
{
|
|
if (volumeUpload.IndexVolume == null)
|
|
worker.Task = Task.Run(() => UploadFileAsync(volumeUpload.BlockEntry, worker, m_cancelTokenSource.Token));
|
|
else
|
|
worker.Task = Task.Run(() => UploadBlockAndIndexAsync(volumeUpload, worker, m_cancelTokenSource.Token));
|
|
|
|
lastSize = volumeUpload.BlockVolume.SourceSize;
|
|
uploadsInProgress++;
|
|
}
|
|
else if (req is FilesetUploadRequest filesetUpload)
|
|
{
|
|
filesetsToUpload.Add(filesetUpload);
|
|
}
|
|
else if (req is IndexVolumeUploadRequest indexUpload)
|
|
{
|
|
worker.Task = Task.Run(() => UploadVolumeWriter(indexUpload.IndexVolume, worker, m_cancelTokenSource.Token));
|
|
uploadsInProgress++;
|
|
}
|
|
else if (req is FlushRequest flush)
|
|
{
|
|
await WaitForWorkers(workers, flush).ConfigureAwait(false);
|
|
UploadFilesets(workers, filesetsToUpload);
|
|
await WaitForWorkers(workers, flush).ConfigureAwait(false);
|
|
|
|
flush.SetFlushed(lastSize);
|
|
uploadsInProgress = 0;
|
|
break;
|
|
}
|
|
|
|
if (uploadsInProgress >= m_maxConcurrentUploads)
|
|
{
|
|
await Task.WhenAny(workers.Select(w => w.Task)).ConfigureAwait(false);
|
|
uploadsInProgress--;
|
|
|
|
var failedUploads = workers.Where(w => w.Task.IsFaulted).Select(w => GetInnerMostException(w.Task.Exception)).ToList();
|
|
if (failedUploads.Any())
|
|
{
|
|
if (failedUploads.Count == 1)
|
|
ExceptionDispatchInfo.Capture(failedUploads.First()).Throw();
|
|
else
|
|
throw new AggregateException(failedUploads);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex) when (!ex.IsRetiredException())
|
|
{
|
|
m_cancelTokenSource.Cancel();
|
|
try
|
|
{
|
|
await Task.WhenAll(workers.Select(w => w.Task));
|
|
}
|
|
catch
|
|
{
|
|
// As we are cancelling all threads we do not need to alert the user to any of these exceptions.
|
|
}
|
|
finally
|
|
{
|
|
workers.ForEach(w => w.Dispose());
|
|
}
|
|
throw;
|
|
}
|
|
|
|
try
|
|
{
|
|
m_stats.SetBlocking(true);
|
|
await Task.WhenAll(workers.Select(w => w.Task));
|
|
}
|
|
finally
|
|
{
|
|
m_stats.SetBlocking(false);
|
|
workers.ForEach(w => w.Dispose());
|
|
}
|
|
});
|
|
}
|
|
|
|
private Worker GetWorker(List<Worker> workers)
|
|
{
|
|
var worker = workers.FirstOrDefault(w => w.Task.IsCompleted && !w.Task.IsFaulted);
|
|
if (worker == null)
|
|
{
|
|
worker = new Worker(m_backendFactory());
|
|
workers.Add(worker);
|
|
}
|
|
|
|
return worker;
|
|
}
|
|
|
|
private async Task WaitForWorkers(List<Worker> workers, FlushRequest flush)
|
|
{
|
|
while (workers.Any())
|
|
{
|
|
Task finishedTask = await Task.WhenAny(workers.Select(w => w.Task)).ConfigureAwait(false);
|
|
if (finishedTask.IsFaulted)
|
|
{
|
|
flush.TrySetCanceled();
|
|
ExceptionDispatchInfo.Capture(finishedTask.Exception).Throw();
|
|
}
|
|
|
|
Worker finishedWorker = workers.Single(w => w.Task == finishedTask);
|
|
workers.Remove(finishedWorker);
|
|
finishedWorker.Dispose();
|
|
}
|
|
}
|
|
|
|
private void UploadFilesets(List<Worker> workers, List<FilesetUploadRequest> filesets)
|
|
{
|
|
foreach (var fileset in filesets)
|
|
{
|
|
Worker worker = GetWorker(workers);
|
|
worker.Task = Task.Run(() => UploadVolumeWriter(fileset.Fileset, worker, m_cancelTokenSource.Token));
|
|
}
|
|
|
|
filesets.Clear();
|
|
}
|
|
|
|
private static Exception GetInnerMostException(Exception ex)
|
|
{
|
|
while (ex.InnerException != null)
|
|
ex = ex.InnerException;
|
|
return ex;
|
|
}
|
|
|
|
private async Task UploadBlockAndIndexAsync(VolumeUploadRequest upload, Worker worker, CancellationToken cancelToken)
|
|
{
|
|
if (await UploadFileAsync(upload.BlockEntry, worker, cancelToken).ConfigureAwait(false))
|
|
{
|
|
// We must use the BlockEntry's RemoteFilename and not the BlockVolume's since the
|
|
// BlockEntry's' RemoteFilename reflects renamed files due to retries after errors.
|
|
IndexVolumeWriter indexVolumeWriter = await upload.IndexVolume.CreateVolume(upload.BlockEntry.RemoteFilename, upload.BlockEntry.Hash, upload.BlockEntry.Size, upload.Options, upload.Database);
|
|
FileEntryItem indexEntry = indexVolumeWriter.CreateFileEntryForUpload(upload.Options);
|
|
if (await UploadFileAsync(indexEntry, worker, cancelToken).ConfigureAwait(false))
|
|
{
|
|
await m_database.AddIndexBlockLinkAsync(indexVolumeWriter.VolumeID, upload.BlockVolume.VolumeID).ConfigureAwait(false);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task UploadVolumeWriter(VolumeWriterBase volumeWriter, Worker worker, CancellationToken cancelToken)
|
|
{
|
|
var fileEntry = new FileEntryItem(BackendActionType.Put, volumeWriter.RemoteFilename);
|
|
fileEntry.SetLocalfilename(volumeWriter.LocalFilename);
|
|
fileEntry.Encrypt(m_options);
|
|
fileEntry.UpdateHashAndSize(m_options);
|
|
|
|
await UploadFileAsync(fileEntry, worker, cancelToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task<bool> UploadFileAsync(FileEntryItem item, Worker worker, CancellationToken cancelToken)
|
|
{
|
|
if (cancelToken.IsCancellationRequested)
|
|
return false;
|
|
|
|
return await DoWithRetry(async () =>
|
|
{
|
|
if (item.IsRetry)
|
|
await RenameFileAfterErrorAsync(item).ConfigureAwait(false);
|
|
await DoPut(item, worker.Backend, cancelToken).ConfigureAwait(false);
|
|
},
|
|
item, worker, cancelToken).ConfigureAwait(false);
|
|
}
|
|
|
|
private async Task<bool> DoWithRetry(Func<Task> method, FileEntryItem item, Worker worker, CancellationToken cancelToken)
|
|
{
|
|
item.IsRetry = false;
|
|
var retryCount = 0;
|
|
|
|
for (retryCount = 0; retryCount <= m_options.NumberOfRetries; retryCount++)
|
|
{
|
|
if (m_options.RetryDelay.Ticks != 0 && retryCount != 0)
|
|
{
|
|
var delay = Library.Utility.Utility.GetRetryDelay(m_options.RetryDelay, retryCount, m_options.RetryWithExponentialBackoff);
|
|
await Task.Delay(delay).ConfigureAwait(false);
|
|
}
|
|
|
|
if (cancelToken.IsCancellationRequested)
|
|
return false;
|
|
|
|
try
|
|
{
|
|
if (worker.Backend == null)
|
|
worker.Backend = m_backendFactory();
|
|
await method().ConfigureAwait(false);
|
|
return true;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
item.IsRetry = true;
|
|
Logging.Log.WriteRetryMessage(LOGTAG, $"Retry{item.Operation}", ex, "Operation {0} with file {1} attempt {2} of {3} failed with message: {4}", item.Operation, item.RemoteFilename, retryCount + 1, m_options.NumberOfRetries + 1, ex.Message);
|
|
if (ex is ThreadAbortException || ex is OperationCanceledException)
|
|
break;
|
|
|
|
await m_stats.SendEventAsync(item.Operation, retryCount < m_options.NumberOfRetries ? BackendEventType.Retrying : BackendEventType.Failed, item.RemoteFilename, item.Size);
|
|
|
|
bool recovered = false;
|
|
if (m_options.AutocreateFolders && ex is FolderMissingException)
|
|
{
|
|
try
|
|
{
|
|
// If we successfully create the folder, we can re-use the connection
|
|
await worker.Backend.CreateFolderAsync(cancelToken).ConfigureAwait(false);
|
|
recovered = true;
|
|
}
|
|
catch (Exception dex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "FolderCreateError", dex, "Failed to create folder: {0}", ex.Message);
|
|
}
|
|
}
|
|
|
|
if (!recovered)
|
|
ResetBackend(ex, worker);
|
|
|
|
if (retryCount == m_options.NumberOfRetries)
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
if (m_options.NoConnectionReuse)
|
|
ResetBackend(null, worker);
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private void ResetBackend(Exception ex, Worker worker)
|
|
{
|
|
try
|
|
{
|
|
worker.Backend?.Dispose();
|
|
}
|
|
catch (Exception dex)
|
|
{
|
|
Logging.Log.WriteWarningMessage(LOGTAG, "BackendDisposeError", dex, "Failed to dispose backend instance: {0}", ex?.Message);
|
|
}
|
|
finally
|
|
{
|
|
worker.Backend = null;
|
|
}
|
|
}
|
|
|
|
private async Task RenameFileAfterErrorAsync(FileEntryItem item)
|
|
{
|
|
var p = VolumeBase.ParseFilename(item.RemoteFilename);
|
|
var guid = VolumeWriterBase.GenerateGuid();
|
|
var time = p.Time.Ticks == 0 ? p.Time : p.Time.AddSeconds(1);
|
|
var newname = VolumeBase.GenerateFilename(p.FileType, p.Prefix, guid, time, p.CompressionModule, p.EncryptionModule);
|
|
var oldname = item.RemoteFilename;
|
|
|
|
await m_stats.SendEventAsync(item.Operation, BackendEventType.Rename, oldname, item.Size);
|
|
await m_stats.SendEventAsync(item.Operation, BackendEventType.Rename, newname, item.Size);
|
|
Logging.Log.WriteInformationMessage(LOGTAG, "RenameRemoteTargetFile", "Renaming \"{0}\" to \"{1}\"", oldname, newname);
|
|
await m_database.RenameRemoteFileAsync(oldname, newname);
|
|
item.RemoteFilename = newname;
|
|
}
|
|
|
|
private async Task DoPut(FileEntryItem item, IBackend backend, CancellationToken cancelToken)
|
|
{
|
|
if (cancelToken.IsCancellationRequested)
|
|
return;
|
|
|
|
if (item.TrackedInDb)
|
|
await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploading, item.Size, item.Hash);
|
|
|
|
if (m_options.Dryrun)
|
|
{
|
|
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadVolume", "Would upload volume: {0}, size: {1}", item.RemoteFilename, Library.Utility.Utility.FormatSizeString(new FileInfo(item.LocalFilename).Length));
|
|
item.DeleteLocalFile();
|
|
return;
|
|
}
|
|
|
|
await m_database.LogRemoteOperationAsync("put", item.RemoteFilename, JsonConvert.SerializeObject(new { Size = item.Size, Hash = item.Hash }));
|
|
await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Started, item.RemoteFilename, item.Size, updateProgress: false);
|
|
m_progressUpdater.StartFileProgress(item.RemoteFilename, item.Size);
|
|
|
|
var begin = DateTime.Now;
|
|
|
|
if (!m_options.DisableStreamingTransfers && backend is IStreamingBackend streamingBackend)
|
|
{
|
|
// A download throttle speed is not given to the ThrottledStream as we are only uploading data here
|
|
using (var fs = File.OpenRead(item.LocalFilename))
|
|
using (var ts = new ThrottledStream(fs, m_initialUploadThrottleSpeed, 0))
|
|
using (var pgs = new ProgressReportingStream(ts, pg => HandleProgress(ts, pg, item.RemoteFilename)))
|
|
await streamingBackend.PutAsync(item.RemoteFilename, pgs, cancelToken).ConfigureAwait(false);
|
|
}
|
|
else
|
|
await backend.PutAsync(item.RemoteFilename, item.LocalFilename, cancelToken).ConfigureAwait(false);
|
|
|
|
var duration = DateTime.Now - begin;
|
|
m_progressUpdater.EndFileProgress(item.RemoteFilename);
|
|
Logging.Log.WriteProfilingMessage(LOGTAG, "UploadSpeed", "Uploaded {0} in {1}, {2}/s", Library.Utility.Utility.FormatSizeString(item.Size), duration, Library.Utility.Utility.FormatSizeString((long)(item.Size / duration.TotalSeconds)));
|
|
|
|
if (item.TrackedInDb)
|
|
await m_database.UpdateRemoteVolumeAsync(item.RemoteFilename, RemoteVolumeState.Uploaded, item.Size, item.Hash);
|
|
|
|
await m_stats.SendEventAsync(BackendActionType.Put, BackendEventType.Completed, item.RemoteFilename, item.Size);
|
|
|
|
if (m_options.ListVerifyUploads)
|
|
{
|
|
var f = backend.List().FirstOrDefault(n => n.Name.Equals(item.RemoteFilename, StringComparison.OrdinalIgnoreCase));
|
|
if (f == null)
|
|
throw new Exception(string.Format("List verify failed, file was not found after upload: {0}", item.RemoteFilename));
|
|
else if (f.Size != item.Size && f.Size >= 0)
|
|
throw new Exception(string.Format("List verify failed for file: {0}, size was {1} but expected to be {2}", f.Name, f.Size, item.Size));
|
|
}
|
|
|
|
item.DeleteLocalFile();
|
|
await m_database.CommitTransactionAsync("CommitAfterUpload");
|
|
}
|
|
|
|
private void HandleProgress(ThrottledStream stream, long progress, string path)
|
|
{
|
|
UpdateThrottleSpeed();
|
|
m_progressUpdater.UpdateFileProgress(path, progress, stream);
|
|
}
|
|
|
|
private void UpdateThrottleSpeed()
|
|
{
|
|
var updateThrottleSpeed = false;
|
|
|
|
lock (m_progressUpdater)
|
|
{
|
|
var currentTick = Environment.TickCount;
|
|
if (currentTick - m_lastThrottleCheckTime > 2000)
|
|
{
|
|
m_lastThrottleCheckTime = currentTick;
|
|
updateThrottleSpeed = true;
|
|
}
|
|
}
|
|
|
|
if (!updateThrottleSpeed)
|
|
return;
|
|
|
|
var maxUploadPerSecond = m_options.MaxUploadPrSecond;
|
|
if (maxUploadPerSecond != m_lastUploadThrottleSpeed)
|
|
{
|
|
m_lastUploadThrottleSpeed = maxUploadPerSecond;
|
|
m_initialUploadThrottleSpeed = maxUploadPerSecond / m_maxConcurrentUploads;
|
|
m_progressUpdater.UpdateThrottleSpeeds(maxUploadPerSecond);
|
|
}
|
|
}
|
|
|
|
private class Worker : IDisposable
|
|
{
|
|
public Task Task;
|
|
public IBackend Backend;
|
|
|
|
public Worker(IBackend backend)
|
|
{
|
|
Backend = backend;
|
|
Task = Task.FromResult(true);
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
this.Task?.Dispose();
|
|
this.Backend?.Dispose();
|
|
}
|
|
}
|
|
}
|
|
} |