Merge branch 'master' into performance/restoreopt
This commit is contained in:
Carl Johnsen
@@ -14,7 +14,7 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentFTP" Version="50.1.0" />
|
||||
<PackageReference Include="FluentFTP" Version="52.0.0" />
|
||||
<PackageReference Include="Microsoft.DotNet.Analyzers.Compatibility" Version="0.2.12-alpha">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using CoCoL;
|
||||
@@ -123,11 +123,11 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
m_progressUpdater = new FileProgressThrottler(stats, options.MaxUploadPrSecond);
|
||||
}
|
||||
|
||||
public Task Run()
|
||||
public Task Run(Channels channels)
|
||||
{
|
||||
return AutomationExtensions.RunTask(new
|
||||
{
|
||||
Input = Channels.BackendRequest.ForRead,
|
||||
Input = channels.BackendRequest.AsRead()
|
||||
},
|
||||
|
||||
async self =>
|
||||
|
||||
@@ -1,25 +1,24 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using CoCoL;
|
||||
|
||||
namespace Duplicati.Library.Main.Operation.Backup
|
||||
@@ -27,40 +26,40 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// <summary>
|
||||
/// Static typed definitions of channels used for the backup process
|
||||
/// </summary>
|
||||
internal static class Channels
|
||||
internal class Channels
|
||||
{
|
||||
/// <summary>
|
||||
/// Requests to the backend are send over this channel and picked up by the <see cref="BackendUploader" /> class
|
||||
/// </summary>
|
||||
public static readonly ChannelMarkerWrapper<IUploadRequest> BackendRequest = new ChannelMarkerWrapper<IUploadRequest>(new ChannelNameAttribute("BackendRequests"));
|
||||
public readonly IChannel<IUploadRequest> BackendRequest = ChannelManager.CreateChannel<IUploadRequest>();
|
||||
/// <summary>
|
||||
/// When the backup completes, all in-progress archives are sent from the <see cref="DataBlockProcessor"/> to the <see cref="SpillCollectorProcess"/>
|
||||
/// </summary>
|
||||
public static readonly ChannelMarkerWrapper<SpillVolumeRequest> SpillPickup = new ChannelMarkerWrapper<SpillVolumeRequest>(new ChannelNameAttribute("SpillPickup"));
|
||||
public readonly IChannel<SpillVolumeRequest> SpillPickup = ChannelManager.CreateChannel<SpillVolumeRequest>();
|
||||
/// <summary>
|
||||
/// All data blocks are sent during the scanning to the <see cref="DataBlockProcessor"/> who bundles them in compressed archives
|
||||
/// </summary>
|
||||
public static readonly ChannelMarkerWrapper<DataBlock> OutputBlocks = new ChannelMarkerWrapper<DataBlock>(new ChannelNameAttribute("OutputBlocks"));
|
||||
public readonly IChannel<DataBlock> OutputBlocks = ChannelManager.CreateChannel<DataBlock>();
|
||||
/// <summary>
|
||||
/// If a file has changes in the metadata, it is sent to the <see cref="FileBlockProcessor"/> where it is read
|
||||
/// </summary>
|
||||
public static readonly ChannelMarkerWrapper<MetadataPreProcess.FileEntry> AcceptedChangedFile = new ChannelMarkerWrapper<MetadataPreProcess.FileEntry>(new ChannelNameAttribute("AcceptedChangedFile"));
|
||||
public readonly IChannel<MetadataPreProcess.FileEntry> AcceptedChangedFile = ChannelManager.CreateChannel<MetadataPreProcess.FileEntry>();
|
||||
/// <summary>
|
||||
/// If a file has changes in the metadata, it is sent to the <see cref="FileBlockProcessor"/> where it is read
|
||||
/// </summary>
|
||||
public static readonly ChannelMarkerWrapper<StreamBlock> StreamBlock = new ChannelMarkerWrapper<StreamBlock>(new ChannelNameAttribute("StreamBlockSplitter"));
|
||||
public readonly IChannel<StreamBlock> StreamBlock = ChannelManager.CreateChannel<StreamBlock>();
|
||||
/// <summary>
|
||||
/// After metadata has been processed and collected, the <see cref="MetadataPreProcess"/> sends the file data to the <see cref="FilePreFilterProcess"/>
|
||||
/// </summary>
|
||||
public static readonly ChannelMarkerWrapper<MetadataPreProcess.FileEntry> ProcessedFiles = new ChannelMarkerWrapper<MetadataPreProcess.FileEntry>(new ChannelNameAttribute("ProcessedFiles"));
|
||||
public readonly IChannel<MetadataPreProcess.FileEntry> ProcessedFiles = ChannelManager.CreateChannel<MetadataPreProcess.FileEntry>();
|
||||
/// <summary>
|
||||
/// When enumerating the source folders, all discovered paths are sent by the <see cref="FileEnumerationProcess"/> to the <see cref="MetadataPreProcess"/>
|
||||
/// When enumerating the source folders, all discovered paths are sent by the <see cref="FileEnumerationProcess"/> to the <see cref="MetadataPreProcess"/>
|
||||
/// </summary>
|
||||
public static readonly ChannelMarkerWrapper<string> SourcePaths = new ChannelMarkerWrapper<string>(new ChannelNameAttribute("SourcePaths"));
|
||||
public readonly IChannel<string> SourcePaths = ChannelManager.CreateChannel<string>();
|
||||
/// <summary>
|
||||
/// All progress events are communicated over this channel, to ensure that parallel progress is reported as a if it was sequential
|
||||
/// </summary>
|
||||
public static readonly ChannelMarkerWrapper<Common.ProgressEvent> ProgressEvents = new ChannelMarkerWrapper<Common.ProgressEvent>(new ChannelNameAttribute("ProgressEvents"));
|
||||
public readonly IChannel<Common.ProgressEvent> ProgressEvents = ChannelManager.CreateChannel<Common.ProgressEvent>();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -32,15 +32,14 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
public static async Task Run(IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, BackupResults result, Options options, IFilter sourcefilter, IFilter filter, HashSet<string> blacklistPaths, Common.ITaskReader taskreader, System.Threading.CancellationToken token)
|
||||
{
|
||||
// Make sure we create the enumeration process in a separate scope,
|
||||
// but keep the log channel from the parent scope
|
||||
// Keep the log channel from the parent scope
|
||||
using (Logging.Log.StartIsolatingScope(true))
|
||||
using (new IsolatedChannelScope())
|
||||
{
|
||||
var enumeratorTask = Backup.FileEnumerationProcess.Run(sources, snapshot, journalService, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, blacklistPaths, options.ChangedFilelist, taskreader, token);
|
||||
Channels channels = new ();
|
||||
var enumeratorTask = Backup.FileEnumerationProcess.Run(channels, sources, snapshot, journalService, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, blacklistPaths, options.ChangedFilelist, taskreader, token);
|
||||
var counterTask = AutomationExtensions.RunTask(new
|
||||
{
|
||||
Input = Backup.Channels.SourcePaths.ForRead
|
||||
Input = channels.SourcePaths.AsRead()
|
||||
},
|
||||
|
||||
async self =>
|
||||
|
||||
@@ -35,14 +35,14 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// </summary>
|
||||
internal static class DataBlockProcessor
|
||||
{
|
||||
public static Task Run(BackupDatabase database, Options options, ITaskReader taskreader)
|
||||
public static Task Run(Channels channels, BackupDatabase database, Options options, ITaskReader taskreader)
|
||||
{
|
||||
return AutomationExtensions.RunTask(
|
||||
new
|
||||
{
|
||||
Input = Channels.OutputBlocks.ForRead,
|
||||
Output = Channels.BackendRequest.ForWrite,
|
||||
SpillPickup = Channels.SpillPickup.ForWrite,
|
||||
Input = channels.OutputBlocks.AsRead(),
|
||||
Output = channels.BackendRequest.AsWrite(),
|
||||
SpillPickup = channels.SpillPickup.AsWrite(),
|
||||
},
|
||||
|
||||
async self =>
|
||||
|
||||
@@ -1,23 +1,23 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using CoCoL;
|
||||
@@ -37,13 +37,13 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// </summary>
|
||||
private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FileBlockProcessor)) + ".FileEntry";
|
||||
|
||||
public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader, CancellationToken token)
|
||||
public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader, CancellationToken token)
|
||||
{
|
||||
return AutomationExtensions.RunTask(
|
||||
new
|
||||
new
|
||||
{
|
||||
Input = Channels.AcceptedChangedFile.ForRead,
|
||||
StreamBlockChannel = Channels.StreamBlock.ForWrite,
|
||||
Input = channels.AcceptedChangedFile.AsRead(),
|
||||
StreamBlockChannel = channels.StreamBlock.AsWrite(),
|
||||
},
|
||||
|
||||
async self =>
|
||||
@@ -82,7 +82,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2;
|
||||
});
|
||||
|
||||
using (var fs = snapshot.OpenRead(e.Path))
|
||||
using (var fs = snapshot.OpenRead(e.Path))
|
||||
filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Path, fs, false, hint);
|
||||
|
||||
await stats.AddOpenedFile(filestreamdata.Streamlength);
|
||||
@@ -97,7 +97,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "NewFile", "New file {0}", e.Path);
|
||||
else
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "ChangedFile", "File has changed {0}", e.Path);
|
||||
|
||||
|
||||
if (e.OldId < 0)
|
||||
{
|
||||
await stats.AddAddedFile(filesize);
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -44,12 +44,12 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// </summary>
|
||||
private static readonly string FILTER_LOGTAG = Logging.Log.LogTagFromType(typeof(FileEnumerationProcess));
|
||||
|
||||
public static Task Run(IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, HashSet<string> blacklistPaths, string[] changedfilelist, ITaskReader taskreader, CancellationToken token)
|
||||
public static Task Run(Channels channels, IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, HashSet<string> blacklistPaths, string[] changedfilelist, ITaskReader taskreader, CancellationToken token)
|
||||
{
|
||||
return AutomationExtensions.RunTask(
|
||||
new
|
||||
{
|
||||
Output = Backup.Channels.SourcePaths.ForWrite
|
||||
Output = channels.SourcePaths.AsWrite()
|
||||
},
|
||||
|
||||
async self =>
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -29,7 +29,7 @@ using Duplicati.Library.Main.Operation.Common;
|
||||
namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
/// <summary>
|
||||
/// This process takes files that are processed for metadata,
|
||||
/// This process takes files that are processed for metadata,
|
||||
/// and checks if anything indicates that the file has changed
|
||||
/// and submits potentially changed files for scanning
|
||||
/// </summary>
|
||||
@@ -40,14 +40,14 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// </summary>
|
||||
private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FilePreFilterProcess)) + ".FileEntry";
|
||||
|
||||
public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupStatsCollector stats, BackupDatabase database)
|
||||
public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupStatsCollector stats, BackupDatabase database)
|
||||
{
|
||||
return AutomationExtensions.RunTask(
|
||||
new
|
||||
{
|
||||
Input = Channels.ProcessedFiles.ForRead,
|
||||
ProgressChannel = Channels.ProgressEvents.ForWrite,
|
||||
Output = Channels.AcceptedChangedFile.ForWrite
|
||||
Input = channels.ProcessedFiles.AsRead(),
|
||||
ProgressChannel = channels.ProgressEvents.AsWrite(),
|
||||
Output = channels.AcceptedChangedFile.AsWrite()
|
||||
},
|
||||
|
||||
async self =>
|
||||
@@ -102,7 +102,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
// and we only check for timestamp changes
|
||||
if (CHECKFILETIMEONLY && !timestampChanged && !isNewFile)
|
||||
{
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path);
|
||||
try
|
||||
{
|
||||
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -64,16 +64,16 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
public IMetahash MetaHashAndSize;
|
||||
public bool MetadataChanged;
|
||||
}
|
||||
|
||||
public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, CancellationToken token)
|
||||
|
||||
public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, CancellationToken token)
|
||||
{
|
||||
return AutomationExtensions.RunTask(new
|
||||
{
|
||||
Input = Backup.Channels.SourcePaths.ForRead,
|
||||
StreamBlockChannel = Channels.StreamBlock.ForWrite,
|
||||
Output = Backup.Channels.ProcessedFiles.ForWrite,
|
||||
Input = channels.SourcePaths.AsRead(),
|
||||
StreamBlockChannel = channels.StreamBlock.AsWrite(),
|
||||
Output = channels.ProcessedFiles.AsWrite(),
|
||||
},
|
||||
|
||||
|
||||
async self =>
|
||||
{
|
||||
var emptymetadata = Utility.WrapMetadata(new Dictionary<string, string>(), options);
|
||||
@@ -88,22 +88,22 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
|
||||
var lastwrite = new DateTime(0, DateTimeKind.Utc);
|
||||
var attributes = default(FileAttributes);
|
||||
try
|
||||
{
|
||||
lastwrite = snapshot.GetLastWriteTimeUtc(path);
|
||||
try
|
||||
{
|
||||
lastwrite = snapshot.GetLastWriteTimeUtc(path);
|
||||
}
|
||||
catch (Exception ex)
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteWarningMessage(FILELOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
attributes = snapshot.GetAttributes(path);
|
||||
}
|
||||
catch (Exception ex)
|
||||
try
|
||||
{
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message);
|
||||
attributes = snapshot.GetAttributes(path);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message);
|
||||
}
|
||||
|
||||
// If we only have metadata, stop here
|
||||
@@ -258,13 +258,13 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
StreamProcessResult res;
|
||||
using (var ms = new MemoryStream(meta.Blob))
|
||||
res = await StreamBlock.ProcessStream(streamblockchannel, path, ms, true, CompressionHint.Default);
|
||||
res = await StreamBlock.ProcessStream(streamblockchannel, path, ms, true, CompressionHint.Default);
|
||||
|
||||
return await database.AddMetadatasetAsync(res.Streamhash, res.Streamlength, res.Blocksetid);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a file to the output,
|
||||
/// Adds a file to the output,
|
||||
/// </summary>
|
||||
/// <param name="filename">The name of the file to record</param>
|
||||
/// <param name="lastModified">The value of the lastModified timestamp</param>
|
||||
@@ -275,7 +275,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a file to the output,
|
||||
/// Adds a file to the output,
|
||||
/// </summary>
|
||||
/// <param name="filename">The name of the file to record</param>
|
||||
/// <param name="lastModified">The value of the lastModified timestamp</param>
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -33,13 +33,13 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// </summary>
|
||||
internal static class ProgressHandler
|
||||
{
|
||||
public static Task Run(BackupResults stat)
|
||||
public static Task Run(Channels channels, BackupResults stat)
|
||||
{
|
||||
return AutomationExtensions.RunTask(new
|
||||
{
|
||||
Input = Channels.ProgressEvents.ForRead
|
||||
Input = channels.ProgressEvents.AsRead()
|
||||
},
|
||||
|
||||
|
||||
async self =>
|
||||
{
|
||||
var filesStarted = new Dictionary<string, long>();
|
||||
@@ -64,7 +64,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
case EventType.FileClosed:
|
||||
if (fileProgress.ContainsKey(t.Filepath))
|
||||
fileProgress[t.Filepath] = t.Length;
|
||||
|
||||
|
||||
if (t.Filepath == current)
|
||||
{
|
||||
stat.OperationProgressUpdater.UpdateFileProgress(t.Length);
|
||||
@@ -94,7 +94,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
stat.OperationProgressUpdater.StartFile(current, filesStarted[current]);
|
||||
if (fileProgress.ContainsKey(current) && fileProgress[current] > 0)
|
||||
stat.OperationProgressUpdater.UpdateFileProgress(fileProgress[current]);
|
||||
stat.OperationProgressUpdater.UpdateFileProgress(fileProgress[current]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -33,11 +33,11 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(RecreateMissingIndexFiles));
|
||||
|
||||
public static Task Run(BackupDatabase database, Options options, ITaskReader taskreader)
|
||||
public static Task Run(Channels channels, BackupDatabase database, Options options, ITaskReader taskreader)
|
||||
{
|
||||
return AutomationExtensions.RunTask(new
|
||||
{
|
||||
UploadChannel = Channels.BackendRequest.ForWrite
|
||||
UploadChannel = channels.BackendRequest.AsWrite()
|
||||
},
|
||||
|
||||
async self =>
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using CoCoL;
|
||||
@@ -49,13 +49,13 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// </summary>
|
||||
internal static class SpillCollectorProcess
|
||||
{
|
||||
public static Task Run(Options options, BackupDatabase database, ITaskReader taskreader)
|
||||
public static Task Run(Channels channels, Options options, BackupDatabase database, ITaskReader taskreader)
|
||||
{
|
||||
return AutomationExtensions.RunTask(
|
||||
new
|
||||
{
|
||||
Input = Channels.SpillPickup.ForRead,
|
||||
Output = Channels.BackendRequest.ForWrite,
|
||||
Input = channels.SpillPickup.AsRead(),
|
||||
Output = channels.BackendRequest.AsWrite(),
|
||||
},
|
||||
|
||||
async self =>
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -40,14 +40,14 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(StreamBlockSplitter)) ;
|
||||
private static readonly string FILELOGTAG = LOGTAG + ".FileEntry";
|
||||
|
||||
public static Task Run(Options options, BackupDatabase database, ITaskReader taskreader)
|
||||
public static Task Run(Channels channels, Options options, BackupDatabase database, ITaskReader taskreader)
|
||||
{
|
||||
return AutomationExtensions.RunTask(
|
||||
new
|
||||
{
|
||||
Input = Channels.StreamBlock.ForRead,
|
||||
ProgressChannel = Channels.ProgressEvents.ForWrite,
|
||||
BlockOutput = Channels.OutputBlocks.ForWrite
|
||||
Input = channels.StreamBlock.AsRead(),
|
||||
ProgressChannel = channels.ProgressEvents.AsWrite(),
|
||||
BlockOutput = channels.OutputBlocks.AsWrite()
|
||||
},
|
||||
|
||||
async self =>
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -33,23 +33,23 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadRealFilelist));
|
||||
|
||||
public static Task Run(BackupResults result, BackupDatabase db, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader)
|
||||
public static Task Run(Channels channels, BackupResults result, BackupDatabase db, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader)
|
||||
{
|
||||
return AutomationExtensions.RunTask(new
|
||||
{
|
||||
Output = Channels.BackendRequest.ForWrite,
|
||||
Output = channels.BackendRequest.AsWrite()
|
||||
},
|
||||
|
||||
async self =>
|
||||
{
|
||||
if (!await taskreader.ProgressAsync)
|
||||
return;
|
||||
|
||||
|
||||
// Update the reported source and backend changes
|
||||
using(new Logging.Timer(LOGTAG, "UpdateChangeStatistics", "UpdateChangeStatistics"))
|
||||
await db.UpdateChangeStatisticsAsync(result);
|
||||
|
||||
var changeCount =
|
||||
var changeCount =
|
||||
result.AddedFiles + result.ModifiedFiles + result.DeletedFiles +
|
||||
result.AddedFolders + result.ModifiedFolders + result.DeletedFolders +
|
||||
result.AddedSymlinks + result.ModifiedSymlinks + result.DeletedSymlinks;
|
||||
@@ -71,7 +71,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
|
||||
if (!await taskreader.ProgressAsync)
|
||||
return;
|
||||
|
||||
|
||||
await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
|
||||
await db.CommitTransactionAsync("CommitUpdateRemoteVolume");
|
||||
await self.Output.WriteAsync(new FilesetUploadRequest(filesetvolume));
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using CoCoL;
|
||||
@@ -38,11 +38,11 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadSyntheticFilelist));
|
||||
|
||||
public static Task Run(BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader, string lastTempFilelist, long lastTempFilesetId)
|
||||
public static Task Run(Channels channels, BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader, string lastTempFilelist, long lastTempFilesetId)
|
||||
{
|
||||
return AutomationExtensions.RunTask(new
|
||||
{
|
||||
UploadChannel = Channels.BackendRequest.ForWrite
|
||||
UploadChannel = channels.BackendRequest.AsWrite()
|
||||
},
|
||||
|
||||
async self =>
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -32,7 +32,6 @@ using Duplicati.Library.Snapshots;
|
||||
using Duplicati.Library.Utility;
|
||||
using Duplicati.Library.Common.IO;
|
||||
using Duplicati.Library.Logging;
|
||||
using System.IO;
|
||||
|
||||
namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
@@ -179,11 +178,11 @@ namespace Duplicati.Library.Main.Operation
|
||||
/// The logic here is needed to check that the database is in a state
|
||||
/// where it can be used for the backup, and that the backend is also
|
||||
/// in the same state as the database.
|
||||
///
|
||||
///
|
||||
/// If the auto-repair option is enabled, this method will attempt to
|
||||
/// call the repair method, which requires that the database is closed
|
||||
/// and re-opened.
|
||||
///
|
||||
///
|
||||
/// For efficiency, the database is only closed if the repair is needed,
|
||||
/// and returned to the caller in an open state in either case.
|
||||
/// </summary>
|
||||
@@ -292,43 +291,39 @@ namespace Duplicati.Library.Main.Operation
|
||||
/// <summary>
|
||||
/// Performs the bulk of work by starting all relevant processes
|
||||
/// </summary>
|
||||
private static async Task RunMainOperation(IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, Common.ITaskReader taskreader, long filesetid, long lastfilesetid, CancellationToken token)
|
||||
private static async Task RunMainOperation(Backup.Channels channels, IEnumerable<string> sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, Common.ITaskReader taskreader, long filesetid, long lastfilesetid, CancellationToken token)
|
||||
{
|
||||
using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation"))
|
||||
{
|
||||
// Make sure the CompressionHints table is initialized, otherwise all workers will initialize it
|
||||
var unused = options.CompressionHints.Count;
|
||||
|
||||
Task all;
|
||||
using (new ChannelScope())
|
||||
{
|
||||
all = Task.WhenAll(
|
||||
new[]
|
||||
{
|
||||
Backup.DataBlockProcessor.Run(database, options, taskreader),
|
||||
Backup.FileBlockProcessor.Run(snapshot, options, database, stats, taskreader, token),
|
||||
Backup.StreamBlockSplitter.Run(options, database, taskreader),
|
||||
Backup.FileEnumerationProcess.Run(sources, snapshot, journalService,
|
||||
options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy,
|
||||
options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames,
|
||||
GetBlacklistedPaths(options), options.ChangedFilelist, taskreader, token),
|
||||
Backup.FilePreFilterProcess.Run(snapshot, options, stats, database),
|
||||
Backup.MetadataPreProcess.Run(snapshot, options, database, lastfilesetid, token),
|
||||
Backup.SpillCollectorProcess.Run(options, database, taskreader),
|
||||
Backup.ProgressHandler.Run(result)
|
||||
}
|
||||
// Spawn additional block hashers
|
||||
.Union(
|
||||
Enumerable.Range(0, options.ConcurrencyBlockHashers - 1).Select(x =>
|
||||
Backup.StreamBlockSplitter.Run(options, database, taskreader))
|
||||
)
|
||||
// Spawn additional compressors
|
||||
.Union(
|
||||
Enumerable.Range(0, options.ConcurrencyCompressors - 1).Select(x =>
|
||||
Backup.DataBlockProcessor.Run(database, options, taskreader))
|
||||
)
|
||||
);
|
||||
}
|
||||
Task all = Task.WhenAll(
|
||||
new[]
|
||||
{
|
||||
Backup.DataBlockProcessor.Run(channels, database, options, taskreader),
|
||||
Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader, token),
|
||||
Backup.StreamBlockSplitter.Run(channels, options, database, taskreader),
|
||||
Backup.FileEnumerationProcess.Run(channels, sources, snapshot, journalService,
|
||||
options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy,
|
||||
options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames,
|
||||
GetBlacklistedPaths(options), options.ChangedFilelist, taskreader, token),
|
||||
Backup.FilePreFilterProcess.Run(channels, snapshot, options, stats, database),
|
||||
Backup.MetadataPreProcess.Run(channels, snapshot, options, database, lastfilesetid, token),
|
||||
Backup.SpillCollectorProcess.Run(channels, options, database, taskreader),
|
||||
Backup.ProgressHandler.Run(channels, result)
|
||||
}
|
||||
// Spawn additional block hashers
|
||||
.Union(
|
||||
Enumerable.Range(0, options.ConcurrencyBlockHashers - 1).Select(x =>
|
||||
Backup.StreamBlockSplitter.Run(channels, options, database, taskreader))
|
||||
)
|
||||
// Spawn additional compressors
|
||||
.Union(
|
||||
Enumerable.Range(0, options.ConcurrencyCompressors - 1).Select(x =>
|
||||
Backup.DataBlockProcessor.Run(channels, database, options, taskreader))
|
||||
)
|
||||
);
|
||||
|
||||
await all.ConfigureAwait(false);
|
||||
|
||||
@@ -517,189 +512,186 @@ namespace Duplicati.Library.Main.Operation
|
||||
// Do a remote verification, unless disabled
|
||||
var (database, backendManager, lastTempFilelist, lastTempFilesetId) = await PreBackupVerify(m_backendurl, m_options, m_result);
|
||||
|
||||
// New isolated scope for each operation
|
||||
using (new IsolatedChannelScope())
|
||||
Backup.Channels channels = new ();
|
||||
|
||||
// If there is no filter, we set an empty filter to simplify the code
|
||||
// If there is a filter, we make sure that the sources are included
|
||||
m_filter = filter ?? new Library.Utility.FilterExpression();
|
||||
m_sourceFilter = new Library.Utility.FilterExpression(sources, true);
|
||||
|
||||
Task parallelScanner = null;
|
||||
Task uploaderTask = null;
|
||||
try
|
||||
{
|
||||
// If there is no filter, we set an empty filter to simplify the code
|
||||
// If there is a filter, we make sure that the sources are included
|
||||
m_filter = filter ?? new Library.Utility.FilterExpression();
|
||||
m_sourceFilter = new Library.Utility.FilterExpression(sources, true);
|
||||
|
||||
Task parallelScanner = null;
|
||||
Task uploaderTask = null;
|
||||
try
|
||||
using (m_database = database)
|
||||
using (backendManager)
|
||||
using (var db = new Backup.BackupDatabase(m_database, m_options))
|
||||
// Setup runners and instances here
|
||||
using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
|
||||
using (var stats = new Backup.BackupStatsCollector(m_result))
|
||||
using (var uploadtarget = ChannelManager.GetChannel(channels.BackendRequest.AsWrite()))
|
||||
{
|
||||
using (m_database = database)
|
||||
using (backendManager)
|
||||
using (var db = new Backup.BackupDatabase(m_database, m_options))
|
||||
// Setup runners and instances here
|
||||
using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
|
||||
using (var stats = new Backup.BackupStatsCollector(m_result))
|
||||
// Keep a reference to these channels to avoid shutdown
|
||||
using (var uploadtarget = ChannelManager.GetChannel(Backup.Channels.BackendRequest.ForWrite))
|
||||
long filesetid;
|
||||
var counterToken = new CancellationTokenSource();
|
||||
var uploader = new Backup.BackendUploader(() => DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions), m_options, db, m_result.TaskReader, stats);
|
||||
using (var snapshot = GetSnapshot(sources, m_options))
|
||||
{
|
||||
long filesetid;
|
||||
var counterToken = new CancellationTokenSource();
|
||||
var uploader = new Backup.BackendUploader(() => DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions), m_options, db, m_result.TaskReader, stats);
|
||||
using (var snapshot = GetSnapshot(sources, m_options))
|
||||
{
|
||||
try
|
||||
{
|
||||
// Start the uploader process
|
||||
uploaderTask = uploader.Run();
|
||||
|
||||
// If the previous backup was interrupted, send a synthetic list
|
||||
await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskReader, lastTempFilelist, lastTempFilesetId);
|
||||
|
||||
// Grab the previous backup ID, if any
|
||||
var prevfileset = m_database.FilesetTimes.FirstOrDefault();
|
||||
if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime())
|
||||
throw new Exception(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime()));
|
||||
|
||||
var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key;
|
||||
|
||||
// Rebuild any index files that are missing
|
||||
await Backup.RecreateMissingIndexFiles.Run(db, m_options, m_result.TaskReader);
|
||||
|
||||
// Prepare the operation by registering the filelist
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);
|
||||
|
||||
var repcnt = 0;
|
||||
while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
|
||||
filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++));
|
||||
|
||||
if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
|
||||
throw new Exception("Unable to generate a unique fileset name");
|
||||
|
||||
var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
|
||||
filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time);
|
||||
|
||||
// create USN-based scanner if enabled
|
||||
var journalService = GetJournalService(sources, snapshot, filter, lastfilesetid);
|
||||
|
||||
// Start parallel scan, or use the database
|
||||
if (m_options.DisableFileScanner)
|
||||
{
|
||||
var d = m_database.GetLastBackupFileCountAndSize();
|
||||
m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true);
|
||||
}
|
||||
else
|
||||
{
|
||||
parallelScanner = Backup.CountFilesHandler.Run(sources, snapshot, journalService, m_result, m_options, m_sourceFilter, m_filter, GetBlacklistedPaths(), m_result.TaskReader, counterToken.Token);
|
||||
}
|
||||
|
||||
// Run the backup operation
|
||||
if (await m_result.TaskReader.ProgressAsync)
|
||||
{
|
||||
await RunMainOperation(sources, snapshot, journalService, db, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskReader, filesetid, lastfilesetid, token).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
//If the scanner is still running for some reason, make sure we kill it now
|
||||
counterToken.Cancel();
|
||||
}
|
||||
}
|
||||
|
||||
// Add the fileset file to the dlist file
|
||||
filesetvolume.CreateFilesetFile(!token.IsCancellationRequested);
|
||||
|
||||
// Ensure the database is in a sane state after adding data
|
||||
using (new Logging.Timer(LOGTAG, "VerifyConsistency", "VerifyConsistency"))
|
||||
await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false);
|
||||
|
||||
// Send the actual filelist
|
||||
if (await m_result.TaskReader.ProgressAsync)
|
||||
await Backup.UploadRealFilelist.Run(m_result, db, m_options, filesetvolume, filesetid, m_result.TaskReader);
|
||||
|
||||
// Wait for upload completion
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
|
||||
var lastVolumeSize = await FlushBackend(m_result, uploadtarget, uploaderTask).ConfigureAwait(false);
|
||||
|
||||
// Make sure we have the database up-to-date
|
||||
await db.CommitTransactionAsync("CommitAfterUpload", false);
|
||||
|
||||
// TODO: Remove this later
|
||||
m_transaction = m_database.BeginTransaction();
|
||||
|
||||
try
|
||||
{
|
||||
// If this throws, we should roll back the transaction
|
||||
// Start the uploader process
|
||||
uploaderTask = uploader.Run(channels);
|
||||
|
||||
// If the previous backup was interrupted, send a synthetic list
|
||||
await Backup.UploadSyntheticFilelist.Run(channels, db, m_options, m_result, m_result.TaskReader, lastTempFilelist, lastTempFilesetId);
|
||||
|
||||
// Grab the previous backup ID, if any
|
||||
var prevfileset = m_database.FilesetTimes.FirstOrDefault();
|
||||
if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime())
|
||||
throw new Exception(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime()));
|
||||
|
||||
var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key;
|
||||
|
||||
// Rebuild any index files that are missing
|
||||
await Backup.RecreateMissingIndexFiles.Run(channels, db, m_options, m_result.TaskReader);
|
||||
|
||||
// Prepare the operation by registering the filelist
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);
|
||||
|
||||
var repcnt = 0;
|
||||
while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
|
||||
filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++));
|
||||
|
||||
if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
|
||||
throw new Exception("Unable to generate a unique fileset name");
|
||||
|
||||
var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
|
||||
filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time);
|
||||
|
||||
// create USN-based scanner if enabled
|
||||
var journalService = GetJournalService(sources, snapshot, filter, lastfilesetid);
|
||||
|
||||
// Start parallel scan, or use the database
|
||||
if (m_options.DisableFileScanner)
|
||||
{
|
||||
var d = m_database.GetLastBackupFileCountAndSize();
|
||||
m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true);
|
||||
}
|
||||
else
|
||||
{
|
||||
parallelScanner = Backup.CountFilesHandler.Run(sources, snapshot, journalService, m_result, m_options, m_sourceFilter, m_filter, GetBlacklistedPaths(), m_result.TaskReader, counterToken.Token);
|
||||
}
|
||||
|
||||
// Run the backup operation
|
||||
if (await m_result.TaskReader.ProgressAsync)
|
||||
CompactIfRequired(backendManager, lastVolumeSize);
|
||||
|
||||
if (m_options.UploadVerificationFile && await m_result.TaskReader.ProgressAsync)
|
||||
{
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
|
||||
FilelistProcessor.UploadVerificationFile(backendManager.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction);
|
||||
await RunMainOperation(channels, sources, snapshot, journalService, db, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskReader, filesetid, lastfilesetid, token).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
m_transaction.Rollback();
|
||||
m_transaction.Dispose();
|
||||
m_transaction = null;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "RollbackError", ex, "Rollback error: {0}", ex.Message);
|
||||
}
|
||||
|
||||
// Re-throw the original exception
|
||||
throw;
|
||||
//If the scanner is still running for some reason, make sure we kill it now
|
||||
counterToken.Cancel();
|
||||
}
|
||||
}
|
||||
|
||||
if (m_options.Dryrun)
|
||||
// Add the fileset file to the dlist file
|
||||
filesetvolume.CreateFilesetFile(!token.IsCancellationRequested);
|
||||
|
||||
// Ensure the database is in a sane state after adding data
|
||||
using (new Logging.Timer(LOGTAG, "VerifyConsistency", "VerifyConsistency"))
|
||||
await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false);
|
||||
|
||||
// Send the actual filelist
|
||||
if (await m_result.TaskReader.ProgressAsync)
|
||||
await Backup.UploadRealFilelist.Run(channels, m_result, db, m_options, filesetvolume, filesetid, m_result.TaskReader);
|
||||
|
||||
// Wait for upload completion
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
|
||||
var lastVolumeSize = await FlushBackend(m_result, uploadtarget, uploaderTask).ConfigureAwait(false);
|
||||
|
||||
// Make sure we have the database up-to-date
|
||||
await db.CommitTransactionAsync("CommitAfterUpload", false);
|
||||
|
||||
// TODO: Remove this later
|
||||
m_transaction = m_database.BeginTransaction();
|
||||
|
||||
try
|
||||
{
|
||||
// If this throws, we should roll back the transaction
|
||||
if (await m_result.TaskReader.ProgressAsync)
|
||||
CompactIfRequired(backendManager, lastVolumeSize);
|
||||
|
||||
if (m_options.UploadVerificationFile && await m_result.TaskReader.ProgressAsync)
|
||||
{
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
|
||||
FilelistProcessor.UploadVerificationFile(backendManager.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
try
|
||||
{
|
||||
m_transaction.Rollback();
|
||||
m_transaction.Dispose();
|
||||
m_transaction = null;
|
||||
}
|
||||
else
|
||||
catch (Exception ex)
|
||||
{
|
||||
using (new Logging.Timer(LOGTAG, "CommitFinalizingBackup", "CommitFinalizingBackup"))
|
||||
m_transaction.Commit();
|
||||
|
||||
m_transaction = null;
|
||||
|
||||
if (m_result.TaskControlRendevouz() != TaskControlState.Abort)
|
||||
{
|
||||
if (m_options.NoBackendverification)
|
||||
UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager);
|
||||
else
|
||||
PostBackupVerification(filesetvolume.RemoteFilename);
|
||||
}
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "RollbackError", ex, "Rollback error: {0}", ex.Message);
|
||||
}
|
||||
|
||||
m_database.WriteResults();
|
||||
m_database.PurgeLogData(m_options.LogRetention);
|
||||
m_database.PurgeDeletedVolumes(DateTime.UtcNow);
|
||||
|
||||
if (m_options.AutoVacuum)
|
||||
{
|
||||
m_result.VacuumResults = new VacuumResults(m_result);
|
||||
new VacuumHandler(m_options, (VacuumResults)m_result.VacuumResults).Run();
|
||||
}
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete);
|
||||
return;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var aex = BuildException(ex, uploaderTask, parallelScanner);
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "FatalError", ex, "Fatal error");
|
||||
if (aex == ex)
|
||||
// Re-throw the original exception
|
||||
throw;
|
||||
}
|
||||
|
||||
throw aex;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (parallelScanner != null && !parallelScanner.IsCompleted)
|
||||
parallelScanner.Wait(500);
|
||||
if (m_options.Dryrun)
|
||||
{
|
||||
m_transaction.Rollback();
|
||||
m_transaction = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
using (new Logging.Timer(LOGTAG, "CommitFinalizingBackup", "CommitFinalizingBackup"))
|
||||
m_transaction.Commit();
|
||||
|
||||
m_transaction = null;
|
||||
|
||||
if (m_result.TaskControlRendevouz() != TaskControlState.Abort)
|
||||
{
|
||||
if (m_options.NoBackendverification)
|
||||
UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager);
|
||||
else
|
||||
PostBackupVerification(filesetvolume.RemoteFilename);
|
||||
}
|
||||
}
|
||||
|
||||
m_database.WriteResults();
|
||||
m_database.PurgeLogData(m_options.LogRetention);
|
||||
m_database.PurgeDeletedVolumes(DateTime.UtcNow);
|
||||
|
||||
if (m_options.AutoVacuum)
|
||||
{
|
||||
m_result.VacuumResults = new VacuumResults(m_result);
|
||||
new VacuumHandler(m_options, (VacuumResults)m_result.VacuumResults).Run();
|
||||
}
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete);
|
||||
return;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
var aex = BuildException(ex, uploaderTask, parallelScanner);
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "FatalError", ex, "Fatal error");
|
||||
if (aex == ex)
|
||||
throw;
|
||||
|
||||
throw aex;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (parallelScanner != null && !parallelScanner.IsCompleted)
|
||||
parallelScanner.Wait(500);
|
||||
}
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -48,15 +48,17 @@ namespace Duplicati.Library.Main.Operation
|
||||
var sourcefilter = new Library.Utility.FilterExpression(sources, true);
|
||||
|
||||
using (var snapshot = BackupHandler.GetSnapshot(sources, m_options))
|
||||
using (new IsolatedChannelScope())
|
||||
{
|
||||
var source = Operation.Backup.FileEnumerationProcess.Run(sources, snapshot, null,
|
||||
Backup.Channels channels = new ();
|
||||
var source = Backup.FileEnumerationProcess.Run(channels, sources, snapshot, null,
|
||||
m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy,
|
||||
m_options.HardlinkPolicy, m_options.ExcludeEmptyFolders, m_options.IgnoreFilenames,
|
||||
BackupHandler.GetBlacklistedPaths(m_options), null, m_result.TaskReader, token);
|
||||
|
||||
var sink = CoCoL.AutomationExtensions.RunTask(
|
||||
new { source = Operation.Backup.Channels.SourcePaths.ForRead },
|
||||
var sink = CoCoL.AutomationExtensions.RunTask(new
|
||||
{
|
||||
source = channels.SourcePaths.AsRead()
|
||||
},
|
||||
async self =>
|
||||
{
|
||||
while (true)
|
||||
|
||||
@@ -257,7 +257,7 @@
|
||||
</ul>
|
||||
</div>
|
||||
<div class="themelink">
|
||||
<a href="https://duplicati.readthedocs.io/en/latest/" target="_blank" rel="external">{{'Manual' | translate}}<div class="external-link-image"></div></a>
|
||||
<a href="https://docs.duplicati.com" target="_blank" rel="external">{{'Manual' | translate}}<div class="external-link-image"></div></a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -55,6 +55,16 @@ TESTCREDENTIAL_AZURE_ACCESSKEY
|
||||
TESTCREDENTIAL_AZURE_CONTAINERNAME
|
||||
```
|
||||
|
||||
## pCloud Native API
|
||||
|
||||
```
|
||||
TESTCREDENTIAL_PCLOUD_SERVER
|
||||
TESTCREDENTIAL_PCLOUD_TOKEN
|
||||
TESTCREDENTIAL_PCLOUD_FOLDER
|
||||
```
|
||||
For PCloud the server is the API server(eapi.pcloud.com for EU hosted account or api.pcloud.com for non EU). The token is the OAuth token.
|
||||
|
||||
|
||||
## Running the tests
|
||||
|
||||
[TestContainers](https://testcontainers.org/) is a pre-requisite for SSH, FTP and Webdav tests. It is not required for the other tests.
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
// Copyright (C) 2024, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
namespace Duplicati.Backend.Tests.pCloud;
|
||||
|
||||
/// <summary>
|
||||
/// pCloud Tests for CI.
|
||||
///
|
||||
/// These tests cannot be parallelized as they will share credentials and folders
|
||||
/// </summary>
|
||||
[TestClass]
|
||||
[DoNotParallelize]
|
||||
public sealed class pCloudTests : BaseTest
|
||||
{
|
||||
/// <summary>
|
||||
/// Perform tests using pCloud API
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
[TestMethod]
|
||||
public Task TestpCoudAPI()
|
||||
{
|
||||
CheckRequiredEnvironment([
|
||||
"TESTCREDENTIAL_PCLOUD_SERVER",
|
||||
"TESTCREDENTIAL_PCLOUD_TOKEN",
|
||||
"TESTCREDENTIAL_PCLOUD_FOLDER"
|
||||
]);
|
||||
|
||||
var testURI = $"pcloud://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_SERVER")}/{Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_FOLDER")}?&authid={Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_TOKEN")}";
|
||||
|
||||
// Specifically disable streaming to pass tests.
|
||||
var exitCode = CommandLine.BackendTester.Program.Main(new[] {testURI, "--disable-streaming-transfers"}.Concat(Parameters.GlobalTestParameters).ToArray());
|
||||
|
||||
if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details");
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Perform tests using pCloud API
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
[TestMethod]
|
||||
public Task TestpCoudAPIWithStreaming()
|
||||
{
|
||||
CheckRequiredEnvironment([
|
||||
"TESTCREDENTIAL_PCLOUD_SERVER",
|
||||
"TESTCREDENTIAL_PCLOUD_TOKEN",
|
||||
"TESTCREDENTIAL_PCLOUD_FOLDER"
|
||||
]);
|
||||
|
||||
var testURI = $"pcloud://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_SERVER")}/{Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_FOLDER")}?&authid={Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_TOKEN")}";
|
||||
|
||||
var exitCode = CommandLine.BackendTester.Program.Main(new[] {testURI}.Concat(Parameters.GlobalTestParameters).ToArray());
|
||||
|
||||
if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details");
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -22,7 +22,7 @@ Gitterは削除済
|
||||
|
||||
Duplicatiは、フリー（自由）でオープンソースのバックアップ用クライアントです。圧縮し、暗号化した増分バックアップを、クラウドストレージサービスや遠隔のファイルサーバー上に安全に保存できます。Duplicatiは、主に以下のサービスやソフトウェアで使うことができます。
|
||||
|
||||
*Amazon S3、[IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage")、Box、Dropbox、FTP、Googleクラウド、Googleドライブ、MEGA、Microsoft Azure、Microsoft OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、Tencentクラウドオブジェクトストレージ（COS）、Aliyun OSS、[その他にも対応しています！](https://duplicati.readthedocs.io/en/latest/01-introduction/#supported-backends)*
|
||||
*Amazon S3、[IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage")、Box、Dropbox、FTP、Googleクラウド、Googleドライブ、MEGA、Microsoft Azure、Microsoft OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、Tencentクラウドオブジェクトストレージ（COS）、Aliyun OSS、[その他にも対応しています！](https://docs.duplicati.com/backup-destinations/destination-overview)*
|
||||
|
||||
DuplicatiはMITライセンスで公開されており、Windows、OSX、Linuxで利用できます（.NET 4.7.1以上、またはMono 5.10.0以上が必要です）。
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ Removed Gitter
|
||||
|
||||
Duplicati is a free, open source, backup client that securely stores encrypted, incremental, compressed backups on cloud storage services and remote file servers. It works with:
|
||||
|
||||
*Amazon S3, [IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2"), [Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage"), Box, Dropbox, FTP, Google Cloud and Drive, MEGA, Microsoft Azure and OneDrive, Rackspace Cloud Files, OpenStack Storage (Swift), Sia, Storj DCS, SSH (SFTP), WebDAV, Tencent Cloud Object Storage (COS), Aliyun OSS, [and more!](https://duplicati.readthedocs.io/en/latest/01-introduction/#supported-backends)*
|
||||
*Amazon S3, [IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2"), [Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage"), Box, Dropbox, FTP, Google Cloud and Drive, MEGA, Microsoft Azure and OneDrive, Rackspace Cloud Files, OpenStack Storage (Swift), Sia, Storj DCS, SSH (SFTP), WebDAV, Tencent Cloud Object Storage (COS), Aliyun OSS, [and more!](https://docs.duplicati.com/backup-destinations/destination-overview)*
|
||||
|
||||
Duplicati is licensed under the MIT license and available for Windows, OSX and Linux (.NET 4.7.1+ or Mono 5.10.0+ required).
|
||||
|
||||
|
||||
+1
-1
@@ -26,7 +26,7 @@
|
||||
|
||||
Duplicati 是一个免费、开源的备份客户端，可以安全地将加密、增量、压缩的备份存储在云存储服务和远程文件服务器上。它与以下服务兼容：
|
||||
|
||||
*亚马逊 S3、[IDrive e2](https://www.idrive.com/e2/duplicati "使用 Duplicati 与 IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati 与 Backblaze B2 云存储")、Box、Dropbox、FTP、Google Cloud 和 Drive、MEGA、Microsoft Azure 和 OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、阿里云对象存储(OSS)、百度网盘、阿里云盘、腾讯云对象存储 (COS)、[以及更多！](https://duplicati.readthedocs.io/en/latest/01-introduction/#supported-backends)*
|
||||
*亚马逊 S3、[IDrive e2](https://www.idrive.com/e2/duplicati "使用 Duplicati 与 IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati 与 Backblaze B2 云存储")、Box、Dropbox、FTP、Google Cloud 和 Drive、MEGA、Microsoft Azure 和 OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、阿里云对象存储(OSS)、百度网盘、阿里云盘、腾讯云对象存储 (COS)、[以及更多！](https://docs.duplicati.com/backup-destinations/destination-overview)*
|
||||
|
||||
Duplicati 根据 LGPL 许可证授权，并可用于 Windows、OSX 和 Linux (.NET 4.7.1+ 或 Mono 5.10.0+ 需要)。
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ public static partial class Command
|
||||
}, workingDirectory: baseDir);
|
||||
|
||||
// Then push the release
|
||||
await ProcessHelper.Execute(new[] { "git", "push" }, workingDirectory: baseDir);
|
||||
await ProcessHelper.Execute(new[] { "git", "push", "--tags" }, workingDirectory: baseDir);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -220,7 +220,7 @@ public static partial class Command
|
||||
);
|
||||
|
||||
var changelogFileOption = new Option<FileInfo>(
|
||||
name: "--changelog-file",
|
||||
name: "--changelog-news-file",
|
||||
description: "The path to the changelog news file. Contents from this file are prepended to the changelog.",
|
||||
getDefaultValue: () => new FileInfo(Path.GetFullPath("changelog-news.txt"))
|
||||
);
|
||||
@@ -319,7 +319,7 @@ public static partial class Command
|
||||
/// <param name="DisableDockerPush">If the docker push should be disabled</param>
|
||||
/// <param name="MacOSAppName">The name of the MacOS app bundle</param>
|
||||
/// <param name="DockerRepo">The docker repository to push to</param>
|
||||
/// <param name="ChangelogFile">The path to the changelog file</param>
|
||||
/// <param name="ChangelogNewsFile">The path to the changelog news file</param>
|
||||
/// <param name="DisableNotarizeSigning">If notarize signing should be disabled</param>
|
||||
/// <param name="DisableGpgSigning">If GPG signing should be disabled</param>
|
||||
/// <param name="DisableS3Upload">If S3 upload should be disabled</param>
|
||||
@@ -342,7 +342,7 @@ public static partial class Command
|
||||
bool DisableDockerPush,
|
||||
string MacOSAppName,
|
||||
string DockerRepo,
|
||||
FileInfo ChangelogFile,
|
||||
FileInfo ChangelogNewsFile,
|
||||
bool DisableNotarizeSigning,
|
||||
bool DisableGpgSigning,
|
||||
bool DisableS3Upload,
|
||||
@@ -408,28 +408,28 @@ public static partial class Command
|
||||
if (!File.Exists(primaryCLI))
|
||||
throw new Exception($"Failed to locate project file: {primaryCLI}");
|
||||
|
||||
if (!input.ChangelogFile.Exists)
|
||||
if (!input.ChangelogNewsFile.Exists)
|
||||
{
|
||||
Console.WriteLine($"Changelog news file not found: {input.ChangelogFile.FullName}");
|
||||
Console.WriteLine($"Changelog news file not found: {input.ChangelogNewsFile.FullName}");
|
||||
Console.WriteLine($"Create an empty file if you want a release without changes");
|
||||
if (OperatingSystem.IsWindows())
|
||||
Console.WriteLine($"> type nul > {input.ChangelogFile.FullName}");
|
||||
Console.WriteLine($"> type nul > {input.ChangelogNewsFile.FullName}");
|
||||
else
|
||||
Console.WriteLine($"> touch {input.ChangelogFile.FullName}");
|
||||
Console.WriteLine($"> touch {input.ChangelogNewsFile.FullName}");
|
||||
|
||||
Program.ReturnCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (input.ChangelogFile.LastAccessTimeUtc < DateTime.UtcNow.AddDays(-1))
|
||||
if (input.ChangelogNewsFile.LastAccessTimeUtc < DateTime.UtcNow.AddDays(-1))
|
||||
{
|
||||
Console.WriteLine($"Changelog news file was last modified {input.ChangelogFile.LastAccessTimeUtc:yyyy-MM-dd}, indicating a stale file");
|
||||
Console.WriteLine($"Changelog news file was last modified {input.ChangelogNewsFile.LastAccessTimeUtc:yyyy-MM-dd}, indicating a stale file");
|
||||
Console.WriteLine($"Please update the file with the changelog news for the release");
|
||||
Program.ReturnCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
var changelogNews = File.ReadAllText(input.ChangelogFile.FullName);
|
||||
var changelogNews = File.ReadAllText(input.ChangelogNewsFile.FullName);
|
||||
|
||||
var releaseInfo = string.IsNullOrWhiteSpace(input.Version)
|
||||
? ReleaseInfo.Create(input.Channel, Version.Parse(File.ReadAllText(versionFilePath)), 1)
|
||||
@@ -472,6 +472,12 @@ public static partial class Command
|
||||
throw new Exception($"The following packages cannot be built without Docker: {string.Join(", ", unsupportedBuilds.Select(x => x.PackageTargetString))}");
|
||||
}
|
||||
|
||||
if (rtcfg.UseGithubUpload && !input.GitStashPush)
|
||||
{
|
||||
Console.WriteLine("Github upload requested, but pushing the tag is disabled");
|
||||
Program.ReturnCode = 1;
|
||||
}
|
||||
|
||||
if (!input.KeepBuilds && Directory.Exists(input.BuildPath.FullName))
|
||||
{
|
||||
Console.WriteLine($"Deleting build folder: {input.BuildPath.FullName}");
|
||||
@@ -576,6 +582,11 @@ public static partial class Command
|
||||
files.Add(sigfile);
|
||||
}
|
||||
|
||||
// Ensure the tag is pushed before uploading, so the uploaded files
|
||||
// are associated with the release tag
|
||||
if (input.GitStashPush)
|
||||
await GitPush.TagAndPush(baseDir, releaseInfo);
|
||||
|
||||
if (rtcfg.UseS3Upload || rtcfg.UseGithubUpload)
|
||||
{
|
||||
Console.WriteLine("Build completed, uploading packages ...");
|
||||
@@ -597,10 +608,18 @@ public static partial class Command
|
||||
}
|
||||
|
||||
if (rtcfg.UseGithubUpload)
|
||||
{
|
||||
if (!rtcfg.UseS3Upload && input.GitStashPush)
|
||||
{
|
||||
Console.WriteLine("Waiting for Github to create the release tag ...");
|
||||
await Task.Delay(TimeSpan.FromSeconds(30));
|
||||
}
|
||||
|
||||
await Upload.UploadToGithub(
|
||||
files.Select(x => new Upload.UploadFile(x, Path.GetFileName(x))),
|
||||
rtcfg
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (rtcfg.UseUpdateServerReload)
|
||||
@@ -617,10 +636,8 @@ public static partial class Command
|
||||
|
||||
if (input.GitStashPush)
|
||||
{
|
||||
await GitPush.TagAndPush(baseDir, releaseInfo);
|
||||
|
||||
// Contents are added to changelog, so we can remove the file
|
||||
input.ChangelogFile.Delete();
|
||||
input.ChangelogNewsFile.Delete();
|
||||
}
|
||||
|
||||
// Clean up the source tree
|
||||
@@ -628,10 +645,18 @@ public static partial class Command
|
||||
if (Directory.Exists(folder))
|
||||
Directory.Delete(folder, true);
|
||||
|
||||
await ProcessHelper.Execute(new[] {
|
||||
// This often fails and should be fixed
|
||||
try
|
||||
{
|
||||
await ProcessHelper.Execute(new[] {
|
||||
"git", "checkout",
|
||||
}.Concat(revertableFiles.Select(x => Path.GetRelativePath(baseDir, x)))
|
||||
, workingDirectory: baseDir);
|
||||
, workingDirectory: baseDir);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Warning: Failed to clean source directory: {ex.Message}");
|
||||
}
|
||||
|
||||
Console.WriteLine("All done!");
|
||||
}
|
||||
|
||||
@@ -1 +1 @@
|
||||
2.1.0.100
|
||||
2.1.0.101
|
||||
@@ -1,3 +1,31 @@
|
||||
2024-12-06 - 2.1.0.101_canary_2024-12-06
|
||||
==========
|
||||
This release is a canary release intended to be used for testing.
|
||||
|
||||
**Note that this build is using .NET8, unlike the 2.0.8.1 build**
|
||||
|
||||
## Breaking change from previous canary
|
||||
The pCloud backend is updated to support subfolders.
|
||||
Paths with `/` will now be treated as a folder structure on pCloud.
|
||||
Previous version would treat `folder/subfolder` as a single foldername,
|
||||
this version treats it as `folder` / `subfolder`.
|
||||
|
||||
## Detailed list of changes:
|
||||
- Fixed a case where the TrayIcon would hang in detached mode
|
||||
- Improved OpenAPI description for filter groups
|
||||
- Fixed an issue with websocket requests through a proxy
|
||||
- Updated icons for Linux and dark-mode MacOS
|
||||
- Added logging of unobserved exceptions
|
||||
- Fixed a compatibility issue with FTP not correctly detecting explicit TLS
|
||||
- Fixed an issue where the quota was reported incorrectly
|
||||
- Removed the unused `--use-block-cache` option
|
||||
- Agent now has a dedicated `register` command
|
||||
- Sizes can now be supplied as kikibytes, eg. `20kib`
|
||||
- Fixed supporting subfolders in pCloud
|
||||
- Added option to send http requests with relaxed TLS security
|
||||
- Agent can now be asked to retain the previous UI password
|
||||
- Updated SSH.NET to v2024.2.0
|
||||
|
||||
2024-11-25 - 2.1.0.100_canary_2024-11-25
|
||||
==========
|
||||
This release is a canary release intended to be used for testing.
|
||||
|
||||
Reference in New Issue
Block a user