diff --git a/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj b/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj index 28a2a7698..5b88e6b19 100644 --- a/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj +++ b/Duplicati/Library/Backend/FTP/Duplicati.Library.Backend.FTP.csproj @@ -14,7 +14,7 @@ - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs b/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs index 2fc6230be..0508a04d3 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CoCoL; @@ -123,11 +123,11 @@ namespace Duplicati.Library.Main.Operation.Backup m_progressUpdater = new FileProgressThrottler(stats, options.MaxUploadPrSecond); } - public Task Run() + public Task Run(Channels channels) { return AutomationExtensions.RunTask(new { - Input = Channels.BackendRequest.ForRead, + Input = channels.BackendRequest.AsRead() }, async self => diff --git a/Duplicati/Library/Main/Operation/Backup/Channels.cs b/Duplicati/Library/Main/Operation/Backup/Channels.cs index d7c5ff88e..40a442fe4 100644 --- a/Duplicati/Library/Main/Operation/Backup/Channels.cs +++ b/Duplicati/Library/Main/Operation/Backup/Channels.cs @@ -1,25 +1,24 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -using System; using CoCoL; namespace Duplicati.Library.Main.Operation.Backup @@ -27,40 +26,40 @@ namespace Duplicati.Library.Main.Operation.Backup /// /// Static typed definitions of channels used for the backup process /// - internal static class Channels + internal class Channels { /// /// Requests to the backend are send over this channel and picked up by the class /// - public static readonly ChannelMarkerWrapper BackendRequest = new ChannelMarkerWrapper(new ChannelNameAttribute("BackendRequests")); + public readonly IChannel BackendRequest = ChannelManager.CreateChannel(); /// /// When the backup completes, all in-progress archives are sent from the to the /// - public static readonly ChannelMarkerWrapper SpillPickup = new ChannelMarkerWrapper(new ChannelNameAttribute("SpillPickup")); + public readonly IChannel SpillPickup = ChannelManager.CreateChannel(); /// /// All data blocks are sent during the scanning to the who bundles them in compressed archives /// - public static readonly ChannelMarkerWrapper OutputBlocks = new ChannelMarkerWrapper(new ChannelNameAttribute("OutputBlocks")); + public readonly IChannel OutputBlocks = ChannelManager.CreateChannel(); /// /// If a file has changes in the metadata, it is sent to the where it is read /// - public static readonly ChannelMarkerWrapper AcceptedChangedFile = new ChannelMarkerWrapper(new ChannelNameAttribute("AcceptedChangedFile")); + public readonly IChannel AcceptedChangedFile = ChannelManager.CreateChannel(); /// /// If a file has changes in the metadata, it is sent to the where it is read /// - public static readonly ChannelMarkerWrapper StreamBlock = new ChannelMarkerWrapper(new ChannelNameAttribute("StreamBlockSplitter")); + public readonly IChannel StreamBlock = ChannelManager.CreateChannel(); /// /// After metadata has been processed and collected, the sends the file data to the /// - public static readonly ChannelMarkerWrapper ProcessedFiles = new ChannelMarkerWrapper(new ChannelNameAttribute("ProcessedFiles")); + public readonly IChannel ProcessedFiles = ChannelManager.CreateChannel(); /// - /// When enumerating the source folders, all discovered paths are sent by the to the + /// When enumerating the source folders, all discovered paths are sent by the to the /// - public static readonly ChannelMarkerWrapper SourcePaths = new ChannelMarkerWrapper(new ChannelNameAttribute("SourcePaths")); + public readonly IChannel SourcePaths = ChannelManager.CreateChannel(); /// /// All progress events are communicated over this channel, to ensure that parallel progress is reported as a if it was sequential /// - public static readonly ChannelMarkerWrapper ProgressEvents = new ChannelMarkerWrapper(new ChannelNameAttribute("ProgressEvents")); + public readonly IChannel ProgressEvents = ChannelManager.CreateChannel(); } } diff --git a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs index 479bc0674..47e660277 100644 --- a/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/Backup/CountFilesHandler.cs @@ -32,15 +32,14 @@ namespace Duplicati.Library.Main.Operation.Backup { public static async Task Run(IEnumerable sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, BackupResults result, Options options, IFilter sourcefilter, IFilter filter, HashSet blacklistPaths, Common.ITaskReader taskreader, System.Threading.CancellationToken token) { - // Make sure we create the enumeration process in a separate scope, - // but keep the log channel from the parent scope + // Keep the log channel from the parent scope using (Logging.Log.StartIsolatingScope(true)) - using (new IsolatedChannelScope()) { - var enumeratorTask = Backup.FileEnumerationProcess.Run(sources, snapshot, journalService, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, blacklistPaths, options.ChangedFilelist, taskreader, token); + Channels channels = new (); + var enumeratorTask = Backup.FileEnumerationProcess.Run(channels, sources, snapshot, journalService, options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, blacklistPaths, options.ChangedFilelist, taskreader, token); var counterTask = AutomationExtensions.RunTask(new { - Input = Backup.Channels.SourcePaths.ForRead + Input = channels.SourcePaths.AsRead() }, async self => diff --git a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs index 49545a46f..5624651ba 100644 --- a/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/DataBlockProcessor.cs @@ -35,14 +35,14 @@ namespace Duplicati.Library.Main.Operation.Backup /// internal static class DataBlockProcessor { - public static Task Run(BackupDatabase database, Options options, ITaskReader taskreader) + public static Task Run(Channels channels, BackupDatabase database, Options options, ITaskReader taskreader) { return AutomationExtensions.RunTask( new { - Input = Channels.OutputBlocks.ForRead, - Output = Channels.BackendRequest.ForWrite, - SpillPickup = Channels.SpillPickup.ForWrite, + Input = channels.OutputBlocks.AsRead(), + Output = channels.BackendRequest.AsWrite(), + SpillPickup = channels.SpillPickup.AsWrite(), }, async self => diff --git a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs index 42096e655..c1ab00934 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileBlockProcessor.cs @@ -1,23 +1,23 @@ -// Copyright (C) 2024, The Duplicati Team -// https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. using System; using CoCoL; @@ -37,13 +37,13 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FileBlockProcessor)) + ".FileEntry"; - public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader, CancellationToken token) + public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, BackupStatsCollector stats, ITaskReader taskreader, CancellationToken token) { return AutomationExtensions.RunTask( - new + new { - Input = Channels.AcceptedChangedFile.ForRead, - StreamBlockChannel = Channels.StreamBlock.ForWrite, + Input = channels.AcceptedChangedFile.AsRead(), + StreamBlockChannel = channels.StreamBlock.AsWrite(), }, async self => @@ -82,7 +82,7 @@ namespace Duplicati.Library.Main.Operation.Backup return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2; }); - using (var fs = snapshot.OpenRead(e.Path)) + using (var fs = snapshot.OpenRead(e.Path)) filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Path, fs, false, hint); await stats.AddOpenedFile(filestreamdata.Streamlength); @@ -97,7 +97,7 @@ namespace Duplicati.Library.Main.Operation.Backup Logging.Log.WriteVerboseMessage(FILELOGTAG, "NewFile", "New file {0}", e.Path); else Logging.Log.WriteVerboseMessage(FILELOGTAG, "ChangedFile", "File has changed {0}", e.Path); - + if (e.OldId < 0) { await stats.AddAddedFile(filesize); diff --git a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs index a96ae5576..21fd6039a 100644 --- a/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FileEnumerationProcess.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -44,12 +44,12 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string FILTER_LOGTAG = Logging.Log.LogTagFromType(typeof(FileEnumerationProcess)); - public static Task Run(IEnumerable sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, HashSet blacklistPaths, string[] changedfilelist, ITaskReader taskreader, CancellationToken token) + public static Task Run(Channels channels, IEnumerable sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, FileAttributes fileAttributes, Duplicati.Library.Utility.IFilter sourcefilter, Duplicati.Library.Utility.IFilter emitfilter, Options.SymlinkStrategy symlinkPolicy, Options.HardlinkStrategy hardlinkPolicy, bool excludeemptyfolders, string[] ignorenames, HashSet blacklistPaths, string[] changedfilelist, ITaskReader taskreader, CancellationToken token) { return AutomationExtensions.RunTask( new { - Output = Backup.Channels.SourcePaths.ForWrite + Output = channels.SourcePaths.AsWrite() }, async self => diff --git a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs index 8e84c651a..af1a52e4f 100644 --- a/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/FilePreFilterProcess.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -29,7 +29,7 @@ using Duplicati.Library.Main.Operation.Common; namespace Duplicati.Library.Main.Operation.Backup { /// - /// This process takes files that are processed for metadata, + /// This process takes files that are processed for metadata, /// and checks if anything indicates that the file has changed /// and submits potentially changed files for scanning /// @@ -40,14 +40,14 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FilePreFilterProcess)) + ".FileEntry"; - public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupStatsCollector stats, BackupDatabase database) + public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupStatsCollector stats, BackupDatabase database) { return AutomationExtensions.RunTask( new { - Input = Channels.ProcessedFiles.ForRead, - ProgressChannel = Channels.ProgressEvents.ForWrite, - Output = Channels.AcceptedChangedFile.ForWrite + Input = channels.ProcessedFiles.AsRead(), + ProgressChannel = channels.ProgressEvents.AsWrite(), + Output = channels.AcceptedChangedFile.AsWrite() }, async self => @@ -102,7 +102,7 @@ namespace Duplicati.Library.Main.Operation.Backup // and we only check for timestamp changes if (CHECKFILETIMEONLY && !timestampChanged && !isNewFile) { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path); + Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Path); try { await database.AddUnmodifiedAsync(e.OldId, e.LastWrite); diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index a449b03c9..9512e42ee 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -64,16 +64,16 @@ namespace Duplicati.Library.Main.Operation.Backup public IMetahash MetaHashAndSize; public bool MetadataChanged; } - - public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, CancellationToken token) + + public static Task Run(Channels channels, Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, CancellationToken token) { return AutomationExtensions.RunTask(new { - Input = Backup.Channels.SourcePaths.ForRead, - StreamBlockChannel = Channels.StreamBlock.ForWrite, - Output = Backup.Channels.ProcessedFiles.ForWrite, + Input = channels.SourcePaths.AsRead(), + StreamBlockChannel = channels.StreamBlock.AsWrite(), + Output = channels.ProcessedFiles.AsWrite(), }, - + async self => { var emptymetadata = Utility.WrapMetadata(new Dictionary(), options); @@ -88,22 +88,22 @@ namespace Duplicati.Library.Main.Operation.Backup var lastwrite = new DateTime(0, DateTimeKind.Utc); var attributes = default(FileAttributes); - try - { - lastwrite = snapshot.GetLastWriteTimeUtc(path); + try + { + lastwrite = snapshot.GetLastWriteTimeUtc(path); } - catch (Exception ex) + catch (Exception ex) { Logging.Log.WriteWarningMessage(FILELOGTAG, "TimestampReadFailed", ex, "Failed to read timestamp on \"{0}\"", path); } - try - { - attributes = snapshot.GetAttributes(path); - } - catch (Exception ex) + try { - Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); + attributes = snapshot.GetAttributes(path); + } + catch (Exception ex) + { + Logging.Log.WriteVerboseMessage(FILELOGTAG, "FailedAttributeRead", "Failed to read attributes from {0}: {1}", path, ex.Message); } // If we only have metadata, stop here @@ -258,13 +258,13 @@ namespace Duplicati.Library.Main.Operation.Backup { StreamProcessResult res; using (var ms = new MemoryStream(meta.Blob)) - res = await StreamBlock.ProcessStream(streamblockchannel, path, ms, true, CompressionHint.Default); + res = await StreamBlock.ProcessStream(streamblockchannel, path, ms, true, CompressionHint.Default); return await database.AddMetadatasetAsync(res.Streamhash, res.Streamlength, res.Blocksetid); } /// - /// Adds a file to the output, + /// Adds a file to the output, /// /// The name of the file to record /// The value of the lastModified timestamp @@ -275,7 +275,7 @@ namespace Duplicati.Library.Main.Operation.Backup } /// - /// Adds a file to the output, + /// Adds a file to the output, /// /// The name of the file to record /// The value of the lastModified timestamp diff --git a/Duplicati/Library/Main/Operation/Backup/ProgressHandler.cs b/Duplicati/Library/Main/Operation/Backup/ProgressHandler.cs index 46f3f354b..be4d804ae 100644 --- a/Duplicati/Library/Main/Operation/Backup/ProgressHandler.cs +++ b/Duplicati/Library/Main/Operation/Backup/ProgressHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -33,13 +33,13 @@ namespace Duplicati.Library.Main.Operation.Backup /// internal static class ProgressHandler { - public static Task Run(BackupResults stat) + public static Task Run(Channels channels, BackupResults stat) { return AutomationExtensions.RunTask(new { - Input = Channels.ProgressEvents.ForRead + Input = channels.ProgressEvents.AsRead() }, - + async self => { var filesStarted = new Dictionary(); @@ -64,7 +64,7 @@ namespace Duplicati.Library.Main.Operation.Backup case EventType.FileClosed: if (fileProgress.ContainsKey(t.Filepath)) fileProgress[t.Filepath] = t.Length; - + if (t.Filepath == current) { stat.OperationProgressUpdater.UpdateFileProgress(t.Length); @@ -94,7 +94,7 @@ namespace Duplicati.Library.Main.Operation.Backup { stat.OperationProgressUpdater.StartFile(current, filesStarted[current]); if (fileProgress.ContainsKey(current) && fileProgress[current] > 0) - stat.OperationProgressUpdater.UpdateFileProgress(fileProgress[current]); + stat.OperationProgressUpdater.UpdateFileProgress(fileProgress[current]); } } } diff --git a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs index a1242260a..e1050c624 100644 --- a/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs +++ b/Duplicati/Library/Main/Operation/Backup/RecreateMissingIndexFiles.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -33,11 +33,11 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(RecreateMissingIndexFiles)); - public static Task Run(BackupDatabase database, Options options, ITaskReader taskreader) + public static Task Run(Channels channels, BackupDatabase database, Options options, ITaskReader taskreader) { return AutomationExtensions.RunTask(new { - UploadChannel = Channels.BackendRequest.ForWrite + UploadChannel = channels.BackendRequest.AsWrite() }, async self => diff --git a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs index 25cdf4b13..005f5a1cd 100644 --- a/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/SpillCollectorProcess.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CoCoL; @@ -49,13 +49,13 @@ namespace Duplicati.Library.Main.Operation.Backup /// internal static class SpillCollectorProcess { - public static Task Run(Options options, BackupDatabase database, ITaskReader taskreader) + public static Task Run(Channels channels, Options options, BackupDatabase database, ITaskReader taskreader) { return AutomationExtensions.RunTask( new { - Input = Channels.SpillPickup.ForRead, - Output = Channels.BackendRequest.ForWrite, + Input = channels.SpillPickup.AsRead(), + Output = channels.BackendRequest.AsWrite(), }, async self => diff --git a/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs b/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs index 1dd4f6666..871f71ae4 100644 --- a/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs +++ b/Duplicati/Library/Main/Operation/Backup/StreamBlockSplitter.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -40,14 +40,14 @@ namespace Duplicati.Library.Main.Operation.Backup private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(StreamBlockSplitter)) ; private static readonly string FILELOGTAG = LOGTAG + ".FileEntry"; - public static Task Run(Options options, BackupDatabase database, ITaskReader taskreader) + public static Task Run(Channels channels, Options options, BackupDatabase database, ITaskReader taskreader) { return AutomationExtensions.RunTask( new { - Input = Channels.StreamBlock.ForRead, - ProgressChannel = Channels.ProgressEvents.ForWrite, - BlockOutput = Channels.OutputBlocks.ForWrite + Input = channels.StreamBlock.AsRead(), + ProgressChannel = channels.ProgressEvents.AsWrite(), + BlockOutput = channels.OutputBlocks.AsWrite() }, async self => diff --git a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs index 292eac1a5..0cb8b2f98 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadRealFilelist.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -33,23 +33,23 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadRealFilelist)); - public static Task Run(BackupResults result, BackupDatabase db, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader) + public static Task Run(Channels channels, BackupResults result, BackupDatabase db, Options options, FilesetVolumeWriter filesetvolume, long filesetid, Common.ITaskReader taskreader) { return AutomationExtensions.RunTask(new { - Output = Channels.BackendRequest.ForWrite, + Output = channels.BackendRequest.AsWrite() }, async self => { if (!await taskreader.ProgressAsync) return; - + // Update the reported source and backend changes using(new Logging.Timer(LOGTAG, "UpdateChangeStatistics", "UpdateChangeStatistics")) await db.UpdateChangeStatisticsAsync(result); - var changeCount = + var changeCount = result.AddedFiles + result.ModifiedFiles + result.DeletedFiles + result.AddedFolders + result.ModifiedFolders + result.DeletedFolders + result.AddedSymlinks + result.ModifiedSymlinks + result.DeletedSymlinks; @@ -71,7 +71,7 @@ namespace Duplicati.Library.Main.Operation.Backup if (!await taskreader.ProgressAsync) return; - + await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null); await db.CommitTransactionAsync("CommitUpdateRemoteVolume"); await self.Output.WriteAsync(new FilesetUploadRequest(filesetvolume)); diff --git a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs index 1da10d502..49c57540a 100644 --- a/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs +++ b/Duplicati/Library/Main/Operation/Backup/UploadSyntheticFilelist.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using CoCoL; @@ -38,11 +38,11 @@ namespace Duplicati.Library.Main.Operation.Backup /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(UploadSyntheticFilelist)); - public static Task Run(BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader, string lastTempFilelist, long lastTempFilesetId) + public static Task Run(Channels channels, BackupDatabase database, Options options, BackupResults result, ITaskReader taskreader, string lastTempFilelist, long lastTempFilesetId) { return AutomationExtensions.RunTask(new { - UploadChannel = Channels.BackendRequest.ForWrite + UploadChannel = channels.BackendRequest.AsWrite() }, async self => diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index 7db790590..bcc263476 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -32,7 +32,6 @@ using Duplicati.Library.Snapshots; using Duplicati.Library.Utility; using Duplicati.Library.Common.IO; using Duplicati.Library.Logging; -using System.IO; namespace Duplicati.Library.Main.Operation { @@ -179,11 +178,11 @@ namespace Duplicati.Library.Main.Operation /// The logic here is needed to check that the database is in a state /// where it can be used for the backup, and that the backend is also /// in the same state as the database. - /// + /// /// If the auto-repair option is enabled, this method will attempt to /// call the repair method, which requires that the database is closed /// and re-opened. - /// + /// /// For efficiency, the database is only closed if the repair is needed, /// and returned to the caller in an open state in either case. /// @@ -292,43 +291,39 @@ namespace Duplicati.Library.Main.Operation /// /// Performs the bulk of work by starting all relevant processes /// - private static async Task RunMainOperation(IEnumerable sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, Common.ITaskReader taskreader, long filesetid, long lastfilesetid, CancellationToken token) + private static async Task RunMainOperation(Backup.Channels channels, IEnumerable sources, Snapshots.ISnapshotService snapshot, UsnJournalService journalService, Backup.BackupDatabase database, Backup.BackupStatsCollector stats, Options options, IFilter sourcefilter, IFilter filter, BackupResults result, Common.ITaskReader taskreader, long filesetid, long lastfilesetid, CancellationToken token) { using (new Logging.Timer(LOGTAG, "BackupMainOperation", "BackupMainOperation")) { // Make sure the CompressionHints table is initialized, otherwise all workers will initialize it var unused = options.CompressionHints.Count; - Task all; - using (new ChannelScope()) - { - all = Task.WhenAll( - new[] - { - Backup.DataBlockProcessor.Run(database, options, taskreader), - Backup.FileBlockProcessor.Run(snapshot, options, database, stats, taskreader, token), - Backup.StreamBlockSplitter.Run(options, database, taskreader), - Backup.FileEnumerationProcess.Run(sources, snapshot, journalService, - options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, - options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, - GetBlacklistedPaths(options), options.ChangedFilelist, taskreader, token), - Backup.FilePreFilterProcess.Run(snapshot, options, stats, database), - Backup.MetadataPreProcess.Run(snapshot, options, database, lastfilesetid, token), - Backup.SpillCollectorProcess.Run(options, database, taskreader), - Backup.ProgressHandler.Run(result) - } - // Spawn additional block hashers - .Union( - Enumerable.Range(0, options.ConcurrencyBlockHashers - 1).Select(x => - Backup.StreamBlockSplitter.Run(options, database, taskreader)) - ) - // Spawn additional compressors - .Union( - Enumerable.Range(0, options.ConcurrencyCompressors - 1).Select(x => - Backup.DataBlockProcessor.Run(database, options, taskreader)) - ) - ); - } + Task all = Task.WhenAll( + new[] + { + Backup.DataBlockProcessor.Run(channels, database, options, taskreader), + Backup.FileBlockProcessor.Run(channels, snapshot, options, database, stats, taskreader, token), + Backup.StreamBlockSplitter.Run(channels, options, database, taskreader), + Backup.FileEnumerationProcess.Run(channels, sources, snapshot, journalService, + options.FileAttributeFilter, sourcefilter, filter, options.SymlinkPolicy, + options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames, + GetBlacklistedPaths(options), options.ChangedFilelist, taskreader, token), + Backup.FilePreFilterProcess.Run(channels, snapshot, options, stats, database), + Backup.MetadataPreProcess.Run(channels, snapshot, options, database, lastfilesetid, token), + Backup.SpillCollectorProcess.Run(channels, options, database, taskreader), + Backup.ProgressHandler.Run(channels, result) + } + // Spawn additional block hashers + .Union( + Enumerable.Range(0, options.ConcurrencyBlockHashers - 1).Select(x => + Backup.StreamBlockSplitter.Run(channels, options, database, taskreader)) + ) + // Spawn additional compressors + .Union( + Enumerable.Range(0, options.ConcurrencyCompressors - 1).Select(x => + Backup.DataBlockProcessor.Run(channels, database, options, taskreader)) + ) + ); await all.ConfigureAwait(false); @@ -517,189 +512,186 @@ namespace Duplicati.Library.Main.Operation // Do a remote verification, unless disabled var (database, backendManager, lastTempFilelist, lastTempFilesetId) = await PreBackupVerify(m_backendurl, m_options, m_result); - // New isolated scope for each operation - using (new IsolatedChannelScope()) + Backup.Channels channels = new (); + + // If there is no filter, we set an empty filter to simplify the code + // If there is a filter, we make sure that the sources are included + m_filter = filter ?? new Library.Utility.FilterExpression(); + m_sourceFilter = new Library.Utility.FilterExpression(sources, true); + + Task parallelScanner = null; + Task uploaderTask = null; + try { - // If there is no filter, we set an empty filter to simplify the code - // If there is a filter, we make sure that the sources are included - m_filter = filter ?? new Library.Utility.FilterExpression(); - m_sourceFilter = new Library.Utility.FilterExpression(sources, true); - - Task parallelScanner = null; - Task uploaderTask = null; - try + using (m_database = database) + using (backendManager) + using (var db = new Backup.BackupDatabase(m_database, m_options)) + // Setup runners and instances here + using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp)) + using (var stats = new Backup.BackupStatsCollector(m_result)) + using (var uploadtarget = ChannelManager.GetChannel(channels.BackendRequest.AsWrite())) { - using (m_database = database) - using (backendManager) - using (var db = new Backup.BackupDatabase(m_database, m_options)) - // Setup runners and instances here - using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp)) - using (var stats = new Backup.BackupStatsCollector(m_result)) - // Keep a reference to these channels to avoid shutdown - using (var uploadtarget = ChannelManager.GetChannel(Backup.Channels.BackendRequest.ForWrite)) + long filesetid; + var counterToken = new CancellationTokenSource(); + var uploader = new Backup.BackendUploader(() => DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions), m_options, db, m_result.TaskReader, stats); + using (var snapshot = GetSnapshot(sources, m_options)) { - long filesetid; - var counterToken = new CancellationTokenSource(); - var uploader = new Backup.BackendUploader(() => DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions), m_options, db, m_result.TaskReader, stats); - using (var snapshot = GetSnapshot(sources, m_options)) - { - try - { - // Start the uploader process - uploaderTask = uploader.Run(); - - // If the previous backup was interrupted, send a synthetic list - await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskReader, lastTempFilelist, lastTempFilesetId); - - // Grab the previous backup ID, if any - var prevfileset = m_database.FilesetTimes.FirstOrDefault(); - if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime()) - throw new Exception(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime())); - - var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key; - - // Rebuild any index files that are missing - await Backup.RecreateMissingIndexFiles.Run(db, m_options, m_result.TaskReader); - - // Prepare the operation by registering the filelist - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles); - - var repcnt = 0; - while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0) - filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++)); - - if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0) - throw new Exception("Unable to generate a unique fileset name"); - - var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); - filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time); - - // create USN-based scanner if enabled - var journalService = GetJournalService(sources, snapshot, filter, lastfilesetid); - - // Start parallel scan, or use the database - if (m_options.DisableFileScanner) - { - var d = m_database.GetLastBackupFileCountAndSize(); - m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true); - } - else - { - parallelScanner = Backup.CountFilesHandler.Run(sources, snapshot, journalService, m_result, m_options, m_sourceFilter, m_filter, GetBlacklistedPaths(), m_result.TaskReader, counterToken.Token); - } - - // Run the backup operation - if (await m_result.TaskReader.ProgressAsync) - { - await RunMainOperation(sources, snapshot, journalService, db, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskReader, filesetid, lastfilesetid, token).ConfigureAwait(false); - } - } - finally - { - //If the scanner is still running for some reason, make sure we kill it now - counterToken.Cancel(); - } - } - - // Add the fileset file to the dlist file - filesetvolume.CreateFilesetFile(!token.IsCancellationRequested); - - // Ensure the database is in a sane state after adding data - using (new Logging.Timer(LOGTAG, "VerifyConsistency", "VerifyConsistency")) - await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false); - - // Send the actual filelist - if (await m_result.TaskReader.ProgressAsync) - await Backup.UploadRealFilelist.Run(m_result, db, m_options, filesetvolume, filesetid, m_result.TaskReader); - - // Wait for upload completion - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload); - var lastVolumeSize = await FlushBackend(m_result, uploadtarget, uploaderTask).ConfigureAwait(false); - - // Make sure we have the database up-to-date - await db.CommitTransactionAsync("CommitAfterUpload", false); - - // TODO: Remove this later - m_transaction = m_database.BeginTransaction(); - try { - // If this throws, we should roll back the transaction + // Start the uploader process + uploaderTask = uploader.Run(channels); + + // If the previous backup was interrupted, send a synthetic list + await Backup.UploadSyntheticFilelist.Run(channels, db, m_options, m_result, m_result.TaskReader, lastTempFilelist, lastTempFilesetId); + + // Grab the previous backup ID, if any + var prevfileset = m_database.FilesetTimes.FirstOrDefault(); + if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime()) + throw new Exception(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime())); + + var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key; + + // Rebuild any index files that are missing + await Backup.RecreateMissingIndexFiles.Run(channels, db, m_options, m_result.TaskReader); + + // Prepare the operation by registering the filelist + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles); + + var repcnt = 0; + while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0) + filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++)); + + if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0) + throw new Exception("Unable to generate a unique fileset name"); + + var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary); + filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time); + + // create USN-based scanner if enabled + var journalService = GetJournalService(sources, snapshot, filter, lastfilesetid); + + // Start parallel scan, or use the database + if (m_options.DisableFileScanner) + { + var d = m_database.GetLastBackupFileCountAndSize(); + m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true); + } + else + { + parallelScanner = Backup.CountFilesHandler.Run(sources, snapshot, journalService, m_result, m_options, m_sourceFilter, m_filter, GetBlacklistedPaths(), m_result.TaskReader, counterToken.Token); + } + + // Run the backup operation if (await m_result.TaskReader.ProgressAsync) - CompactIfRequired(backendManager, lastVolumeSize); - - if (m_options.UploadVerificationFile && await m_result.TaskReader.ProgressAsync) { - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); - FilelistProcessor.UploadVerificationFile(backendManager.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction); + await RunMainOperation(channels, sources, snapshot, journalService, db, stats, m_options, m_sourceFilter, m_filter, m_result, m_result.TaskReader, filesetid, lastfilesetid, token).ConfigureAwait(false); } } - catch + finally { - try - { - m_transaction.Rollback(); - m_transaction.Dispose(); - m_transaction = null; - } - catch (Exception ex) - { - Logging.Log.WriteErrorMessage(LOGTAG, "RollbackError", ex, "Rollback error: {0}", ex.Message); - } - - // Re-throw the original exception - throw; + //If the scanner is still running for some reason, make sure we kill it now + counterToken.Cancel(); } + } - if (m_options.Dryrun) + // Add the fileset file to the dlist file + filesetvolume.CreateFilesetFile(!token.IsCancellationRequested); + + // Ensure the database is in a sane state after adding data + using (new Logging.Timer(LOGTAG, "VerifyConsistency", "VerifyConsistency")) + await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false); + + // Send the actual filelist + if (await m_result.TaskReader.ProgressAsync) + await Backup.UploadRealFilelist.Run(channels, m_result, db, m_options, filesetvolume, filesetid, m_result.TaskReader); + + // Wait for upload completion + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload); + var lastVolumeSize = await FlushBackend(m_result, uploadtarget, uploaderTask).ConfigureAwait(false); + + // Make sure we have the database up-to-date + await db.CommitTransactionAsync("CommitAfterUpload", false); + + // TODO: Remove this later + m_transaction = m_database.BeginTransaction(); + + try + { + // If this throws, we should roll back the transaction + if (await m_result.TaskReader.ProgressAsync) + CompactIfRequired(backendManager, lastVolumeSize); + + if (m_options.UploadVerificationFile && await m_result.TaskReader.ProgressAsync) + { + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload); + FilelistProcessor.UploadVerificationFile(backendManager.BackendUrl, m_options, m_result.BackendWriter, m_database, m_transaction); + } + } + catch + { + try { m_transaction.Rollback(); + m_transaction.Dispose(); m_transaction = null; } - else + catch (Exception ex) { - using (new Logging.Timer(LOGTAG, "CommitFinalizingBackup", "CommitFinalizingBackup")) - m_transaction.Commit(); - - m_transaction = null; - - if (m_result.TaskControlRendevouz() != TaskControlState.Abort) - { - if (m_options.NoBackendverification) - UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager); - else - PostBackupVerification(filesetvolume.RemoteFilename); - } + Logging.Log.WriteErrorMessage(LOGTAG, "RollbackError", ex, "Rollback error: {0}", ex.Message); } - m_database.WriteResults(); - m_database.PurgeLogData(m_options.LogRetention); - m_database.PurgeDeletedVolumes(DateTime.UtcNow); - - if (m_options.AutoVacuum) - { - m_result.VacuumResults = new VacuumResults(m_result); - new VacuumHandler(m_options, (VacuumResults)m_result.VacuumResults).Run(); - } - m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete); - return; - } - } - catch (Exception ex) - { - var aex = BuildException(ex, uploaderTask, parallelScanner); - Logging.Log.WriteErrorMessage(LOGTAG, "FatalError", ex, "Fatal error"); - if (aex == ex) + // Re-throw the original exception throw; + } - throw aex; - } - finally - { - if (parallelScanner != null && !parallelScanner.IsCompleted) - parallelScanner.Wait(500); + if (m_options.Dryrun) + { + m_transaction.Rollback(); + m_transaction = null; + } + else + { + using (new Logging.Timer(LOGTAG, "CommitFinalizingBackup", "CommitFinalizingBackup")) + m_transaction.Commit(); + + m_transaction = null; + + if (m_result.TaskControlRendevouz() != TaskControlState.Abort) + { + if (m_options.NoBackendverification) + UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager); + else + PostBackupVerification(filesetvolume.RemoteFilename); + } + } + + m_database.WriteResults(); + m_database.PurgeLogData(m_options.LogRetention); + m_database.PurgeDeletedVolumes(DateTime.UtcNow); + + if (m_options.AutoVacuum) + { + m_result.VacuumResults = new VacuumResults(m_result); + new VacuumHandler(m_options, (VacuumResults)m_result.VacuumResults).Run(); + } + m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete); + return; } } + catch (Exception ex) + { + var aex = BuildException(ex, uploaderTask, parallelScanner); + Logging.Log.WriteErrorMessage(LOGTAG, "FatalError", ex, "Fatal error"); + if (aex == ex) + throw; + + throw aex; + } + finally + { + if (parallelScanner != null && !parallelScanner.IsCompleted) + parallelScanner.Wait(500); + } } public void Dispose() diff --git a/Duplicati/Library/Main/Operation/TestFilterHandler.cs b/Duplicati/Library/Main/Operation/TestFilterHandler.cs index 2e80bebc8..93f68d4d2 100644 --- a/Duplicati/Library/Main/Operation/TestFilterHandler.cs +++ b/Duplicati/Library/Main/Operation/TestFilterHandler.cs @@ -1,22 +1,22 @@ // Copyright (C) 2024, The Duplicati Team // https://duplicati.com, hello@duplicati.com -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in +// +// The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. using System; @@ -48,15 +48,17 @@ namespace Duplicati.Library.Main.Operation var sourcefilter = new Library.Utility.FilterExpression(sources, true); using (var snapshot = BackupHandler.GetSnapshot(sources, m_options)) - using (new IsolatedChannelScope()) { - var source = Operation.Backup.FileEnumerationProcess.Run(sources, snapshot, null, + Backup.Channels channels = new (); + var source = Backup.FileEnumerationProcess.Run(channels, sources, snapshot, null, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_options.ExcludeEmptyFolders, m_options.IgnoreFilenames, BackupHandler.GetBlacklistedPaths(m_options), null, m_result.TaskReader, token); - var sink = CoCoL.AutomationExtensions.RunTask( - new { source = Operation.Backup.Channels.SourcePaths.ForRead }, + var sink = CoCoL.AutomationExtensions.RunTask(new + { + source = channels.SourcePaths.AsRead() + }, async self => { while (true) diff --git a/Duplicati/Server/webroot/ngax/index.html b/Duplicati/Server/webroot/ngax/index.html index deb30ad95..b3e984013 100755 --- a/Duplicati/Server/webroot/ngax/index.html +++ b/Duplicati/Server/webroot/ngax/index.html @@ -257,7 +257,7 @@
- {{'Manual' | translate}}
 + {{'Manual' | translate}}
diff --git a/LiveTests/Duplicati.Backend.Tests/README.md b/LiveTests/Duplicati.Backend.Tests/README.md index 6c21d81dc..a9a845b4e 100644 --- a/LiveTests/Duplicati.Backend.Tests/README.md +++ b/LiveTests/Duplicati.Backend.Tests/README.md @@ -55,6 +55,16 @@ TESTCREDENTIAL_AZURE_ACCESSKEY TESTCREDENTIAL_AZURE_CONTAINERNAME ``` +## pCloud Native API + +``` +TESTCREDENTIAL_PCLOUD_SERVER +TESTCREDENTIAL_PCLOUD_TOKEN +TESTCREDENTIAL_PCLOUD_FOLDER +``` +For PCloud the server is the API server(eapi.pcloud.com for EU hosted account or api.pcloud.com for non EU). The token is the OAuth token. + + ## Running the tests [TestContainers](https://testcontainers.org/) is a pre-requisite for SSH, FTP and Webdav tests. It is not required for the other tests. diff --git a/LiveTests/Duplicati.Backend.Tests/pCloud/pCloudTests.cs b/LiveTests/Duplicati.Backend.Tests/pCloud/pCloudTests.cs new file mode 100644 index 000000000..9b3d7b933 --- /dev/null +++ b/LiveTests/Duplicati.Backend.Tests/pCloud/pCloudTests.cs @@ -0,0 +1,78 @@ +// Copyright (C) 2024, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.Backend.Tests.pCloud; + +/// +/// pCloud Tests for CI. +/// +/// These tests cannot be parallelized as they will share credentials and folders +/// +[TestClass] +[DoNotParallelize] +public sealed class pCloudTests : BaseTest +{ + /// + /// Perform tests using pCloud API + /// + /// + [TestMethod] + public Task TestpCoudAPI() + { + CheckRequiredEnvironment([ + "TESTCREDENTIAL_PCLOUD_SERVER", + "TESTCREDENTIAL_PCLOUD_TOKEN", + "TESTCREDENTIAL_PCLOUD_FOLDER" + ]); + + var testURI = $"pcloud://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_SERVER")}/{Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_FOLDER")}?&authid={Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_TOKEN")}"; + + // Specifically disable streaming to pass tests. + var exitCode = CommandLine.BackendTester.Program.Main(new[] {testURI, "--disable-streaming-transfers"}.Concat(Parameters.GlobalTestParameters).ToArray()); + + if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details"); + + return Task.CompletedTask; + } + + /// + /// Perform tests using pCloud API + /// + /// + [TestMethod] + public Task TestpCoudAPIWithStreaming() + { + CheckRequiredEnvironment([ + "TESTCREDENTIAL_PCLOUD_SERVER", + "TESTCREDENTIAL_PCLOUD_TOKEN", + "TESTCREDENTIAL_PCLOUD_FOLDER" + ]); + + var testURI = $"pcloud://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_SERVER")}/{Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_FOLDER")}?&authid={Environment.GetEnvironmentVariable("TESTCREDENTIAL_PCLOUD_TOKEN")}"; + + var exitCode = CommandLine.BackendTester.Program.Main(new[] {testURI}.Concat(Parameters.GlobalTestParameters).ToArray()); + + if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details"); + + return Task.CompletedTask; + } + +} \ No newline at end of file diff --git a/README.ja-JP.md b/README.ja-JP.md index c44107140..2f21bc824 100644 --- a/README.ja-JP.md +++ b/README.ja-JP.md @@ -22,7 +22,7 @@ Gitterは削除済 Duplicatiは、フリー（自由）でオープンソースのバックアップ用クライアントです。圧縮し、暗号化した増分バックアップを、クラウドストレージサービスや遠隔のファイルサーバー上に安全に保存できます。Duplicatiは、主に以下のサービスやソフトウェアで使うことができます。 -   *Amazon S3、[IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage")、Box、Dropbox、FTP、Googleクラウド、Googleドライブ、MEGA、Microsoft Azure、Microsoft OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、Tencentクラウドオブジェクトストレージ（COS）、Aliyun OSS、[その他にも対応しています！](https://duplicati.readthedocs.io/en/latest/01-introduction/#supported-backends)* +   *Amazon S3、[IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage")、Box、Dropbox、FTP、Googleクラウド、Googleドライブ、MEGA、Microsoft Azure、Microsoft OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、Tencentクラウドオブジェクトストレージ（COS）、Aliyun OSS、[その他にも対応しています！](https://docs.duplicati.com/backup-destinations/destination-overview)* DuplicatiはMITライセンスで公開されており、Windows、OSX、Linuxで利用できます（.NET 4.7.1以上、またはMono 5.10.0以上が必要です）。 diff --git a/README.md b/README.md index c49dfa2a0..94240146a 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Removed Gitter Duplicati is a free, open source, backup client that securely stores encrypted, incremental, compressed backups on cloud storage services and remote file servers. It works with: -   *Amazon S3, [IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2"), [Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage"), Box, Dropbox, FTP, Google Cloud and Drive, MEGA, Microsoft Azure and OneDrive, Rackspace Cloud Files, OpenStack Storage (Swift), Sia, Storj DCS, SSH (SFTP), WebDAV, Tencent Cloud Object Storage (COS), Aliyun OSS, [and more!](https://duplicati.readthedocs.io/en/latest/01-introduction/#supported-backends)* +   *Amazon S3, [IDrive e2](https://www.idrive.com/e2/duplicati "Using Duplicati with IDrive e2"), [Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati with Backblaze B2 Cloud Storage"), Box, Dropbox, FTP, Google Cloud and Drive, MEGA, Microsoft Azure and OneDrive, Rackspace Cloud Files, OpenStack Storage (Swift), Sia, Storj DCS, SSH (SFTP), WebDAV, Tencent Cloud Object Storage (COS), Aliyun OSS, [and more!](https://docs.duplicati.com/backup-destinations/destination-overview)* Duplicati is licensed under the MIT license and available for Windows, OSX and Linux (.NET 4.7.1+ or Mono 5.10.0+ required). diff --git a/README.zh-CN.md b/README.zh-CN.md index 567e22c35..9c81224bb 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -26,7 +26,7 @@ Duplicati 是一个免费、开源的备份客户端，可以安全地将加密、增量、压缩的备份存储在云存储服务和远程文件服务器上。它与以下服务兼容： -   *亚马逊 S3、[IDrive e2](https://www.idrive.com/e2/duplicati "使用 Duplicati 与 IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati 与 Backblaze B2 云存储")、Box、Dropbox、FTP、Google Cloud 和 Drive、MEGA、Microsoft Azure 和 OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、阿里云对象存储(OSS)、百度网盘、阿里云盘、腾讯云对象存储 (COS)、[以及更多！](https://duplicati.readthedocs.io/en/latest/01-introduction/#supported-backends)* +   *亚马逊 S3、[IDrive e2](https://www.idrive.com/e2/duplicati "使用 Duplicati 与 IDrive e2")、[Backblaze (B2)](https://www.backblaze.com/blog/duplicati-backups-cloud-storage/ "Duplicati 与 Backblaze B2 云存储")、Box、Dropbox、FTP、Google Cloud 和 Drive、MEGA、Microsoft Azure 和 OneDrive、Rackspace Cloud Files、OpenStack Storage (Swift)、Sia、Storj DCS、SSH (SFTP)、WebDAV、阿里云对象存储(OSS)、百度网盘、阿里云盘、腾讯云对象存储 (COS)、[以及更多！](https://docs.duplicati.com/backup-destinations/destination-overview)* Duplicati 根据 LGPL 许可证授权，并可用于 Windows、OSX 和 Linux (.NET 4.7.1+ 或 Mono 5.10.0+ 需要)。 diff --git a/ReleaseBuilder/Build/Command.GitPush.cs b/ReleaseBuilder/Build/Command.GitPush.cs index 2212b4735..3ba8e30b8 100644 --- a/ReleaseBuilder/Build/Command.GitPush.cs +++ b/ReleaseBuilder/Build/Command.GitPush.cs @@ -40,6 +40,7 @@ public static partial class Command }, workingDirectory: baseDir); // Then push the release + await ProcessHelper.Execute(new[] { "git", "push" }, workingDirectory: baseDir); await ProcessHelper.Execute(new[] { "git", "push", "--tags" }, workingDirectory: baseDir); } } diff --git a/ReleaseBuilder/Build/Command.cs b/ReleaseBuilder/Build/Command.cs index 9c6094747..627dcd7d5 100644 --- a/ReleaseBuilder/Build/Command.cs +++ b/ReleaseBuilder/Build/Command.cs @@ -220,7 +220,7 @@ public static partial class Command ); var changelogFileOption = new Option( - name: "--changelog-file", + name: "--changelog-news-file", description: "The path to the changelog news file. Contents from this file are prepended to the changelog.", getDefaultValue: () => new FileInfo(Path.GetFullPath("changelog-news.txt")) ); @@ -319,7 +319,7 @@ public static partial class Command /// If the docker push should be disabled /// The name of the MacOS app bundle /// The docker repository to push to - /// The path to the changelog file + /// The path to the changelog news file /// If notarize signing should be disabled /// If GPG signing should be disabled /// If S3 upload should be disabled @@ -342,7 +342,7 @@ public static partial class Command bool DisableDockerPush, string MacOSAppName, string DockerRepo, - FileInfo ChangelogFile, + FileInfo ChangelogNewsFile, bool DisableNotarizeSigning, bool DisableGpgSigning, bool DisableS3Upload, @@ -408,28 +408,28 @@ public static partial class Command if (!File.Exists(primaryCLI)) throw new Exception($"Failed to locate project file: {primaryCLI}"); - if (!input.ChangelogFile.Exists) + if (!input.ChangelogNewsFile.Exists) { - Console.WriteLine($"Changelog news file not found: {input.ChangelogFile.FullName}"); + Console.WriteLine($"Changelog news file not found: {input.ChangelogNewsFile.FullName}"); Console.WriteLine($"Create an empty file if you want a release without changes"); if (OperatingSystem.IsWindows()) - Console.WriteLine($"> type nul > {input.ChangelogFile.FullName}"); + Console.WriteLine($"> type nul > {input.ChangelogNewsFile.FullName}"); else - Console.WriteLine($"> touch {input.ChangelogFile.FullName}"); + Console.WriteLine($"> touch {input.ChangelogNewsFile.FullName}"); Program.ReturnCode = 1; return; } - if (input.ChangelogFile.LastAccessTimeUtc < DateTime.UtcNow.AddDays(-1)) + if (input.ChangelogNewsFile.LastAccessTimeUtc < DateTime.UtcNow.AddDays(-1)) { - Console.WriteLine($"Changelog news file was last modified {input.ChangelogFile.LastAccessTimeUtc:yyyy-MM-dd}, indicating a stale file"); + Console.WriteLine($"Changelog news file was last modified {input.ChangelogNewsFile.LastAccessTimeUtc:yyyy-MM-dd}, indicating a stale file"); Console.WriteLine($"Please update the file with the changelog news for the release"); Program.ReturnCode = 1; return; } - var changelogNews = File.ReadAllText(input.ChangelogFile.FullName); + var changelogNews = File.ReadAllText(input.ChangelogNewsFile.FullName); var releaseInfo = string.IsNullOrWhiteSpace(input.Version) ? ReleaseInfo.Create(input.Channel, Version.Parse(File.ReadAllText(versionFilePath)), 1) @@ -472,6 +472,12 @@ public static partial class Command throw new Exception($"The following packages cannot be built without Docker: {string.Join(", ", unsupportedBuilds.Select(x => x.PackageTargetString))}"); } + if (rtcfg.UseGithubUpload && !input.GitStashPush) + { + Console.WriteLine("Github upload requested, but pushing the tag is disabled"); + Program.ReturnCode = 1; + } + if (!input.KeepBuilds && Directory.Exists(input.BuildPath.FullName)) { Console.WriteLine($"Deleting build folder: {input.BuildPath.FullName}"); @@ -576,6 +582,11 @@ public static partial class Command files.Add(sigfile); } + // Ensure the tag is pushed before uploading, so the uploaded files + // are associated with the release tag + if (input.GitStashPush) + await GitPush.TagAndPush(baseDir, releaseInfo); + if (rtcfg.UseS3Upload || rtcfg.UseGithubUpload) { Console.WriteLine("Build completed, uploading packages ..."); @@ -597,10 +608,18 @@ public static partial class Command } if (rtcfg.UseGithubUpload) + { + if (!rtcfg.UseS3Upload && input.GitStashPush) + { + Console.WriteLine("Waiting for Github to create the release tag ..."); + await Task.Delay(TimeSpan.FromSeconds(30)); + } + await Upload.UploadToGithub( files.Select(x => new Upload.UploadFile(x, Path.GetFileName(x))), rtcfg ); + } } if (rtcfg.UseUpdateServerReload) @@ -617,10 +636,8 @@ public static partial class Command if (input.GitStashPush) { - await GitPush.TagAndPush(baseDir, releaseInfo); - // Contents are added to changelog, so we can remove the file - input.ChangelogFile.Delete(); + input.ChangelogNewsFile.Delete(); } // Clean up the source tree @@ -628,10 +645,18 @@ public static partial class Command if (Directory.Exists(folder)) Directory.Delete(folder, true); - await ProcessHelper.Execute(new[] { + // This often fails and should be fixed + try + { + await ProcessHelper.Execute(new[] { "git", "checkout", }.Concat(revertableFiles.Select(x => Path.GetRelativePath(baseDir, x))) - , workingDirectory: baseDir); + , workingDirectory: baseDir); + } + catch (Exception ex) + { + Console.WriteLine($"Warning: Failed to clean source directory: {ex.Message}"); + } Console.WriteLine("All done!"); } diff --git a/ReleaseBuilder/build_version.txt b/ReleaseBuilder/build_version.txt index 298378dc2..bbe27ff9e 100644 --- a/ReleaseBuilder/build_version.txt +++ b/ReleaseBuilder/build_version.txt @@ -1 +1 @@ -2.1.0.100 \ No newline at end of file +2.1.0.101 \ No newline at end of file diff --git a/changelog.txt b/changelog.txt index 221afd310..9a572bf18 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,31 @@ +2024-12-06 - 2.1.0.101_canary_2024-12-06 +========== +This release is a canary release intended to be used for testing. + +**Note that this build is using .NET8, unlike the 2.0.8.1 build** + +## Breaking change from previous canary +The pCloud backend is updated to support subfolders. +Paths with `/` will now be treated as a folder structure on pCloud. +Previous version would treat `folder/subfolder` as a single foldername, +this version treats it as `folder` / `subfolder`. + +## Detailed list of changes: +- Fixed a case where the TrayIcon would hang in detached mode +- Improved OpenAPI description for filter groups +- Fixed an issue with websocket requests through a proxy +- Updated icons for Linux and dark-mode MacOS +- Added logging of unobserved exceptions +- Fixed a compatibility issue with FTP not correctly detecting explicit TLS +- Fixed an issue where the quota was reported incorrectly +- Removed the unused `--use-block-cache` option +- Agent now has a dedicated `register` command +- Sizes can now be supplied as kikibytes, eg. `20kib` +- Fixed supporting subfolders in pCloud +- Added option to send http requests with relaxed TLS security +- Agent can now be asked to retain the previous UI password +- Updated SSH.NET to v2024.2.0 + 2024-11-25 - 2.1.0.100_canary_2024-11-25 ========== This release is a canary release intended to be used for testing.