diff --git a/Duplicati/Library/Interface/CustomExceptions.cs b/Duplicati/Library/Interface/CustomExceptions.cs index a67496de3..0d7f6e55d 100644 --- a/Duplicati/Library/Interface/CustomExceptions.cs +++ b/Duplicati/Library/Interface/CustomExceptions.cs @@ -273,4 +273,35 @@ namespace Duplicati.Library.Interface : base(message, helpId, innerException) { } } + + /// + /// Exception indicating that the backup database is currently locked by another running operation + /// and cannot be accessed concurrently. + /// + [Serializable] + public class DatabaseLockedException : UserInformationException + { + /// + /// The path to the database file that is locked. + /// + public readonly string DatabasePath; + + public DatabaseLockedException(string databasePath) + : base(LC.L($"The backup database is currently in use by another operation and cannot be accessed concurrently. Please wait for the running operation to finish and try again."), "DatabaseLocked") + { + DatabasePath = databasePath; + } + + public DatabaseLockedException(string databasePath, Exception innerException) + : base(LC.L($"The backup database is currently in use by another operation and cannot be accessed concurrently. Please wait for the running operation to finish and try again."), "DatabaseLocked", innerException) + { + DatabasePath = databasePath; + } + + public DatabaseLockedException(string message, string helpId, Exception innerException) + : base(message, helpId, innerException) + { + DatabasePath = string.Empty; + } + } } diff --git a/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs b/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs index 98a615683..4fe3797c9 100644 --- a/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs +++ b/Duplicati/Library/RestAPI/Abstractions/IQueueRunnerService.cs @@ -99,10 +99,32 @@ public interface IQueueRunnerService /// A list of tuples containing the task ID and backup ID IList> GetQueueWithIds(); + /// + /// Cancels the database lock wait for the currently running task, if its ID matches + /// and it is still waiting to acquire the database lock. + /// If the task has already acquired the lock and is executing, this has no effect — + /// call or + /// to interrupt a running task. This method is intended to be called *before* + /// / so that + /// a task blocked on lock acquisition is unblocked rather than left stuck. + /// + /// The ID of the currently running task whose lock wait should be cancelled. + void CancelCurrentTaskLockWait(long taskID); + /// /// Runs a task immediately, bypassing the queue. - /// Note that the task will run concurrently with the queue tasks and may cause database lock issues. + /// If the task's backup database is already in use by a queued task (e.g. a running backup), + /// this method throws immediately + /// rather than waiting or causing a concurrent "database is locked" error. + /// + /// Only tasks that expose a database path via + /// (i.e. -backed tasks) are protected by the + /// database lock. In production all tasks are + /// instances; non-runner implementations (test mocks) return a + /// null path and bypass locking. + /// /// /// The task to run + /// Thrown when the backup database is already in use. Task RunImmediatelyAsync(IQueuedTask task); -} \ No newline at end of file +} diff --git a/Duplicati/Library/RestAPI/Runner.cs b/Duplicati/Library/RestAPI/Runner.cs index b0fedf6c0..67853c0be 100644 --- a/Duplicati/Library/RestAPI/Runner.cs +++ b/Duplicati/Library/RestAPI/Runner.cs @@ -1422,6 +1422,49 @@ namespace Duplicati.Server options["remote-sync-json-config"] = JsonSerializer.Serialize(config, jsonOptions); } + /// + /// Returns the effective local database path for an instance. + /// In production, all queued tasks are instances created by the + /// Runner.Create* factories, so this delegates to + /// . If the task is not an + /// (only possible in tests with mock + /// implementations), the result is null. + /// + /// The effective path, or null + public static string? GetEffectiveDBPath(IQueuedTask? runnerData) + => GetEffectiveDBPath(runnerData as IRunnerData); + + /// + /// Returns the effective local database path for a instance. + /// The precedence mirrors : the dbpath extra option takes + /// priority, then the backup's --dbpath advanced setting, then the stored + /// . If no-local-db is set + /// (either as an extra option or as a backup advanced setting), the result is null. + /// + /// The effective path, or null + public static string? GetEffectiveDBPath(IRunnerData? runnerData) + { + if (runnerData == null) + return null; + + if (runnerData.ExtraOptions != null) + { + if (runnerData.ExtraOptions.ContainsKey("no-local-db") && Utility.ParseBoolOption(runnerData.ExtraOptions.AsReadOnly(), "no-local-db")) + return null; + if (runnerData.ExtraOptions.TryGetValue("dbpath", out var dbpath) && !string.IsNullOrWhiteSpace(dbpath)) + return dbpath; + } + + if (runnerData.Backup == null) + return null; + + var setting = runnerData.Backup.Settings?.FirstOrDefault(s => s.Name.Equals($"--no-local-db", StringComparison.OrdinalIgnoreCase)); + if (setting != null && Utility.ParseBool(setting.Value, true)) + return null; + + return GetEffectiveDBPath(runnerData.Backup); + } + /// /// Returns the effective local database path for a backup: the "--dbpath" advanced option if /// it is set, otherwise the stored . This diff --git a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs index 58d3fd7c8..2898de067 100644 --- a/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs +++ b/Duplicati/Library/SQLiteHelper/SQLiteLoader.cs @@ -42,6 +42,13 @@ namespace Duplicati.Library.SQLiteHelper /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(SQLiteLoader)); + /// + /// The default busy timeout (in milliseconds) applied to every connection opened by + /// via the connection string. This is a defense-in-depth + /// measure that ensures an active reader does not cause failed commits. + /// + private const int DefaultBusyTimeoutMs = 10000; + /// /// Helper method with logic to handle opening a database in possibly encrypted format. /// @@ -281,6 +288,12 @@ namespace Duplicati.Library.SQLiteHelper con.ConnectionString = $"Data Source={path};Pooling=false"; await con.OpenAsync().ConfigureAwait(false); + using (var cmd = con.CreateCommand()) + { + cmd.CommandText = $"PRAGMA busy_timeout={DefaultBusyTimeoutMs}"; + await cmd.ExecuteNonQueryAsync().ConfigureAwait(false); + } + // Make the file only accessible by the current user, unless opting out if (!SystemIO.IO_OS.FileExists(SystemIO.IO_OS.PathCombine(SystemIO.IO_OS.PathGetDirectoryName(path), Util.InsecurePermissionsMarkerFile))) try { SystemIO.IO_OS.FileSetPermissionUserRWOnly(path); } diff --git a/Duplicati/UnitTest/DatabaseLockTrackerTests.cs b/Duplicati/UnitTest/DatabaseLockTrackerTests.cs new file mode 100644 index 000000000..7f403a48b --- /dev/null +++ b/Duplicati/UnitTest/DatabaseLockTrackerTests.cs @@ -0,0 +1,225 @@ +// Copyright (C) 2026, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +#nullable enable + +using System; +using System.Threading; +using System.Threading.Tasks; +using Duplicati.WebserverCore.Services; +using NUnit.Framework; +using Assert = NUnit.Framework.Legacy.ClassicAssert; + +namespace Duplicati.UnitTest +{ + /// + /// Tests for , which coordinates access to local + /// backup database files between queued and immediate operations. + /// + [TestFixture] + public class DatabaseLockTrackerTests + { + /// + /// A simple value-returning async disposable used in tests. + /// + private static async Task DisposeAsync(IAsyncDisposable disposable) + => await disposable.DisposeAsync().ConfigureAwait(false); + + [Test] + public async Task TryAcquireReturnsLockWhenFree() + { + var tracker = new DatabaseLockTracker(); + var handle = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNotNull(handle, "TryAcquire should succeed when the database is not locked"); + await DisposeAsync(handle!); + } + + [Test] + public async Task TryAcquireReturnsNullWhenAlreadyLocked() + { + var tracker = new DatabaseLockTracker(); + var first = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNotNull(first, "First TryAcquire should succeed"); + + var second = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNull(second, "Second TryAcquire should fail when the database is locked"); + + await DisposeAsync(first!); + } + + [Test] + public async Task AcquireAsyncWaitsForExistingLock() + { + var tracker = new DatabaseLockTracker(); + var first = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNotNull(first); + + var acquireTask = tracker.AcquireAsync("/tmp/test.sqlite", CancellationToken.None); + Assert.IsFalse(acquireTask.IsCompleted, "AcquireAsync should be waiting for the lock"); + + await DisposeAsync(first!); + var second = await acquireTask; + Assert.IsNotNull(second, "AcquireAsync should complete after the lock is released"); + await DisposeAsync(second); + } + + [Test] + public async Task ReleaseAllowsSubsequentAcquire() + { + var tracker = new DatabaseLockTracker(); + var first = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNotNull(first); + await DisposeAsync(first!); + + var second = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNotNull(second, "TryAcquire should succeed after the lock is released"); + await DisposeAsync(second!); + } + + [Test] + public async Task DifferentPathsAreIndependent() + { + var tracker = new DatabaseLockTracker(); + var lockA = tracker.TryAcquire("/tmp/a.sqlite"); + Assert.IsNotNull(lockA); + + var lockB = tracker.TryAcquire("/tmp/b.sqlite"); + Assert.IsNotNull(lockB, "TryAcquire on a different path should succeed independently"); + + await DisposeAsync(lockA!); + await DisposeAsync(lockB!); + } + + [Test] + public async Task AcquireAsyncCancelsWhenTokenFires() + { + var tracker = new DatabaseLockTracker(); + var first = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNotNull(first); + + using var cts = new CancellationTokenSource(); + var acquireTask = tracker.AcquireAsync("/tmp/test.sqlite", cts.Token); + Assert.IsFalse(acquireTask.IsCompleted); + + cts.Cancel(); + + Assert.ThrowsAsync(async () => await acquireTask); + + await DisposeAsync(first!); + } + + [Test] + public async Task DoubleDisposeIsSafe() + { + var tracker = new DatabaseLockTracker(); + var handle = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNotNull(handle); + + await DisposeAsync(handle!); + await DisposeAsync(handle!); + + var next = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNotNull(next, "After double-dispose, the lock should be releasable exactly once"); + await DisposeAsync(next!); + } + + [Test] + public async Task TryAcquireOnLockedReleasesCorrectly() + { + var tracker = new DatabaseLockTracker(); + var first = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNotNull(first); + + var second = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNull(second); + + await DisposeAsync(first!); + + var third = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNotNull(third, "After releasing the first lock, TryAcquire should succeed"); + await DisposeAsync(third!); + } + + [Test] + public async Task RelativeAndAbsolutePathMapToSameLock() + { + var tracker = new DatabaseLockTracker(); + var absolute = tracker.TryAcquire(System.IO.Path.GetFullPath("/tmp/test.sqlite")); + Assert.IsNotNull(absolute); + + var relative = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNull(relative, "Path normalization should map relative and absolute paths to the same lock"); + + await DisposeAsync(absolute!); + } + + [Test] + public async Task AcquireAsyncPreservesOrder() + { + var tracker = new DatabaseLockTracker(); + var first = tracker.TryAcquire("/tmp/test.sqlite"); + Assert.IsNotNull(first); + + var order = new System.Collections.Generic.List(); + + var t2 = Task.Run(async () => + { + await using var h = await tracker.AcquireAsync("/tmp/test.sqlite", CancellationToken.None); + order.Add(2); + }); + + var t3 = Task.Run(async () => + { + await using var h = await tracker.AcquireAsync("/tmp/test.sqlite", CancellationToken.None); + order.Add(3); + }); + + await Task.Delay(50); + order.Add(1); + await DisposeAsync(first!); + + await Task.WhenAll(t2, t3); + + Assert.AreEqual(1, order[0], "First lock holder should complete first"); + } + + [Test] + public void TryAcquireEmptyPathThrows() + { + var tracker = new DatabaseLockTracker(); + Assert.Throws(() => tracker.TryAcquire("")); + } + + [Test] + public void TryAcquireWhitespacePathThrows() + { + var tracker = new DatabaseLockTracker(); + Assert.Throws(() => tracker.TryAcquire(" ")); + } + + [Test] + public void AcquireAsyncEmptyPathThrows() + { + var tracker = new DatabaseLockTracker(); + Assert.ThrowsAsync(async () => await tracker.AcquireAsync("", CancellationToken.None)); + } + } +} diff --git a/Duplicati/UnitTest/FolderStatusServiceTests.cs b/Duplicati/UnitTest/FolderStatusServiceTests.cs index ec461c954..b40db298c 100644 --- a/Duplicati/UnitTest/FolderStatusServiceTests.cs +++ b/Duplicati/UnitTest/FolderStatusServiceTests.cs @@ -62,6 +62,7 @@ public class FolderStatusServiceTests public void Resume() { } public void Pause() { } public IList> GetQueueWithIds() => new List>(); + public void CancelCurrentTaskLockWait(long taskID) { } public Task RunImmediatelyAsync(IQueuedTask task) => Task.FromResult(null); } diff --git a/Duplicati/UnitTest/Issue1698.cs b/Duplicati/UnitTest/Issue1698.cs index bb571e04f..307807472 100644 --- a/Duplicati/UnitTest/Issue1698.cs +++ b/Duplicati/UnitTest/Issue1698.cs @@ -24,6 +24,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using Duplicati.Server.Database; using Duplicati.Server.Serialization.Interface; using NUnit.Framework; @@ -87,5 +88,100 @@ namespace Duplicati.UnitTest var backup = CreateBackup("/stored/path.sqlite", ("--dbpath", " ")); Assert.AreEqual("/stored/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(backup)); } + + // ---- Tests for GetEffectiveDBPath(IQueuedTask) / GetEffectiveDBPath(IRunnerData) ---- + + private static Duplicati.Server.Runner.IRunnerData CreateRunnerTask( + Backup backup, + IDictionary? extraOptions = null) + => Duplicati.Server.Runner.CreateTask( + Duplicati.Server.Serialization.DuplicatiOperation.BackupOrSync, + backup, + extraOptions ?? new Dictionary()); + + [Test] + public void RunnerDataFallsBackToBackupDbPath() + { + var backup = CreateBackup("/stored/path.sqlite"); + var task = CreateRunnerTask(backup); + Assert.AreEqual("/stored/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task)); + } + + [Test] + public void RunnerDataRespectsBackupAdvancedDbPath() + { + var backup = CreateBackup("/stored/path.sqlite", ("--dbpath", "/override/path.sqlite")); + var task = CreateRunnerTask(backup); + Assert.AreEqual("/override/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task)); + } + + [Test] + public void RunnerDataExtraDbPathOverridesBackup() + { + var backup = CreateBackup("/stored/path.sqlite", ("--dbpath", "/backup-override.sqlite")); + var task = CreateRunnerTask(backup, new Dictionary { ["dbpath"] = "/extra-override.sqlite" }); + Assert.AreEqual("/extra-override.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task)); + } + + [Test] + public void RunnerDataExtraNoLocalDbReturnsNull() + { + var backup = CreateBackup("/stored/path.sqlite"); + var task = CreateRunnerTask(backup, new Dictionary { ["no-local-db"] = "true" }); + Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath(task)); + } + + [Test] + public void RunnerDataBackupNoLocalDbSettingReturnsNull() + { + var backup = CreateBackup("/stored/path.sqlite", ("--no-local-db", "true")); + var task = CreateRunnerTask(backup); + Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath(task)); + } + + [Test] + public void RunnerDataExtraNoLocalDbFalseReturnsDbPath() + { + var backup = CreateBackup("/stored/path.sqlite"); + var task = CreateRunnerTask(backup, new Dictionary { ["no-local-db"] = "false" }); + Assert.AreEqual("/stored/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task)); + } + + [Test] + public void RunnerDataNullReturnsNull() + { + Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath((Duplicati.Server.Runner.IRunnerData?)null)); + Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath((Duplicati.Server.Serialization.Interface.IQueuedTask?)null)); + } + + [Test] + public void RunnerDataNonRunnerDataQueuedTaskReturnsNull() + { + // A non-IRunnerData IQueuedTask implementation should cast to null and return null. + var nonRunnerTask = new NonRunnerQueuedTask(); + Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath(nonRunnerTask)); + } + + /// + /// A minimal implementation that is NOT an + /// , used to verify that + /// safely returns + /// null for non-runner tasks. + /// + private sealed class NonRunnerQueuedTask : Duplicati.Server.Serialization.Interface.IQueuedTask + { + public long TaskID => 0; + public string? BackupID => null; + public Duplicati.Server.Serialization.DuplicatiOperation Operation => Duplicati.Server.Serialization.DuplicatiOperation.BackupOrSync; + public Func? OnStarting { get; set; } + public Func? OnFinished { get; set; } + public DateTime? TaskStarted { get; set; } + public DateTime? TaskFinished { get; set; } + public Task UpdateThrottleSpeedsAsync(string? uploadSpeed, string? downloadSpeed) => Task.CompletedTask; + public Task StopAsync() => Task.CompletedTask; + public Task AbortAsync() => Task.CompletedTask; + public Task PauseAsync(bool alsoTransfers) => Task.CompletedTask; + public Task ResumeAsync() => Task.CompletedTask; + } } } diff --git a/Duplicati/WebserverCore/Abstractions/IDatabaseLockTracker.cs b/Duplicati/WebserverCore/Abstractions/IDatabaseLockTracker.cs new file mode 100644 index 000000000..4637b1d63 --- /dev/null +++ b/Duplicati/WebserverCore/Abstractions/IDatabaseLockTracker.cs @@ -0,0 +1,48 @@ +// Copyright (C) 2026, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +namespace Duplicati.WebserverCore.Abstractions; + +/// +/// Tracks which local backup database files are currently in use by running operations, +/// preventing concurrent access that would cause SQLite "database is locked" errors. +/// +public interface IDatabaseLockTracker +{ + /// + /// Attempts to acquire a lock on the given database path, waiting until the lock + /// is available. Used by queued tasks (backup, restore, etc.) which should wait + /// for any concurrent immediate operations to finish. + /// + /// The normalized, full path to the database file. + /// A cancellation token to cancel the wait. + /// A disposable handle that releases the lock when disposed. + Task AcquireAsync(string dbPath, CancellationToken cancellationToken); + + /// + /// Attempts to acquire a lock on the given database path, failing immediately if + /// the database is already in use. Used by immediate (non-queued) operations such + /// as listing filesets from the restore UI. + /// + /// The normalized, full path to the database file. + /// A disposable handle that releases the lock when disposed, or null if the database is already locked. + IAsyncDisposable? TryAcquire(string dbPath); +} diff --git a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs index 3228fba0e..29c167110 100644 --- a/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs +++ b/Duplicati/WebserverCore/Endpoints/V1/Backup/BackupGet.cs @@ -45,12 +45,12 @@ public class BackupGet : IEndpointV1 => ExecuteGetFilesAsync(queueRunnerService, GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary())) .RequireAuthorization(); - group.MapGet("/backup/{id}/log", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize) - => ExecuteGetLog(connection, GetBackup(connection, id), offset, pagesize ?? 100)) + group.MapGet("/backup/{id}/log", ([FromServices] Connection connection, [FromServices] IDatabaseLockTracker databaseLockTracker, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize) + => ExecuteGetLog(connection, databaseLockTracker, GetBackup(connection, id), offset, pagesize ?? 100)) .RequireAuthorization(); - group.MapGet("/backup/{id}/remotelog", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize) - => ExecuteGetRemotelog(connection, GetBackup(connection, id), offset, pagesize ?? 100)) + group.MapGet("/backup/{id}/remotelog", ([FromServices] Connection connection, [FromServices] IDatabaseLockTracker databaseLockTracker, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize) + => ExecuteGetRemotelog(connection, databaseLockTracker, GetBackup(connection, id), offset, pagesize ?? 100)) .RequireAuthorization(); group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly) @@ -185,7 +185,7 @@ public class BackupGet : IEndpointV1 private static async Task> ExecuteGetFilesAsync(IQueueRunnerService queueRunnerService, IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary extraValues) => await SearchFilesAsync(queueRunnerService, bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues).ConfigureAwait(false); - private static List> ExecuteGetLog(Connection connection, IBackup bk, long? offset, long pagesize) + private static async Task>> ExecuteGetLog(Connection connection, IDatabaseLockTracker databaseLockTracker, IBackup bk, long? offset, long pagesize) { // Use the effective database path (honoring a "--dbpath" advanced option) so that the log is // read from the same database the backup actually uses (see issue #1698). @@ -193,28 +193,48 @@ public class BackupGet : IEndpointV1 if (!File.Exists(dbpath)) return new List>(); - using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath)) - using (var cmd = con.CreateCommand()) - return LogData.DumpTable(cmd, "LogData", "ID", offset, pagesize); + var dbLock = databaseLockTracker.TryAcquire(dbpath) + ?? throw new DatabaseLockedException(dbpath); + + try + { + using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath)) + using (var cmd = con.CreateCommand()) + return LogData.DumpTable(cmd, "LogData", "ID", offset, pagesize); + } + finally + { + await dbLock.DisposeAsync().ConfigureAwait(false); + } } - private static List> ExecuteGetRemotelog(Connection connection, IBackup bk, long? offset, long pagesize) + private static async Task>> ExecuteGetRemotelog(Connection connection, IDatabaseLockTracker databaseLockTracker, IBackup bk, long? offset, long pagesize) { var dbpath = Runner.GetEffectiveDBPath(bk); if (!File.Exists(dbpath)) return new List>(); - using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath)) - using (var cmd = con.CreateCommand()) + var dbLock = databaseLockTracker.TryAcquire(dbpath) + ?? throw new DatabaseLockedException(dbpath); + + try { - var dt = LogData.DumpTable(cmd, "RemoteOperation", "ID", offset, pagesize); + using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath)) + using (var cmd = con.CreateCommand()) + { + var dt = LogData.DumpTable(cmd, "RemoteOperation", "ID", offset, pagesize); - // Unwrap raw data to a string - foreach (var n in dt) - try { n["Data"] = System.Text.Encoding.UTF8.GetString((byte[])n["Data"]); } - catch { } + // Unwrap raw data to a string + foreach (var n in dt) + try { n["Data"] = System.Text.Encoding.UTF8.GetString((byte[])n["Data"]); } + catch { } - return dt; + return dt; + } + } + finally + { + await dbLock.DisposeAsync().ConfigureAwait(false); } } diff --git a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs index b10255e9f..5ade6e3ef 100644 --- a/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs +++ b/Duplicati/WebserverCore/Extensions/ServiceCollectionsExtensions.cs @@ -67,6 +67,7 @@ public static class ServiceCollectionsExtensions .AddSingleton() .AddSingleton() .AddSingleton() + .AddSingleton() .AddSingleton() .AddTransient() .AddTransient() diff --git a/Duplicati/WebserverCore/Services/DatabaseLockTracker.cs b/Duplicati/WebserverCore/Services/DatabaseLockTracker.cs new file mode 100644 index 000000000..935bb3c3b --- /dev/null +++ b/Duplicati/WebserverCore/Services/DatabaseLockTracker.cs @@ -0,0 +1,103 @@ +// Copyright (C) 2026, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or +// sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +using System.Collections.Concurrent; +using Duplicati.WebserverCore.Abstractions; + +namespace Duplicati.WebserverCore.Services; + +/// +/// Tracks which local backup database files are currently in use by running operations, +/// preventing concurrent access that would cause SQLite "database is locked" errors. +/// +/// +/// Each database path gets its own (initial count 1). +/// Queued tasks call which waits for the lock. +/// Immediate tasks call which fails instantly if locked. +/// +/// Semaphore entries are intentionally not removed on release. Removing them is racy: a +/// concurrent acquirer could take the lock between a release and a dictionary removal, +/// after which a new acquirer would create a fresh semaphore for the same path, breaking +/// mutual exclusion. The dictionary is bounded by the number of unique database paths +/// (one per backup), which is a small, long-lived set, so the lack of cleanup does not +/// cause unbounded growth in practice. +/// +/// +public class DatabaseLockTracker : IDatabaseLockTracker +{ + private const int MaxLockCount = 1; + + private readonly ConcurrentDictionary _locks = new(Library.Utility.Utility.ClientFilenameStringComparer); + + /// + public async Task AcquireAsync(string dbPath, CancellationToken cancellationToken) + { + var semaphore = GetOrCreateSemaphore(dbPath); + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + return new LockRelease(semaphore); + } + + /// + public IAsyncDisposable? TryAcquire(string dbPath) + { + var semaphore = GetOrCreateSemaphore(dbPath); + if (!semaphore.Wait(0)) + return null; + return new LockRelease(semaphore); + } + + /// + /// Normalizes the database path and gets (or creates) the semaphore for it. + /// + private SemaphoreSlim GetOrCreateSemaphore(string dbPath) + { + var normalized = NormalizePath(dbPath); + if (string.IsNullOrEmpty(normalized)) + throw new ArgumentException("Database path cannot be null or empty", nameof(dbPath)); + return _locks.GetOrAdd(normalized, _ => new SemaphoreSlim(MaxLockCount, MaxLockCount)); + } + + /// + /// Normalizes a file path so that the same database file always maps to the same key, + /// regardless of path separators or relative vs. absolute forms. + /// + private static string NormalizePath(string dbPath) + { + if (string.IsNullOrWhiteSpace(dbPath)) + return string.Empty; + return Path.GetFullPath(dbPath); + } + + /// + /// A disposable handle that releases the semaphore when disposed. + /// + private sealed class LockRelease(SemaphoreSlim semaphore) : IAsyncDisposable + { + private int _released; + + public ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _released, 1) == 0) + semaphore.Release(); + return ValueTask.CompletedTask; + } + } +} diff --git a/Duplicati/WebserverCore/Services/QueueRunnerService.cs b/Duplicati/WebserverCore/Services/QueueRunnerService.cs index 7d4b860d2..b89b23272 100644 --- a/Duplicati/WebserverCore/Services/QueueRunnerService.cs +++ b/Duplicati/WebserverCore/Services/QueueRunnerService.cs @@ -37,7 +37,8 @@ public class QueueRunnerService( EventPollNotify eventPollNotify, INotificationUpdateService notificationUpdateService, IProgressStateProviderService progressStateProviderService, - IApplicationSettings applicationSettings) : IQueueRunnerService + IApplicationSettings applicationSettings, + IDatabaseLockTracker databaseLockTracker) : IQueueRunnerService, IDisposable { private readonly object _lock = new(); /// @@ -51,9 +52,10 @@ public class QueueRunnerService( private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100; private readonly List _tasks = new(); - private (Task? Task, IQueuedTask? QueuedTask) _current; + private (Task? Task, IQueuedTask? QueuedTask, CancellationTokenSource? LockCts) _current; private bool _isPaused; private bool _isTerminated; + private readonly CancellationTokenSource _terminateCts = new(); public long AddTask(IQueuedTask task) => AddTask(task, false); @@ -101,6 +103,7 @@ public class QueueRunnerService( public void Terminate(bool wait) { _isTerminated = true; + _terminateCts.Cancel(); if (wait) { var task = _current.Task; @@ -118,20 +121,27 @@ public class QueueRunnerService( // Clean up completed tasks if (_current.Task != null && _current.Task.IsCompleted) - _current = (null, null); + { + _current.LockCts?.Dispose(); + _current = (null, null, null); + } if (_tasks.Count == 0) return; var nextTask = _tasks[0]; _tasks.RemoveAt(0); - _current = (Task.Run(() => RunTaskAsync(nextTask), CancellationToken.None), nextTask); + _current = (Task.Run(() => RunTaskAsync(nextTask), CancellationToken.None), nextTask, null); } } private async Task RunTaskAsync(IQueuedTask task) { var completed = false; + IAsyncDisposable? dbLock = null; + var lockCts = CancellationTokenSource.CreateLinkedTokenSource(_terminateCts.Token); + lock (_lock) + _current = (_current.Task, _current.QueuedTask, lockCts); try { eventPollNotify.SignalNewEvent(); @@ -140,6 +150,8 @@ public class QueueRunnerService( if (task.OnStarting != null) await task.OnStarting().ConfigureAwait(false); + dbLock = await AcquireDatabaseLockAsync(task, lockCts.Token).ConfigureAwait(false); + await Runner.RunAsync(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, true).ConfigureAwait(false); // If the task is completed, don't call OnFinished again @@ -160,9 +172,14 @@ public class QueueRunnerService( } finally { + if (dbLock != null) + await dbLock.DisposeAsync().ConfigureAwait(false); task.TaskFinished = DateTime.UtcNow; lock (_lock) - _current = (null, null); + { + lockCts.Dispose(); + _current = (null, null, null); + } eventPollNotify.SignalNewEvent(); eventPollNotify.SignalTaskQueueUpdate(); eventPollNotify.SignalTaskCompleted(task.TaskID); @@ -215,6 +232,55 @@ public class QueueRunnerService( /// public async Task RunImmediatelyAsync(IQueuedTask task) { - return await Runner.RunAsync(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, false).ConfigureAwait(false); + IAsyncDisposable? dbLock = null; + var dbPath = Runner.GetEffectiveDBPath(task); + + // If we have a database to work with, make sure we can get the lock + if (!string.IsNullOrWhiteSpace(dbPath)) + dbLock = databaseLockTracker.TryAcquire(dbPath) + ?? throw new DatabaseLockedException(dbPath); + + try + { + return await Runner.RunAsync(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, false).ConfigureAwait(false); + } + finally + { + if (dbLock != null) + await dbLock.DisposeAsync().ConfigureAwait(false); + } + } + + /// + public void CancelCurrentTaskLockWait(long taskID) + { + lock (_lock) + { + if (_current.QueuedTask?.TaskID == taskID && _current.LockCts != null) + _current.LockCts.Cancel(); + } + } + + /// + /// Acquires the database lock for a queued task, waiting until the lock is available + /// or the cancellation token is cancelled. Tasks without a database path (e.g. custom + /// runner tasks) skip locking. + /// + private async Task AcquireDatabaseLockAsync(IQueuedTask task, CancellationToken cancellationToken) + { + var dbPath = Runner.GetEffectiveDBPath(task); + if (string.IsNullOrWhiteSpace(dbPath)) + return null; + + return await databaseLockTracker.AcquireAsync(dbPath, cancellationToken).ConfigureAwait(false); + } + + /// + /// Disposes resources held by the queue runner, including the termination + /// cancellation token source used to cancel tasks waiting on the database lock. + /// + public void Dispose() + { + _terminateCts.Dispose(); } } \ No newline at end of file diff --git a/Duplicati/WebserverCore/Services/TaskQueueService.cs b/Duplicati/WebserverCore/Services/TaskQueueService.cs index 3616ea2a2..3f36d0cb4 100644 --- a/Duplicati/WebserverCore/Services/TaskQueueService.cs +++ b/Duplicati/WebserverCore/Services/TaskQueueService.cs @@ -108,6 +108,8 @@ public class TaskQueueService(IQueueRunnerService queueRunnerService) : ITaskQue if (task == null) throw new NotFoundException("No such task found"); + queueRunnerService.CancelCurrentTaskLockWait(task.TaskID); + if (abort) await task.AbortAsync().ConfigureAwait(false); else