Merge pull request #7035 from duplicati/feature/server-db-lock
Server database lock
This commit is contained in:
@@ -273,4 +273,35 @@ namespace Duplicati.Library.Interface
|
||||
: base(message, helpId, innerException)
|
||||
{ }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception indicating that the backup database is currently locked by another running operation
|
||||
/// and cannot be accessed concurrently.
|
||||
/// </summary>
|
||||
[Serializable]
|
||||
public class DatabaseLockedException : UserInformationException
|
||||
{
|
||||
/// <summary>
|
||||
/// The path to the database file that is locked.
|
||||
/// </summary>
|
||||
public readonly string DatabasePath;
|
||||
|
||||
public DatabaseLockedException(string databasePath)
|
||||
: base(LC.L($"The backup database is currently in use by another operation and cannot be accessed concurrently. Please wait for the running operation to finish and try again."), "DatabaseLocked")
|
||||
{
|
||||
DatabasePath = databasePath;
|
||||
}
|
||||
|
||||
public DatabaseLockedException(string databasePath, Exception innerException)
|
||||
: base(LC.L($"The backup database is currently in use by another operation and cannot be accessed concurrently. Please wait for the running operation to finish and try again."), "DatabaseLocked", innerException)
|
||||
{
|
||||
DatabasePath = databasePath;
|
||||
}
|
||||
|
||||
public DatabaseLockedException(string message, string helpId, Exception innerException)
|
||||
: base(message, helpId, innerException)
|
||||
{
|
||||
DatabasePath = string.Empty;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,10 +99,32 @@ public interface IQueueRunnerService
|
||||
/// <returns>A list of tuples containing the task ID and backup ID</returns>
|
||||
IList<Tuple<long, string?>> GetQueueWithIds();
|
||||
|
||||
/// <summary>
|
||||
/// Cancels the database lock wait for the currently running task, if its ID matches
|
||||
/// <paramref name="taskID"/> and it is still waiting to acquire the database lock.
|
||||
/// If the task has already acquired the lock and is executing, this has no effect —
|
||||
/// call <see cref="IQueuedTask.AbortAsync"/> or <see cref="IQueuedTask.StopAsync"/>
|
||||
/// to interrupt a running task. This method is intended to be called *before*
|
||||
/// <see cref="IQueuedTask.AbortAsync"/>/<see cref="IQueuedTask.StopAsync"/> so that
|
||||
/// a task blocked on lock acquisition is unblocked rather than left stuck.
|
||||
/// </summary>
|
||||
/// <param name="taskID">The ID of the currently running task whose lock wait should be cancelled.</param>
|
||||
void CancelCurrentTaskLockWait(long taskID);
|
||||
|
||||
/// <summary>
|
||||
/// Runs a task immediately, bypassing the queue.
|
||||
/// Note that the task will run concurrently with the queue tasks and may cause database lock issues.
|
||||
/// If the task's backup database is already in use by a queued task (e.g. a running backup),
|
||||
/// this method throws <see cref="Duplicati.Library.Interface.DatabaseLockedException"/> immediately
|
||||
/// rather than waiting or causing a concurrent "database is locked" error.
|
||||
/// <para>
|
||||
/// Only tasks that expose a database path via <see cref="Duplicati.Library.RestAPI.Runner.GetEffectiveDBPath"/>
|
||||
/// (i.e. <see cref="Duplicati.Library.RestAPI.IRunnerData"/>-backed tasks) are protected by the
|
||||
/// database lock. In production all tasks are <see cref="Duplicati.Library.RestAPI.IRunnerData"/>
|
||||
/// instances; non-runner <see cref="IQueuedTask"/> implementations (test mocks) return a
|
||||
/// <c>null</c> path and bypass locking.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
/// <param name="task">The task to run</param>
|
||||
/// <exception cref="Duplicati.Library.Interface.DatabaseLockedException">Thrown when the backup database is already in use.</exception>
|
||||
Task<IBasicResults?> RunImmediatelyAsync(IQueuedTask task);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1422,6 +1422,49 @@ namespace Duplicati.Server
|
||||
options["remote-sync-json-config"] = JsonSerializer.Serialize(config, jsonOptions);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the effective local database path for an <see cref="IQueuedTask"/> instance.
|
||||
/// In production, all queued tasks are <see cref="IRunnerData"/> instances created by the
|
||||
/// <c>Runner.Create*</c> factories, so this delegates to
|
||||
/// <see cref="GetEffectiveDBPath(IRunnerData?)"/>. If the task is not an
|
||||
/// <see cref="IRunnerData"/> (only possible in tests with mock <see cref="IQueuedTask"/>
|
||||
/// implementations), the result is <c>null</c>.
|
||||
/// </summary>
|
||||
/// <returns>The effective path, or <c>null</c></returns>
|
||||
public static string? GetEffectiveDBPath(IQueuedTask? runnerData)
|
||||
=> GetEffectiveDBPath(runnerData as IRunnerData);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the effective local database path for a <see cref="IRunnerData"/> instance.
|
||||
/// The precedence mirrors <see cref="ApplyOptions"/>: the <c>dbpath</c> extra option takes
|
||||
/// priority, then the backup's <c>--dbpath</c> advanced setting, then the stored
|
||||
/// <see cref="Serialization.Interface.IBackup.DBPath"/>. If <c>no-local-db</c> is set
|
||||
/// (either as an extra option or as a backup advanced setting), the result is <c>null</c>.
|
||||
/// </summary>
|
||||
/// <returns>The effective path, or <c>null</c></returns>
|
||||
public static string? GetEffectiveDBPath(IRunnerData? runnerData)
|
||||
{
|
||||
if (runnerData == null)
|
||||
return null;
|
||||
|
||||
if (runnerData.ExtraOptions != null)
|
||||
{
|
||||
if (runnerData.ExtraOptions.ContainsKey("no-local-db") && Utility.ParseBoolOption(runnerData.ExtraOptions.AsReadOnly(), "no-local-db"))
|
||||
return null;
|
||||
if (runnerData.ExtraOptions.TryGetValue("dbpath", out var dbpath) && !string.IsNullOrWhiteSpace(dbpath))
|
||||
return dbpath;
|
||||
}
|
||||
|
||||
if (runnerData.Backup == null)
|
||||
return null;
|
||||
|
||||
var setting = runnerData.Backup.Settings?.FirstOrDefault(s => s.Name.Equals($"--no-local-db", StringComparison.OrdinalIgnoreCase));
|
||||
if (setting != null && Utility.ParseBool(setting.Value, true))
|
||||
return null;
|
||||
|
||||
return GetEffectiveDBPath(runnerData.Backup);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the effective local database path for a backup: the "--dbpath" advanced option if
|
||||
/// it is set, otherwise the stored <see cref="Serialization.Interface.IBackup.DBPath"/>. This
|
||||
|
||||
@@ -42,6 +42,13 @@ namespace Duplicati.Library.SQLiteHelper
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(SQLiteLoader));
|
||||
|
||||
/// <summary>
|
||||
/// The default busy timeout (in milliseconds) applied to every connection opened by
|
||||
/// <see cref="OpenSQLiteFileAsync"/> via the connection string. This is a defense-in-depth
|
||||
/// measure that ensures an active reader does not cause failed commits.
|
||||
/// </summary>
|
||||
private const int DefaultBusyTimeoutMs = 10000;
|
||||
|
||||
/// <summary>
|
||||
/// Helper method with logic to handle opening a database in possibly encrypted format.
|
||||
/// </summary>
|
||||
@@ -281,6 +288,12 @@ namespace Duplicati.Library.SQLiteHelper
|
||||
con.ConnectionString = $"Data Source={path};Pooling=false";
|
||||
await con.OpenAsync().ConfigureAwait(false);
|
||||
|
||||
using (var cmd = con.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = $"PRAGMA busy_timeout={DefaultBusyTimeoutMs}";
|
||||
await cmd.ExecuteNonQueryAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Make the file only accessible by the current user, unless opting out
|
||||
if (!SystemIO.IO_OS.FileExists(SystemIO.IO_OS.PathCombine(SystemIO.IO_OS.PathGetDirectoryName(path), Util.InsecurePermissionsMarkerFile)))
|
||||
try { SystemIO.IO_OS.FileSetPermissionUserRWOnly(path); }
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
// Copyright (C) 2026, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
// sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.WebserverCore.Services;
|
||||
using NUnit.Framework;
|
||||
using Assert = NUnit.Framework.Legacy.ClassicAssert;
|
||||
|
||||
namespace Duplicati.UnitTest
|
||||
{
|
||||
/// <summary>
|
||||
/// Tests for <see cref="DatabaseLockTracker"/>, which coordinates access to local
|
||||
/// backup database files between queued and immediate operations.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public class DatabaseLockTrackerTests
|
||||
{
|
||||
/// <summary>
|
||||
/// A simple value-returning async disposable used in tests.
|
||||
/// </summary>
|
||||
private static async Task DisposeAsync(IAsyncDisposable disposable)
|
||||
=> await disposable.DisposeAsync().ConfigureAwait(false);
|
||||
|
||||
[Test]
|
||||
public async Task TryAcquireReturnsLockWhenFree()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
var handle = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNotNull(handle, "TryAcquire should succeed when the database is not locked");
|
||||
await DisposeAsync(handle!);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TryAcquireReturnsNullWhenAlreadyLocked()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
var first = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNotNull(first, "First TryAcquire should succeed");
|
||||
|
||||
var second = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNull(second, "Second TryAcquire should fail when the database is locked");
|
||||
|
||||
await DisposeAsync(first!);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AcquireAsyncWaitsForExistingLock()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
var first = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNotNull(first);
|
||||
|
||||
var acquireTask = tracker.AcquireAsync("/tmp/test.sqlite", CancellationToken.None);
|
||||
Assert.IsFalse(acquireTask.IsCompleted, "AcquireAsync should be waiting for the lock");
|
||||
|
||||
await DisposeAsync(first!);
|
||||
var second = await acquireTask;
|
||||
Assert.IsNotNull(second, "AcquireAsync should complete after the lock is released");
|
||||
await DisposeAsync(second);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ReleaseAllowsSubsequentAcquire()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
var first = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNotNull(first);
|
||||
await DisposeAsync(first!);
|
||||
|
||||
var second = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNotNull(second, "TryAcquire should succeed after the lock is released");
|
||||
await DisposeAsync(second!);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DifferentPathsAreIndependent()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
var lockA = tracker.TryAcquire("/tmp/a.sqlite");
|
||||
Assert.IsNotNull(lockA);
|
||||
|
||||
var lockB = tracker.TryAcquire("/tmp/b.sqlite");
|
||||
Assert.IsNotNull(lockB, "TryAcquire on a different path should succeed independently");
|
||||
|
||||
await DisposeAsync(lockA!);
|
||||
await DisposeAsync(lockB!);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AcquireAsyncCancelsWhenTokenFires()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
var first = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNotNull(first);
|
||||
|
||||
using var cts = new CancellationTokenSource();
|
||||
var acquireTask = tracker.AcquireAsync("/tmp/test.sqlite", cts.Token);
|
||||
Assert.IsFalse(acquireTask.IsCompleted);
|
||||
|
||||
cts.Cancel();
|
||||
|
||||
Assert.ThrowsAsync<System.OperationCanceledException>(async () => await acquireTask);
|
||||
|
||||
await DisposeAsync(first!);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DoubleDisposeIsSafe()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
var handle = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNotNull(handle);
|
||||
|
||||
await DisposeAsync(handle!);
|
||||
await DisposeAsync(handle!);
|
||||
|
||||
var next = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNotNull(next, "After double-dispose, the lock should be releasable exactly once");
|
||||
await DisposeAsync(next!);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task TryAcquireOnLockedReleasesCorrectly()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
var first = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNotNull(first);
|
||||
|
||||
var second = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNull(second);
|
||||
|
||||
await DisposeAsync(first!);
|
||||
|
||||
var third = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNotNull(third, "After releasing the first lock, TryAcquire should succeed");
|
||||
await DisposeAsync(third!);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RelativeAndAbsolutePathMapToSameLock()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
var absolute = tracker.TryAcquire(System.IO.Path.GetFullPath("/tmp/test.sqlite"));
|
||||
Assert.IsNotNull(absolute);
|
||||
|
||||
var relative = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNull(relative, "Path normalization should map relative and absolute paths to the same lock");
|
||||
|
||||
await DisposeAsync(absolute!);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AcquireAsyncPreservesOrder()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
var first = tracker.TryAcquire("/tmp/test.sqlite");
|
||||
Assert.IsNotNull(first);
|
||||
|
||||
var order = new System.Collections.Generic.List<int>();
|
||||
|
||||
var t2 = Task.Run(async () =>
|
||||
{
|
||||
await using var h = await tracker.AcquireAsync("/tmp/test.sqlite", CancellationToken.None);
|
||||
order.Add(2);
|
||||
});
|
||||
|
||||
var t3 = Task.Run(async () =>
|
||||
{
|
||||
await using var h = await tracker.AcquireAsync("/tmp/test.sqlite", CancellationToken.None);
|
||||
order.Add(3);
|
||||
});
|
||||
|
||||
await Task.Delay(50);
|
||||
order.Add(1);
|
||||
await DisposeAsync(first!);
|
||||
|
||||
await Task.WhenAll(t2, t3);
|
||||
|
||||
Assert.AreEqual(1, order[0], "First lock holder should complete first");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TryAcquireEmptyPathThrows()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
Assert.Throws<ArgumentException>(() => tracker.TryAcquire(""));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TryAcquireWhitespacePathThrows()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
Assert.Throws<ArgumentException>(() => tracker.TryAcquire(" "));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AcquireAsyncEmptyPathThrows()
|
||||
{
|
||||
var tracker = new DatabaseLockTracker();
|
||||
Assert.ThrowsAsync<ArgumentException>(async () => await tracker.AcquireAsync("", CancellationToken.None));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -62,6 +62,7 @@ public class FolderStatusServiceTests
|
||||
public void Resume() { }
|
||||
public void Pause() { }
|
||||
public IList<Tuple<long, string?>> GetQueueWithIds() => new List<Tuple<long, string?>>();
|
||||
public void CancelCurrentTaskLockWait(long taskID) { }
|
||||
public Task<IBasicResults?> RunImmediatelyAsync(IQueuedTask task) => Task.FromResult<IBasicResults?>(null);
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Server.Database;
|
||||
using Duplicati.Server.Serialization.Interface;
|
||||
using NUnit.Framework;
|
||||
@@ -87,5 +88,100 @@ namespace Duplicati.UnitTest
|
||||
var backup = CreateBackup("/stored/path.sqlite", ("--dbpath", " "));
|
||||
Assert.AreEqual("/stored/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(backup));
|
||||
}
|
||||
|
||||
// ---- Tests for GetEffectiveDBPath(IQueuedTask) / GetEffectiveDBPath(IRunnerData) ----
|
||||
|
||||
private static Duplicati.Server.Runner.IRunnerData CreateRunnerTask(
|
||||
Backup backup,
|
||||
IDictionary<string, string?>? extraOptions = null)
|
||||
=> Duplicati.Server.Runner.CreateTask(
|
||||
Duplicati.Server.Serialization.DuplicatiOperation.BackupOrSync,
|
||||
backup,
|
||||
extraOptions ?? new Dictionary<string, string?>());
|
||||
|
||||
[Test]
|
||||
public void RunnerDataFallsBackToBackupDbPath()
|
||||
{
|
||||
var backup = CreateBackup("/stored/path.sqlite");
|
||||
var task = CreateRunnerTask(backup);
|
||||
Assert.AreEqual("/stored/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RunnerDataRespectsBackupAdvancedDbPath()
|
||||
{
|
||||
var backup = CreateBackup("/stored/path.sqlite", ("--dbpath", "/override/path.sqlite"));
|
||||
var task = CreateRunnerTask(backup);
|
||||
Assert.AreEqual("/override/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RunnerDataExtraDbPathOverridesBackup()
|
||||
{
|
||||
var backup = CreateBackup("/stored/path.sqlite", ("--dbpath", "/backup-override.sqlite"));
|
||||
var task = CreateRunnerTask(backup, new Dictionary<string, string?> { ["dbpath"] = "/extra-override.sqlite" });
|
||||
Assert.AreEqual("/extra-override.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RunnerDataExtraNoLocalDbReturnsNull()
|
||||
{
|
||||
var backup = CreateBackup("/stored/path.sqlite");
|
||||
var task = CreateRunnerTask(backup, new Dictionary<string, string?> { ["no-local-db"] = "true" });
|
||||
Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath(task));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RunnerDataBackupNoLocalDbSettingReturnsNull()
|
||||
{
|
||||
var backup = CreateBackup("/stored/path.sqlite", ("--no-local-db", "true"));
|
||||
var task = CreateRunnerTask(backup);
|
||||
Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath(task));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RunnerDataExtraNoLocalDbFalseReturnsDbPath()
|
||||
{
|
||||
var backup = CreateBackup("/stored/path.sqlite");
|
||||
var task = CreateRunnerTask(backup, new Dictionary<string, string?> { ["no-local-db"] = "false" });
|
||||
Assert.AreEqual("/stored/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RunnerDataNullReturnsNull()
|
||||
{
|
||||
Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath((Duplicati.Server.Runner.IRunnerData?)null));
|
||||
Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath((Duplicati.Server.Serialization.Interface.IQueuedTask?)null));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RunnerDataNonRunnerDataQueuedTaskReturnsNull()
|
||||
{
|
||||
// A non-IRunnerData IQueuedTask implementation should cast to null and return null.
|
||||
var nonRunnerTask = new NonRunnerQueuedTask();
|
||||
Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath(nonRunnerTask));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A minimal <see cref="IQueuedTask"/> implementation that is NOT an
|
||||
/// <see cref="Duplicati.Server.Runner.IRunnerData"/>, used to verify that
|
||||
/// <see cref="Duplicati.Server.Runner.GetEffectiveDBPath(IQueuedTask)"/> safely returns
|
||||
/// null for non-runner tasks.
|
||||
/// </summary>
|
||||
private sealed class NonRunnerQueuedTask : Duplicati.Server.Serialization.Interface.IQueuedTask
|
||||
{
|
||||
public long TaskID => 0;
|
||||
public string? BackupID => null;
|
||||
public Duplicati.Server.Serialization.DuplicatiOperation Operation => Duplicati.Server.Serialization.DuplicatiOperation.BackupOrSync;
|
||||
public Func<Task>? OnStarting { get; set; }
|
||||
public Func<Exception?, Task>? OnFinished { get; set; }
|
||||
public DateTime? TaskStarted { get; set; }
|
||||
public DateTime? TaskFinished { get; set; }
|
||||
public Task UpdateThrottleSpeedsAsync(string? uploadSpeed, string? downloadSpeed) => Task.CompletedTask;
|
||||
public Task StopAsync() => Task.CompletedTask;
|
||||
public Task AbortAsync() => Task.CompletedTask;
|
||||
public Task PauseAsync(bool alsoTransfers) => Task.CompletedTask;
|
||||
public Task ResumeAsync() => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright (C) 2026, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
// sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
namespace Duplicati.WebserverCore.Abstractions;
|
||||
|
||||
/// <summary>
|
||||
/// Tracks which local backup database files are currently in use by running operations,
|
||||
/// preventing concurrent access that would cause SQLite "database is locked" errors.
|
||||
/// </summary>
|
||||
public interface IDatabaseLockTracker
|
||||
{
|
||||
/// <summary>
|
||||
/// Attempts to acquire a lock on the given database path, waiting until the lock
|
||||
/// is available. Used by queued tasks (backup, restore, etc.) which should wait
|
||||
/// for any concurrent immediate operations to finish.
|
||||
/// </summary>
|
||||
/// <param name="dbPath">The normalized, full path to the database file.</param>
|
||||
/// <param name="cancellationToken">A cancellation token to cancel the wait.</param>
|
||||
/// <returns>A disposable handle that releases the lock when disposed.</returns>
|
||||
Task<IAsyncDisposable> AcquireAsync(string dbPath, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to acquire a lock on the given database path, failing immediately if
|
||||
/// the database is already in use. Used by immediate (non-queued) operations such
|
||||
/// as listing filesets from the restore UI.
|
||||
/// </summary>
|
||||
/// <param name="dbPath">The normalized, full path to the database file.</param>
|
||||
/// <returns>A disposable handle that releases the lock when disposed, or <c>null</c> if the database is already locked.</returns>
|
||||
IAsyncDisposable? TryAcquire(string dbPath);
|
||||
}
|
||||
@@ -45,12 +45,12 @@ public class BackupGet : IEndpointV1
|
||||
=> ExecuteGetFilesAsync(queueRunnerService, GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary<string, string>()))
|
||||
.RequireAuthorization();
|
||||
|
||||
group.MapGet("/backup/{id}/log", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize)
|
||||
=> ExecuteGetLog(connection, GetBackup(connection, id), offset, pagesize ?? 100))
|
||||
group.MapGet("/backup/{id}/log", ([FromServices] Connection connection, [FromServices] IDatabaseLockTracker databaseLockTracker, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize)
|
||||
=> ExecuteGetLog(connection, databaseLockTracker, GetBackup(connection, id), offset, pagesize ?? 100))
|
||||
.RequireAuthorization();
|
||||
|
||||
group.MapGet("/backup/{id}/remotelog", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize)
|
||||
=> ExecuteGetRemotelog(connection, GetBackup(connection, id), offset, pagesize ?? 100))
|
||||
group.MapGet("/backup/{id}/remotelog", ([FromServices] Connection connection, [FromServices] IDatabaseLockTracker databaseLockTracker, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize)
|
||||
=> ExecuteGetRemotelog(connection, databaseLockTracker, GetBackup(connection, id), offset, pagesize ?? 100))
|
||||
.RequireAuthorization();
|
||||
|
||||
group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly)
|
||||
@@ -185,7 +185,7 @@ public class BackupGet : IEndpointV1
|
||||
private static async Task<Dictionary<string, object>> ExecuteGetFilesAsync(IQueueRunnerService queueRunnerService, IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary<string, string> extraValues)
|
||||
=> await SearchFilesAsync(queueRunnerService, bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues).ConfigureAwait(false);
|
||||
|
||||
private static List<Dictionary<string, object>> ExecuteGetLog(Connection connection, IBackup bk, long? offset, long pagesize)
|
||||
private static async Task<List<Dictionary<string, object>>> ExecuteGetLog(Connection connection, IDatabaseLockTracker databaseLockTracker, IBackup bk, long? offset, long pagesize)
|
||||
{
|
||||
// Use the effective database path (honoring a "--dbpath" advanced option) so that the log is
|
||||
// read from the same database the backup actually uses (see issue #1698).
|
||||
@@ -193,28 +193,48 @@ public class BackupGet : IEndpointV1
|
||||
if (!File.Exists(dbpath))
|
||||
return new List<Dictionary<string, object>>();
|
||||
|
||||
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath))
|
||||
using (var cmd = con.CreateCommand())
|
||||
return LogData.DumpTable(cmd, "LogData", "ID", offset, pagesize);
|
||||
var dbLock = databaseLockTracker.TryAcquire(dbpath)
|
||||
?? throw new DatabaseLockedException(dbpath);
|
||||
|
||||
try
|
||||
{
|
||||
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath))
|
||||
using (var cmd = con.CreateCommand())
|
||||
return LogData.DumpTable(cmd, "LogData", "ID", offset, pagesize);
|
||||
}
|
||||
finally
|
||||
{
|
||||
await dbLock.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static List<Dictionary<string, object>> ExecuteGetRemotelog(Connection connection, IBackup bk, long? offset, long pagesize)
|
||||
private static async Task<List<Dictionary<string, object>>> ExecuteGetRemotelog(Connection connection, IDatabaseLockTracker databaseLockTracker, IBackup bk, long? offset, long pagesize)
|
||||
{
|
||||
var dbpath = Runner.GetEffectiveDBPath(bk);
|
||||
if (!File.Exists(dbpath))
|
||||
return new List<Dictionary<string, object>>();
|
||||
|
||||
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath))
|
||||
using (var cmd = con.CreateCommand())
|
||||
var dbLock = databaseLockTracker.TryAcquire(dbpath)
|
||||
?? throw new DatabaseLockedException(dbpath);
|
||||
|
||||
try
|
||||
{
|
||||
var dt = LogData.DumpTable(cmd, "RemoteOperation", "ID", offset, pagesize);
|
||||
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath))
|
||||
using (var cmd = con.CreateCommand())
|
||||
{
|
||||
var dt = LogData.DumpTable(cmd, "RemoteOperation", "ID", offset, pagesize);
|
||||
|
||||
// Unwrap raw data to a string
|
||||
foreach (var n in dt)
|
||||
try { n["Data"] = System.Text.Encoding.UTF8.GetString((byte[])n["Data"]); }
|
||||
catch { }
|
||||
// Unwrap raw data to a string
|
||||
foreach (var n in dt)
|
||||
try { n["Data"] = System.Text.Encoding.UTF8.GetString((byte[])n["Data"]); }
|
||||
catch { }
|
||||
|
||||
return dt;
|
||||
return dt;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
await dbLock.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ public static class ServiceCollectionsExtensions
|
||||
.AddSingleton<IRemoteControllerRegistration, RemoteControllerRegistrationService>()
|
||||
.AddSingleton<ISystemInfoProvider, SystemInfoProvider>()
|
||||
.AddSingleton<IQueueRunnerService, QueueRunnerService>()
|
||||
.AddSingleton<IDatabaseLockTracker, DatabaseLockTracker>()
|
||||
.AddSingleton<IProgressStateProviderService, ProgressStateProviderService>()
|
||||
.AddTransient<INotificationService, NotificationService>()
|
||||
.AddTransient<IBackupListService, BackupListService>()
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
// Copyright (C) 2026, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||
// sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
||||
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
using Duplicati.WebserverCore.Abstractions;
|
||||
|
||||
namespace Duplicati.WebserverCore.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Tracks which local backup database files are currently in use by running operations,
|
||||
/// preventing concurrent access that would cause SQLite "database is locked" errors.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Each database path gets its own <see cref="SemaphoreSlim"/> (initial count 1).
|
||||
/// Queued tasks call <see cref="AcquireAsync"/> which waits for the lock.
|
||||
/// Immediate tasks call <see cref="TryAcquire"/> which fails instantly if locked.
|
||||
/// <para>
|
||||
/// Semaphore entries are intentionally not removed on release. Removing them is racy: a
|
||||
/// concurrent acquirer could take the lock between a release and a dictionary removal,
|
||||
/// after which a new acquirer would create a fresh semaphore for the same path, breaking
|
||||
/// mutual exclusion. The dictionary is bounded by the number of unique database paths
|
||||
/// (one per backup), which is a small, long-lived set, so the lack of cleanup does not
|
||||
/// cause unbounded growth in practice.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public class DatabaseLockTracker : IDatabaseLockTracker
|
||||
{
|
||||
private const int MaxLockCount = 1;
|
||||
|
||||
private readonly ConcurrentDictionary<string, SemaphoreSlim> _locks = new(Library.Utility.Utility.ClientFilenameStringComparer);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IAsyncDisposable> AcquireAsync(string dbPath, CancellationToken cancellationToken)
|
||||
{
|
||||
var semaphore = GetOrCreateSemaphore(dbPath);
|
||||
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
return new LockRelease(semaphore);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IAsyncDisposable? TryAcquire(string dbPath)
|
||||
{
|
||||
var semaphore = GetOrCreateSemaphore(dbPath);
|
||||
if (!semaphore.Wait(0))
|
||||
return null;
|
||||
return new LockRelease(semaphore);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes the database path and gets (or creates) the semaphore for it.
|
||||
/// </summary>
|
||||
private SemaphoreSlim GetOrCreateSemaphore(string dbPath)
|
||||
{
|
||||
var normalized = NormalizePath(dbPath);
|
||||
if (string.IsNullOrEmpty(normalized))
|
||||
throw new ArgumentException("Database path cannot be null or empty", nameof(dbPath));
|
||||
return _locks.GetOrAdd(normalized, _ => new SemaphoreSlim(MaxLockCount, MaxLockCount));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes a file path so that the same database file always maps to the same key,
|
||||
/// regardless of path separators or relative vs. absolute forms.
|
||||
/// </summary>
|
||||
private static string NormalizePath(string dbPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(dbPath))
|
||||
return string.Empty;
|
||||
return Path.GetFullPath(dbPath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A disposable handle that releases the semaphore when disposed.
|
||||
/// </summary>
|
||||
private sealed class LockRelease(SemaphoreSlim semaphore) : IAsyncDisposable
|
||||
{
|
||||
private int _released;
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _released, 1) == 0)
|
||||
semaphore.Release();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,8 @@ public class QueueRunnerService(
|
||||
EventPollNotify eventPollNotify,
|
||||
INotificationUpdateService notificationUpdateService,
|
||||
IProgressStateProviderService progressStateProviderService,
|
||||
IApplicationSettings applicationSettings) : IQueueRunnerService
|
||||
IApplicationSettings applicationSettings,
|
||||
IDatabaseLockTracker databaseLockTracker) : IQueueRunnerService, IDisposable
|
||||
{
|
||||
private readonly object _lock = new();
|
||||
/// <summary>
|
||||
@@ -51,9 +52,10 @@ public class QueueRunnerService(
|
||||
private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100;
|
||||
|
||||
private readonly List<IQueuedTask> _tasks = new();
|
||||
private (Task? Task, IQueuedTask? QueuedTask) _current;
|
||||
private (Task? Task, IQueuedTask? QueuedTask, CancellationTokenSource? LockCts) _current;
|
||||
private bool _isPaused;
|
||||
private bool _isTerminated;
|
||||
private readonly CancellationTokenSource _terminateCts = new();
|
||||
|
||||
public long AddTask(IQueuedTask task)
|
||||
=> AddTask(task, false);
|
||||
@@ -101,6 +103,7 @@ public class QueueRunnerService(
|
||||
public void Terminate(bool wait)
|
||||
{
|
||||
_isTerminated = true;
|
||||
_terminateCts.Cancel();
|
||||
if (wait)
|
||||
{
|
||||
var task = _current.Task;
|
||||
@@ -118,20 +121,27 @@ public class QueueRunnerService(
|
||||
|
||||
// Clean up completed tasks
|
||||
if (_current.Task != null && _current.Task.IsCompleted)
|
||||
_current = (null, null);
|
||||
{
|
||||
_current.LockCts?.Dispose();
|
||||
_current = (null, null, null);
|
||||
}
|
||||
|
||||
if (_tasks.Count == 0)
|
||||
return;
|
||||
|
||||
var nextTask = _tasks[0];
|
||||
_tasks.RemoveAt(0);
|
||||
_current = (Task.Run(() => RunTaskAsync(nextTask), CancellationToken.None), nextTask);
|
||||
_current = (Task.Run(() => RunTaskAsync(nextTask), CancellationToken.None), nextTask, null);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunTaskAsync(IQueuedTask task)
|
||||
{
|
||||
var completed = false;
|
||||
IAsyncDisposable? dbLock = null;
|
||||
var lockCts = CancellationTokenSource.CreateLinkedTokenSource(_terminateCts.Token);
|
||||
lock (_lock)
|
||||
_current = (_current.Task, _current.QueuedTask, lockCts);
|
||||
try
|
||||
{
|
||||
eventPollNotify.SignalNewEvent();
|
||||
@@ -140,6 +150,8 @@ public class QueueRunnerService(
|
||||
if (task.OnStarting != null)
|
||||
await task.OnStarting().ConfigureAwait(false);
|
||||
|
||||
dbLock = await AcquireDatabaseLockAsync(task, lockCts.Token).ConfigureAwait(false);
|
||||
|
||||
await Runner.RunAsync(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, true).ConfigureAwait(false);
|
||||
|
||||
// If the task is completed, don't call OnFinished again
|
||||
@@ -160,9 +172,14 @@ public class QueueRunnerService(
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (dbLock != null)
|
||||
await dbLock.DisposeAsync().ConfigureAwait(false);
|
||||
task.TaskFinished = DateTime.UtcNow;
|
||||
lock (_lock)
|
||||
_current = (null, null);
|
||||
{
|
||||
lockCts.Dispose();
|
||||
_current = (null, null, null);
|
||||
}
|
||||
eventPollNotify.SignalNewEvent();
|
||||
eventPollNotify.SignalTaskQueueUpdate();
|
||||
eventPollNotify.SignalTaskCompleted(task.TaskID);
|
||||
@@ -215,6 +232,55 @@ public class QueueRunnerService(
|
||||
/// <inheritdoc/>
|
||||
public async Task<IBasicResults?> RunImmediatelyAsync(IQueuedTask task)
|
||||
{
|
||||
return await Runner.RunAsync(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, false).ConfigureAwait(false);
|
||||
IAsyncDisposable? dbLock = null;
|
||||
var dbPath = Runner.GetEffectiveDBPath(task);
|
||||
|
||||
// If we have a database to work with, make sure we can get the lock
|
||||
if (!string.IsNullOrWhiteSpace(dbPath))
|
||||
dbLock = databaseLockTracker.TryAcquire(dbPath)
|
||||
?? throw new DatabaseLockedException(dbPath);
|
||||
|
||||
try
|
||||
{
|
||||
return await Runner.RunAsync(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, false).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (dbLock != null)
|
||||
await dbLock.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void CancelCurrentTaskLockWait(long taskID)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
if (_current.QueuedTask?.TaskID == taskID && _current.LockCts != null)
|
||||
_current.LockCts.Cancel();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Acquires the database lock for a queued task, waiting until the lock is available
|
||||
/// or the cancellation token is cancelled. Tasks without a database path (e.g. custom
|
||||
/// runner tasks) skip locking.
|
||||
/// </summary>
|
||||
private async Task<IAsyncDisposable?> AcquireDatabaseLockAsync(IQueuedTask task, CancellationToken cancellationToken)
|
||||
{
|
||||
var dbPath = Runner.GetEffectiveDBPath(task);
|
||||
if (string.IsNullOrWhiteSpace(dbPath))
|
||||
return null;
|
||||
|
||||
return await databaseLockTracker.AcquireAsync(dbPath, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Disposes resources held by the queue runner, including the termination
|
||||
/// cancellation token source used to cancel tasks waiting on the database lock.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
_terminateCts.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -108,6 +108,8 @@ public class TaskQueueService(IQueueRunnerService queueRunnerService) : ITaskQue
|
||||
if (task == null)
|
||||
throw new NotFoundException("No such task found");
|
||||
|
||||
queueRunnerService.CancelCurrentTaskLockWait(task.TaskID);
|
||||
|
||||
if (abort)
|
||||
await task.AbortAsync().ConfigureAwait(false);
|
||||
else
|
||||
|
||||
Reference in New Issue
Block a user