Merge pull request #7035 from duplicati/feature/server-db-lock

Server database lock
This commit is contained in:
Kenneth Skovhede
2026-07-11 14:55:02 +02:00
committed by GitHub
13 changed files with 696 additions and 25 deletions
@@ -273,4 +273,35 @@ namespace Duplicati.Library.Interface
: base(message, helpId, innerException)
{ }
}
/// <summary>
/// Exception indicating that the backup database is currently locked by another running operation
/// and cannot be accessed concurrently.
/// </summary>
[Serializable]
public class DatabaseLockedException : UserInformationException
{
/// <summary>
/// The path to the database file that is locked.
/// </summary>
public readonly string DatabasePath;
public DatabaseLockedException(string databasePath)
: base(LC.L($"The backup database is currently in use by another operation and cannot be accessed concurrently. Please wait for the running operation to finish and try again."), "DatabaseLocked")
{
DatabasePath = databasePath;
}
public DatabaseLockedException(string databasePath, Exception innerException)
: base(LC.L($"The backup database is currently in use by another operation and cannot be accessed concurrently. Please wait for the running operation to finish and try again."), "DatabaseLocked", innerException)
{
DatabasePath = databasePath;
}
public DatabaseLockedException(string message, string helpId, Exception innerException)
: base(message, helpId, innerException)
{
DatabasePath = string.Empty;
}
}
}
@@ -99,10 +99,32 @@ public interface IQueueRunnerService
/// <returns>A list of tuples containing the task ID and backup ID</returns>
IList<Tuple<long, string?>> GetQueueWithIds();
/// <summary>
/// Cancels the database lock wait for the currently running task, if its ID matches
/// <paramref name="taskID"/> and it is still waiting to acquire the database lock.
/// If the task has already acquired the lock and is executing, this has no effect —
/// call <see cref="IQueuedTask.AbortAsync"/> or <see cref="IQueuedTask.StopAsync"/>
/// to interrupt a running task. This method is intended to be called *before*
/// <see cref="IQueuedTask.AbortAsync"/>/<see cref="IQueuedTask.StopAsync"/> so that
/// a task blocked on lock acquisition is unblocked rather than left stuck.
/// </summary>
/// <param name="taskID">The ID of the currently running task whose lock wait should be cancelled.</param>
void CancelCurrentTaskLockWait(long taskID);
/// <summary>
/// Runs a task immediately, bypassing the queue.
/// Note that the task will run concurrently with the queue tasks and may cause database lock issues.
/// If the task's backup database is already in use by a queued task (e.g. a running backup),
/// this method throws <see cref="Duplicati.Library.Interface.DatabaseLockedException"/> immediately
/// rather than waiting or causing a concurrent "database is locked" error.
/// <para>
/// Only tasks that expose a database path via <see cref="Duplicati.Library.RestAPI.Runner.GetEffectiveDBPath"/>
/// (i.e. <see cref="Duplicati.Library.RestAPI.IRunnerData"/>-backed tasks) are protected by the
/// database lock. In production all tasks are <see cref="Duplicati.Library.RestAPI.IRunnerData"/>
/// instances; non-runner <see cref="IQueuedTask"/> implementations (test mocks) return a
/// <c>null</c> path and bypass locking.
/// </para>
/// </summary>
/// <param name="task">The task to run</param>
/// <exception cref="Duplicati.Library.Interface.DatabaseLockedException">Thrown when the backup database is already in use.</exception>
Task<IBasicResults?> RunImmediatelyAsync(IQueuedTask task);
}
}
+43
View File
@@ -1422,6 +1422,49 @@ namespace Duplicati.Server
options["remote-sync-json-config"] = JsonSerializer.Serialize(config, jsonOptions);
}
/// <summary>
/// Returns the effective local database path for an <see cref="IQueuedTask"/> instance.
/// In production, all queued tasks are <see cref="IRunnerData"/> instances created by the
/// <c>Runner.Create*</c> factories, so this delegates to
/// <see cref="GetEffectiveDBPath(IRunnerData?)"/>. If the task is not an
/// <see cref="IRunnerData"/> (only possible in tests with mock <see cref="IQueuedTask"/>
/// implementations), the result is <c>null</c>.
/// </summary>
/// <returns>The effective path, or <c>null</c></returns>
public static string? GetEffectiveDBPath(IQueuedTask? runnerData)
=> GetEffectiveDBPath(runnerData as IRunnerData);
/// <summary>
/// Returns the effective local database path for a <see cref="IRunnerData"/> instance.
/// The precedence mirrors <see cref="ApplyOptions"/>: the <c>dbpath</c> extra option takes
/// priority, then the backup's <c>--dbpath</c> advanced setting, then the stored
/// <see cref="Serialization.Interface.IBackup.DBPath"/>. If <c>no-local-db</c> is set
/// (either as an extra option or as a backup advanced setting), the result is <c>null</c>.
/// </summary>
/// <returns>The effective path, or <c>null</c></returns>
public static string? GetEffectiveDBPath(IRunnerData? runnerData)
{
if (runnerData == null)
return null;
if (runnerData.ExtraOptions != null)
{
if (runnerData.ExtraOptions.ContainsKey("no-local-db") && Utility.ParseBoolOption(runnerData.ExtraOptions.AsReadOnly(), "no-local-db"))
return null;
if (runnerData.ExtraOptions.TryGetValue("dbpath", out var dbpath) && !string.IsNullOrWhiteSpace(dbpath))
return dbpath;
}
if (runnerData.Backup == null)
return null;
var setting = runnerData.Backup.Settings?.FirstOrDefault(s => s.Name.Equals($"--no-local-db", StringComparison.OrdinalIgnoreCase));
if (setting != null && Utility.ParseBool(setting.Value, true))
return null;
return GetEffectiveDBPath(runnerData.Backup);
}
/// <summary>
/// Returns the effective local database path for a backup: the "--dbpath" advanced option if
/// it is set, otherwise the stored <see cref="Serialization.Interface.IBackup.DBPath"/>. This
@@ -42,6 +42,13 @@ namespace Duplicati.Library.SQLiteHelper
/// </summary>
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(SQLiteLoader));
/// <summary>
/// The default busy timeout (in milliseconds) applied to every connection opened by
/// <see cref="OpenSQLiteFileAsync"/> via the connection string. This is a defense-in-depth
/// measure that ensures an active reader does not cause failed commits.
/// </summary>
private const int DefaultBusyTimeoutMs = 10000;
/// <summary>
/// Helper method with logic to handle opening a database in possibly encrypted format.
/// </summary>
@@ -281,6 +288,12 @@ namespace Duplicati.Library.SQLiteHelper
con.ConnectionString = $"Data Source={path};Pooling=false";
await con.OpenAsync().ConfigureAwait(false);
using (var cmd = con.CreateCommand())
{
cmd.CommandText = $"PRAGMA busy_timeout={DefaultBusyTimeoutMs}";
await cmd.ExecuteNonQueryAsync().ConfigureAwait(false);
}
// Make the file only accessible by the current user, unless opting out
if (!SystemIO.IO_OS.FileExists(SystemIO.IO_OS.PathCombine(SystemIO.IO_OS.PathGetDirectoryName(path), Util.InsecurePermissionsMarkerFile)))
try { SystemIO.IO_OS.FileSetPermissionUserRWOnly(path); }
@@ -0,0 +1,225 @@
// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
#nullable enable
using System;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.WebserverCore.Services;
using NUnit.Framework;
using Assert = NUnit.Framework.Legacy.ClassicAssert;
namespace Duplicati.UnitTest
{
/// <summary>
/// Tests for <see cref="DatabaseLockTracker"/>, which coordinates access to local
/// backup database files between queued and immediate operations.
/// </summary>
[TestFixture]
public class DatabaseLockTrackerTests
{
/// <summary>
/// A simple value-returning async disposable used in tests.
/// </summary>
private static async Task DisposeAsync(IAsyncDisposable disposable)
=> await disposable.DisposeAsync().ConfigureAwait(false);
[Test]
public async Task TryAcquireReturnsLockWhenFree()
{
var tracker = new DatabaseLockTracker();
var handle = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNotNull(handle, "TryAcquire should succeed when the database is not locked");
await DisposeAsync(handle!);
}
[Test]
public async Task TryAcquireReturnsNullWhenAlreadyLocked()
{
var tracker = new DatabaseLockTracker();
var first = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNotNull(first, "First TryAcquire should succeed");
var second = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNull(second, "Second TryAcquire should fail when the database is locked");
await DisposeAsync(first!);
}
[Test]
public async Task AcquireAsyncWaitsForExistingLock()
{
var tracker = new DatabaseLockTracker();
var first = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNotNull(first);
var acquireTask = tracker.AcquireAsync("/tmp/test.sqlite", CancellationToken.None);
Assert.IsFalse(acquireTask.IsCompleted, "AcquireAsync should be waiting for the lock");
await DisposeAsync(first!);
var second = await acquireTask;
Assert.IsNotNull(second, "AcquireAsync should complete after the lock is released");
await DisposeAsync(second);
}
[Test]
public async Task ReleaseAllowsSubsequentAcquire()
{
var tracker = new DatabaseLockTracker();
var first = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNotNull(first);
await DisposeAsync(first!);
var second = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNotNull(second, "TryAcquire should succeed after the lock is released");
await DisposeAsync(second!);
}
[Test]
public async Task DifferentPathsAreIndependent()
{
var tracker = new DatabaseLockTracker();
var lockA = tracker.TryAcquire("/tmp/a.sqlite");
Assert.IsNotNull(lockA);
var lockB = tracker.TryAcquire("/tmp/b.sqlite");
Assert.IsNotNull(lockB, "TryAcquire on a different path should succeed independently");
await DisposeAsync(lockA!);
await DisposeAsync(lockB!);
}
[Test]
public async Task AcquireAsyncCancelsWhenTokenFires()
{
var tracker = new DatabaseLockTracker();
var first = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNotNull(first);
using var cts = new CancellationTokenSource();
var acquireTask = tracker.AcquireAsync("/tmp/test.sqlite", cts.Token);
Assert.IsFalse(acquireTask.IsCompleted);
cts.Cancel();
Assert.ThrowsAsync<System.OperationCanceledException>(async () => await acquireTask);
await DisposeAsync(first!);
}
[Test]
public async Task DoubleDisposeIsSafe()
{
var tracker = new DatabaseLockTracker();
var handle = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNotNull(handle);
await DisposeAsync(handle!);
await DisposeAsync(handle!);
var next = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNotNull(next, "After double-dispose, the lock should be releasable exactly once");
await DisposeAsync(next!);
}
[Test]
public async Task TryAcquireOnLockedReleasesCorrectly()
{
var tracker = new DatabaseLockTracker();
var first = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNotNull(first);
var second = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNull(second);
await DisposeAsync(first!);
var third = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNotNull(third, "After releasing the first lock, TryAcquire should succeed");
await DisposeAsync(third!);
}
[Test]
public async Task RelativeAndAbsolutePathMapToSameLock()
{
var tracker = new DatabaseLockTracker();
var absolute = tracker.TryAcquire(System.IO.Path.GetFullPath("/tmp/test.sqlite"));
Assert.IsNotNull(absolute);
var relative = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNull(relative, "Path normalization should map relative and absolute paths to the same lock");
await DisposeAsync(absolute!);
}
[Test]
public async Task AcquireAsyncPreservesOrder()
{
var tracker = new DatabaseLockTracker();
var first = tracker.TryAcquire("/tmp/test.sqlite");
Assert.IsNotNull(first);
var order = new System.Collections.Generic.List<int>();
var t2 = Task.Run(async () =>
{
await using var h = await tracker.AcquireAsync("/tmp/test.sqlite", CancellationToken.None);
order.Add(2);
});
var t3 = Task.Run(async () =>
{
await using var h = await tracker.AcquireAsync("/tmp/test.sqlite", CancellationToken.None);
order.Add(3);
});
await Task.Delay(50);
order.Add(1);
await DisposeAsync(first!);
await Task.WhenAll(t2, t3);
Assert.AreEqual(1, order[0], "First lock holder should complete first");
}
[Test]
public void TryAcquireEmptyPathThrows()
{
var tracker = new DatabaseLockTracker();
Assert.Throws<ArgumentException>(() => tracker.TryAcquire(""));
}
[Test]
public void TryAcquireWhitespacePathThrows()
{
var tracker = new DatabaseLockTracker();
Assert.Throws<ArgumentException>(() => tracker.TryAcquire(" "));
}
[Test]
public void AcquireAsyncEmptyPathThrows()
{
var tracker = new DatabaseLockTracker();
Assert.ThrowsAsync<ArgumentException>(async () => await tracker.AcquireAsync("", CancellationToken.None));
}
}
}
@@ -62,6 +62,7 @@ public class FolderStatusServiceTests
public void Resume() { }
public void Pause() { }
public IList<Tuple<long, string?>> GetQueueWithIds() => new List<Tuple<long, string?>>();
public void CancelCurrentTaskLockWait(long taskID) { }
public Task<IBasicResults?> RunImmediatelyAsync(IQueuedTask task) => Task.FromResult<IBasicResults?>(null);
}
+96
View File
@@ -24,6 +24,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Duplicati.Server.Database;
using Duplicati.Server.Serialization.Interface;
using NUnit.Framework;
@@ -87,5 +88,100 @@ namespace Duplicati.UnitTest
var backup = CreateBackup("/stored/path.sqlite", ("--dbpath", " "));
Assert.AreEqual("/stored/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(backup));
}
// ---- Tests for GetEffectiveDBPath(IQueuedTask) / GetEffectiveDBPath(IRunnerData) ----
private static Duplicati.Server.Runner.IRunnerData CreateRunnerTask(
Backup backup,
IDictionary<string, string?>? extraOptions = null)
=> Duplicati.Server.Runner.CreateTask(
Duplicati.Server.Serialization.DuplicatiOperation.BackupOrSync,
backup,
extraOptions ?? new Dictionary<string, string?>());
[Test]
public void RunnerDataFallsBackToBackupDbPath()
{
var backup = CreateBackup("/stored/path.sqlite");
var task = CreateRunnerTask(backup);
Assert.AreEqual("/stored/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task));
}
[Test]
public void RunnerDataRespectsBackupAdvancedDbPath()
{
var backup = CreateBackup("/stored/path.sqlite", ("--dbpath", "/override/path.sqlite"));
var task = CreateRunnerTask(backup);
Assert.AreEqual("/override/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task));
}
[Test]
public void RunnerDataExtraDbPathOverridesBackup()
{
var backup = CreateBackup("/stored/path.sqlite", ("--dbpath", "/backup-override.sqlite"));
var task = CreateRunnerTask(backup, new Dictionary<string, string?> { ["dbpath"] = "/extra-override.sqlite" });
Assert.AreEqual("/extra-override.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task));
}
[Test]
public void RunnerDataExtraNoLocalDbReturnsNull()
{
var backup = CreateBackup("/stored/path.sqlite");
var task = CreateRunnerTask(backup, new Dictionary<string, string?> { ["no-local-db"] = "true" });
Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath(task));
}
[Test]
public void RunnerDataBackupNoLocalDbSettingReturnsNull()
{
var backup = CreateBackup("/stored/path.sqlite", ("--no-local-db", "true"));
var task = CreateRunnerTask(backup);
Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath(task));
}
[Test]
public void RunnerDataExtraNoLocalDbFalseReturnsDbPath()
{
var backup = CreateBackup("/stored/path.sqlite");
var task = CreateRunnerTask(backup, new Dictionary<string, string?> { ["no-local-db"] = "false" });
Assert.AreEqual("/stored/path.sqlite", Duplicati.Server.Runner.GetEffectiveDBPath(task));
}
[Test]
public void RunnerDataNullReturnsNull()
{
Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath((Duplicati.Server.Runner.IRunnerData?)null));
Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath((Duplicati.Server.Serialization.Interface.IQueuedTask?)null));
}
[Test]
public void RunnerDataNonRunnerDataQueuedTaskReturnsNull()
{
// A non-IRunnerData IQueuedTask implementation should cast to null and return null.
var nonRunnerTask = new NonRunnerQueuedTask();
Assert.IsNull(Duplicati.Server.Runner.GetEffectiveDBPath(nonRunnerTask));
}
/// <summary>
/// A minimal <see cref="IQueuedTask"/> implementation that is NOT an
/// <see cref="Duplicati.Server.Runner.IRunnerData"/>, used to verify that
/// <see cref="Duplicati.Server.Runner.GetEffectiveDBPath(IQueuedTask)"/> safely returns
/// null for non-runner tasks.
/// </summary>
private sealed class NonRunnerQueuedTask : Duplicati.Server.Serialization.Interface.IQueuedTask
{
public long TaskID => 0;
public string? BackupID => null;
public Duplicati.Server.Serialization.DuplicatiOperation Operation => Duplicati.Server.Serialization.DuplicatiOperation.BackupOrSync;
public Func<Task>? OnStarting { get; set; }
public Func<Exception?, Task>? OnFinished { get; set; }
public DateTime? TaskStarted { get; set; }
public DateTime? TaskFinished { get; set; }
public Task UpdateThrottleSpeedsAsync(string? uploadSpeed, string? downloadSpeed) => Task.CompletedTask;
public Task StopAsync() => Task.CompletedTask;
public Task AbortAsync() => Task.CompletedTask;
public Task PauseAsync(bool alsoTransfers) => Task.CompletedTask;
public Task ResumeAsync() => Task.CompletedTask;
}
}
}
@@ -0,0 +1,48 @@
// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
namespace Duplicati.WebserverCore.Abstractions;
/// <summary>
/// Tracks which local backup database files are currently in use by running operations,
/// preventing concurrent access that would cause SQLite "database is locked" errors.
/// </summary>
public interface IDatabaseLockTracker
{
/// <summary>
/// Attempts to acquire a lock on the given database path, waiting until the lock
/// is available. Used by queued tasks (backup, restore, etc.) which should wait
/// for any concurrent immediate operations to finish.
/// </summary>
/// <param name="dbPath">The normalized, full path to the database file.</param>
/// <param name="cancellationToken">A cancellation token to cancel the wait.</param>
/// <returns>A disposable handle that releases the lock when disposed.</returns>
Task<IAsyncDisposable> AcquireAsync(string dbPath, CancellationToken cancellationToken);
/// <summary>
/// Attempts to acquire a lock on the given database path, failing immediately if
/// the database is already in use. Used by immediate (non-queued) operations such
/// as listing filesets from the restore UI.
/// </summary>
/// <param name="dbPath">The normalized, full path to the database file.</param>
/// <returns>A disposable handle that releases the lock when disposed, or <c>null</c> if the database is already locked.</returns>
IAsyncDisposable? TryAcquire(string dbPath);
}
@@ -45,12 +45,12 @@ public class BackupGet : IEndpointV1
=> ExecuteGetFilesAsync(queueRunnerService, GetBackup(connection, id), filter, time, allVersions ?? false, prefixOnly ?? false, folderContents ?? false, new Dictionary<string, string>()))
.RequireAuthorization();
group.MapGet("/backup/{id}/log", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize)
=> ExecuteGetLog(connection, GetBackup(connection, id), offset, pagesize ?? 100))
group.MapGet("/backup/{id}/log", ([FromServices] Connection connection, [FromServices] IDatabaseLockTracker databaseLockTracker, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize)
=> ExecuteGetLog(connection, databaseLockTracker, GetBackup(connection, id), offset, pagesize ?? 100))
.RequireAuthorization();
group.MapGet("/backup/{id}/remotelog", ([FromServices] Connection connection, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize)
=> ExecuteGetRemotelog(connection, GetBackup(connection, id), offset, pagesize ?? 100))
group.MapGet("/backup/{id}/remotelog", ([FromServices] Connection connection, [FromServices] IDatabaseLockTracker databaseLockTracker, [FromRoute] string id, [FromQuery] long? offset, [FromQuery] long? pagesize)
=> ExecuteGetRemotelog(connection, databaseLockTracker, GetBackup(connection, id), offset, pagesize ?? 100))
.RequireAuthorization();
group.MapGet("/backup/{id}/filesets", ([FromServices] Connection connection, [FromServices] IQueueRunnerService queueRunnerService, [FromRoute] string id, [FromQuery(Name = "include-metadata")] bool? includeMetadata, [FromQuery(Name = "from-remote-only")] bool? fromRemoteOnly)
@@ -185,7 +185,7 @@ public class BackupGet : IEndpointV1
private static async Task<Dictionary<string, object>> ExecuteGetFilesAsync(IQueueRunnerService queueRunnerService, IBackup bk, string? filter, string? timestring, bool allVersions, bool prefixOnly, bool folderContents, Dictionary<string, string> extraValues)
=> await SearchFilesAsync(queueRunnerService, bk, filter, timestring, allVersions, prefixOnly, folderContents, extraValues).ConfigureAwait(false);
private static List<Dictionary<string, object>> ExecuteGetLog(Connection connection, IBackup bk, long? offset, long pagesize)
private static async Task<List<Dictionary<string, object>>> ExecuteGetLog(Connection connection, IDatabaseLockTracker databaseLockTracker, IBackup bk, long? offset, long pagesize)
{
// Use the effective database path (honoring a "--dbpath" advanced option) so that the log is
// read from the same database the backup actually uses (see issue #1698).
@@ -193,28 +193,48 @@ public class BackupGet : IEndpointV1
if (!File.Exists(dbpath))
return new List<Dictionary<string, object>>();
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath))
using (var cmd = con.CreateCommand())
return LogData.DumpTable(cmd, "LogData", "ID", offset, pagesize);
var dbLock = databaseLockTracker.TryAcquire(dbpath)
?? throw new DatabaseLockedException(dbpath);
try
{
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath))
using (var cmd = con.CreateCommand())
return LogData.DumpTable(cmd, "LogData", "ID", offset, pagesize);
}
finally
{
await dbLock.DisposeAsync().ConfigureAwait(false);
}
}
private static List<Dictionary<string, object>> ExecuteGetRemotelog(Connection connection, IBackup bk, long? offset, long pagesize)
private static async Task<List<Dictionary<string, object>>> ExecuteGetRemotelog(Connection connection, IDatabaseLockTracker databaseLockTracker, IBackup bk, long? offset, long pagesize)
{
var dbpath = Runner.GetEffectiveDBPath(bk);
if (!File.Exists(dbpath))
return new List<Dictionary<string, object>>();
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath))
using (var cmd = con.CreateCommand())
var dbLock = databaseLockTracker.TryAcquire(dbpath)
?? throw new DatabaseLockedException(dbpath);
try
{
var dt = LogData.DumpTable(cmd, "RemoteOperation", "ID", offset, pagesize);
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(dbpath))
using (var cmd = con.CreateCommand())
{
var dt = LogData.DumpTable(cmd, "RemoteOperation", "ID", offset, pagesize);
// Unwrap raw data to a string
foreach (var n in dt)
try { n["Data"] = System.Text.Encoding.UTF8.GetString((byte[])n["Data"]); }
catch { }
// Unwrap raw data to a string
foreach (var n in dt)
try { n["Data"] = System.Text.Encoding.UTF8.GetString((byte[])n["Data"]); }
catch { }
return dt;
return dt;
}
}
finally
{
await dbLock.DisposeAsync().ConfigureAwait(false);
}
}
@@ -67,6 +67,7 @@ public static class ServiceCollectionsExtensions
.AddSingleton<IRemoteControllerRegistration, RemoteControllerRegistrationService>()
.AddSingleton<ISystemInfoProvider, SystemInfoProvider>()
.AddSingleton<IQueueRunnerService, QueueRunnerService>()
.AddSingleton<IDatabaseLockTracker, DatabaseLockTracker>()
.AddSingleton<IProgressStateProviderService, ProgressStateProviderService>()
.AddTransient<INotificationService, NotificationService>()
.AddTransient<IBackupListService, BackupListService>()
@@ -0,0 +1,103 @@
// Copyright (C) 2026, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or
// sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
using System.Collections.Concurrent;
using Duplicati.WebserverCore.Abstractions;
namespace Duplicati.WebserverCore.Services;
/// <summary>
/// Tracks which local backup database files are currently in use by running operations,
/// preventing concurrent access that would cause SQLite "database is locked" errors.
/// </summary>
/// <remarks>
/// Each database path gets its own <see cref="SemaphoreSlim"/> (initial count 1).
/// Queued tasks call <see cref="AcquireAsync"/> which waits for the lock.
/// Immediate tasks call <see cref="TryAcquire"/> which fails instantly if locked.
/// <para>
/// Semaphore entries are intentionally not removed on release. Removing them is racy: a
/// concurrent acquirer could take the lock between a release and a dictionary removal,
/// after which a new acquirer would create a fresh semaphore for the same path, breaking
/// mutual exclusion. The dictionary is bounded by the number of unique database paths
/// (one per backup), which is a small, long-lived set, so the lack of cleanup does not
/// cause unbounded growth in practice.
/// </para>
/// </remarks>
public class DatabaseLockTracker : IDatabaseLockTracker
{
private const int MaxLockCount = 1;
private readonly ConcurrentDictionary<string, SemaphoreSlim> _locks = new(Library.Utility.Utility.ClientFilenameStringComparer);
/// <inheritdoc />
public async Task<IAsyncDisposable> AcquireAsync(string dbPath, CancellationToken cancellationToken)
{
var semaphore = GetOrCreateSemaphore(dbPath);
await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
return new LockRelease(semaphore);
}
/// <inheritdoc />
public IAsyncDisposable? TryAcquire(string dbPath)
{
var semaphore = GetOrCreateSemaphore(dbPath);
if (!semaphore.Wait(0))
return null;
return new LockRelease(semaphore);
}
/// <summary>
/// Normalizes the database path and gets (or creates) the semaphore for it.
/// </summary>
private SemaphoreSlim GetOrCreateSemaphore(string dbPath)
{
var normalized = NormalizePath(dbPath);
if (string.IsNullOrEmpty(normalized))
throw new ArgumentException("Database path cannot be null or empty", nameof(dbPath));
return _locks.GetOrAdd(normalized, _ => new SemaphoreSlim(MaxLockCount, MaxLockCount));
}
/// <summary>
/// Normalizes a file path so that the same database file always maps to the same key,
/// regardless of path separators or relative vs. absolute forms.
/// </summary>
private static string NormalizePath(string dbPath)
{
if (string.IsNullOrWhiteSpace(dbPath))
return string.Empty;
return Path.GetFullPath(dbPath);
}
/// <summary>
/// A disposable handle that releases the semaphore when disposed.
/// </summary>
private sealed class LockRelease(SemaphoreSlim semaphore) : IAsyncDisposable
{
private int _released;
public ValueTask DisposeAsync()
{
if (Interlocked.Exchange(ref _released, 1) == 0)
semaphore.Release();
return ValueTask.CompletedTask;
}
}
}
@@ -37,7 +37,8 @@ public class QueueRunnerService(
EventPollNotify eventPollNotify,
INotificationUpdateService notificationUpdateService,
IProgressStateProviderService progressStateProviderService,
IApplicationSettings applicationSettings) : IQueueRunnerService
IApplicationSettings applicationSettings,
IDatabaseLockTracker databaseLockTracker) : IQueueRunnerService, IDisposable
{
private readonly object _lock = new();
/// <summary>
@@ -51,9 +52,10 @@ public class QueueRunnerService(
private static readonly int MAX_TASK_RESULT_CACHE_SIZE = 100;
private readonly List<IQueuedTask> _tasks = new();
private (Task? Task, IQueuedTask? QueuedTask) _current;
private (Task? Task, IQueuedTask? QueuedTask, CancellationTokenSource? LockCts) _current;
private bool _isPaused;
private bool _isTerminated;
private readonly CancellationTokenSource _terminateCts = new();
public long AddTask(IQueuedTask task)
=> AddTask(task, false);
@@ -101,6 +103,7 @@ public class QueueRunnerService(
public void Terminate(bool wait)
{
_isTerminated = true;
_terminateCts.Cancel();
if (wait)
{
var task = _current.Task;
@@ -118,20 +121,27 @@ public class QueueRunnerService(
// Clean up completed tasks
if (_current.Task != null && _current.Task.IsCompleted)
_current = (null, null);
{
_current.LockCts?.Dispose();
_current = (null, null, null);
}
if (_tasks.Count == 0)
return;
var nextTask = _tasks[0];
_tasks.RemoveAt(0);
_current = (Task.Run(() => RunTaskAsync(nextTask), CancellationToken.None), nextTask);
_current = (Task.Run(() => RunTaskAsync(nextTask), CancellationToken.None), nextTask, null);
}
}
private async Task RunTaskAsync(IQueuedTask task)
{
var completed = false;
IAsyncDisposable? dbLock = null;
var lockCts = CancellationTokenSource.CreateLinkedTokenSource(_terminateCts.Token);
lock (_lock)
_current = (_current.Task, _current.QueuedTask, lockCts);
try
{
eventPollNotify.SignalNewEvent();
@@ -140,6 +150,8 @@ public class QueueRunnerService(
if (task.OnStarting != null)
await task.OnStarting().ConfigureAwait(false);
dbLock = await AcquireDatabaseLockAsync(task, lockCts.Token).ConfigureAwait(false);
await Runner.RunAsync(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, true).ConfigureAwait(false);
// If the task is completed, don't call OnFinished again
@@ -160,9 +172,14 @@ public class QueueRunnerService(
}
finally
{
if (dbLock != null)
await dbLock.DisposeAsync().ConfigureAwait(false);
task.TaskFinished = DateTime.UtcNow;
lock (_lock)
_current = (null, null);
{
lockCts.Dispose();
_current = (null, null, null);
}
eventPollNotify.SignalNewEvent();
eventPollNotify.SignalTaskQueueUpdate();
eventPollNotify.SignalTaskCompleted(task.TaskID);
@@ -215,6 +232,55 @@ public class QueueRunnerService(
/// <inheritdoc/>
public async Task<IBasicResults?> RunImmediatelyAsync(IQueuedTask task)
{
return await Runner.RunAsync(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, false).ConfigureAwait(false);
IAsyncDisposable? dbLock = null;
var dbPath = Runner.GetEffectiveDBPath(task);
// If we have a database to work with, make sure we can get the lock
if (!string.IsNullOrWhiteSpace(dbPath))
dbLock = databaseLockTracker.TryAcquire(dbPath)
?? throw new DatabaseLockedException(dbPath);
try
{
return await Runner.RunAsync(connection, eventPollNotify, notificationUpdateService, progressStateProviderService, applicationSettings, task, false).ConfigureAwait(false);
}
finally
{
if (dbLock != null)
await dbLock.DisposeAsync().ConfigureAwait(false);
}
}
/// <inheritdoc/>
public void CancelCurrentTaskLockWait(long taskID)
{
lock (_lock)
{
if (_current.QueuedTask?.TaskID == taskID && _current.LockCts != null)
_current.LockCts.Cancel();
}
}
/// <summary>
/// Acquires the database lock for a queued task, waiting until the lock is available
/// or the cancellation token is cancelled. Tasks without a database path (e.g. custom
/// runner tasks) skip locking.
/// </summary>
private async Task<IAsyncDisposable?> AcquireDatabaseLockAsync(IQueuedTask task, CancellationToken cancellationToken)
{
var dbPath = Runner.GetEffectiveDBPath(task);
if (string.IsNullOrWhiteSpace(dbPath))
return null;
return await databaseLockTracker.AcquireAsync(dbPath, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Disposes resources held by the queue runner, including the termination
/// cancellation token source used to cancel tasks waiting on the database lock.
/// </summary>
public void Dispose()
{
_terminateCts.Dispose();
}
}
@@ -108,6 +108,8 @@ public class TaskQueueService(IQueueRunnerService queueRunnerService) : ITaskQue
if (task == null)
throw new NotFoundException("No such task found");
queueRunnerService.CancelCurrentTaskLockWait(task.TaskID);
if (abort)
await task.AbortAsync().ConfigureAwait(false);
else