Merge pull request #6360 from carljohnsen/performance/mssqlite-async
Performance/Asynchronous database backend
This commit is contained in:
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System.CommandLine;
|
||||
using System.CommandLine.NamingConventionBinder;
|
||||
@@ -24,6 +24,7 @@ using Duplicati.Library.AutoUpdater;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Main.Database;
|
||||
using Duplicati.Library.SQLiteHelper;
|
||||
using Duplicati.Library.Utility;
|
||||
|
||||
namespace Duplicati.CommandLine.DatabaseTool.Commands;
|
||||
|
||||
@@ -50,7 +51,7 @@ public static class Downgrade
|
||||
}
|
||||
.WithHandler(CommandHandler.Create<string[], DirectoryInfo, int, int, bool, bool>((databases, serverdatafolder, serverversion, localversion, nobackups, includeuntrackeddatabases) =>
|
||||
{
|
||||
databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases);
|
||||
databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases).Await();
|
||||
if (databases.Length == 0)
|
||||
{
|
||||
Console.WriteLine("No databases found to downgrade");
|
||||
@@ -100,7 +101,7 @@ public static class Downgrade
|
||||
bool isserverdb;
|
||||
try
|
||||
{
|
||||
(version, isserverdb) = Helper.ExamineDatabase(db);
|
||||
(version, isserverdb) = Helper.ExamineDatabase(db).Await();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -112,7 +113,8 @@ public static class Downgrade
|
||||
ApplyDowngrade(db, version,
|
||||
isserverdb ? serverversion : localversion,
|
||||
isserverdb ? serverVersions : localVersions,
|
||||
nobackups);
|
||||
nobackups)
|
||||
.Await();
|
||||
}
|
||||
}));
|
||||
|
||||
@@ -132,7 +134,8 @@ public static class Downgrade
|
||||
/// <param name="targetversion">The target database version</param>
|
||||
/// <param name="scripts">The downgrade scripts</param>
|
||||
/// <param name="nobackups">Flag to disable backups</param>
|
||||
private static void ApplyDowngrade(string db, int dbversion, int targetversion, IEnumerable<DowngradeScript> scripts, bool nobackups)
|
||||
/// <returns>A task that completes when the downgrade is done.</returns>
|
||||
private static async Task ApplyDowngrade(string db, int dbversion, int targetversion, IEnumerable<DowngradeScript> scripts, bool nobackups)
|
||||
{
|
||||
if (dbversion > targetversion)
|
||||
{
|
||||
@@ -158,21 +161,27 @@ public static class Downgrade
|
||||
if (!nobackups)
|
||||
Helper.CreateFileBackup(db);
|
||||
|
||||
using var con = SQLiteLoader.LoadConnection(db, 0);
|
||||
using var tr = con.BeginTransaction();
|
||||
using var cmd = con.CreateCommand(tr);
|
||||
await using var con = await SQLiteLoader.LoadConnectionAsync(db)
|
||||
.ConfigureAwait(false);
|
||||
await using var tr = con.BeginTransaction();
|
||||
await using var cmd = con.CreateCommand(tr);
|
||||
foreach (var script in downgradeScripts)
|
||||
{
|
||||
Console.WriteLine($"Applying downgrade script {script.Filename} ...");
|
||||
try
|
||||
{
|
||||
cmd.SetCommandAndParameters(script.Content);
|
||||
var r = cmd.ExecuteScalar();
|
||||
var r = await cmd.ExecuteScalarAsync().ConfigureAwait(false);
|
||||
if (r != null)
|
||||
throw new UserInformationException($"{r}", "DowngradeScriptFailure");
|
||||
cmd.SetCommandAndParameters("UPDATE Version SET Version = @Version")
|
||||
.SetParameterValue("@Version", script.Version - 1);
|
||||
cmd.ExecuteNonQuery();
|
||||
|
||||
await cmd.SetCommandAndParameters(@"
|
||||
UPDATE ""Version""
|
||||
SET ""Version"" = @Version
|
||||
")
|
||||
.SetParameterValue("@Version", script.Version - 1)
|
||||
.ExecuteNonQueryAsync()
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -180,7 +189,7 @@ public static class Downgrade
|
||||
}
|
||||
}
|
||||
|
||||
tr.Commit();
|
||||
await tr.CommitAsync().ConfigureAwait(false);
|
||||
Console.WriteLine($"Downgraded {db} to version {targetversion}");
|
||||
}
|
||||
else if (dbversion == targetversion)
|
||||
|
||||
@@ -1,23 +1,23 @@
|
||||
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System.CommandLine;
|
||||
using System.CommandLine.NamingConventionBinder;
|
||||
@@ -48,7 +48,7 @@ public static class Execute
|
||||
if (createbackup)
|
||||
Helper.CreateFileBackup(database);
|
||||
|
||||
using var connection = SQLiteLoader.LoadConnection(database, 0);
|
||||
using var connection = SQLiteLoader.LoadConnection(database);
|
||||
try
|
||||
{
|
||||
if (command.IndexOfAny(Path.GetInvalidPathChars()) < 0 && File.Exists(command))
|
||||
|
||||
@@ -1,23 +1,23 @@
|
||||
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System.CommandLine;
|
||||
using System.CommandLine.NamingConventionBinder;
|
||||
@@ -54,7 +54,7 @@ public static class List
|
||||
return;
|
||||
}
|
||||
|
||||
using var con = SQLiteLoader.LoadConnection(database, 0);
|
||||
using var con = SQLiteLoader.LoadConnection(database);
|
||||
using var cmd = con.CreateCommand();
|
||||
|
||||
if (tables == null || tables.Length == 0)
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System.CommandLine;
|
||||
using System.CommandLine.NamingConventionBinder;
|
||||
@@ -24,6 +24,7 @@ using Duplicati.Library.AutoUpdater;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Main.Database;
|
||||
using Duplicati.Library.SQLiteHelper;
|
||||
using Duplicati.Library.Utility;
|
||||
|
||||
namespace Duplicati.CommandLine.DatabaseTool.Commands;
|
||||
|
||||
@@ -50,7 +51,7 @@ public static class Upgrade
|
||||
}
|
||||
.WithHandler(CommandHandler.Create<string[], DirectoryInfo, int, int, bool, bool>((databases, serverdatafolder, serverversion, localversion, nobackups, includeuntrackeddatabases) =>
|
||||
{
|
||||
databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases);
|
||||
databases = Helper.FindAllDatabases(databases, serverdatafolder.FullName, includeuntrackeddatabases).Await();
|
||||
if (databases.Length == 0)
|
||||
{
|
||||
Console.WriteLine("No databases found to upgrade");
|
||||
@@ -83,7 +84,7 @@ public static class Upgrade
|
||||
bool isserverdb;
|
||||
try
|
||||
{
|
||||
(version, isserverdb) = Helper.ExamineDatabase(db);
|
||||
(version, isserverdb) = Helper.ExamineDatabase(db).Await();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -95,7 +96,8 @@ public static class Upgrade
|
||||
ApplyUpgrade(db, version,
|
||||
isserverdb ? serverversion : localversion,
|
||||
isserverdb ? serverVersions : localVersions,
|
||||
nobackups);
|
||||
nobackups)
|
||||
.Await();
|
||||
|
||||
}
|
||||
}));
|
||||
@@ -145,7 +147,8 @@ public static class Upgrade
|
||||
/// <param name="targetversion">The target database version</param>
|
||||
/// <param name="scripts">The upgrade scripts</param>
|
||||
/// <param name="nobackups">Flag to disable backups</param>
|
||||
private static void ApplyUpgrade(string db, int dbversion, int targetversion, IEnumerable<UpgradeScript> scripts, bool nobackups)
|
||||
/// <returns>A task that completes when the upgrade is done.</returns>
|
||||
private static async Task ApplyUpgrade(string db, int dbversion, int targetversion, IEnumerable<UpgradeScript> scripts, bool nobackups)
|
||||
{
|
||||
if (targetversion > dbversion)
|
||||
{
|
||||
@@ -170,21 +173,27 @@ public static class Upgrade
|
||||
if (!nobackups)
|
||||
Helper.CreateFileBackup(db);
|
||||
|
||||
using var con = SQLiteLoader.LoadConnection(db, 0);
|
||||
using var tr = con.BeginTransaction();
|
||||
using var cmd = con.CreateCommand(tr);
|
||||
await using var con = await SQLiteLoader.LoadConnectionAsync(db)
|
||||
.ConfigureAwait(false);
|
||||
await using var tr = con.BeginTransaction();
|
||||
await using var cmd = con.CreateCommand(tr);
|
||||
foreach (var script in upgradeScripts)
|
||||
{
|
||||
Console.WriteLine($"Applying upgrade script {script.Filename} ...");
|
||||
try
|
||||
{
|
||||
cmd.SetCommandAndParameters(script.Content);
|
||||
var r = cmd.ExecuteScalar();
|
||||
var r = await cmd.ExecuteScalarAsync()
|
||||
.ConfigureAwait(false);
|
||||
if (r != null)
|
||||
throw new UserInformationException($"{r}", "UpgradeScriptFailure");
|
||||
cmd.SetCommandAndParameters("UPDATE Version SET Version = @Version")
|
||||
.SetParameterValue("@Version", script.Version);
|
||||
cmd.ExecuteNonQuery();
|
||||
await cmd.SetCommandAndParameters(@"
|
||||
UPDATE ""Version""
|
||||
SET ""Version"" = @Version
|
||||
")
|
||||
.SetParameterValue("@Version", script.Version)
|
||||
.ExecuteNonQueryAsync()
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -41,8 +41,8 @@ public static class Helper
|
||||
/// <param name="databases">The input databases</param>
|
||||
/// <param name="datafolder">The folder to scan</param>
|
||||
/// <param name="scanExtra">Whether to scan for extra databases</param>
|
||||
/// <returns>The list of databases</returns>
|
||||
public static string[] FindAllDatabases(string[]? databases, string datafolder, bool scanExtra)
|
||||
/// <returns>A task that when awaited contains the list of databases.</returns>
|
||||
public static async Task<string[]> FindAllDatabases(string[]? databases, string datafolder, bool scanExtra)
|
||||
{
|
||||
databases ??= [];
|
||||
if (databases.Length != 0)
|
||||
@@ -60,9 +60,13 @@ public static class Helper
|
||||
{
|
||||
try
|
||||
{
|
||||
using var con = SQLiteLoader.LoadConnection(serverdb, 0);
|
||||
using var cmd = con.CreateCommand();
|
||||
foreach (var rd in cmd.ExecuteReaderEnumerable(@"SELECT ""DBPath"" FROM ""Backup"""))
|
||||
await using var con = await SQLiteLoader.LoadConnectionAsync(serverdb)
|
||||
.ConfigureAwait(false);
|
||||
await using var cmd = con.CreateCommand(@"
|
||||
SELECT ""DBPath""
|
||||
FROM ""Backup""
|
||||
");
|
||||
await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(CancellationToken.None).ConfigureAwait(false))
|
||||
dbpaths.Add(rd.ConvertValueToString(0) ?? "");
|
||||
}
|
||||
catch
|
||||
@@ -86,16 +90,30 @@ public static class Helper
|
||||
/// Examines the database and returns the version and whether it is a server database
|
||||
/// </summary>
|
||||
/// <param name="db">The path to the database</param>
|
||||
/// <returns>A tuple containing the version and whether it is a server database</returns>
|
||||
public static (int Version, bool isserver) ExamineDatabase(string db)
|
||||
/// <returns>A task that when awaited contains a tuple with the version and whether it is a server database</returns>
|
||||
public static async Task<(int Version, bool isserver)> ExamineDatabase(string db)
|
||||
{
|
||||
using (var con = SQLiteLoader.LoadConnection(db, 0))
|
||||
{
|
||||
using var cmd = con.CreateCommand();
|
||||
var isserverdb = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='Backup' OR name='Schedule'") == 2;
|
||||
var version = (int)cmd.ExecuteScalarInt64("SELECT MAX(Version) FROM Version");
|
||||
return (version, isserverdb);
|
||||
}
|
||||
await using var con = await SQLiteLoader.LoadConnectionAsync(db)
|
||||
.ConfigureAwait(false);
|
||||
await using var cmd = con.CreateCommand();
|
||||
|
||||
var isserverdb = await cmd.ExecuteScalarInt64Async(@"
|
||||
SELECT COUNT(*)
|
||||
FROM sqlite_master
|
||||
WHERE
|
||||
type='table'
|
||||
AND name='Backup'
|
||||
OR name='Schedule'
|
||||
", CancellationToken.None)
|
||||
.ConfigureAwait(false) == 2;
|
||||
|
||||
var version = (int)await cmd.ExecuteScalarInt64Async(@"
|
||||
SELECT MAX(Version)
|
||||
FROM Version
|
||||
", CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return (version, isserverdb);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -10,6 +10,8 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="SQLitePCLRaw.bundle_green" Version="2.1.10" />
|
||||
<PackageReference Include="sqlite-net-pcl" Version="1.9.172" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||
<PackageReference Include="uplink.NET" Version="2.13.3484" />
|
||||
<PackageReference Include="uplink.NET.Win" Version="2.13.3484" />
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Logging;
|
||||
using Duplicati.Library.Main.Database;
|
||||
|
||||
@@ -17,7 +19,7 @@ partial class BackendManager
|
||||
/// Since the backend communication is done in parallel with the actual operations,
|
||||
/// we store the performed operations in a queue and flush them to the database when the database is available.
|
||||
/// The main operations are responsible for flushing the messages when comitting a transaction.
|
||||
/// If the operation fails, the logged messages here should still be flushed to the database,
|
||||
/// If the operation fails, the logged messages here should still be flushed to the database,
|
||||
/// as they have already been performed on the remote destination.
|
||||
/// </summary>
|
||||
private class DatabaseCollector
|
||||
@@ -111,12 +113,12 @@ partial class BackendManager
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flushes all messages to the database
|
||||
/// Flushes all messages to the database.
|
||||
/// </summary>
|
||||
/// <param name="db">The database to write pending messages to</param>
|
||||
/// <param name="transaction">The transaction to use, if any</param>
|
||||
/// <returns>Whether any messages were flushed</returns>
|
||||
public bool FlushPendingMessages(LocalDatabase db, System.Data.IDbTransaction? transaction)
|
||||
/// <param name="db">The database to write pending messages to.</param>
|
||||
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains whether any messages were flushed.</returns>
|
||||
public async Task<bool> FlushPendingMessages(LocalDatabase db, CancellationToken cancellationToken)
|
||||
{
|
||||
List<IRemoteOperationEntry> entries;
|
||||
lock (m_dbqueuelock)
|
||||
@@ -134,22 +136,32 @@ partial class BackendManager
|
||||
// As we replace the list, we can now freely access the elements without locking
|
||||
foreach (var e in entries)
|
||||
if (e is RemoteOperationLogEntry operation)
|
||||
db.LogRemoteOperation(operation.Action, operation.File, operation.Result, transaction);
|
||||
await db.LogRemoteOperation(operation.Action, operation.File, operation.Result, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
else if (e is RemoteVolumeUpdate update && update.State == RemoteVolumeState.Deleted)
|
||||
{
|
||||
db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), null, transaction);
|
||||
await db.UpdateRemoteVolume(update.Remotename, RemoteVolumeState.Deleted, update.Size, update.Hash, true, TimeSpan.FromHours(2), null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
volsRemoved.Add(update.Remotename);
|
||||
}
|
||||
else if (e is RemoteVolumeUpdate dbUpdate)
|
||||
db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash, transaction);
|
||||
await db.UpdateRemoteVolume(dbUpdate.Remotename, dbUpdate.State, dbUpdate.Size, dbUpdate.Hash, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
else if (e is RenameRemoteVolume rename)
|
||||
db.RenameRemoteFile(rename.Oldname, rename.Newname, transaction);
|
||||
await db.RenameRemoteFile(rename.Oldname, rename.Newname, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
else if (e != null)
|
||||
Log.WriteErrorMessage(LOGTAG, "InvalidQueueElement", null, "Queue had element of type: {0}, {1}", e.GetType(), e);
|
||||
|
||||
// Finally remove volumes from DB.
|
||||
if (volsRemoved.Count > 0)
|
||||
db.RemoveRemoteVolumes(volsRemoved);
|
||||
{
|
||||
await db.RemoveRemoteVolumes(volsRemoved, cancellationToken).ConfigureAwait(false);
|
||||
// This commit is to mimic the pre Microsoft.Data.Sqlite backend
|
||||
// behavior, in which RemoveRemoteVolumes started a new
|
||||
// transaction, if none was passed down.
|
||||
await db.Transaction.CommitAsync(token: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -98,14 +98,14 @@ partial class BackendManager
|
||||
/// <summary>
|
||||
/// A callback that is invoked when the index volume is finished
|
||||
/// </summary>
|
||||
public required Action? IndexVolumeFinishedCallback { get; init; }
|
||||
public required Func<Task>? IndexVolumeFinishedCallback { get; init; }
|
||||
/// <summary>
|
||||
/// The default callback for database updates (no-op)
|
||||
/// </summary>
|
||||
public static Func<Task> OnDbUpdateDefault = () => Task.CompletedTask;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new put operation
|
||||
/// Creates a new put operation
|
||||
/// </summary>
|
||||
/// <param name="context">The execution context</param>
|
||||
/// <param name="waitForComplete">True if the operation should wait for completion</param>
|
||||
@@ -234,7 +234,8 @@ partial class BackendManager
|
||||
|
||||
|
||||
OriginalIndexFile.FinishVolume(hash, size);
|
||||
IndexVolumeFinishedCallback?.Invoke();
|
||||
if (IndexVolumeFinishedCallback != null)
|
||||
await IndexVolumeFinishedCallback();
|
||||
OriginalIndexFile.Close();
|
||||
|
||||
indexOperation = (OriginalIndexFile, req2);
|
||||
@@ -344,7 +345,7 @@ partial class BackendManager
|
||||
Context.Database.LogRemoteVolumeRenamed(oldname, newname);
|
||||
remoteFilename = newname;
|
||||
|
||||
// If there is an index file attached to the block file,
|
||||
// If there is an index file attached to the block file,
|
||||
// it references the block filename, so we create a new index file
|
||||
// which is a copy of the current, but with the new name
|
||||
if (indexOperation != null)
|
||||
|
||||
@@ -246,7 +246,7 @@ internal partial class BackendManager : IBackendManager
|
||||
/// <param name="onDbUpdate">The callback to call when the database is updated</param>
|
||||
/// <param name="cancelToken">The cancellation token</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
public async Task PutAsync(VolumeWriterBase volume, IndexVolumeWriter? indexVolume, Action? indexVolumeFinished, bool waitForComplete, Func<Task>? onDbUpdate, CancellationToken cancelToken)
|
||||
public async Task PutAsync(VolumeWriterBase volume, IndexVolumeWriter? indexVolume, Func<Task>? indexVolumeFinished, bool waitForComplete, Func<Task>? onDbUpdate, CancellationToken cancelToken)
|
||||
{
|
||||
volume.Close();
|
||||
|
||||
@@ -292,23 +292,21 @@ internal partial class BackendManager : IBackendManager
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Flushes the database messages to the database
|
||||
/// Flushes the database messages to the database.
|
||||
/// </summary>
|
||||
/// <param name="database">The database to write to</param>
|
||||
/// <param name="transaction">The transaction to use</param>
|
||||
/// <param name="cancellationToken">The cancellation token</param>
|
||||
/// <returns></returns>
|
||||
public Task FlushPendingMessagesAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken)
|
||||
/// <param name="database">The database to write to.</param>
|
||||
/// <param name="cancellationToken">The cancellation token.</param>
|
||||
/// <returns>A task that completes when the messages are flushed.</returns>
|
||||
public async Task FlushPendingMessagesAsync(LocalDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
context.Database.FlushPendingMessages(database, transaction);
|
||||
return Task.CompletedTask;
|
||||
await context.Database.FlushPendingMessages(database, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Waits for the backend queue to be empty
|
||||
/// Waits for the backend queue to be empty.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">The cancellation token</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
/// <param name="cancellationToken">The cancellation token.</param>
|
||||
/// <returns>An awaitable task.</returns>
|
||||
public async Task WaitForEmptyAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var op = new WaitForEmptyOperation(context, cancellationToken);
|
||||
@@ -323,22 +321,22 @@ internal partial class BackendManager : IBackendManager
|
||||
/// <param name="transaction">The transaction to use</param>
|
||||
/// <param name="cancellationToken">The cancellation token</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
public async Task WaitForEmptyAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken)
|
||||
public async Task WaitForEmptyAsync(LocalDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
await FlushPendingMessagesAsync(database, transaction, cancellationToken).ConfigureAwait(false);
|
||||
await FlushPendingMessagesAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
await WaitForEmptyAsync(cancellationToken).ConfigureAwait(false);
|
||||
await FlushPendingMessagesAsync(database, transaction, cancellationToken).ConfigureAwait(false);
|
||||
await FlushPendingMessagesAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops the backend manager and flushes any pending messages to the database
|
||||
/// Stops the backend manager and flushes any pending messages to the database.
|
||||
/// </summary>
|
||||
/// <param name="database">The database to write pending messages to</param>
|
||||
/// <param name="transaction">The transaction to use, if any</param>
|
||||
public async Task StopRunnerAndFlushMessages(LocalDatabase database, IDbTransaction? transaction)
|
||||
/// <param name="database">The database to write pending messages to.</param>
|
||||
/// <returns>A task that completes when the runner is stopped and messages are flushed.</returns>
|
||||
public async Task StopRunnerAndFlushMessages(LocalDatabase database)
|
||||
{
|
||||
await requestChannel.RetireAsync().ConfigureAwait(false);
|
||||
await FlushPendingMessagesAsync(database, transaction, CancellationToken.None).ConfigureAwait(false);
|
||||
await FlushPendingMessagesAsync(database, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
if (queueRunner.IsFaulted)
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, "BackendManagerShutdown", queueRunner.Exception, "Backend manager queue runner crashed");
|
||||
@@ -368,7 +366,8 @@ internal partial class BackendManager : IBackendManager
|
||||
yield break;
|
||||
|
||||
// Get the first volume, so we do not have pending parallel transfers
|
||||
var prevResult = await GetWithInfoAsync(prevVolume.Name, prevVolume.Hash, prevVolume.Size, cancelToken);
|
||||
var prevResult = await GetWithInfoAsync(prevVolume.Name, prevVolume.Hash, prevVolume.Size, cancelToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
foreach (var volume in volumes.Skip(1))
|
||||
{
|
||||
@@ -381,7 +380,7 @@ internal partial class BackendManager : IBackendManager
|
||||
|
||||
// Set up for next iteration
|
||||
prevVolume = volume;
|
||||
prevResult = await nextTask;
|
||||
prevResult = await nextTask.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Return the last result
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -252,9 +252,9 @@ namespace Duplicati.Library.Main
|
||||
{
|
||||
return RunAction(new ListRemoteResults(), async (result, backendManager) =>
|
||||
{
|
||||
using (var tf = File.Exists(m_options.Dbpath) ? null : new TempFile())
|
||||
using (var db = new LocalDatabase(((string)tf) ?? m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache))
|
||||
result.SetResult(await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false));
|
||||
using var tf = File.Exists(m_options.Dbpath) ? null : new TempFile();
|
||||
await using var db = await LocalDatabase.CreateLocalDatabaseAsync(((string)tf) ?? m_options.Dbpath, "list-remote", true, null, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
result.SetResult(await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -277,12 +277,15 @@ namespace Duplicati.Library.Main
|
||||
// and uses the same prefix (see issues #2678, #3845, and #4244).
|
||||
if (File.Exists(m_options.Dbpath))
|
||||
{
|
||||
using (LocalDatabase db = new LocalDatabase(m_options.Dbpath, "list-remote", true, m_options.SqlitePageCache))
|
||||
{
|
||||
var dbRemoteVolumes = db.GetRemoteVolumes();
|
||||
var dbRemoteFiles = dbRemoteVolumes.Select(x => x.Name).ToHashSet();
|
||||
list = list.Where(x => dbRemoteFiles.Contains(x.File.Name)).ToList();
|
||||
}
|
||||
await using LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "list-remote", true, null, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
var dbRemoteVolumes = db.GetRemoteVolumes(result.TaskControl.ProgressToken);
|
||||
var dbRemoteFiles = await dbRemoteVolumes
|
||||
.Select(x => x.Name)
|
||||
.ToHashSetAsync(cancellationToken: result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
list = list.Where(x =>
|
||||
dbRemoteFiles.Contains(x.File.Name)
|
||||
).ToList();
|
||||
}
|
||||
|
||||
result.OperationProgressUpdater.UpdatePhase(OperationPhase.Delete_Deleting);
|
||||
@@ -517,10 +520,10 @@ namespace Duplicati.Library.Main
|
||||
// This would also allow us to control the unclean shutdown flag,
|
||||
// by toggling this on start and completion of transfers in the manager,
|
||||
// instead of relying on the operations to correctly toggle the flag
|
||||
if (LocalDatabase.Exists(m_options.Dbpath) && !m_options.NoLocalDb)
|
||||
if (File.Exists(m_options.Dbpath) && !m_options.NoLocalDb)
|
||||
{
|
||||
using (var db = new LocalDatabase(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache))
|
||||
backend.StopRunnerAndFlushMessages(db, null).Await();
|
||||
using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, null, CancellationToken.None).Await();
|
||||
backend.StopRunnerAndFlushMessages(db).Await();
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -533,29 +536,27 @@ namespace Duplicati.Library.Main
|
||||
resultSetter.EndTime = DateTime.UtcNow;
|
||||
resultSetter.Interrupted = false;
|
||||
|
||||
if (LocalDatabase.Exists(m_options.Dbpath) && !m_options.Dryrun)
|
||||
if (File.Exists(m_options.Dbpath) && !m_options.Dryrun)
|
||||
{
|
||||
using (var db = new LocalDatabase(m_options.Dbpath, null, true, m_options.SqlitePageCache))
|
||||
{
|
||||
db.WriteResults(result);
|
||||
db.PurgeLogData(m_options.LogRetention);
|
||||
db.PurgeDeletedVolumes(DateTime.UtcNow);
|
||||
using var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, null, CancellationToken.None).Await();
|
||||
db.WriteResults(result, CancellationToken.None).Await();
|
||||
db.PurgeLogData(m_options.LogRetention, CancellationToken.None).Await();
|
||||
db.PurgeDeletedVolumes(DateTime.UtcNow, CancellationToken.None).Await();
|
||||
|
||||
// Vacuum is done AFTER the results are written to the database
|
||||
// This means that the information about the vacuum is not stored in the database,
|
||||
// but will be reported in the log output and messages sent with any of the reporting modules
|
||||
if (m_options.AutoVacuum && result is IResultsWithVacuum vacuumResults && result is BasicResults basicResults)
|
||||
// Vacuum is done AFTER the results are written to the database
|
||||
// This means that the information about the vacuum is not stored in the database,
|
||||
// but will be reported in the log output and messages sent with any of the reporting modules
|
||||
if (m_options.AutoVacuum && result is IResultsWithVacuum vacuumResults && result is BasicResults basicResults)
|
||||
{
|
||||
try
|
||||
{
|
||||
try
|
||||
{
|
||||
vacuumResults.VacuumResults = new VacuumResults(basicResults);
|
||||
new Operation.VacuumHandler(m_options, (VacuumResults)vacuumResults.VacuumResults)
|
||||
.RunAsync().Await();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToVacuum", ex, "Failed to vacuum database");
|
||||
}
|
||||
vacuumResults.VacuumResults = new VacuumResults(basicResults);
|
||||
new Operation.VacuumHandler(m_options, (VacuumResults)vacuumResults.VacuumResults)
|
||||
.RunAsync().Await();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteVerboseMessage(LOGTAG, "FailedToVacuum", ex, "Failed to vacuum database");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -578,9 +579,9 @@ namespace Duplicati.Library.Main
|
||||
try
|
||||
{
|
||||
// No operation was started in database, so write logs to new operation
|
||||
if (LocalDatabase.Exists(m_options.Dbpath) && !m_options.Dryrun)
|
||||
using (var db = new LocalDatabase(m_options.Dbpath, result.MainOperation.ToString(), true, m_options.SqlitePageCache))
|
||||
db.WriteResults(result);
|
||||
if (File.Exists(m_options.Dbpath) && !m_options.Dryrun)
|
||||
using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, result.MainOperation.ToString(), true, null, CancellationToken.None).Await())
|
||||
db.WriteResults(result, CancellationToken.None).Await();
|
||||
}
|
||||
catch (Exception we)
|
||||
{
|
||||
@@ -603,9 +604,9 @@ namespace Duplicati.Library.Main
|
||||
|
||||
resultSetter.Fatal = true;
|
||||
// Write logs to previous operation if database exists
|
||||
if (LocalDatabase.Exists(m_options.Dbpath) && !m_options.Dryrun)
|
||||
using (var db = new LocalDatabase(m_options.Dbpath, null, true, m_options.SqlitePageCache))
|
||||
db.WriteResults(result);
|
||||
if (File.Exists(m_options.Dbpath) && !m_options.Dryrun)
|
||||
using (var db = LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, null, true, null, CancellationToken.None).Await())
|
||||
db.WriteResults(result, CancellationToken.None).Await();
|
||||
}
|
||||
catch (Exception we)
|
||||
{
|
||||
@@ -756,7 +757,7 @@ namespace Duplicati.Library.Main
|
||||
private async Task ApplySecretProvider(CancellationToken cancellationToken)
|
||||
{
|
||||
var args = new[] { new Library.Utility.Uri(m_backendUrl) };
|
||||
await SecretProviderHelper.ApplySecretProviderAsync([], args, m_options.RawOptions, TempFolder.SystemTempPath, SecretProvider, cancellationToken);
|
||||
await SecretProviderHelper.ApplySecretProviderAsync([], args, m_options.RawOptions, TempFolder.SystemTempPath, SecretProvider, cancellationToken).ConfigureAwait(false);
|
||||
// Write back the backend argument, if it was modified by the secret provider
|
||||
m_backendUrl = args[0].ToString();
|
||||
}
|
||||
|
||||
@@ -1,27 +1,29 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.Data;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace Duplicati.Library.Main.Database;
|
||||
|
||||
@@ -36,132 +38,197 @@ public static class ChangeStatistics
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ChangeStatistics));
|
||||
|
||||
/// <summary>
|
||||
/// Calculates the change statistics for the current and previous fileset
|
||||
/// Calculates the change statistics for the current and previous fileset.
|
||||
/// </summary>
|
||||
/// <param name="cmd">The database command to use</param>
|
||||
/// <param name="results">The results object to update with the statistics</param>
|
||||
/// <param name="currentFilesetId">The ID of the current fileset</param>
|
||||
/// <param name="previousFilesetId">The ID of the previous fileset</param>
|
||||
internal static void UpdateChangeStatistics(IDbCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId)
|
||||
/// <param name="cmd">The database command to use.</param>
|
||||
/// <param name="results">The results object to update with the statistics.</param>
|
||||
/// <param name="currentFilesetId">The ID of the current fileset.</param>
|
||||
/// <param name="previousFilesetId">The ID of the previous fileset.</param>
|
||||
/// <param name="token">The cancellation token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that completes when the statistics are updated.</returns>
|
||||
internal static async Task UpdateChangeStatistics(SqliteCommand cmd, BackupResults results, long currentFilesetId, long previousFilesetId, CancellationToken token)
|
||||
{
|
||||
var tmpName = $"TmpFileState_{Guid.NewGuid():N}";
|
||||
|
||||
try
|
||||
{
|
||||
// Create temp table
|
||||
cmd.SetCommandAndParameters(LocalDatabase.FormatInvariant($@"
|
||||
CREATE TEMP TABLE ""{tmpName}"" AS
|
||||
SELECT
|
||||
FL.""PrefixID"",
|
||||
FL.""Path"",
|
||||
FL.""BlocksetID"",
|
||||
BS_Meta.""Fullhash"" AS ""Metahash"",
|
||||
CASE FE.""FilesetID""
|
||||
WHEN @LastFilesetId THEN 0
|
||||
WHEN @CurrentFilesetId THEN 1
|
||||
END AS Source
|
||||
FROM ""FileLookup"" FL
|
||||
JOIN ""FilesetEntry"" FE ON FL.""ID"" = FE.""FileID""
|
||||
LEFT JOIN ""Blockset"" BS_Data ON FL.""BlocksetID"" = BS_Data.""ID""
|
||||
LEFT JOIN ""Metadataset"" M ON FL.""MetadataID"" = M.""ID""
|
||||
LEFT JOIN ""Blockset"" BS_Meta ON M.""BlocksetID"" = BS_Meta.""ID""
|
||||
WHERE FE.""FilesetID"" IN (@LastFilesetId, @CurrentFilesetId);
|
||||
"))
|
||||
.SetParameterValue("@LastFilesetId", previousFilesetId)
|
||||
.SetParameterValue("@CurrentFilesetId", currentFilesetId)
|
||||
.ExecuteNonQuery();
|
||||
await cmd.SetCommandAndParameters($@"
|
||||
CREATE TEMP TABLE ""{tmpName}"" AS
|
||||
SELECT
|
||||
""FL"".""PrefixID"",
|
||||
""FL"".""Path"",
|
||||
""FL"".""BlocksetID"",
|
||||
""BS_Meta"".""Fullhash"" AS ""Metahash"",
|
||||
CASE ""FE"".""FilesetID""
|
||||
WHEN @LastFilesetId THEN 0
|
||||
WHEN @CurrentFilesetId THEN 1
|
||||
END AS ""Source""
|
||||
FROM ""FileLookup"" ""FL""
|
||||
JOIN ""FilesetEntry"" ""FE""
|
||||
ON ""FL"".""ID"" = ""FE"".""FileID""
|
||||
LEFT JOIN ""Blockset"" ""BS_Data""
|
||||
ON ""FL"".""BlocksetID"" = ""BS_Data"".""ID""
|
||||
LEFT JOIN ""Metadataset"" M
|
||||
ON ""FL"".""MetadataID"" = M.""ID""
|
||||
LEFT JOIN ""Blockset"" ""BS_Meta""
|
||||
ON M.""BlocksetID"" = ""BS_Meta"".""ID""
|
||||
WHERE ""FE"".""FilesetID"" IN (
|
||||
@LastFilesetId,
|
||||
@CurrentFilesetId
|
||||
);
|
||||
")
|
||||
.SetParameterValue("@LastFilesetId", previousFilesetId)
|
||||
.SetParameterValue("@CurrentFilesetId", currentFilesetId)
|
||||
.ExecuteNonQueryAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Index for fast comparison
|
||||
cmd.ExecuteNonQuery(LocalDatabase.FormatInvariant($@"CREATE INDEX ""idx_{tmpName}"" ON ""{tmpName}"" (""PrefixID"", ""Path"")"));
|
||||
await cmd.ExecuteNonQueryAsync($@"
|
||||
CREATE INDEX ""idx_{tmpName}""
|
||||
ON ""{tmpName}"" (
|
||||
""PrefixID"",
|
||||
""Path""
|
||||
)
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Added
|
||||
results.AddedFolders = CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID);
|
||||
results.AddedSymlinks = CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID);
|
||||
results.AddedFiles = CountAdded(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID });
|
||||
results.AddedFolders = await CountAdded(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token)
|
||||
.ConfigureAwait(false);
|
||||
results.AddedSymlinks = await CountAdded(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token)
|
||||
.ConfigureAwait(false);
|
||||
results.AddedFiles = await CountAdded(cmd, tmpName, null, [
|
||||
LocalDatabase.FOLDER_BLOCKSET_ID,
|
||||
LocalDatabase.SYMLINK_BLOCKSET_ID
|
||||
], token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Deleted
|
||||
results.DeletedFolders = CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID);
|
||||
results.DeletedSymlinks = CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID);
|
||||
results.DeletedFiles = CountDeleted(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID });
|
||||
results.DeletedFolders = await CountDeleted(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token)
|
||||
.ConfigureAwait(false);
|
||||
results.DeletedSymlinks = await CountDeleted(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token)
|
||||
.ConfigureAwait(false);
|
||||
results.DeletedFiles = await CountDeleted(cmd, tmpName, null, [
|
||||
LocalDatabase.FOLDER_BLOCKSET_ID,
|
||||
LocalDatabase.SYMLINK_BLOCKSET_ID
|
||||
], token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Modified
|
||||
results.ModifiedFolders = CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID);
|
||||
results.ModifiedSymlinks = CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID);
|
||||
results.ModifiedFiles = CountModified(cmd, tmpName, null, new[] { LocalDatabase.FOLDER_BLOCKSET_ID, LocalDatabase.SYMLINK_BLOCKSET_ID });
|
||||
results.ModifiedFolders = await CountModified(cmd, tmpName, LocalDatabase.FOLDER_BLOCKSET_ID, null, token)
|
||||
.ConfigureAwait(false);
|
||||
results.ModifiedSymlinks = await CountModified(cmd, tmpName, LocalDatabase.SYMLINK_BLOCKSET_ID, null, token)
|
||||
.ConfigureAwait(false);
|
||||
results.ModifiedFiles = await CountModified(cmd, tmpName, null, [
|
||||
LocalDatabase.FOLDER_BLOCKSET_ID,
|
||||
LocalDatabase.SYMLINK_BLOCKSET_ID
|
||||
], token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
try { cmd.ExecuteNonQuery(LocalDatabase.FormatInvariant($@"DROP TABLE IF EXISTS ""{tmpName}"";")); }
|
||||
catch (Exception ex) { Logging.Log.WriteWarningMessage(LOGTAG, "DropTemp", ex, $"Failed to drop {tmpName}"); }
|
||||
try
|
||||
{
|
||||
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tmpName}"";", token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, "DropTemp", ex, $"Failed to drop {tmpName}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Counts the number of added files or folders
|
||||
/// Counts the number of added files or folders.
|
||||
/// </summary>
|
||||
/// <param name="cmd">The database command to use</param>
|
||||
/// <param name="tmpName">The name of the temporary table</param>
|
||||
/// <param name="blocksetId">The ID of the blockset to count, or null for all</param>
|
||||
/// <param name="excludeBlocksets">The blocksets to exclude from the count</param>
|
||||
/// <returns>The number of added files or folders</returns>
|
||||
private static long CountAdded(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null)
|
||||
/// <param name="cmd">The database command to use.</param>
|
||||
/// <param name="tmpName">The name of the temporary table.</param>
|
||||
/// <param name="blocksetId">The ID of the blockset to count, or null for all.</param>
|
||||
/// <param name="excludeBlocksets">The blocksets to exclude from the count.</param>
|
||||
/// <param name="token">The cancellation token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that when awaited contains the number of added files or folders.</returns>
|
||||
private static async Task<long> CountAdded(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token)
|
||||
{
|
||||
var conditions = LocalDatabase.FormatInvariant($@"Source = 1 AND NOT EXISTS (
|
||||
SELECT 1 FROM ""{tmpName}"" B
|
||||
WHERE B.Source = 0
|
||||
AND B.PrefixID = A.PrefixID
|
||||
AND B.Path = A.Path
|
||||
)");
|
||||
var conditions = $@"
|
||||
""Source"" = 1
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM ""{tmpName}"" ""B""
|
||||
WHERE
|
||||
""B"".""Source"" = 0
|
||||
AND ""B"".""PrefixID"" = ""A"".""PrefixID""
|
||||
AND ""B"".""Path"" = ""A"".""Path""
|
||||
)
|
||||
";
|
||||
|
||||
return CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets);
|
||||
return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets, token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Counts the number of deleted files or folders
|
||||
/// Counts the number of deleted files or folders.
|
||||
/// </summary>
|
||||
/// <param name="cmd">The database command to use</param>
|
||||
/// <param name="tmpName">The name of the temporary table</param>
|
||||
/// <param name="blocksetId">The ID of the blockset to count, or null for all</param>
|
||||
/// <param name="excludeBlocksets">The blocksets to exclude from the count</param>
|
||||
/// <returns>The number of deleted files or folders</returns>
|
||||
private static long CountDeleted(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null)
|
||||
/// <param name="cmd">The database command to use.</param>
|
||||
/// <param name="tmpName">The name of the temporary table.</param>
|
||||
/// <param name="blocksetId">The ID of the blockset to count, or null for all.</param>
|
||||
/// <param name="excludeBlocksets">The blocksets to exclude from the count.</param>
|
||||
/// <param name="token">The cancellation token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that when awaited contains the number of deleted files or folders.</returns>
|
||||
private static async Task<long> CountDeleted(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token)
|
||||
{
|
||||
var conditions = @"Source = 0 AND NOT EXISTS (
|
||||
SELECT 1 FROM """ + tmpName + @""" B
|
||||
WHERE B.Source = 1
|
||||
AND B.PrefixID = A.PrefixID
|
||||
AND B.Path = A.Path
|
||||
)";
|
||||
var conditions = @$"
|
||||
""Source"" = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM ""{tmpName}"" ""B""
|
||||
WHERE
|
||||
""B"".""Source"" = 1
|
||||
AND ""B"".""PrefixID"" = ""A"".""PrefixID""
|
||||
AND ""B"".""Path"" = ""A"".""Path""
|
||||
)
|
||||
";
|
||||
|
||||
return CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets);
|
||||
return await CountWithCondition(cmd, tmpName, "A", conditions, blocksetId, excludeBlocksets, token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Counts the number of modified files or folders
|
||||
/// Counts the number of modified files or folders.
|
||||
/// </summary>
|
||||
/// <param name="cmd">The database command to use</param>
|
||||
/// <param name="tmpName">The name of the temporary table</param>
|
||||
/// <param name="blocksetId">The ID of the blockset to count, or null for all</param>
|
||||
/// <param name="hash1">The hash column to compare</param>
|
||||
/// <param name="excludeBlocksets">The blocksets to exclude from the count</param>
|
||||
/// <returns>The number of modified files or folders</returns>
|
||||
private static long CountModified(IDbCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets = null)
|
||||
/// <param name="cmd">The database command to use.</param>
|
||||
/// <param name="tmpName">The name of the temporary table.</param>
|
||||
/// <param name="blocksetId">The ID of the blockset to count, or null for all.</param>
|
||||
/// <param name="excludeBlocksets">The blocksets to exclude from the count.</param>
|
||||
/// <param name="token">The cancellation token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that when awaited contains the number of modified files or folders.</returns>
|
||||
private static async Task<long> CountModified(SqliteCommand cmd, string tmpName, long? blocksetId, long[]? excludeBlocksets, CancellationToken token)
|
||||
{
|
||||
string conditions;
|
||||
|
||||
if (blocksetId == LocalDatabase.FOLDER_BLOCKSET_ID || blocksetId == LocalDatabase.SYMLINK_BLOCKSET_ID)
|
||||
{
|
||||
conditions = @"
|
||||
A.Source = 0 AND B.Source = 1
|
||||
AND A.PrefixID = B.PrefixID AND A.Path = B.Path
|
||||
AND A.Metahash IS NOT B.Metahash";
|
||||
""A"".""Source"" = 0
|
||||
AND ""B"".""Source"" = 1
|
||||
AND ""A"".""PrefixID"" = ""B"".""PrefixID""
|
||||
AND ""A"".""Path"" = ""B"".""Path""
|
||||
AND ""A"".""Metahash"" IS NOT ""B"".""Metahash""
|
||||
";
|
||||
}
|
||||
else
|
||||
{
|
||||
conditions = @"
|
||||
A.Source = 0 AND B.Source = 1
|
||||
AND A.PrefixID = B.PrefixID AND A.Path = B.Path
|
||||
AND (A.BlocksetID IS NOT B.BlocksetID OR A.Metahash IS NOT B.Metahash)";
|
||||
""A"".""Source"" = 0
|
||||
AND ""B"".""Source"" = 1
|
||||
AND ""A"".""PrefixID"" = ""B"".""PrefixID""
|
||||
AND ""A"".""Path"" = ""B"".""Path""
|
||||
AND (
|
||||
""A"".""BlocksetID"" IS NOT ""B"".""BlocksetID""
|
||||
OR ""A"".""Metahash"" IS NOT ""B"".""Metahash""
|
||||
)
|
||||
";
|
||||
}
|
||||
|
||||
string blocksetCondition = GetBlocksetCondition("A", blocksetId, excludeBlocksets);
|
||||
@@ -169,33 +236,48 @@ public static class ChangeStatistics
|
||||
conditions += $" AND {blocksetCondition}";
|
||||
|
||||
string sql = $@"
|
||||
SELECT COUNT(*)
|
||||
FROM ""{tmpName}"" A
|
||||
JOIN ""{tmpName}"" B ON A.PrefixID = B.PrefixID AND A.Path = B.Path
|
||||
WHERE {conditions}";
|
||||
SELECT COUNT(*)
|
||||
FROM ""{tmpName}"" ""A""
|
||||
JOIN ""{tmpName}"" ""B""
|
||||
ON ""A"".""PrefixID"" = ""B"".""PrefixID""
|
||||
AND ""A"".""Path"" = ""B"".""Path""
|
||||
WHERE {conditions}
|
||||
";
|
||||
|
||||
return cmd.SetCommandAndParameters(sql).ExecuteScalarInt64(0);
|
||||
return await cmd
|
||||
.SetCommandAndParameters(sql)
|
||||
.ExecuteScalarInt64Async(0, token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Counts the number of files or folders with a specific condition
|
||||
/// Counts the number of files or folders with a specific condition.
|
||||
/// </summary>
|
||||
/// <param name="cmd">The database command to use</param>
|
||||
/// <param name="tmpName">The name of the temporary table</param>
|
||||
/// <param name="alias">The alias for the temporary table</param>
|
||||
/// <param name="baseCondition">The base condition for the count</param>
|
||||
/// <param name="blocksetId">The ID of the blockset to count, or null for all</param>
|
||||
/// <param name="excludeBlocksets">The blocksets to exclude from the count</param>
|
||||
/// <returns>The number of files or folders that match the condition</returns>
|
||||
private static long CountWithCondition(IDbCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets)
|
||||
/// <param name="cmd">The database command to use.</param>
|
||||
/// <param name="tmpName">The name of the temporary table.</param>
|
||||
/// <param name="alias">The alias for the temporary table.</param>
|
||||
/// <param name="baseCondition">The base condition for the count.</param>
|
||||
/// <param name="blocksetId">The ID of the blockset to count, or null for all.</param>
|
||||
/// <param name="excludeBlocksets">The blocksets to exclude from the count.</param>
|
||||
/// <param name="token">The cancellation token to monitor for cancellation requests.</param>
|
||||
/// <returns>A task that when awaited contains the number of files or folders that match the condition.</returns>
|
||||
private static async Task<long> CountWithCondition(SqliteCommand cmd, string tmpName, string alias, string baseCondition, long? blocksetId, long[]? excludeBlocksets, CancellationToken token)
|
||||
{
|
||||
var fullCondition = baseCondition;
|
||||
var blocksetCondition = GetBlocksetCondition(alias, blocksetId, excludeBlocksets);
|
||||
if (!string.IsNullOrEmpty(blocksetCondition))
|
||||
fullCondition += " AND " + blocksetCondition;
|
||||
|
||||
var sql = LocalDatabase.FormatInvariant($@"SELECT COUNT(*) FROM ""{tmpName}"" {alias} WHERE {fullCondition}");
|
||||
return cmd.SetCommandAndParameters(sql).ExecuteScalarInt64(0);
|
||||
var sql = $@"
|
||||
SELECT COUNT(*)
|
||||
FROM ""{tmpName}"" ""{alias}""
|
||||
WHERE {fullCondition}
|
||||
";
|
||||
|
||||
return await cmd
|
||||
.SetCommandAndParameters(sql)
|
||||
.ExecuteScalarInt64Async(0, token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -208,9 +290,9 @@ public static class ChangeStatistics
|
||||
private static string GetBlocksetCondition(string alias, long? blocksetId, long[]? exclude)
|
||||
{
|
||||
if (blocksetId.HasValue)
|
||||
return LocalDatabase.FormatInvariant($"{alias}.\"BlocksetID\" = {blocksetId.Value}");
|
||||
return @$"{alias}.""BlocksetID"" = {blocksetId.Value}";
|
||||
if (exclude?.Length > 0)
|
||||
return LocalDatabase.FormatInvariant($"{alias}.\"BlocksetID\" NOT IN ({string.Join(",", exclude)})");
|
||||
return @$"{alias}.""BlocksetID"" NOT IN ({string.Join(",", exclude)})";
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
@@ -1,28 +1,32 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Utility;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
#nullable enable
|
||||
|
||||
@@ -31,13 +35,674 @@ namespace Duplicati.Library.Main.Database;
|
||||
/// <summary>
|
||||
/// Extension method for <see cref="IDbCommand"/>
|
||||
/// </summary>
|
||||
public static class ExtensionMethods
|
||||
public static partial class ExtensionMethods
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// Converts the value at the given index in the reader to an Int64.
|
||||
/// </summary>
|
||||
/// <param name="reader">The <see cref="SqliteDataReader"/> to read from.</param>
|
||||
/// <param name="index">The index of the value to convert.</param>
|
||||
/// <param name="defaultvalue">The default value to return if the value is null or cannot be converted.</param>
|
||||
/// <returns>The converted Int64 value, or the default value if the conversion fails.</returns>
|
||||
public static long ConvertValueToInt64(this SqliteDataReader reader, int index, long defaultvalue = -1)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!reader.IsDBNull(index))
|
||||
return reader.GetInt64(index);
|
||||
}
|
||||
catch { }
|
||||
|
||||
return defaultvalue;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts the value at the given index in the reader to a string.
|
||||
/// </summary>
|
||||
/// <param name="reader">The <see cref="SqliteDataReader"/> to read from.</param>
|
||||
/// <param name="index">The index of the value to convert.</param>
|
||||
/// <returns>The converted string value, or null if the value is null or cannot be converted.</returns>
|
||||
public static string? ConvertValueToString(this SqliteDataReader reader, int index)
|
||||
{
|
||||
var v = reader.GetValue(index);
|
||||
if (v == null || v == DBNull.Value)
|
||||
return null;
|
||||
|
||||
return v.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new <see cref="SqliteCommand"/> with the given command text.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteConnection"/> to create the command for.</param>
|
||||
/// <param name="cmdtext">The command text to set for the command.</param>
|
||||
/// <returns>A new <see cref="SqliteCommand"/> with the command text set.</returns>
|
||||
public static SqliteCommand CreateCommand(this SqliteConnection self, string cmdtext)
|
||||
{
|
||||
return CreateCommandAsync(self, cmdtext, default).Await();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new <see cref="SqliteCommand"/> with the given command text and prepares it asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteConnection"/> to create the command for.</param>
|
||||
/// <param name="cmdtext">The command text to set for the command.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A new <see cref="SqliteCommand"/> with the command text set and prepared asynchronously.</returns>
|
||||
public static async Task<SqliteCommand> CreateCommandAsync(this SqliteConnection self, string cmdtext, CancellationToken cancellationToken)
|
||||
{
|
||||
var cmd = self.CreateCommand()
|
||||
.SetCommandAndParameters(cmdtext);
|
||||
|
||||
await cmd.PrepareAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return cmd;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new <see cref="SqliteCommand"/> with the given transaction.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteConnection"/> to create the command for.</param>
|
||||
/// <param name="transaction">The <see cref="SqliteTransaction"/> to set for the command.</param>
|
||||
/// <returns>A new <see cref="SqliteCommand"/> with the transaction set.</returns>
|
||||
public static SqliteCommand CreateCommand(this SqliteConnection self, SqliteTransaction transaction)
|
||||
{
|
||||
var cmd = self.CreateCommand();
|
||||
cmd.SetTransaction(transaction);
|
||||
|
||||
return cmd;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new <see cref="SqliteCommand"/> with the given reusable transaction.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteConnection"/> to create the command for.</param>
|
||||
/// <param name="rtr">The <see cref="ReusableTransaction"/> to set for the command.</param>
|
||||
/// <returns>A new <see cref="SqliteCommand"/> with the transaction set.</returns>
|
||||
internal static SqliteCommand CreateCommand(this SqliteConnection self, ReusableTransaction rtr)
|
||||
{
|
||||
return self.CreateCommand(rtr.Transaction);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the number of rows affected.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmdtext">The command text to execute.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the number of rows affected.</returns>
|
||||
public static async Task<int> ExecuteNonQueryAsync(this SqliteCommand self, string? cmdtext, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteNonQueryAsync(self, true, cmdtext, null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the number of rows affected.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmd">The command text to execute.</param>
|
||||
/// <param name="values">The values to use as parameters.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the number of rows affected.</returns>
|
||||
public static async Task<int> ExecuteNonQueryAsync(this SqliteCommand self, string cmd, Dictionary<string, object?> values, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteNonQueryAsync(self, true, cmd, values, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the number of rows affected.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="writeLog">Whether to write a log entry.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the number of rows affected.</returns>
|
||||
public static async Task<int> ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteNonQueryAsync(self, writeLog, null, null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the number of rows affected.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="writeLog">Whether to write a log entry.</param>
|
||||
/// <param name="cmd">The command text to execute.</param>
|
||||
/// <param name="values">The values to set as parameters.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the number of rows affected.</returns>
|
||||
public static async Task<int> ExecuteNonQueryAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary<string, object?>? values, CancellationToken cancellationToken)
|
||||
{
|
||||
if (cmd != null)
|
||||
self.SetCommandAndParameters(cmd);
|
||||
|
||||
if (values != null && values.Count > 0)
|
||||
self.SetParameterValues(values);
|
||||
|
||||
using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteNonQueryAsync", string.Format("ExecuteNonQueryAsync: {0}", self.GetPrintableCommandText())) : null)
|
||||
return await self.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns a <see cref="SqliteDataReader"/>.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmdtext">The command text to execute.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the <see cref="SqliteDataReader"/>.</returns>
|
||||
public static async Task<SqliteDataReader> ExecuteReaderAsync(this SqliteCommand self, string cmdtext, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteReaderAsync(self, true, cmdtext, null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns a <see cref="SqliteDataReader"/>.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="writeLog">Whether to write a log entry.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the <see cref="SqliteDataReader"/>.</returns>
|
||||
public static async Task<SqliteDataReader> ExecuteReaderAsync(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteReaderAsync(self, writeLog, null, null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns a <see cref="SqliteDataReader"/>.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmdtext">The command text to execute.</param>
|
||||
/// <param name="values">The values to use as parameters.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the <see cref="SqliteDataReader"/>.</returns>
|
||||
public static async Task<SqliteDataReader> ExecuteReaderAsync(this SqliteCommand self, string cmdtext, Dictionary<string, object?>? values, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteReaderAsync(self, true, cmdtext, values, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns a <see cref="SqliteDataReader"/>.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="writeLog">Whether to write a log entry.</param>
|
||||
/// <param name="cmdtext">The command text to execute.</param>
|
||||
/// <param name="values">The values to set as parameters.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the <see cref="SqliteDataReader"/>.</returns>
|
||||
public static async Task<SqliteDataReader> ExecuteReaderAsync(this SqliteCommand self, bool writeLog, string? cmdtext, Dictionary<string, object?>? values, CancellationToken cancellationToken)
|
||||
{
|
||||
if (cmdtext != null)
|
||||
self.SetCommandAndParameters(cmdtext);
|
||||
|
||||
if (values != null && values.Count > 0)
|
||||
self.SetParameterValues(values);
|
||||
|
||||
using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReader", string.Format("ExecuteReader: {0}", self.GetPrintableCommandText())) : null)
|
||||
return await self.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns an enumerable of <see cref="SqliteDataReader"/>.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>An asynchronous enumerable of <see cref="SqliteDataReader"/>.</returns>
|
||||
public static IAsyncEnumerable<SqliteDataReader> ExecuteReaderEnumerableAsync(this SqliteCommand self, CancellationToken cancellationToken)
|
||||
{
|
||||
return ExecuteReaderEnumerableAsync(self, true, null, null, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns an enumerable of <see cref="SqliteDataReader"/>.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmdtext">The command text to execute.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>An asynchronous enumerable of <see cref="SqliteDataReader"/>.</returns>
|
||||
public static IAsyncEnumerable<SqliteDataReader> ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext, CancellationToken cancellationToken)
|
||||
{
|
||||
return ExecuteReaderEnumerableAsync(self, true, cmdtext, null, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns an enumerable of <see cref="SqliteDataReader"/>.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmdtext">The command text to execute.</param>
|
||||
/// <param name="values">The values to use as parameters.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>An asynchronous enumerable of <see cref="SqliteDataReader"/>.</returns>
|
||||
public static IAsyncEnumerable<SqliteDataReader> ExecuteReaderEnumerableAsync(this SqliteCommand self, string cmdtext, Dictionary<string, object?>? values, CancellationToken cancellationToken)
|
||||
{
|
||||
return ExecuteReaderEnumerableAsync(self, true, cmdtext, values, cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns an enumerable of <see cref="SqliteDataReader"/>.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="writeLog">Whether to write a log entry.</param>
|
||||
/// <param name="cmdtext">The command text to execute.</param>
|
||||
/// <param name="values">The values to set as parameters.</param>
|
||||
/// <returns>An asynchronous enumerable of <see cref="SqliteDataReader"/>.</returns>
|
||||
public static async IAsyncEnumerable<SqliteDataReader> ExecuteReaderEnumerableAsync(this SqliteCommand self, bool writeLog, string? cmdtext, Dictionary<string, object?>? values, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
if (cmdtext != null)
|
||||
self.SetCommandAndParameters(cmdtext);
|
||||
|
||||
if (values != null && values.Count > 0)
|
||||
self.SetParameterValues(values);
|
||||
|
||||
using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteReaderEnumerableAsync", $"ExecuteReaderEnumerableAsync: {self.GetPrintableCommandText()}") : null)
|
||||
await using (var rd = await self.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false))
|
||||
while (await rd.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
yield return rd;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned.</returns>
|
||||
public static async Task<object?> ExecuteScalarAsync(this SqliteCommand self, CancellationToken cancellationToken)
|
||||
{
|
||||
return await self.ExecuteScalarAsync(true, null, null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmdtext">The command text to execute.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned.</returns>
|
||||
public static async Task<object?> ExecuteScalarAsync(this SqliteCommand self, string cmdtext, CancellationToken cancellationToken)
|
||||
{
|
||||
return await self.ExecuteScalarAsync(true, cmdtext, null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="writeLog">Whether to write a log entry.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned.</returns>
|
||||
public static async Task<object?> ExecuteScalarAsync(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken)
|
||||
{
|
||||
return await self.ExecuteScalarAsync(writeLog, null, null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmdtext">The command text to execute.</param>
|
||||
/// <param name="values">The values to use as parameters.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned.</returns>
|
||||
public static async Task<object?> ExecuteScalarAsync(this SqliteCommand self, string cmdtext, Dictionary<string, object?>? values, CancellationToken cancellationToken)
|
||||
{
|
||||
return await self.ExecuteScalarAsync(true, cmdtext, values, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="writeLog">Whether to write a log entry.</param>
|
||||
/// <param name="cmd">The command text to execute.</param>
|
||||
/// <param name="values">The values to set as parameters.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set, or null if no rows are returned.</returns>
|
||||
public static async Task<object?> ExecuteScalarAsync(this SqliteCommand self, bool writeLog, string? cmd, Dictionary<string, object?>? values, CancellationToken cancellationToken)
|
||||
{
|
||||
if (cmd != null)
|
||||
self.SetCommandAndParameters(cmd);
|
||||
|
||||
if (values != null && values.Count > 0)
|
||||
self.SetParameterValues(values);
|
||||
|
||||
using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarAsync", string.Format("ExecuteScalarAsync: {0}", self.GetPrintableCommandText())) : null)
|
||||
return await self.ExecuteScalarAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set as an Int64, or -1 if no rows are returned or the value cannot be converted.</returns>
|
||||
public static async Task<long> ExecuteScalarInt64Async(this SqliteCommand self, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteScalarInt64Async(self, true, null, null, -1, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="defaultvalue">The default value to return if no rows are returned or the value cannot be converted.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set as an Int64, or -1 if no rows are returned or the value cannot be converted.</returns>
|
||||
public static async Task<long> ExecuteScalarInt64Async(this SqliteCommand self, long defaultvalue, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteScalarInt64Async(self, true, null, null, defaultvalue, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="writeLog">Whether to write a log entry.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set as an Int64, or -1 if no rows are returned or the value cannot be converted.</returns>
|
||||
public static async Task<long> ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteScalarInt64Async(self, writeLog, null, null, -1, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="writeLog">Whether to write a log entry.</param>
|
||||
/// <param name="defaultvalue">The default value to return if no rows are returned or the value cannot be converted.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted.</returns>
|
||||
public static async Task<long> ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, long defaultvalue, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteScalarInt64Async(self, writeLog, null, null, defaultvalue, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmd">The command text to execute.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted.</returns>
|
||||
public static async Task<long> ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteScalarInt64Async(self, true, cmd, null, -1, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmd">The command text to execute.</param>
|
||||
/// <param name="defaultvalue">The default value to return if no rows are returned or the value cannot be converted.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted.</returns>
|
||||
public static async Task<long> ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, long defaultvalue, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteScalarInt64Async(self, true, cmd, null, defaultvalue, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmd">The command text to execute.</param>
|
||||
/// <param name="values">The values to use as parameters.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted.</returns>
|
||||
public static async Task<long> ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, Dictionary<string, object?>? values, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteScalarInt64Async(self, true, cmd, values, -1, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="cmd">The command text to execute.</param>
|
||||
/// <param name="values">The values to use as parameters.</param>
|
||||
/// <param name="defaultvalue">The default value to return if no rows are returned or the value cannot be converted.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted.</returns>
|
||||
public static async Task<long> ExecuteScalarInt64Async(this SqliteCommand self, string? cmd, Dictionary<string, object?>? values, long defaultvalue, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteScalarInt64Async(self, true, cmd, values, defaultvalue, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="writeLog">Whether to write a log entry.</param>
|
||||
/// <param name="cmd">The command text to execute.</param>
|
||||
/// <param name="values">The values to set as parameters.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted.</returns>
|
||||
public static async Task<long> ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd, Dictionary<string, object?>? values, CancellationToken cancellationToken)
|
||||
{
|
||||
return await ExecuteScalarInt64Async(self, writeLog, cmd, values, -1, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Executes the command asynchronously and returns the first column of the first row in the result set as an Int64.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> instance to execute on.</param>
|
||||
/// <param name="writeLog">Whether to write a log entry.</param>
|
||||
/// <param name="cmd">The command text to execute.</param>
|
||||
/// <param name="values">The values to set as parameters.</param>
|
||||
/// <param name="defaultvalue">The default value to return if no rows are returned or the value cannot be converted.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the first column of the first row in the result set as an Int64, or the default value if no rows are returned or the value cannot be converted.</returns>
|
||||
public static async Task<long> ExecuteScalarInt64Async(this SqliteCommand self, bool writeLog, string? cmd, Dictionary<string, object?>? values, long defaultvalue, CancellationToken cancellationToken)
|
||||
{
|
||||
if (cmd != null)
|
||||
self.SetCommandAndParameters(cmd);
|
||||
|
||||
if (values != null && values.Count > 0)
|
||||
self.SetParameterValues(values);
|
||||
|
||||
using (writeLog ? new Logging.Timer(LOGTAG, "ExecuteScalarInt64Async", string.Format("ExecuteScalarInt64Async: {0}", self.GetPrintableCommandText())) : null)
|
||||
await using (var rd = await self.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false))
|
||||
if (await rd.ReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
return ConvertValueToInt64(rd, 0, defaultvalue);
|
||||
|
||||
return defaultvalue;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expands an IN clause parameter in a SqliteCommand by replacing the parameter with multiple parameters for each value in the collection.
|
||||
/// </summary>
|
||||
/// <param name="cmd">The <see cref="SqliteCommand"/> to expand the IN clause parameter for.</param>
|
||||
/// <param name="originalParamName">The name of the original parameter to replace, must start with '@'.</param>
|
||||
/// <param name="values">The collection of values to expand the IN clause with.</param>
|
||||
/// <typeparam name="T">The type of the values in the collection.</typeparam>
|
||||
/// <returns>The modified <see cref="SqliteCommand"/> with the IN clause expanded.</returns>
|
||||
public static SqliteCommand ExpandInClauseParameterMssqlite<T>(this SqliteCommand cmd, string originalParamName, IEnumerable<T> values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@"))
|
||||
throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName));
|
||||
|
||||
foreach (var p in cmd.Parameters)
|
||||
if (p is IDataParameter parameter && parameter.ParameterName.Equals(originalParamName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
cmd.Parameters.Remove(parameter);
|
||||
break;
|
||||
}
|
||||
|
||||
foreach ((var value, var index) in values.Select((value, index) => (value, index)))
|
||||
cmd.AddNamedParameter($"{originalParamName}{index}", value);
|
||||
|
||||
var inClause = string.Join(", ", values.Select((_, index) => $"{originalParamName}{index}"));
|
||||
if (string.IsNullOrWhiteSpace(inClause) && values.Any())
|
||||
throw new ArgumentException("IN clause cannot be empty", nameof(values));
|
||||
|
||||
#if DEBUG
|
||||
if (!cmd.CommandText.Contains(originalParamName, StringComparison.OrdinalIgnoreCase))
|
||||
throw new ArgumentException($"Command text does not contain parameter '{originalParamName}'", nameof(originalParamName));
|
||||
#endif
|
||||
cmd.CommandText = cmd.CommandText.Replace(originalParamName, inClause, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
return cmd;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Expands an IN clause parameter in a SqliteCommand by replacing the parameter with multiple parameters for each value in the collection.
|
||||
/// If a temporary table is used, it replaces the parameter with the table name.
|
||||
/// </summary>
|
||||
/// <param name="cmd">The <see cref="SqliteCommand"/> to expand the IN clause parameter for.</param>
|
||||
/// <param name="originalParamName">The name of the original parameter to replace, must start with '@'.</param>
|
||||
/// <param name="values">The collection of values to expand the IN clause with.</param>
|
||||
/// <param name="cancellationToken">Cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the modified <see cref="SqliteCommand"/> with the IN clause expanded.</returns>
|
||||
internal static async Task<SqliteCommand> ExpandInClauseParameterMssqliteAsync(this SqliteCommand cmd, string originalParamName, TemporaryDbValueList values, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@"))
|
||||
throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName));
|
||||
|
||||
if (!values.IsTableCreated)
|
||||
return ExpandInClauseParameterMssqlite(cmd, originalParamName, values.Values);
|
||||
|
||||
// We have a temporary table, so we need to replace the parameter with the table name
|
||||
cmd.CommandText = cmd.CommandText.Replace(
|
||||
originalParamName,
|
||||
await values.GetInClause(cancellationToken).ConfigureAwait(false),
|
||||
StringComparison.OrdinalIgnoreCase
|
||||
);
|
||||
|
||||
return cmd;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the command text and parameters for the given <see cref="SqliteCommand"/>.
|
||||
/// </summary>
|
||||
/// <param name="cmd">The <see cref="SqliteCommand"/> to set the command text and parameters for.</param>
|
||||
/// <param name="cmdtext">The command text to set for the command, must not contain '?' as a parameter placeholder.</param>
|
||||
/// <returns>The modified <see cref="SqliteCommand"/> with the command text and parameters set.</returns>
|
||||
/// <exception cref="ArgumentException">If the command text contains '?' as a parameter placeholder.</exception>
|
||||
public static SqliteCommand SetCommandAndParameters(this SqliteCommand cmd, string cmdtext)
|
||||
{
|
||||
cmd.CommandText = Library.Utility.Utility.FormatInvariant(cmdtext);
|
||||
cmd.Parameters.Clear();
|
||||
|
||||
#if DEBUG
|
||||
if (cmd.CommandText.Contains('?', StringComparison.OrdinalIgnoreCase))
|
||||
throw new ArgumentException("Command text cannot contain '?' as a parameter placeholder, use '@' instead.", nameof(cmdtext));
|
||||
#endif
|
||||
|
||||
var parameters = MyRegex().Matches(cmdtext);
|
||||
var found = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (Match match in parameters)
|
||||
{
|
||||
if (found.Contains(match.Value))
|
||||
continue;
|
||||
found.Add(match.Value);
|
||||
cmd.AddNamedParameter(match.Value);
|
||||
}
|
||||
|
||||
return cmd;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the value of a parameter in the command.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> to set the parameter value for.</param>
|
||||
/// <param name="name">The name of the parameter to set the value for.</param>
|
||||
/// <param name="value">The value to set for the parameter, or null to set it to DBNull.Value.</param>
|
||||
/// <returns>The <see cref="SqliteCommand"/> with the parameter value set.</returns>
|
||||
public static SqliteCommand SetParameterValue(this SqliteCommand self, string name, object? value)
|
||||
{
|
||||
#if DEBUG
|
||||
if (value is not null && value is System.Collections.IEnumerable && value is not string)
|
||||
throw new ArgumentException($"Cannot set parameter '{name}' to an array or enumerable type, as the SQLite bindings does not support it.", nameof(value));
|
||||
#endif
|
||||
self.Parameters[name].Value = value ?? DBNull.Value;
|
||||
|
||||
return self;
|
||||
}
|
||||
|
||||
// Special case for DateTime, as we need to convert it to a long
|
||||
/// <summary>
|
||||
/// Sets the value of a DateTime parameter in the command.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> to set the parameter value for.</param>
|
||||
/// <param name="name">The name of the parameter to set the value for.</param>
|
||||
/// <param name="value">The DateTime value to set for the parameter.</param>
|
||||
/// <returns>The <see cref="SqliteCommand"/> with the parameter value set.</returns>
|
||||
public static SqliteCommand SetParameterValue(this SqliteCommand self, string name, DateTime value)
|
||||
{
|
||||
self.Parameters[name].Value = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(value);
|
||||
|
||||
return self;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the parameter values for the command, the parameters must already be added.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> to set the parameter values for.</param>
|
||||
/// <param name="values">The values to set the parameters to, where the key is the parameter name.</param>
|
||||
/// <returns>The <see cref="SqliteCommand"/> with the parameter values set.</returns>
|
||||
public static SqliteCommand SetParameterValues(this SqliteCommand self, Dictionary<string, object?> values)
|
||||
{
|
||||
foreach (var kvp in values)
|
||||
self.Parameters[kvp.Key].Value = kvp.Value;
|
||||
|
||||
return self;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the transaction for the command.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> to set the transaction for.</param>
|
||||
/// <param name="transaction">The <see cref="SqliteTransaction"/> to set for the command.</param>
|
||||
/// <returns>The <see cref="SqliteCommand"/> with the transaction set.</returns>
|
||||
public static SqliteCommand SetTransaction(this SqliteCommand self, SqliteTransaction transaction)
|
||||
{
|
||||
self.Transaction = transaction;
|
||||
|
||||
return self;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the transaction for the command using a <see cref="ReusableTransaction"/>.
|
||||
/// </summary>
|
||||
/// <param name="self">The <see cref="SqliteCommand"/> to set the transaction for.</param>
|
||||
/// <param name="rtr">The <see cref="ReusableTransaction"/> to set for the command.</param>
|
||||
/// <returns>The <see cref="SqliteCommand"/> with the transaction set.</returns>
|
||||
internal static SqliteCommand SetTransaction(this SqliteCommand self, ReusableTransaction rtr)
|
||||
{
|
||||
self.Transaction = rtr.Transaction;
|
||||
|
||||
return self;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The tag used for logging
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ExtensionMethods));
|
||||
|
||||
// This regex matches named parameters in the form of @paramName
|
||||
[GeneratedRegex(@"@\w+")]
|
||||
private static partial Regex MyRegex();
|
||||
|
||||
/// <summary>
|
||||
/// Adds a parameter to the command with the given value.
|
||||
/// </summary>
|
||||
@@ -46,8 +711,28 @@ public static class ExtensionMethods
|
||||
/// <param name="name">The name of the parameter</param>
|
||||
/// <param name="value">The optional value of the parameter</param>
|
||||
/// <returns>The command with the parameter added</returns>
|
||||
public static T AddNamedParameter<T>(this T self, string name, object? value = null)
|
||||
where T : IDbCommand
|
||||
public static SqliteCommand AddNamedParameter(this SqliteCommand self, string name, object? value = null)
|
||||
{
|
||||
var p = self.CreateParameter();
|
||||
p.ParameterName = name;
|
||||
if (value != null)
|
||||
p.Value = value;
|
||||
self.Parameters.Add(p);
|
||||
return self;
|
||||
}
|
||||
|
||||
// TODO At some point, when the entire codebase is migrated to
|
||||
// Microsoft.Data.Sqlite, the extension methods using IDb* should be
|
||||
// removed. Currently, they're kept for compatibility.
|
||||
|
||||
/// <summary>
|
||||
/// Adds a named parameter to the command with the given value.
|
||||
/// </summary>
|
||||
/// <param name="self">The command to add the parameter to</param>
|
||||
/// <param name="name">The name of the parameter</param>
|
||||
/// <param name="value">The optional value of the parameter</param>
|
||||
/// <returns>The command with the parameter added</returns>
|
||||
public static IDbCommand AddNamedParameter(this IDbCommand self, string name, object? value = null)
|
||||
{
|
||||
var p = self.CreateParameter();
|
||||
p.ParameterName = name;
|
||||
@@ -97,7 +782,7 @@ public static class ExtensionMethods
|
||||
/// <typeparam name="T">The type of the command</typeparam>
|
||||
/// <param name="self">The command to set the transaction for</param>
|
||||
/// <param name="transaction">The transaction to set for the command</param>
|
||||
/// <returns>>The command with the transaction set</returns>
|
||||
/// <returns>The command with the transaction set</returns>
|
||||
public static T SetTransaction<T>(this T self, IDbTransaction? transaction)
|
||||
where T : IDbCommand
|
||||
{
|
||||
@@ -114,21 +799,27 @@ public static class ExtensionMethods
|
||||
{
|
||||
var txt = self.CommandText;
|
||||
|
||||
foreach (var p in self.Parameters.Cast<IDbDataParameter>())
|
||||
// Replace each named parameter with its value
|
||||
foreach (IDataParameter p in self.Parameters)
|
||||
{
|
||||
var ix = txt.IndexOf('?');
|
||||
if (ix >= 0)
|
||||
{
|
||||
string v;
|
||||
if (p.Value is string)
|
||||
v = string.Format("\"{0}\"", p.Value);
|
||||
else if (p.Value == null)
|
||||
v = "NULL";
|
||||
else
|
||||
v = string.Format("{0}", p.Value);
|
||||
var paramName = p.ParameterName;
|
||||
if (string.IsNullOrEmpty(paramName))
|
||||
continue;
|
||||
|
||||
txt = txt.Substring(0, ix) + v + txt.Substring(ix + 1);
|
||||
}
|
||||
string v;
|
||||
if (p.Value == null || p.Value == DBNull.Value)
|
||||
v = "NULL";
|
||||
else if (p.Value is string)
|
||||
v = $"\"{p.Value}\"";
|
||||
else if (p.Value is DateTime dt)
|
||||
v = $"\"{dt:O}\"";
|
||||
else if (p.Value is byte[] bytes)
|
||||
v = $"X'{BitConverter.ToString(bytes).Replace("-", "")}'";
|
||||
else
|
||||
v = p.Value.ToString() ?? "NULL";
|
||||
|
||||
// Replace all occurrences of the parameter (with word boundary)
|
||||
txt = Regex.Replace(txt, $@"\B{Regex.Escape(paramName)}\b", v, RegexOptions.IgnoreCase);
|
||||
}
|
||||
|
||||
return txt;
|
||||
@@ -500,15 +1191,17 @@ public static class ExtensionMethods
|
||||
/// </summary>
|
||||
/// <param name="self">The connection to create the transaction on</param>
|
||||
/// <returns>The transaction</returns>
|
||||
public static IDbTransaction BeginTransactionSafe(this IDbConnection self)
|
||||
{
|
||||
var transaction = self.BeginTransaction();
|
||||
// using (var cmd = self.CreateCommand(transaction, "BEGIN IMMEDIATE;"))
|
||||
// try { cmd.ExecuteNonQuery(); }
|
||||
// catch { /* ignore */ }
|
||||
|
||||
return transaction;
|
||||
}
|
||||
//public static IDbTransaction BeginTransactionSafe(this IDbConnection self)
|
||||
//{
|
||||
// var transaction = self.BeginTransaction();
|
||||
// // using (var cmd = self.CreateCommand(transaction, "BEGIN IMMEDIATE;"))
|
||||
// // try { cmd.ExecuteNonQuery(); }
|
||||
// // catch {
|
||||
// // ignore
|
||||
// }
|
||||
//
|
||||
// return transaction;
|
||||
//}
|
||||
|
||||
|
||||
/// <summary>
|
||||
@@ -528,8 +1221,7 @@ public static class ExtensionMethods
|
||||
/// <param name="cmd">The command to expand the parameter for</param>
|
||||
/// <param name="originalParamName">The original parameter name to expand</param>
|
||||
/// <param name="values">The values to expand the parameter for</param>
|
||||
public static TCommand ExpandInClauseParameter<TValue, TCommand>(this TCommand cmd, string originalParamName, IEnumerable<TValue> values)
|
||||
where TCommand : IDbCommand
|
||||
public static IDbCommand ExpandInClauseParameter<TValue>(this IDbCommand cmd, string originalParamName, IEnumerable<TValue> values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@"))
|
||||
throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName));
|
||||
@@ -564,8 +1256,7 @@ public static class ExtensionMethods
|
||||
/// <param name="cmd">The command to expand the parameter for</param>
|
||||
/// <param name="originalParamName">The original parameter name to expand</param>
|
||||
/// <param name="values">The values to expand the parameter for</param>
|
||||
public static T ExpandInClauseParameter<T>(this T cmd, string originalParamName, TemporaryDbValueList values)
|
||||
where T : IDbCommand
|
||||
internal static IDbCommand ExpandInClauseParameter(this IDbCommand cmd, string originalParamName, TemporaryDbValueList values)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(originalParamName) || !originalParamName.StartsWith("@"))
|
||||
throw new ArgumentException("Parameter name must start with '@'", nameof(originalParamName));
|
||||
@@ -574,7 +1265,8 @@ public static class ExtensionMethods
|
||||
return ExpandInClauseParameter(cmd, originalParamName, values.Values);
|
||||
|
||||
// We have a temporary table, so we need to replace the parameter with the table name
|
||||
cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause(), StringComparison.OrdinalIgnoreCase);
|
||||
cmd.CommandText = cmd.CommandText.Replace(originalParamName, values.GetInClause(default).Await(), StringComparison.OrdinalIgnoreCase);
|
||||
return cmd;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,29 +1,41 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
namespace Duplicati.Library.Main.Database
|
||||
{
|
||||
/// <summary>
|
||||
/// Interface for a remote volume, containing its name, hash, and size.
|
||||
/// </summary>
|
||||
public interface IRemoteVolume
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the name of the remote volume.
|
||||
/// </summary>
|
||||
string Name { get; }
|
||||
/// <summary>
|
||||
/// Gets the hash of the remote volume.
|
||||
/// </summary>
|
||||
string Hash { get; }
|
||||
/// <summary>
|
||||
/// Gets the size of the remote volume in bytes.
|
||||
/// </summary>
|
||||
long Size { get; }
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,31 +1,37 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Common.IO;
|
||||
|
||||
namespace Duplicati.Library.Main.Database
|
||||
{
|
||||
|
||||
/// <summary>
|
||||
/// A local database for bug reports, which obfuscates sensitive data before generating a bug report.
|
||||
/// </summary>
|
||||
internal class LocalBugReportDatabase : LocalDatabase
|
||||
{
|
||||
/// <summary>
|
||||
@@ -33,66 +39,144 @@ namespace Duplicati.Library.Main.Database
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(LocalBugReportDatabase));
|
||||
|
||||
public LocalBugReportDatabase(string path, long pagecachesize)
|
||||
: base(path, "BugReportCreate", false, pagecachesize)
|
||||
/// <summary>
|
||||
/// Creates a new instance of the <see cref="LocalBugReportDatabase"/> class.
|
||||
/// </summary>
|
||||
/// <param name="path">The path to the database file.</param>
|
||||
/// <param name="dbnew">An optional existing database instance to use. Used to mimic constructor chaining.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains a new instance of <see cref="LocalBugReportDatabase"/>.</returns>
|
||||
public static async Task<LocalBugReportDatabase> CreateAsync(string path, LocalBugReportDatabase? dbnew, CancellationToken token)
|
||||
{
|
||||
ShouldCloseConnection = true;
|
||||
dbnew ??= new LocalBugReportDatabase();
|
||||
|
||||
dbnew = (LocalBugReportDatabase)
|
||||
await CreateLocalDatabaseAsync(path, "BugReportCreate", false, dbnew, token)
|
||||
.ConfigureAwait(false);
|
||||
dbnew.ShouldCloseConnection = true;
|
||||
|
||||
return dbnew;
|
||||
}
|
||||
|
||||
public void Fix()
|
||||
/// <summary>
|
||||
/// Obfuscates sensitive data in the database, readying it for a bug report.
|
||||
/// </summary>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the obfuscation is finished.</returns>
|
||||
public async Task Fix(CancellationToken token)
|
||||
{
|
||||
using (var tr = m_connection.BeginTransactionSafe())
|
||||
using (var cmd = m_connection.CreateCommand())
|
||||
await using var cmd = m_connection.CreateCommand(m_rtr);
|
||||
var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
||||
|
||||
// TODO: Rewrite this to use PathPrefix
|
||||
// TODO: Needs to be much faster
|
||||
await using (var upcmd = m_connection.CreateCommand(m_rtr))
|
||||
{
|
||||
cmd.Transaction = tr;
|
||||
var tablename = "PathMap-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
||||
await upcmd.ExecuteNonQueryAsync($@"
|
||||
CREATE TEMPORARY TABLE ""{tablename}"" (
|
||||
""ID"" INTEGER PRIMARY KEY,
|
||||
""RealPath"" TEXT NOT NULL,
|
||||
""Obfuscated"" TEXT NULL
|
||||
)
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// TODO: Rewrite this to use PathPrefix
|
||||
// TODO: Needs to be much faster
|
||||
using (var upcmd = m_connection.CreateCommand())
|
||||
{
|
||||
await upcmd.ExecuteNonQueryAsync($@"
|
||||
INSERT INTO ""{tablename}"" (""RealPath"")
|
||||
SELECT DISTINCT ""Path""
|
||||
FROM ""File""
|
||||
ORDER BY ""Path""
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
upcmd.Transaction = tr;
|
||||
upcmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{tablename}"" (""ID"" INTEGER PRIMARY KEY, ""RealPath"" TEXT NOT NULL, ""Obfuscated"" TEXT NULL)"));
|
||||
upcmd.ExecuteNonQuery(FormatInvariant($@"INSERT INTO ""{tablename}"" (""RealPath"") SELECT DISTINCT ""Path"" FROM ""File"" ORDER BY ""Path"" "));
|
||||
upcmd.SetCommandAndParameters(FormatInvariant($@"UPDATE ""{tablename}"" SET ""Obfuscated"" = @StartPath || length(""RealPath"") || @DirSep || ""ID"" || (CASE WHEN substr(""RealPath"", length(""RealPath"")) = @DirSep THEN @DirSep ELSE @FileExt END) "))
|
||||
.SetParameterValue("@StartPath", !OperatingSystem.IsWindows() ? "/" : "X:\\")
|
||||
.SetParameterValue("@DirSep", Util.DirectorySeparatorString)
|
||||
.SetParameterValue("@FileExt", ".bin")
|
||||
.ExecuteNonQuery();
|
||||
await upcmd.SetCommandAndParameters($@"
|
||||
UPDATE ""{tablename}""
|
||||
SET ""Obfuscated"" =
|
||||
@StartPath
|
||||
|| length(""RealPath"")
|
||||
|| @DirSep
|
||||
|| ""ID""
|
||||
|| (
|
||||
CASE
|
||||
WHEN substr(""RealPath"", length(""RealPath"")) = @DirSep
|
||||
THEN @DirSep
|
||||
ELSE @FileExt
|
||||
END
|
||||
)
|
||||
")
|
||||
.SetParameterValue("@StartPath", !OperatingSystem.IsWindows() ? "/" : "X:\\")
|
||||
.SetParameterValue("@DirSep", Util.DirectorySeparatorString)
|
||||
.SetParameterValue("@FileExt", ".bin")
|
||||
.ExecuteNonQueryAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
/*long id = 1;
|
||||
using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""RealPath"", ""Obfuscated"" FROM ""{0}"" ", tablename)))
|
||||
while(rd.Read())
|
||||
{
|
||||
upcmd.SetCommandAndParameters(@"UPDATE ""LogData"" SET ""Message"" = replace(""Message"", @RealPath, @Obfuscated), ""Exception"" = replace(""Exception"", @RealPath, @Obfuscated)")
|
||||
.SetParameterValue("@RealPath", rd.GetValue(0))
|
||||
.SetParameterValue("@Obfuscated", rd.GetValue(1))
|
||||
.ExecuteNonQuery();
|
||||
id++;
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Message"" = 'ERASED!' WHERE ""Message"" LIKE '%/%' OR ""Message"" LIKE '%:\%' ");
|
||||
cmd.ExecuteNonQuery(@"UPDATE ""LogData"" SET ""Exception"" = 'ERASED!' WHERE ""Exception"" LIKE '%/%' OR ""Exception"" LIKE '%:\%' ");
|
||||
|
||||
cmd.ExecuteNonQuery(@"UPDATE ""Configuration"" SET ""Value"" = 'ERASED!' WHERE ""Key"" = 'passphrase' ");
|
||||
|
||||
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TABLE ""FixedFile"" AS SELECT ""B"".""ID"" AS ""ID"", ""A"".""Obfuscated"" AS ""Path"", ""B"".""BlocksetID"" AS ""BlocksetID"", ""B"".""MetadataID"" AS ""MetadataID"" FROM ""{tablename}"" ""A"", ""File"" ""B"" WHERE ""A"".""RealPath"" = ""B"".""Path"" "));
|
||||
cmd.ExecuteNonQuery(@"DROP VIEW ""File"" ");
|
||||
cmd.ExecuteNonQuery(@"DROP TABLE ""FileLookup"" ");
|
||||
cmd.ExecuteNonQuery(@"DROP TABLE ""PathPrefix"" ");
|
||||
|
||||
cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{tablename}"" "));
|
||||
|
||||
using (new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport"))
|
||||
tr.Commit();
|
||||
|
||||
cmd.Transaction = null;
|
||||
|
||||
cmd.ExecuteNonQuery("VACUUM");
|
||||
/*long id = 1;
|
||||
using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""RealPath"", ""Obfuscated"" FROM ""{0}"" ", tablename)))
|
||||
while(rd.Read())
|
||||
{
|
||||
upcmd.SetCommandAndParameters(@"UPDATE ""LogData"" SET ""Message"" = replace(""Message"", @RealPath, @Obfuscated), ""Exception"" = replace(""Exception"", @RealPath, @Obfuscated)")
|
||||
.SetParameterValue("@RealPath", rd.GetValue(0))
|
||||
.SetParameterValue("@Obfuscated", rd.GetValue(1))
|
||||
.ExecuteNonQuery();
|
||||
id++;
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
await cmd.ExecuteNonQueryAsync(@"
|
||||
UPDATE ""LogData""
|
||||
SET ""Message"" = 'ERASED!'
|
||||
WHERE
|
||||
""Message"" LIKE '%/%'
|
||||
OR ""Message"" LIKE '%:\%'
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await cmd.ExecuteNonQueryAsync(@"
|
||||
UPDATE ""LogData""
|
||||
SET ""Exception"" = 'ERASED!'
|
||||
WHERE
|
||||
""Exception"" LIKE '%/%'
|
||||
OR ""Exception"" LIKE '%:\%'
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await cmd.ExecuteNonQueryAsync(@"
|
||||
UPDATE ""Configuration""
|
||||
SET ""Value"" = 'ERASED!'
|
||||
WHERE ""Key"" = 'passphrase'
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await cmd.ExecuteNonQueryAsync($@"
|
||||
CREATE TABLE ""FixedFile"" AS
|
||||
SELECT
|
||||
""B"".""ID"" AS ""ID"",
|
||||
""A"".""Obfuscated"" AS ""Path"",
|
||||
""B"".""BlocksetID"" AS ""BlocksetID"",
|
||||
""B"".""MetadataID"" AS ""MetadataID""
|
||||
FROM
|
||||
""{tablename}"" ""A"",
|
||||
""File"" ""B""
|
||||
WHERE ""A"".""RealPath"" = ""B"".""Path""
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await cmd.ExecuteNonQueryAsync(@"DROP VIEW ""File"" ", token)
|
||||
.ConfigureAwait(false);
|
||||
await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""FileLookup"" ", token)
|
||||
.ConfigureAwait(false);
|
||||
await cmd.ExecuteNonQueryAsync(@"DROP TABLE ""PathPrefix"" ", token)
|
||||
.ConfigureAwait(false);
|
||||
await cmd.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{tablename}"" ", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
using (new Logging.Timer(LOGTAG, "CommitUpdateBugReport", "CommitUpdateBugReport"))
|
||||
await m_rtr.CommitAsync(token: token).ConfigureAwait(false);
|
||||
|
||||
cmd.SetTransaction(m_rtr);
|
||||
await cmd.ExecuteNonQueryAsync("VACUUM", token).ConfigureAwait(false);
|
||||
await m_rtr.CommitAsync(token: token).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
@@ -24,131 +24,396 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using System.Threading;
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
namespace Duplicati.Library.Main.Database
|
||||
{
|
||||
/// <summary>
|
||||
/// LocalListAffectedDatabase is a specialized database class for listing
|
||||
/// affected filesets, files, remote logs, and remote volumes.
|
||||
/// </summary>
|
||||
internal class LocalListAffectedDatabase : LocalDatabase
|
||||
{
|
||||
public LocalListAffectedDatabase(string path, long pagecachesize)
|
||||
: base(path, "ListAffected", false, pagecachesize)
|
||||
/// <summary>
|
||||
/// Creates a new instance of <see cref="LocalListAffectedDatabase"/>.
|
||||
/// </summary>
|
||||
/// <param name="path">The path to the database file.</param>
|
||||
/// <param name="dbnew">An optional existing database instance to use. Used to mimic constructor chaining.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited returns a new instance of <see cref="LocalListAffectedDatabase"/>.</returns>
|
||||
public static async Task<LocalListAffectedDatabase> CreateAsync(string path, LocalListAffectedDatabase? dbnew, CancellationToken token)
|
||||
{
|
||||
ShouldCloseConnection = true;
|
||||
dbnew ??= new LocalListAffectedDatabase();
|
||||
|
||||
dbnew = (LocalListAffectedDatabase)
|
||||
await CreateLocalDatabaseAsync(path, "ListAffected", false, dbnew, token)
|
||||
.ConfigureAwait(false);
|
||||
dbnew.ShouldCloseConnection = true;
|
||||
|
||||
return dbnew;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a fileset result for listing affected filesets.
|
||||
/// </summary>
|
||||
private class ListResultFileset : Interface.IListResultFileset
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the version of the fileset.
|
||||
/// </summary>
|
||||
public long Version { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets whether the backup related to this fileset is a full backup.
|
||||
/// </summary>
|
||||
public int IsFullBackup { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the time when the fileset was created.
|
||||
/// </summary>
|
||||
public DateTime Time { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the number of files in the fileset.
|
||||
/// </summary>
|
||||
public long FileCount { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the total size of files in the fileset.
|
||||
/// </summary>
|
||||
public long FileSizes { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a file result for listing affected files.
|
||||
/// </summary>
|
||||
private class ListResultFile : Interface.IListResultFile
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the path of the file reperesented by this result entry.
|
||||
/// </summary>
|
||||
public required string Path { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the sizes of the file blocks, if available.
|
||||
/// This can be null if the sizes are not available or not applicable.
|
||||
/// </summary>
|
||||
public required IEnumerable<long>? Sizes { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a remote log entry for listing affected remote logs.
|
||||
/// </summary>
|
||||
private class ListResultRemoteLog : Interface.IListResultRemoteLog
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the timestamp of the log entry.
|
||||
/// </summary>
|
||||
public required DateTime Timestamp { get; set; }
|
||||
/// <summary>
|
||||
/// Gets or sets the message of the log entry.
|
||||
/// </summary>
|
||||
public required string Message { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Represents a remote volume entry for listing affected remote volumes.
|
||||
/// </summary>
|
||||
private class ListResultRemoteVolume : Interface.IListResultRemoteVolume
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets or sets the name of the remote volume.
|
||||
/// </summary>
|
||||
public required string Name { get; set; }
|
||||
}
|
||||
|
||||
public IEnumerable<Interface.IListResultFileset> GetFilesets(IEnumerable<string> items)
|
||||
/// <summary>
|
||||
/// Retrieves the fileset times from the database.
|
||||
/// </summary>
|
||||
/// <param name="items">The items to filter the filesets by.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>An asynchronous enumerable of fileset times.</returns>
|
||||
public async IAsyncEnumerable<Interface.IListResultFileset> GetFilesets(IEnumerable<string> items, [EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
var filesets = FilesetTimes.ToArray();
|
||||
var filesets = await FilesetTimes(token)
|
||||
.ToArrayAsync(cancellationToken: token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var dict = new Dictionary<long, long>();
|
||||
for (var i = 0; i < filesets.Length; i++)
|
||||
dict[filesets[i].Key] = i;
|
||||
|
||||
var sql = FormatInvariant($@"SELECT DISTINCT ""FilesetID"" FROM (
|
||||
SELECT ""FilesetID"" FROM ""FilesetEntry"" WHERE ""FileID"" IN ( SELECT ""ID"" FROM ""FileLookup"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN ( SELECT ""ID"" From ""Block"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)))))
|
||||
UNION
|
||||
SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names))
|
||||
)");
|
||||
using (var tmptable = new TemporaryDbValueList(m_connection, null, items))
|
||||
using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameter("@Names", tmptable))
|
||||
using (var rd = cmd.ExecuteReader())
|
||||
while (rd.Read())
|
||||
{
|
||||
var v = dict[rd.ConvertValueToInt64(0)];
|
||||
yield return new ListResultFileset()
|
||||
{
|
||||
Version = v,
|
||||
Time = filesets[v].Value
|
||||
};
|
||||
}
|
||||
}
|
||||
var sql = $@"
|
||||
SELECT DISTINCT ""FilesetID""
|
||||
FROM (
|
||||
SELECT ""FilesetID""
|
||||
FROM ""FilesetEntry""
|
||||
WHERE ""FileID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""FileLookup""
|
||||
WHERE ""BlocksetID"" IN (
|
||||
SELECT ""BlocksetID""
|
||||
FROM ""BlocksetEntry""
|
||||
WHERE ""BlockID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""Block""
|
||||
WHERE ""VolumeID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""RemoteVolume""
|
||||
WHERE ""Name"" IN (@Names)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
UNION
|
||||
SELECT ""ID""
|
||||
FROM ""Fileset""
|
||||
WHERE ""VolumeID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""RemoteVolume""
|
||||
WHERE ""Name"" IN (@Names)
|
||||
)
|
||||
)
|
||||
";
|
||||
|
||||
public IEnumerable<Interface.IListResultFile> GetFiles(IEnumerable<string> items)
|
||||
{
|
||||
var sql = FormatInvariant($@"SELECT DISTINCT ""Path"" FROM (
|
||||
SELECT ""Path"" FROM ""File"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN (@Names))))
|
||||
UNION
|
||||
SELECT ""Path"" FROM ""File"" WHERE ""MetadataID"" IN (SELECT ""ID"" FROM ""Metadataset"" WHERE ""BlocksetID"" IN (SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" IN (SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN (SELECT ""ID"" from ""RemoteVolume"" WHERE ""Name"" IN (@Names)))))
|
||||
UNION
|
||||
SELECT ""Path"" FROM ""File"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names))))
|
||||
) ORDER BY ""Path""");
|
||||
|
||||
using (var tmptable = new TemporaryDbValueList(m_connection, null, items))
|
||||
using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameter("@Names", tmptable))
|
||||
using (var rd = cmd.ExecuteReader())
|
||||
while (rd.Read())
|
||||
yield return new ListResultFile()
|
||||
{
|
||||
Path = rd.ConvertValueToString(0) ?? throw new InvalidOperationException("Path is null"),
|
||||
Sizes = null
|
||||
};
|
||||
}
|
||||
|
||||
public IEnumerable<Interface.IListResultRemoteLog> GetLogLines(IEnumerable<string> items)
|
||||
{
|
||||
using var cmd = m_connection.CreateCommand();
|
||||
|
||||
foreach (var slice in items.Chunk(CHUNK_SIZE / 2))
|
||||
await using var tmptable = await TemporaryDbValueList
|
||||
.CreateAsync(this, items, token)
|
||||
.ConfigureAwait(false);
|
||||
await using var cmd = await m_connection
|
||||
.CreateCommand(sql)
|
||||
.SetTransaction(m_rtr)
|
||||
.ExpandInClauseParameterMssqliteAsync("@Names", tmptable, token)
|
||||
.ConfigureAwait(false);
|
||||
await using var rd = await cmd
|
||||
.ExecuteReaderAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
while (await rd.ReadAsync(token).ConfigureAwait(false))
|
||||
{
|
||||
var sql = FormatInvariant($@"
|
||||
SELECT ""TimeStamp"", ""Message"" || ' ' || CASE WHEN ""Exception"" IS NULL THEN '' ELSE ""Exception"" END FROM ""LogData"" WHERE {string.Join(" OR ", slice.Select((x, i) => @$"""Message"" LIKE @Message{i}"))}
|
||||
UNION
|
||||
SELECT ""Timestamp"", ""Data"" FROM ""RemoteOperation"" WHERE ""Path"" IN (@Paths)");
|
||||
|
||||
cmd.SetCommandAndParameters(sql);
|
||||
cmd.ExpandInClauseParameter("@Paths", items.ToArray());
|
||||
foreach ((var x, var i) in items.Select((x, i) => (x, i)))
|
||||
cmd.SetParameterValue($"@Message{i}", "%" + x + "%");
|
||||
|
||||
using (var rd = cmd.ExecuteReader())
|
||||
while (rd.Read())
|
||||
yield return new ListResultRemoteLog()
|
||||
{
|
||||
Timestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(0)),
|
||||
Message = rd.ConvertValueToString(1) ?? ""
|
||||
};
|
||||
var v = dict[rd.ConvertValueToInt64(0)];
|
||||
yield return new ListResultFileset()
|
||||
{
|
||||
Version = v,
|
||||
Time = filesets[v].Value
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public IEnumerable<Interface.IListResultRemoteVolume> GetVolumes(IEnumerable<string> items)
|
||||
/// <summary>
|
||||
/// Retrieves the list of files from the database.
|
||||
/// </summary>
|
||||
/// <param name="items">The items to filter the files by.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>An asynchronous enumerable of file results.</returns>
|
||||
public async IAsyncEnumerable<Interface.IListResultFile> GetFiles(IEnumerable<string> items, [EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
var sql = FormatInvariant($@"SELECT DISTINCT ""Name"" FROM (
|
||||
SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)))))))
|
||||
UNION
|
||||
SELECT ""Name"" FROM ""Remotevolume"" WHERE ""ID"" IN ( SELECT ""VolumeID"" FROM ""Block"" WHERE ""ID"" IN ( SELECT ""BlockID"" FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ( SELECT ""BlocksetID"" FROM ""Metadataset"" WHERE ""ID"" IN ( SELECT ""MetadataID"" FROM ""FileLookup"" WHERE ""ID"" IN ( SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" IN ( SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ( SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names))))))))
|
||||
)");
|
||||
var sql = $@"
|
||||
SELECT DISTINCT ""Path""
|
||||
FROM (
|
||||
SELECT ""Path""
|
||||
FROM ""File""
|
||||
WHERE ""BlocksetID"" IN (
|
||||
SELECT ""BlocksetID""
|
||||
FROM ""BlocksetEntry""
|
||||
WHERE ""BlockID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""Block""
|
||||
WHERE ""VolumeID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""RemoteVolume""
|
||||
WHERE ""Name"" IN (@Names)
|
||||
)
|
||||
)
|
||||
)
|
||||
UNION
|
||||
SELECT ""Path""
|
||||
FROM ""File""
|
||||
WHERE ""MetadataID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""Metadataset""
|
||||
WHERE ""BlocksetID"" IN (
|
||||
SELECT ""BlocksetID""
|
||||
FROM ""BlocksetEntry""
|
||||
WHERE ""BlockID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""Block""
|
||||
WHERE ""VolumeID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""RemoteVolume""
|
||||
WHERE ""Name"" IN (@Names)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
UNION
|
||||
SELECT ""Path""
|
||||
FROM ""File""
|
||||
WHERE ""ID"" IN (
|
||||
SELECT ""FileID""
|
||||
FROM ""FilesetEntry""
|
||||
WHERE ""FilesetID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""Fileset""
|
||||
WHERE ""VolumeID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""RemoteVolume""
|
||||
WHERE ""Name"" IN (@Names)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
ORDER BY ""Path""
|
||||
";
|
||||
|
||||
using (var cmd = m_connection.CreateCommand(sql).ExpandInClauseParameter("@Names", items.ToArray()))
|
||||
using (var rd = cmd.ExecuteReader())
|
||||
while (rd.Read())
|
||||
yield return new ListResultRemoteVolume()
|
||||
await using var tmptable = await TemporaryDbValueList
|
||||
.CreateAsync(this, items, token)
|
||||
.ConfigureAwait(false);
|
||||
await using var cmd = await m_connection
|
||||
.CreateCommand(sql)
|
||||
.ExpandInClauseParameterMssqliteAsync("@Names", tmptable, token)
|
||||
.ConfigureAwait(false);
|
||||
await using var rd = await cmd
|
||||
.ExecuteReaderAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
while (await rd.ReadAsync(token).ConfigureAwait(false))
|
||||
yield return new ListResultFile()
|
||||
{
|
||||
Path = rd.ConvertValueToString(0) ?? throw new InvalidOperationException("Path is null"),
|
||||
Sizes = null
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves the log lines from the database that match the specified items.
|
||||
/// The log lines are retrieved from both the LogData and RemoteOperation tables.
|
||||
/// </summary>
|
||||
/// <param name="items">The items to filter the log lines by.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>An asynchronous enumerable of log line results.</returns>
|
||||
public async IAsyncEnumerable<Interface.IListResultRemoteLog> GetLogLines(IEnumerable<string> items, [EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
await using var cmd = m_connection.CreateCommand();
|
||||
|
||||
foreach (var slice in items.Chunk(CHUNK_SIZE / 2))
|
||||
{
|
||||
var sql = $@"
|
||||
SELECT
|
||||
""TimeStamp"",
|
||||
""Message""
|
||||
|| ' '
|
||||
||
|
||||
CASE
|
||||
WHEN ""Exception"" IS NULL
|
||||
THEN ''
|
||||
ELSE ""Exception""
|
||||
END
|
||||
FROM ""LogData""
|
||||
WHERE
|
||||
{string.Join(" OR ", slice.Select((x, i) => @$"""Message"" LIKE @Message{i}"))}
|
||||
UNION
|
||||
SELECT
|
||||
""Timestamp"",
|
||||
""Data""
|
||||
FROM ""RemoteOperation""
|
||||
WHERE ""Path"" IN (@Paths)
|
||||
";
|
||||
|
||||
cmd.SetCommandAndParameters(sql)
|
||||
.ExpandInClauseParameterMssqlite("@Paths", items.ToArray());
|
||||
foreach ((var x, var i) in items.Select((x, i) => (x, i)))
|
||||
cmd.SetParameterValue($"@Message{i}", "%" + x + "%");
|
||||
|
||||
await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false);
|
||||
while (await rd.ReadAsync(token).ConfigureAwait(false))
|
||||
yield return new ListResultRemoteLog()
|
||||
{
|
||||
Name = rd.ConvertValueToString(0) ?? ""
|
||||
Timestamp = ParseFromEpochSeconds(rd.ConvertValueToInt64(0)),
|
||||
Message = rd.ConvertValueToString(1) ?? ""
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Retrieves the remote volumes that match the specified items.
|
||||
/// This method queries the database for remote volumes associated with filesets and metadata datasets.
|
||||
/// </summary>
|
||||
/// <param name="items">The names of the remote volumes to filter by.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>An asynchronous enumerable of remote volume results.</returns>
|
||||
public async IAsyncEnumerable<Interface.IListResultRemoteVolume> GetVolumes(IEnumerable<string> items, [EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
var sql = $@"
|
||||
SELECT DISTINCT ""Name""
|
||||
FROM (
|
||||
SELECT ""Name""
|
||||
FROM ""Remotevolume""
|
||||
WHERE ""ID"" IN (
|
||||
SELECT ""VolumeID""
|
||||
FROM ""Block""
|
||||
WHERE ""ID"" IN (
|
||||
SELECT ""BlockID""
|
||||
FROM ""BlocksetEntry""
|
||||
WHERE ""BlocksetID"" IN (
|
||||
SELECT ""BlocksetID""
|
||||
FROM ""FileLookup""
|
||||
WHERE ""ID"" IN (
|
||||
SELECT ""FileID""
|
||||
FROM ""FilesetEntry""
|
||||
WHERE ""FilesetID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""Fileset""
|
||||
WHERE ""VolumeID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""RemoteVolume""
|
||||
WHERE ""Name"" IN (@Names)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
UNION
|
||||
SELECT ""Name""
|
||||
FROM ""Remotevolume""
|
||||
WHERE ""ID"" IN (
|
||||
SELECT ""VolumeID""
|
||||
FROM ""Block""
|
||||
WHERE ""ID"" IN (
|
||||
SELECT ""BlockID""
|
||||
FROM ""BlocksetEntry""
|
||||
WHERE ""BlocksetID"" IN (
|
||||
SELECT ""BlocksetID""
|
||||
FROM ""Metadataset""
|
||||
WHERE ""ID"" IN (
|
||||
SELECT ""MetadataID""
|
||||
FROM ""FileLookup""
|
||||
WHERE ""ID"" IN (
|
||||
SELECT ""FileID""
|
||||
FROM ""FilesetEntry""
|
||||
WHERE ""FilesetID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""Fileset""
|
||||
WHERE ""VolumeID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""RemoteVolume""
|
||||
WHERE ""Name"" IN (@Names)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
";
|
||||
|
||||
await using var cmd = m_connection
|
||||
.CreateCommand(sql)
|
||||
.ExpandInClauseParameterMssqlite("@Names", items.ToArray());
|
||||
await using var rd = await cmd
|
||||
.ExecuteReaderAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
while (await rd.ReadAsync(token).ConfigureAwait(false))
|
||||
yield return new ListResultRemoteVolume()
|
||||
{
|
||||
Name = rd.ConvertValueToString(0) ?? ""
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,214 +1,450 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Linq;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Duplicati.Library.Main.Database
|
||||
{
|
||||
internal class LocalListBrokenFilesDatabase : LocalDatabase
|
||||
{
|
||||
private static readonly string BLOCK_VOLUME_IDS = FormatInvariant($@"SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Blocks.ToString()}'");
|
||||
|
||||
// Invalid blocksets include those that:
|
||||
// - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available)
|
||||
// - Invalid blocks include those that appear to be in non-Blocks volumes (e.g., are listed as being in an Index or Files volume) or that appear in an unknown volume (-1)
|
||||
// - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available)
|
||||
// - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files)
|
||||
// However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed.
|
||||
private static readonly string BROKEN_FILE_IDS = FormatInvariant($@"
|
||||
SELECT DISTINCT ""ID"" FROM (
|
||||
SELECT ""ID"" AS ""ID"", ""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" WHERE ""BlocksetID"" != {FOLDER_BLOCKSET_ID} AND ""BlocksetID"" != {SYMLINK_BLOCKSET_ID}
|
||||
UNION
|
||||
SELECT ""A"".""ID"" AS ""ID"", ""B"".""BlocksetID"" AS ""BlocksetID"" FROM ""FileLookup"" A LEFT JOIN ""Metadataset"" B ON ""A"".""MetadataID"" = ""B"".""ID""
|
||||
)
|
||||
WHERE ""BlocksetID"" IS NULL OR ""BlocksetID"" IN
|
||||
(
|
||||
SELECT DISTINCT ""BlocksetID"" FROM
|
||||
(
|
||||
SELECT ""BlocksetID"" FROM ""BlocksetEntry"" WHERE ""BlockID"" NOT IN
|
||||
(SELECT ""ID"" FROM ""Block"" WHERE ""VolumeID"" IN
|
||||
({BLOCK_VOLUME_IDS}))
|
||||
UNION
|
||||
SELECT ""BlocksetID"" FROM ""BlocklistHash"" WHERE ""Hash"" NOT IN
|
||||
(SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN
|
||||
({BLOCK_VOLUME_IDS}))
|
||||
UNION
|
||||
SELECT ""A"".""ID"" AS ""BlocksetID"" FROM ""Blockset"" A LEFT JOIN ""BlocksetEntry"" B ON ""A"".""ID"" = ""B"".""BlocksetID"" WHERE ""A"".""Length"" > 0 AND ""B"".""BlocksetID"" IS NULL
|
||||
)
|
||||
WHERE ""BlocksetID"" NOT IN (SELECT ""ID"" FROM ""Blockset"" WHERE ""Length"" == 0)
|
||||
)
|
||||
");
|
||||
private static readonly string BROKEN_FILE_SETS = FormatInvariant($@"SELECT DISTINCT ""B"".""Timestamp"", ""A"".""FilesetID"", COUNT(""A"".""FileID"") AS ""FileCount"" FROM ""FilesetEntry"" A, ""Fileset"" B WHERE ""A"".""FilesetID"" = ""B"".""ID"" AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})");
|
||||
|
||||
private static readonly string BROKEN_FILE_NAMES = FormatInvariant($@"SELECT ""A"".""Path"", ""B"".""Length"" FROM ""File"" A LEFT JOIN ""Blockset"" B ON (""A"".""BlocksetID"" = ""B"".""ID"") WHERE ""A"".""ID"" IN ({BROKEN_FILE_IDS}) AND ""A"".""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)");
|
||||
|
||||
private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => FormatInvariant($@"INSERT INTO ""{tablename}"" (""{IDfieldname}"") {BROKEN_FILE_IDS} AND ""ID"" IN (SELECT ""FileID"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId)");
|
||||
|
||||
public LocalListBrokenFilesDatabase(string path, long pagecachesize)
|
||||
: base(path, "ListBrokenFiles", false, pagecachesize)
|
||||
{
|
||||
ShouldCloseConnection = true;
|
||||
}
|
||||
|
||||
public LocalListBrokenFilesDatabase(LocalDatabase parent)
|
||||
: base(parent)
|
||||
{
|
||||
ShouldCloseConnection = false;
|
||||
}
|
||||
|
||||
public IEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[]? versions, IDbTransaction transaction)
|
||||
{
|
||||
var query = BROKEN_FILE_SETS;
|
||||
var clause = GetFilelistWhereClause(time, versions);
|
||||
if (!string.IsNullOrWhiteSpace(clause.Item1))
|
||||
query += FormatInvariant($@" AND ""A"".""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" {clause.Item1})");
|
||||
|
||||
query += @" GROUP BY ""A"".""FilesetID""";
|
||||
|
||||
using (var cmd = Connection.CreateCommand(transaction))
|
||||
foreach (var rd in cmd.SetCommandAndParameters(query).SetParameterValues(clause.Values).ExecuteReaderEnumerable())
|
||||
if (!rd.IsDBNull(0))
|
||||
yield return (ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)), rd.ConvertValueToInt64(1, -1), rd.ConvertValueToInt64(2, 0));
|
||||
}
|
||||
|
||||
public IEnumerable<Tuple<string, long>> GetBrokenFilenames(long filesetid, IDbTransaction transaction)
|
||||
{
|
||||
using (var cmd = Connection.CreateCommand(transaction, BROKEN_FILE_NAMES).SetParameterValue("@FilesetId", filesetid))
|
||||
foreach (var rd in cmd.ExecuteReaderEnumerable())
|
||||
if (!rd.IsDBNull(0))
|
||||
yield return new Tuple<string, long>(rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"), rd.ConvertValueToInt64(1));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns all index files that are orphaned, i.e., not referenced by any block files.
|
||||
/// A database for listing broken files, i.e., files that reference blocksets or blocks that are not available.
|
||||
/// </summary>
|
||||
/// <param name="transaction">Transaction to use for the query.</param>
|
||||
/// <returns>>All index files that are orphaned.</returns>
|
||||
public IEnumerable<RemoteVolume> GetOrphanedIndexFiles(IDbTransaction transaction)
|
||||
internal class LocalListBrokenFilesDatabase : LocalDatabase
|
||||
{
|
||||
using var cmd = Connection.CreateCommand(transaction, FormatInvariant($@"SELECT ""Name"", ""Hash"", ""Size"" FROM ""RemoteVolume"" WHERE ""Type"" = '{RemoteVolumeType.Index.ToString()}' AND ""ID"" NOT IN (SELECT ""IndexVolumeID"" FROM ""IndexBlockLink"")"));
|
||||
/// <summary>
|
||||
/// SQL query to get the IDs of all block volumes.
|
||||
/// </summary>
|
||||
private static readonly string BLOCK_VOLUME_IDS = $@"
|
||||
SELECT ""ID""
|
||||
FROM ""RemoteVolume""
|
||||
WHERE ""Type"" = '{RemoteVolumeType.Blocks}'
|
||||
";
|
||||
|
||||
foreach (var rd in cmd.ExecuteReaderEnumerable())
|
||||
yield return new RemoteVolume(
|
||||
rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"),
|
||||
rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"),
|
||||
rd.ConvertValueToInt64(2, -1)
|
||||
);
|
||||
}
|
||||
// Invalid blocksets include those that:
|
||||
// - Have BlocksetEntries with unknown/invalid blocks (meaning the data to rebuild the blockset isn't available)
|
||||
// - Invalid blocks include those that appear to be in non-Blocks volumes (e.g., are listed as being in an Index or Files volume) or that appear in an unknown volume (-1)
|
||||
// - Have BlocklistHash entries with unknown/invalid blocks (meaning the data which defines the list of hashes that makes up the blockset isn't available)
|
||||
// - Are defined in the Blockset table but have no entries in the BlocksetEntries table (this can happen during recreate if Files volumes reference blocksets that are not found in any Index files)
|
||||
// However, blocksets with a length of 0 are excluded from this check, as the corresponding blocks for these are not needed.
|
||||
|
||||
/// <summary>
|
||||
/// Inserts the broken file IDs into the given table. The table must have a single column with the same name as the ID field name.
|
||||
/// </summary>
|
||||
/// <param name="filesetid">The filset id for the current operation</param>
|
||||
/// <param name="tablename">The name of the table to insert into</param>
|
||||
/// <param name="IDfieldname">The name of the ID field in the table</param>
|
||||
/// <param name="transaction">The transaction to use for the query</param>
|
||||
public void InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, IDbTransaction? transaction)
|
||||
{
|
||||
using var cmd = Connection.CreateCommand(transaction, INSERT_BROKEN_IDS(tablename, IDfieldname))
|
||||
.SetParameterValue("@FilesetId", filesetid);
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
/// <summary>
|
||||
/// SQL query to get the IDs of broken files.
|
||||
/// A broken file is one that has a BlocksetID that is not in the list of valid blocksets, or that has a MetadataID that is not in the list of valid metadata blocksets.
|
||||
/// </summary>
|
||||
private static readonly string BROKEN_FILE_IDS = $@"
|
||||
SELECT DISTINCT ""ID""
|
||||
FROM (
|
||||
SELECT
|
||||
""ID"" AS ""ID"",
|
||||
""BlocksetID"" AS ""BlocksetID""
|
||||
FROM ""FileLookup""
|
||||
WHERE
|
||||
""BlocksetID"" != {FOLDER_BLOCKSET_ID}
|
||||
AND ""BlocksetID"" != {SYMLINK_BLOCKSET_ID}
|
||||
UNION
|
||||
SELECT
|
||||
""A"".""ID"" AS ""ID"",
|
||||
""B"".""BlocksetID"" AS ""BlocksetID""
|
||||
FROM ""FileLookup"" ""A""
|
||||
LEFT JOIN ""Metadataset"" ""B""
|
||||
ON ""A"".""MetadataID"" = ""B"".""ID""
|
||||
)
|
||||
WHERE
|
||||
""BlocksetID"" IS NULL
|
||||
OR ""BlocksetID"" IN (
|
||||
SELECT DISTINCT ""BlocksetID""
|
||||
FROM (
|
||||
SELECT ""BlocksetID""
|
||||
FROM ""BlocksetEntry""
|
||||
WHERE ""BlockID"" NOT IN (
|
||||
SELECT ""ID""
|
||||
FROM ""Block""
|
||||
WHERE ""VolumeID"" IN ({BLOCK_VOLUME_IDS})
|
||||
)
|
||||
UNION
|
||||
SELECT ""BlocksetID""
|
||||
FROM ""BlocklistHash""
|
||||
WHERE ""Hash"" NOT IN (
|
||||
SELECT ""Hash""
|
||||
FROM ""Block""
|
||||
WHERE ""VolumeID"" IN ({BLOCK_VOLUME_IDS})
|
||||
)
|
||||
UNION
|
||||
SELECT ""A"".""ID"" AS ""BlocksetID""
|
||||
FROM ""Blockset"" ""A""
|
||||
LEFT JOIN ""BlocksetEntry"" ""B""
|
||||
ON ""A"".""ID"" = ""B"".""BlocksetID""
|
||||
WHERE
|
||||
""A"".""Length"" > 0
|
||||
AND ""B"".""BlocksetID"" IS NULL
|
||||
)
|
||||
WHERE ""BlocksetID"" NOT IN (
|
||||
SELECT ""ID""
|
||||
FROM ""Blockset""
|
||||
WHERE ""Length"" == 0
|
||||
)
|
||||
)
|
||||
";
|
||||
|
||||
/// <summary>
|
||||
/// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume.
|
||||
/// This is used to clean up the metadata blocksets that are now missing.
|
||||
/// </summary>
|
||||
/// <param name="filesetId">The filesetId to target</param>
|
||||
/// <param name="emptyBlocksetId">The empty blockset ID to replace with</param>
|
||||
/// <param name="transaction">The transaction to use for the query</param>
|
||||
/// <returns>The number of rows affected</returns>
|
||||
public int ReplaceMetadata(long filesetId, long emptyBlocksetId, IDbTransaction? transaction)
|
||||
{
|
||||
using var cmd = m_connection.CreateCommand(transaction, @"
|
||||
UPDATE ""Metadataset""
|
||||
SET ""BlocksetID"" = @EmptyBlocksetID
|
||||
WHERE
|
||||
""ID"" IN (
|
||||
SELECT ""FileLookup"".""MetadataID""
|
||||
FROM ""FileLookup"", ""FilesetEntry""
|
||||
WHERE
|
||||
""FilesetEntry"".""FilesetId"" = @FilesetId
|
||||
AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID""
|
||||
)
|
||||
AND ""BlocksetID"" NOT IN (
|
||||
SELECT ""BlocksetID""
|
||||
FROM ""BlocksetEntry"", ""Block""
|
||||
WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID""
|
||||
)")
|
||||
.SetParameterValue("@EmptyBlocksetId", emptyBlocksetId)
|
||||
.SetParameterValue("@FilesetId", filesetId);
|
||||
return cmd.ExecuteNonQuery();
|
||||
}
|
||||
/// <summary>
|
||||
/// SQL query to get the broken filesets, i.e., filesets that contain broken files.
|
||||
/// </summary>
|
||||
private static readonly string BROKEN_FILE_SETS = $@"
|
||||
SELECT DISTINCT
|
||||
""B"".""Timestamp"",
|
||||
""A"".""FilesetID"",
|
||||
COUNT(""A"".""FileID"") AS ""FileCount""
|
||||
FROM
|
||||
""FilesetEntry"" ""A"",
|
||||
""Fileset"" ""B""
|
||||
WHERE
|
||||
""A"".""FilesetID"" = ""B"".""ID""
|
||||
AND ""A"".""FileID"" IN ({BROKEN_FILE_IDS})
|
||||
";
|
||||
|
||||
public void RemoveMissingBlocks(IEnumerable<string> names, IDbTransaction transaction)
|
||||
{
|
||||
if (names == null || !names.Any()) return;
|
||||
if (transaction == null)
|
||||
throw new Exception("This function cannot be called when not in a transaction, as it leaves the database in an inconsistent state");
|
||||
/// <summary>
|
||||
/// SQL query to get the names and lengths of broken files.
|
||||
/// </summary>
|
||||
private static readonly string BROKEN_FILE_NAMES = $@"
|
||||
SELECT
|
||||
""A"".""Path"",
|
||||
""B"".""Length""
|
||||
FROM ""File"" ""A""
|
||||
LEFT JOIN ""Blockset"" ""B""
|
||||
ON (""A"".""BlocksetID"" = ""B"".""ID"")
|
||||
WHERE
|
||||
""A"".""ID"" IN ({BROKEN_FILE_IDS})
|
||||
AND ""A"".""ID"" IN (
|
||||
SELECT ""FileID""
|
||||
FROM ""FilesetEntry""
|
||||
WHERE ""FilesetID"" = @FilesetId
|
||||
)
|
||||
";
|
||||
|
||||
using (var deletecmd = m_connection.CreateCommand(transaction))
|
||||
{
|
||||
var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
||||
var volidstable = "DelVolSetIds-" + temptransguid;
|
||||
/// <summary>
|
||||
/// SQL query to insert broken file IDs into a specified table.
|
||||
/// The table must have a single column with the same name as the ID field name.
|
||||
/// </summary>
|
||||
private static string INSERT_BROKEN_IDS(string tablename, string IDfieldname) => $@"
|
||||
INSERT INTO ""{tablename}"" (
|
||||
""{IDfieldname}""
|
||||
)
|
||||
{BROKEN_FILE_IDS}
|
||||
AND ""ID"" IN (
|
||||
SELECT ""FileID""
|
||||
FROM ""FilesetEntry""
|
||||
WHERE ""FilesetID"" = @FilesetId
|
||||
)
|
||||
";
|
||||
|
||||
// Create and fill a temp table with the volids to delete. We avoid using too many parameters that way.
|
||||
deletecmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMP TABLE ""{volidstable}"" (""ID"" INTEGER PRIMARY KEY)"));
|
||||
|
||||
using (var tmptable = new TemporaryDbValueList(m_connection, transaction, names))
|
||||
deletecmd.SetCommandAndParameters(FormatInvariant($@"INSERT OR IGNORE INTO ""{volidstable}"" (""ID"") SELECT ""ID"" FROM ""RemoteVolume"" WHERE ""Name"" IN (@Names)"))
|
||||
.ExpandInClauseParameter("@Names", tmptable)
|
||||
.ExecuteNonQuery();
|
||||
|
||||
var volIdsSubQuery = FormatInvariant($@"SELECT ""ID"" FROM ""{volidstable}"" ");
|
||||
deletecmd.Parameters.Clear();
|
||||
|
||||
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({volIdsSubQuery}) OR ""IndexVolumeID"" IN ({volIdsSubQuery})"));
|
||||
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"));
|
||||
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"));
|
||||
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""DuplicateBlock"" WHERE ""VolumeID"" IN ({volIdsSubQuery})"));
|
||||
|
||||
// Clean up temp tables for subqueries. We truncate content and then try to delete.
|
||||
// Drop in try-block, as it fails in nested transactions (SQLite problem)
|
||||
// SQLite.SQLiteException (0x80004005): database table is locked
|
||||
deletecmd.ExecuteNonQuery(FormatInvariant($@"DELETE FROM ""{volidstable}"" "));
|
||||
try
|
||||
/// <summary>
|
||||
/// Creates a new instance of the <see cref="LocalListBrokenFilesDatabase"/> class.
|
||||
/// </summary>
|
||||
/// <param name="path">The path to the database file.</param>
|
||||
/// <param name="dbnew">An optional existing database instance to use. Used to mimic constructor chaining.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains a new instance of <see cref="LocalListBrokenFilesDatabase"/>.</returns>
|
||||
public static async Task<LocalListBrokenFilesDatabase> CreateAsync(string path, LocalListBrokenFilesDatabase? dbnew, CancellationToken token)
|
||||
{
|
||||
deletecmd.CommandTimeout = 2;
|
||||
deletecmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{volidstable}"" "));
|
||||
}
|
||||
catch { /* Ignore, will be deleted on close anyway. */ }
|
||||
}
|
||||
}
|
||||
dbnew ??= new LocalListBrokenFilesDatabase();
|
||||
|
||||
public long GetFilesetFileCount(long filesetid, IDbTransaction transaction)
|
||||
{
|
||||
using var cmd = m_connection.CreateCommand(transaction, @"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = @FilesetId")
|
||||
.SetParameterValue("@FilesetId", filesetid);
|
||||
return cmd.ExecuteScalarInt64(0);
|
||||
dbnew = (LocalListBrokenFilesDatabase)
|
||||
await CreateLocalDatabaseAsync(path, "ListBrokenFiles", false, dbnew, token)
|
||||
.ConfigureAwait(false);
|
||||
dbnew.ShouldCloseConnection = true;
|
||||
|
||||
return dbnew;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of the <see cref="LocalListBrokenFilesDatabase"/> class.
|
||||
/// </summary>
|
||||
/// <param name="dbparent">The parent database to use for the new database.</param>
|
||||
/// <param name="dbnew">An optional existing database instance to use. Used to mimic constructor chaining.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains a new instance of <see cref="LocalListBrokenFilesDatabase"/>.</returns>
|
||||
public static async Task<LocalListBrokenFilesDatabase> CreateAsync(LocalDatabase dbparent, LocalListBrokenFilesDatabase? dbnew, CancellationToken token)
|
||||
{
|
||||
dbnew ??= new LocalListBrokenFilesDatabase();
|
||||
|
||||
dbnew = (LocalListBrokenFilesDatabase)
|
||||
await CreateLocalDatabaseAsync(dbparent, dbnew, token)
|
||||
.ConfigureAwait(false);
|
||||
dbnew.ShouldCloseConnection = false;
|
||||
|
||||
return dbnew;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns all broken file IDs, i.e., files that reference blocksets or blocks that are not available.
|
||||
/// </summary>
|
||||
/// <param name="time">The time to filter filesets by.</param>
|
||||
/// <param name="versions">Optional array of versions to filter filesets by. If null, all versions are considered.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>An asynchronous enumerable of broken file IDs.</returns>
|
||||
public async IAsyncEnumerable<(DateTime FilesetTime, long FilesetID, long RemoveFileCount)> GetBrokenFilesets(DateTime time, long[]? versions, [EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
var query = BROKEN_FILE_SETS;
|
||||
var clause = await GetFilelistWhereClause(time, versions, null, false, token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(clause.Query))
|
||||
query += $@"
|
||||
AND ""A"".""FilesetID"" IN (
|
||||
SELECT ""ID""
|
||||
FROM ""Fileset""
|
||||
{clause.Query}
|
||||
)
|
||||
";
|
||||
|
||||
query += @" GROUP BY ""A"".""FilesetID""";
|
||||
|
||||
await using var cmd = Connection.CreateCommand(m_rtr)
|
||||
.SetCommandAndParameters(query)
|
||||
.SetParameterValues(clause.Values);
|
||||
|
||||
await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false))
|
||||
if (!rd.IsDBNull(0))
|
||||
yield return (
|
||||
ParseFromEpochSeconds(rd.ConvertValueToInt64(0, 0)),
|
||||
rd.ConvertValueToInt64(1, -1),
|
||||
rd.ConvertValueToInt64(2, 0)
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns all broken file IDs, i.e., files that reference blocksets or blocks that are not available.
|
||||
/// </summary>
|
||||
/// <param name="filesetid">The fileset ID to filter by.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>An asynchronous enumerable of broken file IDs.</returns>
|
||||
public async IAsyncEnumerable<Tuple<string, long>> GetBrokenFilenames(long filesetid, [EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
await using var cmd = Connection.CreateCommand(m_rtr)
|
||||
.SetCommandAndParameters(BROKEN_FILE_NAMES)
|
||||
.SetParameterValue("@FilesetId", filesetid);
|
||||
|
||||
await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false))
|
||||
if (!rd.IsDBNull(0))
|
||||
yield return new Tuple<string, long>(
|
||||
rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"),
|
||||
rd.ConvertValueToInt64(1)
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns all index files that are orphaned, i.e., not referenced by any block files.
|
||||
/// </summary>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>An asynchronous enumerable of <see cref="RemoteVolume"/> representing the orphaned index files.</returns>
|
||||
public async IAsyncEnumerable<RemoteVolume> GetOrphanedIndexFiles([EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
await using var cmd = Connection.CreateCommand($@"
|
||||
SELECT
|
||||
""Name"",
|
||||
""Hash"",
|
||||
""Size""
|
||||
FROM ""RemoteVolume""
|
||||
WHERE
|
||||
""Type"" = '{RemoteVolumeType.Index}'
|
||||
AND ""ID"" NOT IN (
|
||||
SELECT ""IndexVolumeID""
|
||||
FROM ""IndexBlockLink""
|
||||
)
|
||||
")
|
||||
.SetTransaction(m_rtr);
|
||||
|
||||
await foreach (var rd in cmd.ExecuteReaderEnumerableAsync(token).ConfigureAwait(false))
|
||||
yield return new RemoteVolume(
|
||||
rd.ConvertValueToString(0) ?? throw new Exception("Filename was null"),
|
||||
rd.ConvertValueToString(1) ?? throw new Exception("Hash was null"),
|
||||
rd.ConvertValueToInt64(2, -1)
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Inserts the broken file IDs into the given table. The table must have a single column with the same name as the ID field name.
|
||||
/// </summary>
|
||||
/// <param name="filesetid">The filset id for the current operation.</param>
|
||||
/// <param name="tablename">The name of the table to insert into.</param>
|
||||
/// <param name="IDfieldname">The name of the ID field in the table.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the insertion is finished.</returns>
|
||||
public async Task InsertBrokenFileIDsIntoTable(long filesetid, string tablename, string IDfieldname, CancellationToken token)
|
||||
{
|
||||
await using var cmd = Connection.CreateCommand(m_rtr)
|
||||
.SetCommandAndParameters(INSERT_BROKEN_IDS(tablename, IDfieldname))
|
||||
.SetParameterValue("@FilesetId", filesetid);
|
||||
|
||||
await cmd.ExecuteNonQueryAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replaces the metadata blockset ID in the Metadataset table with the empty blockset ID for all fileset entries that are not in any block volume.
|
||||
/// This is used to clean up the metadata blocksets that are now missing.
|
||||
/// </summary>
|
||||
/// <param name="filesetId">The filesetId to target.</param>
|
||||
/// <param name="emptyBlocksetId">The empty blockset ID to replace with.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the number of rows affected</returns>
|
||||
public async Task<int> ReplaceMetadata(long filesetId, long emptyBlocksetId, CancellationToken token)
|
||||
{
|
||||
await using var cmd = m_connection.CreateCommand(@"
|
||||
UPDATE ""Metadataset""
|
||||
SET ""BlocksetID"" = @EmptyBlocksetID
|
||||
WHERE
|
||||
""ID"" IN (
|
||||
SELECT ""FileLookup"".""MetadataID""
|
||||
FROM
|
||||
""FileLookup"",
|
||||
""FilesetEntry""
|
||||
WHERE
|
||||
""FilesetEntry"".""FilesetId"" = @FilesetID
|
||||
AND ""FileLookup"".""ID"" = ""FilesetEntry"".""FileID""
|
||||
)
|
||||
AND ""BlocksetID"" NOT IN (
|
||||
SELECT ""BlocksetID""
|
||||
FROM
|
||||
""BlocksetEntry"",
|
||||
""Block""
|
||||
WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID""
|
||||
)
|
||||
")
|
||||
.SetTransaction(m_rtr)
|
||||
.SetParameterValue("@EmptyBlocksetID", emptyBlocksetId)
|
||||
.SetParameterValue("@FilesetID", filesetId);
|
||||
|
||||
return await cmd.ExecuteNonQueryAsync(token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes all blocks, blocksets, and index links that are missing from the specified volumes.
|
||||
/// </summary>
|
||||
/// <param name="names">The names of the volumes to check for missing blocks.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the removal is finished.</returns>
|
||||
public async Task RemoveMissingBlocks(IEnumerable<string> names, CancellationToken token)
|
||||
{
|
||||
if (names == null || !names.Any()) return;
|
||||
|
||||
await using var deletecmd = m_connection.CreateCommand(m_rtr);
|
||||
var temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
||||
var volidstable = "DelVolSetIds-" + temptransguid;
|
||||
|
||||
// Create and fill a temp table with the volids to delete. We avoid using too many parameters that way.
|
||||
await deletecmd.ExecuteNonQueryAsync($@"
|
||||
CREATE TEMP TABLE ""{volidstable}"" (
|
||||
""ID"" INTEGER PRIMARY KEY
|
||||
)
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await using (var tmptable = await TemporaryDbValueList.CreateAsync(this, names, token).ConfigureAwait(false))
|
||||
await (
|
||||
await deletecmd.SetCommandAndParameters($@"
|
||||
INSERT OR IGNORE INTO ""{volidstable}"" (""ID"")
|
||||
SELECT ""ID""
|
||||
FROM ""RemoteVolume""
|
||||
WHERE ""Name"" IN (@Names)
|
||||
")
|
||||
.ExpandInClauseParameterMssqliteAsync("@Names", tmptable, token)
|
||||
.ConfigureAwait(false)
|
||||
)
|
||||
.ExecuteNonQueryAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var volIdsSubQuery = $@"
|
||||
SELECT ""ID""
|
||||
FROM ""{volidstable}""
|
||||
";
|
||||
|
||||
await deletecmd.ExecuteNonQueryAsync($@"
|
||||
DELETE FROM ""IndexBlockLink""
|
||||
WHERE
|
||||
""BlockVolumeID"" IN ({volIdsSubQuery})
|
||||
OR ""IndexVolumeID"" IN ({volIdsSubQuery})
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await deletecmd.ExecuteNonQueryAsync($@"
|
||||
DELETE FROM ""Block""
|
||||
WHERE ""VolumeID"" IN ({volIdsSubQuery})
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await deletecmd.ExecuteNonQueryAsync($@"
|
||||
DELETE FROM ""DeletedBlock""
|
||||
WHERE ""VolumeID"" IN ({volIdsSubQuery})
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await deletecmd.ExecuteNonQueryAsync($@"
|
||||
DELETE FROM ""DuplicateBlock""
|
||||
WHERE ""VolumeID"" IN ({volIdsSubQuery})
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Clean up temp tables for subqueries. We truncate content and then try to delete.
|
||||
// Drop in try-block, as it fails in nested transactions (SQLite problem)
|
||||
// SQLite.SQLiteException (0x80004005): database table is locked
|
||||
await deletecmd
|
||||
.ExecuteNonQueryAsync($@"DELETE FROM ""{volidstable}"" ", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
deletecmd.CommandTimeout = 2;
|
||||
await deletecmd
|
||||
.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{volidstable}"" ", token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch { /* Ignore, will be deleted on close anyway. */ }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the count of files in a specific fileset.
|
||||
/// </summary>
|
||||
/// <param name="filesetid">The ID of the fileset to count files in.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the count of files in the specified fileset.</returns>
|
||||
public async Task<long> GetFilesetFileCount(long filesetid, CancellationToken token)
|
||||
{
|
||||
await using var cmd = m_connection.CreateCommand(@"
|
||||
SELECT COUNT(*)
|
||||
FROM ""FilesetEntry""
|
||||
WHERE ""FilesetID"" = @FilesetId
|
||||
")
|
||||
.SetTransaction(m_rtr)
|
||||
.SetParameterValue("@FilesetId", filesetid);
|
||||
|
||||
return await cmd
|
||||
.ExecuteScalarInt64Async(0, token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,119 +1,277 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.Data;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Utility;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace Duplicati.Library.Main.Database
|
||||
{
|
||||
/// <summary>
|
||||
/// A local database for purging files from a fileset.
|
||||
/// </summary>
|
||||
internal class LocalPurgeDatabase : LocalDeleteDatabase
|
||||
{
|
||||
public LocalPurgeDatabase(string path, long pagecachesize)
|
||||
: base(path, "Purge", pagecachesize)
|
||||
/// <summary>
|
||||
/// Creates a new instance of the <see cref="LocalPurgeDatabase"/> class.
|
||||
/// </summary>
|
||||
/// <param name="path">The path to the database file.</param>
|
||||
/// <param name="dbnew">The optional existing database instance to use. Used to mimic constructor chaining.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains a new instance of <see cref="LocalPurgeDatabase"/>.</returns>
|
||||
public static async Task<LocalPurgeDatabase> CreateAsync(string path, LocalPurgeDatabase? dbnew, CancellationToken token)
|
||||
{
|
||||
ShouldCloseConnection = true;
|
||||
dbnew ??= new LocalPurgeDatabase();
|
||||
|
||||
dbnew = (LocalPurgeDatabase)
|
||||
await LocalDeleteDatabase.CreateAsync(path, "Purge", dbnew, token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
dbnew.ShouldCloseConnection = true;
|
||||
|
||||
return dbnew;
|
||||
}
|
||||
|
||||
public LocalPurgeDatabase(LocalDatabase db)
|
||||
: base(db)
|
||||
/// <summary>
|
||||
/// Creates a new instance of the <see cref="LocalPurgeDatabase"/> class.
|
||||
/// </summary>
|
||||
/// <param name="dbparent">The parent database to use.</param>
|
||||
/// <param name="dbnew">An optional existing database instance to use. Used to mimic constructor chaining.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains a new instance of <see cref="LocalPurgeDatabase"/>.</returns>
|
||||
public static async Task<LocalPurgeDatabase> CreateAsync(LocalDatabase dbparent, LocalPurgeDatabase? dbnew, CancellationToken token)
|
||||
{
|
||||
dbnew ??= new LocalPurgeDatabase();
|
||||
|
||||
dbnew = (LocalPurgeDatabase)
|
||||
await LocalDeleteDatabase.CreateAsync(dbparent, dbnew, token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return dbnew;
|
||||
}
|
||||
|
||||
public ITemporaryFileset CreateTemporaryFileset(long parentid, IDbTransaction transaction)
|
||||
/// <summary>
|
||||
/// Creates a new instance of the <see cref="ITemporaryFileset"/> class.
|
||||
/// </summary>
|
||||
/// <param name="parentid">The ID of the parent fileset.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains a new instance of <see cref="ITemporaryFileset"/>.</returns>
|
||||
public async Task<ITemporaryFileset> CreateTemporaryFileset(long parentid, CancellationToken token)
|
||||
{
|
||||
return new TemporaryFileset(parentid, this, m_connection, transaction);
|
||||
return await TemporaryFileset.CreateAsync(parentid, this, token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public string GetRemoteVolumeNameForFileset(long id, IDbTransaction transaction)
|
||||
/// <summary>
|
||||
/// Gets the remote volume name for a fileset with the specified ID.
|
||||
/// </summary>
|
||||
/// <param name="id">The ID of the fileset.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the name of the remote volume.</returns>
|
||||
public async Task<string> GetRemoteVolumeNameForFileset(long id, CancellationToken token)
|
||||
{
|
||||
using var cmd = m_connection.CreateCommand(transaction, @"SELECT ""B"".""Name"" FROM ""Fileset"" A, ""RemoteVolume"" B WHERE ""A"".""VolumeID"" = ""B"".""ID"" AND ""A"".""ID"" = @FilesetId ")
|
||||
.SetParameterValue("@FilesetId", id);
|
||||
using (var rd = cmd.ExecuteReader())
|
||||
if (!rd.Read())
|
||||
throw new Exception($"No remote volume found for fileset with id {id}");
|
||||
else
|
||||
return rd.ConvertValueToString(0) ?? throw new Exception($"Remote volume name for fileset with id {id} is null");
|
||||
await using var cmd = m_connection.CreateCommand(@"
|
||||
SELECT
|
||||
""B"".""Name""
|
||||
FROM
|
||||
""Fileset"" ""A"",
|
||||
""RemoteVolume"" ""B""
|
||||
WHERE
|
||||
""A"".""VolumeID"" = ""B"".""ID""
|
||||
AND ""A"".""ID"" = @FilesetId
|
||||
")
|
||||
.SetParameterValue("@FilesetId", id);
|
||||
|
||||
await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false);
|
||||
if (!await rd.ReadAsync(token).ConfigureAwait(false))
|
||||
throw new Exception($"No remote volume found for fileset with id {id}");
|
||||
else
|
||||
return rd.ConvertValueToString(0) ?? throw new Exception($"Remote volume name for fileset with id {id} is null");
|
||||
}
|
||||
|
||||
internal long CountOrphanFiles(IDbTransaction transaction)
|
||||
/// <summary>
|
||||
/// Counts the number of orphan files in the database.
|
||||
/// </summary>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the count of orphan files.</returns>
|
||||
internal async Task<long> CountOrphanFiles(CancellationToken token)
|
||||
{
|
||||
using (var cmd = m_connection.CreateCommand(transaction))
|
||||
using (var rd = cmd.ExecuteReader(@"SELECT COUNT(*) FROM ""FileLookup"" WHERE ""ID"" NOT IN (SELECT DISTINCT ""FileID"" FROM ""FilesetEntry"")"))
|
||||
if (rd.Read())
|
||||
return rd.ConvertValueToInt64(0, 0);
|
||||
else
|
||||
return 0;
|
||||
await using var cmd = m_connection.CreateCommand(m_rtr);
|
||||
cmd.SetCommandAndParameters(@"
|
||||
SELECT COUNT(*)
|
||||
FROM ""FileLookup""
|
||||
WHERE ""ID"" NOT IN (
|
||||
SELECT DISTINCT ""FileID""
|
||||
FROM ""FilesetEntry""
|
||||
)
|
||||
");
|
||||
await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false);
|
||||
if (await rd.ReadAsync(token).ConfigureAwait(false))
|
||||
return rd.ConvertValueToInt64(0, 0);
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
public interface ITemporaryFileset : IDisposable
|
||||
/// <summary>
|
||||
/// Interface for a temporary fileset used for purging files from a fileset.
|
||||
/// </summary>
|
||||
public interface ITemporaryFileset : IDisposable, IAsyncDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the ID of the parent fileset.
|
||||
/// </summary>
|
||||
long ParentID { get; }
|
||||
/// <summary>
|
||||
/// Gets the count of removed files.
|
||||
/// </summary>
|
||||
long RemovedFileCount { get; }
|
||||
/// <summary>
|
||||
/// Gets the total size of removed files.
|
||||
/// </summary>
|
||||
long RemovedFileSize { get; }
|
||||
/// <summary>
|
||||
/// Gets the count of updated files.
|
||||
/// </summary>
|
||||
long UpdatedFileCount { get; }
|
||||
|
||||
void ApplyFilter(Library.Utility.IFilter filter);
|
||||
void ApplyFilter(Func<IDbCommand, long, string, int> filtercommand);
|
||||
Tuple<long, long> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup);
|
||||
IEnumerable<KeyValuePair<string, long>> ListAllDeletedFiles();
|
||||
/// <summary>
|
||||
/// Applies a filter to the temporary fileset.
|
||||
/// </summary>
|
||||
/// <param name="filter">The filter to apply.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the filter has been applied.</returns>
|
||||
Task ApplyFilter(Library.Utility.IFilter filter, CancellationToken token);
|
||||
/// <summary>
|
||||
/// Applies a filter using a custom command.
|
||||
/// </summary>
|
||||
/// <param name="filtercommand">The command to execute for filtering.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the filter has been applied.</returns>
|
||||
Task ApplyFilter(Func<SqliteCommand, long, string, Task<int>> filtercommand, CancellationToken token);
|
||||
/// <summary>
|
||||
/// Converts the temporary fileset to a permanent fileset.
|
||||
/// </summary>
|
||||
/// <param name="name">The name of the new fileset.</param>
|
||||
/// <param name="timestamp">The timestamp for the new fileset.</param>
|
||||
/// <param name="isFullBackup">Indicates if this is a full backup.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains a tuple with the remote volume ID and the new fileset ID.</returns>
|
||||
Task<Tuple<long, long>> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup, CancellationToken token);
|
||||
/// <summary>
|
||||
/// Lists all deleted files in the temporary fileset.
|
||||
/// </summary>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>An asynchronous enumerable of key-value pairs where the key is the file path and the value is the file size.</returns>
|
||||
IAsyncEnumerable<KeyValuePair<string, long>> ListAllDeletedFiles(CancellationToken token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A temporary fileset implementation for purging files from a fileset.
|
||||
/// </summary>
|
||||
private class TemporaryFileset : ITemporaryFileset
|
||||
{
|
||||
private readonly IDbConnection m_connection;
|
||||
private readonly IDbTransaction m_transaction;
|
||||
private readonly string m_tablename;
|
||||
private readonly LocalPurgeDatabase m_parentdb;
|
||||
/// <summary>
|
||||
/// The name of the temporary table used for storing deleted files.
|
||||
/// </summary>
|
||||
private string m_tablename = null!;
|
||||
/// <summary>
|
||||
/// The database instance used for queries.
|
||||
/// </summary>
|
||||
private LocalDatabase m_db = null!;
|
||||
|
||||
public long ParentID { get; private set; }
|
||||
public long RemovedFileCount { get; private set; }
|
||||
public long RemovedFileSize { get; private set; }
|
||||
public long UpdatedFileCount { get; private set; }
|
||||
|
||||
public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, IDbConnection connection, IDbTransaction transaction)
|
||||
/// <summary>
|
||||
/// Calling this constructor will throw an exception. Use CreateAsync instead.
|
||||
/// </summary>
|
||||
[Obsolete("Calling this constructor will throw an exception. Use CreateAsync instead.")]
|
||||
public TemporaryFileset(long parentid, LocalPurgeDatabase parentdb, SqliteConnection connection, SqliteTransaction transaction)
|
||||
{
|
||||
ParentID = parentid;
|
||||
m_parentdb = parentdb;
|
||||
m_connection = connection;
|
||||
m_transaction = transaction;
|
||||
m_tablename = "TempDeletedFilesTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
||||
|
||||
using (var cmd = m_connection.CreateCommand(m_transaction))
|
||||
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{m_tablename}"" (""FileID"" INTEGER PRIMARY KEY) "));
|
||||
throw new NotImplementedException("Use CreateAsync instead");
|
||||
}
|
||||
|
||||
public void ApplyFilter(Func<IDbCommand, long, string, int> filtercommand)
|
||||
/// <summary>
|
||||
/// Private constructor to prevent direct instantiation.
|
||||
/// </summary>
|
||||
private TemporaryFileset() { }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new instance of the <see cref="TemporaryFileset"/> class asynchronously.
|
||||
/// </summary>
|
||||
/// <param name="parentid">The ID of the parent fileset.</param>
|
||||
/// <param name="db">The database instance to use.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains a new instance of <see cref="TemporaryFileset"/>.</returns>
|
||||
public static async Task<TemporaryFileset> CreateAsync(long parentid, LocalDatabase db, CancellationToken token)
|
||||
{
|
||||
var tempf = new TemporaryFileset()
|
||||
{
|
||||
ParentID = parentid,
|
||||
m_db = db,
|
||||
m_tablename = "TempDeletedFilesTable-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray())
|
||||
};
|
||||
|
||||
await using (var cmd = db.Connection.CreateCommand())
|
||||
await cmd.ExecuteNonQueryAsync($@"
|
||||
CREATE TEMPORARY TABLE ""{tempf.m_tablename}"" (
|
||||
""FileID"" INTEGER PRIMARY KEY
|
||||
)
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return tempf;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies a filter to the temporary fileset using a custom command.
|
||||
/// </summary>
|
||||
/// <param name="filtercommand">The command to execute for filtering.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the filter has been applied.</returns>
|
||||
public async Task ApplyFilter(Func<SqliteCommand, long, string, Task<int>> filtercommand, CancellationToken token)
|
||||
{
|
||||
int updated;
|
||||
using (var cmd = m_connection.CreateCommand(m_transaction))
|
||||
updated = filtercommand(cmd, ParentID, m_tablename);
|
||||
await using (var cmd = m_db.Connection.CreateCommand())
|
||||
updated = await filtercommand(cmd, ParentID, m_tablename)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
PostFilterChecks(updated);
|
||||
await PostFilterChecks(updated, token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public void ApplyFilter(Library.Utility.IFilter filter)
|
||||
/// <summary>
|
||||
/// Applies a filter to the temporary fileset.
|
||||
/// </summary>
|
||||
/// <param name="filter">The filter to apply.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the filter has been applied.</returns>
|
||||
public async Task ApplyFilter(Library.Utility.IFilter filter, CancellationToken token)
|
||||
{
|
||||
if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple)
|
||||
{
|
||||
@@ -122,103 +280,219 @@ namespace Duplicati.Library.Main.Database
|
||||
// SQLite only supports ASCII compares
|
||||
var p = expression.GetSimpleList();
|
||||
var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
|
||||
using (var cmd = m_connection.CreateCommand(m_transaction))
|
||||
{
|
||||
cmd.ExecuteNonQuery(FormatInvariant($@"CREATE TEMPORARY TABLE ""{filenamestable}"" (""Path"" TEXT NOT NULL) "));
|
||||
cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{filenamestable}"" (""Path"") VALUES (@Path)"));
|
||||
foreach (var s in p)
|
||||
cmd.SetParameterValue("@Path", s)
|
||||
.ExecuteNonQuery();
|
||||
await using var cmd = m_db.Connection.CreateCommand();
|
||||
await cmd.ExecuteNonQueryAsync($@"
|
||||
CREATE TEMPORARY TABLE ""{filenamestable}"" (
|
||||
""Path"" TEXT NOT NULL
|
||||
)
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""FileID"") SELECT DISTINCT ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""Path"" IN ""{filenamestable}"""))
|
||||
.SetParameterValue("@FilesetId", ParentID)
|
||||
.ExecuteNonQuery();
|
||||
cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{filenamestable}"" "));
|
||||
}
|
||||
await cmd.SetCommandAndParameters($@"
|
||||
INSERT INTO ""{filenamestable}"" (""Path"")
|
||||
VALUES (@Path)
|
||||
")
|
||||
.PrepareAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
foreach (var s in p)
|
||||
await cmd.SetParameterValue("@Path", s)
|
||||
.ExecuteNonQueryAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await cmd.SetCommandAndParameters($@"
|
||||
INSERT INTO ""{m_tablename}"" (""FileID"")
|
||||
SELECT DISTINCT ""A"".""FileID""
|
||||
FROM
|
||||
""FilesetEntry"" ""A"",
|
||||
""File"" ""B""
|
||||
WHERE
|
||||
""A"".""FilesetID"" = @FilesetId
|
||||
AND ""A"".""FileID"" = ""B"".""ID""
|
||||
AND ""B"".""Path"" IN ""{filenamestable}""
|
||||
")
|
||||
.SetParameterValue("@FilesetId", ParentID)
|
||||
.ExecuteNonQueryAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await cmd
|
||||
.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{filenamestable}"" ", token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Do row-wise iteration
|
||||
var values = new object[2];
|
||||
using (var cmd = m_connection.CreateCommand(m_transaction))
|
||||
using (var cmd2 = m_connection.CreateCommand(m_transaction))
|
||||
await using var cmd = m_db.Connection.CreateCommand();
|
||||
await using var cmd2 = m_db.Connection.CreateCommand();
|
||||
await cmd2.SetCommandAndParameters($@"
|
||||
INSERT INTO ""{m_tablename}"" (""FileID"")
|
||||
VALUES (@FileId)
|
||||
")
|
||||
.PrepareAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
cmd.SetCommandAndParameters(@"
|
||||
SELECT
|
||||
""B"".""Path"",
|
||||
""A"".""FileID""
|
||||
FROM
|
||||
""FilesetEntry"" ""A"",
|
||||
""File"" ""B""
|
||||
WHERE
|
||||
""A"".""FilesetID"" = @FilesetId
|
||||
AND ""A"".""FileID"" = ""B"".""ID""
|
||||
")
|
||||
.SetParameterValue("@FilesetId", ParentID);
|
||||
|
||||
await using var rd = await cmd.ExecuteReaderAsync(token).ConfigureAwait(false);
|
||||
while (await rd.ReadAsync(token).ConfigureAwait(false))
|
||||
{
|
||||
cmd2.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""{m_tablename}"" (""FileID"") VALUES (@FileId)"));
|
||||
cmd.SetCommandAndParameters(@"SELECT ""B"".""Path"", ""A"".""FileID"" FROM ""FilesetEntry"" A, ""File"" B WHERE ""A"".""FilesetID"" = @FilesetId AND ""A"".""FileID"" = ""B"".""ID"" ")
|
||||
.SetParameterValue("@FilesetId", ParentID);
|
||||
using (var rd = cmd.ExecuteReader())
|
||||
while (rd.Read())
|
||||
{
|
||||
rd.GetValues(values);
|
||||
var path = values[0] as string;
|
||||
if (path != null && Library.Utility.FilterExpression.Matches(filter, path.ToString()))
|
||||
{
|
||||
cmd2.SetParameterValue("@FileId", values[1])
|
||||
.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
rd.GetValues(values);
|
||||
var path = values[0] as string;
|
||||
if (path != null && Library.Utility.FilterExpression.Matches(filter, path.ToString()))
|
||||
{
|
||||
await cmd2
|
||||
.SetParameterValue("@FileId", values[1])
|
||||
.ExecuteNonQueryAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
PostFilterChecks(0);
|
||||
await PostFilterChecks(0, token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private void PostFilterChecks(int updated)
|
||||
/// <summary>
|
||||
/// Applies the filter checks after filtering.
|
||||
/// </summary>
|
||||
/// <param name="updated">The number of files updated by the filter.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the checks have been applied.</returns>
|
||||
private async Task PostFilterChecks(int updated, CancellationToken token)
|
||||
{
|
||||
using (var cmd = m_connection.CreateCommand(m_transaction))
|
||||
{
|
||||
UpdatedFileCount = updated;
|
||||
RemovedFileCount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""{m_tablename}"""), 0);
|
||||
RemovedFileSize = cmd.ExecuteScalarInt64(FormatInvariant($@"
|
||||
SELECT
|
||||
SUM(""C"".""Length"")
|
||||
await using var cmd = m_db.Connection.CreateCommand();
|
||||
UpdatedFileCount = updated;
|
||||
|
||||
RemovedFileCount = await cmd.ExecuteScalarInt64Async($@"
|
||||
SELECT COUNT(*)
|
||||
FROM ""{m_tablename}""
|
||||
", 0, token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
RemovedFileSize = await cmd.ExecuteScalarInt64Async($@"
|
||||
SELECT SUM(""C"".""Length"")
|
||||
FROM
|
||||
""{m_tablename}"" A, ""FileLookup"" B, ""Blockset"" C, ""Metadataset"" D
|
||||
""{m_tablename}"" ""A"",
|
||||
""FileLookup"" ""B"",
|
||||
""Blockset"" ""C"",
|
||||
""Metadataset"" ""D""
|
||||
WHERE
|
||||
""A"".""FileID"" = ""B"".""ID""
|
||||
AND(""B"".""BlocksetID"" = ""C"".""ID"" OR(""B"".""MetadataID"" = ""D"".""ID"" AND ""D"".""BlocksetID"" = ""C"".""ID""))
|
||||
"), 0);
|
||||
var filesetcount = cmd.ExecuteScalarInt64(FormatInvariant($@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetID"" = {ParentID}"), 0);
|
||||
if (filesetcount == RemovedFileCount)
|
||||
throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset");
|
||||
}
|
||||
AND (
|
||||
""B"".""BlocksetID"" = ""C"".""ID""
|
||||
OR (
|
||||
""B"".""MetadataID"" = ""D"".""ID""
|
||||
AND ""D"".""BlocksetID"" = ""C"".""ID""
|
||||
)
|
||||
)
|
||||
", 0, token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var filesetcount = await cmd.SetCommandAndParameters($@"
|
||||
SELECT COUNT(*)
|
||||
FROM ""FilesetEntry""
|
||||
WHERE ""FilesetID"" = @ParentID
|
||||
")
|
||||
.SetParameterValue("@ParentID", ParentID)
|
||||
.ExecuteScalarInt64Async(0, token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (filesetcount == RemovedFileCount)
|
||||
throw new Interface.UserInformationException($"Refusing to purge {RemovedFileCount} files from fileset with ID {ParentID}, as that would remove the entire fileset.\nTo delete a fileset, use the \"delete\" command.", "PurgeWouldRemoveEntireFileset");
|
||||
}
|
||||
|
||||
public Tuple<long, long> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup)
|
||||
/// <inheritdoc/>
|
||||
public async Task<Tuple<long, long>> ConvertToPermanentFileset(string name, DateTime timestamp, bool isFullBackup, CancellationToken token)
|
||||
{
|
||||
var remotevolid = m_parentdb.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_transaction);
|
||||
var filesetid = m_parentdb.CreateFileset(remotevolid, timestamp, m_transaction);
|
||||
m_parentdb.UpdateFullBackupStateInFileset(filesetid, isFullBackup);
|
||||
var remotevolid =
|
||||
await m_db
|
||||
.RegisterRemoteVolume(name, RemoteVolumeType.Files, RemoteVolumeState.Temporary, token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
using (var cmd = m_connection.CreateCommand(m_transaction))
|
||||
cmd.SetCommandAndParameters(FormatInvariant($@"INSERT INTO ""FilesetEntry"" (""FilesetID"", ""FileID"", ""Lastmodified"") SELECT @TargetFilesetId, ""FileID"", ""LastModified"" FROM ""FilesetEntry"" WHERE ""FilesetID"" = @SourceFilesetId AND ""FileID"" NOT IN ""{m_tablename}"" "))
|
||||
.SetParameterValue("@TargetFilesetId", filesetid)
|
||||
.SetParameterValue("@SourceFilesetId", ParentID)
|
||||
.ExecuteNonQuery();
|
||||
var filesetid = await m_db
|
||||
.CreateFileset(remotevolid, timestamp, token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await m_db
|
||||
.UpdateFullBackupStateInFileset(filesetid, isFullBackup, token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await using (var cmd = m_db.Connection.CreateCommand(m_db.Transaction))
|
||||
await cmd.SetCommandAndParameters($@"
|
||||
INSERT INTO ""FilesetEntry"" (
|
||||
""FilesetID"",
|
||||
""FileID"",
|
||||
""Lastmodified""
|
||||
)
|
||||
SELECT
|
||||
@TargetFilesetId,
|
||||
""FileID"",
|
||||
""LastModified""
|
||||
FROM ""FilesetEntry""
|
||||
WHERE
|
||||
""FilesetID"" = @SourceFilesetId
|
||||
AND ""FileID"" NOT IN ""{m_tablename}""
|
||||
")
|
||||
.SetParameterValue("@TargetFilesetId", filesetid)
|
||||
.SetParameterValue("@SourceFilesetId", ParentID)
|
||||
.ExecuteNonQueryAsync(token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return new Tuple<long, long>(remotevolid, filesetid);
|
||||
}
|
||||
|
||||
public IEnumerable<KeyValuePair<string, long>> ListAllDeletedFiles()
|
||||
/// <inheritdoc/>
|
||||
public async IAsyncEnumerable<KeyValuePair<string, long>> ListAllDeletedFiles([EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
using (var cmd = m_connection.CreateCommand(m_transaction))
|
||||
using (var rd = cmd.ExecuteReader(FormatInvariant($@"SELECT ""B"".""Path"", ""C"".""Length"" FROM ""{m_tablename}"" A, ""File"" B, ""Blockset"" C WHERE ""A"".""FileID"" = ""B"".""ID"" AND ""B"".""BlocksetID"" = ""C"".""ID"" ")))
|
||||
while (rd.Read())
|
||||
yield return new KeyValuePair<string, long>(rd.ConvertValueToString(0) ?? "", rd.ConvertValueToInt64(1));
|
||||
await using var cmd = m_db.Connection.CreateCommand();
|
||||
await using var rd = await cmd.ExecuteReaderAsync($@"
|
||||
SELECT
|
||||
""B"".""Path"",
|
||||
""C"".""Length""
|
||||
FROM
|
||||
""{m_tablename}"" ""A"",
|
||||
""File"" ""B"",
|
||||
""Blockset"" ""C""
|
||||
WHERE
|
||||
""A"".""FileID"" = ""B"".""ID""
|
||||
AND ""B"".""BlocksetID"" = ""C"".""ID""
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
while (await rd.ReadAsync(token).ConfigureAwait(false))
|
||||
yield return new KeyValuePair<string, long>(
|
||||
rd.ConvertValueToString(0) ?? "",
|
||||
rd.ConvertValueToInt64(1)
|
||||
);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
DisposeAsync().AsTask().Await();
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
using (var cmd = m_connection.CreateCommand(m_transaction))
|
||||
cmd.ExecuteNonQuery(FormatInvariant($@"DROP TABLE IF EXISTS ""{m_tablename}"""));
|
||||
}
|
||||
catch
|
||||
{
|
||||
await using var cmd = m_db.Connection.CreateCommand();
|
||||
await cmd
|
||||
.ExecuteNonQueryAsync($@"DROP TABLE IF EXISTS ""{m_tablename}""", default)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch { }
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,28 +1,38 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
namespace Duplicati.Library.Main.Database
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a remote volume with its name, hash, size, and an optional file entry.
|
||||
/// Implements the IRemoteVolume interface.
|
||||
/// </summary>
|
||||
public class RemoteVolume : IRemoteVolume
|
||||
{
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="RemoteVolume"/> class with a file
|
||||
/// entry and an optional hash.
|
||||
/// </summary>
|
||||
/// <param name="file">The file entry associated with the remote volume.</param>
|
||||
/// <param name="hash">The hash of the remote volume, can be null.</param>
|
||||
public RemoteVolume(Interface.IFileEntry file, string hash = null)
|
||||
{
|
||||
Name = file.Name;
|
||||
@@ -31,6 +41,13 @@ namespace Duplicati.Library.Main.Database
|
||||
File = file;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="RemoteVolume"/> class with a name,
|
||||
/// hash, and size.
|
||||
/// </summary>
|
||||
/// <param name="name">The name of the remote volume.</param>
|
||||
/// <param name="hash">The hash of the remote volume.</param>
|
||||
/// <param name="size">The size of the remote volume in bytes.</param>
|
||||
public RemoteVolume(string name, string hash, long size)
|
||||
{
|
||||
Name = name;
|
||||
|
||||
@@ -1,40 +1,75 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System;
|
||||
|
||||
namespace Duplicati.Library.Main.Database
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents a remote volume entry with its ID, name, hash, size, type, state,
|
||||
/// delete grace period, and archive time.
|
||||
/// Implements the IRemoteVolume interface.
|
||||
/// </summary>
|
||||
public struct RemoteVolumeEntry : IRemoteVolume
|
||||
{
|
||||
/// <summary>
|
||||
/// The ID of the remote volume entry.
|
||||
/// </summary>
|
||||
public long ID { get; private set; }
|
||||
public string Name { get; private set; }
|
||||
public string Hash { get; private set; }
|
||||
public long Size { get; private set; }
|
||||
/// <summary>
|
||||
/// The type of the remote volume, indicating whether it is a file or a directory.
|
||||
/// </summary>
|
||||
public RemoteVolumeType Type { get; private set; }
|
||||
/// <summary>
|
||||
/// The state of the remote volume, indicating its current status.
|
||||
/// </summary>
|
||||
public RemoteVolumeState State { get; private set; }
|
||||
/// <summary>
|
||||
/// The time period during which the remote volume can be deleted without permanent loss.
|
||||
/// </summary>
|
||||
public DateTime DeleteGracePeriod { get; private set; }
|
||||
/// <summary>
|
||||
/// The time when the remote volume was archived, indicating when it was moved to a less accessible state.
|
||||
/// </summary>
|
||||
public DateTime ArchiveTime { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Represents an empty remote volume entry with default values.
|
||||
/// This is useful for initializing or resetting remote volume entries.
|
||||
/// </summary>
|
||||
public static readonly RemoteVolumeEntry Empty = new RemoteVolumeEntry(-1, null, null, -1, (RemoteVolumeType)(-1), (RemoteVolumeState)(-1), default(DateTime), default(DateTime));
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the RemoteVolumeEntry struct with specified values.
|
||||
/// </summary>
|
||||
/// <param name="id">The ID of the remote volume entry.</param>
|
||||
/// <param name="name">The name of the remote volume entry.</param>
|
||||
/// <param name="hash">The hash of the remote volume entry.</param>
|
||||
/// <param name="size">The size of the remote volume entry.</param>
|
||||
/// <param name="type">The type of the remote volume entry (file or directory).</param>
|
||||
/// <param name="state">The state of the remote volume entry.</param>
|
||||
/// <param name="deleteGracePeriod">The delete grace period for the remote volume entry.</param>
|
||||
/// <param name="archiveTime">The archive time for the remote volume entry.</param>
|
||||
public RemoteVolumeEntry(long id, string name, string hash, long size, RemoteVolumeType type, RemoteVolumeState state, DateTime deleteGracePeriod, DateTime archiveTime)
|
||||
{
|
||||
ID = id;
|
||||
|
||||
@@ -1,90 +1,170 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using System.Data;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Utility;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
#nullable enable
|
||||
|
||||
namespace Duplicati.Library.Main.Database;
|
||||
|
||||
/// <summary>
|
||||
/// Wraps a transaction so it can be comitted and restarted
|
||||
/// Wraps a transaction so it can be comitted and restarted.
|
||||
/// </summary>
|
||||
internal class ReusableTransaction : IDisposable
|
||||
/// <remarks>
|
||||
/// Creates a new reusable transaction.
|
||||
/// </remarks>
|
||||
/// <param name="db">The database to use.</param>
|
||||
/// <param name="transaction">The transaction to use. If null, a new transaction is created.</param>
|
||||
internal class ReusableTransaction(SqliteConnection con, SqliteTransaction? transaction = null) : IDisposable, IAsyncDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// The tag used for logging
|
||||
/// The tag used for logging.
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(ReusableTransaction));
|
||||
|
||||
/// <summary>
|
||||
/// The database to use
|
||||
/// The database to use.
|
||||
/// </summary>
|
||||
private readonly LocalDatabase m_db;
|
||||
private readonly SqliteConnection m_con = con;
|
||||
/// <summary>
|
||||
/// The current transaction
|
||||
/// The current transaction.
|
||||
/// </summary>
|
||||
private IDbTransaction? m_transaction;
|
||||
private SqliteTransaction m_transaction = transaction ?? con.BeginTransaction(deferred: true);
|
||||
/// <summary>
|
||||
/// True if the transaction is disposed.
|
||||
/// </summary>
|
||||
private bool m_disposed = false;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new reusable transaction
|
||||
/// Creates a new reusable transaction.
|
||||
/// </summary>
|
||||
/// <param name="db">The database to use</param>
|
||||
public ReusableTransaction(LocalDatabase db, IDbTransaction? transaction = null)
|
||||
/// <param name="db">The database this transaction relates to.</param>
|
||||
/// <param name="transaction">An optional existing transaction to use. If null, a new transaction is created.</param>
|
||||
public ReusableTransaction(LocalDatabase db, SqliteTransaction? transaction = null) : this(db.Connection, transaction) { }
|
||||
|
||||
/// <summary>
|
||||
/// The current transaction.
|
||||
/// </summary>
|
||||
public SqliteTransaction Transaction => m_disposed ? throw new InvalidOperationException("Transaction is disposed") : m_transaction;
|
||||
|
||||
/// <summary>
|
||||
/// Commits the transaction and restarts it.
|
||||
/// </summary>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the commit is done and a new transaction has been started.</returns>
|
||||
public async Task CommitAsync(CancellationToken token)
|
||||
{
|
||||
m_db = db;
|
||||
m_transaction = transaction ?? db.BeginTransaction();
|
||||
await CommitAsync(null, true, token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The current transaction
|
||||
/// Async version of Commit: <inheritdoc cref="Commit(string?, bool)"/>
|
||||
/// </summary>
|
||||
public IDbTransaction Transaction => m_transaction ?? throw new InvalidOperationException("Transaction is disposed");
|
||||
|
||||
/// <summary>
|
||||
/// Commits the current transaction and optionally restarts it
|
||||
/// </summary>
|
||||
/// <param name="message">The log message to use</param>
|
||||
/// <param name="restart">True if the transaction should be restarted</param>
|
||||
public void Commit(string? message, bool restart = true)
|
||||
/// <param name="message">The log message to use.</param>
|
||||
/// <param name="restart">True if the transaction should be restarted.</param>
|
||||
/// <returns>A task that completes when the commit is done and (potentially) a new transaction has been started.</returns>
|
||||
/// <exception cref="InvalidOperationException">If the transaction is already Disposed.</exception>
|
||||
public async Task CommitAsync(string? message = null, bool restart = true, CancellationToken token = default)
|
||||
{
|
||||
if (m_transaction == null)
|
||||
if (m_disposed)
|
||||
throw new InvalidOperationException("Transaction is already disposed");
|
||||
|
||||
if (m_transaction != null)
|
||||
{
|
||||
using (var timer = string.IsNullOrWhiteSpace(message) ? null : new Logging.Timer(LOGTAG, message, $"CommitTransaction: {message}"))
|
||||
m_transaction.Commit();
|
||||
m_transaction.Dispose();
|
||||
m_transaction = null;
|
||||
}
|
||||
message ??= "Unnamed commit";
|
||||
using (var timer = new Logging.Timer(LOGTAG, message, $"CommitTransaction: {message}"))
|
||||
await m_transaction.CommitAsync().ConfigureAwait(false);
|
||||
await m_transaction.DisposeAsync().ConfigureAwait(false);
|
||||
|
||||
if (restart)
|
||||
m_transaction = m_db.BeginTransaction();
|
||||
m_transaction = m_con.BeginTransaction(deferred: true);
|
||||
else
|
||||
m_disposed = true;
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Dispose()
|
||||
{
|
||||
m_transaction?.Dispose();
|
||||
m_transaction = null;
|
||||
DisposeAsync().AsTask().Await();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (!m_disposed)
|
||||
{
|
||||
try
|
||||
{
|
||||
using (var timer = new Logging.Timer(LOGTAG, "Dispose", "Rollback during transaction dispose"))
|
||||
await m_transaction.RollbackAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "ReusableTransaction dispose", ex, "Transaction disposed with error: {0}", ex.Message);
|
||||
throw;
|
||||
}
|
||||
finally
|
||||
{
|
||||
m_disposed = true;
|
||||
await m_transaction.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Rolls back the transaction and restarts it.
|
||||
/// </summary>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the rollback is done and a new transaction has been started.</returns>
|
||||
public async Task RollBackAsync(CancellationToken token)
|
||||
{
|
||||
await RollBackAsync(null, true, token).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Rolls back the transaction and optionally restarts it.
|
||||
/// </summary>
|
||||
/// <param name="message">Message to log.</param>
|
||||
/// <param name="restart">Whether to restart the transaction after rolling back.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the rollback is done and (potentially) a new transaction has been started.</returns>
|
||||
/// <exception cref="InvalidOperationException">If the transaction has already been disposed.</exception>
|
||||
public async Task RollBackAsync(string? message = null, bool restart = true, CancellationToken token = default)
|
||||
{
|
||||
if (m_disposed)
|
||||
throw new InvalidOperationException("Transaction is already disposed");
|
||||
|
||||
using (var timer = new Logging.Timer(LOGTAG, message, $"RollbackTransaction: {message}"))
|
||||
await m_transaction.RollbackAsync().ConfigureAwait(false);
|
||||
await m_transaction.DisposeAsync().ConfigureAwait(false);
|
||||
|
||||
if (restart)
|
||||
{
|
||||
m_transaction = m_con.BeginTransaction();
|
||||
}
|
||||
else
|
||||
{
|
||||
m_disposed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,140 +1,200 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Utility;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
#nullable enable
|
||||
|
||||
namespace Duplicati.Library.Main.Database;
|
||||
|
||||
/// <summary>
|
||||
/// A list of values that can be used in a query
|
||||
/// A list of values that can be used in a query.
|
||||
/// </summary>
|
||||
public class TemporaryDbValueList : IDisposable
|
||||
internal class TemporaryDbValueList : IDisposable, IAsyncDisposable
|
||||
{
|
||||
/// <summary>
|
||||
/// The tag used for logging
|
||||
/// The tag used for logging.
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(TemporaryDbValueList));
|
||||
|
||||
/// <summary>
|
||||
/// The command to use
|
||||
/// The command to use.
|
||||
/// </summary>
|
||||
private IDbCommand? _cmd;
|
||||
private SqliteCommand? _cmd;
|
||||
/// <summary>
|
||||
/// The connection to use
|
||||
/// The connection to use.
|
||||
/// </summary>
|
||||
private readonly IDbConnection _con;
|
||||
private SqliteConnection _con;
|
||||
/// <summary>
|
||||
/// The transaction to use, if any
|
||||
/// The reusable transaction to use.
|
||||
/// </summary>
|
||||
private readonly IDbTransaction? _transaction;
|
||||
private ReusableTransaction _rtr;
|
||||
/// <summary>
|
||||
/// The table to use
|
||||
/// The table to use.
|
||||
/// </summary>
|
||||
private readonly string _tableName = $"TemporaryList-{Guid.NewGuid():N}";
|
||||
|
||||
/// <summary>
|
||||
/// Flag indicating if the object has been disposed
|
||||
/// Flag indicating if the object has been disposed.
|
||||
/// </summary>
|
||||
private bool _disposed = false;
|
||||
/// <summary>
|
||||
/// Flag indicating if the values are in a table
|
||||
/// Flag indicating if the values are in a table.
|
||||
/// </summary>
|
||||
private bool _isTable = false;
|
||||
/// <summary>
|
||||
/// The values to use
|
||||
/// The values to use.
|
||||
/// </summary>
|
||||
private IEnumerable<object> _values;
|
||||
private readonly IEnumerable<object> _values;
|
||||
/// <summary>
|
||||
/// The type of the values
|
||||
/// The type of the values.
|
||||
/// </summary>
|
||||
private readonly string _valuesType;
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new reusable transaction
|
||||
/// Creates a new TemporaryDbValueList with the given values.
|
||||
/// </summary>
|
||||
/// <param name="con">The connection to use</param>
|
||||
/// <param name="transaction">The transaction to use</param>
|
||||
/// <param name="values">The values to use</param>
|
||||
public TemporaryDbValueList(IDbConnection con, IDbTransaction? transaction, IEnumerable<long> values)
|
||||
/// <param name="con">The connection to use.</param>
|
||||
/// <param name="rtr">The reusable transaction to use.</param>
|
||||
/// <param name="values">The values to use.</param>
|
||||
/// <param name="valuesType">The type of the values (e.g., "INTEGER", "TEXT").</param>
|
||||
private TemporaryDbValueList(SqliteConnection con, ReusableTransaction rtr, IEnumerable<object> values, string valuesType)
|
||||
{
|
||||
_con = con;
|
||||
_transaction = transaction;
|
||||
_valuesType = "INTEGER";
|
||||
if (values == null)
|
||||
throw new ArgumentNullException(nameof(values));
|
||||
_values = values.Cast<object>();
|
||||
if (values.Count() > LocalDatabase.CHUNK_SIZE)
|
||||
ForceToTable();
|
||||
_rtr = rtr;
|
||||
_valuesType = valuesType;
|
||||
_values = values;
|
||||
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new reusable transaction
|
||||
/// Asynchronously creates a new TemporaryDbValueList with the given values.
|
||||
/// </summary>
|
||||
/// <param name="cmd">The command to use</param>
|
||||
/// <param name="con">The connection to use</param>
|
||||
/// <param name="values">The values to use</param>
|
||||
public TemporaryDbValueList(IDbConnection con, IDbTransaction? transaction, IEnumerable<string> values)
|
||||
/// <param name="db">The database to use.</param>
|
||||
/// <param name="values">The values to use.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the created TemporaryDbValueList.</returns>
|
||||
internal static async Task<TemporaryDbValueList> CreateAsync(LocalDatabase db, IEnumerable<long> values, CancellationToken token)
|
||||
{
|
||||
_con = con;
|
||||
_transaction = transaction;
|
||||
_valuesType = "TEXT";
|
||||
if (values == null)
|
||||
throw new ArgumentNullException(nameof(values));
|
||||
_values = values.Cast<object>();
|
||||
if (values.Count() > LocalDatabase.CHUNK_SIZE)
|
||||
ForceToTable();
|
||||
return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast<object>(), "INTEGER"), token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The values in the list
|
||||
/// Asynchronously creates a new TemporaryDbValueList with the given values.
|
||||
/// </summary>
|
||||
/// <param name="db">The database to use.</param>
|
||||
/// <param name="values">The values to use.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the created TemporaryDbValueList.</returns>
|
||||
internal static async Task<TemporaryDbValueList> CreateAsync(LocalDatabase db, IEnumerable<string> values, CancellationToken token)
|
||||
{
|
||||
return await DoCreateAsync(new TemporaryDbValueList(db.Connection, db.Transaction, values.Cast<object>(), "TEXT"), token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Asynchronously creates a new TemporaryDbValueList with the given values.
|
||||
/// </summary>
|
||||
/// <param name="con">The connection to use.</param>
|
||||
/// <param name="rtr">The reusable transaction to use.</param>
|
||||
/// <param name="values">The values to use.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the created TemporaryDbValueList.</returns>
|
||||
internal static async Task<TemporaryDbValueList> CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable<long> values, CancellationToken token)
|
||||
{
|
||||
return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast<object>(), "INTEGER"), token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Asynchronously creates a new TemporaryDbValueList with the given values.
|
||||
/// </summary>
|
||||
/// <param name="con">The connection to use.</param>
|
||||
/// <param name="rtr">The reusable transaction to use.</param>
|
||||
/// <param name="values">The values to use.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the created TemporaryDbValueList.</returns>
|
||||
internal static async Task<TemporaryDbValueList> CreateAsync(SqliteConnection con, ReusableTransaction rtr, IEnumerable<string> values, CancellationToken token)
|
||||
{
|
||||
return await DoCreateAsync(new TemporaryDbValueList(con, rtr, values.Cast<object>(), "TEXT"), token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Performs the actual creation of the TemporaryDbValueList, ensuring that if the number of values exceeds the chunk size,
|
||||
/// the values are forced into a table.
|
||||
/// </summary>
|
||||
/// <param name="valueList">The TemporaryDbValueList to create.</param>
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the created TemporaryDbValueList.</returns>
|
||||
private static async Task<TemporaryDbValueList> DoCreateAsync(TemporaryDbValueList valueList, CancellationToken token)
|
||||
{
|
||||
if (valueList._values.Count() > LocalDatabase.CHUNK_SIZE)
|
||||
await valueList.ForceToTable(token).ConfigureAwait(false);
|
||||
|
||||
return valueList;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The values in the list.
|
||||
/// </summary>
|
||||
public IEnumerable<object> Values => _values;
|
||||
|
||||
/// <summary>
|
||||
/// Flag indicating if the table has been created
|
||||
/// Flag indicating if the table has been created.
|
||||
/// </summary>
|
||||
public bool IsTableCreated => _isTable;
|
||||
|
||||
/// <summary>
|
||||
/// Get the in clause for the values, creating the table if needed
|
||||
/// Get the in clause for the values, creating the table if needed.
|
||||
/// </summary>
|
||||
/// <returns>The in clause for the values</returns>
|
||||
public string GetInClause()
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited contains the SQL in clause string.</returns>
|
||||
public async Task<string> GetInClause(CancellationToken token)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(TemporaryDbValueList));
|
||||
|
||||
ForceToTable();
|
||||
await ForceToTable(token).ConfigureAwait(false);
|
||||
|
||||
return $@"SELECT ""Value"" FROM ""{_tableName}""";
|
||||
return $@"
|
||||
SELECT ""Value""
|
||||
FROM ""{_tableName}""
|
||||
";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Force the values to be written to a table, if not already done
|
||||
/// Force the values to be written to a table, if not already done.
|
||||
/// </summary>
|
||||
public void ForceToTable()
|
||||
/// <param name="token">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the table has been created and the values inserted.</returns>
|
||||
public async Task ForceToTable(CancellationToken token)
|
||||
{
|
||||
if (_disposed)
|
||||
throw new ObjectDisposedException(nameof(TemporaryDbValueList));
|
||||
@@ -142,29 +202,42 @@ public class TemporaryDbValueList : IDisposable
|
||||
if (_isTable)
|
||||
return;
|
||||
|
||||
_cmd = _con.CreateCommand(_transaction);
|
||||
_cmd.ExecuteNonQuery($@"CREATE TEMPORARY TABLE ""{_tableName}"" (""Value"" {_valuesType})");
|
||||
_cmd = _con.CreateCommand(_rtr);
|
||||
await _cmd.ExecuteNonQueryAsync($@"
|
||||
CREATE TEMPORARY TABLE ""{_tableName}"" (""Value"" {_valuesType})
|
||||
", token)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
_isTable = true;
|
||||
foreach (var slice in _values.Chunk(LocalDatabase.CHUNK_SIZE))
|
||||
{
|
||||
var parameterNames = slice.Select((_, i) => $"@p{i}").ToArray();
|
||||
var sql = $@"INSERT INTO ""{_tableName}"" (""Value"") VALUES {string.Join(", ", parameterNames.Select(p => $"({p})"))}";
|
||||
var sql = $@"
|
||||
INSERT INTO ""{_tableName}"" (""Value"")
|
||||
VALUES {string.Join(", ", parameterNames.Select(p => $"({p})"))}
|
||||
";
|
||||
|
||||
_cmd.CommandText = sql;
|
||||
_cmd.SetCommandAndParameters(sql);
|
||||
for (int i = 0; i < slice.Length; i++)
|
||||
_cmd.AddNamedParameter(parameterNames[i], slice[i]);
|
||||
_cmd.SetParameterValue(parameterNames[i], slice[i]);
|
||||
|
||||
_cmd.ExecuteNonQuery();
|
||||
await _cmd.ExecuteNonQueryAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The table name to use
|
||||
/// The table name to use.
|
||||
/// </summary>
|
||||
public string TableName => _tableName;
|
||||
|
||||
/// <inheritdoc/>
|
||||
public void Dispose()
|
||||
{
|
||||
DisposeAsync().AsTask().Await();
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed)
|
||||
return;
|
||||
@@ -175,8 +248,11 @@ public class TemporaryDbValueList : IDisposable
|
||||
try
|
||||
{
|
||||
if (_cmd != null)
|
||||
_cmd.ExecuteNonQuery($@"DROP TABLE ""{_tableName}""");
|
||||
_cmd?.Dispose();
|
||||
{
|
||||
await _cmd.ExecuteNonQueryAsync($@"DROP TABLE ""{_tableName}""", default)
|
||||
.ConfigureAwait(false);
|
||||
await _cmd.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
_cmd = null;
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -28,7 +28,7 @@ internal interface IBackendManager : IDisposable
|
||||
/// <param name="onDbUpdate">The action to call when the database should be updated</param>
|
||||
/// <param name="cancelToken">The cancellation token</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
Task PutAsync(VolumeWriterBase blockVolume, IndexVolumeWriter? indexVolume, Action? indexVolumeFinished, bool waitForComplete, Func<Task>? onDbUpdate, CancellationToken cancelToken);
|
||||
Task PutAsync(VolumeWriterBase blockVolume, IndexVolumeWriter? indexVolume, Func<Task>? indexVolumeFinished, bool waitForComplete, Func<Task>? onDbUpdate, CancellationToken cancelToken);
|
||||
|
||||
/// <summary>
|
||||
/// Uploads a file to the backend without encryption
|
||||
@@ -50,10 +50,9 @@ internal interface IBackendManager : IDisposable
|
||||
/// Waits for the backend queue to be empty and flushes any pending messages to the database
|
||||
/// </summary>
|
||||
/// <param name="database">The database to write pending messages to</param>
|
||||
/// <param name="transaction">The transaction to use, if any</param>
|
||||
/// <param name="cancellationToken">The cancellation token</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
Task WaitForEmptyAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken);
|
||||
Task WaitForEmptyAsync(LocalDatabase database, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Lists the files on the backend
|
||||
@@ -130,10 +129,9 @@ internal interface IBackendManager : IDisposable
|
||||
/// Flushes the database messages to the database
|
||||
/// </summary>
|
||||
/// <param name="database">The database to write to</param>
|
||||
/// <param name="transaction">The transaction to use</param>
|
||||
/// <param name="cancellationToken">The cancellation token</param>
|
||||
/// <returns></returns>
|
||||
Task FlushPendingMessagesAsync(LocalDatabase database, IDbTransaction? transaction, CancellationToken cancellationToken);
|
||||
Task FlushPendingMessagesAsync(LocalDatabase database, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// Updates the throttle values for upload and download
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -28,7 +28,7 @@ using System.Collections.Generic;
|
||||
using Duplicati.Library.Main.Volumes;
|
||||
using System.Linq;
|
||||
using Duplicati.Library.Interface;
|
||||
|
||||
using System.Threading;
|
||||
|
||||
namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
@@ -55,255 +55,400 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
m_database = database;
|
||||
}
|
||||
|
||||
public Task<long> FindBlockIDAsync(string hash, long size)
|
||||
public Task<long> FindBlockIDAsync(string hash, long size, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.FindBlockID(hash, size, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.FindBlockID(hash, size, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<bool> AddBlockAsync(string hash, long size, long volumeid)
|
||||
public Task<bool> AddBlockAsync(string hash, long size, long volumeid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.AddBlock(hash, size, volumeid, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.AddBlock(hash, size, volumeid, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<string> GetFileHashAsync(long fileid)
|
||||
public Task<string> GetFileHashAsync(long fileid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.GetFileHash(fileid, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.GetFileHash(fileid, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<Tuple<bool, long>> AddMetadatasetAsync(string hash, long size, long blocksetid)
|
||||
public Task<(bool, long)> AddMetadatasetAsync(string hash, long size, long blocksetid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() =>
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.AddMetadataset(hash, size, blocksetid, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<(bool, long)> GetMetadataIDAsync(string hash, long size, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.GetMetadatasetID(hash, size, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task AddDirectoryEntryAsync(string filename, long metadataid, DateTime lastModified, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.AddDirectoryEntry(filename, metadataid, lastModified, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task AddSymlinkEntryAsync(string filename, long metadataid, DateTime lastModified, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.AddSymlinkEntry(filename, metadataid, lastModified, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<(string MetadataHash, long Size)?> GetMetadataHashAndSizeForFileAsync(long fileid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.GetMetadataHashAndSizeForFile(fileid, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<(long, DateTime, long)> GetFileLastModifiedAsync(long prefixid, string path, long lastfilesetid, bool includeLength, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.GetFileLastModified(prefixid, path, lastfilesetid, includeLength, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<FileEntryData> GetFileEntryAsync(long prefixid, string path, long lastfilesetid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(async () =>
|
||||
{
|
||||
long metadataid;
|
||||
var n = m_database.AddMetadataset(hash, size, blocksetid, out metadataid, GetTransaction());
|
||||
return new Tuple<bool, long>(n, metadataid);
|
||||
});
|
||||
}
|
||||
var (id, oldModified, lastFileSize, oldMetahash, oldMetasize) =
|
||||
await m_database.GetFileEntry(prefixid, path, lastfilesetid, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
public Task<Tuple<bool, long>> GetMetadataIDAsync(string hash, long size)
|
||||
{
|
||||
return RunOnMain(() =>
|
||||
{
|
||||
long metadataid;
|
||||
var r = m_database.GetMetadatasetID(hash, size, out metadataid, GetTransaction());
|
||||
return new Tuple<bool, long>(r, metadataid);
|
||||
});
|
||||
}
|
||||
|
||||
public Task AddDirectoryEntryAsync(string filename, long metadataid, DateTime lastModified)
|
||||
{
|
||||
return RunOnMain(() => m_database.AddDirectoryEntry(filename, metadataid, lastModified, GetTransaction()));
|
||||
}
|
||||
|
||||
public Task AddSymlinkEntryAsync(string filename, long metadataid, DateTime lastModified)
|
||||
{
|
||||
return RunOnMain(() => m_database.AddSymlinkEntry(filename, metadataid, lastModified, GetTransaction()));
|
||||
}
|
||||
|
||||
public Task<(long Size, string MetadataHash)?> GetMetadataHashAndSizeForFileAsync(long fileid)
|
||||
{
|
||||
return RunOnMain(() => m_database.GetMetadataHashAndSizeForFile(fileid, GetTransaction()));
|
||||
}
|
||||
|
||||
public Task<Tuple<long, DateTime, long>> GetFileLastModifiedAsync(long prefixid, string path, long lastfilesetid, bool includeLength)
|
||||
{
|
||||
return RunOnMain(() =>
|
||||
{
|
||||
var id = m_database.GetFileLastModified(prefixid, path, lastfilesetid, includeLength, out var lastModified, out var length, GetTransaction());
|
||||
return new Tuple<long, DateTime, long>(id, lastModified, length);
|
||||
});
|
||||
}
|
||||
|
||||
public Task<FileEntryData> GetFileEntryAsync(long prefixid, string path, long lastfilesetid)
|
||||
{
|
||||
return RunOnMain(() =>
|
||||
{
|
||||
DateTime oldModified;
|
||||
long lastFileSize;
|
||||
string oldMetahash;
|
||||
long oldMetasize;
|
||||
|
||||
var id = m_database.GetFileEntry(prefixid, path, lastfilesetid, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize, GetTransaction());
|
||||
return
|
||||
id < 0 ?
|
||||
null :
|
||||
new FileEntryData() { id = id, modified = oldModified, filesize = lastFileSize, metahash = oldMetahash, metasize = oldMetasize };
|
||||
null
|
||||
:
|
||||
new FileEntryData()
|
||||
{
|
||||
id = id,
|
||||
modified = oldModified,
|
||||
filesize = lastFileSize,
|
||||
metahash = oldMetahash,
|
||||
metasize = oldMetasize
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
public Task<long> AddBlocksetAsync(string filehash, long size, int blocksize, IEnumerable<string> hashlist, IEnumerable<string> blocklisthashes)
|
||||
public Task<long> AddBlocksetAsync(string filehash, long size, int blocksize, IEnumerable<string> hashlist, IEnumerable<string> blocklisthashes, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() =>
|
||||
return RunOnMain(async () =>
|
||||
{
|
||||
long blocksetid;
|
||||
var (_, blocksetid) = await m_database
|
||||
.AddBlockset(filehash, size, blocksize, hashlist, blocklisthashes, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
m_database.AddBlockset(filehash, size, blocksize, hashlist, blocklisthashes, out blocksetid, GetTransaction());
|
||||
return blocksetid;
|
||||
});
|
||||
}
|
||||
|
||||
public Task<long> GetOrCreatePathPrefix(string prefix)
|
||||
public Task<long> GetOrCreatePathPrefix(string prefix, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.GetOrCreatePathPrefix(prefix, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.GetOrCreatePathPrefix(prefix, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task AddFileAsync(long prefixid, string filename, DateTime lastmodified, long blocksetid, long metadataid)
|
||||
public Task AddFileAsync(long prefixid, string filename, DateTime lastmodified, long blocksetid, long metadataid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.AddFile(prefixid, filename, lastmodified, blocksetid, metadataid, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.AddFile(prefixid, filename, lastmodified, blocksetid, metadataid, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task AddUnmodifiedAsync(long fileid, DateTime lastModified)
|
||||
public Task AddUnmodifiedAsync(long fileid, DateTime lastModified, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.AddKnownFile(fileid, lastModified, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.AddKnownFile(fileid, lastModified, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
public Task MoveBlockToVolumeAsync(string blockkey, long size, long sourcevolumeid, long targetvolumeid)
|
||||
public Task MoveBlockToVolumeAsync(string blockkey, long size, long sourcevolumeid, long targetvolumeid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.MoveBlockToVolume(blockkey, size, sourcevolumeid, targetvolumeid, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.MoveBlockToVolume(blockkey, size, sourcevolumeid, targetvolumeid, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task SafeDeleteRemoteVolumeAsync(string remotename)
|
||||
public Task SafeDeleteRemoteVolumeAsync(string remotename, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.SafeDeleteRemoteVolume(remotename, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.SafeDeleteRemoteVolume(remotename, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<string[]> GetBlocklistHashesAsync(string remotename)
|
||||
public Task<string[]> GetBlocklistHashesAsync(string remotename, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => Task.FromResult(m_database.GetBlocklistHashes(remotename, GetTransaction())));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.GetBlocklistHashes(remotename, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task UpdateIndexVolumeAsync(IndexVolumeWriter indexvolume, BlockVolumeWriter blockvolume)
|
||||
public Task UpdateIndexVolumeAsync(IndexVolumeWriter indexvolume, BlockVolumeWriter blockvolume, CancellationToken cancellationToken)
|
||||
{
|
||||
if (indexvolume != null)
|
||||
return Task.FromResult<bool>(true);
|
||||
|
||||
return RunOnMain(() =>
|
||||
return RunOnMain(async () =>
|
||||
{
|
||||
m_database.AddIndexBlockLink(indexvolume.VolumeID, blockvolume.VolumeID, GetTransaction());
|
||||
await m_database
|
||||
.AddIndexBlockLink(indexvolume.VolumeID, blockvolume.VolumeID, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
indexvolume.StartVolume(blockvolume.RemoteFilename);
|
||||
|
||||
foreach (var b in m_database.GetBlocks(blockvolume.VolumeID, GetTransaction()))
|
||||
await foreach (var b in m_database.GetBlocks(blockvolume.VolumeID, cancellationToken).ConfigureAwait(false))
|
||||
indexvolume.AddBlock(b.Hash, b.Size);
|
||||
|
||||
m_database.UpdateRemoteVolume(indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, GetTransaction());
|
||||
await m_database
|
||||
.UpdateRemoteVolume(indexvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
});
|
||||
}
|
||||
|
||||
public Task AppendFilesFromPreviousSetAsync()
|
||||
public Task AppendFilesFromPreviousSetAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.AppendFilesFromPreviousSet(GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.AppendFilesFromPreviousSet(null, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist)
|
||||
public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.AppendFilesFromPreviousSet(GetTransaction(), deletedFilelist));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.AppendFilesFromPreviousSet(deletedFilelist, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, long filesetid, long prevId, DateTime timestamp)
|
||||
public Task AppendFilesFromPreviousSetAsync(string[] deletedFilelist, long filesetid, long prevId, DateTime timestamp, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.AppendFilesFromPreviousSet(GetTransaction(), deletedFilelist, filesetid, prevId, timestamp));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.AppendFilesFromPreviousSet(deletedFilelist, filesetid, prevId, timestamp, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Populates FilesetEntry table with files from previous fileset, which aren't
|
||||
/// yet part of the new fileset, and which aren't excluded by the (optional) exclusion
|
||||
/// Populates FilesetEntry table with files from previous fileset, which aren't
|
||||
/// yet part of the new fileset, and which aren't excluded by the (optional) exclusion
|
||||
/// predicate.
|
||||
/// </summary>
|
||||
/// <param name="exclusionPredicate">Optional exclusion predicate (true = exclude file)</param>
|
||||
public Task AppendFilesFromPreviousSetWithPredicateAsync(Func<string, long, bool> exclusionPredicate)
|
||||
public Task AppendFilesFromPreviousSetWithPredicateAsync(Func<string, long, bool> exclusionPredicate, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.AppendFilesFromPreviousSetWithPredicate(GetTransaction(), exclusionPredicate));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Populates FilesetEntry table with files from previous fileset, which aren't
|
||||
/// yet part of the new fileset, and which aren't excluded by the (optional) exclusion
|
||||
/// Populates FilesetEntry table with files from previous fileset, which aren't
|
||||
/// yet part of the new fileset, and which aren't excluded by the (optional) exclusion
|
||||
/// predicate.
|
||||
/// </summary>
|
||||
/// <param name="exclusionPredicate">Optional exclusion predicate (true = exclude file)</param>
|
||||
/// <param name="fileSetId">Current fileset ID</param>
|
||||
/// <param name="prevFileSetId">Source fileset ID</param>
|
||||
/// <param name="timestamp">If <c>prevFileSetId</c> == -1, used to locate previous fileset</param>
|
||||
public Task AppendFilesFromPreviousSetWithPredicateAsync(Func<string, long, bool> exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp)
|
||||
/// <param name="exclusionPredicate">Optional exclusion predicate (true = exclude file).</param>
|
||||
/// <param name="fileSetId">Current fileset ID.</param>
|
||||
/// <param name="prevFileSetId">Source fileset ID.</param>
|
||||
/// <param name="timestamp">If <c>prevFileSetId</c> == -1, used to locate previous fileset.</param>
|
||||
/// <param name="cancellationToken">The cancellation token to cancel the operation.</param>
|
||||
/// <returns>Task that completes when the files have been appended.</returns>
|
||||
public Task AppendFilesFromPreviousSetWithPredicateAsync(Func<string, long, bool> exclusionPredicate, long fileSetId, long prevFileSetId, DateTime timestamp, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.AppendFilesFromPreviousSetWithPredicate(GetTransaction(), exclusionPredicate, fileSetId, prevFileSetId, timestamp));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.AppendFilesFromPreviousSetWithPredicate(exclusionPredicate, fileSetId, prevFileSetId, timestamp, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<KeyValuePair<long, DateTime>[]> GetIncompleteFilesetsAsync()
|
||||
public Task<KeyValuePair<long, DateTime>[]> GetIncompleteFilesetsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.GetIncompleteFilesets(GetTransaction()).OrderBy(x => x.Value).ToArray());
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.GetIncompleteFilesets(cancellationToken)
|
||||
.OrderBy(x => x.Value)
|
||||
.ToArrayAsync(cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<KeyValuePair<long, DateTime>[]> GetFilesetTimesAsync()
|
||||
public Task<KeyValuePair<long, DateTime>[]> GetFilesetTimesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.FilesetTimes.ToArray());
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.FilesetTimes(cancellationToken)
|
||||
.ToArrayAsync(cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<long> CreateFilesetAsync(long volumeID, DateTime fileTime)
|
||||
public Task<long> CreateFilesetAsync(long volumeID, DateTime fileTime, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.CreateFileset(volumeID, fileTime, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
{
|
||||
var fs = await m_database
|
||||
.CreateFileset(volumeID, fileTime, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await m_database.Transaction
|
||||
.CommitAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return fs;
|
||||
});
|
||||
}
|
||||
|
||||
public Task LinkFilesetToVolumeAsync(long filesetid, long volumeid)
|
||||
public Task LinkFilesetToVolumeAsync(long filesetid, long volumeid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.LinkFilesetToVolume(filesetid, volumeid, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.LinkFilesetToVolume(filesetid, volumeid, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task WriteFilesetAsync(FilesetVolumeWriter fsw, long filesetid)
|
||||
public Task WriteFilesetAsync(FilesetVolumeWriter fsw, long filesetid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.WriteFileset(fsw, filesetid, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.WriteFileset(fsw, filesetid, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task PushTimestampChangesToPreviousVersionAsync(long filesetid)
|
||||
public Task PushTimestampChangesToPreviousVersionAsync(long filesetid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.PushTimestampChangesToPreviousVersion(filesetid, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.PushTimestampChangesToPreviousVersion(filesetid, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task UpdateFilesetAndMarkAsFullBackupAsync(long filesetid)
|
||||
public Task UpdateFilesetAndMarkAsFullBackupAsync(long filesetid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.UpdateFullBackupStateInFileset(filesetid, true, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.UpdateFullBackupStateInFileset(filesetid, true, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
// TODO: Make IAsyncEnumerable or get rid of the SingleRunner setup
|
||||
public Task<string[]> GetMissingIndexFilesAsync()
|
||||
public Task<IAsyncEnumerable<string>> GetMissingIndexFilesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.GetMissingIndexFiles(GetTransaction()).ToArray());
|
||||
return RunOnMain(() => m_database.GetMissingIndexFiles(cancellationToken));
|
||||
}
|
||||
|
||||
public Task UpdateChangeStatisticsAsync(BackupResults result)
|
||||
public Task UpdateChangeStatisticsAsync(BackupResults result, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.UpdateChangeStatistics(result, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.UpdateChangeStatistics(result, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task VerifyConsistencyAsync(int blocksize, int blockhashSize, bool verifyfilelists)
|
||||
public Task VerifyConsistencyAsync(int blocksize, int blockhashSize, bool verifyfilelists, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.VerifyConsistency(blocksize, blockhashSize, verifyfilelists, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.VerifyConsistency(blocksize, blockhashSize, verifyfilelists, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task RemoveRemoteVolumeAsync(string remoteFilename)
|
||||
public Task RemoveRemoteVolumeAsync(string remoteFilename, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.RemoveRemoteVolume(remoteFilename, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.RemoveRemoteVolume(remoteFilename, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<RemoteVolumeEntry> GetRemoteVolumeFromFilesetIDAsync(long filesetID)
|
||||
public Task<RemoteVolumeEntry> GetRemoteVolumeFromFilesetIDAsync(long filesetID, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.GetRemoteVolumeFromFilesetID(filesetID, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.GetRemoteVolumeFromFilesetID(filesetID, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task CreateChangeJournalDataAsync(IEnumerable<USNJournalDataEntry> journalData)
|
||||
public Task CreateChangeJournalDataAsync(IEnumerable<USNJournalDataEntry> journalData, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.CreateChangeJournalData(journalData, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.CreateChangeJournalData(journalData, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task UpdateChangeJournalDataAsync(IEnumerable<USNJournalDataEntry> journalData, long lastfilesetid)
|
||||
public Task UpdateChangeJournalDataAsync(IEnumerable<USNJournalDataEntry> journalData, long lastfilesetid, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.UpdateChangeJournalData(journalData, lastfilesetid, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.UpdateChangeJournalData(journalData, lastfilesetid, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<bool> IsBlocklistHashKnownAsync(string hash)
|
||||
public Task<bool> IsBlocklistHashKnownAsync(string hash, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_database.IsBlocklistHashKnown(hash, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_database
|
||||
.IsBlocklistHashKnown(hash, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using CoCoL;
|
||||
@@ -82,7 +82,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
// There can be a race, such that two workers determine that
|
||||
// the block is missing, but this will be solved by the AddBlock call
|
||||
// which runs atomically
|
||||
if (!isMandatoryBlocklistHash && await database.FindBlockIDAsync(b.HashKey, b.Size) >= 0)
|
||||
if (!isMandatoryBlocklistHash && await database.FindBlockIDAsync(b.HashKey, b.Size, taskreader.ProgressToken) >= 0)
|
||||
{
|
||||
b.TaskCompletion.TrySetResult(false);
|
||||
sw_workload.Stop();
|
||||
@@ -90,13 +90,13 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
}
|
||||
|
||||
blockvolume = new BlockVolumeWriter(options);
|
||||
blockvolume.VolumeID = await database.RegisterRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary);
|
||||
blockvolume.VolumeID = await database.RegisterRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, taskreader.ProgressToken);
|
||||
|
||||
indexvolume = noIndexFiles ? null : new TemporaryIndexVolume(options);
|
||||
blocklistHashesAdded = 0;
|
||||
}
|
||||
|
||||
var newBlock = await database.AddBlockAsync(b.HashKey, b.Size, blockvolume.VolumeID);
|
||||
var newBlock = await database.AddBlockAsync(b.HashKey, b.Size, blockvolume.VolumeID, taskreader.ProgressToken);
|
||||
b.TaskCompletion.TrySetResult(newBlock);
|
||||
|
||||
// If we are recording blocklist hashes, add them to the index file,
|
||||
@@ -105,7 +105,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
// This can cause a race between workers,
|
||||
// but the side-effect is that the index files are slightly larger
|
||||
if (newBlock || !await database.IsBlocklistHashKnownAsync(b.HashKey))
|
||||
if (newBlock || !await database.IsBlocklistHashKnownAsync(b.HashKey, taskreader.ProgressToken))
|
||||
{
|
||||
blocklistHashesAdded++;
|
||||
indexvolume.AddBlockListHash(b.HashKey, b.Size, b.Data);
|
||||
@@ -114,7 +114,8 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
|
||||
if (newBlock)
|
||||
{
|
||||
blockvolume.AddBlock(b.HashKey, b.Data, b.Offset, (int)b.Size, b.Hint);
|
||||
await blockvolume.AddBlock(b.HashKey, b.Data, b.Offset, (int)b.Size, b.Hint)
|
||||
.ConfigureAwait(false);
|
||||
if (indexvolume != null)
|
||||
indexvolume.AddBlock(b.HashKey, b.Size);
|
||||
|
||||
@@ -123,7 +124,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
//When uploading a new volume, we register the volumes and then flush the transaction
|
||||
// this ensures that the local database and remote storage are as closely related as possible
|
||||
await database.UpdateRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
|
||||
await database.UpdateRemoteVolumeAsync(blockvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken);
|
||||
|
||||
blockvolume.Close();
|
||||
|
||||
@@ -132,17 +133,17 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
// TODO: It is much easier to let the BackendManager deal with index files,
|
||||
// but it adds a bit of strain to the database
|
||||
indexVolumeCopy = await indexvolume.CreateVolume(blockvolume.RemoteFilename, options, database);
|
||||
indexVolumeCopy = await indexvolume.CreateVolume(blockvolume.RemoteFilename, options, database, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
// Create link before upload is started, it will be removed later if upload fails
|
||||
await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, blockvolume.VolumeID).ConfigureAwait(false);
|
||||
await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, blockvolume.VolumeID, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var blockVolumeCopy = blockvolume;
|
||||
blockvolume = null;
|
||||
indexvolume = null;
|
||||
|
||||
await database.CommitTransactionAsync("CommitAddBlockToOutputFlush");
|
||||
await backendManager.PutAsync(blockVolumeCopy, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken);
|
||||
await database.CommitTransactionAsync("CommitAddBlockToOutputFlush", true, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.PutAsync(blockVolumeCopy, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
}
|
||||
|
||||
@@ -163,11 +164,15 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
if (blockvolume.SourceSize > 0 || blocklistHashesAdded > 0)
|
||||
{
|
||||
await self.SpillPickup.WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume));
|
||||
await self.SpillPickup
|
||||
.WriteAsync(new SpillVolumeRequest(blockvolume, indexvolume))
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await database.RemoveRemoteVolumeAsync(blockvolume.RemoteFilename);
|
||||
await database
|
||||
.RemoveRemoteVolumeAsync(blockvolume.RemoteFilename, taskreader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -49,7 +49,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
var e = await self.Input.ReadAsync();
|
||||
var e = await self.Input.ReadAsync().ConfigureAwait(false);
|
||||
|
||||
// We ignore the stop signal, but not the pause and terminate
|
||||
await taskreader.ProgressRendevouz().ConfigureAwait(false);
|
||||
@@ -57,7 +57,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
try
|
||||
{
|
||||
var hint = options.GetCompressionHintFromFilename(e.Entry.Path);
|
||||
var oldHash = e.OldId < 0 ? null : await database.GetFileHashAsync(e.OldId);
|
||||
var oldHash = e.OldId < 0 ? null : await database.GetFileHashAsync(e.OldId, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
StreamProcessResult filestreamdata;
|
||||
|
||||
@@ -68,7 +68,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
// If we have determined that metadata has not changed, just grab the ID
|
||||
if (!e.MetadataChanged)
|
||||
{
|
||||
var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length);
|
||||
var res = await database.GetMetadataIDAsync(e.MetaHashAndSize.FileHash, e.MetaHashAndSize.Blob.Length, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
if (res.Item1)
|
||||
return res.Item2;
|
||||
|
||||
@@ -76,15 +76,15 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
e.MetadataChanged = true;
|
||||
}
|
||||
|
||||
return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Entry.Path, e.MetaHashAndSize, database, self.StreamBlockChannel)).Item2;
|
||||
return (await MetadataPreProcess.AddMetadataToOutputAsync(e.Entry.Path, e.MetaHashAndSize, database, self.StreamBlockChannel, taskreader.ProgressToken).ConfigureAwait(false)).Item2;
|
||||
});
|
||||
|
||||
using (var fs = await e.Entry.OpenRead(taskreader.ProgressToken))
|
||||
filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Entry.Path, fs, false, hint);
|
||||
using (var fs = await e.Entry.OpenRead(taskreader.ProgressToken).ConfigureAwait(false))
|
||||
filestreamdata = await StreamBlock.ProcessStream(self.StreamBlockChannel, e.Entry.Path, fs, false, hint).ConfigureAwait(false);
|
||||
|
||||
await stats.AddOpenedFile(filestreamdata.Streamlength);
|
||||
await stats.AddOpenedFile(filestreamdata.Streamlength).ConfigureAwait(false);
|
||||
|
||||
var metadataid = await metatask;
|
||||
var metadataid = await metatask.ConfigureAwait(false);
|
||||
var filekey = filestreamdata.Streamhash;
|
||||
var filesize = filestreamdata.Streamlength;
|
||||
|
||||
@@ -110,18 +110,18 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "WouldAddChangedFile", "Would add changed file {0}, size {1}", e.Entry.Path, Library.Utility.Utility.FormatSizeString(filesize));
|
||||
}
|
||||
|
||||
await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid);
|
||||
await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
else if (e.MetadataChanged)
|
||||
{
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileMetadataChanged", "File has only metadata changes {0}", e.Entry.Path);
|
||||
await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid);
|
||||
await database.AddFileAsync(e.PathPrefixID, e.Filename, e.LastWrite, filestreamdata.Blocksetid, metadataid, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
else if (e.TimestampChanged)
|
||||
{
|
||||
await stats.AddTimestampChangedFile();
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "FileTimestampChanged", "File has only timestamp changes {0}", e.Entry.Path);
|
||||
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
|
||||
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
else /*if (e.OldId >= 0)*/
|
||||
{
|
||||
@@ -130,7 +130,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
|
||||
try
|
||||
{
|
||||
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
|
||||
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -40,7 +40,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// </summary>
|
||||
private static readonly string FILELOGTAG = Logging.Log.LogTagFromType(typeof(FilePreFilterProcess)) + ".FileEntry";
|
||||
|
||||
public static Task Run(Channels channels, Options options, BackupStatsCollector stats, BackupDatabase database)
|
||||
public static Task Run(Channels channels, Options options, BackupStatsCollector stats, BackupDatabase database, ITaskReader taskreader)
|
||||
{
|
||||
return AutomationExtensions.RunTask(
|
||||
new
|
||||
@@ -105,7 +105,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoTimestampChange", "Skipped checking file, because timestamp was not updated {0}", e.Entry.Path);
|
||||
try
|
||||
{
|
||||
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
|
||||
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -121,7 +121,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
// but we want to know if the metadata is potentially changed
|
||||
if (!isNewFile && DISABLEFILETIMECHECK)
|
||||
{
|
||||
var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId);
|
||||
var tp = await database.GetMetadataHashAndSizeForFileAsync(e.OldId, taskreader.ProgressToken);
|
||||
if (tp != null)
|
||||
{
|
||||
e.OldMetaSize = tp.Value.Size;
|
||||
@@ -145,7 +145,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "SkipCheckNoMetadataChange", "Skipped checking file, because no metadata was updated {0}", e.Entry.Path);
|
||||
try
|
||||
{
|
||||
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite);
|
||||
await database.AddUnmodifiedAsync(e.OldId, e.LastWrite, taskreader.ProgressToken);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -26,6 +26,7 @@ using System.IO;
|
||||
using System.Collections.Generic;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Main.Operation.Common;
|
||||
using System.Threading;
|
||||
|
||||
namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
@@ -63,7 +64,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
public bool TimestampChanged;
|
||||
}
|
||||
|
||||
public static Task Run(Channels channels, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskReader)
|
||||
public static Task Run(Channels channels, Options options, BackupDatabase database, long lastfilesetid, ITaskReader taskreader)
|
||||
{
|
||||
return AutomationExtensions.RunTask(new
|
||||
{
|
||||
@@ -85,7 +86,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
var entry = await self.Input.ReadAsync();
|
||||
|
||||
// We ignore the stop signal, but not the pause and terminate
|
||||
await taskReader.ProgressRendevouz().ConfigureAwait(false);
|
||||
await taskreader.ProgressRendevouz().ConfigureAwait(false);
|
||||
|
||||
var lastwrite = new DateTime(0, DateTimeKind.Utc);
|
||||
var attributes = entry.IsFolder
|
||||
@@ -111,7 +112,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
}
|
||||
|
||||
// If we only have metadata, stop here
|
||||
if (await ProcessMetadata(entry, attributes, lastwrite, options, emptymetadata, database, self.StreamBlockChannel).ConfigureAwait(false))
|
||||
if (await ProcessMetadata(entry, attributes, lastwrite, options, emptymetadata, database, self.StreamBlockChannel, taskreader.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -122,13 +123,13 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
prefixid = prevprefix.Value;
|
||||
else
|
||||
{
|
||||
prefixid = await database.GetOrCreatePathPrefix(split.Key);
|
||||
prefixid = await database.GetOrCreatePathPrefix(split.Key, taskreader.ProgressToken);
|
||||
prevprefix = new KeyValuePair<string, long>(split.Key, prefixid);
|
||||
}
|
||||
|
||||
if (CHECKFILETIMEONLY || DISABLEFILETIMECHECK)
|
||||
{
|
||||
var tmp = await database.GetFileLastModifiedAsync(prefixid, split.Value, lastfilesetid, false);
|
||||
var tmp = await database.GetFileLastModifiedAsync(prefixid, split.Value, lastfilesetid, false, taskreader.ProgressToken);
|
||||
await self.Output.WriteAsync(new FileEntry
|
||||
{
|
||||
OldId = tmp.Item1,
|
||||
@@ -145,7 +146,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
}
|
||||
else
|
||||
{
|
||||
var res = await database.GetFileEntryAsync(prefixid, split.Value, lastfilesetid);
|
||||
var res = await database.GetFileEntryAsync(prefixid, split.Value, lastfilesetid, taskreader.ProgressToken);
|
||||
await self.Output.WriteAsync(new FileEntry
|
||||
{
|
||||
OldId = res == null ? -1 : res.id,
|
||||
@@ -178,8 +179,16 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// <summary>
|
||||
/// Processes the metadata for the given path.
|
||||
/// </summary>
|
||||
/// <param name="entry">The source provider entry to process.</param>
|
||||
/// <param name="attributes">The file attributes of the entry.</param>
|
||||
/// <param name="lastwrite">The last write timestamp of the entry.</param>
|
||||
/// <param name="options">The options for the backup operation.</param>
|
||||
/// <param name="emptymetadata">An empty metadata object to use if no metadata is generated.</param>
|
||||
/// <param name="database">The backup database to use for storing metadata.</param>
|
||||
/// <param name="streamblockchannel">The channel to write stream blocks to.</param>
|
||||
/// <param name="cancellationToken">The cancellation token to use for the operation.</param>
|
||||
/// <returns><c>True</c> if the path should be submitted to more analysis, <c>false</c> if there is nothing else to do</returns>
|
||||
private static async Task<bool> ProcessMetadata(ISourceProviderEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel)
|
||||
private static async Task<bool> ProcessMetadata(ISourceProviderEntry entry, FileAttributes attributes, DateTime lastwrite, Options options, IMetahash emptymetadata, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel, CancellationToken cancellationToken)
|
||||
{
|
||||
if (entry.IsSymlink)
|
||||
{
|
||||
@@ -213,7 +222,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
metadata["CoreSymlinkTarget"] = symlinkTarget;
|
||||
|
||||
var metahash = Utility.WrapMetadata(metadata, options);
|
||||
await AddSymlinkToOutputAsync(entry.Path, DateTime.UtcNow, metahash, database, streamblockchannel).ConfigureAwait(false);
|
||||
await AddSymlinkToOutputAsync(entry.Path, DateTime.UtcNow, metahash, database, streamblockchannel, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "StoreSymlink", "Stored symlink {0}", entry.Path);
|
||||
// Don't process further
|
||||
@@ -241,7 +250,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
}
|
||||
|
||||
Logging.Log.WriteVerboseMessage(FILELOGTAG, "AddDirectory", "Adding directory {0}", entry.Path);
|
||||
await AddFolderToOutputAsync(entry.Path, lastwrite, metahash, database, streamblockchannel).ConfigureAwait(false);
|
||||
await AddFolderToOutputAsync(entry.Path, lastwrite, metahash, database, streamblockchannel, cancellationToken).ConfigureAwait(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -252,29 +261,19 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// <summary>
|
||||
/// Adds metadata to output, and returns the metadataset ID
|
||||
/// </summary>
|
||||
/// <returns>The metadataset ID.</returns>
|
||||
/// <param name="path">The path for which metadata is processed.</param>
|
||||
/// <param name="meta">The metadata entry.</param>
|
||||
/// <param name="database">The database connection.</param>
|
||||
/// <param name="streamblockchannel">The channel to write streams to.</param>
|
||||
internal static async Task<Tuple<bool, long>> AddMetadataToOutputAsync(string path, IMetahash meta, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel)
|
||||
/// <param name="cancellationToken">The cancellation token to use.</param>
|
||||
/// <returns>A task that when awaited returns a tuple with the first value indicating if the metadata set was added, and the second value being the metadata ID.</returns>
|
||||
internal static async Task<(bool, long)> AddMetadataToOutputAsync(string path, IMetahash meta, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel, CancellationToken cancellationToken)
|
||||
{
|
||||
StreamProcessResult res;
|
||||
using (var ms = new MemoryStream(meta.Blob))
|
||||
res = await StreamBlock.ProcessStream(streamblockchannel, path, ms, true, CompressionHint.Default);
|
||||
|
||||
return await database.AddMetadatasetAsync(res.Streamhash, res.Streamlength, res.Blocksetid);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a file to the output,
|
||||
/// </summary>
|
||||
/// <param name="filename">The name of the file to record</param>
|
||||
/// <param name="lastModified">The value of the lastModified timestamp</param>
|
||||
private static async Task AddFolderToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel)
|
||||
{
|
||||
var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel).ConfigureAwait(false);
|
||||
await database.AddDirectoryEntryAsync(filename, metadataid.Item2, lastModified);
|
||||
return await database.AddMetadatasetAsync(res.Streamhash, res.Streamlength, res.Blocksetid, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -282,13 +281,31 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
/// </summary>
|
||||
/// <param name="filename">The name of the file to record</param>
|
||||
/// <param name="lastModified">The value of the lastModified timestamp</param>
|
||||
/// <param name="meta">The metadata to record</param>
|
||||
/// <param name="database">The database to use</param>
|
||||
/// <param name="streamblockchannel">The channel to write blocks to</param>
|
||||
/// <param name="meta">The metadata ti record</param>
|
||||
private static async Task AddSymlinkToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel)
|
||||
/// <param name="cancellationToken">The cancellation token to use</param>
|
||||
/// <returns>A task that completes when the file has been added to the output.</returns>
|
||||
private static async Task AddFolderToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel, CancellationToken cancellationToken)
|
||||
{
|
||||
var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel).ConfigureAwait(false);
|
||||
await database.AddSymlinkEntryAsync(filename, metadataid.Item2, lastModified);
|
||||
var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel, cancellationToken).ConfigureAwait(false);
|
||||
await database.AddDirectoryEntryAsync(filename, metadataid.Item2, lastModified, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds a file to the output,
|
||||
/// </summary>
|
||||
/// <param name="filename">The name of the file to record</param>
|
||||
/// <param name="lastModified">The value of the lastModified timestamp</param>
|
||||
/// <param name="meta">The metadata ti record</param>
|
||||
/// <param name="database">The database to use</param>
|
||||
/// <param name="streamblockchannel">The channel to write blocks to</param>
|
||||
/// <param name="cancellationToken">The cancellation token to use</param>
|
||||
/// <returns>A task that completes when the file has been added to the output.</returns>
|
||||
private static async Task AddSymlinkToOutputAsync(string filename, DateTime lastModified, IMetahash meta, BackupDatabase database, IWriteChannel<StreamBlock> streamblockchannel, CancellationToken cancellationToken)
|
||||
{
|
||||
var metadataid = await AddMetadataToOutputAsync(filename, meta, database, streamblockchannel, cancellationToken).ConfigureAwait(false);
|
||||
await database.AddSymlinkEntryAsync(filename, metadataid.Item2, lastModified, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System.Threading.Tasks;
|
||||
@@ -35,20 +35,20 @@ internal static class RecreateMissingIndexFiles
|
||||
{
|
||||
if (options.IndexfilePolicy != Options.IndexFileStrategy.None)
|
||||
{
|
||||
foreach (var blockfile in await database.GetMissingIndexFilesAsync().ConfigureAwait(false))
|
||||
await foreach (var blockfile in (await database.GetMissingIndexFilesAsync(taskreader.ProgressToken).ConfigureAwait(false)).ConfigureAwait(false))
|
||||
{
|
||||
if (!await taskreader.ProgressRendevouz().ConfigureAwait(false))
|
||||
return;
|
||||
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "RecreateMissingIndexFile", "Re-creating missing index file for {0}", blockfile);
|
||||
var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database).ConfigureAwait(false);
|
||||
var w = await Common.IndexVolumeCreator.CreateIndexVolume(blockfile, options, database, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
if (!await taskreader.ProgressRendevouz().ConfigureAwait(false))
|
||||
return;
|
||||
|
||||
await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null).ConfigureAwait(false);
|
||||
await database.CommitTransactionAsync("RecreateMissingIndexFile").ConfigureAwait(false);
|
||||
await backendManager.PutAsync(w, null, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await database.UpdateRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await database.CommitTransactionAsync("RecreateMissingIndexFile", true, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.PutAsync(w, null, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using CoCoL;
|
||||
@@ -79,7 +79,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
// No more targets, make one
|
||||
target = new SpillVolumeRequest(new BlockVolumeWriter(options), source.IndexVolume == null ? null : new TemporaryIndexVolume(options));
|
||||
target.BlockVolume.VolumeID = await database.RegisterRemoteVolumeAsync(target.BlockVolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary).ConfigureAwait(false);
|
||||
target.BlockVolume.VolumeID = await database.RegisterRemoteVolumeAsync(target.BlockVolume.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -124,8 +124,8 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
target = await GetNextTarget(source).ConfigureAwait(false);
|
||||
|
||||
var len = rd.ReadBlock(file.Key, buffer);
|
||||
target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default);
|
||||
await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID).ConfigureAwait(false);
|
||||
await target.BlockVolume.AddBlock(file.Key, buffer, 0, len, Duplicati.Library.Interface.CompressionHint.Default).ConfigureAwait(false);
|
||||
await database.MoveBlockToVolumeAsync(file.Key, len, source.BlockVolume.VolumeID, target.BlockVolume.VolumeID, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
if (target.IndexVolume != null)
|
||||
target.IndexVolume.AddBlock(file.Key, len);
|
||||
@@ -141,7 +141,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
|
||||
// Make sure they are out of the database
|
||||
System.IO.File.Delete(source.BlockVolume.LocalFilename);
|
||||
await database.SafeDeleteRemoteVolumeAsync(source.BlockVolume.RemoteFilename).ConfigureAwait(false);
|
||||
await database.SafeDeleteRemoteVolumeAsync(source.BlockVolume.RemoteFilename, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
// Re-inject the target if it has content
|
||||
if (target != null)
|
||||
@@ -167,13 +167,13 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
{
|
||||
// TODO: It is much easier to let the BackendManager deal with index files,
|
||||
// but it adds a bit of strain to the database
|
||||
indexVolumeCopy = await target.IndexVolume.CreateVolume(target.BlockVolume.RemoteFilename, options, database).ConfigureAwait(false);
|
||||
indexVolumeCopy = await target.IndexVolume.CreateVolume(target.BlockVolume.RemoteFilename, options, database, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
// Create link before upload is started, it will be removed later if upload fails
|
||||
await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, target.BlockVolume.VolumeID).ConfigureAwait(false);
|
||||
await database.AddIndexBlockLinkAsync(indexVolumeCopy.VolumeID, target.BlockVolume.VolumeID, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await database.CommitTransactionAsync("UploadSpillVolume").ConfigureAwait(false);
|
||||
await backendManager.PutAsync(target.BlockVolume, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await database.CommitTransactionAsync("UploadSpillVolume", true, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.PutAsync(target.BlockVolume, indexVolumeCopy, null, false, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -162,7 +162,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
|
||||
filehasher.TransformFinalBlock(new byte[0], 0, 0);
|
||||
var filehash = Convert.ToBase64String(filehasher.Hash);
|
||||
var blocksetid = await database.AddBlocksetAsync(filehash, filesize, blocksize, hashcollector, blocklisthashes);
|
||||
var blocksetid = await database.AddBlocksetAsync(filehash, filesize, blocksize, hashcollector, blocklisthashes, taskreader.ProgressToken);
|
||||
cur.SetResult(new StreamProcessResult() { Streamlength = filesize, Streamhash = filehash, Blocksetid = blocksetid });
|
||||
cur = null;
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -39,7 +39,7 @@ internal static class UploadRealFilelist
|
||||
|
||||
// Update the reported source and backend changes
|
||||
using (new Logging.Timer(LOGTAG, "UpdateChangeStatistics", "UpdateChangeStatistics"))
|
||||
await db.UpdateChangeStatisticsAsync(result);
|
||||
await db.UpdateChangeStatisticsAsync(result, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
var changeCount =
|
||||
result.AddedFiles + result.ModifiedFiles + result.DeletedFiles +
|
||||
@@ -58,16 +58,16 @@ internal static class UploadRealFilelist
|
||||
// We ignore the stop signal, but not the pause and terminate
|
||||
await taskreader.ProgressRendevouz().ConfigureAwait(false);
|
||||
|
||||
await db.WriteFilesetAsync(filesetvolume, filesetid).ConfigureAwait(false);
|
||||
await db.WriteFilesetAsync(filesetvolume, filesetid, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
filesetvolume.Close();
|
||||
|
||||
// We ignore the stop signal, but not the pause and terminate
|
||||
await taskreader.ProgressRendevouz().ConfigureAwait(false);
|
||||
|
||||
await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null).ConfigureAwait(false);
|
||||
await db.CommitTransactionAsync("CommitUpdateRemoteVolume").ConfigureAwait(false);
|
||||
await db.UpdateRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await db.CommitTransactionAsync("CommitUpdateRemoteVolume", true, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
await backendManager.PutAsync(filesetvolume, null, null, false, () => db.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.PutAsync(filesetvolume, null, null, false, () => db.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -75,14 +75,14 @@ internal static class UploadRealFilelist
|
||||
if (result.TimestampChangedFiles != 0)
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "DetectedTimestampChanges", "Detected timestamp changes, but no data needs to be uploaded, pushing timestamp changes to the latest fileset");
|
||||
await db.PushTimestampChangesToPreviousVersionAsync(filesetid).ConfigureAwait(false);
|
||||
await db.PushTimestampChangesToPreviousVersionAsync(filesetid, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
Logging.Log.WriteVerboseMessage(LOGTAG, "RemovingLeftoverTempFile", "removing temp files, as no data needs to be uploaded");
|
||||
await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename);
|
||||
await db.RemoveRemoteVolumeAsync(filesetvolume.RemoteFilename, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await db.CommitTransactionAsync("CommitUpdateRemoteVolume");
|
||||
await db.CommitTransactionAsync("CommitUpdateRemoteVolume", true, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using Duplicati.Library.Main.Database;
|
||||
@@ -62,8 +62,8 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
}
|
||||
|
||||
// Ready to build and upload the synthetic list
|
||||
await database.CommitTransactionAsync("PreSyntheticFilelist");
|
||||
var incompleteFilesets = (await database.GetIncompleteFilesetsAsync()).OrderBy(x => x.Value).ToList();
|
||||
await database.CommitTransactionAsync("PreSyntheticFilelist", true, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
var incompleteFilesets = (await database.GetIncompleteFilesetsAsync(taskreader.ProgressToken)).OrderBy(x => x.Value).ToList();
|
||||
|
||||
if (!incompleteFilesets.Any())
|
||||
return;
|
||||
@@ -77,7 +77,7 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
var incompleteSet = incompleteFilesets.Last();
|
||||
var badIds = incompleteFilesets.Select(n => n.Key);
|
||||
|
||||
var prevs = (await database.GetFilesetTimesAsync())
|
||||
var prevs = (await database.GetFilesetTimesAsync(taskreader.ProgressToken))
|
||||
.Where(n => n.Key < incompleteSet.Key && !badIds.Contains(n.Key))
|
||||
.OrderBy(n => n.Key)
|
||||
.Select(n => n.Key)
|
||||
@@ -88,9 +88,9 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
FilesetVolumeWriter fsw = null;
|
||||
try
|
||||
{
|
||||
var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(database, options, incompleteSet.Value).ConfigureAwait(false);
|
||||
var fileTime = await FilesetVolumeWriter.ProbeUnusedFilenameName(database, options, incompleteSet.Value, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
fsw = new FilesetVolumeWriter(options, fileTime);
|
||||
fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
|
||||
fsw.VolumeID = await database.RegisterRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
if (!string.IsNullOrEmpty(options.ControlFiles))
|
||||
foreach (var p in options.ControlFiles.Split(new char[] { System.IO.Path.PathSeparator }, StringSplitOptions.RemoveEmptyEntries))
|
||||
@@ -99,27 +99,27 @@ namespace Duplicati.Library.Main.Operation.Backup
|
||||
// We declare this to be a partial backup since the synthetic filelist is only created
|
||||
// when a backup is interrupted.
|
||||
fsw.CreateFilesetFile(false);
|
||||
var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime);
|
||||
await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID);
|
||||
await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime);
|
||||
var newFilesetID = await database.CreateFilesetAsync(fsw.VolumeID, fileTime, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await database.LinkFilesetToVolumeAsync(newFilesetID, fsw.VolumeID, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await database.AppendFilesFromPreviousSetAsync(null, newFilesetID, prevId, fileTime, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
await database.WriteFilesetAsync(fsw, newFilesetID);
|
||||
await database.WriteFilesetAsync(fsw, newFilesetID, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
fsw.Close();
|
||||
|
||||
if (!await taskreader.ProgressRendevouz().ConfigureAwait(false))
|
||||
return;
|
||||
|
||||
await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null);
|
||||
await database.UpdateRemoteVolumeAsync(fsw.RemoteFilename, RemoteVolumeState.Uploading, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
// If the previous filelist was not uploaded, we register it for deletion, as we have created a new synthetic one
|
||||
// Because it is registered as "Deleting", it will be removed from remote storage by the cleanup process if it exists
|
||||
if (!string.IsNullOrWhiteSpace(lastTempFilelist.Name) && (lastTempFilelist.State == RemoteVolumeState.Uploading || lastTempFilelist.State == RemoteVolumeState.Temporary))
|
||||
await database.UpdateRemoteVolumeAsync(lastTempFilelist.Name, RemoteVolumeState.Deleting, -1, null);
|
||||
await database.CommitTransactionAsync("CommitUpdateFilelistVolume");
|
||||
await backendManager.PutAsync(fsw, null, null, true, () => database.FlushBackendMessagesAndCommitAsync(backendManager), taskreader.ProgressToken);
|
||||
await database.UpdateRemoteVolumeAsync(lastTempFilelist.Name, RemoteVolumeState.Deleting, -1, null, false, default, null, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await database.CommitTransactionAsync("CommitUpdateFilelistVolume", true, taskreader.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.PutAsync(fsw, null, null, true, () => database.FlushBackendMessagesAndCommitAsync(backendManager, taskreader.ProgressToken), taskreader.ProgressToken);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await database.RollbackTransactionAsync();
|
||||
await database.RollbackTransactionAsync(taskreader.ProgressToken).ConfigureAwait(false);
|
||||
fsw?.Dispose();
|
||||
throw;
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -208,7 +208,13 @@ namespace Duplicati.Library.Main.Operation
|
||||
throw new UserInformationException("Multiple USN journal services are not supported", "MultipleUSNJournalServices");
|
||||
var fileProvider = providers.First().Provider as LocalFileSource;
|
||||
|
||||
var journalData = m_database.GetChangeJournalData(lastfilesetid);
|
||||
// TODO because of the new SQLite backend, this method returns an
|
||||
// async enumerable. The proper way to handle this would be to
|
||||
// change the UsnJournalService to use factory construction (see how
|
||||
// all of the local database classes handle this), as the constructor
|
||||
// is not async and shouldn't be executing "heavy" code, such as
|
||||
// database queries.
|
||||
var journalData = m_database.GetChangeJournalData(lastfilesetid, m_taskReader.ProgressToken).ToEnumerable();
|
||||
var service = new UsnJournalService(fileProvider.SnapshotService, filter, m_options.FileAttributeFilter, m_options.SkipFilesLargerThan,
|
||||
journalData, cancellationTokenSource.Token);
|
||||
|
||||
@@ -294,12 +300,14 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
try
|
||||
{
|
||||
database = new LocalBackupDatabase(options.Dbpath, options);
|
||||
var lastFilestTime = database.FilesetTimes
|
||||
database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
var lastFilestTime = await database.FilesetTimes(result.TaskControl.ProgressToken)
|
||||
.Select(x => x.Value)
|
||||
.Append(DateTime.UnixEpoch)
|
||||
.Select(Library.Utility.Utility.NormalizeDateTimeToEpochSeconds)
|
||||
.Max();
|
||||
.MaxAsync()
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var currentStamp = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(database.OperationTimestamp);
|
||||
if (lastFilestTime > currentStamp)
|
||||
@@ -311,43 +319,53 @@ namespace Duplicati.Library.Main.Operation
|
||||
Log.WriteVerboseMessage(LOGTAG, "DatabaseTimestampError", "The database has a timestamp of {0}, but the last fileset has a timestamp of {1}. Sleeping for {2} seconds to allow the clock to catch up.", database.OperationTimestamp.ToLocalTime(), Library.Utility.Utility.EPOCH.AddSeconds(lastFilestTime).ToLocalTime(), sleepSeconds);
|
||||
Thread.Sleep(TimeSpan.FromSeconds(sleepSeconds));
|
||||
database.Dispose();
|
||||
database = new LocalBackupDatabase(options.Dbpath, options);
|
||||
database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
result.Dryrun = options.Dryrun;
|
||||
|
||||
// Check the database integrity
|
||||
Utility.UpdateOptionsFromDb(database, options);
|
||||
Utility.VerifyOptionsAndUpdateDatabase(database, options);
|
||||
await Utility.UpdateOptionsFromDb(database, options, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
await Utility.VerifyOptionsAndUpdateDatabase(database, options, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var probe_path = await database
|
||||
.GetFirstPath(result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var probe_path = database.GetFirstPath();
|
||||
if (probe_path != null && Util.GuessDirSeparator(probe_path) != Util.DirectorySeparatorString)
|
||||
throw new UserInformationException(string.Format("The backup contains files that belong to another operating system. Proceeding with a backup would cause the database to contain paths from two different operation systems, which is not supported. To proceed without losing remote data, delete all filesets and make sure the --{0} option is set, then run the backup again to re-use the existing data on the remote store.", "no-auto-compact"), "CrossOsDatabaseReuseNotSupported");
|
||||
|
||||
if (database.PartiallyRecreated)
|
||||
if (await database.PartiallyRecreated(result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
throw new UserInformationException("The database was only partially recreated. This database may be incomplete and the repair process is not allowed to alter remote files as that could result in data loss.", "DatabaseIsPartiallyRecreated");
|
||||
|
||||
if (database.RepairInProgress)
|
||||
if (await database.RepairInProgress(result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
throw new UserInformationException("The database was attempted repaired, but the repair did not complete. This database may be incomplete and the backup process cannot continue. You may delete the local database and attempt to repair it again.", "DatabaseRepairInProgress");
|
||||
|
||||
// Make sure the database is sane
|
||||
database.VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks, null);
|
||||
await database
|
||||
.VerifyConsistency(options.Blocksize, options.BlockhashSize, !options.DisableFilelistConsistencyChecks, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Guard the last filelist
|
||||
if (!options.DisableSyntheticFilelist)
|
||||
lastTempFilelist = database.GetLastIncompleteFilesetVolume(null);
|
||||
lastTempFilelist = await database
|
||||
.GetLastIncompleteFilesetVolume(result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
if (options.NoBackendverification)
|
||||
{
|
||||
await FilelistProcessor.VerifyLocalList(backendManager, database, null, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await FilelistProcessor.VerifyLocalList(backendManager, database, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await UpdateStorageStatsFromDatabase(result, database, options, backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, null, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean).ConfigureAwait(false);
|
||||
var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : database.GetRemoteVolume(lastTempFilelist.Name, null);
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: false, verifyMode: FilelistProcessor.VerifyMode.VerifyAndClean, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
var updatedLastTemp = string.IsNullOrWhiteSpace(lastTempFilelist.Name) ? default : await database.GetRemoteVolume(lastTempFilelist.Name, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
if (!string.IsNullOrWhiteSpace(updatedLastTemp.Name) && updatedLastTemp.State == RemoteVolumeState.Deleting)
|
||||
{
|
||||
// The last temporary filelist was emptied, and scheduled for deletion, so we need to delete it
|
||||
@@ -360,9 +378,11 @@ namespace Duplicati.Library.Main.Operation
|
||||
Log.WriteMessage(LogMessageType.Information, LOGTAG, "BackendDeleteFailed", ex, "Backend delete failed: {0}", ex.Message);
|
||||
}
|
||||
|
||||
await backendManager.WaitForEmptyAsync(database, null, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(database, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
// In case the backend delete failed, manually set the state to deleted
|
||||
database.UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null, null);
|
||||
await database
|
||||
.UpdateRemoteVolume(updatedLastTemp.Name, RemoteVolumeState.Deleted, updatedLastTemp.Size, updatedLastTemp.Hash, false, TimeSpan.FromHours(2), null, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
// The last temporary filelist is no more
|
||||
lastTempFilelist = default;
|
||||
}
|
||||
@@ -382,17 +402,18 @@ namespace Duplicati.Library.Main.Operation
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Re-open the database and backend manager
|
||||
database = new LocalBackupDatabase(options.Dbpath, options);
|
||||
database = await LocalBackupDatabase.CreateAsync(options.Dbpath, options, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
Log.WriteInformationMessage(LOGTAG, "BackendCleanupFinished", "Backend cleanup finished, retrying verification");
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, null, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, options, database, result.BackendWriter, [lastTempFilelist.Name], [], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
throw;
|
||||
}
|
||||
|
||||
// Return the updated version of the fileset, in case it was actually complete
|
||||
return new PreBackupVerifyResult(database, database.GetLastIncompleteFilesetVolume(null));
|
||||
return new PreBackupVerifyResult(database, await database.GetLastIncompleteFilesetVolume(result.TaskControl.ProgressToken).ConfigureAwait(false));
|
||||
}
|
||||
catch
|
||||
{
|
||||
@@ -424,7 +445,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
options.HardlinkPolicy, options.ExcludeEmptyFolders, options.IgnoreFilenames,
|
||||
GetBlacklistedPaths(options), options.ChangedFilelist, taskreader,
|
||||
() => result.PartialBackup = true, CancellationToken.None),
|
||||
Backup.FilePreFilterProcess.Run(channels, options, stats, database),
|
||||
Backup.FilePreFilterProcess.Run(channels, options, stats, database, taskreader),
|
||||
Backup.MetadataPreProcess.Run(channels, options, database, lastfilesetid, taskreader),
|
||||
Backup.SpillCollectorProcess.Run(channels, options, database, backendManager, taskreader),
|
||||
Backup.ProgressHandler.Run(channels, result)
|
||||
@@ -451,7 +472,9 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
if (options.ChangedFilelist != null && options.ChangedFilelist.Length >= 1)
|
||||
{
|
||||
await database.AppendFilesFromPreviousSetAsync(options.DeletedFilelist);
|
||||
await database
|
||||
.AppendFilesFromPreviousSetAsync(options.DeletedFilelist, taskreader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else if (journalService != null)
|
||||
{
|
||||
@@ -472,7 +495,8 @@ namespace Duplicati.Library.Main.Operation
|
||||
stats.AddExaminedFile(fileSize);
|
||||
|
||||
return false;
|
||||
});
|
||||
}, taskreader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// store journal data in database, unless job is being canceled
|
||||
if (!result.PartialBackup)
|
||||
@@ -481,10 +505,14 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (data.Any())
|
||||
{
|
||||
// always record change journal data for current fileset (entry may be dropped later if nothing is uploaded)
|
||||
await database.CreateChangeJournalDataAsync(data);
|
||||
await database
|
||||
.CreateChangeJournalDataAsync(data, taskreader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// update the previous fileset's change journal entry to resume at this point in case nothing was backed up
|
||||
await database.UpdateChangeJournalDataAsync(data, lastfilesetid);
|
||||
await database
|
||||
.UpdateChangeJournalDataAsync(data, lastfilesetid, taskreader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -495,14 +523,16 @@ namespace Duplicati.Library.Main.Operation
|
||||
}
|
||||
else
|
||||
{
|
||||
await database.UpdateFilesetAndMarkAsFullBackupAsync(filesetid);
|
||||
await database
|
||||
.UpdateFilesetAndMarkAsFullBackupAsync(filesetid, taskreader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
result.OperationProgressUpdater.UpdatefileCount(result.ExaminedFiles, result.SizeOfExaminedFiles, true);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task CompactIfRequired(IBackendManager backendManager, ReusableTransaction transaction, long lastVolumeSize)
|
||||
private async Task CompactIfRequired(IBackendManager backendManager, long lastVolumeSize)
|
||||
{
|
||||
var currentIsSmall = lastVolumeSize != -1 && lastVolumeSize <= m_options.SmallFileSize;
|
||||
|
||||
@@ -510,36 +540,41 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Delete);
|
||||
m_result.DeleteResults = new DeleteResults(m_result);
|
||||
using (var db = new LocalDeleteDatabase(m_database))
|
||||
await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults)
|
||||
.DoRunAsync(db, transaction, true, currentIsSmall, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
await using var db = await LocalDeleteDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await new DeleteHandler(m_options, (DeleteResults)m_result.DeleteResults)
|
||||
.DoRunAsync(db, true, currentIsSmall, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
}
|
||||
else if (currentIsSmall && !m_options.NoAutoCompact)
|
||||
{
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Compact);
|
||||
m_result.CompactResults = new CompactResults(m_result);
|
||||
using (var db = new LocalDeleteDatabase(m_database))
|
||||
await new CompactHandler(m_options, (CompactResults)m_result.CompactResults)
|
||||
.DoCompactAsync(db, true, transaction, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
await using var db = await LocalDeleteDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await new CompactHandler(m_options, (CompactResults)m_result.CompactResults)
|
||||
.DoCompactAsync(db, true, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task PostBackupVerification(string currentFilelistVolume, string previousTemporaryFilelist, IBackendManager backendManager, ReusableTransaction rtr)
|
||||
private async Task PostBackupVerification(string currentFilelistVolume, string previousTemporaryFilelist, IBackendManager backendManager)
|
||||
{
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupVerify);
|
||||
using (new Logging.Timer(LOGTAG, "AfterBackupVerify", "AfterBackupVerify"))
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, rtr.Transaction, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(m_database, rtr.Transaction, m_taskReader.ProgressToken);
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, m_database, m_result.BackendWriter, [currentFilelistVolume], [previousTemporaryFilelist], logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
await backendManager
|
||||
.WaitForEmptyAsync(m_database, m_taskReader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Calculate the number of samples to test, using the largest number of file of a given type
|
||||
long remoteVolumeCount = m_database.GetRemoteVolumes()
|
||||
long remoteVolumeCount = await m_database
|
||||
.GetRemoteVolumes(m_result.TaskControl.ProgressToken)
|
||||
.Where(x => x.State == RemoteVolumeState.Verified)
|
||||
.GroupBy(x => x.Type)
|
||||
.Select(x => x.LongCount())
|
||||
.Max();
|
||||
.SelectAwait(async x => await x.LongCountAsync())
|
||||
.MaxAsync()
|
||||
.ConfigureAwait(false);
|
||||
|
||||
long samplesToTest = Math.Max(m_options.BackupTestSampleCount, (long)Math.Round(remoteVolumeCount * (m_options.BackupTestPercentage / 100m), MidpointRounding.AwayFromZero));
|
||||
if (samplesToTest > 0 && remoteVolumeCount > 0)
|
||||
@@ -547,10 +582,10 @@ namespace Duplicati.Library.Main.Operation
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_PostBackupTest);
|
||||
m_result.TestResults = new TestResults(m_result);
|
||||
|
||||
using (var testdb = new LocalTestDatabase(m_database))
|
||||
await new TestHandler(m_options, (TestResults)m_result.TestResults)
|
||||
.DoRunAsync(samplesToTest, testdb, rtr, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
await using var testdb = await LocalTestDatabase.CreateAsync(m_database, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await new TestHandler(m_options, (TestResults)m_result.TestResults)
|
||||
.DoRunAsync(samplesToTest, testdb, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -561,15 +596,37 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
if (result.BackendWriter != null)
|
||||
{
|
||||
result.BackendWriter.KnownFileCount = database.GetRemoteVolumes().Count();
|
||||
result.BackendWriter.KnownFilesets = database.GetRemoteVolumes().Count(x => x.Type == RemoteVolumeType.Files);
|
||||
result.BackendWriter.KnownFileSize = database.GetRemoteVolumes().Select(x => Math.Max(0, x.Size)).Sum();
|
||||
result.BackendWriter.KnownFileCount = await database
|
||||
.GetRemoteVolumes(cancelToken)
|
||||
.CountAsync(cancelToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
result.BackendWriter.KnownFilesets = await database
|
||||
.GetRemoteVolumes(cancelToken)
|
||||
.CountAsync(x => x.Type == RemoteVolumeType.Files)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
result.BackendWriter.KnownFileSize = await database
|
||||
.GetRemoteVolumes(cancelToken)
|
||||
.Select(x => Math.Max(0, x.Size))
|
||||
.SumAsync(cancelToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
result.BackendWriter.UnknownFileCount = 0;
|
||||
result.BackendWriter.UnknownFileSize = 0;
|
||||
|
||||
result.BackendWriter.BackupListCount = database.FilesetTimes.Count();
|
||||
result.BackendWriter.LastBackupDate = database.FilesetTimes.FirstOrDefault().Value.ToLocalTime();
|
||||
result.BackendWriter.BackupListCount = await database
|
||||
.FilesetTimes(cancelToken)
|
||||
.CountAsync(cancelToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
result.BackendWriter.LastBackupDate = (
|
||||
await database
|
||||
.FilesetTimes(cancelToken)
|
||||
.FirstOrDefaultAsync(cancelToken)
|
||||
.ConfigureAwait(false)
|
||||
)
|
||||
.Value.ToLocalTime();
|
||||
|
||||
if (!options.QuotaDisable)
|
||||
{
|
||||
@@ -618,9 +675,9 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
try
|
||||
{
|
||||
await database.FlushBackendMessagesAndCommitAsync(backendManager).ConfigureAwait(false);
|
||||
await database.FlushBackendMessagesAndCommitAsync(backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await database.FlushBackendMessagesAndCommitAsync(backendManager).ConfigureAwait(false);
|
||||
await database.FlushBackendMessagesAndCommitAsync(backendManager, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (RetiredException)
|
||||
{
|
||||
@@ -632,7 +689,9 @@ namespace Duplicati.Library.Main.Operation
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Begin);
|
||||
|
||||
// Do a remote verification, unless disabled
|
||||
var (database, lastTempFilelist) = await PreBackupVerify(m_options, m_result, backendManager);
|
||||
var (database, lastTempFilelist) = await PreBackupVerify(m_options, m_result, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var lastTempVolumeIncomplete = false;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(lastTempFilelist.Name))
|
||||
@@ -651,9 +710,11 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
// If we crash now, there is a chance we leave behind partial remote files
|
||||
if (!m_options.Dryrun)
|
||||
database.TerminatedWithActiveUploads = true;
|
||||
await database
|
||||
.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
using (m_database = database)
|
||||
await using (m_database = database)
|
||||
using (var db = new Backup.BackupDatabase(m_database, m_options))
|
||||
// Setup runners and instances here
|
||||
using (var filesetvolume = new FilesetVolumeWriter(m_options, m_database.OperationTimestamp))
|
||||
@@ -665,37 +726,51 @@ namespace Duplicati.Library.Main.Operation
|
||||
try
|
||||
{
|
||||
// If the previous backup was interrupted, send a synthetic list
|
||||
await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskControl, backendManager, lastTempFilelist);
|
||||
await Backup.UploadSyntheticFilelist.Run(db, m_options, m_result, m_result.TaskControl, backendManager, lastTempFilelist)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Grab the previous backup ID, if any
|
||||
var prevfileset = m_database.FilesetTimes.FirstOrDefault();
|
||||
var prevfileset = await m_database
|
||||
.FilesetTimes(m_result.TaskControl.ProgressToken)
|
||||
.FirstOrDefaultAsync()
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (prevfileset.Value.ToUniversalTime() > m_database.OperationTimestamp.ToUniversalTime())
|
||||
throw new Exception(string.Format("The previous backup has time {0}, but this backup has time {1}. Something is wrong with the clock.", prevfileset.Value.ToLocalTime(), m_database.OperationTimestamp.ToLocalTime()));
|
||||
|
||||
var lastfilesetid = prevfileset.Value.Ticks == 0 ? -1 : prevfileset.Key;
|
||||
|
||||
// Rebuild any index files that are missing
|
||||
await Backup.RecreateMissingIndexFiles.Run(db, backendManager, m_options, m_result.TaskControl);
|
||||
await Backup.RecreateMissingIndexFiles.Run(db, backendManager, m_options, m_result.TaskControl)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Prepare the operation by registering the filelist
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_ProcessingFiles);
|
||||
|
||||
var repcnt = 0;
|
||||
while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
|
||||
while (repcnt < 100 && await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename, m_taskReader.ProgressToken).ConfigureAwait(false) >= 0)
|
||||
filesetvolume.ResetRemoteFilename(m_options, m_database.OperationTimestamp.AddSeconds(repcnt++));
|
||||
|
||||
if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename) >= 0)
|
||||
if (await db.GetRemoteVolumeIDAsync(filesetvolume.RemoteFilename, m_taskReader.ProgressToken).ConfigureAwait(false) >= 0)
|
||||
throw new Exception("Unable to generate a unique fileset name");
|
||||
|
||||
var filesetvolumeid = await db.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary);
|
||||
filesetid = await db.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time);
|
||||
var filesetvolumeid = await db
|
||||
.RegisterRemoteVolumeAsync(filesetvolume.RemoteFilename, RemoteVolumeType.Files, RemoteVolumeState.Temporary, m_taskReader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
filesetid = await db
|
||||
.CreateFilesetAsync(filesetvolumeid, VolumeBase.ParseFilename(filesetvolume.RemoteFilename).Time, m_taskReader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var journalService = GetJournalService(source, filter, lastfilesetid);
|
||||
|
||||
// Start parallel scan, or use the database
|
||||
if (m_options.DisableFileScanner)
|
||||
{
|
||||
var d = m_database.GetLastBackupFileCountAndSize();
|
||||
var d = await m_database
|
||||
.GetLastBackupFileCountAndSize(m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true);
|
||||
}
|
||||
else
|
||||
@@ -722,49 +797,68 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
// Ensure the database is in a sane state after adding data
|
||||
using (new Logging.Timer(LOGTAG, "VerifyConsistency", "VerifyConsistency"))
|
||||
await db.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false);
|
||||
await db
|
||||
.VerifyConsistencyAsync(m_options.Blocksize, m_options.BlockhashSize, false, m_taskReader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await FlushBackend(db, m_result, backendManager).ConfigureAwait(false);
|
||||
await FlushBackend(db, m_result, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Send the actual filelist
|
||||
await Backup.UploadRealFilelist.Run(m_result, db, backendManager, m_options, filesetvolume, filesetid, m_result.TaskControl, lastTempVolumeIncomplete);
|
||||
await Backup.UploadRealFilelist.Run(m_result, db, backendManager, m_options, filesetvolume, filesetid, m_result.TaskControl, lastTempVolumeIncomplete)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Wait for upload completion
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_WaitForUpload);
|
||||
await FlushBackend(db, m_result, backendManager).ConfigureAwait(false);
|
||||
await FlushBackend(db, m_result, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
database.TerminatedWithActiveUploads = false;
|
||||
// Make sure we have the database up-to-date
|
||||
await db.CommitTransactionAsync("CommitAfterUpload", false);
|
||||
await database
|
||||
.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
using (var rtr = new ReusableTransaction(m_database))
|
||||
// Make sure we have the database up-to-date
|
||||
await db
|
||||
.CommitTransactionAsync("CommitAfterUpload", true, m_taskReader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// If this throws, we should roll back the transaction
|
||||
if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
// If this throws, we should roll back the transaction
|
||||
var lastVolumeSize = await m_database
|
||||
.GetLastWrittenDBlockVolumeSize(m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await CompactIfRequired(backendManager, lastVolumeSize)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
|
||||
await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
{
|
||||
await database.Transaction
|
||||
.CommitAsync("CommitFinalizingBackup")
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
var lastVolumeSize = m_database.GetLastWrittenDBlockVolumeSize(rtr.Transaction);
|
||||
await CompactIfRequired(backendManager, rtr, lastVolumeSize);
|
||||
}
|
||||
|
||||
if (m_options.UploadVerificationFile && await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_VerificationUpload);
|
||||
await FilelistProcessor.UploadVerificationFile(backendManager, m_options, m_database, rtr.Transaction);
|
||||
}
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
{
|
||||
rtr.Commit("CommitFinalizingBackup");
|
||||
if (await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
if (m_options.NoBackendverification)
|
||||
await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken).ConfigureAwait(false);
|
||||
else
|
||||
await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager, rtr).ConfigureAwait(false);
|
||||
}
|
||||
rtr.Commit("CommitAfterVerification", restart: false);
|
||||
if (m_options.NoBackendverification)
|
||||
await UpdateStorageStatsFromDatabase(m_result, m_database, m_options, backendManager, m_taskReader.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
else
|
||||
await PostBackupVerification(filesetvolume.RemoteFilename, lastTempFilelist.Name, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
await database.Transaction
|
||||
.CommitAsync("CommitAfterVerification")
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete);
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -40,7 +40,6 @@ namespace Duplicati.Library.Main.Operation.Common
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType<DatabaseCommon>();
|
||||
|
||||
protected readonly LocalDatabase m_db;
|
||||
private System.Data.IDbTransaction m_transaction;
|
||||
protected readonly Options m_options;
|
||||
|
||||
public DatabaseCommon(LocalDatabase db, Options options)
|
||||
@@ -48,112 +47,131 @@ namespace Duplicati.Library.Main.Operation.Common
|
||||
{
|
||||
m_db = db;
|
||||
m_options = options;
|
||||
m_transaction = db.BeginTransaction();
|
||||
}
|
||||
|
||||
protected System.Data.IDbTransaction GetTransaction()
|
||||
public Task<long> RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, RemoteVolumeState state, CancellationToken cancellationToken)
|
||||
{
|
||||
m_workerSource.Token.ThrowIfCancellationRequested();
|
||||
|
||||
if (m_transaction == null)
|
||||
m_transaction = m_db.BeginTransaction();
|
||||
return m_transaction;
|
||||
return RunOnMain(async () =>
|
||||
await m_db
|
||||
.RegisterRemoteVolume(name, type, state, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<long> RegisterRemoteVolumeAsync(string name, RemoteVolumeType type, RemoteVolumeState state)
|
||||
public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup, TimeSpan deleteGraceTime, bool? setArchived, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_db.RegisterRemoteVolume(name, type, state, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_db
|
||||
.UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, setArchived, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task UpdateRemoteVolumeAsync(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup = false, TimeSpan deleteGraceTime = default(TimeSpan), bool? setArchived = null)
|
||||
public async Task FlushBackendMessagesAndCommitAsync(IBackendManager backendManager, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_db.UpdateRemoteVolume(name, state, size, hash, suppressCleanup, deleteGraceTime, setArchived, GetTransaction()));
|
||||
await FlushPendingBackendMessagesAsync(backendManager, cancellationToken).ConfigureAwait(false);
|
||||
await CommitTransactionAsync("FlushBackendMessagesAndCommitAsync", true, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task FlushBackendMessagesAndCommitAsync(IBackendManager backendManager)
|
||||
{
|
||||
await FlushPendingBackendMessagesAsync(backendManager).ConfigureAwait(false);
|
||||
await CommitTransactionAsync("FlushBackendMessagesAndCommitAsync").ConfigureAwait(false);
|
||||
}
|
||||
private Task FlushPendingBackendMessagesAsync(IBackendManager backendManager, CancellationToken cancellationToken)
|
||||
=> RunOnMain(async () => await backendManager.FlushPendingMessagesAsync(m_db, cancellationToken).ConfigureAwait(false));
|
||||
|
||||
private Task FlushPendingBackendMessagesAsync(IBackendManager backendManager)
|
||||
=> RunOnMain(() => backendManager.FlushPendingMessagesAsync(m_db, GetTransaction(), CancellationToken.None).ConfigureAwait(false));
|
||||
|
||||
public Task CommitTransactionAsync(string message, bool restart = true)
|
||||
public Task CommitTransactionAsync(string message, bool restart, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() =>
|
||||
return RunOnMain(async () =>
|
||||
{
|
||||
if (m_options.Dryrun)
|
||||
{
|
||||
if (!restart)
|
||||
{
|
||||
if (m_transaction != null)
|
||||
m_transaction.Rollback();
|
||||
m_transaction = null;
|
||||
}
|
||||
return;
|
||||
await m_db.Transaction
|
||||
.RollBackAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (m_transaction != null)
|
||||
else
|
||||
{
|
||||
using (new Logging.Timer(LOGTAG, "CommitTransactionAsync", message))
|
||||
m_transaction.Commit();
|
||||
m_transaction = null;
|
||||
await m_db.Transaction
|
||||
.CommitAsync(message, restart, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public Task RollbackTransactionAsync()
|
||||
public Task RollbackTransactionAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() =>
|
||||
{
|
||||
if (m_transaction != null)
|
||||
{
|
||||
m_transaction.Rollback();
|
||||
m_transaction = null;
|
||||
}
|
||||
});
|
||||
return RunOnMain(async () =>
|
||||
await m_db.Transaction
|
||||
.RollBackAsync(cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
|
||||
public Task RenameRemoteFileAsync(string oldname, string newname)
|
||||
public Task RenameRemoteFileAsync(string oldname, string newname, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_db.RenameRemoteFile(oldname, newname, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_db
|
||||
.RenameRemoteFile(oldname, newname, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task LogRemoteOperationAsync(string operation, string path, string data)
|
||||
public Task LogRemoteOperationAsync(string operation, string path, string data, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_db.LogRemoteOperation(operation, path, data, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_db
|
||||
.LogRemoteOperation(operation, path, data, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<LocalDatabase.IBlock[]> GetBlocksAsync(long volumeid)
|
||||
public Task<LocalDatabase.IBlock[]> GetBlocksAsync(long volumeid, CancellationToken cancellationToken)
|
||||
{
|
||||
// TODO: Figure out how to return the enumerable, while keeping the lock
|
||||
// and not creating the entire result in memory
|
||||
return RunOnMain(() => m_db.GetBlocks(volumeid, GetTransaction()).ToArray());
|
||||
return RunOnMain(async () =>
|
||||
await m_db
|
||||
.GetBlocks(volumeid, cancellationToken)
|
||||
.ToArrayAsync(cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<RemoteVolumeEntry> GetVolumeInfoAsync(string remotename)
|
||||
public Task<RemoteVolumeEntry> GetVolumeInfoAsync(string remotename, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_db.GetRemoteVolume(remotename, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_db
|
||||
.GetRemoteVolume(remotename, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<(string Hash, byte[] Buffer, int Size)[]> GetBlocklistsAsync(long volumeid, int blocksize, int hashsize)
|
||||
public Task<(string Hash, byte[] Buffer, int Size)[]> GetBlocklistsAsync(long volumeid, int blocksize, int hashsize, CancellationToken cancellationToken)
|
||||
{
|
||||
// TODO: Figure out how to return the enumerable, while keeping the lock
|
||||
// and not creating the entire result in memory
|
||||
return RunOnMain(() => m_db.GetBlocklists(volumeid, blocksize, hashsize, GetTransaction()).ToArray());
|
||||
return RunOnMain(async () =>
|
||||
await m_db
|
||||
.GetBlocklists(volumeid, blocksize, hashsize, cancellationToken)
|
||||
.ToArrayAsync(cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task<long> GetRemoteVolumeIDAsync(string remotename)
|
||||
public Task<long> GetRemoteVolumeIDAsync(string remotename, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_db.GetRemoteVolumeID(remotename, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_db
|
||||
.GetRemoteVolumeID(remotename, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
public Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID)
|
||||
public Task AddIndexBlockLinkAsync(long indexVolumeID, long blockVolumeID, CancellationToken cancellationToken)
|
||||
{
|
||||
return RunOnMain(() => m_db.AddIndexBlockLink(indexVolumeID, blockVolumeID, GetTransaction()));
|
||||
return RunOnMain(async () =>
|
||||
await m_db
|
||||
.AddIndexBlockLink(indexVolumeID, blockVolumeID, cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -162,14 +180,7 @@ namespace Duplicati.Library.Main.Operation.Common
|
||||
if (m_workerSource.IsCancellationRequested)
|
||||
return;
|
||||
|
||||
RunOnMain(() =>
|
||||
{
|
||||
base.Dispose(isDisposing);
|
||||
|
||||
var tr = Interlocked.Exchange(ref m_transaction, null);
|
||||
tr?.Commit();
|
||||
tr?.Dispose();
|
||||
}).Await();
|
||||
base.Dispose(isDisposing);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,26 +1,27 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Main.Volumes;
|
||||
using Duplicati.Library.Utility;
|
||||
@@ -67,14 +68,15 @@ namespace Duplicati.Library.Main.Operation.Common
|
||||
/// <summary>
|
||||
/// Creates an index volume with the temporary contents
|
||||
/// </summary>
|
||||
/// <returns>The index volume.</returns>
|
||||
/// <param name="blockfilename">The name of the block file.</param>
|
||||
/// <param name="options">The options used in this run.</param>
|
||||
/// <param name="database">The database to use.</param>
|
||||
public async Task<IndexVolumeWriter> CreateVolume(string blockfilename, Options options, DatabaseCommon database)
|
||||
/// <param name="cancellationToken">The cancellation token to use.</param>
|
||||
/// <returns>The index volume.</returns>
|
||||
public async Task<IndexVolumeWriter> CreateVolume(string blockfilename, Options options, DatabaseCommon database, CancellationToken cancellationToken)
|
||||
{
|
||||
var w = new IndexVolumeWriter(options);
|
||||
w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary);
|
||||
w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var enumerator = blockListHashes.GetEnumerator();
|
||||
while (enumerator.MoveNext())
|
||||
@@ -158,23 +160,23 @@ namespace Duplicati.Library.Main.Operation.Common
|
||||
/// </summary>
|
||||
internal static class IndexVolumeCreator
|
||||
{
|
||||
public static async Task<IndexVolumeWriter> CreateIndexVolume(string blockname, Options options, Common.DatabaseCommon database)
|
||||
public static async Task<IndexVolumeWriter> CreateIndexVolume(string blockname, Options options, Common.DatabaseCommon database, CancellationToken cancellationToken)
|
||||
{
|
||||
using (var h = HashFactory.CreateHasher(options.BlockHashAlgorithm))
|
||||
{
|
||||
var w = new IndexVolumeWriter(options);
|
||||
w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary);
|
||||
w.VolumeID = await database.RegisterRemoteVolumeAsync(w.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var blockvolume = await database.GetVolumeInfoAsync(blockname);
|
||||
var blockvolume = await database.GetVolumeInfoAsync(blockname, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
w.StartVolume(blockname);
|
||||
foreach (var b in await database.GetBlocksAsync(blockvolume.ID))
|
||||
foreach (var b in await database.GetBlocksAsync(blockvolume.ID, cancellationToken).ConfigureAwait(false))
|
||||
w.AddBlock(b.Hash, b.Size);
|
||||
|
||||
w.FinishVolume(blockvolume.Hash, blockvolume.Size);
|
||||
|
||||
if (options.IndexfilePolicy == Options.IndexFileStrategy.Full)
|
||||
foreach (var b in await database.GetBlocklistsAsync(blockvolume.ID, options.Blocksize, options.BlockhashSize))
|
||||
foreach (var b in await database.GetBlocklistsAsync(blockvolume.ID, options.Blocksize, options.BlockhashSize, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var bh = Convert.ToBase64String(h.ComputeHash(b.Item2, 0, b.Item3));
|
||||
if (bh != b.Item1)
|
||||
@@ -185,7 +187,7 @@ namespace Duplicati.Library.Main.Operation.Common
|
||||
w.Close();
|
||||
|
||||
// Register that the index file is tracking the block file
|
||||
await database.AddIndexBlockLinkAsync(w.VolumeID, blockvolume.ID);
|
||||
await database.AddIndexBlockLinkAsync(w.VolumeID, blockvolume.ID, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return w;
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using Duplicati.Library.Main.Database;
|
||||
@@ -52,23 +52,28 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (!System.IO.File.Exists(m_options.Dbpath))
|
||||
throw new Exception(string.Format("Database file does not exist: {0}", m_options.Dbpath));
|
||||
|
||||
using (var db = new LocalDeleteDatabase(m_options.Dbpath, "Compact", m_options.SqlitePageCache))
|
||||
using (var tr = new ReusableTransaction(db))
|
||||
await using (var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Compact", null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
Utility.UpdateOptionsFromDb(db, m_options);
|
||||
Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
|
||||
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var changed = await DoCompactAsync(db, false, tr, backendManager).ConfigureAwait(false);
|
||||
var changed = await DoCompactAsync(db, false, backendManager).ConfigureAwait(false);
|
||||
|
||||
if (changed && m_options.UploadVerificationFile)
|
||||
await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, null);
|
||||
await FilelistProcessor.UploadVerificationFile(backendManager, m_options, db, m_result.TaskControl.ProgressToken);
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
{
|
||||
tr.Commit("CommitCompact", restart: false);
|
||||
await db.Transaction
|
||||
.CommitAsync("CommitCompact")
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (changed)
|
||||
{
|
||||
db.WriteResults(m_result);
|
||||
await db.WriteResults(m_result, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
if (m_options.AutoVacuum)
|
||||
{
|
||||
m_result.VacuumResults = new VacuumResults(m_result);
|
||||
@@ -79,31 +84,45 @@ namespace Duplicati.Library.Main.Operation
|
||||
}
|
||||
}
|
||||
|
||||
internal async Task<bool> DoCompactAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, ReusableTransaction rtr, IBackendManager backendManager)
|
||||
internal async Task<bool> DoCompactAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, IBackendManager backendManager)
|
||||
{
|
||||
var report = db.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, rtr.Transaction);
|
||||
var report = await db
|
||||
.GetCompactReport(m_options.VolumeSize, m_options.Threshold, m_options.SmallFileSize, m_options.SmallFileMaxCount, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
report.ReportCompactData();
|
||||
|
||||
if (report.ShouldReclaim || report.ShouldCompact)
|
||||
{
|
||||
// Workaround where we allow a running backendmanager to be used
|
||||
if (!hasVerifiedBackend)
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict, rtr.Transaction).ConfigureAwait(false);
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, true, FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
var newvol = new BlockVolumeWriter(m_options);
|
||||
newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, rtr.Transaction);
|
||||
newvol.VolumeID = await db
|
||||
.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
IndexVolumeWriter newvolindex = null;
|
||||
if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
|
||||
{
|
||||
newvolindex = new IndexVolumeWriter(m_options);
|
||||
newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction);
|
||||
db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, rtr.Transaction);
|
||||
newvolindex.VolumeID = await db
|
||||
.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await db
|
||||
.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var blocksInVolume = 0L;
|
||||
var buffer = new byte[m_options.Blocksize];
|
||||
var remoteList = db.GetRemoteVolumes().Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified).ToArray();
|
||||
var remoteList = await db
|
||||
.GetRemoteVolumes(m_result.TaskControl.ProgressToken)
|
||||
.Where(n => n.State == RemoteVolumeState.Uploaded || n.State == RemoteVolumeState.Verified)
|
||||
.ToArrayAsync(m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
//These are for bookkeeping
|
||||
var uploadedVolumes = new List<KeyValuePair<string, long>>();
|
||||
@@ -121,7 +140,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
.Cast<IRemoteVolume>()
|
||||
.ToList();
|
||||
}
|
||||
await foreach (var d in DoDelete(db, backendManager, fullyDeleteable, rtr, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
await foreach (var d in DoDelete(db, backendManager, fullyDeleteable, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
deletedVolumes.Add(d);
|
||||
|
||||
// This list is used to pick up unused volumes,
|
||||
@@ -133,7 +152,9 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
// If we crash now, we may leave partial files
|
||||
if (!m_options.Dryrun)
|
||||
db.TerminatedWithActiveUploads = true;
|
||||
await db
|
||||
.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
newvolindex?.StartVolume(newvol.RemoteFilename);
|
||||
List<IRemoteVolume> volumesToDownload = [];
|
||||
@@ -147,7 +168,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
.ToList();
|
||||
}
|
||||
|
||||
using (var q = db.CreateBlockQueryHelper(rtr.Transaction))
|
||||
using (var q = await db.CreateBlockQueryHelper(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
await foreach (var (tmpfile, hash, size, name) in backendManager.GetFilesOverlappedAsync(volumesToDownload, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
@@ -156,59 +177,79 @@ namespace Duplicati.Library.Main.Operation
|
||||
var entry = new RemoteVolume(name, hash, size);
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
return false;
|
||||
}
|
||||
|
||||
downloadedVolumes.Add(new KeyValuePair<string, long>(entry.Name, entry.Size));
|
||||
var volumeid = db.GetRemoteVolumeID(entry.Name, rtr.Transaction);
|
||||
var volumeid = await db
|
||||
.GetRemoteVolumeID(entry.Name, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var inst = VolumeBase.ParseFilename(entry.Name);
|
||||
using (var f = new BlockVolumeReader(inst.CompressionModule, tmpfile, m_options))
|
||||
{
|
||||
foreach (var e in f.Blocks)
|
||||
{
|
||||
if (q.UseBlock(e.Key, e.Value, volumeid, rtr.Transaction))
|
||||
if (await q.UseBlock(e.Key, e.Value, volumeid, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
//TODO: How do we get the compression hint? Reverse query for filename in db?
|
||||
var s = f.ReadBlock(e.Key, buffer);
|
||||
if (s != e.Value)
|
||||
throw new Exception(string.Format("Size mismatch problem for block {0}, {1} vs {2}", e.Key, s, e.Value));
|
||||
|
||||
newvol.AddBlock(e.Key, buffer, 0, s, Interface.CompressionHint.Compressible);
|
||||
await newvol
|
||||
.AddBlock(e.Key, buffer, 0, s, Interface.CompressionHint.Compressible)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (newvolindex != null)
|
||||
newvolindex.AddBlock(e.Key, e.Value);
|
||||
|
||||
db.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, rtr.Transaction);
|
||||
await db
|
||||
.RegisterDuplicatedBlock(e.Key, e.Value, newvol.VolumeID, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
blocksInVolume++;
|
||||
|
||||
if (newvol.Filesize > (m_options.VolumeSize - m_options.Blocksize))
|
||||
{
|
||||
await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes, rtr);
|
||||
await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
newvol = new BlockVolumeWriter(m_options);
|
||||
newvol.VolumeID = db.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, rtr.Transaction);
|
||||
newvol.VolumeID = await db
|
||||
.RegisterRemoteVolume(newvol.RemoteFilename, RemoteVolumeType.Blocks, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (m_options.IndexfilePolicy != Options.IndexFileStrategy.None)
|
||||
{
|
||||
newvolindex = new IndexVolumeWriter(m_options);
|
||||
newvolindex.VolumeID = db.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, rtr.Transaction);
|
||||
db.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, rtr.Transaction);
|
||||
newvolindex.VolumeID = await db
|
||||
.RegisterRemoteVolume(newvolindex.RemoteFilename, RemoteVolumeType.Index, RemoteVolumeState.Temporary, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await db
|
||||
.AddIndexBlockLink(newvolindex.VolumeID, newvol.VolumeID, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
newvolindex.StartVolume(newvol.RemoteFilename);
|
||||
}
|
||||
|
||||
blocksInVolume = 0;
|
||||
|
||||
// Wait for the backend to catch up
|
||||
await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
// Commit as we have uploaded a volume
|
||||
if (!m_options.Dryrun)
|
||||
rtr.Commit("CommitCompact");
|
||||
await db.Transaction
|
||||
.CommitAsync("CommitCompact", true, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (deleteableVolumes.Any())
|
||||
{
|
||||
// Preserve space by deleting the old volume
|
||||
await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, rtr, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
deletedVolumes.Add(d);
|
||||
deleteableVolumes.Clear();
|
||||
}
|
||||
@@ -223,14 +264,20 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
if (blocksInVolume > 0)
|
||||
{
|
||||
await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes, rtr).ConfigureAwait(false);
|
||||
await FinishVolumeAndUpload(db, backendManager, newvol, newvolindex, uploadedVolumes).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
db.RemoveRemoteVolume(newvol.RemoteFilename, rtr.Transaction);
|
||||
await db
|
||||
.RemoveRemoteVolume(newvol.RemoteFilename, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (newvolindex != null)
|
||||
{
|
||||
db.RemoveRemoteVolume(newvolindex.RemoteFilename, rtr.Transaction);
|
||||
await db
|
||||
.RemoveRemoteVolume(newvolindex.RemoteFilename, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
newvolindex.FinishVolume(null, 0);
|
||||
}
|
||||
}
|
||||
@@ -238,7 +285,9 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
// The remainder of the operation cannot leave partial files
|
||||
if (!m_options.Dryrun)
|
||||
db.TerminatedWithActiveUploads = false;
|
||||
await db
|
||||
.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -246,7 +295,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
newvol.Dispose();
|
||||
}
|
||||
|
||||
await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, rtr, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
await foreach (var d in DoDelete(db, backendManager, deleteableVolumes, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
deletedVolumes.Add(d);
|
||||
|
||||
var downloadSize = downloadedVolumes.Where(x => x.Value >= 0).Aggregate(0L, (a, x) => a + x.Value);
|
||||
@@ -291,7 +340,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
Library.Utility.Utility.FormatSizeString(m_result.DeletedFileSize - m_result.UploadedFileSize));
|
||||
}
|
||||
|
||||
await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
m_result.EndTime = DateTime.UtcNow;
|
||||
return (m_result.DeletedFileCount + m_result.UploadedFileCount) > 0;
|
||||
@@ -303,41 +352,53 @@ namespace Duplicati.Library.Main.Operation
|
||||
}
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<KeyValuePair<string, long>> DoDelete(LocalDeleteDatabase db, IBackendManager backend, IEnumerable<IRemoteVolume> deleteableVolumes, ReusableTransaction rtr, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
private async IAsyncEnumerable<KeyValuePair<string, long>> DoDelete(LocalDeleteDatabase db, IBackendManager backend, IEnumerable<IRemoteVolume> deleteableVolumes, [EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
// Find volumes that can be deleted
|
||||
var remoteFilesToRemove = db.ReOrderDeleteableVolumes(deleteableVolumes, rtr.Transaction).ToList();
|
||||
var remoteFilesToRemove = await db
|
||||
.ReOrderDeleteableVolumes(deleteableVolumes, cancellationToken)
|
||||
.ToListAsync(cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Make sure we do not re-assign blocks to any of the volumes we are about to delete
|
||||
var toRemoveVolumeIds = db.GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name), rtr.Transaction)
|
||||
var toRemoveVolumeIds = await db
|
||||
.GetRemoteVolumeIDs(remoteFilesToRemove.Select(x => x.Name), cancellationToken)
|
||||
.Select(x => x.Value)
|
||||
.Distinct()
|
||||
.ToList();
|
||||
.ToListAsync(cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Mark all volumes and relevant index files as disposable
|
||||
foreach (var f in remoteFilesToRemove)
|
||||
{
|
||||
db.PrepareForDelete(f.Name, toRemoveVolumeIds, rtr.Transaction);
|
||||
db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, rtr.Transaction);
|
||||
await db
|
||||
.PrepareForDelete(f.Name, toRemoveVolumeIds, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await db
|
||||
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Before we commit the current state, make sure the backend has caught up
|
||||
await backend.WaitForEmptyAsync(db, rtr.Transaction, cancellationToken).ConfigureAwait(false);
|
||||
await backend.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
rtr.Commit("CommitDelete");
|
||||
await db.Transaction
|
||||
.CommitAsync("CommitDelete", token: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await foreach (var d in PerformDelete(backend, remoteFilesToRemove, cancellationToken).ConfigureAwait(false))
|
||||
yield return d;
|
||||
}
|
||||
|
||||
private async Task FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List<KeyValuePair<string, long>> uploadedVolumes, ReusableTransaction rtr)
|
||||
private async Task FinishVolumeAndUpload(LocalDeleteDatabase db, IBackendManager backendManager, BlockVolumeWriter newvol, IndexVolumeWriter newvolindex, List<KeyValuePair<string, long>> uploadedVolumes)
|
||||
{
|
||||
Action indexVolumeFinished = null;
|
||||
Func<Task> indexVolumeFinished = null;
|
||||
if (newvolindex != null && m_options.IndexfilePolicy == Options.IndexFileStrategy.Full)
|
||||
indexVolumeFinished = () =>
|
||||
indexVolumeFinished = async () =>
|
||||
{
|
||||
foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize, rtr.Transaction))
|
||||
await foreach (var blocklist in db.GetBlocklists(newvol.VolumeID, m_options.Blocksize, m_options.BlockhashSize, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
newvolindex.WriteBlocklist(blocklist.Hash, blocklist.Buffer, 0, blocklist.Size);
|
||||
};
|
||||
|
||||
@@ -347,8 +408,10 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
// We can handle at most one in-flight upload at a time,
|
||||
// because the transaction is not thread-safe, and shared with the upload
|
||||
await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
db.UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, rtr.Transaction);
|
||||
await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await db
|
||||
.UpdateRemoteVolume(newvol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// TODO: The upload here does not flush the database messages,
|
||||
// and this can leave the database in a state where it does not know of the remote file
|
||||
@@ -356,7 +419,10 @@ namespace Duplicati.Library.Main.Operation
|
||||
// Once fixed, we can perhaps let he backend manager simply call the database directly
|
||||
if (!m_options.Dryrun)
|
||||
{
|
||||
rtr.Commit("CommitUpload");
|
||||
await db.Transaction
|
||||
.CommitAsync("CommitUpload")
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await backendManager.PutAsync(newvol, newvolindex, indexVolumeFinished, false, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -68,11 +68,11 @@ namespace Duplicati.Library.Main.Operation
|
||||
using (var tmp = new TempFile())
|
||||
{
|
||||
File.Copy(m_options.Dbpath, tmp, true);
|
||||
using (var db = new LocalBugReportDatabase(tmp, m_options.SqlitePageCache))
|
||||
await using (var db = await LocalBugReportDatabase.CreateAsync(tmp, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
db.Fix();
|
||||
await db.Fix(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
if (m_options.AutoVacuum)
|
||||
db.Vacuum();
|
||||
await db.Vacuum(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Apply zip64 option if not already set
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -51,25 +51,33 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (!System.IO.File.Exists(m_options.Dbpath))
|
||||
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseFileMissing");
|
||||
|
||||
using (var db = new LocalDeleteDatabase(m_options.Dbpath, "Delete", m_options.SqlitePageCache))
|
||||
using (var tr = new ReusableTransaction(db))
|
||||
{
|
||||
Utility.UpdateOptionsFromDb(db, m_options);
|
||||
Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
|
||||
await using var db = await LocalDeleteDatabase.CreateAsync(m_options.Dbpath, "Delete", null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await DoRunAsync(db, tr, false, false, backendManager).ConfigureAwait(false);
|
||||
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
tr.Commit("ComitDelete", restart: false);
|
||||
}
|
||||
await DoRunAsync(db, false, false, backendManager).ConfigureAwait(false);
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
await db.Transaction
|
||||
.CommitAsync("ComitDelete")
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task DoRunAsync(LocalDeleteDatabase db, ReusableTransaction rtr, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager)
|
||||
public async Task DoRunAsync(LocalDeleteDatabase db, bool hasVerifiedBackend, bool forceCompact, IBackendManager backendManager)
|
||||
{
|
||||
if (!hasVerifiedBackend)
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, rtr.Transaction).ConfigureAwait(false);
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
// We collapse the async enumerablo into a array to avoid multiple
|
||||
// enumerations (and thus multiple database queries)
|
||||
var filesets = await db
|
||||
.FilesetsWithBackupVersion(m_result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var filesets = db.FilesetsWithBackupVersion.ToArray();
|
||||
List<IListResultFileset> versionsToDelete =
|
||||
[
|
||||
.. new SpecificVersionsRemover(m_options).GetFilesetsToDelete(filesets),
|
||||
@@ -97,18 +105,31 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "DeleteRemoteFileset", "Deleting {0} remote fileset(s) ...", versionsToDelete.Count);
|
||||
|
||||
var lst = db.DropFilesetsFromTable(versionsToDelete.Select(x => x.Time).ToArray(), rtr.Transaction).ToArray();
|
||||
var lst = await db
|
||||
.DropFilesetsFromTable(
|
||||
versionsToDelete
|
||||
.Select(x => x.Time)
|
||||
.ToArray(),
|
||||
m_result.TaskControl.ProgressToken
|
||||
)
|
||||
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
foreach (var f in lst)
|
||||
db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, rtr.Transaction);
|
||||
await db
|
||||
.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
rtr.Commit("CommitBeforeDelete");
|
||||
await db.Transaction
|
||||
.CommitAsync("CommitBeforeDelete", true, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
foreach (var f in lst)
|
||||
{
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -118,7 +139,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldDeleteRemoteFileset", "Would delete remote fileset: {0}", f.Key);
|
||||
}
|
||||
|
||||
await backendManager.WaitForEmptyAsync(db, rtr.Transaction, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
var count = lst.Length;
|
||||
if (!m_options.Dryrun)
|
||||
@@ -145,7 +166,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
m_result.CompactResults = new CompactResults(m_result);
|
||||
await new CompactHandler(m_options, (CompactResults)m_result.CompactResults)
|
||||
.DoCompactAsync(db, true, rtr, backendManager).ConfigureAwait(false);
|
||||
.DoCompactAsync(db, true, backendManager).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
m_result.SetResults(versionsToDelete.Select(v => new Tuple<long, DateTime>(v.Version, v.Time)), m_options.Dryrun);
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -63,13 +63,16 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
/// <summary>
|
||||
/// Helper method that verifies uploaded volumes and updates their state in the database.
|
||||
/// Throws an error if there are issues with the remote storage
|
||||
/// Throws an error if there are issues with the remote storage.
|
||||
/// </summary>
|
||||
/// <param name="database">The database to compare with</param>
|
||||
public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, IDbTransaction transaction, CancellationToken cancellationToken)
|
||||
/// <param name="backendManager"> The backend manager instance to use.</param>
|
||||
/// <param name="database">The database to compare with.</param>
|
||||
/// <param name="cancellationToken">The cancellation token to use.</param>
|
||||
/// <returns>A task that completes when the verification is done.</returns>
|
||||
public static async Task VerifyLocalList(IBackendManager backendManager, LocalDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
var locallist = database.GetRemoteVolumes(transaction);
|
||||
foreach (var i in locallist)
|
||||
var locallist = database.GetRemoteVolumes(cancellationToken);
|
||||
await foreach (var i in locallist.ConfigureAwait(false))
|
||||
{
|
||||
switch (i.State)
|
||||
{
|
||||
@@ -98,48 +101,55 @@ namespace Duplicati.Library.Main.Operation
|
||||
break;
|
||||
}
|
||||
|
||||
await backendManager.WaitForEmptyAsync(database, null, cancellationToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(database, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method that verifies uploaded volumes and updates their state in the database.
|
||||
/// Throws an error if there are issues with the remote storage
|
||||
/// Throws an error if there are issues with the remote storage.
|
||||
/// </summary>
|
||||
/// <param name="backend">The backend instance to use</param>
|
||||
/// <param name="options">The options used</param>
|
||||
/// <param name="database">The database to compare with</param>
|
||||
/// <param name="backendWriter">The backend writer to use</param>
|
||||
/// <param name="latestVolumesOnly">True if only the latest volumes should be verified</param>
|
||||
/// <param name="verifyMode">The mode to use for verification</param>
|
||||
/// <param name="transaction">An optional transaction object</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode, IDbTransaction transaction)
|
||||
/// <param name="backend">The backend instance to use.</param>
|
||||
/// <param name="options">The options used.</param>
|
||||
/// <param name="database">The database to compare with.</param>
|
||||
/// <param name="backendWriter">The backend writer to use.</param>
|
||||
/// <param name="latestVolumesOnly">True if only the latest volumes should be verified.</param>
|
||||
/// <param name="verifyMode">The mode to use for verification.</param>
|
||||
/// <param name="cancellationToken">The cancellation token to use.</param>
|
||||
/// <returns>An awaitable task.</returns>
|
||||
public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter backendWriter, bool latestVolumesOnly, VerifyMode verifyMode, CancellationToken cancellationToken)
|
||||
{
|
||||
if (!options.NoBackendverification)
|
||||
{
|
||||
LocalBackupDatabase backupDatabase = new LocalBackupDatabase(database, options);
|
||||
IEnumerable<string> protectedFiles = backupDatabase.GetTemporaryFilelistVolumeNames(latestVolumesOnly, transaction);
|
||||
await VerifyRemoteList(backend, options, database, transaction, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode).ConfigureAwait(false);
|
||||
LocalBackupDatabase backupDatabase =
|
||||
await LocalBackupDatabase.CreateAsync(database, options, null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
IEnumerable<string> protectedFiles = await backupDatabase
|
||||
.GetTemporaryFilelistVolumeNames(latestVolumesOnly, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await VerifyRemoteList(backend, options, database, backendWriter, protectedFiles, [], logErrors: true, verifyMode: verifyMode, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method that verifies uploaded volumes and updates their state in the database.
|
||||
/// Throws an error if there are issues with the remote storage
|
||||
/// Throws an error if there are issues with the remote storage.
|
||||
/// </summary>
|
||||
/// <param name="backend">The backend instance to use</param>
|
||||
/// <param name="options">The options used</param>
|
||||
/// <param name="database">The database to compare with</param>
|
||||
/// <param name="log">The log instance to use</param>
|
||||
/// <param name="protectedFiles">Filenames that should be exempted from deletion</param>
|
||||
/// <param name="strictExcemptFiles">Filenames that should be exempted from strict verification</param>
|
||||
/// <param name="verifyMode">The mode to use for verification</param>
|
||||
/// <param name="logErrors">Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IDbTransaction transaction, IBackendWriter log, IEnumerable<string> protectedFiles, IEnumerable<string> strictExcemptFiles, bool logErrors, VerifyMode verifyMode)
|
||||
/// <param name="backend">The backend instance to use.</param>
|
||||
/// <param name="options">The options used.</param>
|
||||
/// <param name="database">The database to compare with.</param>
|
||||
/// <param name="log">The log instance to use.</param>
|
||||
/// <param name="protectedFiles">Filenames that should be exempted from deletion.</param>
|
||||
/// <param name="strictExcemptFiles">Filenames that should be exempted from strict verification.</param>
|
||||
/// <param name="logErrors">Disable the logging of errors to prevent spamming the log; exceptions will be thrown regardless.</param>
|
||||
/// <param name="verifyMode">The mode to use for verification.</param>
|
||||
/// <param name="cancellationToken">The cancellation token to use.</param>
|
||||
/// <returns>An awaitable task.</returns>
|
||||
public static async Task VerifyRemoteList(IBackendManager backend, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles, IEnumerable<string> strictExcemptFiles, bool logErrors, VerifyMode verifyMode, CancellationToken cancellationToken)
|
||||
{
|
||||
var tp = await RemoteListAnalysis(backend, options, database, transaction, log, protectedFiles, strictExcemptFiles, verifyMode).ConfigureAwait(false);
|
||||
var tp = await RemoteListAnalysis(backend, options, database, log, protectedFiles, strictExcemptFiles, verifyMode, cancellationToken).ConfigureAwait(false);
|
||||
long extraCount = 0;
|
||||
long missingCount = 0;
|
||||
|
||||
@@ -205,26 +215,34 @@ namespace Duplicati.Library.Main.Operation
|
||||
/// <returns>The verification file.</returns>
|
||||
/// <param name="db">The database instance</param>
|
||||
/// <param name="stream">The stream to write to</param>
|
||||
public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream)
|
||||
/// <param name="cancellationToken">The cancellation token to use.</param>
|
||||
public static void CreateVerificationFile(LocalDatabase db, System.IO.StreamWriter stream, CancellationToken cancellationToken)
|
||||
{
|
||||
var s = new Newtonsoft.Json.JsonSerializer();
|
||||
s.Serialize(stream, db.GetRemoteVolumes().Where(x => x.State != RemoteVolumeState.Temporary).Cast<IRemoteVolume>().ToArray());
|
||||
s.Serialize(stream,
|
||||
db.GetRemoteVolumes(cancellationToken)
|
||||
.Where(x => x.State != RemoteVolumeState.Temporary)
|
||||
.ToEnumerable()
|
||||
.Cast<IRemoteVolume>()
|
||||
.ToArray()
|
||||
);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uploads the verification file.
|
||||
/// </summary>
|
||||
/// <param name="backendurl">The backend url</param>
|
||||
/// <param name="options">The options to use</param>
|
||||
/// <param name="db">The attached database</param>
|
||||
/// <param name="transaction">An optional transaction object</param>
|
||||
public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db, IDbTransaction transaction)
|
||||
/// <param name="backendManager">The backend manager instance to use.</param>
|
||||
/// <param name="options">The options to use.</param>
|
||||
/// <param name="db">The attached database.</param>
|
||||
/// <param name="cancellationToken">The cancellation token to use.</param>
|
||||
/// <returns>A task that completes when the upload is done.</returns>
|
||||
public static async Task UploadVerificationFile(IBackendManager backendManager, Options options, LocalDatabase db, CancellationToken cancellationToken)
|
||||
{
|
||||
using (var tempfile = new Library.Utility.TempFile())
|
||||
{
|
||||
var remotename = options.Prefix + "-verification.json";
|
||||
await using (var stream = new System.IO.StreamWriter(tempfile, false, System.Text.Encoding.UTF8))
|
||||
CreateVerificationFile(db, stream);
|
||||
CreateVerificationFile(db, stream, cancellationToken);
|
||||
|
||||
if (options.Dryrun)
|
||||
{
|
||||
@@ -232,34 +250,36 @@ namespace Duplicati.Library.Main.Operation
|
||||
}
|
||||
else
|
||||
{
|
||||
await backendManager.PutVerificationFileAsync(remotename, tempfile, CancellationToken.None).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(db, transaction, CancellationToken.None).ConfigureAwait(false);
|
||||
await backendManager.PutVerificationFileAsync(remotename, tempfile, cancellationToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Helper method that verifies uploaded volumes and updates their state in the database.
|
||||
/// Throws an error if there are issues with the remote storage
|
||||
/// Throws an error if there are issues with the remote storage.
|
||||
/// </summary>
|
||||
/// <param name="backend">The backend instance to use</param>
|
||||
/// <param name="options">The options used</param>
|
||||
/// <param name="database">The database to compare with</param>
|
||||
/// <param name="protectedFiles">Filenames that should be exempted from deletion</param>
|
||||
/// <param name="strictExcemptFiles">Filenames that should be exempted from strict verification</param>
|
||||
/// <param name="verifyMode">The mode to use for verification</param>
|
||||
/// <returns>The result of the analysis</returns>
|
||||
public static async Task<RemoteAnalysisResult> RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IDbTransaction transaction, IBackendWriter log, IEnumerable<string> protectedFiles, IEnumerable<string> strictExcemptFiles, VerifyMode verifyMode)
|
||||
/// <param name="backendManager">The backend manager instance to use.</param>
|
||||
/// <param name="options">The options used.</param>
|
||||
/// <param name="database">The database to compare with.</param>
|
||||
/// <param name="log">The log instance to use.</param>
|
||||
/// <param name="protectedFiles">Filenames that should be exempted from deletion.</param>
|
||||
/// <param name="strictExcemptFiles">Filenames that should be exempted from strict verification.</param>
|
||||
/// <param name="verifyMode">The mode to use for verification.</param>
|
||||
/// <param name="cancellationToken">The cancellation token to use.</param>
|
||||
/// <returns>A task that when awaited contains the analysis result.</returns>
|
||||
public static async Task<RemoteAnalysisResult> RemoteListAnalysis(IBackendManager backendManager, Options options, LocalDatabase database, IBackendWriter log, IEnumerable<string> protectedFiles, IEnumerable<string> strictExcemptFiles, VerifyMode verifyMode, CancellationToken cancellationToken)
|
||||
{
|
||||
// If the last operation completed, no cleanup should be required
|
||||
if (verifyMode == VerifyMode.VerifyAndClean && !database.TerminatedWithActiveUploads)
|
||||
if (verifyMode == VerifyMode.VerifyAndClean && !await database.TerminatedWithActiveUploads(cancellationToken).ConfigureAwait(false))
|
||||
verifyMode = VerifyMode.VerifyStrict;
|
||||
|
||||
// Force cleanup should set the mode to cleanup
|
||||
if (verifyMode == VerifyMode.VerifyAndCleanForced)
|
||||
verifyMode = VerifyMode.VerifyAndClean;
|
||||
|
||||
var rawlist = await backendManager.ListAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
var rawlist = await backendManager.ListAsync(cancellationToken).ConfigureAwait(false);
|
||||
var lookup = new Dictionary<string, Volumes.IParsedVolume>();
|
||||
protectedFiles = protectedFiles ?? [];
|
||||
strictExcemptFiles = strictExcemptFiles ?? [];
|
||||
@@ -290,7 +310,10 @@ namespace Duplicati.Library.Main.Operation
|
||||
log.KnownFileSize = knownFileSize;
|
||||
log.UnknownFileCount = unknownlist.Count;
|
||||
log.UnknownFileSize = unknownlist.Select(x => Math.Max(0, x.Size)).Sum();
|
||||
log.BackupListCount = database.FilesetTimes.Count();
|
||||
log.BackupListCount = await database
|
||||
.FilesetTimes(cancellationToken)
|
||||
.CountAsync(cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
log.LastBackupDate = filesets.Count == 0 ? new DateTime(0) : filesets[0].Time.ToLocalTime();
|
||||
|
||||
await CheckQuota(backendManager, options, log, knownFileSize).ConfigureAwait(false);
|
||||
@@ -304,16 +327,18 @@ namespace Duplicati.Library.Main.Operation
|
||||
var missingUploadingVolumes = new HashSet<string>();
|
||||
var protectedVolumes = new HashSet<string>();
|
||||
|
||||
foreach (var e in database.DuplicateRemoteVolumes(transaction))
|
||||
await foreach (var e in database.DuplicateRemoteVolumes(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
if (e.Value == RemoteVolumeState.Uploading || e.Value == RemoteVolumeState.Temporary)
|
||||
database.UnlinkRemoteVolume(e.Key, e.Value, transaction);
|
||||
await database
|
||||
.UnlinkRemoteVolume(e.Key, e.Value, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
else
|
||||
throw new RemoteListVerificationException(string.Format("The remote volume {0} appears in the database with state {1} and a deleted state, cannot continue", e.Key, e.Value.ToString()), "AmbiguousStateRemoteFiles");
|
||||
}
|
||||
|
||||
var locallist = database.GetRemoteVolumes(transaction);
|
||||
foreach (var i in locallist)
|
||||
var locallist = database.GetRemoteVolumes(cancellationToken);
|
||||
await foreach (var i in locallist.ConfigureAwait(false))
|
||||
{
|
||||
var remoteFound = lookup.TryGetValue(i.Name, out var r);
|
||||
var correctSize = remoteFound && i.Size >= 0 && (i.Size == r.File.Size || r.File.Size < 0);
|
||||
@@ -321,9 +346,13 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (archived && r.File.Size == 0)
|
||||
correctSize = true;
|
||||
if (archived && i.ArchiveTime <= DateTime.UnixEpoch)
|
||||
database.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true, transaction);
|
||||
await database
|
||||
.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, true, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
else if (!archived && i.ArchiveTime > DateTime.UnixEpoch)
|
||||
database.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false, transaction);
|
||||
await database
|
||||
.UpdateRemoteVolume(i.Name, i.State, i.Size, i.Hash, true, TimeSpan.Zero, false, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
lookup.Remove(i.Name);
|
||||
|
||||
@@ -378,7 +407,9 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (remoteFound && correctSize && r.File.Size >= 0)
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "PromotingCompleteFile", "Promoting uploaded complete file from {0} to {2}: {1}", i.State, i.Name, RemoteVolumeState.Uploaded);
|
||||
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash, transaction);
|
||||
await database
|
||||
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Uploaded, i.Size, i.Hash, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else if (!remoteFound)
|
||||
{
|
||||
@@ -386,7 +417,9 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
protectedVolumes.Add(i.Name);
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "KeepIncompleteFile", "Keeping protected incomplete remote file listed as {0}: {1}", i.State, i.Name);
|
||||
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null, transaction);
|
||||
await database
|
||||
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Temporary, i.Size, i.Hash, false, new TimeSpan(0), null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else if (verifyMode == VerifyMode.VerifyStrict && !strictExcemptFiles.Any(pf => pf == i.Name))
|
||||
{
|
||||
@@ -397,7 +430,9 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "SchedulingMissingFileForDelete", "Scheduling missing file for deletion, currently listed as {0}: {1}", i.State, i.Name);
|
||||
missingUploadingVolumes.Add(i.Name);
|
||||
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null, transaction);
|
||||
await database
|
||||
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Deleting, i.Size, i.Hash, false, TimeSpan.FromHours(2), null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -435,7 +470,9 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (!remoteFound)
|
||||
missing.Add(i);
|
||||
else if (correctSize)
|
||||
database.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash, transaction);
|
||||
await database
|
||||
.UpdateRemoteVolume(i.Name, RemoteVolumeState.Verified, i.Size, i.Hash, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
else
|
||||
missingHash.Add(new Tuple<long, RemoteVolumeEntry>(r.File.Size, i));
|
||||
|
||||
@@ -455,19 +492,28 @@ namespace Duplicati.Library.Main.Operation
|
||||
}
|
||||
}
|
||||
|
||||
await backendManager.WaitForEmptyAsync(database, null, CancellationToken.None).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(database, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
// Batch cleanup deleted volumes in DB
|
||||
if (verifyMode == VerifyMode.VerifyStrict && missingUploadingVolumes.Count > 0)
|
||||
throw new RemoteListVerificationException($"The remote volumes {string.Join(", ", missingUploadingVolumes)} are supposed to be deleted, but strict mode is on. Try running the \"repair\" command", "DeleteDuringStrictMode");
|
||||
else if (verifyMode == VerifyMode.VerifyAndClean || verifyMode == VerifyMode.VerifyStrict)
|
||||
{
|
||||
database.RemoveRemoteVolumes(missingUploadingVolumes.Concat(temporaryAndDeletedVolumes), transaction);
|
||||
await database
|
||||
.RemoveRemoteVolumes(
|
||||
missingUploadingVolumes
|
||||
.Concat(temporaryAndDeletedVolumes),
|
||||
cancellationToken
|
||||
)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Clear the flag after we have cleaned up
|
||||
if (!options.Dryrun && protectedVolumes.Count == 0)
|
||||
database.TerminatedWithActiveUploads = false;
|
||||
await database
|
||||
.TerminatedWithActiveUploads(cancellationToken, false)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else if (verifyMode == VerifyMode.VerifyOnly && database.TerminatedWithActiveUploads)
|
||||
else if (verifyMode == VerifyMode.VerifyOnly && await database.TerminatedWithActiveUploads(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, "ActiveUploadsDetected", null, "Active uploads detected, but no cleanup was performed. Run the \"repair\" command to clean up the incomplete files");
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -39,36 +39,50 @@ namespace Duplicati.Library.Main.Operation
|
||||
m_result = result;
|
||||
}
|
||||
|
||||
public Task RunAsync(List<string> args, Action<IListAffectedResults> callback = null)
|
||||
public async Task RunAsync(List<string> args, Action<IListAffectedResults> callback = null)
|
||||
{
|
||||
if (!File.Exists(m_options.Dbpath))
|
||||
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
|
||||
|
||||
using (var db = new Database.LocalListAffectedDatabase(m_options.Dbpath, m_options.SqlitePageCache))
|
||||
await using var db = await Database.LocalListAffectedDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
if (callback == null)
|
||||
{
|
||||
if (callback == null)
|
||||
{
|
||||
m_result.SetResult(
|
||||
db.GetFilesets(args).OrderByDescending(x => x.Time).ToArray(),
|
||||
db.GetFiles(args).ToArray(),
|
||||
db.GetLogLines(args).ToArray(),
|
||||
db.GetVolumes(args).ToArray()
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
m_result.SetResult(
|
||||
db.GetFilesets(args).OrderByDescending(x => x.Time),
|
||||
db.GetFiles(args),
|
||||
db.GetLogLines(args),
|
||||
db.GetVolumes(args)
|
||||
);
|
||||
|
||||
callback(m_result);
|
||||
}
|
||||
m_result.SetResult(
|
||||
await db.GetFilesets(args, m_result.TaskControl.ProgressToken)
|
||||
.OrderByDescending(x => x.Time)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false),
|
||||
await db.GetFiles(args, m_result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false),
|
||||
await db.GetLogLines(args, m_result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false),
|
||||
await db.GetVolumes(args, m_result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
m_result.SetResult(
|
||||
await db.GetFilesets(args, m_result.TaskControl.ProgressToken)
|
||||
.OrderByDescending(x => x.Time)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false),
|
||||
await db.GetFiles(args, m_result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false),
|
||||
await db.GetLogLines(args, m_result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false),
|
||||
await db.GetVolumes(args, m_result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
|
||||
return Task.CompletedTask;
|
||||
callback(m_result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
@@ -52,28 +52,31 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (!File.Exists(m_options.Dbpath))
|
||||
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
|
||||
|
||||
using (var db = new Database.LocalListBrokenFilesDatabase(m_options.Dbpath, m_options.SqlitePageCache))
|
||||
using (var tr = db.BeginTransaction())
|
||||
{
|
||||
Utility.UpdateOptionsFromDb(db, m_options);
|
||||
Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
|
||||
await DoRunAsync(backendManager, db, tr, filter, callbackhandler).ConfigureAwait(false);
|
||||
}
|
||||
await using var db = await Database.LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
await DoRunAsync(backendManager, db, filter, callbackhandler).ConfigureAwait(false);
|
||||
await db.Transaction.RollBackAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public static async Task<((DateTime FilesetTime, long FilesetID, long RemoveCount)[]?, List<Database.RemoteVolumeEntry>? Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, Options options)
|
||||
public static async Task<((DateTime FilesetTime, long FilesetID, long RemoveCount)[]?, List<Database.RemoteVolumeEntry>? Missing)> GetBrokenFilesetsFromRemote(IBackendManager backendManager, BasicResults result, Database.LocalListBrokenFilesDatabase db, Options options)
|
||||
{
|
||||
List<Database.RemoteVolumeEntry>? missing = null;
|
||||
var brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray();
|
||||
var brokensets = await db
|
||||
.GetBrokenFilesets(options.Time, options.Version, result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (brokensets.Length == 0)
|
||||
{
|
||||
if (db.RepairInProgress)
|
||||
if (await db.RepairInProgress(result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
throw new UserInformationException("Cannot continue because the database is marked as being under repair, but does not have broken files.", "CannotListOnDatabaseInRepair");
|
||||
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsInDatabase", "No broken filesets found in database, checking for missing remote files");
|
||||
|
||||
var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, transaction, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly).ConfigureAwait(false);
|
||||
var remotestate = await FilelistProcessor.RemoteListAnalysis(backendManager, options, db, result.BackendWriter, null, null, FilelistProcessor.VerifyMode.VerifyOnly, result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
if (!remotestate.ParsedVolumes.Any())
|
||||
throw new UserInformationException("No remote volumes were found, refusing purge", "CannotPurgeWithNoRemoteVolumes");
|
||||
|
||||
@@ -86,32 +89,41 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
// Mark all volumes as disposable
|
||||
foreach (var f in missing)
|
||||
db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction);
|
||||
await db
|
||||
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "MarkedRemoteFilesForDeletion", "Marked {0} remote files for deletion", missing.Count);
|
||||
|
||||
// Drop all content from tables
|
||||
db.RemoveMissingBlocks(missing.Select(x => x.Name), transaction);
|
||||
await db
|
||||
.RemoveMissingBlocks(missing.Select(x => x.Name), result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// Mark all orphaned index files as disposable after removing the missing block files
|
||||
foreach (var f in db.GetOrphanedIndexFiles(transaction).ToList())
|
||||
db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, transaction);
|
||||
await foreach (var f in db.GetOrphanedIndexFiles(result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
await db
|
||||
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Deleting, f.Size, f.Hash, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
brokensets = db.GetBrokenFilesets(options.Time, options.Version, transaction).ToArray();
|
||||
brokensets = await db
|
||||
.GetBrokenFilesets(options.Time, options.Version, result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return (brokensets, missing);
|
||||
}
|
||||
|
||||
private async Task DoRunAsync(IBackendManager backendManager, Database.LocalListBrokenFilesDatabase db, System.Data.IDbTransaction transaction, IFilter filter, Func<long, DateTime, long, string, long, bool>? callbackhandler)
|
||||
private async Task DoRunAsync(IBackendManager backendManager, Database.LocalListBrokenFilesDatabase db, IFilter filter, Func<long, DateTime, long, string, long, bool>? callbackhandler)
|
||||
{
|
||||
if (filter != null && !filter.Empty)
|
||||
throw new UserInformationException("Filters are not supported for this operation", "FiltersAreNotSupportedForListBrokenFiles");
|
||||
|
||||
if (db.PartiallyRecreated)
|
||||
if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
throw new UserInformationException("The command does not work on partially recreated databases", "ListBrokenFilesDoesNotWorkOnPartialDatabase");
|
||||
|
||||
(var brokensets, var missing) = await GetBrokenFilesetsFromRemote(backendManager, m_result, db, transaction, m_options).ConfigureAwait(false);
|
||||
(var brokensets, var missing) = await GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false);
|
||||
if (brokensets == null)
|
||||
return;
|
||||
|
||||
@@ -129,35 +141,41 @@ namespace Duplicati.Library.Main.Operation
|
||||
return;
|
||||
}
|
||||
|
||||
var fstimes = db.FilesetTimes.ToList();
|
||||
var fstimes = await db
|
||||
.FilesetTimes(m_result.TaskControl.ProgressToken)
|
||||
.ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var brokenfilesets =
|
||||
brokensets.Select(x => new
|
||||
{
|
||||
Version = fstimes.FindIndex(y => y.Key == x.Item2),
|
||||
Timestamp = x.Item1,
|
||||
FilesetID = x.Item2,
|
||||
BrokenCount = x.Item3
|
||||
Version = fstimes.FindIndex(y => y.Key == x.FilesetID),
|
||||
Timestamp = x.FilesetTime,
|
||||
FilesetID = x.FilesetID,
|
||||
BrokenCount = x.RemoveCount
|
||||
}
|
||||
)
|
||||
.ToArray();
|
||||
|
||||
|
||||
m_result.BrokenFiles =
|
||||
brokenfilesets.Select(
|
||||
x => new Tuple<long, DateTime, IEnumerable<Tuple<string, long>>>(
|
||||
await Task.WhenAll(brokenfilesets.Select(
|
||||
async x => new Tuple<long, DateTime, IEnumerable<Tuple<string, long>>>(
|
||||
x.Version,
|
||||
x.Timestamp,
|
||||
callbackhandler == null && !m_options.ListSetsOnly
|
||||
? db.GetBrokenFilenames(x.FilesetID, transaction).ToArray().AsEnumerable()
|
||||
? await db
|
||||
.GetBrokenFilenames(x.FilesetID, m_result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false)
|
||||
: new MockList<Tuple<string, long>>((int)x.BrokenCount)
|
||||
))
|
||||
.ToArray();
|
||||
)))
|
||||
.ConfigureAwait(false);
|
||||
|
||||
|
||||
if (callbackhandler != null)
|
||||
foreach (var bs in brokenfilesets)
|
||||
foreach (var fe in db.GetBrokenFilenames(bs.FilesetID, transaction))
|
||||
await foreach (var fe in db.GetBrokenFilenames(bs.FilesetID, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
if (!callbackhandler(bs.Version, bs.Timestamp, bs.BrokenCount, fe.Item1, fe.Item2))
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -79,103 +79,118 @@ namespace Duplicati.Library.Main.Operation
|
||||
DateTime baseVersionTime;
|
||||
DateTime compareVersionTime;
|
||||
|
||||
using (var tmpdb = useLocalDb ? null : new TempFile())
|
||||
using (var db = new Database.LocalListChangesDatabase(useLocalDb ? m_options.Dbpath : (string)tmpdb, m_options.SqlitePageCache))
|
||||
using (var storageKeeper = db.CreateStorageHelper())
|
||||
using var tmpdb = useLocalDb ? null : new TempFile();
|
||||
await using var db = await Database.LocalListChangesDatabase.CreateAsync(useLocalDb ? m_options.Dbpath : (string)tmpdb, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await using var storageKeeper = await db.CreateStorageHelper(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
if (useLocalDb)
|
||||
{
|
||||
if (useLocalDb)
|
||||
{
|
||||
var dbtimes = db.FilesetTimes.ToList();
|
||||
if (dbtimes.Count < 2)
|
||||
throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", dbtimes.Count), "NeedTwoBackupsToStartDiff");
|
||||
var dbtimes = await db
|
||||
.FilesetTimes(m_result.TaskControl.ProgressToken)
|
||||
.ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
long baseVersionId;
|
||||
long compareVersionId;
|
||||
if (dbtimes.Count < 2)
|
||||
throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", dbtimes.Count), "NeedTwoBackupsToStartDiff");
|
||||
|
||||
var times = dbtimes.Zip(Enumerable.Range(0, dbtimes.Count), (a, b) => new Tuple<long, DateTime, long>(b, a.Value, a.Key)).ToList();
|
||||
var bt = SelectTime(baseVersion, times, out baseVersionIndex, out baseVersionTime, out baseVersionId);
|
||||
times.Remove(bt);
|
||||
SelectTime(compareVersion, times, out compareVersionIndex, out compareVersionTime, out compareVersionId);
|
||||
long baseVersionId;
|
||||
long compareVersionId;
|
||||
|
||||
storageKeeper.AddFromDb(baseVersionId, false, filter);
|
||||
storageKeeper.AddFromDb(compareVersionId, true, filter);
|
||||
}
|
||||
else
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store");
|
||||
var times = dbtimes.Zip(Enumerable.Range(0, dbtimes.Count), (a, b) => new Tuple<long, DateTime, long>(b, a.Value, a.Key)).ToList();
|
||||
var bt = SelectTime(baseVersion, times, out baseVersionIndex, out baseVersionTime, out baseVersionId);
|
||||
times.Remove(bt);
|
||||
SelectTime(compareVersion, times, out compareVersionIndex, out compareVersionTime, out compareVersionId);
|
||||
|
||||
var parsedlist = (await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
.Select(n => Volumes.VolumeBase.ParseFilename(n))
|
||||
.Where(p => p != null && p.FileType == RemoteVolumeType.Files)
|
||||
.OrderByDescending(p => p.Time)
|
||||
.ToArray();
|
||||
|
||||
var numberedList = parsedlist.Zip(Enumerable.Range(0, parsedlist.Length), (a, b) => new Tuple<long, DateTime, Volumes.IParsedVolume>(b, a.Time, a)).ToList();
|
||||
if (numberedList.Count < 2)
|
||||
throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", numberedList.Count), "NeedTwoBackupsToStartDiff");
|
||||
|
||||
Volumes.IParsedVolume baseFile;
|
||||
Volumes.IParsedVolume compareFile;
|
||||
|
||||
var bt = SelectTime(baseVersion, numberedList, out baseVersionIndex, out baseVersionTime, out baseFile);
|
||||
numberedList.Remove(bt);
|
||||
SelectTime(compareVersion, numberedList, out compareVersionIndex, out compareVersionTime, out compareFile);
|
||||
|
||||
Func<FilelistEntryType, ListChangesElementType> conv = (x) =>
|
||||
{
|
||||
switch (x)
|
||||
{
|
||||
case FilelistEntryType.File:
|
||||
return ListChangesElementType.File;
|
||||
case FilelistEntryType.Folder:
|
||||
return ListChangesElementType.Folder;
|
||||
case FilelistEntryType.Symlink:
|
||||
return ListChangesElementType.Symlink;
|
||||
default:
|
||||
return (ListChangesElementType)(-1);
|
||||
}
|
||||
};
|
||||
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
return;
|
||||
|
||||
using (var tmpfile = await backendManager.GetAsync(baseFile.File.Name, null, baseFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(baseFile.File.Name), tmpfile, m_options))
|
||||
foreach (var f in rd.Files)
|
||||
if (FilterExpression.Matches(filter, f.Path))
|
||||
storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false);
|
||||
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
return;
|
||||
|
||||
using (var tmpfile = await backendManager.GetAsync(compareFile.File.Name, null, compareFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(compareFile.File.Name), tmpfile, m_options))
|
||||
foreach (var f in rd.Files)
|
||||
if (FilterExpression.Matches(filter, f.Path))
|
||||
storageKeeper.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true);
|
||||
}
|
||||
|
||||
var changes = storageKeeper.CreateChangeCountReport();
|
||||
var sizes = storageKeeper.CreateChangeSizeReport();
|
||||
|
||||
var lst = (m_options.FullResult || callback != null) ?
|
||||
(from n in storageKeeper.CreateChangedFileReport()
|
||||
select n) : null;
|
||||
|
||||
m_result.SetResult(
|
||||
baseVersionTime, baseVersionIndex, compareVersionTime, compareVersionIndex,
|
||||
changes.AddedFolders, changes.AddedSymlinks, changes.AddedFiles,
|
||||
changes.DeletedFolders, changes.DeletedSymlinks, changes.DeletedFiles,
|
||||
changes.ModifiedFolders, changes.ModifiedSymlinks, changes.ModifiedFiles,
|
||||
sizes.AddedSize, sizes.DeletedSize, sizes.PreviousSize, sizes.CurrentSize,
|
||||
(lst == null || callback == null) ? null : lst.ToArray()
|
||||
);
|
||||
|
||||
if (callback != null)
|
||||
callback(m_result, lst);
|
||||
|
||||
return;
|
||||
await storageKeeper
|
||||
.AddFromDb(baseVersionId, false, filter, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
await storageKeeper
|
||||
.AddFromDb(compareVersionId, true, filter, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store");
|
||||
|
||||
var parsedlist = (await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
.Select(n => Volumes.VolumeBase.ParseFilename(n))
|
||||
.Where(p => p != null && p.FileType == RemoteVolumeType.Files)
|
||||
.OrderByDescending(p => p.Time)
|
||||
.ToArray();
|
||||
|
||||
var numberedList = parsedlist.Zip(Enumerable.Range(0, parsedlist.Length), (a, b) => new Tuple<long, DateTime, Volumes.IParsedVolume>(b, a.Time, a)).ToList();
|
||||
if (numberedList.Count < 2)
|
||||
throw new UserInformationException(string.Format("Need at least two backups to show differences, database contains {0} backups", numberedList.Count), "NeedTwoBackupsToStartDiff");
|
||||
|
||||
Volumes.IParsedVolume baseFile;
|
||||
Volumes.IParsedVolume compareFile;
|
||||
|
||||
var bt = SelectTime(baseVersion, numberedList, out baseVersionIndex, out baseVersionTime, out baseFile);
|
||||
numberedList.Remove(bt);
|
||||
SelectTime(compareVersion, numberedList, out compareVersionIndex, out compareVersionTime, out compareFile);
|
||||
|
||||
Func<FilelistEntryType, ListChangesElementType> conv = (x) =>
|
||||
{
|
||||
switch (x)
|
||||
{
|
||||
case FilelistEntryType.File:
|
||||
return ListChangesElementType.File;
|
||||
case FilelistEntryType.Folder:
|
||||
return ListChangesElementType.Folder;
|
||||
case FilelistEntryType.Symlink:
|
||||
return ListChangesElementType.Symlink;
|
||||
default:
|
||||
return (ListChangesElementType)(-1);
|
||||
}
|
||||
};
|
||||
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
return;
|
||||
|
||||
using (var tmpfile = await backendManager.GetAsync(baseFile.File.Name, null, baseFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(baseFile.File.Name), tmpfile, m_options))
|
||||
foreach (var f in rd.Files)
|
||||
if (FilterExpression.Matches(filter, f.Path))
|
||||
await storageKeeper
|
||||
.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), false, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
return;
|
||||
|
||||
using (var tmpfile = await backendManager.GetAsync(compareFile.File.Name, null, compareFile.File.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
using (var rd = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(compareFile.File.Name), tmpfile, m_options))
|
||||
foreach (var f in rd.Files)
|
||||
if (FilterExpression.Matches(filter, f.Path))
|
||||
await storageKeeper
|
||||
.AddElement(f.Path, f.Hash, f.Metahash, f.Size, conv(f.Type), true, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var changes = await storageKeeper
|
||||
.CreateChangeCountReport(m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var sizes = await storageKeeper
|
||||
.CreateChangeSizeReport(m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var lst = (m_options.FullResult || callback != null) ?
|
||||
(from n in storageKeeper.CreateChangedFileReport(m_result.TaskControl.ProgressToken)
|
||||
select n) : null;
|
||||
|
||||
m_result.SetResult(
|
||||
baseVersionTime, baseVersionIndex, compareVersionTime, compareVersionIndex,
|
||||
changes.AddedFolders, changes.AddedSymlinks, changes.AddedFiles,
|
||||
changes.DeletedFolders, changes.DeletedSymlinks, changes.DeletedFiles,
|
||||
changes.ModifiedFolders, changes.ModifiedSymlinks, changes.ModifiedFiles,
|
||||
sizes.AddedSize, sizes.DeletedSize, sizes.PreviousSize, sizes.CurrentSize,
|
||||
(lst == null || callback == null) ? null : await lst.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken).ConfigureAwait(false)
|
||||
);
|
||||
|
||||
if (callback != null)
|
||||
callback(m_result, lst.ToEnumerable());
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -42,51 +42,51 @@ namespace Duplicati.Library.Main.Operation
|
||||
public async Task RunAsync(IBackendManager backendManager, IEnumerable<string> filterstrings, IFilter compositefilter)
|
||||
{
|
||||
var cancellationToken = CancellationToken.None;
|
||||
using (var tmpdb = new TempFile())
|
||||
using (var db = new LocalDatabase(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, m_options.SqlitePageCache))
|
||||
using var tmpdb = new TempFile();
|
||||
await using var db = await LocalDatabase.CreateLocalDatabaseAsync(System.IO.File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "ListControlFiles", true, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter);
|
||||
|
||||
try
|
||||
{
|
||||
var filter = JoinedFilterExpression.Join(new Library.Utility.FilterExpression(filterstrings), compositefilter);
|
||||
var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options);
|
||||
if (filteredList.Count == 0)
|
||||
throw new Exception("No filesets found on remote target");
|
||||
|
||||
try
|
||||
{
|
||||
var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options);
|
||||
if (filteredList.Count == 0)
|
||||
throw new Exception("No filesets found on remote target");
|
||||
Exception lastEx = new Exception("No suitable files found on remote target");
|
||||
|
||||
Exception lastEx = new Exception("No suitable files found on remote target");
|
||||
foreach (var fileversion in filteredList)
|
||||
try
|
||||
{
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
return;
|
||||
|
||||
foreach (var fileversion in filteredList)
|
||||
try
|
||||
{
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
return;
|
||||
var file = fileversion.Value.File;
|
||||
var entry = await db
|
||||
.GetRemoteVolume(file.Name, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var file = fileversion.Value.File;
|
||||
var entry = db.GetRemoteVolume(file.Name);
|
||||
var files = new List<Library.Interface.IListResultFile>();
|
||||
using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).ConfigureAwait(false))
|
||||
using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options))
|
||||
foreach (var cf in tmp.ControlFiles)
|
||||
if (Library.Utility.FilterExpression.Matches(filter, cf.Key))
|
||||
files.Add(new ListResultFile(cf.Key, null));
|
||||
|
||||
var files = new List<Library.Interface.IListResultFile>();
|
||||
using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, cancellationToken).ConfigureAwait(false))
|
||||
using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options))
|
||||
foreach (var cf in tmp.ControlFiles)
|
||||
if (Library.Utility.FilterExpression.Matches(filter, cf.Key))
|
||||
files.Add(new ListResultFile(cf.Key, null));
|
||||
m_result.SetResult(new Library.Interface.IListResultFileset[] { new ListResultFileset(fileversion.Key, BackupType.PARTIAL_BACKUP, fileversion.Value.Time, -1, -1) }, files);
|
||||
lastEx = null;
|
||||
break;
|
||||
}
|
||||
catch (Exception ex) when (!ex.IsAbortException())
|
||||
{
|
||||
lastEx = ex;
|
||||
}
|
||||
|
||||
m_result.SetResult(new Library.Interface.IListResultFileset[] { new ListResultFileset(fileversion.Key, BackupType.PARTIAL_BACKUP, fileversion.Value.Time, -1, -1) }, files);
|
||||
lastEx = null;
|
||||
break;
|
||||
}
|
||||
catch (Exception ex) when (!ex.IsAbortException())
|
||||
{
|
||||
lastEx = ex;
|
||||
}
|
||||
|
||||
if (lastEx != null)
|
||||
throw lastEx;
|
||||
}
|
||||
finally
|
||||
{
|
||||
await backendManager.WaitForEmptyAsync(db, null, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
if (lastEx != null)
|
||||
throw lastEx;
|
||||
}
|
||||
finally
|
||||
{
|
||||
await backendManager.WaitForEmptyAsync(db, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
@@ -42,16 +42,22 @@ internal static class ListFileVersionsHandler
|
||||
/// <param name="offset">The offset to start listing from</param>
|
||||
/// <param name="limit">The maximum number of results to return</param>
|
||||
/// <returns>A task representing the asynchronous operation</returns>
|
||||
public static Task RunAsync(Options options, ListFileVersionsResults result, string[] paths, long offset, long limit)
|
||||
public async static Task RunAsync(Options options, ListFileVersionsResults result, string[] paths, long offset, long limit)
|
||||
{
|
||||
if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb)
|
||||
throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase");
|
||||
|
||||
using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache);
|
||||
await using var db =
|
||||
await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
long[]? filesetIds = null;
|
||||
if (!options.AllVersions)
|
||||
{
|
||||
filesetIds = db.GetFilesetIDs(options.Time, options.Version).ToArray();
|
||||
filesetIds = await db
|
||||
.GetFilesetIDs(options.Time, options.Version, false, result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (filesetIds.Length == 0)
|
||||
throw new UserInformationException("No filesets found", "NoFilesetsFound");
|
||||
}
|
||||
@@ -60,8 +66,8 @@ internal static class ListFileVersionsHandler
|
||||
throw new UserInformationException("No path specified", "NoPathSpecified");
|
||||
|
||||
paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray();
|
||||
result.FileVersions = db.ListFileVersions(paths, filesetIds, offset, limit);
|
||||
|
||||
return Task.CompletedTask;
|
||||
result.FileVersions = await db
|
||||
.ListFileVersions(paths, filesetIds, offset, limit, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,23 +1,24 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
@@ -54,59 +55,76 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
//Use a speedy local query
|
||||
if (!m_options.NoLocalDb && System.IO.File.Exists(m_options.Dbpath))
|
||||
using (var db = new Database.LocalListDatabase(m_options.Dbpath, m_options.SqlitePageCache))
|
||||
await using (var db = await Database.LocalListDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
using (var filesets = db.SelectFileSets(m_options.Time, m_options.Version))
|
||||
await using var filesets = await db.SelectFileSets(m_options.Time, m_options.Version, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
if (!filter.Empty)
|
||||
{
|
||||
if (!filter.Empty)
|
||||
if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions))
|
||||
{
|
||||
if (simpleList || (m_options.ListFolderContents && !m_options.AllVersions))
|
||||
{
|
||||
filesets.TakeFirst();
|
||||
}
|
||||
await filesets
|
||||
.TakeFirst(m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
IEnumerable<Database.LocalListDatabase.IFileversion> files;
|
||||
if (m_options.ListFolderContents)
|
||||
{
|
||||
files = filesets.SelectFolderContents(filter);
|
||||
}
|
||||
else if (m_options.ListPrefixOnly)
|
||||
{
|
||||
files = filesets.GetLargestPrefix(filter);
|
||||
}
|
||||
else if (filter.Empty)
|
||||
{
|
||||
files = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
files = filesets.SelectFiles(filter);
|
||||
}
|
||||
|
||||
if (m_options.ListSetsOnly)
|
||||
{
|
||||
m_result.SetResult(
|
||||
filesets.QuickSets.Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)).ToArray(),
|
||||
null
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
m_result.SetResult(
|
||||
filesets.Sets.Select(x =>
|
||||
new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes)).ToArray(),
|
||||
files == null
|
||||
? null
|
||||
: (from n in files
|
||||
select (Duplicati.Library.Interface.IListResultFile)(new ListResultFile(n.Path,
|
||||
n.Sizes.ToArray())))
|
||||
.ToArray()
|
||||
);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
IAsyncEnumerable<Database.LocalListDatabase.IFileversion> files;
|
||||
if (m_options.ListFolderContents)
|
||||
{
|
||||
files = filesets.SelectFolderContents(filter, m_result.TaskControl.ProgressToken);
|
||||
}
|
||||
else if (m_options.ListPrefixOnly)
|
||||
{
|
||||
files = filesets.GetLargestPrefix(filter, m_result.TaskControl.ProgressToken);
|
||||
}
|
||||
else if (filter.Empty)
|
||||
{
|
||||
files = null;
|
||||
}
|
||||
else
|
||||
{
|
||||
files = filesets.SelectFiles(filter, m_result.TaskControl.ProgressToken);
|
||||
}
|
||||
|
||||
if (m_options.ListSetsOnly)
|
||||
{
|
||||
m_result.SetResult(
|
||||
await filesets
|
||||
.QuickSets(m_result.TaskControl.ProgressToken)
|
||||
.Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes))
|
||||
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false),
|
||||
null
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
m_result.SetResult(
|
||||
await filesets
|
||||
.Sets(m_result.TaskControl.ProgressToken)
|
||||
.Select(x => new ListResultFileset(x.Version, x.IsFullBackup, x.Time, x.FileCount, x.FileSizes))
|
||||
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false),
|
||||
files == null
|
||||
? null
|
||||
:
|
||||
await files.Select(async n =>
|
||||
new ListResultFile(
|
||||
n.Path,
|
||||
await n
|
||||
.Sizes(m_result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false)
|
||||
)
|
||||
)
|
||||
.Select(x => x.Result)
|
||||
.Cast<IListResultFile>()
|
||||
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NoLocalDatabase", "No local database, accessing remote store");
|
||||
@@ -119,13 +137,14 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
// Otherwise, grab info from remote location
|
||||
using (var tmpdb = new TempFile())
|
||||
using (var db = new LocalDatabase(tmpdb, "List", true, m_options.SqlitePageCache))
|
||||
await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(tmpdb, "List", true, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
var filteredList = ParseAndFilterFilesets(await backendManager.ListAsync(cancellationToken).ConfigureAwait(false), m_options);
|
||||
if (filteredList.Count == 0)
|
||||
throw new UserInformationException("No filesets found on remote target", "EmptyRemoteFolder");
|
||||
|
||||
var numberSeq = await CreateResultSequence(filteredList, backendManager, m_options, cancellationToken);
|
||||
var numberSeq = await CreateResultSequence(filteredList, backendManager, m_options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (filter.Empty)
|
||||
{
|
||||
m_result.SetResult(numberSeq, null);
|
||||
|
||||
@@ -1,27 +1,28 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Interface;
|
||||
|
||||
@@ -46,11 +47,17 @@ internal static class ListFilesetsHandler
|
||||
/// <returns>A task representing the asynchronous operation</returns>
|
||||
public static async Task RunAsync(Options options, ListFilesetResults result, IBackendManager backendManager)
|
||||
{
|
||||
//Use a speedy local query
|
||||
// Use a speedy local query
|
||||
if (System.IO.File.Exists(options.Dbpath) && !options.NoLocalDb)
|
||||
{
|
||||
using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache);
|
||||
result.Filesets = db.ListFilesetsExtended().ToArray();
|
||||
await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
result.Filesets = await db
|
||||
.ListFilesetsExtended(result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
@@ -41,13 +41,19 @@ internal static class ListFolderHandler
|
||||
/// <param name="offset">The offset to start listing from</param>
|
||||
/// <param name="limit">The maximum number of entries to list</param>
|
||||
/// <returns>A task representing the asynchronous operation</returns>
|
||||
public static Task RunAsync(Options options, ListFolderResults result, string[] folders, long offset, long limit)
|
||||
public static async Task RunAsync(Options options, ListFolderResults result, string[] folders, long offset, long limit)
|
||||
{
|
||||
if (!System.IO.File.Exists(options.Dbpath) || options.NoLocalDb)
|
||||
throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase");
|
||||
|
||||
using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache);
|
||||
var filesetIds = db.GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true).ToArray();
|
||||
await using var db = await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var filesetIds = await db
|
||||
.GetFilesetIDs(options.Time, options.Version, singleTimeMatch: true, result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (filesetIds.Length == 0)
|
||||
throw new UserInformationException("No filesets found", "NoFilesetsFound");
|
||||
if (filesetIds.Length > 1)
|
||||
@@ -57,13 +63,23 @@ internal static class ListFolderHandler
|
||||
{
|
||||
if (folders != null && folders.Length > 1)
|
||||
throw new UserInformationException("When no folder is specified, only one folder can be listed", "MultipleFoldersFound");
|
||||
var rootFolders = db.GetMinimalUniquePrefixEntries(filesetIds[0]);
|
||||
var rootFolders = await db
|
||||
.GetMinimalUniquePrefixEntries(filesetIds[0], result.TaskControl.ProgressToken)
|
||||
.ToListAsync(cancellationToken: result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
result.Entries = new PaginatedResults<IListFolderEntry>(0, rootFolders.Count, 1, rootFolders.Count, rootFolders);
|
||||
}
|
||||
else
|
||||
{
|
||||
result.Entries = db.ListFolder(db.GetPrefixIds(folders), filesetIds[0], offset, limit);
|
||||
result.Entries = await db
|
||||
.ListFolder(
|
||||
db.GetPrefixIds(folders, result.TaskControl.ProgressToken).ToEnumerable(),
|
||||
filesetIds[0],
|
||||
offset,
|
||||
limit,
|
||||
result.TaskControl.ProgressToken
|
||||
)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,203 +1,225 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Main.Database;
|
||||
|
||||
namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
internal class PurgeBrokenFilesHandler
|
||||
{
|
||||
/// <summary>
|
||||
/// The tag used for logging
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(PurgeBrokenFilesHandler));
|
||||
protected readonly Options m_options;
|
||||
protected readonly PurgeBrokenFilesResults m_result;
|
||||
|
||||
public PurgeBrokenFilesHandler(Options options, PurgeBrokenFilesResults result)
|
||||
{
|
||||
m_options = options;
|
||||
m_result = result;
|
||||
}
|
||||
|
||||
public async Task RunAsync(IBackendManager backendManager, Library.Utility.IFilter filter)
|
||||
{
|
||||
if (!System.IO.File.Exists(m_options.Dbpath))
|
||||
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
|
||||
|
||||
if (filter != null && !filter.Empty)
|
||||
throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles");
|
||||
|
||||
using (var db = new LocalListBrokenFilesDatabase(m_options.Dbpath, m_options.SqlitePageCache))
|
||||
using (var tr = db.BeginTransaction())
|
||||
{
|
||||
if (db.PartiallyRecreated)
|
||||
throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase");
|
||||
|
||||
Utility.UpdateOptionsFromDb(db, m_options);
|
||||
Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
|
||||
|
||||
(var sets, var missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, tr, m_options).ConfigureAwait(false);
|
||||
if (sets == null)
|
||||
return;
|
||||
|
||||
if (sets.Length == 0)
|
||||
{
|
||||
if (missing == null)
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesets", "Found no broken filesets");
|
||||
else if (missing.Count == 0)
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files");
|
||||
else
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count));
|
||||
}
|
||||
else
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.RemoveCount));
|
||||
|
||||
var pgoffset = 0.0f;
|
||||
var pgspan = 0.95f / sets.Length;
|
||||
|
||||
var filesets = db.FilesetTimes.ToList();
|
||||
|
||||
var compare_list = sets.Select(x => new
|
||||
{
|
||||
FilesetID = x.Item2,
|
||||
Timestamp = x.Item1,
|
||||
RemoveCount = x.Item3,
|
||||
Version = filesets.FindIndex(y => y.Key == x.Item2),
|
||||
SetCount = db.GetFilesetFileCount(x.Item2, tr)
|
||||
}).ToArray();
|
||||
|
||||
var replacementMetadataBlocksetId = -1L;
|
||||
if (!m_options.DisableReplaceMissingMetadata)
|
||||
{
|
||||
var emptymetadata = Utility.WrapMetadata(new Dictionary<string, string>(), m_options);
|
||||
replacementMetadataBlocksetId = db.GetEmptyMetadataBlocksetId((missing ?? []).Select(x => x.ID), emptymetadata.FileHash, emptymetadata.Blob.Length, null);
|
||||
if (replacementMetadataBlocksetId < 0)
|
||||
throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset");
|
||||
}
|
||||
|
||||
var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray();
|
||||
var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray();
|
||||
|
||||
if (fully_emptied.Length == db.FilesetTimes.Count())
|
||||
throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken");
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
tr.Commit();
|
||||
|
||||
|
||||
if (fully_emptied.Length != 0)
|
||||
{
|
||||
if (fully_emptied.Length == 1)
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount);
|
||||
else
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString())));
|
||||
|
||||
m_result.DeleteResults = new DeleteResults(m_result);
|
||||
using (var rmdb = new LocalDeleteDatabase(db))
|
||||
using (var deltr = new ReusableTransaction(rmdb))
|
||||
{
|
||||
var opts = new Options(new Dictionary<string, string?>(m_options.RawOptions));
|
||||
opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString()));
|
||||
opts.RawOptions.Remove("time");
|
||||
opts.RawOptions["no-auto-compact"] = "true";
|
||||
|
||||
await new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults)
|
||||
.DoRunAsync(rmdb, deltr, true, false, backendManager).ConfigureAwait(false);
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
deltr.Commit("CommitDelete", restart: false);
|
||||
}
|
||||
|
||||
pgoffset += (pgspan * fully_emptied.Length);
|
||||
m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
|
||||
}
|
||||
|
||||
if (to_purge.Length > 0)
|
||||
{
|
||||
m_result.PurgeResults = new PurgeFilesResults(m_result);
|
||||
|
||||
foreach (var bs in to_purge)
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime());
|
||||
var opts = new Options(new Dictionary<string, string?>(m_options.RawOptions));
|
||||
|
||||
using (var pgdb = new Database.LocalPurgeDatabase(db))
|
||||
{
|
||||
// Recompute the version number after we deleted the versions before
|
||||
filesets = pgdb.FilesetTimes.ToList();
|
||||
var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID);
|
||||
if (thisversion < 0)
|
||||
throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString()))));
|
||||
|
||||
opts.RawOptions["version"] = thisversion.ToString();
|
||||
opts.RawOptions.Remove("time");
|
||||
opts.RawOptions["no-auto-compact"] = "true";
|
||||
|
||||
await new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).RunAsync(backendManager, pgdb, pgoffset, pgspan, (cmd, filesetid, tablename) =>
|
||||
{
|
||||
if (filesetid != bs.FilesetID)
|
||||
throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID));
|
||||
|
||||
// Update entries that would be removed because of missing metadata
|
||||
var updatedEntries = 0;
|
||||
if (!m_options.DisableReplaceMissingMetadata)
|
||||
updatedEntries = db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId, cmd.Transaction);
|
||||
|
||||
db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", cmd.Transaction);
|
||||
return updatedEntries;
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
pgoffset += pgspan;
|
||||
m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m_result.OperationProgressUpdater.UpdateProgress(0.95f);
|
||||
|
||||
if (!m_options.Dryrun && db.RepairInProgress)
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files");
|
||||
db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null);
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete");
|
||||
db.RepairInProgress = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null);
|
||||
}
|
||||
|
||||
m_result.OperationProgressUpdater.UpdateProgress(1.0f);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Main.Database;
|
||||
using Duplicati.Library.Utility;
|
||||
|
||||
namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
internal class PurgeBrokenFilesHandler
|
||||
{
|
||||
/// <summary>
|
||||
/// The tag used for logging
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(PurgeBrokenFilesHandler));
|
||||
protected readonly Options m_options;
|
||||
protected readonly PurgeBrokenFilesResults m_result;
|
||||
|
||||
public PurgeBrokenFilesHandler(Options options, PurgeBrokenFilesResults result)
|
||||
{
|
||||
m_options = options;
|
||||
m_result = result;
|
||||
}
|
||||
|
||||
public async Task RunAsync(IBackendManager backendManager, Library.Utility.IFilter filter)
|
||||
{
|
||||
if (!System.IO.File.Exists(m_options.Dbpath))
|
||||
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
|
||||
|
||||
if (filter != null && !filter.Empty)
|
||||
throw new UserInformationException("Filters are not supported for this operation", "FiltersNotAllowedOnPurgeBrokenFiles");
|
||||
|
||||
await using var db = await LocalListBrokenFilesDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
throw new UserInformationException("The command does not work on partially recreated databases", "CannotPurgeOnPartialDatabase");
|
||||
|
||||
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var (sets, missing) = await ListBrokenFilesHandler.GetBrokenFilesetsFromRemote(backendManager, m_result, db, m_options).ConfigureAwait(false);
|
||||
if (sets == null)
|
||||
return;
|
||||
|
||||
if (sets.Length == 0)
|
||||
{
|
||||
if (missing == null)
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesets", "Found no broken filesets");
|
||||
else if (missing.Count == 0)
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenFilesetsOrMissingFiles", "Found no broken filesets and no missing remote files");
|
||||
else
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NoBrokenSetsButMissingRemoteFiles", string.Format("Found no broken filesets, but {0} missing remote files. Purging from database.", missing.Count));
|
||||
}
|
||||
else
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "FoundBrokenFilesets", "Found {0} broken filesets with {1} affected files, purging files", sets.Length, sets.Sum(x => x.RemoveCount));
|
||||
|
||||
var pgoffset = 0.0f;
|
||||
var pgspan = 0.95f / sets.Length;
|
||||
|
||||
var filesets = await db
|
||||
.FilesetTimes(m_result.TaskControl.ProgressToken)
|
||||
.ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var compare_list = sets.Select(async x => new
|
||||
{
|
||||
FilesetID = x.FilesetID,
|
||||
Timestamp = x.FilesetTime,
|
||||
RemoveCount = x.RemoveCount,
|
||||
Version = filesets.FindIndex(y => y.Key == x.FilesetID),
|
||||
SetCount = await db
|
||||
.GetFilesetFileCount(x.FilesetID, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false)
|
||||
})
|
||||
.Select(x => x.Result)
|
||||
.ToArray();
|
||||
|
||||
var replacementMetadataBlocksetId = -1L;
|
||||
if (!m_options.DisableReplaceMissingMetadata)
|
||||
{
|
||||
var emptymetadata = Utility.WrapMetadata(new Dictionary<string, string>(), m_options);
|
||||
replacementMetadataBlocksetId = await db
|
||||
.GetEmptyMetadataBlocksetId(
|
||||
(missing ?? []).Select(x => x.ID),
|
||||
emptymetadata.FileHash,
|
||||
emptymetadata.Blob.Length,
|
||||
m_result.TaskControl.ProgressToken
|
||||
)
|
||||
.ConfigureAwait(false);
|
||||
if (replacementMetadataBlocksetId < 0)
|
||||
throw new UserInformationException($"Failed to locate an empty metadata blockset to replace missing metadata. Set the option --disable-replace-missing-metadata=true to ignore this and drop files with missing metadata.", "FailedToLocateEmptyMetadataBlockset");
|
||||
}
|
||||
|
||||
var fully_emptied = compare_list.Where(x => x.RemoveCount == x.SetCount).ToArray();
|
||||
var to_purge = compare_list.Where(x => x.RemoveCount != x.SetCount).ToArray();
|
||||
|
||||
if (fully_emptied.Length == await db.FilesetTimes(m_result.TaskControl.ProgressToken).CountAsync(cancellationToken: m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
throw new UserInformationException("All filesets are fully broken and needs to be removed. To avoid unexpected deletions, you must manually remove the remote files and delete the database.", "AllFilesetsBroken");
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
await db.Transaction.CommitAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
|
||||
if (fully_emptied.Length != 0)
|
||||
{
|
||||
if (fully_emptied.Length == 1)
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing entire fileset {1} as all {0} file(s) are broken", fully_emptied.First().Timestamp, fully_emptied.First().RemoveCount);
|
||||
else
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "RemovingFilesets", "Removing {0} filesets where all file(s) are broken: {1}", fully_emptied.Length, string.Join(", ", fully_emptied.Select(x => x.Timestamp.ToLocalTime().ToString())));
|
||||
|
||||
m_result.DeleteResults = new DeleteResults(m_result);
|
||||
await using (var rmdb = await LocalDeleteDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
var opts = new Options(new Dictionary<string, string?>(m_options.RawOptions));
|
||||
opts.RawOptions["version"] = string.Join(",", fully_emptied.Select(x => x.Version.ToString()));
|
||||
opts.RawOptions.Remove("time");
|
||||
opts.RawOptions["no-auto-compact"] = "true";
|
||||
|
||||
await new DeleteHandler(opts, (DeleteResults)m_result.DeleteResults)
|
||||
.DoRunAsync(rmdb, true, false, backendManager).ConfigureAwait(false);
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
await rmdb.Transaction
|
||||
.CommitAsync("CommitDelete", true, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
pgoffset += (pgspan * fully_emptied.Length);
|
||||
m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
|
||||
}
|
||||
|
||||
if (to_purge.Length > 0)
|
||||
{
|
||||
m_result.PurgeResults = new PurgeFilesResults(m_result);
|
||||
|
||||
foreach (var bs in to_purge)
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "PurgingFiles", "Purging {0} file(s) from fileset {1}", bs.RemoveCount, bs.Timestamp.ToLocalTime());
|
||||
var opts = new Options(new Dictionary<string, string?>(m_options.RawOptions));
|
||||
|
||||
await using (var pgdb = await Database.LocalPurgeDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
// Recompute the version number after we deleted the versions before
|
||||
filesets = await pgdb
|
||||
.FilesetTimes(m_result.TaskControl.ProgressToken)
|
||||
.ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
var thisversion = filesets.FindIndex(y => y.Key == bs.FilesetID);
|
||||
if (thisversion < 0)
|
||||
throw new Exception(string.Format("Failed to find match for {0} ({1}) in {2}", bs.FilesetID, bs.Timestamp.ToLocalTime(), string.Join(", ", filesets.Select(x => x.ToString()))));
|
||||
|
||||
opts.RawOptions["version"] = thisversion.ToString();
|
||||
opts.RawOptions.Remove("time");
|
||||
opts.RawOptions["no-auto-compact"] = "true";
|
||||
|
||||
await new PurgeFilesHandler(opts, (PurgeFilesResults)m_result.PurgeResults).RunAsync(backendManager, pgdb, pgoffset, pgspan, async (cmd, filesetid, tablename) =>
|
||||
{
|
||||
if (filesetid != bs.FilesetID)
|
||||
throw new Exception(string.Format("Unexpected filesetid: {0}, expected {1}", filesetid, bs.FilesetID));
|
||||
|
||||
// Update entries that would be removed because of missing metadata
|
||||
var updatedEntries = 0;
|
||||
if (!m_options.DisableReplaceMissingMetadata)
|
||||
updatedEntries = await db.ReplaceMetadata(filesetid, replacementMetadataBlocksetId, m_result.TaskControl.ProgressToken);
|
||||
|
||||
await db.InsertBrokenFileIDsIntoTable(filesetid, tablename, "FileID", m_result.TaskControl.ProgressToken);
|
||||
return updatedEntries;
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
pgoffset += pgspan;
|
||||
m_result.OperationProgressUpdater.UpdateProgress(pgoffset);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m_result.OperationProgressUpdater.UpdateProgress(0.95f);
|
||||
|
||||
if (!m_options.Dryrun && await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "ValidatingDatabase", "Database was previously marked as in-progress, checking if it is valid after purging files");
|
||||
await db
|
||||
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "UpdatingDatabase", "Purge completed, and consistency checks completed, marking database as complete");
|
||||
await db.RepairInProgress(m_result.TaskControl.ProgressToken, false).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await db.Transaction.RollBackAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await db
|
||||
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
m_result.OperationProgressUpdater.UpdateProgress(1.0f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -25,6 +25,7 @@ using System.Threading.Tasks;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Main.Volumes;
|
||||
using Duplicati.Library.Utility;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
@@ -51,51 +52,62 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (!System.IO.File.Exists(m_options.Dbpath))
|
||||
throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
|
||||
|
||||
using (var db = new Database.LocalPurgeDatabase(m_options.Dbpath, m_options.SqlitePageCache))
|
||||
{
|
||||
await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false);
|
||||
db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, null);
|
||||
}
|
||||
await using var db = await Database.LocalPurgeDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await DoRunAsync(backendManager, db, filter, null, 0, 1).ConfigureAwait(false);
|
||||
await db
|
||||
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, true, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Func<System.Data.IDbCommand, long, string, int> filtercommand)
|
||||
public Task RunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, float pgoffset, float pgspan, Func<SqliteCommand, long, string, Task<int>> filtercommand)
|
||||
=> DoRunAsync(backendManager, db, null, filtercommand, pgoffset, pgspan);
|
||||
|
||||
private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Func<System.Data.IDbCommand, long, string, int> filtercommand, float pgoffset, float pgspan)
|
||||
private async Task DoRunAsync(IBackendManager backendManager, Database.LocalPurgeDatabase db, IFilter filter, Func<SqliteCommand, long, string, Task<int>> filtercommand, float pgoffset, float pgspan)
|
||||
{
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Begin);
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "StartingPurge", "Starting purge operation");
|
||||
|
||||
var doCompactStep = !m_options.NoAutoCompact && filtercommand == null;
|
||||
|
||||
if (db.PartiallyRecreated)
|
||||
if (await db.PartiallyRecreated(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
throw new UserInformationException("The purge command does not work on partially recreated databases", "PurgeNotAllowedOnPartialDatabase");
|
||||
|
||||
if (db.RepairInProgress && filtercommand == null)
|
||||
if (await db.RepairInProgress(m_result.TaskControl.ProgressToken).ConfigureAwait(false) && filtercommand == null)
|
||||
throw new UserInformationException(string.Format("The purge command does not work on an incomplete database, try the {0} operation.", "purge-broken-files"), "PurgeNotAllowedOnIncompleteDatabase");
|
||||
|
||||
var versions = db.GetFilesetIDs(m_options.Time, m_options.Version).OrderByDescending(x => x).ToArray();
|
||||
var versions = await db
|
||||
.GetFilesetIDs(m_options.Time, m_options.Version, false, m_result.TaskControl.ProgressToken)
|
||||
.OrderByDescending(x => x)
|
||||
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
if (versions.Length <= 0)
|
||||
throw new UserInformationException("No filesets matched the supplied time or versions", "NoFilesetFoundForTimeOrVersion");
|
||||
|
||||
var orphans = db.CountOrphanFiles(null);
|
||||
var orphans = await db.CountOrphanFiles(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
if (orphans != 0)
|
||||
throw new UserInformationException(string.Format("Unable to start the purge process as there are {0} orphan file(s)", orphans), "CannotPurgeWithOrphans");
|
||||
|
||||
Utility.UpdateOptionsFromDb(db, m_options);
|
||||
Utility.VerifyOptionsAndUpdateDatabase(db, m_options);
|
||||
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (filtercommand == null)
|
||||
{
|
||||
db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, null);
|
||||
await db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, false, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (m_options.NoBackendverification)
|
||||
await FilelistProcessor.VerifyLocalList(backendManager, db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await FilelistProcessor.VerifyLocalList(backendManager, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
else
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, null, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict).ConfigureAwait(false);
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, null, null, logErrors: true, verifyMode: FilelistProcessor.VerifyMode.VerifyStrict, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var filesets = db.FilesetTimes.OrderByDescending(x => x.Value).ToArray();
|
||||
var filesets = await db
|
||||
.FilesetTimes(m_result.TaskControl.ProgressToken)
|
||||
.OrderByDescending(x => x.Value)
|
||||
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var versionprogress = ((doCompactStep ? 0.75f : 1.0f) / versions.Length) * pgspan;
|
||||
var currentprogress = pgoffset;
|
||||
@@ -106,7 +118,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
|
||||
// If we crash now, it is possible that the remote storage contains partial files
|
||||
if (!m_options.Dryrun)
|
||||
db.TerminatedWithActiveUploads = true;
|
||||
await db.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, true).ConfigureAwait(false);
|
||||
|
||||
// Reverse makes sure we re-write the old versions first
|
||||
foreach (var versionid in versions.Reverse())
|
||||
@@ -114,94 +126,120 @@ namespace Duplicati.Library.Main.Operation
|
||||
progress++;
|
||||
Logging.Log.WriteVerboseMessage(LOGTAG, "ProcessingFilelistVolumes", "Processing filelist volume {0} of {1}", progress, versions.Length);
|
||||
|
||||
using (var tr = db.BeginTransaction())
|
||||
(var _, var tsOriginal, var ix) = filesets.Select((x, i) => (x.Key, x.Value, i)).FirstOrDefault(x => x.Key == versionid);
|
||||
if (ix < 0 || tsOriginal.Ticks == 0)
|
||||
throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid));
|
||||
|
||||
var ts = await FilesetVolumeWriter.ProbeUnusedFilenameName(db, m_options, tsOriginal, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
var prevfilename = await db.GetRemoteVolumeNameForFileset(filesets[ix].Key, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (ix != 0 && filesets[ix - 1].Value <= ts)
|
||||
throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value));
|
||||
|
||||
await using (var tempset = await db.CreateTemporaryFileset(versionid, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
(var _, var tsOriginal, var ix) = filesets.Select((x, i) => (x.Key, x.Value, i)).FirstOrDefault(x => x.Key == versionid);
|
||||
if (ix < 0 || tsOriginal.Ticks == 0)
|
||||
throw new InvalidProgramException(string.Format("Fileset was reported with id {0}, but could not be found?", versionid));
|
||||
if (filtercommand == null)
|
||||
await tempset.ApplyFilter(filter, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
else
|
||||
await tempset.ApplyFilter(filtercommand, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var ts = FilesetVolumeWriter.ProbeUnusedFilenameName(db, tr, m_options, tsOriginal);
|
||||
var prevfilename = db.GetRemoteVolumeNameForFileset(filesets[ix].Key, tr);
|
||||
|
||||
if (ix != 0 && filesets[ix - 1].Value <= ts)
|
||||
throw new Exception(string.Format("Unable to create a new fileset for {0} because the resulting timestamp {1} is larger than the next timestamp {2}", prevfilename, ts, filesets[ix - 1].Value));
|
||||
|
||||
using (var tempset = db.CreateTemporaryFileset(versionid, tr))
|
||||
if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0)
|
||||
{
|
||||
if (filtercommand == null)
|
||||
tempset.ApplyFilter(filter);
|
||||
else
|
||||
tempset.ApplyFilter(filtercommand);
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename);
|
||||
currentprogress += versionprogress;
|
||||
await db.Transaction
|
||||
.RollBackAsync()
|
||||
.ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
using (var tf = new TempFile())
|
||||
using (var vol = new FilesetVolumeWriter(m_options, ts))
|
||||
{
|
||||
var isOriginalFilesetFullBackup = await db
|
||||
.IsFilesetFullBackup(tsOriginal, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
var newids = await tempset
|
||||
.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
vol.VolumeID = newids.Item1;
|
||||
vol.CreateFilesetFile(isOriginalFilesetFullBackup);
|
||||
|
||||
if (tempset.RemovedFileCount + tempset.UpdatedFileCount == 0)
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "NotWritingNewFileset", "Not writing a new fileset for {0} as it was not changed", prevfilename);
|
||||
currentprogress += versionprogress;
|
||||
tr.Rollback();
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
using (var tf = new TempFile())
|
||||
using (var vol = new FilesetVolumeWriter(m_options, ts))
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize));
|
||||
|
||||
await db
|
||||
.WriteFileset(vol, newids.Item2, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
m_result.RemovedFileSize += tempset.RemovedFileSize;
|
||||
m_result.RemovedFileCount += tempset.RemovedFileCount;
|
||||
m_result.UpdatedFileCount += tempset.UpdatedFileCount;
|
||||
m_result.RewrittenFileLists++;
|
||||
|
||||
currentprogress += (versionprogress / 2);
|
||||
m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
|
||||
|
||||
if (m_options.Dryrun || m_options.FullResult)
|
||||
{
|
||||
var isOriginalFilesetFullBackup = db.IsFilesetFullBackup(tsOriginal, tr);
|
||||
var newids = tempset.ConvertToPermanentFileset(vol.RemoteFilename, ts, isOriginalFilesetFullBackup);
|
||||
vol.VolumeID = newids.Item1;
|
||||
vol.CreateFilesetFile(isOriginalFilesetFullBackup);
|
||||
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "ReplacingFileset", "Replacing fileset {0} with {1} which has with {2} fewer file(s) ({3} reduction)", prevfilename, vol.RemoteFilename, tempset.RemovedFileCount, Library.Utility.Utility.FormatSizeString(tempset.RemovedFileSize));
|
||||
|
||||
db.WriteFileset(vol, newids.Item2, tr);
|
||||
|
||||
m_result.RemovedFileSize += tempset.RemovedFileSize;
|
||||
m_result.RemovedFileCount += tempset.RemovedFileCount;
|
||||
m_result.UpdatedFileCount += tempset.UpdatedFileCount;
|
||||
m_result.RewrittenFileLists++;
|
||||
|
||||
currentprogress += (versionprogress / 2);
|
||||
m_result.OperationProgressUpdater.UpdateProgress(currentprogress);
|
||||
|
||||
if (m_options.Dryrun || m_options.FullResult)
|
||||
await foreach (var fe in tempset.ListAllDeletedFiles(m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var fe in tempset.ListAllDeletedFiles())
|
||||
{
|
||||
var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value));
|
||||
var msg = string.Format(" Purging file {0} ({1})", fe.Key, Library.Utility.Utility.FormatSizeString(fe.Value));
|
||||
|
||||
Logging.Log.WriteProfilingMessage(LOGTAG, "PurgeFile", msg);
|
||||
Logging.Log.WriteVerboseMessage(LOGTAG, "PurgeFile", msg);
|
||||
|
||||
if (m_options.Dryrun)
|
||||
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPurgeFile", msg);
|
||||
}
|
||||
Logging.Log.WriteProfilingMessage(LOGTAG, "PurgeFile", msg);
|
||||
Logging.Log.WriteVerboseMessage(LOGTAG, "PurgeFile", msg);
|
||||
|
||||
if (m_options.Dryrun)
|
||||
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldWriteRemoteFiles", "Would write files to remote storage");
|
||||
|
||||
Logging.Log.WriteVerboseMessage(LOGTAG, "WritingRemoteFiles", "Writing files to remote storage");
|
||||
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldPurgeFile", msg);
|
||||
}
|
||||
|
||||
if (m_options.Dryrun)
|
||||
{
|
||||
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount);
|
||||
tr.Rollback();
|
||||
}
|
||||
else
|
||||
{
|
||||
db.UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, tr);
|
||||
var lst = db.DropFilesetsFromTable(new[] { tsOriginal }, tr).ToArray();
|
||||
foreach (var f in lst)
|
||||
db.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, tr);
|
||||
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldWriteRemoteFiles", "Would write files to remote storage");
|
||||
|
||||
tr.Commit();
|
||||
Logging.Log.WriteVerboseMessage(LOGTAG, "WritingRemoteFiles", "Writing files to remote storage");
|
||||
}
|
||||
|
||||
await backendManager.PutAsync(vol, null, null, true, async () =>
|
||||
{
|
||||
await backendManager.FlushPendingMessagesAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
if (m_options.Dryrun)
|
||||
{
|
||||
Logging.Log.WriteDryrunMessage(LOGTAG, "WouldUploadAndDelete", "Would upload file {0} ({1}) and delete file {2}, removing {3} files", vol.RemoteFilename, Library.Utility.Utility.FormatSizeString(vol.Filesize), prevfilename, tempset.RemovedFileCount);
|
||||
await db.Transaction
|
||||
.RollBackAsync()
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
{
|
||||
await db
|
||||
.UpdateRemoteVolume(vol.RemoteFilename, RemoteVolumeState.Uploading, -1, null, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
var lst = await db
|
||||
.DropFilesetsFromTable(new[] { tsOriginal }, m_result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
foreach (var f in lst)
|
||||
await db
|
||||
.UpdateRemoteVolume(f.Key, RemoteVolumeState.Deleting, f.Value, null, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await db.Transaction
|
||||
.CommitAsync(m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await backendManager.PutAsync(vol, null, null, true, async () =>
|
||||
{
|
||||
await backendManager.FlushPendingMessagesAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
// This commit mimics the pre-
|
||||
// Microsoft.Data.Sqlite backend way of not
|
||||
// passing down a transaction, but maybe is
|
||||
// not needed anymore.
|
||||
await db.Transaction
|
||||
.CommitAsync(m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.DeleteAsync(prevfilename, -1, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -212,7 +250,9 @@ namespace Duplicati.Library.Main.Operation
|
||||
}
|
||||
|
||||
if (!m_options.Dryrun)
|
||||
db.TerminatedWithActiveUploads = false;
|
||||
await db
|
||||
.TerminatedWithActiveUploads(m_result.TaskControl.ProgressToken, false)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (doCompactStep)
|
||||
{
|
||||
@@ -225,21 +265,18 @@ namespace Duplicati.Library.Main.Operation
|
||||
m_result.OperationProgressUpdater.UpdateProgress(pgoffset + (0.75f * pgspan));
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Compact);
|
||||
m_result.CompactResults = new CompactResults(m_result);
|
||||
using (var cdb = new Database.LocalDeleteDatabase(db))
|
||||
using (var ctr = new Database.ReusableTransaction(cdb))
|
||||
{
|
||||
await new CompactHandler(m_options, (CompactResults)m_result.CompactResults)
|
||||
.DoCompactAsync(cdb, true, ctr, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
await using var cdb = await Database.LocalDeleteDatabase.CreateAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await new CompactHandler(m_options, (CompactResults)m_result.CompactResults)
|
||||
.DoCompactAsync(cdb, true, backendManager)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
ctr.Commit("PostCompact", restart: false);
|
||||
}
|
||||
await cdb.Transaction.CommitAsync("PostCompact", true, m_result.TaskControl.ProgressToken);
|
||||
}
|
||||
|
||||
m_result.OperationProgressUpdater.UpdateProgress(pgoffset + pgspan);
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.PurgeFiles_Complete);
|
||||
|
||||
await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -131,10 +131,9 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="SleepableDictionary"/> class.
|
||||
/// </summary>
|
||||
/// <param name="db">The database holding information about how many of each block this restore requires.</param>
|
||||
/// <param name="volume_request">Channel for submitting block requests from a volume.</param>
|
||||
/// <param name="readers">Number of readers accessing this dictionary. Used during shutdown / cleanup.</param>
|
||||
public SleepableDictionary(LocalRestoreDatabase db, IWriteChannel<object> volume_request, Options options, int readers)
|
||||
private SleepableDictionary(IWriteChannel<object> volume_request, Options options, int readers)
|
||||
{
|
||||
m_options = options;
|
||||
m_volume_request = volume_request;
|
||||
@@ -171,14 +170,31 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
sw_set_set = options.InternalProfiling ? new() : null;
|
||||
sw_set_wake_get = options.InternalProfiling ? new() : null;
|
||||
sw_set_wake_set = options.InternalProfiling ? new() : null;
|
||||
}
|
||||
|
||||
foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata))
|
||||
/// <summary>
|
||||
/// Asynchronously creates a new instance of the <see cref="SleepableDictionary"/> class.
|
||||
/// This method initializes the block and volume counts based on the data in the database.
|
||||
/// </summary>
|
||||
/// <param name="db">The database holding information about how many of each block this restore requires.</param>
|
||||
/// <param name="volume_request">CoCoL channel for submitting block requests from a volume.</param>
|
||||
/// <param name="options">The restore options.</param>
|
||||
/// <param name="readers">The number of readers accessing this dictionary. Used during shutdown / cleanup.</param>
|
||||
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited returns a new instance of the <see cref="SleepableDictionary"/> class.</returns>
|
||||
public static async Task<SleepableDictionary> CreateAsync(LocalRestoreDatabase db, IWriteChannel<object> volume_request, Options options, int readers, CancellationToken cancellationToken)
|
||||
{
|
||||
var sd = new SleepableDictionary(volume_request, options, readers);
|
||||
|
||||
await foreach (var (block_id, volume_id) in db.GetBlocksAndVolumeIDs(options.SkipMetadata, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
var bc = m_blockcount.TryGetValue(block_id, out var c);
|
||||
m_blockcount[block_id] = bc ? c + 1 : 1;
|
||||
var vc = m_volumecount.TryGetValue(volume_id, out var v);
|
||||
m_volumecount[volume_id] = vc ? v + 1 : 1;
|
||||
var bc = sd.m_blockcount.TryGetValue(block_id, out var c);
|
||||
sd.m_blockcount[block_id] = bc ? c + 1 : 1;
|
||||
var vc = sd.m_volumecount.TryGetValue(volume_id, out var v);
|
||||
sd.m_volumecount[volume_id] = vc ? v + 1 : 1;
|
||||
}
|
||||
|
||||
return sd;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -303,7 +319,7 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
sw_get_wait?.Stop();
|
||||
}
|
||||
|
||||
return await new_tcs.Task;
|
||||
return await new_tcs.Task.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -421,10 +437,11 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
/// </summary>
|
||||
/// <param name="channels">The named channels for the restore operation.</param>
|
||||
/// <param name="db">The database holding information about how many of each block this restore requires.</param>
|
||||
/// <param name="options">The restore options.</param>
|
||||
/// <param name="fp_requests">The channels for reading block requests from the `FileProcessor`.</param>
|
||||
/// <param name="fp_responses">The channels for writing block responses back to the `FileProcessor`.</param>
|
||||
public static Task Run(Channels channels, LocalRestoreDatabase db, Options options, IChannel<BlockRequest>[] fp_requests, IChannel<Task<byte[]>>[] fp_responses)
|
||||
/// <param name="options">The restore options.</param>
|
||||
/// <param name="results">The results of the restore operation.</param>
|
||||
public static Task Run(Channels channels, LocalRestoreDatabase db, IChannel<BlockRequest>[] fp_requests, IChannel<Task<byte[]>>[] fp_responses, Options options, RestoreResults results)
|
||||
{
|
||||
return AutomationExtensions.RunTask(
|
||||
new
|
||||
@@ -435,7 +452,9 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
async self =>
|
||||
{
|
||||
// Create a cache for the blocks,
|
||||
using SleepableDictionary cache = new(db, self.Output, options, fp_requests.Length);
|
||||
using SleepableDictionary cache =
|
||||
await SleepableDictionary.CreateAsync(db, self.Output, options, fp_requests.Length, results.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// The volume consumer will read blocks from the input channel (data blocks from the volumes) and store them in the cache.
|
||||
var volume_consumer = Task.Run(async () =>
|
||||
|
||||
@@ -66,7 +66,12 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
try
|
||||
{
|
||||
sw_get_files?.Start();
|
||||
var files = db.GetFilesAndSymlinksToRestore().ToArray();
|
||||
// The enumerables are cast to arrays to force the query to be executed and release the database lock.
|
||||
var files = await db
|
||||
.GetFilesAndSymlinksToRestore(result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false);
|
||||
|
||||
result.OperationProgressUpdater.UpdatePhase(OperationPhase.Restore_DownloadingRemoteFiles);
|
||||
sw_get_files?.Stop();
|
||||
|
||||
@@ -78,7 +83,11 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
if (!options.SkipMetadata)
|
||||
{
|
||||
sw_get_folders?.Start();
|
||||
var folders = db.GetFolderMetadataToRestore().ToArray();
|
||||
// The enumerables are cast to arrays to force the query to be executed and release the database lock.
|
||||
var folders = await db
|
||||
.GetFolderMetadataToRestore(result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false);
|
||||
sw_get_folders?.Stop();
|
||||
|
||||
sw_write_folder?.Start();
|
||||
|
||||
@@ -53,6 +53,8 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
/// The number of file processors that are still restoring files. It is set by the <see cref="RestoreHandler"/>.
|
||||
/// </summary>
|
||||
public static int file_processors_restoring_files;
|
||||
public static object file_processor_continue_lock = new object();
|
||||
public static TaskCompletionSource file_processor_continue = new();
|
||||
|
||||
/// <summary>
|
||||
/// Runs the file processor process that restores the files that need to be restored.
|
||||
@@ -110,11 +112,16 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
// Check if there are other FileProcessor's still restoring files
|
||||
if (!decremented)
|
||||
{
|
||||
await RendezvousBeforeProcessingFolderMetadata();
|
||||
if (file_processors_restoring_files <= 0 && !file_processor_continue.Task.IsCompleted)
|
||||
file_processor_continue.SetResult();
|
||||
else
|
||||
await RendezvousBeforeProcessingFolderMetadata()
|
||||
.ConfigureAwait(false);
|
||||
decremented = true;
|
||||
}
|
||||
|
||||
await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp);
|
||||
await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp, results.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
continue;
|
||||
}
|
||||
@@ -122,7 +129,10 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
// Get information about the blocks for the file
|
||||
// TODO rather than keeping all of the blocks in memory, we could do a single pass over the blocks using a cursor, only keeping the relevant block requests in memory. Maybe even only a single block request at a time.
|
||||
sw_block?.Start();
|
||||
var blocks = db.GetBlocksFromFile(file.BlocksetID).ToArray();
|
||||
var blocks = await db
|
||||
.GetBlocksFromFile(file.BlocksetID, results.TaskControl.ProgressToken)
|
||||
.ToArrayAsync()
|
||||
.ConfigureAwait(false);
|
||||
sw_block?.Stop();
|
||||
|
||||
sw_work_verify_target?.Start();
|
||||
@@ -362,12 +372,17 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
else
|
||||
{
|
||||
// Not a missing block, so read from the file
|
||||
sw_work_read?.Start();
|
||||
var read = await fs?.ReadAsync(buffer, 0, buffer.Length);
|
||||
sw_work_read?.Stop();
|
||||
sw_work_hash?.Start();
|
||||
filehasher.TransformBlock(buffer, 0, read, buffer, 0);
|
||||
sw_work_hash?.Stop();
|
||||
if (fs != null)
|
||||
{
|
||||
sw_work_read?.Start();
|
||||
var read = await fs
|
||||
.ReadAsync(buffer, 0, buffer.Length)
|
||||
.ConfigureAwait(false);
|
||||
sw_work_read?.Stop();
|
||||
sw_work_hash?.Start();
|
||||
filehasher.TransformBlock(buffer, 0, read, buffer, 0);
|
||||
sw_work_hash?.Stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -420,7 +435,7 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
|
||||
if (!options.SkipMetadata)
|
||||
{
|
||||
empty_file_or_symlink |= await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp).ConfigureAwait(false);
|
||||
empty_file_or_symlink |= await RestoreMetadata(db, file, block_request, block_response, options, sw_meta, sw_work_meta, sw_req, sw_resp, results.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
sw_work_meta?.Stop();
|
||||
}
|
||||
|
||||
@@ -459,7 +474,12 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
finally
|
||||
{
|
||||
if (!decremented)
|
||||
Interlocked.Decrement(ref file_processors_restoring_files);
|
||||
lock (file_processor_continue_lock)
|
||||
{
|
||||
file_processors_restoring_files--;
|
||||
if (file_processors_restoring_files <= 0 && !file_processor_continue.Task.IsCompleted)
|
||||
file_processor_continue.SetResult();
|
||||
}
|
||||
|
||||
block_request.Retire();
|
||||
block_response.Retire();
|
||||
@@ -554,11 +574,18 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
/// <returns>An awaitable task that completes once all of the FileProcessor's have rendezvoused.</returns>
|
||||
private static async Task RendezvousBeforeProcessingFolderMetadata()
|
||||
{
|
||||
Interlocked.Decrement(ref file_processors_restoring_files);
|
||||
|
||||
while (file_processors_restoring_files > 0)
|
||||
var should_wait = false;
|
||||
lock (file_processor_continue_lock)
|
||||
{
|
||||
await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding);
|
||||
file_processors_restoring_files--;
|
||||
if (file_processors_restoring_files <= 0 && !file_processor_continue.Task.IsCompleted)
|
||||
file_processor_continue.SetResult();
|
||||
should_wait = file_processors_restoring_files > 0;
|
||||
}
|
||||
|
||||
if (should_wait)
|
||||
{
|
||||
await file_processor_continue.Task.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -574,15 +601,22 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
/// <param name="sw_work">The stopwatch for internal profiling of the general processing.</param>
|
||||
/// <param name="sw_req">The stopwatch for internal profiling of the block requests.</param>
|
||||
/// <param name="sw_resp">The stopwatch for internal profiling of the block responses.</param>
|
||||
private static async Task<bool> RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel<BlockRequest> block_request, IChannel<Task<byte[]>> block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp)
|
||||
/// <param name="cancellationToken">The cancellation token to cancel the operation.</param>
|
||||
/// <returns>An awaitable `Task`, which returns `true` if the metadata was restored successfully, `false` otherwise.</returns>
|
||||
private static async Task<bool> RestoreMetadata(LocalRestoreDatabase db, FileRequest file, IChannel<BlockRequest> block_request, IChannel<Task<byte[]>> block_response, Options options, Stopwatch sw_meta, Stopwatch sw_work, Stopwatch sw_req, Stopwatch sw_resp, CancellationToken cancellationToken)
|
||||
{
|
||||
sw_meta?.Start();
|
||||
var blocks = db.GetMetadataBlocksFromFile(file.ID);
|
||||
// Since each FileProcessor should have its own connection, it's ok
|
||||
// to keep the lock associated with the read transaction for a long
|
||||
// time, as the other processes should still be able to read.
|
||||
// Therefore, we don't need to read the entire result set into
|
||||
// memory.
|
||||
var blocks = db.GetMetadataBlocksFromFile(file.ID, cancellationToken);
|
||||
sw_meta?.Stop();
|
||||
|
||||
using var ms = new MemoryStream();
|
||||
|
||||
foreach (var block in blocks)
|
||||
await foreach (var block in blocks.ConfigureAwait(false))
|
||||
{
|
||||
sw_work?.Stop();
|
||||
sw_req?.Start();
|
||||
@@ -768,7 +802,7 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
int j = 0;
|
||||
for (long i = 0; i < total_blocks; i++)
|
||||
{
|
||||
int read;
|
||||
int read = 0;
|
||||
if (j < blocks.Count && blocks[j].BlockOffset == i)
|
||||
{
|
||||
// The current block is a missing block
|
||||
@@ -797,18 +831,23 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
{
|
||||
if (!options.Dryrun)
|
||||
{
|
||||
try
|
||||
if (f_target != null)
|
||||
{
|
||||
f_target?.Seek(blocks[j].BlockOffset * options.Blocksize, SeekOrigin.Begin);
|
||||
await f_target?.WriteAsync(buffer, 0, read);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
lock (results)
|
||||
try
|
||||
{
|
||||
results.BrokenLocalFiles.Add(file.TargetPath);
|
||||
f_target.Seek(blocks[j].BlockOffset * options.Blocksize, SeekOrigin.Begin);
|
||||
await f_target
|
||||
.WriteAsync(buffer, 0, read)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
lock (results)
|
||||
{
|
||||
results.BrokenLocalFiles.Add(file.TargetPath);
|
||||
}
|
||||
throw;
|
||||
}
|
||||
throw;
|
||||
}
|
||||
}
|
||||
bytes_read += read;
|
||||
@@ -841,18 +880,23 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
else
|
||||
{
|
||||
// The current block is not a missing block - read from the target file.
|
||||
try
|
||||
if (f_target != null)
|
||||
{
|
||||
f_target?.Seek(i * options.Blocksize, SeekOrigin.Begin);
|
||||
read = await f_target?.ReadAsync(buffer, 0, options.Blocksize);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
lock (results)
|
||||
try
|
||||
{
|
||||
results.BrokenLocalFiles.Add(file.TargetPath);
|
||||
f_target.Seek(i * options.Blocksize, SeekOrigin.Begin);
|
||||
read = await f_target
|
||||
.ReadAsync(buffer, 0, options.Blocksize)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
lock (results)
|
||||
{
|
||||
results.BrokenLocalFiles.Add(file.TargetPath);
|
||||
}
|
||||
throw;
|
||||
}
|
||||
throw;
|
||||
}
|
||||
}
|
||||
filehasher.TransformBlock(buffer, 0, read, buffer, 0);
|
||||
|
||||
@@ -75,10 +75,13 @@ namespace Duplicati.Library.Main.Operation.Restore
|
||||
// Trigger the download.
|
||||
sw_wait?.Start();
|
||||
TempFile f;
|
||||
var (volume_name, size, hash) = db.GetVolumeInfo(volume_id).First();
|
||||
var (volume_name, size, hash) = await db
|
||||
.GetVolumeInfo(volume_id, results.TaskControl.ProgressToken)
|
||||
.FirstAsync()
|
||||
.ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
f = await backend.GetDirectAsync(volume_name, hash, size, CancellationToken.None).ConfigureAwait(false);
|
||||
f = await backend.GetDirectAsync(volume_name, hash, size, results.TaskControl.TransferToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
@@ -45,62 +45,68 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (!Directory.Exists(m_options.Restorepath))
|
||||
Directory.CreateDirectory(m_options.Restorepath);
|
||||
|
||||
using (var tmpdb = new TempFile())
|
||||
using (var db = new Database.LocalDatabase(File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles", true, m_options.SqlitePageCache))
|
||||
using var tmpdb = new TempFile();
|
||||
await using var db = await Database.LocalDatabase.CreateLocalDatabaseAsync(
|
||||
File.Exists(m_options.Dbpath) ? m_options.Dbpath : (string)tmpdb, "RestoreControlFiles",
|
||||
true,
|
||||
null,
|
||||
m_result.TaskControl.ProgressToken
|
||||
).ConfigureAwait(false);
|
||||
|
||||
var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter);
|
||||
|
||||
try
|
||||
{
|
||||
var filter = JoinedFilterExpression.Join(new FilterExpression(filterstrings), compositefilter);
|
||||
var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false), m_options);
|
||||
if (filteredList.Count == 0)
|
||||
throw new Exception("No filesets found on remote target");
|
||||
|
||||
try
|
||||
{
|
||||
var filteredList = ListFilesHandler.ParseAndFilterFilesets(await backendManager.ListAsync(m_result.TaskControl.ProgressToken).ConfigureAwait(false), m_options);
|
||||
if (filteredList.Count == 0)
|
||||
throw new Exception("No filesets found on remote target");
|
||||
var lastEx = new Exception("No suitable files found on remote target");
|
||||
|
||||
var lastEx = new Exception("No suitable files found on remote target");
|
||||
|
||||
foreach (var fileversion in filteredList)
|
||||
try
|
||||
foreach (var fileversion in filteredList)
|
||||
try
|
||||
{
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var file = fileversion.Value.File;
|
||||
var entry = db.GetRemoteVolume(file.Name);
|
||||
|
||||
var res = new List<string>();
|
||||
using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options))
|
||||
foreach (var cf in tmp.ControlFiles)
|
||||
if (FilterExpression.Matches(filter, cf.Key))
|
||||
{
|
||||
var targetpath = Path.Combine(m_options.Restorepath, cf.Key);
|
||||
using (var ts = File.Create(targetpath))
|
||||
await Library.Utility.Utility.CopyStreamAsync(cf.Value, ts, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
res.Add(targetpath);
|
||||
}
|
||||
|
||||
m_result.SetResult(res);
|
||||
|
||||
lastEx = null;
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
lastEx = ex;
|
||||
if (ex.IsAbortException())
|
||||
throw;
|
||||
await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
if (lastEx != null)
|
||||
throw lastEx;
|
||||
}
|
||||
finally
|
||||
{
|
||||
await backendManager.WaitForEmptyAsync(db, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
var file = fileversion.Value.File;
|
||||
var entry = await db
|
||||
.GetRemoteVolume(file.Name, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var res = new List<string>();
|
||||
using (var tmpfile = await backendManager.GetAsync(file.Name, entry.Hash, entry.Size < 0 ? file.Size : entry.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
using (var tmp = new Volumes.FilesetVolumeReader(RestoreHandler.GetCompressionModule(file.Name), tmpfile, m_options))
|
||||
foreach (var cf in tmp.ControlFiles)
|
||||
if (FilterExpression.Matches(filter, cf.Key))
|
||||
{
|
||||
var targetpath = Path.Combine(m_options.Restorepath, cf.Key);
|
||||
using (var ts = File.Create(targetpath))
|
||||
await Library.Utility.Utility.CopyStreamAsync(cf.Value, ts, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
res.Add(targetpath);
|
||||
}
|
||||
|
||||
m_result.SetResult(res);
|
||||
|
||||
lastEx = null;
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
lastEx = ex;
|
||||
if (ex.IsAbortException())
|
||||
throw;
|
||||
}
|
||||
|
||||
if (lastEx != null)
|
||||
throw lastEx;
|
||||
}
|
||||
finally
|
||||
{
|
||||
await backendManager.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
@@ -44,16 +44,22 @@ internal static class SearchEntriesHandler
|
||||
/// <param name="offset">The offset to start searching from</param>
|
||||
/// <param name="limit">The maximum number of results to return</param>
|
||||
/// <returns>A task representing the asynchronous operation</returns>
|
||||
public static Task RunAsync(Options options, SearchFilesResults result, string[] paths, IFilter filter, long offset, long limit)
|
||||
public static async Task RunAsync(Options options, SearchFilesResults result, string[] paths, IFilter filter, long offset, long limit)
|
||||
{
|
||||
if (!System.IO.File.Exists(options.Dbpath))
|
||||
throw new UserInformationException("No local database found, this operation requires a local database", "NoLocalDatabase");
|
||||
|
||||
using var db = new Database.LocalListDatabase(options.Dbpath, options.SqlitePageCache);
|
||||
await using var db =
|
||||
await Database.LocalListDatabase.CreateAsync(options.Dbpath, null, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
long[]? filesetIds = null;
|
||||
if (!options.AllVersions)
|
||||
{
|
||||
filesetIds = db.GetFilesetIDs(options.Time, options.Version).ToArray();
|
||||
filesetIds = await db
|
||||
.GetFilesetIDs(options.Time, options.Version, false, result.TaskControl.ProgressToken)
|
||||
.ToArrayAsync(cancellationToken: result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (filesetIds.Length == 0)
|
||||
throw new UserInformationException("No filesets found", "NoFilesetsFound");
|
||||
}
|
||||
@@ -61,7 +67,8 @@ internal static class SearchEntriesHandler
|
||||
if (paths != null)
|
||||
paths = paths.Select(path => Util.AppendDirSeparator(path)).ToArray();
|
||||
|
||||
result.FileVersions = db.SearchEntries(paths, filter, filesetIds, offset, limit);
|
||||
return Task.CompletedTask;
|
||||
result.FileVersions = await db
|
||||
.SearchEntries(paths, filter, filesetIds, offset, limit, result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -40,12 +40,12 @@ namespace Duplicati.Library.Main.Operation
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType<TestHandler>();
|
||||
|
||||
private readonly Options m_options;
|
||||
private readonly TestResults m_results;
|
||||
private readonly TestResults m_result;
|
||||
|
||||
public TestHandler(Options options, TestResults results)
|
||||
{
|
||||
m_options = options;
|
||||
m_results = results;
|
||||
m_result = results;
|
||||
}
|
||||
|
||||
public async Task RunAsync(long samples, IBackendManager backendManager)
|
||||
@@ -53,47 +53,56 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (!System.IO.File.Exists(m_options.Dbpath))
|
||||
throw new UserInformationException(LC.L("Database file does not exist: {0}", m_options.Dbpath), "DatabaseDoesNotExist");
|
||||
|
||||
using (var db = new LocalTestDatabase(m_options.Dbpath, m_options.SqlitePageCache))
|
||||
using (var rtr = new ReusableTransaction(db))
|
||||
{
|
||||
Utility.UpdateOptionsFromDb(db, m_options, rtr.Transaction);
|
||||
Utility.VerifyOptionsAndUpdateDatabase(db, m_options, rtr.Transaction);
|
||||
db.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, rtr.Transaction);
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_results.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, rtr.Transaction).ConfigureAwait(false);
|
||||
await DoRunAsync(samples, db, rtr, backendManager).ConfigureAwait(false);
|
||||
rtr.Commit("TestHandlerComplete");
|
||||
}
|
||||
await using var db = await LocalTestDatabase.CreateAsync(m_options.Dbpath, null, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await Utility.UpdateOptionsFromDb(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
await Utility.VerifyOptionsAndUpdateDatabase(db, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await db
|
||||
.VerifyConsistency(m_options.Blocksize, m_options.BlockhashSize, !m_options.DisableFilelistConsistencyChecks, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
await FilelistProcessor.VerifyRemoteList(backendManager, m_options, db, m_result.BackendWriter, latestVolumesOnly: true, verifyMode: FilelistProcessor.VerifyMode.VerifyOnly, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await DoRunAsync(samples, db, backendManager).ConfigureAwait(false);
|
||||
await db.Transaction
|
||||
.CommitAsync("TestHandlerComplete")
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task DoRunAsync(long samples, LocalTestDatabase db, ReusableTransaction rtr, IBackendManager backend)
|
||||
public async Task DoRunAsync(long samples, LocalTestDatabase db, IBackendManager backend)
|
||||
{
|
||||
var files = db.SelectTestTargets(samples, m_options, rtr.Transaction).ToList();
|
||||
var files = await db
|
||||
.SelectTestTargets(samples, m_options, m_result.TaskControl.ProgressToken)
|
||||
.ToListAsync(cancellationToken: m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
m_results.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running);
|
||||
m_results.OperationProgressUpdater.UpdateProgress(0);
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Verify_Running);
|
||||
m_result.OperationProgressUpdater.UpdateProgress(0);
|
||||
var progress = 0L;
|
||||
|
||||
if (m_options.FullRemoteVerification != Options.RemoteTestStrategy.False)
|
||||
{
|
||||
var faultyIndexFiles = new List<IRemoteVolume>();
|
||||
await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, m_results.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
await foreach (var (tf, hash, size, name) in backend.GetFilesOverlappedAsync(files, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{
|
||||
var vol = new RemoteVolume(name, hash, size);
|
||||
try
|
||||
{
|
||||
if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
await backend.WaitForEmptyAsync(db, rtr.Transaction, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
m_results.EndTime = DateTime.UtcNow;
|
||||
await backend.WaitForEmptyAsync(db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
m_result.EndTime = DateTime.UtcNow;
|
||||
return;
|
||||
}
|
||||
|
||||
progress++;
|
||||
m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count);
|
||||
m_result.OperationProgressUpdater.UpdateProgress((float)progress / files.Count);
|
||||
|
||||
KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> res;
|
||||
using (tf)
|
||||
res = TestVolumeInternals(db, rtr, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2);
|
||||
res = await TestVolumeInternals(db, vol, tf, m_options, m_options.FullBlockVerification ? 1.0 : 0.2, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var parsedInfo = VolumeBase.ParseFilename(vol.Name);
|
||||
if (parsedInfo.FileType == RemoteVolumeType.Index)
|
||||
@@ -111,13 +120,15 @@ namespace Duplicati.Library.Main.Operation
|
||||
faultyIndexFiles.Add(vol);
|
||||
}
|
||||
|
||||
m_results.AddResult(res.Key, res.Value);
|
||||
m_result.AddResult(res.Key, res.Value);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(vol.Hash) && vol.Size > 0)
|
||||
{
|
||||
if (res.Value == null || !res.Value.Any())
|
||||
{
|
||||
var rv = db.GetRemoteVolume(vol.Name, rtr.Transaction);
|
||||
var rv = await db
|
||||
.GetRemoteVolume(vol.Name, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (rv.ID < 0)
|
||||
{
|
||||
@@ -130,22 +141,26 @@ namespace Duplicati.Library.Main.Operation
|
||||
else
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "CaptureHashAndSize", LC.L("Successfully captured hash and size for {0}, updating database", vol.Name));
|
||||
db.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash, rtr.Transaction);
|
||||
await db
|
||||
.UpdateRemoteVolume(vol.Name, RemoteVolumeState.Verified, vol.Size, vol.Hash, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
db.UpdateVerificationCount(vol.Name, rtr.Transaction);
|
||||
await db
|
||||
.UpdateVerificationCount(vol.Name, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
m_results.AddResult(vol.Name, [new KeyValuePair<TestEntryStatus, string>(TestEntryStatus.Error, ex.Message)]);
|
||||
m_result.AddResult(vol.Name, [new KeyValuePair<TestEntryStatus, string>(TestEntryStatus.Error, ex.Message)]);
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "RemoteFileProcessingFailed", ex, LC.L("Failed to process file {0}", vol.Name));
|
||||
if (ex.IsAbortException())
|
||||
{
|
||||
m_results.EndTime = DateTime.UtcNow;
|
||||
m_result.EndTime = DateTime.UtcNow;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@@ -156,7 +171,7 @@ namespace Duplicati.Library.Main.Operation
|
||||
if (m_options.ReplaceFaultyIndexFiles)
|
||||
{
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, LC.L("Found {0} faulty index files, repairing now", faultyIndexFiles.Count));
|
||||
await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, rtr, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await ReplaceFaultyIndexFilesAsync(faultyIndexFiles, backend, db, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
}
|
||||
else
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, "FaultyIndexFiles", null, LC.L("Found {0} faulty index files, remove the option {1} to repair them", faultyIndexFiles.Count, "--dont-replace-faulty-index-files"));
|
||||
@@ -168,25 +183,26 @@ namespace Duplicati.Library.Main.Operation
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
m_results.EndTime = DateTime.UtcNow;
|
||||
m_result.EndTime = DateTime.UtcNow;
|
||||
return;
|
||||
}
|
||||
|
||||
progress++;
|
||||
m_results.OperationProgressUpdater.UpdateProgress((float)progress / files.Count);
|
||||
m_result.OperationProgressUpdater.UpdateProgress((float)progress / files.Count);
|
||||
|
||||
if (f.Size <= 0 || string.IsNullOrWhiteSpace(f.Hash))
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "MissingRemoteHash", LC.L("No hash or size recorded for {0}, performing full verification", f.Name));
|
||||
KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> res;
|
||||
|
||||
(var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
(var tf, var hash, var size) = await backend.GetWithInfoAsync(f.Name, f.Hash, f.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
|
||||
using (tf)
|
||||
res = TestVolumeInternals(db, rtr, f, tf, m_options, 1);
|
||||
m_results.AddResult(res.Key, res.Value);
|
||||
res = await TestVolumeInternals(db, f, tf, m_options, 1, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
m_result.AddResult(res.Key, res.Value);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(hash) && size > 0)
|
||||
{
|
||||
@@ -199,54 +215,62 @@ namespace Duplicati.Library.Main.Operation
|
||||
else
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "CapturedHashAndSize", LC.L("Successfully captured hash and size for {0}, updating database", f.Name));
|
||||
db.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash, rtr.Transaction);
|
||||
await db
|
||||
.UpdateRemoteVolume(f.Name, RemoteVolumeState.Verified, size, hash, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
using (var tf = await backend.GetAsync(f.Name, f.Hash, f.Size, m_results.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
using (var tf = await backend.GetAsync(f.Name, f.Hash, f.Size, m_result.TaskControl.ProgressToken).ConfigureAwait(false))
|
||||
{ }
|
||||
}
|
||||
|
||||
db.UpdateVerificationCount(f.Name, rtr.Transaction);
|
||||
m_results.AddResult(f.Name, []);
|
||||
await db
|
||||
.UpdateVerificationCount(f.Name, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
m_result.AddResult(f.Name, []);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
m_results.AddResult(f.Name, [new KeyValuePair<TestEntryStatus, string>(TestEntryStatus.Error, ex.Message)]);
|
||||
m_result.AddResult(f.Name, [new KeyValuePair<TestEntryStatus, string>(TestEntryStatus.Error, ex.Message)]);
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "FailedToProcessFile", ex, LC.L("Failed to process file {0}", f.Name));
|
||||
if (ex.IsAbortOrCancelException())
|
||||
{
|
||||
m_results.EndTime = DateTime.UtcNow;
|
||||
m_result.EndTime = DateTime.UtcNow;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m_results.EndTime = DateTime.UtcNow;
|
||||
m_result.EndTime = DateTime.UtcNow;
|
||||
// generate a backup error status when any test is failing - except for 'extra' status
|
||||
// because these problems don't block database rebuilding.
|
||||
var filtered = from n in m_results.Verifications where n.Value.Any(x => x.Key != TestEntryStatus.Extra) select n;
|
||||
var filtered = from n in m_result.Verifications where n.Value.Any(x => x.Key != TestEntryStatus.Extra) select n;
|
||||
if (!filtered.Any())
|
||||
{
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "Test results", LC.L("Successfully verified {0} remote files", m_results.VerificationsActualLength));
|
||||
Logging.Log.WriteInformationMessage(LOGTAG, "Test results", LC.L("Successfully verified {0} remote files", m_result.VerificationsActualLength));
|
||||
}
|
||||
else
|
||||
{
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "Test results", null, LC.L("Verified {0} remote files with {1} problem(s)", m_results.VerificationsActualLength, filtered.Count()));
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "Test results", null, LC.L("Verified {0} remote files with {1} problem(s)", m_result.VerificationsActualLength, filtered.Count()));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests the volume by examining the internal contents
|
||||
/// Tests the volume by examining the internal contents.
|
||||
/// </summary>
|
||||
/// <param name="vol">The remote volume being examined</param>
|
||||
/// <param name="tf">The path to the downloaded copy of the file</param>
|
||||
/// <param name="sample_percent">A value between 0 and 1 that indicates how many blocks are tested in a dblock file</param>
|
||||
public static KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>> TestVolumeInternals(LocalTestDatabase db, ReusableTransaction rtr, IRemoteVolume vol, string tf, Options options, double sample_percent)
|
||||
/// <param name="db">The local test database to use for verification.</param>
|
||||
/// <param name="vol">The remote volume being examined.</param>
|
||||
/// <param name="tf">The path to the downloaded copy of the file.</param>
|
||||
/// <param name="options">The options to use for the test.</param>
|
||||
/// <param name="sample_percent">A value between 0 and 1 that indicates how many blocks are tested in a dblock file.</param>
|
||||
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited returns a key-value pair where the key is the volume name and the value is a list of test results.</returns>
|
||||
public static async Task<KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>> TestVolumeInternals(LocalTestDatabase db, IRemoteVolume vol, string tf, Options options, double sample_percent, CancellationToken cancellationToken)
|
||||
{
|
||||
var hashsize = HashFactory.HashSizeBytes(options.BlockHashAlgorithm);
|
||||
var parsedInfo = Volumes.VolumeBase.ParseFilename(vol.Name);
|
||||
@@ -255,15 +279,17 @@ namespace Duplicati.Library.Main.Operation
|
||||
switch (parsedInfo.FileType)
|
||||
{
|
||||
case RemoteVolumeType.Files:
|
||||
//Compare with db and see if all files are accounted for
|
||||
//Compare with db and see if all files are accounted for
|
||||
// with correct file hashes and blocklist hashes
|
||||
using (var fl = db.CreateFilelist(vol.Name, rtr))
|
||||
await using (var fl = await db.CreateFilelist(vol.Name, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
using (var rd = new Volumes.FilesetVolumeReader(parsedInfo.CompressionModule, tf, options))
|
||||
foreach (var f in rd.Files)
|
||||
fl.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time);
|
||||
await fl
|
||||
.Add(f.Path, f.Size, f.Hash, f.Metasize, f.Metahash, f.BlocklistHashes, f.Type, f.Time, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, fl.Compare().ToList());
|
||||
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, await fl.Compare(cancellationToken).ToListAsync(cancellationToken: cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
case RemoteVolumeType.Index:
|
||||
@@ -276,46 +302,69 @@ namespace Duplicati.Library.Main.Operation
|
||||
foreach (var v in rd.Volumes)
|
||||
{
|
||||
blocklinks.Add(new Tuple<string, string, long>(v.Filename, v.Hash, v.Length));
|
||||
using (var bl = db.CreateBlocklist(v.Filename, rtr))
|
||||
await using (var bl = await db.CreateBlocklist(v.Filename, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var h in v.Blocks)
|
||||
bl.AddBlock(h.Key, h.Value);
|
||||
await bl
|
||||
.AddBlock(h.Key, h.Value, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
combined.AddRange(bl.Compare());
|
||||
combined.AddRange(
|
||||
await bl
|
||||
.Compare(cancellationToken)
|
||||
.ToListAsync(cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (options.IndexfilePolicy == Options.IndexFileStrategy.Full)
|
||||
{
|
||||
var hashesPerBlock = options.Blocksize / options.BlockhashSize;
|
||||
using (var bl = db.CreateBlocklistHashList(vol.Name, rtr))
|
||||
await using (var bl = await db.CreateBlocklistHashList(vol.Name, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var b in rd.BlockLists)
|
||||
bl.AddBlockHash(b.Hash, b.Length);
|
||||
await bl
|
||||
.AddBlockHash(b.Hash, b.Length, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
combined.AddRange(bl.Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize));
|
||||
combined.AddRange(
|
||||
await bl
|
||||
.Compare(hashesPerBlock, options.BlockhashSize, options.Blocksize, cancellationToken)
|
||||
.ToListAsync(cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compare with db and see that all blocklists are listed
|
||||
using (var il = db.CreateIndexlist(vol.Name, rtr))
|
||||
await using (var il = await db.CreateIndexlist(vol.Name, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var t in blocklinks)
|
||||
il.AddBlockLink(t.Item1, t.Item2, t.Item3);
|
||||
await il
|
||||
.AddBlockLink(t.Item1, t.Item2, t.Item3, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
combined.AddRange(il.Compare());
|
||||
combined.AddRange(
|
||||
await il
|
||||
.Compare(cancellationToken)
|
||||
.ToArrayAsync(cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
);
|
||||
}
|
||||
|
||||
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, combined);
|
||||
case RemoteVolumeType.Blocks:
|
||||
using (var blockhasher = HashFactory.CreateHasher(options.BlockHashAlgorithm))
|
||||
using (var bl = db.CreateBlocklist(vol.Name, rtr))
|
||||
await using (var bl = await db.CreateBlocklist(vol.Name, cancellationToken).ConfigureAwait(false))
|
||||
using (var rd = new Volumes.BlockVolumeReader(parsedInfo.CompressionModule, tf, options))
|
||||
{
|
||||
//Verify that all blocks are in the file
|
||||
foreach (var b in rd.Blocks)
|
||||
bl.AddBlock(b.Key, b.Value);
|
||||
await bl
|
||||
.AddBlock(b.Key, b.Value, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
//Select random blocks and verify their hashes match the filename and size
|
||||
var hashsamples = new List<KeyValuePair<string, long>>(rd.Blocks);
|
||||
@@ -340,7 +389,15 @@ namespace Duplicati.Library.Main.Operation
|
||||
}
|
||||
}
|
||||
|
||||
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, changes.Union(bl.Compare().ToList()));
|
||||
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(
|
||||
vol.Name,
|
||||
changes.Union(
|
||||
await bl
|
||||
.Compare(cancellationToken)
|
||||
.ToListAsync(cancellationToken: cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -348,14 +405,17 @@ namespace Duplicati.Library.Main.Operation
|
||||
return new KeyValuePair<string, IEnumerable<KeyValuePair<TestEntryStatus, string>>>(vol.Name, null);
|
||||
}
|
||||
|
||||
private async Task ReplaceFaultyIndexFilesAsync(List<IRemoteVolume> faultyIndexFiles, IBackendManager backendManager, LocalTestDatabase db, ReusableTransaction rtr, CancellationToken cancellationToken)
|
||||
private async Task ReplaceFaultyIndexFilesAsync(List<IRemoteVolume> faultyIndexFiles, IBackendManager backendManager, LocalTestDatabase db, CancellationToken cancellationToken)
|
||||
{
|
||||
using var repairdb = new LocalRepairDatabase(db);
|
||||
await using var repairdb =
|
||||
await LocalRepairDatabase.CreateAsync(db, null, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
foreach (var vol in faultyIndexFiles)
|
||||
{
|
||||
if (!await m_results.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
if (!await m_result.TaskControl.ProgressRendevouz().ConfigureAwait(false))
|
||||
{
|
||||
m_results.EndTime = DateTime.UtcNow;
|
||||
m_result.EndTime = DateTime.UtcNow;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -363,17 +423,19 @@ namespace Duplicati.Library.Main.Operation
|
||||
try
|
||||
{
|
||||
var w = newEntry = new IndexVolumeWriter(m_options);
|
||||
await RepairHandler.RunRepairDindex(backendManager, repairdb, rtr, w, vol, m_options, cancellationToken).ConfigureAwait(false);
|
||||
await RepairHandler.RunRepairDindex(backendManager, repairdb, w, vol, m_options, cancellationToken).ConfigureAwait(false);
|
||||
if (m_options.Dryrun)
|
||||
{
|
||||
Logging.Log.WriteDryrunMessage(LOGTAG, "ReplaceFaultyIndexFile", LC.L("Would replace faulty index file {0} with {1}", vol.Name, w.RemoteFilename));
|
||||
}
|
||||
else
|
||||
{
|
||||
await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(repairdb, rtr.Transaction, m_results.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
rtr.Commit("ReplaceFaultyIndexFileCommit");
|
||||
m_results.RemoveResult(vol.Name);
|
||||
await backendManager.DeleteAsync(vol.Name, vol.Size, true, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await backendManager.WaitForEmptyAsync(repairdb, m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
await repairdb.Transaction
|
||||
.CommitAsync("ReplaceFaultyIndexFileCommit", token: cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
m_result.RemoveResult(vol.Name);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -1,27 +1,25 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace Duplicati.Library.Main.Operation
|
||||
@@ -37,15 +35,14 @@ namespace Duplicati.Library.Main.Operation
|
||||
m_result = result;
|
||||
}
|
||||
|
||||
public virtual Task RunAsync()
|
||||
=> Task.Run(() =>
|
||||
{
|
||||
using (var db = new Database.LocalDatabase(m_options.Dbpath, "Vacuum", false, m_options.SqlitePageCache))
|
||||
{
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running);
|
||||
db.Vacuum();
|
||||
m_result.EndTime = DateTime.UtcNow;
|
||||
}
|
||||
});
|
||||
public virtual async Task RunAsync()
|
||||
{
|
||||
await using var db =
|
||||
await Database.LocalDatabase.CreateLocalDatabaseAsync(m_options.Dbpath, "Vacuum", false, null, m_result.TaskControl.ProgressToken)
|
||||
.ConfigureAwait(false);
|
||||
m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Vacuum_Running);
|
||||
await db.Vacuum(m_result.TaskControl.ProgressToken).ConfigureAwait(false);
|
||||
m_result.EndTime = DateTime.UtcNow;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -176,11 +176,6 @@ namespace Duplicati.Library.Main
|
||||
/// </summary>
|
||||
private static readonly int DEFAULT_RESTORE_CHANNEL_BUFFER_SIZE = Environment.ProcessorCount;
|
||||
|
||||
/// <summary>
|
||||
/// The default value for the size of the SQLite page cache
|
||||
/// </summary>
|
||||
private static readonly string DEFAULT_SQLITE_PAGE_CACHE_SIZE = MemoryInfo.GetTotalMemoryString(0.01, SQLiteLoader.MINIMUM_SQLITE_PAGE_CACHE_SIZE); // 1% of the total memory
|
||||
|
||||
/// <summary>
|
||||
/// An enumeration that describes the supported strategies for an optimization
|
||||
/// </summary>
|
||||
@@ -535,7 +530,6 @@ namespace Duplicati.Library.Main
|
||||
new CommandLineArgument("restore-volume-downloaders", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreVolumeDownloadersShort, Strings.Options.RestoreVolumeDownloadersLong, DEFAULT_RESTORE_VOLUME_DOWNLOADERS.ToString()),
|
||||
new CommandLineArgument("restore-channel-buffer-size", CommandLineArgument.ArgumentType.Integer, Strings.Options.RestoreChannelBufferSizeShort, Strings.Options.RestoreChannelBufferSizeLong, DEFAULT_RESTORE_CHANNEL_BUFFER_SIZE.ToString()),
|
||||
new CommandLineArgument("internal-profiling", CommandLineArgument.ArgumentType.Boolean, Strings.Options.InternalProfilingShort, Strings.Options.InternalProfilingLong, "false"),
|
||||
new CommandLineArgument("sqlite-page-cache", CommandLineArgument.ArgumentType.Size, Strings.Options.SqlitePageCacheShort, Strings.Options.SqlitePageCacheLong(SQLiteLoader.MINIMUM_SQLITE_PAGE_CACHE_SIZE), DEFAULT_SQLITE_PAGE_CACHE_SIZE),
|
||||
new CommandLineArgument("ignore-update-if-version-exists", CommandLineArgument.ArgumentType.Boolean, Strings.Options.IgnoreUpdateIfVersionExistsShort, Strings.Options.IgnoreUpdateIfVersionExistsLong, "false"),
|
||||
];
|
||||
|
||||
@@ -1699,19 +1693,6 @@ namespace Duplicati.Library.Main
|
||||
/// </summary>
|
||||
public bool InternalProfiling => GetBool("internal-profiling");
|
||||
|
||||
/// <summary>
|
||||
/// Gets the size of file-blocks
|
||||
/// </summary>
|
||||
public long SqlitePageCache
|
||||
{
|
||||
get
|
||||
{
|
||||
var pagesize = GetSize("sqlite-page-cache", "kb", DEFAULT_SQLITE_PAGE_CACHE_SIZE);
|
||||
return pagesize <= SQLiteLoader.MINIMUM_SQLITE_PAGE_CACHE_SIZE
|
||||
? 0
|
||||
: pagesize;
|
||||
}
|
||||
}
|
||||
/// <summary>
|
||||
/// Ignores the update if the version already exists in the database.
|
||||
/// </summary>
|
||||
|
||||
@@ -1,33 +1,35 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using Duplicati.Library.Interface;
|
||||
using System.Collections.Generic;
|
||||
using Duplicati.Library.Main.Database;
|
||||
using Duplicati.Library.Logging;
|
||||
using System.Linq;
|
||||
using Newtonsoft.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Logging;
|
||||
using Duplicati.Library.Main.Database;
|
||||
using Duplicati.Library.Main.Operation.Common;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace Duplicati.Library.Main
|
||||
{
|
||||
@@ -179,7 +181,7 @@ namespace Duplicati.Library.Main
|
||||
|
||||
protected readonly BasicResults m_parent;
|
||||
protected System.Threading.Thread m_callerThread;
|
||||
protected readonly object m_lock = new object();
|
||||
protected readonly SemaphoreSlim m_lock = new(1, 1);
|
||||
protected readonly Queue<DbMessage> m_dbqueue;
|
||||
|
||||
public virtual ParsedResultType ParsedResult
|
||||
@@ -258,20 +260,33 @@ namespace Duplicati.Library.Main
|
||||
}
|
||||
}
|
||||
|
||||
public void FlushLog(LocalDatabase db)
|
||||
/// <summary>
|
||||
/// Flushes the log messages to the database.
|
||||
/// </summary>
|
||||
/// <param name="db">The database to flush the log messages to.</param>
|
||||
/// <param name="token">The cancellation token to use.</param>
|
||||
/// <returns>A task that completes when the log messages have been flushed.</returns>
|
||||
public async Task FlushLog(LocalDatabase db, CancellationToken token)
|
||||
{
|
||||
if (m_parent != null)
|
||||
m_parent.FlushLog(db);
|
||||
await m_parent.FlushLog(db, token).ConfigureAwait(false);
|
||||
else
|
||||
{
|
||||
lock (m_lock)
|
||||
await m_lock.WaitAsync(token).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
while (m_dbqueue.Count > 0)
|
||||
{
|
||||
var el = m_dbqueue.Dequeue();
|
||||
db.LogMessage(el.Type, el.Message, el.Exception, null);
|
||||
await db
|
||||
.LogMessage(el.Type, el.Message, el.Exception, token)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
m_lock.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -364,8 +364,6 @@ namespace Duplicati.Library.Main.Strings
|
||||
public static string RestoreChannelBufferSizeLong { get { return LC.L("Use this option to set the size of the buffers of the channels used during restore. The buffers are used to allow for better asynchronous communication between the processes in the restore flow. Increasing the buffer size may improve restore performance."); } }
|
||||
public static string InternalProfilingShort { get { return LC.L("Enable internal profiling"); } }
|
||||
public static string InternalProfilingLong { get { return LC.L("Use this option to enable internal profiling. Profiling is used to measure the performance of the internal code. The profiling data is written to the log file and can be used to identify performance bottlenecks."); } }
|
||||
public static string SqlitePageCacheShort { get { return LC.L("Size of the SQLite page cache"); } }
|
||||
public static string SqlitePageCacheLong(long minimumvalue) { return LC.L("Use this option to set the size of the SQLite page cache. The page cache is used to store the pages of the database in memory. Increasing the page cache size may improve performance, but will also increase memory usage. If the supplied value is the same or less than {0} bytes, the default SQLite cache value is used.", minimumvalue); }
|
||||
public static string IgnoreUpdateIfVersionExistsShort { get { return LC.L("Ignore update if version exists"); } }
|
||||
public static string IgnoreUpdateIfVersionExistsLong { get { return LC.L("Use this option to ignore the update if the version already exists. This can be used to avoid errors if asking to update the database with a version that already exists."); } }
|
||||
}
|
||||
|
||||
@@ -1,32 +1,34 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using Newtonsoft.Json;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Newtonsoft.Json;
|
||||
using Duplicati.Library.Main.Database;
|
||||
using Duplicati.Library.Utility;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
|
||||
namespace Duplicati.Library.Main
|
||||
{
|
||||
@@ -98,15 +100,16 @@ namespace Duplicati.Library.Main
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Updates the options with settings from the data, if any
|
||||
/// Updates the options with settings from the data, if any.
|
||||
/// </summary>
|
||||
/// <param name="db">The database to read from</param>
|
||||
/// <param name="options">The options to update</param>
|
||||
/// <param name="transaction">The transaction to use, if any</param>
|
||||
internal static void UpdateOptionsFromDb(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null)
|
||||
/// <param name="db">The database to read from.</param>
|
||||
/// <param name="options">The options to update.</param>
|
||||
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the options have been updated.</returns>
|
||||
internal static async Task UpdateOptionsFromDb(LocalDatabase db, Options options, CancellationToken cancellationToken)
|
||||
{
|
||||
string n = null;
|
||||
var opts = db.GetDbOptions(transaction);
|
||||
var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false);
|
||||
if (opts.ContainsKey("blocksize") && (!options.RawOptions.TryGetValue("blocksize", out n) || string.IsNullOrEmpty(n)))
|
||||
options.RawOptions["blocksize"] = opts["blocksize"] + "b";
|
||||
|
||||
@@ -117,23 +120,32 @@ namespace Duplicati.Library.Main
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the database contains options that need to be verified, such as the blocksize
|
||||
/// Checks if the database contains options that need to be verified, such as the blocksize.
|
||||
/// </summary>
|
||||
/// <param name="db">The database to check</param>
|
||||
/// <returns><c>true</c> if the database contains options that need to be verified; <c>false</c> otherwise</returns>
|
||||
internal static bool ContainsOptionsForVerification(LocalDatabase db)
|
||||
/// <param name="db">The database to check.</param>
|
||||
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that when awaited returns <c>true</c> if the database contains options that need to be verified; <c>false</c> otherwise.</returns>
|
||||
internal static async Task<bool> ContainsOptionsForVerification(LocalDatabase db, CancellationToken cancellationToken)
|
||||
{
|
||||
var opts = db.GetDbOptions();
|
||||
return new[] { "blocksize", "blockhash", "filehash", "passphrase" }.Any(opts.ContainsKey);
|
||||
var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false);
|
||||
await db.Transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
return new[] {
|
||||
"blocksize",
|
||||
"blockhash",
|
||||
"filehash",
|
||||
"passphrase"
|
||||
}
|
||||
.Any(opts.ContainsKey);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies the parameters in the database, and updates the database if needed
|
||||
/// Verifies the parameters in the database, and updates the database if needed.
|
||||
/// </summary>
|
||||
/// <param name="db">The database to check</param>
|
||||
/// <param name="options">The options to verify</param>
|
||||
/// <param name="transaction">The transaction to use, if any</param>
|
||||
internal static void VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options, System.Data.IDbTransaction transaction = null)
|
||||
/// <param name="db">The database to check.</param>
|
||||
/// <param name="options">The options to verify.</param>
|
||||
/// <param name="cancellationToken">A cancellation token to cancel the operation.</param>
|
||||
/// <returns>A task that completes when the options have been verified and the database has been updated if needed.</returns>
|
||||
internal static async Task VerifyOptionsAndUpdateDatabase(LocalDatabase db, Options options, CancellationToken cancellationToken)
|
||||
{
|
||||
var newDict = new Dictionary<string, string>
|
||||
{
|
||||
@@ -141,7 +153,7 @@ namespace Duplicati.Library.Main
|
||||
{ "blockhash", options.BlockHashAlgorithm },
|
||||
{ "filehash", options.FileHashAlgorithm }
|
||||
};
|
||||
var opts = db.GetDbOptions(transaction);
|
||||
var opts = await db.GetDbOptions(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (options.NoEncryption)
|
||||
{
|
||||
@@ -162,7 +174,7 @@ namespace Duplicati.Library.Main
|
||||
|
||||
newDict["passphrase-salt"] = salt;
|
||||
|
||||
// We avoid storing the passphrase directly,
|
||||
// We avoid storing the passphrase directly,
|
||||
// instead we salt and rehash repeatedly
|
||||
newDict.Add("passphrase", Library.Utility.Utility.ByteArrayAsHexString(Library.Utility.Utility.RepeatedHashWithSalt(options.Passphrase, salt, 1200)));
|
||||
}
|
||||
@@ -191,7 +203,7 @@ namespace Duplicati.Library.Main
|
||||
}
|
||||
|
||||
//Extra sanity check
|
||||
if (db.GetBlocksLargerThan(options.Blocksize) > 0)
|
||||
if (await db.GetBlocksLargerThan(options.Blocksize, cancellationToken).ConfigureAwait(false) > 0)
|
||||
throw new Duplicati.Library.Interface.UserInformationException("You have attempted to change the block-size on an existing backup, which is not supported. Please configure a new clean backup if you want to change the block-size.", "BlockSizeChangeNotSupported");
|
||||
|
||||
if (needsUpdate)
|
||||
@@ -201,7 +213,7 @@ namespace Duplicati.Library.Main
|
||||
if (!newDict.ContainsKey(k.Key))
|
||||
newDict[k.Key] = k.Value;
|
||||
|
||||
db.SetDbOptions(newDict, transaction);
|
||||
await db.SetDbOptions(newDict, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,27 +1,28 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Interface;
|
||||
|
||||
namespace Duplicati.Library.Main.Volumes
|
||||
@@ -33,7 +34,7 @@ namespace Duplicati.Library.Main.Volumes
|
||||
|
||||
public long Blocks { get { return m_blocks; } }
|
||||
public long SourceSize { get { return m_sourcesize; } }
|
||||
|
||||
|
||||
public override RemoteVolumeType FileType { get { return RemoteVolumeType.Blocks; } }
|
||||
|
||||
public BlockVolumeWriter(Options options)
|
||||
@@ -41,14 +42,14 @@ namespace Duplicati.Library.Main.Volumes
|
||||
{
|
||||
}
|
||||
|
||||
public void AddBlock(string hash, byte[] data, int offset, int size, CompressionHint hint)
|
||||
public async Task AddBlock(string hash, byte[] data, int offset, int size, CompressionHint hint)
|
||||
{
|
||||
m_blocks++;
|
||||
m_sourcesize += size;
|
||||
|
||||
//Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications,
|
||||
//Filenames are encoded with "modified Base64 for URL" https://en.wikipedia.org/wiki/Base64#URL_applications,
|
||||
using (var s = m_compression.CreateFile(Library.Utility.Utility.Base64PlainToBase64Url(hash), hint, DateTime.UtcNow))
|
||||
s.Write(data, offset, size);
|
||||
await s.WriteAsync(data.AsMemory(offset, size)).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
@@ -28,6 +28,7 @@ using Duplicati.Library.Main.Operation.Common;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Utility;
|
||||
using System.Data;
|
||||
using System.Threading;
|
||||
|
||||
namespace Duplicati.Library.Main.Volumes
|
||||
{
|
||||
@@ -82,17 +83,19 @@ namespace Duplicati.Library.Main.Volumes
|
||||
}
|
||||
}
|
||||
|
||||
public void AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable<string> blocklisthashes, IEnumerable<string> metablocklisthashes)
|
||||
public async Task AddFile(string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IAsyncEnumerable<string> blocklisthashes, IEnumerable<string> metablocklisthashes)
|
||||
{
|
||||
AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes);
|
||||
await AddFileEntry(FilelistEntryType.File, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public void AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, string blockhash, long blocksize, IEnumerable<string> blocklisthashes, IEnumerable<string> metablocklisthashes)
|
||||
public async Task AddAlternateStream(string name, string filehash, long size, DateTime lastmodified, string metahash, string metablockhash, long metasize, string blockhash, long blocksize, IAsyncEnumerable<string> blocklisthashes, IEnumerable<string> metablocklisthashes)
|
||||
{
|
||||
AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes);
|
||||
await AddFileEntry(FilelistEntryType.AlternateStream, name, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blocklisthashes, metablocklisthashes)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private void AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IEnumerable<string> blocklisthashes, IEnumerable<string> metablocklisthashes)
|
||||
private async Task AddFileEntry(FilelistEntryType type, string name, string filehash, long size, DateTime lastmodified, string metahash, long metasize, string metablockhash, string blockhash, long blocksize, IAsyncEnumerable<string> blocklisthashes, IEnumerable<string> metablocklisthashes)
|
||||
{
|
||||
m_filecount++;
|
||||
m_writer.WriteStartObject();
|
||||
@@ -111,15 +114,15 @@ namespace Duplicati.Library.Main.Volumes
|
||||
|
||||
if (blocklisthashes != null)
|
||||
{
|
||||
//Slightly awkward, but we avoid writing if there are no entries
|
||||
using (var en = blocklisthashes.GetEnumerator())
|
||||
//Slightly awkward, but we avoid writing if there are no entries
|
||||
await using (var en = blocklisthashes.GetAsyncEnumerator())
|
||||
{
|
||||
if (en.MoveNext() && !string.IsNullOrEmpty(en.Current))
|
||||
if (await en.MoveNextAsync().ConfigureAwait(false) && !string.IsNullOrEmpty(en.Current))
|
||||
{
|
||||
m_writer.WritePropertyName("blocklists");
|
||||
m_writer.WriteStartArray();
|
||||
m_writer.WriteValue(en.Current);
|
||||
while (en.MoveNext())
|
||||
while (await en.MoveNextAsync().ConfigureAwait(false))
|
||||
m_writer.WriteValue(en.Current);
|
||||
m_writer.WriteEndArray();
|
||||
}
|
||||
@@ -234,24 +237,25 @@ namespace Duplicati.Library.Main.Volumes
|
||||
/// <param name="database">The database to check for clashes</param>
|
||||
/// <param name="options">The options to use for the filename</param>
|
||||
/// <param name="start">The starting time to probe from</param>
|
||||
/// <param name="cancellationToken">The cancellation token to use for the operation</param>
|
||||
/// <param name="increment">The time to increment by each probe</param>
|
||||
/// <param name="maxTries">The maximum number of tries to probe</param>
|
||||
/// <returns>The first unused filename</returns>
|
||||
internal static Task<DateTime> ProbeUnusedFilenameName(DatabaseCommon database, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60)
|
||||
=> ProbeUnusedFilenameName(name => database.GetRemoteVolumeIDAsync(name), options, start, increment, maxTries);
|
||||
internal static Task<DateTime> ProbeUnusedFilenameName(DatabaseCommon database, Options options, DateTime start, CancellationToken cancellationToken, TimeSpan increment = default, int maxTries = 60)
|
||||
=> ProbeUnusedFilenameName((name, CancellationToken) => database.GetRemoteVolumeIDAsync(name, cancellationToken), options, start, cancellationToken, increment, maxTries);
|
||||
|
||||
/// <summary>
|
||||
/// Probes for an unused filename, using the current time as a starting point
|
||||
/// Probes for an unused filename, using the current time as a starting point.
|
||||
/// </summary>
|
||||
/// <param name="database">The database to check for clashes</param>
|
||||
/// <param name="options">The options to use for the filename</param>
|
||||
/// <param name="transaction">The transaction to use</param>
|
||||
/// <param name="start">The starting time to probe from</param>
|
||||
/// <param name="increment">The time to increment by each probe</param>
|
||||
/// <param name="maxTries">The maximum number of tries to probe</param>
|
||||
/// <returns>The first unused filename</returns>
|
||||
internal static DateTime ProbeUnusedFilenameName(LocalDatabase database, IDbTransaction transaction, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60)
|
||||
=> ProbeUnusedFilenameName((name) => Task.FromResult(database.GetRemoteVolumeID(name, transaction)), options, start, increment, maxTries).Await();
|
||||
/// <param name="database">The database to check for clashes.</param>
|
||||
/// <param name="options">The options to use for the filename.</param>
|
||||
/// <param name="start">The starting time to probe from.</param>
|
||||
/// <param name="increment">The time to increment by each probe.</param>
|
||||
/// <param name="maxTries">The maximum number of tries to probe.</param>
|
||||
/// <returns>A task that when awaited returns the first unused filename.</returns>
|
||||
internal static async Task<DateTime> ProbeUnusedFilenameName(LocalDatabase database, Options options, DateTime start, CancellationToken cancellationToken, TimeSpan increment = default, int maxTries = 60)
|
||||
=> await ProbeUnusedFilenameName(database.GetRemoteVolumeID, options, start, cancellationToken, increment, maxTries)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
/// <summary>
|
||||
/// Probes for an unused filename, using the current time as a starting point
|
||||
@@ -262,7 +266,7 @@ namespace Duplicati.Library.Main.Volumes
|
||||
/// <param name="increment">The time to increment by each probe</param>
|
||||
/// <param name="maxTries">The maximum number of tries to probe</param>
|
||||
/// <returns>The first unused filename</returns>
|
||||
private static async Task<DateTime> ProbeUnusedFilenameName(Func<string, Task<long>> ProbeForId, Options options, DateTime start, TimeSpan increment = default, int maxTries = 60)
|
||||
private static async Task<DateTime> ProbeUnusedFilenameName(Func<string, CancellationToken, Task<long>> ProbeForId, Options options, DateTime start, CancellationToken cancellationToken, TimeSpan increment = default, int maxTries = 60)
|
||||
{
|
||||
if (increment == default)
|
||||
increment = TimeSpan.FromSeconds(1);
|
||||
@@ -272,7 +276,7 @@ namespace Duplicati.Library.Main.Volumes
|
||||
|
||||
while (s < maxTries)
|
||||
{
|
||||
var id = await ProbeForId(GenerateFilename(RemoteVolumeType.Files, options, null, time)).ConfigureAwait(false);
|
||||
var id = await ProbeForId(GenerateFilename(RemoteVolumeType.Files, options, null, time), cancellationToken).ConfigureAwait(false);
|
||||
if (id < 0)
|
||||
return time;
|
||||
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -108,7 +108,7 @@ namespace Duplicati.Server.Database
|
||||
|
||||
public void ReWriteAllFieldsIfEncryptionChanged()
|
||||
{
|
||||
// The token is automatically decrypted when the settings are loaded
|
||||
// The token is automatically decrypted when the settings are loaded
|
||||
// In case the password has changed, this will fail and return the encrypted
|
||||
// hex-string, but will crash before reaching this point
|
||||
if (this.ApplicationSettings.EncryptedFields != m_encryptSensitiveFields)
|
||||
@@ -246,22 +246,25 @@ namespace Duplicati.Server.Database
|
||||
internal void SetMetadata(IDictionary<string, string> values, long id, IDbTransaction? transaction)
|
||||
{
|
||||
lock (m_lock)
|
||||
using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null)
|
||||
{
|
||||
OverwriteAndUpdateDb(
|
||||
tr,
|
||||
cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Metadata"" WHERE ""BackupID"" = @Id")
|
||||
.SetParameterValue("@Id", id),
|
||||
values ?? new Dictionary<string, string>(),
|
||||
cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Metadata"" (""BackupID"", ""Name"", ""Value"") VALUES (@BackupId, @Name, @Value)"),
|
||||
(cmd, f) => cmd.SetParameterValue("@BackupId", id)
|
||||
.SetParameterValue("@Name", f.Key)
|
||||
.SetParameterValue("@Value", f.Value)
|
||||
);
|
||||
{
|
||||
var tr = transaction ?? m_connection.BeginTransaction();
|
||||
OverwriteAndUpdateDb(
|
||||
tr,
|
||||
cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Metadata"" WHERE ""BackupID"" = @Id")
|
||||
.SetParameterValue("@Id", id),
|
||||
values ?? new Dictionary<string, string>(),
|
||||
cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Metadata"" (""BackupID"", ""Name"", ""Value"") VALUES (@BackupId, @Name, @Value)"),
|
||||
(cmd, f) => cmd.SetParameterValue("@BackupId", id)
|
||||
.SetParameterValue("@Name", f.Key)
|
||||
.SetParameterValue("@Value", f.Value)
|
||||
);
|
||||
|
||||
if (tr != null)
|
||||
tr.Commit();
|
||||
if (transaction == null)
|
||||
{
|
||||
tr.Commit();
|
||||
tr.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal IFilter[] GetFilters(long id)
|
||||
@@ -282,23 +285,26 @@ namespace Duplicati.Server.Database
|
||||
internal void SetFilters(IEnumerable<IFilter> values, long id, IDbTransaction? transaction = null)
|
||||
{
|
||||
lock (m_lock)
|
||||
using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null)
|
||||
{
|
||||
OverwriteAndUpdateDb(
|
||||
tr,
|
||||
cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Filter"" WHERE ""BackupID"" = @Id")
|
||||
.SetParameterValue("@Id", id),
|
||||
values,
|
||||
cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Filter"" (""BackupID"", ""Order"", ""Include"", ""Expression"") VALUES (@Id, @Order, @Include, @Expression)"),
|
||||
(cmd, f) => cmd.SetParameterValue("@Id", id)
|
||||
.SetParameterValue("@Order", f.Order)
|
||||
.SetParameterValue("@Include", f.Include)
|
||||
.SetParameterValue("@Expression", f.Expression)
|
||||
);
|
||||
{
|
||||
var tr = transaction ?? m_connection.BeginTransaction();
|
||||
OverwriteAndUpdateDb(
|
||||
tr,
|
||||
cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Filter"" WHERE ""BackupID"" = @Id")
|
||||
.SetParameterValue("@Id", id),
|
||||
values,
|
||||
cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Filter"" (""BackupID"", ""Order"", ""Include"", ""Expression"") VALUES (@Id, @Order, @Include, @Expression)"),
|
||||
(cmd, f) => cmd.SetParameterValue("@Id", id)
|
||||
.SetParameterValue("@Order", f.Order)
|
||||
.SetParameterValue("@Include", f.Include)
|
||||
.SetParameterValue("@Expression", f.Expression)
|
||||
);
|
||||
|
||||
if (tr != null)
|
||||
tr.Commit();
|
||||
if (transaction == null)
|
||||
{
|
||||
tr.Commit();
|
||||
tr.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public ISetting[] GetSettings(long id)
|
||||
@@ -320,37 +326,40 @@ namespace Duplicati.Server.Database
|
||||
internal void SetSettings(IEnumerable<ISetting> values, long id, IDbTransaction? transaction = null)
|
||||
{
|
||||
lock (m_lock)
|
||||
using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null)
|
||||
{
|
||||
var tr = transaction ?? m_connection.BeginTransaction();
|
||||
if (m_encryptSensitiveFields)
|
||||
values = values.Select(x => new Setting
|
||||
{
|
||||
Filter = x.Filter,
|
||||
Name = x.Name,
|
||||
Value = EncryptSensitiveFields(x.Name, x.Value, m_key)
|
||||
}).ToList();
|
||||
|
||||
OverwriteAndUpdateDb(
|
||||
tr,
|
||||
cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Option"" WHERE ""BackupID"" = @Id")
|
||||
.SetParameterValue("@Id", id),
|
||||
values,
|
||||
cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Option"" (""BackupID"", ""Filter"", ""Name"", ""Value"") VALUES (@BackupId, @Filter, @Name, @Value)"),
|
||||
(cmd, f) =>
|
||||
{
|
||||
if (PASSWORD_PLACEHOLDER.Equals(f.Value))
|
||||
throw new Exception("Attempted to save a property with the placeholder password");
|
||||
|
||||
cmd.SetParameterValue("@BackupId", id)
|
||||
.SetParameterValue("@Filter", f.Filter ?? "")
|
||||
.SetParameterValue("@Name", f.Name ?? "")
|
||||
.SetParameterValue("@Value", f.Value ?? "");
|
||||
}
|
||||
);
|
||||
|
||||
if (transaction == null)
|
||||
{
|
||||
if (m_encryptSensitiveFields)
|
||||
values = values.Select(x => new Setting
|
||||
{
|
||||
Filter = x.Filter,
|
||||
Name = x.Name,
|
||||
Value = EncryptSensitiveFields(x.Name, x.Value, m_key)
|
||||
}).ToList();
|
||||
|
||||
OverwriteAndUpdateDb(
|
||||
tr,
|
||||
cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Option"" WHERE ""BackupID"" = @Id")
|
||||
.SetParameterValue("@Id", id),
|
||||
values,
|
||||
cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Option"" (""BackupID"", ""Filter"", ""Name"", ""Value"") VALUES (@BackupId, @Filter, @Name, @Value)"),
|
||||
(cmd, f) =>
|
||||
{
|
||||
if (PASSWORD_PLACEHOLDER.Equals(f.Value))
|
||||
throw new Exception("Attempted to save a property with the placeholder password");
|
||||
|
||||
cmd.SetParameterValue("@BackupId", id)
|
||||
.SetParameterValue("@Filter", f.Filter ?? "")
|
||||
.SetParameterValue("@Name", f.Name ?? "")
|
||||
.SetParameterValue("@Value", f.Value ?? "");
|
||||
}
|
||||
);
|
||||
|
||||
if (tr != null)
|
||||
tr.Commit();
|
||||
tr.Commit();
|
||||
tr.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal string?[] GetSources(long id)
|
||||
@@ -366,21 +375,24 @@ namespace Duplicati.Server.Database
|
||||
internal void SetSources(IEnumerable<string> values, long id, IDbTransaction transaction)
|
||||
{
|
||||
lock (m_lock)
|
||||
using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null)
|
||||
{
|
||||
OverwriteAndUpdateDb(
|
||||
tr,
|
||||
cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Source"" WHERE ""BackupID"" = @Id")
|
||||
.SetParameterValue("@Id", id),
|
||||
values,
|
||||
cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Source"" (""BackupID"", ""Path"") VALUES (@BackupId, @Path)"),
|
||||
(cmd, f) => cmd.SetParameterValue("@BackupId", id)
|
||||
.SetParameterValue("@Path", f)
|
||||
);
|
||||
{
|
||||
var tr = transaction ?? m_connection.BeginTransaction();
|
||||
OverwriteAndUpdateDb(
|
||||
tr,
|
||||
cmd => cmd.SetCommandAndParameters(@"DELETE FROM ""Source"" WHERE ""BackupID"" = @Id")
|
||||
.SetParameterValue("@Id", id),
|
||||
values,
|
||||
cmd => cmd.SetCommandAndParameters(@"INSERT INTO ""Source"" (""BackupID"", ""Path"") VALUES (@BackupId, @Path)"),
|
||||
(cmd, f) => cmd.SetParameterValue("@BackupId", id)
|
||||
.SetParameterValue("@Path", f)
|
||||
);
|
||||
|
||||
if (tr != null)
|
||||
tr.Commit();
|
||||
if (transaction == null)
|
||||
{
|
||||
tr.Commit();
|
||||
tr.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal long[] GetBackupIDsForTags(string[] tags)
|
||||
@@ -624,7 +636,7 @@ namespace Duplicati.Server.Database
|
||||
{
|
||||
lock (m_lock)
|
||||
{
|
||||
using (var tr = m_connection.BeginTransactionSafe())
|
||||
using (var tr = m_connection.BeginTransaction())
|
||||
{
|
||||
using (var cmd = m_connection.CreateCommand(tr, @"UPDATE ""Backup"" SET ""DBPath""= @Dbpath WHERE ""ID""= @Id"))
|
||||
{
|
||||
@@ -665,7 +677,7 @@ namespace Duplicati.Server.Database
|
||||
throw new Exception("Unable to generate a unique database file name");
|
||||
}
|
||||
|
||||
using (var tr = m_connection.BeginTransactionSafe())
|
||||
using (var tr = m_connection.BeginTransaction())
|
||||
{
|
||||
OverwriteAndUpdateDb(
|
||||
tr,
|
||||
@@ -760,7 +772,7 @@ namespace Duplicati.Server.Database
|
||||
internal void AddOrUpdateSchedule(ISchedule item)
|
||||
{
|
||||
lock (m_lock)
|
||||
using (var tr = m_connection.BeginTransactionSafe())
|
||||
using (var tr = m_connection.BeginTransaction())
|
||||
{
|
||||
AddOrUpdateSchedule(item, tr);
|
||||
tr.Commit();
|
||||
@@ -811,7 +823,7 @@ namespace Duplicati.Server.Database
|
||||
|
||||
lock (m_lock)
|
||||
{
|
||||
using (var tr = m_connection.BeginTransactionSafe())
|
||||
using (var tr = m_connection.BeginTransaction())
|
||||
{
|
||||
var existing = GetScheduleIDsFromTags(new string[] { "ID=" + ID.ToString() });
|
||||
if (existing.Any())
|
||||
@@ -1000,7 +1012,7 @@ namespace Duplicati.Server.Database
|
||||
//Workaround to clean up the database after invalid settings update
|
||||
public void FixInvalidBackupId()
|
||||
{
|
||||
using (var tr = m_connection.BeginTransactionSafe())
|
||||
using (var tr = m_connection.BeginTransaction())
|
||||
using (var cmd = m_connection.CreateCommand(tr))
|
||||
{
|
||||
cmd.SetCommandAndParameters(@"DELETE FROM ""Option"" WHERE ""BackupID"" = @BackupId")
|
||||
@@ -1051,7 +1063,7 @@ namespace Duplicati.Server.Database
|
||||
public void SetUISettings(string scheme, IDictionary<string, string?> values, IDbTransaction? transaction = null)
|
||||
{
|
||||
lock (m_lock)
|
||||
using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null)
|
||||
using (var tr = transaction == null ? m_connection.BeginTransaction() : null)
|
||||
{
|
||||
OverwriteAndUpdateDb(
|
||||
tr,
|
||||
@@ -1075,7 +1087,7 @@ namespace Duplicati.Server.Database
|
||||
public void UpdateUISettings(string scheme, IDictionary<string, string?> values, IDbTransaction? transaction = null)
|
||||
{
|
||||
lock (m_lock)
|
||||
using (var tr = transaction == null ? m_connection.BeginTransactionSafe() : null)
|
||||
using (var tr = transaction == null ? m_connection.BeginTransaction() : null)
|
||||
{
|
||||
OverwriteAndUpdateDb(
|
||||
tr,
|
||||
@@ -1128,7 +1140,7 @@ namespace Duplicati.Server.Database
|
||||
{
|
||||
var t = Library.Utility.Utility.NormalizeDateTimeToEpochSeconds(purgeDate);
|
||||
|
||||
using (var tr = m_connection.BeginTransactionSafe())
|
||||
using (var tr = m_connection.BeginTransaction())
|
||||
using (var cmd = m_connection.CreateCommand(tr))
|
||||
{
|
||||
cmd.SetCommandAndParameters(@"DELETE FROM ""ErrorLog"" WHERE ""Timestamp"" < @Time")
|
||||
@@ -1207,7 +1219,7 @@ namespace Duplicati.Server.Database
|
||||
{
|
||||
if (transaction == null)
|
||||
{
|
||||
using (var tr = m_connection.BeginTransactionSafe())
|
||||
using (var tr = m_connection.BeginTransaction())
|
||||
{
|
||||
var r = DeleteFromDb(tablename, id, tr);
|
||||
tr.Commit();
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -36,24 +36,24 @@ namespace Duplicati.Library.SQLiteHelper
|
||||
/// Updates should have the form "1.Sample upgrade.sql".
|
||||
/// When the database schema changes, simply put a new file into the folder
|
||||
/// and set it to be embedded in the binary.
|
||||
///
|
||||
/// Additionally, it's possible to execute custom code before and after
|
||||
///
|
||||
/// Additionally, it's possible to execute custom code before and after
|
||||
/// the SQL is executed. To set up a custom upgrade stage, add your
|
||||
/// code to DbUpgradesRegistry along with the DB version to apply it with.
|
||||
///
|
||||
///
|
||||
/// Even if all the DB upgrade code is handled in C#, you still have to add
|
||||
/// a dummy SQL file to indicate the version ID is already taken.
|
||||
///
|
||||
///
|
||||
/// Each upgrade file should ONLY upgrade from the previous version.
|
||||
/// If done correctly, a user may be upgrade from the very first version
|
||||
/// to the very latest.
|
||||
///
|
||||
/// The Schema.sql file should ALWAYS have the latest schema, as that will
|
||||
///
|
||||
/// The Schema.sql file should ALWAYS have the latest schema, as that will
|
||||
/// ensure that new installs do not run upgrades after installation.
|
||||
/// Also remember to update the last line in Schema.sql to insert the
|
||||
/// Also remember to update the last line in Schema.sql to insert the
|
||||
/// current version number in the version table.
|
||||
///
|
||||
/// Currently no upgrades may contain semicolons, except as the SQL statement
|
||||
///
|
||||
/// Currently no upgrades may contain semicolons, except as the SQL statement
|
||||
/// delimiter.
|
||||
/// </summary>
|
||||
public static class DatabaseUpgrader
|
||||
@@ -203,14 +203,21 @@ namespace Duplicati.Library.SQLiteHelper
|
||||
try
|
||||
{
|
||||
//See if the version table is present,
|
||||
cmd.CommandText = "SELECT COUNT(*) FROM SQLITE_MASTER WHERE Name LIKE 'Version'";
|
||||
cmd.CommandText = @"
|
||||
SELECT COUNT(*)
|
||||
FROM SQLITE_MASTER
|
||||
WHERE Name LIKE 'Version'
|
||||
";
|
||||
int count = Convert.ToInt32(cmd.ExecuteScalar());
|
||||
|
||||
if (count == 0)
|
||||
dbversion = -1; //Empty
|
||||
else if (count == 1)
|
||||
{
|
||||
cmd.CommandText = "SELECT max(Version) FROM Version";
|
||||
cmd.CommandText = @"
|
||||
SELECT max(Version)
|
||||
FROM Version
|
||||
";
|
||||
dbversion = Convert.ToInt32(cmd.ExecuteScalar());
|
||||
}
|
||||
else
|
||||
|
||||
@@ -10,21 +10,7 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<!--
|
||||
MS Data.Sqlite seems much more portable, but does not support unnamed parameters,
|
||||
which are used extensively in Duplicati.
|
||||
Issue tracking progress on the unnamed parameters:
|
||||
https://github.com/dotnet/efcore/issues/24480
|
||||
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.3" />
|
||||
-->
|
||||
|
||||
<PackageReference Include="System.Data.SQLite.Core" Version="1.0.115.0" />
|
||||
<PackageReference Include="System.Data.SQLite.Core.MSIL" Version="1.0.115.0" />
|
||||
<PackageReference Include="System.Data.SQLite.Core.Duplicati.macos.arm64" Version="1.0.116.1" />
|
||||
<PackageReference Include="System.Data.SQLite.Core.Duplicati.linux.arm64" Version="1.0.116" />
|
||||
<PackageReference Include="System.Data.SQLite.Core.Duplicati.linux.armv7" Version="1.0.116" />
|
||||
<PackageReference Include="System.Data.SQLite.Core.Duplicati.windows.arm64" Version="1.0.116.1" />
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.4" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -1,52 +1,56 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
#nullable enable
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Common.IO;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Utility;
|
||||
|
||||
namespace Duplicati.Library.SQLiteHelper
|
||||
{
|
||||
/// <summary>
|
||||
/// Provides methods to load and manage SQLite connections, including handling encrypted databases.
|
||||
/// </summary>
|
||||
public static class SQLiteLoader
|
||||
{
|
||||
/// <summary>
|
||||
/// The minimum value for the SQLite page cache size
|
||||
/// </summary>
|
||||
public const long MINIMUM_SQLITE_PAGE_CACHE_SIZE = 2048000L;
|
||||
|
||||
/// <summary>
|
||||
/// The tag used for logging
|
||||
/// The tag used for logging.
|
||||
/// </summary>
|
||||
private static readonly string LOGTAG = Logging.Log.LogTagFromType(typeof(SQLiteLoader));
|
||||
|
||||
/// <summary>
|
||||
/// Helper method with logic to handle opening a database in possibly encrypted format
|
||||
/// Helper method with logic to handle opening a database in possibly encrypted format.
|
||||
/// </summary>
|
||||
/// <param name="con">The SQLite connection object</param>
|
||||
/// <param name="con">The SQLite connection object.</param>
|
||||
/// <param name="databasePath">The location of Duplicati's database.</param>
|
||||
/// <param name="decryptionPassword">The password to use for decryption.</param>
|
||||
public static void OpenDatabase(System.Data.IDbConnection con, string databasePath, string? decryptionPassword)
|
||||
/// <returns>A task that completes when the database is opened.</returns>
|
||||
/// <exception cref="UserInformationException">Thrown if the database cannot be opened or decrypted.</exception>
|
||||
public static async Task OpenDatabaseAsync(Microsoft.Data.Sqlite.SqliteConnection con, string databasePath, string? decryptionPassword)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(decryptionPassword) && SQLiteRC4Decrypter.IsDatabaseEncrypted(databasePath))
|
||||
{
|
||||
@@ -66,12 +70,13 @@ namespace Duplicati.Library.SQLiteHelper
|
||||
try
|
||||
{
|
||||
//Attempt to open in preferred state
|
||||
OpenSQLiteFile(con, databasePath);
|
||||
TestSQLiteFile(con);
|
||||
await OpenSQLiteFileAsync(con, databasePath)
|
||||
.ConfigureAwait(false);
|
||||
await TestSQLiteFileAsync(con).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
try { con.Dispose(); }
|
||||
try { await con.DisposeAsync().ConfigureAwait(false); }
|
||||
catch { }
|
||||
|
||||
throw;
|
||||
@@ -82,22 +87,34 @@ namespace Duplicati.Library.SQLiteHelper
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads an SQLite connection instance and opening the database
|
||||
/// Loads an SQLite connection instance and opening the database.
|
||||
/// </summary>
|
||||
/// <returns>The SQLite connection instance.</returns>
|
||||
public static System.Data.IDbConnection LoadConnection()
|
||||
/// <remarks>
|
||||
/// This method is synchronous and should be used when you need to load the connection immediately. It calls the asynchronous version and waits for it to complete.
|
||||
/// </remarks>
|
||||
public static Microsoft.Data.Sqlite.SqliteConnection LoadConnection()
|
||||
{
|
||||
System.Data.IDbConnection? con = null;
|
||||
return LoadConnectionAsync().Await();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads an SQLite connection instance and opening the database.
|
||||
/// </summary>
|
||||
/// <returns>A task that when awaited returns the SQLite connection instance.</returns>
|
||||
public static async Task<Microsoft.Data.Sqlite.SqliteConnection> LoadConnectionAsync()
|
||||
{
|
||||
Microsoft.Data.Sqlite.SqliteConnection? con = null;
|
||||
SetEnvironmentVariablesForSQLiteTempDir();
|
||||
|
||||
try
|
||||
{
|
||||
con = (System.Data.IDbConnection?)Activator.CreateInstance(Duplicati.Library.SQLiteHelper.SQLiteLoader.SQLiteConnectionType);
|
||||
con = new Microsoft.Data.Sqlite.SqliteConnection();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "FailedToLoadConnectionSQLite", ex, "Failed to load connection.");
|
||||
DisposeConnection(con);
|
||||
await DisposeConnectionAsync(con).ConfigureAwait(false);
|
||||
|
||||
throw;
|
||||
}
|
||||
@@ -106,79 +123,109 @@ namespace Duplicati.Library.SQLiteHelper
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies user-supplied custom pragmas to the SQLite connection
|
||||
/// Applies user-supplied custom pragmas to the SQLite connection.
|
||||
/// </summary>
|
||||
/// <param name="con">The connection to apply the pragmas to.</param>
|
||||
/// <param name="pagecachesize"> The page cache size to set.</param>
|
||||
/// <returns>The connection with the pragmas applied.</returns>
|
||||
public static System.Data.IDbConnection ApplyCustomPragmas(System.Data.IDbConnection con, long pagecachesize)
|
||||
/// <returns>A task that when awaited returns the connection with the pragmas applied.</returns>
|
||||
public static async Task<Microsoft.Data.Sqlite.SqliteConnection> ApplyCustomPragmasAsync(Microsoft.Data.Sqlite.SqliteConnection con)
|
||||
{
|
||||
var opts = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS_DUPLICATI") ?? "";
|
||||
if (pagecachesize > MINIMUM_SQLITE_PAGE_CACHE_SIZE)
|
||||
opts = string.Format(CultureInfo.InvariantCulture, "cache_size=-{0};{1}", pagecachesize / 1024L, opts);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(opts))
|
||||
return con;
|
||||
Dictionary<string, string> customOptions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
{ "synchronous", "NORMAL" }, // NORMAL is more performant than FULL (the default), but less safe, as it no longer blocks until the disk sync syscall to the OS has completed.
|
||||
{ "temp_store", "MEMORY" }, // Use memory for temporary storage to improve performance.
|
||||
{ "journal_mode", "WAL" }, // Use Write-Ahead Logging for better concurrency and performance.
|
||||
{ "cache_size", "-65536" }, // Set cache size to 64 MB (negative value means in KB, so -64000 = 64 MB). Default is 2000 pages, which is 2 MB (2000 * 1024 bytes).
|
||||
{ "mmap_size", "67108864" }, // 64 MB.
|
||||
{ "threads", "8" }, // Use 8 threads for parallel processing where applicable.
|
||||
{ "shared_cache", "true" } //
|
||||
};
|
||||
// Override the default options with any custom options set in the environment variable.
|
||||
var customOptionsEnv = Environment.GetEnvironmentVariable("CUSTOMSQLITEOPTIONS");
|
||||
if (!string.IsNullOrWhiteSpace(customOptionsEnv))
|
||||
{
|
||||
foreach (var opt in customOptionsEnv.Split([';'], StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var parts = opt.Split(new[] { '=' }, 2);
|
||||
if (parts.Length == 2)
|
||||
customOptions[parts[0].Trim()] = parts[1].Trim();
|
||||
}
|
||||
}
|
||||
|
||||
using (var cmd = con.CreateCommand())
|
||||
foreach (var opt in opts.Split(new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
foreach (var (key, value) in customOptions)
|
||||
{
|
||||
var opt = $"{key}={value}";
|
||||
Logging.Log.WriteVerboseMessage(LOGTAG, "CustomSQLiteOption", @"Setting custom SQLite option '{0}'.", opt);
|
||||
try
|
||||
{
|
||||
cmd.CommandText = string.Format(CultureInfo.InvariantCulture, "PRAGMA {0}", opt);
|
||||
cmd.ExecuteNonQuery();
|
||||
await cmd.ExecuteNonQueryAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteWarningMessage(LOGTAG, "CustomSQLiteOption", ex, @"Error setting custom SQLite option '{0}'.", opt);
|
||||
}
|
||||
}
|
||||
}
|
||||
return con;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads an SQLite connection instance and opening the database
|
||||
/// Loads an SQLite connection instance and opening the database with a specified page cache size.
|
||||
/// </summary>
|
||||
/// <returns>The SQLite connection instance.</returns>
|
||||
/// <param name="targetpath">The optional path to the database.</param>
|
||||
/// <param name="pagecachesize"> The page cache size to set.</param>
|
||||
public static System.Data.IDbConnection LoadConnection(string targetpath, long pagecachesize)
|
||||
/// <returns>The SQLite connection instance.</returns>
|
||||
/// <remarks>
|
||||
/// This method is synchronous and should be used when you need to load the connection immediately. It calls the asynchronous version and waits for it to complete.
|
||||
/// </remarks>
|
||||
public static Microsoft.Data.Sqlite.SqliteConnection LoadConnection(string targetpath)
|
||||
{
|
||||
return LoadConnectionAsync(targetpath).Await();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads an SQLite connection instance and opening the database.
|
||||
/// </summary>
|
||||
/// <param name="targetpath">The optional path to the database.</param>
|
||||
/// <returns>A task that when waited returns the SQLite connection instance.</returns>
|
||||
public static async Task<Microsoft.Data.Sqlite.SqliteConnection> LoadConnectionAsync(string targetpath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(targetpath))
|
||||
throw new ArgumentNullException(nameof(targetpath));
|
||||
|
||||
var con = LoadConnection();
|
||||
var con = await LoadConnectionAsync().ConfigureAwait(false);
|
||||
|
||||
try
|
||||
{
|
||||
OpenSQLiteFile(con, targetpath);
|
||||
await OpenSQLiteFileAsync(con, targetpath).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log.WriteErrorMessage(LOGTAG, "FailedToLoadConnectionSQLite", ex, @"Failed to load connection with path '{0}'.", targetpath);
|
||||
DisposeConnection(con);
|
||||
await DisposeConnectionAsync(con).ConfigureAwait(false);
|
||||
|
||||
throw;
|
||||
}
|
||||
|
||||
// set custom Sqlite options
|
||||
return ApplyCustomPragmas(con, pagecachesize);
|
||||
return await ApplyCustomPragmasAsync(con)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the SQLiteCommand type for the current architecture
|
||||
/// Returns the SQLiteCommand type for the current architecture.
|
||||
/// </summary>
|
||||
public static Type SQLiteConnectionType
|
||||
{
|
||||
get
|
||||
{
|
||||
return typeof(System.Data.SQLite.SQLiteConnection);
|
||||
return typeof(Microsoft.Data.Sqlite.SqliteConnection);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the version string from the SQLite type
|
||||
/// Returns the version string from the SQLite type.
|
||||
/// </summary>
|
||||
public static string? SQLiteVersion
|
||||
{
|
||||
@@ -212,31 +259,27 @@ namespace Duplicati.Library.SQLiteHelper
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Wrapper to dispose the SQLite connection
|
||||
/// Wrapper to dispose the SQLite connection.
|
||||
/// </summary>
|
||||
/// <param name="con">The connection to close.</param>
|
||||
private static void DisposeConnection(System.Data.IDbConnection? con)
|
||||
/// <returns>A task that completes when the connection is disposed.</returns>
|
||||
private static async Task DisposeConnectionAsync(Microsoft.Data.Sqlite.SqliteConnection? con)
|
||||
{
|
||||
if (con != null)
|
||||
try { con.Dispose(); }
|
||||
try { await con.DisposeAsync().ConfigureAwait(false); }
|
||||
catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "ConnectionDisposeError", ex, "Failed to dispose connection"); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Opens the SQLite file in the given connection, creating the file if required
|
||||
/// Opens the SQLite file in the given connection, creating the file if required.
|
||||
/// </summary>
|
||||
/// <param name="con">The connection to use.</param>
|
||||
/// <param name="path">Path to the file to open, which may not exist.</param>
|
||||
private static void OpenSQLiteFile(System.Data.IDbConnection con, string path)
|
||||
/// <returns>A task that completes when the file is opened.</returns>
|
||||
private static async Task OpenSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con, string path)
|
||||
{
|
||||
con.ConnectionString = "Data Source=" + path;
|
||||
con.Open();
|
||||
if (con is System.Data.SQLite.SQLiteConnection sqlitecon && !OperatingSystem.IsMacOS())
|
||||
{
|
||||
// These configuration options crash on MacOS (arm64), but the other platforms should be enough to detect incorrect SQL
|
||||
sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DDL, false);
|
||||
sqlitecon.SetConfigurationOption(System.Data.SQLite.SQLiteConfigDbOpsEnum.SQLITE_DBCONFIG_DQS_DML, false);
|
||||
}
|
||||
con.ConnectionString = $"Data Source={path};Pooling=false";
|
||||
await con.OpenAsync().ConfigureAwait(false);
|
||||
|
||||
// Make the file only accessible by the current user, unless opting out
|
||||
if (!SystemIO.IO_OS.FileExists(SystemIO.IO_OS.PathCombine(SystemIO.IO_OS.PathGetDirectoryName(path), Util.InsecurePermissionsMarkerFile)))
|
||||
@@ -245,17 +288,16 @@ namespace Duplicati.Library.SQLiteHelper
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tests the SQLite connection, throwing an exception if the connection does not work
|
||||
/// Tests the SQLite connection, throwing an exception if the connection does not work.
|
||||
/// </summary>
|
||||
/// <param name="con">The connection to test.</param>
|
||||
private static void TestSQLiteFile(System.Data.IDbConnection con)
|
||||
/// <returns>A task that completes when the test query is executed.</returns>
|
||||
private static async Task TestSQLiteFileAsync(Microsoft.Data.Sqlite.SqliteConnection con)
|
||||
{
|
||||
// Do a dummy query to make sure we have a working db
|
||||
using (var cmd = con.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "SELECT COUNT(*) FROM SQLITE_MASTER";
|
||||
cmd.ExecuteScalar();
|
||||
}
|
||||
using var cmd = con.CreateCommand();
|
||||
cmd.CommandText = "SELECT COUNT(*) FROM SQLITE_MASTER";
|
||||
await cmd.ExecuteScalarAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -762,7 +762,7 @@ namespace Duplicati.Server
|
||||
System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(databasePath));
|
||||
|
||||
// Attempt to open the database, removing any encryption present
|
||||
Library.SQLiteHelper.SQLiteLoader.OpenDatabase(con, databasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions));
|
||||
Library.SQLiteHelper.SQLiteLoader.OpenDatabaseAsync(con, databasePath, Library.SQLiteHelper.SQLiteRC4Decrypter.GetEncryptionPassword(commandlineOptions)).Await();
|
||||
|
||||
Library.SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(con, databasePath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker));
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.CommandLine.DatabaseTool;
|
||||
@@ -35,7 +35,7 @@ namespace Duplicati.UnitTest
|
||||
public async Task TestLocalDbMethods()
|
||||
{
|
||||
using var dbfile = new TempFile();
|
||||
using var db = SQLiteLoader.LoadConnection(dbfile, 0);
|
||||
using var db = SQLiteLoader.LoadConnection(dbfile);
|
||||
using var cmd = db.CreateCommand();
|
||||
cmd.CommandText = LocalSchemaV12;
|
||||
cmd.ExecuteNonQuery();
|
||||
@@ -55,7 +55,7 @@ namespace Duplicati.UnitTest
|
||||
public async Task TestServerDbMethods()
|
||||
{
|
||||
using var dbfile = new TempFile();
|
||||
using var db = SQLiteLoader.LoadConnection(dbfile, 0);
|
||||
using var db = SQLiteLoader.LoadConnection(dbfile);
|
||||
using var cmd = db.CreateCommand();
|
||||
cmd.CommandText = ServerSchemaV6;
|
||||
cmd.ExecuteNonQuery();
|
||||
@@ -142,8 +142,8 @@ CREATE TABLE ""Notification"" (
|
||||
""ID"" INTEGER PRIMARY KEY,
|
||||
""Type"" TEXT NOT NULL,
|
||||
""Title"" TEXT NOT NULL,
|
||||
""Message"" TEXT NOT NULL,
|
||||
""Exception"" TEXT NOT NULL,
|
||||
""Message"" TEXT NOT NULL,
|
||||
""Exception"" TEXT NOT NULL,
|
||||
""BackupID"" TEXT NULL,
|
||||
""Action"" TEXT NOT NULL,
|
||||
""Timestamp"" INTEGER NOT NULL,
|
||||
@@ -153,15 +153,15 @@ CREATE TABLE ""Notification"" (
|
||||
);
|
||||
|
||||
CREATE TABLE ""UIStorage"" (
|
||||
""Scheme"" TEXT NOT NULL,
|
||||
""Key"" TEXT NOT NULL,
|
||||
""Scheme"" TEXT NOT NULL,
|
||||
""Key"" TEXT NOT NULL,
|
||||
""Value"" TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE ""TempFile"" (
|
||||
""ID"" INTEGER PRIMARY KEY,
|
||||
""Origin"" TEXT NOT NULL,
|
||||
""Path"" TEXT NOT NULL,
|
||||
""Origin"" TEXT NOT NULL,
|
||||
""Path"" TEXT NOT NULL,
|
||||
""Timestamp"" INTEGER NOT NULL,
|
||||
""Expires"" INTEGER NOT NULL
|
||||
);
|
||||
@@ -251,7 +251,7 @@ CREATE TABLE ""Blockset"" (
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX ""BlocksetFullHash"" ON ""Blockset"" (""FullHash"", ""Length"");
|
||||
|
||||
|
||||
CREATE TABLE ""BlocksetEntry"" (
|
||||
""BlocksetID"" INTEGER NOT NULL,
|
||||
""Index"" INTEGER NOT NULL,
|
||||
@@ -326,11 +326,11 @@ CREATE TABLE ""Configuration"" (
|
||||
|
||||
CREATE TABLE ""ChangeJournalData"" (
|
||||
""ID"" INTEGER PRIMARY KEY,
|
||||
""FilesetID"" INTEGER NOT NULL,
|
||||
""VolumeName"" TEXT NOT NULL,
|
||||
""JournalID"" INTEGER NOT NULL,
|
||||
""NextUsn"" INTEGER NOT NULL,
|
||||
""ConfigHash"" TEXT NOT NULL
|
||||
""FilesetID"" INTEGER NOT NULL,
|
||||
""VolumeName"" TEXT NOT NULL,
|
||||
""JournalID"" INTEGER NOT NULL,
|
||||
""NextUsn"" INTEGER NOT NULL,
|
||||
""ConfigHash"" TEXT NOT NULL
|
||||
);
|
||||
|
||||
INSERT INTO ""Version"" (""Version"") VALUES (12);
|
||||
|
||||
@@ -2,6 +2,8 @@ using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Main;
|
||||
using Duplicati.Library.Main.Database;
|
||||
using Duplicati.Library.Utility;
|
||||
@@ -321,10 +323,11 @@ namespace Duplicati.UnitTest
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetMinimalUniquePrefixEntries_ShouldReturnCorrectLinuxPrefixes()
|
||||
public async Task GetMinimalUniquePrefixEntries_ShouldReturnCorrectLinuxPrefixes()
|
||||
{
|
||||
using var tempFile = new TempFile();
|
||||
using var db = new LocalListDatabase(tempFile, 1);
|
||||
using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
SeedTestData(db, [
|
||||
"/folder1/",
|
||||
"/folder1/sub1/",
|
||||
@@ -333,7 +336,11 @@ namespace Duplicati.UnitTest
|
||||
"/folder3/"
|
||||
]);
|
||||
|
||||
var result = db.GetMinimalUniquePrefixEntries(1).Select(e => e.Path).ToList();
|
||||
var result = await db
|
||||
.GetMinimalUniquePrefixEntries(1, CancellationToken.None)
|
||||
.Select(e => e.Path)
|
||||
.ToListAsync()
|
||||
.ConfigureAwait(false);
|
||||
|
||||
Assert.That(result, Does.Contain("/folder1/"));
|
||||
Assert.That(result, Does.Contain("/folder2/"));
|
||||
@@ -344,10 +351,11 @@ namespace Duplicati.UnitTest
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetMinimalUniquePrefixEntries_ShouldReturnCorrectWindowsDrivePrefixes()
|
||||
public async Task GetMinimalUniquePrefixEntries_ShouldReturnCorrectWindowsDrivePrefixes()
|
||||
{
|
||||
using var tempFile = new TempFile();
|
||||
using var db = new LocalListDatabase(tempFile, 1);
|
||||
using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
SeedTestData(db, [
|
||||
"C:\\folder1\\",
|
||||
"C:\\folder1\\sub1\\",
|
||||
@@ -355,7 +363,11 @@ namespace Duplicati.UnitTest
|
||||
"D:\\otherfolder\\"
|
||||
]);
|
||||
|
||||
var result = db.GetMinimalUniquePrefixEntries(1).Select(e => e.Path).ToList();
|
||||
var result = await db
|
||||
.GetMinimalUniquePrefixEntries(1, CancellationToken.None)
|
||||
.Select(e => e.Path)
|
||||
.ToListAsync()
|
||||
.ConfigureAwait(false);
|
||||
|
||||
Assert.That(result, Does.Contain("C:\\folder1\\"));
|
||||
Assert.That(result, Does.Contain("C:\\folder2\\"));
|
||||
@@ -365,17 +377,22 @@ namespace Duplicati.UnitTest
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetMinimalUniquePrefixEntries_ShouldReturnCorrectWindowsUncPrefixes()
|
||||
public async Task GetMinimalUniquePrefixEntries_ShouldReturnCorrectWindowsUncPrefixes()
|
||||
{
|
||||
using var tempFile = new TempFile();
|
||||
using var db = new LocalListDatabase(tempFile, 1);
|
||||
using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
SeedTestData(db, [
|
||||
"\\\\server\\share\\folder\\",
|
||||
"\\\\server\\share\\folder\\subfolder\\",
|
||||
"\\\\server\\share\\otherfolder\\"
|
||||
]);
|
||||
|
||||
var result = db.GetMinimalUniquePrefixEntries(1).Select(e => e.Path).ToList();
|
||||
var result = await db
|
||||
.GetMinimalUniquePrefixEntries(1, CancellationToken.None)
|
||||
.Select(e => e.Path)
|
||||
.ToListAsync()
|
||||
.ConfigureAwait(false);
|
||||
|
||||
Assert.That(result, Does.Contain("\\\\server\\share\\folder\\"));
|
||||
Assert.That(result, Does.Contain("\\\\server\\share\\otherfolder\\"));
|
||||
@@ -384,10 +401,11 @@ namespace Duplicati.UnitTest
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetMinimalUniquePrefixEntries_ShouldHandleMixedWindowsDriveAndUncPaths()
|
||||
public async Task GetMinimalUniquePrefixEntries_ShouldHandleMixedWindowsDriveAndUncPaths()
|
||||
{
|
||||
using var tempFile = new TempFile();
|
||||
using var db = new LocalListDatabase(tempFile, 1);
|
||||
using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
SeedTestData(db, [
|
||||
"C:\\data\\",
|
||||
"C:\\data\\sub1\\",
|
||||
@@ -398,7 +416,11 @@ namespace Duplicati.UnitTest
|
||||
"\\\\server\\share\\pictures\\"
|
||||
]);
|
||||
|
||||
var result = db.GetMinimalUniquePrefixEntries(1).Select(e => e.Path).ToList();
|
||||
var result = await db
|
||||
.GetMinimalUniquePrefixEntries(1, CancellationToken.None)
|
||||
.Select(e => e.Path)
|
||||
.ToListAsync()
|
||||
.ConfigureAwait(false);
|
||||
|
||||
Assert.That(result, Does.Contain("C:\\data\\"));
|
||||
Assert.That(result, Does.Contain("C:\\music\\"));
|
||||
@@ -411,7 +433,7 @@ namespace Duplicati.UnitTest
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void GetMinimalUniquePrefixEntries_ShouldReturnExpectedMinimalRoots()
|
||||
public async Task GetMinimalUniquePrefixEntries_ShouldReturnExpectedMinimalRoots()
|
||||
{
|
||||
// Arrange: Prepare prefixes (minimal unique) and contents
|
||||
var testPrefixes = new[]
|
||||
@@ -430,10 +452,15 @@ namespace Duplicati.UnitTest
|
||||
|
||||
// Act
|
||||
using var tempFile = new TempFile();
|
||||
using var db = new LocalListDatabase(tempFile, 1);
|
||||
await using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
SeedTestData(db, testPrefixes);
|
||||
|
||||
var resultItems = db.GetMinimalUniquePrefixEntries(1).ToList();
|
||||
var resultItems = await db
|
||||
.GetMinimalUniquePrefixEntries(1, CancellationToken.None)
|
||||
.ToListAsync()
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var result = resultItems.Select(e => e.Path).ToList();
|
||||
|
||||
Assert.That(result, Does.Contain(@"C:\Downloads\testsource\AA\"));
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
@@ -732,7 +732,7 @@ namespace Duplicati.UnitTest
|
||||
|
||||
[Test]
|
||||
[Category("Disruption")]
|
||||
public void TestUploadExceptionWithNoRetries()
|
||||
public async Task TestUploadExceptionWithNoRetries()
|
||||
{
|
||||
var testopts = TestOptions;
|
||||
testopts["number-of-retries"] = "0";
|
||||
@@ -806,7 +806,7 @@ namespace Duplicati.UnitTest
|
||||
}
|
||||
catch (TestUtils.TestVerificationException e)
|
||||
{
|
||||
using var db = new LocalDatabase(testopts["dbpath"], "test", true, 0);
|
||||
using var db = await LocalDatabase.CreateLocalDatabaseAsync(testopts["dbpath"], "test", true, null, CancellationToken.None).ConfigureAwait(false);
|
||||
using var cmd = db.Connection.CreateCommand();
|
||||
|
||||
var sb = new StringBuilder();
|
||||
@@ -1097,7 +1097,7 @@ namespace Duplicati.UnitTest
|
||||
// If we prevent the file from being uploaded,
|
||||
// it will be a temporary file, and a new synthetic file
|
||||
// will be attempted, but that fails as well
|
||||
// If it is after, then the upload completes,
|
||||
// If it is after, then the upload completes,
|
||||
// and it will be promoted to a regular uploaded backup
|
||||
expectedFilesets += before ? 1 : runs;
|
||||
|
||||
|
||||
@@ -28,6 +28,9 @@ using Duplicati.Library.Main.Database;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.SQLiteHelper;
|
||||
using NUnit.Framework;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Utility;
|
||||
|
||||
namespace Duplicati.UnitTest;
|
||||
|
||||
@@ -35,7 +38,7 @@ public class EmptyMetadataTests : BasicSetupHelper
|
||||
{
|
||||
[Test]
|
||||
[Category("Targeted")]
|
||||
public void ReplaceMissingMetadataRestoresConsistency()
|
||||
public async Task ReplaceMissingMetadataRestoresConsistency()
|
||||
{
|
||||
var testopts = TestOptions.Expand(new { no_encryption = true });
|
||||
|
||||
@@ -49,7 +52,7 @@ public class EmptyMetadataTests : BasicSetupHelper
|
||||
long metaBlocksetId;
|
||||
long filesetId;
|
||||
|
||||
using (var db = SQLiteLoader.LoadConnection(DBFILE, 0))
|
||||
using (var db = SQLiteLoader.LoadConnection(DBFILE))
|
||||
using (var cmd = db.CreateCommand())
|
||||
{
|
||||
// Find metadata blockset ID and fileset ID for the backed up folder
|
||||
@@ -79,26 +82,28 @@ public class EmptyMetadataTests : BasicSetupHelper
|
||||
}
|
||||
|
||||
var opts = new Options(testopts);
|
||||
var emptyMeta = Utility.WrapMetadata(new Dictionary<string, string>(), opts);
|
||||
var emptyMeta = Duplicati.Library.Main.Utility.WrapMetadata(new Dictionary<string, string>(), opts);
|
||||
|
||||
Assert.Throws<DatabaseInconsistencyException>(() =>
|
||||
{
|
||||
using var db = new LocalDatabase(DBFILE, "verify", true, 0);
|
||||
db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, null);
|
||||
using var db = LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "verify", true, null, CancellationToken.None).Await();
|
||||
db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, CancellationToken.None).Await();
|
||||
});
|
||||
|
||||
using (var db = new LocalListBrokenFilesDatabase(DBFILE, 0))
|
||||
await using (var db = await LocalListBrokenFilesDatabase.CreateAsync(DBFILE, null, CancellationToken.None).ConfigureAwait(false))
|
||||
{
|
||||
var blockVolumeIds = Array.Empty<long>();
|
||||
|
||||
var emptyId = db.GetEmptyMetadataBlocksetId(blockVolumeIds, emptyMeta.FileHash, emptyMeta.Blob.Length, null);
|
||||
var emptyId = await db.GetEmptyMetadataBlocksetId(blockVolumeIds, emptyMeta.FileHash, emptyMeta.Blob.Length, CancellationToken.None);
|
||||
Assert.That(emptyId, Is.GreaterThanOrEqualTo(0), "Empty metadata blockset not found");
|
||||
|
||||
var replaced = db.ReplaceMetadata(filesetId, emptyId, null);
|
||||
var replaced = await db.ReplaceMetadata(filesetId, emptyId, CancellationToken.None);
|
||||
Assert.That(replaced, Is.GreaterThan(0), "No metadata rows replaced");
|
||||
|
||||
await db.Transaction.CommitAsync(CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
using (var db = new LocalDatabase(DBFILE, "verify", true, 0))
|
||||
db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, null);
|
||||
await using (var db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "verify", true, null, CancellationToken.None))
|
||||
await db.VerifyConsistency(opts.Blocksize, opts.BlockhashSize, true, CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,27 +1,28 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.RestAPI;
|
||||
using Duplicati.Library.SQLiteHelper;
|
||||
using Duplicati.Server;
|
||||
@@ -124,7 +125,7 @@ namespace Duplicati.UnitTest
|
||||
|
||||
[Test]
|
||||
[Category("ImportExport")]
|
||||
public void RoundTrip()
|
||||
public async Task RoundTrip()
|
||||
{
|
||||
var metadata = new Dictionary<string, string> { { "SourceFilesCount", "1" } };
|
||||
var advancedOptions = new Dictionary<string, string> { { "server-datafolder", this.serverDatafolder } };
|
||||
@@ -133,8 +134,8 @@ namespace Duplicati.UnitTest
|
||||
if (File.Exists(dbpath))
|
||||
File.Delete(dbpath);
|
||||
|
||||
var con = SQLiteLoader.LoadConnection();
|
||||
SQLiteLoader.OpenDatabase(con, dbpath, null);
|
||||
var con = await SQLiteLoader.LoadConnectionAsync().ConfigureAwait(false);
|
||||
await SQLiteLoader.OpenDatabaseAsync(con, dbpath, null).ConfigureAwait(false);
|
||||
DatabaseUpgrader.UpgradeDatabase(con, dbpath, typeof(Library.RestAPI.Database.DatabaseSchemaMarker));
|
||||
|
||||
using (var connection = new Connection(con, true, null, this.serverDatafolder, null))
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -156,7 +156,7 @@ namespace Duplicati.UnitTest
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
|
||||
|
||||
using (var db = SQLiteLoader.LoadConnection(DBFILE, 0))
|
||||
using (var db = SQLiteLoader.LoadConnection(DBFILE))
|
||||
using (var cmd = db.CreateCommand())
|
||||
{
|
||||
var deletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock""");
|
||||
@@ -172,14 +172,14 @@ namespace Duplicati.UnitTest
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
|
||||
|
||||
using (var db = SQLiteLoader.LoadConnection(DBFILE, 0))
|
||||
using (var db = SQLiteLoader.LoadConnection(DBFILE))
|
||||
using (var cmd = db.CreateCommand())
|
||||
{
|
||||
var deletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock""");
|
||||
Assert.That(deletedBlocks, Is.GreaterThan(0), "DeletedBlock table is empty");
|
||||
}
|
||||
|
||||
using (var db = SQLiteLoader.LoadConnection(DBFILE, 0))
|
||||
using (var db = SQLiteLoader.LoadConnection(DBFILE))
|
||||
using (var cmd = db.CreateCommand())
|
||||
{
|
||||
var deletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock""");
|
||||
@@ -193,7 +193,7 @@ namespace Duplicati.UnitTest
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
|
||||
|
||||
using (var db = SQLiteLoader.LoadConnection(DBFILE, 0))
|
||||
using (var db = SQLiteLoader.LoadConnection(DBFILE))
|
||||
using (var cmd = db.CreateCommand())
|
||||
{
|
||||
var alldeletedBlocks = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""DeletedBlock""");
|
||||
|
||||
+150
-150
@@ -1,150 +1,150 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Duplicati.Library.Utility;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace Duplicati.UnitTest
|
||||
{
|
||||
public class Issue5066 : BasicSetupHelper
|
||||
{
|
||||
|
||||
private static string CalculateFileHash(string filename)
|
||||
{
|
||||
using (var fs = File.OpenRead(filename))
|
||||
using (var hasher = HashFactory.CreateHasher("SHA256"))
|
||||
return Convert.ToBase64String(hasher.ComputeHash(fs));
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("Targeted")]
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void TestDuplicatedBlocklists1(bool deleteAllIndexFiles)
|
||||
{
|
||||
var testopts = TestOptions.Expand(new { blocksize = "1kb" });
|
||||
var hashes = new List<string>();
|
||||
|
||||
// Full blocklist with zeroes
|
||||
var data = new byte[32769];
|
||||
data.AsSpan().Fill((byte)'a');
|
||||
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
|
||||
hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a")));
|
||||
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
|
||||
|
||||
// Make the first blocklist different
|
||||
data[0] = (byte)'b';
|
||||
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
|
||||
hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a")));
|
||||
|
||||
// Record existing dindex files
|
||||
var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList();
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
|
||||
|
||||
// Delete new index files
|
||||
foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly))
|
||||
{
|
||||
if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles)
|
||||
File.Delete(file);
|
||||
}
|
||||
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
TestUtils.AssertResults(c.Repair());
|
||||
TestUtils.AssertResults(c.Test());
|
||||
}
|
||||
|
||||
for (var version = 0; version < 2; version++)
|
||||
{
|
||||
File.Delete(Path.Combine(DATAFOLDER, "a"));
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null))
|
||||
{
|
||||
TestUtils.AssertResults(c.Restore(null));
|
||||
var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a"));
|
||||
Assert.That(hashes[version], Is.EqualTo(hash), "Hash mismatch for version " + version);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("Targeted")]
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void TestDuplicatedBlocklists2(bool deleteAllIndexFiles)
|
||||
{
|
||||
var testopts = TestOptions.Expand(new { blocksize = "1kb" });
|
||||
var hashes = new List<string>();
|
||||
|
||||
// Full blocklist with zeroes
|
||||
var data = new byte[32769];
|
||||
data.AsSpan().Fill((byte)'a');
|
||||
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
|
||||
hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a")));
|
||||
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
|
||||
|
||||
// Expand to an new set
|
||||
data = new byte[32769 * 2 + 1];
|
||||
data.AsSpan().Fill((byte)'b');
|
||||
data[data.Length - 1] = (byte)'a';
|
||||
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
|
||||
hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a")));
|
||||
|
||||
// Record existing dindex files
|
||||
var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList();
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
|
||||
|
||||
// Delete new index files
|
||||
foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly))
|
||||
{
|
||||
if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles)
|
||||
File.Delete(file);
|
||||
}
|
||||
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
TestUtils.AssertResults(c.Repair());
|
||||
TestUtils.AssertResults(c.Test());
|
||||
}
|
||||
|
||||
for (var version = 0; version < 2; version++)
|
||||
{
|
||||
File.Delete(Path.Combine(DATAFOLDER, "a"));
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null))
|
||||
{
|
||||
TestUtils.AssertResults(c.Restore(null));
|
||||
var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a"));
|
||||
Assert.That(hashes[version], Is.EqualTo(hash), "Hash mismatch for version " + version);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using Duplicati.Library.Utility;
|
||||
using NUnit.Framework;
|
||||
|
||||
namespace Duplicati.UnitTest
|
||||
{
|
||||
public class Issue5066 : BasicSetupHelper
|
||||
{
|
||||
|
||||
private static string CalculateFileHash(string filename)
|
||||
{
|
||||
using (var fs = File.OpenRead(filename))
|
||||
using (var hasher = HashFactory.CreateHasher("SHA256"))
|
||||
return Convert.ToBase64String(hasher.ComputeHash(fs));
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("Targeted")]
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void TestDuplicatedBlocklists1(bool deleteAllIndexFiles)
|
||||
{
|
||||
var testopts = TestOptions.Expand(new { blocksize = "1kb" });
|
||||
var hashes = new List<string>();
|
||||
|
||||
// Full blocklist with zeroes
|
||||
var data = new byte[32769];
|
||||
data.AsSpan().Fill((byte)'a');
|
||||
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
|
||||
hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a")));
|
||||
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
|
||||
|
||||
// Make the first blocklist different
|
||||
data[0] = (byte)'b';
|
||||
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
|
||||
hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a")));
|
||||
|
||||
// Record existing dindex files
|
||||
var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList();
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
|
||||
|
||||
// Delete new index files
|
||||
foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly))
|
||||
{
|
||||
if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles)
|
||||
File.Delete(file);
|
||||
}
|
||||
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
TestUtils.AssertResults(c.Repair());
|
||||
TestUtils.AssertResults(c.Test());
|
||||
}
|
||||
|
||||
for (var version = 0; version < 2; version++)
|
||||
{
|
||||
File.Delete(Path.Combine(DATAFOLDER, "a"));
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null))
|
||||
{
|
||||
TestUtils.AssertResults(c.Restore(null));
|
||||
var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a"));
|
||||
Assert.That(hashes[version], Is.EqualTo(hash), "Hash mismatch for version " + version);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("Targeted")]
|
||||
[TestCase(true)]
|
||||
[TestCase(false)]
|
||||
public void TestDuplicatedBlocklists2(bool deleteAllIndexFiles)
|
||||
{
|
||||
var testopts = TestOptions.Expand(new { blocksize = "1kb" });
|
||||
var hashes = new List<string>();
|
||||
|
||||
// Full blocklist with zeroes
|
||||
var data = new byte[32769];
|
||||
data.AsSpan().Fill((byte)'a');
|
||||
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
|
||||
hashes.Add(CalculateFileHash(Path.Combine(DATAFOLDER, "a")));
|
||||
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
|
||||
|
||||
// Expand to an new set
|
||||
data = new byte[32769 * 2 + 1];
|
||||
data.AsSpan().Fill((byte)'b');
|
||||
data[data.Length - 1] = (byte)'a';
|
||||
File.WriteAllBytes(Path.Combine(DATAFOLDER, "a"), data);
|
||||
hashes.Insert(0, CalculateFileHash(Path.Combine(DATAFOLDER, "a")));
|
||||
|
||||
// Record existing dindex files
|
||||
var existingDIndexFiles = Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly).ToList();
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
TestUtils.AssertResults(c.Backup([DATAFOLDER]));
|
||||
|
||||
// Delete new index files
|
||||
foreach (var file in Directory.GetFiles(TARGETFOLDER, "*.dindex*", SearchOption.TopDirectoryOnly))
|
||||
{
|
||||
if (existingDIndexFiles.Contains(file) || deleteAllIndexFiles)
|
||||
File.Delete(file);
|
||||
}
|
||||
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts, null))
|
||||
{
|
||||
TestUtils.AssertResults(c.Repair());
|
||||
TestUtils.AssertResults(c.Test());
|
||||
}
|
||||
|
||||
for (var version = 0; version < 2; version++)
|
||||
{
|
||||
File.Delete(Path.Combine(DATAFOLDER, "a"));
|
||||
using (var c = new Library.Main.Controller("file://" + TARGETFOLDER, testopts.Expand(new { version = version }), null))
|
||||
{
|
||||
TestUtils.AssertResults(c.Restore(null));
|
||||
var hash = CalculateFileHash(Path.Combine(DATAFOLDER, "a"));
|
||||
Assert.That(hashes[version], Is.EqualTo(hash), "Hash mismatch for version " + version);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
@@ -25,6 +25,7 @@ using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Interface;
|
||||
using Duplicati.Library.Main;
|
||||
using Duplicati.Library.Main.Database;
|
||||
@@ -86,7 +87,7 @@ namespace Duplicati.UnitTest
|
||||
List<int> expectedBlocksetIDs = new List<int>();
|
||||
List<int> expectedIndexes = new List<int>();
|
||||
List<string> expectedHashes = new List<string>();
|
||||
using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"], 0))
|
||||
using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"]))
|
||||
{
|
||||
// Read the contents of the BlocklistHash table so that we can
|
||||
// compare them to the contents after the repair operation.
|
||||
@@ -123,7 +124,7 @@ namespace Duplicati.UnitTest
|
||||
List<int> repairedBlocksetIDs = new List<int>();
|
||||
List<int> repairedIndexes = new List<int>();
|
||||
List<string> repairedHashes = new List<string>();
|
||||
using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"], 0))
|
||||
using (IDbConnection connection = SQLiteLoader.LoadConnection(options["dbpath"]))
|
||||
{
|
||||
using (IDbCommand command = connection.CreateCommand())
|
||||
{
|
||||
@@ -403,7 +404,7 @@ namespace Duplicati.UnitTest
|
||||
[Category("RepairHandler")]
|
||||
[TestCase("true")]
|
||||
[TestCase("false")]
|
||||
public void RepairExtraIndexFiles(string noEncryption)
|
||||
public async Task RepairExtraIndexFiles(string noEncryption)
|
||||
{
|
||||
// Extra index files will be added to the database and should have a correct link established
|
||||
Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions) { ["no-encryption"] = noEncryption };
|
||||
@@ -427,18 +428,30 @@ namespace Duplicati.UnitTest
|
||||
}
|
||||
|
||||
// Check database block link
|
||||
using (LocalDatabase db = new LocalDatabase(DBFILE, "Test", true, 0))
|
||||
using (LocalDatabase db = await LocalDatabase.CreateLocalDatabaseAsync(DBFILE, "Test", true, null, CancellationToken.None).ConfigureAwait(false))
|
||||
{
|
||||
var indexVolumeId = db.GetRemoteVolumeID(Path.GetFileName(origFile));
|
||||
var duplicateVolumeId = db.GetRemoteVolumeID(Path.GetFileName(dupFile));
|
||||
var indexVolumeId = await db
|
||||
.GetRemoteVolumeID(Path.GetFileName(origFile), CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
var duplicateVolumeId = await db
|
||||
.GetRemoteVolumeID(Path.GetFileName(dupFile), CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
Assert.AreNotEqual(-1, indexVolumeId);
|
||||
Assert.AreNotEqual(-1, duplicateVolumeId);
|
||||
|
||||
using (var cmd = db.Connection.CreateCommand())
|
||||
{
|
||||
var sql = @"SELECT ""BlockVolumeID"" FROM ""IndexBlockLink"" WHERE ""IndexVolumeID"" = @VolumeId";
|
||||
var linkedIndexId = cmd.SetCommandAndParameters(sql).SetParameterValue("@VolumeId", indexVolumeId).ExecuteScalarInt64(-1);
|
||||
var linkedDuplicateId = cmd.SetCommandAndParameters(sql).SetParameterValue("@VolumeId", duplicateVolumeId).ExecuteScalarInt64(-1);
|
||||
var linkedIndexId = await cmd
|
||||
.SetCommandAndParameters(sql)
|
||||
.SetParameterValue("@VolumeId", indexVolumeId)
|
||||
.ExecuteScalarInt64Async(-1, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
var linkedDuplicateId = await cmd
|
||||
.SetCommandAndParameters(sql)
|
||||
.SetParameterValue("@VolumeId", duplicateVolumeId)
|
||||
.ExecuteScalarInt64Async(-1, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
Assert.AreEqual(linkedIndexId, linkedDuplicateId);
|
||||
}
|
||||
}
|
||||
@@ -494,7 +507,7 @@ namespace Duplicati.UnitTest
|
||||
dlistFiles = dlistFiles.Reverse();
|
||||
|
||||
var dlistFile = dlistFiles.First();
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0))
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"]))
|
||||
using (var cmd = con.CreateCommand("DELETE FROM RemoteVolume WHERE Name = @Name"))
|
||||
Assert.AreEqual(1, cmd.SetParameterValue("@Name", Path.GetFileName(dlistFile)).ExecuteNonQuery());
|
||||
|
||||
@@ -514,7 +527,7 @@ namespace Duplicati.UnitTest
|
||||
}
|
||||
|
||||
// Check that the entry was recreated
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0))
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"]))
|
||||
using (var cmd = con.CreateCommand("SELECT COUNT(*) FROM RemoteVolume WHERE Name LIKE '%.dlist.%' AND State != @State"))
|
||||
Assert.AreEqual(2, cmd.SetParameterValue("@State", RemoteVolumeState.Deleted.ToString()).ExecuteScalarInt64(-1));
|
||||
|
||||
@@ -551,7 +564,7 @@ namespace Duplicati.UnitTest
|
||||
TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER }));
|
||||
|
||||
// Remove a fileset
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0))
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"]))
|
||||
using (var cmd = con.CreateCommand())
|
||||
{
|
||||
var filesetId = cmd.ExecuteScalarInt64("SELECT Id FROM Fileset ORDER BY Id DESC LIMIT 1");
|
||||
@@ -581,7 +594,7 @@ namespace Duplicati.UnitTest
|
||||
}
|
||||
|
||||
// Check that entry was recreated
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0))
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"]))
|
||||
using (var cmd = con.CreateCommand())
|
||||
Assert.AreEqual(2, cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM Fileset"));
|
||||
}
|
||||
@@ -605,7 +618,7 @@ namespace Duplicati.UnitTest
|
||||
// Remove a fileset
|
||||
long filesetEntries;
|
||||
long fileLookupEntries;
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0))
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"]))
|
||||
using (var cmd = con.CreateCommand())
|
||||
{
|
||||
filesetEntries = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM FilesetEntry");
|
||||
@@ -628,7 +641,7 @@ namespace Duplicati.UnitTest
|
||||
}
|
||||
|
||||
// Check that entry was recreated
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0))
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"]))
|
||||
using (var cmd = con.CreateCommand())
|
||||
{
|
||||
Assert.AreEqual(filesetEntries, cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM FilesetEntry"));
|
||||
@@ -646,7 +659,7 @@ namespace Duplicati.UnitTest
|
||||
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
|
||||
TestUtils.AssertResults(c.Backup(new[] { this.DATAFOLDER }));
|
||||
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0))
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"]))
|
||||
using (var cmd = con.CreateCommand())
|
||||
{
|
||||
var lookupId = cmd.ExecuteScalarInt64("SELECT ID FROM FileLookup LIMIT 1");
|
||||
@@ -664,7 +677,7 @@ namespace Duplicati.UnitTest
|
||||
Assert.AreEqual(0, res.Errors.Count());
|
||||
Assert.AreEqual(0, res.Warnings.Count());
|
||||
}
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"], 0))
|
||||
using (var con = SQLiteLoader.LoadConnection(options["dbpath"]))
|
||||
using (var cmd = con.CreateCommand())
|
||||
{
|
||||
var remaining = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM Metadataset JOIN Blockset ON Metadataset.BlocksetID = Blockset.ID WHERE Blockset.Length = 0");
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System;
|
||||
@@ -325,7 +325,7 @@ namespace Duplicati.UnitTest
|
||||
|
||||
(long remoteVolumes, long duplicatedBlocks, List<long> blockVolumeIds, List<(long, long)> indexVolumeLinks) GetDbSignature()
|
||||
{
|
||||
using var db = SQLiteLoader.LoadConnection(DBFILE, 0);
|
||||
using var db = SQLiteLoader.LoadConnection(DBFILE);
|
||||
using var cmd = db.CreateCommand();
|
||||
|
||||
var remoteVolumes = cmd.ExecuteScalarInt64("SELECT COUNT(*) FROM RemoteVolume");
|
||||
|
||||
@@ -1,28 +1,31 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Duplicati.Library.Main.Database;
|
||||
using Duplicati.Library.SQLiteHelper;
|
||||
using Duplicati.Library.Utility;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using NUnit.Framework;
|
||||
using Assert = NUnit.Framework.Legacy.ClassicAssert;
|
||||
|
||||
@@ -31,17 +34,17 @@ namespace Duplicati.UnitTest
|
||||
[TestFixture]
|
||||
public class SQLiteTests : BasicSetupHelper
|
||||
{
|
||||
private static IDbConnection CreateDummyDatabase(string path)
|
||||
private static async Task<SqliteConnection> CreateDummyDatabase(string path)
|
||||
{
|
||||
var connection = SQLiteLoader.LoadConnection(path, 0);
|
||||
var connection = SQLiteLoader.LoadConnection(path);
|
||||
|
||||
using (var command = connection.CreateCommand())
|
||||
{
|
||||
command.CommandText = "CREATE TABLE TestTable (ID INTEGER PRIMARY KEY, Name TEXT)";
|
||||
command.ExecuteNonQuery();
|
||||
await command.ExecuteNonQueryAsync().ConfigureAwait(false);
|
||||
|
||||
command.CommandText = "INSERT INTO TestTable (ID, Name) VALUES (1, 'Test1'), (2, 'Test2'), (3, 'Test3')";
|
||||
command.ExecuteNonQuery();
|
||||
await command.ExecuteNonQueryAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return connection;
|
||||
@@ -50,47 +53,49 @@ namespace Duplicati.UnitTest
|
||||
|
||||
[Test]
|
||||
[Category("SQLite")]
|
||||
public void TestEmptyTransaction()
|
||||
public async Task TestEmptyTransaction()
|
||||
{
|
||||
using var tf = new TempFile();
|
||||
using var connection = CreateDummyDatabase(tf);
|
||||
using var connection = await CreateDummyDatabase(tf)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
using var t1 = connection.BeginTransaction();
|
||||
t1.Commit(); // No exception should be thrown
|
||||
await t1.CommitAsync().ConfigureAwait(false); // No exception should be thrown
|
||||
|
||||
using var t2 = connection.BeginTransaction();
|
||||
t2.Rollback(); // No exception should be thrown
|
||||
await t2.RollbackAsync().ConfigureAwait(false); // No exception should be thrown
|
||||
}
|
||||
|
||||
[Test]
|
||||
[Category("SQLite")]
|
||||
public void TestListExpansion()
|
||||
public async Task TestListExpansion()
|
||||
{
|
||||
using var tf = new TempFile();
|
||||
using var connection = CreateDummyDatabase(tf);
|
||||
using var connection = await CreateDummyDatabase(tf).ConfigureAwait(false);
|
||||
using var rtr = new ReusableTransaction(connection);
|
||||
|
||||
using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)"))
|
||||
{
|
||||
command.ExpandInClauseParameter("@List", new long[] { 1, 2, 3 });
|
||||
Assert.AreEqual(3, command.ExecuteScalarInt64());
|
||||
Assert.AreEqual(3, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)"))
|
||||
{
|
||||
command.ExpandInClauseParameter("@List", new long[] { 1, 2 });
|
||||
Assert.AreEqual(2, command.ExecuteScalarInt64());
|
||||
Assert.AreEqual(2, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)"))
|
||||
{
|
||||
command.ExpandInClauseParameter("@List", new long[] { 1 });
|
||||
Assert.AreEqual(1, command.ExecuteScalarInt64());
|
||||
Assert.AreEqual(1, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)"))
|
||||
{
|
||||
command.ExpandInClauseParameter("@List", new long[0]);
|
||||
Assert.AreEqual(0, command.ExecuteScalarInt64());
|
||||
Assert.AreEqual(0, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
var list = new List<long>();
|
||||
@@ -99,9 +104,10 @@ namespace Duplicati.UnitTest
|
||||
|
||||
using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)"))
|
||||
{
|
||||
using var tmplist = new TemporaryDbValueList(connection, null, list);
|
||||
using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
command.ExpandInClauseParameter("@List", list);
|
||||
Assert.AreEqual(3, command.ExecuteScalarInt64());
|
||||
Assert.AreEqual(3, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
using (var command = connection.CreateCommand("SELECT COUNT(*) FROM TestTable WHERE ID IN (@List)"))
|
||||
@@ -109,9 +115,10 @@ namespace Duplicati.UnitTest
|
||||
list.Remove(1);
|
||||
list.Remove(2);
|
||||
|
||||
using var tmplist = new TemporaryDbValueList(connection, null, list);
|
||||
using var tmplist = await TemporaryDbValueList.CreateAsync(connection, rtr, list, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
command.ExpandInClauseParameter("@List", list);
|
||||
Assert.AreEqual(1, command.ExecuteScalarInt64());
|
||||
Assert.AreEqual(1, await command.ExecuteScalarInt64Async(CancellationToken.None).ConfigureAwait(false));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
@@ -179,7 +179,7 @@ public class BackupGet : IEndpointV1
|
||||
if (!File.Exists(bk.DBPath))
|
||||
return new List<Dictionary<string, object>>();
|
||||
|
||||
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath, 0))
|
||||
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath))
|
||||
using (var cmd = con.CreateCommand())
|
||||
return LogData.DumpTable(cmd, "LogData", "ID", offset, pagesize);
|
||||
}
|
||||
@@ -189,7 +189,7 @@ public class BackupGet : IEndpointV1
|
||||
if (!File.Exists(bk.DBPath))
|
||||
return new List<Dictionary<string, object>>();
|
||||
|
||||
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath, 0))
|
||||
using (var con = Library.SQLiteHelper.SQLiteLoader.LoadConnection(bk.DBPath))
|
||||
using (var cmd = con.CreateCommand())
|
||||
{
|
||||
var dt = LogData.DumpTable(cmd, "RemoteOperation", "ID", offset, pagesize);
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
@@ -56,7 +56,6 @@ public static partial class Command
|
||||
|
||||
case OSType.Linux:
|
||||
await ReplaceLibMonoUnix(baseDir, buildDir, target.Arch);
|
||||
await ReplaceSQLiteInterop(baseDir, buildDir, target.Arch);
|
||||
break;
|
||||
|
||||
default:
|
||||
@@ -280,28 +279,6 @@ public static partial class Command
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Replaces the library SQLiteInterop.dll with a version that is built against GLIBC_2.33 for ARM7
|
||||
/// </summary>
|
||||
/// <param name="baseDir">The base directory</param>
|
||||
/// <param name="buildDir">The build directory</param>
|
||||
/// <param name="arch">The architecture to build for</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
static Task ReplaceSQLiteInterop(string baseDir, string buildDir, ArchType arch)
|
||||
{
|
||||
if (arch != ArchType.Arm7)
|
||||
return Task.CompletedTask;
|
||||
|
||||
var sourceFile = Path.Combine(baseDir, "ReleaseBuilder", "Resources", "linux-arm-binary", "SQLite.Interop.dll");
|
||||
var targetFile = Path.Combine(buildDir, "SQLite.Interop.dll");
|
||||
if (!File.Exists(targetFile))
|
||||
throw new Exception($"Expected file \"{targetFile}\" not found, has build changed?");
|
||||
|
||||
File.Copy(sourceFile, targetFile, overwrite: true);
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Signs all .exe and .dll files with Authenticode
|
||||
/// </summary>
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
@@ -54,7 +54,7 @@ public static partial class Command
|
||||
"Google.Apis.Auth.dll",
|
||||
"Google.Apis.Core.dll",
|
||||
"SQLiteHelper.dll",
|
||||
"SQLite.Interop.dll",
|
||||
"Microsoft.Data.Sqlite.dll",
|
||||
"Microsoft.IdentityModel.Abstractions.dll",
|
||||
"System.Reactive.dll"
|
||||
};
|
||||
@@ -248,7 +248,7 @@ public static partial class Command
|
||||
{
|
||||
// Using v3.3 for assembly, but 3.7 in nuget
|
||||
{ "AWSSDK.Core", new Version(3, 3, 0, 0) },
|
||||
|
||||
|
||||
// Using the Framework version, not the package version
|
||||
{ "Microsoft.CSharp", new Version(8, 0, 0, 0) },
|
||||
{ "System.Memory", new Version(8, 0, 0, 0) },
|
||||
@@ -256,7 +256,10 @@ public static partial class Command
|
||||
{ "System.Security.Principal.Windows", new Version(8, 0, 0, 0) },
|
||||
{ "System.Security.Cryptography.Algorithms", new Version(8, 0, 0, 0) },
|
||||
{ "System.Security.Cryptography.Cng", new Version(8, 0, 0, 0) },
|
||||
|
||||
|
||||
// The assembly version also has a revision number, but the nuget version does not.
|
||||
{ "SQLitePCLRaw.core", new Version(2, 1, 10, 2445) },
|
||||
|
||||
// Using v9.0 for assembly, but 9.0.2 in nuget
|
||||
{ "System.IO.Pipelines", new Version(9, 0, 0, 0) }
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user