Files
duplicati/Duplicati/UnitTest/DirectListHandlerTests.cs
T

1014 lines
46 KiB
C#

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Main;
using Duplicati.Library.Main.Database;
using Duplicati.Library.Main.Database.Local;
using Duplicati.Library.Utility;
using NUnit.Framework;
namespace Duplicati.UnitTest
{
[TestFixture]
public class DirectListHandlerTests : BasicSetupHelper
{
[Test]
public async Task ListFilesetsAsync()
{
var options = new Dictionary<string, string>(this.TestOptions)
{
["upload-unchanged-backups"] = "true"
};
for (var i = 0; i < 3; i++)
{
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
TestUtils.AssertResults(await c.BackupAsync(new[] { this.DATAFOLDER }));
var sets = await c.ListFilesetsAsync();
Assert.That(sets.Filesets.Count(), Is.EqualTo(i + 1));
}
}
}
[Test]
public async Task ListFolderContentsAsync()
{
var options = new Dictionary<string, string>(this.TestOptions)
{
["upload-unchanged-backups"] = "true"
};
var initial = new[] {
"folder1/",
"folder2/",
"file1",
"folder1/file1.txt",
"folder1/file2.txt",
"folder1/folder1/",
"folder1/folder2/",
"folder1/folder1/file1.txt",
"folder1/folder2/file2.txt",
"folder2/folder2/"
};
var add2 = new[] {
"folder3/",
"folder1/folder1/file3.txt",
};
var add3 = new[] {
"folder1/folder1/file3.txt",
"folder1/folder1/file4.txt"
};
IEnumerable<string> rootItems(IEnumerable<string> items)
=> items.Where(x => !x.Contains('/') || x.IndexOf('/') == x.Length - 1);
void createStructure(IEnumerable<string> paths)
{
foreach (var p in paths)
{
var path = Path.Combine(this.DATAFOLDER, p.Replace("/", Path.DirectorySeparatorChar.ToString()));
if (p.EndsWith("/"))
Directory.CreateDirectory(path);
else
File.WriteAllText(path, p);
}
}
var sources = initial.Take(3).ToArray();
var rounds = new[] {
initial,
initial.Concat(add2),
initial.Concat(add3)
};
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
foreach (var round in rounds)
{
createStructure(round);
TestUtils.AssertResults(await c.BackupAsync(rootItems(round).Select(x => Path.Combine(this.DATAFOLDER, x.Replace("/", Path.DirectorySeparatorChar.ToString()))).ToArray()));
}
var sets = await c.ListFilesetsAsync();
Assert.That(sets.Filesets.Count(), Is.EqualTo(rounds.Length));
}
var version = 0;
foreach (var round in rounds.Reverse())
{
using (var c = new Controller("file://" + this.TARGETFOLDER, options.Expand(new { version = version }), null))
{
var files = await c.ListFolderAsync([""], 0, 0, false);
var roots = rootItems(round).Select(x => Path.Combine(this.DATAFOLDER, x.Replace("/", Path.DirectorySeparatorChar.ToString()))).ToArray();
Assert.That(files.Entries.Items.Count(), Is.EqualTo(roots.Length));
Assert.That(files.Entries.Items.Count(x => x.IsDirectory), Is.EqualTo(roots.Count(x => x.EndsWith(Path.DirectorySeparatorChar))));
Assert.That(files.Entries.Items.Count(x => !x.IsDirectory), Is.EqualTo(roots.Count(x => !x.EndsWith(Path.DirectorySeparatorChar))));
Assert.That(files.Entries.Items.Count(x => roots.Any(y => x.Path.EndsWith(y))), Is.EqualTo(roots.Length));
var work = new Queue<string>(roots);
while (work.Count > 0)
{
var path = work.Dequeue();
var files2 = await c.ListFolderAsync(new[] { Path.Combine(this.DATAFOLDER, path.Replace("/", Path.DirectorySeparatorChar.ToString())) }, 0, 0, false);
var matches = round.Select(x => Path.Combine(this.DATAFOLDER, x.Replace("/", Path.DirectorySeparatorChar.ToString())))
.Where(x => x.StartsWith(path) && x.Length > path.Length && !x.Substring(path.Length, x.Length - path.Length - 1).Contains(Path.DirectorySeparatorChar))
.ToArray();
Assert.That(files2.Entries.Items.Count(), Is.EqualTo(matches.Length));
Assert.That(files2.Entries.Items.Count(x => x.IsDirectory), Is.EqualTo(matches.Count(x => x.EndsWith(Path.DirectorySeparatorChar))));
Assert.That(files2.Entries.Items.Count(x => !x.IsDirectory), Is.EqualTo(matches.Count(x => !x.EndsWith(Path.DirectorySeparatorChar))));
Assert.That(files2.Entries.Items.Count(x => matches.Contains(x.Path)), Is.EqualTo(matches.Length));
foreach (var match in matches)
if (match.EndsWith("/"))
work.Enqueue(match);
}
}
version++;
}
}
[Test]
public async Task ListFileVersions_LifecycleTestAsync()
{
var options = new Dictionary<string, string>(this.TestOptions)
{
["upload-unchanged-backups"] = "true"
};
var stays = "file_stays.txt";
var modify = "file_modify.txt";
var delete = "file_delete.txt";
var create = "file_create.txt";
void createStructure(IEnumerable<string> files)
{
foreach (var f in files)
{
var path = Path.Combine(this.DATAFOLDER, f.Replace("/", Path.DirectorySeparatorChar.ToString()));
Directory.CreateDirectory(Path.GetDirectoryName(path));
File.WriteAllText(path, f + "-contents");
}
}
void modifyFile(string file, string text)
{
var path = Path.Combine(this.DATAFOLDER, file.Replace("/", Path.DirectorySeparatorChar.ToString()));
File.WriteAllText(path, text);
}
var rounds = new List<string[]> {
new[] { stays, modify, delete }, // Round 0 - initial
new[] { stays, modify, create }, // Round 1 - modify, create, delete 'delete'
new[] { stays, modify, create } // Round 2 - no changes
};
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
// Round 0 - initial
createStructure(rounds[0]);
TestUtils.AssertResults(await c.BackupAsync(rounds[0].Select(x => Path.Combine(this.DATAFOLDER, x.Replace("/", Path.DirectorySeparatorChar.ToString()))).ToArray()));
// Round 1 - modify and create new file
modifyFile(modify, "modified-content");
createStructure(new[] { create });
File.Delete(Path.Combine(this.DATAFOLDER, delete.Replace("/", Path.DirectorySeparatorChar.ToString()))); // Delete file
TestUtils.AssertResults(await c.BackupAsync(rounds[1].Select(x => Path.Combine(this.DATAFOLDER, x.Replace("/", Path.DirectorySeparatorChar.ToString()))).ToArray()));
// Round 2 - no changes
TestUtils.AssertResults(await c.BackupAsync(rounds[2].Select(x => Path.Combine(this.DATAFOLDER, x.Replace("/", Path.DirectorySeparatorChar.ToString()))).ToArray()));
}
var pathsToCheck = new[]
{
Path.Combine(this.DATAFOLDER, stays),
Path.Combine(this.DATAFOLDER, modify),
Path.Combine(this.DATAFOLDER, delete),
Path.Combine(this.DATAFOLDER, create)
};
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
var allVersions = await c.ListFileVersionsAsync(pathsToCheck, 0, 0);
var grouped = allVersions.FileVersions.Items.GroupBy(x => x.Path);
foreach (var group in grouped)
{
var path = group.Key;
var versions = group.OrderByDescending(x => x.Time).ToArray(); // Newest first
if (path.EndsWith(stays))
Assert.That(versions.Length, Is.EqualTo(3), $"File {path} should exist in all 3 versions");
else if (path.EndsWith(modify))
Assert.That(versions.Length, Is.EqualTo(3), $"File {path} should exist in all 3 versions");
else if (path.EndsWith(delete))
Assert.That(versions.Length, Is.EqualTo(1), $"File {path} should exist only in first version before deletion");
else if (path.EndsWith(create))
Assert.That(versions.Length, Is.EqualTo(2), $"File {path} should exist in 2 versions after creation");
else
Assert.Fail($"Unexpected file {path} in file versions list.");
}
}
}
[Test]
public async Task SearchFilesTestAsync()
{
var options = new Dictionary<string, string>(this.TestOptions)
{
["upload-unchanged-backups"] = "true"
};
var initial = new[] {
"file1.txt",
"folder1/",
"folder1/file2.txt",
"folder1/file3.txt",
"folder2/",
"folder2/notes.txt"
};
IEnumerable<string> rootItems(IEnumerable<string> items)
=> items.Where(x => !x.Contains('/') || x.IndexOf('/') == x.Length - 1);
void createStructure(IEnumerable<string> paths)
{
foreach (var p in paths)
{
var path = Path.Combine(this.DATAFOLDER, p.Replace("/", Path.DirectorySeparatorChar.ToString()));
if (p.EndsWith("/"))
Directory.CreateDirectory(path);
else
File.WriteAllText(path, p);
}
}
IFilter ParseFilters(string args)
=> args.Split(";").Select(x => (IFilter)new FilterExpression(x.Substring(1), x.StartsWith("+"))).Aggregate((x, y) => FilterExpression.Combine(x, y));
using (var c = new Controller("file://" + this.TARGETFOLDER, options, null))
{
createStructure(initial);
TestUtils.AssertResults(await c.BackupAsync(rootItems(initial).Select(x => Path.Combine(this.DATAFOLDER, x.Replace("/", Path.DirectorySeparatorChar.ToString()))).ToArray()));
// Simple Search for 'file'
var search = await c.SearchEntriesAsync(null, ParseFilters("+file"), false, 0, 0, false, false);
Assert.That(search.FileVersions.Items.Count(), Is.EqualTo(3)); // file1.txt, file2.txt, file3.txt
// Simple Search for 'notes'
var searchNotes = await c.SearchEntriesAsync(null, ParseFilters("+notes"), false, 0, 0, false, false);
Assert.That(searchNotes.FileVersions.Items.Count(), Is.EqualTo(1));
Assert.That(searchNotes.FileVersions.Items.First().Path.EndsWith("notes.txt"));
// Simple Search for NOT 'notes'
var searchNotNotes = await c.SearchEntriesAsync(null, ParseFilters("-notes"), false, 0, 0, false, false);
Assert.That(searchNotNotes.FileVersions.Items.Count(), Is.EqualTo(5));
Assert.That(searchNotNotes.FileVersions.Items.All(x => !x.Path.Contains("notes")), Is.True);
// Simple Search for 'folder1' folder contents
var searchFolder = await c.SearchEntriesAsync(null, ParseFilters("+folder1"), false, 0, 0, false, false);
Assert.That(searchFolder.FileVersions.Items.All(x => x.Path.Contains("folder1")), Is.True);
// Simple Search with exact file name
var searchExact = await c.SearchEntriesAsync(null, ParseFilters("+file1.txt"), false, 0, 0, false, false);
Assert.That(searchExact.FileVersions.Items.Count(), Is.EqualTo(1));
Assert.That(searchExact.FileVersions.Items.First().Path.EndsWith("file1.txt"));
// Mixed include and exclude
var searchMixed = await c.SearchEntriesAsync(null, ParseFilters("+file;-file2"), false, 0, 0, false, false);
// Include has precedence over exclude, but we include non-matches as well
Assert.That(searchMixed.FileVersions.Items.Count(), Is.EqualTo(6));
Assert.That(searchMixed.FileVersions.Items.Any(x => x.Path.Contains("file2")), Is.True);
// Mixed include and exclude
var searchMixed2 = await c.SearchEntriesAsync(null, ParseFilters("+file2;-file"), false, 0, 0, false, false);
// Include has precedence over exclude, but we include non-matches as well
Assert.That(searchMixed2.FileVersions.Items.Count(), Is.EqualTo(4));
Assert.That(searchMixed2.FileVersions.Items.Any(x => x.Path.Contains("file2")), Is.True);
// Mixed include and exclude, wildcards
var searchMixed3 = await c.SearchEntriesAsync(null, ParseFilters("+file2;-*"), false, 0, 0, false, false);
// Include has precedence over exclude, but we include non-matches as well
Assert.That(searchMixed3.FileVersions.Items.Count(), Is.EqualTo(1));
Assert.That(searchMixed3.FileVersions.Items.All(x => x.Path.Contains("file2")), Is.True);
// NEW: Search inside specific folder prefix: folder1
var searchInFolder1 = await c.SearchEntriesAsync(
new[] { Path.Combine(this.DATAFOLDER, "folder1") },
ParseFilters("+file"), false,
0, 0, false, false);
Assert.That(searchInFolder1.FileVersions.Items.Count(), Is.EqualTo(2)); // file2.txt, file3.txt inside folder1
Assert.That(searchInFolder1.FileVersions.Items.All(x => x.Path.Contains("folder1")), Is.True);
// NEW: Search inside multiple folder prefixes: folder1 and folder2
var searchInFolders = await c.SearchEntriesAsync(
new[]
{
Path.Combine(this.DATAFOLDER, "folder1"),
Path.Combine(this.DATAFOLDER, "folder2")
},
ParseFilters("+file;+notes"), false,
0, 0, false, false);
Assert.That(searchInFolders.FileVersions.Items.Count(), Is.EqualTo(3)); // file2.txt, file3.txt, notes.txt
Assert.That(searchInFolders.FileVersions.Items.Any(x => x.Path.Contains("folder1")), Is.True);
Assert.That(searchInFolders.FileVersions.Items.Any(x => x.Path.Contains("folder2")), Is.True);
}
}
[Test]
public async Task GetMinimalUniquePrefixEntries_ShouldReturnCorrectLinuxPrefixesAsync()
{
using var tempFile = new TempFile();
using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
.ConfigureAwait(false);
SeedTestData(db, [
"/folder1/",
"/folder1/sub1/",
"/folder2/",
"/folder2/sub2/",
"/folder3/"
]);
var result = await db
.GetMinimalUniquePrefixEntriesAsync(1, CancellationToken.None)
.Select(e => e.Path)
.ToListAsync()
.ConfigureAwait(false);
Assert.That(result, Does.Contain("/folder1/"));
Assert.That(result, Does.Contain("/folder2/"));
Assert.That(result, Does.Contain("/folder3/"));
Assert.That(result.Any(x => x.Contains("sub1")), Is.False);
Assert.That(result.Any(x => x.Contains("sub2")), Is.False);
Assert.That(result.Count, Is.EqualTo(3));
}
[Test]
public async Task GetMinimalUniquePrefixEntries_ShouldReturnCorrectWindowsDrivePrefixes_Async()
{
using var tempFile = new TempFile();
using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
.ConfigureAwait(false);
SeedTestData(db, [
"C:\\folder1\\",
"C:\\folder1\\sub1\\",
"C:\\folder2\\",
"D:\\otherfolder\\"
]);
var result = await db
.GetMinimalUniquePrefixEntriesAsync(1, CancellationToken.None)
.Select(e => e.Path)
.ToListAsync()
.ConfigureAwait(false);
Assert.That(result, Does.Contain("C:\\folder1\\"));
Assert.That(result, Does.Contain("C:\\folder2\\"));
Assert.That(result, Does.Contain("D:\\otherfolder\\"));
Assert.That(result.Any(x => x.Contains("sub1")), Is.False);
Assert.That(result.Count, Is.EqualTo(3));
}
[Test]
public async Task GetMinimalUniquePrefixEntries_ShouldReturnCorrectWindowsUncPrefixes_Async()
{
using var tempFile = new TempFile();
using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
.ConfigureAwait(false);
SeedTestData(db, [
"\\\\server\\share\\folder\\",
"\\\\server\\share\\folder\\subfolder\\",
"\\\\server\\share\\otherfolder\\"
]);
var result = await db
.GetMinimalUniquePrefixEntriesAsync(1, CancellationToken.None)
.Select(e => e.Path)
.ToListAsync()
.ConfigureAwait(false);
Assert.That(result, Does.Contain("\\\\server\\share\\folder\\"));
Assert.That(result, Does.Contain("\\\\server\\share\\otherfolder\\"));
Assert.That(result.Any(x => x.Contains("subfolder")), Is.False);
Assert.That(result.Count, Is.EqualTo(2));
}
[Test]
public async Task GetMinimalUniquePrefixEntries_ShouldHandleMixedWindowsDriveAndUncPathsAsync()
{
using var tempFile = new TempFile();
using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
.ConfigureAwait(false);
SeedTestData(db, [
"C:\\data\\",
"C:\\data\\sub1\\",
"C:\\music\\",
"D:\\videos\\",
"\\\\server\\share\\docs\\",
"\\\\server\\share\\docs\\subdoc\\",
"\\\\server\\share\\pictures\\"
]);
var result = await db
.GetMinimalUniquePrefixEntriesAsync(1, CancellationToken.None)
.Select(e => e.Path)
.ToListAsync()
.ConfigureAwait(false);
Assert.That(result, Does.Contain("C:\\data\\"));
Assert.That(result, Does.Contain("C:\\music\\"));
Assert.That(result, Does.Contain("D:\\videos\\"));
Assert.That(result, Does.Contain("\\\\server\\share\\docs\\"));
Assert.That(result, Does.Contain("\\\\server\\share\\pictures\\"));
Assert.That(result.Any(x => x.Contains("sub1")), Is.False);
Assert.That(result.Any(x => x.Contains("subdoc")), Is.False);
Assert.That(result.Count, Is.EqualTo(5));
}
[Test]
public async Task GetMinimalUniquePrefixEntries_ShouldReturnExpectedMinimalRootsAsync()
{
// Arrange: Prepare prefixes (minimal unique) and contents
var testPrefixes = new[]
{
@"C:\Downloads\testsource\AA\",
@"C:\Downloads\testsource\AA\readme.txt",
@"C:\Downloads\testsource\AAA\",
@"C:\Downloads\testsource\AAA\readme.txt",
@"C:\Downloads\testsource\AAAA\",
@"C:\Downloads\testsource\AAAA\readme.txt",
@"C:\Downloads\Kits\",
@"C:\Downloads\Kits\abc.txt",
@"Y:\source\",
@"Y:\source\123.bin"
};
// Act
using var tempFile = new TempFile();
await using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
.ConfigureAwait(false);
SeedTestData(db, testPrefixes);
var resultItems = await db
.GetMinimalUniquePrefixEntriesAsync(1, CancellationToken.None)
.ToListAsync()
.ConfigureAwait(false);
var result = resultItems.Select(e => e.Path).ToList();
Assert.That(result, Does.Contain(@"C:\Downloads\testsource\AA\"));
Assert.That(result, Does.Contain(@"C:\Downloads\testsource\AAA\"));
Assert.That(result, Does.Contain(@"C:\Downloads\testsource\AAAA\"));
Assert.That(result, Does.Contain(@"C:\Downloads\Kits\"));
Assert.That(result, Does.Contain(@"Y:\source\"));
Assert.That(result.Count, Is.EqualTo(5), "Expected 5 unique prefixes");
}
private void SeedTestData(LocalListDatabase db, IEnumerable<string> fullPaths)
{
const long filesetId = 1;
long fileId = 1;
using var cmd = db.Connection.CreateCommand();
// Insert fileset entry
cmd.SetCommandAndParameters(@"
INSERT OR IGNORE INTO Fileset (ID, OperationID, VolumeID, IsFullBackup, Timestamp)
VALUES (@filesetId, 1, 1, 1, 0);")
.SetParameterValue("@filesetId", filesetId)
.ExecuteNonQuery();
foreach (var rawPath in fullPaths)
{
// Normalize path separators but preserve OS style
var path = rawPath.Replace('\\', Path.DirectorySeparatorChar)
.Replace('/', Path.DirectorySeparatorChar);
string prefix;
string relative;
if (rawPath.EndsWith("/") || rawPath.EndsWith("\\"))
{
// Directory: entire path is prefix, no file
prefix = rawPath;
relative = string.Empty;
}
else
{
int lastSep = Math.Max(rawPath.LastIndexOf('/'), rawPath.LastIndexOf('\\'));
if (lastSep < 0)
{
// No separator: file in root
prefix = string.Empty;
relative = rawPath;
}
else
{
prefix = rawPath.Substring(0, lastSep + 1); // include slash
relative = rawPath.Substring(lastSep + 1);
}
}
// Insert prefix if not present
cmd.SetCommandAndParameters(@"
INSERT OR IGNORE INTO PathPrefix (Prefix)
VALUES (@prefix);")
.SetParameterValue("@prefix", prefix)
.ExecuteNonQuery();
long prefixId = cmd.SetCommandAndParameters("SELECT ID FROM PathPrefix WHERE Prefix = @prefix")
.SetParameterValue("@prefix", prefix)
.ExecuteScalarInt64();
// Insert FileLookup
cmd.SetCommandAndParameters(@"
INSERT INTO FileLookup (ID, PrefixID, Path, BlocksetID, MetadataID)
VALUES (@fileId, @prefixId, @path, 1, 1);")
.SetParameterValue("@fileId", fileId)
.SetParameterValue("@prefixId", prefixId)
.SetParameterValue("@path", relative)
.ExecuteNonQuery();
// Insert FilesetEntry
cmd.SetCommandAndParameters(@"
INSERT INTO FilesetEntry (FilesetID, FileID, Lastmodified)
VALUES (@filesetId, @fileId, 0);")
.SetParameterValue("@filesetId", filesetId)
.SetParameterValue("@fileId", fileId)
.ExecuteNonQuery();
fileId++;
}
}
/// <summary>
/// Tests that ListFolder works correctly with a large number of prefix IDs
/// that triggers the temporary table code path (when count > CHUNK_SIZE = 128).
/// </summary>
[Test]
[Category("Database")]
public async Task ListFolder_WithLargePrefixIds_UsesTemporaryTableAsync()
{
using var tempFile = new TempFile();
await using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
.ConfigureAwait(false);
// Insert fileset entry
using (var cmd = db.Connection.CreateCommand())
{
await cmd.SetCommandAndParameters(@"
INSERT OR IGNORE INTO Fileset (ID, OperationID, VolumeID, IsFullBackup, Timestamp)
VALUES (@filesetId, 1, 1, 1, 0);")
.SetParameterValue("@filesetId", 1L)
.ExecuteNonQueryAsync();
}
// Create 150 prefix IDs (exceeds CHUNK_SIZE of 128) to trigger temporary table path
var prefixIds = new List<long>();
for (int i = 0; i < 150; i++)
{
var prefix = $"/test/prefix/{i}/";
using var cmd = db.Connection.CreateCommand();
// Insert prefix
await cmd.SetCommandAndParameters(@"
INSERT OR IGNORE INTO PathPrefix (Prefix)
VALUES (@prefix);")
.SetParameterValue("@prefix", prefix)
.ExecuteNonQueryAsync();
// Get prefix ID
var prefixId = cmd.SetCommandAndParameters("SELECT ID FROM PathPrefix WHERE Prefix = @prefix")
.SetParameterValue("@prefix", prefix)
.ExecuteScalarInt64();
prefixIds.Add(prefixId);
// Insert FileLookup
await cmd.SetCommandAndParameters(@"
INSERT INTO FileLookup (ID, PrefixID, Path, BlocksetID, MetadataID)
VALUES (@fileId, @prefixId, @path, 1, 1);")
.SetParameterValue("@fileId", i + 1)
.SetParameterValue("@prefixId", prefixId)
.SetParameterValue("@path", $"file{i}.txt")
.ExecuteNonQueryAsync();
// Insert FilesetEntry
await cmd.SetCommandAndParameters(@"
INSERT INTO FilesetEntry (FilesetID, FileID, Lastmodified)
VALUES (@filesetId, @fileId, 0);")
.SetParameterValue("@filesetId", 1L)
.SetParameterValue("@fileId", i + 1)
.ExecuteNonQueryAsync();
}
// Act: Call ListFolder with 150 prefix IDs
// This should trigger the temporary table code path
var result = await db.ListFolderAsync(prefixIds, 1, 0, 1000, CancellationToken.None)
.ConfigureAwait(false);
// Assert: Should return all 150 files
Assert.That(result.Items.Count(), Is.EqualTo(150));
}
/// <summary>
/// Tests that <see cref="LocalListDatabase.ListFileVersionsAsync"/> works correctly
/// with a large number of fileset IDs that triggers the temporary table code path
/// (when count > CHUNK_SIZE = 128).
/// </summary>
[Test]
[Category("Database")]
public async Task ListFileVersions_WithLargeFilesetIds_UsesTemporaryTableAsync()
{
// Arrange
using var tempFile = new TempFile();
await using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
.ConfigureAwait(false);
// Insert operation entry
using (var cmd = db.Connection.CreateCommand())
{
await cmd.SetCommandAndParameters(@"
INSERT OR IGNORE INTO Operation (ID, Description, Timestamp)
VALUES (1, 'TestOperation', 0);")
.ExecuteNonQueryAsync();
}
// Create 150 filesets (exceeds CHUNK_SIZE of 128) to trigger temporary table path
var filesetIds = new List<long>();
for (int i = 0; i < 150; i++)
{
using var cmd = db.Connection.CreateCommand();
// Insert RemoteVolume for the fileset
await cmd.SetCommandAndParameters(@"
INSERT INTO RemoteVolume (ID, OperationID, Name, Type, State, Size, VerificationCount, DeleteGraceTime, ArchiveTime, LockExpirationTime)
VALUES (@id, @operationId, @name, @type, @state, @size, @verificationCount, @deleteGraceTime, @archiveTime, @lockExpirationTime);")
.SetParameterValue("@id", i + 1)
.SetParameterValue("@operationId", 1L)
.SetParameterValue("@name", $"volume{i}.zip")
.SetParameterValue("@type", "Files")
.SetParameterValue("@state", "Verified")
.SetParameterValue("@size", 1024L)
.SetParameterValue("@verificationCount", 0)
.SetParameterValue("@deleteGraceTime", 0)
.SetParameterValue("@archiveTime", 0)
.SetParameterValue("@lockExpirationTime", 0)
.ExecuteNonQueryAsync();
// Insert Fileset
await cmd.SetCommandAndParameters(@"
INSERT INTO Fileset (ID, OperationID, VolumeID, IsFullBackup, Timestamp)
VALUES (@id, @operationId, @volumeId, @isFullBackup, @timestamp);")
.SetParameterValue("@id", i + 1)
.SetParameterValue("@operationId", 1L)
.SetParameterValue("@volumeId", i + 1)
.SetParameterValue("@isFullBackup", 1)
.SetParameterValue("@timestamp", i)
.ExecuteNonQueryAsync();
filesetIds.Add(i + 1);
}
// Insert a single file that exists in all filesets
using (var cmd = db.Connection.CreateCommand())
{
// Insert Blockset
await cmd.SetCommandAndParameters(@"
INSERT INTO Blockset (ID, Length, FullHash)
VALUES (1, 1024, 'fullhash');")
.ExecuteNonQueryAsync();
// Insert Metadataset
await cmd.SetCommandAndParameters(@"
INSERT INTO Metadataset (ID, BlocksetID)
VALUES (1, 1);")
.ExecuteNonQueryAsync();
// Insert PathPrefix
await cmd.SetCommandAndParameters(@"
INSERT INTO PathPrefix (ID, Prefix)
VALUES (1, '/test/');")
.ExecuteNonQueryAsync();
// Insert FileLookup
await cmd.SetCommandAndParameters(@"
INSERT INTO FileLookup (ID, PrefixID, Path, BlocksetID, MetadataID)
VALUES (1, 1, 'file.txt', 1, 1);")
.ExecuteNonQueryAsync();
// Insert FilesetEntry for each fileset
for (int i = 0; i < 150; i++)
{
await cmd.SetCommandAndParameters(@"
INSERT INTO FilesetEntry (FilesetID, FileID, Lastmodified)
VALUES (@filesetId, @fileId, @lastModified);")
.SetParameterValue("@filesetId", i + 1)
.SetParameterValue("@fileId", 1)
.SetParameterValue("@lastModified", 0)
.ExecuteNonQueryAsync();
}
}
// Act: Call ListFileVersions with 150 fileset IDs
// This should trigger the temporary table code path
var result = await db.ListFileVersionsAsync(
new[] { "file.txt" },
filesetIds.ToArray(),
0,
1000,
CancellationToken.None)
.ConfigureAwait(false);
// Assert: Should return all 150 file versions
Assert.That(result.Items.Count(), Is.EqualTo(150));
}
/// <summary>
/// Tests that <see cref="LocalListDatabase.SearchEntriesAsync"/> works correctly
/// with a large number of fileset IDs that triggers the temporary table code path
/// (when count > CHUNK_SIZE = 128).
/// </summary>
[Test]
[Category("Database")]
public async Task SearchEntries_WithLargeFilesetIds_UsesTemporaryTableAsync()
{
// Arrange
using var tempFile = new TempFile();
await using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
.ConfigureAwait(false);
// Insert operation entry
using (var cmd = db.Connection.CreateCommand())
{
await cmd.SetCommandAndParameters(@"
INSERT OR IGNORE INTO Operation (ID, Description, Timestamp)
VALUES (1, 'TestOperation', 0);")
.ExecuteNonQueryAsync();
}
// Create 150 filesets (exceeds CHUNK_SIZE of 128) to trigger temporary table path
var filesetIds = new List<long>();
for (int i = 0; i < 150; i++)
{
using var cmd = db.Connection.CreateCommand();
// Insert RemoteVolume for the fileset
await cmd.SetCommandAndParameters(@"
INSERT INTO RemoteVolume (ID, OperationID, Name, Type, State, Size, VerificationCount, DeleteGraceTime, ArchiveTime, LockExpirationTime)
VALUES (@id, @operationId, @name, @type, @state, @size, @verificationCount, @deleteGraceTime, @archiveTime, @lockExpirationTime);")
.SetParameterValue("@id", i + 1)
.SetParameterValue("@operationId", 1L)
.SetParameterValue("@name", $"volume{i}.zip")
.SetParameterValue("@type", "Files")
.SetParameterValue("@state", "Verified")
.SetParameterValue("@size", 1024L)
.SetParameterValue("@verificationCount", 0)
.SetParameterValue("@deleteGraceTime", 0)
.SetParameterValue("@archiveTime", 0)
.SetParameterValue("@lockExpirationTime", 0)
.ExecuteNonQueryAsync();
// Insert Fileset
await cmd.SetCommandAndParameters(@"
INSERT INTO Fileset (ID, OperationID, VolumeID, IsFullBackup, Timestamp)
VALUES (@id, @operationId, @volumeId, @isFullBackup, @timestamp);")
.SetParameterValue("@id", i + 1)
.SetParameterValue("@operationId", 1L)
.SetParameterValue("@volumeId", i + 1)
.SetParameterValue("@isFullBackup", 1)
.SetParameterValue("@timestamp", i)
.ExecuteNonQueryAsync();
filesetIds.Add(i + 1);
}
// Insert files that exist in all filesets
using (var cmd = db.Connection.CreateCommand())
{
// Insert Blockset
await cmd.SetCommandAndParameters(@"
INSERT INTO Blockset (ID, Length, FullHash)
VALUES (1, 1024, 'fullhash');")
.ExecuteNonQueryAsync();
// Insert Metadataset
await cmd.SetCommandAndParameters(@"
INSERT INTO Metadataset (ID, BlocksetID)
VALUES (1, 1);")
.ExecuteNonQueryAsync();
// Insert PathPrefix
await cmd.SetCommandAndParameters(@"
INSERT INTO PathPrefix (ID, Prefix)
VALUES (1, '/test/');")
.ExecuteNonQueryAsync();
// Insert FileLookup entries
for (int i = 0; i < 10; i++)
{
await cmd.SetCommandAndParameters(@"
INSERT INTO FileLookup (ID, PrefixID, Path, BlocksetID, MetadataID)
VALUES (@id, @prefixId, @path, @blocksetId, @metadataId);")
.SetParameterValue("@id", i + 1)
.SetParameterValue("@prefixId", 1)
.SetParameterValue("@path", $"file{i}.txt")
.SetParameterValue("@blocksetId", 1)
.SetParameterValue("@metadataId", 1)
.ExecuteNonQueryAsync();
}
// Insert FilesetEntry for each fileset and file
for (int i = 0; i < 150; i++)
{
for (int j = 0; j < 10; j++)
{
await cmd.SetCommandAndParameters(@"
INSERT INTO FilesetEntry (FilesetID, FileID, Lastmodified)
VALUES (@filesetId, @fileId, @lastModified);")
.SetParameterValue("@filesetId", i + 1)
.SetParameterValue("@fileId", j + 1)
.SetParameterValue("@lastModified", 0)
.ExecuteNonQueryAsync();
}
}
}
// Act: Call SearchEntries with 150 fileset IDs
// This should trigger the temporary table code path
// Use a simple filter that matches all files (empty filter matches everything)
var result = await db.SearchEntriesAsync(
null,
new FilterExpression("*", true),
false,
filesetIds.ToArray(),
false,
0,
2000,
CancellationToken.None)
.ConfigureAwait(false);
// Assert: Should return all 1500 file versions (150 filesets * 10 files)
Assert.That(result.Items.Count(), Is.EqualTo(1500));
}
/// <summary>
/// Tests that <see cref="LocalListDatabase.SearchEntriesAsync"/> can match values inside
/// metadata JSON when <paramref name="searchMetadata"/> is <c>true</c>.
/// </summary>
[Test]
[Category("Database")]
public async Task SearchEntries_WithMetadataSearch_MatchesMetadataValuesAsync()
{
// Arrange
using var tempFile = new TempFile();
await using var db = await LocalListDatabase.CreateAsync(tempFile, null, CancellationToken.None)
.ConfigureAwait(false);
// Insert operation entry
using (var cmd = db.Connection.CreateCommand())
{
await cmd.SetCommandAndParameters(@"
INSERT OR IGNORE INTO Operation (ID, Description, Timestamp)
VALUES (1, 'TestOperation', 0);")
.ExecuteNonQueryAsync();
}
// Insert RemoteVolume and Fileset
using (var cmd = db.Connection.CreateCommand())
{
await cmd.SetCommandAndParameters(@"
INSERT INTO RemoteVolume (ID, OperationID, Name, Type, State, Size, VerificationCount, DeleteGraceTime, ArchiveTime, LockExpirationTime)
VALUES (1, 1, 'volume.zip', 'Files', 'Verified', 1024, 0, 0, 0, 0);")
.ExecuteNonQueryAsync();
await cmd.SetCommandAndParameters(@"
INSERT INTO Fileset (ID, OperationID, VolumeID, IsFullBackup, Timestamp)
VALUES (1, 1, 1, 1, 0);")
.ExecuteNonQueryAsync();
}
using (var cmd = db.Connection.CreateCommand())
{
// Insert Blockset for the Metadataset
await cmd.SetCommandAndParameters(@"
INSERT INTO Blockset (ID, Length, FullHash)
VALUES (1, 1024, 'fullhash');")
.ExecuteNonQueryAsync();
// Insert two metadata entries
await cmd.SetCommandAndParameters(@"
INSERT INTO Metadataset (ID, BlocksetID, Content)
VALUES (1, 1, '{""o365:Name"": ""MyDocument"", ""o365:Type"": ""DriveFile""}');")
.ExecuteNonQueryAsync();
await cmd.SetCommandAndParameters(@"
INSERT INTO Metadataset (ID, BlocksetID, Content)
VALUES (2, 1, '{""gsuite:Name"": ""MySheet"", ""gsuite:Type"": ""Spreadsheet""}');")
.ExecuteNonQueryAsync();
// Insert PathPrefix
await cmd.SetCommandAndParameters(@"
INSERT INTO PathPrefix (ID, Prefix)
VALUES (1, '/test/');")
.ExecuteNonQueryAsync();
// Insert FileLookup entries: one with metadata, one without
// File 1: has metadata matching "MyDocument"
await cmd.SetCommandAndParameters(@"
INSERT INTO FileLookup (ID, PrefixID, Path, BlocksetID, MetadataID)
VALUES (1, 1, 'file1.txt', 1, 1);")
.ExecuteNonQueryAsync();
// File 2: has metadata matching "MySheet" (different key)
await cmd.SetCommandAndParameters(@"
INSERT INTO FileLookup (ID, PrefixID, Path, BlocksetID, MetadataID)
VALUES (2, 1, 'file2.txt', 1, 2);")
.ExecuteNonQueryAsync();
// File 3: no metadata
await cmd.SetCommandAndParameters(@"
INSERT INTO FileLookup (ID, PrefixID, Path, BlocksetID, MetadataID)
VALUES (3, 1, 'file3.txt', 1, 0);")
.ExecuteNonQueryAsync();
// Folder 4: folder with metadata matching "MyDocument"
// BlocksetID = -100 is FOLDER_BLOCKSET_ID
await cmd.SetCommandAndParameters(@"
INSERT INTO FileLookup (ID, PrefixID, Path, BlocksetID, MetadataID)
VALUES (4, 1, 'folder1', -100, 1);")
.ExecuteNonQueryAsync();
// Insert FilesetEntry for all four entries
for (int fileId = 1; fileId <= 4; fileId++)
{
await cmd.SetCommandAndParameters(@"
INSERT INTO FilesetEntry (FilesetID, FileID, Lastmodified)
VALUES (@filesetId, @fileId, @lastModified);")
.SetParameterValue("@filesetId", 1)
.SetParameterValue("@fileId", fileId)
.SetParameterValue("@lastModified", 0)
.ExecuteNonQueryAsync();
}
}
// Act: Search for "MyDocument" with metadata search enabled
var result = await db.SearchEntriesAsync(
null,
new FilterExpression("MyDocument", true),
false,
[1],
true, // searchMetadata
0,
100,
CancellationToken.None)
.ConfigureAwait(false);
// Assert: Both file1.txt and folder1 should match (via o365:Name metadata)
Assert.That(result.Items.Count(), Is.EqualTo(2));
var paths = result.Items.Select(x => x.Path).OrderBy(x => x).ToList();
Assert.That(paths[0], Is.EqualTo("/test/file1.txt"));
Assert.That(paths[1], Is.EqualTo("/test/folder1"));
Assert.That(result.Items.First(x => x.Path == "/test/folder1").IsDirectory, Is.True);
// Act: Search for "MySheet" with metadata search enabled
result = await db.SearchEntriesAsync(
null,
new FilterExpression("MySheet", true),
false,
[1],
true, // searchMetadata
0,
100,
CancellationToken.None)
.ConfigureAwait(false);
// Assert: Only file2.txt should match (via gsuite:Name metadata)
Assert.That(result.Items.Count(), Is.EqualTo(1));
Assert.That(result.Items.First().Path, Is.EqualTo("/test/file2.txt"));
// Act: Search for "MyDocument" with metadata search DISABLED
result = await db.SearchEntriesAsync(
null,
new FilterExpression("MyDocument", true),
false,
[1],
false, // searchMetadata disabled
0,
100,
CancellationToken.None)
.ConfigureAwait(false);
// Assert: No files should match (path doesn't contain "MyDocument")
Assert.That(result.Items.Count(), Is.EqualTo(0));
}
}
}