haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve file backend list speed.

Browse Source 
Directly list FileInfo instead of lookups by filename. This greatly speeds up backup verify on exFAT.

Closes #5061
This commit is contained in:
Jojo-1000
2023-11-19 20:04:37 +01:00
parent ec9d49d684
commit f7e435b1dd
4 changed files with 65 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Backend/File/FileBackend.cs
+1 -9
View File
@@ -176,15 +176,7 @@ namespace Duplicati.Library.Backend
if (!systemIO.DirectoryExists(m_path))
throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path));
foreach (string s in systemIO.EnumerateFiles(m_path))
{
yield return systemIO.FileEntry(s);
}
foreach (string s in systemIO.EnumerateDirectories(m_path))
{
yield return systemIO.DirectoryEntry(s);
}
return systemIO.EnumerateFileEntries(m_path);
}
#if DEBUG_RETRY
Duplicati/Library/Common/IO/ISystemIO.cs
+6
View File
@@ -28,6 +28,7 @@ namespace Duplicati.Library.Common.IO
public interface ISystemIO
{
IFileEntry DirectoryEntry(string path);
IFileEntry DirectoryEntry(DirectoryInfo dirInfo);
void DirectoryCreate(string path);
void DirectoryDelete(string path, bool recursive);
bool DirectoryExists(string path);
@@ -36,6 +37,7 @@ namespace Duplicati.Library.Common.IO
void DirectorySetCreationTimeUtc(string path, DateTime time);
IFileEntry FileEntry(string path);
IFileEntry FileEntry(FileInfo fileInfo);
void FileMove(string source, string target);
void FileDelete(string path);
void FileCopy(string source, string target, bool overwrite);
@@ -69,6 +71,10 @@ namespace Duplicati.Library.Common.IO
IEnumerable<string> EnumerateFiles(string path, string searchPattern, SearchOption searchOption);
IEnumerable<string> EnumerateDirectories(string path);
// Enumerate FileEntries of files and directories
// This is more efficient than enumerating file names when metadata is needed
IEnumerable<IFileEntry> EnumerateFileEntries(string path);
void SetMetadata(string path, Dictionary<string, string> metdata, bool restorePermissions);
Dictionary<string, string> GetMetadata(string path, bool isSymlink, bool followSymlink);
}
Duplicati/Library/Common/IO/SystemIOLinux.cs
+24 -2
View File
@@ -132,6 +132,22 @@ namespace Duplicati.Library.Common.IO
return Directory.EnumerateFiles(path, searchPattern, searchOption);
}
public IEnumerable<IFileEntry> EnumerateFileEntries(string path)
{
// For consistency with previous implementation, enumerate files first and directories after
DirectoryInfo dir = new DirectoryInfo(path);
foreach (FileInfo file in dir.EnumerateFiles())
{
yield return FileEntry(file);
}
foreach (DirectoryInfo d in dir.EnumerateDirectories())
{
yield return DirectoryEntry(d);
}
}
public string PathGetFileName(string path)
{
return Path.GetFileName(path);
@@ -293,7 +309,10 @@ namespace Duplicati.Library.Common.IO
public IFileEntry DirectoryEntry(string path)
{
var dInfo = new DirectoryInfo(path);
return DirectoryEntry(new DirectoryInfo(path));
}
public IFileEntry DirectoryEntry(DirectoryInfo dInfo)
{
return new FileEntry(dInfo.Name, 0, dInfo.LastAccessTime, dInfo.LastWriteTime)
{
IsFolder = true
@@ -302,7 +321,10 @@ namespace Duplicati.Library.Common.IO
public IFileEntry FileEntry(string path)
{
var fileInfo = new FileInfo(path);
return FileEntry(new FileInfo(path));
}
public IFileEntry FileEntry(FileInfo fileInfo)
{
return new FileEntry(fileInfo.Name, fileInfo.Length, fileInfo.LastAccessTime, fileInfo.LastWriteTime);
}
}
Duplicati/Library/Common/IO/SystemIOWindows.cs
+34 -4
View File
@@ -75,7 +75,7 @@ namespace Duplicati.Library.Common.IO
}
}
}
/// <summary>
/// Returns true if prefixed with @"\\" or @"//".
/// </summary>
@@ -361,7 +361,7 @@ namespace Duplicati.Library.Common.IO
public IEnumerable<string> EnumerateFiles(string path, string searchPattern, SearchOption searchOption)
{
return System.IO.Directory.EnumerateFiles(AddExtendedDevicePathPrefix(path), searchPattern, searchOption).Select(RemoveExtendedDevicePathPrefix);
return System.IO.Directory.EnumerateFiles(AddExtendedDevicePathPrefix(path), searchPattern, searchOption).Select(RemoveExtendedDevicePathPrefix);
}
public string PathGetFileName(string path)
@@ -474,6 +474,30 @@ namespace Duplicati.Library.Common.IO
}
}
public IEnumerable<IFileEntry> EnumerateFileEntries(string path)
{
// For consistency with previous implementation, enumerate files first and directories after
DirectoryInfo dir;
if (IsPrefixedWithExtendedDevicePathPrefix(path))
{
dir = new DirectoryInfo(path);
}
else
{
dir = new DirectoryInfo(AddExtendedDevicePathPrefix(path));
}
foreach (FileInfo file in dir.EnumerateFiles())
{
yield return FileEntry(file);
}
foreach (DirectoryInfo d in dir.EnumerateDirectories())
{
yield return DirectoryEntry(d);
}
}
public void FileCopy(string source, string target, bool overwrite)
{
File.Copy(AddExtendedDevicePathPrefix(source), AddExtendedDevicePathPrefix(target), overwrite);
@@ -498,7 +522,10 @@ namespace Duplicati.Library.Common.IO
public IFileEntry DirectoryEntry(string path)
{
var dInfo = new DirectoryInfo(AddExtendedDevicePathPrefix(path));
return DirectoryEntry(new DirectoryInfo(AddExtendedDevicePathPrefix(path)));
}
public IFileEntry DirectoryEntry(DirectoryInfo dInfo)
{
return new FileEntry(dInfo.Name, 0, dInfo.LastAccessTime, dInfo.LastWriteTime)
{
IsFolder = true
@@ -507,7 +534,10 @@ namespace Duplicati.Library.Common.IO
public IFileEntry FileEntry(string path)
{
var fileInfo = new FileInfo(AddExtendedDevicePathPrefix(path));
return FileEntry(new FileInfo(AddExtendedDevicePathPrefix(path)));
}
public IFileEntry FileEntry(FileInfo fileInfo)
{
var lastAccess = new DateTime();
try
{