haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re-implemented the test-filter method with channels such that it calls the same code as the backup process

Browse Source
This commit is contained in:
Kenneth Skovhede
2018-03-06 13:47:47 +01:00
parent ef4a7e75bb
commit 7db715b50b
2 changed files with 110 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs
+72 -43
View File
@@ -50,8 +50,18 @@ namespace Duplicati.Library.Main.Operation.Backup
public IMetahash MetaHashAndSize;
public bool MetadataChanged;
}
public static Task RunOnlyRecord(Snapshots.ISnapshotService snapshot, Options options)
{
return Run(snapshot, options, null, -1, false);
}
public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid)
{
return Run(snapshot, options, database, lastfilesetid, true);
}
private static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, bool storeData)
{
return AutomationExtensions.RunTask(new
{
@@ -60,7 +70,7 @@ namespace Duplicati.Library.Main.Operation.Backup
LogChannel = Common.Channels.LogChannel.ForWrite,
Output = Backup.Channels.ProcessedFiles.ForWrite,
},
async self =>
{
var log = new LogWrapper(self.LogChannel);
@@ -72,67 +82,86 @@ namespace Duplicati.Library.Main.Operation.Backup
var lastwrite = new DateTime(0, DateTimeKind.Utc);
var attributes = default(FileAttributes);
try
{
lastwrite = snapshot.GetLastWriteTimeUtc(path);
try
{
lastwrite = snapshot.GetLastWriteTimeUtc(path);
}
catch (Exception ex)
catch (Exception ex)
{
await log.WriteWarningAsync(string.Format("Failed to read timestamp on \"{0}\"", path), ex);
}
try
{
attributes = snapshot.GetAttributes(path);
try
{
attributes = snapshot.GetAttributes(path);
}
catch (Exception ex)
catch (Exception ex)
{
await log.WriteWarningAsync(string.Format("Failed to read attributes on \"{0}\"", path), ex);
}
// If we only have metadata, stop here
if (await ProcessMetadata(path, attributes, lastwrite, options, log, snapshot, emptymetadata, database, self.StreamBlockChannel))
if (storeData)
{
try
// If we only have metadata, stop here
if (await ProcessMetadata(path, attributes, lastwrite, options, log, snapshot, emptymetadata, database, self.StreamBlockChannel))
{
if (options.CheckFiletimeOnly || options.DisableFiletimeCheck)
try
{
var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid);
await self.Output.WriteAsync(new FileEntry() {
OldId = tmp.Key < 0 ? -1 : tmp.Key,
Path = path,
Attributes = attributes,
LastWrite = lastwrite,
OldModified = tmp.Key < 0 ? new DateTime(0) : tmp.Value,
LastFileSize = -1 ,
OldMetaHash = null,
OldMetaSize = -1
});
if (options.CheckFiletimeOnly || options.DisableFiletimeCheck)
{
var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid);
await self.Output.WriteAsync(new FileEntry()
{
OldId = tmp.Key < 0 ? -1 : tmp.Key,
Path = path,
Attributes = attributes,
LastWrite = lastwrite,
OldModified = tmp.Key < 0 ? new DateTime(0) : tmp.Value,
LastFileSize = -1,
OldMetaHash = null,
OldMetaSize = -1
});
}
else
{
var res = await database.GetFileEntryAsync(path, lastfilesetid);
await self.Output.WriteAsync(new FileEntry()
{
OldId = res == null ? -1 : res.id,
Path = path,
Attributes = attributes,
LastWrite = lastwrite,
OldModified = res == null ? new DateTime(0) : res.modified,
LastFileSize = res == null ? -1 : res.filesize,
OldMetaHash = res == null ? null : res.metahash,
OldMetaSize = res == null ? -1 : res.metasize
});
}
}
else
catch (Exception ex)
{
var res = await database.GetFileEntryAsync(path, lastfilesetid);
await self.Output.WriteAsync(new FileEntry() {
OldId = res == null ? -1 : res.id,
Path = path,
Attributes = attributes,
LastWrite = lastwrite,
OldModified = res == null ? new DateTime(0) : res.modified,
LastFileSize = res == null ? -1 : res.filesize,
OldMetaHash = res == null ? null : res.metahash,
OldMetaSize = res == null ? -1 : res.metasize
});
await log.WriteErrorAsync(string.Format("Failed to process entry, path: {0}", path), ex);
}
}
catch(Exception ex)
{
await log.WriteErrorAsync(string.Format("Failed to process entry, path: {0}", path), ex);
}
}
else
{
// Write a dummy output
await self.Output.WriteAsync(new FileEntry()
{
OldId = -1,
Path = path,
Attributes = attributes,
LastWrite = lastwrite,
OldModified = new DateTime(0),
LastFileSize = -1,
OldMetaHash = null,
OldMetaSize = -1
});
}
}
});
}
/// <summary>
Duplicati/Library/Main/Operation/TestFilterHandler.cs
+38 -32
View File
@@ -16,7 +16,9 @@
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using System;
using CoCoL;
using System.IO;
using System.Threading.Tasks;
using Duplicati.Library.Snapshots;
namespace Duplicati.Library.Main.Operation
@@ -34,49 +36,53 @@ namespace Duplicati.Library.Main.Operation
public void Run(string[] sources, Library.Utility.IFilter filter)
{
var storeSymlinks = m_options.SymlinkPolicy == Duplicati.Library.Main.Options.SymlinkStrategy.Store;
var sourcefilter = new Library.Utility.FilterExpression(sources, true);
using(new IsolatedChannelScope(Common.Channels.LogChannel))
using(var snapshot = BackupHandler.GetSnapshot(sources, m_options, m_result))
{
foreach(var path in new BackupHandler.FilterHandler(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_result).EnumerateFilesAndFolders())
{
var fa = FileAttributes.Normal;
try { fa = snapshot.GetAttributes(path); }
catch { }
if (storeSymlinks && snapshot.IsSymlink(path, fa))
var enumeratorTask = Backup.FileEnumerationProcess.Run(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_options.ChangedFilelist, m_result.TaskReader);
var metadataTask = Backup.MetadataPreProcess.RunOnlyRecord(snapshot, m_options);
var fileCounterTask = AutomationExtensions.RunTask(
new
{
m_result.AddVerboseMessage("Storing symlink: {0}", path);
}
else if ((fa & FileAttributes.Directory) == FileAttributes.Directory)
Input = Backup.Channels.ProcessedFiles.ForRead
},
async self =>
{
m_result.AddVerboseMessage("Including folder: {0}", path);
}
else
{
m_result.FileCount++;
var size = -1L;
try
while (await m_result.TaskReader.ProgressAsync)
{
size = snapshot.GetFileSize(path);
m_result.FileSize += size;
var f = await self.Input.ReadAsync(CoCoL.Timeout.Infinite);
long filestatsize = -1;
try
{
filestatsize = snapshot.GetFileSize(f.Path);
}
catch
{
}
var tooLargeFile = m_options.SkipFilesLargerThan != long.MaxValue && m_options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > m_options.SkipFilesLargerThan;
if (tooLargeFile)
m_result.AddVerboseMessage("Excluding file due to size: {0} ({1})", f.Path, filestatsize < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(filestatsize));
else
m_result.AddVerboseMessage("Including file: {0} ({1})", f.Path, filestatsize < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(filestatsize));
}
catch
{
}
if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan)
m_result.AddVerboseMessage("Including file: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
else
m_result.AddVerboseMessage("Excluding file due to size: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size));
}
}
);
Task.WhenAll(
enumeratorTask,
metadataTask,
fileCounterTask
)
.WaitForTaskOrThrow();
}
}
#region IDisposable implementation
public void Dispose()