diff --git a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs index ff62f5a01..ea0ca266b 100644 --- a/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs +++ b/Duplicati/Library/Main/Operation/Backup/MetadataPreProcess.cs @@ -50,8 +50,18 @@ namespace Duplicati.Library.Main.Operation.Backup public IMetahash MetaHashAndSize; public bool MetadataChanged; } - + + public static Task RunOnlyRecord(Snapshots.ISnapshotService snapshot, Options options) + { + return Run(snapshot, options, null, -1, false); + } + public static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid) + { + return Run(snapshot, options, database, lastfilesetid, true); + } + + private static Task Run(Snapshots.ISnapshotService snapshot, Options options, BackupDatabase database, long lastfilesetid, bool storeData) { return AutomationExtensions.RunTask(new { @@ -60,7 +70,7 @@ namespace Duplicati.Library.Main.Operation.Backup LogChannel = Common.Channels.LogChannel.ForWrite, Output = Backup.Channels.ProcessedFiles.ForWrite, }, - + async self => { var log = new LogWrapper(self.LogChannel); @@ -72,67 +82,86 @@ namespace Duplicati.Library.Main.Operation.Backup var lastwrite = new DateTime(0, DateTimeKind.Utc); var attributes = default(FileAttributes); - try - { - lastwrite = snapshot.GetLastWriteTimeUtc(path); + try + { + lastwrite = snapshot.GetLastWriteTimeUtc(path); } - catch (Exception ex) + catch (Exception ex) { await log.WriteWarningAsync(string.Format("Failed to read timestamp on \"{0}\"", path), ex); } - try - { - attributes = snapshot.GetAttributes(path); + try + { + attributes = snapshot.GetAttributes(path); } - catch (Exception ex) + catch (Exception ex) { await log.WriteWarningAsync(string.Format("Failed to read attributes on \"{0}\"", path), ex); } - // If we only have metadata, stop here - if (await ProcessMetadata(path, attributes, lastwrite, options, log, snapshot, emptymetadata, database, self.StreamBlockChannel)) + if (storeData) { - try + // If we only have metadata, stop here + if (await ProcessMetadata(path, attributes, lastwrite, options, log, snapshot, emptymetadata, database, self.StreamBlockChannel)) { - if (options.CheckFiletimeOnly || options.DisableFiletimeCheck) + try { - var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid); - await self.Output.WriteAsync(new FileEntry() { - OldId = tmp.Key < 0 ? -1 : tmp.Key, - Path = path, - Attributes = attributes, - LastWrite = lastwrite, - OldModified = tmp.Key < 0 ? new DateTime(0) : tmp.Value, - LastFileSize = -1 , - OldMetaHash = null, - OldMetaSize = -1 - }); + if (options.CheckFiletimeOnly || options.DisableFiletimeCheck) + { + var tmp = await database.GetFileLastModifiedAsync(path, lastfilesetid); + await self.Output.WriteAsync(new FileEntry() + { + OldId = tmp.Key < 0 ? -1 : tmp.Key, + Path = path, + Attributes = attributes, + LastWrite = lastwrite, + OldModified = tmp.Key < 0 ? new DateTime(0) : tmp.Value, + LastFileSize = -1, + OldMetaHash = null, + OldMetaSize = -1 + }); + } + else + { + var res = await database.GetFileEntryAsync(path, lastfilesetid); + await self.Output.WriteAsync(new FileEntry() + { + OldId = res == null ? -1 : res.id, + Path = path, + Attributes = attributes, + LastWrite = lastwrite, + OldModified = res == null ? new DateTime(0) : res.modified, + LastFileSize = res == null ? -1 : res.filesize, + OldMetaHash = res == null ? null : res.metahash, + OldMetaSize = res == null ? -1 : res.metasize + }); + } } - else + catch (Exception ex) { - var res = await database.GetFileEntryAsync(path, lastfilesetid); - await self.Output.WriteAsync(new FileEntry() { - OldId = res == null ? -1 : res.id, - Path = path, - Attributes = attributes, - LastWrite = lastwrite, - OldModified = res == null ? new DateTime(0) : res.modified, - LastFileSize = res == null ? -1 : res.filesize, - OldMetaHash = res == null ? null : res.metahash, - OldMetaSize = res == null ? -1 : res.metasize - }); + await log.WriteErrorAsync(string.Format("Failed to process entry, path: {0}", path), ex); } } - catch(Exception ex) - { - await log.WriteErrorAsync(string.Format("Failed to process entry, path: {0}", path), ex); - } + } + else + { + // Write a dummy output + await self.Output.WriteAsync(new FileEntry() + { + OldId = -1, + Path = path, + Attributes = attributes, + LastWrite = lastwrite, + OldModified = new DateTime(0), + LastFileSize = -1, + OldMetaHash = null, + OldMetaSize = -1 + }); + } } - }); - } /// diff --git a/Duplicati/Library/Main/Operation/TestFilterHandler.cs b/Duplicati/Library/Main/Operation/TestFilterHandler.cs index 7b4b4775b..4ab833794 100644 --- a/Duplicati/Library/Main/Operation/TestFilterHandler.cs +++ b/Duplicati/Library/Main/Operation/TestFilterHandler.cs @@ -16,7 +16,9 @@ // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; +using CoCoL; using System.IO; +using System.Threading.Tasks; using Duplicati.Library.Snapshots; namespace Duplicati.Library.Main.Operation @@ -34,49 +36,53 @@ namespace Duplicati.Library.Main.Operation public void Run(string[] sources, Library.Utility.IFilter filter) { - var storeSymlinks = m_options.SymlinkPolicy == Duplicati.Library.Main.Options.SymlinkStrategy.Store; var sourcefilter = new Library.Utility.FilterExpression(sources, true); + using(new IsolatedChannelScope(Common.Channels.LogChannel)) using(var snapshot = BackupHandler.GetSnapshot(sources, m_options, m_result)) { - foreach(var path in new BackupHandler.FilterHandler(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_result).EnumerateFilesAndFolders()) - { - var fa = FileAttributes.Normal; - try { fa = snapshot.GetAttributes(path); } - catch { } - - if (storeSymlinks && snapshot.IsSymlink(path, fa)) + var enumeratorTask = Backup.FileEnumerationProcess.Run(snapshot, m_options.FileAttributeFilter, sourcefilter, filter, m_options.SymlinkPolicy, m_options.HardlinkPolicy, m_options.ChangedFilelist, m_result.TaskReader); + var metadataTask = Backup.MetadataPreProcess.RunOnlyRecord(snapshot, m_options); + + var fileCounterTask = AutomationExtensions.RunTask( + new { - m_result.AddVerboseMessage("Storing symlink: {0}", path); - } - else if ((fa & FileAttributes.Directory) == FileAttributes.Directory) + Input = Backup.Channels.ProcessedFiles.ForRead + }, + async self => { - m_result.AddVerboseMessage("Including folder: {0}", path); - } - else - { - m_result.FileCount++; - var size = -1L; - - try + while (await m_result.TaskReader.ProgressAsync) { - size = snapshot.GetFileSize(path); - m_result.FileSize += size; + var f = await self.Input.ReadAsync(CoCoL.Timeout.Infinite); + long filestatsize = -1; + try + { + filestatsize = snapshot.GetFileSize(f.Path); + } + catch + { + } + + var tooLargeFile = m_options.SkipFilesLargerThan != long.MaxValue && m_options.SkipFilesLargerThan != 0 && filestatsize >= 0 && filestatsize > m_options.SkipFilesLargerThan; + + if (tooLargeFile) + m_result.AddVerboseMessage("Excluding file due to size: {0} ({1})", f.Path, filestatsize < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(filestatsize)); + else + m_result.AddVerboseMessage("Including file: {0} ({1})", f.Path, filestatsize < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(filestatsize)); } - catch - { - } - - - if (m_options.SkipFilesLargerThan == long.MaxValue || m_options.SkipFilesLargerThan == 0 || size < m_options.SkipFilesLargerThan) - m_result.AddVerboseMessage("Including file: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); - else - m_result.AddVerboseMessage("Excluding file due to size: {0} ({1})", path, size < 0 ? "unknown" : Duplicati.Library.Utility.Utility.FormatSizeString(size)); } - - } + ); + + Task.WhenAll( + enumeratorTask, + metadataTask, + fileCounterTask + ) + .WaitForTaskOrThrow(); } } + + #region IDisposable implementation public void Dispose()