// Copyright (C) 2015, The Duplicati Team // http://www.duplicati.com, info@duplicati.com // // This library is free software; you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as // published by the Free Software Foundation; either version 2.1 of the // License, or (at your option) any later version. // // This library is distributed in the hope that it will be useful, but // WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU // Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; using CoCoL; using System.Threading.Tasks; using System.IO; using System.Collections.Generic; using Duplicati.Library.Interface; namespace Duplicati.Library.Main.Operation.Backup { public class MetadataPreProcess : ProcessHelper { public struct FileEntry { public long OldId; public string Path; public DateTime OldModified; public long LastFileSize; public string OldMetaHash; public long OldMetaSize; public DateTime LastWrite; public FileAttributes Attributes; } [ChannelName("SourcePaths")] private IReadChannel m_input; [ChannelName("LogChannel")] private IWriteChannel m_logchannel; [ChannelName("ProcessedFiles")] private IWriteChannel m_output; [ChannelName("OutputBlocks")] private IWriteChannel m_blockoutput; private Snapshots.ISnapshotService m_snapshot; private Options m_options; private Database.LocalBackupDatabase m_database; private readonly IMetahash EMPTY_METADATA; private int m_blocksize; public MetadataPreProcess(Snapshots.ISnapshotService snapshot, Options options, Database.LocalBackupDatabase database) : base() { m_snapshot = snapshot; m_options = options; m_database = database; m_blocksize = m_options.Blocksize; EMPTY_METADATA = Utility.WrapMetadata(new Dictionary(), options); } private async Task ProcessMetadata(string path, FileAttributes attributes, DateTime lastwrite) { if ((attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) { if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Ignore) { await m_logchannel.WriteAsync(LogMessage.Verbose("Ignoring symlink {0}", path)); return false; } if (m_options.SymlinkPolicy == Options.SymlinkStrategy.Store) { var metadata = await MetadataGenerator.GenerateMetadataAsync(path, attributes, m_options, m_snapshot, m_logchannel); if (!metadata.ContainsKey("CoreSymlinkTarget")) metadata["CoreSymlinkTarget"] = m_snapshot.GetSymlinkTarget(path); var metahash = Utility.WrapMetadata(metadata, m_options); AddSymlinkToOutput(path, DateTime.UtcNow, metahash); await m_logchannel.WriteAsync(LogMessage.Verbose("Stored symlink {0}", path)); // Don't process further return false; } } if ((attributes & FileAttributes.Directory) == FileAttributes.Directory) { IMetahash metahash; if (m_options.StoreMetadata) { metahash = Utility.WrapMetadata(await MetadataGenerator.GenerateMetadataAsync(path, attributes, m_options, m_snapshot, m_logchannel), m_options); } else { metahash = EMPTY_METADATA; } await m_logchannel.WriteAsync(LogMessage.Verbose("Adding directory {0}", path)); AddFolderToOutput(path, lastwrite, metahash); return false; } // Regular file, keep going return true; } /// /// Adds a file to the output, /// /// The name of the file to record /// The value of the lastModified timestamp /// The list of hashes that make up the file /// The size of the file /// The offset into a fragment block where the last few bytes are stored /// A lookup table with various metadata values describing the file private async Task AddFolderToOutput(string filename, DateTime lastModified, IMetahash meta) { long metadataid; bool r = false; if (meta.Size > m_blocksize) throw new InvalidDataException(string.Format("Too large metadata, cannot handle more than {0} bytes", m_blocksize)); r |= await FileSplitterProcess.DataBlock.AddBlockToOutputAsync(m_blockoutput, meta.Hash, meta.Blob, 0, (int)meta.Size, CompressionHint.Default, false); lock(m_database.AccessLock) r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid, m_transaction); m_database.AddDirectoryEntry(filename, metadataid, lastModified, m_transaction); return r; } /// /// Adds a file to the output, /// /// The name of the file to record /// The value of the lastModified timestamp /// The list of hashes that make up the file /// The size of the file /// The offset into a fragment block where the last few bytes are stored /// A lookup table with various metadata values describing the file private async Task AddSymlinkToOutput(string filename, DateTime lastModified, IMetahash meta) { long metadataid; bool r = false; if (meta.Size > m_blocksize) throw new InvalidDataException(string.Format("Too large metadata, cannot handle more than {0} bytes", m_blocksize)); r |= await FileSplitterProcess.DataBlock.AddBlockToOutputAsync(m_blockoutput, meta.Hash, meta.Blob, 0, (int)meta.Size, CompressionHint.Default, false); lock(m_database.AccessLock) r |= m_database.AddMetadataset(meta.Hash, meta.Size, out metadataid, m_transaction); lock(m_database.AccessLock) m_database.AddSymlinkEntry(filename, metadataid, lastModified, m_transaction); return r; } protected override async Task Start() { using(var input = m_input.AsReadOnly()) { while (true) { var path = await m_input.ReadAsync(); DateTime lastwrite = new DateTime(0, DateTimeKind.Utc); FileAttributes attributes; try { lastwrite = m_snapshot.GetLastWriteTimeUtc(path); } catch (Exception ex) { await m_logchannel.WriteAsync(LogMessage.Warning(string.Format("Failed to read timestamp on \"{0}\"", path), ex)); } try { attributes = m_snapshot.GetAttributes(path); } catch (Exception ex) { await m_logchannel.WriteAsync(LogMessage.Warning(string.Format("Failed to read attributes on \"{0}\"", path), ex)); } // If we only have metadata, stop here if (await ProcessMetadata(path, attributes, lastwrite)) { try { DateTime oldModified; long lastFileSize = -1; string oldMetahash; long oldMetasize; long oldId; lock(m_database.AccessLock) oldId = m_database.GetFileEntry(path, out oldModified, out lastFileSize, out oldMetahash, out oldMetasize); await m_output.WriteAsync(new FileEntry() { OldId = oldId, Path = path, Attributes = attributes, LastWrite = lastwrite, OldModified = oldModified, LastFileSize = lastFileSize, OldMetaHash = oldMetahash, OldMetaSize = oldMetasize }); } catch(Exception ex) { await m_logchannel.WriteAsync(LogMessage.Error(string.Format("Failed to process entry, path: {0}", path), ex)); } } } } } } }