diff --git a/Duplicati/Library/Main/BackendWrapper.cs b/Duplicati/Library/Main/BackendWrapper.cs index 19ac43d2d..3010ffa42 100644 --- a/Duplicati/Library/Main/BackendWrapper.cs +++ b/Duplicati/Library/Main/BackendWrapper.cs @@ -59,6 +59,16 @@ namespace Duplicati.Library.Main /// private Backupmetadata m_metadata; + /// + /// Size of manifestA + /// + private long m_manifestSizeA = 0; + + /// + /// Size of manifestB + /// + private long m_manifestSizeB = 0; + /// /// The set of options used by Duplicati /// @@ -337,6 +347,8 @@ namespace Duplicati.Library.Main this.m_metadata.TotalSize = totalSize; this.m_metadata.TotalFileCount = result.Count; + if (m_options.QuotaSize > 0) + this.m_metadata.AssignedQuotaSpace = m_options.QuotaSize; if (!filter) return result; @@ -415,7 +427,7 @@ namespace Duplicati.Library.Main for (int i = 0; i < result.Count; i++) if (lookup.ContainsKey(result[i].Name)) { - m_metadata.SourceFileCount++; + m_metadata.OrphanFileCount++; if (result[i].Size > 0) m_metadata.OrphanFileSize += result[i].Size; @@ -463,12 +475,18 @@ namespace Duplicati.Library.Main } + public void ExtractMetadataFromList(List files) + { + try { SortAndPairSets(files, true); } + catch { } + } + /// /// Parses the filelist into a list of full backups, each with a chain of incrementals attached /// /// The list of filenames found on the backend /// A list of full backups - private List SortAndPairSets(List files) + private List SortAndPairSets(List files, bool metadataOnly) { Sorter sortHelper = new Sorter(); files.Sort(sortHelper); @@ -476,7 +494,7 @@ namespace Duplicati.Library.Main for(int i = 1; i < files.Count; i++) if (files[i].Name == files[i - 1].Name) { - if (m_statistics != null) + if (m_statistics != null && !metadataOnly) m_statistics.LogWarning(string.Format(Strings.BackendWrapper.DuplicateFileEntryWarning, files[i].Name), null); files.RemoveAt(i); @@ -489,13 +507,19 @@ namespace Duplicati.Library.Main Dictionary> contents = new Dictionary>(); Dictionary verifications = new Dictionary(); + //We set these properties so they are reported, and they will be updated to correct values if any alien/orphan files are found + m_metadata.AlienFileCount = 0; + m_metadata.AlienFileSize = 0; + m_metadata.OrphanFileCount = 0; + m_metadata.OrphanFileSize = 0; + //First we parse all files into their respective classes foreach (Duplicati.Library.Interface.IFileEntry fe in files) { BackupEntryBase be = m_filenamestrategy.ParseFilename(fe); if (be == null) { - if (m_statistics != null && m_statistics.VerboseErrors && !fe.IsFolder) + if (m_statistics != null && m_statistics.VerboseErrors && !fe.IsFolder && !metadataOnly) m_statistics.LogWarning(string.Format(Strings.BackendWrapper.UnmatchedFilenameWarning, fe.Name), null); m_metadata.AlienFileCount++; @@ -506,10 +530,28 @@ namespace Duplicati.Library.Main } if (!string.IsNullOrEmpty(be.EncryptionMode) && Array.IndexOf(DynamicLoader.EncryptionLoader.Keys, be.EncryptionMode) < 0) - continue; + { + if (m_statistics != null && !metadataOnly) + m_statistics.LogWarning(string.Format(Strings.BackendWrapper.UnmatchedEncryptionModuleWarning, be.Filename, be.EncryptionMode), null); + + m_metadata.AlienFileCount++; + if (be.Filesize > 0) + m_metadata.AlienFileSize += be.Filesize; - if (be is PayloadEntryBase && Array.IndexOf(DynamicLoader.CompressionLoader.Keys,((PayloadEntryBase)be).Compression) < 0) continue; + } + + if (be is PayloadEntryBase && Array.IndexOf(DynamicLoader.CompressionLoader.Keys, ((PayloadEntryBase)be).Compression) < 0) + { + if (m_statistics != null && !metadataOnly) + m_statistics.LogWarning(string.Format(Strings.BackendWrapper.UnmatchedCompressionModuleWarning, be.Filename, ((PayloadEntryBase)be).Compression), null); + + m_metadata.AlienFileCount++; + if (be.Filesize > 0) + m_metadata.AlienFileSize += be.Filesize; + + continue; + } if (be is ManifestEntry) { @@ -537,7 +579,16 @@ namespace Duplicati.Library.Main verifications[be.TimeString] = (VerificationEntry)be; } else - throw new Exception(string.Format(Strings.BackendWrapper.InvalidEntryTypeError, be.GetType().FullName)); + { + if (!metadataOnly) + throw new Exception(string.Format(Strings.BackendWrapper.InvalidEntryTypeError, be.GetType().FullName)); + else + { + m_metadata.AlienFileCount++; + if (be.Filesize > 0) + m_metadata.AlienFileSize += be.Filesize; + } + } } fulls.Sort(sortHelper); @@ -622,10 +673,13 @@ namespace Duplicati.Library.Main if (index >= fulls.Count || be.Time <= fulls[index].Time) { - if (m_orphans == null) - Logging.Log.WriteMessage(string.Format(Strings.BackendWrapper.OrphanIncrementalFoundMessage, be.Filename), Duplicati.Library.Logging.LogMessageType.Warning); - else - m_orphans.Add(be); + if (!metadataOnly) + { + if (m_orphans == null) + Logging.Log.WriteMessage(string.Format(Strings.BackendWrapper.OrphanIncrementalFoundMessage, be.Filename), Duplicati.Library.Logging.LogMessageType.Warning); + else + m_orphans.Add(be); + } continue; } else @@ -636,16 +690,19 @@ namespace Duplicati.Library.Main } } - foreach (VerificationEntry ve in verifications.Values) - AddOrphan(ve); + if (!metadataOnly) + { + foreach (VerificationEntry ve in verifications.Values) + AddOrphan(ve); - foreach (List lb in contents.Values) - foreach (ContentEntry be in lb) - AddOrphan(be); + foreach (List lb in contents.Values) + foreach (ContentEntry be in lb) + AddOrphan(be); - foreach (List lb in signatures.Values) - foreach (SignatureEntry be in lb) - AddOrphan(be); + foreach (List lb in signatures.Values) + foreach (SignatureEntry be in lb) + AddOrphan(be); + } //Assign the manifest to allow traversing the chain of manifests foreach (ManifestEntry me in fulls) @@ -659,7 +716,7 @@ namespace Duplicati.Library.Main } - if (m_statistics != null) + if (m_statistics != null && !metadataOnly) { foreach (ManifestEntry me in fulls) { @@ -672,35 +729,7 @@ namespace Duplicati.Library.Main } } - m_metadata.FullBackupCount = fulls.Count; - long maxlen = 0; - long volumecount = 0; - long fullsize = 0; - foreach (ManifestEntry me in fulls) - { - maxlen = Math.Max(maxlen, me.Incrementals.Count + 1); - volumecount += me.Volumes.Count; - foreach (ManifestEntry mex in me.Incrementals) - volumecount += mex.Volumes.Count; - fullsize += CalculateSizeOfBackupSet(me, true); - } - m_metadata.LongestChainLength = maxlen; - m_metadata.TotalVolumeCount = volumecount; - m_metadata.TotalBackupSize = fullsize; - - if (fulls.Count > 0) - { - ManifestEntry cur = fulls[fulls.Count - 1]; - m_metadata.CurrentChainLength = cur.Incrementals.Count + 1; - m_metadata.CurrentFullDate = cur.Time; - m_metadata.CurrentChainSize = CalculateSizeOfBackupSet(cur, true); - - if (cur.Incrementals.Count > 0) - cur = cur.Incrementals[cur.Incrementals.Count - 1]; - - m_metadata.LastBackupDate = cur.Time; - m_metadata.LastBackupSize = CalculateSizeOfBackupSet(cur, false); - } + RecalculateChainMetadata(fulls); return fulls; } @@ -752,7 +781,7 @@ namespace Duplicati.Library.Main public List GetBackupSets() { using (new Logging.Timer("Getting and sorting filelist from " + m_backend.DisplayName)) - return SortAndPairSets(List(true)); + return SortAndPairSets(List(true), false); } public void Put(BackupEntryBase remote, string filename) @@ -967,14 +996,23 @@ namespace Duplicati.Library.Main BackupEntryBase be = m_filenamestrategy.ParseFilename(s); if (m_options.Force) + { this.Delete(be ?? new SignatureEntry(s.Name, s, DateTime.Now, true, "", "", "", 0)); + m_metadata.OrphanFileCount--; + if (s.Size > 0) + m_metadata.OrphanFileSize -= s.Size; + } } sb.AppendLine(string.Format(Strings.BackendWrapper.DeletingTransactionLeftoverFile, m_transactionFile.Filename)); if (m_options.Force) this.Delete(m_transactionFile); - + + m_metadata.OrphanFileCount--; + if (m_transactionFile.Filesize > 0) + m_metadata.OrphanFileSize -= m_transactionFile.Filesize; + m_transactionFile = null; sb.AppendLine(Strings.BackendWrapper.CompletedDeleteTransaction); @@ -1026,6 +1064,9 @@ namespace Duplicati.Library.Main if (m_statistics != null) m_statistics.LogWarning(string.Format(Strings.BackendWrapper.RemoveOrphanFileWarning, be.Filename), null); this.Delete(be); + m_metadata.OrphanFileCount--; + if (be.Filesize > 0) + m_metadata.OrphanFileSize -= be.Filesize; } if (be is ManifestEntry) @@ -1045,7 +1086,14 @@ namespace Duplicati.Library.Main } this.Delete(bex.Key); + m_metadata.OrphanFileCount--; + if (bex.Key.Filesize > 0) + m_metadata.OrphanFileSize -= bex.Key.Filesize; + this.Delete(bex.Value); + m_metadata.OrphanFileCount--; + if (bex.Value.Filesize > 0) + m_metadata.OrphanFileSize -= bex.Value.Filesize; } } } @@ -1152,6 +1200,10 @@ namespace Duplicati.Library.Main if (lastEx != null) throw new Exception(string.Format(Strings.BackendWrapper.FileDeleteError2, remote.Filename, lastEx.Message), lastEx); + m_metadata.TotalFileCount--; + if (remote.Filesize > 0) + m_metadata.TotalSize -= remote.Filesize; + if (remote is SignatureEntry && !string.IsNullOrEmpty(m_options.SignatureCachePath)) { try @@ -1514,6 +1566,33 @@ namespace Duplicati.Library.Main throw new Exception(string.Format(Strings.BackendWrapper.FileUploadError2, remotename, lastEx == null ? "" : lastEx.Message), lastEx); m_statistics.AddBytesUploaded(sourceFileSize); + if (remote is ManifestEntry) + { + if (remotename.IndexOf("manifestA") > remotename.IndexOf("manifestB")) + { + if (m_manifestSizeA == 0) + m_metadata.TotalFileCount++; + + m_metadata.SourceFileSize += (sourceFileSize - m_manifestSizeA); + m_manifestSizeA = sourceFileSize; + } + else + { + if (m_manifestSizeB == 0) + m_metadata.TotalFileCount++; + + m_metadata.SourceFileSize += (sourceFileSize - m_manifestSizeB); + m_manifestSizeB = sourceFileSize; + } + + m_metadata.TotalVolumeCount++; + } + else + { + m_metadata.TotalSize += sourceFileSize; + m_metadata.TotalBackupSize += sourceFileSize; + m_metadata.TotalFileCount ++; + } } finally { @@ -1791,5 +1870,53 @@ namespace Duplicati.Library.Main } #endregion + + public void RecalculateChainMetadata(List flatlist) + { + List fulls = new List(); + foreach (ManifestEntry mfe in flatlist) + if (mfe.IsFull) + fulls.Add(mfe); + + m_metadata.FullBackupCount = fulls.Count; + long maxlen = 0; + long volumecount = 0; + long fullsize = 0; + long sets = 0; + foreach (ManifestEntry me in fulls) + { + maxlen = Math.Max(maxlen, me.Incrementals.Count + 1); + volumecount += me.Volumes.Count; + sets++; + foreach (ManifestEntry mex in me.Incrementals) + { + volumecount += mex.Volumes.Count; + sets++; + } + fullsize += CalculateSizeOfBackupSet(me, true); + } + m_metadata.LongestChainLength = maxlen; + m_metadata.TotalVolumeCount = volumecount; + m_metadata.TotalBackupSize = fullsize; + m_metadata.TotalBackupSets = sets; + + if (fulls.Count > 0) + { + ManifestEntry cur = fulls[fulls.Count - 1]; + m_metadata.CurrentChainLength = cur.Incrementals.Count + 1; + m_metadata.CurrentFullDate = cur.Time; + m_metadata.CurrentChainSize = CalculateSizeOfBackupSet(cur, true); + + if (cur.Incrementals.Count > 0) + cur = cur.Incrementals[cur.Incrementals.Count - 1]; + + m_metadata.LastBackupDate = cur.Time; + m_metadata.LastBackupSize = CalculateSizeOfBackupSet(cur, false); + } + else + { + m_metadata.RemoveCurrentBackupData(); + } + } } } diff --git a/Duplicati/Library/Main/Backupmetadata.cs b/Duplicati/Library/Main/Backupmetadata.cs index 3d7cae29c..83b154313 100644 --- a/Duplicati/Library/Main/Backupmetadata.cs +++ b/Duplicati/Library/Main/Backupmetadata.cs @@ -33,9 +33,11 @@ namespace Duplicati.Library.Main private long? m_source_folder_count; private long? m_total_backup_size; + private long? m_total_backup_sets; private long? m_total_quota_space; private long? m_free_quota_space; + private long? m_assigned_quota_space; public long TotalSize { get { return m_total_size ?? 0; } set { m_total_size = value; } } public long AlienFileSize { get { return m_alien_file_size ?? 0; } set { m_alien_file_size = value; } } @@ -55,8 +57,10 @@ namespace Duplicati.Library.Main public long SourceFileCount { get { return m_source_file_count ?? 0; } set { m_source_file_count = value; } } public long SourceFolderCount { get { return m_source_folder_count ?? 0; } set { m_source_folder_count = value; } } public long TotalBackupSize { get { return m_total_backup_size ?? 0; } set { m_total_backup_size = value; } } + public long TotalBackupSets { get { return m_total_backup_sets ?? 0; } set { m_total_backup_sets = value; } } public long TotalQuotaSpace { get { return m_total_quota_space ?? 0; } set { m_total_quota_space = value; } } public long FreeQuotaSpace { get { return m_free_quota_space ?? 0; } set { m_free_quota_space = value; } } + public long AssignedQuotaSpace { get { return m_assigned_quota_space ?? 0; } set { m_assigned_quota_space = value; } } public IDictionary AsReport() { @@ -69,12 +73,24 @@ namespace Duplicati.Library.Main if (v != null) { string reportname = fi.Name.Substring("m_".Length).Replace('_', '-'); - result[reportname] = v.ToString(); + if (fi.FieldType == typeof(DateTime?)) + result[reportname] = ((DateTime?)v).Value.ToUniversalTime().ToString("u"); + else + result[reportname] = v.ToString(); } } } return result; } + + public void RemoveCurrentBackupData() + { + m_current_chain_length = null; + m_current_full_date = null; + m_current_chain_size = null; + m_last_backup_date = null; + m_last_backup_size = null; + } } } diff --git a/Duplicati/Library/Main/Interface.cs b/Duplicati/Library/Main/Interface.cs index 8f0fd61c4..76636127c 100644 --- a/Duplicati/Library/Main/Interface.cs +++ b/Duplicati/Library/Main/Interface.cs @@ -381,7 +381,12 @@ namespace Duplicati.Library.Main if (full) { //This will create the target folder - backend.List(false); + List tmp = backend.List(false); + + //If it is possible to parse the chain, we extract some metadata + try { backend.ExtractMetadataFromList(tmp); } + catch { } + backupsets = new List(); } else @@ -575,7 +580,7 @@ namespace Duplicati.Library.Main OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusBuildingFilelist, ""); - bool completedWithoutChanges; + bool completedWithoutChanges = true; using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(manifest.SourceDirs, bs, backend.Metadata, m_options.Filter, patches)) { @@ -722,6 +727,25 @@ namespace Duplicati.Library.Main vol++; } } + + if (!completedWithoutChanges) + { + if (full) + { + backend.Metadata.CurrentChainLength = 1; + backend.Metadata.CurrentFullDate = backupchaintime; + backend.Metadata.FullBackupCount++; + } + else + { + backend.Metadata.CurrentChainLength++; + } + backend.Metadata.LastBackupDate = backupchaintime; + backend.Metadata.LongestChainLength = Math.Max(backend.Metadata.CurrentChainLength, backend.Metadata.LongestChainLength); + backend.Metadata.TotalVolumeCount += vol; + backend.Metadata.TotalBackupSets++; + } + } @@ -1518,6 +1542,7 @@ namespace Duplicati.Library.Main throw new Exception(Strings.Interface.InternalDeleteCountError); sb.Append(RemoveBackupSets(backend, toremove)); + backend.RecalculateChainMetadata(flatlist); } finally { @@ -1569,6 +1594,7 @@ namespace Duplicati.Library.Main throw new Exception(Strings.Interface.InternalDeleteCountError); sb.Append(RemoveBackupSets(backend, toremove)); + backend.RecalculateChainMetadata(entries); } finally { @@ -1641,6 +1667,7 @@ namespace Duplicati.Library.Main throw new Exception(Strings.Interface.InternalDeleteCountError); sb.Append(RemoveBackupSets(backend, toremove)); + backend.RecalculateChainMetadata(entries); } finally { @@ -1711,7 +1738,13 @@ namespace Duplicati.Library.Main { backend.Delete(kx.Key); backend.Delete(kx.Value); + backend.Metadata.TotalVolumeCount--; } + + if (me.IsFull) + backend.Metadata.FullBackupCount--; + + backend.Metadata.TotalBackupSets--; } } diff --git a/Duplicati/Library/Main/Strings/BackendWrapper.Designer.cs b/Duplicati/Library/Main/Strings/BackendWrapper.Designer.cs index 96c014fab..bf6d6a8f0 100644 --- a/Duplicati/Library/Main/Strings/BackendWrapper.Designer.cs +++ b/Duplicati/Library/Main/Strings/BackendWrapper.Designer.cs @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ // // This code was generated by a tool. -// Runtime Version:4.0.30319.261 +// Runtime Version:4.0.30319.269 // // Changes to this file may cause incorrect behavior and will be lost if // the code is regenerated. @@ -386,6 +386,24 @@ namespace Duplicati.Library.Main.Strings { } } + /// + /// Looks up a localized string similar to The file "{0}" has compression module "{1}" which is not supported, ignoring file. + /// + internal static string UnmatchedCompressionModuleWarning { + get { + return ResourceManager.GetString("UnmatchedCompressionModuleWarning", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The file "{0}" has encryption module "{1}" which is not supported, ignoring file. + /// + internal static string UnmatchedEncryptionModuleWarning { + get { + return ResourceManager.GetString("UnmatchedEncryptionModuleWarning", resourceCulture); + } + } + /// /// Looks up a localized string similar to The file {0} was found on the backend but not recognized by Duplicati. /// diff --git a/Duplicati/Library/Main/Strings/BackendWrapper.resx b/Duplicati/Library/Main/Strings/BackendWrapper.resx index d5ae892ec..e37f3ed50 100644 --- a/Duplicati/Library/Main/Strings/BackendWrapper.resx +++ b/Duplicati/Library/Main/Strings/BackendWrapper.resx @@ -112,10 +112,10 @@ 2.0 - System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 Unable to find backend for: {0} @@ -236,4 +236,10 @@ If this problem happens more than once, it is possible that the delete transacti Failed to automatically create folder, error message: {0} + + The file "{0}" has compression module "{1}" which is not supported, ignoring file + + + The file "{0}" has encryption module "{1}" which is not supported, ignoring file + \ No newline at end of file diff --git a/Duplicati/Library/Main/Strings/Interface.Designer.cs b/Duplicati/Library/Main/Strings/Interface.Designer.cs index 3b3162d46..641d0f88c 100644 --- a/Duplicati/Library/Main/Strings/Interface.Designer.cs +++ b/Duplicati/Library/Main/Strings/Interface.Designer.cs @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ // // This code was generated by a tool. -// Runtime Version:4.0.30319.239 +// Runtime Version:4.0.30319.269 // // Changes to this file may cause incorrect behavior and will be lost if // the code is regenerated.