Updated metadata reporting to better reflect the state AFTER the action, instead of before.

git-svn-id: https://duplicati.googlecode.com/svn/trunk@1292 59da171f-624f-0410-aa54-27559c288bec
2012-05-15 17:59:40 +00:00
parent b950d6e342
commit 722abc9d20
6 changed files with 258 additions and 58 deletions
@@ -59,6 +59,16 @@ namespace Duplicati.Library.Main
        /// </summary>
        private Backupmetadata m_metadata;

+        /// <summary>
+        /// Size of manifestA
+        /// </summary>
+        private long m_manifestSizeA = 0;
+
+        /// <summary>
+        /// Size of manifestB
+        /// </summary>
+        private long m_manifestSizeB = 0;
+
        /// <summary>
        /// The set of options used by Duplicati
        /// </summary>
@@ -337,6 +347,8 @@ namespace Duplicati.Library.Main

            this.m_metadata.TotalSize = totalSize;
            this.m_metadata.TotalFileCount = result.Count;
+            if (m_options.QuotaSize > 0)
+                this.m_metadata.AssignedQuotaSpace = m_options.QuotaSize;

            if (!filter)
                return result;
@@ -415,7 +427,7 @@ namespace Duplicati.Library.Main
                for (int i = 0; i < result.Count; i++)
                    if (lookup.ContainsKey(result[i].Name))
                    {
-                        m_metadata.SourceFileCount++;
+                        m_metadata.OrphanFileCount++;
                        if (result[i].Size > 0)
                            m_metadata.OrphanFileSize += result[i].Size;

@@ -463,12 +475,18 @@ namespace Duplicati.Library.Main

        }

+        public void ExtractMetadataFromList(List<Duplicati.Library.Interface.IFileEntry> files)
+        {
+            try { SortAndPairSets(files, true); }
+            catch { }
+        }
+
        /// <summary>
        /// Parses the filelist into a list of full backups, each with a chain of incrementals attached
        /// </summary>
        /// <param name="files">The list of filenames found on the backend</param>
        /// <returns>A list of full backups</returns>
-        private List<ManifestEntry> SortAndPairSets(List<Duplicati.Library.Interface.IFileEntry> files)
+        private List<ManifestEntry> SortAndPairSets(List<Duplicati.Library.Interface.IFileEntry> files, bool metadataOnly)
        {
            Sorter sortHelper = new Sorter();
            files.Sort(sortHelper);
@@ -476,7 +494,7 @@ namespace Duplicati.Library.Main
            for(int i = 1; i < files.Count; i++)
                if (files[i].Name == files[i - 1].Name)
                {
-                    if (m_statistics != null)
+                    if (m_statistics != null && !metadataOnly)
                        m_statistics.LogWarning(string.Format(Strings.BackendWrapper.DuplicateFileEntryWarning, files[i].Name), null);
                    
                    files.RemoveAt(i);
@@ -489,13 +507,19 @@ namespace Duplicati.Library.Main
            Dictionary<string, List<ContentEntry>> contents = new Dictionary<string, List<ContentEntry>>();
            Dictionary<string, VerificationEntry> verifications = new Dictionary<string, VerificationEntry>();

+            //We set these properties so they are reported, and they will be updated to correct values if any alien/orphan files are found
+            m_metadata.AlienFileCount = 0;
+            m_metadata.AlienFileSize = 0;
+            m_metadata.OrphanFileCount = 0; 
+            m_metadata.OrphanFileSize = 0;
+
            //First we parse all files into their respective classes
            foreach (Duplicati.Library.Interface.IFileEntry fe in files)
            {
                BackupEntryBase be = m_filenamestrategy.ParseFilename(fe);
                if (be == null)
                {
-                    if (m_statistics != null && m_statistics.VerboseErrors && !fe.IsFolder)
+                    if (m_statistics != null && m_statistics.VerboseErrors && !fe.IsFolder && !metadataOnly)
                        m_statistics.LogWarning(string.Format(Strings.BackendWrapper.UnmatchedFilenameWarning, fe.Name), null);

                    m_metadata.AlienFileCount++;
@@ -506,10 +530,28 @@ namespace Duplicati.Library.Main
                }

                if (!string.IsNullOrEmpty(be.EncryptionMode) && Array.IndexOf<string>(DynamicLoader.EncryptionLoader.Keys, be.EncryptionMode) < 0)
-                    continue;
+                {
+                    if (m_statistics != null && !metadataOnly)
+                        m_statistics.LogWarning(string.Format(Strings.BackendWrapper.UnmatchedEncryptionModuleWarning, be.Filename, be.EncryptionMode), null);
+
+                    m_metadata.AlienFileCount++;
+                    if (be.Filesize > 0)
+                        m_metadata.AlienFileSize += be.Filesize;

-                if (be is PayloadEntryBase && Array.IndexOf<string>(DynamicLoader.CompressionLoader.Keys,((PayloadEntryBase)be).Compression) < 0)
                    continue;
+                }
+
+                if (be is PayloadEntryBase && Array.IndexOf<string>(DynamicLoader.CompressionLoader.Keys, ((PayloadEntryBase)be).Compression) < 0)
+                {
+                    if (m_statistics != null && !metadataOnly)
+                        m_statistics.LogWarning(string.Format(Strings.BackendWrapper.UnmatchedCompressionModuleWarning, be.Filename, ((PayloadEntryBase)be).Compression), null);
+
+                    m_metadata.AlienFileCount++;
+                    if (be.Filesize > 0)
+                        m_metadata.AlienFileSize += be.Filesize;
+
+                    continue;
+                }

                if (be is ManifestEntry)
                {
@@ -537,7 +579,16 @@ namespace Duplicati.Library.Main
                    verifications[be.TimeString] = (VerificationEntry)be;
                }
                else
-                    throw new Exception(string.Format(Strings.BackendWrapper.InvalidEntryTypeError, be.GetType().FullName));
+                {
+                    if (!metadataOnly)
+                        throw new Exception(string.Format(Strings.BackendWrapper.InvalidEntryTypeError, be.GetType().FullName));
+                    else
+                    {
+                        m_metadata.AlienFileCount++;
+                        if (be.Filesize > 0)
+                            m_metadata.AlienFileSize += be.Filesize;
+                    }
+                }
            }

            fulls.Sort(sortHelper);
@@ -622,10 +673,13 @@ namespace Duplicati.Library.Main

                if (index >= fulls.Count || be.Time <= fulls[index].Time)
                {
-                    if (m_orphans == null)
-                        Logging.Log.WriteMessage(string.Format(Strings.BackendWrapper.OrphanIncrementalFoundMessage, be.Filename), Duplicati.Library.Logging.LogMessageType.Warning);
-                    else
-                        m_orphans.Add(be);
+                    if (!metadataOnly)
+                    {
+                        if (m_orphans == null)
+                            Logging.Log.WriteMessage(string.Format(Strings.BackendWrapper.OrphanIncrementalFoundMessage, be.Filename), Duplicati.Library.Logging.LogMessageType.Warning);
+                        else
+                            m_orphans.Add(be);
+                    }
                    continue;
                }
                else
@@ -636,16 +690,19 @@ namespace Duplicati.Library.Main
                }
            }

-            foreach (VerificationEntry ve in verifications.Values)
-                AddOrphan(ve);
+            if (!metadataOnly)
+            {
+                foreach (VerificationEntry ve in verifications.Values)
+                    AddOrphan(ve);

-            foreach (List<ContentEntry> lb in contents.Values)
-                foreach (ContentEntry be in lb)
-                    AddOrphan(be);
+                foreach (List<ContentEntry> lb in contents.Values)
+                    foreach (ContentEntry be in lb)
+                        AddOrphan(be);

-            foreach (List<SignatureEntry> lb in signatures.Values)
-                foreach (SignatureEntry be in lb)
-                    AddOrphan(be);
+                foreach (List<SignatureEntry> lb in signatures.Values)
+                    foreach (SignatureEntry be in lb)
+                        AddOrphan(be);
+            }

            //Assign the manifest to allow traversing the chain of manifests
            foreach (ManifestEntry me in fulls)
@@ -659,7 +716,7 @@ namespace Duplicati.Library.Main
            }


-            if (m_statistics != null)
+            if (m_statistics != null && !metadataOnly)
            {
                foreach (ManifestEntry me in fulls)
                {
@@ -672,35 +729,7 @@ namespace Duplicati.Library.Main
                }
            }

-            m_metadata.FullBackupCount = fulls.Count;
-            long maxlen = 0;
-            long volumecount = 0;
-            long fullsize = 0;
-            foreach (ManifestEntry me in fulls)
-            {
-                maxlen = Math.Max(maxlen, me.Incrementals.Count + 1);
-                volumecount += me.Volumes.Count;
-                foreach (ManifestEntry mex in me.Incrementals)
-                    volumecount += mex.Volumes.Count;
-                fullsize += CalculateSizeOfBackupSet(me, true);
-            }
-            m_metadata.LongestChainLength = maxlen;
-            m_metadata.TotalVolumeCount = volumecount;
-            m_metadata.TotalBackupSize = fullsize;
-
-            if (fulls.Count > 0)
-            {
-                ManifestEntry cur = fulls[fulls.Count - 1];
-                m_metadata.CurrentChainLength = cur.Incrementals.Count + 1;
-                m_metadata.CurrentFullDate = cur.Time;
-                m_metadata.CurrentChainSize = CalculateSizeOfBackupSet(cur, true);
-
-                if (cur.Incrementals.Count > 0)
-                    cur = cur.Incrementals[cur.Incrementals.Count - 1];
-
-                m_metadata.LastBackupDate = cur.Time;
-                m_metadata.LastBackupSize = CalculateSizeOfBackupSet(cur, false);
-            }
+            RecalculateChainMetadata(fulls);

            return fulls;
        }
@@ -752,7 +781,7 @@ namespace Duplicati.Library.Main
        public List<ManifestEntry> GetBackupSets()
        {
            using (new Logging.Timer("Getting and sorting filelist from " + m_backend.DisplayName))
-                return SortAndPairSets(List(true));
+                return SortAndPairSets(List(true), false);
        }

        public void Put(BackupEntryBase remote, string filename)
@@ -967,14 +996,23 @@ namespace Duplicati.Library.Main

                    BackupEntryBase be = m_filenamestrategy.ParseFilename(s);
                    if (m_options.Force)
+                    {
                        this.Delete(be ?? new SignatureEntry(s.Name, s, DateTime.Now, true, "", "", "", 0));
+                        m_metadata.OrphanFileCount--;
+                        if (s.Size > 0)
+                            m_metadata.OrphanFileSize -= s.Size;
+                    }
                }

                sb.AppendLine(string.Format(Strings.BackendWrapper.DeletingTransactionLeftoverFile, m_transactionFile.Filename));

                if (m_options.Force)
                    this.Delete(m_transactionFile);
-                
+
+                m_metadata.OrphanFileCount--;
+                if (m_transactionFile.Filesize > 0)
+                    m_metadata.OrphanFileSize -= m_transactionFile.Filesize;
+
                m_transactionFile = null;

                sb.AppendLine(Strings.BackendWrapper.CompletedDeleteTransaction);
@@ -1026,6 +1064,9 @@ namespace Duplicati.Library.Main
                    if (m_statistics != null)
                        m_statistics.LogWarning(string.Format(Strings.BackendWrapper.RemoveOrphanFileWarning, be.Filename), null);
                    this.Delete(be);
+                    m_metadata.OrphanFileCount--;
+                    if (be.Filesize > 0)
+                        m_metadata.OrphanFileSize -= be.Filesize;
                }

                if (be is ManifestEntry)
@@ -1045,7 +1086,14 @@ namespace Duplicati.Library.Main
                            }

                            this.Delete(bex.Key);
+                            m_metadata.OrphanFileCount--;
+                            if (bex.Key.Filesize > 0)
+                                m_metadata.OrphanFileSize -= bex.Key.Filesize;
+
                            this.Delete(bex.Value);
+                            m_metadata.OrphanFileCount--;
+                            if (bex.Value.Filesize > 0)
+                                m_metadata.OrphanFileSize -= bex.Value.Filesize;
                        }
                    }
            }
@@ -1152,6 +1200,10 @@ namespace Duplicati.Library.Main
            if (lastEx != null)
                throw new Exception(string.Format(Strings.BackendWrapper.FileDeleteError2, remote.Filename, lastEx.Message), lastEx);

+            m_metadata.TotalFileCount--;
+            if (remote.Filesize > 0)
+                m_metadata.TotalSize -= remote.Filesize;
+
            if (remote is SignatureEntry && !string.IsNullOrEmpty(m_options.SignatureCachePath))
            {
                try 
@@ -1514,6 +1566,33 @@ namespace Duplicati.Library.Main
                    throw new Exception(string.Format(Strings.BackendWrapper.FileUploadError2, remotename, lastEx == null ? "<null>" : lastEx.Message), lastEx);

                m_statistics.AddBytesUploaded(sourceFileSize);
+                if (remote is ManifestEntry)
+                {
+                    if (remotename.IndexOf("manifestA") > remotename.IndexOf("manifestB"))
+                    {
+                        if (m_manifestSizeA == 0)
+                            m_metadata.TotalFileCount++;
+
+                        m_metadata.SourceFileSize += (sourceFileSize - m_manifestSizeA);
+                        m_manifestSizeA = sourceFileSize;
+                    }
+                    else
+                    {
+                        if (m_manifestSizeB == 0)
+                            m_metadata.TotalFileCount++;
+
+                        m_metadata.SourceFileSize += (sourceFileSize - m_manifestSizeB);
+                        m_manifestSizeB = sourceFileSize;
+                    }
+
+                    m_metadata.TotalVolumeCount++;
+                }
+                else
+                {
+                    m_metadata.TotalSize += sourceFileSize;
+                    m_metadata.TotalBackupSize += sourceFileSize;
+                    m_metadata.TotalFileCount ++;
+                }
            }
            finally
            {
@@ -1791,5 +1870,53 @@ namespace Duplicati.Library.Main
        }

        #endregion
+
+        public void RecalculateChainMetadata(List<ManifestEntry> flatlist)
+        {
+            List<ManifestEntry> fulls = new List<ManifestEntry>();
+            foreach (ManifestEntry mfe in flatlist)
+                if (mfe.IsFull)
+                    fulls.Add(mfe);
+
+            m_metadata.FullBackupCount = fulls.Count;
+            long maxlen = 0;
+            long volumecount = 0;
+            long fullsize = 0;
+            long sets = 0;
+            foreach (ManifestEntry me in fulls)
+            {
+                maxlen = Math.Max(maxlen, me.Incrementals.Count + 1);
+                volumecount += me.Volumes.Count;
+                sets++;
+                foreach (ManifestEntry mex in me.Incrementals)
+                {
+                    volumecount += mex.Volumes.Count;
+                    sets++;
+                }
+                fullsize += CalculateSizeOfBackupSet(me, true);
+            }
+            m_metadata.LongestChainLength = maxlen;
+            m_metadata.TotalVolumeCount = volumecount;
+            m_metadata.TotalBackupSize = fullsize;
+            m_metadata.TotalBackupSets = sets;
+
+            if (fulls.Count > 0)
+            {
+                ManifestEntry cur = fulls[fulls.Count - 1];
+                m_metadata.CurrentChainLength = cur.Incrementals.Count + 1;
+                m_metadata.CurrentFullDate = cur.Time;
+                m_metadata.CurrentChainSize = CalculateSizeOfBackupSet(cur, true);
+
+                if (cur.Incrementals.Count > 0)
+                    cur = cur.Incrementals[cur.Incrementals.Count - 1];
+
+                m_metadata.LastBackupDate = cur.Time;
+                m_metadata.LastBackupSize = CalculateSizeOfBackupSet(cur, false);
+            }
+            else
+            {
+                m_metadata.RemoveCurrentBackupData();
+            }
+        }
    }
 }
@@ -33,9 +33,11 @@ namespace Duplicati.Library.Main
        private long? m_source_folder_count;

        private long? m_total_backup_size;
+        private long? m_total_backup_sets;

        private long? m_total_quota_space;
        private long? m_free_quota_space;
+        private long? m_assigned_quota_space;

        public long TotalSize { get { return m_total_size ?? 0; } set { m_total_size = value; } }
        public long AlienFileSize { get { return m_alien_file_size ?? 0; } set { m_alien_file_size = value; } }
@@ -55,8 +57,10 @@ namespace Duplicati.Library.Main
        public long SourceFileCount { get { return m_source_file_count ?? 0; } set { m_source_file_count = value; } }
        public long SourceFolderCount { get { return m_source_folder_count ?? 0; } set { m_source_folder_count = value; } }
        public long TotalBackupSize { get { return m_total_backup_size ?? 0; } set { m_total_backup_size = value; } }
+        public long TotalBackupSets { get { return m_total_backup_sets ?? 0; } set { m_total_backup_sets = value; } }
        public long TotalQuotaSpace { get { return m_total_quota_space ?? 0; } set { m_total_quota_space = value; } }
        public long FreeQuotaSpace { get { return m_free_quota_space ?? 0; } set { m_free_quota_space = value; } }
+        public long AssignedQuotaSpace { get { return m_assigned_quota_space ?? 0; } set { m_assigned_quota_space = value; } }

        public IDictionary<string, string> AsReport()
        {
@@ -69,12 +73,24 @@ namespace Duplicati.Library.Main
                    if (v != null)
                    {
                        string reportname = fi.Name.Substring("m_".Length).Replace('_', '-');
-                        result[reportname] = v.ToString();
+                        if (fi.FieldType == typeof(DateTime?))
+                            result[reportname] = ((DateTime?)v).Value.ToUniversalTime().ToString("u");
+                        else
+                            result[reportname] = v.ToString();
                    }
                }
            }

            return result;
        }
+
+        public void RemoveCurrentBackupData()
+        {
+            m_current_chain_length = null;
+            m_current_full_date = null;
+            m_current_chain_size = null;
+            m_last_backup_date = null;
+            m_last_backup_size = null;
+        }
    }
 }
@@ -381,7 +381,12 @@ namespace Duplicati.Library.Main
                    if (full)
                    {
                        //This will create the target folder
-                        backend.List(false);
+                        List<Duplicati.Library.Interface.IFileEntry> tmp = backend.List(false);
+
+                        //If it is possible to parse the chain, we extract some metadata
+                        try { backend.ExtractMetadataFromList(tmp); }
+                        catch { }
+
                        backupsets = new List<ManifestEntry>();
                    }
                    else
@@ -575,7 +580,7 @@ namespace Duplicati.Library.Main

                        OperationProgress(this, DuplicatiOperation.Backup, bs.OperationMode, -1, -1, Strings.Interface.StatusBuildingFilelist, "");

-                        bool completedWithoutChanges;
+                        bool completedWithoutChanges = true;

                        using (RSync.RSyncDir dir = new Duplicati.Library.Main.RSync.RSyncDir(manifest.SourceDirs, bs, backend.Metadata, m_options.Filter, patches))
                        {
@@ -722,6 +727,25 @@ namespace Duplicati.Library.Main
                                    vol++;
                                }
                            }
+
+                            if (!completedWithoutChanges)
+                            {
+                                if (full)
+                                {
+                                    backend.Metadata.CurrentChainLength = 1;
+                                    backend.Metadata.CurrentFullDate = backupchaintime;
+                                    backend.Metadata.FullBackupCount++;
+                                }
+                                else
+                                {
+                                    backend.Metadata.CurrentChainLength++;
+                                }
+                                backend.Metadata.LastBackupDate = backupchaintime;
+                                backend.Metadata.LongestChainLength = Math.Max(backend.Metadata.CurrentChainLength, backend.Metadata.LongestChainLength);
+                                backend.Metadata.TotalVolumeCount += vol;
+                                backend.Metadata.TotalBackupSets++;
+                            }
+
                        }


@@ -1518,6 +1542,7 @@ namespace Duplicati.Library.Main
                        throw new Exception(Strings.Interface.InternalDeleteCountError);

                    sb.Append(RemoveBackupSets(backend, toremove));
+                    backend.RecalculateChainMetadata(flatlist);
                }
                finally
                {
@@ -1569,6 +1594,7 @@ namespace Duplicati.Library.Main
                    throw new Exception(Strings.Interface.InternalDeleteCountError);

                sb.Append(RemoveBackupSets(backend, toremove));
+                backend.RecalculateChainMetadata(entries);
            }
            finally
            {
@@ -1641,6 +1667,7 @@ namespace Duplicati.Library.Main
                    throw new Exception(Strings.Interface.InternalDeleteCountError);

                sb.Append(RemoveBackupSets(backend, toremove));
+                backend.RecalculateChainMetadata(entries);
            }
            finally
            {
@@ -1711,7 +1738,13 @@ namespace Duplicati.Library.Main
                        {
                            backend.Delete(kx.Key);
                            backend.Delete(kx.Value);
+                            backend.Metadata.TotalVolumeCount--;
                        }
+
+                        if (me.IsFull)
+                            backend.Metadata.FullBackupCount--;
+                        
+                        backend.Metadata.TotalBackupSets--;
                    }
                }

@@ -1,7 +1,7 @@
 //------------------------------------------------------------------------------
 // <auto-generated>
 //     This code was generated by a tool.
-//     Runtime Version:4.0.30319.261
+//     Runtime Version:4.0.30319.269
 //
 //     Changes to this file may cause incorrect behavior and will be lost if
 //     the code is regenerated.
@@ -386,6 +386,24 @@ namespace Duplicati.Library.Main.Strings {
            }
        }
        
+        /// <summary>
+        ///   Looks up a localized string similar to The file &quot;{0}&quot; has compression module &quot;{1}&quot; which is not supported, ignoring file.
+        /// </summary>
+        internal static string UnmatchedCompressionModuleWarning {
+            get {
+                return ResourceManager.GetString("UnmatchedCompressionModuleWarning", resourceCulture);
+            }
+        }
+        
+        /// <summary>
+        ///   Looks up a localized string similar to The file &quot;{0}&quot; has encryption module &quot;{1}&quot; which is not supported, ignoring file.
+        /// </summary>
+        internal static string UnmatchedEncryptionModuleWarning {
+            get {
+                return ResourceManager.GetString("UnmatchedEncryptionModuleWarning", resourceCulture);
+            }
+        }
+        
        /// <summary>
        ///   Looks up a localized string similar to The file {0} was found on the backend but not recognized by Duplicati.
        /// </summary>
@@ -112,10 +112,10 @@
    <value>2.0</value>
  </resheader>
  <resheader name="reader">
-    <value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+    <value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
  </resheader>
  <resheader name="writer">
-    <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=2.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+    <value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
  </resheader>
  <data name="BackendNotFoundError" xml:space="preserve">
    <value>Unable to find backend for: {0}</value>
@@ -236,4 +236,10 @@ If this problem happens more than once, it is possible that the delete transacti
  <data name="AutoCreateFolderFailed" xml:space="preserve">
    <value>Failed to automatically create folder, error message: {0}</value>
  </data>
+  <data name="UnmatchedCompressionModuleWarning" xml:space="preserve">
+    <value>The file "{0}" has compression module "{1}" which is not supported, ignoring file</value>
+  </data>
+  <data name="UnmatchedEncryptionModuleWarning" xml:space="preserve">
+    <value>The file "{0}" has encryption module "{1}" which is not supported, ignoring file</value>
+  </data>
 </root>
@@ -1,7 +1,7 @@
 //------------------------------------------------------------------------------
 // <auto-generated>
 //     This code was generated by a tool.
-//     Runtime Version:4.0.30319.239
+//     Runtime Version:4.0.30319.269
 //
 //     Changes to this file may cause incorrect behavior and will be lost if
 //     the code is regenerated.