From 0e6bb625205231d2802db5636ec352bdb257bc77 Mon Sep 17 00:00:00 2001 From: JonMikelV Date: Wed, 3 Jan 2018 18:18:53 -0600 Subject: [PATCH 1/5] Let AddVerboseMessage() handle null / empty args variable & update PurgeFilesHandler to correctly call AddMessage() --- .../Library/Main/Operation/PurgeFilesHandler.cs | 4 ++-- Duplicati/Library/Main/ResultClasses.cs | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index fc91ddb64..416c09a62 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -173,13 +173,13 @@ namespace Duplicati.Library.Main.Operation if (m_options.Dryrun) m_result.AddDryrunMessage(msg); else if (m_options.Verbose) - m_result.AddVerboseMessage(msg); + m_result.AddMessage(msg); } if (m_options.Dryrun) m_result.AddDryrunMessage("Writing files to remote storage"); else if (m_options.Verbose) - m_result.AddVerboseMessage("Writing files to remote storage"); + m_result.AddMessage("Writing files to remote storage"); } if (m_options.Dryrun) diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 97d7eed70..79fadc9d7 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -420,10 +420,13 @@ namespace Duplicati.Library.Main public void AddVerboseMessage(string message, params object[] args) { if (m_parent != null) - m_parent.AddVerboseMessage(message, args); + { + if ((args != null) && (args.Length > 0)) m_parent.AddVerboseMessage(message, args); + else m_parent.AddMessage(message); + } else { - lock(Logging.Log.Lock) + lock (Logging.Log.Lock) { if (m_is_reporting) return; @@ -431,10 +434,13 @@ namespace Duplicati.Library.Main try { m_is_reporting = true; - Logging.Log.WriteMessage(string.Format(message, args), Duplicati.Library.Logging.LogMessageType.Profiling, null); + Logging.Log.WriteMessage((((args != null) || (args.Length > 0)) ? string.Format(message, args) : message), Duplicati.Library.Logging.LogMessageType.Profiling, null); if (MessageSink != null) - MessageSink.VerboseEvent(message, args); + { + if ((args != null) && (args.Length > 0)) MessageSink.VerboseEvent(message, args); + else MessageSink.MessageEvent(message); + } } finally { From 0d2145fc55a85f9bad1566539538ffe8a7d1c8c7 Mon Sep 17 00:00:00 2001 From: JonMikelV Date: Thu, 4 Jan 2018 22:49:43 -0600 Subject: [PATCH 2/5] Fix copy/paste typo with && vs || --- Duplicati/Library/Main/ResultClasses.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index 79fadc9d7..290c98035 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -434,7 +434,7 @@ namespace Duplicati.Library.Main try { m_is_reporting = true; - Logging.Log.WriteMessage((((args != null) || (args.Length > 0)) ? string.Format(message, args) : message), Duplicati.Library.Logging.LogMessageType.Profiling, null); + Logging.Log.WriteMessage((((args != null) && (args.Length > 0)) ? string.Format(message, args) : message), Duplicati.Library.Logging.LogMessageType.Profiling, null); if (MessageSink != null) { From 95a768feaae8f9fb5df30d231555a886db5b3b50 Mon Sep 17 00:00:00 2001 From: JonMikelV Date: Thu, 4 Jan 2018 23:05:01 -0600 Subject: [PATCH 3/5] FormatSizeString now will also format negative values such as produced when reporting on space savings due to compacting. --- Duplicati/Library/Utility/Utility.cs | 101 ++++++++++++++------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index a1f42e45e..d31dacdf2 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -190,7 +190,7 @@ namespace Duplicati.Library.Utility { IFilter match; filter = filter ?? new FilterExpression(); - return EnumerateFileSystemEntries(basepath, (rootpath, path, attributes) => { + return EnumerateFileSystemEntries(basepath, (rootpath, path, attributes) => { bool result; if (!filter.Matches(path, out result, out match)) result = true; @@ -261,7 +261,7 @@ namespace Duplicati.Library.Utility public static IEnumerable EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader, ReportAccessError errorCallback = null) { Stack lst = new Stack(); - + var isFolder = false; try { @@ -271,15 +271,15 @@ namespace Duplicati.Library.Utility isFolder = (attributeReader(rootpath) & System.IO.FileAttributes.Directory) == System.IO.FileAttributes.Directory; } catch - { + { } - + if (isFolder) { rootpath = AppendDirSeparator(rootpath); try { - + System.IO.FileAttributes attr = attributeReader == null ? System.IO.FileAttributes.Directory : attributeReader(rootpath); if (callback(rootpath, rootpath, attr)) lst.Push(rootpath); @@ -298,12 +298,12 @@ namespace Duplicati.Library.Utility while (lst.Count > 0) { string f = AppendDirSeparator(lst.Pop()); - + yield return f; - + try { - foreach(string s in folderList(f)) + foreach (string s in folderList(f)) { var sf = AppendDirSeparator(s); try @@ -351,9 +351,9 @@ namespace Duplicati.Library.Utility errorCallback(rootpath, f, ex); callback(rootpath, f, System.IO.FileAttributes.Directory | ATTRIBUTE_ERROR); } - + if (files != null) - foreach(var s in files) + foreach (var s in files) { try { @@ -395,7 +395,7 @@ namespace Duplicati.Library.Utility callback(rootpath, rootpath, ATTRIBUTE_ERROR); yield break; } - + yield return rootpath; } } @@ -508,13 +508,13 @@ namespace Duplicati.Library.Utility while ((a1 = ForceStreamRead(stream1, buf1, buf1.Length)) == (a2 = ForceStreamRead(stream2, buf2, buf2.Length))) { int ix = 0; - for(int i = 0; i < a1 / longSize; i++) + for (int i = 0; i < a1 / longSize; i++) if (BitConverter.ToUInt64(buf1, ix) != BitConverter.ToUInt64(buf2, ix)) return false; else ix += longSize; - for(int i = 0; i < a1 % longSize; i++) + for (int i = 0; i < a1 % longSize; i++) if (buf1[ix] != buf2[ix]) return false; else @@ -534,7 +534,7 @@ namespace Duplicati.Library.Utility /// The base64 encoded hash public static string CalculateHash(string path) { - using(System.IO.FileStream fs = System.IO.File.Open(path, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read)) + using (System.IO.FileStream fs = System.IO.File.Open(path, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read)) return CalculateHash(fs); } @@ -559,14 +559,14 @@ namespace Duplicati.Library.Utility // we need to detect the encoding (at least that it's not UTF8). // So we read the first 4096 bytes and try to decode them as UTF8. byte[] buffer = new byte[4096]; - using(System.IO.FileStream file = new System.IO.FileStream(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read)) + using (System.IO.FileStream file = new System.IO.FileStream(filename, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read)) file.Read(buffer, 0, 4096); Encoding enc = Encoding.UTF8; try { // this will throw an error if not really UTF8 - new UTF8Encoding(false, true).GetString(buffer); + new UTF8Encoding(false, true).GetString(buffer); } catch (Exception) { @@ -574,7 +574,7 @@ namespace Duplicati.Library.Utility } // This will load the text using the BOM, or the detected encoding if no BOM. - using(System.IO.StreamReader reader = new System.IO.StreamReader(filename, enc, true)) + using (System.IO.StreamReader reader = new System.IO.StreamReader(filename, enc, true)) { // Remove all \r from the file and split on \n, then pass directly to ExtractOptions return reader.ReadToEnd(); @@ -582,19 +582,20 @@ namespace Duplicati.Library.Utility } /// - /// Formats a size into a human readable format, eg. 2048 becomes "2 KB". + /// Formats a size into a human readable format, eg. 2048 becomes "2 KB" or -2283 becomes &qout;-2.23 KB%quot. /// /// The size to format /// A human readable string representing the size public static string FormatSizeString(long size) { - if (size >= 1024 * 1024 * 1024 * 1024L) + long sizeAbs = Math.Abs(size); // Allow formatting of negative sizes + if (sizeAbs >= 1024 * 1024 * 1024 * 1024L) return Strings.Utility.FormatStringTB((double)size / (1024 * 1024 * 1024 * 1024L)); - else if (size >= 1024 * 1024 * 1024) + else if (sizeAbs >= 1024 * 1024 * 1024) return Strings.Utility.FormatStringGB((double)size / (1024 * 1024 * 1024)); - else if (size >= 1024 * 1024) + else if (sizeAbs >= 1024 * 1024) return Strings.Utility.FormatStringMB((double)size / (1024 * 1024)); - else if (size >= 1024) + else if (sizeAbs >= 1024) return Strings.Utility.FormatStringKB((double)size / 1024); else return Strings.Utility.FormatStringB(size); @@ -712,18 +713,18 @@ namespace Duplicati.Library.Utility return data; } - + public static bool Which(string appname) { if (!IsClientLinux) return false; - + try { var psi = new System.Diagnostics.ProcessStartInfo("which", appname); psi.RedirectStandardOutput = true; psi.UseShellExecute = false; - + var pi = System.Diagnostics.Process.Start(psi); pi.WaitForExit(5000); if (pi.HasExited) @@ -734,7 +735,7 @@ namespace Duplicati.Library.Utility catch { } - + return false; } @@ -752,7 +753,7 @@ namespace Duplicati.Library.Utility if (!IsClientLinux) return false; - + try { if (UNAME == null) @@ -760,7 +761,7 @@ namespace Duplicati.Library.Utility var psi = new System.Diagnostics.ProcessStartInfo("uname"); psi.RedirectStandardOutput = true; psi.UseShellExecute = false; - + var pi = System.Diagnostics.Process.Start(psi); pi.WaitForExit(5000); if (pi.HasExited) @@ -770,9 +771,9 @@ namespace Duplicati.Library.Utility catch { } - + return "Darwin".Equals(UNAME); - + } } /// @@ -784,13 +785,13 @@ namespace Duplicati.Library.Utility { if (!IsClientLinux) return null; - + try { var psi = new System.Diagnostics.ProcessStartInfo("uname", "-a"); psi.RedirectStandardOutput = true; psi.UseShellExecute = false; - + var pi = System.Diagnostics.Process.Start(psi); pi.WaitForExit(5000); if (pi.HasExited) @@ -799,8 +800,8 @@ namespace Duplicati.Library.Utility catch { } - - return null; + + return null; } } /// @@ -869,7 +870,7 @@ namespace Duplicati.Library.Utility var match = regex.Match(v); if (match.Success) return new Version(match.Value); - } + } } catch { @@ -878,13 +879,13 @@ namespace Duplicati.Library.Utility return new Version(); } } - + /// /// Gets the Mono display version, or null if not running Mono /// public static string MonoDisplayVersion { - get + get { try { @@ -900,7 +901,7 @@ namespace Duplicati.Library.Utility { } - return null; + return null; } } @@ -953,11 +954,11 @@ namespace Duplicati.Library.Utility string homedir = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + System.IO.Path.PathSeparator.ToString(); //Look in application base folder and all system path folders - foreach(string s in (homedir + Environment.GetEnvironmentVariable("PATH")).Split(System.IO.Path.PathSeparator)) + foreach (string s in (homedir + Environment.GetEnvironmentVariable("PATH")).Split(System.IO.Path.PathSeparator)) if (!string.IsNullOrEmpty(s) && s.Trim().Length > 0) try { - foreach(string sx in System.IO.Directory.GetFiles(ExpandEnvironmentVariables(s), filename)) + foreach (string sx in System.IO.Directory.GetFiles(ExpandEnvironmentVariables(s), filename)) return sx; } catch @@ -1264,7 +1265,7 @@ namespace Duplicati.Library.Utility /// A lookup table with visited objects, used to avoid inifinite recursion /// The maximum number of items to report from an IEnumerable instance public static void PrintSerializeObject(object item, System.IO.TextWriter writer, Func filter = null, bool recurseobjects = false, int indentation = 0, int collectionlimit = 0, Dictionary visited = null) - { + { visited = visited ?? new Dictionary(); var indentstring = new string(' ', indentation); @@ -1382,7 +1383,7 @@ namespace Duplicati.Library.Utility public static StringBuilder PrintSerializeObject(object item, StringBuilder sb = null, Func filter = null, bool recurseobjects = false, int indentation = 0, int collectionlimit = 10) { sb = sb ?? new StringBuilder(); - using(var sw = new System.IO.StringWriter(sb)) + using (var sw = new System.IO.StringWriter(sb)) PrintSerializeObject(item, sw, filter, recurseobjects, indentation, collectionlimit); return sb; } @@ -1403,7 +1404,7 @@ namespace Duplicati.Library.Utility System.Text.Encoding.UTF8.GetBytes(salt ?? ""), repeats); } - + /// /// Repeatedly hash a value with a salt. /// This effectively masks the original value, @@ -1416,21 +1417,21 @@ namespace Duplicati.Library.Utility { // We avoid storing the passphrase directly, // instead we salt and rehash repeatedly - using(var h = System.Security.Cryptography.SHA256.Create()) + using (var h = System.Security.Cryptography.SHA256.Create()) { h.Initialize(); h.TransformBlock(salt, 0, salt.Length, salt, 0); h.TransformFinalBlock(data, 0, data.Length); var buf = h.Hash; - - for(var i = 0; i < repeats; i++) + + for (var i = 0; i < repeats; i++) { h.Initialize(); h.TransformBlock(salt, 0, salt.Length, salt, 0); h.TransformFinalBlock(buf, 0, buf.Length); buf = h.Hash; } - + return buf; } } @@ -1522,7 +1523,7 @@ namespace Duplicati.Library.Utility // We could consider using single quotes that prevents all expansions //if (!allowEnvExpansion) // return "'" + arg.Replace("'", "\\'") + "'"; - + // Linux is using backslash to escape, except for ! arg = COMMANDLINE_ESCAPED_LINUX.Replace(arg, (match) => { @@ -1531,7 +1532,7 @@ namespace Duplicati.Library.Utility if (match.Value == "$" && allowEnvExpansion) return match.Value; - + return "\\" + match.Value; }); } @@ -1558,7 +1559,7 @@ namespace Duplicati.Library.Utility if (COMMANDLINE_SAFE.Match(arg).Length != arg.Length) return "\"" + arg + "\""; else - return arg; + return arg; } /// From 2c43f389452f30d19e252942d11fcb5a21c11ab3 Mon Sep 17 00:00:00 2001 From: JonMikelV Date: Sun, 21 Jan 2018 02:11:27 -0600 Subject: [PATCH 4/5] Undo unnecessary AddVerboseMessage() change --- Duplicati/Library/Main/Operation/PurgeFilesHandler.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs index 416c09a62..6d56073af 100644 --- a/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs +++ b/Duplicati/Library/Main/Operation/PurgeFilesHandler.cs @@ -40,7 +40,7 @@ namespace Duplicati.Library.Main.Operation if (!System.IO.File.Exists(m_options.Dbpath)) throw new UserInformationException(string.Format("Database file does not exist: {0}", m_options.Dbpath)); - + using (var db = new Database.LocalPurgeDatabase(m_options.Dbpath)) DoRun(db, filter, null, 0, 1); } @@ -173,13 +173,13 @@ namespace Duplicati.Library.Main.Operation if (m_options.Dryrun) m_result.AddDryrunMessage(msg); else if (m_options.Verbose) - m_result.AddMessage(msg); + m_result.AddVerboseMessage(msg); } if (m_options.Dryrun) m_result.AddDryrunMessage("Writing files to remote storage"); else if (m_options.Verbose) - m_result.AddMessage("Writing files to remote storage"); + m_result.AddVerboseMessage("Writing files to remote storage"); } if (m_options.Dryrun) From 4f0cf589ba0d82ba366850b7c0d93c26e37016f2 Mon Sep 17 00:00:00 2001 From: JonMikelV Date: Sun, 21 Jan 2018 02:46:17 -0600 Subject: [PATCH 5/5] Update duplicate error message to better distinguish failure point when error is thrown --- Duplicati/Library/Main/Database/LocalDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 6ffe4bd35..6bd6fe44d 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -185,7 +185,7 @@ namespace Duplicati.Library.Main.Database if (deleteGraceTime.Ticks > 0) using(var cmd = m_connection.CreateCommand(transaction)) if ((c = cmd.ExecuteNonQuery(@"UPDATE ""RemoteVolume"" SET ""DeleteGraceTime"" = ? WHERE ""Name"" = ? ", (DateTime.UtcNow + deleteGraceTime).Ticks, name)) != 1) - throw new Exception(string.Format("Unexpected number of remote volumes detected: {0}!", c)); + throw new Exception(string.Format("Unexpected number of updates when recording remote volume updates: {0}!", c)); if (!suppressCleanup && state == RemoteVolumeState.Deleted)