diff --git a/Duplicati.Library.Modules.Builtin.UnitTest/Duplicati.Library.Modules.Builtin.UnitTest.csproj b/Duplicati.Library.Modules.Builtin.UnitTest/Duplicati.Library.Modules.Builtin.UnitTest.csproj new file mode 100644 index 000000000..2dca5a99d --- /dev/null +++ b/Duplicati.Library.Modules.Builtin.UnitTest/Duplicati.Library.Modules.Builtin.UnitTest.csproj @@ -0,0 +1,60 @@ + + + + Debug + AnyCPU + {3E5B9D4B-962A-4D80-8922-F2150BB48343} + Library + Properties + Duplicati.Library.Modules.Builtin.UnitTest + Duplicati.Library.Modules.Builtin.UnitTest + v4.6.1 + 512 + 15.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages + False + UnitTest + + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\packages\NUnit.3.5.0\lib\net45\nunit.framework.dll + + + + + + + + + + + + + + {52826615-7964-47fe-b4b3-1b2dbdf605b9} + Duplicati.Library.Modules.Builtin + + + + + \ No newline at end of file diff --git a/Duplicati.Library.Modules.Builtin.UnitTest/Properties/AssemblyInfo.cs b/Duplicati.Library.Modules.Builtin.UnitTest/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..83f857c9d --- /dev/null +++ b/Duplicati.Library.Modules.Builtin.UnitTest/Properties/AssemblyInfo.cs @@ -0,0 +1,20 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +[assembly: AssemblyTitle("Duplicati.Library.Modules.Builtin.UnitTest")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Duplicati.Library.Modules.Builtin.UnitTest")] +[assembly: AssemblyCopyright("Copyright © 2018")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +[assembly: ComVisible(false)] + +[assembly: Guid("3e5b9d4b-962a-4d80-8922-f2150bb48343")] + +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/Duplicati.Library.Modules.Builtin.UnitTest/ResultSerialization/ResultFormatSerializerProviderTest.cs b/Duplicati.Library.Modules.Builtin.UnitTest/ResultSerialization/ResultFormatSerializerProviderTest.cs new file mode 100644 index 000000000..fcfbb75fe --- /dev/null +++ b/Duplicati.Library.Modules.Builtin.UnitTest/ResultSerialization/ResultFormatSerializerProviderTest.cs @@ -0,0 +1,23 @@ +using Duplicati.Library.Modules.Builtin.ResultSerialization; +using NUnit.Framework; + +namespace Duplicati.Library.Modules.Builtin.UnitTest +{ + [TestFixture] + public class ResultFormatSerializerProviderTest + { + [Test] + public void TestGetSerializerGivenDuplicatiReturnsDuplicatiSerializer() + { + IResultFormatSerializer serializer = ResultFormatSerializerProvider.GetSerializer(ResultExportFormat.Duplicati); + Assert.AreEqual(typeof(DuplicatiFormatSerializer), serializer.GetType()); + } + + [Test] + public void TestGetSerializerGivenJsonReturnsJsonSerializer() + { + IResultFormatSerializer serializer = ResultFormatSerializerProvider.GetSerializer(ResultExportFormat.Json); + Assert.AreEqual(typeof(JsonFormatSerializer), serializer.GetType()); + } + } +} diff --git a/Duplicati.Library.Modules.Builtin.UnitTest/packages.config b/Duplicati.Library.Modules.Builtin.UnitTest/packages.config new file mode 100644 index 000000000..77de612bc --- /dev/null +++ b/Duplicati.Library.Modules.Builtin.UnitTest/packages.config @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/Duplicati.sln b/Duplicati.sln index 99721e4a6..2fdf32fcf 100644 --- a/Duplicati.sln +++ b/Duplicati.sln @@ -99,6 +99,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.S EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Backend.Rclone", "Duplicati\Library\Backend\Rclone\Duplicati.Library.Backend.Rclone.csproj", "{851A1CB8-3CEB-41B4-956F-34D760D2A8E5}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Duplicati.Library.Modules.Builtin.UnitTest", "Duplicati.Library.Modules.Builtin.UnitTest\Duplicati.Library.Modules.Builtin.UnitTest.csproj", "{3E5B9D4B-962A-4D80-8922-F2150BB48343}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -297,6 +299,10 @@ Global {851A1CB8-3CEB-41B4-956F-34D760D2A8E5}.Debug|Any CPU.Build.0 = Debug|Any CPU {851A1CB8-3CEB-41B4-956F-34D760D2A8E5}.Release|Any CPU.ActiveCfg = Release|Any CPU {851A1CB8-3CEB-41B4-956F-34D760D2A8E5}.Release|Any CPU.Build.0 = Release|Any CPU + {3E5B9D4B-962A-4D80-8922-F2150BB48343}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3E5B9D4B-962A-4D80-8922-F2150BB48343}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3E5B9D4B-962A-4D80-8922-F2150BB48343}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3E5B9D4B-962A-4D80-8922-F2150BB48343}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/Duplicati/Library/Backend/OneDrive/Exceptions.cs b/Duplicati/Library/Backend/OneDrive/Exceptions.cs index fa8e154ea..d31562da2 100644 --- a/Duplicati/Library/Backend/OneDrive/Exceptions.cs +++ b/Duplicati/Library/Backend/OneDrive/Exceptions.cs @@ -11,7 +11,7 @@ namespace Duplicati.Library.Backend.MicrosoftGraph { public class MicrosoftGraphException : Exception { - private static readonly Regex authorizationHeaderRemover = new Regex(@"Authorization:\s*Bearer\s+[A-Za-z0-9+/=]+", RegexOptions.IgnoreCase); + private static readonly Regex authorizationHeaderRemover = new Regex(@"Authorization:\s*Bearer\s+\S+", RegexOptions.IgnoreCase); public MicrosoftGraphException(HttpResponseMessage response) : this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage.RequestUri), response) diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs index 7d94cba02..e8915a629 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs @@ -42,31 +42,31 @@ namespace Duplicati.Library.Backend.OpenStack private const int PAGE_LIMIT = 500; - private string m_container; - private string m_prefix; + private readonly string m_container; + private readonly string m_prefix; private readonly string m_domainName; - private string m_username; - private string m_password; - private string m_authUri; + private readonly string m_username; + private readonly string m_password; + private readonly string m_authUri; private readonly string m_version; - private string m_tenantName; - private string m_apikey; - private string m_region; + private readonly string m_tenantName; + private readonly string m_apikey; + private readonly string m_region; protected string m_simplestorageendpoint; - private WebHelper m_helper = null; + private readonly WebHelper m_helper; private OpenStackAuthResponse.TokenClass m_accessToken; - public static readonly KeyValuePair[] KNOWN_OPENSTACK_PROVIDERS = new KeyValuePair[] { + public static readonly KeyValuePair[] KNOWN_OPENSTACK_PROVIDERS = { new KeyValuePair("Rackspace US", "https://identity.api.rackspacecloud.com/v2.0"), new KeyValuePair("Rackspace UK", "https://lon.identity.api.rackspacecloud.com/v2.0"), new KeyValuePair("OVH Cloud Storage", "https://auth.cloud.ovh.net/v2.0"), new KeyValuePair("Selectel Cloud Storage", "https://auth.selcdn.ru"), }; - public static readonly KeyValuePair[] OPENSTACK_VERSIONS = new KeyValuePair[] { + public static readonly KeyValuePair[] OPENSTACK_VERSIONS = { new KeyValuePair("v2.0", "v2"), new KeyValuePair("v3", "v3"), }; @@ -209,7 +209,7 @@ namespace Duplicati.Library.Backend.OpenStack if (string.IsNullOrEmpty(apikey)) { - this.auth.PasswordCredentials = new PasswordBasedRequest() + this.auth.PasswordCredentials = new PasswordBasedRequest { username = username, password = password, @@ -217,7 +217,7 @@ namespace Duplicati.Library.Backend.OpenStack } else { - this.auth.ApiCredentials = new ApiKeyBasedRequest() + this.auth.ApiCredentials = new ApiKeyBasedRequest { username = username, apiKey = apikey @@ -297,7 +297,7 @@ namespace Duplicati.Library.Backend.OpenStack private class WebHelper : JSONWebHelper { - private OpenStackStorage m_parent; + private readonly OpenStackStorage m_parent; public WebHelper(OpenStackStorage parent) { m_parent = parent; } @@ -376,12 +376,12 @@ namespace Duplicati.Library.Backend.OpenStack } } - private string JoinUrls(string uri, string fragment) + private static string JoinUrls(string uri, string fragment) { fragment = fragment ?? ""; return uri + (uri.EndsWith("/", StringComparison.Ordinal) ? "" : "/") + (fragment.StartsWith("/", StringComparison.Ordinal) ? fragment.Substring(1) : fragment); } - private string JoinUrls(string uri, string fragment1, string fragment2) + private static string JoinUrls(string uri, string fragment1, string fragment2) { return JoinUrls(JoinUrls(uri, fragment1), fragment2); } @@ -610,7 +610,7 @@ namespace Duplicati.Library.Backend.OpenStack foreach(var s in KNOWN_OPENSTACK_PROVIDERS) authuris.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); - return new List(new CommandLineArgument[] { + return new List(new [] { new CommandLineArgument(DOMAINNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.DomainnameOptionShort, Strings.OpenStack.UsernameOptionLong), new CommandLineArgument(USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong), new CommandLineArgument(PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)), diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index ef83945a0..777260d40 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -559,7 +559,6 @@ namespace Duplicati.Library.Main using (new ProcessController(m_options)) using (new Logging.Timer(LOGTAG, string.Format("Run{0}", result.MainOperation), string.Format("Running {0}", result.MainOperation))) - using (new Logging.RepeatingLogScope()) method(result); if (result.EndTime.Ticks == 0) diff --git a/Duplicati/Library/Main/ProcessController.cs b/Duplicati/Library/Main/ProcessController.cs index e56850e3c..8a75c11c9 100755 --- a/Duplicati/Library/Main/ProcessController.cs +++ b/Duplicati/Library/Main/ProcessController.cs @@ -180,15 +180,25 @@ namespace Duplicati.Library.Main if (Duplicati.Library.Utility.Utility.IsClientOSX) { var data = RunProcessAndGetResult("ps", $"-onice -p {pid}"); - m_originalNiceLevel = int.Parse(data.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries).Last()); + if (data.Item1 != 0) + { + Logging.Log.WriteWarningMessage(LOGTAG, "BackgroundPriorityError", null, "Failed to get background IO priority, exitcode: {0}, stderr: {1}", data.Item1, data.Item3); + } + else + { + m_originalNiceLevel = int.Parse(data.Item2.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries).Last()); - data = RunProcessAndGetResult("renice", $"20 {pid}"); - m_hasEnabledBackgroundIOPriority = true; + data = RunProcessAndGetResult("renice", $"20 -p {pid}"); + if (data.Item1 != 0) + Logging.Log.WriteWarningMessage(LOGTAG, "BackgroundPriorityError", null, "Failed to get background IO priority, exitcode: {0}, stderr: {1}", data.Item1, data.Item3); + else + m_hasEnabledBackgroundIOPriority = true; + } } else { var data = RunProcessAndGetResult("ionice", $"-p {pid}"); - var results = data.Split(new char[] { ':', ' ', '\t', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + var results = data.Item2.Split(new char[] { ':', ' ', '\t', '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); var ioclass = results[0]; if (string.Equals(ioclass, "idle", StringComparison.OrdinalIgnoreCase)) { @@ -326,18 +336,28 @@ namespace Duplicati.Library.Main { m_hasEnabledBackgroundIOPriority = false; var pid = System.Diagnostics.Process.GetCurrentProcess().Id; + Tuple data; + if (Duplicati.Library.Utility.Utility.IsClientOSX) { - var data = RunProcessAndGetResult($"renice", $"{m_originalNiceLevel} {pid}"); + // TODO: We can only give lower priority, thus not reset it ... + data = RunProcessAndGetResult($"renice", $"{m_originalNiceLevel} -p {pid}"); + if (data.Item1 != 0) + Logging.Log.WriteWarningMessage(LOGTAG, "BackgroundPriorityError", null, "Failed to reset background IO priority, exitcode: {0}, stderr: {1}", data.Item1, data.Item3); } else { - string data; if (m_originalNiceLevel < 0) data = RunProcessAndGetResult($"ionice", $"-c {m_originalNiceClass} -p {pid}"); else data = RunProcessAndGetResult($"ionice", $"-c {m_originalNiceClass} -n {m_originalNiceLevel} -p {pid}"); + + if (!string.IsNullOrWhiteSpace(data.Item3)) + Logging.Log.WriteWarningMessage(LOGTAG, "BackgroundPriorityError", null, "Failed to reset background IO priority, exitcode: {0}, stderr: {1}", data.Item1, data.Item3); + } + + } } } @@ -358,10 +378,11 @@ namespace Duplicati.Library.Main /// The stdout data. /// The executable to invoke. /// The commandline arguments. - private static string RunProcessAndGetResult(string filename, string arguments) + private static Tuple RunProcessAndGetResult(string filename, string arguments) { var psi = new System.Diagnostics.ProcessStartInfo(filename, arguments); psi.RedirectStandardOutput = true; + psi.RedirectStandardError = true; psi.UseShellExecute = false; Logging.Log.WriteExplicitMessage(LOGTAG, "RunningCommand", null, "Running: {0} {1}", filename, arguments); @@ -369,7 +390,14 @@ namespace Duplicati.Library.Main var pi = System.Diagnostics.Process.Start(psi); pi.WaitForExit(5000); if (pi.HasExited) - return pi.StandardOutput.ReadToEnd().Trim(); + { + return + new Tuple( + pi.ExitCode, + pi.StandardOutput.ReadToEnd().Trim(), + pi.StandardError.ReadToEnd().Trim() + ); + } pi.Kill(); throw new Library.Interface.UserInformationException($"The process {filename} with arguments {arguments} failed to stop", "LaunchProcessFailed"); diff --git a/Duplicati/Library/Main/ResultClasses.cs b/Duplicati/Library/Main/ResultClasses.cs index b08da90f2..e7b004a4d 100644 --- a/Duplicati/Library/Main/ResultClasses.cs +++ b/Duplicati/Library/Main/ResultClasses.cs @@ -21,6 +21,7 @@ using System.Collections.Generic; using Duplicati.Library.Main.Database; using Duplicati.Library.Logging; using System.Linq; +using Newtonsoft.Json; namespace Duplicati.Library.Main { @@ -228,6 +229,8 @@ namespace Duplicati.Library.Main protected Library.Utility.FileBackedStringList m_retryAttempts; protected IMessageSink m_messageSink; + + [JsonIgnore] public IMessageSink MessageSink { get { return m_messageSink; } @@ -385,6 +388,7 @@ namespace Duplicati.Library.Main } } + [JsonIgnore] public IBackendWriter BackendWriter { get { return (IBackendWriter)this.BackendStatistics; } } public event Action StateChangedEvent; diff --git a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj index 3e36679c7..8cc2019b5 100644 --- a/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj +++ b/Duplicati/Library/Modules/Builtin/Duplicati.Library.Modules.Builtin.csproj @@ -67,6 +67,7 @@ + @@ -77,6 +78,10 @@ DnsLite.cs + + + + @@ -129,4 +134,7 @@ PreserveNewest + + + \ No newline at end of file diff --git a/Duplicati/Library/Modules/Builtin/Properties/AssemblyInfo.cs b/Duplicati/Library/Modules/Builtin/Properties/AssemblyInfo.cs index 399ba857a..03f8cb13a 100644 --- a/Duplicati/Library/Modules/Builtin/Properties/AssemblyInfo.cs +++ b/Duplicati/Library/Modules/Builtin/Properties/AssemblyInfo.cs @@ -53,3 +53,4 @@ using System.Runtime.InteropServices; // [assembly: AssemblyVersion("2.0.0.7")] [assembly: AssemblyVersion("2.0.0.7")] [assembly: AssemblyFileVersion("2.0.0.7")] +[assembly: InternalsVisibleTo("Duplicati.Library.Modules.Builtin.UnitTest")] diff --git a/Duplicati/Library/Modules/Builtin/ReportHelper.cs b/Duplicati/Library/Modules/Builtin/ReportHelper.cs index 9bdc302c3..b83c81d3f 100644 --- a/Duplicati/Library/Modules/Builtin/ReportHelper.cs +++ b/Duplicati/Library/Modules/Builtin/ReportHelper.cs @@ -20,6 +20,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; using Duplicati.Library.Interface; +using Duplicati.Library.Modules.Builtin.ResultSerialization; using Duplicati.Library.Utility; namespace Duplicati.Library.Modules.Builtin @@ -68,6 +69,11 @@ namespace Duplicati.Library.Modules.Builtin /// Name of the option used to specify the maximum number of log lines to include /// protected abstract string LogLinesOptionName { get; } + + /// + /// Name of the option used to the output format + /// + protected abstract string ResultFormatOptionName { get; } /// /// The default subject or title line @@ -92,6 +98,11 @@ namespace Duplicati.Library.Modules.Builtin /// protected virtual Logging.LogMessageType DEFAULT_LOG_LEVEL { get; } = Logging.LogMessageType.Warning; + /// + /// The default export format + /// + protected virtual ResultExportFormat DEFAULT_EXPORT_FORMAT { get; } = ResultExportFormat.Duplicati; + /// /// The module key /// @@ -111,7 +122,12 @@ namespace Duplicati.Library.Modules.Builtin /// /// The list of supported commands /// - public abstract IList SupportedCommands { get; } + public abstract IList SupportedCommands { get; } + + /// + /// Returns the format used by the serializer + /// + protected ResultExportFormat ExportFormat => m_resultFormatSerializer.Format; /// /// The cached name of the operation @@ -170,6 +186,10 @@ namespace Duplicati.Library.Modules.Builtin /// The log storage /// private Utility.FileBackedStringList m_logstorage; + /// + /// Serializer to use when serializing the message. + /// + private IResultFormatSerializer m_resultFormatSerializer; /// /// Configures the module @@ -220,6 +240,14 @@ namespace Duplicati.Library.Modules.Builtin m_sendAll = Utility.Utility.ParseBoolOption(commandlineOptions, ActionOnAnyOperationOptionName); + ResultExportFormat resultFormat; + if (!commandlineOptions.TryGetValue(ResultFormatOptionName, out var tmpResultFormat)) + resultFormat = DEFAULT_EXPORT_FORMAT; + else if (!Enum.TryParse(tmpResultFormat, true, out resultFormat)) + resultFormat = DEFAULT_EXPORT_FORMAT; + + m_resultFormatSerializer = ResultFormatSerializerProvider.GetSerializer(resultFormat); + commandlineOptions.TryGetValue(LogLinesOptionName, out var loglinestr); if (!int.TryParse(loglinestr, out m_maxmimumLogLines)) m_maxmimumLogLines = DEFAULT_LOGLINES; @@ -277,12 +305,8 @@ namespace Duplicati.Library.Modules.Builtin } else { - if (input.IndexOf("%RESULT%", StringComparison.OrdinalIgnoreCase) >= 0) - using (TempFile tf = new TempFile()) - { - RunScript.SerializeResult(tf, result, LogLines); - input = Regex.Replace(input, "\\%RESULT\\%", System.IO.File.ReadAllText(tf), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - } + if (input.IndexOf("%RESULT%", StringComparison.OrdinalIgnoreCase) >= 0) + input = Regex.Replace(input, "\\%RESULT\\%", m_resultFormatSerializer.Serialize(result, LogLines), RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); } foreach (KeyValuePair kv in m_options) diff --git a/Duplicati/Library/Modules/Builtin/ResultExportFormat.cs b/Duplicati/Library/Modules/Builtin/ResultExportFormat.cs new file mode 100644 index 000000000..b5d7ba58f --- /dev/null +++ b/Duplicati/Library/Modules/Builtin/ResultExportFormat.cs @@ -0,0 +1,9 @@ + +namespace Duplicati.Library.Modules.Builtin +{ + public enum ResultExportFormat + { + Duplicati, + Json + } +} diff --git a/Duplicati/Library/Modules/Builtin/ResultSerialization/DuplicatiFormatSerializer.cs b/Duplicati/Library/Modules/Builtin/ResultSerialization/DuplicatiFormatSerializer.cs new file mode 100644 index 000000000..a066e370f --- /dev/null +++ b/Duplicati/Library/Modules/Builtin/ResultSerialization/DuplicatiFormatSerializer.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace Duplicati.Library.Modules.Builtin.ResultSerialization +{ + /// + /// Provides a human-readable output for result data + /// + class DuplicatiFormatSerializer : IResultFormatSerializer + { + /// + /// Serialize the specified result and logLines. + /// + /// The serialized result string. + /// The result to serialize. + /// The log lines to serialize. + public string Serialize(object result, IEnumerable loglines) + { + StringBuilder sb = new StringBuilder(); + + if (result == null) + { + sb.Append("null?"); + } + else if (result is IEnumerable) + { + IEnumerable resultEnumerable = (IEnumerable)result; + IEnumerator resultEnumerator = resultEnumerable.GetEnumerator(); + resultEnumerator.Reset(); + + while (resultEnumerator.MoveNext()) + { + object current = resultEnumerator.Current; + if (current == null) + { + continue; + } + + if (current.GetType().IsGenericType && !current.GetType().IsGenericTypeDefinition && current.GetType().GetGenericTypeDefinition() == typeof(KeyValuePair<,>)) + { + object key = current.GetType().GetProperty("Key").GetValue(current, null); + object value = current.GetType().GetProperty("Value").GetValue(current, null); + sb.AppendFormat("{0}: {1}", key, value).AppendLine(); + } + else + { + sb.AppendLine(current.ToString()); + } + } + } + else if (result.GetType().IsArray) + { + Array array = (Array)result; + + for (int i = array.GetLowerBound(0); i <= array.GetUpperBound(0); i++) + { + object c = array.GetValue(i); + + if (c == null) + { + continue; + } + + if (c.GetType().IsGenericType && !c.GetType().IsGenericTypeDefinition && c.GetType().GetGenericTypeDefinition() == typeof(KeyValuePair<,>)) + { + object key = c.GetType().GetProperty("Key").GetValue(c, null); + object value = c.GetType().GetProperty("Value").GetValue(c, null); + sb.AppendFormat("{0}: {1}", key, value).AppendLine(); + } + else + { + sb.AppendLine(c.ToString()); + } + } + } + else if (result is Exception) + { + //No localization, must be parseable by script + Exception exception = (Exception)result; + sb.AppendFormat("Failed: {0}", exception.Message).AppendLine(); + sb.AppendFormat("Details: {0}", exception).AppendLine(); + } + else + { + Utility.Utility.PrintSerializeObject(result, sb); + } + + if (loglines != null && loglines.Any()) + { + sb.AppendLine(); + sb.AppendLine("Log data:"); + foreach (var n in loglines) + sb.AppendLine(n); + } + + return sb.ToString(); + } + + /// + /// Returns the format that the serializer represents + /// + public ResultExportFormat Format => ResultExportFormat.Duplicati; + } +} diff --git a/Duplicati/Library/Modules/Builtin/ResultSerialization/IResultFormatSerializer.cs b/Duplicati/Library/Modules/Builtin/ResultSerialization/IResultFormatSerializer.cs new file mode 100644 index 000000000..4fbfdf3fb --- /dev/null +++ b/Duplicati/Library/Modules/Builtin/ResultSerialization/IResultFormatSerializer.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; + +namespace Duplicati.Library.Modules.Builtin +{ + /// + /// Interface for describing a result serializer + /// + public interface IResultFormatSerializer + { + /// + /// Serialize the specified result and logLines. + /// + /// The serialized result string. + /// The result to serialize. + /// The log lines to serialize. + string Serialize(object result, IEnumerable loglines); + + /// + /// Returns the format that the serializer represents + /// + ResultExportFormat Format { get; } + } +} diff --git a/Duplicati/Library/Modules/Builtin/ResultSerialization/JsonFormatSerializer.cs b/Duplicati/Library/Modules/Builtin/ResultSerialization/JsonFormatSerializer.cs new file mode 100644 index 000000000..1d1bdee49 --- /dev/null +++ b/Duplicati/Library/Modules/Builtin/ResultSerialization/JsonFormatSerializer.cs @@ -0,0 +1,86 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Duplicati.Library.Interface; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using Newtonsoft.Json.Serialization; + +namespace Duplicati.Library.Modules.Builtin.ResultSerialization +{ + /// + /// Implements serialization of results to JSON + /// + class JsonFormatSerializer : IResultFormatSerializer + { + /// + /// Helper to filter the result classes properties + /// + private class DynamicContractResolver : DefaultContractResolver + { + /// + /// List of names to exclude + /// + private readonly HashSet m_excludes; + /// + /// Initializes a new instance of the + /// class. + /// + /// The names to exclude. + public DynamicContractResolver(params string[] names) + { + m_excludes = new HashSet(names); + } + + /// + /// Creates a filtered list of properties + /// + /// The filtered properties. + /// The type to create the list for. + /// Member serialization parameter. + protected override IList CreateProperties(Type type, MemberSerialization memberSerialization) + { + return base + .CreateProperties(type, memberSerialization) + .Where(x => !m_excludes.Contains(x.PropertyName)) + .ToList(); + } + } + + /// + /// Serialize the specified result and logLines. + /// + /// The serialized result string. + /// The result to serialize. + /// The log lines to serialize. + public string Serialize(object result, IEnumerable loglines) + { + return JsonConvert.SerializeObject( + new + { + Data = result, + LogLines = loglines + }, + + new JsonSerializerSettings() + { + ReferenceLoopHandling = ReferenceLoopHandling.Ignore, + ContractResolver = new DynamicContractResolver( + nameof(IBasicResults.Warnings), + nameof(IBasicResults.Errors), + nameof(IBasicResults.Messages) + ), + Converters = new List() + { + new StringEnumConverter() + } + } + ); + } + + /// + /// Returns the format that the serializer represents + /// + public ResultExportFormat Format => ResultExportFormat.Json; + } +} diff --git a/Duplicati/Library/Modules/Builtin/ResultSerialization/ResultFormatSerializerProvider.cs b/Duplicati/Library/Modules/Builtin/ResultSerialization/ResultFormatSerializerProvider.cs new file mode 100644 index 000000000..1d7141f32 --- /dev/null +++ b/Duplicati/Library/Modules/Builtin/ResultSerialization/ResultFormatSerializerProvider.cs @@ -0,0 +1,23 @@ +using System; +using Duplicati.Library.Localization.Short; + +namespace Duplicati.Library.Modules.Builtin.ResultSerialization +{ + /// + /// Factory class to provide result serialization + /// + public static class ResultFormatSerializerProvider + { + public static IResultFormatSerializer GetSerializer(ResultExportFormat format) { + switch (format) + { + case ResultExportFormat.Duplicati: + return new DuplicatiFormatSerializer(); + case ResultExportFormat.Json: + return new JsonFormatSerializer(); + default: + throw new Interface.UserInformationException(LC.L("The format is not supported: {0}", format), "SerializationFormatNotSupported"); + } + } + } +} diff --git a/Duplicati/Library/Modules/Builtin/RunScript.cs b/Duplicati/Library/Modules/Builtin/RunScript.cs index 07f0a4ca1..4c33a44a1 100644 --- a/Duplicati/Library/Modules/Builtin/RunScript.cs +++ b/Duplicati/Library/Modules/Builtin/RunScript.cs @@ -22,8 +22,9 @@ using System.Text; using System.Collections.Generic; using Duplicati.Library.Utility; using Duplicati.Library.Interface; -using System.Linq; - +using System.Linq; +using Duplicati.Library.Modules.Builtin.ResultSerialization; + namespace Duplicati.Library.Modules.Builtin { public class RunScript : Duplicati.Library.Interface.IGenericCallbackModule @@ -50,6 +51,8 @@ namespace Duplicati.Library.Modules.Builtin /// private const string OPTION_LOG_FILTER = "run-script-log-filter"; + private const string RESULT_FORMAT_OPTION = "run-script-result-output-format"; + private const string DEFAULT_TIMEOUT = "60s"; private string m_requiredScript = null; @@ -61,6 +64,7 @@ namespace Duplicati.Library.Modules.Builtin private string m_remoteurl; private string[] m_localpath; private IDictionary m_options; + private IResultFormatSerializer resultFormatSerializer; /// /// The log scope that should be disposed @@ -79,11 +83,23 @@ namespace Duplicati.Library.Modules.Builtin commandlineOptions.TryGetValue(REQUIRED_OPTION, out m_requiredScript); commandlineOptions.TryGetValue(FINISH_OPTION, out m_finishScript); + string tmpResultFormat; + ResultExportFormat resultFormat; + if (!commandlineOptions.TryGetValue(RESULT_FORMAT_OPTION, out tmpResultFormat)) { + resultFormat = ResultExportFormat.Duplicati; + } + else if (!Enum.TryParse(tmpResultFormat, true, out resultFormat)) { + resultFormat = ResultExportFormat.Duplicati; + } + + resultFormatSerializer = ResultFormatSerializerProvider.GetSerializer(resultFormat); + string t; if (!commandlineOptions.TryGetValue(TIMEOUT_OPTION, out t)) t = DEFAULT_TIMEOUT; m_timeout = (int)Utility.Timeparser.ParseTimeSpan(t).TotalMilliseconds; + m_options = commandlineOptions; m_options.TryGetValue(OPTION_LOG_FILTER, out var logfilterstring); @@ -111,10 +127,18 @@ namespace Duplicati.Library.Modules.Builtin { get { + string[] resultOutputFormatOptions = new string[] { ResultExportFormat.Duplicati.ToString(), ResultExportFormat.Json.ToString() }; return new List(new Duplicati.Library.Interface.ICommandLineArgument[] { new Duplicati.Library.Interface.CommandLineArgument(STARTUP_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.RunScript.StartupoptionShort, Strings.RunScript.StartupoptionLong), new Duplicati.Library.Interface.CommandLineArgument(FINISH_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.RunScript.FinishoptionShort, Strings.RunScript.FinishoptionLong), new Duplicati.Library.Interface.CommandLineArgument(REQUIRED_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Path, Strings.RunScript.RequiredoptionShort, Strings.RunScript.RequiredoptionLong), + new CommandLineArgument(RESULT_FORMAT_OPTION, + CommandLineArgument.ArgumentType.Enumeration, + Strings.RunScript.ResultFormatShort, + Strings.RunScript.ResultFormatLong(resultOutputFormatOptions), + ResultExportFormat.Duplicati.ToString(), + null, + resultOutputFormatOptions), new Duplicati.Library.Interface.CommandLineArgument(TIMEOUT_OPTION, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.RunScript.TimeoutoptionShort, Strings.RunScript.TimeoutoptionLong, DEFAULT_TIMEOUT), new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevellShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), @@ -163,91 +187,14 @@ namespace Duplicati.Library.Modules.Builtin using (TempFile tmpfile = new TempFile()) { - SerializeResult(tmpfile, result, m_logstorage); + using (var streamWriter = new StreamWriter(tmpfile)) + streamWriter.Write(resultFormatSerializer.Serialize(result, m_logstorage)); + Execute(m_finishScript, "AFTER", m_operationName, ref m_remoteurl, ref m_localpath, m_timeout, false, m_options, tmpfile, level); } } #endregion - public static void SerializeResult(string file, object result, IEnumerable loglines) - { - using(StreamWriter sw = new StreamWriter(file)) - { - if (result == null) - { - sw.WriteLine("null?"); - } - else if (result is System.Collections.IEnumerable) - { - System.Collections.IEnumerable ie = (System.Collections.IEnumerable)result; - System.Collections.IEnumerator ien = ie.GetEnumerator(); - ien.Reset(); - - while (ien.MoveNext()) - { - object c = ien.Current; - if (c == null) - continue; - - if (c.GetType().IsGenericType && !c.GetType().IsGenericTypeDefinition && c.GetType().GetGenericTypeDefinition() == typeof(KeyValuePair<,>)) - { - object key = c.GetType().GetProperty("Key").GetValue(c, null); - object value = c.GetType().GetProperty("Value").GetValue(c, null); - sw.WriteLine("{0}: {1}", key, value); - } - else - sw.WriteLine(c); - } - } - else if (result.GetType().IsArray) - { - Array a = (Array)result; - - for(int i = a.GetLowerBound(0); i <= a.GetUpperBound(0); i++) - { - object c = a.GetValue(i); - - if (c == null) - continue; - - if (c.GetType().IsGenericType && !c.GetType().IsGenericTypeDefinition && c.GetType().GetGenericTypeDefinition() == typeof(KeyValuePair<,>)) - { - object key = c.GetType().GetProperty("Key").GetValue(c, null); - object value = c.GetType().GetProperty("Value").GetValue(c, null); - sw.WriteLine("{0}: {1}", key, value); - } - else - sw.WriteLine(c); - } - } - else if (result is Exception) - { - //No localization, must be parseable by script - Exception e = (Exception)result; - sw.WriteLine("Failed: {0}", e.Message); - sw.WriteLine("Details: {0}", e); - } - else - { - Utility.Utility.PrintSerializeObject(result, sw, - filter: (p, x) => - !new[] { - nameof(IBasicResults.Warnings), - nameof(IBasicResults.Errors), - nameof(IBasicResults.Messages), - }.Contains(p.Name, StringComparer.Ordinal)); - } - - if (loglines != null) - { - sw.WriteLine(); - sw.WriteLine("Log data:"); - foreach (var n in loglines) - sw.WriteLine(n); - } - } - } - private static void Execute(string scriptpath, string eventname, string operationname, ref string remoteurl, ref string[] localpath, int timeout, bool requiredScript, IDictionary options, string datafile, ParsedResultType? level) { try diff --git a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs index 353136634..e0dc9fdb8 100644 --- a/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendHttpMessage.cs @@ -1,5 +1,6 @@ using Duplicati.Library.Interface; using Duplicati.Library.Logging; +using Duplicati.Library.Modules.Builtin.ResultSerialization; using Duplicati.Library.Utility; using System; using System.Collections.Generic; @@ -47,13 +48,10 @@ namespace Duplicati.Library.Modules.Builtin { /// Option used to specify if reports are sent for other operations than backups /// private const string OPTION_SENDALL = "send-http-any-operation"; - - /// - /// Option used to specify if reports are sent as json-serialized files + /// Option used to specify what format the result is sent in. /// - private const string OPTION_SENDASJSON = "send-http-as-json"; - + private const string OPTION_RESULT_FORMAT = "send-http-result-output-format"; /// /// Option used to set the log level /// @@ -85,8 +83,6 @@ namespace Duplicati.Library.Modules.Builtin { #endregion #region Private variables - - /// /// The HTTP report URL /// @@ -100,10 +96,6 @@ namespace Duplicati.Library.Modules.Builtin { /// private string m_extraParameters; /// - /// Flag indicating if data is sent as JSON body - /// - private bool m_asJson; - /// /// The http verb /// private string m_verb; @@ -151,21 +143,23 @@ namespace Duplicati.Library.Modules.Builtin { new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.SendhttpanyoperationShort, Strings.SendHttpMessage.SendhttpanyoperationLong), new CommandLineArgument(OPTION_VERB, CommandLineArgument.ArgumentType.String, Strings.SendHttpMessage.HttpverbShort, Strings.SendHttpMessage.HttpverbLong, "POST"), - new CommandLineArgument(OPTION_SENDASJSON, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.SendasjsonShort, Strings.SendHttpMessage.SendasjsonLong, "false"), new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevellShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), - }); + + new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), + }); } } - protected override string SubjectOptionName => OPTION_MESSAGE; - protected override string BodyOptionName => OPTION_MESSAGE; - protected override string ActionLevelOptionName => OPTION_SENDLEVEL; - protected override string ActionOnAnyOperationOptionName => OPTION_SENDALL; - protected override string LogLevelOptionName => OPTION_LOG_LEVEL; + protected override string SubjectOptionName => OPTION_MESSAGE; + protected override string BodyOptionName => OPTION_MESSAGE; + protected override string ActionLevelOptionName => OPTION_SENDLEVEL; + protected override string ActionOnAnyOperationOptionName => OPTION_SENDALL; + protected override string LogLevelOptionName => OPTION_LOG_LEVEL; protected override string LogFilterOptionName => OPTION_LOG_FILTER; protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; + protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT; /// /// This method is the interception where the module can interact with the execution environment and modify the settings. @@ -183,8 +177,6 @@ namespace Duplicati.Library.Modules.Builtin { m_messageParameterName = DEFAULT_MESSAGE_PARAMETER_NAME; commandlineOptions.TryGetValue(OPTION_EXTRA_PARAMETERS, out m_extraParameters); - m_asJson = Library.Utility.Utility.ParseBoolOption(commandlineOptions, OPTION_SENDASJSON); - commandlineOptions.TryGetValue(OPTION_VERB, out m_verb); if (string.IsNullOrWhiteSpace(m_verb)) m_verb = "POST"; @@ -192,24 +184,31 @@ namespace Duplicati.Library.Modules.Builtin { return true; } - #endregion + #endregion + protected override string ReplaceTemplate(string input, object result, bool subjectline) + { + // No need to do the expansion as we throw away the result + if (subjectline) + return string.Empty; + return base.ReplaceTemplate(input, result, subjectline); + } - protected override void SendMessage(string subject, string message) { + protected override void SendMessage(string subject, string body) { Exception ex = null; byte[] data; string contenttype; - if (m_asJson) + if (ExportFormat == ResultExportFormat.Json) { contenttype = "application/json"; - data = Encoding.UTF8.GetBytes(message); + data = Encoding.UTF8.GetBytes(body); } else { contenttype = "application/x-www-form-urlencoded"; - var postData = $"{m_messageParameterName}={System.Uri.EscapeDataString(message)}"; + var postData = $"{m_messageParameterName}={System.Uri.EscapeDataString(body)}"; if (!string.IsNullOrEmpty(m_extraParameters)) { postData += $"&{System.Uri.EscapeUriString(m_extraParameters)}"; @@ -262,87 +261,5 @@ namespace Duplicati.Library.Modules.Builtin { if (ex != null) throw ex; } - - /// - /// Helper method to perform template expansion - /// - /// The expanded template. - /// The input template. - /// The result object. - /// If set to true, the result is intended for a subject or title line. - protected override string ReplaceTemplate(string input, object result, bool subjectline) - { - if (subjectline) - return string.Empty; - else - { - if (m_asJson) - return SerializeAsJson(result, LogLines); - else - return base.ReplaceTemplate(input, result, subjectline); - } - } - - /// - /// Writes the result as a JSON formatted file - /// - /// The result object. - /// The log lines to serialize. - public static string SerializeAsJson(object result, IEnumerable loglines) - { - var settings = new Newtonsoft.Json.JsonSerializerSettings - { - ContractResolver = new DynamicContractResolver( - nameof(IBasicResults.Warnings), - nameof(IBasicResults.Errors), - nameof(IBasicResults.Messages) - ), - Formatting = Newtonsoft.Json.Formatting.None, - ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore, - }; - - return Newtonsoft.Json.JsonConvert.SerializeObject( - new { - Data = result, - LogLines = loglines - }, - settings - ); - } - - - /// - /// Helper to filter the result classes properties - /// - private class DynamicContractResolver : Newtonsoft.Json.Serialization.DefaultContractResolver - { - /// - /// List of names to exclude - /// - private readonly HashSet m_excludes; - /// - /// Initializes a new instance of the - /// class. - /// - /// The names to exclude. - public DynamicContractResolver(params string[] names) - { - m_excludes = new HashSet(names); - } - - /// - /// Creates a filtered list of properties - /// - /// The filtered properties. - /// The type to create the list for. - /// Member serialization parameter. - protected override IList CreateProperties(Type type, Newtonsoft.Json.MemberSerialization memberSerialization) - { - return base - .CreateProperties(type, memberSerialization) - .Where(x => !m_excludes.Contains(x.PropertyName)) - .ToList(); - } - } } } diff --git a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs index 71bf7f162..755ae54f9 100644 --- a/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs +++ b/Duplicati/Library/Modules/Builtin/SendJabberMessage.cs @@ -2,13 +2,20 @@ using System.Collections.Generic; using System.Linq; using Duplicati.Library.Interface; +using Duplicati.Library.Logging; +using System.Net.NetworkInformation; +using Duplicati.Library.Modules.Builtin.ResultSerialization; namespace Duplicati.Library.Modules.Builtin { public class SendJabberMessage : ReportHelper { - #region Option names + /// + /// The tag used for log messages + /// + private static readonly string LOGTAG = Logging.Log.LogTagFromType(); + #region Option names /// /// Option used to specify server username /// @@ -33,6 +40,10 @@ namespace Duplicati.Library.Modules.Builtin /// Option used to specify if reports are sent for other operations than backups /// private const string OPTION_SENDALL = "send-xmpp-any-operation"; + /// + /// Option used to specify what format the result is sent in. + /// + private const string OPTION_RESULT_FORMAT = "send-xmpp-result-output-format"; /// /// Option used to set the log level @@ -61,7 +72,6 @@ namespace Duplicati.Library.Modules.Builtin #endregion #region Private variables - /// /// The server username /// @@ -120,6 +130,8 @@ namespace Duplicati.Library.Modules.Builtin new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevellShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), + + new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), }); } } @@ -131,6 +143,7 @@ namespace Duplicati.Library.Modules.Builtin protected override string LogLevelOptionName => OPTION_LOG_LEVEL; protected override string LogFilterOptionName => OPTION_LOG_FILTER; protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; + protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT; /// /// This method is the interception where the module can interact with the execution environment and modify the settings. @@ -151,7 +164,15 @@ namespace Duplicati.Library.Modules.Builtin #endregion - protected override void SendMessage(string subject, string message) + protected override string ReplaceTemplate(string input, object result, bool subjectline) + { + // No need to do the expansion as we throw away the result + if (subjectline) + return string.Empty; + return base.ReplaceTemplate(input, result, subjectline); + } + + protected override void SendMessage(string subject, string body) { Exception ex = null; var waitEvent = new System.Threading.ManualResetEvent(false); @@ -173,10 +194,11 @@ namespace Duplicati.Library.Modules.Builtin try { foreach(var recipient in m_to.Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries)) - con.Send(new agsXMPP.protocol.client.Message(recipient, agsXMPP.protocol.client.MessageType.chat, message)); + con.Send(new agsXMPP.protocol.client.Message(recipient, agsXMPP.protocol.client.MessageType.chat, body)); } catch (Exception e) { + Logging.Log.WriteWarningMessage(LOGTAG, "XMPPSendError", e, "Failed to send to XMPP messages: {0}", e.Message); ex = e; } finally @@ -186,11 +208,15 @@ namespace Duplicati.Library.Modules.Builtin }; agsXMPP.ErrorHandler errorHandler = (sender, e) => { + Logging.Log.WriteWarningMessage(LOGTAG, "XMPPError", e, "An error ocurred in XMPP: {0}", e.Message); + ex = e; waitEvent.Set(); }; agsXMPP.XmppElementHandler loginErroHandler = (sender, e) => { + Logging.Log.WriteWarningMessage(LOGTAG, "XMPPLoginError", null, "Failed to login to XMPP: {0}", e); + ex = new Exception(string.Format("Failed to log in: {0}", e)); waitEvent.Set(); }; @@ -214,8 +240,9 @@ namespace Duplicati.Library.Modules.Builtin { con.Close(); } - catch + catch (Exception lex) { + Logging.Log.WriteExplicitMessage(LOGTAG, "CloseConnectionError", lex, "Failed to close XMPP connection: {0}", lex.Message); } if (ex != null) @@ -223,20 +250,5 @@ namespace Duplicati.Library.Modules.Builtin if (timeout) throw new TimeoutException(Strings.SendJabberMessage.LoginTimeoutError); } - - /// - /// Helper method to perform template expansion - /// - /// The expanded template. - /// The input template. - /// The result object. - /// If set to true, the result is intended for a subject or title line. - protected override string ReplaceTemplate(string input, object result, bool subjectline) - { - if (subjectline) - return string.Empty; - else - return base.ReplaceTemplate(input, result, subjectline); - } } } diff --git a/Duplicati/Library/Modules/Builtin/SendMail.cs b/Duplicati/Library/Modules/Builtin/SendMail.cs index c73a73328..107ba4b80 100644 --- a/Duplicati/Library/Modules/Builtin/SendMail.cs +++ b/Duplicati/Library/Modules/Builtin/SendMail.cs @@ -14,6 +14,7 @@ using System.Net.NetworkInformation; using System.Net.Sockets; using MailKit.Net.Smtp; using MimeKit; +using Duplicati.Library.Modules.Builtin.ResultSerialization; namespace Duplicati.Library.Modules.Builtin { @@ -63,6 +64,10 @@ namespace Duplicati.Library.Modules.Builtin /// private const string OPTION_SENDALL = "send-mail-any-operation"; /// + /// Option used to specify what format the result is sent in. + /// + private const string OPTION_RESULT_FORMAT = "send-mail-result-output-format"; + /// /// Option used to set the log level for mail reports /// private const string OPTION_LOG_LEVEL = "send-mail-log-level"; @@ -146,11 +151,13 @@ namespace Duplicati.Library.Modules.Builtin new CommandLineArgument(OPTION_USERNAME, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionUsernameShort, Strings.SendMail.OptionUsernameLong), new CommandLineArgument(OPTION_PASSWORD, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionPasswordShort, Strings.SendMail.OptionPasswordLong), new CommandLineArgument(OPTION_SENDLEVEL, CommandLineArgument.ArgumentType.String, Strings.SendMail.OptionSendlevelShort, Strings.SendMail.OptionSendlevelLong(ParsedResultType.Success.ToString(), ParsedResultType.Warning.ToString(), ParsedResultType.Error.ToString(), ParsedResultType.Fatal.ToString(), "All"), DEFAULT_LEVEL, null, Enum.GetNames(typeof(ParsedResultType)).Union(new string [] { "All" }).ToArray()), - new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendMail.OptionSendallShort, Strings.SendMail.OptionSendallLong), + new CommandLineArgument(OPTION_SENDALL, CommandLineArgument.ArgumentType.Boolean, Strings.SendHttpMessage.SendhttpanyoperationShort, Strings.SendHttpMessage.SendhttpanyoperationLong), new CommandLineArgument(OPTION_LOG_LEVEL, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.OptionLoglevellShort, Strings.ReportHelper.OptionLoglevelLong, DEFAULT_LOG_LEVEL.ToString(), null, Enum.GetNames(typeof(Logging.LogMessageType))), new CommandLineArgument(OPTION_LOG_FILTER, CommandLineArgument.ArgumentType.String, Strings.ReportHelper.OptionLogfilterShort, Strings.ReportHelper.OptionLogfilterLong), new CommandLineArgument(OPTION_MAX_LOG_LINES, CommandLineArgument.ArgumentType.Integer, Strings.ReportHelper.OptionmaxloglinesShort, Strings.ReportHelper.OptionmaxloglinesLong, DEFAULT_LOGLINES.ToString()), + + new CommandLineArgument(OPTION_RESULT_FORMAT, CommandLineArgument.ArgumentType.Enumeration, Strings.ReportHelper.ResultFormatShort, Strings.ReportHelper.ResultFormatLong(Enum.GetNames(typeof(ResultExportFormat))), DEFAULT_EXPORT_FORMAT.ToString(), null, Enum.GetNames(typeof(ResultExportFormat))), }); } } @@ -160,9 +167,10 @@ namespace Duplicati.Library.Modules.Builtin protected override string ActionLevelOptionName => OPTION_SENDLEVEL; protected override string ActionOnAnyOperationOptionName => OPTION_SENDALL; protected override string LogLevelOptionName => OPTION_LOG_LEVEL; - protected override string LogFilterOptionName => OPTION_LOG_FILTER; - protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; - + protected override string LogFilterOptionName => OPTION_LOG_FILTER; + protected override string LogLinesOptionName => OPTION_MAX_LOG_LINES; + protected override string ResultFormatOptionName => OPTION_RESULT_FORMAT; + /// /// This method is the interception where the module can interact with the execution environment and modify the settings. /// @@ -234,7 +242,7 @@ namespace Duplicati.Library.Modules.Builtin if (dnsAddress.AddressFamily == AddressFamily.InterNetwork) dnslist.Add(dnsAddress.ToString()); } - catch { } + catch (Exception ex) { Logging.Log.WriteExplicitMessage(LOGTAG, "DNSServerLookupFailure", ex, "Failed to get DNS servers from network interface"); } dnslist = dnslist.Distinct().ToList(); diff --git a/Duplicati/Library/Modules/Builtin/Strings.cs b/Duplicati/Library/Modules/Builtin/Strings.cs index 3caffbe11..fe86872eb 100644 --- a/Duplicati/Library/Modules/Builtin/Strings.cs +++ b/Duplicati/Library/Modules/Builtin/Strings.cs @@ -1,7 +1,8 @@ using Duplicati.Library.Localization.Short; using System; - - +using System.Collections; +using System.Collections.Generic; + namespace Duplicati.Library.Modules.Builtin.Strings { internal static class ConsolePasswordInput { public static string ConfirmPassphrasePrompt { get { return LC.L(@"Confirm encryption passphrase"); } } @@ -55,6 +56,8 @@ namespace Duplicati.Library.Modules.Builtin.Strings { public static string InvalidExitCodeError(string script, int exitcode) { return LC.L(@"The script ""{0}"" returned with exit code {1}", script, exitcode); } public static string RequiredoptionLong { get { return LC.L(@"Executes a script before performing an operation. The operation will block until the script has completed or timed out. If the script returns a non-zero error code or times out, the operation will be aborted."); } } public static string RequiredoptionShort { get { return LC.L(@"Run a required script on startup"); } } + public static string ResultFormatShort { get { return LC.L(@"Selects the output format for results"); } } + public static string ResultFormatLong(IEnumerable options) { return LC.L(@"Selects the output format for results. Available formats: {0}", string.Join(", ", options)); } public static string ScriptExecuteError(string script, string message) { return LC.L(@"Error while executing script ""{0}"": {1}", script, message); } public static string ScriptTimeoutError(string script) { return LC.L(@"Execution of the script ""{0}"" timed out", script); } public static string StartupoptionLong { get { return LC.L(@"Executes a script before performing an operation. The operation will block until the script has completed or timed out."); } } @@ -176,5 +179,7 @@ You can supply multiple options with a comma separator, e.g. ""{0},{1}"". The sp public static string OptionLogfilterLong { get { return LC.L("Use this option to set a filter expression that defines what options are included in the report"); } } public static string OptionmaxloglinesShort { get { return LC.L("Limits log lines"); } } public static string OptionmaxloglinesLong { get { return LC.L("Use this option to set the maximum number of log lines to include in the report. Zero or negative values means unlimited."); } } + public static string ResultFormatShort { get { return LC.L(@"Selects the output format for results"); } } + public static string ResultFormatLong(IEnumerable options) { return LC.L(@"Selects the output format for results. Available formats: {0}", string.Join(", ", options)); } } } diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index e4fe7c814..d3c092b9f 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -35,6 +35,11 @@ namespace Duplicati.Library.Utility /// public static long DEFAULT_BUFFER_SIZE => SystemContextSettings.Buffersize; + /// + /// A cache of the FileSystemCaseSensitive property, which is computed upon the first access. + /// + private static bool? CachedIsFSCaseSensitive = null; + /// /// Gets the hash algorithm used for calculating a hash /// @@ -867,13 +872,19 @@ namespace Duplicati.Library.Utility { get { - var str = Environment.GetEnvironmentVariable("FILESYSTEM_CASE_SENSITIVE"); + if (!CachedIsFSCaseSensitive.HasValue) + { + var str = Environment.GetEnvironmentVariable("FILESYSTEM_CASE_SENSITIVE"); - // TODO: This should probably be determined by filesystem rather than OS, - // OSX can actually have the disks formated as Case Sensitive, but insensitive is default - Func defaultReply = () => Utility.IsClientLinux && !Utility.IsClientOSX; + // TODO: This should probably be determined by filesystem rather than OS, + // OSX can actually have the disks formated as Case Sensitive, but insensitive is default - return Utility.ParseBool(str, defaultReply); + Func defaultReply = () => Utility.IsClientLinux && !Utility.IsClientOSX; + + CachedIsFSCaseSensitive = Utility.ParseBool(str, defaultReply); + } + + return CachedIsFSCaseSensitive.Value; } } diff --git a/Duplicati/Server/webroot/ngax/templates/notificationarea.html b/Duplicati/Server/webroot/ngax/templates/notificationarea.html index c73d4c94b..2880f1556 100644 --- a/Duplicati/Server/webroot/ngax/templates/notificationarea.html +++ b/Duplicati/Server/webroot/ngax/templates/notificationarea.html @@ -35,7 +35,7 @@ -
  • +
  • Dismiss all diff --git a/tabs-to-spaces.sh b/tabs-to-spaces.sh index 0f9ddd0d2..1e52ad850 100644 --- a/tabs-to-spaces.sh +++ b/tabs-to-spaces.sh @@ -3,3 +3,5 @@ find Duplicati -type f -name *.js -exec sed -i '' $'s/\t/ /g' {} + find Duplicati -type f -name *.css -exec sed -i '' $'s/\t/ /g' {} + find Duplicati -type f -name *.cs -exec sed -i '' $'s/\t/ /g' {} + find Duplicati -type f -name *.txt -exec sed -i '' $'s/\t/ /g' {} + + +find Duplicati -type f -name *.cs | xargs python unix2dos.py diff --git a/unix2dos.py b/unix2dos.py new file mode 100644 index 000000000..517955040 --- /dev/null +++ b/unix2dos.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +import sys + +for fname in sys.argv[1:]: + infile = open( fname, "rb" ) + instr = infile.read() + infile.close() + outstr = instr.replace( "\r\n", "\n" ).replace( "\r", "\n" ).replace( "\n", "\r\n" ) + + if len(outstr) == len(instr): + continue + + outfile = open( fname, "wb" ) + outfile.write( outstr ) + outfile.close()