From bc0686794afa04d0a08a6953127d80ceab9e8725 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Wed, 30 Oct 2019 17:30:47 -0600 Subject: [PATCH 1/2] Cache to disk --- Duplicati/Library/Main/Options.cs | 7 +++++ Duplicati/Library/Main/Strings.cs | 4 ++- .../Main/Volumes/FilesetVolumeWriter.cs | 26 +++++++++++++++---- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index f07399623..3526ba285 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -292,6 +292,8 @@ namespace Duplicati.Library.Main new CommandLineArgument("asynchronous-concurrent-upload-limit", CommandLineArgument.ArgumentType.Integer, Strings.Options.AsynchronousconcurrentuploadlimitShort, Strings.Options.AsynchronousconcurrentuploadlimitLong, "4"), new CommandLineArgument("asynchronous-upload-folder", CommandLineArgument.ArgumentType.Path, Strings.Options.AsynchronousuploadfolderShort, Strings.Options.AsynchronousuploadfolderLong, System.IO.Path.GetTempPath()), + new CommandLineArgument("cache-fileset-on-disk", CommandLineArgument.ArgumentType.Boolean, Strings.Options.CachefilesetondiskShort,Strings.Options.CachefilesetondiskLong, "false"), + new CommandLineArgument("disable-streaming-transfers", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisableStreamingShort, Strings.Options.DisableStreamingLong, "false"), new CommandLineArgument("throttle-upload", CommandLineArgument.ArgumentType.Size, Strings.Options.ThrottleuploadShort, Strings.Options.ThrottleuploadLong, "0kb"), @@ -1098,6 +1100,11 @@ namespace Duplicati.Library.Main } } + /// + /// A value indicating if filesets should be cached on disk instead of in memory. + /// + public bool CacheFilesetOnDisk { get { return GetBool("cache-fileset-on-disk"); } } + /// /// Gets the logfile filename /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 4ad1339d5..aaa531f3c 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -86,7 +86,7 @@ namespace Duplicati.Library.Main.Strings public static string EncryptionmoduleLong { get { return LC.L(@"Duplicati supports pluggable encryption modules. Use this option to select a module to use for encryption. This is only applied when creating new volumes, when reading an existing file, the filename is used to select the encryption module."); } } public static string EncryptionmoduleShort { get { return LC.L(@"Select what module to use for encryption"); } } public static string DisablemoduleLong { get { return LC.L(@"Supply one or more module names, separated by commas to unload them"); } } - public static string DisablemoduleShort { get { return LC.L(@"Disabled one or more modules"); } } + public static string DisablemoduleShort { get { return LC.L(@"Disables one or more modules"); } } public static string EnablemoduleLong { get { return LC.L(@"Supply one or more module names, separated by commas to load them"); } } public static string EnablemoduleShort { get { return LC.L(@"Enables one or more modules"); } } public static string SnapshotpolicyLong { get { return LC.L(@"This setting controls the usage of snapshots, which allows Duplicati to backup files that are locked by other programs. If this is set to ""off"", Duplicati will not attempt to create a disk snapshot. Setting this to ""auto"" makes Duplicati attempt to create a snapshot, and fail silently if that was not allowed or supported (note that the OS may still log system warnings). A setting of ""on"" will also make Duplicati attempt to create a snapshot, but will produce a warning message in the log if it fails. Setting it to ""required"" will make Duplicati abort the backup if the snapshot creation fails. On windows this uses the Volume Shadow Copy Services (VSS) and requires administrative privileges. On Linux this uses Logical Volume Management (LVM) and requires root privileges."); } } @@ -97,6 +97,8 @@ namespace Duplicati.Library.Main.Strings public static string AsynchronousuploadlimitShort { get { return LC.L(@"The number of volumes to create ahead of time"); } } public static string AsynchronousconcurrentuploadlimitLong { get { return LC.L(@"When performing asynchronous uploads, the maximum number of concurrent uploads allowed. Set to zero to disable the limit."); } } public static string AsynchronousconcurrentuploadlimitShort { get { return LC.L(@"The number of concurrent uploads allowed"); } } + public static string CachefilesetondiskLong { get { return LC.L(@"Causes fileset writer to hold data on the disk instead of holding it in memory. May be useful on devices with limited memory."); } } + public static string CachefilesetondiskShort { get { return LC.L(@"Hold fileset data on disk instead of in memory"); } } public static string DebugoutputLong { get { return LC.L(@"Activating this option will make some error messages more verbose, which may help you track down a particular issue"); } } public static string DebugoutputShort { get { return LC.L(@"Enables debugging output"); } } public static string LogfileShort { get { return LC.L(@"Log internal information to a file"); } } diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index 60a0dff2c..527090b90 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -8,7 +8,8 @@ namespace Duplicati.Library.Main.Volumes { public class FilesetVolumeWriter : VolumeWriterBase { - private readonly MemoryStream m_memorystream; + private readonly Library.Utility.TempFile m_tempFile; + private readonly Stream m_tempStream; private StreamWriter m_streamwriter; private readonly JsonWriter m_writer; private long m_filecount; @@ -19,8 +20,17 @@ namespace Duplicati.Library.Main.Volumes public FilesetVolumeWriter(Options options, DateTime timestamp) : base(options, timestamp) { - m_memorystream = new MemoryStream(); - m_streamwriter = new StreamWriter(m_memorystream, ENCODING); + if (options.CacheFilesetOnDisk) + { + m_tempFile = new Library.Utility.TempFile(); + m_tempStream = File.Open(m_tempFile, FileMode.Create, FileAccess.ReadWrite); + } + else + { + m_tempStream = new MemoryStream(); + } + + m_streamwriter = new StreamWriter(m_tempStream, ENCODING); m_writer = new JsonTextWriter(m_streamwriter); m_writer.WriteStartArray(); } @@ -138,6 +148,12 @@ namespace Duplicati.Library.Main.Volumes m_streamwriter = null; } + if (m_tempFile != null) + { + m_tempStream.Dispose(); + m_tempFile.Dispose(); + } + base.Close(); } @@ -149,8 +165,8 @@ namespace Duplicati.Library.Main.Volumes using (Stream sr = m_compression.CreateFile(FILELIST, CompressionHint.Compressible, DateTime.UtcNow)) { - m_memorystream.Seek(0, SeekOrigin.Begin); - m_memorystream.CopyTo(sr); + m_tempStream.Seek(0, SeekOrigin.Begin); + m_tempStream.CopyTo(sr); sr.Flush(); } } From e14be62c2b23ac742975d989ead7aef6087d98c8 Mon Sep 17 00:00:00 2001 From: Tyler Gill Date: Fri, 1 Nov 2019 19:27:49 -0600 Subject: [PATCH 2/2] Make the temp file the default and only bevavior - fileset volume no longer uses memory stream --- Duplicati/Library/Main/Options.cs | 11 ++--------- Duplicati/Library/Main/Strings.cs | 2 -- .../Main/Volumes/FilesetVolumeWriter.cs | 18 +++++++----------- 3 files changed, 9 insertions(+), 22 deletions(-) diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 3526ba285..6f6c56328 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -291,9 +291,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("asynchronous-upload-limit", CommandLineArgument.ArgumentType.Integer, Strings.Options.AsynchronousuploadlimitShort, Strings.Options.AsynchronousuploadlimitLong, "4"), new CommandLineArgument("asynchronous-concurrent-upload-limit", CommandLineArgument.ArgumentType.Integer, Strings.Options.AsynchronousconcurrentuploadlimitShort, Strings.Options.AsynchronousconcurrentuploadlimitLong, "4"), new CommandLineArgument("asynchronous-upload-folder", CommandLineArgument.ArgumentType.Path, Strings.Options.AsynchronousuploadfolderShort, Strings.Options.AsynchronousuploadfolderLong, System.IO.Path.GetTempPath()), - - new CommandLineArgument("cache-fileset-on-disk", CommandLineArgument.ArgumentType.Boolean, Strings.Options.CachefilesetondiskShort,Strings.Options.CachefilesetondiskLong, "false"), - + new CommandLineArgument("disable-streaming-transfers", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisableStreamingShort, Strings.Options.DisableStreamingLong, "false"), new CommandLineArgument("throttle-upload", CommandLineArgument.ArgumentType.Size, Strings.Options.ThrottleuploadShort, Strings.Options.ThrottleuploadLong, "0kb"), @@ -1099,12 +1097,7 @@ namespace Duplicati.Library.Main return value; } } - - /// - /// A value indicating if filesets should be cached on disk instead of in memory. - /// - public bool CacheFilesetOnDisk { get { return GetBool("cache-fileset-on-disk"); } } - + /// /// Gets the logfile filename /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index aaa531f3c..632d9645b 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -97,8 +97,6 @@ namespace Duplicati.Library.Main.Strings public static string AsynchronousuploadlimitShort { get { return LC.L(@"The number of volumes to create ahead of time"); } } public static string AsynchronousconcurrentuploadlimitLong { get { return LC.L(@"When performing asynchronous uploads, the maximum number of concurrent uploads allowed. Set to zero to disable the limit."); } } public static string AsynchronousconcurrentuploadlimitShort { get { return LC.L(@"The number of concurrent uploads allowed"); } } - public static string CachefilesetondiskLong { get { return LC.L(@"Causes fileset writer to hold data on the disk instead of holding it in memory. May be useful on devices with limited memory."); } } - public static string CachefilesetondiskShort { get { return LC.L(@"Hold fileset data on disk instead of in memory"); } } public static string DebugoutputLong { get { return LC.L(@"Activating this option will make some error messages more verbose, which may help you track down a particular issue"); } } public static string DebugoutputShort { get { return LC.L(@"Enables debugging output"); } } public static string LogfileShort { get { return LC.L(@"Log internal information to a file"); } } diff --git a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs index 527090b90..a5faba88b 100644 --- a/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs +++ b/Duplicati/Library/Main/Volumes/FilesetVolumeWriter.cs @@ -20,16 +20,8 @@ namespace Duplicati.Library.Main.Volumes public FilesetVolumeWriter(Options options, DateTime timestamp) : base(options, timestamp) { - if (options.CacheFilesetOnDisk) - { - m_tempFile = new Library.Utility.TempFile(); - m_tempStream = File.Open(m_tempFile, FileMode.Create, FileAccess.ReadWrite); - } - else - { - m_tempStream = new MemoryStream(); - } - + m_tempFile = new Library.Utility.TempFile(); + m_tempStream = File.Open(m_tempFile, FileMode.Create, FileAccess.ReadWrite); m_streamwriter = new StreamWriter(m_tempStream, ENCODING); m_writer = new JsonTextWriter(m_streamwriter); m_writer.WriteStartArray(); @@ -148,9 +140,13 @@ namespace Duplicati.Library.Main.Volumes m_streamwriter = null; } - if (m_tempFile != null) + if (m_tempStream != null) { m_tempStream.Dispose(); + } + + if (m_tempFile != null) + { m_tempFile.Dispose(); }