Merge branch 'feature/rework_buffer_avoidance'

Duplicati/Library/Backend/CloudFiles/CloudFiles.cs

@@ -346,8 +346,13 @@ namespace Duplicati.Library.Backend
             {
                 string fileHash = null;
 
+                long streamLen = -1;
+                try { streamLen = stream.Length; }
+                catch {}
+
+
                 Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
-                using (System.IO.Stream s = areq.GetRequestStream())
+                using (System.IO.Stream s = areq.GetRequestStream(streamLen))
                 using (var mds = new Utility.MD5CalculatingStream(s))
                 {
                     Utility.Utility.CopyStream(stream, mds, true, m_copybuffer);

Duplicati/Library/Backend/FTP/FTPBackend.cs

@@ -222,7 +222,7 @@ namespace Duplicati.Library.Backend
                 catch {}
 
                 Utility.AsyncHttpRequest areq = new Utility.AsyncHttpRequest(req);
-                using (System.IO.Stream rs = areq.GetRequestStream())
+                using (System.IO.Stream rs = areq.GetRequestStream(streamLen))
                     Utility.Utility.CopyStream(input, rs, true, m_copybuffer);
                 
                 if (m_listVerify) 

Duplicati/Library/Utility/AsyncHttpRequest.cs

@@ -95,8 +95,7 @@ namespace Duplicati.Library.Utility
                 ((HttpWebRequest)m_request).ReadWriteTimeout = System.Threading.Timeout.Infinite;
 
                 // Prevent in-memory buffering causing out-of-memory issues
-                ((HttpWebRequest)m_request).AllowReadStreamBuffering = false;
-                ((HttpWebRequest)m_request).AllowWriteStreamBuffering = false;
+                ((HttpWebRequest)m_request).AllowReadStreamBuffering = false;                    
             }
         }
 
@@ -106,7 +105,7 @@ namespace Duplicati.Library.Utility
         public WebRequest Request { get { return m_request; } }
 
         /// <summary>
-        /// Gets or sets the timeout used to guard the <see cref="GetRequestStream()"/> and <see cref="GetResponse()"/> calls
+        /// Gets or sets the timeout used to guard the <see cref="GetRequestStream(long)"/> and <see cref="GetResponse()"/> calls
         /// </summary>
         public int Timeout { get { return m_timeout; } set { m_timeout = value; } }
 
@@ -114,8 +113,13 @@ namespace Duplicati.Library.Utility
         /// Gets the request stream
         /// </summary>
         /// <returns>The request stream</returns>
-        public Stream GetRequestStream()
+        /// <param name="contentlength">The content length to use</param>
+        public Stream GetRequestStream(long contentlength = -1)
         {
+            // Prevent in-memory buffering causing out-of-memory issues
+            if (contentlength >= 0 || m_request.ContentLength >= 0)
+                ((HttpWebRequest)m_request).AllowWriteStreamBuffering = false;
+
             if (m_state == RequestStates.GetRequest)
                 return (Stream)m_asyncRequest.GetResponseOrStream();