Merge branch 'feature/add_throttle_ui'

# Conflicts:
#	Duplicati/Server/webroot/ngax/styles/style.css

Duplicati/Library/Main/BackendManager.cs

@@ -699,14 +699,33 @@ namespace Duplicati.Library.Main
                 }
             }
         }
-        
-        private void HandleProgress(long pg)
+
+        private string m_lastThrottleUploadValue = null;
+		private string m_lastThrottleDownloadValue = null;
+
+		private void HandleProgress(ThrottledStream ts, long pg)
         {
             // TODO: Should we pause here as well?
             // It might give annoying timeouts for transfers
             if (m_taskControl != null)
                 m_taskControl.TaskControlRendevouz();
 
+            // Update the throttle speeds if they have changed
+            string tmp;
+            m_options.RawOptions.TryGetValue("throttle-upload", out tmp);
+            if (tmp != m_lastThrottleUploadValue)
+            {
+                ts.WriteSpeed = m_options.MaxUploadPrSecond;
+                m_lastThrottleUploadValue = tmp;
+            }
+
+			m_options.RawOptions.TryGetValue("throttle-download", out tmp);
+            if (tmp != m_lastThrottleDownloadValue)
+            {
+                ts.ReadSpeed = m_options.MaxDownloadPrSecond;
+                m_lastThrottleDownloadValue = tmp;
+            }
+
             m_statwriter.BackendProgressUpdater.UpdateProgress(pg);
         }
 
@@ -734,8 +753,8 @@ namespace Duplicati.Library.Main
             if (m_backend is Library.Interface.IStreamingBackend && !m_options.DisableStreamingTransfers)
             {
                 using (var fs = System.IO.File.OpenRead(item.LocalFilename))
-                using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
-                using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
+                using (var ts = new ThrottledStream(fs, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
+                using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
                     ((Library.Interface.IStreamingBackend)m_backend).Put(item.RemoteFilename, pgs);
             }
             else
@@ -829,7 +848,7 @@ namespace Duplicati.Library.Main
                         using (var ss = new ShaderStream(nextTierWriter, false))
                         {
                             using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
-                            using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
+                            using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
                             {
                                 taskHasher.Start(); // We do not start tasks earlier to be sure the input always gets closed. 
                                 if (taskDecrypter != null) taskDecrypter.Start();
@@ -913,8 +932,8 @@ namespace Duplicati.Library.Main
                     using (var hs = GetFileHasherStream(fs, System.Security.Cryptography.CryptoStreamMode.Write, out getFileHash))
                     using (var ss = new ShaderStream(hs, true))
                     {
-                        using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
-                        using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, HandleProgress))
+                        using (var ts = new ThrottledStream(ss, m_options.MaxUploadPrSecond, m_options.MaxDownloadPrSecond))
+                        using (var pgs = new Library.Utility.ProgressReportingStream(ts, item.Size, pg => HandleProgress(ts, pg)))
                         { ((Library.Interface.IStreamingBackend)m_backend).Get(item.RemoteFilename, pgs); }
                         ss.Flush();
                         retDownloadSize = ss.TotalBytesWritten;

Duplicati/Library/Main/Controller.cs

@@ -1017,9 +1017,21 @@ namespace Duplicati.Library.Main
                 t.Abort();
         }
 
-        #region IDisposable Members
-
-        public void Dispose()
+        public long MaxUploadSpeed
+        {
+            get { return m_options.MaxUploadPrSecond; }
+            set { m_options.MaxUploadPrSecond = value; }
+        }
+
+		public long MaxDownloadSpeed
+		{
+			get { return m_options.MaxDownloadPrSecond; }
+			set { m_options.MaxDownloadPrSecond = value; }
+		}
+
+		#region IDisposable Members
+
+		public void Dispose()
         {
         }
 

Duplicati/Library/Utility/ThrottledStream.cs

@@ -66,12 +66,40 @@ namespace Duplicati.Library.Utility
 		/// <summary>
 		/// The current measured read speed
 		/// </summary>
-		private double m_current_write_speed;
-
+		private double m_current_write_speed;
+
 		/// <summary>
-        /// The number of ticks to have passed before a sample is taken
-        /// </summary>
-		private const long SAMPLE_PERIOD = TimeSpan.TicksPerSecond / 4;
+		/// The time the last read was sampled
+		/// </summary>
+		private DateTime m_last_limit_update;
+
+		/// <summary>
+		/// The number of ticks to have passed before a sample is taken
+		/// </summary>
+		private const long SAMPLE_PERIOD = TimeSpan.TicksPerSecond / 4;
+
+		/// <summary>
+		/// The number of ticks to have passed before the limits updater is called
+		/// </summary>
+		private const long UPDATE_PERIOD = TimeSpan.TicksPerSecond;
+
+		/// <summary>
+		/// Callback method used to update limits
+		/// </summary>
+		private readonly Action<ThrottledStream> m_updateLimits;
+
+		/// <summary>
+		/// Creates a throttle around a stream.
+		/// </summary>
+		/// <param name="basestream">The stream to throttle</param>
+		/// <param name="readspeed">The maximum number of bytes pr. second to read. Specify a number less than 1 to allow unlimited speed.</param>
+		/// <param name="writespeed">The maximum number of bytes pr. second to write. Specify a number less than 1 to allow unlimited speed.</param>
+        public ThrottledStream(Stream basestream, Action<ThrottledStream> updateLimits)
+			: base(basestream)
+		{
+            m_last_read_sample = m_last_write_sample = m_last_limit_update = new DateTime(0);
+            m_updateLimits = updateLimits;
+		}
 
         /// <summary>
         /// Creates a throttle around a stream.
@@ -85,7 +113,8 @@ namespace Duplicati.Library.Utility
             m_readspeed = readspeed;
             m_writespeed = writespeed;
 
-			m_last_read_sample = m_last_write_sample = new DateTime(0);
+			m_last_read_sample = m_last_write_sample = m_last_limit_update = new DateTime(0);
+            m_updateLimits = null;
         }
 
 		/// <summary>
@@ -99,9 +128,10 @@ namespace Duplicati.Library.Utility
 			var remaining = count;
 
 			while (remaining > 0)
-			{
-				// To avoid excessive waiting, the delay will wait at most 2 seconds,
-				// so we split the blocks to limit the number of seconds we can wait
+			{
+				// To avoid excessive waiting, the delay will wait at most 2 seconds,
+				// so we split the blocks to limit the number of seconds we can wait
+				UpdateLimits();
 				var chunksize = (int)Math.Min(remaining, m_readspeed <= 0 ? remaining : m_readspeed * 2);
 				DelayIfRequired(ref m_readspeed, chunksize, ref m_last_read_sample, ref m_current_read_counter, ref m_current_read_speed);
 
@@ -128,8 +158,9 @@ namespace Duplicati.Library.Utility
         {
 			while (count > 0)
 			{
-				// To avoid excessive waiting, the delay will wait at most 2 seconds,
-				// so we split the blocks to limit the number of seconds we can wait
+                // To avoid excessive waiting, the delay will wait at most 2 seconds,
+                // so we split the blocks to limit the number of seconds we can wait
+                UpdateLimits();
 				var chunksize = (int)Math.Min(count, m_writespeed <= 0 ? count : m_writespeed * 2);
 				DelayIfRequired(ref m_writespeed, chunksize, ref m_last_write_sample, ref m_current_write_counter, ref m_current_write_speed);
 				m_basestream.Write(buffer, offset, chunksize);
@@ -170,6 +201,19 @@ namespace Duplicati.Library.Utility
 		/// </summary>
 		public double MeasuredWriteSpeed { get { return m_current_write_speed; } }
 
+        /// <summary>
+        /// Helper method called to update the limits
+        /// </summary>
+        private void UpdateLimits()
+        {
+            var now = DateTime.Now;
+            if (m_updateLimits != null && now.Ticks - m_last_limit_update.Ticks > UPDATE_PERIOD)
+            {
+                m_updateLimits(this);
+                m_last_limit_update = now;
+            }
+        }
+
 		/// <summary>
 		/// Calculates the speed, and inserts appropriate delays
 		/// </summary>

Duplicati/Server/Database/ServerSettings.cs

@@ -90,12 +90,9 @@ namespace Duplicati.Server.Database
                         m_values.Remove(k.Key);
                     else
                         m_values[k.Key] = newsettings[k.Key];
-
-                SaveSettings();
-            }
-
-            System.Threading.Interlocked.Increment(ref Program.LastDataUpdateID);
-            Program.StatusEventNotifyer.SignalNewEvent();
+            }
+
+			SaveSettings();
         }
             
         private void SaveSettings()
@@ -106,10 +103,15 @@ namespace Duplicati.Server.Database
                     Filter = "",
                     Name = n.Key,
                     Value = n.Value
-            }, Database.Connection.SERVER_SETTINGS_ID);
-
-            // In case the usage reporter is enabled or disabled, refresh now
-            Program.StartOrStopUsageReporter();
+            }, Database.Connection.SERVER_SETTINGS_ID);
+
+			System.Threading.Interlocked.Increment(ref Program.LastDataUpdateID);
+			Program.StatusEventNotifyer.SignalNewEvent();
+
+			// In case the usage reporter is enabled or disabled, refresh now
+			Program.StartOrStopUsageReporter();
+            // If throttle options were changed, update now
+            Program.UpdateThrottleSpeeds();
         }
         
         public string StartupDelayDuration

Duplicati/Server/Program.cs

@@ -563,6 +563,16 @@ namespace Duplicati.Server
                 Library.UsageReporter.Reporter.SetReportLevel(reportLevel, disableUsageReporter);
         }
 
+        public static void UpdateThrottleSpeeds()
+        {
+            if (Program.WorkThread == null)
+                return;
+            
+            var cur = Program.WorkThread.CurrentTask;
+            if (cur != null)
+                cur.UpdateThrottleSpeed();
+        }
+
         private static void SignalNewEvent(object sender, EventArgs e)
         {
             StatusEventNotifyer.SignalNewEvent();

Duplicati/Server/Runner.cs

@@ -33,6 +33,7 @@ namespace Duplicati.Server
             void Abort();
             void Pause();
             void Resume();
+            void UpdateThrottleSpeed();
             void SetController(Duplicati.Library.Main.Controller controller);
         }
         
@@ -82,7 +83,49 @@ namespace Duplicati.Server
                 if (c != null)
                     c.Resume();
             }
-            
+
+            public long OriginalUploadSpeed { get; set; }
+            public long OriginalDownloadSpeed { get; set; }
+
+			public void UpdateThrottleSpeed()
+			{
+				var controller = this.Controller;
+				if (controller == null)
+					return;
+
+				var job_upload_throttle = this.OriginalUploadSpeed <= 0 ? long.MaxValue : this.OriginalUploadSpeed;
+				var job_download_throttle = this.OriginalDownloadSpeed <= 0 ? long.MaxValue : this.OriginalDownloadSpeed;
+
+				var server_upload_throttle = long.MaxValue;
+				var server_download_throttle = long.MaxValue;
+
+				try
+				{
+					if (!string.IsNullOrWhiteSpace(Program.DataConnection.ApplicationSettings.UploadSpeedLimit))
+						server_upload_throttle = Duplicati.Library.Utility.Sizeparser.ParseSize(Program.DataConnection.ApplicationSettings.UploadSpeedLimit, "kb");
+				}
+				catch { }
+
+				try
+				{
+					if (!string.IsNullOrWhiteSpace(Program.DataConnection.ApplicationSettings.DownloadSpeedLimit))
+						server_download_throttle = Duplicati.Library.Utility.Sizeparser.ParseSize(Program.DataConnection.ApplicationSettings.DownloadSpeedLimit, "kb");
+				}
+				catch { }
+
+				var upload_throttle = Math.Min(job_upload_throttle, server_upload_throttle);
+				var download_throttle = Math.Min(job_download_throttle, server_download_throttle);
+
+				if (upload_throttle <= 0 || upload_throttle == long.MaxValue)
+					upload_throttle = 0;
+
+				if (download_throttle <= 0 || download_throttle == long.MaxValue)
+					download_throttle = 0;
+
+				controller.MaxUploadSpeed = upload_throttle;
+				controller.MaxDownloadSpeed = download_throttle;
+			}
+
             private readonly long m_taskID;
             
             public RunnerData()
@@ -488,9 +531,24 @@ namespace Duplicati.Server
                 using(tempfolder)
                 using(var controller = new Duplicati.Library.Main.Controller(backup.TargetURL, options, sink))
                 {
-                    ((RunnerData)data).Controller = controller;
-                    
-                    switch (data.Operation)
+                    try 
+                    {
+                        if (options.ContainsKey("throttle-upload"))
+                            ((RunnerData)data).OriginalUploadSpeed = Duplicati.Library.Utility.Sizeparser.ParseSize(options["throttle-upload"], "kb");
+                    }
+                    catch { }
+
+					try
+					{
+						if (options.ContainsKey("throttle-download"))
+                            ((RunnerData)data).OriginalDownloadSpeed = Duplicati.Library.Utility.Sizeparser.ParseSize(options["throttle-download"], "kb");
+					}
+					catch { }
+
+					((RunnerData)data).Controller = controller;
+                    data.UpdateThrottleSpeed();
+
+					switch (data.Operation)
                     {
                         case DuplicatiOperation.Backup:
                             {
@@ -843,6 +901,8 @@ namespace Duplicati.Server
                 select (Duplicati.Library.Utility.IFilter)(new Duplicati.Library.Utility.FilterExpression(exp, n.Include)))
                 .Aggregate((a, b) => Duplicati.Library.Utility.FilterExpression.Combine(a, b));
         }
+
+
     }
 }
 

Duplicati/Server/WebServer/RESTMethods/ServerSettings.cs

@@ -77,23 +77,27 @@ namespace Duplicati.Server.WebServer.RESTMethods
 
                 serversettings.Remove("server-ssl-certificate");
 				serversettings.Remove("ServerSSLCertificate");
-
-				Program.DataConnection.ApplicationSettings.UpdateSettings(serversettings, false);
 
-                // Update based on inputs
-                var existing = Program.DataConnection.Settings.ToDictionary(x => x.Name, x => x);
-                foreach (var g in globalsettings)
-                    if (g.Value == null)
-                        existing.Remove(g.Key);
-                    else
-                    {
-                        if (existing.ContainsKey(g.Key))
-                            existing[g.Key].Value = g.Value;
+                if (serversettings.Any())
+				    Program.DataConnection.ApplicationSettings.UpdateSettings(serversettings, false);
+
+                if (globalsettings.Any())
+                {
+                    // Update based on inputs
+                    var existing = Program.DataConnection.Settings.ToDictionary(x => x.Name, x => x);
+                    foreach (var g in globalsettings)
+                        if (g.Value == null)
+                            existing.Remove(g.Key);
                         else
-                            existing[g.Key] = new Setting() { Name = g.Key, Value = g.Value };
-                    }
+                        {
+                            if (existing.ContainsKey(g.Key))
+                                existing[g.Key].Value = g.Value;
+                            else
+                                existing[g.Key] = new Setting() { Name = g.Key, Value = g.Value };
+                        }
 
-                Program.DataConnection.Settings = existing.Select(x => x.Value).ToArray();
+                    Program.DataConnection.Settings = existing.Select(x => x.Value).ToArray();
+                }
 
                 info.OutputOK();
             }

Duplicati/Server/webroot/ngax/index.html

@@ -94,6 +94,7 @@
     <script type="text/javascript" src="scripts/controllers/AddWizardController.js"></script>
     <script type="text/javascript" src="scripts/controllers/PauseController.js"></script>
     <script type="text/javascript" src="scripts/controllers/CommandlineController.js"></script>
+    <script type="text/javascript" src="scripts/controllers/ThrottleController.js"></script>
 
     <script type="text/javascript" src="scripts/filters/timeremaining.js"></script>
     <script type="text/javascript" src="scripts/filters/highlight.js"></script>
@@ -152,7 +153,7 @@
 
             <div class="action-icons-small">
                 <div class="pause" ng-class="{'active': state.programState != 'Running'}" ng-click="pauseOptions()" title="{{state.programState == 'Running' ? 'Click to see the pause options' : 'Click to resume'}}"></div>
-                <!-- <div class="throttle" ng-click="throttleOptions()"></div> -->
+                <div class="throttle {{throttle_active ? 'active' : 'inactive'}}" ng-click="throttleOptions()" title="{{'Click to set throttle options' | translate}}" alt="{{'Click to set throttle options' | translate}}"></div>
             </div>
 
             <div class="statepadding">
@@ -195,7 +196,7 @@
 
                 <div class="action-icons">
                     <div class="pause" ng-class="{'active': state.programState != 'Running'}" ng-click="pauseOptions()" title="{{state.programState == 'Running' ? 'Click to see the pause options' : 'Click to resume'}}"></div>
-                    <!--<div class="throttle" ng-click="throttleOptions()"></div> -->
+                    <div class="throttle {{throttle_active ? 'active' : 'inactive'}}" ng-click="throttleOptions()" title="{{'Click to set throttle options' | translate}}" alt="{{'Click to set throttle options' | translate}}"></div>
                 </div>
             </div>            
         </div>

Duplicati/Server/webroot/ngax/less/style.less

@@ -585,11 +585,11 @@ body {
                     height: 26px;
                     display: inline-block;
                     cursor: pointer;
-                    background: url('../img/throttle-inactive.png');
+                    background: url('../img/throttle.png');
                 }
 
-                > pause.active {
-                    background: url('../img/throttle.png');
+                > .throttle.inactive {
+                    opacity: 0.5;
                 }
             }
 
@@ -2083,6 +2083,40 @@ div.modal-dialog {
     }
 }
 
+.throttlesettings
+{
+    div.multiple {
+
+        select {
+            width: auto;
+            margin-right: 5px;
+        }
+
+        input {
+            width: 100px;
+        }
+
+        input.checkbox {
+            width: auto;
+        }
+
+        label {
+            line-height: 35px;
+            padding: 0 15px;
+            width: auto;
+            min-width: 150px;
+        }
+    }
+
+    .disabled {
+        color: lighten(@tColor, 25%);
+        input, select {
+            color: lighten(@tColor, 25%);
+        }
+    }
+
+}
+
 // Smaller screen sizes
 @media (max-width: 1200px) {
     body {
@@ -2177,7 +2211,7 @@ div.modal-dialog {
                 .content {
                     float: none;
                     padding: 20px 20px;
-                    margin: 0 auto;
+                    margin: 0px auto 30px auto;
 
                     .state {
                         width: auto;
@@ -2331,6 +2365,8 @@ div.modal-dialog {
                 padding-bottom: 10px;
 
                 .content {
+                    margin: 0px auto;
+
                     div.add,
                     div.restore {
                         form {

Duplicati/Server/webroot/ngax/scripts/controllers/AppController.js

@@ -6,6 +6,7 @@ backupApp.controller('AppController', function($scope, $cookies, $location, AppS
     $scope.localized = {};
     $scope.location = $location;
     $scope.saved_theme = $scope.active_theme = $cookies.get('current-theme') || 'default';
+    $scope.throttle_active = false;
 
     // If we want the theme settings
     // to be persisted on the server,
@@ -52,7 +53,23 @@ backupApp.controller('AppController', function($scope, $cookies, $location, AppS
     };
 
     $scope.throttleOptions = function() {
-        alert('Throttle options are not implemented yet');
+        DialogService.htmlDialog(
+            gettextCatalog.getString('Throttle settings'), 
+            'templates/throttle.html', 
+            [gettextCatalog.getString('OK'), gettextCatalog.getString('Cancel')], 
+            function(index, text, cur) {
+                if (index == 0 && cur != null && cur.uploadspeed != null && cur.downloadspeed != null) {
+                    var patchdata = {
+                        'max-download-speed': cur.downloadthrottleenabled ? cur.downloadspeed : '',
+                        'max-upload-speed': cur.uploadthrottleenabled ? cur.uploadspeed : '',
+                    };
+
+                    AppService.patch('/serversettings', patchdata, {headers: {'Content-Type': 'application/json; charset=utf-8'}}).then(function(data) {
+                        $scope.throttle_active = cur.downloadthrottleenabled || cur.uploadthrottleenabled;
+                    }, AppUtils.connectionError);
+                }
+            }
+        );
     };
 
     function updateCurrentPage() {
@@ -139,6 +156,12 @@ backupApp.controller('AppController', function($scope, $cookies, $location, AppS
             $scope.active_theme = $scope.saved_theme;
         else
             $scope.active_theme = args.theme || '';
-
     });
+
+    AppService.get('/serversettings').then(function(data) {
+        var ut = data.data['max-upload-speed'];
+        var dt = data.data['max-download-speed'];
+        $scope.throttle_active = (ut != null && ut.trim().length != 0) || (dt != null && dt.trim().length != 0);
+
+    }, AppUtils.connectionError);
 });

Duplicati/Server/webroot/ngax/scripts/controllers/ThrottleController.js

@@ -0,0 +1,22 @@
+backupApp.controller('ThrottleController', function($scope, AppService, ServerStatus, SystemInfo, AppUtils) {
+    $scope.speedMultipliers = AppUtils.speedMultipliers;
+
+    $scope.selection = $scope.$parent.state.CurrentItem;
+
+    AppService.get('/serversettings').then(function(data) {
+
+		$scope.selection.uploadspeed = data.data['max-upload-speed'];
+		$scope.selection.downloadspeed = data.data['max-download-speed'];
+
+		$scope.selection.uploadthrottleenabled = ($scope.selection.uploadspeed != '');
+		$scope.selection.downloadthrottleenabled = ($scope.selection.downloadspeed != '');
+
+		// Nicer looking UI
+		if (!$scope.selection.uploadthrottleenabled)
+			$scope.selection.uploadspeed = "10MB";
+		if (!$scope.selection.downloadthrottleenabled)
+			$scope.selection.downloadspeed = "10MB";
+
+    }, AppUtils.connectionError);
+
+});

Duplicati/Server/webroot/ngax/templates/throttle.html

@@ -0,0 +1,27 @@
+<div ng-controller="ThrottleController" class="throttlesettings">
+    <form class="styled">
+        <div class="input mixed multiple" ng-class="selection.uploadthrottleenabled ? '' : 'disabled'">
+            <input type="checkbox" class="checkbox" name="uploadthrottleenabled" ng-model="selection.uploadthrottleenabled">
+            <label ng-click="selection.uploadthrottleenabled = !selection.uploadthrottleenabled" for="uploadspeed" translate>Max upload speed</label>
+
+            <input ng-disabled="!selection.uploadthrottleenabled" type="number" id="uploadspeed" name="uploadspeed" parse-size-number="uppercase" ng-model="selection.uploadspeed" />
+            <select ng-disabled="!selection.uploadthrottleenabled" parse-size-multiplier="uppercase" ng-model="selection.uploadspeed" ng-options="item.value as item.name for item in speedMultipliers">
+            </select>
+        </div>
+
+        <div class="input mixed multiple" ng-class="selection.downloadthrottleenabled ? '' : 'disabled'">
+            <input type="checkbox" class="checkbox" name="downloadthrottleenabled" ng-model="selection.downloadthrottleenabled">
+
+            <label ng-click="selection.downloadthrottleenabled = !selection.downloadthrottleenabled" for="downloadspeed" translate>Max download speed</label>
+
+            <input ng-disabled="!selection.downloadthrottleenabled" type="number" id="downloadspeed" name="downloadspeed" parse-size-number="uppercase" ng-model="selection.downloadspeed" />
+            <select ng-disabled="!selection.downloadthrottleenabled" parse-size-multiplier="uppercase" ng-model="selection.downloadspeed" ng-options="item.value as item.name for item in speedMultipliers">
+            </select>
+        </div>
+
+        <div>
+        Note that speeds are entered in bytes, and line speeds are typically reported in bits. Use a factor of 8 to convert, such that an 8 mbit/s line is equivalent to 1 MByte/s.
+        <div>
+	</form>
+
+</div>