haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feature/improve ci testing (#5781)

Browse Source 
* Added options to insert delays in backend tester

* Added wait delays to Azure and GoogleDrive to see it the CI is more stable with small delays

* Added more debug output to the backend tester

* Fixed dropbox reporting OAuth quota errors, when it is likely Dropbox API quota errors

* Implemented exception detection on AzureBlob backend

* Also showing sleep times in output

* Fixed a bug with the dropbox quota response

* Fixed error message to report correct number of files

* Fixed suppressed warning
This commit is contained in:
Kenneth Skovhede
2024-12-20 16:10:08 +01:00
committed by GitHub
parent 68627c06bf
commit 447c2b3238
6 changed files with 68 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Duplicati/CommandLine/BackendTester/Program.cs
+28 -9
View File
@@ -33,13 +33,6 @@ namespace Duplicati.CommandLine.BackendTester
{
public class Program
{
/// <summary>
/// Used to maintain a reference to initialized system settings.
/// </summary>
#pragma warning disable CS0414 // The private field `Duplicati.CommandLine.BackendTester.Program.SystemSettings' is assigned but its value is never used
private static IDisposable SystemSettings;
#pragma warning restore CS0414 // The private field `Duplicati.CommandLine.BackendTester.Program.SystemSettings' is assigned but its value is never used
class TempFile
{
public readonly string remotefilename;
@@ -110,7 +103,7 @@ namespace Duplicati.CommandLine.BackendTester
if (options.ContainsKey("tempdir") && !string.IsNullOrEmpty(options["tempdir"]))
Library.Utility.SystemContextSettings.DefaultTempPath = options["tempdir"];
SystemSettings = Duplicati.Library.Utility.SystemContextSettings.StartSession();
using var SystemSettings = Duplicati.Library.Utility.SystemContextSettings.StartSession();
if (!options.ContainsKey("auth_password") && !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("AUTH_PASSWORD")))
options["auth_password"] = System.Environment.GetEnvironmentVariable("AUTH_PASSWORD");
@@ -211,7 +204,12 @@ namespace Duplicati.CommandLine.BackendTester
else
Console.WriteLine("Specify the --force flag to actually delete files");
Console.WriteLine("*** Remote folder is not empty, aborting");
var fileCount = curlist.Where(x => !x.IsFolder).Count();
var filenames = curlist.Where(x => !x.IsFolder).Select(x => x.Name).Take(10).ToList();
Console.WriteLine($"*** Remote folder contains {fileCount} file(s), aborting");
Console.WriteLine($"*** First {filenames.Count} file(s): {Environment.NewLine}{string.Join(Environment.NewLine, filenames)}");
if (fileCount > filenames.Count)
Console.WriteLine($"*** ... and {fileCount - filenames.Count} more file(s)");
return false;
}
@@ -224,6 +222,8 @@ namespace Duplicati.CommandLine.BackendTester
var disableStreaming = Library.Utility.Utility.ParseBoolOption(options, "disable-streaming-transfers");
var skipOverwriteTest = Library.Utility.Utility.ParseBoolOption(options, "skip-overwrite-test");
var trimFilenameSpaces = Library.Utility.Utility.ParseBoolOption(options, "trim-filename-spaces");
var waitAfterUpload = TimeSpan.Zero;
var waitAfterDelete = TimeSpan.Zero;
var throttleUpload = 0L;
if (options.TryGetValue("throttle-upload", out var throttleUploadString))
@@ -270,6 +270,11 @@ namespace Duplicati.CommandLine.BackendTester
if (options.ContainsKey("max-filename-length"))
max_filename_size = int.Parse(options["max-filename-length"]);
if (options.ContainsKey("wait-after-upload"))
waitAfterUpload = Timeparser.ParseTimeSpan(options["wait-after-upload"]);
if (options.ContainsKey("wait-after-delete"))
waitAfterDelete = Timeparser.ParseTimeSpan(options["wait-after-delete"]);
var rnd = new Random();
var sha = System.Security.Cryptography.SHA256.Create();
@@ -327,6 +332,12 @@ namespace Duplicati.CommandLine.BackendTester
files[renameIndex] = new TempFile(renamedFileNewName, originalRenamedFile.localfilename, originalRenamedFile.hash, originalRenamedFile.length);
}
if (waitAfterUpload > TimeSpan.Zero)
{
Console.WriteLine("Waiting {0} after upload", waitAfterUpload);
Thread.Sleep(waitAfterUpload);
}
Console.WriteLine("Verifying file list ...");
curlist = backend.List();
@@ -421,6 +432,12 @@ namespace Duplicati.CommandLine.BackendTester
Console.WriteLine("*** Failed to delete file {0}, message: {1}", tx.remotefilename, ex);
}
if (waitAfterDelete > TimeSpan.Zero)
{
Console.WriteLine("Waiting {0} after delete", waitAfterDelete);
Thread.Sleep(waitAfterDelete);
}
curlist = backend.List();
foreach (var fe in curlist)
if (!fe.IsFolder)
@@ -612,6 +629,8 @@ namespace Duplicati.CommandLine.BackendTester
new CommandLineArgument("skip-overwrite-test", CommandLineArgument.ArgumentType.Boolean, "Bypass the overwrite test", "A value that indicates if dummy files should be uploaded prior to uploading the real files", "false"),
new CommandLineArgument("auto-clean", CommandLineArgument.ArgumentType.Boolean, "Remove any files found in target folder", "A value that indicates if all files in the target folder should be deleted before starting the first test", "false"),
new CommandLineArgument("force", CommandLineArgument.ArgumentType.Boolean, "Activate file deletion", "A value that indicates if existing files should really be deleted when using auto-clean", "false"),
new CommandLineArgument("wait-after-upload", CommandLineArgument.ArgumentType.Timespan, "Wait after all uploads", "A value that indicates how long to wait after all files are uploaded, to account for the backends eventual consistency", "0s"),
new CommandLineArgument("wait-after-delete", CommandLineArgument.ArgumentType.Timespan, "Wait after all deletes", "A value that indicates how long to wait after each delete operation, to account for the backends eventual consistency", "0s"),
});
}
}
Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs
+28 -7
View File
@@ -20,6 +20,7 @@
// DEALINGS IN THE SOFTWARE.
using Duplicati.Library.Interface;
using Microsoft.Azure.Storage.Shared.Protocol;
using System.Collections.Generic;
using System.IO;
using System.Threading;
@@ -111,9 +112,9 @@ namespace Duplicati.Library.Backend.AzureBlob
}
}
public async Task PutAsync(string remotename, Stream input, CancellationToken cancelToken)
public Task PutAsync(string remotename, Stream input, CancellationToken cancelToken)
{
await _azureBlob.AddFileStream(remotename, input, cancelToken);
return WrapWithExceptionHandler(_azureBlob.AddFileStream(remotename, input, cancelToken));
}
public async Task GetAsync(string remotename, string localname, CancellationToken cancellationToken)
@@ -128,12 +129,12 @@ namespace Duplicati.Library.Backend.AzureBlob
public Task GetAsync(string remotename, Stream output, CancellationToken cancellationToken)
{
return _azureBlob.GetFileStreamAsync(remotename, output, cancellationToken);
return WrapWithExceptionHandler(_azureBlob.GetFileStreamAsync(remotename, output, cancellationToken));
}
public Task DeleteAsync(string remotename, CancellationToken cancellationToken)
{
return _azureBlob.DeleteObjectAsync(remotename, cancellationToken);
return WrapWithExceptionHandler(_azureBlob.DeleteObjectAsync(remotename, cancellationToken));
}
public IList<ICommandLineArgument> SupportedCommands
@@ -197,13 +198,33 @@ namespace Duplicati.Library.Backend.AzureBlob
public Task TestAsync(CancellationToken cancellationToken)
{
this.TestList();
return Task.CompletedTask;
return WrapWithExceptionHandler(Task.Run(() => this.TestList()));
}
public Task CreateFolderAsync(CancellationToken cancellationToken)
{
return _azureBlob.AddContainerAsync(cancellationToken);
return WrapWithExceptionHandler(_azureBlob.AddContainerAsync(cancellationToken));
}
private async Task WrapWithExceptionHandler(Task task)
{
try
{
await task.ConfigureAwait(false);
}
catch (Microsoft.WindowsAzure.Storage.StorageException e)
when (e.RequestInformation.HttpStatusCode == 404
|| e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ResourceNotFound)
{
throw new FileMissingException(e.Message, e);
}
catch (Microsoft.WindowsAzure.Storage.StorageException e)
when (e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerNotFound
|| e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerBeingDeleted
|| e.RequestInformation.HttpStatusMessage == StorageErrorCodeStrings.ContainerDisabled)
{
throw new FolderMissingException(e.Message, e);
}
}
public void Dispose()
Duplicati/Library/Backend/Dropbox/DropboxHelper.cs
+2 -1
View File
@@ -259,8 +259,9 @@ namespace Duplicati.Library.Backend
}
if (httpResp.StatusCode == HttpStatusCode.Unauthorized)
ThrowAuthException(json, exception);
if ((int)httpResp.StatusCode == 429 || (int)httpResp.StatusCode == 507)
ThrowOverQuotaError();
throw new Duplicati.Library.Interface.UserInformationException(Strings.Dropbox.OverQuotaError(string.IsNullOrWhiteSpace(json) ? exception.Message : json), "DropboxOverQuotaError", ex);
}
Duplicati/Library/Backend/Dropbox/Strings.cs
+1
View File
@@ -28,5 +28,6 @@ namespace Duplicati.Library.Backend.Strings
public static string DisplayName { get { return LC.L(@"Dropbox"); } }
public static string AuthidLong(string url) { return LC.L(@"The authorization token retrieved from {0}", url); }
public static string AuthidShort { get { return LC.L(@"The authorization code"); } }
public static string OverQuotaError(string message) { return LC.L(@"The Dropbox account is over quota: {0}", message); }
}
}
LiveTests/Duplicati.Backend.Tests/Azure/AzureTests.cs
+5 -3
View File
@@ -27,8 +27,10 @@ namespace Duplicati.Backend.Tests.Azure;
[TestClass]
public sealed class AzureTests : BaseTest
{
static readonly string[] AdditionalArguments = ["--wait-after-upload=10s", "--wait-after-delete=10s"];
/// <summary>
/// Basic Dropbox test. There are no adicional parameters to be set or tested.
/// Basic Azure test.
/// </summary>
[TestMethod]
public Task TestAzureBlob()
@@ -40,11 +42,11 @@ public sealed class AzureTests : BaseTest
new[]
{
$"azure://{Environment.GetEnvironmentVariable("TESTCREDENTIAL_AZURE_CONTAINERNAME")}?auth-username={Environment.GetEnvironmentVariable("TESTCREDENTIAL_AZURE_ACCOUNTNAME")}&auth-password={Uri.EscapeDataString(Environment.GetEnvironmentVariable("TESTCREDENTIAL_AZURE_ACCESSKEY")!)}"
}.Concat(Parameters.GlobalTestParameters).ToArray());
}.Concat(Parameters.GlobalTestParameters).Concat(AdditionalArguments).ToArray());
if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details");
return Task.CompletedTask;
}
}
LiveTests/Duplicati.Backend.Tests/Google Drive/GoogleDriveTests.cs
+4 -3
View File
@@ -28,17 +28,18 @@ namespace Duplicati.Backend.Tests.Google_Drive;
public sealed class GoogleDriveTests : BaseTest
{
static readonly string[] AdditionalArguments = ["--wait-after-upload=10s", "--wait-after-delete=10s"];
[TestMethod]
public Task TestGoogleDriveSimple()
{
CheckRequiredEnvironment(["TESTCREDENTIAL_GOOGLEDRIVE_TOKEN","TESTCREDENTIAL_GOOGLEDRIVE_FOLDER"]);
CheckRequiredEnvironment(["TESTCREDENTIAL_GOOGLEDRIVE_TOKEN", "TESTCREDENTIAL_GOOGLEDRIVE_FOLDER"]);
var exitCode = CommandLine.BackendTester.Program.Main(
new[]
{
$"googledrive:///{Environment.GetEnvironmentVariable("TESTCREDENTIAL_GOOGLEDRIVE_FOLDER")}?authid={Environment.GetEnvironmentVariable("TESTCREDENTIAL_GOOGLEDRIVE_TOKEN")}",
}.Concat(Parameters.GlobalTestParameters).ToArray());
}.Concat(Parameters.GlobalTestParameters).Concat(AdditionalArguments).ToArray());
if (exitCode != 0) Assert.Fail("BackendTester is returning non-zero exit code, check logs for details");