From 7e7aadc10f10356097f7ef873c41989a374ade42 Mon Sep 17 00:00:00 2001
From: gpatel-fr <44170243+gpatel-fr@users.noreply.github.com>
Date: Sun, 7 May 2023 00:09:23 +0200
Subject: [PATCH] try to work around problems in downloading test files by
 retrying set maximum output for the Linux and Mac tests

---
 .github/workflows/tests.yml                   |  1 +
 .../UnitTest/CommandLineOperationsTests.cs    | 26 ++++++++++++--
 Duplicati/UnitTest/GeneralBlackBoxTesting.cs  | 36 +++++++++++++++++--
 3 files changed, 58 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 6de22f105..67a464737 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -43,6 +43,7 @@ jobs:
     - name: Run Unit Tests (Linux and macOS)
       if: matrix.os != 'windows-latest'
       run: |
+        export DEBUG_OUTPUT=1
         nuget install NUnit.Runners -Version 3.12.0 -OutputDirectory nunit
         mono nunit/NUnit.ConsoleRunner.3.12.0/tools/nunit3-console.exe Duplicati/UnitTest/bin/Release/Duplicati.UnitTest.dll
 
diff --git a/Duplicati/UnitTest/CommandLineOperationsTests.cs b/Duplicati/UnitTest/CommandLineOperationsTests.cs
index 695221ea9..9e28b1bd8 100644
--- a/Duplicati/UnitTest/CommandLineOperationsTests.cs
+++ b/Duplicati/UnitTest/CommandLineOperationsTests.cs
@@ -82,23 +82,43 @@ namespace Duplicati.UnitTest
             try
             {
                 // check if the file should be downloaded, exception if not
-                using ((HttpWebResponse)webRequest.GetResponse())
+                using (var wr = (HttpWebResponse)webRequest.GetResponse())
 
                 using (WebClient client = new WebClient())
-                {
-                    client.DownloadFile(url, destinationFilePath);
+                { // try to workaround a weird intermittent bug where downloaded size
+                  // differs from real size but no error is thrown
+                    Console.WriteLine("downloading test file to: {0}, length: {1}", destinationFilePath, wr.ContentLength);
+                    var maxAttempts = 5;
+                    while (maxAttempts-- > 0) {
+                        DateTime beginTime = DateTime.Now;
+                        client.DownloadFile(url, destinationFilePath);
+                        long length = new System.IO.FileInfo(destinationFilePath).Length;
+                        Console.WriteLine("downloaded test file: {0}: length {1}, duration {2}", destinationFilePath, length, (DateTime.Now - beginTime).TotalSeconds);
+                        if (length == wr.ContentLength) {
+                            maxAttempts = -1;
+                        } else {
+                            Console.WriteLine("invalid downloaded length {0}, should be {1}...", length, wr.ContentLength);
+                            System.Threading.Thread.Sleep(120000);
+                        }
+                    }
+                    if (maxAttempts == 0) {
+                        throw new Exception(string.Format("Unable to download test file from {0}", url));
+                    }
                 }
             }
             catch (WebException ex)
             {
                 if (ex.Response == null){
+                    Console.WriteLine("ex.Response is null !");
                     throw;
                 }
 
                 if (((HttpWebResponse) ex.Response).StatusCode != HttpStatusCode.NotModified)
                 {
+                    Console.WriteLine("file to download {0} already exists and is up to date", destinationFilePath);
                     throw;
                 }
+                Console.WriteLine("download exception: {0}", ex.Response);
             }
         }
 
diff --git a/Duplicati/UnitTest/GeneralBlackBoxTesting.cs b/Duplicati/UnitTest/GeneralBlackBoxTesting.cs
index 50a955957..7b0065dae 100644
--- a/Duplicati/UnitTest/GeneralBlackBoxTesting.cs
+++ b/Duplicati/UnitTest/GeneralBlackBoxTesting.cs
@@ -65,10 +65,42 @@ namespace Duplicati.UnitTest
         public override void OneTimeSetUp()
         {
             base.OneTimeSetUp();
-            
+            var url = $"https://testfiles.duplicati.com/{this.zipFilename}";
+            var destinationFilePath = this.zipFilepath;
+            var webRequest = (HttpWebRequest)WebRequest.Create(url);
+
+            using (var wr = (HttpWebResponse)webRequest.GetResponse())
             using (WebClient client = new WebClient())
             {
-                client.DownloadFile($"https://testfiles.duplicati.com/{this.zipFilename}", this.zipFilepath);
+                Console.WriteLine("downloading test file to: {0}, length: {1}", destinationFilePath, wr.ContentLength);
+                var maxAttempts = 5;
+                while (maxAttempts-- > 0) {
+                    try {
+                        DateTime beginTime = DateTime.Now;
+                        client.DownloadFile(url, destinationFilePath);
+                        long length = new System.IO.FileInfo(destinationFilePath).Length;
+                        Console.WriteLine("downloaded test file: {0}: length {1}, duration {2}", destinationFilePath, length, (DateTime.Now - beginTime).TotalSeconds);
+                        if (length == wr.ContentLength) {
+                            maxAttempts = -1;
+                        } else {
+                            Console.WriteLine("invalid downloaded length {0}, should be {1}...", length, wr.ContentLength);
+                            System.Threading.Thread.Sleep(120000);
+                        }
+                    }
+                    catch (WebException ex)
+                    {
+                        if (ex.Response == null){
+                            Console.WriteLine("ex.Response is null !");
+                        } else {
+                            Console.WriteLine("exception {0}", ex);
+                            throw;
+                        }
+                    }
+                }
+
+                if (maxAttempts == 0) {
+                    throw new Exception(string.Format("Unable to download test file from {0}", url));
+                }
             }
             
             System.IO.Compression.ZipFile.ExtractToDirectory(this.zipFilepath, BASEFOLDER);