haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/barfoo4711/duplicati into master

Browse Source
This commit is contained in:
Kai
2020-10-03 14:20:08 +02:00
parent 8e3a52654b f6e225d62d
commit 8b1d2cfafa
37 changed files with 388 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+2 -2
View File
@@ -57,13 +57,13 @@ jobs:
directories:
- $BUILD_DIR
script:
- ${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories SVNDataLong,SVNData,RecoveryTool,ProblematicPath
- ${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories SVNDataLong,SVNData,RecoveryTool,ProblematicPath,SymLink
- stage: tests
cache:
directories:
- $BUILD_DIR
script:
- ${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories Border
- travis_wait ${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories Border
- stage: tests
cache:
directories:
Duplicati/CommandLine/BackendTester/app.config
+1
View File
@@ -4,6 +4,7 @@
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.1" />
</startup>
<runtime>
<AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
Duplicati/CommandLine/BackendTool/app.config
+1
View File
@@ -4,6 +4,7 @@
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.1" />
</startup>
<runtime>
<AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="System.Net.Http" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
Duplicati/CommandLine/ConfigurationImporter/app.config
+1
View File
@@ -4,6 +4,7 @@
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.1" />
</startup>
<runtime>
<AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="System.Net.Http" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
Duplicati/CommandLine/app.config
+1
View File
@@ -4,6 +4,7 @@
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.1" />
</startup>
<runtime>
<AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
Duplicati/GUI/Duplicati.GUI.TrayIcon/app.config
+1
View File
@@ -2,6 +2,7 @@
<configuration>
<startup useLegacyV2RuntimeActivationPolicy="true"><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.1" /></startup>
<runtime>
<AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
Duplicati/Library/AutoUpdater/app.config
+3
View File
@@ -3,4 +3,7 @@
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.1"/>
</startup>
<runtime>
<AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false" />
</runtime>
</configuration>
Duplicati/Library/Backend/S3/S3Backend.cs
+1
View File
@@ -51,6 +51,7 @@ namespace Duplicati.Library.Backend
{ "dinCloud - Chicago", "d3-ord.dincloud.com" },
{ "dinCloud - Los Angeles", "d3-lax.dincloud.com" },
{ "IBM COS (S3) Public US", "s3-api.us-geo.objectstorage.softlayer.net" },
{ "Storadera", "eu-east-1.s3.storadera.com" },
{ "Wasabi Hot Storage", "s3.wasabisys.com" },
{ "Wasabi Hot Storage (US West)", "s3.us-west-1.wasabisys.com" },
{ "Wasabi Hot Storage (EU Central)", "s3.eu-central-1.wasabisys.com" },
Duplicati/Library/Main/Operation/DeleteHandler.cs
-1
View File
@@ -281,7 +281,6 @@ namespace Duplicati.Library.Main.Operation
/// <summary>
/// Remove backups according to the --retention-policy option.
/// Backups that are not within any of the specified time frames will will NOT be deleted.
/// Partial backups are not removed.
/// </summary>
public class RetentionPolicyRemover : FilesetRemover
Duplicati/Library/Snapshots/app.config
+1
View File
@@ -4,6 +4,7 @@
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.1"/>
</startup>
<runtime>
<AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="AlphaVSS.Common" publicKeyToken="959d3993561034e3" culture="neutral"/>
Duplicati/Library/Utility/Uri.cs
+21 -2
View File
@@ -19,7 +19,6 @@ using System;
using System.Collections.Specialized;
using System.Linq;
using System.Text;
using System.Web;
namespace Duplicati.Library.Utility
{
@@ -445,6 +444,18 @@ namespace Duplicati.Library.Utility
/// <returns>The parsed query string</returns>
/// <param name="query">The query to parse</param>
public static NameValueCollection ParseQueryString(string query)
{
return Library.Utility.Uri.ParseQueryString(query, true);
}
/// <summary>
/// Parses the query string.
/// This is a duplicate of the System.Web.HttpUtility.ParseQueryString that does not work well on Mono
/// </summary>
/// <returns>The parsed query string</returns>
/// <param name="query">The query to parse</param>
/// <param name="decodeValues">Whether to the parameter values should be decoded or not.</param>
public static NameValueCollection ParseQueryString(string query, bool decodeValues)
{
if (query == null)
throw new ArgumentNullException(nameof(query));
@@ -455,7 +466,15 @@ namespace Duplicati.Library.Utility
var result = new NameValueCollection(StringComparer.OrdinalIgnoreCase);
foreach (System.Text.RegularExpressions.Match m in RE_URLPARAM.Matches(query))
result.Add(UrlDecode(m.Groups["key"].Value), UrlDecode(m.Groups["value"].Success ? m.Groups["value"].Value : ""));
{
string value = m.Groups["value"].Success ? m.Groups["value"].Value : "";
if (decodeValues)
{
value = UrlDecode(value);
}
result.Add(UrlDecode(m.Groups["key"].Value), value);
}
return result;
}
Duplicati/Server/Database/Backup.cs
+13 -4
View File
@@ -18,6 +18,7 @@
using System;
using Duplicati.Server.Serialization.Interface;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
namespace Duplicati.Server.Database
@@ -121,11 +122,19 @@ namespace Duplicati.Server.Database
public void SanitizeTargetUrl()
{
var url = new Duplicati.Library.Utility.Uri(this.TargetURL);
var filteredParameters = url.QueryParameters;
foreach (string field in UrlPasswords) {
filteredParameters.Remove(field);
NameValueCollection filteredParameters = new NameValueCollection();
if (url.Query != null)
{
// We cannot use url.QueryParameters since it contains decoded parameter values, which
// breaks assumptions made by the decode_uri function in AppUtils.js. Since we are simply
// removing password parameters, we will leave the parameters as they are in the target URL.
filteredParameters = Library.Utility.Uri.ParseQueryString(url.Query, false);
foreach (string field in this.UrlPasswords)
{
filteredParameters.Remove(field);
}
}
url = url.SetQuery(Duplicati.Library.Utility.Uri.BuildUriQuery(url.QueryParameters));
url = url.SetQuery(Duplicati.Library.Utility.Uri.BuildUriQuery(filteredParameters));
this.TargetURL = url.ToString();
}
Duplicati/Server/WebServer/RESTMethods/Backup.cs
+7 -2
View File
@@ -122,6 +122,12 @@ namespace Duplicati.Server.WebServer.RESTMethods
info.OutputOK(new { inuse = Library.Main.DatabaseLocator.IsDatabasePathInUse(backup.DBPath) });
}
public static void RemovePasswords(IBackup backup)
{
backup.SanitizeSettings();
backup.SanitizeTargetUrl();
}
private void Export(IBackup backup, RequestInfo info)
{
var cmdline = Library.Utility.Utility.ParseBool(info.Request.QueryString["cmdline"].Value, false);
@@ -129,8 +135,7 @@ namespace Duplicati.Server.WebServer.RESTMethods
var exportPasswords = Library.Utility.Utility.ParseBool(info.Request.QueryString["export-passwords"].Value, false);
if (!exportPasswords)
{
backup.SanitizeSettings();
backup.SanitizeTargetUrl();
Backup.RemovePasswords(backup);
}
if (cmdline)
Duplicati/Server/app.config
+1
View File
@@ -2,6 +2,7 @@
<configuration>
<startup useLegacyV2RuntimeActivationPolicy="true"><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.1" /></startup>
<runtime>
<AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
Duplicati/Server/webroot/ngax/less/style.less
+1 -1
View File
@@ -2525,13 +2525,13 @@ div.modal-dialog {
background-image: url(../img/loader-16.gif);
background-repeat: no-repeat;
background-position: 0px 0px;
background-size: 16px 16px;
@media
only screen and (-webkit-min-device-pixel-ratio: 1.25),
only screen and (min-resolution: 192dpi),
only screen and (min-resolution: 1.25dppx) {
background-image: url('../img/loader-32.gif');
background-size: 16px 16px;
}
}
Duplicati/Server/webroot/ngax/scripts/directives/backupEditUri.js
+1 -1
View File
@@ -209,7 +209,7 @@ backupApp.directive('backupEditUri', function(gettextCatalog) {
var message = ((prev || '').trim().length == 0) ?
(gettextCatalog.getString('No certificate was specified previously, please verify with the server administrator that the key is correct: {{key}} \n\nDo you want to approve the reported host key?', { key: key }))
:
(gettextCatalog.getString('The host key has changed, please check with the server administrator if this is correct, otherwise you could be the victim of a MAN-IN-THE-MIDDLE attack.\n\nDo you want to REPLACE your CURRENT host key "{{prev}}" with the REPORTED host key: {{key}}?', { prev: pref, key: key }));
(gettextCatalog.getString('The host key has changed, please check with the server administrator if this is correct, otherwise you could be the victim of a MAN-IN-THE-MIDDLE attack.\n\nDo you want to REPLACE your CURRENT host key "{{prev}}" with the REPORTED host key: {{key}}?', { prev: prev, key: key }));
DialogService.dialog(gettextCatalog.getString('Trust host certificate?'), message, [gettextCatalog.getString('No'), gettextCatalog.getString('Yes')], function(ix) {
if (ix == 1) {
Duplicati/Server/webroot/ngax/styles/style.css
+1 -1
View File
File diff suppressed because one or more lines are too long
Duplicati/Service/app.config
+1
View File
@@ -4,6 +4,7 @@
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.1" />
</startup>
<runtime>
<AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
Duplicati/UnitTest/BasicSetupHelper.cs
+1
View File
@@ -21,6 +21,7 @@ using System.Collections.Generic;
using System.IO.Compression;
using Duplicati.Library.Common;
using Duplicati.Library.Common.IO;
using Duplicati.Library.Utility;
namespace Duplicati.UnitTest
{
Duplicati/UnitTest/BorderTests.cs
+1 -1
View File
@@ -353,7 +353,7 @@ namespace Duplicati.UnitTest
Assert.AreEqual(filenames.Count * 3, r.RestoredFiles);
}
TestUtils.VerifyDir(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"));
TestUtils.AssertDirectoryTreesAreEquivalent(DATAFOLDER, RESTOREFOLDER, !Library.Utility.Utility.ParseBoolOption(testopts, "skip-metadata"), "Restore");
using(var tf = new Library.Utility.TempFolder())
{
Duplicati/UnitTest/CommandLineOperationsTests.cs
+5 -5
View File
@@ -215,8 +215,8 @@ namespace Duplicati.UnitTest
Duplicati.CommandLine.Program.RealMain((new string[] { "restore", target, rf + "*", "--restore-path=\"" + RESTOREFOLDER + "\"" }.Union(opts)).ToArray());
ProgressWriteLine("Verifying partial restore ...");
using (new Library.Logging.Timer(LOGTAG, "VerifiationOfPartialRestore", "Verification of partial restored files"))
TestUtils.VerifyDir(rf, RESTOREFOLDER, true);
using (new Library.Logging.Timer(LOGTAG, "VerificationOfPartialRestore", "Verification of partial restored files"))
TestUtils.AssertDirectoryTreesAreEquivalent(rf, RESTOREFOLDER, true, "VerificationOfPartialRestore");
systemIO.DirectoryDelete(RESTOREFOLDER, true);
@@ -226,7 +226,7 @@ namespace Duplicati.UnitTest
ProgressWriteLine("Verifying partial restore ...");
using (new Library.Logging.Timer(LOGTAG, "VerificationOfPartialRestore", "Verification of partial restored files"))
TestUtils.VerifyDir(rf, RESTOREFOLDER, true);
TestUtils.AssertDirectoryTreesAreEquivalent(rf, RESTOREFOLDER, true, "VerificationOfPartialRestore");
systemIO.DirectoryDelete(RESTOREFOLDER, true);
@@ -237,7 +237,7 @@ namespace Duplicati.UnitTest
ProgressWriteLine("Verifying full restore ...");
using (new Library.Logging.Timer(LOGTAG, "VerificationOfFullRestore", "Verification of restored files"))
foreach (var s in systemIO.EnumerateDirectories(DATAFOLDER))
TestUtils.VerifyDir(s, Path.Combine(RESTOREFOLDER, Path.GetFileName(s)), true);
TestUtils.AssertDirectoryTreesAreEquivalent(s, Path.Combine(RESTOREFOLDER, Path.GetFileName(s)), true, "VerificationOfFullRestore");
systemIO.DirectoryDelete(RESTOREFOLDER, true);
@@ -248,7 +248,7 @@ namespace Duplicati.UnitTest
ProgressWriteLine("Verifying full restore ...");
using (new Library.Logging.Timer(LOGTAG, "VerificationOfFullRestoreWithoutDb", "Verification of restored files"))
foreach (var s in systemIO.EnumerateDirectories(DATAFOLDER))
TestUtils.VerifyDir(s, Path.Combine(RESTOREFOLDER, Path.GetFileName(s)), true);
TestUtils.AssertDirectoryTreesAreEquivalent(s, Path.Combine(RESTOREFOLDER, Path.GetFileName(s)), true, "VerificationOfFullRestoreWithoutDb");
ProgressWriteLine("Testing data ...");
using (new Library.Logging.Timer(LOGTAG, "TestRemoteData", "Test remote data"))
Duplicati/UnitTest/DisruptionTests.cs
+3 -3
View File
@@ -527,7 +527,7 @@ namespace Duplicati.UnitTest
foreach (string filepath in partialVersionFiles)
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), filename, false));
TestUtils.AssertFilesAreEqual(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), false, filename);
}
}
@@ -572,7 +572,7 @@ namespace Duplicati.UnitTest
foreach (string filepath in fullVersionFiles)
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), filename, false));
TestUtils.AssertFilesAreEqual(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), false, filename);
}
}
}
@@ -639,7 +639,7 @@ namespace Duplicati.UnitTest
foreach (string filepath in fullVersionFiles)
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), filename, false));
TestUtils.AssertFilesAreEqual(filepath, Path.Combine(this.RESTOREFOLDER, filename ?? String.Empty), false, filename);
}
}
}
Duplicati/UnitTest/Duplicati.UnitTest.csproj
+2
View File
@@ -42,8 +42,10 @@
<Reference Include="System" />
<Reference Include="System.IO.Compression" />
<Reference Include="System.IO.Compression.FileSystem" />
<Reference Include="System.Web" />
</ItemGroup>
<ItemGroup>
<Compile Include="SymLinkTests.cs" />
<Compile Include="ControllerTests.cs" />
<Compile Include="DeleteHandlerTests.cs" />
<Compile Include="DisruptionTests.cs" />
Duplicati/UnitTest/ImportExportTests.cs
+94 -40
View File
@@ -14,12 +14,18 @@
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
using Duplicati.Server.Database;
using Duplicati.Server.Serialization.Interface;
using NUnit.Framework;
using System;
using System.Collections.Generic;
using System.IO;
using Duplicati.Server;
using Duplicati.Server.Database;
using Duplicati.Server.Serializable;
using Duplicati.Server.Serialization;
using Duplicati.Server.Serialization.Interface;
using Duplicati.Server.WebServer.RESTMethods;
using NUnit.Framework;
using Backup = Duplicati.Server.Database.Backup;
namespace Duplicati.UnitTest
{
@@ -47,62 +53,110 @@ namespace Duplicati.UnitTest
}
}
private IBackup CreateBackup(string name, string username, string password, Dictionary<string, string> metadata)
{
return new Backup
{
Description = "Mock Backup Description",
Filters = new IFilter[0],
ID = "1",
Metadata = metadata,
Name = name,
Settings = new[] {new Setting {Name = "passphrase", Value = "12345"}},
Sources = new[] {"Mock Backup Source"},
Tags = new[] {"Tags"},
TargetURL = $"file:///mock_backup_target?auth-username={username}&auth-password={password}"
};
}
[Test]
[Category("ImportExport")]
[TestCase(true)]
[TestCase(false)]
public void ExportToJSONEncoding(bool removePasswords)
{
Dictionary<string, string> advancedOptions = new Dictionary<string, string> {{"server-datafolder", this.serverDatafolder}};
string usernameKey = "auth-username";
string passwordKey = "auth-password";
string username = @"user%40email.com";
string password = @"abcde12345!@#$%/\";
IBackup backup = this.CreateBackup("backup", username, password, new Dictionary<string, string>());
if (removePasswords)
{
Server.WebServer.RESTMethods.Backup.RemovePasswords(backup);
}
Assert.That(backup.TargetURL, Does.Contain($"{usernameKey}={username}"));
if (removePasswords)
{
Assert.That(backup.TargetURL, Does.Not.Contain(passwordKey));
Assert.That(backup.TargetURL, Does.Not.Contain(password));
}
else
{
Assert.That(backup.TargetURL, Does.Contain(passwordKey));
Assert.That(backup.TargetURL, Does.Contain(password));
}
byte[] jsonByteArray;
using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions))
{
jsonByteArray = Server.WebServer.RESTMethods.Backup.ExportToJSON(backup, null);
}
// The username should not have the '%40' converted to '@' since the import code
// cannot handle it (see issue #3619).
string json = System.Text.Encoding.Default.GetString(jsonByteArray);
Assert.That(json, Does.Not.Contain("user@email.com"));
ImportExportStructure importedConfiguration = Serializer.Deserialize<ImportExportStructure>(new StreamReader(new MemoryStream(jsonByteArray)));
Assert.AreEqual(backup.Description, importedConfiguration.Backup.Description);
Assert.AreEqual(backup.Name, importedConfiguration.Backup.Name);
Assert.AreEqual(backup.TargetURL, importedConfiguration.Backup.TargetURL);
}
[Test]
[Category("ImportExport")]
public void RoundTrip()
{
Dictionary<string, string> metadata = new Dictionary<string, string> { { "SourceFilesCount", "1" } };
IBackup CreateBackup(string name)
{
return new Backup
{
Description = "Mock Backup Description",
Filters = new IFilter[0],
ID = "1",
Metadata = metadata,
Name = name,
Settings = new[] { new Setting { Name = "passphrase", Value = "12345" } },
Sources = new[] { "Mock Backup Source" },
Tags = new[] { "Tags" },
TargetURL = "file:///mock_backup_target"
};
}
Dictionary<string, string> advancedOptions = new Dictionary<string, string> { { "server-datafolder", this.serverDatafolder } };
using (Duplicati.Server.Program.DataConnection = Duplicati.Server.Program.GetDatabaseConnection(advancedOptions))
Dictionary<string, string> metadata = new Dictionary<string, string> {{"SourceFilesCount", "1"}};
Dictionary<string, string> advancedOptions = new Dictionary<string, string> {{"server-datafolder", this.serverDatafolder}};
using (Program.DataConnection = Program.GetDatabaseConnection(advancedOptions))
{
// Unencrypted file, don't import metadata.
string unencryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName());
File.WriteAllBytes(unencryptedWithoutMetadata, Server.WebServer.RESTMethods.Backup.ExportToJSON(CreateBackup("unencrypted without metadata"), null));
Duplicati.Server.WebServer.RESTMethods.Backups.ImportBackup(unencryptedWithoutMetadata, false, () => null, advancedOptions);
Assert.AreEqual(1, Duplicati.Server.Program.DataConnection.Backups.Length);
Assert.AreEqual(0, Duplicati.Server.Program.DataConnection.Backups[0].Metadata.Count);
File.WriteAllBytes(unencryptedWithoutMetadata, Server.WebServer.RESTMethods.Backup.ExportToJSON(this.CreateBackup("unencrypted without metadata", "user", "password", metadata), null));
Backups.ImportBackup(unencryptedWithoutMetadata, false, () => null, advancedOptions);
Assert.AreEqual(1, Program.DataConnection.Backups.Length);
Assert.AreEqual(0, Program.DataConnection.Backups[0].Metadata.Count);
// Unencrypted file, import metadata.
string unencryptedWithMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName());
File.WriteAllBytes(unencryptedWithMetadata, Server.WebServer.RESTMethods.Backup.ExportToJSON(CreateBackup("unencrypted with metadata"), null));
Duplicati.Server.WebServer.RESTMethods.Backups.ImportBackup(unencryptedWithMetadata, true, () => null, advancedOptions);
Assert.AreEqual(2, Duplicati.Server.Program.DataConnection.Backups.Length);
Assert.AreEqual(metadata.Count, Duplicati.Server.Program.DataConnection.Backups[1].Metadata.Count);
File.WriteAllBytes(unencryptedWithMetadata, Server.WebServer.RESTMethods.Backup.ExportToJSON(this.CreateBackup("unencrypted with metadata", "user", "password", metadata), null));
Backups.ImportBackup(unencryptedWithMetadata, true, () => null, advancedOptions);
Assert.AreEqual(2, Program.DataConnection.Backups.Length);
Assert.AreEqual(metadata.Count, Program.DataConnection.Backups[1].Metadata.Count);
// Encrypted file, don't import metadata.
string encryptedWithoutMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName());
string passphrase = "abcde";
File.WriteAllBytes(encryptedWithoutMetadata, Server.WebServer.RESTMethods.Backup.ExportToJSON(CreateBackup("encrypted without metadata"), passphrase));
Duplicati.Server.WebServer.RESTMethods.Backups.ImportBackup(encryptedWithoutMetadata, false, () => passphrase, advancedOptions);
Assert.AreEqual(3, Duplicati.Server.Program.DataConnection.Backups.Length);
Assert.AreEqual(0, Duplicati.Server.Program.DataConnection.Backups[2].Metadata.Count);
File.WriteAllBytes(encryptedWithoutMetadata, Server.WebServer.RESTMethods.Backup.ExportToJSON(this.CreateBackup("encrypted without metadata", "user", "password", metadata), passphrase));
Backups.ImportBackup(encryptedWithoutMetadata, false, () => passphrase, advancedOptions);
Assert.AreEqual(3, Program.DataConnection.Backups.Length);
Assert.AreEqual(0, Program.DataConnection.Backups[2].Metadata.Count);
// Encrypted file, import metadata.
string encryptedWithMetadata = Path.Combine(this.serverDatafolder, Path.GetRandomFileName());
File.WriteAllBytes(encryptedWithMetadata, Server.WebServer.RESTMethods.Backup.ExportToJSON(CreateBackup("encrypted with metadata"), passphrase));
Duplicati.Server.WebServer.RESTMethods.Backups.ImportBackup(encryptedWithMetadata, true, () => passphrase, advancedOptions);
Assert.AreEqual(4, Duplicati.Server.Program.DataConnection.Backups.Length);
Assert.AreEqual(metadata.Count, Duplicati.Server.Program.DataConnection.Backups[3].Metadata.Count);
File.WriteAllBytes(encryptedWithMetadata, Server.WebServer.RESTMethods.Backup.ExportToJSON(this.CreateBackup("encrypted with metadata", "user", "password", metadata), passphrase));
Backups.ImportBackup(encryptedWithMetadata, true, () => passphrase, advancedOptions);
Assert.AreEqual(4, Program.DataConnection.Backups.Length);
Assert.AreEqual(metadata.Count, Program.DataConnection.Backups[3].Metadata.Count);
// Encrypted file, incorrect passphrase.
Assert.Throws(Is.InstanceOf<Exception>(), () => Duplicati.Server.WebServer.RESTMethods.Backups.ImportBackup(encryptedWithMetadata, true, () => passphrase + " ", advancedOptions));
Assert.Throws(Is.InstanceOf<Exception>(), () => Backups.ImportBackup(encryptedWithMetadata, true, () => passphrase + " ", advancedOptions));
}
}
}
}
}
Duplicati/UnitTest/ProblematicPathTests.cs
+14 -28
View File
@@ -14,14 +14,6 @@ namespace Duplicati.UnitTest
[TestFixture]
public class ProblematicPathTests : BasicSetupHelper
{
private static void WriteFile(string path, byte[] contents)
{
using (FileStream fileStream = SystemIO.IO_OS.FileOpenWrite(path))
{
Utility.CopyStream(new MemoryStream(contents), fileStream);
}
}
[Test]
[Category("ProblematicPath")]
public void DirectoriesWithWildcards()
@@ -40,7 +32,7 @@ namespace Duplicati.UnitTest
if (!Platform.IsClientWindows)
{
SystemIO.IO_OS.DirectoryCreate(dirWithAsterisk);
WriteFile(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file), new byte[] {0});
TestUtils.WriteFile(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file), new byte[] {0});
directories.Add(dirWithAsterisk);
questionMarkWildcardShouldMatchCount++;
verbatimAsteriskShouldMatchCount++;
@@ -52,7 +44,7 @@ namespace Duplicati.UnitTest
if (!Platform.IsClientWindows)
{
SystemIO.IO_OS.DirectoryCreate(dirWithQuestionMark);
WriteFile(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file), new byte[] { 1 });
TestUtils.WriteFile(SystemIO.IO_OS.PathCombine(dirWithQuestionMark, file), new byte[] { 1 });
directories.Add(dirWithQuestionMark);
questionMarkWildcardShouldMatchCount++;
}
@@ -62,14 +54,14 @@ namespace Duplicati.UnitTest
const string singleCharacterDir = "X";
string dirWithSingleCharacter = Path.Combine(this.DATAFOLDER, singleCharacterDir);
SystemIO.IO_OS.DirectoryCreate(dirWithSingleCharacter);
WriteFile(SystemIO.IO_OS.PathCombine(dirWithSingleCharacter, file), new byte[] { 2 });
TestUtils.WriteFile(SystemIO.IO_OS.PathCombine(dirWithSingleCharacter, file), new byte[] { 2 });
directories.Add(dirWithSingleCharacter);
questionMarkWildcardShouldMatchCount++;
const string dir = "dir";
string normalDir = Path.Combine(this.DATAFOLDER, dir);
SystemIO.IO_OS.DirectoryCreate(normalDir);
WriteFile(SystemIO.IO_OS.PathCombine(normalDir, file), new byte[] {3});
TestUtils.WriteFile(SystemIO.IO_OS.PathCombine(normalDir, file), new byte[] {3});
directories.Add(normalDir);
// Backup all files.
@@ -89,16 +81,7 @@ namespace Duplicati.UnitTest
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
foreach (string directory in directories)
{
string directoryName = SystemIO.IO_OS.PathGetFileName(directory);
foreach (string expectedFilePath in SystemIO.IO_OS.EnumerateFiles(directory))
{
string fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath);
string restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, directoryName, fileName);
Assert.IsTrue(TestUtils.CompareFiles(expectedFilePath, restoredFilePath, expectedFilePath, false));
}
}
TestUtils.AssertDirectoryTreesAreEquivalent(this.DATAFOLDER, this.RESTOREFOLDER, true, "Restore");
// List results using * should return a match for each directory.
IListResults listResults = c.List(SystemIO.IO_OS.PathCombine(dirWithAsterisk, file));
@@ -140,7 +123,7 @@ namespace Duplicati.UnitTest
string fileName = SystemIO.IO_OS.PathGetFileName(expectedFilePath);
string restoredFilePath = SystemIO.IO_OS.PathCombine(this.RESTOREFOLDER, fileName);
Assert.IsTrue(TestUtils.CompareFiles(expectedFilePath, restoredFilePath, expectedFilePath, false));
TestUtils.AssertFilesAreEqual(expectedFilePath, restoredFilePath, false, expectedFilePath);
SystemIO.IO_OS.FileDelete(restoredFilePath);
}
@@ -157,6 +140,9 @@ namespace Duplicati.UnitTest
Assert.AreEqual(directories.Count, backupResults.ExaminedFiles);
}
// Block for a small amount of time to avoid clock issues when quickly running successive backups.
System.Threading.Thread.Sleep(1000);
// Backup with verbatim asterisk in include filter should include
// one directory in Linux and zero directories in Windows.
filter = new FilterExpression("@" + SystemIO.IO_OS.PathCombine(dirWithAsterisk, file));
@@ -179,7 +165,7 @@ namespace Duplicati.UnitTest
// A long path to exclude.
string longFile = SystemIO.IO_OS.PathCombine(this.DATAFOLDER, new string('y', 255));
WriteFile(longFile, new byte[] {0, 1});
TestUtils.WriteFile(longFile, new byte[] {0, 1});
// A folder that ends with a dot to exclude.
string folderWithDot = Path.Combine(this.DATAFOLDER, "folder_with_dot.");
@@ -191,11 +177,11 @@ namespace Duplicati.UnitTest
// A file that ends with a dot to exclude.
string fileWithDot = Path.Combine(this.DATAFOLDER, "file_with_dot.");
WriteFile(fileWithDot, new byte[] {0, 1});
TestUtils.WriteFile(fileWithDot, new byte[] {0, 1});
// A file that ends with a space to exclude.
string fileWithSpace = Path.Combine(this.DATAFOLDER, "file_with_space ");
WriteFile(fileWithSpace, new byte[] {0, 1});
TestUtils.WriteFile(fileWithSpace, new byte[] {0, 1});
FilterExpression filter = new FilterExpression(longFile, false);
filter = FilterExpression.Combine(filter, new FilterExpression(Util.AppendDirSeparator(folderWithDot), false));
@@ -234,7 +220,7 @@ namespace Duplicati.UnitTest
string fileName = new string('y', 255);
string filePath = SystemIO.IO_OS.PathCombine(folderPath, fileName);
byte[] fileBytes = {0, 1, 2};
WriteFile(filePath, fileBytes);
TestUtils.WriteFile(filePath, fileBytes);
Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
@@ -277,7 +263,7 @@ namespace Duplicati.UnitTest
string filePath = SystemIO.IO_OS.PathCombine(folderPath, pathComponent);
byte[] fileBytes = {0, 1, 2};
WriteFile(filePath, fileBytes);
TestUtils.WriteFile(filePath, fileBytes);
Dictionary<string, string> options = new Dictionary<string, string>(this.TestOptions);
using (Controller c = new Controller("file://" + this.TARGETFOLDER, options, null))
Duplicati/UnitTest/RecoveryToolTests.cs
+2 -2
View File
@@ -93,13 +93,13 @@ namespace Duplicati.UnitTest
foreach (string filepath in Directory.EnumerateFiles(this.DATAFOLDER))
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(restoreFolder, baseFolder, filename ?? String.Empty), filename, false));
TestUtils.AssertFilesAreEqual(filepath, Path.Combine(restoreFolder, baseFolder, filename ?? String.Empty), false, filename);
}
foreach (string filepath in Directory.EnumerateFiles(subdirectoryPath))
{
string filename = Path.GetFileName(filepath);
Assert.IsTrue(TestUtils.CompareFiles(filepath, Path.Combine(restoreFolder, baseFolder, subdirectoryName, filename ?? String.Empty), filename, false));
TestUtils.AssertFilesAreEqual(filepath, Path.Combine(restoreFolder, baseFolder, subdirectoryName, filename ?? String.Empty), false, filename);
}
}
}
Duplicati/UnitTest/SVNCheckoutsTest.cs
+2 -6
View File
@@ -421,11 +421,7 @@ namespace Duplicati.UnitTest
throw new Exception("Unittest is broken");
}
if (!TestUtils.CompareFiles(sourcename, restoredname, s, verifymetadata))
{
Log.WriteErrorMessage(LOGTAG, "PartialRestoreWrongFile", null, "Partial restore file differs: {0}", s);
BasicSetupHelper.ProgressWriteLine("Partial restore file differs: " + s);
}
TestUtils.AssertFilesAreEqual(sourcename, restoredname, verifymetadata, $"Partial restore file differs: {s}");
}
}
}
@@ -435,7 +431,7 @@ namespace Duplicati.UnitTest
using (new Timer(LOGTAG, "SourceVerification", "Verification of " + source))
{
for (int j = 0; j < actualfolders.Length; j++)
TestUtils.VerifyDir(actualfolders[j], restorefoldernames[j], verifymetadata);
TestUtils.AssertDirectoryTreesAreEquivalent(actualfolders[j], restorefoldernames[j], verifymetadata, "VerifyFullRestore");
}
}
Duplicati/UnitTest/SymLinkTests.cs
Executable
+91
View File
@@ -0,0 +1,91 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Duplicati.Library.Interface;
using Duplicati.Library.Main;
using Duplicati.Library.Snapshots;
using NUnit.Framework;
namespace Duplicati.UnitTest
{
[TestFixture]
public class SymLinkTests : BasicSetupHelper
{
[Test]
[Category("SymLink")]
[TestCase(Options.SymlinkStrategy.Store)]
[TestCase(Options.SymlinkStrategy.Follow)]
[TestCase(Options.SymlinkStrategy.Ignore)]
public void SymLinkPolicy(Options.SymlinkStrategy symlinkPolicy)
{
// Create symlink target directory
const string targetDirName = "target";
var targetDir = systemIO.PathCombine(this.DATAFOLDER, targetDirName);
systemIO.DirectoryCreate(targetDir);
// Create files in symlink target directory
var fileNames = new[] { "a.txt", "b.txt", "c.txt" };
foreach (var file in fileNames)
{
var targetFile = systemIO.PathCombine(targetDir, file);
TestUtils.WriteFile(targetFile, Encoding.Default.GetBytes(file));
}
// Create actual symlink directory linking to the target directory
const string symlinkDirName = "symlink";
var symlinkDir = systemIO.PathCombine(this.DATAFOLDER, symlinkDirName);
try
{
systemIO.CreateSymlink(symlinkDir, targetDir, asDir: true);
}
catch (Exception e)
{
// If client cannot create symlinks, mark test as ignored
Assert.Ignore($"Client could not create a symbolic link. Error reported: {e.Message}");
}
// Backup all files with given symlink policy
Dictionary<string, string> restoreOptions = new Dictionary<string, string>(this.TestOptions) { ["restore-path"] = this.RESTOREFOLDER };
Dictionary<string, string> backupOptions = new Dictionary<string, string>(this.TestOptions) { ["symlink-policy"] = symlinkPolicy.ToString() };
using (Controller c = new Controller("file://" + this.TARGETFOLDER, backupOptions, null))
{
IBackupResults backupResults = c.Backup(new[] { this.DATAFOLDER });
Assert.AreEqual(0, backupResults.Errors.Count());
Assert.AreEqual(0, backupResults.Warnings.Count());
}
// Restore all files
using (Controller c = new Controller("file://" + this.TARGETFOLDER, restoreOptions, null))
{
IRestoreResults restoreResults = c.Restore(null);
Assert.AreEqual(0, restoreResults.Errors.Count());
Assert.AreEqual(0, restoreResults.Warnings.Count());
// Verify that symlink policy was followed
var restoreSymlinkDir = systemIO.PathCombine(this.RESTOREFOLDER, symlinkDirName);
switch (symlinkPolicy)
{
case Options.SymlinkStrategy.Store:
// Restore should contain an actual symlink to the original target
Assert.That(systemIO.IsSymlink(restoreSymlinkDir), Is.True);
var restoredSymlinkFullPath = systemIO.PathGetFullPath(systemIO.GetSymlinkTarget(restoreSymlinkDir));
var symlinkTargetFullPath = systemIO.PathGetFullPath(targetDir);
Assert.That(restoredSymlinkFullPath, Is.EqualTo(symlinkTargetFullPath));
break;
case Options.SymlinkStrategy.Follow:
// Restore should contain a regular directory with copies of the files in the symlink target
Assert.That(systemIO.IsSymlink(restoreSymlinkDir), Is.False);
TestUtils.AssertDirectoryTreesAreEquivalent(targetDir, restoreSymlinkDir, true, "Restore");
break;
case Options.SymlinkStrategy.Ignore:
// Restore should not contain the symlink or directory at all
Assert.That(systemIO.DirectoryExists(restoreSymlinkDir), Is.False);
Assert.That(systemIO.FileExists(restoreSymlinkDir), Is.False);
break;
default:
Assert.Fail($"Unexpected symlink policy");
break;
}
}
}
}
}
Duplicati/UnitTest/TestUtils.cs
+77 -105
View File
@@ -24,7 +24,8 @@ using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using Duplicati.Library.Common.IO;
using NUnit.Framework;
namespace Duplicati.UnitTest
{
public static class TestUtils
@@ -169,129 +170,90 @@ namespace Duplicati.UnitTest
}
/// <summary>
/// Verifies the existence of all files and folders, and ensures that all
/// files are binary equal.
/// Asserts that the two directory trees are equivalent; i.e.,
/// that they they contain the same directories and files, recursively.
/// </summary>
/// <param name="f1">One folder</param>
/// <param name="f2">Another folder</param>
public static void VerifyDir(string f1, string f2, bool verifymetadata)
/// <param name="expectedDir">The expected directory tree.</param>
/// <param name="actualDir">The actual directory tree.</param>
/// <param name="verifymetadata">True to also compare file metadata.</param>
/// <param name="contextMessage">Context information to include in an assert message.</param>
public static void AssertDirectoryTreesAreEquivalent(string expectedDir, string actualDir, bool verifymetadata, string contextMessage)
{
var anymissing = false;
f1 = Util.AppendDirSeparator(f1);
f2 = Util.AppendDirSeparator(f2);
var folders1 = Utility.EnumerateFolders(f1);
var folders2 = Utility.EnumerateFolders(f2).ToList();
foreach (string s in folders1)
var localMessage = $"{contextMessage}, in directories {expectedDir} and {actualDir}";
// Assert that expectedDir and actualDir contain the same directories
var expectedSubdirs = SystemIO.IO_OS.EnumerateDirectories(expectedDir).OrderBy(SystemIO.IO_OS.PathGetFileName);
var actualSubdirs = SystemIO.IO_OS.EnumerateDirectories(actualDir).OrderBy(SystemIO.IO_OS.PathGetFileName);
Assert.That(expectedSubdirs.Select(SystemIO.IO_OS.PathGetFileName), Is.EquivalentTo(actualSubdirs.Select(SystemIO.IO_OS.PathGetFileName)), localMessage);
// Recursively compare the contained directories
var expectedSubdirsEnumerator = expectedSubdirs.GetEnumerator();
var actualSubdirsEnumerator = actualSubdirs.GetEnumerator();
while (expectedSubdirsEnumerator.MoveNext() && actualSubdirsEnumerator.MoveNext())
{
string relpath = s.Substring(f1.Length);
string target = System.IO.Path.Combine(f2, relpath);
int ix = IndexOf(folders2, target);
if (ix < 0)
{
Log.WriteErrorMessage(LOGTAG, "MissingFolder", null, "Missing folder: {0}", relpath);
Console.WriteLine("Missing folder: " + relpath);
anymissing = true;
}
else
folders2.RemoveAt(ix);
AssertDirectoryTreesAreEquivalent(expectedSubdirsEnumerator.Current, actualSubdirsEnumerator.Current, verifymetadata, contextMessage);
}
foreach (string s in folders2)
// Assert that expectedDir and actualDir contain the same files
var expectedFiles = SystemIO.IO_OS.EnumerateFiles(expectedDir).OrderBy(SystemIO.IO_OS.PathGetFileName);
var actualFiles = SystemIO.IO_OS.EnumerateFiles(actualDir).OrderBy(SystemIO.IO_OS.PathGetFileName);
Assert.That(expectedFiles.Select(SystemIO.IO_OS.PathGetFileName), Is.EquivalentTo(actualFiles.Select(SystemIO.IO_OS.PathGetFileName)), localMessage);
// Assert that the files are equal
var expectedFilesEnumerator = expectedFiles.GetEnumerator();
var actualFilesEnumerator = actualFiles.GetEnumerator();
while (expectedFilesEnumerator.MoveNext() && actualFilesEnumerator.MoveNext())
{
Log.WriteErrorMessage(LOGTAG, "ExtraFolder", null, "Extra folder: {0}", s.Substring(f2.Length));
Console.WriteLine("Extra folder: " + s.Substring(f2.Length));
AssertFilesAreEqual(expectedFilesEnumerator.Current, actualFilesEnumerator.Current, verifymetadata, contextMessage);
}
var files1 = Utility.EnumerateFiles(f1);
var files2 = Utility.EnumerateFiles(f2).ToList();
foreach (string s in files1)
{
string relpath = s.Substring(f1.Length);
string target = System.IO.Path.Combine(f2, relpath);
int ix = IndexOf(files2, target);
if (ix < 0)
{
Log.WriteErrorMessage(LOGTAG, "MissingFile", null, "Missing file: {0}", relpath);
Console.WriteLine("Missing file: " + relpath);
anymissing = true;
}
else
{
files2.RemoveAt(ix);
if (!CompareFiles(s, target, relpath, verifymetadata))
{
Log.WriteErrorMessage(LOGTAG, "FileDiffers", null, "File differs: {0}", relpath);
Console.WriteLine("File differs: " + relpath);
}
}
}
foreach (string s in files2)
{
Log.WriteErrorMessage(LOGTAG, "ExtraFile", null, "Extra file: {0}", s.Substring(f2.Length));
Console.WriteLine("Extra file: " + s.Substring(f2.Length));
}
if (anymissing)
throw new Exception("Verify failed, some items are missing");
}
/// <summary>
/// Compares two files by reading all bytes, and comparing one by one
/// Asserts that two files are equal by comparing their length, contents, and, optionally, their metadata.
/// </summary>
/// <param name="f1">One file</param>
/// <param name="f2">Another file</param>
/// <param name="display">File display name</param>
/// <returns>True if they are equal, false otherwise</returns>
public static bool CompareFiles(string f1, string f2, string display, bool verifymetadata)
/// <param name="expectedFile">The expected file.</param>
/// <param name="actualFile">The actual file.</param>
/// <param name="verifymetadata">True to also compare file metadata.</param>
/// <param name="contextMessage">Context information to include in an assert message.</param>
public static void AssertFilesAreEqual(string expectedFile, string actualFile, bool verifymetadata, string contextMessage)
{
using (System.IO.FileStream fs1 = System.IO.File.OpenRead(f1))
using (System.IO.FileStream fs2 = System.IO.File.OpenRead(f2))
if (fs1.Length != fs2.Length)
using (var expectedFileStream = SystemIO.IO_OS.FileOpenRead(expectedFile))
using (var actualFileStream = SystemIO.IO_OS.FileOpenRead(actualFile))
{
// Compare file lengths
var expectedFileStreamLength = expectedFileStream.Length;
var actualFileStreamLength = actualFileStream.Length;
Assert.That(actualFileStreamLength, Is.EqualTo(expectedFileStreamLength), $"{contextMessage}, file size mismatch for {expectedFile} and {actualFile}");
// Compare file contents
// The byte-by-byte compare is dog-slow, so we use a fast(-er) check, and then report the first byte diff if required
if (!Utility.CompareStreams(expectedFileStream, actualFileStream, true))
{
Log.WriteErrorMessage(LOGTAG, "LengthsDiffer", null, "Lengths differ: {0}, {1} vs {2}", display, fs1.Length.ToString(), fs2.Length.ToString());
Console.WriteLine("Lengths differ: " + display + ", " + fs1.Length.ToString() + " vs. " + fs2.Length.ToString());
return false;
}
else
{
// The byte-by-byte compare is dog-slow, so we use a fast(-er) check, and then report the first byte diff if required
if (!Library.Utility.Utility.CompareStreams(fs1, fs2, true))
// Reset stream positions
expectedFileStream.Position = 0;
actualFileStream.Position = 0;
for (long i = 0; i < expectedFileStreamLength; i++)
{
fs1.Position = 0;
fs2.Position = 0;
long len = fs1.Length;
for(long l = 0; l < len; l++)
if (fs1.ReadByte() != fs2.ReadByte())
{
Log.WriteErrorMessage(LOGTAG, "MismatchInFile", null, "Mismatch in byte {0} in file {1}", l.ToString(), display);
Console.WriteLine("Mismatch in byte " + l.ToString() + " in file " + display);
return false;
}
var expectedByte = expectedFileStream.ReadByte();
var actualByte = actualFileStream.ReadByte();
// For performance reasons, only exercise Assert mechanism and generate message if byte comparison fails
if (expectedByte != actualByte)
{
var message = $"{contextMessage}, file contents mismatch at position {i} for {expectedFile} and {actualFile}";
Assert.That(actualByte, Is.EqualTo(expectedByte), message);
}
}
}
}
// Compare file metadata
if (verifymetadata)
{
if (System.IO.File.GetLastWriteTime(f1) != System.IO.File.GetLastWriteTime(f2))
{
Log.WriteWarningMessage(LOGTAG, "MismatchInLastModified", null, "Mismatch in lastmodified for {0}, {1} vs {2}", f2, System.IO.File.GetLastWriteTimeUtc(f1), System.IO.File.GetLastWriteTimeUtc(f2));
Console.WriteLine("Mismatch in lastmodified for " + f2 + ", " + System.IO.File.GetLastWriteTimeUtc(f1) + " vs. " + System.IO.File.GetLastWriteTimeUtc(f2));
}
if (System.IO.File.GetCreationTimeUtc(f1) != System.IO.File.GetCreationTimeUtc(f2))
{
Log.WriteWarningMessage(LOGTAG, "MismatchInCreateTime", null, "Mismatch in create-time for {0}, {1} vs {2}", f2, System.IO.File.GetCreationTimeUtc(f1), System.IO.File.GetCreationTimeUtc(f2));
Console.WriteLine("Mismatch in create-time for " + f2 + ", " + System.IO.File.GetCreationTimeUtc(f1) + " vs. " + System.IO.File.GetCreationTimeUtc(f2));
}
Assert.That(
SystemIO.IO_OS.GetLastWriteTimeUtc(actualFile),
Is.EqualTo(SystemIO.IO_OS.GetLastWriteTimeUtc(expectedFile)).Within(1).Milliseconds,
$"{contextMessage}, last write time mismatch for {expectedFile} and {actualFile}");
Assert.That(
SystemIO.IO_OS.GetCreationTimeUtc(actualFile),
Is.EqualTo(SystemIO.IO_OS.GetCreationTimeUtc(expectedFile)).Within(1).Milliseconds,
$"{contextMessage}, creation time mismatch for {expectedFile} and {actualFile}");
}
return true;
}
public static Dictionary<string, string> Expand(this Dictionary<string, string> self, object extra)
{
var res = new Dictionary<string, string>(self);
@@ -312,6 +274,16 @@ namespace Duplicati.UnitTest
return res;
}
/// <summary>
/// Write file <paramref name="path"/> with <paramref name="contents"/>.
/// </summary>
public static void WriteFile(string path, byte[] contents)
{
using (FileStream fileStream = SystemIO.IO_OS.FileOpenWrite(path))
{
Utility.CopyStream(new MemoryStream(contents), fileStream);
}
}
}
}
Duplicati/WindowsService/App.config
+3
View File
@@ -3,4 +3,7 @@
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.7.1"/>
</startup>
<runtime>
<AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false" />
</runtime>
</configuration>
Installer/AssemblyRedirects.xml
+1
View File
@@ -1,6 +1,7 @@
<configuration>
<runtime>
<loadFromRemoteSources enabled="true"/>
<AppContextSwitchOverrides value="Switch.System.IO.UseLegacyPathHandling=false" />
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
Installer/Docker/context/Dockerfile
+1
View File
@@ -3,6 +3,7 @@ FROM --platform=$TARGETPLATFORM mono:5-slim
RUN apt-get update && \
apt-get install -y --no-install-recommends \
curl \
mono-complete \
libmono-sqlite4.0-cil \
libmono-system-drawing4.0-cil \
libmono-system-net-http-webrequest4.0-cil \
Updates/build_version.txt
+1 -1
View File
@@ -1 +1 @@
110
111
build.sh
-53
View File
@@ -1,53 +0,0 @@
#!/bin/bash
quit_on_error() {
echo "Error on line $1, stopping build."
exit 1
}
list_dir() {
echo "listing directory $1 :"
ls -al $1
}
trap 'quit_on_error $LINENO' ERR
CATEGORY=$1
TRAVIS_BUILD_DIR=${2:-.}
if id travis &> /dev/null
then
TESTUSER=travis
else
TESTUSER=$(whoami)
fi
echo "Build script starting with parameters TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR and CATEGORY=$CATEGORY"
# build duplicati
list_dir "${TRAVIS_BUILD_DIR}"/packages/
echo "travis_fold:start:build_duplicati"
msbuild /p:Configuration=Release Duplicati.sln
cp -r ./Duplicati/Server/webroot ./Duplicati/GUI/Duplicati.GUI.TrayIcon/bin/Release/webroot
echo "travis_fold:end:build_duplicati"
rm -rf ~/duplicati_testdata && mkdir ~/duplicati_testdata
chown -R $TESTUSER ~/duplicati_testdata/
chmod -R 755 ~/duplicati_testdata
# run unit tests
echo "travis_fold:start:unit_test"
if [[ "$CATEGORY" != "GUI" && "$CATEGORY" != "" ]]; then
mono ./testrunner/NUnit.ConsoleRunner.3.10.0/tools/nunit3-console.exe \
./Duplicati/UnitTest/bin/Release/Duplicati.UnitTest.dll --where:cat==$CATEGORY --workers=1
fi
echo "travis_fold:end:unit_test"
# start server and run gui tests
echo "travis_fold:start:gui_unit_test"
if [[ "$CATEGORY" == "GUI" ]]; then
mono ./Duplicati/GUI/Duplicati.GUI.TrayIcon/bin/Release/Duplicati.Server.exe &
python guiTests/guiTest.py
fi
echo "travis_fold:end:gui_unit_test"
changelog.txt
+14
View File
@@ -1,3 +1,17 @@
2020-09-26 - 2.0.5.111_canary_2020-09-26
==========
* Support for Windows paths starting with "\\?\", thanks @dferreyra
* Improved performance of USN, thanks @dgehri
* Improved performance for restoring, thanks @pcmind
* Added support for filters with literals that would normally be treated as wildcards, thanks @dferreyra
* Improved build and tests, thanks @warwickmm and @dferreyra
* Switched to the new built-in .Net support for long paths, thanks @dferreyra
* Fixed issues with the Docker images by including mono-complete, thanks @warwickmm
* Fixed an issue with exporting passwords with url-encoding characters, thanks @warwickmm
* Fixed an issue with displaying icons in the restore tree, thanks @warwickmm
* Fixed a type that prevented a warning box from showing, thanks @warwickmm
* Added Storadera as an S3 destination, thanks @warwickmm
2020-08-10 - 2.0.5.110_canary_2020-08-10
==========
* Fixed an issue with snapshot paths, thanks @dferreyra
pipeline/stage_build/job.sh
+17 -18
View File
@@ -3,31 +3,30 @@
. /pipeline/shared/duplicati.sh
function build () {
nuget restore "${DUPLICATI_ROOT}"/Duplicati.sln
nuget restore "${DUPLICATI_ROOT}"/Duplicati.sln
if [ ! -d "${DUPLICATI_ROOT}"/packages/SharpCompress.0.23.0 ]; then
ln -s "${DUPLICATI_ROOT}"/packages/sharpcompress.0.23.0 "${DUPLICATI_ROOT}"/packages/SharpCompress.0.23.0
fi
if [ ! -d "${DUPLICATI_ROOT}"/packages/SharpCompress.0.23.0 ]; then
ln -s "${DUPLICATI_ROOT}"/packages/sharpcompress.0.23.0 "${DUPLICATI_ROOT}"/packages/SharpCompress.0.23.0
fi
# build version stamper
msbuild -p:Configuration=Release -v:minimal "${DUPLICATI_ROOT}/BuildTools/UpdateVersionStamp/UpdateVersionStamp.csproj"
mono "${DUPLICATI_ROOT}/BuildTools/UpdateVersionStamp/bin/Release/UpdateVersionStamp.exe" --version="${releaseversion}"
# build version stamper
msbuild -p:Configuration=Release -v:minimal "${DUPLICATI_ROOT}/BuildTools/UpdateVersionStamp/UpdateVersionStamp.csproj"
mono "${DUPLICATI_ROOT}/BuildTools/UpdateVersionStamp/bin/Release/UpdateVersionStamp.exe" --version="${releaseversion}"
# build autoupdate
nuget restore "${DUPLICATI_ROOT}/BuildTools/AutoUpdateBuilder/AutoUpdateBuilder.sln"
msbuild -p:Configuration=Release -v:minimal "${DUPLICATI_ROOT}/BuildTools/AutoUpdateBuilder/AutoUpdateBuilder.sln"
# build autoupdate
nuget restore "${DUPLICATI_ROOT}/BuildTools/AutoUpdateBuilder/AutoUpdateBuilder.sln"
msbuild -p:Configuration=Release -v:minimal "${DUPLICATI_ROOT}/BuildTools/AutoUpdateBuilder/AutoUpdateBuilder.sln"
# build gpg signing tool
nuget restore "${DUPLICATI_ROOT}/BuildTools/GnupgSigningTool/GnupgSigningTool.sln"
msbuild -p:Configuration=Release -v:minimal "${DUPLICATI_ROOT}/BuildTools/GnupgSigningTool/GnupgSigningTool.sln"
# build gpg signing tool
nuget restore "${DUPLICATI_ROOT}/BuildTools/GnupgSigningTool/GnupgSigningTool.sln"
msbuild -p:Configuration=Release -v:minimal "${DUPLICATI_ROOT}/BuildTools/GnupgSigningTool/GnupgSigningTool.sln"
# build duplicati
msbuild -p:DefineConstants=__MonoCS__ -p:DefineConstants=ENABLE_GTK /p:Configuration=Release "${DUPLICATI_ROOT}/Duplicati.sln"
# build duplicati
msbuild -p:DefineConstants=ENABLE_GTK /p:Configuration=Release "${DUPLICATI_ROOT}/Duplicati.sln"
msbuild -p:Configuration=Release -v:minimal "${DUPLICATI_ROOT}"/Duplicati.sln
cp -r "${DUPLICATI_ROOT}"/Duplicati/Server/webroot "${DUPLICATI_ROOT}"/Duplicati/GUI/Duplicati.GUI.TrayIcon/bin/Release/webroot
cp -r "${DUPLICATI_ROOT}"/Duplicati/Server/webroot "${DUPLICATI_ROOT}"/Duplicati/GUI/Duplicati.GUI.TrayIcon/bin/Release/webroot
}
travis_mark_begin "BUILDING BINARIES"
build
travis_mark_end "BUILDING BINARIES"
travis_mark_end "BUILDING BINARIES"
pipeline/start.sh
+1 -1
View File
@@ -6,6 +6,6 @@ SCRIPT_DIR="$( cd "$(dirname "$0")" ; pwd -P )"
${ROOT_DIR}/pipeline/jobs/build_job.sh
${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories BulkNormal
${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories BulkNoSize
${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories SVNDataLong,SVNData,RecoveryTool,ProblematicPath
${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories SVNDataLong,SVNData,RecoveryTool,ProblematicPath,SymLink
${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories Border
${ROOT_DIR}/pipeline/jobs/unittest_job.sh --testcategories Filter,Targeted,Purge,Serialization,WebApi,Utility,UriUtility,IO,ImportExport,Disruption,RestoreHandler,RepairHandler,DeleteHandler,Controller