kenneth.skovhede@gmail.com
b81d63ea73
Fixed a protential race condition with warning messages and file counts (cosmetic only) when using async transfers. Spelling correction, Seperator -> Separator. Changed so a restore operation can also supply advanced options. Changed the format of timestamps in filenames to follow the Duplicity convention, which removes the need for --time-separator and --use-short-filenames. Moved the URL utility into Core, to avoid compilation errors with Monodevelop. Reorganized the runner to better support the LiveControls on all operations rather than only for backup/restore. Update issue #210 Status: Fixed I have now changed the time format to follow the Duplicity naming convention. This means that all filenames are now generated with the time set as "yyyyMMddTHHmmssZ". This format is more portable across all platforms, and easily sortable/readable. The drawback is that the Z indicates UTC time, so if one looks at the filenames, the times are not local. The motivation for this is to avoid trouble with daylight savings time offsets. There is full backwards compatibility as long as backups have not used more than one character for --time-separator (only possible via commandline). This new format means that the options --time-separator and --use-short-filenames are now deprecated and removed from the user interface. It is possible to access them from the advanced options page, along with the --use-old-filenames option. If anyone needs these options for some reason, let me know, otherwise they will be removed in a later version. git-svn-id: https://duplicati.googlecode.com/svn/trunk@523 59da171f-624f-0410-aa54-27559c288bec
328 lines
15 KiB
C#
328 lines
15 KiB
C#
#region Disclaimer / License
|
|
// Copyright (C) 2010, Kenneth Skovhede
|
|
// http://www.hexad.dk, opensource@hexad.dk
|
|
//
|
|
// This library is free software; you can redistribute it and/or
|
|
// modify it under the terms of the GNU Lesser General Public
|
|
// License as published by the Free Software Foundation; either
|
|
// version 2.1 of the License, or (at your option) any later version.
|
|
//
|
|
// This library is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
// Lesser General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Lesser General Public
|
|
// License along with this library; if not, write to the Free Software
|
|
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
//
|
|
#endregion
|
|
|
|
#if DEBUG
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Text;
|
|
using Duplicati.Library.Logging;
|
|
using Duplicati.Library.Core;
|
|
|
|
namespace Duplicati.CommandLine
|
|
{
|
|
/// <summary>
|
|
/// This class encapsulates a simple method for testing the correctness of duplicati.
|
|
/// </summary>
|
|
public class UnitTest
|
|
{
|
|
/// <summary>
|
|
/// A helper class to write debug messages to the log file
|
|
/// </summary>
|
|
private class LogHelper : StreamLog
|
|
{
|
|
private string m_backupset;
|
|
|
|
public string Backupset
|
|
{
|
|
get { return m_backupset; }
|
|
set { m_backupset = value; }
|
|
}
|
|
|
|
public LogHelper(string file)
|
|
: base(file)
|
|
{
|
|
this.Backupset = "none";
|
|
}
|
|
|
|
public override void WriteMessage(string message, LogMessageType type, Exception exception)
|
|
{
|
|
base.WriteMessage(this.Backupset + ", " + message, type, exception);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Running the unit test confirms the correctness of duplicati
|
|
/// </summary>
|
|
/// <param name="folders">The folders to backup. Folder at index 0 is the base, all others are incrementals</param>
|
|
public static void RunTest(string[] folders, Dictionary<string, string> options)
|
|
{
|
|
//Place a file called "unittest_target.txt" in the bin folder, and enter a connection string like "ftp://username:password@example.com"
|
|
|
|
string ftp_password = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "unittest_ftppassword.txt");
|
|
if (System.IO.File.Exists(ftp_password))
|
|
options["ftp-password"] = System.IO.File.ReadAllText(ftp_password).Trim();
|
|
|
|
string alttarget = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "unittest_target.txt");
|
|
if (System.IO.File.Exists(alttarget))
|
|
RunTest(folders, options, System.IO.File.ReadAllText(alttarget).Trim());
|
|
else
|
|
RunTest(folders, options, null);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Running the unit test confirms the correctness of duplicati
|
|
/// </summary>
|
|
/// <param name="folders">The folders to backup. Folder at index 0 is the base, all others are incrementals</param>
|
|
/// <param name="target">The target destination for the backups</param>
|
|
public static void RunTest(string[] folders, Dictionary<string, string> options, string target)
|
|
{
|
|
LogHelper log = new LogHelper("unittest.log");
|
|
Log.CurrentLog = log; ;
|
|
Log.LogLevel = Duplicati.Library.Logging.LogMessageType.Profiling;
|
|
|
|
string tempdir = System.IO.Path.Combine(System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), "tempdir");
|
|
|
|
try
|
|
{
|
|
if (System.IO.Directory.Exists(tempdir))
|
|
System.IO.Directory.Delete(tempdir, true);
|
|
|
|
System.IO.Directory.CreateDirectory(tempdir);
|
|
}
|
|
catch(Exception ex)
|
|
{
|
|
Log.WriteMessage("Failed to clean tempdir", LogMessageType.Error, ex);
|
|
}
|
|
|
|
Duplicati.Library.Core.TempFolder.SystemTempPath = tempdir;
|
|
|
|
//Set some defaults
|
|
if (!options.ContainsKey("passphrase"))
|
|
options["passphrase"] = "secret password!";
|
|
|
|
if (!options.ContainsKey("backup-prefix"))
|
|
options["backup-prefix"] = "duplicati_unittest";
|
|
|
|
//This would break the test, because the data is not modified the normal way
|
|
options["disable-filetime-check"] = null;
|
|
//We do not use the same folder, so we need this option
|
|
options["allow-sourcefolder-change"] = null;
|
|
|
|
using(new Timer("Total unittest"))
|
|
using(TempFolder tf = new TempFolder())
|
|
{
|
|
//The code below tests for a race condition in the ssh backend.
|
|
/*string[] list = null;
|
|
string[] prevList = null;
|
|
|
|
for (int i = 0; i < 1000; i++)
|
|
{
|
|
Console.WriteLine(string.Format("Listing, test {0}", i));
|
|
list = Duplicati.Library.Main.Interface.List(target, options);
|
|
if (i != 0 && list.Length != prevList.Length)
|
|
Console.WriteLine(string.Format("Count mismatch {0} vs {1}", list.Length, prevList.Length));
|
|
|
|
prevList = list;
|
|
}*/
|
|
|
|
if (string.IsNullOrEmpty(target))
|
|
{
|
|
target = "file://" + tf;
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("Removing old backups");
|
|
Dictionary<string, string> tmp = new Dictionary<string, string>(options);
|
|
tmp["delete-all-but-n-full"] = "0";
|
|
tmp["force"] = "";
|
|
tmp["allow-full-removal"] = "";
|
|
|
|
using (new Timer("Cleaning up any existing backups"))
|
|
Console.WriteLine(Duplicati.Library.Main.Interface.DeleteAllButNFull(target, tmp));
|
|
}
|
|
|
|
log.Backupset = "Backup " + folders[0];
|
|
Console.WriteLine("Backing up the full copy: " + folders[0]);
|
|
using (new Timer("Full backup of " + folders[0]))
|
|
{
|
|
options["full"] = "";
|
|
Log.WriteMessage(Duplicati.Library.Main.Interface.Backup(folders[0].Split(System.IO.Path.PathSeparator), target, options), LogMessageType.Information);
|
|
options.Remove("full");
|
|
}
|
|
|
|
for (int i = 1; i < folders.Length; i++)
|
|
{
|
|
//options["passphrase"] = "bad password";
|
|
//If the backups are too close, we can't pick the right one :(
|
|
System.Threading.Thread.Sleep(1000 * 5);
|
|
log.Backupset = "Backup " + folders[i];
|
|
Console.WriteLine("Backing up the incremental copy: " + folders[i]);
|
|
using (new Timer("Incremental backup of " + folders[i]))
|
|
Log.WriteMessage(Duplicati.Library.Main.Interface.Backup(folders[i].Split(System.IO.Path.PathSeparator), target, options), LogMessageType.Information);
|
|
}
|
|
|
|
Duplicati.Library.Main.Options opts = new Duplicati.Library.Main.Options(options);
|
|
using (Duplicati.Library.Interface.IBackend bk = Duplicati.Library.DynamicLoader.BackendLoader.GetBackend(target, options))
|
|
foreach (Duplicati.Library.Interface.IFileEntry fe in bk.List())
|
|
if (fe.Size > opts.VolumeSize)
|
|
{
|
|
string msg = string.Format("The file {0} is {1} bytes larger than allowed", fe.Name, fe.Size - opts.VolumeSize);
|
|
Console.WriteLine(msg);
|
|
Log.WriteMessage(msg, LogMessageType.Error);
|
|
}
|
|
|
|
List<Duplicati.Library.Main.ManifestEntry> entries = Duplicati.Library.Main.Interface.ParseFileList(target, options);
|
|
|
|
if (entries.Count != 1 || entries[0].Incrementals.Count != folders.Length - 1)
|
|
{
|
|
StringBuilder sb = new StringBuilder();
|
|
sb.AppendLine("Entry count: " + entries.Count.ToString());
|
|
if (entries.Count == 1)
|
|
sb.Append(string.Format("Found {0} incrementals but there were {1} source folders", entries[0].Incrementals.Count, folders.Length));
|
|
throw new Exception("Filename parsing problem, or corrupt storage: " + sb.ToString());
|
|
}
|
|
|
|
List<Duplicati.Library.Main.ManifestEntry> t = new List<Duplicati.Library.Main.ManifestEntry>();
|
|
t.Add(entries[0]);
|
|
t.AddRange(entries[0].Incrementals);
|
|
entries = t;
|
|
|
|
for (int i = 0; i < entries.Count; i++)
|
|
{
|
|
using (TempFolder ttf = new TempFolder())
|
|
{
|
|
log.Backupset = "Restore " + folders[i];
|
|
Console.WriteLine("Restoring the copy: " + folders[i]);
|
|
|
|
options["restore-time"] = entries[i].Time.ToString();
|
|
|
|
using (new Timer("Restore of " + folders[i]))
|
|
Log.WriteMessage(Duplicati.Library.Main.Interface.Restore(target, new string[] { ttf }, options), LogMessageType.Information);
|
|
|
|
Console.WriteLine("Verifying the copy: " + folders[i]);
|
|
|
|
using (new Timer("Verification of " + folders[i]))
|
|
{
|
|
string[] actualfolders = folders[i].Split(System.IO.Path.PathSeparator);
|
|
if (actualfolders.Length == 1)
|
|
VerifyDir(System.IO.Path.GetFullPath(folders[i]), ttf);
|
|
else
|
|
{
|
|
for (int j = 0; j < actualfolders.Length; j++)
|
|
VerifyDir(actualfolders[j], System.IO.Path.Combine(ttf, j.ToString()));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
(Log.CurrentLog as StreamLog).Dispose();
|
|
Log.CurrentLog = null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Verifies the existence of all files and folders, and ensures that all
|
|
/// files are binary equal.
|
|
/// </summary>
|
|
/// <param name="f1">One folder</param>
|
|
/// <param name="f2">Another folder</param>
|
|
private static void VerifyDir(string f1, string f2)
|
|
{
|
|
f1 = Utility.AppendDirSeparator(f1);
|
|
f2 = Utility.AppendDirSeparator(f2);
|
|
|
|
List<string> folders1 = Utility.EnumerateFolders(f1);
|
|
List<string> folders2 = Utility.EnumerateFolders(f2);
|
|
|
|
foreach (string s in folders1)
|
|
{
|
|
string relpath = s.Substring(f1.Length);
|
|
string target = System.IO.Path.Combine(f2, relpath);
|
|
if (!folders2.Contains(target))
|
|
{
|
|
Log.WriteMessage("Missing folder: " + relpath, LogMessageType.Error);
|
|
Console.WriteLine("Missing folder: " + relpath);
|
|
}
|
|
else
|
|
folders2.Remove(target);
|
|
}
|
|
|
|
foreach (string s in folders2)
|
|
{
|
|
Log.WriteMessage("Extra folder: " + s.Substring(f2.Length), LogMessageType.Error);
|
|
Console.WriteLine("Extra folder: " + s.Substring(f2.Length));
|
|
}
|
|
|
|
List<string> files1 = Utility.EnumerateFiles(f1);
|
|
List<string> files2 = Utility.EnumerateFiles(f2);
|
|
foreach (string s in files1)
|
|
{
|
|
string relpath = s.Substring(f1.Length);
|
|
string target = System.IO.Path.Combine(f2, relpath);
|
|
if (!files2.Contains(target))
|
|
{
|
|
Log.WriteMessage("Missing file: " + relpath, LogMessageType.Error);
|
|
Console.WriteLine("Missing file: " + relpath);
|
|
}
|
|
else
|
|
{
|
|
files2.Remove(target);
|
|
if (!CompareFiles(s, target, relpath))
|
|
{
|
|
Log.WriteMessage("File differs: " + relpath, LogMessageType.Error);
|
|
Console.WriteLine("File differs: " + relpath);
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach (string s in files2)
|
|
{
|
|
Log.WriteMessage("Extra file: " + s.Substring(f2.Length), LogMessageType.Error);
|
|
Console.WriteLine("Extra file: " + s.Substring(f2.Length));
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Compares two files by reading all bytes, and comparing one by one
|
|
/// </summary>
|
|
/// <param name="f1">One file</param>
|
|
/// <param name="f2">Another file</param>
|
|
/// <param name="display">File display name</param>
|
|
/// <returns>True if they are equal, false otherwise</returns>
|
|
private static bool CompareFiles(string f1, string f2, string display)
|
|
{
|
|
using (System.IO.FileStream fs1 = System.IO.File.OpenRead(f1))
|
|
using (System.IO.FileStream fs2 = System.IO.File.OpenRead(f2))
|
|
if (fs1.Length != fs2.Length)
|
|
{
|
|
Log.WriteMessage("Lengths differ: " + display + ", " + fs1.Length.ToString() + " vs. " + fs2.Length.ToString(), LogMessageType.Error);
|
|
Console.WriteLine("Lengths differ: " + display + ", " + fs1.Length.ToString() + " vs. " + fs2.Length.ToString());
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
long len = fs1.Length;
|
|
for (long l = 0; l < len; l++)
|
|
if (fs1.ReadByte() != fs2.ReadByte())
|
|
{
|
|
Log.WriteMessage("Mismatch in byte " + l.ToString() + " in file " + display, LogMessageType.Error);
|
|
Console.WriteLine("Mismatch in byte " + l.ToString() + " in file " + display);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
|
|
#endif |