From 7e1ea9176018a5e4ca4c8b1ab1237b5faee8a612 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Tue, 30 Jul 2013 22:42:59 +0200 Subject: [PATCH] Added rolling checks to ensure equal testing of all files. --- .../Main/Database/Database schema/Schema.sql | 3 +- .../Library/Main/Database/LocalDatabase.cs | 5 +- .../Main/Database/LocalTestDatabase.cs | 89 +++++++++++++++---- .../Library/Main/Operation/TestHandler.cs | 2 + 4 files changed, 78 insertions(+), 21 deletions(-) diff --git a/Duplicati/Library/Main/Database/Database schema/Schema.sql b/Duplicati/Library/Main/Database/Database schema/Schema.sql index 872a7767f..5f4c2c9a9 100644 --- a/Duplicati/Library/Main/Database/Database schema/Schema.sql +++ b/Duplicati/Library/Main/Database/Database schema/Schema.sql @@ -21,7 +21,8 @@ CREATE TABLE "Remotevolume" ( "Type" TEXT NOT NULL, "Size" INTEGER NULL, "Hash" TEXT NULL, - "State" TEXT NOT NULL + "State" TEXT NOT NULL, + "VerificationCount" INTEGER NOT NULL ); /* diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 57cba7db3..d17beb8d0 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -117,8 +117,8 @@ namespace Duplicati.Library.Main.Database m_selectremotevolumeIdCommand.CommandText = @"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = ?"; - m_createremotevolumeCommand.CommandText = @"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"") VALUES (?, ?, ?, ?); SELECT last_insert_rowid();"; - m_createremotevolumeCommand.AddParameters(4); + m_createremotevolumeCommand.CommandText = @"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"", ""VerificationCount"") VALUES (?, ?, ?, ?, ?); SELECT last_insert_rowid();"; + m_createremotevolumeCommand.AddParameters(5); m_insertIndexBlockLink.CommandText = @"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)"; m_insertIndexBlockLink.AddParameters(2); @@ -352,6 +352,7 @@ namespace Duplicati.Library.Main.Database m_createremotevolumeCommand.SetParameterValue(1, name); m_createremotevolumeCommand.SetParameterValue(2, type.ToString()); m_createremotevolumeCommand.SetParameterValue(3, state.ToString()); + m_createremotevolumeCommand.SetParameterValue(4, 0); m_createremotevolumeCommand.Transaction = tr.Parent; var r = Convert.ToInt64(m_createremotevolumeCommand.ExecuteScalar()); tr.Commit(); diff --git a/Duplicati/Library/Main/Database/LocalTestDatabase.cs b/Duplicati/Library/Main/Database/LocalTestDatabase.cs index 4256f1820..b22239a0e 100644 --- a/Duplicati/Library/Main/Database/LocalTestDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalTestDatabase.cs @@ -16,6 +16,7 @@ // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; +using System.Linq; using System.Collections.Generic; namespace Duplicati.Library.Main.Database @@ -27,12 +28,19 @@ namespace Duplicati.Library.Main.Database { } + public void UpdateVerificationCount(string name) + { + using(var cmd = m_connection.CreateCommand()) + cmd.ExecuteNonQuery(@"UPDATE ""RemoteVolume"" SET ""VerificationCount"" = MAX(1, CASE WHEN ""VerificationCount"" <= 0 THEN (SELECT MAX(""VerificationCount"") FROM ""RemoteVolume"") ELSE ""VerificationCount"" + 1 END) WHERE ""Name"" = ?", name); + } + private class RemoteVolume : IRemoteVolume { public long ID { get; private set; } public string Name { get; private set; } public long Size { get; private set; } public string Hash { get; private set; } + public long VerificationCount { get; private set; } public RemoteVolume(System.Data.IDataReader rd) { @@ -40,21 +48,76 @@ namespace Duplicati.Library.Main.Database this.Name = rd.GetValue(1).ToString(); this.Size = Convert.ToInt64(rd.GetValue(2)); this.Hash = rd.GetValue(3).ToString(); + this.VerificationCount = Convert.ToInt64(rd.GetValue(4)); } } + private IEnumerable FilterByVerificationCount(IEnumerable volumes, long samples, long maxverification) + { + var rnd = new Random(); + + // First round is the new items + var res = (from n in volumes where n.VerificationCount == 0 select n).ToList(); + while (res.Count > samples) + res.RemoveAt(rnd.Next(0, res.Count)); + + // Quick exit if we are done + if (res.Count == samples) + return res; + + // Next is the volumes that are not + // verified as much, with preference for low verification count + var starved = (from n in volumes where n.VerificationCount != 0 && n.VerificationCount < maxverification orderby n.VerificationCount select n); + if (starved.Any()) + { + var max = starved.Select(x => x.VerificationCount).Max(); + var min = starved.Select(x => x.VerificationCount).Min(); + + for(var i = min; i <= max; i++) + { + var p = starved.Where(x => x.VerificationCount == i).ToList(); + while (res.Count < samples && p.Count > 0) + { + var n = rnd.Next(0, p.Count); + res.Add(p[n]); + p.RemoveAt(n); + } + } + + // Quick exit if we are done + if (res.Count == samples) + return res; + } + + if (maxverification > 0) + { + // Last is the items that are verified mostly + var remainder = (from n in volumes where n.VerificationCount >= maxverification select n).ToList(); + while (res.Count < samples && remainder.Count > 0) + { + var n = rnd.Next(0, remainder.Count); + res.Add(remainder[n]); + remainder.RemoveAt(n); + } + } + + return res; + } + public IEnumerable SelectTestTargets(long samples, Options options) { var tp = GetFilelistWhereClause(options.Time, options.Version); - var rnd = new Random(); samples = Math.Max(1, samples); using(var cmd = m_connection.CreateCommand()) { + //Grab the max value + var max = Convert.ToInt64(cmd.ExecuteScalar(@"SELECT MAX(""VerificationCount"") FROM ""RemoteVolume""")); + //First we select some filesets var files = new List(); var whereClause = string.IsNullOrEmpty(tp.Item1) ? " WHERE " : (" " + tp.Item1 + " AND "); - using(var rd = cmd.ExecuteReader(@"SELECT ""A"".""VolumeID"", ""A"".""Name"", ""A"".""Size"", ""A"".""Hash"" FROM (SELECT ""ID"" AS ""VolumeID"", ""Name"", ""Size"", ""Hash"" FROM ""Remotevolume"") A, ""Fileset"" " + whereClause + @" ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" ORDER BY ""Fileset"".""Timestamp"" " , tp.Item2)) + using(var rd = cmd.ExecuteReader(@"SELECT ""A"".""VolumeID"", ""A"".""Name"", ""A"".""Size"", ""A"".""Hash"", ""A"".""VerificationCount"" FROM (SELECT ""ID"" AS ""VolumeID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"") A, ""Fileset"" " + whereClause + @" ""A"".""VolumeID"" = ""Fileset"".""VolumeID"" ORDER BY ""Fileset"".""Timestamp"" " , tp.Item2)) while (rd.Read()) files.Add(new RemoteVolume(rd)); @@ -62,11 +125,7 @@ namespace Duplicati.Library.Main.Database yield break; if (string.IsNullOrEmpty(tp.Item1)) - { - //No explicit fileset(s) selected, choose some samples - while (files.Count > samples) - files.RemoveAt(rnd.Next(files.Count)); - } + files = FilterByVerificationCount(files, samples, max).ToList(); foreach(var f in files) yield return f; @@ -74,26 +133,20 @@ namespace Duplicati.Library.Main.Database //Then we select some index files files.Clear(); - using(var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"" FROM ""Remotevolume"" WHERE ""Type"" = ? ", RemoteVolumeType.Index.ToString())) + using(var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""Type"" = ? ", RemoteVolumeType.Index.ToString())) while (rd.Read()) files.Add(new RemoteVolume(rd)); - - while (files.Count > samples) - files.RemoveAt(rnd.Next(files.Count)); - - foreach(var f in files) + + foreach(var f in FilterByVerificationCount(files, samples, max)) yield return f; //And finally some block files files.Clear(); - using(var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"" FROM ""Remotevolume"" WHERE ""Type"" = ? ", RemoteVolumeType.Blocks.ToString())) + using(var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Name"", ""Size"", ""Hash"", ""VerificationCount"" FROM ""Remotevolume"" WHERE ""Type"" = ? ", RemoteVolumeType.Blocks.ToString())) while (rd.Read()) files.Add(new RemoteVolume(rd)); - while (files.Count > samples) - files.RemoveAt(rnd.Next(files.Count)); - - foreach(var f in files) + foreach(var f in FilterByVerificationCount(files, samples, max)) yield return f; } } diff --git a/Duplicati/Library/Main/Operation/TestHandler.cs b/Duplicati/Library/Main/Operation/TestHandler.cs index 3a38f29f9..f39b6fc3b 100644 --- a/Duplicati/Library/Main/Operation/TestHandler.cs +++ b/Duplicati/Library/Main/Operation/TestHandler.cs @@ -146,6 +146,8 @@ namespace Duplicati.Library.Main.Operation } } + + db.UpdateVerificationCount(vol.Name); } catch (Exception ex) {