haeaeae/duplicati
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
b73e94de7ee24c5261e14f29bcceecb39b519aa8
duplicati/Duplicati/Library/Main/Database/LocalListChangesDatabase.cs
T

354 lines
22 KiB
C#
Raw Normal View History

Updated all license mentions to MIT
2024-02-28 15:45:30 +01:00
// Copyright (C) 2024, The Duplicati Team
// https://duplicati.com, hello@duplicati.com
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal in the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
Implemented list-changes
2013-06-20 20:17:10 +02:00
using System;
using System.Collections.Generic;
Use pattern matching to simplify casts.
2019-09-29 20:16:28 -07:00
using Duplicati.Library.Utility;
Implemented list-changes
2013-06-20 20:17:10 +02:00
namespace Duplicati.Library.Main.Database
{
Cleaned up some output with Backup and compacting.
2013-08-22 20:52:54 +02:00
internal class LocalListChangesDatabase : LocalDatabase
Implemented list-changes
2013-06-20 20:17:10 +02:00
{
public LocalListChangesDatabase(string path)
Fixed multiple cases where connections and command instances were not disposed.
2016-04-06 20:40:34 +02:00
: base(path, "ListChanges", false)
Implemented list-changes
2013-06-20 20:17:10 +02:00
{
Fixed multiple cases where connections and command instances were not disposed.
2016-04-06 20:40:34 +02:00
ShouldCloseConnection = true;
Implemented list-changes
2013-06-20 20:17:10 +02:00
}
public interface IStorageHelper : IDisposable
{
void AddElement(string path, string filehash, string metahash, long size, Library.Interface.ListChangesElementType type, bool asNew);
void AddFromDb(long filesetId, bool asNew, Library.Utility.IFilter filter);
IChangeCountReport CreateChangeCountReport();
IChangeSizeReport CreateChangeSizeReport();
IEnumerable<Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>> CreateChangedFileReport();
}
public interface IChangeCountReport
{
long AddedFolders { get; }
long AddedSymlinks { get; }
long AddedFiles { get; }
long DeletedFolders { get; }
long DeletedSymlinks { get; }
long DeletedFiles { get; }
long ModifiedFolders { get; }
long ModifiedSymlinks { get; }
long ModifiedFiles { get; }
}
public interface IChangeSizeReport
{
long AddedSize { get; }
long DeletedSize { get; }
long PreviousSize { get; }
long CurrentSize { get; }
}
internal class ChangeCountReport : IChangeCountReport
{
public long AddedFolders { get; internal set; }
public long AddedSymlinks { get; internal set; }
public long AddedFiles { get; internal set; }
public long DeletedFolders { get; internal set; }
public long DeletedSymlinks { get; internal set; }
public long DeletedFiles { get; internal set; }
public long ModifiedFolders { get; internal set; }
public long ModifiedSymlinks { get; internal set; }
public long ModifiedFiles { get; internal set; }
}
internal class ChangeSizeReport : IChangeSizeReport
{
public long AddedSize { get; internal set; }
public long DeletedSize { get; internal set; }
public long PreviousSize { get; internal set; }
public long CurrentSize { get; internal set; }
}
private class StorageHelper : IStorageHelper
{
Mark fields that shouldn't be reassigned as readonly.
2018-05-23 21:18:01 -07:00
private readonly System.Data.IDbConnection m_connection;
Implemented list-changes
2013-06-20 20:17:10 +02:00
private System.Data.IDbTransaction m_transaction;
private System.Data.IDbCommand m_insertPreviousElementCommand;
private System.Data.IDbCommand m_insertCurrentElementCommand;
private string m_previousTable;
private string m_currentTable;
public StorageHelper(System.Data.IDbConnection con)
{
m_connection = con;
m_previousTable = "Previous-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
m_currentTable = "Current-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
m_transaction = m_connection.BeginTransaction();
using(var cmd = m_connection.CreateCommand())
{
cmd.Transaction = m_transaction;
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_previousTable));
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL, ""FileHash"" TEXT NULL, ""MetaHash"" TEXT NOT NULL, ""Size"" INTEGER NOT NULL, ""Type"" INTEGER NOT NULL) ", m_currentTable));
}
m_insertPreviousElementCommand = m_connection.CreateCommand();
m_insertPreviousElementCommand.Transaction = m_transaction;
m_insertPreviousElementCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_previousTable);
m_insertPreviousElementCommand.AddParameters(5);
m_insertCurrentElementCommand = m_connection.CreateCommand();
m_insertCurrentElementCommand.Transaction = m_transaction;
m_insertCurrentElementCommand.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", m_currentTable);
m_insertCurrentElementCommand.AddParameters(5);
}
public void AddFromDb(long filesetId, bool asNew, Library.Utility.IFilter filter)
{
var tablename = asNew ? m_currentTable : m_previousTable;
var folders = string.Format(@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -100 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" ", (int)Library.Interface.ListChangesElementType.Folder);
var symlinks = string.Format(@"SELECT ""File"".""Path"" AS ""Path"", NULL AS ""FileHash"", ""Blockset"".""Fullhash"" AS ""MetaHash"", -1 AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = -200 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""Blockset"".""ID"" ", (int)Library.Interface.ListChangesElementType.Symlink);
var files = string.Format(@"SELECT ""File"".""Path"" AS ""Path"", ""FB"".""FullHash"" AS ""FileHash"", ""MB"".""Fullhash"" AS ""MetaHash"", ""FB"".""Length"" AS ""Size"", {0} AS ""Type"", ""FilesetEntry"".""FilesetID"" AS ""FilesetID"" FROM ""File"",""FilesetEntry"",""Metadataset"",""Blockset"" MB, ""Blockset"" FB WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" >= 0 AND ""Metadataset"".""ID""=""File"".""MetadataID"" AND ""Metadataset"".""BlocksetID"" = ""MB"".""ID"" AND ""File"".""BlocksetID"" = ""FB"".""ID"" ", (int)Library.Interface.ListChangesElementType.File);
var combined = "(" + folders + " UNION " + symlinks + " UNION " + files + ")";
using(var cmd = m_connection.CreateCommand())
{
cmd.Transaction = m_transaction;
if (filter == null || filter.Empty)
{
// Simple case, select everything
cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {1} A WHERE ""A"".""FilesetID"" = ? ", tablename, combined), filesetId);
}
Use pattern matching to simplify casts.
2019-09-29 20:16:28 -07:00
else if (Library.Utility.Utility.IsFSCaseSensitive && filter is FilterExpression expression && expression.Type == Duplicati.Library.Utility.FilterType.Simple)
Implemented list-changes
2013-06-20 20:17:10 +02:00
{
// File list based
Fixed issues with filtering/selecting files on case-insensitive filesystems.
2015-09-15 19:29:58 +02:00
// unfortunately we cannot do this if the filesystem is case sensitive as
// SQLite only supports ASCII compares
Use pattern matching to simplify casts.
2019-09-29 20:16:28 -07:00
var p = expression.GetSimpleList();
Implemented list-changes
2013-06-20 20:17:10 +02:00
var filenamestable = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL) ", filenamestable));
cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", filenamestable);
cmd.AddParameter();
foreach(var s in p)
{
cmd.SetParameterValue(0, s);
cmd.ExecuteNonQuery();
}
cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") SELECT ""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"" FROM {1} A WHERE ""A"".""FilesetID"" = ? AND ""A"".""Path"" IN (SELECT DISTINCT ""Path"" FROM ""{2}"") ", tablename, combined, filenamestable), filesetId);
Fixed a number of dispose errors that would mask real errors
2013-08-23 23:36:25 +02:00
cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", filenamestable));
Implemented list-changes
2013-06-20 20:17:10 +02:00
}
else
{
// Do row-wise iteration
object[] values = new object[5];
using(var cmd2 = m_connection.CreateCommand())
{
cmd2.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"", ""FileHash"", ""MetaHash"", ""Size"", ""Type"") VALUES (?,?,?,?,?)", tablename);
cmd2.AddParameters(5);
cmd2.Transaction = m_transaction;
Update LocalListChangesDatabase.cs
2017-10-03 11:08:43 +02:00
using(var rd = cmd.ExecuteReader(string.Format(@"SELECT ""A"".""Path"", ""A"".""FileHash"", ""A"".""MetaHash"", ""A"".""Size"", ""A"".""Type"" FROM {0} A WHERE ""A"".""FilesetID"" = ?", combined), filesetId))
Implemented list-changes
2013-06-20 20:17:10 +02:00
while (rd.Read())
{
rd.GetValues(values);
Fixed filters to include everything if empty.
2013-06-30 12:42:13 +02:00
if (values[0] != null && values[0] != DBNull.Value && Library.Utility.FilterExpression.Matches(filter, values[0].ToString()))
Implemented list-changes
2013-06-20 20:17:10 +02:00
{
cmd2.SetParameterValue(0, values[0]);
cmd2.SetParameterValue(1, values[1]);
cmd2.SetParameterValue(2, values[2]);
cmd2.SetParameterValue(3, values[3]);
cmd2.SetParameterValue(4, values[4]);
cmd2.ExecuteNonQuery();
}
}
}
}
}
}
public void AddElement(string path, string filehash, string metahash, long size, Library.Interface.ListChangesElementType type, bool asNew)
{
var cmd = asNew ? m_insertCurrentElementCommand : m_insertPreviousElementCommand;
cmd.SetParameterValue(0, path);
cmd.SetParameterValue(1, filehash);
cmd.SetParameterValue(2, metahash);
cmd.SetParameterValue(3, size);
cmd.SetParameterValue(4, (int)type);
cmd.ExecuteNonQuery();
}
private static IEnumerable<string> ReaderToStringList(System.Data.IDataReader rd)
{
using(rd)
while (rd.Read())
{
var v = rd.GetValue(0);
if (v == null || v == DBNull.Value)
yield return null;
else
yield return v.ToString();
}
}
private Tuple<string, string, string> GetSqls(bool allTypes)
{
return new Tuple<string, string, string>(
string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE {2} ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_currentTable, m_previousTable, string.Format(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_currentTable)),
string.Format(@"SELECT ""Path"" FROM ""{0}"" WHERE {2} ""{0}"".""Path"" NOT IN (SELECT ""Path"" FROM ""{1}"")", m_previousTable, m_currentTable, string.Format(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_previousTable)),
string.Format(@"SELECT ""{0}"".""Path"" FROM ""{0}"",""{1}"" WHERE {2} ""{0}"".""Path"" = ""{1}"".""Path"" AND (""{0}"".""FileHash"" != ""{1}"".""FileHash"" OR ""{0}"".""MetaHash"" != ""{1}"".""MetaHash"" OR ""{0}"".""Type"" != ""{1}"".""Type"") ", m_currentTable, m_previousTable, string.Format(allTypes ? "" : @" ""{0}"".""Type"" = ? AND ", m_currentTable))
);
}
Now using the file modification timestamp and the filesize as indicators that trigger file scanning.
2015-01-24 21:59:53 +01:00
Implemented list-changes
2013-06-20 20:17:10 +02:00
public IChangeSizeReport CreateChangeSizeReport()
{
var sqls = GetSqls(true);
var added = sqls.Item1;
var deleted = sqls.Item2;
//var modified = sqls.Item3;
using(var cmd = m_connection.CreateCommand())
{
cmd.Transaction = m_transaction;
var result = new ChangeSizeReport();
Now using the file modification timestamp and the filesize as indicators that trigger file scanning.
2015-01-24 21:59:53 +01:00
result.PreviousSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""Size"") FROM ""{0}"" ", m_previousTable), 0);
result.CurrentSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""Size"") FROM ""{0}"" ", m_currentTable), 0);
Implemented list-changes
2013-06-20 20:17:10 +02:00
Now using the file modification timestamp and the filesize as indicators that trigger file scanning.
2015-01-24 21:59:53 +01:00
result.AddedSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""Size"") FROM ""{0}"" WHERE ""{0}"".""Path"" IN ({1}) ", m_currentTable, added), 0);
result.DeletedSize = cmd.ExecuteScalarInt64(string.Format(@"SELECT SUM(""Size"") FROM ""{0}"" WHERE ""{0}"".""Path"" IN ({1}) ", m_previousTable, deleted), 0);
Implemented list-changes
2013-06-20 20:17:10 +02:00
return result;
}
}
public IChangeCountReport CreateChangeCountReport()
{
var sqls = GetSqls(false);
var added = @"SELECT COUNT(*) FROM (" + sqls.Item1 + ")";
var deleted = @"SELECT COUNT(*) FROM (" + sqls.Item2 + ")";
var modified = @"SELECT COUNT(*) FROM (" + sqls.Item3 + ")";
using(var cmd = m_connection.CreateCommand())
{
cmd.Transaction = m_transaction;
var result = new ChangeCountReport();
Now using the file modification timestamp and the filesize as indicators that trigger file scanning.
2015-01-24 21:59:53 +01:00
result.AddedFolders = cmd.ExecuteScalarInt64(added, 0, (int)Library.Interface.ListChangesElementType.Folder);
result.AddedSymlinks = cmd.ExecuteScalarInt64(added, 0, (int)Library.Interface.ListChangesElementType.Symlink);
result.AddedFiles = cmd.ExecuteScalarInt64(added, 0, (int)Library.Interface.ListChangesElementType.File);
Implemented list-changes
2013-06-20 20:17:10 +02:00
Now using the file modification timestamp and the filesize as indicators that trigger file scanning.
2015-01-24 21:59:53 +01:00
result.DeletedFolders = cmd.ExecuteScalarInt64(deleted, 0, (int)Library.Interface.ListChangesElementType.Folder);
result.DeletedSymlinks = cmd.ExecuteScalarInt64(deleted, 0, (int)Library.Interface.ListChangesElementType.Symlink);
result.DeletedFiles = cmd.ExecuteScalarInt64(deleted, 0, (int)Library.Interface.ListChangesElementType.File);
Implemented list-changes
2013-06-20 20:17:10 +02:00
Now using the file modification timestamp and the filesize as indicators that trigger file scanning.
2015-01-24 21:59:53 +01:00
result.ModifiedFolders = cmd.ExecuteScalarInt64(modified, 0, (int)Library.Interface.ListChangesElementType.Folder);
result.ModifiedSymlinks = cmd.ExecuteScalarInt64(modified, 0, (int)Library.Interface.ListChangesElementType.Symlink);
result.ModifiedFiles = cmd.ExecuteScalarInt64(modified, 0, (int)Library.Interface.ListChangesElementType.File);
Implemented list-changes
2013-06-20 20:17:10 +02:00
return result;
}
}
public IEnumerable<Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>> CreateChangedFileReport()
{
var sqls = GetSqls(false);
var added = sqls.Item1;
var deleted = sqls.Item2;
var modified = sqls.Item3;
using(var cmd = m_connection.CreateCommand())
{
cmd.Transaction = m_transaction;
foreach(var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.Folder)))
yield return new Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>(Library.Interface.ListChangesChangeType.Added, Library.Interface.ListChangesElementType.Folder, s);
foreach(var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.Symlink)))
yield return new Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>(Library.Interface.ListChangesChangeType.Added, Library.Interface.ListChangesElementType.Symlink, s);
foreach(var s in ReaderToStringList(cmd.ExecuteReader(added, (int)Library.Interface.ListChangesElementType.File)))
yield return new Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>(Library.Interface.ListChangesChangeType.Added, Library.Interface.ListChangesElementType.File, s);
foreach(var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.Folder)))
yield return new Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>(Library.Interface.ListChangesChangeType.Deleted, Library.Interface.ListChangesElementType.Folder, s);
foreach(var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.Symlink)))
yield return new Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>(Library.Interface.ListChangesChangeType.Deleted, Library.Interface.ListChangesElementType.Symlink, s);
foreach(var s in ReaderToStringList(cmd.ExecuteReader(deleted, (int)Library.Interface.ListChangesElementType.File)))
yield return new Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>(Library.Interface.ListChangesChangeType.Deleted, Library.Interface.ListChangesElementType.File, s);
foreach(var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.Folder)))
yield return new Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>(Library.Interface.ListChangesChangeType.Modified, Library.Interface.ListChangesElementType.Folder, s);
foreach(var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.Symlink)))
yield return new Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>(Library.Interface.ListChangesChangeType.Modified, Library.Interface.ListChangesElementType.Symlink, s);
foreach(var s in ReaderToStringList(cmd.ExecuteReader(modified, (int)Library.Interface.ListChangesElementType.File)))
yield return new Tuple<Library.Interface.ListChangesChangeType, Library.Interface.ListChangesElementType, string>(Library.Interface.ListChangesChangeType.Modified, Library.Interface.ListChangesElementType.File, s);
}
}
public void Dispose()
{
if (m_insertPreviousElementCommand != null)
{
try { m_insertPreviousElementCommand.Dispose(); }
catch {}
finally { m_insertPreviousElementCommand = null; }
}
if (m_insertCurrentElementCommand != null)
{
try { m_insertCurrentElementCommand.Dispose(); }
catch {}
finally { m_insertCurrentElementCommand = null; }
}
if (m_transaction != null)
{
try { m_transaction.Rollback(); }
catch {}
finally
{
m_previousTable = null;
m_currentTable= null;
m_transaction = null;
}
}
}
}
public IStorageHelper CreateStorageHelper()
{
return new StorageHelper(m_connection);
}
}
}
Reference in New Issue Copy Permalink