duplicati/Duplicati/Library/Main/Database/LocalDatabase.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

namespace Duplicati.Library.Main.Database
{
    internal class LocalDatabase : IDisposable
    {        
        protected readonly System.Data.IDbConnection m_connection;
        protected readonly long m_operationid = -1;

        private readonly System.Data.IDbCommand m_updateremotevolumeCommand;
        private readonly System.Data.IDbCommand m_selectremotevolumesCommand;
        private readonly System.Data.IDbCommand m_selectremotevolumeCommand;
        private readonly System.Data.IDbCommand m_removeremotevolumeCommand;
        private readonly System.Data.IDbCommand m_selectremotevolumeIdCommand;
        private readonly System.Data.IDbCommand m_createremotevolumeCommand;
        private readonly System.Data.IDbCommand m_selectduplicateRemoteVolumesCommand;

        private readonly System.Data.IDbCommand m_insertlogCommand;
        private readonly System.Data.IDbCommand m_insertremotelogCommand;
        private readonly System.Data.IDbCommand m_insertIndexBlockLink;

        protected BasicResults m_result;

        public const long FOLDER_BLOCKSET_ID = -100;
        public const long SYMLINK_BLOCKSET_ID = -200;

        public DateTime OperationTimestamp { get; private set; }

        internal System.Data.IDbConnection Connection { get { return m_connection; } }
        
        public bool IsDisposed { get; private set; }

        public bool ShouldCloseConnection { get; set; }

        private System.Data.IDbTransaction m_transaction;
        private LocalDatabase m_parent;

        internal System.Data.IDbTransaction Transaction { get { return m_parent?.Transaction ?? m_transaction; } }

        protected static System.Data.IDbConnection CreateConnection(string path)
        {
            path = System.IO.Path.GetFullPath(path);
            if (!System.IO.Directory.Exists(System.IO.Path.GetDirectoryName(path)))
                System.IO.Directory.CreateDirectory(System.IO.Path.GetDirectoryName(path));
            
            var c = Duplicati.Library.SQLiteHelper.SQLiteLoader.LoadConnection(path);

            Library.SQLiteHelper.DatabaseUpgrader.UpgradeDatabase(c, path, typeof(LocalDatabase));
            
            return c;
        }

        /// <summary>
        /// Creates a new database instance and starts a new operation
        /// </summary>
        /// <param name="path">The path to the database</param>
        /// <param name="operation">The name of the operation</param>
        public LocalDatabase(string path, string operation, bool shouldclose)
            : this(CreateConnection(path), operation)
        {
            ShouldCloseConnection = shouldclose;
            this.m_transaction = m_connection.BeginTransaction();
        }

        /// <summary>
        /// Creates a new database instance and starts a new operation
        /// </summary>
        public LocalDatabase(LocalDatabase db)
            : this(db.m_connection)
        {
            this.OperationTimestamp = db.OperationTimestamp;
            this.m_connection = db.m_connection;
            this.m_operationid = db.m_operationid;
            this.m_result = db.m_result;
            this.m_parent = db;
        }
        
        /// <summary>
        /// Creates a new database instance and starts a new operation
        /// </summary>
        /// <param name="operation">The name of the operation</param>
        private LocalDatabase(System.Data.IDbConnection connection, string operation)
            : this(connection)
        {
            this.OperationTimestamp = DateTime.UtcNow;
            m_connection = connection;

            if (m_connection.State != System.Data.ConnectionState.Open)
                m_connection.Open();

            using (var cmd = m_connection.CreateCommand())
                m_operationid = cmd.ExecuteScalarInt64( @"INSERT INTO ""Operation"" (""Description"", ""Timestamp"") VALUES (?, ?); SELECT last_insert_rowid();", -1, operation, NormalizeDateTimeToEpochSeconds(OperationTimestamp));
        }
        
        private LocalDatabase(System.Data.IDbConnection connection)
        {
            m_updateremotevolumeCommand = connection.CreateCommand();
            m_selectremotevolumesCommand = connection.CreateCommand();
            m_selectduplicateRemoteVolumesCommand = connection.CreateCommand();
            m_selectremotevolumeCommand = connection.CreateCommand();
            m_insertlogCommand = connection.CreateCommand();
            m_insertremotelogCommand = connection.CreateCommand();
            m_removeremotevolumeCommand = connection.CreateCommand();
            m_selectremotevolumeIdCommand = connection.CreateCommand();
            m_createremotevolumeCommand = connection.CreateCommand();
            m_insertIndexBlockLink = connection.CreateCommand();

            m_insertlogCommand.CommandText = @"INSERT INTO ""LogData"" (""OperationID"", ""Timestamp"", ""Type"", ""Message"", ""Exception"") VALUES (?, ?, ?, ?, ?)";
            m_insertlogCommand.AddParameters(5);

            m_insertremotelogCommand.CommandText = @"INSERT INTO ""RemoteOperation"" (""OperationID"", ""Timestamp"", ""Operation"", ""Path"", ""Data"") VALUES (?, ?, ?, ?, ?)";
            m_insertremotelogCommand.AddParameters(5);

            m_updateremotevolumeCommand.CommandText = @"UPDATE ""Remotevolume"" SET ""OperationID"" = ?, ""State"" = ?, ""Hash"" = ?, ""Size"" = ? WHERE ""Name"" = ?";
            m_updateremotevolumeCommand.AddParameters(5);

            m_selectremotevolumesCommand.CommandText = @"SELECT ""ID"", ""Name"", ""Type"", ""Size"", ""Hash"", ""State"", ""DeleteGraceTime"" FROM ""Remotevolume""";

            m_selectremotevolumeCommand.CommandText = m_selectremotevolumesCommand.CommandText + @" WHERE ""Name"" = ?";
            m_selectduplicateRemoteVolumesCommand.CommandText = string.Format(@"SELECT DISTINCT ""Name"", ""State"" FROM ""Remotevolume"" WHERE ""Name"" IN (SELECT ""Name"" FROM ""Remotevolume"" WHERE ""State"" IN (""{0}"", ""{1}"")) AND NOT ""State"" IN (""{0}"", ""{1}"")", RemoteVolumeState.Deleted.ToString(), RemoteVolumeState.Deleting.ToString());

            m_selectremotevolumeCommand.AddParameter();

            m_removeremotevolumeCommand.CommandText = @"DELETE FROM ""Remotevolume"" WHERE ""Name"" = ? AND (""DeleteGraceTime"" < ? OR ""State"" != ?)";
            m_removeremotevolumeCommand.AddParameters(3);

            m_selectremotevolumeIdCommand.CommandText = @"SELECT ""ID"" FROM ""Remotevolume"" WHERE ""Name"" = ?";

            m_createremotevolumeCommand.CommandText = @"INSERT INTO ""Remotevolume"" (""OperationID"", ""Name"", ""Type"", ""State"", ""Size"", ""VerificationCount"", ""DeleteGraceTime"") VALUES (?, ?, ?, ?, ?, ?, ?); SELECT last_insert_rowid();";
            m_createremotevolumeCommand.AddParameters(7);

            m_insertIndexBlockLink.CommandText = @"INSERT INTO ""IndexBlockLink"" (""IndexVolumeID"", ""BlockVolumeID"") VALUES (?, ?)";
            m_insertIndexBlockLink.AddParameters(2);
        }

        internal void SetResult(BasicResults result)
        {
            m_result = result;
        }
        
        /// <summary>
        /// Normalizes a DateTime instance floor'ed to seconds and in UTC
        /// </summary>
        /// <returns>The normalised date time</returns>
        /// <param name="input">The input time</param>
        public static DateTime NormalizeDateTime(DateTime input)
        {
            var ticks = input.ToUniversalTime().Ticks;
            ticks -= ticks % TimeSpan.TicksPerSecond;
            return new DateTime(ticks, DateTimeKind.Utc);
        }
        
        public static long NormalizeDateTimeToEpochSeconds(DateTime input)
        {
            return (long)Math.Floor((NormalizeDateTime(input) - Library.Utility.Utility.EPOCH).TotalSeconds);
        }
        
        /// <summary>
        /// Creates a DateTime instance by adding the specified number of seconds to the EPOCH value
        /// </summary>        
        public static DateTime ParseFromEpochSeconds(long seconds)
        {
            return Library.Utility.Utility.EPOCH.AddSeconds(seconds);
        }

        public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash)
        { 
            UpdateRemoteVolume(name, state, size, hash, false); 
        }

        public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup)
        {
            UpdateRemoteVolume(name, state, size, hash, suppressCleanup, new TimeSpan(0)); 
        }

        public void UpdateRemoteVolume(string name, RemoteVolumeState state, long size, string hash, bool suppressCleanup, TimeSpan deleteGraceTime)
        {
            m_updateremotevolumeCommand.Transaction = Transaction;
            m_updateremotevolumeCommand.SetParameterValue(0, m_operationid);
            m_updateremotevolumeCommand.SetParameterValue(1, state.ToString());
            m_updateremotevolumeCommand.SetParameterValue(2, hash);
            m_updateremotevolumeCommand.SetParameterValue(3, size);
            m_updateremotevolumeCommand.SetParameterValue(4, name);
            var c = m_updateremotevolumeCommand.ExecuteNonQuery();
            if (c != 1)
                throw new Exception(string.Format("Unexpected number of remote volumes detected: {0}!", c));

            if (deleteGraceTime.Ticks > 0)
                using(var cmd = m_connection.CreateCommand(Transaction))
                    if ((c = cmd.ExecuteNonQuery(@"UPDATE ""RemoteVolume"" SET ""DeleteGraceTime"" = ? WHERE ""Name"" = ? ", (DateTime.UtcNow + deleteGraceTime).Ticks, name)) != 1)
                        throw new Exception(string.Format("Unexpected number of updates when recording remote volume updates: {0}!", c));


            if (!suppressCleanup && state == RemoteVolumeState.Deleted)
                   RemoveRemoteVolume(name);
        }
        
        public IEnumerable<KeyValuePair<long, DateTime>> FilesetTimes
        { 
            get 
            {
                using(var cmd = m_connection.CreateCommand(Transaction))
                using(var rd = cmd.ExecuteReader(@"SELECT ""ID"", ""Timestamp"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC"))
                    while (rd.Read())
                        yield return new KeyValuePair<long, DateTime>(rd.GetInt64(0), ParseFromEpochSeconds(rd.GetInt64(1)).ToLocalTime());
            }
        }

        public Tuple<string, object[]> GetFilelistWhereClause(DateTime time, long[] versions, IEnumerable<KeyValuePair<long, DateTime>> filesetslist = null, bool singleTimeMatch = false)
        {
            var filesets = (filesetslist ?? this.FilesetTimes).ToArray();
            string query = "";
            var args = new List<object>();
            if (time.Ticks > 0 || (versions != null && versions.Length > 0))
            {
                var hasTime = false;
                if (time.Ticks > 0)
                {
                    if (time.Kind == DateTimeKind.Unspecified)
                        throw new Exception("Invalid DateTime given, must be either local or UTC");
            
                    query += singleTimeMatch ? @" ""Timestamp"" = ?" : @" ""Timestamp"" <= ?";
                    // Make sure the resolution is the same (i.e. no milliseconds)
                    args.Add(NormalizeDateTimeToEpochSeconds(time));
                    hasTime = true;
                }
                
                if (versions != null && versions.Length > 0)
                {
                    var qs = "";
                    
                    foreach(var v in versions)
                        if (v >= 0 && v < filesets.Length)
                        {
                            args.Add(filesets[v].Key);
                            qs += "?,";
                        }
                        else
                            m_result.AddWarning(string.Format("Skipping invalid version: {0}", v), null);
                            
                        
                    if (qs.Length > 0)
                    {
                        qs = qs.Substring(0, qs.Length - 1);
                        
                        if (hasTime)
                            query += " OR ";
                                            
                        query += @" ""ID"" IN (" + qs + ")";
                    }
                }
                
                if (!string.IsNullOrEmpty(query))
                    query = " WHERE " + query;

            }
            
            return new Tuple<string, object[]>(query, args.ToArray());
        }

        public long GetRemoteVolumeID(string file)
        {
            m_selectremotevolumeIdCommand.Transaction = Transaction;
            return m_selectremotevolumeIdCommand.ExecuteScalarInt64(null, -1, file);
        }

        public RemoteVolumeEntry GetRemoteVolume(string file)
        {
            m_selectremotevolumeCommand.Transaction = Transaction;
            m_selectremotevolumeCommand.SetParameterValue(0, file);
            using(var rd = m_selectremotevolumeCommand.ExecuteReader())
                if (rd.Read())
                    return new RemoteVolumeEntry(
                        rd.ConvertValueToInt64(0),
                        rd.GetValue(1).ToString(),
                        (rd.GetValue(4) == null || rd.GetValue(4) == DBNull.Value) ? null : rd.GetValue(4).ToString(),
                        rd.ConvertValueToInt64(3, -1),
                        (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.GetValue(2).ToString()),
                        (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(5).ToString()),
                        new DateTime(rd.ConvertValueToInt64(6, 0), DateTimeKind.Utc)
                    );
            
            return RemoteVolumeEntry.Empty;
        }

        public IEnumerable<KeyValuePair<string, RemoteVolumeState>> DuplicateRemoteVolumes()
        {
            m_selectduplicateRemoteVolumesCommand.Transaction = Transaction;
            foreach(var rd in m_selectduplicateRemoteVolumesCommand.ExecuteReaderEnumerable(null))
            {
                yield return new KeyValuePair<string, RemoteVolumeState>(
                    rd.GetValue(0).ToString(),
                    (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(1).ToString())
                );
            }
        }

        public IEnumerable<RemoteVolumeEntry> GetRemoteVolumes()
        {
            m_selectremotevolumesCommand.Transaction = Transaction;
            using (var rd = m_selectremotevolumesCommand.ExecuteReader())
            {
                while (rd.Read())
                {
                    yield return new RemoteVolumeEntry(
                        rd.ConvertValueToInt64(0),
                        rd.GetValue(1).ToString(),
                        (rd.GetValue(4) == null || rd.GetValue(4) == DBNull.Value) ? null : rd.GetValue(4).ToString(),
                        rd.ConvertValueToInt64(3, -1),
                        (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), rd.GetValue(2).ToString()),
                        (RemoteVolumeState)Enum.Parse(typeof(RemoteVolumeState), rd.GetValue(5).ToString()),
                        new DateTime(rd.ConvertValueToInt64(6, 0), DateTimeKind.Utc)
                    );
                }
            }
        }

        /// <summary>
        /// Log an operation performed on the remote backend
        /// </summary>
        /// <param name="operation">The operation performed</param>
        /// <param name="path">The path involved</param>
        /// <param name="data">Any data relating to the operation</param>
        public void LogRemoteOperation(string operation, string path, string data)
        {
            m_insertremotelogCommand.Transaction = Transaction;
            m_insertremotelogCommand.SetParameterValue(0, m_operationid);
            m_insertremotelogCommand.SetParameterValue(1, NormalizeDateTimeToEpochSeconds(DateTime.UtcNow));
            m_insertremotelogCommand.SetParameterValue(2, operation);
            m_insertremotelogCommand.SetParameterValue(3, path);
            m_insertremotelogCommand.SetParameterValue(4, data);
            m_insertremotelogCommand.ExecuteNonQuery();
        }

        /// <summary>
        /// Log a debug message
        /// </summary>
        /// <param name="type">The message type</param>
        /// <param name="message">The message</param>
        /// <param name="exception">An optional exception</param>
        public void LogMessage(string type, string message, Exception exception)
        {
            m_insertlogCommand.Transaction = Transaction;
            m_insertlogCommand.SetParameterValue(0, m_operationid);
            m_insertlogCommand.SetParameterValue(1, NormalizeDateTimeToEpochSeconds(DateTime.UtcNow));
            m_insertlogCommand.SetParameterValue(2, type);
            m_insertlogCommand.SetParameterValue(3, message);
            m_insertlogCommand.SetParameterValue(4, exception == null ? null : exception.ToString());
            m_insertlogCommand.ExecuteNonQuery();
        }

        public void UnlinkRemoteVolume(string name, RemoteVolumeState state)
        {
            using(var cmd = m_connection.CreateCommand())
            {
                cmd.Transaction = Transaction;
                var c = cmd.ExecuteNonQuery(@"DELETE FROM ""RemoteVolume"" WHERE ""Name"" = ? AND ""State"" = ? ", name, state.ToString());
                if (c != 1)
                    throw new Exception(string.Format("Unexpected number of remote volumes deleted: {0}, expected {1}", c, 1));
            }
        }

        public void RemoveRemoteVolume(string name)
        {
            RemoveRemoteVolumes(new string[] { name });
        }

        public void RemoveRemoteVolumes(IEnumerable<string> names)
        {
            if (names == null || !names.Any()) return;

            using (var deletecmd = m_connection.CreateCommand())
            {
                deletecmd.Transaction = Transaction;

                string temptransguid = Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
                var volidstable = "DelVolSetIds-" + temptransguid;
                var blocksetidstable = "DelBlockSetIds-" + temptransguid;

                // Create and fill a temp table with the volids to delete. We avoid using too many parameters that way.
                deletecmd.ExecuteNonQuery(string.Format(@"CREATE TEMP TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY)", volidstable));
                deletecmd.CommandText = string.Format(@"INSERT OR IGNORE INTO ""{0}"" (""ID"") VALUES (?)", volidstable);
                deletecmd.Parameters.Clear();
                deletecmd.AddParameters(1);
                foreach (var name in names)
                {
                    var volumeid = GetRemoteVolumeID(name);
                    deletecmd.SetParameterValue(0, volumeid);
                    deletecmd.ExecuteNonQuery();
                }
                var volIdsSubQuery = string.Format(@"SELECT ""ID"" FROM ""{0}"" ", volidstable);
                deletecmd.Parameters.Clear();
                
                // If the volume is a block or index volume, this will update the crosslink table, otherwise nothing will happen
                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""IndexBlockLink"" WHERE ""BlockVolumeID"" IN ({0}) OR ""IndexVolumeID"" IN ({0})", volIdsSubQuery));
                
                // If the volume is a fileset, this will remove the fileset, otherwise nothing will happen
                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""FilesetEntry"" WHERE ""FilesetID"" IN (SELECT ""ID"" FROM ""Fileset"" WHERE ""VolumeID"" IN ({0}))", volIdsSubQuery));
                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Fileset"" WHERE ""VolumeID""  IN ({0})", volIdsSubQuery));
                                                
                var bsIdsSubQuery = string.Format(
                      @"SELECT ""BlocksetEntry"".""BlocksetID"" FROM ""BlocksetEntry"", ""Block"" "
                    + @" WHERE ""BlocksetEntry"".""BlockID"" = ""Block"".""ID"" AND ""Block"".""VolumeID"" IN ({0}) "
                    + @"UNION ALL "
                    + @"SELECT ""BlocksetID"" FROM ""BlocklistHash"" "
                    + @"WHERE ""Hash"" IN (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN ({0}))"
                    , volIdsSubQuery);

                // Create a temporary table to cache subquery result, as it might take long (SQLite does not cache at all). 
                deletecmd.ExecuteNonQuery(string.Format(@"CREATE TEMP TABLE ""{0}"" (""ID"" INTEGER PRIMARY KEY)", blocksetidstable));
                deletecmd.ExecuteNonQuery(string.Format(@"INSERT OR IGNORE INTO ""{0}"" (""ID"") {1}", blocksetidstable, bsIdsSubQuery));
                bsIdsSubQuery = string.Format(@"SELECT ""ID"" FROM ""{0}"" ", blocksetidstable);
                deletecmd.Parameters.Clear();

                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""File"" WHERE ""BlocksetID"" IN ({0}) OR ""MetadataID"" IN ({0})", bsIdsSubQuery));
                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Metadataset"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery));
                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Blockset"" WHERE ""ID"" IN ({0})", bsIdsSubQuery));
                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""BlocksetEntry"" WHERE ""BlocksetID"" IN ({0})", bsIdsSubQuery));

                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""BlocklistHash"" WHERE ""Hash"" IN (SELECT ""Hash"" FROM ""Block"" WHERE ""VolumeID"" IN ({0}))", volIdsSubQuery));
                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""Block"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery));
                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""DeletedBlock"" WHERE ""VolumeID"" IN ({0})", volIdsSubQuery));

                // Clean up temp tables for subqueries. We truncate content and then try to delete.
                // Drop in try-block, as it fails in nested transactions (SQLite problem)
                // System.Data.SQLite.SQLiteException (0x80004005): database table is locked
                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}"" ", blocksetidstable));
                deletecmd.ExecuteNonQuery(string.Format(@"DELETE FROM ""{0}"" ", volidstable));
                try
                {
                    deletecmd.CommandTimeout = 2;
                    deletecmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", blocksetidstable));
                    deletecmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", volidstable));
                }
                catch { /* Ignore, will be deleted on close anyway. */ }

                m_removeremotevolumeCommand.Transaction = Transaction;
                m_removeremotevolumeCommand.SetParameterValue(1, DateTime.UtcNow.Ticks);
				m_removeremotevolumeCommand.SetParameterValue(2, RemoteVolumeState.Deleted.ToString());
				foreach (var name in names)
                {
                    m_removeremotevolumeCommand.SetParameterValue(0, name);
                    m_removeremotevolumeCommand.ExecuteNonQuery();
                }
            }
        }
        
        public void Vacuum()
        {
            using(var cmd = m_connection.CreateCommand())
                cmd.ExecuteNonQuery("VACUUM");
        }

        public long RegisterRemoteVolume(string name, RemoteVolumeType type, long size, RemoteVolumeState state)
        {
            return RegisterRemoteVolume(name, type, state, size, new TimeSpan(0));
        }

        public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state)
        {
            return RegisterRemoteVolume(name, type, state, new TimeSpan(0));
        }

        public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, TimeSpan deleteGraceTime)
        {
            return RegisterRemoteVolume(name, type, state, -1, deleteGraceTime);
        }
            
        public long RegisterRemoteVolume(string name, RemoteVolumeType type, RemoteVolumeState state, long size, TimeSpan deleteGraceTime)
        {
            m_createremotevolumeCommand.SetParameterValue(0, m_operationid);
            m_createremotevolumeCommand.SetParameterValue(1, name);
            m_createremotevolumeCommand.SetParameterValue(2, type.ToString());
            m_createremotevolumeCommand.SetParameterValue(3, state.ToString());
            m_createremotevolumeCommand.SetParameterValue(4, size);
            m_createremotevolumeCommand.SetParameterValue(5, 0);

            if (deleteGraceTime.Ticks <= 0)
                m_createremotevolumeCommand.SetParameterValue(6, 0);
            else
                m_createremotevolumeCommand.SetParameterValue(6, (DateTime.UtcNow + deleteGraceTime).Ticks);
            
            m_createremotevolumeCommand.Transaction = Transaction;
            return m_createremotevolumeCommand.ExecuteScalarInt64();
        }

        public long GetFilesetID(DateTime restoretime, long[] versions)
        {
            return GetFilesetIDs(restoretime, versions).First();
        }        

        public IEnumerable<long> GetFilesetIDs(DateTime restoretime, long[] versions)
        {
            if (restoretime.Kind == DateTimeKind.Unspecified)
                throw new Exception("Invalid DateTime given, must be either local or UTC");

            var tmp = GetFilelistWhereClause(restoretime, versions);
            string query = tmp.Item1;
            var args = tmp.Item2;

            var res = new List<long>();
            using(var cmd = m_connection.CreateCommand())
            {            
                using(var rd = cmd.ExecuteReader(@"SELECT ""ID"" FROM ""Fileset"" " + query  + @" ORDER BY ""Timestamp"" DESC", args))
                    while (rd.Read())
                        res.Add(rd.GetInt64(0));
                        
                if (res.Count == 0)
                {
                    cmd.Parameters.Clear();
                    using(var rd = cmd.ExecuteReader(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC "))
                    while (rd.Read())
                        res.Add(rd.ConvertValueToInt64(0));
                    
                    if (res.Count == 0)
                        throw new Duplicati.Library.Interface.UserInformationException("No backup at the specified date");
                    else
                        m_result.AddWarning(string.Format("Restore time or version did not match any existing backups, selecting newest backup"), null);
                }

                return res;
            }
        }

        public IEnumerable<long> FindMatchingFilesets(DateTime restoretime, long[] versions)
        {
            if (restoretime.Kind == DateTimeKind.Unspecified)
                throw new Exception("Invalid DateTime given, must be either local or UTC");

            var tmp = GetFilelistWhereClause(restoretime, versions, singleTimeMatch: true);
            string query = tmp.Item1;
            var args = tmp.Item2;

            var res = new List<long>();
            using(var cmd = m_connection.CreateCommand())
            using(var rd = cmd.ExecuteReader(@"SELECT ""ID"" FROM ""Fileset"" " + query  + @" ORDER BY ""Timestamp"" DESC", args))
                while (rd.Read())
                    res.Add(rd.GetInt64(0));

            return res;
        }

        /// <summary>
        /// Writes the current changes to the database
        /// </summary>
        /// <returns>An awaitable task.</returns>
        /// <param name="message">The message to use for logging the time spent in this operation.</param>
        /// <param name="restart">If set to <c>true</c>, a transaction will be started again after this call.</param>
        public void CommitTransaction(string message, bool restart = true)
        {
            if (m_parent != null)
                m_parent.CommitTransaction(message, restart);
            else
            {
                using (new Logging.Timer(message))
                {
                    m_transaction.Commit();
                    m_transaction = restart ? m_connection.BeginTransaction() : null;
                }
            }
        }

        public void RollbackTransaction(bool restart = true)
        {
            if (m_parent != null)
                m_parent.RollbackTransaction();
            else
            {
                m_transaction.Rollback();
                if (restart)
                    m_transaction = m_connection.BeginTransaction();
            }
        }


        private class LocalFileEntry : ILocalFileEntry
        {
            private System.Data.IDataReader m_reader;
            public LocalFileEntry(System.Data.IDataReader reader)
            {
                m_reader = reader;
            }

            public string Path
            {
                get 
                {
                    var c = m_reader.GetValue(0);
                    if (c == null || c == DBNull.Value)
                        return null;
                    return c.ToString();
                }
            }

            public long Length
            {
                get
                {
                    return m_reader.ConvertValueToInt64(1);;
                }
            }

            public string Hash
            {
                get
                {
                    var c = m_reader.GetValue(2);
                    if (c == null || c == DBNull.Value)
                        return null;
                    return c.ToString();
                }
            }

            public string Metahash
            {
                get
                {
                    var c = m_reader.GetValue(3);
                    if (c == null || c == DBNull.Value)
                        return null;
                    return c.ToString();
                }
            }
        }
        
        public IEnumerable<ILocalFileEntry> GetFiles(long filesetId)
        {
            using(var cmd = m_connection.CreateCommand())
            using(var rd = cmd.ExecuteReader(@"SELECT ""A"".""Path"", ""B"".""Length"", ""B"".""FullHash"", ""D"".""FullHash"" FROM ""File"" A, ""Blockset"" B, ""Metadataset"" C, ""Blockset"" D, ""FilesetEntry"" E WHERE ""A"".""BlocksetID"" = ""B"".""ID"" AND ""A"".""MetadataID"" = ""C"".""ID"" AND ""C"".""BlocksetID"" = ""D"".""ID"" AND ""A"".""ID"" = ""E"".""FileID"" AND ""E"".""FilesetID"" = ? ", filesetId))
            while(rd.Read())
                yield return new LocalFileEntry(rd);
        }

        private IEnumerable<KeyValuePair<string, string>> GetDbOptionList()
        {
            using(var cmd = m_connection.CreateCommand(Transaction))
            using(var rd = cmd.ExecuteReader(@"SELECT ""Key"", ""Value"" FROM ""Configuration"" "))
            while(rd.Read())
                yield return new KeyValuePair<string, string>(rd.GetValue(0).ToString(), rd.GetValue(1).ToString());
        }
                    
        public IDictionary<string, string> GetDbOptions()
        {
            return GetDbOptionList().ToDictionary(x => x.Key, x => x.Value);    
        }

        public bool RepairInProgress
        {
            get
            {
                return GetDbOptions().ContainsKey("repair-in-progress");
            }
            set
            {
                var opts = GetDbOptions();

                if (value)
                    opts["repair-in-progress"] = "true";
                else
                    opts.Remove("repair-in-progress");
                
                SetDbOptions(opts);
            }
        }

        public bool PartiallyRecreated
        {
            get
            {
                return GetDbOptions().ContainsKey("partially-recreated");
            }
            set
            {
                var opts = GetDbOptions();

                if (value)
                    opts["partially-recreated"] = "true";
                else
                    opts.Remove("partially-recreated");

                SetDbOptions(opts);
            }
        }
        
        public void SetDbOptions(IDictionary<string, string> options)
        {
            using(var cmd = m_connection.CreateCommand())
            {
                cmd.Transaction = Transaction;
                cmd.ExecuteNonQuery(@"DELETE FROM ""Configuration"" ");
                foreach(var kp in options)
                    cmd.ExecuteNonQuery(@"INSERT INTO ""Configuration"" (""Key"", ""Value"") VALUES (?, ?) ", kp.Key, kp.Value);
            }
        }

        public long GetBlocksLargerThan(long fhblocksize)
        {
            using(var cmd = m_connection.CreateCommand())
                return cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Block"" WHERE ""Size"" > ?", -1, fhblocksize);
        }

        public void VerifyConsistency(long blocksize, long hashsize, bool verifyfilelists)
        {
            using (var cmd = m_connection.CreateCommand(Transaction))
            {
                // Calculate the lengths for each blockset                
                var combinedLengths = @"
SELECT 
    ""A"".""ID"" AS ""BlocksetID"", 
    IFNULL(""B"".""CalcLen"", 0) AS ""CalcLen"", 
    ""A"".""Length""
FROM
    ""Blockset"" A
LEFT OUTER JOIN
    (
        SELECT 
            ""BlocksetEntry"".""BlocksetID"",
            SUM(""Block"".""Size"") AS ""CalcLen""
        FROM
            ""BlocksetEntry""
        LEFT OUTER JOIN
            ""Block""
        ON
            ""Block"".""ID"" = ""BlocksetEntry"".""BlockID""
        GROUP BY ""BlocksetEntry"".""BlocksetID""
    ) B
ON
    ""A"".""ID"" = ""B"".""BlocksetID""

";
                // For each blockset with wrong lengths, fetch the file path
                var reportDetails = @"SELECT ""CalcLen"", ""Length"", ""A"".""BlocksetID"", ""File"".""Path"" FROM (" + combinedLengths + @") A, ""File"" WHERE ""A"".""BlocksetID"" = ""File"".""BlocksetID"" AND ""A"".""CalcLen"" != ""A"".""Length"" ";
                
                using(var rd = cmd.ExecuteReader(reportDetails))
                    if (rd.Read())
                    {
                        var sb = new StringBuilder();
                        sb.AppendLine("Found inconsistency in the following files while validating database: ");
                        var c = 0;
                        do
                        {
                            if (c < 5)
                                sb.AppendFormat("{0}, actual size {1}, dbsize {2}, blocksetid: {3}{4}", rd.GetValue(3), rd.GetValue(1), rd.GetValue(0), rd.GetValue(2), Environment.NewLine);
                            c++;
                        } while(rd.Read());
                        
                        c -= 5;
                        if (c > 0)
                            sb.AppendFormat("... and {0} more", c);
                        
                        sb.Append(". Run repair to fix it.");
                        throw new InvalidDataException(sb.ToString());
                    }

                var real_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM ""BlocklistHash""", 0);
                var unique_count = cmd.ExecuteScalarInt64(@"SELECT Count(*) FROM (SELECT DISTINCT ""BlocksetID"", ""Index"" FROM ""BlocklistHash"")", 0);

                if (real_count != unique_count)
                    throw new InvalidDataException(string.Format("Found {0} blocklist hashes, but there should be {1}. Run repair to fix it.", real_count, unique_count));

                var itemswithnoblocklisthash = cmd.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM (SELECT * FROM (SELECT ""N"".""BlocksetID"", ((""N"".""BlockCount"" + {0} - 1) / {0}) AS ""BlocklistHashCountExpected"", CASE WHEN ""G"".""BlocklistHashCount"" IS NULL THEN 0 ELSE ""G"".""BlocklistHashCount"" END AS ""BlocklistHashCountActual"" FROM (SELECT ""BlocksetID"", COUNT(*) AS ""BlockCount"" FROM ""BlocksetEntry"" GROUP BY ""BlocksetID"") ""N"" LEFT OUTER JOIN (SELECT ""BlocksetID"", COUNT(*) AS ""BlocklistHashCount"" FROM ""BlocklistHash"" GROUP BY ""BlocksetID"") ""G"" ON ""N"".""BlocksetID"" = ""G"".""BlocksetID"" WHERE ""N"".""BlockCount"" > 1) WHERE ""BlocklistHashCountExpected"" != ""BlocklistHashCountActual"")", blocksize / hashsize), 0);
                if (itemswithnoblocklisthash != 0)
                    throw new InvalidDataException(string.Format("Found {0} file(s) with missing blocklist hashes", itemswithnoblocklisthash));

                if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Blockset"" WHERE ""Length"" > 0 AND ""ID"" NOT IN (SELECT ""BlocksetId"" FROM ""BlocksetEntry"")") != 0)
                {
                    throw new Exception("Detected non-empty blocksets with no associated blocks!");
                }

                if (cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" WHERE ""BlocksetID"" != ? AND ""BlocksetID"" != ? AND NOT ""BlocksetID"" IN (SELECT ""ID"" FROM ""Blockset"")", 0, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID) != 0)
                    throw new Exception("Detected files associated with non-existing blocksets!");

                if (verifyfilelists)
                {
                    using(var cmd2 = m_connection.CreateCommand(Transaction))
                    foreach(var filesetid in cmd.ExecuteReaderEnumerable(@"SELECT ""ID"" FROM ""Fileset"" ").Select(x => x.ConvertValueToInt64(0, -1)))
                    {
                            var expandedCmd = string.Format(@"SELECT COUNT(*) FROM (SELECT DISTINCT ""Path"" FROM ({0}) UNION SELECT DISTINCT ""Path"" FROM ({1}))", LocalDatabase.LIST_FILESETS, LocalDatabase.LIST_FOLDERS_AND_SYMLINKS);
                        var expandedlist = cmd2.ExecuteScalarInt64(expandedCmd, 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID, filesetid);
                        //var storedfilelist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"", ""File"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" != ? AND ""File"".""BlocksetID"" != ?"), 0, filesetid, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);
                        var storedlist = cmd2.ExecuteScalarInt64(string.Format(@"SELECT COUNT(*) FROM ""FilesetEntry"" WHERE ""FilesetEntry"".""FilesetID"" = ?"), 0, filesetid);

                        if (expandedlist != storedlist)
                            throw new Exception(string.Format("Unexpected difference in fileset {0}, found {1} entries, but expected {2}", filesetid, expandedlist, storedlist));
                    }
                }
            }
        }

		public interface IBlock
		{
			string Hash { get; }
			long Size { get; }
		}
		
		internal class Block : IBlock
		{
			public string Hash { get; private set; }
			public long Size { get; private set; }
			
			public Block(string hash, long size)
			{
				this.Hash = hash;
				this.Size = size;
			}
		}		

        public IEnumerable<IBlock> GetBlocks(long volumeid)
		{
            using(var cmd = m_connection.CreateCommand(Transaction))
			using(var rd = cmd.ExecuteReader(@"SELECT DISTINCT ""Hash"", ""Size"" FROM ""Block"" WHERE ""VolumeID"" = ?", volumeid))
				while (rd.Read())
                    yield return new Block(rd.GetValue(0).ToString(), rd.GetInt64(1));
        }

        private class BlocklistHashEnumerable : IEnumerable<string>
        {
            private class BlocklistHashEnumerator : IEnumerator<string>
            {
                private System.Data.IDataReader m_reader;
                private BlocklistHashEnumerable m_parent;
                private string m_path = null;
                private bool m_first = true;
                private string m_current = null;

                public BlocklistHashEnumerator(BlocklistHashEnumerable parent, System.Data.IDataReader reader)
                {
                    m_reader = reader;
                    m_parent = parent;
                }

                public string Current { get{ return m_current; } }

                public void Dispose()
                {
                }

                object System.Collections.IEnumerator.Current { get { return this.Current; } }

                public bool MoveNext()
                {
                    m_first = false;

                    if (m_path == null)
                    {
                        m_path = m_reader.GetValue(0).ToString();
                        m_current = m_reader.GetValue(6).ToString();
                        return true;
                    }
                    else
                    {
                        if (m_current == null)
                            return false;

                        if (!m_reader.Read())
                        {
                            m_current = null;
                            m_parent.MoreData = false;
                            return false;
                        }

                        var np = m_reader.GetValue(0).ToString();
                        if (m_path != np)
                        {
                            m_current = null;
                            return false;
                        }

                        m_current = m_reader.GetValue(6).ToString();
                        return true;
                    }
                }

                public void Reset()
                {
                    if (!m_first)
                        throw new Exception("Iterator reset not supported");

                    m_first = false;
                }
            }

            private System.Data.IDataReader m_reader;

            public BlocklistHashEnumerable(System.Data.IDataReader reader)
            {
                m_reader = reader;
                this.MoreData = true;
            }

            public bool MoreData { get; protected set; }

            public IEnumerator<string> GetEnumerator()
            {
                return new BlocklistHashEnumerator(this, m_reader);
            }

            System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
            {
                return this.GetEnumerator();
            }
        }

        public const string LIST_FILESETS = @"
SELECT
    ""L"".""Path"", 
    ""L"".""Lastmodified"", 
    ""L"".""Filelength"", 
    ""L"".""Filehash"", 
    ""L"".""Metahash"", 
    ""L"".""Metalength"",
    ""L"".""BlocklistHash"", 
    ""L"".""FirstBlockHash"",
    ""L"".""FirstBlockSize"",
    ""L"".""FirstMetaBlockHash"",
    ""L"".""FirstMetaBlockSize"",
    ""M"".""Hash"" AS ""MetaBlocklistHash""
FROM
    (
    SELECT 
        ""J"".""Path"", 
        ""J"".""Lastmodified"", 
        ""J"".""Filelength"", 
        ""J"".""Filehash"", 
        ""J"".""Metahash"", 
        ""J"".""Metalength"",
        ""K"".""Hash"" AS ""BlocklistHash"", 
        ""J"".""FirstBlockHash"",
        ""J"".""FirstBlockSize"",
        ""J"".""FirstMetaBlockHash"",
        ""J"".""FirstMetaBlockSize"",
        ""J"".""MetablocksetID""
    FROM 
        (
        SELECT 
	        ""A"".""Path"" AS ""Path"", 
	        ""D"".""Lastmodified"" AS ""Lastmodified"", 
	        ""B"".""Length"" AS ""Filelength"", 
	        ""B"".""FullHash"" AS ""Filehash"", 
	        ""E"".""FullHash"" AS ""Metahash"", 
	        ""E"".""Length"" AS ""Metalength"",
	        ""A"".""BlocksetID"" AS ""BlocksetID"",
	        ""F"".""Hash"" AS ""FirstBlockHash"",
	        ""F"".""Size"" AS ""FirstBlockSize"",
	        ""H"".""Hash"" AS ""FirstMetaBlockHash"",
	        ""H"".""Size"" AS ""FirstMetaBlockSize"",
	        ""C"".""BlocksetID"" AS ""MetablocksetID""
        FROM 
	        ""File"" A	
        LEFT JOIN ""Blockset"" B
          ON ""A"".""BlocksetID"" = ""B"".""ID"" 
        LEFT JOIN ""Metadataset"" C  
          ON ""A"".""MetadataID"" = ""C"".""ID""
        LEFT JOIN ""FilesetEntry"" D
          ON ""A"".""ID"" = ""D"".""FileID""
        LEFT JOIN ""Blockset"" E
          ON ""E"".""ID"" = ""C"".""BlocksetID""
        LEFT JOIN ""BlocksetEntry"" G
          ON ""B"".""ID"" = ""G"".""BlocksetID""
        LEFT JOIN ""Block"" F 
          ON ""G"".""BlockID"" = ""F"".""ID""  
        LEFT JOIN ""BlocksetEntry"" I
          ON ""E"".""ID"" = ""I"".""BlocksetID""
        LEFT JOIN ""Block"" H 
          ON ""I"".""BlockID"" = ""H"".""ID""
        WHERE 
          ""A"".""BlocksetId"" >= 0 AND
          ""D"".""FilesetID"" = ? AND
          (""I"".""Index"" = 0 OR ""I"".""Index"" IS NULL) AND  
          (""G"".""Index"" = 0 OR ""G"".""Index"" IS NULL)
        ) J
    LEFT OUTER JOIN 
        ""BlocklistHash"" K 
    ON 
        ""K"".""BlocksetID"" = ""J"".""BlocksetID"" 
    ORDER BY ""J"".""Path"", ""K"".""Index""
    ) L

LEFT OUTER JOIN
    ""BlocklistHash"" M
ON
    ""M"".""BlocksetID"" = ""L"".""MetablocksetID""
";

        public const string LIST_FOLDERS_AND_SYMLINKS = @"
SELECT
    ""G"".""BlocksetID"",
    ""G"".""ID"",
    ""G"".""Path"",
    ""G"".""Length"",
    ""G"".""FullHash"",
    ""G"".""Lastmodified"",
    ""G"".""FirstMetaBlockHash"",
    ""H"".""Hash"" AS ""MetablocklistHash""
FROM
    (
    SELECT
        ""B"".""BlocksetID"",
        ""B"".""ID"",
        ""B"".""Path"",
        ""D"".""Length"",
        ""D"".""FullHash"",
        ""A"".""Lastmodified"",
        ""F"".""Hash"" AS ""FirstMetaBlockHash"",
        ""C"".""BlocksetID"" AS ""MetaBlocksetID""
    FROM
        ""FilesetEntry"" A, 
        ""File"" B, 
        ""Metadataset"" C, 
        ""Blockset"" D,
        ""BlocksetEntry"" E,
        ""Block"" F
    WHERE 
        ""A"".""FileID"" = ""B"".""ID"" 
        AND ""B"".""MetadataID"" = ""C"".""ID"" 
        AND ""C"".""BlocksetID"" = ""D"".""ID"" 
        AND ""E"".""BlocksetID"" = ""C"".""BlocksetID""
        AND ""E"".""BlockID"" = ""F"".""ID""
        AND ""E"".""Index"" = 0
        AND (""B"".""BlocksetID"" = ? OR ""B"".""BlocksetID"" = ?) 
        AND ""A"".""FilesetID"" = ?
    ) G
LEFT OUTER JOIN
   ""BlocklistHash"" H
ON
   ""H"".""BlocksetID"" = ""G"".""MetaBlocksetID""
ORDER BY
   ""G"".""Path"", ""H"".""Index""

";

        public void WriteFileset(Volumes.FilesetVolumeWriter filesetvolume, long filesetId)
        {
            using (var cmd = m_connection.CreateCommand(Transaction))
            {
                cmd.CommandText = LIST_FOLDERS_AND_SYMLINKS;
                cmd.AddParameter(FOLDER_BLOCKSET_ID);
                cmd.AddParameter(SYMLINK_BLOCKSET_ID);
                cmd.AddParameter(filesetId);

                string lastpath = null;
                using (var rd = cmd.ExecuteReader())
                while(rd.Read())
                {
                    var blocksetID = rd.ConvertValueToInt64(0, -1);
                    var path = rd.GetValue(2).ToString();
                    var metalength = rd.ConvertValueToInt64(3, -1);
                    var metahash = rd.GetValue(4).ToString();
                    var metablockhash = rd.GetValue(6).ToString();
                    var metablocklisthash = rd.GetValue(7).ToString();

                    if (path == lastpath)
                        m_result.AddWarning(string.Format("Duplicate path detected: {0}!", path), null);

                    lastpath = path;

                    if (blocksetID == FOLDER_BLOCKSET_ID)
                        filesetvolume.AddDirectory(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
                    else if (blocksetID == SYMLINK_BLOCKSET_ID)
                        filesetvolume.AddSymlink(path, metahash, metalength, metablockhash, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
                }
                        
                // TODO: Perhaps run the above query after recreate and compare count(*) with count(*) from filesetentry where id = x

                cmd.CommandText = LIST_FILESETS;
                cmd.Parameters.Clear();
                cmd.AddParameter(filesetId);

                using (var rd = cmd.ExecuteReader())
                if (rd.Read())
                {
                    var more = false;
                    do
                    {
                        var path = rd.GetValue(0).ToString();
                        var filehash = rd.GetValue(3).ToString();
                        var size = rd.ConvertValueToInt64(2);
                        var lastmodified = new DateTime(rd.ConvertValueToInt64(1, 0), DateTimeKind.Utc);
                        var metahash = rd.GetValue(4).ToString();
                        var metasize = rd.ConvertValueToInt64(5, -1);
                        var p = rd.GetValue(6);
                        var blrd = (p == null || p == DBNull.Value) ? null : new BlocklistHashEnumerable(rd);
                        var blockhash = rd.GetValue(7).ToString();
                        var blocksize = rd.ConvertValueToInt64(8, -1);
                        var metablockhash = rd.GetValue(9).ToString();
                        //var metablocksize = rd.ConvertValueToInt64(10, -1);
                        var metablocklisthash = rd.GetValue(11).ToString();

                        if (blockhash == filehash)
                            blockhash = null;

                        if (metablockhash == metahash)
                            metablockhash = null;

                        filesetvolume.AddFile(path, filehash, size, lastmodified, metahash, metasize, metablockhash, blockhash, blocksize, blrd, string.IsNullOrWhiteSpace(metablocklisthash) ? null : new string[] { metablocklisthash });
                        if (blrd == null)
                            more = rd.Read();
                        else
                            more = blrd.MoreData;

                    } while (more);
                }
            }
        }
        
        /// <summary>
        /// Keeps a list of filenames in a temporary table with a single columne Path
        ///</summary>
        public class FilteredFilenameTable : IDisposable
        {
            public string Tablename { get; private set; }
            private System.Data.IDbConnection m_connection;
            
            public FilteredFilenameTable(System.Data.IDbConnection connection, Library.Utility.IFilter filter, System.Data.IDbTransaction transaction)
            {
                m_connection = connection;
                Tablename = "Filenames-" + Library.Utility.Utility.ByteArrayAsHexString(Guid.NewGuid().ToByteArray());
                var type = Library.Utility.FilterType.Regexp;
                if (filter is Library.Utility.FilterExpression)
                    type = ((Library.Utility.FilterExpression)filter).Type;

                // Bugfix: SQLite does not handle case-insensitive LIKE with non-ascii characters
                if (type != Library.Utility.FilterType.Regexp && !Library.Utility.Utility.IsFSCaseSensitive && filter.ToString().Any(x => x > 127))
                    type = Library.Utility.FilterType.Regexp;
                
                if (type == Library.Utility.FilterType.Regexp)
                {
                    using(var cmd = m_connection.CreateCommand(transaction))
                    {
                        cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename));
                        cmd.CommandText = string.Format(@"INSERT INTO ""{0}"" (""Path"") VALUES (?)", Tablename);
                        cmd.AddParameter();
                        using(var c2 = m_connection.CreateCommand(transaction))
                        using(var rd = c2.ExecuteReader(@"SELECT DISTINCT ""Path"" FROM ""File"" "))
                            while(rd.Read())
                            {
                                var p = rd.GetValue(0).ToString();
                                if(Library.Utility.FilterExpression.Matches(filter, p))
                                {
                                    cmd.SetParameterValue(0, p);
                                    cmd.ExecuteNonQuery();
                                }
                            }
                    }
                }
                else
                {
                    var sb = new StringBuilder();
                    var args = new List<object>();
                    foreach(var f in ((Library.Utility.FilterExpression)filter).GetSimpleList())
                    {
                        if (f.Contains('*') || f.Contains('?'))
                        {
                            sb.Append(@"""Path"" LIKE ? OR ");
                            args.Add(f.Replace('*', '%').Replace('?', '_'));
                        }
                        else
                        {
                            sb.Append(@"""Path"" = ? OR ");
                            args.Add(f);
                        }
                    }
                    
                    sb.Length = sb.Length - " OR ".Length;

                    using (var cmd = m_connection.CreateCommand(transaction))
                    {
                        cmd.ExecuteNonQuery(string.Format(@"CREATE TEMPORARY TABLE ""{0}"" (""Path"" TEXT NOT NULL)", Tablename));
                        cmd.ExecuteNonQuery(string.Format(@"INSERT INTO ""{0}"" SELECT DISTINCT ""Path"" FROM ""File"" WHERE " + sb, Tablename), args.ToArray());
                    }
                }
            }
            
            public void Dispose()
            {
                if (Tablename != null)
                    try 
                    { 
                        using(var cmd = m_connection.CreateCommand())
                            cmd.ExecuteNonQuery(string.Format(@"DROP TABLE IF EXISTS ""{0}"" ", Tablename));
                    }
                    catch { }
                    finally { Tablename = null; }
            }                
        }
        
        public void RenameRemoteFile(string oldname, string newname)
        {
            using(var cmd = m_connection.CreateCommand(Transaction))
            {
                //Rename the old entry, to preserve ID links
                var c = cmd.ExecuteNonQuery(@"UPDATE ""Remotevolume"" SET ""Name"" = ? WHERE ""Name"" = ?", newname, oldname);
                if (c != 1)
                    throw new Exception(string.Format("Unexpected result from renaming \"{0}\" to \"{1}\", expected {2} got {3}", oldname, newname, 1, c));
                
                // Grab the type of entry
                var type = (RemoteVolumeType)Enum.Parse(typeof(RemoteVolumeType), cmd.ExecuteScalar(@"SELECT ""Type"" FROM ""Remotevolume"" WHERE ""Name"" = ?", newname).ToString(), true);
                
                //Create a fake new entry with the old name and mark as deleting
                // as this ensures we will remove it, if it shows up in some later listing
                RegisterRemoteVolume(oldname, type, RemoteVolumeState.Deleting);
            }
        }
        
        /// <summary>
        /// Creates a timestamped backup operation to correctly associate the fileset with the time it was created.
        /// </summary>
        /// <param name="volumeid">The ID of the fileset volume to update</param>
        /// <param name="timestamp">The timestamp of the operation to create</param>
        /// <param name="transaction">An optional external transaction</param>
        public virtual long CreateFileset(long volumeid, DateTime timestamp)
        {
            using (var cmd = m_connection.CreateCommand(Transaction))
                return cmd.ExecuteScalarInt64(@"INSERT INTO ""Fileset"" (""OperationID"", ""Timestamp"", ""VolumeID"") VALUES (?, ?, ?); SELECT last_insert_rowid();", -1, m_operationid, NormalizeDateTimeToEpochSeconds(timestamp), volumeid);
        }
        
        public void AddIndexBlockLink(long indexVolumeID, long blockVolumeID)
        {
            m_insertIndexBlockLink.Transaction = Transaction;
            m_insertIndexBlockLink.SetParameterValue(0, indexVolumeID);
            m_insertIndexBlockLink.SetParameterValue(1, blockVolumeID);
            m_insertIndexBlockLink.ExecuteNonQuery();
        }

        public IEnumerable<Tuple<string, byte[], int>> GetBlocklists(long volumeid, long blocksize, int hashsize)
        {
            using(var cmd = m_connection.CreateCommand(Transaction))
            {
                var sql = string.Format(@"SELECT ""A"".""Hash"", ""C"".""Hash"" FROM " + 
                    @"(SELECT ""BlocklistHash"".""BlocksetID"", ""Block"".""Hash"", * FROM  ""BlocklistHash"",""Block"" WHERE  ""BlocklistHash"".""Hash"" = ""Block"".""Hash"" AND ""Block"".""VolumeID"" = ?) A, " + 
                    @" ""BlocksetEntry"" B, ""Block"" C WHERE ""B"".""BlocksetID"" = ""A"".""BlocksetID"" AND " + 
                    @" ""B"".""Index"" >= (""A"".""Index"" * {0}) AND ""B"".""Index"" < ((""A"".""Index"" + 1) * {0}) AND ""C"".""ID"" = ""B"".""BlockID"" " + 
                    @" ORDER BY ""A"".""BlocksetID"", ""B"".""Index""",
                    blocksize / hashsize
                );

                string curHash = null;
                int index = 0;
                byte[] buffer = new byte[blocksize];

                using(var rd = cmd.ExecuteReader(sql, volumeid))
                    while (rd.Read())
                    {
                        var blockhash = rd.GetValue(0).ToString();
                        if ((blockhash != curHash && curHash != null) || index + hashsize > buffer.Length)
                        {
                            yield return new Tuple<string, byte[], int>(curHash, buffer, index);
                            curHash = null;
                            index = 0;
                            buffer = new byte[blocksize];
                        }

                        var hash = Convert.FromBase64String(rd.GetValue(1).ToString());
                        Array.Copy(hash, 0, buffer, index, hashsize);
                        curHash = blockhash;
                        index += hashsize;

                    }

                if (curHash != null)
                    yield return new Tuple<string, byte[], int>(curHash, buffer, index);


            }
        }

        public void PurgeLogData(DateTime threshold)
        {
            using(var cmd = m_connection.CreateCommand(Transaction))
            {
                var t = NormalizeDateTimeToEpochSeconds(threshold);
                cmd.ExecuteNonQuery(@"DELETE FROM ""LogData"" WHERE ""Timestamp"" < ?", t);
                cmd.ExecuteNonQuery(@"DELETE FROM ""RemoteOperation"" WHERE ""Timestamp"" < ?", t);
            }
        }
        
        public virtual void Dispose()
        {
            if (IsDisposed)
                return;

            DisposeAllFields<System.Data.IDbCommand>(this, false);

            if (ShouldCloseConnection && m_connection != null)
            {
                if (m_connection.State == System.Data.ConnectionState.Open)
                    m_connection.Close();
                m_connection.Dispose();
            }

            IsDisposed = true;
        }

        /// <summary>
        /// Disposes all fields of a certain type, in the instance and its bases
        /// </summary>
        /// <typeparam name="T">The type of fields to find</typeparam>
        /// <param name="item">The item to dispose</param>
        /// <param name="throwExceptions"><c>True</c> if an aggregate exception should be thrown, or <c>false</c> if exceptions are silently captured</param>
        public static void DisposeAllFields<T>(object item, bool throwExceptions)
            where T : IDisposable
        {
            var typechain = new List<Type>();
            var cur = item.GetType();
            var exceptions = new List<Exception>();

            while (cur != null && cur != typeof(object))
            {
                typechain.Add(cur);
                cur = cur.BaseType;
            }

            var fields =
                typechain.SelectMany(x =>
                    x.GetFields(System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.FlattenHierarchy)
                ).Distinct().Where(x => x.FieldType.IsAssignableFrom(typeof(T)));

            foreach (var p in fields)
                try
                {
                    var val = p.GetValue(item);
                    if (val != null)
                        ((T)val).Dispose();
                }
                catch (Exception ex)
                {
                    if (throwExceptions)
                        exceptions.Add(ex);
                }

            if (exceptions.Count > 0)
                throw new AggregateException(exceptions);
        }

        public void WriteResults()
        {
            if (IsDisposed)
                return;

            if (m_connection != null && m_result != null)
            {
                m_result.FlushLog();
                if (m_result.EndTime.Ticks == 0)
                    m_result.EndTime = DateTime.UtcNow;

                LogMessage("Result", 
                    Library.Utility.Utility.PrintSerializeObject(
                        m_result, 
                        (StringBuilder)null, 
                        (prop, item) => 
                            !typeof(IBackendProgressUpdater).IsAssignableFrom(prop.PropertyType) && 
                            !typeof(IMessageSink).IsAssignableFrom(prop.PropertyType) && 
                            !typeof(ILogWriter).IsAssignableFrom(prop.PropertyType) &&
                            prop.Name != "VerboseOutput" && 
                            prop.Name != "VerboseErrors" &&
                           !(prop.Name == "MainOperation" && item is BackendWriter) &&
                           !(prop.Name == "EndTime" && item is BackendWriter) &&
                           !(prop.Name == "Duration" && item is BackendWriter) &&
                           !(prop.Name == "BeginTime" && item is BackendWriter), 
                        recurseobjects: true, 
                        collectionlimit: 5
                    ).ToString(),
                    null
              );
            }

            CommitTransaction("WriteMessage");
        }
    }
}