From 0d57f2ceeaddeeaf358dd9c25e95d3adb0c8d94e Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Wed, 9 Jul 2025 14:31:21 +0200 Subject: [PATCH] Rewrote HyperV and MSSQL parser to support paths under each instance. This also cleans up the filters slightly, so the logic for "first match" still holds. If an item is excluded, and subfolders are also automatically excluded. --- .../Library/Modules/Builtin/HyperVOptions.cs | 122 +++++++++++------- .../Library/Modules/Builtin/MSSQLOptions.cs | 119 ++++++++++------- .../Snapshots/Windows/HyperVUtility.cs | 42 +++++- .../Library/Snapshots/Windows/MSSQLUtility.cs | 32 ++++- 4 files changed, 217 insertions(+), 98 deletions(-) diff --git a/Duplicati/Library/Modules/Builtin/HyperVOptions.cs b/Duplicati/Library/Modules/Builtin/HyperVOptions.cs index 403cdcbe5..9f1eeba30 100644 --- a/Duplicati/Library/Modules/Builtin/HyperVOptions.cs +++ b/Duplicati/Library/Modules/Builtin/HyperVOptions.cs @@ -35,8 +35,8 @@ namespace Duplicati.Library.Modules.Builtin /// The tag used for logging /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - private const string m_HyperVPathGuidRegExp = @"\%HYPERV\%\\([0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12})"; - private const string m_HyperVPathAllRegExp = @"%HYPERV%"; + private const string m_HyperVPathGuidRegExp = @"\%HYPERV\%\\(?[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12})(\\(?.*))?"; + private const string m_HyperVPathAllMarker = @"%HYPERV%"; private const string IGNORE_CONSISTENCY_WARNING_OPTION = "hyperv-ignore-client-warning"; @@ -84,7 +84,7 @@ namespace Duplicati.Library.Modules.Builtin Logging.Log.WriteWarningMessage(LOGTAG, "HyperVWindowsOnly", null, "Hyper-V backup works only on Windows OS"); if (paths != null) - paths = paths.Where(x => !x.Equals(m_HyperVPathAllRegExp, StringComparison.OrdinalIgnoreCase) && !Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToArray(); + paths = paths.Where(x => !x.Equals(m_HyperVPathAllMarker, StringComparison.OrdinalIgnoreCase) && !Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToArray(); if (!string.IsNullOrEmpty(filter)) { @@ -104,26 +104,21 @@ namespace Duplicati.Library.Modules.Builtin // referenced types from System.Management here [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] [SupportedOSPlatform("windows")] - private Dictionary RealParseSourcePaths(ref string[] paths, ref string filter, Dictionary commandlineOptions) + public Dictionary RealParseSourcePaths(ref string[] paths, ref string filter, Dictionary commandlineOptions, IHyperVUtility hypervUtility = null) { var changedOptions = new Dictionary(); - var filtersInclude = new List(); - var filtersExclude = new List(); + var hypervFilters = new List(); if (!string.IsNullOrEmpty(filter)) { var filters = filter.Split(new string[] { System.IO.Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries); - filtersInclude = filters.Where(x => x.StartsWith("+", StringComparison.Ordinal) && Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) - .Select(x => Regex.Match(x.Substring(1), m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant).Groups[1].Value).ToList(); - filtersExclude = filters.Where(x => x.StartsWith("-", StringComparison.Ordinal) && Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) - .Select(x => Regex.Match(x.Substring(1), m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant).Groups[1].Value).ToList(); - + hypervFilters = filters.Where(x => Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToList(); var remainingfilters = filters.Where(x => !Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToArray(); filter = string.Join(System.IO.Path.PathSeparator.ToString(), remainingfilters); } - var hypervUtility = new HyperVUtility(); + hypervUtility ??= new HyperVUtility(); if (paths == null || !ContainFilesForBackup(paths) || !hypervUtility.IsHyperVInstalled) return changedOptions; @@ -132,10 +127,10 @@ namespace Duplicati.Library.Modules.Builtin { var excludedWriters = commandlineOptions["vss-exclude-writers"].Split(';').Where(x => !string.IsNullOrWhiteSpace(x) && x.Trim().Length > 0).Select(x => new Guid(x)).ToArray(); - if (excludedWriters.Contains(HyperVUtility.HyperVWriterGuid)) + if (excludedWriters.Contains(hypervUtility.HyperVWriterGuid)) { - Logging.Log.WriteWarningMessage(LOGTAG, "CannotExcludeHyperVVSSWriter", null, "Excluded writers for VSS cannot contain Hyper-V writer when backuping Hyper-V virtual machines. Removing \"{0}\" to continue", HyperVUtility.HyperVWriterGuid.ToString()); - changedOptions["vss-exclude-writers"] = string.Join(";", excludedWriters.Where(x => x != HyperVUtility.HyperVWriterGuid)); + Logging.Log.WriteWarningMessage(LOGTAG, "CannotExcludeHyperVVSSWriter", null, "Excluded writers for VSS cannot contain Hyper-V writer when backuping Hyper-V virtual machines. Removing \"{0}\" to continue", hypervUtility.HyperVWriterGuid.ToString()); + changedOptions["vss-exclude-writers"] = string.Join(";", excludedWriters.Where(x => x != hypervUtility.HyperVWriterGuid)); } } @@ -151,56 +146,89 @@ namespace Duplicati.Library.Modules.Builtin Logging.Log.WriteInformationMessage(LOGTAG, "StartingHyperVQuery", "Starting to gather Hyper-V information"); var provider = Utility.Utility.ParseEnumOption(changedOptions.AsReadOnly(), "snapshot-provider", WindowsSnapshot.DEFAULT_WINDOWS_SNAPSHOT_QUERY_PROVIDER); hypervUtility.QueryHyperVGuestsInfo(provider, true); - Logging.Log.WriteInformationMessage(LOGTAG, "HyperVMachineCount", "Found {0} virtual machines on Hyper-V", hypervUtility.Guests.Count); + if (hypervUtility.Guests == null || hypervUtility.Guests.Count == 0) + { + Logging.Log.WriteWarningMessage(LOGTAG, "NoHyperVMachinesFound", null, "No Hyper-V virtual machines found."); + return changedOptions; + } + + Logging.Log.WriteInformationMessage(LOGTAG, "HyperVMachineCount", "Found {0} virtual machines on Hyper-V", hypervUtility.Guests?.Count); foreach (var guest in hypervUtility.Guests) Logging.Log.WriteProfilingMessage(LOGTAG, "FoundHyperVMachine", "Found VM name {0}, ID {1}, files {2}", guest.Name, guest.ID, string.Join(";", guest.DataPaths)); - List guestsForBackup = new List(); + var guestsForBackup = new List(); + var conditionalPathsForBackup = new List<(string ID, string Path)>(); - if (paths.Contains(m_HyperVPathAllRegExp, StringComparer.OrdinalIgnoreCase)) + if (paths.Contains(m_HyperVPathAllMarker, StringComparer.OrdinalIgnoreCase)) guestsForBackup = hypervUtility.Guests; else - foreach (var guestID in paths.Where(x => Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) - .Select(x => Regex.Match(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant).Groups[1].Value).ToArray()) + { + var guestEntries = paths.Where(x => Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) + .Select(x => + { + var m = Regex.Match(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + return (ID: m.Groups["id"].Value, Path: m.Groups["path"].Value); + }); + + foreach ((var guestID, var path) in guestEntries) { var foundGuest = hypervUtility.Guests.Where(x => x.ID == new Guid(guestID)); if (foundGuest.Count() != 1) throw new Duplicati.Library.Interface.UserInformationException(string.Format("Hyper-V guest specified in source with ID {0} cannot be found", guestID), "HyperVGuestNotFound"); - guestsForBackup.Add(foundGuest.First()); + if (string.IsNullOrWhiteSpace(path)) + guestsForBackup.Add(foundGuest.First()); + else + conditionalPathsForBackup.Add((guestID, path)); } + } - if (filtersInclude.Count > 0) - foreach (var guestID in filtersInclude) + var guestStatus = new Dictionary(StringComparer.OrdinalIgnoreCase); + var pathFilters = new List<(string ID, string Filter)>(); + + foreach (var filterExp in hypervFilters) + { + var m = Regex.Match(filterExp, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + (var id, var path) = (m.Groups["id"].Value, m.Groups["path"].Value); + if (string.IsNullOrWhiteSpace(path)) { - var foundGuest = hypervUtility.Guests.Where(x => x.ID == new Guid(guestID)); - - if (foundGuest.Count() != 1) - throw new Duplicati.Library.Interface.UserInformationException(string.Format("Hyper-V guest specified in include filter with ID {0} cannot be found", guestID), "HyperVGuestNotFound"); - - guestsForBackup.Add(foundGuest.First()); - Logging.Log.WriteInformationMessage(LOGTAG, "IncludeByFilter", "Including {0} based on including filters", guestID); + if (!guestStatus.ContainsKey(id)) + guestStatus[id] = filterExp.StartsWith("-", StringComparison.Ordinal) ? false : true; } - - guestsForBackup = guestsForBackup.Distinct().ToList(); - - if (filtersExclude.Count > 0) - foreach (var guestID in filtersExclude) + else { - var foundGuest = guestsForBackup.Where(x => x.ID == new Guid(guestID)); - - if (foundGuest.Count() != 1) - throw new Duplicati.Library.Interface.UserInformationException(string.Format("Hyper-V guest specified in exclude filter with ID {0} cannot be found", guestID), "HyperVGuestNotFound"); - - guestsForBackup.Remove(foundGuest.First()); - Logging.Log.WriteInformationMessage(LOGTAG, "ExcludeByFilter", "Excluding {0} based on excluding filters", guestID); + pathFilters.Add((id, filterExp[0] + path)); } + } - var pathsForBackup = new List(paths); - var filterhandler = new Utility.FilterExpression( - filter.Split(new string[] { System.IO.Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries).Where(x => x.StartsWith("-", StringComparison.Ordinal)).Select(x => x.Substring(1)).ToList()); + guestsForBackup = guestsForBackup + .Where(x => !guestStatus.ContainsKey(x.ID.ToString()) || guestStatus[x.ID.ToString()]) + .DistinctBy(x => x.ID) + .ToList(); + + var includedGuestIds = guestsForBackup.Select(x => x.ID.ToString()).ToHashSet(StringComparer.OrdinalIgnoreCase); + + pathFilters = pathFilters + .Where(x => includedGuestIds.Contains(x.ID)) + .DistinctBy(x => x.ID) + .ToList(); + + conditionalPathsForBackup = conditionalPathsForBackup + .Where(x => includedGuestIds.Contains(x.ID)) + .DistinctBy(x => x.ID) + .ToList(); + + var filterhandler = pathFilters.Select(x => x.Filter) + .Concat(filter.Split([System.IO.Path.PathSeparator], StringSplitOptions.RemoveEmptyEntries)) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => Utility.FilterExpression.StringToIFilter(x)) + .Append(new Utility.FilterExpression()) + .Aggregate((a, b) => Utility.FilterExpression.Combine(a, b)); + + var pathsForBackup = new List(paths.Where(x => !x.Equals(m_HyperVPathAllMarker, StringComparison.OrdinalIgnoreCase) && !Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant))); + pathsForBackup.AddRange(conditionalPathsForBackup.Select(x => x.Path).Where(x => !string.IsNullOrWhiteSpace(x))); foreach (var guestForBackup in guestsForBackup) foreach (var pathForBackup in guestForBackup.DataPaths) @@ -214,7 +242,7 @@ namespace Duplicati.Library.Modules.Builtin Logging.Log.WriteInformationMessage(LOGTAG, "ExcludeByFilter", "Excluding {0} based on excluding filters", pathForBackup); } - paths = pathsForBackup.Where(x => !x.Equals(m_HyperVPathAllRegExp, StringComparison.OrdinalIgnoreCase) && !Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) + paths = pathsForBackup .Distinct(Utility.Utility.ClientFilenameStringComparer).OrderBy(a => a).ToArray(); return changedOptions; @@ -225,7 +253,7 @@ namespace Duplicati.Library.Modules.Builtin if (paths == null || !OperatingSystem.IsWindows()) return false; - return paths.Where(x => !string.IsNullOrWhiteSpace(x)).Any(x => x.Equals(m_HyperVPathAllRegExp, StringComparison.OrdinalIgnoreCase) || Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); + return paths.Where(x => !string.IsNullOrWhiteSpace(x)).Any(x => x.Equals(m_HyperVPathAllMarker, StringComparison.OrdinalIgnoreCase) || Regex.IsMatch(x, m_HyperVPathGuidRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); } #endregion diff --git a/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs b/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs index 120a6fe72..c99cbbf2d 100644 --- a/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs +++ b/Duplicati/Library/Modules/Builtin/MSSQLOptions.cs @@ -36,8 +36,8 @@ namespace Duplicati.Library.Modules.Builtin /// private static readonly string LOGTAG = Logging.Log.LogTagFromType(); - private const string m_MSSQLPathDBRegExp = @"\%MSSQL\%\\(.+)"; - private const string m_MSSQLPathAllRegExp = @"%MSSQL%"; + private const string m_MSSQLPathDBRegExp = @"\%MSSQL\%\\(?^\\+)(?\\.*)"; + private const string m_MSSQLPathAllMarker = @"%MSSQL%"; #region IGenericModule Members @@ -82,7 +82,7 @@ namespace Duplicati.Library.Modules.Builtin Logging.Log.WriteWarningMessage(LOGTAG, "MSSqlWindowsOnly", null, "Microsoft SQL Server databases backup works only on Windows OS"); if (paths != null) - paths = paths.Where(x => !x.Equals(m_MSSQLPathAllRegExp, StringComparison.OrdinalIgnoreCase) && !Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToArray(); + paths = paths.Where(x => !x.Equals(m_MSSQLPathAllMarker, StringComparison.OrdinalIgnoreCase) && !Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToArray(); if (!string.IsNullOrEmpty(filter)) { @@ -102,26 +102,21 @@ namespace Duplicati.Library.Modules.Builtin // referenced types from System.Management here [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] [SupportedOSPlatform("windows")] - private Dictionary RealParseSourcePaths(ref string[] paths, ref string filter, Dictionary commandlineOptions) + public Dictionary RealParseSourcePaths(ref string[] paths, ref string filter, Dictionary commandlineOptions, IMSSQLUtility mssqlUtility = null) { var changedOptions = new Dictionary(); - var filtersInclude = new List(); - var filtersExclude = new List(); + var mssqlFilters = new List(); if (!string.IsNullOrEmpty(filter)) { var filters = filter.Split(new string[] { System.IO.Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries); - filtersInclude = filters.Where(x => x.StartsWith("+", StringComparison.Ordinal) && Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) - .Select(x => Regex.Match(x.Substring(1), m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant).Groups[1].Value).ToList(); - filtersExclude = filters.Where(x => x.StartsWith("-", StringComparison.Ordinal) && Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) - .Select(x => Regex.Match(x.Substring(1), m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant).Groups[1].Value).ToList(); - + mssqlFilters = filters.Where(x => Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToList(); var remainingfilters = filters.Where(x => !Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)).ToArray(); filter = string.Join(System.IO.Path.PathSeparator.ToString(), remainingfilters); } - var mssqlUtility = new MSSQLUtility(); + mssqlUtility ??= new MSSQLUtility(); if (paths == null || !ContainFilesForBackup(paths) || !mssqlUtility.IsMSSQLInstalled) return changedOptions; @@ -130,10 +125,10 @@ namespace Duplicati.Library.Modules.Builtin { var excludedWriters = commandlineOptions["vss-exclude-writers"].Split(';').Where(x => !string.IsNullOrWhiteSpace(x) && x.Trim().Length > 0).Select(x => new Guid(x)).ToArray(); - if (excludedWriters.Contains(MSSQLUtility.MSSQLWriterGuid)) + if (excludedWriters.Contains(mssqlUtility.MSSQLWriterGuid)) { - Logging.Log.WriteWarningMessage(LOGTAG, "CannotExcludeMsSqlVSSWriter", null, "Excluded writers for VSS cannot contain MS SQL writer when backuping Microsoft SQL Server databases. Removing \"{0}\" to continue", MSSQLUtility.MSSQLWriterGuid.ToString()); - changedOptions["vss-exclude-writers"] = string.Join(";", excludedWriters.Where(x => x != MSSQLUtility.MSSQLWriterGuid)); + Logging.Log.WriteWarningMessage(LOGTAG, "CannotExcludeMsSqlVSSWriter", null, "Excluded writers for VSS cannot contain MS SQL writer when backuping Microsoft SQL Server databases. Removing \"{0}\" to continue", mssqlUtility.MSSQLWriterGuid.ToString()); + changedOptions["vss-exclude-writers"] = string.Join(";", excludedWriters.Where(x => x != mssqlUtility.MSSQLWriterGuid)); } } @@ -146,56 +141,90 @@ namespace Duplicati.Library.Modules.Builtin Logging.Log.WriteInformationMessage(LOGTAG, "StartingMsSqlQuery", "Starting to gather Microsoft SQL Server information", Logging.LogMessageType.Information); var provider = Utility.Utility.ParseEnumOption(changedOptions.AsReadOnly(), "snapshot-provider", WindowsSnapshot.DEFAULT_WINDOWS_SNAPSHOT_QUERY_PROVIDER); mssqlUtility.QueryDBsInfo(provider); + + if (mssqlUtility.DBs == null || mssqlUtility.DBs.Count == 0) + { + Logging.Log.WriteWarningMessage(LOGTAG, "NoMsSqlDatabasesFound", null, "No Microsoft SQL Server databases found."); + return changedOptions; + } + Logging.Log.WriteInformationMessage(LOGTAG, "MsSqlDatabaseCount", "Found {0} databases on Microsoft SQL Server", mssqlUtility.DBs.Count); foreach (var db in mssqlUtility.DBs) Logging.Log.WriteProfilingMessage(LOGTAG, "MsSqlDatabaseName", "Found DB name {0}, ID {1}, files {2}", db.Name, db.ID, string.Join(";", db.DataPaths)); List dbsForBackup = new List(); + var conditionalPathsForBackup = new List<(string ID, string Path)>(); - if (paths.Contains(m_MSSQLPathAllRegExp, StringComparer.OrdinalIgnoreCase)) + if (paths.Contains(m_MSSQLPathAllMarker, StringComparer.OrdinalIgnoreCase)) dbsForBackup = mssqlUtility.DBs; else - foreach (var dbID in paths.Where(x => Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) - .Select(x => Regex.Match(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant).Groups[1].Value).ToArray()) + { + var guestEntries = paths.Where(x => Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) + .Select(x => + { + var m = Regex.Match(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + return (ID: m.Groups["id"].Value, Path: m.Groups["path"].Value); + }); + + foreach ((var dbID, var path) in guestEntries) { var foundDB = mssqlUtility.DBs.Where(x => x.ID.Equals(dbID, StringComparison.OrdinalIgnoreCase)); if (foundDB.Count() != 1) throw new Duplicati.Library.Interface.UserInformationException(string.Format("DB name specified in source with ID {0} cannot be found", dbID), "MsSqlDatabaseNotFound"); - dbsForBackup.Add(foundDB.First()); + if (string.IsNullOrWhiteSpace(path)) + dbsForBackup.Add(foundDB.First()); + else + conditionalPathsForBackup.Add((dbID, path)); } + } - if (filtersInclude.Count > 0) - foreach (var dbID in filtersInclude) + var dbStatus = new Dictionary(StringComparer.OrdinalIgnoreCase); + var pathFilters = new List<(string ID, string Filter)>(); + + foreach (var filterExp in mssqlFilters) + { + var m = Regex.Match(filterExp, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + (var id, var path) = (m.Groups["id"].Value, m.Groups["path"].Value); + if (string.IsNullOrWhiteSpace(path)) { - var foundDB = mssqlUtility.DBs.Where(x => x.ID.Equals(dbID, StringComparison.OrdinalIgnoreCase)); - - if (foundDB.Count() != 1) - throw new Duplicati.Library.Interface.UserInformationException(string.Format("DB name specified in include filter with ID {0} cannot be found", dbID), "MsSqlDatabaseNotFound"); - - dbsForBackup.Add(foundDB.First()); - Logging.Log.WriteInformationMessage(LOGTAG, "IncludeByFilter", "Including {0} based on including filters", dbID); + if (!dbStatus.ContainsKey(id)) + dbStatus[id] = filterExp.StartsWith("-", StringComparison.Ordinal) ? false : true; } - - dbsForBackup = dbsForBackup.Distinct().ToList(); - - if (filtersExclude.Count > 0) - foreach (var dbID in filtersExclude) + else { - var foundDB = dbsForBackup.Where(x => x.ID.Equals(dbID, StringComparison.OrdinalIgnoreCase)); - - if (foundDB.Count() != 1) - throw new Duplicati.Library.Interface.UserInformationException(string.Format("DB name specified in exclude filter with ID {0} cannot be found", dbID), "MsSqlDatabaseNotFound"); - - dbsForBackup.Remove(foundDB.First()); - Logging.Log.WriteInformationMessage(LOGTAG, "ExcludeByFilter", "Excluding {0} based on excluding filters", dbID); + pathFilters.Add((id, filterExp[0] + path)); } + } - var pathsForBackup = new List(paths); - var filterhandler = new Utility.FilterExpression( - filter.Split(new string[] { System.IO.Path.PathSeparator.ToString() }, StringSplitOptions.RemoveEmptyEntries).Where(x => x.StartsWith("-", StringComparison.Ordinal)).Select(x => x.Substring(1)).ToList()); + dbsForBackup = dbsForBackup + .Where(x => !dbStatus.ContainsKey(x.ID.ToString()) || dbStatus[x.ID.ToString()]) + .DistinctBy(x => x.ID) + .ToList(); + + var includedDbIds = dbsForBackup.Select(x => x.ID.ToString()).ToHashSet(StringComparer.OrdinalIgnoreCase); + + pathFilters = pathFilters + .Where(x => includedDbIds.Contains(x.ID)) + .DistinctBy(x => x.ID) + .ToList(); + + conditionalPathsForBackup = conditionalPathsForBackup + .Where(x => includedDbIds.Contains(x.ID)) + .DistinctBy(x => x.ID) + .ToList(); + + var filterhandler = pathFilters.Select(x => x.Filter) + .Concat(filter.Split([System.IO.Path.PathSeparator], StringSplitOptions.RemoveEmptyEntries)) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => Utility.FilterExpression.StringToIFilter(x)) + .Append(new Utility.FilterExpression()) + .Aggregate((a, b) => Utility.FilterExpression.Combine(a, b)); + + var pathsForBackup = new List(paths.Where(x => !x.Equals(m_MSSQLPathAllMarker, StringComparison.OrdinalIgnoreCase) && !Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant))); + pathsForBackup.AddRange(conditionalPathsForBackup.Select(x => x.Path).Where(x => !string.IsNullOrWhiteSpace(x))); foreach (var dbForBackup in dbsForBackup) foreach (var pathForBackup in dbForBackup.DataPaths) @@ -209,7 +238,7 @@ namespace Duplicati.Library.Modules.Builtin Logging.Log.WriteInformationMessage(LOGTAG, "ExcludeByFilter", "Excluding {0} based on excluding filters", pathForBackup); } - paths = pathsForBackup.Where(x => !x.Equals(m_MSSQLPathAllRegExp, StringComparison.OrdinalIgnoreCase) && !Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)) + paths = pathsForBackup .Distinct(Utility.Utility.ClientFilenameStringComparer).OrderBy(a => a).ToArray(); return changedOptions; @@ -220,7 +249,7 @@ namespace Duplicati.Library.Modules.Builtin if (paths == null || !OperatingSystem.IsWindows()) return false; - return paths.Where(x => !string.IsNullOrWhiteSpace(x)).Any(x => x.Equals(m_MSSQLPathAllRegExp, StringComparison.OrdinalIgnoreCase) || Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); + return paths.Where(x => !string.IsNullOrWhiteSpace(x)).Any(x => x.Equals(m_MSSQLPathAllMarker, StringComparison.OrdinalIgnoreCase) || Regex.IsMatch(x, m_MSSQLPathDBRegExp, RegexOptions.IgnoreCase | RegexOptions.CultureInvariant)); } #endregion diff --git a/Duplicati/Library/Snapshots/Windows/HyperVUtility.cs b/Duplicati/Library/Snapshots/Windows/HyperVUtility.cs index c300ded2a..02e4624f7 100644 --- a/Duplicati/Library/Snapshots/Windows/HyperVUtility.cs +++ b/Duplicati/Library/Snapshots/Windows/HyperVUtility.cs @@ -82,8 +82,40 @@ namespace Duplicati.Library.Snapshots.Windows } } + /// + /// Interface for Hyper-V utility to query Hyper-V guests and their paths + /// + public interface IHyperVUtility + { + /// + /// The Hyper-V VSS Writer Guid + /// + Guid HyperVWriterGuid { get; } + /// + /// Hyper-V is supported only on Windows platform + /// + bool IsHyperVInstalled { get; } + /// + /// Hyper-V writer is supported only on Server version of Windows + /// + bool IsVSSWriterSupported { get; } + + /// + /// Enumerated Hyper-V guests + /// + List Guests { get; } + + /// + /// Query Hyper-V for all Virtual Machines info + /// + /// Specify if returned data should contain VM paths + /// The provider to use for VSS + /// List of Hyper-V Machines + void QueryHyperVGuestsInfo(WindowsSnapshotProvider provider, bool bIncludePaths = false); + } + [SupportedOSPlatform("windows")] - public class HyperVUtility + public class HyperVUtility : IHyperVUtility { /// /// The tag used for logging @@ -102,7 +134,11 @@ namespace Duplicati.Library.Snapshots.Windows /// /// The Hyper-V VSS Writer Guid /// - public static readonly Guid HyperVWriterGuid = new Guid("66841cd4-6ded-4f4b-8f17-fd23f8ddc3de"); + internal static readonly Guid _HyperVWriterGuid = new Guid("66841cd4-6ded-4f4b-8f17-fd23f8ddc3de"); + /// + /// The Hyper-V VSS Writer Guid + /// + public Guid HyperVWriterGuid => _HyperVWriterGuid; /// /// Hyper-V is supported only on Windows platform /// @@ -224,7 +260,7 @@ namespace Duplicati.Library.Snapshots.Windows { using (var vssBackupComponents = new SnapshotManager(provider)) { - var writerGUIDS = new[] { HyperVWriterGuid }; + var writerGUIDS = new[] { _HyperVWriterGuid }; try { diff --git a/Duplicati/Library/Snapshots/Windows/MSSQLUtility.cs b/Duplicati/Library/Snapshots/Windows/MSSQLUtility.cs index fc2f316e0..5be13e517 100644 --- a/Duplicati/Library/Snapshots/Windows/MSSQLUtility.cs +++ b/Duplicati/Library/Snapshots/Windows/MSSQLUtility.cs @@ -81,8 +81,30 @@ namespace Duplicati.Library.Snapshots.Windows } } + public interface IMSSQLUtility + { + /// + /// The MS SQL VSS Writer Guid + /// + Guid MSSQLWriterGuid { get; } + /// + /// MS SQL is supported only on Windows platform + /// + bool IsMSSQLInstalled { get; } + /// + /// Enumerated MS SQL DBs + /// + List DBs { get; } + + /// + /// For all MS SQL databases it enumerate all associated paths using VSS data + /// + /// A collection of DBs and paths + void QueryDBsInfo(WindowsSnapshotProvider provider); + } + [SupportedOSPlatform("windows")] - public class MSSQLUtility + public class MSSQLUtility : IMSSQLUtility { /// /// The tag used for logging @@ -91,7 +113,11 @@ namespace Duplicati.Library.Snapshots.Windows /// /// The MS SQL VSS Writer Guid /// - public static readonly Guid MSSQLWriterGuid = new Guid("a65faa63-5ea8-4ebc-9dbd-a0c4db26912a"); + private static readonly Guid _MSSQLWriterGuid = new Guid("a65faa63-5ea8-4ebc-9dbd-a0c4db26912a"); + /// + /// The MS SQL VSS Writer Guid + /// + public Guid MSSQLWriterGuid => _MSSQLWriterGuid; /// /// MS SQL is supported only on Windows platform /// @@ -160,7 +186,7 @@ namespace Duplicati.Library.Snapshots.Windows using (var vssBackupComponents = new SnapshotManager(provider)) { - var writerGUIDS = new[] { MSSQLWriterGuid }; + var writerGUIDS = new[] { _MSSQLWriterGuid }; try { vssBackupComponents.SetupWriters(writerGUIDS, null);