// Copyright (C) 2015, The Duplicati Team // http://www.duplicati.com, info@duplicati.com // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public // License as published by the Free Software Foundation; either // version 2.1 of the License, or (at your option) any later version. // // This library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU // Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; using System.Text; using System.Text.RegularExpressions; namespace Duplicati.Library.Utility { public static class Utility { /// /// Size of buffers for copying stream /// public static long DEFAULT_BUFFER_SIZE => SystemContextSettings.Buffersize; /// /// A cache of the FileSystemCaseSensitive property, which is computed upon the first access. /// private static bool? CachedIsFSCaseSensitive; /// /// Gets the hash algorithm used for calculating a hash /// public static string HashAlgorithm => "SHA256"; /// /// The EPOCH offset (unix style) /// public static readonly DateTime EPOCH = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); /// /// The attribute value used to indicate error /// public const FileAttributes ATTRIBUTE_ERROR = (FileAttributes)(1 << 30); /// /// The callback delegate type used to collecting file information /// /// The path that the file enumeration started at /// The current element /// The attributes of the element /// A value indicating if the folder should be recursed, ignored for other types public delegate bool EnumerationFilterDelegate(string rootpath, string path, FileAttributes attributes); /// /// Copies the content of one stream into another /// /// The stream to read from /// The stream to write to public static void CopyStream(Stream source, Stream target) { CopyStream(source, target, true); } /// /// Copies the content of one stream into another /// /// The stream to read from /// The stream to write to /// True if an attempt should be made to rewind the source stream, false otherwise /// Temporary buffer to use (optional) public static void CopyStream(Stream source, Stream target, bool tryRewindSource, byte[] buf = null) { if (tryRewindSource && source.CanSeek) try { source.Position = 0; } catch { // ignored } buf = buf ?? new byte[DEFAULT_BUFFER_SIZE]; int read; while ((read = source.Read(buf, 0, buf.Length)) != 0) target.Write(buf, 0, read); } /// /// These are characters that must be escaped when using a globbing expression /// private static readonly string BADCHARS = @"\\|\+|\||\{|\[|\(|\)|\]|\}|\^|\$|\#|\."; /// /// Most people will probably want to use fileglobbing, but RegExp's are more flexible. /// By converting from the weak globbing to the stronger regexp, we support both. /// /// /// public static string ConvertGlobbingToRegExp(string globexp) { //First escape all special characters globexp = Regex.Replace(globexp, BADCHARS, @"\$&"); //Replace the globbing expressions with the corresponding regular expressions globexp = globexp.Replace('?', '.').Replace("*", ".*"); return globexp; } /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// A list of the full filenames public static IEnumerable EnumerateFiles(string basepath) { return EnumerateFiles(basepath, null); } /// /// Returns a list of folder names found in the given folder. /// The search is recursive. /// /// The folder to look in /// A list of the full paths public static IEnumerable EnumerateFolders(string basepath) { return EnumerateFolders(basepath, null); } /// /// Returns a list of all files and subfolders found in the given folder. /// The search is recursive. /// /// The folder to look in. /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char public static IEnumerable EnumerateFileSystemEntries(string basepath) { return EnumerateFileSystemEntries(basepath, (IFilter)null); } /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// The filter to apply. /// A list of the full filenames public static IEnumerable EnumerateFiles(string basepath, IFilter filter) { return EnumerateFileSystemEntries(basepath, filter).Where(x => !x.EndsWith(DirectorySeparatorString, StringComparison.Ordinal)); } /// /// Returns a list of folder names found in the given folder. /// The search is recursive. /// /// The folder to look in /// The filter to apply. /// A list of the full paths public static IEnumerable EnumerateFolders(string basepath, IFilter filter) { return EnumerateFileSystemEntries(basepath, filter).Where(x => x.EndsWith(DirectorySeparatorString, StringComparison.Ordinal)); } /// /// Returns a list of all files and subfolders found in the given folder. /// The search is recursive. /// /// The folder to look in. /// The filter to apply. /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char public static IEnumerable EnumerateFileSystemEntries(string basepath, IFilter filter) { filter = filter ?? new FilterExpression(); return EnumerateFileSystemEntries(basepath, (rootpath, path, attributes) => { if (!filter.Matches(path, out var result, out _)) result = true; return result; }); } /// /// A callback delegate used for applying alternate enumeration of filesystems /// /// The path to return data from /// A list of paths public delegate string[] FileSystemInteraction(string path); /// /// A callback delegate used for extracting attributes from a file or folder /// /// The path to return data from /// Attributes for the file or folder public delegate FileAttributes ExtractFileAttributes(string path); /// /// A callback delegate used for extracting attributes from a file or folder /// /// The root folder where the path was found /// The path that produced the error /// The exception for the error public delegate void ReportAccessError(string rootpath, string path, Exception ex); /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// The function to call with the filenames /// A list of the full filenames public static IEnumerable EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback) { return EnumerateFileSystemEntries(rootpath, callback, Directory.GetDirectories, Directory.GetFiles); } /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// The function to call with the filenames /// A function to call that lists all folders in the supplied folder /// A function to call that lists all files in the supplied folder /// A list of the full filenames public static IEnumerable EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList) { return EnumerateFileSystemEntries(rootpath, callback, folderList, fileList, null); } /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// The function to call with the filenames /// A function to call that lists all folders in the supplied folder /// A function to call that lists all files in the supplied folder /// A function to call that obtains the attributes for an element, set to null to avoid reading attributes /// An optional function to call with error messages. /// A list of the full filenames public static IEnumerable EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader, ReportAccessError errorCallback = null) { var lst = new Stack(); if (IsFolder(rootpath, attributeReader)) { rootpath = AppendDirSeparator(rootpath); try { var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Directory; if (callback(rootpath, rootpath, attr)) lst.Push(rootpath); } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, rootpath, ex); callback(rootpath, rootpath, FileAttributes.Directory | ATTRIBUTE_ERROR); } while (lst.Count > 0) { var f = AppendDirSeparator(lst.Pop()); yield return f; try { foreach (var s in folderList(f)) { var sf = AppendDirSeparator(s); try { var attr = attributeReader?.Invoke(sf) ?? FileAttributes.Directory; if (callback(rootpath, sf, attr)) lst.Push(sf); } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, sf, ex); callback(rootpath, sf, FileAttributes.Directory | ATTRIBUTE_ERROR); } } } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, f, ex); callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR); } string[] files = null; if (fileList != null) { try { files = fileList(f); } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, f, ex); callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR); } } if (files != null) { foreach (var s in files) { try { var attr = attributeReader?.Invoke(s) ?? FileAttributes.Normal; if (!callback(rootpath, s, attr)) continue; } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, s, ex); callback(rootpath, s, ATTRIBUTE_ERROR); continue; } yield return s; } } } } else { try { var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Normal; if (!callback(rootpath, rootpath, attr)) yield break; } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, rootpath, ex); callback(rootpath, rootpath, ATTRIBUTE_ERROR); yield break; } yield return rootpath; } } /// /// Test if specified path is a folder /// /// Path to test /// Function to use for testing path /// True if path is refers to a folder public static bool IsFolder(string path, ExtractFileAttributes attributeReader) { if (attributeReader == null) return true; try { return attributeReader(path).HasFlag(FileAttributes.Directory); } catch { return false; } } /// /// Tests if path refers to a file, or folder, below the parent folder /// /// File or folder to test /// Candidate parent folder /// True if below parent folder, false otherwise /// (note that this returns false if the two argument paths are identical!) public static bool IsPathBelowFolder(string fileOrFolderPath, string parentFolder) { var sanitizedParentFolder = AppendDirSeparator(parentFolder); return fileOrFolderPath.StartsWith(sanitizedParentFolder, ClientFilenameStringComparison) && !fileOrFolderPath.Equals(sanitizedParentFolder, ClientFilenameStringComparison); } /// /// Returns parent folder of path /// /// Full file or folder path /// If true, return value always has trailing separator /// Parent folder of path (containing folder for file paths, parent folder for folder paths) public static string GetParent(string path, bool forceTrailingDirectorySeparator) { var len = path.Length - 1; if (len > 1 && path[len] == Path.DirectorySeparatorChar) { len--; } var last = path.LastIndexOf(Path.DirectorySeparatorChar, len); if (last == -1 || last == 0 && len == 0) return null; if (last == 0 && !IsClientWindows) return DirectorySeparatorString; var parent = path.Substring(0, last); if (forceTrailingDirectorySeparator || IsClientWindows && parent.Length == 2 && parent[1] == ':' && char.IsLetter(parent[0])) { parent += Path.DirectorySeparatorChar; } return parent; } /// /// Given a collection of unique folders, returns only parent-most folders /// /// Collection of unique folders /// Parent-most folders of input collection public static IEnumerable SimplifyFolderList(ICollection folders) { if (!folders.Any()) return folders; var result = new LinkedList(); result.AddFirst(folders.First()); foreach (var folder1 in folders) { bool addFolder = true; LinkedListNode next; for (var node = result.First; node != null; node = next) { next = node.Next; var folder2 = node.Value; if (IsPathBelowFolder(folder1, folder2)) { // higher-level folder already present addFolder = false; break; } if (IsPathBelowFolder(folder2, folder1)) { // retain folder1 result.Remove(node); } } if (addFolder) { result.AddFirst(folder1); } } return result.Distinct(); } /// /// Given a collection of file paths, return those NOT contained within specified collection of folders /// /// Collection of files to filter /// Collection of folders to use as filter /// Files not in any of specified folders public static IEnumerable GetFilesNotInFolders(IEnumerable files, IEnumerable folders) { return files.Where(x => folders.All(folder => !IsPathBelowFolder(x, folder))); } /// /// Calculates the size of files in a given folder /// /// The folder to examine /// A filter to apply /// The combined size of all files that match the filter public static long GetDirectorySize(string folder, IFilter filter) { return EnumerateFolders(folder, filter).Sum((path) => new FileInfo(path).Length); } /// /// A cached instance of the directory separator as a string /// public static readonly string DirectorySeparatorString = Path.DirectorySeparatorChar.ToString(); /// /// Appends the appropriate directory separator to paths, depending on OS. /// Does not append the separator if the path already ends with it. /// /// The path to append to /// The path with the directory separator appended public static string AppendDirSeparator(string path) { return AppendDirSeparator(path, DirectorySeparatorString); } /// /// Appends the appropriate directory separator to paths, depending on OS. /// Does not append the separator if the path already ends with it. /// /// The path to append to /// The directory separator to use /// The path with the directory separator appended public static string AppendDirSeparator(string path, string separator) { return !path.EndsWith(DirectorySeparatorString, StringComparison.Ordinal) ? path + separator : path; } /// /// Guesses the directory separator from the path /// /// The path to guess the separator from /// The guessed directory separator public static string GuessDirSeparator(string path) { return string.IsNullOrWhiteSpace(path) || path.StartsWith("/", StringComparison.Ordinal) ? "/" : "\\"; } /// /// Some streams can return a number that is less than the requested number of bytes. /// This is usually due to fragmentation, and is solved by issuing a new read. /// This function wraps that functionality. /// /// The stream to read /// The buffer to read into /// The amout of bytes to read /// The actual number of bytes read public static int ForceStreamRead(Stream stream, byte[] buf, int count) { int a; int index = 0; do { a = stream.Read(buf, index, count); index += a; count -= a; } while (a != 0 && count > 0); return index; } /// /// Some streams can return a number that is less than the requested number of bytes. /// This is usually due to fragmentation, and is solved by issuing a new read. /// This function wraps that functionality. /// /// The stream to read. /// The buffer to read into. /// The amout of bytes to read. /// The number of bytes read public static async Task ForceStreamReadAsync(this System.IO.Stream stream, byte[] buf, int count) { int a; int index = 0; do { a = await stream.ReadAsync(buf, index, count); index += a; count -= a; } while (a != 0 && count > 0); return index; } /// /// Compares two streams to see if they are binary equals /// /// One stream /// Another stream /// True if the length of the two streams should be compared /// True if they are equal, false otherwise public static bool CompareStreams(Stream stream1, Stream stream2, bool checkLength) { if (checkLength) { try { if (stream1.Length != stream2.Length) return false; } catch { //We must read along, trying to determine if they are equals } } int longSize = BitConverter.GetBytes((long)0).Length; byte[] buf1 = new byte[longSize * 512]; byte[] buf2 = new byte[buf1.Length]; int a1, a2; while ((a1 = ForceStreamRead(stream1, buf1, buf1.Length)) == (a2 = ForceStreamRead(stream2, buf2, buf2.Length))) { int ix = 0; for (int i = 0; i < a1 / longSize; i++) if (BitConverter.ToUInt64(buf1, ix) != BitConverter.ToUInt64(buf2, ix)) return false; else ix += longSize; for (int i = 0; i < a1 % longSize; i++) if (buf1[ix] != buf2[ix]) return false; else ix++; if (a1 == 0) break; } return a1 == a2; } /// /// Calculates the hash of a given file, and returns the results as an base64 encoded string /// /// The path to the file to calculate the hash for /// The base64 encoded hash public static string CalculateHash(string path) { using (FileStream fs = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.Read)) return CalculateHash(fs); } /// /// Calculates the hash of a given stream, and returns the results as an base64 encoded string /// /// The stream to calculate the hash for /// The base64 encoded hash public static string CalculateHash(Stream stream) { return Convert.ToBase64String(HashAlgorithmHelper.Create(HashAlgorithm).ComputeHash(stream)); } /// /// Reads a file, attempts to detect encoding /// /// The path to the file to read /// The file contents public static string ReadFileWithDefaultEncoding(string filename) { // Since StreamReader defaults to UTF8 and most text files will NOT be UTF8 without BOM, // we need to detect the encoding (at least that it's not UTF8). // So we read the first 4096 bytes and try to decode them as UTF8. var buffer = new byte[4096]; using (var file = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) file.Read(buffer, 0, 4096); var enc = Encoding.UTF8; try { // this will throw an error if not really UTF8 // ReSharper disable once ReturnValueOfPureMethodIsNotUsed new UTF8Encoding(false, true).GetString(buffer); } catch (Exception) { enc = Encoding.Default; } // This will load the text using the BOM, or the detected encoding if no BOM. using (var reader = new StreamReader(filename, enc, true)) { // Remove all \r from the file and split on \n, then pass directly to ExtractOptions return reader.ReadToEnd(); } } /// /// Formats a size into a human readable format, eg. 2048 becomes "2 KB" or -2283 becomes "-2.23 KB%quot. /// /// The size to format /// A human readable string representing the size public static string FormatSizeString(double size) { double sizeAbs = Math.Abs(size); // Allow formatting of negative sizes if (sizeAbs >= 1024 * 1024 * 1024 * 1024L) return Strings.Utility.FormatStringTB(size / (1024 * 1024 * 1024 * 1024L)); else if (sizeAbs >= 1024 * 1024 * 1024) return Strings.Utility.FormatStringGB(size / (1024 * 1024 * 1024)); else if (sizeAbs >= 1024 * 1024) return Strings.Utility.FormatStringMB(size / (1024 * 1024)); else if (sizeAbs >= 1024) return Strings.Utility.FormatStringKB(size / 1024); else return Strings.Utility.FormatStringB((long) size); // safe to cast because lower than 1024 and thus well within range of long } public static System.Threading.ThreadPriority ParsePriority(string value) { if (string.IsNullOrEmpty(value) || value.Trim().Length == 0) return System.Threading.ThreadPriority.Normal; switch (value.ToLower().Trim()) { case "+2": case "high": case "highest": return System.Threading.ThreadPriority.Highest; case "+1": case "abovenormal": case "above normal": return System.Threading.ThreadPriority.AboveNormal; case "-1": case "belownormal": case "below normal": return System.Threading.ThreadPriority.BelowNormal; case "-2": case "low": case "lowest": case "idle": return System.Threading.ThreadPriority.Lowest; default: return System.Threading.ThreadPriority.Normal; } } /// /// Parses a string into a boolean value. /// /// The value to parse. /// A delegate that returns the default value if is not a valid boolean value. /// The parsed value, or the value returned by . public static bool ParseBool(string value, Func defaultFunc) { if (String.IsNullOrWhiteSpace(value)) { return defaultFunc(); } switch (value.Trim().ToLower()) { case "1": case "on": case "true": case "yes": return true; case "0": case "off": case "false": case "no": return false; default: return defaultFunc(); } } /// /// Parses a string into a boolean value. /// /// The value to parse. /// The default value, in case is not a valid boolean value. /// The parsed value, or the default value. public static bool ParseBool(string value, bool @default) { return ParseBool(value, () => @default); } /// /// Parses an option from the option set, using the convention that if the option is set, it is true unless it parses to false, and false otherwise /// /// The set of options to look for the setting in /// The value to look for in the settings /// public static bool ParseBoolOption(IDictionary options, string value) { string opt; if (options.TryGetValue(value, out opt)) return ParseBool(opt, true); else return false; } /// /// Parses an enum found in the options dictionary /// /// The parsed or default enum value. /// The set of options to look for the setting in /// The value to look for in the settings /// The default value to return if there are no matches. /// The enum type parameter. public static T ParseEnumOption(IDictionary options, string value, T @default) { return options.TryGetValue(value, out var opt) ? ParseEnum(opt, @default) : @default; } /// /// Attempts to parse an enum with case-insensitive lookup, returning the default value if there was no match /// /// The parsed or default enum value. /// The string to parse. /// The default value to return if there are no matches. /// The enum type parameter. public static T ParseEnum(string value, T @default) { foreach (var s in Enum.GetNames(typeof(T))) if (s.Equals(value, StringComparison.OrdinalIgnoreCase)) return (T)Enum.Parse(typeof(T), s); return @default; } /// /// Converts a sequence of bytes to a hex string /// /// The array as hex string. /// The data to convert public static string ByteArrayAsHexString(byte[] data) { return BitConverter.ToString(data).Replace("-", string.Empty); } /// /// Converts a hex string to a byte array /// /// The string as byte array. /// The hex string /// The parsed data public static byte[] HexStringAsByteArray(string hex, byte[] data) { for (var i = 0; i < hex.Length; i += 2) data[i / 2] = Convert.ToByte(hex.Substring(i, 2), 16); return data; } public static bool Which(string appname) { if (!IsClientLinux) return false; try { var psi = new System.Diagnostics.ProcessStartInfo("which", appname); psi.RedirectStandardOutput = true; psi.UseShellExecute = false; var pi = System.Diagnostics.Process.Start(psi); pi.WaitForExit(5000); if (pi.HasExited) return pi.ExitCode == 0; else return false; } catch { } return false; } private static string UNAME; /// /// Gets or sets a value indicating if the client is running OSX /// public static bool IsClientOSX { get { // Sadly, Mono returns Unix when running on OSX //return Environment.OSVersion.Platform == PlatformID.MacOSX; if (!IsClientLinux) return false; try { if (UNAME == null) { var psi = new System.Diagnostics.ProcessStartInfo("uname") { RedirectStandardOutput = true, UseShellExecute = false }; var pi = System.Diagnostics.Process.Start(psi); pi.WaitForExit(5000); if (pi.HasExited) UNAME = pi.StandardOutput.ReadToEnd().Trim(); } } catch { } return "Darwin".Equals(UNAME); } } /// /// Gets the output of "uname -a" on Linux, or null on Windows /// public static string UnameAll { get { if (!IsClientLinux) return null; try { var psi = new System.Diagnostics.ProcessStartInfo("uname", "-a"); psi.RedirectStandardOutput = true; psi.UseShellExecute = false; var pi = System.Diagnostics.Process.Start(psi); pi.WaitForExit(5000); if (pi.HasExited) return pi.StandardOutput.ReadToEnd().Trim(); } catch { // ignored } return null; } } /// /// Gets or sets a value indicating if the client is Linux/Unix based /// public static bool IsClientLinux => Environment.OSVersion.Platform == PlatformID.Unix || Environment.OSVersion.Platform == PlatformID.MacOSX; /// /// Gets a value indicating if the client is Windows based /// public static bool IsClientWindows => !IsClientLinux; /// /// Returns a value indicating if the filesystem, is case sensitive /// public static bool IsFSCaseSensitive { get { if (!CachedIsFSCaseSensitive.HasValue) { var str = Environment.GetEnvironmentVariable("FILESYSTEM_CASE_SENSITIVE"); // TODO: This should probably be determined by filesystem rather than OS, // OSX can actually have the disks formated as Case Sensitive, but insensitive is default CachedIsFSCaseSensitive = ParseBool(str, () => IsClientLinux && !IsClientOSX); } return CachedIsFSCaseSensitive.Value; } } /// /// Returns a value indicating if the app is running under Mono /// public static bool IsMono => Type.GetType("Mono.Runtime") != null; /// /// Gets the current Mono runtime version, will return 0.0 if not running Mono /// public static Version MonoVersion { get { try { var v = MonoDisplayVersion; if (v != null) { var regex = new Regex(@"\d+\.\d+(\.\d+)?(\.\d+)?"); var match = regex.Match(v); if (match.Success) return new Version(match.Value); } } catch { // ignored } return new Version(); } } /// /// Gets the Mono display version, or null if not running Mono /// public static string MonoDisplayVersion { get { try { var t = Type.GetType("Mono.Runtime"); if (t != null) { var mi = t.GetMethod("GetDisplayName", System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic); if (mi != null) return (string)mi.Invoke(null, null); } } catch { // ignored } return null; } } /// /// Gets the users default UI language /// public static System.Globalization.CultureInfo DefaultCulture => new System.Threading.Thread(() => { }).CurrentUICulture; /// /// Gets a string comparer that matches the client filesystems case sensitivity /// public static StringComparer ClientFilenameStringComparer => IsFSCaseSensitive ? StringComparer.CurrentCulture : StringComparer.CurrentCultureIgnoreCase; /// /// Gets the string comparision that matches the client filesystems case sensitivity /// public static StringComparison ClientFilenameStringComparison => IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase; /// /// Searches the system paths for the file specified /// /// The file to locate /// The full path to the file, or null if the file was not found public static string LocateFileInSystemPath(string filename) { try { if (Path.IsPathRooted(filename)) return File.Exists(filename) ? filename : null; try { filename = Path.GetFileName(filename); } catch { // ignored } string homedir = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + Path.PathSeparator.ToString(); //Look in application base folder and all system path folders foreach (string s in (homedir + Environment.GetEnvironmentVariable("PATH")).Split(Path.PathSeparator)) if (!string.IsNullOrEmpty(s) && s.Trim().Length > 0) try { foreach (string sx in Directory.GetFiles(ExpandEnvironmentVariables(s), filename)) return sx; } catch { // ignored } } catch { // ignored } return null; } /// /// The path to the users home directory /// public static readonly string HOME_PATH = Environment.GetFolderPath(IsClientLinux ? Environment.SpecialFolder.Personal : Environment.SpecialFolder.UserProfile); /// /// Expands environment variables. /// /// The expanded string. /// The string to expand. public static string ExpandEnvironmentVariables(string str) { return Environment.ExpandEnvironmentVariables(str); } /// /// Regexp for matching environment variables on Windows (%VAR%) /// private static readonly Regex ENVIRONMENT_VARIABLE_MATCHER_WINDOWS = new Regex(@"\%(?\w+)\%"); /// /// Regexp for matching environment variables on Linux ($VAR or ${VAR}) /// private static readonly Regex ENVIRONMENT_VARIABLE_MATCHER_LINUX = new Regex(@"\$(?\w+)|(\{(?[^\}]+)\})"); /// /// Expands environment variables in a RegExp safe format /// /// The expanded string. /// The string to expand. /// A lookup method that converts an environment key to an expanded string public static string ExpandEnvironmentVariablesRegexp(string str, Func lookup = null) { if (lookup == null) lookup = Environment.GetEnvironmentVariable; return // TODO: Should we switch to using the native format, instead of following the Windows scheme? //IsClientLinux ? ENVIRONMENT_VARIABLE_MATCHER_LINUX : ENVIRONMENT_VARIABLE_MATCHER_WINDOWS ENVIRONMENT_VARIABLE_MATCHER_WINDOWS.Replace(str, m => Regex.Escape(lookup(m.Groups["name"].Value))); } /// /// Checks that a hostname is valid /// /// The hostname to verify /// True if the hostname is valid, false otherwise public static bool IsValidHostname(string hostname) { try { return System.Uri.CheckHostName(hostname) != UriHostNameType.Unknown; } catch { return false; } } /// /// The format string for a DateTime /// //Note: Actually the K should be Z which is more correct as it is forced to be Z, but Z as a format specifier is fairly undocumented public static string SERIALIZED_DATE_TIME_FORMAT = "yyyyMMdd'T'HHmmssK"; /// /// Returns a string representation of a in UTC format /// /// The instance /// A string representing the time public static string SerializeDateTime(DateTime dt) { return dt.ToUniversalTime().ToString(SERIALIZED_DATE_TIME_FORMAT, System.Globalization.CultureInfo.InvariantCulture); } /// /// Parses a serialized instance /// /// The string to parse /// The parsed instance public static bool TryDeserializeDateTime(string str, out DateTime dt) { return DateTime.TryParseExact(str, SERIALIZED_DATE_TIME_FORMAT, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AssumeUniversal, out dt); } /// /// Parses a serialized instance /// /// The string to parse /// The parsed instance public static DateTime DeserializeDateTime(string str) { if (!TryDeserializeDateTime(str, out var dt)) throw new Exception(Strings.Utility.InvalidDateError(str)); return dt; } /// /// Gets the unique items from a collection. /// /// The type of the elements in . /// The collection to remove duplicate items from. /// The duplicate items in . /// The unique items from . public static ISet GetUniqueItems(IEnumerable collection, out ISet duplicateItems) { return GetUniqueItems(collection, EqualityComparer.Default, out duplicateItems); } /// /// Gets the unique items from a collection. /// /// The type of the elements in . /// The collection to remove duplicate items from. /// The implementation to use when comparing values in the collection. /// The duplicate items in . /// The unique items from . public static ISet GetUniqueItems(IEnumerable collection, IEqualityComparer comparer, out ISet duplicateItems) { var uniqueItems = new HashSet(comparer); duplicateItems = new HashSet(comparer); foreach (var item in collection) { if (!uniqueItems.Add(item)) duplicateItems.Add(item); } return uniqueItems; } /// /// Helper method that replaces one file with another /// /// The file to replace /// The file to replace with public static void ReplaceFile(string target, string sourcefile) { if (File.Exists(target)) File.Delete(target); //Nasty workaround for the fact that a recently deleted file occasionally blocks a new write long i = 5; do { try { File.Move(sourcefile, target); break; } catch (Exception ex) { if (i == 0) throw new Exception( $"Failed to replace the file \"{target}\" volume with the \"{sourcefile}\", error: {ex.Message}"); System.Threading.Thread.Sleep(250); } } while (i-- > 0); } // // Returns the entry assembly or reasonable approximation if no entry assembly is available. // This is the case in NUnit tests. The following approach does not work w/ Mono due to unimplemented members: // http://social.msdn.microsoft.com/Forums/nb-NO/clr/thread/db44fe1a-3bb4-41d4-a0e0-f3021f30e56f // so this layer of indirection is necessary // // entry assembly or reasonable approximation public static System.Reflection.Assembly getEntryAssembly() { return System.Reflection.Assembly.GetEntryAssembly() ?? System.Reflection.Assembly.GetExecutingAssembly(); } /// /// Converts a Base64 encoded string to "base64 for url" /// See https://en.wikipedia.org/wiki/Base64#URL_applications /// /// The base64 encoded string /// The base64 for url encoded string public static string Base64PlainToBase64Url(string data) { return data.Replace('+', '-').Replace('/', '_'); } /// /// Converts a "base64 for url" encoded string to a Base64 encoded string. /// See https://en.wikipedia.org/wiki/Base64#URL_applications /// /// The base64 for url encoded string /// The base64 encoded string public static string Base64UrlToBase64Plain(string data) { return data.Replace('-', '+').Replace('_', '/'); } /// /// Encodes a byte array into a "base64 for url" encoded string. /// See https://en.wikipedia.org/wiki/Base64#URL_applications /// /// The data to encode /// The base64 for url encoded string public static string Base64UrlEncode(byte[] data) { return Base64PlainToBase64Url(Convert.ToBase64String(data)); } /// /// Decodes a "base64 for url" encoded string into the raw byte array. /// See https://en.wikipedia.org/wiki/Base64#URL_applications /// /// The data to decode /// The raw data public static byte[] Base64UrlDecode(string data) { return Convert.FromBase64String(Base64UrlToBase64Plain(data)); } /// /// Converts a DateTime instance to a Unix timestamp /// /// The Unix timestamp. /// The DateTime instance to convert. public static long ToUnixTimestamp(DateTime input) { var ticks = input.ToUniversalTime().Ticks; ticks -= ticks % TimeSpan.TicksPerSecond; input = new DateTime(ticks, DateTimeKind.Utc); return (long)Math.Floor((input - EPOCH).TotalSeconds); } /// /// Converts a Unix timestamp to a DateTime instance /// /// The DateTime instance represented by the timestamp. /// The Unix timestamp to use. public static DateTime ToUnixTimestamp(long input) { return EPOCH.AddSeconds(input); } /// /// Returns a value indicating if the given type should be treated as a primitive /// /// true, if type is primitive for serialization, false otherwise. /// The type to check. private static bool IsPrimitiveTypeForSerialization(Type t) { return t.IsPrimitive || t.IsEnum || t == typeof(string) || t == typeof(DateTime) || t == typeof(TimeSpan); } /// /// Writes a primitive to the output, or returns false if the input is not primitive /// /// true, the item was printed, false otherwise. /// The item to write. /// The target writer. private static bool PrintSerializeIfPrimitive(object item, TextWriter writer) { if (item == null) { writer.Write("null"); return true; } if (IsPrimitiveTypeForSerialization(item.GetType())) { if (item is DateTime) { writer.Write(((DateTime)item).ToLocalTime()); writer.Write(" ("); writer.Write(ToUnixTimestamp((DateTime)item)); writer.Write(")"); } else writer.Write(item); return true; } return false; } /// /// Prints the object to a stream, which can be used for display or logging /// /// The serialized object /// The object to serialize /// The writer to write the results to /// A filter applied to properties to decide if they are omitted or not /// A value indicating if non-primitive values are recursed /// The string indentation /// A lookup table with visited objects, used to avoid inifinite recursion /// The maximum number of items to report from an IEnumerable instance public static void PrintSerializeObject(object item, TextWriter writer, Func filter = null, bool recurseobjects = false, int indentation = 0, int collectionlimit = 0, Dictionary visited = null) { visited = visited ?? new Dictionary(); var indentstring = new string(' ', indentation); var first = true; if (item == null || IsPrimitiveTypeForSerialization(item.GetType())) { writer.Write(indentstring); if (PrintSerializeIfPrimitive(item, writer)) return; } foreach (var p in item.GetType().GetProperties()) { if (filter != null && !filter(p, item)) continue; if (IsPrimitiveTypeForSerialization(p.PropertyType)) { if (first) first = false; else writer.WriteLine(); writer.Write("{0}{1}: ", indentstring, p.Name); PrintSerializeIfPrimitive(p.GetValue(item, null), writer); } else if (typeof(System.Collections.IEnumerable).IsAssignableFrom(p.PropertyType)) { var enumerable = (System.Collections.IEnumerable)p.GetValue(item, null); var any = false; if (enumerable != null) { var enumerator = enumerable.GetEnumerator(); if (enumerator != null) { var remain = collectionlimit; if (first) first = false; else writer.WriteLine(); writer.Write("{0}{1}: [", indentstring, p.Name); if (enumerator.MoveNext()) { any = true; writer.WriteLine(); PrintSerializeObject(enumerator.Current, writer, filter, recurseobjects, indentation + 4, collectionlimit, visited); remain--; while (enumerator.MoveNext()) { writer.WriteLine(","); if (remain == 0) { writer.Write("..."); break; } PrintSerializeObject(enumerator.Current, writer, filter, recurseobjects, indentation + 4, collectionlimit, visited); remain--; } } if (any) { writer.WriteLine(); writer.Write(indentstring); } writer.Write("]"); } } } else if (recurseobjects) { var value = p.GetValue(item, null); if (value == null) { if (first) first = false; else writer.WriteLine(); writer.Write("{0}{1}: null", indentstring, p.Name); } else if (!visited.ContainsKey(value)) { if (first) first = false; else writer.WriteLine(); writer.WriteLine("{0}{1}:", indentstring, p.Name); visited[value] = null; PrintSerializeObject(value, writer, filter, recurseobjects, indentation + 4, collectionlimit, visited); } } } writer.Flush(); } /// /// Returns a string representing the object, which can be used for display or logging /// /// The serialized object /// The object to serialize /// A filter applied to properties to decide if they are omitted or not /// A value indicating if non-primitive values are recursed /// The string indentation /// The maximum number of items to report from an IEnumerable instance, set to zero or less for reporting all public static StringBuilder PrintSerializeObject(object item, StringBuilder sb = null, Func filter = null, bool recurseobjects = false, int indentation = 0, int collectionlimit = 10) { sb = sb ?? new StringBuilder(); using (var sw = new StringWriter(sb)) PrintSerializeObject(item, sw, filter, recurseobjects, indentation, collectionlimit); return sb; } /// /// Repeatedly hash a value with a salt. /// This effectively masks the original value, /// and destroys lookup methods, like rainbow tables /// /// The data to hash /// The salt to apply /// The number of times to repeat the hashing /// The salted hash public static byte[] RepeatedHashWithSalt(string data, string salt, int repeats = 1200) { return RepeatedHashWithSalt( Encoding.UTF8.GetBytes(data ?? ""), Encoding.UTF8.GetBytes(salt ?? ""), repeats); } /// /// Repeatedly hash a value with a salt. /// This effectively masks the original value, /// and destroys lookup methods, like rainbow tables /// /// The data to hash /// The salt to apply /// The salted hash public static byte[] RepeatedHashWithSalt(byte[] data, byte[] salt, int repeats = 1200) { // We avoid storing the passphrase directly, // instead we salt and rehash repeatedly using (var h = System.Security.Cryptography.SHA256.Create()) { h.Initialize(); h.TransformBlock(salt, 0, salt.Length, salt, 0); h.TransformFinalBlock(data, 0, data.Length); var buf = h.Hash; for (var i = 0; i < repeats; i++) { h.Initialize(); h.TransformBlock(salt, 0, salt.Length, salt, 0); h.TransformFinalBlock(buf, 0, buf.Length); buf = h.Hash; } return buf; } } /// /// Gets the drive letter from the given volume guid. /// This method cannot be inlined since the System.Management types are not implemented in Mono /// /// Volume guid /// Drive letter, as a single character, or null if the volume wasn't found [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public static string GetDriveLetterFromVolumeGuid(Guid volumeGuid) { // Based on this answer: // https://stackoverflow.com/questions/10186277/how-to-get-drive-information-by-volume-id using (System.Management.ManagementObjectSearcher searcher = new System.Management.ManagementObjectSearcher("Select * from Win32_Volume")) { string targetId = string.Format(@"\\?\Volume{{{0}}}\", volumeGuid); foreach (System.Management.ManagementObject obj in searcher.Get()) { if (string.Equals(obj["DeviceID"].ToString(), targetId, StringComparison.OrdinalIgnoreCase)) { object driveLetter = obj["DriveLetter"]; if (driveLetter != null) { return obj["DriveLetter"].ToString(); } else { // The volume was found, but doesn't have a drive letter associated with it. break; } } } return null; } } /// /// Gets all volume guids and their associated drive letters. /// This method cannot be inlined since the System.Management types are not implemented in Mono /// /// Pairs of drive letter to volume guids [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public static IEnumerable> GetVolumeGuidsAndDriveLetters() { using (var searcher = new System.Management.ManagementObjectSearcher("Select * from Win32_Volume")) { foreach (var obj in searcher.Get()) { var deviceIdObj = obj["DeviceID"]; var driveLetterObj = obj["DriveLetter"]; if (deviceIdObj != null && driveLetterObj != null) { var deviceId = deviceIdObj.ToString(); var driveLetter = driveLetterObj.ToString(); if (!string.IsNullOrEmpty(deviceId) && !string.IsNullOrEmpty(driveLetter)) { yield return new KeyValuePair(driveLetter + @"\", deviceId); } } } } } /// /// The regular expression matching all know non-quoted commandline characters /// private static readonly Regex COMMANDLINE_SAFE = new Regex(@"[A-Za-z0-9\-_/:\.]*"); /// /// Special characters that needs to be escaped on Linux /// private static readonly Regex COMMANDLINE_ESCAPED_LINUX = new Regex(@"[""|$|`|\\|!]"); /// /// Wraps a single argument in quotes suitable for the passing on the commandline /// /// The wrapped commandline element. /// The argument to wrap. /// A flag indicating if environment variables are allowed to be expanded public static string WrapCommandLineElement(string arg, bool allowEnvExpansion) { if (string.IsNullOrWhiteSpace(arg)) return arg; if (!IsClientWindows) { // We could consider using single quotes that prevents all expansions //if (!allowEnvExpansion) // return "'" + arg.Replace("'", "\\'") + "'"; // Linux is using backslash to escape, except for ! arg = COMMANDLINE_ESCAPED_LINUX.Replace(arg, (match) => { if (match.Value == "!") return @"""'!'"""; if (match.Value == "$" && allowEnvExpansion) return match.Value; return @"\" + match.Value; }); } else { // Windows needs only needs " replaced with "", // but is prone to %var% expansion when used in // immediate mode (i.e. from command prompt) // Fortunately it does not expand when processes // are started from within .Net // TODO: I have not found a way to avoid escaping %varname%, // and sadly it expands only if the variable exists // making it even rarer and harder to diagnose when // it happens arg = arg.Replace(@"""", @""""""); // Also fix the case where the argument ends with a slash if (arg[arg.Length - 1] == '\\') arg += @"\"; } // Check that all characters are in the safe set if (COMMANDLINE_SAFE.Match(arg).Length != arg.Length) return @"""" + arg + @""""; else return arg; } /// /// Wrap a set of commandline arguments suitable for the commandline /// /// A commandline string. /// The arguments to create into a commandline. /// A flag indicating if environment variables are allowed to be expanded public static string WrapAsCommandLine(IEnumerable args, bool allowEnvExpansion = false) { return string.Join(" ", args.Select(x => WrapCommandLineElement(x, allowEnvExpansion))); } /// /// Utility method that emulates C#'s built in await keyword without requiring the calling method to be async. /// This method should be preferred over using Task.Result, as it doesn't wrap singular exceptions in AggregateExceptions. /// (It uses Task.GetAwaiter().GetResult(), which is the same thing that await uses under the covers.) /// https://stackoverflow.com/questions/17284517/is-task-result-the-same-as-getawaiter-getresult /// /// Task to await public static void Await(this Task task) { task.GetAwaiter().GetResult(); } /// /// Utility method that emulates C#'s built in await keyword without requiring the calling method to be async. /// This method should be preferred over using Task.Result, as it doesn't wrap singular exceptions in AggregateExceptions. /// (It uses Task.GetAwaiter().GetResult(), which is the same thing that await uses under the covers.) /// https://stackoverflow.com/questions/17284517/is-task-result-the-same-as-getawaiter-getresult /// /// Result type /// Task to await /// Task result public static T Await(this Task task) { return task.GetAwaiter().GetResult(); } } }