#region Disclaimer / License // Copyright (C) 2011, Kenneth Skovhede // http://www.hexad.dk, opensource@hexad.dk // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public // License as published by the Free Software Foundation; either // version 2.1 of the License, or (at your option) any later version. // // This library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU // Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA // #endregion using System; using System.Collections.Generic; using System.Text; namespace Duplicati.Library.Utility { public static class Utility { /// /// Size of buffers for copying stream /// public static long DEFAULT_BUFFER_SIZE = 64 * 1024; /// /// A value indicating if the current process is running in 64bit mode /// public static readonly bool Is64BitProcess = IntPtr.Size == 8; /// /// Gets the hash algorithm used for calculating a hash /// public static string HashAlgorithm { get { return "SHA256"; } } /// /// The EPOCH offset (unix style) /// public static readonly DateTime EPOCH = new DateTime(1970, 1, 1, 0, 0, 0); /// /// The attribute value used to indicate error /// public const System.IO.FileAttributes ATTRIBUTE_ERROR = (System.IO.FileAttributes)(1 << 30); /// /// The callback delegate type used to collecting file information /// /// The path that the file enumeration started at /// The current element /// The attributes of the element /// A value indicating if the folder should be recursed, ignored for other types public delegate bool EnumerationCallbackDelegate(string rootpath, string path, System.IO.FileAttributes attributes); /// /// Copies the content of one stream into another /// /// The stream to read from /// The stream to write to public static void CopyStream(System.IO.Stream source, System.IO.Stream target) { CopyStream(source, target, true); } /// /// Copies the content of one stream into another /// /// The stream to read from /// The stream to write to /// True if an attempt should be made to rewind the source stream, false otherwise public static void CopyStream(System.IO.Stream source, System.IO.Stream target, bool tryRewindSource) { if (tryRewindSource && source.CanSeek) try { source.Position = 0; } catch { } byte[] buf = new byte[DEFAULT_BUFFER_SIZE]; int read; while ((read = source.Read(buf, 0, buf.Length)) != 0) target.Write(buf, 0, read); } /// /// Returns an IEnumerable with all filesystem entries, recursive /// /// The folder to look in /// An optional filter to apply to the filenames /// The function to call with the filenames /// A function to call that lists all folders in the supplied folder /// A function to call that lists all files in the supplied folder /// A function to call that obtains the attributes for an element, set to null to skip symlink detection /// The file system entries public static IEnumerable EnumerateFileSystemEntriesBlocked(string rootpath, FilenameFilter filter, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader) { BlockingQueue queue = new BlockingQueue(); new BlockedCollector(queue, rootpath, filter, folderList, fileList); return new BlockingQueueAsEnumerable(queue); } /// /// Class used to implement blocked collection of files and folders /// private class BlockedCollector { private BlockingQueue m_queue; public BlockedCollector(BlockingQueue queue, string rootpath, FilenameFilter filter, FileSystemInteraction folderList, FileSystemInteraction fileList) { m_queue = queue; System.Threading.Thread t = new System.Threading.Thread(BlockedThreadRunner); t.Start(new object[] { rootpath, filter, folderList, fileList}); } private bool Callback(string rootpath, string path, System.IO.FileAttributes attributes) { if ((attributes & ATTRIBUTE_ERROR) == 0) m_queue.Enqueue(path); return true; } private void BlockedThreadRunner(object _args) { object[] args = (object[])_args; string roothpath = (string)args[0]; FilenameFilter filter = (FilenameFilter)args[1]; FileSystemInteraction folderList = (FileSystemInteraction)args[2]; FileSystemInteraction fileList = (FileSystemInteraction)args[3]; EnumerateFileSystemEntries(roothpath, Callback, folderList, fileList); m_queue.SetCompleted(); } } /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// The filter to apply. /// A list of the full filenames public static List EnumerateFiles(string basepath) { return EnumerateFiles(basepath, null); } /// /// Returns a list of folder names found in the given folder. /// The search is recursive. /// /// The folder to look in /// A list of the full paths public static List EnumerateFolders(string basepath) { return EnumerateFolders(basepath, null); } /// /// Returns a list of all files and subfolders found in the given folder. /// The search is recursive. /// /// The folder to look in. /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char public static List EnumerateFileSystemEntries(string basepath) { return EnumerateFileSystemEntries(basepath, (FilenameFilter)null); } /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// The filter to apply. /// A list of the full filenames public static List EnumerateFiles(string basepath, FilenameFilter filter) { PathCollector c = new PathCollector(false, true, filter); EnumerateFileSystemEntries(basepath, new EnumerationCallbackDelegate(c.Callback)); return c.Files; } /// /// Returns a list of folder names found in the given folder. /// The search is recursive. /// /// The folder to look in /// The filter to apply. /// A list of the full paths public static List EnumerateFolders(string basepath, FilenameFilter filter) { PathCollector c = new PathCollector(true, false, filter); EnumerateFileSystemEntries(basepath, new EnumerationCallbackDelegate(c.Callback)); return c.Files; } /// /// Returns a list of all files and subfolders found in the given folder. /// The search is recursive. /// /// The folder to look in. /// The filter to apply. /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char public static List EnumerateFileSystemEntries(string basepath, FilenameFilter filter) { PathCollector c = new PathCollector(true, true, filter); EnumerateFileSystemEntries(basepath, new EnumerationCallbackDelegate(c.Callback)); return c.Files; } /// /// A callback delegate used for applying alternate enumeration of filesystems /// /// The path to return data from /// A list of paths public delegate string[] FileSystemInteraction(string path); /// /// A callback delegate used for extracting attributes from a file or folder /// /// The path to return data from /// Attributes for the file or folder public delegate System.IO.FileAttributes ExtractFileAttributes(string path); /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// An optional filter to apply to the filenames /// The function to call with the filenames /// A list of the full filenames public static void EnumerateFileSystemEntries(string rootpath, EnumerationCallbackDelegate callback) { EnumerateFileSystemEntries(rootpath, callback, new FileSystemInteraction(System.IO.Directory.GetDirectories), new FileSystemInteraction(System.IO.Directory.GetFiles)); } /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// The function to call with the filenames /// A function to call that lists all folders in the supplied folder /// A function to call that lists all files in the supplied folder /// A list of the full filenames public static void EnumerateFileSystemEntries(string rootpath, EnumerationCallbackDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList) { EnumerateFileSystemEntries(rootpath, callback, folderList, fileList, null); } /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// The function to call with the filenames /// A function to call that lists all folders in the supplied folder /// A function to call that lists all files in the supplied folder /// A function to call that obtains the attributes for an element, set to null to avoid reading attributes /// A list of the full filenames public static void EnumerateFileSystemEntries(string rootpath, EnumerationCallbackDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader) { if (!System.IO.Directory.Exists(rootpath)) return; Queue lst = new Queue(); lst.Enqueue(rootpath); while (lst.Count > 0) { string f = AppendDirSeparator(lst.Dequeue()); try { System.IO.FileAttributes attr = attributeReader == null ? System.IO.FileAttributes.Directory : attributeReader(f); if (!callback(rootpath, f, attr)) continue; foreach (string s in folderList(f)) lst.Enqueue(s); } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception) { callback(rootpath, f, ATTRIBUTE_ERROR | System.IO.FileAttributes.Directory); } try { if (fileList != null) foreach (string s in fileList(f)) { try { System.IO.FileAttributes attr = attributeReader == null ? System.IO.FileAttributes.Normal : attributeReader(s); callback(rootpath, s, attr); } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception) { callback(rootpath, s, ATTRIBUTE_ERROR); } } } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception) { callback(rootpath, f, ATTRIBUTE_ERROR); } } } /// /// An internal helper class to collect filenames from the enumeration callback /// private class PathCollector { private List m_files; private FilenameFilter m_filter; private bool m_includeFolders; private bool m_includeFiles; public PathCollector(bool includeFolders, bool includeFiles, FilenameFilter filter) { m_files = new List(); m_filter = filter; m_includeFolders = includeFolders; m_includeFiles = includeFiles; } public bool Callback(string rootpath, string path, System.IO.FileAttributes attributes) { if ((attributes & ATTRIBUTE_ERROR) != 0) { //Don't care } else { if (m_filter != null && !m_filter.ShouldInclude(rootpath, path)) return false; bool isDirectory = (attributes & System.IO.FileAttributes.Directory) != 0; if (m_includeFolders && isDirectory) m_files.Add(path); else if (m_includeFiles && !isDirectory) m_files.Add(path); } return true; } public List Files { get { return m_files; } } } /// /// An internal helper class to calculate the size of a folders files /// private class PathSizeCalculator { private long m_size = 0; private FilenameFilter m_filter; public PathSizeCalculator(FilenameFilter filter) { m_filter = filter; } public bool Callback(string rootpath, string path, System.IO.FileAttributes attributes) { if (m_filter != null && !m_filter.ShouldInclude(rootpath, path)) return false; if ((attributes & System.IO.FileAttributes.Directory) == 0) try { m_size += new System.IO.FileInfo(path).Length; } catch { } return true; } public long Size { get { return m_size; } } } /// /// Calculates the size of files in a given folder /// /// The folder to examine /// A filter to apply /// The combined size of all files that match the filter public static long GetDirectorySize(string folder, FilenameFilter filter) { PathSizeCalculator c = new PathSizeCalculator(filter); EnumerateFileSystemEntries(folder, new EnumerationCallbackDelegate(c.Callback)); return c.Size; } /// /// A cached instance of the directory separator as a string /// public static readonly string DirectorySeparatorString = System.IO.Path.DirectorySeparatorChar.ToString(); /// /// Appends the appropriate directory separator to paths, depending on OS. /// Does not append the separator if the path already ends with it. /// /// The path to append to /// The path with the directory separator appended public static string AppendDirSeparator(string path) { if (!path.EndsWith(DirectorySeparatorString)) return path += DirectorySeparatorString; else return path; } /// /// Some streams can return a number that is less than the requested number of bytes. /// This is usually due to fragmentation, and is solved by issuing a new read. /// This function wraps that functionality. /// /// The stream to read /// The buffer to read into /// The amout of bytes to read /// The actual number of bytes read public static int ForceStreamRead(System.IO.Stream stream, byte[] buf, int count) { int a; int index = 0; do { a = stream.Read(buf, index, count); index += a; count -= a; } while (a != 0 && count > 0); return index; } /// /// Compares two streams to see if they are binary equals /// /// One stream /// Another stream /// True if the length of the two streams should be compared /// True if they are equal, false otherwise public static bool CompareStreams(System.IO.Stream stream1, System.IO.Stream stream2, bool checkLength) { if (checkLength) { try { if (stream1.Length != stream2.Length) return false; } catch { //We must read along, trying to determine if they are equals } } int longSize = BitConverter.GetBytes((long)0).Length; byte[] buf1 = new byte[longSize * 512]; byte[] buf2 = new byte[buf1.Length]; int a1, a2; while ((a1 = ForceStreamRead(stream1, buf1, buf1.Length)) == (a2 = ForceStreamRead(stream2, buf2, buf2.Length))) { int ix = 0; for (int i = 0; i < a1 / longSize; i++) if (BitConverter.ToUInt64(buf1, ix) != BitConverter.ToUInt64(buf2, ix)) return false; else ix += longSize; for (int i = 0; i < a1 % longSize; i++) if (buf1[ix] != buf2[ix]) return false; else ix++; if (a1 == 0) break; } return a1 == a2; } /// /// Removes an entire folder, and its contents. /// Equal to System.IO.Directory.Delete /// /// The folder to remove public static void DeleteFolder(string path) { if (!System.IO.Directory.Exists(path)) return; foreach (string s in EnumerateFiles(path)) { System.IO.File.SetAttributes(s, System.IO.FileAttributes.Normal); System.IO.File.Delete(s); } List folders = Utility.EnumerateFolders(path); folders.Sort(); folders.Reverse(); foreach (string s in folders) System.IO.Directory.Delete(s); System.IO.Directory.Delete(path); } /// /// Calculates the hash of a given file, and returns the results as an base64 encoded string /// /// The path to the file to calculate the hash for /// The base64 encoded hash public static string CalculateHash(string path) { using (System.IO.FileStream fs = System.IO.File.Open(path, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read)) return CalculateHash(fs); } /// /// Calculates the hash of a given stream, and returns the results as an base64 encoded string /// /// The stream to calculate the hash for /// The base64 encoded hash public static string CalculateHash(System.IO.Stream stream) { return Convert.ToBase64String(System.Security.Cryptography.HashAlgorithm.Create(HashAlgorithm).ComputeHash(stream)); } /// /// Formats a size into a human readable format, eg. 2048 becomes "2 KB". /// /// The size to format /// A human readable string representing the size public static string FormatSizeString(long size) { if (size >= 1024 * 1024 * 1024) return string.Format(Strings.Utility.FormatStringGB, (double)size / (1024 * 1024 * 1024)); else if (size >= 1024 * 1024) return string.Format(Strings.Utility.FormatStringMB, (double)size / (1024 * 1024)); else if (size >= 1024) return string.Format(Strings.Utility.FormatStringKB, (double)size / 1024); else return string.Format(Strings.Utility.FormatStringB, size); } public static System.Threading.ThreadPriority ParsePriority(string value) { if (string.IsNullOrEmpty(value) || value.Trim().Length == 0) return System.Threading.ThreadPriority.Normal; switch (value.ToLower().Trim()) { case "+2": case "high": case "highest": return System.Threading.ThreadPriority.Highest; case "+1": case "abovenormal": case "above normal": return System.Threading.ThreadPriority.AboveNormal; case "-1": case "belownormal": case "below normal": return System.Threading.ThreadPriority.BelowNormal; case "-2": case "low": case "lowest": case "idle": return System.Threading.ThreadPriority.Lowest; default: return System.Threading.ThreadPriority.Normal; } } /// /// Parses a string into a boolean value /// /// The value to parse /// The default value, in case the string is not a valid boolean value /// The parsed value or the default value public static bool ParseBool(string value, bool @default) { if (value == null) value = ""; switch (value.Trim().ToLower()) { case "1": case "on": case "true": case "yes": return true; case "0": case "off": case "false": case "no": return false; default: return @default; } } /// /// Parses an option from the option set, using the convention that if the option is set, it is true unless it parses to false, and false otherwise /// /// The set of options to look for the setting in /// The value to look for in the settings /// public static bool ParseBoolOption(IDictionary options, string value) { string opt; if (options.TryGetValue(value, out opt)) return ParseBool(opt, true); else return false; } /// /// A helper for converting byte arrays to hex, vice versa /// private const string HEX_DIGITS_UPPER = "0123456789ABCDEF"; /// /// Converts the byte array to hex digits /// /// The data to convert /// The data as a string of hex digits public static string ByteArrayAsHexString(byte[] data) { if (data == null || data.Length == 0) return ""; StringBuilder sb = new StringBuilder(); foreach (byte b in data) { sb.Append(HEX_DIGITS_UPPER[(b >> 4) & 0xF]); sb.Append(HEX_DIGITS_UPPER[b & 0xF]); } return sb.ToString(); } /// /// Converts the given hex string into a byte array /// /// The hex string /// The byte array public static byte[] HexStringAsByteArray(string hex) { if (string.IsNullOrEmpty(hex)) return new byte[0]; hex = hex.Trim().ToUpper(); if (hex.Length % 2 != 0) throw new Exception(Strings.Utility.InvalidHexStringLengthError); byte[] data = new byte[hex.Length]; for (int i = 0; i < hex.Length; i+= 2) { int upper = HEX_DIGITS_UPPER.IndexOf(hex[i]); int lower = HEX_DIGITS_UPPER.IndexOf(hex[i + 1]); if (upper < 0) throw new Exception(string.Format(Strings.Utility.InvalidHexDigitError, hex[i])); if (lower < 0) throw new Exception(string.Format(Strings.Utility.InvalidHexDigitError, hex[i + 1])); data[i % 2] = (byte)((upper << 4) | lower); } return data; } /// /// Gets or sets a value indicating if the client is Linux/Unix based /// public static bool IsClientLinux { get { #if __MonoCS__ if (Environment.OSVersion.Platform == PlatformID.Unix || (int)Environment.OSVersion.Platform == 6) return true; #else if (Environment.OSVersion.Platform == PlatformID.Unix || Environment.OSVersion.Platform == PlatformID.MacOSX) return true; #endif return false; } } /// /// Returns a value indicating if the filesystem, is case sensitive /// public static bool IsFSCaseSensitive { get { //TODO: This should probably be determined by filesystem rather than OS //In case MS decides to support case sensitive filesystems (yeah right :)) return IsClientLinux; } } /// /// Returns a value indicating if the app is running under Mono /// public static bool IsMono { get { return Type.GetType("Mono.Runtime") != null; } } /// /// Gets the current Mono runtime version, will return 0.0 if not running Mono /// public static Version MonoVersion { get { try { Type t = Type.GetType("Mono.Runtime"); if (t != null) { System.Reflection.MethodInfo mi = t.GetMethod("GetDisplayName", System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic); if (mi != null) { string version = (string)mi.Invoke(null, null); return new Version(version.Substring(version.Trim().LastIndexOf(' '))); } } } catch { } return new Version(); } } /// /// Gets the users default UI language /// public static System.Globalization.CultureInfo DefaultCulture { get { System.Threading.Thread t = new System.Threading.Thread(new System.Threading.ThreadStart(DummyMethod)); return t.CurrentUICulture; } } //Unused function, used to create a dummy thread private static void DummyMethod() { } /// /// Gets a string comparer that matches the client filesystems case sensitivity /// public static StringComparer ClientFilenameStringComparer { get { return Utility.IsFSCaseSensitive ? StringComparer.CurrentCulture : StringComparer.CurrentCultureIgnoreCase; } } /// /// Gets the string comparision that matches the client filesystems case sensitivity /// public static StringComparison ClientFilenameStringComparision { get { return Utility.IsFSCaseSensitive ? StringComparison.CurrentCulture : StringComparison.CurrentCultureIgnoreCase; } } /// /// Searches the system paths for the file specified /// /// The file to locate /// The full path to the file, or null if the file was not found public static string LocateFileInSystemPath(string filename) { try { if (System.IO.Path.IsPathRooted(filename)) return System.IO.File.Exists(filename) ? filename : null; try { filename = System.IO.Path.GetFileName(filename); } catch { } string homedir = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) + System.IO.Path.PathSeparator.ToString(); //Look in application base folder and all system path folders foreach (string s in (homedir + Environment.GetEnvironmentVariable("PATH")).Split(System.IO.Path.PathSeparator)) if (!string.IsNullOrEmpty(s) && s.Trim().Length > 0) try { foreach (string sx in System.IO.Directory.GetFiles(Environment.ExpandEnvironmentVariables(s), filename)) return sx; } catch { } } catch { } return null; } /// /// Gets a stream using the GetRequestStream method, but with a timeout. /// This is a workaround for the fact that GetRequestStream() hangs if /// Timeout is set to infinity, but setting it to something else than /// inifinity may abort the request if it is slow. Setting it to something /// large is just as bad, as it would hang for a long time. /// /// The WebRequest to invoke GetRequestStream() on /// The request stream public static System.IO.Stream SafeGetRequestStream(System.Net.WebRequest req) { return (System.IO.Stream)SafeGetRequestOrResponseStream(req, true); } /// /// Gets a response using the GetResponse method, but with a timeout. /// This is a workaround for the fact that GetResponse() hangs if /// Timeout is set to infinity, but setting it to something else than /// inifinity may abort the request if it is slow. Setting it to something /// large is just as bad, as it would hang for a long time. /// /// The WebRequest to invoke GetResponse() on /// The WebResponse public static System.Net.WebResponse SafeGetResponse(System.Net.WebRequest req) { return (System.Net.WebResponse)SafeGetRequestOrResponseStream(req, false); } /// /// The default timeout for a connection response, either for creating the connection or for starting to deliver the results /// private static readonly int DEFAULT_RESPONSE_TIMEOUT = (int)TimeSpan.FromMinutes(2).TotalMilliseconds; /// /// Helper function that invokes a thread and the GetRequestStream() or GetResponse() method /// /// The request to invoke the method on /// A value indicating if the invoked method should be GetRequestStream() or GetResponse() /// Either a System.IO.Stream or a System.Net.WebResponse object private static object SafeGetRequestOrResponseStream(System.Net.WebRequest req, bool getRequest) { object[] args = new object[] { req, getRequest, null }; System.Threading.Thread t = new System.Threading.Thread(new System.Threading.ParameterizedThreadStart(RunSafeGetRequest)); try { //We use the timeout to determine how long we should wait int waitTime = req.Timeout == System.Threading.Timeout.Infinite ? DEFAULT_RESPONSE_TIMEOUT : req.Timeout; t.Start(args); if (t.Join(waitTime)) { if (args[2] == null) throw new Exception(string.Format(Strings.Utility.UnexpectedRequestResultError, "null", "")); else if (args[2] is Exception) throw (Exception)args[2]; if (getRequest && args[2] is System.IO.Stream) return (System.IO.Stream)args[2]; else if (!getRequest && args[2] is System.Net.WebResponse) return (System.Net.WebResponse)args[2]; throw new Exception(string.Format(Strings.Utility.UnexpectedRequestResultError, args[2].GetType(), args[2].ToString())); } else { t.Abort(); throw new System.Net.WebException(Strings.Utility.TimeoutException, null, System.Net.WebExceptionStatus.Timeout, null); } } catch { try { t.Abort(); } catch { } throw; } } /// /// The thread method that is invoked to perform the actual invocation, /// this ensures that we can abort the thread should we want to /// /// An object array with the required parameters. Index 0 is the System.Net.WebRequest, index 1 is the flag indicating if it should use GetRequestStream() or GetResponse(), index 2 is an empty slot for returning the result or an exception private static void RunSafeGetRequest(object data) { object[] args = (object[])data; try { System.Net.WebRequest req = (System.Net.WebRequest)args[0]; req.Timeout = System.Threading.Timeout.Infinite; bool getRequest = (bool)args[1]; if (getRequest) { args[2] = req.GetRequestStream(); } else { args[2] = req.GetResponse(); } } catch (System.Threading.ThreadAbortException) { } catch (Exception ex) { args[2] = ex; } } /// /// Checks that a hostname is valid /// /// The hostname to verify /// True if the hostname is valid, false otherwise public static bool IsValidHostname(string hostname) { try { return Uri.CheckHostName(hostname) != UriHostNameType.Unknown; } catch { return false; } } } }