diff --git a/ReleaseBuilder/Build/Command.Compile.Post.cs b/ReleaseBuilder/Build/Command.Compile.Post.cs index 60ada0ba7..07f430ea0 100644 --- a/ReleaseBuilder/Build/Command.Compile.Post.cs +++ b/ReleaseBuilder/Build/Command.Compile.Post.cs @@ -64,42 +64,6 @@ public static partial class Command } } - /// - /// Verify that some files that are expected to be present in the target directory are there - /// - /// The build directory to verify - /// The target to verify for - /// An awaitable task - public static Task VerifyTargetDirectory(string buildDir, PackageTarget target) - { - var rootFiles = Directory.EnumerateFiles(buildDir, "*", SearchOption.TopDirectoryOnly) - .Where(x => x.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) || x.EndsWith(".exe", StringComparison.OrdinalIgnoreCase)) - .Select(x => Path.GetFileName(x)) - .ToHashSet(Duplicati.Library.Utility.Utility.ClientFilenameStringComparer); - - // Random sample of files we expect - var probeFiles = new string[] { - "System.CommandLine.dll", - "System.CommandLine.NamingConventionBinder.dll", - "AWSSDK.S3.dll", - "CoCoL.dll", - "Duplicati.Library.Interface.dll", - "Google.Apis.Auth.dll", - "Google.Apis.Core.dll", - "SQLiteHelper.dll", - "SQLite.Interop.dll", - "Microsoft.IdentityModel.Abstractions.dll", - "System.Reactive.dll", - "System.Reactive.Linq.dll" - }; - - foreach (var f in probeFiles) - if (!rootFiles.Contains(f)) - throw new Exception($"Expected file {f} for {target.BuildTargetString}, but was not found in build directory {buildDir}"); - - return Task.CompletedTask; - } - /// /// Set of files that are unwanted despite the OS /// @@ -150,8 +114,7 @@ public static partial class Command /// /// The operating system to get the unwanted filenames for /// The list of unwanted filenames - static IEnumerable UnwantedFileGlobExps(OSType os) - => new[] { + static IEnumerable UnwantedFileGlobExps(OSType os) => [ "Thumbs.db", "desktop.ini", ".DS_Store", @@ -159,8 +122,9 @@ public static partial class Command "*.pdb", "*.mdb", "._*", - os == OSType.Windows ? "*.sh" : "*.bat" - }; + os == OSType.Windows ? "*.sh" : "*.bat", + os == OSType.Windows ? "*.sh" : "*.ps1" + ]; /// /// Returns a regular expression mapping files that are not wanted in the build folders diff --git a/ReleaseBuilder/Build/Command.Compile.Verify.cs b/ReleaseBuilder/Build/Command.Compile.Verify.cs new file mode 100644 index 000000000..91790e1b2 --- /dev/null +++ b/ReleaseBuilder/Build/Command.Compile.Verify.cs @@ -0,0 +1,292 @@ +// Copyright (C) 2025, The Duplicati Team +// https://duplicati.com, hello@duplicati.com +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +using System.Reflection; +using System.Text.Json; + +namespace ReleaseBuilder.Build; + +public static partial class Command +{ + /// + /// Main compilation of projects + /// + private static class Verify + { + /// + /// Verify that some files that are expected to be present in the target directory are there + /// + /// The build directory to verify + /// The target to verify for + /// An awaitable task + public static Task VerifyTargetDirectory(string buildDir, PackageTarget target) + { + var rootFiles = Directory.EnumerateFiles(buildDir, "*", SearchOption.TopDirectoryOnly) + .Where(x => x.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) || x.EndsWith(".exe", StringComparison.OrdinalIgnoreCase)) + .Select(x => Path.GetFileName(x)) + .ToHashSet(Duplicati.Library.Utility.Utility.ClientFilenameStringComparer); + + // Random sample of files we expect + var probeFiles = new string[] { + "System.CommandLine.dll", + "System.CommandLine.NamingConventionBinder.dll", + "AWSSDK.S3.dll", + "CoCoL.dll", + "Duplicati.Library.Interface.dll", + "Google.Apis.Auth.dll", + "Google.Apis.Core.dll", + "SQLiteHelper.dll", + "SQLite.Interop.dll", + "Microsoft.IdentityModel.Abstractions.dll", + "System.Reactive.dll", + "System.Reactive.Linq.dll" + }; + + foreach (var f in probeFiles) + if (!rootFiles.Contains(f)) + throw new Exception($"Expected file {f} for {target.BuildTargetString}, but was not found in build directory {buildDir}"); + + return Task.CompletedTask; + } + + /// + /// Verifies that all expected executables are in the output + /// + /// The build directory to verify + /// The project files to verify + /// The target to verify for + /// An awaitable task + public static Task VerifyExecutables(string buildDir, IEnumerable projectFiles, PackageTarget target) + { + var expected = projectFiles.Select(x => Path.GetFileNameWithoutExtension(x)) + .Select(x => target.OS == OSType.Windows ? $"{x}.exe" : x); + + foreach (var f in expected) + if (!File.Exists(Path.Combine(buildDir, f))) + throw new Exception($"Expected file {f} for {target.BuildTargetString}, but was not found in build directory {buildDir}"); + + return Task.CompletedTask; + } + + /// + /// Root entry from the dotnet list output + /// + /// The version of the output format + /// The parameters used to generate the output + /// The list of projects found + public sealed record RootJson( + int Version, + string Parameters, + IEnumerable Projects + ); + + /// + /// A single project + /// + /// Full path to the csproj file + /// The frameworks found + public sealed record ProjectJson( + string Path, + IEnumerable Frameworks + ); + + /// + /// Contents of a framework + /// + /// The framework name + /// Directly included packages + /// Packages included due to the top-level packages + public sealed record FrameworkJson( + string Framework, + IEnumerable TopLevelPackages, + IEnumerable TransitivePackages + ); + + /// + /// A top-level package + /// + /// The package id + /// The version requested + /// The resolved version + public sealed record TopLevelJson( + string Id, + string RequestedVersion, + string ResolvedVersion + ); + + /// + /// A transitive package + /// + /// The package id + /// The resolved version + public sealed record PackageJson( + string Id, + string ResolvedVersion + ); + + /// + /// Executes the dotnet list command and parses the output + /// + /// The path to the sln file to analyze + /// The parsed output + public static async Task AnalyzeProject(string slnpath) + { + var output = await ProcessHelper.ExecuteWithOutput([ + "dotnet", "list", + slnpath, "package", + "--include-transitive", + "--format", "json" + ]).ConfigureAwait(false); + + var root = JsonSerializer.Deserialize(output, new JsonSerializerOptions(JsonSerializerOptions.Default) { PropertyNameCaseInsensitive = true }) + ?? throw new Exception("Failed to parse JSON output from dotnet list"); + if (root.Version != 1) + throw new Exception($"Unexpected version {root.Version} from dotnet list"); + + return root; + } + + /// + /// Parses a version string into a Version object + /// + /// The nuget version string + /// A .NET version number + private static Version ParseVersion(string version) + { + var v = new Version(version.Split("-")[0]); + return new Version(v.Major, v.Minor, v.Build, Math.Max(0, v.Revision)); + } + + /// + /// A version that is duplicated in multiple projects + /// + /// The source project + /// The resolved nuget version string + /// The resolved parsed version + public sealed record DuplicatedVersion( + string Project, + string Version, + Version ParsedVersion + ); + + /// + /// Parses the output of the dotnet list command and returns a dictionary of duplicated versions + /// + /// The parsed output from the dotnet list command + /// A dictionary of duplicated versions, where the key is the package id and the value is a list of projects that use that version + public static Dictionary> GetDuplicatedVersions(RootJson input) + => input.Projects + .SelectMany(x => x.Frameworks.Select(y => new + { + Framework = y, + Project = x.Path + })) + .SelectMany(x => + (x.Framework.TopLevelPackages? + .Select(y => new + { + TopLevel = true, + y.Id, + y.ResolvedVersion, + x.Project + }) ?? []) + .Concat(x.Framework.TransitivePackages?.Select(y => new { TopLevel = false, y.Id, y.ResolvedVersion, x.Project }) ?? []) + ) + .GroupBy(x => x.Id, x => new DuplicatedVersion(x.Project, x.ResolvedVersion, ParseVersion(x.ResolvedVersion))) + .Where(x => x.DistinctBy(y => y.ParsedVersion).Count() > 1) + .ToDictionary( + x => x.Key, + x => x.ToList() + ); + + /// + /// Finds the maximum nuget versions of packages + /// + /// The parsed output from the dotnet list command + /// A list of nuget versions for each package + public static Dictionary FindMaxNugetVersions(RootJson input) + => input.Projects + .SelectMany(x => x.Frameworks) + .SelectMany(x => + (x.TopLevelPackages? + .Select(x => new { TopLevel = true, x.Id, x.ResolvedVersion }) ?? []) + .Concat(x.TransitivePackages?.Select(x => new { TopLevel = false, x.Id, x.ResolvedVersion }) ?? []) + ) + .Where(x => !x.TopLevel) + .GroupBy(x => x.Id, x => x.ResolvedVersion) + .Select(x => new { x.Key, Version = x.MaxBy(ParseVersion) }) + .ToDictionary( + x => x.Key, + x => ParseVersion(x.Version!) + ); + + /// + /// List of known wrong versions, where the assembly version is not the same as the nuget version + /// + private static Dictionary ManuallyFixedVersions = new Dictionary + { + // Using v3.3 for assembly, but 3.7 in nuget + { "AWSSDK.Core", new Version(3, 3, 0, 0) }, + + // Using the Framework version, not the package version + { "Microsoft.CSharp", new Version(8, 0, 0, 0) }, + { "System.Memory", new Version(8, 0, 0, 0) }, + { "System.Security.AccessControl", new Version(8, 0, 0, 0) }, + { "System.Security.Principal.Windows", new Version(8, 0, 0, 0) }, + { "System.Security.Cryptography.Algorithms", new Version(8, 0, 0, 0) }, + { "System.Security.Cryptography.Cng", new Version(8, 0, 0, 0) }, + + // Using v9.0 for assembly, but 9.0.2 in nuget + { "System.IO.Pipelines", new Version(9, 0, 0, 0) } + }; + + /// + /// Verifies that the versions of the assemblies in the output folder are the maximum versions + /// + /// The folder to check + /// The parsed output from the dotnet list command + /// An awaitable task + public static Task VerifyDuplicatedVersionsAreMaxVersions(string folder, RootJson input) + { + var duplicatedVersions = GetDuplicatedVersions(input) + .Select(x => + { + if (ManuallyFixedVersions.TryGetValue(x.Key, out var version)) + return new KeyValuePair>(x.Key, [new DuplicatedVersion(x.Value.First().Project, x.Value.First().Version, version)]); + return new KeyValuePair>(x.Key, x.Value); + }); + + foreach (var entry in duplicatedVersions) + { + var maxVersion = entry.Value.MaxBy(x => x.ParsedVersion) + ?? throw new Exception($"Failed to find max version for {entry.Key}"); + var filename = Path.Combine(folder, $"{entry.Key}.dll"); + if (!File.Exists(filename)) + continue; + + var assemblyVersion = AssemblyName.GetAssemblyName(filename).Version; + if (assemblyVersion != maxVersion.ParsedVersion) + throw new Exception($"Version mismatch for {filename}: expected {maxVersion.ParsedVersion}, got {assemblyVersion}"); + } + + return Task.CompletedTask; + } + } +} \ No newline at end of file diff --git a/ReleaseBuilder/Build/Command.Compile.cs b/ReleaseBuilder/Build/Command.Compile.cs index 7efd87c36..6640a7efb 100644 --- a/ReleaseBuilder/Build/Command.Compile.cs +++ b/ReleaseBuilder/Build/Command.Compile.cs @@ -53,56 +53,65 @@ public static partial class Command else Console.WriteLine($"Building {buildArchTargets.Length} versions"); + var buildOutputFolders = buildArchTargets.ToDictionary(x => x, x => Path.Combine(buildDir, x.BuildTargetString)); var temporarySolutionFiles = new Dictionary(); + var targetExecutables = new Dictionary>(); var distinctSolutions = buildArchTargets.GroupBy(x => $"{x.InterfaceString}{(x.OS == OSType.Windows ? $"-{x.OSString}" : "")}").ToArray(); Console.WriteLine($"Creating {distinctSolutions.Length} temporary solution files"); foreach (var tk in distinctSolutions) { + // Don't create a solution if all the output folders exist + if (tk.All(x => Directory.Exists(buildOutputFolders[x]))) + continue; + var tmpslnfile = Path.Combine(buildDir, $"Duplicati-{tk.Key}.sln"); - if (File.Exists(tmpslnfile)) - File.Delete(tmpslnfile); - - var logOut = Path.Combine(logFolder, $"create-{tmpslnfile}.log"); - using var logStream = new FileStream(logOut, FileMode.Create, FileAccess.Write, FileShare.Read); - - await ProcessHelper.ExecuteWithOutput([ - "dotnet", "new", "sln", - "--name", Path.GetFileNameWithoutExtension(tmpslnfile), - "--output", buildDir - ], - logStream - ).ConfigureAwait(false); var target = tk.First(); if (!sourceProjects.TryGetValue(target.Interface, out var buildProjects)) throw new InvalidOperationException($"No projects found for {tk.Key}"); - // Add the projects to the solution - foreach (var proj in buildProjects) - { - if (target.OS != OSType.Windows && windowsOnlyProjects.Contains(proj)) - continue; + var actualBuildProjects = buildProjects + .Where(x => target.OS == OSType.Windows || !windowsOnlyProjects.Contains(x)) + .ToList(); - var projpath = Path.Combine(baseDir, proj); - if (!File.Exists(projpath)) - throw new FileNotFoundException($"Project file {projpath} not found"); + // Faster debugging, keep the solution file + if (!File.Exists(tmpslnfile)) + { + var logOut = Path.Combine(logFolder, $"create-{Path.GetFileName(tmpslnfile)}.log"); + using var logStream = new FileStream(logOut, FileMode.Create, FileAccess.Write, FileShare.Read); await ProcessHelper.ExecuteWithOutput([ - "dotnet", "sln", tmpslnfile, - "add", projpath - ], - logStream - ).ConfigureAwait(false); + "dotnet", "new", "sln", + "--name", Path.GetFileNameWithoutExtension(tmpslnfile), + "--output", buildDir + ], logStream).ConfigureAwait(false); + + // Add the projects to the solution + foreach (var proj in actualBuildProjects) + { + var projpath = Path.Combine(baseDir, proj); + if (!File.Exists(projpath)) + throw new FileNotFoundException($"Project file {projpath} not found"); + + await ProcessHelper.ExecuteWithOutput([ + "dotnet", "sln", tmpslnfile, + "add", projpath + ], logStream).ConfigureAwait(false); + } } foreach (var s in tk) + { temporarySolutionFiles[s] = tmpslnfile; + targetExecutables[s] = actualBuildProjects; + } } - foreach (var target in buildArchTargets) - { - var outputFolder = Path.Combine(buildDir, target.BuildTargetString); + // Start the task now + var verifyRootJson = Verify.AnalyzeProject(Path.Combine(baseDir, "Duplicati.sln")); + foreach ((var target, var outputFolder) in buildOutputFolders) + { // Faster iteration for debugging is to keep the build folder if (keepBuilds && Directory.Exists(outputFolder)) { @@ -145,7 +154,9 @@ public static partial class Command // Perform any post-build steps, cleaning and signing as needed await PostCompile.PrepareTargetDirectory(baseDir, tmpfolder, target, rtcfg, keepBuilds); - await PostCompile.VerifyTargetDirectory(tmpfolder, target); + await Verify.VerifyTargetDirectory(tmpfolder, target); + await Verify.VerifyExecutables(tmpfolder, targetExecutables[target], target); + await Verify.VerifyDuplicatedVersionsAreMaxVersions(tmpfolder, await verifyRootJson.ConfigureAwait(false)); // Move the final build to the output folder Directory.Move(tmpfolder, outputFolder);