Improved validation by checking the nuget package versions against the actual binary versions in the output folder.
Added check for executables to avoid shipping with missing executables.
This commit is contained in:
@@ -64,42 +64,6 @@ public static partial class Command
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verify that some files that are expected to be present in the target directory are there
|
||||
/// </summary>
|
||||
/// <param name="buildDir">The build directory to verify</param>
|
||||
/// <param name="target">The target to verify for</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
public static Task VerifyTargetDirectory(string buildDir, PackageTarget target)
|
||||
{
|
||||
var rootFiles = Directory.EnumerateFiles(buildDir, "*", SearchOption.TopDirectoryOnly)
|
||||
.Where(x => x.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) || x.EndsWith(".exe", StringComparison.OrdinalIgnoreCase))
|
||||
.Select(x => Path.GetFileName(x))
|
||||
.ToHashSet(Duplicati.Library.Utility.Utility.ClientFilenameStringComparer);
|
||||
|
||||
// Random sample of files we expect
|
||||
var probeFiles = new string[] {
|
||||
"System.CommandLine.dll",
|
||||
"System.CommandLine.NamingConventionBinder.dll",
|
||||
"AWSSDK.S3.dll",
|
||||
"CoCoL.dll",
|
||||
"Duplicati.Library.Interface.dll",
|
||||
"Google.Apis.Auth.dll",
|
||||
"Google.Apis.Core.dll",
|
||||
"SQLiteHelper.dll",
|
||||
"SQLite.Interop.dll",
|
||||
"Microsoft.IdentityModel.Abstractions.dll",
|
||||
"System.Reactive.dll",
|
||||
"System.Reactive.Linq.dll"
|
||||
};
|
||||
|
||||
foreach (var f in probeFiles)
|
||||
if (!rootFiles.Contains(f))
|
||||
throw new Exception($"Expected file {f} for {target.BuildTargetString}, but was not found in build directory {buildDir}");
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Set of files that are unwanted despite the OS
|
||||
/// </summary>
|
||||
@@ -150,8 +114,7 @@ public static partial class Command
|
||||
/// </summary>
|
||||
/// <param name="os">The operating system to get the unwanted filenames for</param>
|
||||
/// <returns>The list of unwanted filenames</returns>
|
||||
static IEnumerable<string> UnwantedFileGlobExps(OSType os)
|
||||
=> new[] {
|
||||
static IEnumerable<string> UnwantedFileGlobExps(OSType os) => [
|
||||
"Thumbs.db",
|
||||
"desktop.ini",
|
||||
".DS_Store",
|
||||
@@ -159,8 +122,9 @@ public static partial class Command
|
||||
"*.pdb",
|
||||
"*.mdb",
|
||||
"._*",
|
||||
os == OSType.Windows ? "*.sh" : "*.bat"
|
||||
};
|
||||
os == OSType.Windows ? "*.sh" : "*.bat",
|
||||
os == OSType.Windows ? "*.sh" : "*.ps1"
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Returns a regular expression mapping files that are not wanted in the build folders
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
// Copyright (C) 2025, The Duplicati Team
|
||||
// https://duplicati.com, hello@duplicati.com
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a
|
||||
// copy of this software and associated documentation files (the "Software"),
|
||||
// to deal in the Software without restriction, including without limitation
|
||||
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
// and/or sell copies of the Software, and to permit persons to whom the
|
||||
// Software is furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
// DEALINGS IN THE SOFTWARE.
|
||||
using System.Reflection;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace ReleaseBuilder.Build;
|
||||
|
||||
public static partial class Command
|
||||
{
|
||||
/// <summary>
|
||||
/// Main compilation of projects
|
||||
/// </summary>
|
||||
private static class Verify
|
||||
{
|
||||
/// <summary>
|
||||
/// Verify that some files that are expected to be present in the target directory are there
|
||||
/// </summary>
|
||||
/// <param name="buildDir">The build directory to verify</param>
|
||||
/// <param name="target">The target to verify for</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
public static Task VerifyTargetDirectory(string buildDir, PackageTarget target)
|
||||
{
|
||||
var rootFiles = Directory.EnumerateFiles(buildDir, "*", SearchOption.TopDirectoryOnly)
|
||||
.Where(x => x.EndsWith(".dll", StringComparison.OrdinalIgnoreCase) || x.EndsWith(".exe", StringComparison.OrdinalIgnoreCase))
|
||||
.Select(x => Path.GetFileName(x))
|
||||
.ToHashSet(Duplicati.Library.Utility.Utility.ClientFilenameStringComparer);
|
||||
|
||||
// Random sample of files we expect
|
||||
var probeFiles = new string[] {
|
||||
"System.CommandLine.dll",
|
||||
"System.CommandLine.NamingConventionBinder.dll",
|
||||
"AWSSDK.S3.dll",
|
||||
"CoCoL.dll",
|
||||
"Duplicati.Library.Interface.dll",
|
||||
"Google.Apis.Auth.dll",
|
||||
"Google.Apis.Core.dll",
|
||||
"SQLiteHelper.dll",
|
||||
"SQLite.Interop.dll",
|
||||
"Microsoft.IdentityModel.Abstractions.dll",
|
||||
"System.Reactive.dll",
|
||||
"System.Reactive.Linq.dll"
|
||||
};
|
||||
|
||||
foreach (var f in probeFiles)
|
||||
if (!rootFiles.Contains(f))
|
||||
throw new Exception($"Expected file {f} for {target.BuildTargetString}, but was not found in build directory {buildDir}");
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that all expected executables are in the output
|
||||
/// </summary>
|
||||
/// <param name="buildDir">The build directory to verify</param>
|
||||
/// <param name="sourceProjects">The project files to verify</param>
|
||||
/// <param name="target">The target to verify for</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
public static Task VerifyExecutables(string buildDir, IEnumerable<string> projectFiles, PackageTarget target)
|
||||
{
|
||||
var expected = projectFiles.Select(x => Path.GetFileNameWithoutExtension(x))
|
||||
.Select(x => target.OS == OSType.Windows ? $"{x}.exe" : x);
|
||||
|
||||
foreach (var f in expected)
|
||||
if (!File.Exists(Path.Combine(buildDir, f)))
|
||||
throw new Exception($"Expected file {f} for {target.BuildTargetString}, but was not found in build directory {buildDir}");
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Root entry from the dotnet list output
|
||||
/// </summary>
|
||||
/// <param name="Version">The version of the output format</param>
|
||||
/// <param name="Parameters">The parameters used to generate the output</param>
|
||||
/// <param name="Projects">The list of projects found</param>
|
||||
public sealed record RootJson(
|
||||
int Version,
|
||||
string Parameters,
|
||||
IEnumerable<ProjectJson> Projects
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// A single project
|
||||
/// </summary>
|
||||
/// <param name="Path">Full path to the csproj file</param>
|
||||
/// <param name="Frameworks">The frameworks found</param>
|
||||
public sealed record ProjectJson(
|
||||
string Path,
|
||||
IEnumerable<FrameworkJson> Frameworks
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Contents of a framework
|
||||
/// </summary>
|
||||
/// <param name="Framework">The framework name</param>
|
||||
/// <param name="TopLevelPackages">Directly included packages</param>
|
||||
/// <param name="TransitivePackages">Packages included due to the top-level packages</param>
|
||||
public sealed record FrameworkJson(
|
||||
string Framework,
|
||||
IEnumerable<TopLevelJson> TopLevelPackages,
|
||||
IEnumerable<PackageJson> TransitivePackages
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// A top-level package
|
||||
/// </summary>
|
||||
/// <param name="Id">The package id</param>
|
||||
/// <param name="RequestedVersion">The version requested</param>
|
||||
/// <param name="ResolvedVersion">The resolved version</param>
|
||||
public sealed record TopLevelJson(
|
||||
string Id,
|
||||
string RequestedVersion,
|
||||
string ResolvedVersion
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// A transitive package
|
||||
/// </summary>
|
||||
/// <param name="Id">The package id</param>
|
||||
/// <param name="ResolvedVersion">The resolved version</param>
|
||||
public sealed record PackageJson(
|
||||
string Id,
|
||||
string ResolvedVersion
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Executes the dotnet list command and parses the output
|
||||
/// </summary>
|
||||
/// <param name="slnpath">The path to the sln file to analyze</param>
|
||||
/// <returns>The parsed output</returns>
|
||||
public static async Task<RootJson> AnalyzeProject(string slnpath)
|
||||
{
|
||||
var output = await ProcessHelper.ExecuteWithOutput([
|
||||
"dotnet", "list",
|
||||
slnpath, "package",
|
||||
"--include-transitive",
|
||||
"--format", "json"
|
||||
]).ConfigureAwait(false);
|
||||
|
||||
var root = JsonSerializer.Deserialize<RootJson>(output, new JsonSerializerOptions(JsonSerializerOptions.Default) { PropertyNameCaseInsensitive = true })
|
||||
?? throw new Exception("Failed to parse JSON output from dotnet list");
|
||||
if (root.Version != 1)
|
||||
throw new Exception($"Unexpected version {root.Version} from dotnet list");
|
||||
|
||||
return root;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses a version string into a Version object
|
||||
/// </summary>
|
||||
/// <param name="version">The nuget version string</param>
|
||||
/// <returns>A .NET version number</returns>
|
||||
private static Version ParseVersion(string version)
|
||||
{
|
||||
var v = new Version(version.Split("-")[0]);
|
||||
return new Version(v.Major, v.Minor, v.Build, Math.Max(0, v.Revision));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A version that is duplicated in multiple projects
|
||||
/// </summary>
|
||||
/// <param name="Project">The source project</param>
|
||||
/// <param name="Version">The resolved nuget version string</param>
|
||||
/// <param name="ParsedVersion">The resolved parsed version</param>
|
||||
public sealed record DuplicatedVersion(
|
||||
string Project,
|
||||
string Version,
|
||||
Version ParsedVersion
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Parses the output of the dotnet list command and returns a dictionary of duplicated versions
|
||||
/// </summary>
|
||||
/// <param name="input">The parsed output from the dotnet list command</param>
|
||||
/// <returns>A dictionary of duplicated versions, where the key is the package id and the value is a list of projects that use that version</returns>
|
||||
public static Dictionary<string, List<DuplicatedVersion>> GetDuplicatedVersions(RootJson input)
|
||||
=> input.Projects
|
||||
.SelectMany(x => x.Frameworks.Select(y => new
|
||||
{
|
||||
Framework = y,
|
||||
Project = x.Path
|
||||
}))
|
||||
.SelectMany(x =>
|
||||
(x.Framework.TopLevelPackages?
|
||||
.Select(y => new
|
||||
{
|
||||
TopLevel = true,
|
||||
y.Id,
|
||||
y.ResolvedVersion,
|
||||
x.Project
|
||||
}) ?? [])
|
||||
.Concat(x.Framework.TransitivePackages?.Select(y => new { TopLevel = false, y.Id, y.ResolvedVersion, x.Project }) ?? [])
|
||||
)
|
||||
.GroupBy(x => x.Id, x => new DuplicatedVersion(x.Project, x.ResolvedVersion, ParseVersion(x.ResolvedVersion)))
|
||||
.Where(x => x.DistinctBy(y => y.ParsedVersion).Count() > 1)
|
||||
.ToDictionary(
|
||||
x => x.Key,
|
||||
x => x.ToList()
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// Finds the maximum nuget versions of packages
|
||||
/// </summary>
|
||||
/// <param name="input">The parsed output from the dotnet list command</param>
|
||||
/// <returns>A list of nuget versions for each package</returns>
|
||||
public static Dictionary<string, Version> FindMaxNugetVersions(RootJson input)
|
||||
=> input.Projects
|
||||
.SelectMany(x => x.Frameworks)
|
||||
.SelectMany(x =>
|
||||
(x.TopLevelPackages?
|
||||
.Select(x => new { TopLevel = true, x.Id, x.ResolvedVersion }) ?? [])
|
||||
.Concat(x.TransitivePackages?.Select(x => new { TopLevel = false, x.Id, x.ResolvedVersion }) ?? [])
|
||||
)
|
||||
.Where(x => !x.TopLevel)
|
||||
.GroupBy(x => x.Id, x => x.ResolvedVersion)
|
||||
.Select(x => new { x.Key, Version = x.MaxBy(ParseVersion) })
|
||||
.ToDictionary(
|
||||
x => x.Key,
|
||||
x => ParseVersion(x.Version!)
|
||||
);
|
||||
|
||||
/// <summary>
|
||||
/// List of known wrong versions, where the assembly version is not the same as the nuget version
|
||||
/// </summary>
|
||||
private static Dictionary<string, Version> ManuallyFixedVersions = new Dictionary<string, Version>
|
||||
{
|
||||
// Using v3.3 for assembly, but 3.7 in nuget
|
||||
{ "AWSSDK.Core", new Version(3, 3, 0, 0) },
|
||||
|
||||
// Using the Framework version, not the package version
|
||||
{ "Microsoft.CSharp", new Version(8, 0, 0, 0) },
|
||||
{ "System.Memory", new Version(8, 0, 0, 0) },
|
||||
{ "System.Security.AccessControl", new Version(8, 0, 0, 0) },
|
||||
{ "System.Security.Principal.Windows", new Version(8, 0, 0, 0) },
|
||||
{ "System.Security.Cryptography.Algorithms", new Version(8, 0, 0, 0) },
|
||||
{ "System.Security.Cryptography.Cng", new Version(8, 0, 0, 0) },
|
||||
|
||||
// Using v9.0 for assembly, but 9.0.2 in nuget
|
||||
{ "System.IO.Pipelines", new Version(9, 0, 0, 0) }
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the versions of the assemblies in the output folder are the maximum versions
|
||||
/// </summary>
|
||||
/// <param name="folder">The folder to check</param>
|
||||
/// <param name="input">The parsed output from the dotnet list command</param>
|
||||
/// <returns>An awaitable task</returns>
|
||||
public static Task VerifyDuplicatedVersionsAreMaxVersions(string folder, RootJson input)
|
||||
{
|
||||
var duplicatedVersions = GetDuplicatedVersions(input)
|
||||
.Select(x =>
|
||||
{
|
||||
if (ManuallyFixedVersions.TryGetValue(x.Key, out var version))
|
||||
return new KeyValuePair<string, List<DuplicatedVersion>>(x.Key, [new DuplicatedVersion(x.Value.First().Project, x.Value.First().Version, version)]);
|
||||
return new KeyValuePair<string, List<DuplicatedVersion>>(x.Key, x.Value);
|
||||
});
|
||||
|
||||
foreach (var entry in duplicatedVersions)
|
||||
{
|
||||
var maxVersion = entry.Value.MaxBy(x => x.ParsedVersion)
|
||||
?? throw new Exception($"Failed to find max version for {entry.Key}");
|
||||
var filename = Path.Combine(folder, $"{entry.Key}.dll");
|
||||
if (!File.Exists(filename))
|
||||
continue;
|
||||
|
||||
var assemblyVersion = AssemblyName.GetAssemblyName(filename).Version;
|
||||
if (assemblyVersion != maxVersion.ParsedVersion)
|
||||
throw new Exception($"Version mismatch for {filename}: expected {maxVersion.ParsedVersion}, got {assemblyVersion}");
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -53,56 +53,65 @@ public static partial class Command
|
||||
else
|
||||
Console.WriteLine($"Building {buildArchTargets.Length} versions");
|
||||
|
||||
var buildOutputFolders = buildArchTargets.ToDictionary(x => x, x => Path.Combine(buildDir, x.BuildTargetString));
|
||||
var temporarySolutionFiles = new Dictionary<PackageTarget, string>();
|
||||
var targetExecutables = new Dictionary<PackageTarget, List<string>>();
|
||||
var distinctSolutions = buildArchTargets.GroupBy(x => $"{x.InterfaceString}{(x.OS == OSType.Windows ? $"-{x.OSString}" : "")}").ToArray();
|
||||
Console.WriteLine($"Creating {distinctSolutions.Length} temporary solution files");
|
||||
foreach (var tk in distinctSolutions)
|
||||
{
|
||||
// Don't create a solution if all the output folders exist
|
||||
if (tk.All(x => Directory.Exists(buildOutputFolders[x])))
|
||||
continue;
|
||||
|
||||
var tmpslnfile = Path.Combine(buildDir, $"Duplicati-{tk.Key}.sln");
|
||||
if (File.Exists(tmpslnfile))
|
||||
File.Delete(tmpslnfile);
|
||||
|
||||
var logOut = Path.Combine(logFolder, $"create-{tmpslnfile}.log");
|
||||
using var logStream = new FileStream(logOut, FileMode.Create, FileAccess.Write, FileShare.Read);
|
||||
|
||||
await ProcessHelper.ExecuteWithOutput([
|
||||
"dotnet", "new", "sln",
|
||||
"--name", Path.GetFileNameWithoutExtension(tmpslnfile),
|
||||
"--output", buildDir
|
||||
],
|
||||
logStream
|
||||
).ConfigureAwait(false);
|
||||
|
||||
var target = tk.First();
|
||||
if (!sourceProjects.TryGetValue(target.Interface, out var buildProjects))
|
||||
throw new InvalidOperationException($"No projects found for {tk.Key}");
|
||||
|
||||
// Add the projects to the solution
|
||||
foreach (var proj in buildProjects)
|
||||
{
|
||||
if (target.OS != OSType.Windows && windowsOnlyProjects.Contains(proj))
|
||||
continue;
|
||||
var actualBuildProjects = buildProjects
|
||||
.Where(x => target.OS == OSType.Windows || !windowsOnlyProjects.Contains(x))
|
||||
.ToList();
|
||||
|
||||
var projpath = Path.Combine(baseDir, proj);
|
||||
if (!File.Exists(projpath))
|
||||
throw new FileNotFoundException($"Project file {projpath} not found");
|
||||
// Faster debugging, keep the solution file
|
||||
if (!File.Exists(tmpslnfile))
|
||||
{
|
||||
var logOut = Path.Combine(logFolder, $"create-{Path.GetFileName(tmpslnfile)}.log");
|
||||
using var logStream = new FileStream(logOut, FileMode.Create, FileAccess.Write, FileShare.Read);
|
||||
|
||||
await ProcessHelper.ExecuteWithOutput([
|
||||
"dotnet", "sln", tmpslnfile,
|
||||
"add", projpath
|
||||
],
|
||||
logStream
|
||||
).ConfigureAwait(false);
|
||||
"dotnet", "new", "sln",
|
||||
"--name", Path.GetFileNameWithoutExtension(tmpslnfile),
|
||||
"--output", buildDir
|
||||
], logStream).ConfigureAwait(false);
|
||||
|
||||
// Add the projects to the solution
|
||||
foreach (var proj in actualBuildProjects)
|
||||
{
|
||||
var projpath = Path.Combine(baseDir, proj);
|
||||
if (!File.Exists(projpath))
|
||||
throw new FileNotFoundException($"Project file {projpath} not found");
|
||||
|
||||
await ProcessHelper.ExecuteWithOutput([
|
||||
"dotnet", "sln", tmpslnfile,
|
||||
"add", projpath
|
||||
], logStream).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var s in tk)
|
||||
{
|
||||
temporarySolutionFiles[s] = tmpslnfile;
|
||||
targetExecutables[s] = actualBuildProjects;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var target in buildArchTargets)
|
||||
{
|
||||
var outputFolder = Path.Combine(buildDir, target.BuildTargetString);
|
||||
// Start the task now
|
||||
var verifyRootJson = Verify.AnalyzeProject(Path.Combine(baseDir, "Duplicati.sln"));
|
||||
|
||||
foreach ((var target, var outputFolder) in buildOutputFolders)
|
||||
{
|
||||
// Faster iteration for debugging is to keep the build folder
|
||||
if (keepBuilds && Directory.Exists(outputFolder))
|
||||
{
|
||||
@@ -145,7 +154,9 @@ public static partial class Command
|
||||
|
||||
// Perform any post-build steps, cleaning and signing as needed
|
||||
await PostCompile.PrepareTargetDirectory(baseDir, tmpfolder, target, rtcfg, keepBuilds);
|
||||
await PostCompile.VerifyTargetDirectory(tmpfolder, target);
|
||||
await Verify.VerifyTargetDirectory(tmpfolder, target);
|
||||
await Verify.VerifyExecutables(tmpfolder, targetExecutables[target], target);
|
||||
await Verify.VerifyDuplicatedVersionsAreMaxVersions(tmpfolder, await verifyRootJson.ConfigureAwait(false));
|
||||
|
||||
// Move the final build to the output folder
|
||||
Directory.Move(tmpfolder, outputFolder);
|
||||
|
||||
Reference in New Issue
Block a user