using System.CommandLine; using System.CommandLine.NamingConventionBinder; using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; using System.Text.RegularExpressions; using Duplicati.Library.AutoUpdater; namespace ReleaseBuilder.Build; /// /// The build command implementation /// public static partial class Command { /// /// The primary project to build for GUI builds /// private const string PrimaryGUIProject = "Duplicati.GUI.TrayIcon.csproj"; /// /// The secondary project to build for CLI builds /// private const string PrimaryCLIProject = "Duplicati.CommandLine.csproj"; /// /// Projects that only makes sense for Windows /// private static readonly IReadOnlySet WindowsOnlyProjects = new HashSet(StringComparer.InvariantCultureIgnoreCase) { "Duplicati.WindowsService.csproj" }; /// /// Projects the pull in GUI dependencies /// private static readonly IReadOnlySet GUIProjects = new HashSet(StringComparer.InvariantCultureIgnoreCase) { "Duplicati.GUI.TrayIcon.csproj" }; /// /// Some executables have shorter names that follow the Linux convention of all-lowercase /// private static readonly IDictionary ExecutableRenames = new Dictionary(StringComparer.InvariantCultureIgnoreCase) { { "Duplicati.CommandLine", "duplicati-cli" }, { "Duplicati.Server", "duplicati-server"}, { "Duplicati.CommandLine.BackendTester", "duplicati-backend-tester"}, { "Duplicati.CommandLine.BackendTool", "duplicati-backend-tool" }, { "Duplicati.CommandLine.RecoveryTool", "duplicati-recovery-tool" }, { "Duplicati.GUI.TrayIcon", "duplicati" } }; /// /// The packages that are required for GUI builds /// private static readonly IReadOnlyList DebianGUIDepends = ["libice6", "libsm6", "libfontconfig1"]; /// /// The packages that are required for CLI builds /// private static readonly IReadOnlyList DebianCLIDepends = []; /// /// The packages that are required for GUI builds /// private static readonly IReadOnlyList FedoraGUIDepends = ["libice6", "libsm6", "libfontconfig1"]; /// /// The packages that are required for CLI builds /// private static readonly IReadOnlyList FedoraCLIDepends = []; /// /// Structure for keeping all variables for a single release /// /// The version to use /// The release channel /// The release timestamp private record ReleaseInfo(Version Version, ReleaseChannel Channel, DateTime Timestamp) { /// /// Gets the string name for the release /// public string ReleaseName => $"{Version}_{Channel.ToString().ToLowerInvariant()}_{Timestamp:yyy-MM-dd}"; /// /// Create a new release info /// /// The release type /// The incremental version /// The release info public static ReleaseInfo Create(ReleaseChannel type, int incVersion) => new ReleaseInfo(new Version(2, 0, 0, incVersion), type, DateTime.Today); /// /// Create a new release info /// /// The release type /// The version /// The release info public static ReleaseInfo Create(ReleaseChannel type, Version version) => new ReleaseInfo(version, type, DateTime.Today); } /// /// Creates the build command /// /// The command public static System.CommandLine.Command Create() { var releaseChannelArgument = new Argument( name: "channel", description: "The release channel", getDefaultValue: () => ReleaseChannel.Canary ); var buildTargetOption = new Option( name: "--targets", description: "The possible build targets, multiple arguments supported. Use the format os-arch.package, example: x64-win.msi.", parseArgument: arg => { if (!arg.Tokens.Any()) return Program.SupportedPackageTargets.ToArray(); var requested = arg.Tokens.Select(x => PackageTarget.ParsePackageId(x.Value)).Distinct().ToArray(); var invalid = requested.Where(x => !Program.SupportedPackageTargets.Contains(x)).ToList(); if (invalid.Any()) throw new Exception($"Following targets are not supported: {string.Join(", ", invalid.Select(x => x.PackageTargetString))}"); return requested; }); var gitStashPushOption = new Option( name: "--git-stash-push", description: "Performs a git stash command before running the build, and a git commit after updating files", getDefaultValue: () => true ); var keepBuildsOption = new Option( name: "--keep-builds", description: "Do not delete the build folders if they already exist (re-use build)", getDefaultValue: () => false ); var buildTempOption = new Option( name: "--build-path", description: "The path to the temporary folder used for builds; will be deleted on startup", getDefaultValue: () => new DirectoryInfo(Path.GetFullPath("build-temp")) ); var solutionFileOption = new Option( name: "--solution-file", description: "Path to the Duplicati.sln file", getDefaultValue: () => new FileInfo(Path.GetFullPath(Path.Combine("..", "Duplicati.sln"))) ); var disableAuthenticodeOption = new Option( name: "--disable-authenticode", description: "Disables authenticode signing", getDefaultValue: () => false ); var disableCodeSignOption = new Option( name: "--disable-signcode", description: "Disables Apple signcode signing", getDefaultValue: () => false ); var passwordOption = SharedOptions.passwordOption; var disableDockerPushOption = new Option( name: "--disable-docker-push", description: "Disables pushing the docker image to the repository", getDefaultValue: () => false ); var macOsAppNameOption = new Option( name: "--macos-app-name", description: "The name of the MacOS app bundle", getDefaultValue: () => "Duplicati.app" ); var dockerRepoOption = new Option( name: "--docker-repo", description: "The docker repository to push to", getDefaultValue: () => "duplicati/duplicati" ); var changelogFileOption = new Option( name: "--changelog-file", description: "The path to the changelog news file. Contents from this file are prepended to the changelog.", getDefaultValue: () => new FileInfo(Path.GetFullPath("changelog-news.txt")) ); var disableNotarizeSigningOption = new Option( name: "--disable-notarize-signing", description: "Disables notarize signing for MacOS packages", getDefaultValue: () => false ); var disableGpgSigningOption = new Option( name: "--disable-gpg-signing", description: "Disables GPG signing of packages", getDefaultValue: () => false ); var versionOverrideOption = new Option( name: "--version", description: "Sets a custom version to use", getDefaultValue: () => null ); var command = new System.CommandLine.Command("build", "Builds the packages for a release") { gitStashPushOption, releaseChannelArgument, buildTempOption, buildTargetOption, solutionFileOption, keepBuildsOption, disableAuthenticodeOption, disableCodeSignOption, passwordOption, macOsAppNameOption, disableDockerPushOption, dockerRepoOption, changelogFileOption, disableNotarizeSigningOption, disableGpgSigningOption, versionOverrideOption }; command.Handler = CommandHandler.Create(DoBuild); return command; } /// /// The input for the build command /// /// The build targets /// The build path /// The solution path /// If the git stash should be performed /// The release channel /// The version override to use /// If the builds should be kept /// If authenticode signing should be disabled /// If signcode should be disabled /// The password to use for the keyfile /// If the docker push should be disabled /// The name of the MacOS app bundle /// The docker repository to push to /// The path to the changelog file /// If notarize signing should be disabled /// If GPG signing should be disabled record CommandInput( PackageTarget[] Targets, DirectoryInfo BuildPath, FileInfo SolutionFile, bool GitStashPush, ReleaseChannel Channel, string? Version, bool KeepBuilds, bool DisableAuthenticode, bool DisableSignCode, string Password, bool DisableDockerPush, string MacOSAppName, string DockerRepo, FileInfo ChangelogFile, bool DisableNotarizeSigning, bool DisableGpgSigning ); static async Task DoBuild(CommandInput input) { Console.WriteLine($"Building {input.Channel} release ..."); var buildTargets = input.Targets; if (!buildTargets.Any()) buildTargets = Program.SupportedPackageTargets.ToArray(); if (!input.SolutionFile.Exists) throw new FileNotFoundException($"Solution file not found: {input.SolutionFile.FullName}"); // This could be fixed, so we will throw an exception if the build is not possible if (buildTargets.Any(x => x.Package == PackageType.MSI) && !Program.Configuration.IsMSIBuildPossible()) throw new Exception("WiX toolset not configured, cannot build MSI files"); // This will be fixed in the future, but requires a new http-interface for Synology DSM if (buildTargets.Any(x => x.Package == PackageType.SynologySpk) && !Program.Configuration.IsSynologyPkgPossible()) throw new Exception("Synology SPK files are currently not supported"); // This will not work, so to make it easier for non-MacOS developers, we will remove the MacOS packages if (buildTargets.Any(x => x.Package == PackageType.MacPkg || x.Package == PackageType.DMG) && !Program.Configuration.IsMacPkgBuildPossible()) { Console.WriteLine("MacOS packages requested but not running on MacOS, removing from build targets"); buildTargets = buildTargets.Where(x => x.Package != PackageType.MacPkg && x.Package != PackageType.DMG).ToArray(); } var baseDir = Path.GetDirectoryName(input.SolutionFile.FullName) ?? throw new Exception("Path to solution file was invalid"); var versionFilePath = Path.Combine(baseDir, "Updates", "build_version.txt"); if (!File.Exists(versionFilePath)) throw new FileNotFoundException($"Version file not found: {versionFilePath}"); var sourceProjects = Directory.EnumerateDirectories(Path.Combine(baseDir, "Executables", "net8"), "*", SearchOption.TopDirectoryOnly) .SelectMany(x => Directory.EnumerateFiles(x, "*.csproj", SearchOption.TopDirectoryOnly)) .ToList(); var primaryGUI = sourceProjects.FirstOrDefault(x => string.Equals(Path.GetFileName(x), PrimaryGUIProject, StringComparison.OrdinalIgnoreCase)) ?? throw new Exception("Failed to find tray icon executable"); var primaryCLI = sourceProjects.FirstOrDefault(x => string.Equals(Path.GetFileName(x), PrimaryCLIProject, StringComparison.OrdinalIgnoreCase)) ?? throw new Exception("Failed to find cli executable"); var windowsOnly = sourceProjects.Where(x => WindowsOnlyProjects.Contains(Path.GetFileName(x))).ToHashSet(StringComparer.OrdinalIgnoreCase); // Put primary at the end sourceProjects.Remove(primaryGUI); sourceProjects.Remove(primaryCLI); sourceProjects.Add(primaryCLI); sourceProjects.Add(primaryGUI); if (!File.Exists(primaryGUI)) throw new Exception($"Failed to locate project file: {primaryGUI}"); if (!File.Exists(primaryCLI)) throw new Exception($"Failed to locate project file: {primaryCLI}"); if (!input.ChangelogFile.Exists) { Console.WriteLine($"Changelog news file not found: {input.ChangelogFile.FullName}"); Console.WriteLine($"Create an empty file if you want a release without changes"); if (OperatingSystem.IsWindows()) Console.WriteLine($"> type nul > {input.ChangelogFile.FullName}"); else Console.WriteLine($"> touch {input.ChangelogFile.FullName}"); Program.ReturnCode = 1; return; } var changelogNews = File.ReadAllText(input.ChangelogFile.FullName); var releaseInfo = string.IsNullOrWhiteSpace(input.Version) ? ReleaseInfo.Create(input.Channel, int.Parse(File.ReadAllText(versionFilePath)) + 1) : ReleaseInfo.Create(input.Channel, Version.Parse(input.Version)); Console.WriteLine($"Building {releaseInfo.ReleaseName} ..."); var keyfilePassword = string.IsNullOrEmpty(input.Password) ? ConsoleHelper.ReadPassword("Enter keyfile password") : input.Password; var primarySignKey = LoadKeyFile(Program.Configuration.ConfigFiles.UpdaterKeyfile.FirstOrDefault(), keyfilePassword, false); var additionalKeys = Program.Configuration.ConfigFiles.UpdaterKeyfile .Skip(1) .Select(x => LoadKeyFile(x, keyfilePassword, true)); // Configure runtime environment var rtcfg = new RuntimeConfig( releaseInfo, additionalKeys.Prepend(primarySignKey).ToList(), keyfilePassword, changelogNews, input); rtcfg.ToggleAuthenticodeSigning(); rtcfg.ToggleSignCodeSigning(); rtcfg.ToggleNotarizeSigning(); rtcfg.ToggleGpgSigning(); await rtcfg.ToggleDockerBuild(); if (!rtcfg.UseDockerBuild) { var unsupportedBuilds = buildTargets.Where(x => x.Package == PackageType.Docker || x.Package == PackageType.Deb || x.Package == PackageType.RPM).ToList(); if (unsupportedBuilds.Any()) throw new Exception($"The following packages cannot be built without Docker: {string.Join(", ", unsupportedBuilds.Select(x => x.PackageTargetString))}"); } if (!input.KeepBuilds && Directory.Exists(input.BuildPath.FullName)) { Console.WriteLine($"Deleting build folder: {input.BuildPath.FullName}"); Directory.Delete(input.BuildPath.FullName, true); } if (!Directory.Exists(input.BuildPath.FullName)) Directory.CreateDirectory(input.BuildPath.FullName); // Generally, the builds should happen with a clean source tree, // but this can be disabled for debugging if (input.GitStashPush) await ProcessHelper.Execute(["git", "stash", "save", $"auto-build-{releaseInfo.Timestamp:yyyy-MM-dd}"], workingDirectory: baseDir); // Inject various files that will be embedded into the build artifacts await PrepareSourceDirectory(baseDir, releaseInfo, rtcfg); // Inject a version tag into the html files var revertableFiles = InjectVersionIntoFiles(baseDir, releaseInfo); // Perform the main compilations await Compile.BuildProjects(baseDir, input.BuildPath.FullName, sourceProjects, windowsOnly, GUIProjects, buildTargets, releaseInfo, input.KeepBuilds, rtcfg); // Create the packages var builtPackages = await CreatePackage.BuildPackages(baseDir, input.BuildPath.FullName, buildTargets, input.KeepBuilds, rtcfg); if (rtcfg.UseNotarizeSigning && builtPackages.Any(x => x.Target.Package == PackageType.DMG || x.Target.Package == PackageType.MacPkg)) { // # Notarize and staple takes a while... Console.WriteLine("Performing notarize and staple ..."); foreach (var p in builtPackages.Where(x => x.Target.Package == PackageType.DMG || x.Target.Package == PackageType.MacPkg)) { await ProcessHelper.Execute(["xcrun", "notarytool", "submit", p.CreatedFile, "--keychain-profile", Program.Configuration.ConfigFiles.NotarizeProfile, "--wait"]); await ProcessHelper.Execute(["xcrun", "stapler", "staple", p.CreatedFile]); } } // Build the signed manifest to be uploaded to remote storage Console.WriteLine("Build completed, creating signed manifest ..."); var manifestfile = Path.Combine(input.BuildPath.FullName, "packages", "autoupdate.manifest"); if (File.Exists(manifestfile)) File.Delete(manifestfile); UpdaterManager.CreateSignedManifest( rtcfg.SignKeys.First(), null, Path.Combine(input.BuildPath.FullName, "packages"), version: releaseInfo.Version.ToString(), updateFromV1Url: Program.Configuration.ExtraSettings.UpdateFromV1Url, genericUpdatePageUrl: Program.Configuration.ExtraSettings.GenericUpdatePageUrl, releaseType: releaseInfo.Channel.ToString().ToLowerInvariant(), packages: builtPackages.Select(x => new PackageEntry() { RemoteUrls = Program.Configuration.ExtraSettings.PackageUrls .Select(u => u.Replace("${RELEASE_TYPE}", releaseInfo.Channel.ToString().ToLowerInvariant()) .Replace("${RELEASE_VERSION}", releaseInfo.Version.ToString()) .Replace("${RELEASE_TIMESTAMP}", releaseInfo.Timestamp.ToString("yyyy-MM-dd")) .Replace("${FILENAME}", x.CreatedFile) ).ToArray(), PackageTypeId = x.Target.PackageTargetString, Length = new FileInfo(Path.Combine(input.BuildPath.FullName, x.CreatedFile)).Length, MD5 = CalculateHash(Path.Combine(input.BuildPath.FullName, x.CreatedFile), "md5"), SHA256 = CalculateHash(Path.Combine(input.BuildPath.FullName, x.CreatedFile), "sha256") }) ); File.Move( Path.Combine(input.BuildPath.FullName, "packages", "autoupdate.manifest"), Path.Combine(input.BuildPath.FullName, "packages", "latest-v2.manifest"), true ); var files = builtPackages.Select(x => x.CreatedFile).Append("latest-v2.manifest").ToList(); // Create the GPG signatures for the files if (rtcfg.UseGPGSigning) { Console.WriteLine("Creating GPG signatures ..."); await GpgSign.SignReleaseFiles(files, rtcfg); } Console.WriteLine("Build completed, uploading packages ..."); Console.WriteLine("Upload completed, releasing packages ..."); Console.WriteLine("Release completed, posting release notes ..."); // Clean up the source tree await ProcessHelper.Execute(new[] { "git", "checkout", "Duplicati/License/VersionTag.txt", "Duplicati/Library/AutoUpdater/AutoUpdateURL.txt", "Duplicati/Library/AutoUpdater/AutoUpdateBuildChannel.txt", "Duplicati/Library/AutoUpdater/AutoUpdateSignKeys.txt", }.Concat(revertableFiles.Select(x => Path.GetRelativePath(baseDir, x))) , workingDirectory: baseDir); if (input.GitStashPush) await GitPush.TagAndPush(baseDir, releaseInfo); Console.WriteLine("All done"); } /// /// Injects the version number into some html files /// /// The base directory /// The release info /// The paths that were modified private static string[] InjectVersionIntoFiles(string baseDir, ReleaseInfo releaseInfo) { var targetfiles = Directory.EnumerateFiles(Path.Combine(baseDir, "Duplicati", "Server", "webroot"), "*", SearchOption.AllDirectories) .Where(x => x.EndsWith(".html") || x.EndsWith(".js")) .ToArray(); var versionre = @"(?\d+\.\d+\.(\*|(\d+(\.(\*|\d+)))?))"; var regex = new Regex(@"\?v\=" + versionre); foreach (var file in targetfiles) File.WriteAllText( file, regex.Replace(File.ReadAllText(file), $"?v={releaseInfo.Version}") ); //FILEMAP.Add("AssemblyRedirects.xml", new Regex(@"newVersion\=\""" + versionre + @"\""")); return targetfiles; } /// /// Updates the source directory prior to building. /// This writes a stub package manifest inside the source folder, /// which will be embedded in the excutable to indicate which version it was built from. /// /// The source folder base /// The release info to use /// The runtime configuration /// An awaitable task static Task PrepareSourceDirectory(string baseDir, ReleaseInfo releaseInfo, RuntimeConfig rtcfg) { var urlstring = string.Join(";", Program.Configuration.ExtraSettings.UpdaterUrls.Select(x => x.Replace("${RELEASE_TYPE}", releaseInfo.Channel.ToString().ToLowerInvariant()) .Replace("${RELEASE_VERSION}", releaseInfo.Version.ToString()) .Replace("${RELEASE_TIMESTAMP}", releaseInfo.Timestamp.ToString("yyyy-MM-dd")) .Replace("${FILENAME}", "latest-v2.manifest") )); File.WriteAllText(Path.Combine(baseDir, "Duplicati", "License", "VersionTag.txt"), releaseInfo.Version.ToString()); File.WriteAllText(Path.Combine(baseDir, "Duplicati", "Library", "AutoUpdater", "AutoUpdateBuildChannel.txt"), releaseInfo.Channel.ToString().ToLowerInvariant()); File.WriteAllText(Path.Combine(baseDir, "Duplicati", "Library", "AutoUpdater", "AutoUpdateURL.txt"), urlstring); File.WriteAllLines(Path.Combine(baseDir, "Duplicati", "Library", "AutoUpdater", "AutoUpdateSignKeys.txt"), rtcfg.SignKeys.Select(x => x.ToXmlString(false))); if (!string.IsNullOrWhiteSpace(rtcfg.ChangelogNews)) File.WriteAllText( Path.Combine(baseDir, "changelog.txt"), rtcfg.ChangelogNews + Environment.NewLine + File.ReadAllText(Path.Combine(baseDir, "changelog.txt")) ); // Previous versions used to install the assembly redirects after the build // but it looks like .Net now handles this automatically // If not, we need to adjust the .csproj files before building // find "${UPDATE_SOURCE}" - type f - name Duplicati.*.exe - maxdepth 1 - exec cp Installer/ AssemblyRedirects.xml { }.config \; var manifestFile = Path.Combine(baseDir, "Duplicati", "Library", "AutoUpdater", "autoupdate.manifest"); if (File.Exists(manifestFile)) File.Delete(manifestFile); UpdaterManager.CreateSignedManifest( rtcfg.SignKeys.First(), null, Path.Combine(baseDir, "Duplicati", "Library", "AutoUpdater"), version: releaseInfo.Version.ToString(), updateFromV1Url: Program.Configuration.ExtraSettings.UpdateFromV1Url, genericUpdatePageUrl: Program.Configuration.ExtraSettings.GenericUpdatePageUrl, releaseType: releaseInfo.Channel.ToString().ToLowerInvariant() ); return Task.CompletedTask; } /// /// Loads a keyfile and decrypts the RSA key inside /// /// The keyfile to decrypt /// The keyfile password /// Allow asking for a new password if the password did not match /// The matching key static RSA LoadKeyFile(string? keyfile, string password, bool askForNewPassword) { if (string.IsNullOrWhiteSpace(keyfile)) throw new Exception("Unable to load keyfile, no keyfile specified"); if (!File.Exists(keyfile)) throw new FileNotFoundException($"Keyfile not found: {keyfile}"); try { using var ms = new MemoryStream(); using var fs = File.OpenRead(keyfile); SharpAESCrypt.SharpAESCrypt.Decrypt(password, fs, ms); var rsa = RSA.Create(); rsa.FromXmlString(System.Text.Encoding.UTF8.GetString(ms.ToArray())); return rsa; } catch (SharpAESCrypt.SharpAESCrypt.WrongPasswordException) { if (!askForNewPassword) throw; } password = ConsoleHelper.ReadPassword($"Enter password for {keyfile}"); return LoadKeyFile(keyfile, password, false); } /// /// Calculates the hash of a file and returns the hash as a base64 string /// /// The file to calculate the has for /// The hash algorithm /// The base64 encoded hash static string CalculateHash(string file, string algorithm) { using var fs = File.OpenRead(file); using var hash = algorithm.ToLowerInvariant() switch { "md5" => (HashAlgorithm)MD5.Create(), "sha256" => SHA256.Create(), _ => throw new Exception($"Unknown hash algorithm: {algorithm}") }; return Convert.ToBase64String(hash.ComputeHash(fs)); } }