diff --git a/.github/workflows/build-packages.yml b/.github/workflows/build-packages.yml
new file mode 100644
index 000000000..205bb3ab4
--- /dev/null
+++ b/.github/workflows/build-packages.yml
@@ -0,0 +1,42 @@
+name: Tests
+
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  build_packages:
+    name: Build packages
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, windows-latest]
+        # Super slow on ubuntu-latest
+        # os: [ubuntu-latest, windows-latest, macos-latest]
+
+    steps:
+      - name: Set up .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 8.x
+
+      - name: Checkout source
+        uses: actions/checkout@v4
+
+      - name: Create dummy changelog file
+        working-directory: ./ReleaseBuilder
+        run: echo "## Changed" > ./changelog-news.txt
+
+      - name: Compile releases
+        working-directory: ./ReleaseBuilder
+        env:
+          UPDATER_KEYFILE: ./testfile.key1
+          WIX:
+        run: dotnet run -- build debug --compile-only=true --git-stash-push=false --disable-docker-push=true --keep-builds=true --disable-authenticode=true --disable-signcode=true --disable-notarize-signing=true --disable-gpg-signing=true --disable-s3-upload=true --disable-github-upload=true --disable-update-server-reload=true --disable-discourse-announce=true --disable-clean-source=true --password=test1234 --version=2.0.0.99
+
+  # Disabled, as a new test needs to be written for the new UI
+  # selenium:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Selenium
+  #       run: pipeline/selenium/test.sh
diff --git a/Duplicati/Agent/Program.cs b/Duplicati/Agent/Program.cs
index a2dc73769..53df9f5c4 100644
--- a/Duplicati/Agent/Program.cs
+++ b/Duplicati/Agent/Program.cs
@@ -23,7 +23,6 @@ using System.CommandLine;
 using System.CommandLine.Builder;
 using System.CommandLine.NamingConventionBinder;
 using System.CommandLine.Parsing;
-using System.Reactive.Linq;
 using System.Security.Cryptography;
 using Duplicati.Library.AutoUpdater;
 using Duplicati.Library.Encryption;
diff --git a/ReleaseBuilder/Build/Command.Compile.Post.cs b/ReleaseBuilder/Build/Command.Compile.Post.cs
index 07f430ea0..0383b4b75 100644
--- a/ReleaseBuilder/Build/Command.Compile.Post.cs
+++ b/ReleaseBuilder/Build/Command.Compile.Post.cs
@@ -242,7 +242,8 @@ public static partial class Command
 
             // Rename the executables, as symlinks are not supported in DMG files
             await PackageSupport.RenameExecutables(binDir);
-            await PackageSupport.SetPermissionFlags(binDir, rtcfg);
+            if (!OperatingSystem.IsWindows())
+                await PackageSupport.SetPermissionFlags(binDir, rtcfg);
 
             // Move the licenses out of the code folder as the signing tool trips on it
             var licenseTarget = Path.Combine(tmpApp, "Contents", "Licenses");
diff --git a/ReleaseBuilder/Build/Command.RuntimeConfig.cs b/ReleaseBuilder/Build/Command.RuntimeConfig.cs
index b78a40504..97f556936 100644
--- a/ReleaseBuilder/Build/Command.RuntimeConfig.cs
+++ b/ReleaseBuilder/Build/Command.RuntimeConfig.cs
@@ -211,6 +211,14 @@ public static partial class Command
             }
         }
 
+        /// <summary>
+        /// Disables docker builds
+        /// </summary>
+        public void DisableDockerBuilds()
+        {
+            _dockerBuild = false;
+        }
+
         /// <summary>
         /// Cache value for checking if notarize is enabled
         /// </summary>
diff --git a/ReleaseBuilder/Build/Command.cs b/ReleaseBuilder/Build/Command.cs
index 6ec431a55..d9ee5c90e 100644
--- a/ReleaseBuilder/Build/Command.cs
+++ b/ReleaseBuilder/Build/Command.cs
@@ -41,7 +41,7 @@ public static partial class Command
     /// <summary>
     /// The primary project to build for CLI builds
     /// </summary>
-    private const string PrimaryCLIProject = "Duplicati.CommandLine.csproj";
+    private const string PrimaryCLIProject = "Duplicati.Server.csproj";
 
     /// <summary>
     /// The primary project to build for Agent builds
@@ -191,8 +191,8 @@ public static partial class Command
             getDefaultValue: () => false
         );
 
-        var buildOnlyOption = new Option<bool>(
-            name: "--build-only",
+        var compileOnlyOption = new Option<bool>(
+            name: "--compile-only",
             description: "Only build the binaries, do not create packages",
             getDefaultValue: () => false
         );
@@ -332,7 +332,7 @@ public static partial class Command
             buildTargetOption,
             solutionFileOption,
             keepBuildsOption,
-            buildOnlyOption,
+            compileOnlyOption,
             disableAuthenticodeOption,
             disableCodeSignOption,
             passwordOption,
@@ -369,7 +369,7 @@ public static partial class Command
     /// <param name="Channel">The release channel</param>
     /// <param name="Version">The version override to use</param>
     /// <param name="KeepBuilds">If the builds should be kept</param>
-    /// <param name="BuildOnly">If only the build should be performed</param>
+    /// <param name="CompileOnly">If only the build should be performed</param>
     /// <param name="DisableAuthenticode">If authenticode signing should be disabled</param>
     /// <param name="DisableSignCode">If signcode should be disabled</param>
     /// <param name="Password">The password to use for the keyfile</param>
@@ -397,7 +397,7 @@ public static partial class Command
         ReleaseChannel Channel,
         string? Version,
         bool KeepBuilds,
-        bool BuildOnly,
+        bool CompileOnly,
         bool DisableAuthenticode,
         bool DisableSignCode,
         string Password,
@@ -433,7 +433,7 @@ public static partial class Command
             throw new FileNotFoundException($"Solution file not found: {input.SolutionFile.FullName}");
 
         // This could be fixed, so we will throw an exception if the build is not possible
-        if (buildTargets.Any(x => x.Package == PackageType.MSI) && !configuration.IsMSIBuildPossible())
+        if (buildTargets.Any(x => x.Package == PackageType.MSI) && !configuration.IsMSIBuildPossible() && !input.CompileOnly)
             throw new Exception("WiX toolset not configured, cannot build MSI files");
 
         // This will be fixed in the future, but requires a new http-interface for Synology DSM
@@ -441,7 +441,7 @@ public static partial class Command
             throw new Exception("Synology SPK files are currently not supported");
 
         // This will not work, so to make it easier for non-MacOS developers, we will remove the MacOS packages
-        if (buildTargets.Any(x => x.Package == PackageType.MacPkg || x.Package == PackageType.DMG) && !configuration.IsMacPkgBuildPossible())
+        if (buildTargets.Any(x => x.Package == PackageType.MacPkg || x.Package == PackageType.DMG) && !configuration.IsMacPkgBuildPossible() && !input.CompileOnly)
         {
             Console.WriteLine("MacOS packages requested but not running on MacOS, removing from build targets");
             buildTargets = buildTargets.Where(x => x.Package != PackageType.MacPkg && x.Package != PackageType.DMG).ToArray();
@@ -532,13 +532,20 @@ public static partial class Command
         rtcfg.ToggleGithubUpload(rtcfg.ReleaseInfo.Channel);
         rtcfg.ToogleDiscourseAnnounce(rtcfg.ReleaseInfo.Channel);
         rtcfg.ToggleUpdateServerReload();
-        await rtcfg.ToggleDockerBuild();
-
-        if (!rtcfg.UseDockerBuild)
+        if (input.CompileOnly)
         {
-            var unsupportedBuilds = buildTargets.Where(x => x.Package == PackageType.Docker || x.Package == PackageType.Deb || x.Package == PackageType.RPM).ToList();
-            if (unsupportedBuilds.Any())
-                throw new Exception($"The following packages cannot be built without Docker: {string.Join(", ", unsupportedBuilds.Select(x => x.PackageTargetString))}");
+            rtcfg.DisableDockerBuilds();
+        }
+        else
+        {
+            await rtcfg.ToggleDockerBuild();
+
+            if (!rtcfg.UseDockerBuild)
+            {
+                var unsupportedBuilds = buildTargets.Where(x => x.Package == PackageType.Docker || x.Package == PackageType.Deb || x.Package == PackageType.RPM).ToList();
+                if (unsupportedBuilds.Any())
+                    throw new Exception($"The following packages cannot be built without Docker: {string.Join(", ", unsupportedBuilds.Select(x => x.PackageTargetString))}");
+            }
         }
 
         // Prevent the system from going to sleep during the build
@@ -588,7 +595,7 @@ public static partial class Command
 
         await Compile.BuildProjects(baseDir, input.BuildPath.FullName, sourceBuildMap, windowsOnly, buildTargets, releaseInfo, input.KeepBuilds, rtcfg, input.UseHostedBuilds, input.DisableCleanSource, input.AllowAssemblyMismatch);
 
-        if (input.BuildOnly)
+        if (input.CompileOnly)
         {
             Console.WriteLine("Build completed, skipping package creation ...");
             return;
diff --git a/ReleaseBuilder/KeepAliveAssertion.cs b/ReleaseBuilder/KeepAliveAssertion.cs
index 493db70b2..b2fd44d62 100644
--- a/ReleaseBuilder/KeepAliveAssertion.cs
+++ b/ReleaseBuilder/KeepAliveAssertion.cs
@@ -32,7 +32,7 @@ public class KeepAliveAssertion : IDisposable
     /// <summary>
     /// The previous state of the system, on Windows
     /// </summary>
-    private NativeMethods.EXECUTION_STATE _previousState;
+    private NativeMethods.EXECUTION_STATE? _previousState;
     /// <summary>
     /// The process that keeps the system awake, on Linux and MacOS
     /// </summary>
@@ -45,17 +45,23 @@ public class KeepAliveAssertion : IDisposable
     {
         if (OperatingSystem.IsWindows())
         {
-            _previousState = NativeMethods.SetThreadExecutionState(NativeMethods.EXECUTION_STATE.ES_CONTINUOUS | NativeMethods.EXECUTION_STATE.ES_SYSTEM_REQUIRED | NativeMethods.EXECUTION_STATE.ES_AWAYMODE_REQUIRED);
+            try { _previousState = NativeMethods.SetThreadExecutionState(NativeMethods.EXECUTION_STATE.ES_CONTINUOUS | NativeMethods.EXECUTION_STATE.ES_SYSTEM_REQUIRED | NativeMethods.EXECUTION_STATE.ES_AWAYMODE_REQUIRED); }
+            catch { }
         }
         else if (OperatingSystem.IsLinux())
         {
-            _keepAliveProcess = Process.Start("xset", "s off -dpms");
+            try { _keepAliveProcess = Process.Start("xset", "s off -dpms"); }
+            catch { }
         }
         else if (OperatingSystem.IsMacOS())
         {
-            // Keep the display awake until this process exits
-            var selfProcessId = Process.GetCurrentProcess().Id;
-            _keepAliveProcess = Process.Start("caffeinate", ["-d", "-w", selfProcessId.ToString()]);
+            try
+            {
+                // Keep the display awake until this process exits
+                var selfProcessId = Process.GetCurrentProcess().Id;
+                _keepAliveProcess = Process.Start("caffeinate", ["-d", "-w", selfProcessId.ToString()]);
+            }
+            catch { }
         }
     }
 
@@ -82,7 +88,8 @@ public class KeepAliveAssertion : IDisposable
     {
         if (OperatingSystem.IsWindows())
         {
-            NativeMethods.SetThreadExecutionState(_previousState);
+            if (_previousState != null)
+                NativeMethods.SetThreadExecutionState(_previousState.Value);
         }
         else if (OperatingSystem.IsLinux())
         {
diff --git a/ReleaseBuilder/Program.cs b/ReleaseBuilder/Program.cs
index 290476c2f..3349c6d62 100644
--- a/ReleaseBuilder/Program.cs
+++ b/ReleaseBuilder/Program.cs
@@ -102,12 +102,20 @@ class Program
     /// <returns></returns>
     static async Task<int> Main(string[] args)
     {
-        var r = await new RootCommand("Build tool for Duplicati")
+        try
         {
-            Build.Command.Create(),
-            CreateKey.Command.Create(),
-        }.InvokeAsync(args);
+            ReturnCode = await new RootCommand("Build tool for Duplicati")
+            {
+                Build.Command.Create(),
+                CreateKey.Command.Create(),
+            }.InvokeAsync(args);
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine(ex.ToString());
+            ReturnCode = 1;
+        }
 
-        return ReturnCode ?? r;
+        return ReturnCode ?? 2;
     }
 }
\ No newline at end of file