Compare commits
32
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0c79202da9 | ||
|
|
d21efcf64b | ||
|
|
4fdc02681d | ||
|
|
2d4970ce55 | ||
|
|
3853ad2400 | ||
|
|
509229fd6a | ||
|
|
5c430ee6be | ||
|
|
748c2813eb | ||
|
|
1e2d884e93 | ||
|
|
5005f751a1 | ||
|
|
c7158ab969 | ||
|
|
15b9f99c2e | ||
|
|
d3731d2cf6 | ||
|
|
89cc0b54cc | ||
|
|
5c2618b14a | ||
|
|
8ae03535af | ||
|
|
3dee3a8d30 | ||
|
|
81c7b59562 | ||
|
|
72df9b4bea | ||
|
|
1e1e03e6cb | ||
|
|
d86c1820df | ||
|
|
930242c68b | ||
|
|
17cdd3a887 | ||
|
|
6b1991a26d | ||
|
|
0c488e960e | ||
|
|
79358ed60d | ||
|
|
7c28828cfe | ||
|
|
57bcb08bac | ||
|
|
c26eadedf2 | ||
|
|
376d09d6e3 | ||
|
|
a7322727b6 | ||
|
|
5e064f4b6b |
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(chmod:*)",
|
||||
"Bash(mkdir:*)",
|
||||
"Bash(./gradlew:*)",
|
||||
"Bash(grep:*)",
|
||||
"Bash(cat:*)"
|
||||
],
|
||||
"deny": []
|
||||
}
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
|
||||
// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
|
||||
{
|
||||
"name": "Stirling-PDF Dev Container",
|
||||
"build": {
|
||||
// Sets the run context to one level up instead of the .devcontainer folder.
|
||||
"context": "..",
|
||||
// Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename.
|
||||
"dockerfile": "../Dockerfile.dev"
|
||||
},
|
||||
"runArgs": [
|
||||
"-e",
|
||||
"GIT_EDITOR=code --wait",
|
||||
"--security-opt",
|
||||
"label=disable"
|
||||
],
|
||||
// Use 'forwardPorts' to make a list of ports inside the container available locally.
|
||||
"forwardPorts": [8080, 2002, 2003],
|
||||
"portsAttributes": {
|
||||
"8080": {
|
||||
"label": "Stirling-PDF Dev Port"
|
||||
},
|
||||
"2002": {
|
||||
"label": "unoserver Port"
|
||||
},
|
||||
"2003": {
|
||||
"label": "UnoConvert Port"
|
||||
}
|
||||
},
|
||||
"workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=delegated",
|
||||
"mounts": [
|
||||
"source=logs-volume,target=/workspace/logs,type=volume",
|
||||
"source=build-volume,target=/workspace/build,type=volume"
|
||||
],
|
||||
"workspaceFolder": "/workspace",
|
||||
// Configure tool-specific properties.
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"settings": {
|
||||
"terminal.integrated.shell.linux": "/bin/bash",
|
||||
"editor.wordSegmenterLocales": "",
|
||||
"editor.guides.bracketPairs": "active",
|
||||
"editor.guides.bracketPairsHorizontal": "active",
|
||||
"cSpell.enabled": false,
|
||||
"[java]": {
|
||||
"editor.defaultFormatter": "josevseb.google-java-format-for-vs-code"
|
||||
},
|
||||
"java.compile.nullAnalysis.mode": "automatic",
|
||||
"java.configuration.updateBuildConfiguration": "interactive",
|
||||
"java.format.enabled": true,
|
||||
"java.format.settings.profile": "GoogleStyle",
|
||||
"java.format.settings.google.version": "1.26.0",
|
||||
"java.format.settings.google.extra": "--aosp --skip-sorting-imports --skip-javadoc-formatting",
|
||||
"java.saveActions.cleanup": true,
|
||||
"java.cleanup.actions": [
|
||||
"invertEquals",
|
||||
"instanceofPatternMatch"
|
||||
],
|
||||
"java.completion.engine": "dom",
|
||||
"java.completion.enabled": true,
|
||||
"java.completion.importOrder": [
|
||||
"java",
|
||||
"javax",
|
||||
"org",
|
||||
"com",
|
||||
"net",
|
||||
"io",
|
||||
"jakarta",
|
||||
"lombok",
|
||||
"me",
|
||||
"stirling"
|
||||
],
|
||||
"java.project.resourceFilters": [
|
||||
".devcontainer/",
|
||||
".git/",
|
||||
".github/",
|
||||
".gradle/",
|
||||
".venv/",
|
||||
".venv*/",
|
||||
".vscode/",
|
||||
"bin/",
|
||||
"build/",
|
||||
"configs/",
|
||||
"customFiles/",
|
||||
"docs/",
|
||||
"exampleYmlFiles",
|
||||
"gradle/",
|
||||
"images/",
|
||||
"logs/",
|
||||
"pipeline/",
|
||||
"scripts/",
|
||||
"testings/",
|
||||
".git-blame-ignore-revs",
|
||||
".gitattributes",
|
||||
".gitignore",
|
||||
".pre-commit-config.yaml"
|
||||
],
|
||||
"java.signatureHelp.enabled": true,
|
||||
"java.signatureHelp.description.enabled": true,
|
||||
"java.maven.downloadSources": true,
|
||||
"java.import.gradle.enabled": true,
|
||||
"java.eclipse.downloadSources": true,
|
||||
"java.import.gradle.wrapper.enabled": true,
|
||||
"spring.initializr.defaultLanguage": "Java",
|
||||
"spring.initializr.defaultGroupId": "stirling.software.SPDF",
|
||||
"spring.initializr.defaultArtifactId": "SPDF"
|
||||
},
|
||||
"extensions": [
|
||||
"elagil.pre-commit-helper", // Support for pre-commit hooks to enforce code quality
|
||||
"josevseb.google-java-format-for-vs-code", // Google Java code formatter to follow the Google Java Style Guide
|
||||
"ms-python.black-formatter", // Python code formatter using Black
|
||||
"ms-python.flake8", // Flake8 linter for Python to enforce code quality
|
||||
"ms-python.python", // Official Microsoft Python extension with IntelliSense, debugging, and Jupyter support
|
||||
"ms-vscode-remote.vscode-remote-extensionpack", // Remote Development Pack for SSH, WSL, and Containers
|
||||
// "Oracle.oracle-java", // Oracle Java extension with additional features for Java development
|
||||
"streetsidesoftware.code-spell-checker", // Spell checker for code to avoid typos
|
||||
"vmware.vscode-boot-dev-pack", // Developer tools for Spring Boot by VMware
|
||||
"vscjava.vscode-java-pack", // Java Extension Pack with essential Java tools for VS Code
|
||||
"EditorConfig.EditorConfig", // EditorConfig support for maintaining consistent coding styles
|
||||
"ms-azuretools.vscode-docker", // Docker extension for Visual Studio Code
|
||||
"charliermarsh.ruff", // Ruff extension for Ruff language support
|
||||
"github.vscode-github-actions", // GitHub Actions extension for Visual Studio Code
|
||||
"stylelint.vscode-stylelint", // Stylelint extension for CSS and SCSS linting
|
||||
"redhat.vscode-yaml" // YAML extension for Visual Studio Code
|
||||
]
|
||||
}
|
||||
},
|
||||
// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
|
||||
"remoteUser": "devuser",
|
||||
"shutdownAction": "stopContainer",
|
||||
"initializeCommand": "bash ./.devcontainer/git-init.sh",
|
||||
"postStartCommand": "./.devcontainer/init-setup.sh"
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
GIT_USER=$(git config --get user.name)
|
||||
GIT_EMAIL=$(git config --get user.email)
|
||||
|
||||
# Exit if GIT_USER or GIT_EMAIL is empty
|
||||
if [ -z "$GIT_USER" ] || [ -z "$GIT_EMAIL" ]; then
|
||||
echo "GIT_USER or GIT_EMAIL is not set. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git config --local user.name "$GIT_USER"
|
||||
git config --local user.email "$GIT_EMAIL"
|
||||
|
||||
# This directory should contain custom Git hooks for the repository
|
||||
# Set the path for Git hooks to /workspace/hooks
|
||||
git config --local core.hooksPath '%(prefix)/workspace/hooks'
|
||||
# Set the safe directory to the workspace path
|
||||
git config --local --add safe.directory /workspace
|
||||
@@ -1,75 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
# =============================================================================
|
||||
# Dev Container Initialization Script (init-setup.sh)
|
||||
#
|
||||
# This script runs when the Dev Container starts and provides guidance on
|
||||
# how to interact with the project. It prints an ASCII logo, displays the
|
||||
# current user, changes to the project root, and then shows helpful command
|
||||
# instructions.
|
||||
#
|
||||
# Instructions for future developers:
|
||||
#
|
||||
# - To start the application, use:
|
||||
# ./gradlew bootRun --no-daemon -Dspring-boot.run.fork=true -Dserver.address=0.0.0.0
|
||||
#
|
||||
# - To run tests, use:
|
||||
# ./gradlew test
|
||||
#
|
||||
# - To build the project, use:
|
||||
# ./gradlew build
|
||||
#
|
||||
# - For running pre-commit hooks (if configured), use:
|
||||
# pre-commit run --all-files
|
||||
#
|
||||
# Make sure you are in the project root directory after this script executes.
|
||||
# =============================================================================
|
||||
|
||||
echo "Devcontainer started successfully!"
|
||||
|
||||
VERSION=$(grep "^version =" build.gradle | awk -F'"' '{print $2}')
|
||||
GRADLE_VERSION=$(gradle -version | grep "^Gradle " | awk '{print $2}')
|
||||
GRADLE_PATH=$(which gradle)
|
||||
JAVA_VERSION=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}')
|
||||
JAVA_PATH=$(which java)
|
||||
|
||||
echo """
|
||||
____ _____ ___ ____ _ ___ _ _ ____ ____ ____ _____
|
||||
/ ___|_ _|_ _| _ \| | |_ _| \ | |/ ___| | _ \| _ \| ___|
|
||||
\___ \ | | | || |_) | | | || \| | | _ _____| |_) | | | | |_
|
||||
___) || | | || _ <| |___ | || |\ | |_| |_____| __/| |_| | _|
|
||||
|____/ |_| |___|_| \_\_____|___|_| \_|\____| |_| |____/|_|
|
||||
"""
|
||||
echo -e "Stirling-PDF Version: \e[32m$VERSION\e[0m"
|
||||
echo -e "Gradle Version: \e[32m$GRADLE_VERSION\e[0m"
|
||||
echo -e "Gradle Path: \e[32m$GRADLE_PATH\e[0m"
|
||||
echo -e "Java Version: \e[32m$JAVA_VERSION\e[0m"
|
||||
echo -e "Java Path: \e[32m$JAVA_PATH\e[0m"
|
||||
|
||||
# Display current active user (for permission/debugging purposes)
|
||||
echo -e "Current user: \e[32m$(whoami)\e[0m"
|
||||
|
||||
# Change directory to the project root (parent directory of the script)
|
||||
cd "$(dirname "$0")/.."
|
||||
echo -e "Changed to project root: \e[32m$(pwd)\e[0m"
|
||||
|
||||
# Display available commands for developers
|
||||
echo "=================================================================="
|
||||
echo "Available commands:"
|
||||
echo ""
|
||||
echo " To start unoserver: "
|
||||
echo -e "\e[34m nohup /opt/venv/bin/unoserver --port 2003 --interface 0.0.0.0 > /tmp/unoserver.log 2>&1 &\e[0m"
|
||||
echo
|
||||
echo " To start the application: "
|
||||
echo -e "\e[34m gradle bootRun\e[0m"
|
||||
echo ""
|
||||
echo " To run tests: "
|
||||
echo -e "\e[34m gradle test\e[0m"
|
||||
echo ""
|
||||
echo " To build the project: "
|
||||
echo -e "\e[34m gradle build\e[0m"
|
||||
echo ""
|
||||
echo " To run pre-commit hooks (if configured):"
|
||||
echo -e "\e[34m pre-commit run --all-files -c .pre-commit-config.yaml\e[0m"
|
||||
echo "=================================================================="
|
||||
@@ -0,0 +1,29 @@
|
||||
spec:
|
||||
name: stirling-pdf
|
||||
services:
|
||||
- name: stirling-pdf
|
||||
git:
|
||||
branch: digitalOcean
|
||||
repo_clone_url: https://github.com/Stirling-Tools/Stirling-PDF.git
|
||||
dockerfile_path: Dockerfile2
|
||||
envs:
|
||||
- key: APP_HOME_NAME
|
||||
value: "Stirling PDF"
|
||||
- key: APP_HOME_DESCRIPTION
|
||||
value: "Your locally hosted one-stop-shop for all your PDF needs."
|
||||
- key: APP_NAVBAR_NAME
|
||||
value: "Stirling PDF"
|
||||
- key: ALLOW_GOOGLE_VISIBILITY
|
||||
value: "false"
|
||||
- key: APP_ROOT_PATH
|
||||
value: "/"
|
||||
- key: APP_LOCALE
|
||||
value: "en_GB"
|
||||
- key: TESSERACT_LANGS
|
||||
value: "eng"
|
||||
routes:
|
||||
- path: /
|
||||
http_port: 8080
|
||||
instance_count: 1
|
||||
instance_size_slug: basic-xxs
|
||||
source_dir: "/"
|
||||
@@ -1,52 +0,0 @@
|
||||
root = true
|
||||
|
||||
[*]
|
||||
charset = utf-8
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
end_of_line = lf
|
||||
max_line_length = 127
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
|
||||
[*.java]
|
||||
indent_size = 4
|
||||
max_line_length = 100
|
||||
|
||||
[*.py]
|
||||
indent_size = 2
|
||||
|
||||
[*.gradle]
|
||||
indent_size = 4
|
||||
|
||||
[*.html]
|
||||
indent_size = 2
|
||||
insert_final_newline = false
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[*.js]
|
||||
indent_size = 2
|
||||
|
||||
[*.css]
|
||||
# CSS files typically use an indent size of 2 spaces for better readability and alignment with community standards.
|
||||
indent_size = 2
|
||||
|
||||
[*.yaml]
|
||||
# YAML files use an indent size of 2 spaces to maintain consistency with common YAML formatting practices.
|
||||
indent_size = 2
|
||||
insert_final_newline = false
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[*.yml]
|
||||
# YML files follow the same conventions as YAML files, using an indent size of 2 spaces.
|
||||
indent_size = 2
|
||||
insert_final_newline = false
|
||||
trim_trailing_whitespace = false
|
||||
|
||||
[*.json]
|
||||
# JSON files use an indent size of 2 spaces, which is the standard for JSON formatting.
|
||||
indent_size = 2
|
||||
|
||||
[*.jsonc]
|
||||
# JSONC (JSON with comments) files also follow the standard JSON formatting with an indent size of 2 spaces.
|
||||
indent_size = 2
|
||||
@@ -1,9 +1,5 @@
|
||||
# Formatting
|
||||
5f771b785130154ed47952635b7acef371ffe0ec
|
||||
7fa5e130d99227c2202ebddfdd91348176ec0c7b
|
||||
14d4fbb2a36195eedb034785e5a5ff6a47f268c6
|
||||
ee8030c1c4148062cde15c49c67d04ef03930c55
|
||||
fcd41924f5f261febfa9d9a92994671f3ebc97d6
|
||||
|
||||
# Normalize files
|
||||
55d4fda01b2f39f5b7d7b4fda5214bd7ff0fd5dd
|
||||
|
||||
+7
-7
@@ -1,10 +1,10 @@
|
||||
* text=auto eol=lf
|
||||
|
||||
# Ignore all JavaScript files in a directory
|
||||
app/core/src/main/resources/static/pdfjs/* linguist-vendored
|
||||
app/core/src/main/resources/static/pdfjs/** linguist-vendored
|
||||
app/core/src/main/resources/static/pdfjs-legacy/* linguist-vendored
|
||||
app/core/src/main/resources/static/pdfjs-legacy/** linguist-vendored
|
||||
app/core/src/main/resources/static/css/bootstrap-icons.css linguist-vendored
|
||||
app/core/src/main/resources/static/css/bootstrap.min.css linguist-vendored
|
||||
app/core/src/main/resources/static/css/fonts/* linguist-vendored
|
||||
src/main/resources/static/pdfjs/* linguist-vendored
|
||||
src/main/resources/static/pdfjs/** linguist-vendored
|
||||
src/main/resources/static/pdfjs-legacy/* linguist-vendored
|
||||
src/main/resources/static/pdfjs-legacy/** linguist-vendored
|
||||
src/main/resources/static/css/bootstrap-icons.css linguist-vendored
|
||||
src/main/resources/static/css/bootstrap.min.css linguist-vendored
|
||||
src/main/resources/static/css/fonts/* linguist-vendored
|
||||
|
||||
+1
-1
@@ -1,2 +1,2 @@
|
||||
# All PRs to V1 must be approved by Frooodle
|
||||
* @Frooodle @reecebrowne @Ludy87 @DarioGii @ConnorYoh @EthanHealy01
|
||||
* @Frooodle @reecebrowne @Ludy87 @DarioGii @ConnorYoh
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
name: 'Setup GitHub App Bot'
|
||||
description: 'Generates a GitHub App Token and configures Git for a bot'
|
||||
inputs:
|
||||
app-id:
|
||||
description: 'GitHub App ID'
|
||||
required: True
|
||||
private-key:
|
||||
description: 'GitHub App Private Key'
|
||||
required: True
|
||||
outputs:
|
||||
token:
|
||||
description: 'Generated GitHub App Token'
|
||||
value: ${{ steps.generate-token.outputs.token }}
|
||||
committer:
|
||||
description: 'Committer string for Git'
|
||||
value: "${{ steps.generate-token.outputs.app-slug }}[bot] <${{ steps.generate-token.outputs.app-slug }}[bot]@users.noreply.github.com>"
|
||||
app-slug:
|
||||
description: 'GitHub App slug'
|
||||
value: ${{ steps.generate-token.outputs.app-slug }}
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- name: Generate a GitHub App Token
|
||||
id: generate-token
|
||||
uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
|
||||
with:
|
||||
app-id: ${{ inputs.app-id }}
|
||||
private-key: ${{ inputs.private-key }}
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config --global user.name "${{ steps.generate-token.outputs.app-slug }}[bot]"
|
||||
git config --global user.email "${{ steps.generate-token.outputs.app-slug }}[bot]@users.noreply.github.com"
|
||||
shell: bash
|
||||
@@ -1,29 +0,0 @@
|
||||
build: &build
|
||||
- build.gradle
|
||||
- app/(common|core|proprietary)/build.gradle
|
||||
|
||||
app: &app
|
||||
- app/(common|core|proprietary)/src/main/java/**
|
||||
|
||||
openapi: &openapi
|
||||
- build.gradle
|
||||
- app/(common|core|proprietary)/build.gradle
|
||||
- app/(common|core|proprietary)/src/main/java/**
|
||||
|
||||
project: &project
|
||||
- app/(common|core|proprietary)/src/(main|test)/java/**
|
||||
- app/(common|core|proprietary)/build.gradle
|
||||
- 'app/(common|core|proprietary)/src/(main|test)/resources/**/!(messages_*.properties|*.md)*'
|
||||
- exampleYmlFiles/**
|
||||
- gradle/**
|
||||
- libs/**
|
||||
- testing/**
|
||||
- build.gradle
|
||||
- Dockerfile
|
||||
- Dockerfile.fat
|
||||
- Dockerfile.ultra-lite
|
||||
- gradle.properties
|
||||
- gradlew
|
||||
- gradlew.bat
|
||||
- launch4jConfig.xml
|
||||
- settings.gradle
|
||||
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"repo_devs": [
|
||||
"Frooodle",
|
||||
"sf298",
|
||||
"Ludy87",
|
||||
"LaserKaspar",
|
||||
"sbplat",
|
||||
"reecebrowne",
|
||||
"DarioGii",
|
||||
"ConnorYoh",
|
||||
"EthanHealy01"
|
||||
]
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
You are a professional software engineer specializing in reviewing pull request titles.
|
||||
|
||||
Your job is to analyze a git diff and an existing PR title, then evaluate and improve the PR title.
|
||||
|
||||
You must:
|
||||
- Always return valid JSON
|
||||
- Only return the JSON response (no Markdown, no formatting)
|
||||
- Use one of these conventional commit types at the beginning of the title: build, chore, ci, docs, feat, fix, perf, refactor, revert, style, test
|
||||
- Use lowercase only, no emojis, no trailing period
|
||||
- Ensure the title is between 5 and 72 printable ASCII characters
|
||||
- Never let spelling or grammar errors affect the rating
|
||||
- If the PR title is rated 6 or higher and only contains spelling or grammar mistakes, correct it - do not rephrase it
|
||||
- If the PR title is rated below 6, generate a new, better title based on the diff
|
||||
@@ -1,158 +0,0 @@
|
||||
version: 1
|
||||
labels:
|
||||
|
||||
- label: "Bugfix"
|
||||
title: '^fix(\([^)]*\))?:|^fix:.*'
|
||||
|
||||
- label: "enhancement"
|
||||
title: '^feat(\([^)]*\))?:|^feat:.*'
|
||||
|
||||
- label: "build"
|
||||
title: '^build(\([^)]*\))?:|^build:.*'
|
||||
|
||||
- label: "chore"
|
||||
title: '^chore(\([^)]*\))?:|^chore:.*'
|
||||
|
||||
- label: "ci"
|
||||
title: '^ci(\([^)]*\))?:|^ci:.*'
|
||||
|
||||
- label: "ci"
|
||||
title: '^.*\(ci\):.*'
|
||||
|
||||
- label: "perf"
|
||||
title: '^perf(\([^)]*\))?:|^perf:.*'
|
||||
|
||||
- label: "refactor"
|
||||
title: '^refactor(\([^)]*\))?:|^refactor:.*'
|
||||
|
||||
- label: "revert"
|
||||
title: '^revert(\([^)]*\))?:|^revert:.*'
|
||||
|
||||
- label: "style"
|
||||
title: '^style(\([^)]*\))?:|^style:.*'
|
||||
|
||||
- label: "Documentation"
|
||||
title: '^docs(\([^)]*\))?:|^docs:.*'
|
||||
|
||||
- label: "Documentation"
|
||||
title: '^.*\(docs\):.*'
|
||||
|
||||
- label: "dependencies"
|
||||
title: '^deps(\([^)]*\))?:|^deps:.*'
|
||||
|
||||
- label: "dependencies"
|
||||
title: '^.*\(deps\):.*'
|
||||
|
||||
- label: 'API'
|
||||
title: '.*openapi.*|.*swagger.*|.*api.*'
|
||||
|
||||
- label: 'Translation'
|
||||
files:
|
||||
- 'app/core/src/main/resources/messages_[a-zA-Z_]{2}_[a-zA-Z_]{2,7}.properties'
|
||||
- 'scripts/ignore_translation.toml'
|
||||
- 'app/core/src/main/resources/templates/fragments/languages.html'
|
||||
- '.github/scripts/check_language_properties.py'
|
||||
|
||||
- label: 'Front End'
|
||||
files:
|
||||
- 'app/core/src/main/resources/templates/.*'
|
||||
- 'app/proprietary/src/main/resources/templates/.*'
|
||||
- 'app/core/src/main/resources/static/.*'
|
||||
- 'app/proprietary/src/main/resources/static/.*'
|
||||
- 'app/core/src/main/java/stirling/software/SPDF/controller/web/.*'
|
||||
- 'app/core/src/main/java/stirling/software/SPDF/UI/.*'
|
||||
- 'app/proprietary/src/main/java/stirling/software/proprietary/security/controller/web/.*'
|
||||
|
||||
- label: 'Java'
|
||||
files:
|
||||
- 'app/common/src/main/java/.*.java'
|
||||
- 'app/proprietary/src/main/java/.*.java'
|
||||
- 'app/core/src/main/java/.*.java'
|
||||
|
||||
- label: 'Back End'
|
||||
files:
|
||||
- 'app/core/src/main/java/stirling/software/SPDF/config/.*'
|
||||
- 'app/core/src/main/java/stirling/software/SPDF/controller/.*'
|
||||
- 'app/core/src/main/resources/settings.yml.template'
|
||||
- 'app/core/src/main/resources/application.properties'
|
||||
- 'app/core/src/main/resources/banner.txt'
|
||||
- 'app/core/src/main/resources/static/python/png_to_webp.py'
|
||||
- 'app/core/src/main/resources/static/python/split_photos.py'
|
||||
- 'app/core/src/main/resources/static/pipeline/defaultWebUIConfigs/**'
|
||||
- 'application.properties'
|
||||
|
||||
- label: 'Security'
|
||||
files:
|
||||
- 'app/proprietary/src/main/java/stirling/software/proprietary/security/.*'
|
||||
- 'scripts/download-security-jar.sh'
|
||||
- '.github/workflows/dependency-review.yml'
|
||||
- '.github/workflows/scorecards.yml'
|
||||
|
||||
- label: 'API'
|
||||
files:
|
||||
- 'app/core/src/main/java/stirling/software/SPDF/config/OpenApiConfig.java'
|
||||
- 'app/core/src/main/java/stirling/software/SPDF/controller/web/MetricsController.java'
|
||||
- 'app/core/src/main/java/stirling/software/SPDF/controller/api/.*'
|
||||
- 'app/core/src/main/java/stirling/software/SPDF/model/api/.*'
|
||||
- 'app/core/src/main/java/stirling/software/SPDF/service/ApiDocService.java'
|
||||
- 'app/proprietary/src/main/java/stirling/software/proprietary/security/controller/api/.*'
|
||||
- 'app/core/src/main/resources/static/python/png_to_webp.py'
|
||||
- 'app/core/src/main/resources/static/python/split_photos.py'
|
||||
- '.github/workflows/swagger.yml'
|
||||
|
||||
- label: 'Documentation'
|
||||
files:
|
||||
- '.*.md'
|
||||
- 'scripts/counter_translation.py'
|
||||
- 'scripts/ignore_translation.toml'
|
||||
|
||||
- label: 'Docker'
|
||||
files:
|
||||
- '.github/workflows/build.yml'
|
||||
- '.github/workflows/push-docker.yml'
|
||||
- 'Dockerfile'
|
||||
- 'Dockerfile.fat'
|
||||
- 'Dockerfile.ultra-lite'
|
||||
- 'exampleYmlFiles/.*.yml'
|
||||
- 'scripts/download-security-jar.sh'
|
||||
- 'scripts/init.sh'
|
||||
- 'scripts/init-without-ocr.sh'
|
||||
- 'scripts/installFonts.sh'
|
||||
- 'test.sh'
|
||||
- 'test2.sh'
|
||||
|
||||
- label: 'Devtools'
|
||||
files:
|
||||
- '.devcontainer/.*'
|
||||
- 'Dockerfile.dev'
|
||||
- '.vscode/.*'
|
||||
- '.editorconfig'
|
||||
- '.pre-commit-config'
|
||||
- '.github/workflows/pre_commit.yml'
|
||||
- 'devGuide/.*'
|
||||
- 'devTools/.*'
|
||||
- 'devTools/.*'
|
||||
|
||||
- label: 'Test'
|
||||
files:
|
||||
- 'app/common/src/test/.*'
|
||||
- 'app/proprietary/src/test/.*'
|
||||
- 'app/core/src/test/.*'
|
||||
- 'testing/.*'
|
||||
- '.github/workflows/scorecards.yml'
|
||||
- 'exampleYmlFiles/test_cicd.yml'
|
||||
|
||||
- label: 'Github'
|
||||
files:
|
||||
- '.github/.*'
|
||||
|
||||
- label: 'Gradle'
|
||||
files:
|
||||
- 'gradle/.*'
|
||||
- 'gradlew'
|
||||
- 'gradlew.bat'
|
||||
- 'settings.gradle'
|
||||
- 'build.gradle'
|
||||
- 'app/common/build.gradle'
|
||||
- 'app/proprietary/build.gradle'
|
||||
- 'app/core/build.gradle'
|
||||
@@ -0,0 +1,76 @@
|
||||
Translation:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: 'src/main/resources/messages_*_*.properties'
|
||||
- any-glob-to-any-file: 'scripts/ignore_translation.toml'
|
||||
- any-glob-to-any-file: 'src/main/resources/templates/fragments/languages.html'
|
||||
|
||||
Front End:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: 'src/main/resources/templates/**/*'
|
||||
- any-glob-to-any-file: 'src/main/resources/static/**/*'
|
||||
- any-glob-to-any-file: 'src/main/java/stirling/software/SPDF/controller/web/**'
|
||||
- any-glob-to-any-file: 'src/main/java/stirling/software/SPDF/UI/**/*'
|
||||
|
||||
Java:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: 'src/main/java/**/*.java'
|
||||
|
||||
Back End:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: 'src/main/java/stirling/software/SPDF/config/**/*'
|
||||
- any-glob-to-any-file: 'src/main/java/stirling/software/SPDF/controller/**/*'
|
||||
- any-glob-to-any-file: 'src/main/resources/settings.yml.template'
|
||||
- any-glob-to-any-file: 'src/main/resources/application.properties'
|
||||
- any-glob-to-any-file: 'src/main/resources/banner.txt'
|
||||
- any-glob-to-any-file: 'scripts/png_to_webp.py'
|
||||
- any-glob-to-any-file: 'split_photos.py'
|
||||
|
||||
Security:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: 'src/main/java/stirling/software/SPDF/config/security/**/*'
|
||||
- any-glob-to-any-file: 'src/main/java/stirling/software/SPDF/model/provider/**/*'
|
||||
- any-glob-to-any-file: 'src/main/java/stirling/software/SPDF/model/AuthenticationType.java'
|
||||
- any-glob-to-any-file: 'src/main/java/stirling/software/SPDF/model/BackupNotFoundException.java'
|
||||
- any-glob-to-any-file: 'scripts/download-security-jar.sh'
|
||||
- any-glob-to-any-file: '.github/workflows/dependency-review.yml'
|
||||
- any-glob-to-any-file: '.github/workflows/scorecards.yml'
|
||||
|
||||
API:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: 'src/main/java/stirling/software/SPDF/controller/web/MetricsController.java'
|
||||
- any-glob-to-any-file: 'src/main/java/stirling/software/SPDF/controller/api/**/*'
|
||||
- any-glob-to-any-file: 'scripts/png_to_webp.py'
|
||||
- any-glob-to-any-file: 'split_photos.py'
|
||||
- any-glob-to-any-file: '.github/workflows/swagger.yml'
|
||||
|
||||
Documentation:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: '**/*.md'
|
||||
- any-glob-to-any-file: 'scripts/counter_translation.py'
|
||||
- any-glob-to-any-file: 'scripts/ignore_translation.toml'
|
||||
|
||||
Docker:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: '.github/workflows/build.yml'
|
||||
- any-glob-to-any-file: '.github/workflows/push-docker.yml'
|
||||
- any-glob-to-any-file: 'Dockerfile'
|
||||
- any-glob-to-any-file: 'Dockerfile.*'
|
||||
- any-glob-to-any-file: 'exampleYmlFiles/*.yml'
|
||||
- any-glob-to-any-file: 'scripts/download-security-jar.sh'
|
||||
- any-glob-to-any-file: 'scripts/init.sh'
|
||||
- any-glob-to-any-file: 'scripts/init-without-ocr.sh'
|
||||
- any-glob-to-any-file: 'scripts/installFonts.sh'
|
||||
- any-glob-to-any-file: 'test.sh'
|
||||
- any-glob-to-any-file: 'test2.sh'
|
||||
|
||||
Test:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: 'cucumber/**/*'
|
||||
- any-glob-to-any-file: 'src/test**/*'
|
||||
- any-glob-to-any-file: '.pre-commit-config'
|
||||
- any-glob-to-any-file: '.github/workflows/pre_commit.yml'
|
||||
- any-glob-to-any-file: '.github/workflows/scorecards.yml'
|
||||
|
||||
Github:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file: '.github/**/*'
|
||||
@@ -108,73 +108,3 @@
|
||||
- name: "Priority: Low"
|
||||
color: "00FF00"
|
||||
description: "Issues or pull requests with low priority"
|
||||
- name: "Devtools"
|
||||
color: "FF9E1F"
|
||||
description: "Development tools"
|
||||
- name: "Bugfix"
|
||||
color: "FF9E1F"
|
||||
description: "Pull requests that fix bugs"
|
||||
- name: "Gradle"
|
||||
color: "FF9E1F"
|
||||
description: "Pull requests that update Gradle code"
|
||||
- name: "build"
|
||||
color: "1E90FF"
|
||||
description: "Changes that affect the build system or external dependencies"
|
||||
- name: "chore"
|
||||
color: "FFD700"
|
||||
description: "Routine tasks or maintenance that don't modify src or test files"
|
||||
- name: "ci"
|
||||
color: "4682B4"
|
||||
description: "Changes to CI configuration files and scripts"
|
||||
- name: "perf"
|
||||
color: "FF69B4"
|
||||
description: "Changes that improve performance"
|
||||
- name: "refactor"
|
||||
color: "9932CC"
|
||||
description: "Code changes that neither fix a bug nor add a feature"
|
||||
- name: "revert"
|
||||
color: "DC143C"
|
||||
description: "Reverts a previous commit"
|
||||
- name: "style"
|
||||
color: "FFA500"
|
||||
description: "Changes that do not affect the meaning of the code (formatting, etc.)"
|
||||
- name: "admin"
|
||||
color: "195055"
|
||||
- name: "codex"
|
||||
color: "ededed"
|
||||
description: null
|
||||
- name: "Github"
|
||||
color: "0052CC"
|
||||
- name: "github_actions"
|
||||
color: "000000"
|
||||
description: "Pull requests that update GitHub Actions code"
|
||||
- name: "needs-changes"
|
||||
color: "A65A86"
|
||||
- name: "on-hold"
|
||||
color: "2526F9"
|
||||
- name: "python"
|
||||
color: "2b67c6"
|
||||
description: "Pull requests that update Python code"
|
||||
- name: "size:L"
|
||||
color: "eb9500"
|
||||
description: "This PR changes 100-499 lines ignoring generated files."
|
||||
- name: "size:M"
|
||||
color: "ebb800"
|
||||
description: "This PR changes 30-99 lines ignoring generated files."
|
||||
- name: "size:S"
|
||||
color: "77b800"
|
||||
description: "This PR changes 10-29 lines ignoring generated files."
|
||||
- name: "size:XL"
|
||||
color: "ff823f"
|
||||
description: "This PR changes 500-999 lines ignoring generated files."
|
||||
- name: "size:XS"
|
||||
color: "00ff00"
|
||||
description: "This PR changes 0-9 lines ignoring generated files."
|
||||
- name: "size:XXL"
|
||||
color: "ffb8b8"
|
||||
description: "This PR changes 1000+ lines ignoring generated files."
|
||||
- name: "to research"
|
||||
color: "FBCA04"
|
||||
- name: "pr-deployed"
|
||||
color: "00FF00"
|
||||
description: "Pull request has been deployed to a test environment"
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
# Description of Changes
|
||||
|
||||
<!--
|
||||
Please provide a summary of the changes, including:
|
||||
|
||||
- What was changed
|
||||
@@ -8,7 +7,6 @@ Please provide a summary of the changes, including:
|
||||
- Any challenges encountered
|
||||
|
||||
Closes #(issue_number)
|
||||
-->
|
||||
|
||||
---
|
||||
|
||||
@@ -17,15 +15,15 @@ Closes #(issue_number)
|
||||
### General
|
||||
|
||||
- [ ] I have read the [Contribution Guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md)
|
||||
- [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/DeveloperGuide.md) (if applicable)
|
||||
- [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md) (if applicable)
|
||||
- [ ] I have read the [Stirling-PDF Developer Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md) (if applicable)
|
||||
- [ ] I have read the [How to add new languages to Stirling-PDF](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/HowToAddNewLanguage.md) (if applicable)
|
||||
- [ ] I have performed a self-review of my own code
|
||||
- [ ] My changes generate no new warnings
|
||||
|
||||
### Documentation
|
||||
|
||||
- [ ] I have updated relevant docs on [Stirling-PDF's doc repo](https://github.com/Stirling-Tools/Stirling-Tools.github.io/blob/main/docs/) (if functionality has heavily changed)
|
||||
- [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only)
|
||||
- [ ] I have read the section [Add New Translation Tags](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/HowToAddNewLanguage.md#add-new-translation-tags) (for new translation tags only)
|
||||
|
||||
### UI Changes (if applicable)
|
||||
|
||||
@@ -33,4 +31,4 @@ Closes #(issue_number)
|
||||
|
||||
### Testing (if applicable)
|
||||
|
||||
- [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/devGuide/DeveloperGuide.md#6-testing) for more details.
|
||||
- [ ] I have tested my changes locally. Refer to the [Testing Guide](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/DeveloperGuide.md#6-testing) for more details.
|
||||
|
||||
@@ -21,10 +21,6 @@ changelog:
|
||||
labels:
|
||||
- Translation
|
||||
|
||||
- title: Development Tools
|
||||
labels:
|
||||
- Devtools
|
||||
|
||||
- title: Other Changes
|
||||
labels:
|
||||
- "*"
|
||||
|
||||
@@ -164,7 +164,7 @@ def update_missing_keys(reference_file, file_list, branch=""):
|
||||
if current_entry["type"] == "entry":
|
||||
if ref_entry_copy["type"] != "entry":
|
||||
continue
|
||||
if ref_entry_copy["key"].lower() == current_entry["key"].lower():
|
||||
if ref_entry_copy["key"] == current_entry["key"]:
|
||||
ref_entry_copy["value"] = current_entry["value"]
|
||||
updated_properties.append(ref_entry_copy)
|
||||
write_json_file(os.path.join(branch, file_path), updated_properties)
|
||||
@@ -196,45 +196,32 @@ def check_for_differences(reference_file, file_list, branch, actor):
|
||||
|
||||
if len(file_list) == 1:
|
||||
file_arr = file_list[0].split()
|
||||
base_dir = os.path.abspath(
|
||||
os.path.join(os.getcwd(), "app", "core", "src", "main", "resources")
|
||||
)
|
||||
base_dir = os.path.abspath(os.path.join(os.getcwd(), "src", "main", "resources"))
|
||||
|
||||
for file_path in file_arr:
|
||||
file_normpath = os.path.normpath(file_path)
|
||||
absolute_path = os.path.abspath(file_normpath)
|
||||
absolute_path = os.path.abspath(file_path)
|
||||
# Verify that file is within the expected directory
|
||||
if not absolute_path.startswith(base_dir):
|
||||
raise ValueError(f"Unsafe file found: {file_normpath}")
|
||||
raise ValueError(f"Unsafe file found: {file_path}")
|
||||
# Verify file size before processing
|
||||
if os.path.getsize(os.path.join(branch, file_normpath)) > MAX_FILE_SIZE:
|
||||
if os.path.getsize(os.path.join(branch, file_path)) > MAX_FILE_SIZE:
|
||||
raise ValueError(
|
||||
f"The file {file_normpath} is too large and could pose a security risk."
|
||||
f"The file {file_path} is too large and could pose a security risk."
|
||||
)
|
||||
|
||||
basename_current_file = os.path.basename(os.path.join(branch, file_normpath))
|
||||
basename_current_file = os.path.basename(os.path.join(branch, file_path))
|
||||
if (
|
||||
basename_current_file == basename_reference_file
|
||||
or (
|
||||
# only local windows command
|
||||
not file_normpath.startswith(
|
||||
os.path.join(
|
||||
"", "app", "core", "src", "main", "resources", "messages_"
|
||||
)
|
||||
not file_path.startswith(
|
||||
os.path.join("", "src", "main", "resources", "messages_")
|
||||
)
|
||||
and not file_normpath.startswith(
|
||||
os.path.join(
|
||||
os.getcwd(),
|
||||
"app",
|
||||
"core",
|
||||
"src",
|
||||
"main",
|
||||
"resources",
|
||||
"messages_",
|
||||
)
|
||||
and not file_path.startswith(
|
||||
os.path.join(os.getcwd(), "src", "main", "resources", "messages_")
|
||||
)
|
||||
)
|
||||
or not file_normpath.endswith(".properties")
|
||||
or not file_path.endswith(".properties")
|
||||
or not basename_current_file.startswith("messages_")
|
||||
):
|
||||
continue
|
||||
@@ -305,13 +292,13 @@ def check_for_differences(reference_file, file_list, branch, actor):
|
||||
else:
|
||||
report.append("2. **Test Status:** ✅ **_Passed_**")
|
||||
|
||||
if find_duplicate_keys(os.path.join(branch, file_normpath)):
|
||||
if find_duplicate_keys(os.path.join(branch, file_path)):
|
||||
has_differences = True
|
||||
output = "\n".join(
|
||||
[
|
||||
f" - `{key}`: first at line {first}, duplicate at `line {duplicate}`"
|
||||
for key, first, duplicate in find_duplicate_keys(
|
||||
os.path.join(branch, file_normpath)
|
||||
os.path.join(branch, file_path)
|
||||
)
|
||||
]
|
||||
)
|
||||
@@ -329,7 +316,7 @@ def check_for_differences(reference_file, file_list, branch, actor):
|
||||
report.append("## ❌ Overall Check Status: **_Failed_**")
|
||||
report.append("")
|
||||
report.append(
|
||||
f"@{actor} please check your translation if it conforms to the standard. Follow the format of [messages_en_GB.properties](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/app/core/src/main/resources/messages_en_GB.properties)"
|
||||
f"@{actor} please check your translation if it conforms to the standard. Follow the format of [messages_en_GB.properties](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/src/main/resources/messages_en_GB.properties)"
|
||||
)
|
||||
else:
|
||||
report.append("## ✅ Overall Check Status: **_Success_**")
|
||||
@@ -389,13 +376,7 @@ if __name__ == "__main__":
|
||||
else:
|
||||
file_list = glob.glob(
|
||||
os.path.join(
|
||||
os.getcwd(),
|
||||
"app",
|
||||
"core",
|
||||
"src",
|
||||
"main",
|
||||
"resources",
|
||||
"messages_*.properties",
|
||||
os.getcwd(), "src", "main", "resources", "messages_*.properties"
|
||||
)
|
||||
)
|
||||
update_missing_keys(args.reference_file, file_list)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# This file is autogenerated by pip-compile with Python 3.10
|
||||
# by the following command:
|
||||
#
|
||||
# pip-compile --generate-hashes --output-file='.github\scripts\requirements_pre_commit.txt' --strip-extras '.github\scripts\requirements_pre_commit.in'
|
||||
# pip-compile --generate-hashes --output-file='.github\scripts\requirements_pre_commit.txt' '.github\scripts\requirements_pre_commit.in'
|
||||
#
|
||||
cfgv==3.4.0 \
|
||||
--hash=sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9 \
|
||||
@@ -12,25 +12,25 @@ distlib==0.3.9 \
|
||||
--hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \
|
||||
--hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403
|
||||
# via virtualenv
|
||||
filelock==3.18.0 \
|
||||
--hash=sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2 \
|
||||
--hash=sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de
|
||||
filelock==3.16.1 \
|
||||
--hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \
|
||||
--hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435
|
||||
# via virtualenv
|
||||
identify==2.6.12 \
|
||||
--hash=sha256:ad9672d5a72e0d2ff7c5c8809b62dfa60458626352fb0eb7b55e69bdc45334a2 \
|
||||
--hash=sha256:d8de45749f1efb108badef65ee8386f0f7bb19a7f26185f74de6367bffbaf0e6
|
||||
identify==2.6.5 \
|
||||
--hash=sha256:14181a47091eb75b337af4c23078c9d09225cd4c48929f521f3bf16b09d02566 \
|
||||
--hash=sha256:c10b33f250e5bba374fae86fb57f3adcebf1161bce7cdf92031915fd480c13bc
|
||||
# via pre-commit
|
||||
nodeenv==1.9.1 \
|
||||
--hash=sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f \
|
||||
--hash=sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9
|
||||
# via pre-commit
|
||||
platformdirs==4.3.8 \
|
||||
--hash=sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc \
|
||||
--hash=sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4
|
||||
platformdirs==4.3.6 \
|
||||
--hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \
|
||||
--hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb
|
||||
# via virtualenv
|
||||
pre-commit==4.2.0 \
|
||||
--hash=sha256:601283b9757afd87d40c4c4a9b2b5de9637a8ea02eaff7adc2d0fb4e04841146 \
|
||||
--hash=sha256:a009ca7205f1eb497d10b845e52c838a98b6cdd2102a6c8e4540e94ee75c58bd
|
||||
pre-commit==4.0.1 \
|
||||
--hash=sha256:80905ac375958c0444c65e9cebebd948b3cdb518f335a091a670a89d652139d2 \
|
||||
--hash=sha256:efde913840816312445dc98787724647c65473daefe420785f885e8ed9a06878
|
||||
# via -r .github\scripts\requirements_pre_commit.in
|
||||
pyyaml==6.0.2 \
|
||||
--hash=sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff \
|
||||
@@ -87,7 +87,7 @@ pyyaml==6.0.2 \
|
||||
--hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \
|
||||
--hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4
|
||||
# via pre-commit
|
||||
virtualenv==20.31.2 \
|
||||
--hash=sha256:36efd0d9650ee985f0cad72065001e66d49a6f24eb44d98980f630686243cf11 \
|
||||
--hash=sha256:e10c0a9d02835e592521be48b332b6caee6887f332c111aa79a09b9e79efc2af
|
||||
virtualenv==20.28.1 \
|
||||
--hash=sha256:412773c85d4dab0409b83ec36f7a6499e72eaf08c80e81e9576bca61831c71cb \
|
||||
--hash=sha256:5d34ab240fdb5d21549b76f9e8ff3af28252f5499fb6d6f031adac4e5a8c5329
|
||||
# via pre-commit
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
# This file is autogenerated by pip-compile with Python 3.10
|
||||
# by the following command:
|
||||
#
|
||||
# pip-compile --generate-hashes --output-file='.github\scripts\requirements_sync_readme.txt' --strip-extras '.github\scripts\requirements_sync_readme.in'
|
||||
# pip-compile --generate-hashes --output-file='.github\scripts\requirements_sync_readme.txt' '.github\scripts\requirements_sync_readme.in'
|
||||
#
|
||||
tomlkit==0.13.3 \
|
||||
--hash=sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1 \
|
||||
--hash=sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0
|
||||
tomlkit==0.13.2 \
|
||||
--hash=sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde \
|
||||
--hash=sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79
|
||||
# via -r .github\scripts\requirements_sync_readme.in
|
||||
|
||||
@@ -1,383 +0,0 @@
|
||||
name: PR Deployment via Comment
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
check-comment:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
if: |
|
||||
github.event.issue.pull_request &&
|
||||
(
|
||||
contains(github.event.comment.body, 'prdeploy') ||
|
||||
contains(github.event.comment.body, 'deploypr')
|
||||
)
|
||||
&&
|
||||
(
|
||||
github.event.comment.user.login == 'frooodle' ||
|
||||
github.event.comment.user.login == 'sf298' ||
|
||||
github.event.comment.user.login == 'Ludy87' ||
|
||||
github.event.comment.user.login == 'LaserKaspar' ||
|
||||
github.event.comment.user.login == 'sbplat' ||
|
||||
github.event.comment.user.login == 'reecebrowne' ||
|
||||
github.event.comment.user.login == 'DarioGii' ||
|
||||
github.event.comment.user.login == 'EthanHealy01' ||
|
||||
github.event.comment.user.login == 'ConnorYoh'
|
||||
)
|
||||
outputs:
|
||||
pr_number: ${{ steps.get-pr.outputs.pr_number }}
|
||||
pr_repository: ${{ steps.get-pr-info.outputs.repository }}
|
||||
pr_ref: ${{ steps.get-pr-info.outputs.ref }}
|
||||
comment_id: ${{ github.event.comment.id }}
|
||||
disable_security: ${{ steps.check-security-flag.outputs.disable_security }}
|
||||
enable_pro: ${{ steps.check-pro-flag.outputs.enable_pro }}
|
||||
enable_enterprise: ${{ steps.check-pro-flag.outputs.enable_enterprise }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout PR
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Setup GitHub App Bot
|
||||
if: github.actor != 'dependabot[bot]'
|
||||
id: setup-bot
|
||||
uses: ./.github/actions/setup-bot
|
||||
continue-on-error: true
|
||||
with:
|
||||
app-id: ${{ secrets.GH_APP_ID }}
|
||||
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
|
||||
|
||||
- name: Get PR data
|
||||
id: get-pr
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
script: |
|
||||
const prNumber = context.payload.issue.number;
|
||||
console.log(`PR Number: ${prNumber}`);
|
||||
core.setOutput('pr_number', prNumber);
|
||||
|
||||
- name: Get PR repository and ref
|
||||
id: get-pr-info
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
script: |
|
||||
const { owner, repo } = context.repo;
|
||||
const prNumber = context.payload.issue.number;
|
||||
|
||||
const { data: pr } = await github.rest.pulls.get({
|
||||
owner,
|
||||
repo,
|
||||
pull_number: prNumber,
|
||||
});
|
||||
|
||||
// For forks, use the full repository name, for internal PRs use the current repo
|
||||
const repository = pr.head.repo.fork ? pr.head.repo.full_name : `${owner}/${repo}`;
|
||||
|
||||
console.log(`PR Repository: ${repository}`);
|
||||
console.log(`PR Branch: ${pr.head.ref}`);
|
||||
|
||||
core.setOutput('repository', repository);
|
||||
core.setOutput('ref', pr.head.ref);
|
||||
|
||||
- name: Check for security/login flag
|
||||
id: check-security-flag
|
||||
env:
|
||||
COMMENT_BODY: ${{ github.event.comment.body }}
|
||||
run: |
|
||||
if [[ "$COMMENT_BODY" == *"security"* ]] || [[ "$COMMENT_BODY" == *"login"* ]]; then
|
||||
echo "Security flags detected in comment"
|
||||
echo "disable_security=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "No security flags detected in comment"
|
||||
echo "disable_security=true" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Check for pro flag
|
||||
id: check-pro-flag
|
||||
env:
|
||||
COMMENT_BODY: ${{ github.event.comment.body }}
|
||||
run: |
|
||||
if [[ "$COMMENT_BODY" == *"pro"* ]] || [[ "$COMMENT_BODY" == *"premium"* ]]; then
|
||||
echo "pro flags detected in comment"
|
||||
echo "enable_pro=true" >> $GITHUB_OUTPUT
|
||||
echo "enable_enterprise=false" >> $GITHUB_OUTPUT
|
||||
elif [[ "$COMMENT_BODY" == *"enterprise"* ]]; then
|
||||
echo "enterprise flags detected in comment"
|
||||
echo "enable_enterprise=true" >> $GITHUB_OUTPUT
|
||||
echo "enable_pro=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "No pro or enterprise flags detected in comment"
|
||||
echo "enable_pro=false" >> $GITHUB_OUTPUT
|
||||
echo "enable_enterprise=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Add 'in_progress' reaction to comment
|
||||
id: add-eyes-reaction
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
github-token: ${{ steps.setup-bot.outputs.token }}
|
||||
script: |
|
||||
console.log(`Adding eyes reaction to comment ID: ${context.payload.comment.id}`);
|
||||
try {
|
||||
const { data: reaction } = await github.rest.reactions.createForIssueComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: context.payload.comment.id,
|
||||
content: 'eyes'
|
||||
});
|
||||
console.log(`Added reaction with ID: ${reaction.id}`);
|
||||
return { success: true, id: reaction.id };
|
||||
} catch (error) {
|
||||
console.error(`Failed to add reaction: ${error.message}`);
|
||||
console.error(error);
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
|
||||
deploy-pr:
|
||||
needs: check-comment
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout PR
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Setup GitHub App Bot
|
||||
if: github.actor != 'dependabot[bot]'
|
||||
id: setup-bot
|
||||
uses: ./.github/actions/setup-bot
|
||||
continue-on-error: true
|
||||
with:
|
||||
app-id: ${{ secrets.GH_APP_ID }}
|
||||
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
|
||||
|
||||
- name: Checkout PR
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
repository: ${{ needs.check-comment.outputs.pr_repository }}
|
||||
ref: ${{ needs.check-comment.outputs.pr_ref }}
|
||||
token: ${{ steps.setup-bot.outputs.token }}
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Run Gradle Command
|
||||
run: |
|
||||
if [ "${{ needs.check-comment.outputs.disable_security }}" == "true" ]; then
|
||||
export DISABLE_ADDITIONAL_FEATURES=true
|
||||
else
|
||||
export DISABLE_ADDITIONAL_FEATURES=false
|
||||
fi
|
||||
./gradlew clean build
|
||||
env:
|
||||
STIRLING_PDF_DESKTOP_UI: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Build and push PR-specific image
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
push: true
|
||||
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:pr-${{ needs.check-comment.outputs.pr_number }}
|
||||
build-args: VERSION_TAG=alpha
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Set up SSH
|
||||
run: |
|
||||
mkdir -p ~/.ssh/
|
||||
echo "${{ secrets.VPS_SSH_KEY }}" > ../private.key
|
||||
sudo chmod 600 ../private.key
|
||||
|
||||
- name: Deploy to VPS
|
||||
id: deploy
|
||||
run: |
|
||||
# Set security settings based on flags
|
||||
if [ "${{ needs.check-comment.outputs.disable_security }}" == "false" ]; then
|
||||
DISABLE_ADDITIONAL_FEATURES="false"
|
||||
LOGIN_SECURITY="true"
|
||||
SECURITY_STATUS="🔒 Security Enabled"
|
||||
else
|
||||
DISABLE_ADDITIONAL_FEATURES="true"
|
||||
LOGIN_SECURITY="false"
|
||||
SECURITY_STATUS="Security Disabled"
|
||||
fi
|
||||
|
||||
# Set pro/enterprise settings (enterprise implies pro)
|
||||
if [ "${{ needs.check-comment.outputs.enable_enterprise }}" == "true" ]; then
|
||||
PREMIUM_ENABLED="true"
|
||||
PREMIUM_KEY="${{ secrets.ENTERPRISE_KEY }}"
|
||||
PREMIUM_PROFEATURES_AUDIT_ENABLED="true"
|
||||
elif [ "${{ needs.check-comment.outputs.enable_pro }}" == "true" ]; then
|
||||
PREMIUM_ENABLED="true"
|
||||
PREMIUM_KEY="${{ secrets.PREMIUM_KEY }}"
|
||||
PREMIUM_PROFEATURES_AUDIT_ENABLED="true"
|
||||
else
|
||||
PREMIUM_ENABLED="false"
|
||||
PREMIUM_KEY=""
|
||||
PREMIUM_PROFEATURES_AUDIT_ENABLED="false"
|
||||
fi
|
||||
|
||||
# First create the docker-compose content locally
|
||||
cat > docker-compose.yml << EOF
|
||||
version: '3.3'
|
||||
services:
|
||||
stirling-pdf:
|
||||
container_name: stirling-pdf-pr-${{ needs.check-comment.outputs.pr_number }}
|
||||
image: ${{ secrets.DOCKER_HUB_USERNAME }}/test:pr-${{ needs.check-comment.outputs.pr_number }}
|
||||
ports:
|
||||
- "${{ needs.check-comment.outputs.pr_number }}:8080"
|
||||
volumes:
|
||||
- /stirling/PR-${{ needs.check-comment.outputs.pr_number }}/data:/usr/share/tessdata:rw
|
||||
- /stirling/PR-${{ needs.check-comment.outputs.pr_number }}/config:/configs:rw
|
||||
- /stirling/PR-${{ needs.check-comment.outputs.pr_number }}/logs:/logs:rw
|
||||
environment:
|
||||
DISABLE_ADDITIONAL_FEATURES: "${DISABLE_ADDITIONAL_FEATURES}"
|
||||
SECURITY_ENABLELOGIN: "${LOGIN_SECURITY}"
|
||||
SYSTEM_DEFAULTLOCALE: en-GB
|
||||
UI_APPNAME: "Stirling-PDF PR#${{ needs.check-comment.outputs.pr_number }}"
|
||||
UI_HOMEDESCRIPTION: "PR#${{ needs.check-comment.outputs.pr_number }} for Stirling-PDF Latest"
|
||||
UI_APPNAMENAVBAR: "PR#${{ needs.check-comment.outputs.pr_number }}"
|
||||
SYSTEM_MAXFILESIZE: "100"
|
||||
METRICS_ENABLED: "true"
|
||||
SYSTEM_GOOGLEVISIBILITY: "false"
|
||||
PREMIUM_KEY: "${PREMIUM_KEY}"
|
||||
PREMIUM_ENABLED: "${PREMIUM_ENABLED}"
|
||||
PREMIUM_PROFEATURES_AUDIT_ENABLED: "${PREMIUM_PROFEATURES_AUDIT_ENABLED}"
|
||||
restart: on-failure:5
|
||||
EOF
|
||||
|
||||
# Then copy the file and execute commands
|
||||
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null docker-compose.yml ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }}:/tmp/docker-compose.yml
|
||||
|
||||
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }} << ENDSSH
|
||||
# Create PR-specific directories
|
||||
mkdir -p /stirling/PR-${{ needs.check-comment.outputs.pr_number }}/{data,config,logs}
|
||||
|
||||
# Move docker-compose file to correct location
|
||||
mv /tmp/docker-compose.yml /stirling/PR-${{ needs.check-comment.outputs.pr_number }}/docker-compose.yml
|
||||
|
||||
# Start or restart the container
|
||||
cd /stirling/PR-${{ needs.check-comment.outputs.pr_number }}
|
||||
docker-compose pull
|
||||
docker-compose up -d
|
||||
ENDSSH
|
||||
|
||||
# Set output for use in PR comment
|
||||
echo "security_status=${SECURITY_STATUS}" >> $GITHUB_ENV
|
||||
|
||||
- name: Add success reaction to comment
|
||||
if: success()
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
github-token: ${{ steps.setup-bot.outputs.token }}
|
||||
script: |
|
||||
console.log(`Adding rocket reaction to comment ID: ${{ needs.check-comment.outputs.comment_id }}`);
|
||||
try {
|
||||
const { data: reaction } = await github.rest.reactions.createForIssueComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: ${{ needs.check-comment.outputs.comment_id }},
|
||||
content: 'rocket'
|
||||
});
|
||||
console.log(`Added rocket reaction with ID: ${reaction.id}`);
|
||||
} catch (error) {
|
||||
console.error(`Failed to add reaction: ${error.message}`);
|
||||
console.error(error);
|
||||
}
|
||||
|
||||
// add label to PR
|
||||
const prNumber = ${{ needs.check-comment.outputs.pr_number }};
|
||||
try {
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
labels: ['pr-deployed']
|
||||
});
|
||||
console.log(`Added 'pr-deployed' label to PR #${prNumber}`);
|
||||
} catch (error) {
|
||||
console.error(`Failed to add label to PR: ${error.message}`);
|
||||
console.error(error);
|
||||
}
|
||||
|
||||
- name: Add failure reaction to comment
|
||||
if: failure()
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
github-token: ${{ steps.setup-bot.outputs.token }}
|
||||
script: |
|
||||
console.log(`Adding -1 reaction to comment ID: ${{ needs.check-comment.outputs.comment_id }}`);
|
||||
try {
|
||||
const { data: reaction } = await github.rest.reactions.createForIssueComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: ${{ needs.check-comment.outputs.comment_id }},
|
||||
content: '-1'
|
||||
});
|
||||
console.log(`Added -1 reaction with ID: ${reaction.id}`);
|
||||
} catch (error) {
|
||||
console.error(`Failed to add reaction: ${error.message}`);
|
||||
console.error(error);
|
||||
}
|
||||
|
||||
- name: Post deployment URL to PR
|
||||
if: success()
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
github-token: ${{ steps.setup-bot.outputs.token }}
|
||||
script: |
|
||||
const { GITHUB_REPOSITORY } = process.env;
|
||||
const [repoOwner, repoName] = GITHUB_REPOSITORY.split('/');
|
||||
const prNumber = ${{ needs.check-comment.outputs.pr_number }};
|
||||
const securityStatus = process.env.security_status || "Security Disabled";
|
||||
|
||||
const deploymentUrl = `http://${{ secrets.VPS_HOST }}:${prNumber}`;
|
||||
const commentBody = `## 🚀 PR Test Deployment\n\n` +
|
||||
`Your PR has been deployed for testing!\n\n` +
|
||||
`🔗 **Test URL:** [${deploymentUrl}](${deploymentUrl})\n` +
|
||||
`${securityStatus}\n\n` +
|
||||
`This deployment will be automatically cleaned up when the PR is closed.\n\n`;
|
||||
|
||||
await github.rest.issues.createComment({
|
||||
owner: repoOwner,
|
||||
repo: repoName,
|
||||
issue_number: prNumber,
|
||||
body: commentBody
|
||||
});
|
||||
|
||||
- name: Cleanup temporary files
|
||||
if: always()
|
||||
run: |
|
||||
echo "Cleaning up temporary files..."
|
||||
rm -f ../private.key docker-compose.yml
|
||||
echo "Cleanup complete."
|
||||
continue-on-error: true
|
||||
@@ -1,141 +0,0 @@
|
||||
name: PR Deployment cleanup
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened, closed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
SERVER_IP: ${{ secrets.VPS_IP }} # Add this to your GitHub secrets
|
||||
CLEANUP_PERFORMED: "false" # Add flag to track if cleanup occurred
|
||||
|
||||
jobs:
|
||||
cleanup:
|
||||
if: github.event.action == 'closed'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout PR
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Setup GitHub App Bot
|
||||
if: github.actor != 'dependabot[bot]'
|
||||
id: setup-bot
|
||||
uses: ./.github/actions/setup-bot
|
||||
continue-on-error: true
|
||||
with:
|
||||
app-id: ${{ secrets.GH_APP_ID }}
|
||||
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
|
||||
|
||||
- name: Remove 'pr-deployed' label if present
|
||||
id: remove-label-comment
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
github-token: ${{ steps.setup-bot.outputs.token }}
|
||||
script: |
|
||||
const prNumber = ${{ github.event.pull_request.number }};
|
||||
const owner = context.repo.owner;
|
||||
const repo = context.repo.repo;
|
||||
|
||||
// Get all labels on the PR
|
||||
const { data: labels } = await github.rest.issues.listLabelsOnIssue({
|
||||
owner,
|
||||
repo,
|
||||
issue_number: prNumber
|
||||
});
|
||||
|
||||
const hasLabel = labels.some(label => label.name === 'pr-deployed');
|
||||
|
||||
if (hasLabel) {
|
||||
console.log("Label 'pr-deployed' found. Removing...");
|
||||
await github.rest.issues.removeLabel({
|
||||
owner,
|
||||
repo,
|
||||
issue_number: prNumber,
|
||||
name: 'pr-deployed'
|
||||
});
|
||||
} else {
|
||||
console.log("Label 'pr-deployed' not found. Nothing to do.");
|
||||
}
|
||||
|
||||
// Find existing bot comments about the deployment
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner,
|
||||
repo,
|
||||
issue_number: prNumber
|
||||
});
|
||||
const deploymentComments = comments.filter(c =>
|
||||
c.body?.includes("## 🚀 PR Test Deployment") &&
|
||||
c.user?.type === "Bot"
|
||||
);
|
||||
|
||||
if (deploymentComments.length > 0) {
|
||||
for (const comment of deploymentComments) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner,
|
||||
repo,
|
||||
comment_id: comment.id
|
||||
});
|
||||
console.log(`Deleted deployment comment (ID: ${comment.id})`);
|
||||
}
|
||||
} else {
|
||||
console.log("No matching deployment comments found.");
|
||||
}
|
||||
|
||||
// Set flag if either label or comment was present
|
||||
const hasDeploymentComment = deploymentComments.length > 0;
|
||||
core.setOutput('present', (hasLabel || hasDeploymentComment) ? 'true' : 'false');
|
||||
|
||||
|
||||
- name: Set up SSH
|
||||
if: steps.remove-label-comment.outputs.present == 'true'
|
||||
run: |
|
||||
mkdir -p ~/.ssh/
|
||||
echo "${{ secrets.VPS_SSH_KEY }}" > ../private.key
|
||||
sudo chmod 600 ../private.key
|
||||
|
||||
- name: Cleanup PR deployment
|
||||
if: steps.remove-label-comment.outputs.present == 'true'
|
||||
id: cleanup
|
||||
run: |
|
||||
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -T ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }} << 'ENDSSH'
|
||||
if [ -d "/stirling/PR-${{ github.event.pull_request.number }}" ]; then
|
||||
echo "Found PR directory, proceeding with cleanup..."
|
||||
|
||||
# Stop and remove containers
|
||||
cd /stirling/PR-${{ github.event.pull_request.number }}
|
||||
docker-compose down || true
|
||||
|
||||
# Go back to root before removal
|
||||
cd /
|
||||
|
||||
# Remove PR-specific directories
|
||||
rm -rf /stirling/PR-${{ github.event.pull_request.number }}
|
||||
|
||||
# Remove the Docker image
|
||||
docker rmi --no-prune ${{ secrets.DOCKER_HUB_USERNAME }}/test:pr-${{ github.event.pull_request.number }} || true
|
||||
|
||||
echo "PERFORMED_CLEANUP"
|
||||
else
|
||||
echo "PR directory not found, nothing to clean up"
|
||||
echo "NO_CLEANUP_NEEDED"
|
||||
fi
|
||||
ENDSSH
|
||||
|
||||
- name: Cleanup temporary files
|
||||
if: always()
|
||||
run: |
|
||||
echo "Cleaning up temporary files..."
|
||||
rm -f ../private.key
|
||||
echo "Cleanup complete."
|
||||
continue-on-error: true
|
||||
@@ -1,228 +0,0 @@
|
||||
name: AI - PR Title Review
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, edited]
|
||||
branches: [main]
|
||||
|
||||
permissions: # required for secure-repo hardening
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
ai-title-review:
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
models: read
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Configure Git to suppress detached HEAD warning
|
||||
run: git config --global advice.detachedHead false
|
||||
|
||||
- name: Setup GitHub App Bot
|
||||
if: github.actor != 'dependabot[bot]'
|
||||
id: setup-bot
|
||||
uses: ./.github/actions/setup-bot
|
||||
continue-on-error: true
|
||||
with:
|
||||
app-id: ${{ secrets.GH_APP_ID }}
|
||||
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
|
||||
|
||||
- name: Check if actor is repo developer
|
||||
id: actor
|
||||
run: |
|
||||
if [[ "${{ github.actor }}" == *"[bot]" ]]; then
|
||||
echo "PR opened by a bot – skipping AI title review."
|
||||
echo "is_repo_dev=false" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f .github/config/repo_devs.json ]; then
|
||||
echo "Error: .github/config/repo_devs.json not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Validate JSON and extract repo_devs
|
||||
REPO_DEVS=$(jq -r '.repo_devs[]' .github/config/repo_devs.json 2>/dev/null || { echo "Error: Invalid JSON in repo_devs.json" >&2; exit 1; })
|
||||
# Convert developer list into Bash array
|
||||
mapfile -t DEVS_ARRAY <<< "$REPO_DEVS"
|
||||
if [[ " ${DEVS_ARRAY[*]} " == *" ${{ github.actor }} "* ]]; then
|
||||
echo "is_repo_dev=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "is_repo_dev=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Get PR diff
|
||||
if: steps.actor.outputs.is_repo_dev == 'true'
|
||||
id: get_diff
|
||||
run: |
|
||||
git fetch origin ${{ github.base_ref }}
|
||||
git diff origin/${{ github.base_ref }}...HEAD | head -n 10000 | grep -vP '[\x00-\x08\x0B\x0C\x0E-\x1F\x7F\x{202E}\x{200B}]' > pr.diff
|
||||
echo "diff<<EOF" >> $GITHUB_OUTPUT
|
||||
cat pr.diff >> $GITHUB_OUTPUT
|
||||
echo "EOF" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Check and sanitize PR title
|
||||
if: steps.actor.outputs.is_repo_dev == 'true'
|
||||
id: sanitize_pr_title
|
||||
env:
|
||||
PR_TITLE_RAW: ${{ github.event.pull_request.title }}
|
||||
run: |
|
||||
# Sanitize PR title: max 72 characters, only printable characters
|
||||
PR_TITLE=$(echo "$PR_TITLE_RAW" | tr -d '\n\r' | head -c 72 | sed 's/[^[:print:]]//g')
|
||||
if [[ ${#PR_TITLE} -lt 5 ]]; then
|
||||
echo "PR title is too short. Must be at least 5 characters." >&2
|
||||
fi
|
||||
echo "pr_title=$PR_TITLE" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: AI PR Title Analysis
|
||||
if: steps.actor.outputs.is_repo_dev == 'true'
|
||||
id: ai-title-analysis
|
||||
uses: actions/ai-inference@9693b137b6566bb66055a713613bf4f0493701eb # v1.2.3
|
||||
with:
|
||||
model: openai/gpt-4o
|
||||
system-prompt-file: ".github/config/system-prompt.txt"
|
||||
prompt: |
|
||||
Based on the following input data:
|
||||
|
||||
{
|
||||
"diff": "${{ steps.get_diff.outputs.diff }}",
|
||||
"pr_title": "${{ steps.sanitize_pr_title.outputs.pr_title }}"
|
||||
}
|
||||
|
||||
Respond ONLY with valid JSON in the format:
|
||||
{
|
||||
"improved_rating": <0-10>,
|
||||
"improved_ai_title_rating": <0-10>,
|
||||
"improved_title": "<ai generated title>"
|
||||
}
|
||||
|
||||
- name: Validate and set SCRIPT_OUTPUT
|
||||
if: steps.actor.outputs.is_repo_dev == 'true'
|
||||
run: |
|
||||
cat <<EOF > ai_response.json
|
||||
${{ steps.ai-title-analysis.outputs.response }}
|
||||
EOF
|
||||
|
||||
# Validate JSON structure
|
||||
jq -e '
|
||||
(keys | sort) == ["improved_ai_title_rating", "improved_rating", "improved_title"] and
|
||||
(.improved_rating | type == "number" and . >= 0 and . <= 10) and
|
||||
(.improved_ai_title_rating | type == "number" and . >= 0 and . <= 10) and
|
||||
(.improved_title | type == "string")
|
||||
' ai_response.json
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Invalid AI response format" >&2
|
||||
cat ai_response.json >&2
|
||||
exit 1
|
||||
fi
|
||||
# Parse JSON fields
|
||||
IMPROVED_RATING=$(jq -r '.improved_rating' ai_response.json)
|
||||
IMPROVED_TITLE=$(jq -r '.improved_title' ai_response.json)
|
||||
# Limit comment length to 1000 characters
|
||||
COMMENT=$(cat <<EOF
|
||||
## 🤖 AI PR Title Suggestion
|
||||
|
||||
**PR-Title Rating**: $IMPROVED_RATING/10
|
||||
|
||||
### ⬇️ Suggested Title (copy & paste):
|
||||
|
||||
\`\`\`
|
||||
$IMPROVED_TITLE
|
||||
\`\`\`
|
||||
|
||||
---
|
||||
*Generated by GitHub Models AI*
|
||||
EOF
|
||||
)
|
||||
echo "$COMMENT" > /tmp/ai-title-comment.md
|
||||
# Log input and output to the GitHub Step Summary
|
||||
echo "### 🤖 AI PR Title Analysis" >> $GITHUB_STEP_SUMMARY
|
||||
echo "### Input PR Title" >> $GITHUB_STEP_SUMMARY
|
||||
echo '```bash' >> $GITHUB_STEP_SUMMARY
|
||||
echo "${{ steps.sanitize_pr_title.outputs.pr_title }}" >> $GITHUB_STEP_SUMMARY
|
||||
echo '```' >> $GITHUB_STEP_SUMMARY
|
||||
echo '### AI Response (raw JSON)' >> $GITHUB_STEP_SUMMARY
|
||||
echo '```json' >> $GITHUB_STEP_SUMMARY
|
||||
cat ai_response.json >> $GITHUB_STEP_SUMMARY
|
||||
echo '```' >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
- name: Post comment on PR if needed
|
||||
if: steps.actor.outputs.is_repo_dev == 'true'
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
continue-on-error: true
|
||||
with:
|
||||
github-token: ${{ steps.setup-bot.outputs.token }}
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const body = fs.readFileSync('/tmp/ai-title-comment.md', 'utf8');
|
||||
const { GITHUB_REPOSITORY } = process.env;
|
||||
const [owner, repo] = GITHUB_REPOSITORY.split('/');
|
||||
const issue_number = context.issue.number;
|
||||
|
||||
const ratingMatch = body.match(/\*\*PR-Title Rating\*\*: (\d+)\/10/);
|
||||
const rating = ratingMatch ? parseInt(ratingMatch[1], 10) : null;
|
||||
|
||||
const expectedActor = "${{ steps.setup-bot.outputs.app-slug }}[bot]";
|
||||
const comments = await github.rest.issues.listComments({ owner, repo, issue_number });
|
||||
|
||||
const existing = comments.data.find(c =>
|
||||
c.user?.login === expectedActor &&
|
||||
c.body.includes("## 🤖 AI PR Title Suggestion")
|
||||
);
|
||||
|
||||
if (rating === null) {
|
||||
console.log("No rating found in AI response – skipping.");
|
||||
return;
|
||||
}
|
||||
|
||||
if (rating <= 5) {
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner, repo,
|
||||
comment_id: existing.id,
|
||||
body
|
||||
});
|
||||
console.log("Updated existing suggestion comment.");
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner, repo, issue_number,
|
||||
body
|
||||
});
|
||||
console.log("Created new suggestion comment.");
|
||||
}
|
||||
} else {
|
||||
const praise = `## 🤖 AI PR Title Suggestion\n\nGreat job! The current PR title is clear and well-structured.\n\n✅ No suggestions needed.\n\n---\n*Generated by GitHub Models AI*`;
|
||||
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner, repo,
|
||||
comment_id: existing.id,
|
||||
body: praise
|
||||
});
|
||||
console.log("Replaced suggestion with praise.");
|
||||
} else {
|
||||
console.log("Rating > 5 and no existing comment – skipping comment.");
|
||||
}
|
||||
}
|
||||
|
||||
- name: is not repo dev
|
||||
if: steps.actor.outputs.is_repo_dev != 'true'
|
||||
run: |
|
||||
exit 0 # Skip the AI title review for non-repo developers
|
||||
|
||||
- name: Clean up
|
||||
if: always()
|
||||
run: |
|
||||
rm -f pr.diff ai_response.json /tmp/ai-title-comment.md
|
||||
echo "Cleaned up temporary files."
|
||||
continue-on-error: true # Ensure cleanup runs even if previous steps fail
|
||||
@@ -1,35 +0,0 @@
|
||||
name: "Auto Pull Request Labeler V2"
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
labeler:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Setup GitHub App Bot
|
||||
id: setup-bot
|
||||
uses: ./.github/actions/setup-bot
|
||||
with:
|
||||
app-id: ${{ secrets.GH_APP_ID }}
|
||||
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
|
||||
|
||||
- uses: srvaroa/labeler@0a20eccb8c94a1ee0bed5f16859aece1c45c3e55 # v1.13.0
|
||||
with:
|
||||
config_path: .github/labeler-config-srvaroa.yml
|
||||
use_local_config: false
|
||||
fail_on_error: true
|
||||
env:
|
||||
GITHUB_TOKEN: "${{ steps.setup-bot.outputs.token }}"
|
||||
@@ -1,312 +0,0 @@
|
||||
name: Build and Test Workflow
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
# push:
|
||||
# branches: ["main"]
|
||||
pull_request:
|
||||
branches: ["main"]
|
||||
|
||||
# cancel in-progress jobs if a new job is triggered
|
||||
# This is useful to avoid running multiple builds for the same branch if a new commit is pushed
|
||||
# or a pull request is updated.
|
||||
# It helps to save resources and time by ensuring that only the latest commit is built and tested
|
||||
# This is particularly useful for long-running jobs that may take a while to complete.
|
||||
# The `group` is set to a combination of the workflow name, event name, and branch name.
|
||||
# This ensures that jobs are grouped by the workflow and branch, allowing for cancellation of
|
||||
# in-progress jobs when a new commit is pushed to the same branch or a new pull request is opened.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref_name || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
files-changed:
|
||||
name: detect what files changed
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 3
|
||||
# Map a step output to a job output
|
||||
outputs:
|
||||
build: ${{ steps.changes.outputs.build }}
|
||||
app: ${{ steps.changes.outputs.app }}
|
||||
project: ${{ steps.changes.outputs.project }}
|
||||
openapi: ${{ steps.changes.outputs.openapi }}
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Check for file changes
|
||||
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
|
||||
id: changes
|
||||
with:
|
||||
filters: ".github/config/.files.yaml"
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
security-events: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
jdk-version: [17, 21]
|
||||
spring-security: [true, false]
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up JDK ${{ matrix.jdk-version }}
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: ${{ matrix.jdk-version }}
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
|
||||
with:
|
||||
gradle-version: 8.14
|
||||
|
||||
- name: Build with Gradle and spring security ${{ matrix.spring-security }}
|
||||
run: ./gradlew clean build
|
||||
env:
|
||||
DISABLE_ADDITIONAL_FEATURES: ${{ matrix.spring-security }}
|
||||
|
||||
- name: Check Test Reports Exist
|
||||
id: check-reports
|
||||
if: always()
|
||||
run: |
|
||||
declare -a dirs=(
|
||||
"app/core/build/reports/tests/"
|
||||
"app/core/build/test-results/"
|
||||
"app/common/build/reports/tests/"
|
||||
"app/common/build/test-results/"
|
||||
"app/proprietary/build/reports/tests/"
|
||||
"app/proprietary/build/test-results/"
|
||||
)
|
||||
missing_reports=()
|
||||
for dir in "${dirs[@]}"; do
|
||||
if [ ! -d "$dir" ]; then
|
||||
missing_reports+=("$dir")
|
||||
fi
|
||||
done
|
||||
if [ ${#missing_reports[@]} -gt 0 ]; then
|
||||
echo "ERROR: The following required test report directories are missing:"
|
||||
printf '%s\n' "${missing_reports[@]}"
|
||||
exit 1
|
||||
fi
|
||||
echo "All required test report directories are present"
|
||||
|
||||
- name: Upload Test Reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: test-reports-jdk-${{ matrix.jdk-version }}-spring-security-${{ matrix.spring-security }}
|
||||
path: |
|
||||
app/core/build/reports/tests/
|
||||
app/core/build/test-results/
|
||||
app/core/build/reports/problems/
|
||||
app/common/build/reports/tests/
|
||||
app/common/build/test-results/
|
||||
app/common/build/reports/problems/
|
||||
app/proprietary/build/reports/tests/
|
||||
app/proprietary/build/test-results/
|
||||
app/proprietary/build/reports/problems/
|
||||
build/reports/problems/
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
check-generateOpenApiDocs:
|
||||
if: needs.files-changed.outputs.openapi == 'true'
|
||||
needs: [files-changed, build]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
|
||||
|
||||
- name: Generate OpenAPI documentation
|
||||
run: ./gradlew :stirling-pdf:generateOpenApiDocs
|
||||
|
||||
- name: Upload OpenAPI Documentation
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: openapi-docs
|
||||
path: ./SwaggerDoc.json
|
||||
|
||||
check-licence:
|
||||
if: needs.files-changed.outputs.build == 'true'
|
||||
needs: [files-changed, build]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: check the licenses for compatibility
|
||||
run: ./gradlew clean checkLicense
|
||||
|
||||
- name: FAILED - check the licenses for compatibility
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: dependencies-without-allowed-license.json
|
||||
path: |
|
||||
build/reports/dependency-license/dependencies-without-allowed-license.json
|
||||
retention-days: 3
|
||||
|
||||
docker-compose-tests:
|
||||
if: needs.files-changed.outputs.project == 'true'
|
||||
needs: files-changed
|
||||
# if: github.event_name == 'push' && github.ref == 'refs/heads/main' ||
|
||||
# (github.event_name == 'pull_request' &&
|
||||
# contains(github.event.pull_request.labels.*.name, 'licenses') == false &&
|
||||
# (
|
||||
# contains(github.event.pull_request.labels.*.name, 'Front End') ||
|
||||
# contains(github.event.pull_request.labels.*.name, 'Java') ||
|
||||
# contains(github.event.pull_request.labels.*.name, 'Back End') ||
|
||||
# contains(github.event.pull_request.labels.*.name, 'Security') ||
|
||||
# contains(github.event.pull_request.labels.*.name, 'API') ||
|
||||
# contains(github.event.pull_request.labels.*.name, 'Docker') ||
|
||||
# contains(github.event.pull_request.labels.*.name, 'Test')
|
||||
# )
|
||||
# )
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up Java 17
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Install Docker Compose
|
||||
run: |
|
||||
sudo curl -SL "https://github.com/docker/compose/releases/download/v2.37.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
|
||||
sudo chmod +x /usr/local/bin/docker-compose
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: 'pip' # caching pip dependencies
|
||||
cache-dependency-path: ./testing/cucumber/requirements.txt
|
||||
|
||||
- name: Pip requirements
|
||||
run: |
|
||||
pip install --require-hashes -r ./testing/cucumber/requirements.txt
|
||||
|
||||
- name: Run Docker Compose Tests
|
||||
run: |
|
||||
chmod +x ./testing/test_webpages.sh
|
||||
chmod +x ./testing/test.sh
|
||||
chmod +x ./testing/test_disabledEndpoints.sh
|
||||
./testing/test.sh
|
||||
|
||||
test-build-docker-images:
|
||||
if: github.event_name == 'pull_request' && needs.files-changed.outputs.project == 'true'
|
||||
needs: [files-changed, build, check-generateOpenApiDocs, check-licence]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
docker-rev: ["Dockerfile", "Dockerfile.ultra-lite", "Dockerfile.fat"]
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Set up Gradle
|
||||
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
|
||||
with:
|
||||
gradle-version: 8.14
|
||||
|
||||
- name: Build application
|
||||
run: ./gradlew clean build
|
||||
env:
|
||||
DISABLE_ADDITIONAL_FEATURES: true
|
||||
STIRLING_PDF_DESKTOP_UI: false
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Build ${{ matrix.docker-rev }}
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
file: ./${{ matrix.docker-rev }}
|
||||
push: false
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
platforms: linux/amd64,linux/arm64/v8
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
- name: Upload Reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: reports-docker-${{ matrix.docker-rev }}
|
||||
path: |
|
||||
build/reports/tests/
|
||||
build/test-results/
|
||||
build/reports/problems/
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
@@ -1,292 +0,0 @@
|
||||
name: Check Properties Files on PR
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- "app/core/src/main/resources/messages_*.properties"
|
||||
|
||||
# cancel in-progress jobs if a new job is triggered
|
||||
# This is useful to avoid running multiple builds for the same branch if a new commit is pushed
|
||||
# or a pull request is updated.
|
||||
# It helps to save resources and time by ensuring that only the latest commit is built and tested
|
||||
# This is particularly useful for long-running jobs that may take a while to complete.
|
||||
# The `group` is set to a combination of the workflow name, event name, and branch name.
|
||||
# This ensures that jobs are grouped by the workflow and branch, allowing for cancellation of
|
||||
# in-progress jobs when a new commit is pushed to the same branch or a new pull request is opened.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref_name || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read # Allow read access to repository content
|
||||
|
||||
jobs:
|
||||
check-files:
|
||||
if: github.event_name == 'pull_request_target'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write # Allow posting comments on issues/PRs
|
||||
pull-requests: write # Allow writing to pull requests
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout main branch first
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Setup GitHub App Bot
|
||||
id: setup-bot
|
||||
uses: ./.github/actions/setup-bot
|
||||
with:
|
||||
app-id: ${{ secrets.GH_APP_ID }}
|
||||
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
|
||||
|
||||
- name: Get PR data
|
||||
id: get-pr-data
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
github-token: ${{ steps.setup-bot.outputs.token }}
|
||||
script: |
|
||||
const prNumber = context.payload.pull_request.number;
|
||||
const repoOwner = context.payload.repository.owner.login;
|
||||
const repoName = context.payload.repository.name;
|
||||
const branch = context.payload.pull_request.head.ref;
|
||||
|
||||
console.log(`PR Number: ${prNumber}`);
|
||||
console.log(`Repo Owner: ${repoOwner}`);
|
||||
console.log(`Repo Name: ${repoName}`);
|
||||
console.log(`Branch: ${branch}`);
|
||||
|
||||
core.setOutput("pr_number", prNumber);
|
||||
core.setOutput("repo_owner", repoOwner);
|
||||
core.setOutput("repo_name", repoName);
|
||||
core.setOutput("branch", branch);
|
||||
continue-on-error: true
|
||||
|
||||
- name: Fetch PR changed files
|
||||
id: fetch-pr-changes
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.setup-bot.outputs.token }}
|
||||
run: |
|
||||
echo "Fetching PR changed files..."
|
||||
echo "Getting list of changed files from PR..."
|
||||
# Check if PR number exists
|
||||
if [ -z "${{ steps.get-pr-data.outputs.pr_number }}" ]; then
|
||||
echo "Error: PR number is empty"
|
||||
exit 1
|
||||
fi
|
||||
# Get changed files and filter for properties files, handle case where no matches are found
|
||||
gh pr view ${{ steps.get-pr-data.outputs.pr_number }} --json files -q ".files[].path" | grep -E '^app/core/src/main/resources/messages_[a-zA-Z_]{2}_[a-zA-Z_]{2,7}\.properties$' > changed_files.txt || echo "No matching properties files found in PR"
|
||||
# Check if any files were found
|
||||
if [ ! -s changed_files.txt ]; then
|
||||
echo "No properties files changed in this PR"
|
||||
echo "Workflow will exit early as no relevant files to check"
|
||||
exit 0
|
||||
fi
|
||||
echo "Found $(wc -l < changed_files.txt) matching properties files"
|
||||
|
||||
- name: Determine reference file test
|
||||
id: determine-file
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
github-token: ${{ steps.setup-bot.outputs.token }}
|
||||
script: |
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
|
||||
const prNumber = ${{ steps.get-pr-data.outputs.pr_number }};
|
||||
const repoOwner = "${{ steps.get-pr-data.outputs.repo_owner }}";
|
||||
const repoName = "${{ steps.get-pr-data.outputs.repo_name }}";
|
||||
|
||||
const prRepoOwner = "${{ github.event.pull_request.head.repo.owner.login }}";
|
||||
const prRepoName = "${{ github.event.pull_request.head.repo.name }}";
|
||||
const branch = "${{ steps.get-pr-data.outputs.branch }}";
|
||||
|
||||
console.log(`Determining reference file for PR #${prNumber}`);
|
||||
|
||||
// Validate inputs
|
||||
const validateInput = (input, regex, name) => {
|
||||
if (!regex.test(input)) {
|
||||
throw new Error(`Invalid ${name}: ${input}`);
|
||||
}
|
||||
};
|
||||
|
||||
validateInput(repoOwner, /^[a-zA-Z0-9_-]+$/, "repository owner");
|
||||
validateInput(repoName, /^[a-zA-Z0-9._-]+$/, "repository name");
|
||||
validateInput(branch, /^[a-zA-Z0-9._/-]+$/, "branch name");
|
||||
|
||||
// Get the list of changed files in the PR
|
||||
const { data: files } = await github.rest.pulls.listFiles({
|
||||
owner: repoOwner,
|
||||
repo: repoName,
|
||||
pull_number: prNumber,
|
||||
});
|
||||
|
||||
// Filter for relevant files based on the PR changes
|
||||
const changedFiles = files
|
||||
.filter(file =>
|
||||
file.status !== "removed" &&
|
||||
/^app\/core\/src\/main\/resources\/messages_[a-zA-Z_]{2}_[a-zA-Z_]{2,7}\.properties$/.test(file.filename)
|
||||
)
|
||||
.map(file => file.filename);
|
||||
|
||||
console.log("Changed files:", changedFiles);
|
||||
|
||||
// Create a temporary directory for PR files
|
||||
const tempDir = "pr-branch";
|
||||
if (!fs.existsSync(tempDir)) {
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Download and save each changed file
|
||||
for (const file of changedFiles) {
|
||||
const { data: fileContent } = await github.rest.repos.getContent({
|
||||
owner: prRepoOwner,
|
||||
repo: prRepoName,
|
||||
path: file,
|
||||
ref: branch,
|
||||
});
|
||||
|
||||
const content = Buffer.from(fileContent.content, "base64").toString("utf-8");
|
||||
const filePath = path.join(tempDir, file);
|
||||
const dirPath = path.dirname(filePath);
|
||||
|
||||
if (!fs.existsSync(dirPath)) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
|
||||
fs.writeFileSync(filePath, content);
|
||||
console.log(`Saved file: ${filePath}`);
|
||||
}
|
||||
|
||||
// Output the list of changed files for further processing
|
||||
const fileList = changedFiles.join(" ");
|
||||
core.exportVariable("FILES_LIST", fileList);
|
||||
console.log("Files saved and listed in FILES_LIST.");
|
||||
|
||||
// Determine reference file
|
||||
let referenceFilePath;
|
||||
if (changedFiles.includes("app/core/src/main/resources/messages_en_GB.properties")) {
|
||||
console.log("Using PR branch reference file.");
|
||||
const { data: fileContent } = await github.rest.repos.getContent({
|
||||
owner: prRepoOwner,
|
||||
repo: prRepoName,
|
||||
path: "app/core/src/main/resources/messages_en_GB.properties",
|
||||
ref: branch,
|
||||
});
|
||||
|
||||
referenceFilePath = "pr-branch-messages_en_GB.properties";
|
||||
const content = Buffer.from(fileContent.content, "base64").toString("utf-8");
|
||||
fs.writeFileSync(referenceFilePath, content);
|
||||
} else {
|
||||
console.log("Using main branch reference file.");
|
||||
const { data: fileContent } = await github.rest.repos.getContent({
|
||||
owner: repoOwner,
|
||||
repo: repoName,
|
||||
path: "app/core/src/main/resources/messages_en_GB.properties",
|
||||
ref: "main",
|
||||
});
|
||||
|
||||
referenceFilePath = "main-branch-messages_en_GB.properties";
|
||||
const content = Buffer.from(fileContent.content, "base64").toString("utf-8");
|
||||
fs.writeFileSync(referenceFilePath, content);
|
||||
}
|
||||
|
||||
console.log(`Reference file path: ${referenceFilePath}`);
|
||||
core.exportVariable("REFERENCE_FILE", referenceFilePath);
|
||||
|
||||
- name: Run Python script to check files
|
||||
id: run-check
|
||||
run: |
|
||||
echo "Running Python script to check files..."
|
||||
python .github/scripts/check_language_properties.py \
|
||||
--actor ${{ github.event.pull_request.user.login }} \
|
||||
--reference-file "${REFERENCE_FILE}" \
|
||||
--branch "pr-branch" \
|
||||
--files "${FILES_LIST[@]}" > result.txt
|
||||
continue-on-error: true # Continue the job even if this step fails
|
||||
|
||||
- name: Capture output
|
||||
id: capture-output
|
||||
run: |
|
||||
if [ -f result.txt ] && [ -s result.txt ]; then
|
||||
echo "Test, capturing output..."
|
||||
SCRIPT_OUTPUT=$(cat result.txt)
|
||||
echo "SCRIPT_OUTPUT<<EOF" >> $GITHUB_ENV
|
||||
echo "$SCRIPT_OUTPUT" >> $GITHUB_ENV
|
||||
echo "EOF" >> $GITHUB_ENV
|
||||
echo "${SCRIPT_OUTPUT}"
|
||||
|
||||
# Determine job failure based on script output
|
||||
if [[ "$SCRIPT_OUTPUT" == *"❌"* ]]; then
|
||||
echo "FAIL_JOB=true" >> $GITHUB_ENV
|
||||
else
|
||||
echo "FAIL_JOB=false" >> $GITHUB_ENV
|
||||
fi
|
||||
else
|
||||
echo "No update found."
|
||||
echo "SCRIPT_OUTPUT=" >> $GITHUB_ENV
|
||||
echo "FAIL_JOB=false" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Post comment on PR
|
||||
if: env.SCRIPT_OUTPUT != ''
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
with:
|
||||
github-token: ${{ steps.setup-bot.outputs.token }}
|
||||
script: |
|
||||
const { GITHUB_REPOSITORY, SCRIPT_OUTPUT } = process.env;
|
||||
const [repoOwner, repoName] = GITHUB_REPOSITORY.split('/');
|
||||
const issueNumber = context.issue.number;
|
||||
|
||||
// Find existing comment
|
||||
const comments = await github.rest.issues.listComments({
|
||||
owner: repoOwner,
|
||||
repo: repoName,
|
||||
issue_number: issueNumber
|
||||
});
|
||||
|
||||
const comment = comments.data.find(c => c.body.includes("## 🚀 Translation Verification Summary"));
|
||||
|
||||
// Only update or create comments by the action user
|
||||
const expectedActor = "${{ steps.setup-bot.outputs.app-slug }}[bot]";
|
||||
|
||||
if (comment && comment.user.login === expectedActor) {
|
||||
// Update existing comment
|
||||
await github.rest.issues.updateComment({
|
||||
owner: repoOwner,
|
||||
repo: repoName,
|
||||
comment_id: comment.id,
|
||||
body: `## 🚀 Translation Verification Summary\n\n\n${SCRIPT_OUTPUT}\n`
|
||||
});
|
||||
console.log("Updated existing comment.");
|
||||
} else if (!comment) {
|
||||
// Create new comment if no existing comment is found
|
||||
await github.rest.issues.createComment({
|
||||
owner: repoOwner,
|
||||
repo: repoName,
|
||||
issue_number: issueNumber,
|
||||
body: `## 🚀 Translation Verification Summary\n\n\n${SCRIPT_OUTPUT}\n`
|
||||
});
|
||||
console.log("Created new comment.");
|
||||
} else {
|
||||
console.log("Comment update attempt denied. Actor does not match.");
|
||||
}
|
||||
|
||||
- name: Fail job if errors found
|
||||
if: env.FAIL_JOB == 'true'
|
||||
run: |
|
||||
echo "Failing the job because errors were detected."
|
||||
exit 1
|
||||
|
||||
- name: Cleanup temporary files
|
||||
if: always()
|
||||
run: |
|
||||
echo "Cleaning up temporary files..."
|
||||
rm -rf pr-branch
|
||||
rm -f pr-branch-messages_en_GB.properties main-branch-messages_en_GB.properties changed_files.txt result.txt
|
||||
echo "Cleanup complete."
|
||||
continue-on-error: true # Ensure cleanup runs even if previous steps fail
|
||||
@@ -1,79 +0,0 @@
|
||||
# For most projects, this workflow file will not need changing; you simply need
|
||||
# to commit it to your repository.
|
||||
#
|
||||
# You may wish to alter this file to override the set of languages analyzed,
|
||||
# or to provide custom queries or build logic.
|
||||
#
|
||||
# ******** NOTE ********
|
||||
# We have attempted to detect the languages in your repository. Please check
|
||||
# the `language` matrix defined below to confirm you have the correct set of
|
||||
# supported CodeQL languages.
|
||||
#
|
||||
name: "CodeQL"
|
||||
|
||||
#disable for now
|
||||
#on:
|
||||
# push:
|
||||
# branches: ["main"]
|
||||
# pull_request:
|
||||
# The branches below must be a subset of the branches above
|
||||
# branches: ["main"]
|
||||
# schedule:
|
||||
# - cron: "0 0 * * 1"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Analyze
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
language: ["java"]
|
||||
# CodeQL supports [ $supported-codeql-languages ]
|
||||
# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@c95a14d0e5bab51a9f56296a4eb0e416910cd350 # v2.10.3
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
# By default, queries listed here will override any specified in a config file.
|
||||
# Prefix the list here with "+" to use these queries and those in the config file.
|
||||
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
|
||||
# If the Autobuild fails above, remove it and uncomment the following three lines.
|
||||
# modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
|
||||
|
||||
# - run: |
|
||||
# echo "Run, Build Application using script"
|
||||
# ./location_of_script_within_repo/buildscript.sh
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@48ab28a6f5dbc2a99bf1e0131198dd8f1df78169 # v3.28.0
|
||||
with:
|
||||
category: "/language:${{matrix.language}}"
|
||||
@@ -1,27 +0,0 @@
|
||||
# Dependency Review Action
|
||||
#
|
||||
# This Action will scan dependency manifest files that change as part of a Pull Request,
|
||||
# surfacing known-vulnerable versions of the packages declared or updated in the PR.
|
||||
# Once installed, if the workflow run is marked as required,
|
||||
# PRs introducing known-vulnerable packages will be blocked from merging.
|
||||
#
|
||||
# Source repository: https://github.com/actions/dependency-review-action
|
||||
name: "Dependency Review"
|
||||
on: [pull_request]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
dependency-review:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: "Checkout Repository"
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
- name: "Dependency Review"
|
||||
uses: actions/dependency-review-action@da24556b548a50705dd671f47852072ea4c105d9 # v4.7.1
|
||||
@@ -1,102 +0,0 @@
|
||||
name: License Report Workflow
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "build.gradle"
|
||||
|
||||
# cancel in-progress jobs if a new job is triggered
|
||||
# This is useful to avoid running multiple builds for the same branch if a new commit is pushed
|
||||
# or a pull request is updated.
|
||||
# It helps to save resources and time by ensuring that only the latest commit is built and tested
|
||||
# This is particularly useful for long-running jobs that may take a while to complete.
|
||||
# The `group` is set to a combination of the workflow name, event name, and branch name.
|
||||
# This ensures that jobs are grouped by the workflow and branch, allowing for cancellation of
|
||||
# in-progress jobs when a new commit is pushed to the same branch or a new pull request is opened.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref_name || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
generate-license-report:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
repository-projects: write # Required for enabling automerge
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Check out code
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup GitHub App Bot
|
||||
id: setup-bot
|
||||
uses: ./.github/actions/setup-bot
|
||||
with:
|
||||
app-id: ${{ secrets.GH_APP_ID }}
|
||||
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
|
||||
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
|
||||
|
||||
- name: Check licenses for compatibility
|
||||
run: ./gradlew clean checkLicense
|
||||
|
||||
- name: Upload artifact on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: dependencies-without-allowed-license.json
|
||||
path: build/reports/dependency-license/dependencies-without-allowed-license.json
|
||||
retention-days: 3
|
||||
|
||||
- name: Move and rename license file
|
||||
run: |
|
||||
mv build/reports/dependency-license/index.json app/core/src/main/resources/static/3rdPartyLicenses.json
|
||||
|
||||
- name: Commit changes
|
||||
run: |
|
||||
git add app/core/src/main/resources/static/3rdPartyLicenses.json
|
||||
git diff --staged --quiet || echo "CHANGES_DETECTED=true" >> $GITHUB_ENV
|
||||
|
||||
- name: Create Pull Request
|
||||
id: cpr
|
||||
if: env.CHANGES_DETECTED == 'true'
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
token: ${{ steps.setup-bot.outputs.token }}
|
||||
commit-message: "Update 3rd Party Licenses"
|
||||
committer: ${{ steps.setup-bot.outputs.committer }}
|
||||
author: ${{ steps.setup-bot.outputs.committer }}
|
||||
signoff: true
|
||||
branch: update-3rd-party-licenses
|
||||
title: "Update 3rd Party Licenses"
|
||||
body: |
|
||||
Auto-generated by ${{ steps.setup-bot.outputs.app-slug }}[bot]
|
||||
labels: Licenses,github-actions
|
||||
draft: false
|
||||
delete-branch: true
|
||||
sign-commits: true
|
||||
|
||||
- name: Enable Pull Request Automerge
|
||||
if: steps.cpr.outputs.pull-request-operation == 'created'
|
||||
run: gh pr merge --squash --auto "${{ steps.cpr.outputs.pull-request-number }}"
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.setup-bot.outputs.token }}
|
||||
@@ -1,30 +0,0 @@
|
||||
name: Manage labels
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "30 20 * * *"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
labeler:
|
||||
name: Labeler
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Check out the repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Run Labeler
|
||||
uses: crazy-max/ghaction-github-labeler@24d110aa46a59976b8a7f35518cb7f14f434c916 # v5.3.0
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
yaml-file: .github/labels.yml
|
||||
skip-delete: true
|
||||
@@ -1,318 +0,0 @@
|
||||
name: Test Installers Build
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
test_mode:
|
||||
description: "Run in test mode (skip release step)"
|
||||
required: false
|
||||
default: "false"
|
||||
release:
|
||||
types: [created]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
read_versions:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.versionNumber.outputs.versionNumber }}
|
||||
versionMac: ${{ steps.versionNumberMac.outputs.versionNumberMac }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
distribution: 'temurin'
|
||||
java-version: '21'
|
||||
|
||||
# ✅ Get version from Gradle
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
run: |
|
||||
VERSION=$(./gradlew printVersion --quiet | tail -1)
|
||||
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
# ✅ Get Mac-specific version from Gradle
|
||||
- name: Get version number mac
|
||||
id: versionNumberMac
|
||||
run: |
|
||||
VERSION_MAC=$(./gradlew printMacVersion --quiet | tail -1)
|
||||
echo "versionNumberMac=$VERSION_MAC" >> $GITHUB_OUTPUT
|
||||
|
||||
build-portable:
|
||||
needs: read_versions
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
disable_security: [true, false]
|
||||
include:
|
||||
- disable_security: false
|
||||
file_suffix: "-with-login"
|
||||
- disable_security: true
|
||||
file_suffix: ""
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: "21"
|
||||
distribution: "temurin"
|
||||
|
||||
- uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
|
||||
with:
|
||||
gradle-version: 8.14
|
||||
|
||||
- name: Generate jar (Disable Security=${{ matrix.disable_security }})
|
||||
run: ./gradlew clean createExe
|
||||
env:
|
||||
DISABLE_ADDITIONAL_FEATURES: ${{ matrix.disable_security }}
|
||||
STIRLING_PDF_DESKTOP_UI: false
|
||||
|
||||
- name: Rename binaries
|
||||
run: |
|
||||
mkdir ./binaries
|
||||
mv ./build/launch4j/Stirling-PDF.exe ./binaries/win-Stirling-PDF-portable-Server${{ matrix.file_suffix }}.exe
|
||||
mv ./app/core/build/libs/stirling-pdf-${{ needs.read_versions.outputs.version }}.jar ./binaries/Stirling-PDF${{ matrix.file_suffix }}.jar
|
||||
|
||||
- name: Upload build artifacts
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
retention-days: 1
|
||||
if-no-files-found: error
|
||||
name: stirling${{ matrix.file_suffix }}-binaries
|
||||
path: |
|
||||
./binaries/*
|
||||
|
||||
sign_verify-portable:
|
||||
needs: [build-portable, read_versions]
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
disable_security: [true, false]
|
||||
include:
|
||||
- disable_security: false
|
||||
file_suffix: "with-login-"
|
||||
- disable_security: true
|
||||
file_suffix: ""
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Download build artifacts
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
with:
|
||||
name: stirling-${{ matrix.file_suffix }}binaries
|
||||
|
||||
- name: Display structure of downloaded files
|
||||
run: ls -R
|
||||
|
||||
- name: Upload signed artifacts
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
retention-days: 1
|
||||
if-no-files-found: error
|
||||
name: stirling-${{ matrix.file_suffix }}signed
|
||||
path: |
|
||||
./*
|
||||
!cosign.*
|
||||
|
||||
build-installers:
|
||||
needs: read_versions
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: windows-latest
|
||||
platform: win-
|
||||
- os: macos-latest
|
||||
platform: mac-
|
||||
# - os: ubuntu-latest
|
||||
# platform: linux-
|
||||
runs-on: ${{ matrix.os }}
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: "21"
|
||||
distribution: "temurin"
|
||||
|
||||
- uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
|
||||
with:
|
||||
gradle-version: 8.14
|
||||
|
||||
# Install Windows dependencies
|
||||
- name: Install WiX Toolset
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: |
|
||||
curl -L -o wix.exe https://github.com/wixtoolset/wix3/releases/download/wix3141rtm/wix314.exe
|
||||
.\wix.exe /install /quiet
|
||||
|
||||
# Build installer
|
||||
- name: Build Installer
|
||||
run: ./gradlew build jpackage -x test --info
|
||||
env:
|
||||
DISABLE_ADDITIONAL_FEATURES: true
|
||||
STIRLING_PDF_DESKTOP_UI: true
|
||||
BROWSER_OPEN: true
|
||||
|
||||
- name: Set up JDK (x86_64)
|
||||
if: matrix.os == 'macos-latest'
|
||||
run: |
|
||||
curl -L -o jdk.tar.gz https://cdn.azul.com/zulu/bin/zulu17.56.15-ca-jdk17.0.14-macosx_x64.tar.gz
|
||||
mkdir -p zulu17
|
||||
tar -xzf jdk.tar.gz -C zulu17 --strip-components=1
|
||||
echo "JAVA_HOME=$PWD/zulu17" >> $GITHUB_ENV
|
||||
echo "$PWD/zulu17/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Verify JDK architecture
|
||||
if: matrix.os == 'macos-latest'
|
||||
run: file $JAVA_HOME/bin/java
|
||||
|
||||
- name: Build project and run jpackage (x86_64)
|
||||
if: matrix.os == 'macos-latest'
|
||||
run: arch -x86_64 ./gradlew jpackageMacX64
|
||||
|
||||
# Rename and collect artifacts based on OS
|
||||
- name: Prepare artifacts
|
||||
id: prepare
|
||||
shell: bash
|
||||
run: |
|
||||
ls -lah ./build/jpackage/
|
||||
mkdir ./binaries
|
||||
if [ "${{ matrix.os }}" = "windows-latest" ]; then
|
||||
mv "./build/jpackage/Stirling PDF-${{ needs.read_versions.outputs.version }}.exe" "./binaries/Stirling-PDF-win-installer.exe"
|
||||
elif [ "${{ matrix.os }}" = "macos-latest" ]; then
|
||||
mv "./build/jpackage/Stirling PDF-${{ needs.read_versions.outputs.versionMac }}.dmg" "./binaries/Stirling-PDF-mac-installer.dmg"
|
||||
mv "./build/jpackage/x86_64/Stirling PDF (x86_64)-${{ needs.read_versions.outputs.versionMac }}.dmg" "./binaries/Stirling-PDF-mac-x86_64-installer.dmg"
|
||||
else
|
||||
mv "./build/jpackage/stirling-pdf_${{ needs.read_versions.outputs.version }}-1_amd64.deb" "./binaries/Stirling-PDF-linux-installer.deb"
|
||||
fi
|
||||
|
||||
- name: Display structure of downloaded files
|
||||
run: ls -R ./binaries
|
||||
|
||||
- name: Upload build artifacts
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
retention-days: 1
|
||||
if-no-files-found: error
|
||||
name: ${{ matrix.platform }}binaries
|
||||
path: |
|
||||
./binaries/*
|
||||
|
||||
sign_verify:
|
||||
needs: [read_versions, build-installers]
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- os: windows-latest
|
||||
platform: win-
|
||||
- os: macos-latest
|
||||
platform: mac-
|
||||
# - os: ubuntu-latest
|
||||
# platform: linux-
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Download build artifacts
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
with:
|
||||
name: ${{ matrix.platform }}binaries
|
||||
|
||||
- name: Display structure of downloaded files
|
||||
run: ls -R
|
||||
|
||||
- name: Install Cosign
|
||||
if: matrix.os == 'windows-latest'
|
||||
uses: sigstore/cosign-installer@d58896d6a1865668819e1d91763c7751a165e159 # v3.9.2
|
||||
|
||||
- name: Generate key pair
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: cosign generate-key-pair
|
||||
|
||||
- name: Sign and generate attestations
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: |
|
||||
cosign sign-blob \
|
||||
--key ./cosign.key \
|
||||
--yes \
|
||||
--output-signature ./Stirling-PDF-win-installer.exe.sig \
|
||||
./Stirling-PDF-win-installer.exe
|
||||
|
||||
cosign attest-blob \
|
||||
--predicate - \
|
||||
--key ./cosign.key \
|
||||
--yes \
|
||||
--output-attestation ./Stirling-PDF-win-installer.exe.intoto.jsonl \
|
||||
./Stirling-PDF-win-installer.exe
|
||||
|
||||
cosign verify-blob \
|
||||
--key ./cosign.pub \
|
||||
--signature ./Stirling-PDF-win-installer.exe.sig \
|
||||
./Stirling-PDF-win-installer.exe
|
||||
|
||||
- name: Display structure of downloaded files
|
||||
run: ls -R
|
||||
|
||||
- name: Upload signed artifacts
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
retention-days: 1
|
||||
if-no-files-found: error
|
||||
name: ${{ matrix.platform }}signed
|
||||
path: |
|
||||
./Stirling-PDF-${{ matrix.platform }}installer.*
|
||||
./Stirling-PDF-${{ matrix.platform }}x86_64-installer.*
|
||||
!cosign.*
|
||||
|
||||
create-release:
|
||||
if: github.event_name != 'workflow_dispatch' || github.event.inputs.test_mode != 'true'
|
||||
needs: [read_versions, sign_verify, sign_verify-portable]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Download signed artifacts
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
- name: Display structure of downloaded files
|
||||
run: ls -R
|
||||
- name: Upload binaries, attestations and signatures to Release and create GitHub Release
|
||||
uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2
|
||||
with:
|
||||
tag_name: v${{ needs.read_versions.outputs.version }}
|
||||
generate_release_notes: true
|
||||
files: |
|
||||
./*signed/*
|
||||
@@ -1,82 +0,0 @@
|
||||
name: Pre-commit
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup GitHub App Bot
|
||||
id: setup-bot
|
||||
uses: ./.github/actions/setup-bot
|
||||
with:
|
||||
app-id: ${{ secrets.GH_APP_ID }}
|
||||
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: 3.12
|
||||
cache: 'pip' # caching pip dependencies
|
||||
cache-dependency-path: ./.github/scripts/requirements_pre_commit.txt
|
||||
|
||||
- name: Run Pre-Commit Hooks
|
||||
run: |
|
||||
pip install --require-hashes -r ./.github/scripts/requirements_pre_commit.txt
|
||||
|
||||
- run: pre-commit run --all-files -c .pre-commit-config.yaml
|
||||
continue-on-error: true
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: 17
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Build with Gradle
|
||||
run: ./gradlew clean build
|
||||
|
||||
- name: git add
|
||||
run: |
|
||||
git add .
|
||||
git diff --staged --quiet || echo "CHANGES_DETECTED=true" >> $GITHUB_ENV
|
||||
|
||||
- name: Create Pull Request
|
||||
if: env.CHANGES_DETECTED == 'true'
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
token: ${{ steps.setup-bot.outputs.token }}
|
||||
commit-message: ":file_folder: pre-commit"
|
||||
committer: ${{ steps.setup-bot.outputs.committer }}
|
||||
author: ${{ steps.setup-bot.outputs.committer }}
|
||||
signoff: true
|
||||
branch: pre-commit
|
||||
title: "🤖 format everything with pre-commit by ${{ steps.setup-bot.outputs.app-slug }}"
|
||||
body: |
|
||||
Auto-generated by [create-pull-request][1] with **${{ steps.setup-bot.outputs.app-slug }}**
|
||||
|
||||
[1]: https://github.com/peter-evans/create-pull-request
|
||||
draft: false
|
||||
delete-branch: true
|
||||
labels: github-actions
|
||||
sign-commits: true
|
||||
@@ -1,207 +0,0 @@
|
||||
name: Push Docker Image with VersionNumber
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- main
|
||||
|
||||
# cancel in-progress jobs if a new job is triggered
|
||||
# This is useful to avoid running multiple builds for the same branch if a new commit is pushed
|
||||
# or a pull request is updated.
|
||||
# It helps to save resources and time by ensuring that only the latest commit is built and tested
|
||||
# This is particularly useful for long-running jobs that may take a while to complete.
|
||||
# The `group` is set to a combination of the workflow name, event name, and branch name.
|
||||
# This ensures that jobs are grouped by the workflow and branch, allowing for cancellation of
|
||||
# in-progress jobs when a new commit is pushed to the same branch or a new pull request is opened.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref_name || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
push:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: "temurin"
|
||||
|
||||
- uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
|
||||
with:
|
||||
gradle-version: 8.14
|
||||
|
||||
- name: Run Gradle Command
|
||||
run: ./gradlew clean build
|
||||
env:
|
||||
DISABLE_ADDITIONAL_FEATURES: true
|
||||
STIRLING_PDF_DESKTOP_UI: false
|
||||
|
||||
- name: Install cosign
|
||||
if: github.ref == 'refs/heads/master'
|
||||
uses: sigstore/cosign-installer@d58896d6a1865668819e1d91763c7751a165e159 # v3.9.2
|
||||
with:
|
||||
cosign-release: "v2.4.1"
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
run: echo "versionNumber=$(./gradlew printVersion --quiet | tail -1)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
|
||||
|
||||
- name: Convert repository owner to lowercase
|
||||
id: repoowner
|
||||
run: echo "lowercase=$(echo ${{ github.repository_owner }} | awk '{print tolower($0)}')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Generate tags
|
||||
id: meta
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
|
||||
if: github.ref != 'refs/heads/main'
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
|
||||
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/s-pdf
|
||||
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf
|
||||
${{ secrets.DOCKER_HUB_ORG_USERNAME }}/stirling-pdf
|
||||
tags: |
|
||||
type=raw,value=${{ steps.versionNumber.outputs.versionNumber }},enable=${{ github.ref == 'refs/heads/master' }}
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Build and push main Dockerfile
|
||||
id: build-push-regular
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
if: github.ref != 'refs/heads/main'
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
push: true
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
build-args: VERSION_TAG=${{ steps.versionNumber.outputs.versionNumber }}
|
||||
platforms: linux/amd64,linux/arm64/v8
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
- name: Sign regular images
|
||||
if: github.ref == 'refs/heads/master'
|
||||
env:
|
||||
DIGEST: ${{ steps.build-push-regular.outputs.digest }}
|
||||
TAGS: ${{ steps.meta.outputs.tags }}
|
||||
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
||||
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
|
||||
run: |
|
||||
echo "$TAGS" | tr ',' '\n' | while read -r tag; do
|
||||
cosign sign --yes \
|
||||
--key env://COSIGN_PRIVATE_KEY \
|
||||
"${tag}@${DIGEST}"
|
||||
done
|
||||
|
||||
- name: Generate tags ultra-lite
|
||||
id: meta2
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
|
||||
if: github.ref != 'refs/heads/main'
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
|
||||
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/s-pdf
|
||||
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf
|
||||
${{ secrets.DOCKER_HUB_ORG_USERNAME }}/stirling-pdf
|
||||
tags: |
|
||||
type=raw,value=${{ steps.versionNumber.outputs.versionNumber }}-ultra-lite,enable=${{ github.ref == 'refs/heads/master' }}
|
||||
type=raw,value=latest-ultra-lite,enable=${{ github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Build and push Dockerfile-ultra-lite
|
||||
id: build-push-lite
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
if: github.ref != 'refs/heads/main'
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.ultra-lite
|
||||
push: true
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
tags: ${{ steps.meta2.outputs.tags }}
|
||||
labels: ${{ steps.meta2.outputs.labels }}
|
||||
build-args: VERSION_TAG=${{ steps.versionNumber.outputs.versionNumber }}
|
||||
platforms: linux/amd64,linux/arm64/v8
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
- name: Generate tags fat
|
||||
id: meta3
|
||||
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/s-pdf
|
||||
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/s-pdf
|
||||
ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf
|
||||
${{ secrets.DOCKER_HUB_ORG_USERNAME }}/stirling-pdf
|
||||
tags: |
|
||||
type=raw,value=${{ steps.versionNumber.outputs.versionNumber }}-fat,enable=${{ github.ref == 'refs/heads/master' }}
|
||||
type=raw,value=latest-fat,enable=${{ github.ref == 'refs/heads/master' }}
|
||||
type=raw,value=alpha,enable=${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Build and push main Dockerfile fat
|
||||
id: build-push-fat
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
context: .
|
||||
file: ./Dockerfile.fat
|
||||
push: true
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
tags: ${{ steps.meta3.outputs.tags }}
|
||||
labels: ${{ steps.meta3.outputs.labels }}
|
||||
build-args: VERSION_TAG=${{ steps.versionNumber.outputs.versionNumber }}
|
||||
platforms: linux/amd64,linux/arm64/v8
|
||||
provenance: true
|
||||
sbom: true
|
||||
|
||||
- name: Sign fat images
|
||||
if: github.ref == 'refs/heads/master'
|
||||
env:
|
||||
DIGEST: ${{ steps.build-push-fat.outputs.digest }}
|
||||
TAGS: ${{ steps.meta3.outputs.tags }}
|
||||
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
|
||||
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
|
||||
run: |
|
||||
echo "$TAGS" | tr ',' '\n' | while read -r tag; do
|
||||
cosign sign --key env://COSIGN_PRIVATE_KEY --yes "${tag}@${DIGEST}"
|
||||
done
|
||||
@@ -1,180 +0,0 @@
|
||||
name: Release Artifacts
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
release:
|
||||
types: [created]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
disable_security: [true, false]
|
||||
include:
|
||||
- disable_security: false
|
||||
file_suffix: "-with-login"
|
||||
- disable_security: true
|
||||
file_suffix: ""
|
||||
outputs:
|
||||
version: ${{ steps.versionNumber.outputs.versionNumber }}
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: "temurin"
|
||||
|
||||
- uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
|
||||
with:
|
||||
gradle-version: 8.14
|
||||
|
||||
- name: Generate jar (Disable Security=${{ matrix.disable_security }})
|
||||
run: ./gradlew clean createExe
|
||||
env:
|
||||
DISABLE_ADDITIONAL_FEATURES: ${{ matrix.disable_security }}
|
||||
STIRLING_PDF_DESKTOP_UI: false
|
||||
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
run: |
|
||||
VERSION=$(grep "^version =" build.gradle | awk -F'"' '{print $2}')
|
||||
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Rename binaries
|
||||
run: |
|
||||
mv ./build/launch4j/Stirling-PDF.exe ./build/launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe
|
||||
mv ./build/libs/Stirling-PDF-${{ steps.versionNumber.outputs.versionNumber }}.jar ./build/libs/Stirling-PDF${{ matrix.file_suffix }}.jar
|
||||
|
||||
- name: Debug build artifacts
|
||||
run: |
|
||||
echo "Current Directory: $(pwd)"
|
||||
ls -R ./build/libs
|
||||
ls -R ./build/launch4j
|
||||
|
||||
- name: Upload build artifacts
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: binaries${{ matrix.file_suffix }}
|
||||
path: |
|
||||
./build/launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.*
|
||||
./build/libs/Stirling-PDF${{ matrix.file_suffix }}.*
|
||||
|
||||
sign_verify:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
disable_security: [true, false]
|
||||
include:
|
||||
- disable_security: false
|
||||
file_suffix: "-with-login"
|
||||
- disable_security: true
|
||||
file_suffix: ""
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Download build artifacts
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
with:
|
||||
name: binaries${{ matrix.file_suffix }}
|
||||
- name: Display structure of downloaded files
|
||||
run: ls -R
|
||||
|
||||
- name: Install Cosign
|
||||
uses: sigstore/cosign-installer@d58896d6a1865668819e1d91763c7751a165e159 # v3.9.2
|
||||
|
||||
- name: Generate key pair
|
||||
run: cosign generate-key-pair
|
||||
|
||||
- name: Sign and generate attestations
|
||||
run: |
|
||||
cosign sign-blob \
|
||||
--key ./cosign.key \
|
||||
--yes \
|
||||
--output-signature ./libs/Stirling-PDF${{ matrix.file_suffix }}.jar.sig \
|
||||
./libs/Stirling-PDF${{ matrix.file_suffix }}.jar
|
||||
|
||||
cosign attest-blob \
|
||||
--predicate - \
|
||||
--key ./cosign.key \
|
||||
--yes \
|
||||
--output-attestation ./libs/Stirling-PDF${{ matrix.file_suffix }}.jar.intoto.jsonl \
|
||||
./libs/Stirling-PDF${{ matrix.file_suffix }}.jar
|
||||
|
||||
cosign verify-blob \
|
||||
--key ./cosign.pub \
|
||||
--signature ./libs/Stirling-PDF${{ matrix.file_suffix }}.jar.sig \
|
||||
./libs/Stirling-PDF${{ matrix.file_suffix }}.jar
|
||||
|
||||
cosign sign-blob \
|
||||
--key ./cosign.key \
|
||||
--yes \
|
||||
--output-signature ./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe.sig \
|
||||
./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe
|
||||
|
||||
cosign attest-blob \
|
||||
--predicate - \
|
||||
--key ./cosign.key \
|
||||
--yes \
|
||||
--output-attestation ./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe.intoto.jsonl \
|
||||
./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe
|
||||
|
||||
cosign verify-blob \
|
||||
--key ./cosign.pub \
|
||||
--signature ./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe.sig \
|
||||
./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.exe
|
||||
|
||||
- name: Upload signed artifacts
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: signed${{ matrix.file_suffix }}
|
||||
path: |
|
||||
./libs/Stirling-PDF${{ matrix.file_suffix }}.*
|
||||
./launch4j/Stirling-PDF-Server${{ matrix.file_suffix }}.*
|
||||
|
||||
release:
|
||||
needs: [build, sign_verify]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
strategy:
|
||||
matrix:
|
||||
disable_security: [true, false]
|
||||
include:
|
||||
- disable_security: false
|
||||
file_suffix: "-with-login"
|
||||
- disable_security: true
|
||||
file_suffix: ""
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Download signed artifacts
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
with:
|
||||
name: signed${{ matrix.file_suffix }}
|
||||
|
||||
- name: Upload binaries, attestations and signatures to Release and create GitHub Release
|
||||
uses: softprops/action-gh-release@72f2c25fcb47643c292f7107632f7a47c1df5cd8 # v2.3.2
|
||||
with:
|
||||
tag_name: v${{ needs.build.outputs.version }}
|
||||
generate_release_notes: true
|
||||
files: |
|
||||
./libs/Stirling-PDF*
|
||||
./launch4j/Stirling-PDF-Server*
|
||||
@@ -1,79 +0,0 @@
|
||||
# This workflow uses actions that are not certified by GitHub. They are provided
|
||||
# by a third-party and are governed by separate terms of service, privacy
|
||||
# policy, and support documentation.
|
||||
|
||||
name: Scorecard supply-chain security
|
||||
on:
|
||||
# For Branch-Protection check. Only the default branch is supported. See
|
||||
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
|
||||
branch_protection_rule:
|
||||
# To guarantee Maintained check is occasionally updated. See
|
||||
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
|
||||
schedule:
|
||||
- cron: "20 7 * * 2"
|
||||
push:
|
||||
branches: ["main"]
|
||||
permissions: read-all
|
||||
|
||||
jobs:
|
||||
analysis:
|
||||
name: Scorecard analysis
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
# Needed to upload the results to code-scanning dashboard.
|
||||
security-events: write
|
||||
# Needed to publish results and get a badge (see publish_results below).
|
||||
id-token: write
|
||||
contents: read
|
||||
actions: read
|
||||
# To allow GraphQL ListCommits to work
|
||||
issues: read
|
||||
pull-requests: read
|
||||
# To detect SAST tools
|
||||
checks: read
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: "Checkout code"
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: "Run analysis"
|
||||
uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2
|
||||
with:
|
||||
results_file: results.sarif
|
||||
results_format: sarif
|
||||
# (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
|
||||
# - you want to enable the Branch-Protection check on a *public* repository, or
|
||||
# - you are installing Scorecards on a *private* repository
|
||||
# To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
|
||||
# repo_token: ${{ secrets.SCORECARD_TOKEN }}
|
||||
|
||||
# Public repositories:
|
||||
# - Publish results to OpenSSF REST API for easy access by consumers
|
||||
# - Allows the repository to include the Scorecard badge.
|
||||
# - See https://github.com/ossf/scorecard-action#publishing-results.
|
||||
# For private repositories:
|
||||
# - `publish_results` will always be set to `false`, regardless
|
||||
# of the value entered here.
|
||||
publish_results: true
|
||||
|
||||
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
|
||||
# format to the repository Actions tab.
|
||||
- name: "Upload artifact"
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: SARIF file
|
||||
path: results.sarif
|
||||
retention-days: 5
|
||||
|
||||
# Upload the results to GitHub's code scanning dashboard.
|
||||
- name: "Upload to code-scanning"
|
||||
uses: github/codeql-action/upload-sarif@51f77329afa6477de8c49fc9c7046c15b9a4e79d # v3.29.5
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
@@ -1,75 +0,0 @@
|
||||
name: Run Sonarqube
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
pull_request_target:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
# cancel in-progress jobs if a new job is triggered
|
||||
# This is useful to avoid running multiple builds for the same branch if a new commit is pushed
|
||||
# or a pull request is updated.
|
||||
# It helps to save resources and time by ensuring that only the latest commit is built and tested
|
||||
# This is particularly useful for long-running jobs that may take a while to complete.
|
||||
# The `group` is set to a combination of the workflow name, event name, and branch name.
|
||||
# This ensures that jobs are grouped by the workflow and branch, allowing for cancellation of
|
||||
# in-progress jobs when a new commit is pushed to the same branch or a new pull request is opened.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref_name || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
pull-requests: read
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
sonarqube:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
|
||||
|
||||
- name: Build and analyze with Gradle
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
|
||||
DISABLE_ADDITIONAL_FEATURES: false
|
||||
STIRLING_PDF_DESKTOP_UI: true
|
||||
run: |
|
||||
./gradlew clean build sonar \
|
||||
-Dsonar.projectKey=Stirling-Tools_Stirling-PDF \
|
||||
-Dsonar.organization=stirling-tools \
|
||||
-Dsonar.host.url=https://sonarcloud.io \
|
||||
-Dsonar.login=${SONAR_TOKEN} \
|
||||
-Dsonar.log.level=DEBUG \
|
||||
--info
|
||||
|
||||
- name: Upload Problems Report on Failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: gradle-problems-report
|
||||
path: build/reports/problems/problems-report.html
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload Sonar Logs on Failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: sonar-logs
|
||||
path: |
|
||||
.scannerwork/report-task.txt
|
||||
build/sonar/
|
||||
retention-days: 7
|
||||
@@ -1,40 +0,0 @@
|
||||
name: Close stale issues
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "30 0 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: 30 days stale issues
|
||||
uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
days-before-stale: 30
|
||||
days-before-close: 7
|
||||
stale-issue-message: >
|
||||
This issue has been automatically marked as stale because it has had no recent activity.
|
||||
It will be closed if no further activity occurs. Thank you for your contributions.
|
||||
close-issue-message: >
|
||||
This issue has been automatically closed because it has had no recent activity after being marked as stale.
|
||||
Please reopen if you need further assistance.
|
||||
stale-issue-label: "Stale"
|
||||
remove-stale-when-updated: true
|
||||
only-issue-labels: "more-info-needed"
|
||||
days-before-pr-stale: -1 # Prevents PRs from being marked as stale
|
||||
days-before-pr-close: -1 # Prevents PRs from being closed
|
||||
start-date: "2024-07-06T00:00:00Z" # ISO 8601 Format
|
||||
@@ -1,61 +0,0 @@
|
||||
name: Update Swagger
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
# cancel in-progress jobs if a new job is triggered
|
||||
# This is useful to avoid running multiple builds for the same branch if a new commit is pushed
|
||||
# or a pull request is updated.
|
||||
# It helps to save resources and time by ensuring that only the latest commit is built and tested
|
||||
# This is particularly useful for long-running jobs that may take a while to complete.
|
||||
# The `group` is set to a combination of the workflow name, event name, and branch name.
|
||||
# This ensures that jobs are grouped by the workflow and branch, allowing for cancellation of
|
||||
# in-progress jobs when a new commit is pushed to the same branch or a new pull request is opened.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref_name || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
push:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up JDK 17
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: "17"
|
||||
distribution: "temurin"
|
||||
|
||||
- uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
|
||||
|
||||
- name: Generate Swagger documentation
|
||||
run: ./gradlew :stirling-pdf:generateOpenApiDocs
|
||||
|
||||
- name: Upload Swagger Documentation to SwaggerHub
|
||||
run: ./gradlew swaggerhubUpload
|
||||
env:
|
||||
SWAGGERHUB_API_KEY: ${{ secrets.SWAGGERHUB_API_KEY }}
|
||||
SWAGGERHUB_USER: "Frooodle"
|
||||
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
run: echo "versionNumber=$(./gradlew printVersion --quiet | tail -1)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Set API version as published and default on SwaggerHub
|
||||
run: |
|
||||
curl -X PUT -H "Authorization: ${SWAGGERHUB_API_KEY}" "https://api.swaggerhub.com/apis/${SWAGGERHUB_USER}/Stirling-PDF/${{ steps.versionNumber.outputs.versionNumber }}/settings/lifecycle" -H "accept: application/json" -H "Content-Type: application/json" -d "{\"published\":true,\"default\":true}"
|
||||
env:
|
||||
SWAGGERHUB_API_KEY: ${{ secrets.SWAGGERHUB_API_KEY }}
|
||||
SWAGGERHUB_USER: "Frooodle"
|
||||
@@ -1,116 +0,0 @@
|
||||
name: Sync Files
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "build.gradle"
|
||||
- "README.md"
|
||||
- "app/core/src/main/resources/messages_*.properties"
|
||||
- "app/core/src/main/resources/static/3rdPartyLicenses.json"
|
||||
- "scripts/ignore_translation.toml"
|
||||
|
||||
# cancel in-progress jobs if a new job is triggered
|
||||
# This is useful to avoid running multiple builds for the same branch if a new commit is pushed
|
||||
# or a pull request is updated.
|
||||
# It helps to save resources and time by ensuring that only the latest commit is built and tested
|
||||
# This is particularly useful for long-running jobs that may take a while to complete.
|
||||
# The `group` is set to a combination of the workflow name, event name, and branch name.
|
||||
# This ensures that jobs are grouped by the workflow and branch, allowing for cancellation of
|
||||
# in-progress jobs when a new commit is pushed to the same branch or a new pull request is opened.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref_name || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sync-files:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Setup GitHub App Bot
|
||||
id: setup-bot
|
||||
uses: ./.github/actions/setup-bot
|
||||
with:
|
||||
app-id: ${{ secrets.GH_APP_ID }}
|
||||
private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: "pip" # caching pip dependencies
|
||||
|
||||
- name: Sync translation property files
|
||||
run: |
|
||||
python .github/scripts/check_language_properties.py --reference-file "app/core/src/main/resources/messages_en_GB.properties" --branch main
|
||||
|
||||
- name: Commit translation files
|
||||
run: |
|
||||
git add app/core/src/main/resources/messages_*.properties
|
||||
git diff --staged --quiet || git commit -m ":memo: Sync translation files" || echo "No changes detected"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install --require-hashes -r ./.github/scripts/requirements_sync_readme.txt
|
||||
|
||||
- name: Sync README.md
|
||||
run: |
|
||||
python scripts/counter_translation.py
|
||||
|
||||
- name: Run git add
|
||||
run: |
|
||||
git add README.md scripts/ignore_translation.toml
|
||||
git diff --staged --quiet || git commit -m ":memo: Sync README.md & scripts/ignore_translation.toml" || echo "No changes detected"
|
||||
|
||||
- name: Create Pull Request
|
||||
if: always()
|
||||
uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
|
||||
with:
|
||||
token: ${{ steps.setup-bot.outputs.token }}
|
||||
commit-message: Update files
|
||||
committer: ${{ steps.setup-bot.outputs.committer }}
|
||||
author: ${{ steps.setup-bot.outputs.committer }}
|
||||
signoff: true
|
||||
branch: sync_readme
|
||||
title: ":globe_with_meridians: Sync Translations + Update README Progress Table"
|
||||
body: |
|
||||
### Description of Changes
|
||||
|
||||
This Pull Request was automatically generated to synchronize updates to translation files and documentation. Below are the details of the changes made:
|
||||
|
||||
#### **1. Synchronization of Translation Files**
|
||||
- Updated translation files (`messages_*.properties`) to reflect changes in the reference file `messages_en_GB.properties`.
|
||||
- Ensured consistency and synchronization across all supported language files.
|
||||
- Highlighted any missing or incomplete translations.
|
||||
|
||||
#### **2. Update README.md**
|
||||
- Generated the translation progress table in `README.md`.
|
||||
- Added a summary of the current translation status for all supported languages.
|
||||
- Included up-to-date statistics on translation coverage.
|
||||
|
||||
#### **Why these changes are necessary**
|
||||
- Keeps translation files aligned with the latest reference updates.
|
||||
- Ensures the documentation reflects the current translation progress.
|
||||
|
||||
---
|
||||
|
||||
Auto-generated by [create-pull-request][1].
|
||||
|
||||
[1]: https://github.com/peter-evans/create-pull-request
|
||||
draft: false
|
||||
delete-branch: true
|
||||
labels: github-actions
|
||||
sign-commits: true
|
||||
add-paths: |
|
||||
README.md
|
||||
app/core/src/main/resources/messages_*.properties
|
||||
@@ -1,176 +0,0 @@
|
||||
name: UI test with TestDriverAI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["master", "UITest", "testdriver"]
|
||||
|
||||
# cancel in-progress jobs if a new job is triggered
|
||||
# This is useful to avoid running multiple builds for the same branch if a new commit is pushed
|
||||
# or a pull request is updated.
|
||||
# It helps to save resources and time by ensuring that only the latest commit is built and tested
|
||||
# This is particularly useful for long-running jobs that may take a while to complete.
|
||||
# The `group` is set to a combination of the workflow name, event name, and branch name.
|
||||
# This ensures that jobs are grouped by the workflow and branch, allowing for cancellation of
|
||||
# in-progress jobs when a new commit is pushed to the same branch or a new pull request is opened.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref_name || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
java-version: '17'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@ac638b010cf58a27ee6c972d7336334ccaf61c96 # v4.4.1
|
||||
with:
|
||||
gradle-version: 8.14
|
||||
|
||||
- name: Build with Gradle
|
||||
run: ./gradlew clean build
|
||||
env:
|
||||
DISABLE_ADDITIONAL_FEATURES: true
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||
|
||||
- name: Get version number
|
||||
id: versionNumber
|
||||
run: |
|
||||
VERSION=$(grep "^version =" build.gradle | awk -F'"' '{print $2}')
|
||||
echo "versionNumber=$VERSION" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_API }}
|
||||
|
||||
- name: Build and push test image
|
||||
uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
push: true
|
||||
tags: ${{ secrets.DOCKER_HUB_USERNAME }}/test:test-${{ github.sha }}
|
||||
build-args: VERSION_TAG=${{ steps.versionNumber.outputs.versionNumber }}
|
||||
platforms: linux/amd64
|
||||
|
||||
- name: Set up SSH
|
||||
run: |
|
||||
mkdir -p ~/.ssh/
|
||||
echo "${{ secrets.VPS_SSH_KEY }}" > ../private.key
|
||||
sudo chmod 600 ../private.key
|
||||
|
||||
- name: Deploy to VPS
|
||||
run: |
|
||||
cat > docker-compose.yml << EOF
|
||||
version: '3.3'
|
||||
services:
|
||||
stirling-pdf:
|
||||
container_name: stirling-pdf-test-${{ github.sha }}
|
||||
image: ${{ secrets.DOCKER_HUB_USERNAME }}/test:test-${{ github.sha }}
|
||||
ports:
|
||||
- "1337:8080"
|
||||
volumes:
|
||||
- /stirling/test-${{ github.sha }}/data:/usr/share/tessdata:rw
|
||||
- /stirling/test-${{ github.sha }}/config:/configs:rw
|
||||
- /stirling/test-${{ github.sha }}/logs:/logs:rw
|
||||
environment:
|
||||
DISABLE_ADDITIONAL_FEATURES: "true"
|
||||
SECURITY_ENABLELOGIN: "false"
|
||||
SYSTEM_DEFAULTLOCALE: en-GB
|
||||
UI_APPNAME: "Stirling-PDF Test"
|
||||
UI_HOMEDESCRIPTION: "Test Deployment"
|
||||
UI_APPNAMENAVBAR: "Test"
|
||||
SYSTEM_MAXFILESIZE: "100"
|
||||
METRICS_ENABLED: "true"
|
||||
SYSTEM_GOOGLEVISIBILITY: "false"
|
||||
SYSTEM_ENABLEANALYTICS: "false"
|
||||
restart: on-failure:5
|
||||
EOF
|
||||
|
||||
scp -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null docker-compose.yml ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }}:/tmp/docker-compose.yml
|
||||
|
||||
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }} << EOF
|
||||
mkdir -p /stirling/test-${{ github.sha }}/{data,config,logs}
|
||||
mv /tmp/docker-compose.yml /stirling/test-${{ github.sha }}/docker-compose.yml
|
||||
cd /stirling/test-${{ github.sha }}
|
||||
docker-compose pull
|
||||
docker-compose up -d
|
||||
EOF
|
||||
|
||||
test:
|
||||
needs: deploy
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
cache: 'npm'
|
||||
|
||||
- name: Run TestDriver.ai
|
||||
uses: testdriverai/action@f0d0f45fdd684db628baa843fe9313f3ca3a8aa8 #1.1.3
|
||||
with:
|
||||
key: ${{secrets.TESTDRIVER_API_KEY}}
|
||||
prerun: |
|
||||
npm install
|
||||
npm run build
|
||||
npm install dashcam-chrome --save
|
||||
Start-Process "C:/Program Files/Google/Chrome/Application/chrome.exe" -ArgumentList "--start-maximized", "--load-extension=$(pwd)/node_modules/dashcam-chrome/build", "http://${{ secrets.VPS_HOST }}:1337"
|
||||
Start-Sleep -Seconds 20
|
||||
prompt: |
|
||||
1. /run testing/testdriver/test.yml
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
FORCE_COLOR: "3"
|
||||
|
||||
cleanup:
|
||||
needs: [deploy, test]
|
||||
runs-on: ubuntu-latest
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Harden Runner
|
||||
uses: step-security/harden-runner@ec9f2d5744a09debf3a187a3f4f675c53b671911 # v2.13.0
|
||||
with:
|
||||
egress-policy: audit
|
||||
|
||||
- name: Set up SSH
|
||||
run: |
|
||||
mkdir -p ~/.ssh/
|
||||
echo "${{ secrets.VPS_SSH_KEY }}" > ../private.key
|
||||
sudo chmod 600 ../private.key
|
||||
|
||||
- name: Cleanup deployment
|
||||
run: |
|
||||
ssh -i ../private.key -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${{ secrets.VPS_USERNAME }}@${{ secrets.VPS_HOST }} << EOF
|
||||
cd /stirling/test-${{ github.sha }}
|
||||
docker-compose down
|
||||
cd /stirling
|
||||
rm -rf test-${{ github.sha }}
|
||||
EOF
|
||||
+1
-13
@@ -13,7 +13,6 @@ local.properties
|
||||
.recommenders
|
||||
.classpath
|
||||
.project
|
||||
*.local.json
|
||||
version.properties
|
||||
|
||||
#### Stirling-PDF Files ###
|
||||
@@ -27,8 +26,6 @@ clientWebUI/
|
||||
!cucumber/exampleFiles/
|
||||
!cucumber/exampleFiles/example_html.zip
|
||||
exampleYmlFiles/stirling/
|
||||
/testing/file_snapshots
|
||||
SwaggerDoc.json
|
||||
|
||||
# Gradle
|
||||
.gradle
|
||||
@@ -124,13 +121,7 @@ SwaggerDoc.json
|
||||
*.tar.gz
|
||||
*.rar
|
||||
*.db
|
||||
build
|
||||
app/core/build
|
||||
app/common/build
|
||||
app/proprietary/build
|
||||
common/build
|
||||
proprietary/build
|
||||
stirling-pdf/build
|
||||
/build
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
@@ -197,6 +188,3 @@ id_ed25519.pub
|
||||
.ipynb_checkpoints
|
||||
|
||||
**/jcef-bundle/
|
||||
|
||||
# node_modules
|
||||
node_modules/
|
||||
|
||||
+7
-17
@@ -1,28 +1,28 @@
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.12.7
|
||||
rev: v0.8.4
|
||||
hooks:
|
||||
- id: ruff
|
||||
args:
|
||||
- --fix
|
||||
- --line-length=127
|
||||
files: ^((\.github/scripts|scripts|app/core/src/main/resources/static/python)/.+)?[^/]+\.py$
|
||||
files: ^((\.github/scripts|scripts)/.+)?[^/]+\.py$
|
||||
exclude: (split_photos.py)
|
||||
- id: ruff-format
|
||||
files: ^((\.github/scripts|scripts|app/core/src/main/resources/static/python)/.+)?[^/]+\.py$
|
||||
files: ^((\.github/scripts|scripts)/.+)?[^/]+\.py$
|
||||
exclude: (split_photos.py)
|
||||
- repo: https://github.com/codespell-project/codespell
|
||||
rev: v2.4.1
|
||||
rev: v2.3.0
|
||||
hooks:
|
||||
- id: codespell
|
||||
args:
|
||||
- --ignore-words-list=thirdParty,tabEl,tabEls
|
||||
- --ignore-words-list=
|
||||
- --skip="./.*,*.csv,*.json,*.ambr"
|
||||
- --quiet-level=2
|
||||
files: \.(html|css|js|py|md)$
|
||||
exclude: (.vscode|.devcontainer|app/core/src/main/resources|app/proprietary/src/main/resources|Dockerfile|.*/pdfjs.*|.*/thirdParty.*|bootstrap.*|.*\.min\..*|.*diff\.js)
|
||||
exclude: (.vscode|.devcontainer|src/main/resources|Dockerfile|.*/pdfjs.*|.*/thirdParty.*|bootstrap.*|.*\.min\..*|.*diff\.js)
|
||||
- repo: https://github.com/gitleaks/gitleaks
|
||||
rev: v8.28.0
|
||||
rev: v8.22.0
|
||||
hooks:
|
||||
- id: gitleaks
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
@@ -34,13 +34,3 @@ repos:
|
||||
- id: trailing-whitespace
|
||||
files: ^.*(\.js|\.java|\.py|\.yml)$
|
||||
exclude: ^(.*/pdfjs.*|.*/thirdParty.*|bootstrap.*|.*\.min\..*|.*diff\.js|\.github/workflows/.*$)
|
||||
# - repo: https://github.com/thibaudcolas/pre-commit-stylelint
|
||||
# rev: v16.21.1
|
||||
# hooks:
|
||||
# - id: stylelint
|
||||
# additional_dependencies:
|
||||
# - stylelint@16.21.1
|
||||
# - stylelint-config-standard@38.0.0
|
||||
# - "@stylistic/stylelint-plugin@3.1.3"
|
||||
# files: \.(css)$
|
||||
# args: [--fix]
|
||||
|
||||
Vendored
+11
-10
@@ -5,19 +5,20 @@
|
||||
"ms-python.black-formatter", // Python code formatter using Black
|
||||
"ms-python.flake8", // Flake8 linter for Python to enforce code quality
|
||||
"ms-python.python", // Official Microsoft Python extension with IntelliSense, debugging, and Jupyter support
|
||||
"ms-vscode-remote.vscode-remote-extensionpack", // Remote Development Pack for SSH, WSL, and Containers
|
||||
// "Oracle.oracle-java", // Oracle Java extension with additional features for Java development
|
||||
// "ms-vscode-remote.remote-containers", // Support for remote development with containers (Docker, Dev Containers)
|
||||
// "ms-vscode-remote.vscode-remote-extensionpack", // Remote Development Pack for SSH, WSL, and Containers
|
||||
"Oracle.oracle-java", // Oracle Java extension with additional features for Java development
|
||||
"redhat.java", // Java support by Red Hat with IntelliSense, debugging, and code navigation
|
||||
"shengchen.vscode-checkstyle", // Checkstyle integration for Java code quality checks
|
||||
"streetsidesoftware.code-spell-checker", // Spell checker for code to avoid typos
|
||||
"vmware.vscode-boot-dev-pack", // Developer tools for Spring Boot by VMware
|
||||
"vmware.vscode-spring-boot", // Spring Boot tools by VMware for enhanced Spring development
|
||||
"vscjava.vscode-gradle", // Gradle extension for build and automation support
|
||||
"vscjava.vscode-java-debug", // Debugging support for Java projects
|
||||
"vscjava.vscode-java-dependency", // Java dependency management within VS Code
|
||||
"vscjava.vscode-java-pack", // Java Extension Pack with essential Java tools for VS Code
|
||||
"vscjava.vscode-java-test", // Java test framework for running and debugging tests in VS Code
|
||||
"vscjava.vscode-spring-boot-dashboard", // Spring Boot dashboard for managing and visualizing Spring Boot applications
|
||||
"EditorConfig.EditorConfig", // EditorConfig support for maintaining consistent coding styles
|
||||
"ms-azuretools.vscode-docker", // Docker extension for Visual Studio Code
|
||||
"GitHub.copilot", // GitHub Copilot AI pair programmer for Visual Studio Code
|
||||
"GitHub.vscode-pull-request-github", // GitHub Pull Requests extension for Visual Studio Code
|
||||
"charliermarsh.ruff", // Ruff code formatter for Python to follow the Ruff Style Guide
|
||||
"yzhang.markdown-all-in-one", // Markdown All-in-One extension for enhanced Markdown editing
|
||||
"stylelint.vscode-stylelint", // Stylelint extension for CSS and SCSS linting
|
||||
"redhat.vscode-yaml", // YAML extension for Visual Studio Code
|
||||
"vscjava.vscode-spring-initializr" // Support for Spring Initializr to create new Spring projects
|
||||
]
|
||||
}
|
||||
|
||||
Vendored
+43
-130
@@ -1,142 +1,55 @@
|
||||
{
|
||||
"editor.wordSegmenterLocales": "",
|
||||
"editor.guides.bracketPairs": "active",
|
||||
"editor.guides.bracketPairsHorizontal": "active",
|
||||
"cSpell.enabled": false,
|
||||
"[feature]": {
|
||||
"editor.defaultFormatter": "alexkrechik.cucumberautocomplete"
|
||||
},
|
||||
"java.compile.nullAnalysis.mode": "automatic",
|
||||
"files.eol": "auto",
|
||||
"java.configuration.updateBuildConfiguration": "interactive",
|
||||
"black-formatter.args": ["--line-length", "127"],
|
||||
"flake8.args": ["--max-line-length", "127"],
|
||||
"pylint.args": ["max-line-length", "127"],
|
||||
"[java]": {
|
||||
"editor.defaultFormatter": "josevseb.google-java-format-for-vs-code"
|
||||
},
|
||||
"[jsonc]": {
|
||||
"editor.defaultFormatter": "vscode.json-language-features"
|
||||
},
|
||||
"[css]": {
|
||||
"editor.defaultFormatter": "stylelint.vscode-stylelint"
|
||||
},
|
||||
"[json]": {
|
||||
"editor.defaultFormatter": "vscode.json-language-features"
|
||||
"editor.tabSize": 4,
|
||||
"editor.detectIndentation": false,
|
||||
"editor.rulers": [127]
|
||||
},
|
||||
"[python]": {
|
||||
"editor.defaultFormatter": "ms-python.black-formatter"
|
||||
},
|
||||
"[gradle-kotlin-dsl]": {
|
||||
"editor.defaultFormatter": "vscjava.vscode-gradle"
|
||||
},
|
||||
"[markdown]": {
|
||||
"editor.defaultFormatter": "yzhang.markdown-all-in-one"
|
||||
"editor.tabSize": 2,
|
||||
"editor.detectIndentation": false,
|
||||
"editor.rulers": [127]
|
||||
},
|
||||
"[gradle-build]": {
|
||||
"editor.defaultFormatter": "vscjava.vscode-gradle"
|
||||
"editor.tabSize": 4,
|
||||
"editor.detectIndentation": false,
|
||||
"editor.rulers": [127]
|
||||
},
|
||||
"[gradle]": {
|
||||
"editor.defaultFormatter": "vscjava.vscode-gradle"
|
||||
"editor.tabSize": 4,
|
||||
"editor.detectIndentation": false,
|
||||
"editor.rulers": [127]
|
||||
},
|
||||
"[html]": {
|
||||
"editor.tabSize": 2,
|
||||
"editor.rulers": [127],
|
||||
"files.trimFinalNewlines": false,
|
||||
"files.insertFinalNewline": false
|
||||
},
|
||||
"[javascript]": {
|
||||
"editor.tabSize": 2,
|
||||
"editor.rulers": [127]
|
||||
},
|
||||
"[yaml]": {
|
||||
"editor.defaultFormatter": "redhat.vscode-yaml"
|
||||
"files.trimFinalNewlines": false,
|
||||
"files.insertFinalNewline": false
|
||||
},
|
||||
"java.compile.nullAnalysis.mode": "automatic",
|
||||
"java.configuration.updateBuildConfiguration": "interactive",
|
||||
"java.format.enabled": true,
|
||||
"java.format.settings.profile": "GoogleStyle",
|
||||
"java.format.settings.google.version": "1.27.0",
|
||||
"java.format.settings.google.extra": "--aosp --skip-sorting-imports --skip-javadoc-formatting",
|
||||
// (DE) Aktiviert Kommentare im Java-Format.
|
||||
// (EN) Enables comments in Java formatting.
|
||||
// "java.format.comments.enabled": true,
|
||||
// (DE) Generiert automatisch Kommentare im Code.
|
||||
// (EN) Automatically generates comments in code.
|
||||
// "java.codeGeneration.generateComments": true,
|
||||
// https://github.com/redhat-developer/vscode-java/blob/master/document/_java.learnMoreAboutCleanUps.md#java-clean-ups
|
||||
"java.saveActions.cleanup": true,
|
||||
"java.cleanup.actions": [
|
||||
"invertEquals", // Inverts calls to Object.equals(Object) and String.equalsIgnoreCase(String) to avoid useless null pointer exception.
|
||||
"instanceofPatternMatch" // Replaces instanceof checks with pattern matching.
|
||||
],
|
||||
// (DE) Aktiviert die Code-Vervollständigung für Java.
|
||||
// (EN) Enables code completion for Java.
|
||||
"java.completion.engine": "dom",
|
||||
"java.completion.enabled": true,
|
||||
"java.completion.importOrder": [
|
||||
"java",
|
||||
"javax",
|
||||
"org",
|
||||
"com",
|
||||
"net",
|
||||
"io",
|
||||
"jakarta",
|
||||
"lombok",
|
||||
"me",
|
||||
"stirling",
|
||||
],
|
||||
"java.project.resourceFilters": [
|
||||
".devcontainer/",
|
||||
".git/",
|
||||
".github/",
|
||||
".gradle/",
|
||||
".venv/",
|
||||
".venv*/",
|
||||
".vscode/",
|
||||
"bin/",
|
||||
"app/core/bin/",
|
||||
"app/common/bin/",
|
||||
"app/proprietary/bin/",
|
||||
"build/",
|
||||
"app/core/build/",
|
||||
"app/common/build/",
|
||||
"app/proprietary/build/",
|
||||
"configs/",
|
||||
"app/core/configs/",
|
||||
"customFiles/",
|
||||
"app/core/customFiles/",
|
||||
"docs/",
|
||||
"exampleYmlFiles",
|
||||
"gradle/",
|
||||
"images/",
|
||||
"logs/",
|
||||
"pipeline/",
|
||||
"scripts/",
|
||||
"testings/",
|
||||
".git-blame-ignore-revs",
|
||||
".gitattributes",
|
||||
".gitignore",
|
||||
"app/core/.gitignore",
|
||||
"app/common/.gitignore",
|
||||
"app/proprietary/.gitignore",
|
||||
".pre-commit-config.yaml",
|
||||
],
|
||||
// Enables signature help in Java.
|
||||
"java.signatureHelp.enabled": true,
|
||||
// Enables detailed signature help descriptions.
|
||||
"java.signatureHelp.description.enabled": true,
|
||||
// Downloads sources for Maven dependencies.
|
||||
"java.maven.downloadSources": true,
|
||||
// Enables Gradle project import.
|
||||
"java.import.gradle.enabled": true,
|
||||
// Downloads sources for Eclipse projects.
|
||||
"java.eclipse.downloadSources": true,
|
||||
// Enables import of the Gradle wrapper.
|
||||
"java.import.gradle.wrapper.enabled": true,
|
||||
"spring.initializr.defaultLanguage": "Java",
|
||||
"spring.initializr.defaultGroupId": "stirling.software.SPDF",
|
||||
"spring.initializr.defaultArtifactId": "SPDF",
|
||||
"java.jdt.ls.lombokSupport.enabled": true,
|
||||
"html.format.wrapLineLength": 127,
|
||||
"html.format.enable": true,
|
||||
"html.format.indentInnerHtml": true,
|
||||
"html.format.unformatted": "script,style,textarea",
|
||||
"html.format.contentUnformatted": "pre,code",
|
||||
"html.format.extraLiners": "head,body,/html",
|
||||
"html.format.wrapAttributes": "force",
|
||||
"html.format.wrapAttributesIndentSize": 2,
|
||||
"html.format.indentHandlebars": true,
|
||||
"html.format.preserveNewLines": true,
|
||||
"html.format.maxPreserveNewLines": 2,
|
||||
"stylelint.configFile": "devTools/.stylelintrc.json",
|
||||
"java.project.sourcePaths": [
|
||||
"app/core/src/main/java",
|
||||
"app/common/src/main/java",
|
||||
"app/proprietary/src/main/java"
|
||||
]
|
||||
"diffEditor.maxComputationTime": 0,
|
||||
"editor.wordSegmenterLocales": null,
|
||||
"editor.guides.bracketPairs": "active",
|
||||
"editor.guides.bracketPairsHorizontal": "active",
|
||||
"files.insertFinalNewline": true,
|
||||
"files.trimFinalNewlines": true,
|
||||
"files.trimTrailingWhitespace": true,
|
||||
"editor.indentSize": "tabSize",
|
||||
"editor.stickyScroll.enabled": false,
|
||||
"editor.minimap.enabled": false,
|
||||
"editor.formatOnSave": true,
|
||||
"java.format.settings.google.mode": "jar-file",
|
||||
"java.format.settings.google.extra": "--aosp --skip-sorting-imports"
|
||||
}
|
||||
|
||||
+2
-13
@@ -25,7 +25,7 @@ Please make sure your Pull Request adheres to the following guidelines:
|
||||
|
||||
## Translations
|
||||
|
||||
If you would like to add or modify a translation, please see [How to add new languages to Stirling-PDF](devGuide/HowToAddNewLanguage.md). Also, please create a Pull Request so others can use it!
|
||||
If you would like to add or modify a translation, please see [How to add new languages to Stirling-PDF](HowToAddNewLanguage.md). Also, please create a Pull Request so others can use it!
|
||||
|
||||
## Docs
|
||||
|
||||
@@ -37,18 +37,7 @@ First, make sure you've read the section [Pull Requests](#pull-requests).
|
||||
|
||||
If, at any point in time, you have a question, please feel free to ask in the same issue thread or in our [Discord](https://discord.gg/FJUSXUSYec).
|
||||
|
||||
## Developer Documentation
|
||||
|
||||
For technical guides, setup instructions, and development resources, please see our [Developer Documentation](devGuide/) which includes:
|
||||
|
||||
- [Developer Guide](devGuide/DeveloperGuide.md) - Main setup and architecture guide
|
||||
- [Exception Handling Guide](devGuide/EXCEPTION_HANDLING_GUIDE.md) - Error handling patterns and i18n
|
||||
- [Translation Guide](devGuide/HowToAddNewLanguage.md) - Adding new languages
|
||||
- And more in the [devGuide folder](devGuide/)
|
||||
|
||||
For configuration and usage guides, see:
|
||||
- [Database Guide](DATABASE.md) - Database setup and configuration
|
||||
- [OCR Guide](HowToUseOCR.md) - OCR setup and configuration
|
||||
Developers should review our [Developer Guide](DeveloperGuide.md)
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ Stirling-PDF uses Lombok to reduce boilerplate code. Some IDEs, like Eclipse, do
|
||||
Visit the [Lombok website](https://projectlombok.org/setup/) for installation instructions specific to your IDE.
|
||||
|
||||
5. Add environment variable
|
||||
For local testing, you should generally be testing the full 'Security' version of Stirling PDF. To do this, you must add the environment flag DISABLE_ADDITIONAL_FEATURES=false to your system and/or IDE build/run step.
|
||||
For local testing, you should generally be testing the full 'Security' version of Stirling-PDF. To do this, you must add the environment flag DOCKER_ENABLE_SECURITY=true to your system and/or IDE build/run step.
|
||||
|
||||
## 4. Project Structure
|
||||
|
||||
@@ -114,9 +114,9 @@ Stirling-PDF offers several Docker versions:
|
||||
|
||||
Stirling-PDF provides several example Docker Compose files in the `exampleYmlFiles` directory, such as:
|
||||
|
||||
- `docker-compose-latest.yml`: Latest version without login and security features
|
||||
- `docker-compose-latest-security.yml`: Latest version with login and security features enabled
|
||||
- `docker-compose-latest-fat-security.yml`: Fat version with login and security features enabled
|
||||
- `docker-compose-latest.yml`: Latest version without security features
|
||||
- `docker-compose-latest-security.yml`: Latest version with security features enabled
|
||||
- `docker-compose-latest-fat-security.yml`: Fat version with security features enabled
|
||||
|
||||
These files provide pre-configured setups for different scenarios. For example, here's a snippet from `docker-compose-latest-security.yml`:
|
||||
|
||||
@@ -124,7 +124,7 @@ These files provide pre-configured setups for different scenarios. For example,
|
||||
services:
|
||||
stirling-pdf:
|
||||
container_name: Stirling-PDF-Security
|
||||
image: docker.stirlingpdf.com/stirlingtools/stirling-pdf:latest
|
||||
image: stirlingtools/stirling-pdf:latest
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
@@ -137,11 +137,11 @@ services:
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- ./stirling/latest/data:/usr/share/tessdata:rw
|
||||
- ./stirling/latest/config:/configs:rw
|
||||
- ./stirling/latest/logs:/logs:rw
|
||||
- /stirling/latest/data:/usr/share/tessdata:rw
|
||||
- /stirling/latest/config:/configs:rw
|
||||
- /stirling/latest/logs:/logs:rw
|
||||
environment:
|
||||
DISABLE_ADDITIONAL_FEATURES: "false"
|
||||
DOCKER_ENABLE_SECURITY: "true"
|
||||
SECURITY_ENABLELOGIN: "true"
|
||||
PUID: 1002
|
||||
PGID: 1002
|
||||
@@ -153,7 +153,6 @@ services:
|
||||
SYSTEM_MAXFILESIZE: "100"
|
||||
METRICS_ENABLED: "true"
|
||||
SYSTEM_GOOGLEVISIBILITY: "true"
|
||||
SHOW_SURVEY: "true"
|
||||
restart: on-failure:5
|
||||
```
|
||||
|
||||
@@ -170,7 +169,7 @@ Stirling-PDF uses different Docker images for various configurations. The build
|
||||
1. Set the security environment variable:
|
||||
|
||||
```bash
|
||||
export DISABLE_ADDITIONAL_FEATURES=true # or false for to enable login and security features for builds
|
||||
export DOCKER_ENABLE_SECURITY=false # or true for security-enabled builds
|
||||
```
|
||||
|
||||
2. Build the project with Gradle:
|
||||
@@ -193,10 +192,10 @@ Stirling-PDF uses different Docker images for various configurations. The build
|
||||
docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t stirlingtools/stirling-pdf:latest-ultra-lite -f ./Dockerfile.ultra-lite .
|
||||
```
|
||||
|
||||
For the fat version (with login and security features enabled):
|
||||
For the fat version (with security enabled):
|
||||
|
||||
```bash
|
||||
export DISABLE_ADDITIONAL_FEATURES=false
|
||||
export DOCKER_ENABLE_SECURITY=true
|
||||
docker build --no-cache --pull --build-arg VERSION_TAG=alpha -t stirlingtools/stirling-pdf:latest-fat -f ./Dockerfile.fat .
|
||||
```
|
||||
|
||||
@@ -272,7 +271,7 @@ Important notes:
|
||||
|
||||
6. Push your changes to your fork.
|
||||
7. Submit a pull request to the main repository.
|
||||
8. See additional [contributing guidelines](../CONTRIBUTING.md).
|
||||
8. See additional [contributing guidelines](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/CONTRIBUTING.md).
|
||||
|
||||
When you raise a PR:
|
||||
|
||||
@@ -332,7 +331,7 @@ Thymeleaf is a server-side Java HTML template engine. It is used in Stirling-PDF
|
||||
|
||||
### Thymeleaf overview
|
||||
|
||||
In Stirling-PDF, Thymeleaf is used to create HTML templates that are rendered on the server side. These templates are located in the `app/core/src/main/resources/templates` directory. Thymeleaf templates use a combination of HTML and special Thymeleaf attributes to dynamically generate content.
|
||||
In Stirling-PDF, Thymeleaf is used to create HTML templates that are rendered on the server side. These templates are located in the `src/main/resources/templates` directory. Thymeleaf templates use a combination of HTML and special Thymeleaf attributes to dynamically generate content.
|
||||
|
||||
Some examples of this are:
|
||||
|
||||
@@ -384,7 +383,7 @@ This would generate n entries of tr for each person in exampleData
|
||||
### Adding a New Feature to the Backend (API)
|
||||
|
||||
1. **Create a New Controller:**
|
||||
- Create a new Java class in the `app/core/src/main/java/stirling/software/SPDF/controller/api` directory.
|
||||
- Create a new Java class in the `src/main/java/stirling/software/SPDF/controller/api` directory.
|
||||
- Annotate the class with `@RestController` and `@RequestMapping` to define the API endpoint.
|
||||
- Ensure to add API documentation annotations like `@Tag(name = "General", description = "General APIs")` and `@Operation(summary = "Crops a PDF document", description = "This operation takes an input PDF file and crops it according to the given coordinates. Input:PDF Output:PDF Type:SISO")`.
|
||||
|
||||
@@ -411,7 +410,7 @@ This would generate n entries of tr for each person in exampleData
|
||||
```
|
||||
|
||||
2. **Define the Service Layer:** (Not required but often useful)
|
||||
- Create a new service class in the `app/core/src/main/java/stirling/software/SPDF/service` directory.
|
||||
- Create a new service class in the `src/main/java/stirling/software/SPDF/service` directory.
|
||||
- Implement the business logic for the new feature.
|
||||
|
||||
```java
|
||||
@@ -463,7 +462,7 @@ This would generate n entries of tr for each person in exampleData
|
||||
### Adding a New Feature to the Frontend (UI)
|
||||
|
||||
1. **Create a New Thymeleaf Template:**
|
||||
- Create a new HTML file in the `app/core/src/main/resources/templates` directory.
|
||||
- Create a new HTML file in the `src/main/resources/templates` directory.
|
||||
- Use Thymeleaf attributes to dynamically generate content.
|
||||
- Use `extract-page.html` as a base example for the HTML template, which is useful to ensure importing of the general layout, navbar, and footer.
|
||||
|
||||
@@ -507,7 +506,7 @@ This would generate n entries of tr for each person in exampleData
|
||||
```
|
||||
|
||||
2. **Create a New Controller for the UI:**
|
||||
- Create a new Java class in the `app/core/src/main/java/stirling/software/SPDF/controller/ui` directory.
|
||||
- Create a new Java class in the `src/main/java/stirling/software/SPDF/controller/ui` directory.
|
||||
- Annotate the class with `@Controller` and `@RequestMapping` to define the UI endpoint.
|
||||
|
||||
```java
|
||||
@@ -537,11 +536,11 @@ This would generate n entries of tr for each person in exampleData
|
||||
|
||||
3. **Update the Navigation Bar:**
|
||||
- Add a link to the new feature page in the navigation bar.
|
||||
- Update the `app/core/src/main/resources/templates/fragments/navbar.html` file.
|
||||
- Update the `src/main/resources/templates/fragments/navbar.html` file.
|
||||
|
||||
```html
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" th:href="@{'/new-feature'}">New Feature</a>
|
||||
<a class="nav-link" th:href="@{/new-feature}">New Feature</a>
|
||||
</li>
|
||||
```
|
||||
|
||||
@@ -551,7 +550,7 @@ When adding a new feature or modifying existing ones in Stirling-PDF, you'll nee
|
||||
|
||||
### 1. Locate Existing Language Files
|
||||
|
||||
Find the existing `messages.properties` files in the `app/core/src/main/resources` directory. You'll see files like:
|
||||
Find the existing `messages.properties` files in the `src/main/resources` directory. You'll see files like:
|
||||
|
||||
- `messages.properties` (default, usually English)
|
||||
- `messages_en_GB.properties`
|
||||
+47
-61
@@ -1,10 +1,12 @@
|
||||
# Main stage
|
||||
FROM alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1
|
||||
FROM alpine:3.21.2@sha256:56fa17d2a7e7f168a043a2712e63aed1f8543aeafdcee47c58dcffe38ed51099
|
||||
|
||||
# Copy necessary files
|
||||
COPY scripts /scripts
|
||||
COPY app/core/src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/
|
||||
COPY app/core/build/libs/*.jar app.jar
|
||||
COPY pipeline /pipeline
|
||||
COPY src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/
|
||||
#COPY src/main/resources/static/fonts/*.otf /usr/share/fonts/opentype/noto/
|
||||
COPY build/libs/*.jar app.jar
|
||||
|
||||
ARG VERSION_TAG
|
||||
|
||||
@@ -21,81 +23,65 @@ LABEL org.opencontainers.image.version="${VERSION_TAG}"
|
||||
LABEL org.opencontainers.image.keywords="PDF, manipulation, merge, split, convert, OCR, watermark"
|
||||
|
||||
# Set Environment Variables
|
||||
ENV DISABLE_ADDITIONAL_FEATURES=true \
|
||||
ENV DOCKER_ENABLE_SECURITY=false \
|
||||
VERSION_TAG=$VERSION_TAG \
|
||||
JAVA_BASE_OPTS="-XX:+UnlockExperimentalVMOptions -XX:MaxRAMPercentage=75 -XX:InitiatingHeapOccupancyPercent=20 -XX:+G1PeriodicGCInvokesConcurrent -XX:G1PeriodicGCInterval=10000 -XX:+UseStringDeduplication -XX:G1PeriodicGCSystemLoadThreshold=70" \
|
||||
JAVA_CUSTOM_OPTS="" \
|
||||
JAVA_TOOL_OPTIONS="-XX:+UnlockExperimentalVMOptions \
|
||||
-XX:MaxRAMPercentage=75 \
|
||||
-XX:InitiatingHeapOccupancyPercent=20 \
|
||||
-XX:+G1PeriodicGCInvokesConcurrent \
|
||||
-XX:G1PeriodicGCInterval=10000 \
|
||||
-XX:+UseStringDeduplication \
|
||||
-XX:G1PeriodicGCSystemLoadThreshold=70" \
|
||||
HOME=/home/stirlingpdfuser \
|
||||
PUID=1000 \
|
||||
PGID=1000 \
|
||||
UMASK=022 \
|
||||
PYTHONPATH=/usr/lib/libreoffice/program:/opt/venv/lib/python3.12/site-packages \
|
||||
UNO_PATH=/usr/lib/libreoffice/program \
|
||||
URE_BOOTSTRAP=file:///usr/lib/libreoffice/program/fundamentalrc \
|
||||
PATH=$PATH:/opt/venv/bin \
|
||||
STIRLING_TEMPFILES_DIRECTORY=/tmp/stirling-pdf \
|
||||
TMPDIR=/tmp/stirling-pdf \
|
||||
TEMP=/tmp/stirling-pdf \
|
||||
TMP=/tmp/stirling-pdf
|
||||
UMASK=022
|
||||
|
||||
|
||||
# JDK for app
|
||||
RUN echo "@main https://dl-cdn.alpinelinux.org/alpine/edge/main" | tee -a /etc/apk/repositories && \
|
||||
echo "@community https://dl-cdn.alpinelinux.org/alpine/edge/community" | tee -a /etc/apk/repositories && \
|
||||
RUN echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/main" | tee -a /etc/apk/repositories && \
|
||||
echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/community" | tee -a /etc/apk/repositories && \
|
||||
echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" | tee -a /etc/apk/repositories && \
|
||||
apk upgrade --no-cache -a && \
|
||||
apk add --no-cache \
|
||||
ca-certificates \
|
||||
tzdata \
|
||||
tini \
|
||||
bash \
|
||||
curl \
|
||||
shadow \
|
||||
su-exec \
|
||||
openssl \
|
||||
openssl-dev \
|
||||
openjdk21-jre \
|
||||
# Doc conversion
|
||||
gcompat \
|
||||
libc6-compat \
|
||||
libreoffice \
|
||||
# pdftohtml
|
||||
poppler-utils \
|
||||
# OCR MY PDF (unpaper for descew and other advanced features)
|
||||
tesseract-ocr-data-eng \
|
||||
tesseract-ocr-data-chi_sim \
|
||||
tesseract-ocr-data-deu \
|
||||
tesseract-ocr-data-fra \
|
||||
tesseract-ocr-data-por \
|
||||
unpaper \
|
||||
# CV
|
||||
py3-opencv \
|
||||
python3 \
|
||||
ocrmypdf \
|
||||
py3-pip \
|
||||
py3-pillow@testing \
|
||||
py3-pdf2image@testing \
|
||||
# URW Base 35 fonts for better PDF rendering
|
||||
font-urw-base35 && \
|
||||
python3 -m venv /opt/venv && \
|
||||
/opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools && \
|
||||
/opt/venv/bin/pip install --no-cache-dir --upgrade unoserver weasyprint && \
|
||||
ln -s /usr/lib/libreoffice/program/uno.py /opt/venv/lib/python3.12/site-packages/ && \
|
||||
ln -s /usr/lib/libreoffice/program/unohelper.py /opt/venv/lib/python3.12/site-packages/ && \
|
||||
ln -s /usr/lib/libreoffice/program /opt/venv/lib/python3.12/site-packages/LibreOffice && \
|
||||
ca-certificates \
|
||||
tzdata \
|
||||
tini \
|
||||
bash \
|
||||
curl \
|
||||
qpdf \
|
||||
shadow \
|
||||
su-exec \
|
||||
openssl \
|
||||
openssl-dev \
|
||||
openjdk21-jre \
|
||||
# Doc conversion
|
||||
gcompat \
|
||||
libc6-compat \
|
||||
libreoffice \
|
||||
# pdftohtml
|
||||
poppler-utils \
|
||||
# OCR MY PDF (unpaper for descew and other advanced features)
|
||||
tesseract-ocr-data-eng \
|
||||
# CV
|
||||
py3-opencv \
|
||||
# python3/pip
|
||||
python3 \
|
||||
py3-pip && \
|
||||
# uno unoconv and HTML
|
||||
pip install --break-system-packages --no-cache-dir --upgrade unoconv WeasyPrint pdf2image pillow && \
|
||||
mv /usr/share/tessdata /usr/share/tessdata-original && \
|
||||
mkdir -p $HOME /configs /logs /customFiles /pipeline/watchedFolders /pipeline/finishedFolders /tmp/stirling-pdf && \
|
||||
# Configure URW Base 35 fonts
|
||||
ln -s /usr/share/fontconfig/conf.avail/69-urw-*.conf /etc/fonts/conf.d/ && \
|
||||
mkdir -p $HOME /configs /logs /customFiles /pipeline/watchedFolders /pipeline/finishedFolders && \
|
||||
fc-cache -f -v && \
|
||||
chmod +x /scripts/* && \
|
||||
# User permissions
|
||||
chmod +x /scripts/init.sh && \
|
||||
# User permissions
|
||||
addgroup -S stirlingpdfgroup && adduser -S stirlingpdfuser -G stirlingpdfgroup && \
|
||||
chown -R stirlingpdfuser:stirlingpdfgroup $HOME /scripts /usr/share/fonts/opentype/noto /configs /customFiles /pipeline /tmp/stirling-pdf && \
|
||||
chown -R stirlingpdfuser:stirlingpdfgroup $HOME /scripts /usr/share/fonts/opentype/noto /configs /customFiles /pipeline && \
|
||||
chown stirlingpdfuser:stirlingpdfgroup /app.jar
|
||||
|
||||
EXPOSE 8080/tcp
|
||||
|
||||
# Set user and run command
|
||||
ENTRYPOINT ["tini", "--", "/scripts/init.sh"]
|
||||
CMD ["sh", "-c", "java -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/tmp/stirling-pdf -jar /app.jar & /opt/venv/bin/unoserver --port 2003 --interface 127.0.0.1"]
|
||||
CMD ["java", "-Dfile.encoding=UTF-8", "-jar", "/app.jar"]
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
# dockerfile.dev
|
||||
|
||||
# Basisimage: Gradle mit JDK 17 (Debian-basiert)
|
||||
FROM gradle:8.14-jdk17
|
||||
|
||||
# Als Root-Benutzer arbeiten, um benötigte Pakete zu installieren
|
||||
USER root
|
||||
|
||||
# Set GRADLE_HOME und füge Gradle zum PATH hinzu
|
||||
ENV GRADLE_HOME=/opt/gradle
|
||||
ENV PATH="$GRADLE_HOME/bin:$PATH"
|
||||
|
||||
# Update und Installation zusätzlicher Pakete (Debian/Ubuntu-basiert)
|
||||
RUN apt-get update && apt-get install -y \
|
||||
sudo \
|
||||
libreoffice \
|
||||
poppler-utils \
|
||||
qpdf \
|
||||
# settings.yml | tessdataDir: /usr/share/tesseract-ocr/5/tessdata
|
||||
tesseract-ocr \
|
||||
tesseract-ocr-eng \
|
||||
fonts-terminus fonts-dejavu fonts-font-awesome fonts-noto fonts-noto-core fonts-noto-cjk fonts-noto-extra fonts-liberation fonts-linuxlibertine fonts-urw-base35 \
|
||||
python3-uno \
|
||||
python3-venv \
|
||||
# ss -tln
|
||||
iproute2 \
|
||||
&& apt-get clean && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Setze die Environment Variable für setuptools
|
||||
ENV SETUPTOOLS_USE_DISTUTILS=local \
|
||||
STIRLING_TEMPFILES_DIRECTORY=/tmp/stirling-pdf \
|
||||
TMPDIR=/tmp/stirling-pdf \
|
||||
TEMP=/tmp/stirling-pdf \
|
||||
TMP=/tmp/stirling-pdf
|
||||
|
||||
# Installation der benötigten Python-Pakete
|
||||
RUN python3 -m venv --system-site-packages /opt/venv \
|
||||
&& . /opt/venv/bin/activate \
|
||||
&& pip install --no-cache-dir --upgrade pip setuptools \
|
||||
&& pip install --no-cache-dir WeasyPrint pdf2image pillow unoserver opencv-python-headless pre-commit
|
||||
|
||||
# Füge den venv-Pfad zur globalen PATH-Variable hinzu, damit die Tools verfügbar sind
|
||||
ENV PATH="/opt/venv/bin:$PATH"
|
||||
|
||||
COPY . /workspace
|
||||
|
||||
RUN mkdir -p /tmp/stirling-pdf \
|
||||
&& fc-cache -f -v \
|
||||
&& adduser --disabled-password --gecos '' devuser \
|
||||
&& chown -R devuser:devuser /home/devuser /workspace /tmp/stirling-pdf
|
||||
RUN echo "devuser ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/devuser \
|
||||
&& chmod 0440 /etc/sudoers.d/devuser
|
||||
|
||||
# Setze das Arbeitsverzeichnis (wird später per Bind-Mount überschrieben)
|
||||
WORKDIR /workspace
|
||||
|
||||
RUN chmod +x /workspace/.devcontainer/git-init.sh /workspace/.devcontainer/init-setup.sh
|
||||
|
||||
# Wechsel zum Nicht‑Root Benutzer
|
||||
USER devuser
|
||||
+52
-75
@@ -1,14 +1,5 @@
|
||||
# Build the application
|
||||
FROM gradle:8.14-jdk21 AS build
|
||||
|
||||
COPY build.gradle .
|
||||
COPY settings.gradle .
|
||||
COPY gradlew .
|
||||
COPY gradle gradle/
|
||||
COPY app/core/build.gradle core/.
|
||||
COPY app/common/build.gradle common/.
|
||||
COPY app/proprietary/build.gradle proprietary/.
|
||||
RUN ./gradlew build -x spotlessApply -x spotlessCheck -x test -x sonarqube || return 0
|
||||
FROM gradle:8.12-jdk17 AS build
|
||||
|
||||
# Set the working directory
|
||||
WORKDIR /app
|
||||
@@ -16,98 +7,84 @@ WORKDIR /app
|
||||
# Copy the entire project to the working directory
|
||||
COPY . .
|
||||
|
||||
# Build the application with DISABLE_ADDITIONAL_FEATURES=false
|
||||
RUN DISABLE_ADDITIONAL_FEATURES=false \
|
||||
STIRLING_PDF_DESKTOP_UI=false \
|
||||
./gradlew clean build -x spotlessApply -x spotlessCheck -x test -x sonarqube
|
||||
# Build the application with DOCKER_ENABLE_SECURITY=false
|
||||
RUN DOCKER_ENABLE_SECURITY=true \
|
||||
./gradlew clean build
|
||||
|
||||
# Main stage
|
||||
FROM alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1
|
||||
FROM alpine:3.21.2@sha256:56fa17d2a7e7f168a043a2712e63aed1f8543aeafdcee47c58dcffe38ed51099
|
||||
|
||||
# Copy necessary files
|
||||
COPY scripts /scripts
|
||||
COPY app/core/src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/
|
||||
# first /app directory is for the build stage, second is for the final image
|
||||
COPY --from=build /app/app/core/build/libs/*.jar app.jar
|
||||
COPY pipeline /pipeline
|
||||
COPY src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/
|
||||
COPY --from=build /app/build/libs/*.jar app.jar
|
||||
|
||||
ARG VERSION_TAG
|
||||
|
||||
# Set Environment Variables
|
||||
ENV DISABLE_ADDITIONAL_FEATURES=true \
|
||||
ENV DOCKER_ENABLE_SECURITY=false \
|
||||
VERSION_TAG=$VERSION_TAG \
|
||||
JAVA_BASE_OPTS="-XX:+UnlockExperimentalVMOptions -XX:MaxRAMPercentage=75 -XX:InitiatingHeapOccupancyPercent=20 -XX:+G1PeriodicGCInvokesConcurrent -XX:G1PeriodicGCInterval=10000 -XX:+UseStringDeduplication -XX:G1PeriodicGCSystemLoadThreshold=70" \
|
||||
JAVA_CUSTOM_OPTS="" \
|
||||
JAVA_TOOL_OPTIONS="-XX:+UnlockExperimentalVMOptions \
|
||||
-XX:MaxRAMPercentage=75 \
|
||||
-XX:InitiatingHeapOccupancyPercent=20 \
|
||||
-XX:+G1PeriodicGCInvokesConcurrent \
|
||||
-XX:G1PeriodicGCInterval=10000 \
|
||||
-XX:+UseStringDeduplication \
|
||||
-XX:G1PeriodicGCSystemLoadThreshold=70" \
|
||||
HOME=/home/stirlingpdfuser \
|
||||
PUID=1000 \
|
||||
PGID=1000 \
|
||||
UMASK=022 \
|
||||
FAT_DOCKER=true \
|
||||
INSTALL_BOOK_AND_ADVANCED_HTML_OPS=false \
|
||||
PYTHONPATH=/usr/lib/libreoffice/program:/opt/venv/lib/python3.12/site-packages \
|
||||
UNO_PATH=/usr/lib/libreoffice/program \
|
||||
URE_BOOTSTRAP=file:///usr/lib/libreoffice/program/fundamentalrc \
|
||||
PATH=$PATH:/opt/venv/bin \
|
||||
STIRLING_TEMPFILES_DIRECTORY=/tmp/stirling-pdf \
|
||||
TMPDIR=/tmp/stirling-pdf \
|
||||
TEMP=/tmp/stirling-pdf \
|
||||
TMP=/tmp/stirling-pdf
|
||||
INSTALL_BOOK_AND_ADVANCED_HTML_OPS=false
|
||||
|
||||
|
||||
# JDK for app
|
||||
RUN echo "@main https://dl-cdn.alpinelinux.org/alpine/edge/main" | tee -a /etc/apk/repositories && \
|
||||
echo "@community https://dl-cdn.alpinelinux.org/alpine/edge/community" | tee -a /etc/apk/repositories && \
|
||||
RUN echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/main" | tee -a /etc/apk/repositories && \
|
||||
echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/community" | tee -a /etc/apk/repositories && \
|
||||
echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" | tee -a /etc/apk/repositories && \
|
||||
apk upgrade --no-cache -a && \
|
||||
apk add --no-cache \
|
||||
ca-certificates \
|
||||
tzdata \
|
||||
tini \
|
||||
bash \
|
||||
curl \
|
||||
shadow \
|
||||
su-exec \
|
||||
openssl \
|
||||
openssl-dev \
|
||||
openjdk21-jre \
|
||||
# Doc conversion
|
||||
gcompat \
|
||||
libc6-compat \
|
||||
libreoffice \
|
||||
# pdftohtml
|
||||
poppler-utils \
|
||||
# OCR MY PDF (unpaper for descew and other advanced featues)
|
||||
tesseract-ocr-data-eng \
|
||||
tesseract-ocr-data-chi_sim \
|
||||
tesseract-ocr-data-deu \
|
||||
tesseract-ocr-data-fra \
|
||||
tesseract-ocr-data-por \
|
||||
unpaper \
|
||||
font-terminus font-dejavu font-noto font-noto-cjk font-awesome font-noto-extra font-liberation font-linux-libertine font-urw-base35 \
|
||||
# CV
|
||||
py3-opencv \
|
||||
python3 \
|
||||
ocrmypdf \
|
||||
py3-pip \
|
||||
py3-pillow@testing \
|
||||
py3-pdf2image@testing && \
|
||||
python3 -m venv /opt/venv && \
|
||||
/opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools && \
|
||||
/opt/venv/bin/pip install --no-cache-dir --upgrade unoserver weasyprint && \
|
||||
ln -s /usr/lib/libreoffice/program/uno.py /opt/venv/lib/python3.12/site-packages/ && \
|
||||
ln -s /usr/lib/libreoffice/program/unohelper.py /opt/venv/lib/python3.12/site-packages/ && \
|
||||
ln -s /usr/lib/libreoffice/program /opt/venv/lib/python3.12/site-packages/LibreOffice && \
|
||||
ca-certificates \
|
||||
tzdata \
|
||||
tini \
|
||||
bash \
|
||||
curl \
|
||||
shadow \
|
||||
su-exec \
|
||||
openssl \
|
||||
openssl-dev \
|
||||
openjdk21-jre \
|
||||
# Doc conversion
|
||||
gcompat \
|
||||
libc6-compat \
|
||||
libreoffice \
|
||||
# pdftohtml
|
||||
poppler-utils \
|
||||
# OCR MY PDF (unpaper for descew and other advanced featues)
|
||||
qpdf \
|
||||
tesseract-ocr-data-eng \
|
||||
font-terminus font-dejavu font-noto font-noto-cjk font-awesome font-noto-extra \
|
||||
# CV
|
||||
py3-opencv \
|
||||
# python3/pip
|
||||
python3 \
|
||||
py3-pip && \
|
||||
# uno unoconv and HTML
|
||||
pip install --break-system-packages --no-cache-dir --upgrade unoconv WeasyPrint pdf2image pillow && \
|
||||
mv /usr/share/tessdata /usr/share/tessdata-original && \
|
||||
mkdir -p $HOME /configs /logs /customFiles /pipeline/watchedFolders /pipeline/finishedFolders /tmp/stirling-pdf && \
|
||||
# Configure URW Base 35 fonts
|
||||
ln -s /usr/share/fontconfig/conf.avail/69-urw-*.conf /etc/fonts/conf.d/ && \
|
||||
mkdir -p $HOME /configs /logs /customFiles /pipeline/watchedFolders /pipeline/finishedFolders && \
|
||||
fc-cache -f -v && \
|
||||
chmod +x /scripts/* && \
|
||||
# User permissions
|
||||
chmod +x /scripts/init.sh && \
|
||||
# User permissions
|
||||
addgroup -S stirlingpdfgroup && adduser -S stirlingpdfuser -G stirlingpdfgroup && \
|
||||
chown -R stirlingpdfuser:stirlingpdfgroup $HOME /scripts /usr/share/fonts/opentype/noto /configs /customFiles /pipeline /tmp/stirling-pdf && \
|
||||
chown -R stirlingpdfuser:stirlingpdfgroup $HOME /scripts /usr/share/fonts/opentype/noto /configs /customFiles /pipeline && \
|
||||
chown stirlingpdfuser:stirlingpdfgroup /app.jar
|
||||
|
||||
EXPOSE 8080/tcp
|
||||
|
||||
# Set user and run command
|
||||
ENTRYPOINT ["tini", "--", "/scripts/init.sh"]
|
||||
CMD ["sh", "-c", "java -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/tmp/stirling-pdf -jar /app.jar & /opt/venv/bin/unoserver --port 2003 --interface 127.0.0.1"]
|
||||
CMD ["java", "-Dfile.encoding=UTF-8", "-jar", "/app.jar"]
|
||||
|
||||
+15
-13
@@ -1,27 +1,29 @@
|
||||
# use alpine
|
||||
FROM alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1
|
||||
FROM alpine:3.21.2@sha256:56fa17d2a7e7f168a043a2712e63aed1f8543aeafdcee47c58dcffe38ed51099
|
||||
|
||||
ARG VERSION_TAG
|
||||
|
||||
# Set Environment Variables
|
||||
ENV DISABLE_ADDITIONAL_FEATURES=true \
|
||||
ENV DOCKER_ENABLE_SECURITY=false \
|
||||
HOME=/home/stirlingpdfuser \
|
||||
VERSION_TAG=$VERSION_TAG \
|
||||
JAVA_BASE_OPTS="-XX:+UnlockExperimentalVMOptions -XX:MaxRAMPercentage=75 -XX:InitiatingHeapOccupancyPercent=20 -XX:+G1PeriodicGCInvokesConcurrent -XX:G1PeriodicGCInterval=10000 -XX:+UseStringDeduplication -XX:G1PeriodicGCSystemLoadThreshold=70" \
|
||||
JAVA_CUSTOM_OPTS="" \
|
||||
JAVA_TOOL_OPTIONS="-XX:+UnlockExperimentalVMOptions \
|
||||
-XX:MaxRAMPercentage=75 \
|
||||
-XX:InitiatingHeapOccupancyPercent=20 \
|
||||
-XX:+G1PeriodicGCInvokesConcurrent \
|
||||
-XX:G1PeriodicGCInterval=10000 \
|
||||
-XX:+UseStringDeduplication \
|
||||
-XX:G1PeriodicGCSystemLoadThreshold=70" \
|
||||
PUID=1000 \
|
||||
PGID=1000 \
|
||||
UMASK=022 \
|
||||
STIRLING_TEMPFILES_DIRECTORY=/tmp/stirling-pdf \
|
||||
TMPDIR=/tmp/stirling-pdf \
|
||||
TEMP=/tmp/stirling-pdf \
|
||||
TMP=/tmp/stirling-pdf
|
||||
UMASK=022
|
||||
|
||||
# Copy necessary files
|
||||
COPY scripts/download-security-jar.sh /scripts/download-security-jar.sh
|
||||
COPY scripts/init-without-ocr.sh /scripts/init-without-ocr.sh
|
||||
COPY scripts/installFonts.sh /scripts/installFonts.sh
|
||||
COPY app/core/build/libs/*.jar app.jar
|
||||
COPY pipeline /pipeline
|
||||
COPY build/libs/*.jar app.jar
|
||||
|
||||
# Set up necessary directories and permissions
|
||||
RUN echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/main" | tee -a /etc/apk/repositories && \
|
||||
@@ -38,10 +40,10 @@ RUN echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/main" | tee -a /et
|
||||
su-exec \
|
||||
openjdk21-jre && \
|
||||
# User permissions
|
||||
mkdir -p /configs /logs /customFiles /usr/share/fonts/opentype/noto /tmp/stirling-pdf /pipeline/watchedFolders /pipeline/finishedFolders && \
|
||||
mkdir -p /configs /logs /customFiles /usr/share/fonts/opentype/noto && \
|
||||
chmod +x /scripts/*.sh && \
|
||||
addgroup -S stirlingpdfgroup && adduser -S stirlingpdfuser -G stirlingpdfgroup && \
|
||||
chown -R stirlingpdfuser:stirlingpdfgroup $HOME /scripts /pipeline /configs /customFiles /tmp/stirling-pdf && \
|
||||
chown -R stirlingpdfuser:stirlingpdfgroup $HOME /scripts /configs /customFiles /pipeline && \
|
||||
chown stirlingpdfuser:stirlingpdfgroup /app.jar
|
||||
|
||||
# Set environment variables
|
||||
@@ -51,4 +53,4 @@ EXPOSE 8080/tcp
|
||||
|
||||
# Run the application
|
||||
ENTRYPOINT ["tini", "--", "/scripts/init-without-ocr.sh"]
|
||||
CMD ["java", "-Dfile.encoding=UTF-8", "-Djava.io.tmpdir=/tmp/stirling-pdf", "-jar", "/app.jar"]
|
||||
CMD ["java", "-Dfile.encoding=UTF-8", "-jar", "/app.jar"]
|
||||
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
# Intermediate stage for running dos2unix
|
||||
FROM alpine:3.14 AS dos2unix-converter
|
||||
|
||||
# Install dos2unix
|
||||
RUN apk add --no-cache dos2unix
|
||||
|
||||
# Copy script files
|
||||
COPY ./scripts/* /dos2unix-scripts/
|
||||
|
||||
# Convert line endings
|
||||
RUN dos2unix /dos2unix-scripts/*
|
||||
|
||||
|
||||
|
||||
# Build stage
|
||||
FROM alpine:3.14 AS downloader
|
||||
|
||||
# Install necessary utilities
|
||||
RUN apk add --no-cache curl jq
|
||||
|
||||
# Set work directory
|
||||
WORKDIR /downloader
|
||||
|
||||
# Fetch the latest release jar from GitHub
|
||||
RUN curl -s https://api.github.com/repos/Stirling-Tools/Stirling-PDF/releases/latest \
|
||||
| jq -r ".assets[] | select(.name == \"Stirling-PDF.jar\") | .browser_download_url" \
|
||||
| xargs curl -L -o Stirling-PDF.jar
|
||||
|
||||
|
||||
# Main stage
|
||||
FROM alpine:3.21.2@sha256:56fa17d2a7e7f168a043a2712e63aed1f8543aeafdcee47c58dcffe38ed51099
|
||||
|
||||
# Copy necessary files
|
||||
COPY scripts /scripts
|
||||
COPY pipeline /pipeline
|
||||
COPY src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/
|
||||
|
||||
# Copy the application JAR file from the download stage
|
||||
COPY --from=downloader /downloader/Stirling-PDF.jar /app.jar
|
||||
|
||||
ARG VERSION_TAG
|
||||
|
||||
# Set Environment Variables
|
||||
ENV DOCKER_ENABLE_SECURITY=false \
|
||||
VERSION_TAG=$VERSION_TAG \
|
||||
JAVA_TOOL_OPTIONS="-XX:+UnlockExperimentalVMOptions \
|
||||
-XX:MaxRAMPercentage=75 \
|
||||
-XX:InitiatingHeapOccupancyPercent=20 \
|
||||
-XX:+G1PeriodicGCInvokesConcurrent \
|
||||
-XX:G1PeriodicGCInterval=10000 \
|
||||
-XX:+UseStringDeduplication \
|
||||
-XX:G1PeriodicGCSystemLoadThreshold=70" \
|
||||
HOME=/home/stirlingpdfuser \
|
||||
PUID=1000 \
|
||||
PGID=1000 \
|
||||
UMASK=022 \
|
||||
FAT_DOCKER=true \
|
||||
INSTALL_BOOK_AND_ADVANCED_HTML_OPS=false
|
||||
|
||||
|
||||
# JDK for app
|
||||
RUN echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/main" | tee -a /etc/apk/repositories && \
|
||||
echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/community" | tee -a /etc/apk/repositories && \
|
||||
echo "@testing https://dl-cdn.alpinelinux.org/alpine/edge/testing" | tee -a /etc/apk/repositories && \
|
||||
apk upgrade --no-cache -a && \
|
||||
apk add --no-cache \
|
||||
ca-certificates \
|
||||
tzdata \
|
||||
tini \
|
||||
bash \
|
||||
curl \
|
||||
shadow \
|
||||
su-exec \
|
||||
openssl \
|
||||
openssl-dev \
|
||||
openjdk21-jre \
|
||||
# Doc conversion
|
||||
gcompat \
|
||||
libc6-compat \
|
||||
libreoffice \
|
||||
# pdftohtml
|
||||
poppler-utils \
|
||||
# OCR MY PDF (unpaper for descew and other advanced featues)
|
||||
qpdf \
|
||||
tesseract-ocr-data-eng \
|
||||
font-terminus font-dejavu font-noto font-noto-cjk font-awesome font-noto-extra \
|
||||
# CV
|
||||
# py3-opencv \
|
||||
# python3/pip
|
||||
python3 \
|
||||
py3-pip && \
|
||||
# uno unoconv and HTML
|
||||
pip install --break-system-packages --no-cache-dir --upgrade unoconv WeasyPrint pdf2image pillow && \
|
||||
mv /usr/share/tessdata /usr/share/tessdata-original && \
|
||||
mkdir -p $HOME /configs /logs /customFiles /pipeline/watchedFolders /pipeline/finishedFolders && \
|
||||
fc-cache -f -v && \
|
||||
chmod +x /scripts/* && \
|
||||
chmod +x /scripts/init.sh && \
|
||||
# User permissions
|
||||
addgroup -S stirlingpdfgroup && adduser -S stirlingpdfuser -G stirlingpdfgroup && \
|
||||
chown -R stirlingpdfuser:stirlingpdfgroup $HOME /scripts /usr/share/fonts/opentype/noto /configs /customFiles /pipeline && \
|
||||
chown stirlingpdfuser:stirlingpdfgroup /app.jar
|
||||
|
||||
EXPOSE 8080/tcp
|
||||
|
||||
# Set user and run command
|
||||
ENTRYPOINT ["tini", "--", "/scripts/init.sh"]
|
||||
CMD ["java", "-Dfile.encoding=UTF-8", "-jar", "/app.jar"]
|
||||
@@ -10,13 +10,15 @@ Fork Stirling-PDF and create a new branch out of `main`.
|
||||
|
||||
Then add a reference to the language in the navbar by adding a new language entry to the dropdown:
|
||||
|
||||
- Edit the file: [languages.html](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/app/core/src/main/resources/templates/fragments/languages.html)
|
||||
- Edit the file: [languages.html](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/src/main/resources/templates/fragments/languages.html)
|
||||
- Add a flag SVG file to: [flags directory](https://github.com/Stirling-Tools/Stirling-PDF/tree/main/src/main/resources/static/images/flags)
|
||||
|
||||
Any SVG flags are fine; most of the current ones were sourced from [here](https://flagicons.lipis.dev/). If your language isn't represented by a flag, choose a similar one, such as Saudi Arabia's flag for Arabic.
|
||||
|
||||
For example, to add Polish, you would add:
|
||||
|
||||
```html
|
||||
<div th:replace="~{fragments/languageEntry :: languageEntry ('pl_PL', 'Polski')}" ></div>
|
||||
<a th:if="${#lists.isEmpty(@languages) or #lists.contains(@languages, 'pl_PL')}" class="dropdown-item lang_dropdown-item" href="" data-bs-language-code="pl_PL"> <img th:src="@{'/images/flags/pl.svg'}" alt="icon" width="20" height="15"> Polski</a>
|
||||
```
|
||||
|
||||
The `data-bs-language-code` is the code used to reference the file in the next step.
|
||||
@@ -25,7 +27,7 @@ The `data-bs-language-code` is the code used to reference the file in the next s
|
||||
|
||||
Start by copying the existing English property file:
|
||||
|
||||
- [messages_en_GB.properties](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/app/core/src/main/resources/messages_en_GB.properties)
|
||||
- [messages_en_GB.properties](https://github.com/Stirling-Tools/Stirling-PDF/blob/main/src/main/resources/messages_en_GB.properties)
|
||||
|
||||
Copy and rename it to `messages_{your data-bs-language-code here}.properties`. In the Polish example, you would set the name to `messages_pl_PL.properties`.
|
||||
|
||||
@@ -61,16 +63,8 @@ Make sure to place the entry under the correct language section. This helps main
|
||||
|
||||
#### Windows command
|
||||
|
||||
```powershell
|
||||
python .github/scripts/check_language_properties.py --reference-file app\core\src\main\resources\messages_en_GB.properties --branch "" --files app\core\src\main\resources\messages_pl_PL.properties
|
||||
```ps
|
||||
python .github/scripts/check_language_properties.py --reference-file src\main\resources\messages_en_GB.properties --branch "" --files src\main\resources\messages_pl_PL.properties
|
||||
|
||||
python .github/scripts/check_language_properties.py --reference-file app\core\src\main\resources\messages_en_GB.properties --branch "" --check-file app\core\src\main\resources\messages_pl_PL.properties
|
||||
```
|
||||
|
||||
#### Linux command
|
||||
|
||||
```bash
|
||||
python3 .github/scripts/check_language_properties.py --reference-file app/core/src/main/resources/messages_en_GB.properties --branch "" --files app/core/src/main/resources/messages_pl_PL.properties
|
||||
|
||||
python3 .github/scripts/check_language_properties.py --reference-file app/core/src/main/resources/messages_en_GB.properties --branch "" --check-file app/core/src/main/resources/messages_pl_PL.properties
|
||||
python .github/scripts/check_language_properties.py --reference-file src\main\resources\messages_en_GB.properties --branch "" --check-file src\main\resources\messages_pl_PL.properties
|
||||
```
|
||||
@@ -1,13 +1,6 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2025 Stirling PDF Inc.
|
||||
|
||||
Portions of this software are licensed as follows:
|
||||
|
||||
* All content that resides under the "app/proprietary/" directory of this repository,
|
||||
if that directory exists, is licensed under the license defined in "app/proprietary/LICENSE".
|
||||
* Content outside of the above mentioned directories or restrictions above is
|
||||
available under the MIT License as defined below.
|
||||
Copyright (c) 2024 Stirling Tools
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
[](https://hub.docker.com/r/frooodle/s-pdf)
|
||||
[](https://discord.gg/HYmhKj45pU)
|
||||
[](https://github.com/Stirling-Tools/Stirling-PDF/)
|
||||
[](https://scorecard.dev/viewer/?uri=github.com/Stirling-Tools/Stirling-PDF)
|
||||
[](https://github.com/Stirling-Tools/stirling-pdf)
|
||||
|
||||
@@ -29,7 +30,7 @@ All documentation available at [https://docs.stirlingpdf.com/](https://docs.stir
|
||||
- API for integration with external scripts
|
||||
- Optional Login and Authentication support (see [here](https://docs.stirlingpdf.com/Advanced%20Configuration/System%20and%20Security) for documentation)
|
||||
- Database Backup and Import (see [here](https://docs.stirlingpdf.com/Advanced%20Configuration/DATABASE) for documentation)
|
||||
- Enterprise features like SSO (see [here](https://docs.stirlingpdf.com/Advanced%20Configuration/Single%20Sign-On%20Configuration) for documentation)
|
||||
- Enterprise features like SSO see [here](https://docs.stirlingpdf.com/Enterprise%20Edition)
|
||||
|
||||
## PDF Features
|
||||
|
||||
@@ -112,63 +113,63 @@ Visit our comprehensive documentation at [docs.stirlingpdf.com](https://docs.sti
|
||||
|
||||
## Supported Languages
|
||||
|
||||
Stirling-PDF currently supports 40 languages!
|
||||
Stirling-PDF currently supports 39 languages!
|
||||
|
||||
| Language | Progress |
|
||||
| -------------------------------------------- | -------------------------------------- |
|
||||
| Arabic (العربية) (ar_AR) |  |
|
||||
| Azerbaijani (Azərbaycan Dili) (az_AZ) |  |
|
||||
| Basque (Euskara) (eu_ES) |  |
|
||||
| Bulgarian (Български) (bg_BG) |  |
|
||||
| Catalan (Català) (ca_CA) |  |
|
||||
| Croatian (Hrvatski) (hr_HR) |  |
|
||||
| Czech (Česky) (cs_CZ) |  |
|
||||
| Danish (Dansk) (da_DK) |  |
|
||||
| Dutch (Nederlands) (nl_NL) |  |
|
||||
| Arabic (العربية) (ar_AR) |  |
|
||||
| Azerbaijani (Azərbaycan Dili) (az_AZ) |  |
|
||||
| Basque (Euskara) (eu_ES) |  |
|
||||
| Bulgarian (Български) (bg_BG) |  |
|
||||
| Catalan (Català) (ca_CA) |  |
|
||||
| Croatian (Hrvatski) (hr_HR) |  |
|
||||
| Czech (Česky) (cs_CZ) |  |
|
||||
| Danish (Dansk) (da_DK) |  |
|
||||
| Dutch (Nederlands) (nl_NL) |  |
|
||||
| English (English) (en_GB) |  |
|
||||
| English (US) (en_US) |  |
|
||||
| French (Français) (fr_FR) |  |
|
||||
| German (Deutsch) (de_DE) |  |
|
||||
| Greek (Ελληνικά) (el_GR) |  |
|
||||
| Hindi (हिंदी) (hi_IN) |  |
|
||||
| Hungarian (Magyar) (hu_HU) |  |
|
||||
| Indonesian (Bahasa Indonesia) (id_ID) |  |
|
||||
| Irish (Gaeilge) (ga_IE) |  |
|
||||
| Italian (Italiano) (it_IT) |  |
|
||||
| Japanese (日本語) (ja_JP) |  |
|
||||
| Korean (한국어) (ko_KR) |  |
|
||||
| Norwegian (Norsk) (no_NB) |  |
|
||||
| Persian (فارسی) (fa_IR) |  |
|
||||
| Polish (Polski) (pl_PL) |  |
|
||||
| Portuguese (Português) (pt_PT) |  |
|
||||
| Portuguese Brazilian (Português) (pt_BR) |  |
|
||||
| Romanian (Română) (ro_RO) |  |
|
||||
| Russian (Русский) (ru_RU) |  |
|
||||
| Serbian Latin alphabet (Srpski) (sr_LATN_RS) |  |
|
||||
| Simplified Chinese (简体中文) (zh_CN) |  |
|
||||
| Slovakian (Slovensky) (sk_SK) |  |
|
||||
| Slovenian (Slovenščina) (sl_SI) |  |
|
||||
| Spanish (Español) (es_ES) |  |
|
||||
| Swedish (Svenska) (sv_SE) |  |
|
||||
| Thai (ไทย) (th_TH) |  |
|
||||
| Tibetan (བོད་ཡིག་) (bo_CN) |  |
|
||||
| Traditional Chinese (繁體中文) (zh_TW) |  |
|
||||
| French (Français) (fr_FR) |  |
|
||||
| German (Deutsch) (de_DE) |  |
|
||||
| Greek (Ελληνικά) (el_GR) |  |
|
||||
| Hindi (हिंदी) (hi_IN) |  |
|
||||
| Hungarian (Magyar) (hu_HU) |  |
|
||||
| Indonesian (Bahasa Indonesia) (id_ID) |  |
|
||||
| Irish (Gaeilge) (ga_IE) |  |
|
||||
| Italian (Italiano) (it_IT) |  |
|
||||
| Japanese (日本語) (ja_JP) |  |
|
||||
| Korean (한국어) (ko_KR) |  |
|
||||
| Norwegian (Norsk) (no_NB) |  |
|
||||
| Persian (فارسی) (fa_IR) |  |
|
||||
| Polish (Polski) (pl_PL) |  |
|
||||
| Portuguese (Português) (pt_PT) |  |
|
||||
| Portuguese Brazilian (Português) (pt_BR) |  |
|
||||
| Romanian (Română) (ro_RO) |  |
|
||||
| Russian (Русский) (ru_RU) |  |
|
||||
| Serbian Latin alphabet (Srpski) (sr_LATN_RS) |  |
|
||||
| Simplified Chinese (简体中文) (zh_CN) |  |
|
||||
| Slovakian (Slovensky) (sk_SK) |  |
|
||||
| Slovenian (Slovenščina) (sl_SI) |  |
|
||||
| Spanish (Español) (es_ES) |  |
|
||||
| Swedish (Svenska) (sv_SE) |  |
|
||||
| Thai (ไทย) (th_TH) |  |
|
||||
| Tibetan (བོད་ཡིག་) (zh_BO) |  |
|
||||
| Traditional Chinese (繁體中文) (zh_TW) |  |
|
||||
| Turkish (Türkçe) (tr_TR) |  |
|
||||
| Ukrainian (Українська) (uk_UA) |  |
|
||||
| Vietnamese (Tiếng Việt) (vi_VN) |  |
|
||||
| Malayalam (മലയാളം) (ml_IN) |  |
|
||||
| Vietnamese (Tiếng Việt) (vi_VN) |  |
|
||||
|
||||
|
||||
## Stirling PDF Enterprise
|
||||
|
||||
Stirling PDF offers an Enterprise edition of its software. This is the same great software but with added features, support and comforts.
|
||||
Check out our [Enterprise docs](https://docs.stirlingpdf.com/Pro)
|
||||
Check out our [Enterprise docs](https://docs.stirlingpdf.com/Enterprise%20Edition)
|
||||
|
||||
|
||||
## 🤝 Looking to contribute?
|
||||
|
||||
Join our community:
|
||||
- [Contribution Guidelines](CONTRIBUTING.md)
|
||||
- [Translation Guide (How to add custom languages)](devGuide/HowToAddNewLanguage.md)
|
||||
- [Developer Guide](devGuide/DeveloperGuide.md)
|
||||
- [Translation Guide (How to add custom languages)](HowToAddNewLanguage.md)
|
||||
- [Issue Tracker](https://github.com/Stirling-Tools/Stirling-PDF/issues)
|
||||
- [Discord Community](https://discord.gg/HYmhKj45pU)
|
||||
- [Developer Guide](DeveloperGuide.md)
|
||||
|
||||
@@ -124,18 +124,10 @@
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Eclipse Public License 1.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Eclipse Public License - v 1.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Eclipse Public License v2.0"
|
||||
},
|
||||
{
|
||||
"moduleName": ".*",
|
||||
"moduleLicense": "Eclipse Public License v. 2.0"
|
||||
@@ -1,196 +0,0 @@
|
||||
### Eclipse ###
|
||||
.metadata
|
||||
bin/
|
||||
tmp/
|
||||
*.tmp
|
||||
*.bak
|
||||
*.exe
|
||||
*.swp
|
||||
*~.nib
|
||||
local.properties
|
||||
.settings/
|
||||
.loadpath
|
||||
.recommenders
|
||||
.classpath
|
||||
.project
|
||||
version.properties
|
||||
|
||||
#### Stirling-PDF Files ###
|
||||
pipeline/watchedFolders/
|
||||
pipeline/finishedFolders/
|
||||
customFiles/
|
||||
configs/
|
||||
watchedFolders/
|
||||
clientWebUI/
|
||||
!cucumber/
|
||||
!cucumber/exampleFiles/
|
||||
!cucumber/exampleFiles/example_html.zip
|
||||
exampleYmlFiles/stirling/
|
||||
/testing/file_snapshots
|
||||
SwaggerDoc.json
|
||||
|
||||
# Gradle
|
||||
.gradle
|
||||
.lock
|
||||
|
||||
# External tool builders
|
||||
.externalToolBuilders/
|
||||
|
||||
# Locally stored "Eclipse launch configurations"
|
||||
*.launch
|
||||
|
||||
# PyDev specific (Python IDE for Eclipse)
|
||||
*.pydevproject
|
||||
|
||||
# CDT-specific (C/C++ Development Tooling)
|
||||
.cproject
|
||||
|
||||
# CDT- autotools
|
||||
.autotools
|
||||
|
||||
# Java annotation processor (APT)
|
||||
.factorypath
|
||||
|
||||
# PDT-specific (PHP Development Tools)
|
||||
.buildpath
|
||||
|
||||
# sbteclipse plugin
|
||||
.target
|
||||
|
||||
# Tern plugin
|
||||
.tern-project
|
||||
|
||||
# TeXlipse plugin
|
||||
.texlipse
|
||||
|
||||
# STS (Spring Tool Suite)
|
||||
.springBeans
|
||||
|
||||
# Code Recommenders
|
||||
.recommenders/
|
||||
|
||||
# Annotation Processing
|
||||
.apt_generated/
|
||||
.apt_generated_test/
|
||||
|
||||
# Scala IDE specific (Scala & Java development for Eclipse)
|
||||
.cache-main
|
||||
.scala_dependencies
|
||||
.worksheet
|
||||
|
||||
# Uncomment this line if you wish to ignore the project description file.
|
||||
# Typically, this file would be tracked if it contains build/dependency configurations:
|
||||
#.project
|
||||
|
||||
### Eclipse Patch ###
|
||||
# Spring Boot Tooling
|
||||
.sts4-cache/
|
||||
|
||||
### Git ###
|
||||
# Created by git for backups. To disable backups in Git:
|
||||
# $ git config --global mergetool.keepBackup false
|
||||
*.orig
|
||||
|
||||
# Created by git when using merge tools for conflicts
|
||||
*.BACKUP.*
|
||||
*.BASE.*
|
||||
*.LOCAL.*
|
||||
*.REMOTE.*
|
||||
*_BACKUP_*.txt
|
||||
*_BASE_*.txt
|
||||
*_LOCAL_*.txt
|
||||
*_REMOTE_*.txt
|
||||
|
||||
### Java ###
|
||||
# Compiled class file
|
||||
*.class
|
||||
|
||||
# Log file
|
||||
*.log
|
||||
|
||||
# BlueJ files
|
||||
*.ctxt
|
||||
|
||||
# Mobile Tools for Java (J2ME)
|
||||
.mtj.tmp/
|
||||
|
||||
# Package Files #
|
||||
*.jar
|
||||
*.war
|
||||
*.nar
|
||||
*.ear
|
||||
*.zip
|
||||
*.tar.gz
|
||||
*.rar
|
||||
*.db
|
||||
/build
|
||||
/app/common/build/
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*.pyo
|
||||
|
||||
# Virtual environments
|
||||
.env*
|
||||
.venv*
|
||||
env*/
|
||||
venv*/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# VS Code
|
||||
/.vscode/**/*
|
||||
!/.vscode/settings.json
|
||||
!/.vscode/extensions.json
|
||||
|
||||
# IntelliJ IDEA
|
||||
.idea/
|
||||
*.iml
|
||||
out/
|
||||
|
||||
# Ignore Mac DS_Store files
|
||||
.DS_Store
|
||||
**/.DS_Store
|
||||
|
||||
# cucumber
|
||||
/cucumber/reports/**
|
||||
|
||||
# Certs and Security Files
|
||||
*.p12
|
||||
*.pk8
|
||||
*.pem
|
||||
*.crt
|
||||
*.cer
|
||||
*.cert
|
||||
*.der
|
||||
*.key
|
||||
*.csr
|
||||
*.kdbx
|
||||
*.jks
|
||||
*.asc
|
||||
|
||||
# SSH Keys
|
||||
*.pub
|
||||
*.priv
|
||||
id_rsa
|
||||
id_rsa.pub
|
||||
id_ecdsa
|
||||
id_ecdsa.pub
|
||||
id_ed25519
|
||||
id_ed25519.pub
|
||||
.ssh/
|
||||
*ssh
|
||||
|
||||
# cache
|
||||
.cache
|
||||
.ruff_cache
|
||||
.mypy_cache
|
||||
.pytest_cache
|
||||
.ipynb_checkpoints
|
||||
|
||||
**/jcef-bundle/
|
||||
|
||||
# node_modules
|
||||
node_modules/
|
||||
@@ -1,45 +0,0 @@
|
||||
// Configure bootRun to disable it or point to a main class
|
||||
bootRun {
|
||||
enabled = false
|
||||
}
|
||||
spotless {
|
||||
java {
|
||||
target 'src/**/java/**/*.java'
|
||||
googleJavaFormat(googleJavaFormatVersion).aosp().reorderImports(false)
|
||||
|
||||
importOrder("java", "javax", "org", "com", "net", "io", "jakarta", "lombok", "me", "stirling")
|
||||
toggleOffOn()
|
||||
trimTrailingWhitespace()
|
||||
leadingTabsToSpaces()
|
||||
endWithNewline()
|
||||
}
|
||||
yaml {
|
||||
target '**/*.yml', '**/*.yaml'
|
||||
trimTrailingWhitespace()
|
||||
leadingTabsToSpaces()
|
||||
endWithNewline()
|
||||
}
|
||||
format 'gradle', {
|
||||
target '**/gradle/*.gradle', '**/*.gradle'
|
||||
trimTrailingWhitespace()
|
||||
leadingTabsToSpaces()
|
||||
endWithNewline()
|
||||
}
|
||||
}
|
||||
dependencies {
|
||||
api 'org.springframework.boot:spring-boot-starter-web'
|
||||
api 'org.springframework.boot:spring-boot-starter-aop'
|
||||
api 'org.springframework.boot:spring-boot-starter-thymeleaf'
|
||||
api 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20240325.1'
|
||||
api 'com.fathzer:javaluator:3.0.6'
|
||||
api 'com.posthog.java:posthog:1.2.0'
|
||||
api 'org.apache.commons:commons-lang3:3.18.0'
|
||||
api 'com.drewnoakes:metadata-extractor:2.19.0' // Image metadata extractor
|
||||
api 'com.vladsch.flexmark:flexmark-html2md-converter:0.64.8'
|
||||
api "org.apache.pdfbox:pdfbox:$pdfboxVersion"
|
||||
api 'jakarta.servlet:jakarta.servlet-api:6.1.0'
|
||||
api 'org.snakeyaml:snakeyaml-engine:2.10'
|
||||
api "org.springdoc:springdoc-openapi-starter-webmvc-ui:2.8.9"
|
||||
api 'jakarta.mail:jakarta.mail-api:2.1.3'
|
||||
runtimeOnly 'org.eclipse.angus:angus-mail:2.0.3'
|
||||
}
|
||||
-39
@@ -1,39 +0,0 @@
|
||||
package org.apache.pdfbox.examples.util;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.apache.pdfbox.io.RandomAccessReadBufferedFile;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/** A custom RandomAccessRead implementation that deletes the file when closed */
|
||||
@Slf4j
|
||||
public class DeletingRandomAccessFile extends RandomAccessReadBufferedFile {
|
||||
private final Path tempFilePath;
|
||||
|
||||
public DeletingRandomAccessFile(File file) throws IOException {
|
||||
super(file);
|
||||
this.tempFilePath = file.toPath();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException {
|
||||
try {
|
||||
super.close();
|
||||
} finally {
|
||||
try {
|
||||
boolean deleted = Files.deleteIfExists(tempFilePath);
|
||||
if (deleted) {
|
||||
log.info("Successfully deleted temp file: {}", tempFilePath);
|
||||
} else {
|
||||
log.warn("Failed to delete temp file (may not exist): {}", tempFilePath);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Error deleting temp file: {}", tempFilePath, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package stirling.software.common.annotations;
|
||||
|
||||
import java.lang.annotation.*;
|
||||
|
||||
import org.springframework.core.annotation.AliasFor;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestMethod;
|
||||
|
||||
/**
|
||||
* Shortcut for a POST endpoint that is executed through the Stirling "auto‑job" framework.
|
||||
*
|
||||
* <p>Behaviour notes:
|
||||
*
|
||||
* <ul>
|
||||
* <li>The endpoint is registered with {@code POST} and, by default, consumes {@code
|
||||
* multipart/form-data} unless you override {@link #consumes()}.
|
||||
* <li>When the client supplies {@code ?async=true} the call is handed to {@link
|
||||
* stirling.software.common.service.JobExecutorService JobExecutorService} where it may be
|
||||
* queued, retried, tracked and subject to time‑outs. For synchronous (default) invocations
|
||||
* these advanced options are ignored.
|
||||
* <li>Progress information (see {@link #trackProgress()}) is stored in {@link
|
||||
* stirling.software.common.service.TaskManager TaskManager} and can be polled via <code>
|
||||
* GET /api/v1/general/job/{id}</code>.
|
||||
* </ul>
|
||||
*
|
||||
* <p>Unless stated otherwise an attribute only affects <em>async</em> execution.
|
||||
*/
|
||||
@Target(ElementType.METHOD)
|
||||
@Retention(RetentionPolicy.RUNTIME)
|
||||
@Documented
|
||||
@RequestMapping(method = RequestMethod.POST)
|
||||
public @interface AutoJobPostMapping {
|
||||
|
||||
/** Alias for {@link RequestMapping#value} – the path mapping of the endpoint. */
|
||||
@AliasFor(annotation = RequestMapping.class, attribute = "value")
|
||||
String[] value() default {};
|
||||
|
||||
/** MIME types this endpoint accepts. Defaults to {@code multipart/form-data}. */
|
||||
@AliasFor(annotation = RequestMapping.class, attribute = "consumes")
|
||||
String[] consumes() default {"multipart/form-data"};
|
||||
|
||||
/**
|
||||
* Maximum execution time in milliseconds before the job is aborted. A negative value means "use
|
||||
* the application default".
|
||||
*
|
||||
* <p>Only honoured when {@code async=true}.
|
||||
*/
|
||||
long timeout() default -1;
|
||||
|
||||
/**
|
||||
* Total number of attempts (initial + retries). Must be at least 1. Retries are executed
|
||||
* with exponential back‑off.
|
||||
*
|
||||
* <p>Only honoured when {@code async=true}.
|
||||
*/
|
||||
int retryCount() default 1;
|
||||
|
||||
/**
|
||||
* Record percentage / note updates so they can be retrieved via the REST status endpoint.
|
||||
*
|
||||
* <p>Only honoured when {@code async=true}.
|
||||
*/
|
||||
boolean trackProgress() default true;
|
||||
|
||||
/**
|
||||
* If {@code true} the job may be placed in a queue instead of being rejected when resources are
|
||||
* scarce.
|
||||
*
|
||||
* <p>Only honoured when {@code async=true}.
|
||||
*/
|
||||
boolean queueable() default false;
|
||||
|
||||
/**
|
||||
* Relative resource weight (1–100) used by the scheduler to prioritise / throttle jobs. Values
|
||||
* below 1 are clamped to 1, values above 100 to 100.
|
||||
*/
|
||||
int resourceWeight() default 50;
|
||||
}
|
||||
@@ -1,289 +0,0 @@
|
||||
package stirling.software.common.aop;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.Duration;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.aspectj.lang.ProceedingJoinPoint;
|
||||
import org.aspectj.lang.annotation.*;
|
||||
import org.springframework.core.annotation.Order;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.annotations.AutoJobPostMapping;
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
import stirling.software.common.service.FileOrUploadService;
|
||||
import stirling.software.common.service.FileStorage;
|
||||
import stirling.software.common.service.JobExecutorService;
|
||||
|
||||
@Aspect
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
@Order(0) // Highest precedence - executes before audit aspects
|
||||
public class AutoJobAspect {
|
||||
|
||||
private static final Duration RETRY_BASE_DELAY = Duration.ofMillis(100);
|
||||
|
||||
private final JobExecutorService jobExecutorService;
|
||||
private final HttpServletRequest request;
|
||||
private final FileOrUploadService fileOrUploadService;
|
||||
private final FileStorage fileStorage;
|
||||
|
||||
@Around("@annotation(autoJobPostMapping)")
|
||||
public Object wrapWithJobExecution(
|
||||
ProceedingJoinPoint joinPoint, AutoJobPostMapping autoJobPostMapping) {
|
||||
// This aspect will run before any audit aspects due to @Order(0)
|
||||
// Extract parameters from the request and annotation
|
||||
boolean async = Boolean.parseBoolean(request.getParameter("async"));
|
||||
log.debug(
|
||||
"AutoJobAspect: Processing {} {} with async={}",
|
||||
request.getMethod(),
|
||||
request.getRequestURI(),
|
||||
async);
|
||||
long timeout = autoJobPostMapping.timeout();
|
||||
int retryCount = autoJobPostMapping.retryCount();
|
||||
boolean trackProgress = autoJobPostMapping.trackProgress();
|
||||
|
||||
log.debug(
|
||||
"AutoJobPostMapping execution with async={}, timeout={}, retryCount={}, trackProgress={}",
|
||||
async,
|
||||
timeout > 0 ? timeout : "default",
|
||||
retryCount,
|
||||
trackProgress);
|
||||
|
||||
// Process arguments in-place to avoid type mismatch issues
|
||||
Object[] args = processArgsInPlace(joinPoint.getArgs(), async);
|
||||
|
||||
// Extract queueable and resourceWeight parameters and validate
|
||||
boolean queueable = autoJobPostMapping.queueable();
|
||||
int resourceWeight = Math.max(1, Math.min(100, autoJobPostMapping.resourceWeight()));
|
||||
|
||||
// Integrate with the JobExecutorService
|
||||
if (retryCount <= 1) {
|
||||
// No retries needed, simple execution
|
||||
return jobExecutorService.runJobGeneric(
|
||||
async,
|
||||
() -> {
|
||||
try {
|
||||
// Note: Progress tracking is handled in TaskManager/JobExecutorService
|
||||
// The trackProgress flag controls whether detailed progress is stored
|
||||
// for REST API queries, not WebSocket notifications
|
||||
return joinPoint.proceed(args);
|
||||
} catch (Throwable ex) {
|
||||
log.error(
|
||||
"AutoJobAspect caught exception during job execution: {}",
|
||||
ex.getMessage(),
|
||||
ex);
|
||||
throw new RuntimeException(ex);
|
||||
}
|
||||
},
|
||||
timeout,
|
||||
queueable,
|
||||
resourceWeight);
|
||||
} else {
|
||||
// Use retry logic
|
||||
return executeWithRetries(
|
||||
joinPoint,
|
||||
args,
|
||||
async,
|
||||
timeout,
|
||||
retryCount,
|
||||
trackProgress,
|
||||
queueable,
|
||||
resourceWeight);
|
||||
}
|
||||
}
|
||||
|
||||
private Object executeWithRetries(
|
||||
ProceedingJoinPoint joinPoint,
|
||||
Object[] args,
|
||||
boolean async,
|
||||
long timeout,
|
||||
int maxRetries,
|
||||
boolean trackProgress,
|
||||
boolean queueable,
|
||||
int resourceWeight) {
|
||||
|
||||
// Keep jobId reference for progress tracking in TaskManager
|
||||
AtomicReference<String> jobIdRef = new AtomicReference<>();
|
||||
|
||||
return jobExecutorService.runJobGeneric(
|
||||
async,
|
||||
() -> {
|
||||
// Use iterative approach instead of recursion to avoid stack overflow
|
||||
Throwable lastException = null;
|
||||
|
||||
// Attempt counter starts at 1 for first try
|
||||
for (int currentAttempt = 1; currentAttempt <= maxRetries; currentAttempt++) {
|
||||
try {
|
||||
if (trackProgress && async) {
|
||||
// Get jobId for progress tracking in TaskManager
|
||||
// This enables REST API progress queries, not WebSocket
|
||||
if (jobIdRef.get() == null) {
|
||||
jobIdRef.set(getJobIdFromContext());
|
||||
}
|
||||
String jobId = jobIdRef.get();
|
||||
if (jobId != null) {
|
||||
log.debug(
|
||||
"Tracking progress for job {} (attempt {}/{})",
|
||||
jobId,
|
||||
currentAttempt,
|
||||
maxRetries);
|
||||
// Progress is tracked in TaskManager for REST API access
|
||||
// No WebSocket notifications sent here
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to execute the operation
|
||||
return joinPoint.proceed(args);
|
||||
|
||||
} catch (Throwable ex) {
|
||||
lastException = ex;
|
||||
log.error(
|
||||
"AutoJobAspect caught exception during job execution (attempt {}/{}): {}",
|
||||
currentAttempt,
|
||||
maxRetries,
|
||||
ex.getMessage(),
|
||||
ex);
|
||||
|
||||
// Check if we should retry
|
||||
if (currentAttempt < maxRetries) {
|
||||
log.info(
|
||||
"Retrying operation, attempt {}/{}",
|
||||
currentAttempt + 1,
|
||||
maxRetries);
|
||||
|
||||
if (trackProgress && async) {
|
||||
String jobId = jobIdRef.get();
|
||||
if (jobId != null) {
|
||||
log.debug(
|
||||
"Recording retry attempt for job {} in TaskManager",
|
||||
jobId);
|
||||
// Retry info is tracked in TaskManager for REST API access
|
||||
}
|
||||
}
|
||||
|
||||
// Use non-blocking delay for all retry attempts to avoid blocking
|
||||
// threads
|
||||
// For sync jobs this avoids starving the tomcat thread pool under
|
||||
// load
|
||||
long delayMs = RETRY_BASE_DELAY.toMillis() * currentAttempt;
|
||||
|
||||
// Execute the retry after a delay through the JobExecutorService
|
||||
// rather than blocking the current thread with sleep
|
||||
CompletableFuture<Object> delayedRetry = new CompletableFuture<>();
|
||||
|
||||
// Use a delayed executor for non-blocking delay
|
||||
CompletableFuture.delayedExecutor(delayMs, TimeUnit.MILLISECONDS)
|
||||
.execute(
|
||||
() -> {
|
||||
// Continue the retry loop in the next iteration
|
||||
// We can't return from here directly since
|
||||
// we're in a Runnable
|
||||
delayedRetry.complete(null);
|
||||
});
|
||||
|
||||
// Wait for the delay to complete before continuing
|
||||
try {
|
||||
delayedRetry.join();
|
||||
} catch (Exception e) {
|
||||
Thread.currentThread().interrupt();
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// No more retries, we'll throw the exception after the loop
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If we get here, all retries failed
|
||||
if (lastException != null) {
|
||||
throw new RuntimeException(
|
||||
"Job failed after "
|
||||
+ maxRetries
|
||||
+ " attempts: "
|
||||
+ lastException.getMessage(),
|
||||
lastException);
|
||||
}
|
||||
|
||||
// This should never happen if lastException is properly tracked
|
||||
throw new RuntimeException("Job failed but no exception was recorded");
|
||||
},
|
||||
timeout,
|
||||
queueable,
|
||||
resourceWeight);
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes arguments in-place to handle file resolution and async file persistence. This
|
||||
* approach avoids type mismatch issues by modifying the original objects directly.
|
||||
*
|
||||
* @param originalArgs The original arguments
|
||||
* @param async Whether this is an async operation
|
||||
* @return The original array with processed arguments
|
||||
*/
|
||||
private Object[] processArgsInPlace(Object[] originalArgs, boolean async) {
|
||||
if (originalArgs == null || originalArgs.length == 0) {
|
||||
return originalArgs;
|
||||
}
|
||||
|
||||
// Process all arguments in-place
|
||||
for (int i = 0; i < originalArgs.length; i++) {
|
||||
Object arg = originalArgs[i];
|
||||
|
||||
if (arg instanceof PDFFile pdfFile) {
|
||||
// Case 1: fileId is provided but no fileInput
|
||||
if (pdfFile.getFileInput() == null && pdfFile.getFileId() != null) {
|
||||
try {
|
||||
log.debug("Using fileId {} to get file content", pdfFile.getFileId());
|
||||
MultipartFile file = fileStorage.retrieveFile(pdfFile.getFileId());
|
||||
pdfFile.setFileInput(file);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(
|
||||
"Failed to resolve file by ID: " + pdfFile.getFileId(), e);
|
||||
}
|
||||
}
|
||||
// Case 2: For async requests, we need to make a copy of the MultipartFile
|
||||
else if (async && pdfFile.getFileInput() != null) {
|
||||
try {
|
||||
log.debug("Making persistent copy of uploaded file for async processing");
|
||||
MultipartFile originalFile = pdfFile.getFileInput();
|
||||
String fileId = fileStorage.storeFile(originalFile);
|
||||
|
||||
// Store the fileId for later reference
|
||||
pdfFile.setFileId(fileId);
|
||||
|
||||
// Replace the original MultipartFile with our persistent copy
|
||||
MultipartFile persistentFile = fileStorage.retrieveFile(fileId);
|
||||
pdfFile.setFileInput(persistentFile);
|
||||
|
||||
log.debug("Created persistent file copy with fileId: {}", fileId);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(
|
||||
"Failed to create persistent copy of uploaded file", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return originalArgs;
|
||||
}
|
||||
|
||||
private String getJobIdFromContext() {
|
||||
try {
|
||||
return (String) request.getAttribute("jobId");
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not retrieve job ID from context: {}", e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package stirling.software.common.config;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.util.TempFileRegistry;
|
||||
|
||||
/**
|
||||
* Configuration for the temporary file management system. Sets up the necessary beans and
|
||||
* configures system properties.
|
||||
*/
|
||||
@Slf4j
|
||||
@Configuration
|
||||
@RequiredArgsConstructor
|
||||
public class TempFileConfiguration {
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
/**
|
||||
* Create the TempFileRegistry bean.
|
||||
*
|
||||
* @return A new TempFileRegistry instance
|
||||
*/
|
||||
@Bean
|
||||
public TempFileRegistry tempFileRegistry() {
|
||||
return new TempFileRegistry();
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void initTempFileConfig() {
|
||||
try {
|
||||
ApplicationProperties.TempFileManagement tempFiles =
|
||||
applicationProperties.getSystem().getTempFileManagement();
|
||||
String customTempDirectory = tempFiles.getBaseTmpDir();
|
||||
|
||||
// Create the temp directory if it doesn't exist
|
||||
Path tempDir = Path.of(customTempDirectory);
|
||||
if (!Files.exists(tempDir)) {
|
||||
Files.createDirectories(tempDir);
|
||||
log.info("Created temporary directory: {}", tempDir);
|
||||
}
|
||||
|
||||
log.debug("Temporary file configuration initialized");
|
||||
log.debug("Using temp directory: {}", customTempDirectory);
|
||||
log.debug("Temp file prefix: {}", tempFiles.getPrefix());
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to initialize temporary file configuration", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
package stirling.software.common.config;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Set;
|
||||
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFileRegistry;
|
||||
|
||||
/**
|
||||
* Handles cleanup of temporary files on application shutdown. Implements Spring's DisposableBean
|
||||
* interface to ensure cleanup happens during normal application shutdown.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class TempFileShutdownHook implements DisposableBean {
|
||||
|
||||
private final TempFileRegistry registry;
|
||||
|
||||
public TempFileShutdownHook(TempFileRegistry registry) {
|
||||
this.registry = registry;
|
||||
|
||||
// Register a JVM shutdown hook as a backup in case Spring's
|
||||
// DisposableBean mechanism doesn't trigger (e.g., during a crash)
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(this::cleanupTempFiles));
|
||||
}
|
||||
|
||||
/** Spring's DisposableBean interface method. Called during normal application shutdown. */
|
||||
@Override
|
||||
public void destroy() {
|
||||
log.info("Application shutting down, cleaning up temporary files");
|
||||
cleanupTempFiles();
|
||||
}
|
||||
|
||||
/** Clean up all registered temporary files and directories. */
|
||||
private void cleanupTempFiles() {
|
||||
try {
|
||||
// Clean up all registered files
|
||||
Set<Path> files = registry.getAllRegisteredFiles();
|
||||
int deletedCount = 0;
|
||||
|
||||
for (Path file : files) {
|
||||
try {
|
||||
if (Files.exists(file)) {
|
||||
Files.deleteIfExists(file);
|
||||
deletedCount++;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to delete temp file during shutdown: {}", file, e);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up all registered directories
|
||||
Set<Path> directories = registry.getTempDirectories();
|
||||
for (Path dir : directories) {
|
||||
try {
|
||||
if (Files.exists(dir)) {
|
||||
GeneralUtils.deleteDirectory(dir);
|
||||
deletedCount++;
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to delete temp directory during shutdown: {}", dir, e);
|
||||
}
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Shutdown cleanup complete. Deleted {} temporary files/directories",
|
||||
deletedCount);
|
||||
|
||||
// Clear the registry
|
||||
registry.clear();
|
||||
} catch (Exception e) {
|
||||
log.error("Error during shutdown cleanup", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
-126
@@ -1,126 +0,0 @@
|
||||
package stirling.software.common.configuration;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.List;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.util.YamlHelper;
|
||||
|
||||
/**
|
||||
* A naive, line-based approach to merging "settings.yml" with "settings.yml.template" while
|
||||
* preserving exact whitespace, blank lines, and inline comments -- but we only rewrite the file if
|
||||
* the merged content actually differs.
|
||||
*/
|
||||
@Slf4j
|
||||
public class ConfigInitializer {
|
||||
|
||||
public void ensureConfigExists() throws IOException, URISyntaxException {
|
||||
// 1) If settings file doesn't exist, create from template
|
||||
Path destPath = Paths.get(InstallationPathConfig.getSettingsPath());
|
||||
if (Files.notExists(destPath)) {
|
||||
Files.createDirectories(destPath.getParent());
|
||||
try (InputStream in =
|
||||
getClass().getClassLoader().getResourceAsStream("settings.yml.template")) {
|
||||
if (in == null) {
|
||||
throw new FileNotFoundException(
|
||||
"Resource file not found: settings.yml.template");
|
||||
}
|
||||
Files.copy(in, destPath);
|
||||
}
|
||||
log.info("Created settings file from template");
|
||||
} else {
|
||||
// 2) Merge existing file with the template
|
||||
URL templateResource = getClass().getClassLoader().getResource("settings.yml.template");
|
||||
if (templateResource == null) {
|
||||
throw new IOException("Resource not found: settings.yml.template");
|
||||
}
|
||||
|
||||
// Copy template to a temp location so we can read lines
|
||||
Path tempTemplatePath = Files.createTempFile("settings.yml", ".template");
|
||||
try (InputStream in = templateResource.openStream()) {
|
||||
Files.copy(in, tempTemplatePath, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
|
||||
// Copy setting.yaml to a temp location so we can read lines
|
||||
Path settingTempPath = Files.createTempFile("settings", ".yaml");
|
||||
try (InputStream in = Files.newInputStream(destPath)) {
|
||||
Files.copy(in, settingTempPath, StandardCopyOption.REPLACE_EXISTING);
|
||||
}
|
||||
|
||||
YamlHelper settingsTemplateFile = new YamlHelper(tempTemplatePath);
|
||||
YamlHelper settingsFile = new YamlHelper(settingTempPath);
|
||||
|
||||
migrateEnterpriseEditionToPremium(settingsFile, settingsTemplateFile);
|
||||
|
||||
boolean changesMade =
|
||||
settingsTemplateFile.updateValuesFromYaml(settingsFile, settingsTemplateFile);
|
||||
if (changesMade) {
|
||||
settingsTemplateFile.save(destPath);
|
||||
log.info("Settings file updated based on template changes.");
|
||||
} else {
|
||||
log.info("No changes detected; settings file left as-is.");
|
||||
}
|
||||
|
||||
Files.deleteIfExists(tempTemplatePath);
|
||||
Files.deleteIfExists(settingTempPath);
|
||||
}
|
||||
|
||||
// 3) Ensure custom settings file exists
|
||||
Path customSettingsPath = Paths.get(InstallationPathConfig.getCustomSettingsPath());
|
||||
if (Files.notExists(customSettingsPath)) {
|
||||
Files.createFile(customSettingsPath);
|
||||
log.info("Created custom_settings file: {}", customSettingsPath);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Remove post migration
|
||||
private void migrateEnterpriseEditionToPremium(YamlHelper yaml, YamlHelper template) {
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "enabled") != null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "enabled"),
|
||||
yaml.getValueByExactKeyPath("enterpriseEdition", "enabled"));
|
||||
}
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "key") != null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "key"),
|
||||
yaml.getValueByExactKeyPath("enterpriseEdition", "key"));
|
||||
}
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "SSOAutoLogin") != null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "proFeatures", "SSOAutoLogin"),
|
||||
yaml.getValueByExactKeyPath("enterpriseEdition", "SSOAutoLogin"));
|
||||
}
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "autoUpdateMetadata")
|
||||
!= null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "proFeatures", "CustomMetadata", "autoUpdateMetadata"),
|
||||
yaml.getValueByExactKeyPath(
|
||||
"enterpriseEdition", "CustomMetadata", "autoUpdateMetadata"));
|
||||
}
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "author") != null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "proFeatures", "CustomMetadata", "author"),
|
||||
yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "author"));
|
||||
}
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "creator") != null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "proFeatures", "CustomMetadata", "creator"),
|
||||
yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "creator"));
|
||||
}
|
||||
if (yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "producer")
|
||||
!= null) {
|
||||
template.updateValue(
|
||||
List.of("premium", "proFeatures", "CustomMetadata", "producer"),
|
||||
yaml.getValueByExactKeyPath("enterpriseEdition", "CustomMetadata", "producer"));
|
||||
}
|
||||
}
|
||||
}
|
||||
-79
@@ -1,79 +0,0 @@
|
||||
package stirling.software.common.configuration;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.model.ApplicationProperties.CustomPaths.Operations;
|
||||
import stirling.software.common.model.ApplicationProperties.CustomPaths.Pipeline;
|
||||
|
||||
@Slf4j
|
||||
@Configuration
|
||||
@Getter
|
||||
public class RuntimePathConfig {
|
||||
private final ApplicationProperties properties;
|
||||
private final String basePath;
|
||||
private final String weasyPrintPath;
|
||||
private final String unoConvertPath;
|
||||
|
||||
// Pipeline paths
|
||||
private final String pipelineWatchedFoldersPath;
|
||||
private final String pipelineFinishedFoldersPath;
|
||||
private final String pipelineDefaultWebUiConfigs;
|
||||
private final String pipelinePath;
|
||||
|
||||
public RuntimePathConfig(ApplicationProperties properties) {
|
||||
this.properties = properties;
|
||||
this.basePath = InstallationPathConfig.getPath();
|
||||
|
||||
this.pipelinePath = Path.of(basePath, "pipeline").toString();
|
||||
String defaultWatchedFolders = Path.of(this.pipelinePath, "watchedFolders").toString();
|
||||
String defaultFinishedFolders = Path.of(this.pipelinePath, "finishedFolders").toString();
|
||||
String defaultWebUIConfigs = Path.of(this.pipelinePath, "defaultWebUIConfigs").toString();
|
||||
|
||||
Pipeline pipeline = properties.getSystem().getCustomPaths().getPipeline();
|
||||
|
||||
this.pipelineWatchedFoldersPath =
|
||||
resolvePath(
|
||||
defaultWatchedFolders,
|
||||
pipeline != null ? pipeline.getWatchedFoldersDir() : null);
|
||||
this.pipelineFinishedFoldersPath =
|
||||
resolvePath(
|
||||
defaultFinishedFolders,
|
||||
pipeline != null ? pipeline.getFinishedFoldersDir() : null);
|
||||
this.pipelineDefaultWebUiConfigs =
|
||||
resolvePath(
|
||||
defaultWebUIConfigs,
|
||||
pipeline != null ? pipeline.getWebUIConfigsDir() : null);
|
||||
|
||||
boolean isDocker = isRunningInDocker();
|
||||
|
||||
// Initialize Operation paths
|
||||
String defaultWeasyPrintPath = isDocker ? "/opt/venv/bin/weasyprint" : "weasyprint";
|
||||
String defaultUnoConvertPath = isDocker ? "/opt/venv/bin/unoconvert" : "unoconvert";
|
||||
|
||||
Operations operations = properties.getSystem().getCustomPaths().getOperations();
|
||||
this.weasyPrintPath =
|
||||
resolvePath(
|
||||
defaultWeasyPrintPath,
|
||||
operations != null ? operations.getWeasyprint() : null);
|
||||
this.unoConvertPath =
|
||||
resolvePath(
|
||||
defaultUnoConvertPath,
|
||||
operations != null ? operations.getUnoconvert() : null);
|
||||
}
|
||||
|
||||
private String resolvePath(String defaultPath, String customPath) {
|
||||
return StringUtils.isNotBlank(customPath) ? customPath : defaultPath;
|
||||
}
|
||||
|
||||
private boolean isRunningInDocker() {
|
||||
return Files.exists(Path.of("/.dockerenv"));
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
package stirling.software.common.model.api;
|
||||
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@EqualsAndHashCode
|
||||
public class PDFFile {
|
||||
@Schema(
|
||||
description = "The input PDF file",
|
||||
contentMediaType = "application/pdf",
|
||||
format = "binary")
|
||||
private MultipartFile fileInput;
|
||||
|
||||
@Schema(
|
||||
description = "File ID for server-side files (can be used instead of fileInput)",
|
||||
example = "a1b2c3d4-5678-90ab-cdef-ghijklmnopqr")
|
||||
private String fileId;
|
||||
}
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
package stirling.software.common.model.api.converters;
|
||||
|
||||
import io.swagger.v3.oas.annotations.media.Schema;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.EqualsAndHashCode;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
|
||||
@Data
|
||||
@EqualsAndHashCode(callSuper = true)
|
||||
public class EmlToPdfRequest extends PDFFile {
|
||||
|
||||
// fileInput is inherited from PDFFile
|
||||
|
||||
@Schema(
|
||||
description = "Include email attachments in the PDF output",
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED,
|
||||
example = "false")
|
||||
private boolean includeAttachments = false;
|
||||
|
||||
@Schema(
|
||||
description = "Maximum attachment size in MB to include (default 10MB, range: 1-100)",
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED,
|
||||
example = "10",
|
||||
minimum = "1",
|
||||
maximum = "100")
|
||||
private int maxAttachmentSizeMB = 10;
|
||||
|
||||
@Schema(
|
||||
description = "Download HTML intermediate file instead of PDF",
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED,
|
||||
example = "false")
|
||||
private boolean downloadHtml = false;
|
||||
|
||||
@Schema(
|
||||
description = "Include CC and BCC recipients in header (if available)",
|
||||
requiredMode = Schema.RequiredMode.NOT_REQUIRED,
|
||||
example = "true")
|
||||
private boolean includeAllRecipients = true;
|
||||
}
|
||||
-26
@@ -1,26 +0,0 @@
|
||||
package stirling.software.common.model.enumeration;
|
||||
|
||||
import lombok.Getter;
|
||||
|
||||
@Getter
|
||||
public enum UsernameAttribute {
|
||||
MAIL("mail"),
|
||||
EMAIL("email"),
|
||||
LOGIN("login"),
|
||||
PROFILE("profile"),
|
||||
NAME("name"),
|
||||
UID("uid"),
|
||||
USERNAME("username"),
|
||||
NICKNAME("nickname"),
|
||||
GIVEN_NAME("given_name"),
|
||||
MIDDLE_NAME("middle_name"),
|
||||
FAMILY_NAME("family_name"),
|
||||
PREFERRED_NAME("preferred_name"),
|
||||
PREFERRED_USERNAME("preferred_username");
|
||||
|
||||
private final String name;
|
||||
|
||||
UsernameAttribute(final String name) {
|
||||
this.name = name;
|
||||
}
|
||||
}
|
||||
-7
@@ -1,7 +0,0 @@
|
||||
package stirling.software.common.model.exception;
|
||||
|
||||
public class UnsupportedClaimException extends RuntimeException {
|
||||
public UnsupportedClaimException(String message) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package stirling.software.common.model.job;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class JobProgress {
|
||||
private String jobId;
|
||||
private String status;
|
||||
private int percentComplete;
|
||||
private String message;
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
package stirling.software.common.model.job;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class JobResponse<T> {
|
||||
private boolean async;
|
||||
private String jobId;
|
||||
private T result;
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
package stirling.software.common.model.job;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/** Represents the result of a job execution. Used by the TaskManager to store job results. */
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class JobResult {
|
||||
|
||||
/** The job ID */
|
||||
private String jobId;
|
||||
|
||||
/** Flag indicating if the job is complete */
|
||||
private boolean complete;
|
||||
|
||||
/** Error message if the job failed */
|
||||
private String error;
|
||||
|
||||
/** List of result files for jobs that produce files */
|
||||
@JsonIgnore private List<ResultFile> resultFiles;
|
||||
|
||||
/** Time when the job was created */
|
||||
private LocalDateTime createdAt;
|
||||
|
||||
/** Time when the job was completed */
|
||||
private LocalDateTime completedAt;
|
||||
|
||||
/** The actual result object, if not a file */
|
||||
private Object result;
|
||||
|
||||
/**
|
||||
* Notes attached to this job for tracking purposes. Uses CopyOnWriteArrayList for thread safety
|
||||
* when notes are added concurrently.
|
||||
*/
|
||||
private final List<String> notes = new CopyOnWriteArrayList<>();
|
||||
|
||||
/**
|
||||
* Create a new JobResult with the given job ID
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @return A new JobResult
|
||||
*/
|
||||
public static JobResult createNew(String jobId) {
|
||||
return JobResult.builder()
|
||||
.jobId(jobId)
|
||||
.complete(false)
|
||||
.createdAt(LocalDateTime.now())
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark this job as complete with a general result
|
||||
*
|
||||
* @param result The result object
|
||||
*/
|
||||
public void completeWithResult(Object result) {
|
||||
this.complete = true;
|
||||
this.result = result;
|
||||
this.completedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark this job as failed with an error message
|
||||
*
|
||||
* @param error The error message
|
||||
*/
|
||||
public void failWithError(String error) {
|
||||
this.complete = true;
|
||||
this.error = error;
|
||||
this.completedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark this job as complete with multiple file results
|
||||
*
|
||||
* @param resultFiles The list of result files
|
||||
*/
|
||||
public void completeWithFiles(List<ResultFile> resultFiles) {
|
||||
this.complete = true;
|
||||
this.resultFiles = new ArrayList<>(resultFiles);
|
||||
this.completedAt = LocalDateTime.now();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark this job as complete with a single file result (convenience method)
|
||||
*
|
||||
* @param fileId The file ID of the result
|
||||
* @param fileName The file name
|
||||
* @param contentType The content type of the file
|
||||
* @param fileSize The size of the file in bytes
|
||||
*/
|
||||
public void completeWithSingleFile(
|
||||
String fileId, String fileName, String contentType, long fileSize) {
|
||||
ResultFile resultFile =
|
||||
ResultFile.builder()
|
||||
.fileId(fileId)
|
||||
.fileName(fileName)
|
||||
.contentType(contentType)
|
||||
.fileSize(fileSize)
|
||||
.build();
|
||||
completeWithFiles(List.of(resultFile));
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if this job has file results
|
||||
*
|
||||
* @return true if this job has file results, false otherwise
|
||||
*/
|
||||
public boolean hasFiles() {
|
||||
return resultFiles != null && !resultFiles.isEmpty();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if this job has multiple file results
|
||||
*
|
||||
* @return true if this job has multiple file results, false otherwise
|
||||
*/
|
||||
public boolean hasMultipleFiles() {
|
||||
return resultFiles != null && resultFiles.size() > 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all result files
|
||||
*
|
||||
* @return List of result files
|
||||
*/
|
||||
public List<ResultFile> getAllResultFiles() {
|
||||
if (resultFiles != null && !resultFiles.isEmpty()) {
|
||||
return Collections.unmodifiableList(resultFiles);
|
||||
}
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a note to this job
|
||||
*
|
||||
* @param note The note to add
|
||||
*/
|
||||
public void addNote(String note) {
|
||||
this.notes.add(note);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all notes attached to this job
|
||||
*
|
||||
* @return An unmodifiable view of the notes list
|
||||
*/
|
||||
public List<String> getNotes() {
|
||||
return Collections.unmodifiableList(notes);
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package stirling.software.common.model.job;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/** Represents statistics about jobs in the system */
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class JobStats {
|
||||
|
||||
/** Total number of jobs (active and completed) */
|
||||
private int totalJobs;
|
||||
|
||||
/** Number of active (incomplete) jobs */
|
||||
private int activeJobs;
|
||||
|
||||
/** Number of completed jobs */
|
||||
private int completedJobs;
|
||||
|
||||
/** Number of failed jobs */
|
||||
private int failedJobs;
|
||||
|
||||
/** Number of successful jobs */
|
||||
private int successfulJobs;
|
||||
|
||||
/** Number of jobs with file results */
|
||||
private int fileResultJobs;
|
||||
|
||||
/** The oldest active job's creation timestamp */
|
||||
private LocalDateTime oldestActiveJobTime;
|
||||
|
||||
/** The newest active job's creation timestamp */
|
||||
private LocalDateTime newestActiveJobTime;
|
||||
|
||||
/** The average processing time for completed jobs in milliseconds */
|
||||
private long averageProcessingTimeMs;
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
package stirling.software.common.model.job;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/** Represents a single file result from a job execution */
|
||||
@Data
|
||||
@Builder
|
||||
@NoArgsConstructor
|
||||
@AllArgsConstructor
|
||||
public class ResultFile {
|
||||
|
||||
/** The file ID for accessing the file */
|
||||
private String fileId;
|
||||
|
||||
/** The original file name */
|
||||
private String fileName;
|
||||
|
||||
/** MIME type of the file */
|
||||
private String contentType;
|
||||
|
||||
/** Size of the file in bytes */
|
||||
private long fileSize;
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
package stirling.software.common.model.oauth2;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import stirling.software.common.model.enumeration.UsernameAttribute;
|
||||
|
||||
@NoArgsConstructor
|
||||
public class GitHubProvider extends Provider {
|
||||
|
||||
private static final String NAME = "github";
|
||||
private static final String CLIENT_NAME = "GitHub";
|
||||
private static final String AUTHORIZATION_URI = "https://github.com/login/oauth/authorize";
|
||||
private static final String TOKEN_URI = "https://github.com/login/oauth/access_token";
|
||||
private static final String USER_INFO_URI = "https://api.github.com/user";
|
||||
|
||||
public GitHubProvider(
|
||||
String clientId,
|
||||
String clientSecret,
|
||||
Collection<String> scopes,
|
||||
UsernameAttribute useAsUsername) {
|
||||
super(
|
||||
null,
|
||||
NAME,
|
||||
CLIENT_NAME,
|
||||
clientId,
|
||||
clientSecret,
|
||||
scopes,
|
||||
useAsUsername != null ? useAsUsername : UsernameAttribute.LOGIN,
|
||||
AUTHORIZATION_URI,
|
||||
TOKEN_URI,
|
||||
USER_INFO_URI);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getAuthorizationUri() {
|
||||
return AUTHORIZATION_URI;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTokenUri() {
|
||||
return TOKEN_URI;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getUserInfoUri() {
|
||||
return USER_INFO_URI;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getClientName() {
|
||||
return CLIENT_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<String> getScopes() {
|
||||
Collection<String> scopes = super.getScopes();
|
||||
|
||||
if (scopes == null || scopes.isEmpty()) {
|
||||
scopes = new ArrayList<>();
|
||||
scopes.add("read:user");
|
||||
}
|
||||
|
||||
return scopes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "GitHub [clientId="
|
||||
+ getClientId()
|
||||
+ ", clientSecret="
|
||||
+ (getClientSecret() != null && !getClientSecret().isEmpty() ? "*****" : "NULL")
|
||||
+ ", scopes="
|
||||
+ getScopes()
|
||||
+ ", useAsUsername="
|
||||
+ getUseAsUsername()
|
||||
+ "]";
|
||||
}
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
package stirling.software.common.model.oauth2;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import stirling.software.common.model.enumeration.UsernameAttribute;
|
||||
|
||||
@NoArgsConstructor
|
||||
public class GoogleProvider extends Provider {
|
||||
|
||||
private static final String NAME = "google";
|
||||
private static final String CLIENT_NAME = "Google";
|
||||
private static final String AUTHORIZATION_URI = "https://accounts.google.com/o/oauth2/v2/auth";
|
||||
private static final String TOKEN_URI = "https://www.googleapis.com/oauth2/v4/token";
|
||||
private static final String USER_INFO_URI =
|
||||
"https://www.googleapis.com/oauth2/v3/userinfo?alt=json";
|
||||
|
||||
public GoogleProvider(
|
||||
String clientId,
|
||||
String clientSecret,
|
||||
Collection<String> scopes,
|
||||
UsernameAttribute useAsUsername) {
|
||||
super(
|
||||
null,
|
||||
NAME,
|
||||
CLIENT_NAME,
|
||||
clientId,
|
||||
clientSecret,
|
||||
scopes,
|
||||
useAsUsername,
|
||||
AUTHORIZATION_URI,
|
||||
TOKEN_URI,
|
||||
USER_INFO_URI);
|
||||
}
|
||||
|
||||
public String getAuthorizationUri() {
|
||||
return AUTHORIZATION_URI;
|
||||
}
|
||||
|
||||
public String getTokenUri() {
|
||||
return TOKEN_URI;
|
||||
}
|
||||
|
||||
public String getUserinfoUri() {
|
||||
return USER_INFO_URI;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getClientName() {
|
||||
return CLIENT_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<String> getScopes() {
|
||||
Collection<String> scopes = super.getScopes();
|
||||
|
||||
if (scopes == null || scopes.isEmpty()) {
|
||||
scopes = new ArrayList<>();
|
||||
scopes.add("https://www.googleapis.com/auth/userinfo.email");
|
||||
scopes.add("https://www.googleapis.com/auth/userinfo.profile");
|
||||
}
|
||||
|
||||
return scopes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Google [clientId="
|
||||
+ getClientId()
|
||||
+ ", clientSecret="
|
||||
+ (getClientSecret() != null && !getClientSecret().isEmpty() ? "*****" : "NULL")
|
||||
+ ", scopes="
|
||||
+ getScopes()
|
||||
+ ", useAsUsername="
|
||||
+ getUseAsUsername()
|
||||
+ "]";
|
||||
}
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package stirling.software.common.model.oauth2;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import stirling.software.common.model.enumeration.UsernameAttribute;
|
||||
|
||||
@NoArgsConstructor
|
||||
public class KeycloakProvider extends Provider {
|
||||
|
||||
private static final String NAME = "keycloak";
|
||||
private static final String CLIENT_NAME = "Keycloak";
|
||||
|
||||
public KeycloakProvider(
|
||||
String issuer,
|
||||
String clientId,
|
||||
String clientSecret,
|
||||
Collection<String> scopes,
|
||||
UsernameAttribute useAsUsername) {
|
||||
super(
|
||||
issuer,
|
||||
NAME,
|
||||
CLIENT_NAME,
|
||||
clientId,
|
||||
clientSecret,
|
||||
scopes,
|
||||
useAsUsername,
|
||||
null,
|
||||
null,
|
||||
null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getClientName() {
|
||||
return CLIENT_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Collection<String> getScopes() {
|
||||
Collection<String> scopes = super.getScopes();
|
||||
|
||||
if (scopes == null || scopes.isEmpty()) {
|
||||
scopes = new ArrayList<>();
|
||||
scopes.add("profile");
|
||||
scopes.add("email");
|
||||
}
|
||||
|
||||
return scopes;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Keycloak [issuer="
|
||||
+ getIssuer()
|
||||
+ ", clientId="
|
||||
+ getClientId()
|
||||
+ ", clientSecret="
|
||||
+ (getClientSecret() != null && !getClientSecret().isBlank() ? "*****" : "NULL")
|
||||
+ ", scopes="
|
||||
+ getScopes()
|
||||
+ ", useAsUsername="
|
||||
+ getUseAsUsername()
|
||||
+ "]";
|
||||
}
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
package stirling.software.common.model.oauth2;
|
||||
|
||||
import static stirling.software.common.model.enumeration.UsernameAttribute.EMAIL;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import stirling.software.common.model.enumeration.UsernameAttribute;
|
||||
import stirling.software.common.model.exception.UnsupportedClaimException;
|
||||
|
||||
@Data
|
||||
@NoArgsConstructor
|
||||
public class Provider {
|
||||
|
||||
public static final String EXCEPTION_MESSAGE = "The attribute %s is not supported for %s.";
|
||||
|
||||
private String issuer;
|
||||
private String name;
|
||||
private String clientName;
|
||||
private String clientId;
|
||||
private String clientSecret;
|
||||
private Collection<String> scopes;
|
||||
private UsernameAttribute useAsUsername;
|
||||
private String authorizationUri;
|
||||
private String tokenUri;
|
||||
private String userInfoUri;
|
||||
|
||||
public Provider(
|
||||
String issuer,
|
||||
String name,
|
||||
String clientName,
|
||||
String clientId,
|
||||
String clientSecret,
|
||||
Collection<String> scopes,
|
||||
UsernameAttribute useAsUsername,
|
||||
String authorizationUri,
|
||||
String tokenUri,
|
||||
String userInfoUri) {
|
||||
this.issuer = issuer;
|
||||
this.name = name;
|
||||
this.clientName = clientName;
|
||||
this.clientId = clientId;
|
||||
this.clientSecret = clientSecret;
|
||||
this.scopes = scopes == null ? new ArrayList<>() : scopes;
|
||||
this.useAsUsername =
|
||||
useAsUsername != null ? validateUsernameAttribute(useAsUsername) : EMAIL;
|
||||
this.authorizationUri = authorizationUri;
|
||||
this.tokenUri = tokenUri;
|
||||
this.userInfoUri = userInfoUri;
|
||||
}
|
||||
|
||||
public void setScopes(String scopes) {
|
||||
if (scopes != null && !scopes.isBlank()) {
|
||||
this.scopes = Arrays.stream(scopes.split(",")).map(String::trim).toList();
|
||||
}
|
||||
}
|
||||
|
||||
private UsernameAttribute validateUsernameAttribute(UsernameAttribute usernameAttribute) {
|
||||
switch (name) {
|
||||
case "google" -> {
|
||||
return validateGoogleUsernameAttribute(usernameAttribute);
|
||||
}
|
||||
case "github" -> {
|
||||
return validateGitHubUsernameAttribute(usernameAttribute);
|
||||
}
|
||||
case "keycloak" -> {
|
||||
return validateKeycloakUsernameAttribute(usernameAttribute);
|
||||
}
|
||||
default -> {
|
||||
return usernameAttribute;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private UsernameAttribute validateKeycloakUsernameAttribute(
|
||||
UsernameAttribute usernameAttribute) {
|
||||
switch (usernameAttribute) {
|
||||
case EMAIL, NAME, GIVEN_NAME, FAMILY_NAME, PREFERRED_USERNAME -> {
|
||||
return usernameAttribute;
|
||||
}
|
||||
default ->
|
||||
throw new UnsupportedClaimException(
|
||||
String.format(EXCEPTION_MESSAGE, usernameAttribute, clientName));
|
||||
}
|
||||
}
|
||||
|
||||
private UsernameAttribute validateGoogleUsernameAttribute(UsernameAttribute usernameAttribute) {
|
||||
switch (usernameAttribute) {
|
||||
case EMAIL, NAME, GIVEN_NAME, FAMILY_NAME -> {
|
||||
return usernameAttribute;
|
||||
}
|
||||
default ->
|
||||
throw new UnsupportedClaimException(
|
||||
String.format(EXCEPTION_MESSAGE, usernameAttribute, clientName));
|
||||
}
|
||||
}
|
||||
|
||||
private UsernameAttribute validateGitHubUsernameAttribute(UsernameAttribute usernameAttribute) {
|
||||
switch (usernameAttribute) {
|
||||
case LOGIN, EMAIL, NAME -> {
|
||||
return usernameAttribute;
|
||||
}
|
||||
default ->
|
||||
throw new UnsupportedClaimException(
|
||||
String.format(EXCEPTION_MESSAGE, usernameAttribute, clientName));
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Provider [name="
|
||||
+ getName()
|
||||
+ ", clientName="
|
||||
+ getClientName()
|
||||
+ ", clientId="
|
||||
+ getClientId()
|
||||
+ ", clientSecret="
|
||||
+ (getClientSecret() != null && !getClientSecret().isEmpty() ? "*****" : "NULL")
|
||||
+ ", scopes="
|
||||
+ getScopes()
|
||||
+ ", useAsUsername="
|
||||
+ getUseAsUsername()
|
||||
+ "]";
|
||||
}
|
||||
}
|
||||
-542
@@ -1,542 +0,0 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.StandardCopyOption;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
import org.apache.pdfbox.Loader;
|
||||
import org.apache.pdfbox.examples.util.DeletingRandomAccessFile;
|
||||
import org.apache.pdfbox.io.IOUtils;
|
||||
import org.apache.pdfbox.io.MemoryUsageSetting;
|
||||
import org.apache.pdfbox.io.RandomAccessStreamCache.StreamCacheCreateFunction;
|
||||
import org.apache.pdfbox.io.ScratchFile;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.api.PDFFile;
|
||||
import stirling.software.common.util.ApplicationContextProvider;
|
||||
import stirling.software.common.util.ExceptionUtils;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.TempFileRegistry;
|
||||
|
||||
/**
|
||||
* Adaptive PDF document factory that optimizes memory usage based on file size and available system
|
||||
* resources.
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
public class CustomPDFDocumentFactory {
|
||||
|
||||
private final PdfMetadataService pdfMetadataService;
|
||||
|
||||
// Memory thresholds and limits
|
||||
|
||||
private static final long SMALL_FILE_THRESHOLD = 10 * 1024 * 1024; // 10 MB
|
||||
// Files smaller than this threshold are loaded entirely in memory for better performance.
|
||||
// These files use IOUtils.createMemoryOnlyStreamCache() which keeps all document data in RAM.
|
||||
// No temp files are created for document data, reducing I/O operations but consuming more
|
||||
// memory.
|
||||
|
||||
private static final long LARGE_FILE_THRESHOLD = 50 * 1024 * 1024; // 50 MB
|
||||
// Files between SMALL and LARGE thresholds use file-based caching with ScratchFile,
|
||||
// but are loaded directly from byte arrays if provided that way.
|
||||
// When loading from byte arrays, once size exceeds this threshold, bytes are first
|
||||
// written to temp files before loading to reduce memory pressure.
|
||||
|
||||
private static final long LARGE_FILE_USAGE = 10 * 1024 * 1024;
|
||||
|
||||
private static final long EXTREMELY_LARGE_THRESHOLD = 100 * 1024 * 1024; // 100 MB
|
||||
// Files exceeding this threshold use specialized loading with RandomAccessReadBufferedFile
|
||||
// which provides buffered access to the file without loading the entire content at once.
|
||||
// These files are always processed using file-based caching with minimal memory footprint,
|
||||
// trading some performance for significantly reduced memory usage.
|
||||
// For extremely large PDFs, this prevents OutOfMemoryErrors at the cost of being more I/O
|
||||
// bound.
|
||||
|
||||
private static final double MIN_FREE_MEMORY_PERCENTAGE = 30.0; // 30%
|
||||
private static final long MIN_FREE_MEMORY_BYTES = 4L * 1024 * 1024 * 1024; // 4 GB
|
||||
|
||||
// Counter for tracking temporary resources
|
||||
private static final AtomicLong tempCounter = new AtomicLong(0);
|
||||
|
||||
/**
|
||||
* Main entry point for loading a PDF document from a file. Automatically selects the most
|
||||
* appropriate loading strategy.
|
||||
*/
|
||||
public PDDocument load(File file) throws IOException {
|
||||
return load(file, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Main entry point for loading a PDF document from a file with read-only option. Automatically
|
||||
* selects the most appropriate loading strategy.
|
||||
*/
|
||||
public PDDocument load(File file, boolean readOnly) throws IOException {
|
||||
if (file == null) {
|
||||
throw ExceptionUtils.createNullArgumentException("File");
|
||||
}
|
||||
|
||||
long fileSize = file.length();
|
||||
log.debug("Loading PDF from file, size: {}MB", fileSize / (1024 * 1024));
|
||||
|
||||
PDDocument doc = loadAdaptively(file, fileSize);
|
||||
if (!readOnly) {
|
||||
postProcessDocument(doc);
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/**
|
||||
* Main entry point for loading a PDF document from a Path. Automatically selects the most
|
||||
* appropriate loading strategy.
|
||||
*/
|
||||
public PDDocument load(Path path) throws IOException {
|
||||
return load(path, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Main entry point for loading a PDF document from a Path with read-only option. Automatically
|
||||
* selects the most appropriate loading strategy.
|
||||
*/
|
||||
public PDDocument load(Path path, boolean readOnly) throws IOException {
|
||||
if (path == null) {
|
||||
throw ExceptionUtils.createNullArgumentException("File");
|
||||
}
|
||||
|
||||
long fileSize = Files.size(path);
|
||||
log.debug("Loading PDF from file, size: {}MB", fileSize / (1024 * 1024));
|
||||
|
||||
PDDocument doc = loadAdaptively(path.toFile(), fileSize);
|
||||
if (!readOnly) {
|
||||
postProcessDocument(doc);
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/** Load a PDF from byte array with automatic optimization. */
|
||||
public PDDocument load(byte[] input) throws IOException {
|
||||
return load(input, false);
|
||||
}
|
||||
|
||||
/** Load a PDF from byte array with automatic optimization and read-only option. */
|
||||
public PDDocument load(byte[] input, boolean readOnly) throws IOException {
|
||||
if (input == null) {
|
||||
throw ExceptionUtils.createNullArgumentException("Input bytes");
|
||||
}
|
||||
|
||||
long dataSize = input.length;
|
||||
log.debug("Loading PDF from byte array, size: {}MB", dataSize / (1024 * 1024));
|
||||
|
||||
PDDocument doc = loadAdaptively(input, dataSize);
|
||||
if (!readOnly) {
|
||||
postProcessDocument(doc);
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/** Load a PDF from InputStream with automatic optimization. */
|
||||
public PDDocument load(InputStream input) throws IOException {
|
||||
return load(input, false);
|
||||
}
|
||||
|
||||
/** Load a PDF from InputStream with automatic optimization and read-only option. */
|
||||
public PDDocument load(InputStream input, boolean readOnly) throws IOException {
|
||||
if (input == null) {
|
||||
throw ExceptionUtils.createNullArgumentException("InputStream");
|
||||
}
|
||||
|
||||
// Since we don't know the size upfront, buffer to a temp file
|
||||
Path tempFile = createTempFile("pdf-stream-");
|
||||
|
||||
Files.copy(input, tempFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
PDDocument doc = loadAdaptively(tempFile.toFile(), Files.size(tempFile));
|
||||
if (!readOnly) {
|
||||
postProcessDocument(doc);
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/** Load with password from InputStream */
|
||||
public PDDocument load(InputStream input, String password) throws IOException {
|
||||
return load(input, password, false);
|
||||
}
|
||||
|
||||
/** Load with password from InputStream and read-only option */
|
||||
public PDDocument load(InputStream input, String password, boolean readOnly)
|
||||
throws IOException {
|
||||
if (input == null) {
|
||||
throw ExceptionUtils.createNullArgumentException("InputStream");
|
||||
}
|
||||
|
||||
// Since we don't know the size upfront, buffer to a temp file
|
||||
Path tempFile = createTempFile("pdf-stream-");
|
||||
|
||||
Files.copy(input, tempFile, StandardCopyOption.REPLACE_EXISTING);
|
||||
PDDocument doc =
|
||||
loadAdaptivelyWithPassword(tempFile.toFile(), Files.size(tempFile), password);
|
||||
if (!readOnly) {
|
||||
postProcessDocument(doc);
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
/** Load from a file path string */
|
||||
public PDDocument load(String path) throws IOException {
|
||||
return load(path, false);
|
||||
}
|
||||
|
||||
/** Load from a file path string with read-only option */
|
||||
public PDDocument load(String path, boolean readOnly) throws IOException {
|
||||
return load(new File(path), readOnly);
|
||||
}
|
||||
|
||||
/** Load from a PDFFile object */
|
||||
public PDDocument load(PDFFile pdfFile) throws IOException {
|
||||
return load(pdfFile, false);
|
||||
}
|
||||
|
||||
/** Load from a PDFFile object with read-only option */
|
||||
public PDDocument load(PDFFile pdfFile, boolean readOnly) throws IOException {
|
||||
return load(pdfFile.getFileInput(), readOnly);
|
||||
}
|
||||
|
||||
/** Load from a MultipartFile */
|
||||
public PDDocument load(MultipartFile pdfFile) throws IOException {
|
||||
return load(pdfFile, false);
|
||||
}
|
||||
|
||||
/** Load from a MultipartFile with read-only option */
|
||||
public PDDocument load(MultipartFile pdfFile, boolean readOnly) throws IOException {
|
||||
return load(pdfFile.getInputStream(), readOnly);
|
||||
}
|
||||
|
||||
/** Load with password from MultipartFile */
|
||||
public PDDocument load(MultipartFile fileInput, String password) throws IOException {
|
||||
return load(fileInput, password, false);
|
||||
}
|
||||
|
||||
/** Load with password from MultipartFile with read-only option */
|
||||
public PDDocument load(MultipartFile fileInput, String password, boolean readOnly)
|
||||
throws IOException {
|
||||
return load(fileInput.getInputStream(), password, readOnly);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine the appropriate caching strategy based on file size and available memory. This
|
||||
* common method is used by both password and non-password loading paths.
|
||||
*/
|
||||
public StreamCacheCreateFunction getStreamCacheFunction(long contentSize) {
|
||||
long maxMemory = Runtime.getRuntime().maxMemory();
|
||||
long freeMemory = Runtime.getRuntime().freeMemory();
|
||||
long totalMemory = Runtime.getRuntime().totalMemory();
|
||||
long usedMemory = totalMemory - freeMemory;
|
||||
|
||||
// Calculate percentage of free memory
|
||||
double freeMemoryPercent = (double) (maxMemory - usedMemory) / maxMemory * 100;
|
||||
long actualFreeMemory = maxMemory - usedMemory;
|
||||
|
||||
// Log memory status
|
||||
log.debug(
|
||||
"Memory status - Free: {}MB ({}%), Used: {}MB, Max: {}MB",
|
||||
actualFreeMemory / (1024 * 1024),
|
||||
String.format("%.2f", freeMemoryPercent),
|
||||
usedMemory / (1024 * 1024),
|
||||
maxMemory / (1024 * 1024));
|
||||
|
||||
// If free memory is critically low, always use file-based caching
|
||||
if (freeMemoryPercent < MIN_FREE_MEMORY_PERCENTAGE
|
||||
|| actualFreeMemory < MIN_FREE_MEMORY_BYTES) {
|
||||
log.debug(
|
||||
"Low memory detected ({}%), forcing file-based cache",
|
||||
String.format("%.2f", freeMemoryPercent));
|
||||
return createScratchFileCacheFunction(MemoryUsageSetting.setupTempFileOnly());
|
||||
} else if (contentSize < SMALL_FILE_THRESHOLD) {
|
||||
log.debug("Using memory-only cache for small document ({}KB)", contentSize / 1024);
|
||||
return IOUtils.createMemoryOnlyStreamCache();
|
||||
} else if (contentSize < LARGE_FILE_THRESHOLD) {
|
||||
// For medium files (10-50MB), use a mixed approach
|
||||
log.debug(
|
||||
"Using mixed memory/file cache for medium document ({}MB)",
|
||||
contentSize / (1024 * 1024));
|
||||
return createScratchFileCacheFunction(MemoryUsageSetting.setupMixed(LARGE_FILE_USAGE));
|
||||
} else {
|
||||
log.debug("Using file-based cache for large document");
|
||||
return createScratchFileCacheFunction(MemoryUsageSetting.setupTempFileOnly());
|
||||
}
|
||||
}
|
||||
|
||||
/** Update the existing loadAdaptively method to use the common function */
|
||||
private PDDocument loadAdaptively(Object source, long contentSize) throws IOException {
|
||||
// Get the appropriate caching strategy
|
||||
StreamCacheCreateFunction cacheFunction = getStreamCacheFunction(contentSize);
|
||||
|
||||
// If small handle as bytes and remove original file
|
||||
if (contentSize <= SMALL_FILE_THRESHOLD && source instanceof File file) {
|
||||
source = Files.readAllBytes(file.toPath());
|
||||
file.delete();
|
||||
}
|
||||
PDDocument document;
|
||||
if (source instanceof File file) {
|
||||
document = loadFromFile(file, contentSize, cacheFunction);
|
||||
} else if (source instanceof byte[] bytes) {
|
||||
document = loadFromBytes(bytes, contentSize, cacheFunction);
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported source type: " + source.getClass());
|
||||
}
|
||||
|
||||
configureResourceCacheIfNeeded(document, contentSize);
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure resource cache based on content size and memory constraints. Disables resource
|
||||
* cache for large files or when memory is low to prevent OOM errors.
|
||||
*/
|
||||
private void configureResourceCacheIfNeeded(PDDocument document, long contentSize) {
|
||||
if (contentSize > LARGE_FILE_THRESHOLD) {
|
||||
document.setResourceCache(null);
|
||||
} else {
|
||||
// Check current memory status for smaller files
|
||||
long maxMemory = Runtime.getRuntime().maxMemory();
|
||||
long usedMemory =
|
||||
Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
|
||||
double freeMemoryPercent = (double) (maxMemory - usedMemory) / maxMemory * 100;
|
||||
|
||||
if (freeMemoryPercent < MIN_FREE_MEMORY_PERCENTAGE) {
|
||||
document.setResourceCache(null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Load a PDF with password protection using adaptive loading strategies */
|
||||
private PDDocument loadAdaptivelyWithPassword(Object source, long contentSize, String password)
|
||||
throws IOException {
|
||||
// Get the appropriate caching strategy
|
||||
StreamCacheCreateFunction cacheFunction = getStreamCacheFunction(contentSize);
|
||||
// If small handle as bytes and remove original file
|
||||
if (contentSize <= SMALL_FILE_THRESHOLD && source instanceof File file) {
|
||||
source = Files.readAllBytes(file.toPath());
|
||||
file.delete();
|
||||
}
|
||||
PDDocument document;
|
||||
if (source instanceof File file) {
|
||||
document = loadFromFileWithPassword(file, contentSize, cacheFunction, password);
|
||||
} else if (source instanceof byte[] bytes) {
|
||||
document = loadFromBytesWithPassword(bytes, contentSize, cacheFunction, password);
|
||||
} else {
|
||||
throw new IllegalArgumentException("Unsupported source type: " + source.getClass());
|
||||
}
|
||||
|
||||
configureResourceCacheIfNeeded(document, contentSize);
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
/** Load a file with password */
|
||||
private PDDocument loadFromFileWithPassword(
|
||||
File file, long size, StreamCacheCreateFunction cache, String password)
|
||||
throws IOException {
|
||||
return Loader.loadPDF(new DeletingRandomAccessFile(file), password, null, null, cache);
|
||||
}
|
||||
|
||||
/** Load bytes with password */
|
||||
private PDDocument loadFromBytesWithPassword(
|
||||
byte[] bytes, long size, StreamCacheCreateFunction cache, String password)
|
||||
throws IOException {
|
||||
if (size >= SMALL_FILE_THRESHOLD) {
|
||||
log.debug("Writing large byte array to temp file for password-protected PDF");
|
||||
Path tempFile = createTempFile("pdf-bytes-");
|
||||
|
||||
Files.write(tempFile, bytes);
|
||||
return Loader.loadPDF(tempFile.toFile(), password, null, null, cache);
|
||||
}
|
||||
return Loader.loadPDF(bytes, password, null, null, cache);
|
||||
}
|
||||
|
||||
private StreamCacheCreateFunction createScratchFileCacheFunction(MemoryUsageSetting settings) {
|
||||
return () -> {
|
||||
try {
|
||||
return new ScratchFile(settings);
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("ScratchFile initialization failed", e);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private void postProcessDocument(PDDocument doc) throws IOException {
|
||||
pdfMetadataService.setDefaultMetadata(doc);
|
||||
removePassword(doc);
|
||||
}
|
||||
|
||||
private PDDocument loadFromFile(File file, long size, StreamCacheCreateFunction cache)
|
||||
throws IOException {
|
||||
try {
|
||||
return Loader.loadPDF(new DeletingRandomAccessFile(file), "", null, null, cache);
|
||||
} catch (IOException e) {
|
||||
ExceptionUtils.logException("PDF loading from file", e);
|
||||
throw ExceptionUtils.handlePdfException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private PDDocument loadFromBytes(byte[] bytes, long size, StreamCacheCreateFunction cache)
|
||||
throws IOException {
|
||||
if (size >= SMALL_FILE_THRESHOLD) {
|
||||
log.debug("Writing large byte array to temp file");
|
||||
Path tempFile = createTempFile("pdf-bytes-");
|
||||
|
||||
Files.write(tempFile, bytes);
|
||||
return loadFromFile(tempFile.toFile(), size, cache);
|
||||
}
|
||||
|
||||
try {
|
||||
return Loader.loadPDF(bytes, "", null, null, cache);
|
||||
} catch (IOException e) {
|
||||
ExceptionUtils.logException("PDF loading from bytes", e);
|
||||
throw ExceptionUtils.handlePdfException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public PDDocument createNewDocument(MemoryUsageSetting settings) throws IOException {
|
||||
PDDocument doc = new PDDocument(createScratchFileCacheFunction(settings));
|
||||
pdfMetadataService.setDefaultMetadata(doc);
|
||||
return doc;
|
||||
}
|
||||
|
||||
public PDDocument createNewDocument() throws IOException {
|
||||
return createNewDocument(MemoryUsageSetting.setupTempFileOnly());
|
||||
}
|
||||
|
||||
public byte[] saveToBytes(PDDocument document) throws IOException {
|
||||
if (document.getNumberOfPages() < 10) { // Simple heuristic
|
||||
try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
|
||||
document.save(baos);
|
||||
return baos.toByteArray();
|
||||
}
|
||||
} else {
|
||||
Path tempFile = createTempFile("pdf-save-");
|
||||
|
||||
document.save(tempFile.toFile());
|
||||
return Files.readAllBytes(tempFile);
|
||||
}
|
||||
}
|
||||
|
||||
// Improved password handling
|
||||
private void removePassword(PDDocument document) throws IOException {
|
||||
if (document.isEncrypted()) {
|
||||
try {
|
||||
document.setAllSecurityToBeRemoved(true);
|
||||
} catch (Exception e) {
|
||||
ExceptionUtils.logException("PDF decryption", e);
|
||||
throw new IOException("PDF decryption failed", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Temp file handling with enhanced logging and registry integration
|
||||
private Path createTempFile(String prefix) throws IOException {
|
||||
// Check if TempFileManager is available in the application context
|
||||
try {
|
||||
TempFileManager tempFileManager =
|
||||
ApplicationContextProvider.getBean(TempFileManager.class);
|
||||
if (tempFileManager != null) {
|
||||
// Use TempFileManager to create and register the temp file
|
||||
File file = tempFileManager.createTempFile(".tmp");
|
||||
log.debug("Created and registered temp file via TempFileManager: {}", file);
|
||||
return file.toPath();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("TempFileManager not available, falling back to standard temp file creation");
|
||||
}
|
||||
|
||||
// Fallback to standard temp file creation
|
||||
Path file = Files.createTempFile(prefix + tempCounter.incrementAndGet() + "-", ".tmp");
|
||||
log.debug("Created temp file: {}", file);
|
||||
|
||||
// Try to register the file with a static registry if possible
|
||||
try {
|
||||
TempFileRegistry registry = ApplicationContextProvider.getBean(TempFileRegistry.class);
|
||||
if (registry != null) {
|
||||
registry.register(file);
|
||||
log.debug("Registered fallback temp file with registry: {}", file);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not register fallback temp file with registry: {}", file);
|
||||
}
|
||||
|
||||
return file;
|
||||
}
|
||||
|
||||
/** Create a uniquely named temporary directory */
|
||||
private Path createTempDirectory(String prefix) throws IOException {
|
||||
return Files.createTempDirectory(prefix + tempCounter.incrementAndGet() + "-");
|
||||
}
|
||||
|
||||
/** Create new document bytes based on an existing document */
|
||||
public byte[] createNewBytesBasedOnOldDocument(byte[] oldDocument) throws IOException {
|
||||
try (PDDocument document = load(oldDocument)) {
|
||||
return saveToBytes(document);
|
||||
}
|
||||
}
|
||||
|
||||
/** Create new document bytes based on an existing document file */
|
||||
public byte[] createNewBytesBasedOnOldDocument(File oldDocument) throws IOException {
|
||||
try (PDDocument document = load(oldDocument)) {
|
||||
return saveToBytes(document);
|
||||
}
|
||||
}
|
||||
|
||||
/** Create new document bytes based on an existing PDDocument */
|
||||
public byte[] createNewBytesBasedOnOldDocument(PDDocument oldDocument) throws IOException {
|
||||
pdfMetadataService.setMetadataToPdf(
|
||||
oldDocument, pdfMetadataService.extractMetadataFromPdf(oldDocument), true);
|
||||
return saveToBytes(oldDocument);
|
||||
}
|
||||
|
||||
/** Create a new document based on an existing document bytes */
|
||||
public PDDocument createNewDocumentBasedOnOldDocument(byte[] oldDocument) throws IOException {
|
||||
try (PDDocument document = load(oldDocument)) {
|
||||
return createNewDocumentBasedOnOldDocument(document);
|
||||
}
|
||||
}
|
||||
|
||||
/** Create a new document based on an existing document file */
|
||||
public PDDocument createNewDocumentBasedOnOldDocument(File oldDocument) throws IOException {
|
||||
try (PDDocument document = load(oldDocument)) {
|
||||
return createNewDocumentBasedOnOldDocument(document);
|
||||
}
|
||||
}
|
||||
|
||||
/** Create a new document based on an existing PDDocument */
|
||||
public PDDocument createNewDocumentBasedOnOldDocument(PDDocument oldDocument)
|
||||
throws IOException {
|
||||
PDDocument document = createNewDocument();
|
||||
pdfMetadataService.setMetadataToPdf(
|
||||
document, pdfMetadataService.extractMetadataFromPdf(oldDocument), true);
|
||||
return document;
|
||||
}
|
||||
|
||||
/** Load document from a file and convert it to bytes */
|
||||
public byte[] loadToBytes(File file) throws IOException {
|
||||
try (PDDocument document = load(file)) {
|
||||
return saveToBytes(document);
|
||||
}
|
||||
}
|
||||
|
||||
/** Load document from bytes and convert it back to bytes */
|
||||
public byte[] loadToBytes(byte[] bytes) throws IOException {
|
||||
try (PDDocument document = load(bytes)) {
|
||||
return saveToBytes(document);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.*;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class FileOrUploadService {
|
||||
|
||||
@Value("${stirling.tempDir:/tmp/stirling-files}")
|
||||
private String tempDirPath;
|
||||
|
||||
public Path resolveFilePath(String fileId) {
|
||||
return Path.of(tempDirPath).resolve(fileId);
|
||||
}
|
||||
|
||||
public MultipartFile toMockMultipartFile(String name, byte[] data) throws IOException {
|
||||
return new CustomMultipartFile(name, data);
|
||||
}
|
||||
|
||||
// Custom implementation of MultipartFile
|
||||
private static class CustomMultipartFile implements MultipartFile {
|
||||
private final String name;
|
||||
private final byte[] content;
|
||||
|
||||
public CustomMultipartFile(String name, byte[] content) {
|
||||
this.name = name;
|
||||
this.content = content;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getOriginalFilename() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getContentType() {
|
||||
return "application/pdf";
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isEmpty() {
|
||||
return content == null || content.length == 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getSize() {
|
||||
return content.length;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getBytes() throws IOException {
|
||||
return content;
|
||||
}
|
||||
|
||||
@Override
|
||||
public java.io.InputStream getInputStream() throws IOException {
|
||||
return new ByteArrayInputStream(content);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void transferTo(java.io.File dest) throws IOException, IllegalStateException {
|
||||
Files.write(dest.toPath(), content);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Service for storing and retrieving files with unique file IDs. Used by the AutoJobPostMapping
|
||||
* system to handle file references.
|
||||
*/
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class FileStorage {
|
||||
|
||||
@Value("${stirling.tempDir:/tmp/stirling-files}")
|
||||
private String tempDirPath;
|
||||
|
||||
private final FileOrUploadService fileOrUploadService;
|
||||
|
||||
/**
|
||||
* Store a file and return its unique ID
|
||||
*
|
||||
* @param file The file to store
|
||||
* @return The unique ID assigned to the file
|
||||
* @throws IOException If there is an error storing the file
|
||||
*/
|
||||
public String storeFile(MultipartFile file) throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
// Ensure the directory exists
|
||||
Files.createDirectories(filePath.getParent());
|
||||
|
||||
// Transfer the file to the storage location
|
||||
file.transferTo(filePath.toFile());
|
||||
|
||||
log.debug("Stored file with ID: {}", fileId);
|
||||
return fileId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a byte array as a file and return its unique ID
|
||||
*
|
||||
* @param bytes The byte array to store
|
||||
* @param originalName The original name of the file (for extension)
|
||||
* @return The unique ID assigned to the file
|
||||
* @throws IOException If there is an error storing the file
|
||||
*/
|
||||
public String storeBytes(byte[] bytes, String originalName) throws IOException {
|
||||
String fileId = generateFileId();
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
// Ensure the directory exists
|
||||
Files.createDirectories(filePath.getParent());
|
||||
|
||||
// Write the bytes to the file
|
||||
Files.write(filePath, bytes);
|
||||
|
||||
log.debug("Stored byte array with ID: {}", fileId);
|
||||
return fileId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a file by its ID as a MultipartFile
|
||||
*
|
||||
* @param fileId The ID of the file to retrieve
|
||||
* @return The file as a MultipartFile
|
||||
* @throws IOException If the file doesn't exist or can't be read
|
||||
*/
|
||||
public MultipartFile retrieveFile(String fileId) throws IOException {
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
}
|
||||
|
||||
byte[] fileData = Files.readAllBytes(filePath);
|
||||
return fileOrUploadService.toMockMultipartFile(fileId, fileData);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve a file by its ID as a byte array
|
||||
*
|
||||
* @param fileId The ID of the file to retrieve
|
||||
* @return The file as a byte array
|
||||
* @throws IOException If the file doesn't exist or can't be read
|
||||
*/
|
||||
public byte[] retrieveBytes(String fileId) throws IOException {
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
}
|
||||
|
||||
return Files.readAllBytes(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file by its ID
|
||||
*
|
||||
* @param fileId The ID of the file to delete
|
||||
* @return true if the file was deleted, false otherwise
|
||||
*/
|
||||
public boolean deleteFile(String fileId) {
|
||||
try {
|
||||
Path filePath = getFilePath(fileId);
|
||||
return Files.deleteIfExists(filePath);
|
||||
} catch (IOException e) {
|
||||
log.error("Error deleting file with ID: {}", fileId, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file exists by its ID
|
||||
*
|
||||
* @param fileId The ID of the file to check
|
||||
* @return true if the file exists, false otherwise
|
||||
*/
|
||||
public boolean fileExists(String fileId) {
|
||||
Path filePath = getFilePath(fileId);
|
||||
return Files.exists(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the size of a file by its ID without loading the content into memory
|
||||
*
|
||||
* @param fileId The ID of the file
|
||||
* @return The size of the file in bytes
|
||||
* @throws IOException If the file doesn't exist or can't be read
|
||||
*/
|
||||
public long getFileSize(String fileId) throws IOException {
|
||||
Path filePath = getFilePath(fileId);
|
||||
|
||||
if (!Files.exists(filePath)) {
|
||||
throw new IOException("File not found with ID: " + fileId);
|
||||
}
|
||||
|
||||
return Files.size(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the path for a file ID
|
||||
*
|
||||
* @param fileId The ID of the file
|
||||
* @return The path to the file
|
||||
* @throws IllegalArgumentException if fileId contains path traversal characters or resolves
|
||||
* outside base directory
|
||||
*/
|
||||
private Path getFilePath(String fileId) {
|
||||
// Validate fileId to prevent path traversal
|
||||
if (fileId.contains("..") || fileId.contains("/") || fileId.contains("\\")) {
|
||||
throw new IllegalArgumentException("Invalid file ID");
|
||||
}
|
||||
|
||||
Path basePath = Path.of(tempDirPath).normalize().toAbsolutePath();
|
||||
Path resolvedPath = basePath.resolve(fileId).normalize();
|
||||
|
||||
// Ensure resolved path is within the base directory
|
||||
if (!resolvedPath.startsWith(basePath)) {
|
||||
throw new IllegalArgumentException("File ID resolves to an invalid path");
|
||||
}
|
||||
|
||||
return resolvedPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a unique file ID
|
||||
*
|
||||
* @return A unique file ID
|
||||
*/
|
||||
private String generateFileId() {
|
||||
return UUID.randomUUID().toString();
|
||||
}
|
||||
}
|
||||
@@ -1,476 +0,0 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.job.JobResponse;
|
||||
import stirling.software.common.util.ExecutorFactory;
|
||||
|
||||
/** Service for executing jobs asynchronously or synchronously */
|
||||
@Service
|
||||
@Slf4j
|
||||
public class JobExecutorService {
|
||||
|
||||
private final TaskManager taskManager;
|
||||
private final FileStorage fileStorage;
|
||||
private final HttpServletRequest request;
|
||||
private final ResourceMonitor resourceMonitor;
|
||||
private final JobQueue jobQueue;
|
||||
private final ExecutorService executor = ExecutorFactory.newVirtualOrCachedThreadExecutor();
|
||||
private final long effectiveTimeoutMs;
|
||||
|
||||
public JobExecutorService(
|
||||
TaskManager taskManager,
|
||||
FileStorage fileStorage,
|
||||
HttpServletRequest request,
|
||||
ResourceMonitor resourceMonitor,
|
||||
JobQueue jobQueue,
|
||||
@Value("${spring.mvc.async.request-timeout:1200000}") long asyncRequestTimeoutMs,
|
||||
@Value("${server.servlet.session.timeout:30m}") String sessionTimeout) {
|
||||
this.taskManager = taskManager;
|
||||
this.fileStorage = fileStorage;
|
||||
this.request = request;
|
||||
this.resourceMonitor = resourceMonitor;
|
||||
this.jobQueue = jobQueue;
|
||||
|
||||
// Parse session timeout and calculate effective timeout once during initialization
|
||||
long sessionTimeoutMs = parseSessionTimeout(sessionTimeout);
|
||||
this.effectiveTimeoutMs = Math.min(asyncRequestTimeoutMs, sessionTimeoutMs);
|
||||
log.debug(
|
||||
"Job executor configured with effective timeout of {} ms", this.effectiveTimeoutMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a job either asynchronously or synchronously
|
||||
*
|
||||
* @param async Whether to run the job asynchronously
|
||||
* @param work The work to be done
|
||||
* @return The response
|
||||
*/
|
||||
public ResponseEntity<?> runJobGeneric(boolean async, Supplier<Object> work) {
|
||||
return runJobGeneric(async, work, -1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a job either asynchronously or synchronously with a custom timeout
|
||||
*
|
||||
* @param async Whether to run the job asynchronously
|
||||
* @param work The work to be done
|
||||
* @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
|
||||
* @return The response
|
||||
*/
|
||||
public ResponseEntity<?> runJobGeneric(
|
||||
boolean async, Supplier<Object> work, long customTimeoutMs) {
|
||||
return runJobGeneric(async, work, customTimeoutMs, false, 50);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a job either asynchronously or synchronously with custom parameters
|
||||
*
|
||||
* @param async Whether to run the job asynchronously
|
||||
* @param work The work to be done
|
||||
* @param customTimeoutMs Custom timeout in milliseconds, or -1 to use the default
|
||||
* @param queueable Whether this job can be queued when system resources are limited
|
||||
* @param resourceWeight The resource weight of this job (1-100)
|
||||
* @return The response
|
||||
*/
|
||||
public ResponseEntity<?> runJobGeneric(
|
||||
boolean async,
|
||||
Supplier<Object> work,
|
||||
long customTimeoutMs,
|
||||
boolean queueable,
|
||||
int resourceWeight) {
|
||||
String jobId = UUID.randomUUID().toString();
|
||||
|
||||
// Store the job ID in the request for potential use by other components
|
||||
if (request != null) {
|
||||
request.setAttribute("jobId", jobId);
|
||||
|
||||
// Also track this job ID in the user's session for authorization purposes
|
||||
// This ensures users can only cancel their own jobs
|
||||
if (request.getSession() != null) {
|
||||
@SuppressWarnings("unchecked")
|
||||
java.util.Set<String> userJobIds =
|
||||
(java.util.Set<String>) request.getSession().getAttribute("userJobIds");
|
||||
|
||||
if (userJobIds == null) {
|
||||
userJobIds = new java.util.concurrent.ConcurrentSkipListSet<>();
|
||||
request.getSession().setAttribute("userJobIds", userJobIds);
|
||||
}
|
||||
|
||||
userJobIds.add(jobId);
|
||||
log.debug("Added job ID {} to user session", jobId);
|
||||
}
|
||||
}
|
||||
|
||||
// Determine which timeout to use
|
||||
long timeoutToUse = customTimeoutMs > 0 ? customTimeoutMs : effectiveTimeoutMs;
|
||||
|
||||
log.debug(
|
||||
"Running job with ID: {}, async: {}, timeout: {}ms, queueable: {}, weight: {}",
|
||||
jobId,
|
||||
async,
|
||||
timeoutToUse,
|
||||
queueable,
|
||||
resourceWeight);
|
||||
|
||||
// Check if we need to queue this job based on resource availability
|
||||
boolean shouldQueue =
|
||||
queueable
|
||||
&& async
|
||||
&& // Only async jobs can be queued
|
||||
resourceMonitor.shouldQueueJob(resourceWeight);
|
||||
|
||||
if (shouldQueue) {
|
||||
// Queue the job instead of executing immediately
|
||||
log.debug(
|
||||
"Queueing job {} due to resource constraints (weight: {})",
|
||||
jobId,
|
||||
resourceWeight);
|
||||
|
||||
taskManager.createTask(jobId);
|
||||
|
||||
// Create a specialized wrapper that updates the TaskManager
|
||||
Supplier<Object> wrappedWork =
|
||||
() -> {
|
||||
try {
|
||||
Object result = work.get();
|
||||
processJobResult(jobId, result);
|
||||
return result;
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Error executing queued job {}: {}", jobId, e.getMessage(), e);
|
||||
taskManager.setError(jobId, e.getMessage());
|
||||
throw e;
|
||||
}
|
||||
};
|
||||
|
||||
// Queue the job and get the future
|
||||
CompletableFuture<ResponseEntity<?>> future =
|
||||
jobQueue.queueJob(jobId, resourceWeight, wrappedWork, timeoutToUse);
|
||||
|
||||
// Return immediately with job ID
|
||||
return ResponseEntity.ok().body(new JobResponse<>(true, jobId, null));
|
||||
} else if (async) {
|
||||
taskManager.createTask(jobId);
|
||||
executor.execute(
|
||||
() -> {
|
||||
try {
|
||||
log.debug(
|
||||
"Running async job {} with timeout {} ms", jobId, timeoutToUse);
|
||||
|
||||
// Execute with timeout
|
||||
Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
|
||||
processJobResult(jobId, result);
|
||||
} catch (TimeoutException te) {
|
||||
log.error("Job {} timed out after {} ms", jobId, timeoutToUse);
|
||||
taskManager.setError(jobId, "Job timed out");
|
||||
} catch (Exception e) {
|
||||
log.error("Error executing job {}: {}", jobId, e.getMessage(), e);
|
||||
taskManager.setError(jobId, e.getMessage());
|
||||
}
|
||||
});
|
||||
|
||||
return ResponseEntity.ok().body(new JobResponse<>(true, jobId, null));
|
||||
} else {
|
||||
try {
|
||||
log.debug("Running sync job with timeout {} ms", timeoutToUse);
|
||||
|
||||
// Execute with timeout
|
||||
Object result = executeWithTimeout(() -> work.get(), timeoutToUse);
|
||||
|
||||
// If the result is already a ResponseEntity, return it directly
|
||||
if (result instanceof ResponseEntity) {
|
||||
return (ResponseEntity<?>) result;
|
||||
}
|
||||
|
||||
// Process different result types
|
||||
return handleResultForSyncJob(result);
|
||||
} catch (TimeoutException te) {
|
||||
log.error("Synchronous job timed out after {} ms", timeoutToUse);
|
||||
return ResponseEntity.internalServerError()
|
||||
.body(Map.of("error", "Job timed out after " + timeoutToUse + " ms"));
|
||||
} catch (Exception e) {
|
||||
log.error("Error executing synchronous job: {}", e.getMessage(), e);
|
||||
// Construct a JSON error response
|
||||
return ResponseEntity.internalServerError()
|
||||
.body(Map.of("error", "Job failed: " + e.getMessage()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the result of an asynchronous job
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @param result The result
|
||||
*/
|
||||
private void processJobResult(String jobId, Object result) {
|
||||
try {
|
||||
if (result instanceof byte[]) {
|
||||
// Store byte array directly to disk to avoid double memory consumption
|
||||
String fileId = fileStorage.storeBytes((byte[]) result, "result.pdf");
|
||||
taskManager.setFileResult(jobId, fileId, "result.pdf", "application/pdf");
|
||||
log.debug("Stored byte[] result with fileId: {}", fileId);
|
||||
|
||||
// Let the byte array get collected naturally in the next GC cycle
|
||||
// We don't need to force System.gc() which can be harmful
|
||||
} else if (result instanceof ResponseEntity) {
|
||||
ResponseEntity<?> response = (ResponseEntity<?>) result;
|
||||
Object body = response.getBody();
|
||||
|
||||
if (body instanceof byte[]) {
|
||||
// Extract filename from content-disposition header if available
|
||||
String filename = "result.pdf";
|
||||
String contentType = "application/pdf";
|
||||
|
||||
if (response.getHeaders().getContentDisposition() != null) {
|
||||
String disposition =
|
||||
response.getHeaders().getContentDisposition().toString();
|
||||
if (disposition.contains("filename=")) {
|
||||
filename =
|
||||
disposition.substring(
|
||||
disposition.indexOf("filename=") + 9,
|
||||
disposition.lastIndexOf("\""));
|
||||
}
|
||||
}
|
||||
|
||||
if (response.getHeaders().getContentType() != null) {
|
||||
contentType = response.getHeaders().getContentType().toString();
|
||||
}
|
||||
|
||||
// Store byte array directly to disk
|
||||
String fileId = fileStorage.storeBytes((byte[]) body, filename);
|
||||
taskManager.setFileResult(jobId, fileId, filename, contentType);
|
||||
log.debug("Stored ResponseEntity<byte[]> result with fileId: {}", fileId);
|
||||
|
||||
// Let the GC handle the memory naturally
|
||||
} else {
|
||||
// Check if the response body contains a fileId
|
||||
if (body != null && body.toString().contains("fileId")) {
|
||||
try {
|
||||
// Try to extract fileId using reflection
|
||||
java.lang.reflect.Method getFileId =
|
||||
body.getClass().getMethod("getFileId");
|
||||
String fileId = (String) getFileId.invoke(body);
|
||||
|
||||
if (fileId != null && !fileId.isEmpty()) {
|
||||
// Try to get filename and content type
|
||||
String filename = "result.pdf";
|
||||
String contentType = "application/pdf";
|
||||
|
||||
try {
|
||||
java.lang.reflect.Method getOriginalFileName =
|
||||
body.getClass().getMethod("getOriginalFilename");
|
||||
String origName = (String) getOriginalFileName.invoke(body);
|
||||
if (origName != null && !origName.isEmpty()) {
|
||||
filename = origName;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"Could not get original filename: {}", e.getMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
java.lang.reflect.Method getContentType =
|
||||
body.getClass().getMethod("getContentType");
|
||||
String ct = (String) getContentType.invoke(body);
|
||||
if (ct != null && !ct.isEmpty()) {
|
||||
contentType = ct;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not get content type: {}", e.getMessage());
|
||||
}
|
||||
|
||||
taskManager.setFileResult(jobId, fileId, filename, contentType);
|
||||
log.debug("Extracted fileId from response body: {}", fileId);
|
||||
|
||||
taskManager.setComplete(jobId);
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"Failed to extract fileId from response body: {}",
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// Store generic result
|
||||
taskManager.setResult(jobId, body);
|
||||
}
|
||||
} else if (result instanceof MultipartFile) {
|
||||
MultipartFile file = (MultipartFile) result;
|
||||
String fileId = fileStorage.storeFile(file);
|
||||
taskManager.setFileResult(
|
||||
jobId, fileId, file.getOriginalFilename(), file.getContentType());
|
||||
log.debug("Stored MultipartFile result with fileId: {}", fileId);
|
||||
} else {
|
||||
// Check if result has a fileId field
|
||||
if (result != null) {
|
||||
try {
|
||||
// Try to extract fileId using reflection
|
||||
java.lang.reflect.Method getFileId =
|
||||
result.getClass().getMethod("getFileId");
|
||||
String fileId = (String) getFileId.invoke(result);
|
||||
|
||||
if (fileId != null && !fileId.isEmpty()) {
|
||||
// Try to get filename and content type
|
||||
String filename = "result.pdf";
|
||||
String contentType = "application/pdf";
|
||||
|
||||
try {
|
||||
java.lang.reflect.Method getOriginalFileName =
|
||||
result.getClass().getMethod("getOriginalFilename");
|
||||
String origName = (String) getOriginalFileName.invoke(result);
|
||||
if (origName != null && !origName.isEmpty()) {
|
||||
filename = origName;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not get original filename: {}", e.getMessage());
|
||||
}
|
||||
|
||||
try {
|
||||
java.lang.reflect.Method getContentType =
|
||||
result.getClass().getMethod("getContentType");
|
||||
String ct = (String) getContentType.invoke(result);
|
||||
if (ct != null && !ct.isEmpty()) {
|
||||
contentType = ct;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("Could not get content type: {}", e.getMessage());
|
||||
}
|
||||
|
||||
taskManager.setFileResult(jobId, fileId, filename, contentType);
|
||||
log.debug("Extracted fileId from result object: {}", fileId);
|
||||
|
||||
taskManager.setComplete(jobId);
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug(
|
||||
"Failed to extract fileId from result object: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// Default case: store the result as is
|
||||
taskManager.setResult(jobId, result);
|
||||
}
|
||||
|
||||
taskManager.setComplete(jobId);
|
||||
} catch (Exception e) {
|
||||
log.error("Error processing job result: {}", e.getMessage(), e);
|
||||
taskManager.setError(jobId, "Error processing result: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle different result types for synchronous jobs
|
||||
*
|
||||
* @param result The result object
|
||||
* @return The appropriate ResponseEntity
|
||||
* @throws IOException If there is an error processing the result
|
||||
*/
|
||||
private ResponseEntity<?> handleResultForSyncJob(Object result) throws IOException {
|
||||
if (result instanceof byte[]) {
|
||||
// Return byte array as PDF
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.APPLICATION_PDF)
|
||||
.header(
|
||||
HttpHeaders.CONTENT_DISPOSITION,
|
||||
"form-data; name=\"attachment\"; filename=\"result.pdf\"")
|
||||
.body(result);
|
||||
} else if (result instanceof MultipartFile) {
|
||||
// Return MultipartFile content
|
||||
MultipartFile file = (MultipartFile) result;
|
||||
return ResponseEntity.ok()
|
||||
.contentType(MediaType.parseMediaType(file.getContentType()))
|
||||
.header(
|
||||
HttpHeaders.CONTENT_DISPOSITION,
|
||||
"form-data; name=\"attachment\"; filename=\""
|
||||
+ file.getOriginalFilename()
|
||||
+ "\"")
|
||||
.body(file.getBytes());
|
||||
} else {
|
||||
// Default case: return as JSON
|
||||
return ResponseEntity.ok(result);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse session timeout string (e.g., "30m", "1h") to milliseconds
|
||||
*
|
||||
* @param timeout The timeout string
|
||||
* @return The timeout in milliseconds
|
||||
*/
|
||||
private long parseSessionTimeout(String timeout) {
|
||||
if (timeout == null || timeout.isEmpty()) {
|
||||
return 30 * 60 * 1000; // Default: 30 minutes
|
||||
}
|
||||
|
||||
try {
|
||||
String value = timeout.replaceAll("[^\\d.]", "");
|
||||
String unit = timeout.replaceAll("[\\d.]", "");
|
||||
|
||||
double numericValue = Double.parseDouble(value);
|
||||
|
||||
return switch (unit.toLowerCase()) {
|
||||
case "s" -> (long) (numericValue * 1000);
|
||||
case "m" -> (long) (numericValue * 60 * 1000);
|
||||
case "h" -> (long) (numericValue * 60 * 60 * 1000);
|
||||
case "d" -> (long) (numericValue * 24 * 60 * 60 * 1000);
|
||||
default -> (long) (numericValue * 60 * 1000); // Default to minutes
|
||||
};
|
||||
} catch (Exception e) {
|
||||
log.warn("Could not parse session timeout '{}', using default", timeout);
|
||||
return 30 * 60 * 1000; // Default: 30 minutes
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a supplier with a timeout
|
||||
*
|
||||
* @param supplier The supplier to execute
|
||||
* @param timeoutMs The timeout in milliseconds
|
||||
* @return The result from the supplier
|
||||
* @throws TimeoutException If the execution times out
|
||||
* @throws Exception If the supplier throws an exception
|
||||
*/
|
||||
private <T> T executeWithTimeout(Supplier<T> supplier, long timeoutMs)
|
||||
throws TimeoutException, Exception {
|
||||
// Use the same executor as other async jobs for consistency
|
||||
// This ensures all operations run on the same thread pool
|
||||
java.util.concurrent.CompletableFuture<T> future =
|
||||
java.util.concurrent.CompletableFuture.supplyAsync(supplier, executor);
|
||||
|
||||
try {
|
||||
return future.get(timeoutMs, TimeUnit.MILLISECONDS);
|
||||
} catch (java.util.concurrent.TimeoutException e) {
|
||||
future.cancel(true);
|
||||
throw new TimeoutException("Execution timed out after " + timeoutMs + " ms");
|
||||
} catch (java.util.concurrent.ExecutionException e) {
|
||||
throw (Exception) e.getCause();
|
||||
} catch (java.util.concurrent.CancellationException e) {
|
||||
throw new Exception("Execution was cancelled", e);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new Exception("Execution was interrupted", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,495 +0,0 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.*;
|
||||
import java.util.function.Supplier;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.context.SmartLifecycle;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.util.ExecutorFactory;
|
||||
import stirling.software.common.util.SpringContextHolder;
|
||||
|
||||
/**
|
||||
* Manages a queue of jobs with dynamic sizing based on system resources. Used when system resources
|
||||
* are limited to prevent overloading.
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class JobQueue implements SmartLifecycle {
|
||||
|
||||
private volatile boolean running = false;
|
||||
|
||||
private final ResourceMonitor resourceMonitor;
|
||||
|
||||
@Value("${stirling.job.queue.base-capacity:10}")
|
||||
private int baseQueueCapacity = 10;
|
||||
|
||||
@Value("${stirling.job.queue.min-capacity:2}")
|
||||
private int minQueueCapacity = 2;
|
||||
|
||||
@Value("${stirling.job.queue.check-interval-ms:1000}")
|
||||
private long queueCheckIntervalMs = 1000;
|
||||
|
||||
@Value("${stirling.job.queue.max-wait-time-ms:600000}")
|
||||
private long maxWaitTimeMs = 600000; // 10 minutes
|
||||
|
||||
private volatile BlockingQueue<QueuedJob> jobQueue;
|
||||
private final Map<String, QueuedJob> jobMap = new ConcurrentHashMap<>();
|
||||
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
|
||||
private final ExecutorService jobExecutor = ExecutorFactory.newVirtualOrCachedThreadExecutor();
|
||||
private final Object queueLock = new Object(); // Lock for synchronizing queue operations
|
||||
|
||||
private boolean shuttingDown = false;
|
||||
|
||||
@Getter private int rejectedJobs = 0;
|
||||
|
||||
@Getter private int totalQueuedJobs = 0;
|
||||
|
||||
@Getter private int currentQueueSize = 0;
|
||||
|
||||
/** Represents a job waiting in the queue. */
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
private static class QueuedJob {
|
||||
private final String jobId;
|
||||
private final int resourceWeight;
|
||||
private final Supplier<Object> work;
|
||||
private final long timeoutMs;
|
||||
private final Instant queuedAt;
|
||||
private CompletableFuture<ResponseEntity<?>> future;
|
||||
private volatile boolean cancelled = false;
|
||||
}
|
||||
|
||||
public JobQueue(ResourceMonitor resourceMonitor) {
|
||||
this.resourceMonitor = resourceMonitor;
|
||||
|
||||
// Initialize with dynamic capacity
|
||||
int capacity =
|
||||
resourceMonitor.calculateDynamicQueueCapacity(baseQueueCapacity, minQueueCapacity);
|
||||
this.jobQueue = new LinkedBlockingQueue<>(capacity);
|
||||
}
|
||||
|
||||
// Remove @PostConstruct to let SmartLifecycle control startup
|
||||
private void initializeSchedulers() {
|
||||
log.debug(
|
||||
"Starting job queue with base capacity {}, min capacity {}",
|
||||
baseQueueCapacity,
|
||||
minQueueCapacity);
|
||||
|
||||
// Periodically process the job queue
|
||||
scheduler.scheduleWithFixedDelay(
|
||||
this::processQueue, 0, queueCheckIntervalMs, TimeUnit.MILLISECONDS);
|
||||
|
||||
// Periodically update queue capacity based on resource usage
|
||||
scheduler.scheduleWithFixedDelay(
|
||||
this::updateQueueCapacity,
|
||||
10000, // Initial delay
|
||||
30000, // 30 second interval
|
||||
TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
// Remove @PreDestroy to let SmartLifecycle control shutdown
|
||||
private void shutdownSchedulers() {
|
||||
log.info("Shutting down job queue");
|
||||
shuttingDown = true;
|
||||
|
||||
// Complete any futures that are still waiting
|
||||
jobMap.forEach(
|
||||
(id, job) -> {
|
||||
if (!job.future.isDone()) {
|
||||
job.future.completeExceptionally(
|
||||
new RuntimeException("Server shutting down, job cancelled"));
|
||||
}
|
||||
});
|
||||
|
||||
// Shutdown schedulers and wait for termination
|
||||
try {
|
||||
scheduler.shutdown();
|
||||
if (!scheduler.awaitTermination(5, TimeUnit.SECONDS)) {
|
||||
scheduler.shutdownNow();
|
||||
}
|
||||
|
||||
jobExecutor.shutdown();
|
||||
if (!jobExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
|
||||
jobExecutor.shutdownNow();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
scheduler.shutdownNow();
|
||||
jobExecutor.shutdownNow();
|
||||
}
|
||||
|
||||
log.info(
|
||||
"Job queue shutdown complete. Stats: total={}, rejected={}",
|
||||
totalQueuedJobs,
|
||||
rejectedJobs);
|
||||
}
|
||||
|
||||
// SmartLifecycle methods
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
log.info("Starting JobQueue lifecycle");
|
||||
if (!running) {
|
||||
initializeSchedulers();
|
||||
running = true;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void stop() {
|
||||
log.info("Stopping JobQueue lifecycle");
|
||||
shutdownSchedulers();
|
||||
running = false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isRunning() {
|
||||
return running;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getPhase() {
|
||||
// Start earlier than most components, but shutdown later
|
||||
return 10;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isAutoStartup() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues a job for execution when resources permit.
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @param resourceWeight The resource weight of the job (1-100)
|
||||
* @param work The work to be done
|
||||
* @param timeoutMs The timeout in milliseconds
|
||||
* @return A CompletableFuture that will complete when the job is executed
|
||||
*/
|
||||
public CompletableFuture<ResponseEntity<?>> queueJob(
|
||||
String jobId, int resourceWeight, Supplier<Object> work, long timeoutMs) {
|
||||
|
||||
// Create a CompletableFuture to track this job's completion
|
||||
CompletableFuture<ResponseEntity<?>> future = new CompletableFuture<>();
|
||||
|
||||
// Create the queued job
|
||||
QueuedJob job =
|
||||
new QueuedJob(jobId, resourceWeight, work, timeoutMs, Instant.now(), future, false);
|
||||
|
||||
// Store in our map for lookup
|
||||
jobMap.put(jobId, job);
|
||||
|
||||
// Update stats
|
||||
totalQueuedJobs++;
|
||||
|
||||
// Synchronize access to the queue
|
||||
synchronized (queueLock) {
|
||||
currentQueueSize = jobQueue.size();
|
||||
|
||||
// Try to add to the queue
|
||||
try {
|
||||
boolean added = jobQueue.offer(job, 5, TimeUnit.SECONDS);
|
||||
if (!added) {
|
||||
log.warn("Queue full, rejecting job {}", jobId);
|
||||
rejectedJobs++;
|
||||
future.completeExceptionally(
|
||||
new RuntimeException("Job queue full, please try again later"));
|
||||
jobMap.remove(jobId);
|
||||
return future;
|
||||
}
|
||||
|
||||
log.debug(
|
||||
"Job {} queued for execution (weight: {}, queue size: {})",
|
||||
jobId,
|
||||
resourceWeight,
|
||||
jobQueue.size());
|
||||
|
||||
return future;
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
future.completeExceptionally(new RuntimeException("Job queue interrupted"));
|
||||
jobMap.remove(jobId);
|
||||
return future;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the current capacity of the job queue.
|
||||
*
|
||||
* @return The current capacity
|
||||
*/
|
||||
public int getQueueCapacity() {
|
||||
synchronized (queueLock) {
|
||||
return ((LinkedBlockingQueue<QueuedJob>) jobQueue).remainingCapacity()
|
||||
+ jobQueue.size();
|
||||
}
|
||||
}
|
||||
|
||||
/** Updates the capacity of the job queue based on available system resources. */
|
||||
private void updateQueueCapacity() {
|
||||
try {
|
||||
// Calculate new capacity once and cache the result
|
||||
int newCapacity =
|
||||
resourceMonitor.calculateDynamicQueueCapacity(
|
||||
baseQueueCapacity, minQueueCapacity);
|
||||
|
||||
int currentCapacity = getQueueCapacity();
|
||||
if (newCapacity != currentCapacity) {
|
||||
log.debug(
|
||||
"Updating job queue capacity from {} to {}", currentCapacity, newCapacity);
|
||||
|
||||
synchronized (queueLock) {
|
||||
// Double-check that capacity still needs to be updated
|
||||
// Use the cached currentCapacity to avoid calling getQueueCapacity() again
|
||||
if (newCapacity != currentCapacity) {
|
||||
// Create new queue with updated capacity
|
||||
BlockingQueue<QueuedJob> newQueue = new LinkedBlockingQueue<>(newCapacity);
|
||||
|
||||
// Transfer jobs from old queue to new queue
|
||||
jobQueue.drainTo(newQueue);
|
||||
jobQueue = newQueue;
|
||||
|
||||
currentQueueSize = jobQueue.size();
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error updating queue capacity: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Processes jobs in the queue, executing them when resources permit. */
|
||||
private void processQueue() {
|
||||
// Jobs to execute after releasing the lock
|
||||
java.util.List<QueuedJob> jobsToExecute = new java.util.ArrayList<>();
|
||||
|
||||
// First synchronized block: poll jobs from the queue and prepare them for execution
|
||||
synchronized (queueLock) {
|
||||
if (shuttingDown || jobQueue.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Get current resource status
|
||||
ResourceMonitor.ResourceStatus status = resourceMonitor.getCurrentStatus().get();
|
||||
|
||||
// Check if we should execute any jobs
|
||||
boolean canExecuteJobs = (status != ResourceMonitor.ResourceStatus.CRITICAL);
|
||||
|
||||
if (!canExecuteJobs) {
|
||||
// Under critical load, don't execute any jobs
|
||||
log.debug("System under critical load, delaying job execution");
|
||||
return;
|
||||
}
|
||||
|
||||
// Get jobs from the queue, up to a limit based on resource availability
|
||||
int jobsToProcess =
|
||||
Math.max(
|
||||
1,
|
||||
switch (status) {
|
||||
case OK -> 3;
|
||||
case WARNING -> 1;
|
||||
case CRITICAL -> 0;
|
||||
});
|
||||
|
||||
for (int i = 0; i < jobsToProcess && !jobQueue.isEmpty(); i++) {
|
||||
QueuedJob job = jobQueue.poll();
|
||||
if (job == null) break;
|
||||
|
||||
// Check if it's been waiting too long
|
||||
long waitTimeMs = Instant.now().toEpochMilli() - job.queuedAt.toEpochMilli();
|
||||
if (waitTimeMs > maxWaitTimeMs) {
|
||||
log.warn(
|
||||
"Job {} exceeded maximum wait time ({} ms), executing anyway",
|
||||
job.jobId,
|
||||
waitTimeMs);
|
||||
|
||||
// Add a specific status to the job context that can be tracked
|
||||
// This will be visible in the job status API
|
||||
try {
|
||||
TaskManager taskManager =
|
||||
SpringContextHolder.getBean(TaskManager.class);
|
||||
if (taskManager != null) {
|
||||
taskManager.addNote(
|
||||
job.jobId,
|
||||
"QUEUED_TIMEOUT: Job waited in queue for "
|
||||
+ (waitTimeMs / 1000)
|
||||
+ " seconds, exceeding the maximum wait time of "
|
||||
+ (maxWaitTimeMs / 1000)
|
||||
+ " seconds.");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Failed to add timeout note to job {}: {}",
|
||||
job.jobId,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from our map
|
||||
jobMap.remove(job.jobId);
|
||||
currentQueueSize = jobQueue.size();
|
||||
|
||||
// Add to the list of jobs to execute outside the synchronized block
|
||||
jobsToExecute.add(job);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error processing job queue: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
// Now execute the jobs outside the synchronized block to avoid holding the lock
|
||||
for (QueuedJob job : jobsToExecute) {
|
||||
executeJob(job);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes a job from the queue.
|
||||
*
|
||||
* @param job The job to execute
|
||||
*/
|
||||
private void executeJob(QueuedJob job) {
|
||||
if (job.cancelled) {
|
||||
log.debug("Job {} was cancelled, not executing", job.jobId);
|
||||
return;
|
||||
}
|
||||
|
||||
jobExecutor.execute(
|
||||
() -> {
|
||||
log.debug("Executing queued job {} (queued at {})", job.jobId, job.queuedAt);
|
||||
|
||||
try {
|
||||
// Execute with timeout
|
||||
Object result = executeWithTimeout(job.work, job.timeoutMs);
|
||||
|
||||
// Process the result
|
||||
if (result instanceof ResponseEntity) {
|
||||
job.future.complete((ResponseEntity<?>) result);
|
||||
} else {
|
||||
job.future.complete(ResponseEntity.ok(result));
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error(
|
||||
"Error executing queued job {}: {}", job.jobId, e.getMessage(), e);
|
||||
job.future.completeExceptionally(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a supplier with a timeout.
|
||||
*
|
||||
* @param supplier The supplier to execute
|
||||
* @param timeoutMs The timeout in milliseconds
|
||||
* @return The result from the supplier
|
||||
* @throws Exception If there is an execution error
|
||||
*/
|
||||
private <T> T executeWithTimeout(Supplier<T> supplier, long timeoutMs) throws Exception {
|
||||
CompletableFuture<T> future = CompletableFuture.supplyAsync(supplier);
|
||||
|
||||
try {
|
||||
if (timeoutMs <= 0) {
|
||||
// No timeout
|
||||
return future.join();
|
||||
} else {
|
||||
// With timeout
|
||||
return future.get(timeoutMs, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
} catch (TimeoutException e) {
|
||||
future.cancel(true);
|
||||
throw new TimeoutException("Job timed out after " + timeoutMs + "ms");
|
||||
} catch (ExecutionException e) {
|
||||
throw (Exception) e.getCause();
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new InterruptedException("Job was interrupted");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a job is queued.
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @return true if the job is queued
|
||||
*/
|
||||
public boolean isJobQueued(String jobId) {
|
||||
return jobMap.containsKey(jobId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the current position of a job in the queue.
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @return The position (0-based) or -1 if not found
|
||||
*/
|
||||
public int getJobPosition(String jobId) {
|
||||
if (!jobMap.containsKey(jobId)) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
// Count positions
|
||||
int position = 0;
|
||||
for (QueuedJob job : jobQueue) {
|
||||
if (job.jobId.equals(jobId)) {
|
||||
return position;
|
||||
}
|
||||
position++;
|
||||
}
|
||||
|
||||
// If we didn't find it in the queue but it's in the map,
|
||||
// it might be executing already
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancels a queued job.
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @return true if the job was cancelled, false if not found
|
||||
*/
|
||||
public boolean cancelJob(String jobId) {
|
||||
QueuedJob job = jobMap.remove(jobId);
|
||||
if (job != null) {
|
||||
job.cancelled = true;
|
||||
job.future.completeExceptionally(new RuntimeException("Job cancelled by user"));
|
||||
|
||||
// Try to remove from queue if it's still there
|
||||
jobQueue.remove(job);
|
||||
currentQueueSize = jobQueue.size();
|
||||
|
||||
log.debug("Job {} cancelled", jobId);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get queue statistics.
|
||||
*
|
||||
* @return A map containing queue statistics
|
||||
*/
|
||||
public Map<String, Object> getQueueStats() {
|
||||
return Map.of(
|
||||
"queuedJobs", jobQueue.size(),
|
||||
"queueCapacity", getQueueCapacity(),
|
||||
"totalQueuedJobs", totalQueuedJobs,
|
||||
"rejectedJobs", rejectedJobs,
|
||||
"resourceStatus", resourceMonitor.getCurrentStatus().get().name());
|
||||
}
|
||||
}
|
||||
@@ -1,279 +0,0 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.lang.management.MemoryMXBean;
|
||||
import java.lang.management.OperatingSystemMXBean;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Monitors system resources (CPU, memory) to inform job scheduling decisions. Provides information
|
||||
* about available resources to prevent overloading the system.
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class ResourceMonitor {
|
||||
|
||||
@Value("${stirling.resource.memory.critical-threshold:0.9}")
|
||||
private double memoryCriticalThreshold = 0.9; // 90% usage is critical
|
||||
|
||||
@Value("${stirling.resource.memory.high-threshold:0.75}")
|
||||
private double memoryHighThreshold = 0.75; // 75% usage is high
|
||||
|
||||
@Value("${stirling.resource.cpu.critical-threshold:0.9}")
|
||||
private double cpuCriticalThreshold = 0.9; // 90% usage is critical
|
||||
|
||||
@Value("${stirling.resource.cpu.high-threshold:0.75}")
|
||||
private double cpuHighThreshold = 0.75; // 75% usage is high
|
||||
|
||||
@Value("${stirling.resource.monitor.interval-ms:60000}")
|
||||
private long monitorIntervalMs = 60000; // 60 seconds
|
||||
|
||||
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
|
||||
private final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
|
||||
private final OperatingSystemMXBean osMXBean = ManagementFactory.getOperatingSystemMXBean();
|
||||
|
||||
@Getter
|
||||
private final AtomicReference<ResourceStatus> currentStatus =
|
||||
new AtomicReference<>(ResourceStatus.OK);
|
||||
|
||||
@Getter
|
||||
private final AtomicReference<ResourceMetrics> latestMetrics =
|
||||
new AtomicReference<>(new ResourceMetrics());
|
||||
|
||||
/** Represents the current status of system resources. */
|
||||
public enum ResourceStatus {
|
||||
/** Resources are available, normal operations can proceed */
|
||||
OK,
|
||||
|
||||
/** Resources are under strain, consider queueing high-resource operations */
|
||||
WARNING,
|
||||
|
||||
/** Resources are critically low, queue all operations */
|
||||
CRITICAL
|
||||
}
|
||||
|
||||
/** Detailed metrics about system resources. */
|
||||
@Getter
|
||||
public static class ResourceMetrics {
|
||||
private final double cpuUsage;
|
||||
private final double memoryUsage;
|
||||
private final long freeMemoryBytes;
|
||||
private final long totalMemoryBytes;
|
||||
private final long maxMemoryBytes;
|
||||
private final Instant timestamp;
|
||||
|
||||
public ResourceMetrics() {
|
||||
this(0, 0, 0, 0, 0, Instant.now());
|
||||
}
|
||||
|
||||
public ResourceMetrics(
|
||||
double cpuUsage,
|
||||
double memoryUsage,
|
||||
long freeMemoryBytes,
|
||||
long totalMemoryBytes,
|
||||
long maxMemoryBytes,
|
||||
Instant timestamp) {
|
||||
this.cpuUsage = cpuUsage;
|
||||
this.memoryUsage = memoryUsage;
|
||||
this.freeMemoryBytes = freeMemoryBytes;
|
||||
this.totalMemoryBytes = totalMemoryBytes;
|
||||
this.maxMemoryBytes = maxMemoryBytes;
|
||||
this.timestamp = timestamp;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the age of these metrics.
|
||||
*
|
||||
* @return Duration since these metrics were collected
|
||||
*/
|
||||
public Duration getAge() {
|
||||
return Duration.between(timestamp, Instant.now());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if these metrics are stale (older than threshold).
|
||||
*
|
||||
* @param thresholdMs Staleness threshold in milliseconds
|
||||
* @return true if metrics are stale
|
||||
*/
|
||||
public boolean isStale(long thresholdMs) {
|
||||
return getAge().toMillis() > thresholdMs;
|
||||
}
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void initialize() {
|
||||
log.debug("Starting resource monitoring with interval of {}ms", monitorIntervalMs);
|
||||
scheduler.scheduleAtFixedRate(
|
||||
this::updateResourceMetrics, 0, monitorIntervalMs, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
log.info("Shutting down resource monitoring");
|
||||
scheduler.shutdownNow();
|
||||
}
|
||||
|
||||
/** Updates the resource metrics by sampling current system state. */
|
||||
private void updateResourceMetrics() {
|
||||
try {
|
||||
// Get CPU usage
|
||||
double cpuUsage = osMXBean.getSystemLoadAverage() / osMXBean.getAvailableProcessors();
|
||||
if (cpuUsage < 0) cpuUsage = getAlternativeCpuLoad(); // Fallback if not available
|
||||
|
||||
// Get memory usage
|
||||
long heapUsed = memoryMXBean.getHeapMemoryUsage().getUsed();
|
||||
long nonHeapUsed = memoryMXBean.getNonHeapMemoryUsage().getUsed();
|
||||
long totalUsed = heapUsed + nonHeapUsed;
|
||||
|
||||
long maxMemory = Runtime.getRuntime().maxMemory();
|
||||
long totalMemory = Runtime.getRuntime().totalMemory();
|
||||
long freeMemory = Runtime.getRuntime().freeMemory();
|
||||
|
||||
double memoryUsage = (double) totalUsed / maxMemory;
|
||||
|
||||
// Create new metrics
|
||||
ResourceMetrics metrics =
|
||||
new ResourceMetrics(
|
||||
cpuUsage,
|
||||
memoryUsage,
|
||||
freeMemory,
|
||||
totalMemory,
|
||||
maxMemory,
|
||||
Instant.now());
|
||||
latestMetrics.set(metrics);
|
||||
|
||||
// Determine system status
|
||||
ResourceStatus newStatus;
|
||||
if (cpuUsage > cpuCriticalThreshold || memoryUsage > memoryCriticalThreshold) {
|
||||
newStatus = ResourceStatus.CRITICAL;
|
||||
} else if (cpuUsage > cpuHighThreshold || memoryUsage > memoryHighThreshold) {
|
||||
newStatus = ResourceStatus.WARNING;
|
||||
} else {
|
||||
newStatus = ResourceStatus.OK;
|
||||
}
|
||||
|
||||
// Update status if it changed
|
||||
ResourceStatus oldStatus = currentStatus.getAndSet(newStatus);
|
||||
if (oldStatus != newStatus) {
|
||||
log.info("System resource status changed from {} to {}", oldStatus, newStatus);
|
||||
log.info(
|
||||
"Current metrics - CPU: {}%, Memory: {}%, Free Memory: {} MB",
|
||||
String.format("%.1f", cpuUsage * 100),
|
||||
String.format("%.1f", memoryUsage * 100),
|
||||
freeMemory / (1024 * 1024));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error updating resource metrics: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Alternative method to estimate CPU load if getSystemLoadAverage() is not available. This is a
|
||||
* fallback and less accurate than the official JMX method.
|
||||
*
|
||||
* @return Estimated CPU load as a value between 0.0 and 1.0
|
||||
*/
|
||||
private double getAlternativeCpuLoad() {
|
||||
try {
|
||||
// Try to get CPU time if available through reflection
|
||||
// This is a fallback since we can't directly cast to platform-specific classes
|
||||
try {
|
||||
java.lang.reflect.Method m =
|
||||
osMXBean.getClass().getDeclaredMethod("getProcessCpuLoad");
|
||||
m.setAccessible(true);
|
||||
return (double) m.invoke(osMXBean);
|
||||
} catch (Exception e) {
|
||||
// Try the older method
|
||||
try {
|
||||
java.lang.reflect.Method m =
|
||||
osMXBean.getClass().getDeclaredMethod("getSystemCpuLoad");
|
||||
m.setAccessible(true);
|
||||
return (double) m.invoke(osMXBean);
|
||||
} catch (Exception e2) {
|
||||
log.trace(
|
||||
"Could not get CPU load through reflection, assuming moderate load (0.5)");
|
||||
return 0.5;
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.trace("Could not get CPU load, assuming moderate load (0.5)");
|
||||
return 0.5; // Default to moderate load
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculates the dynamic job queue capacity based on current resource usage.
|
||||
*
|
||||
* @param baseCapacity The base capacity when system is under minimal load
|
||||
* @param minCapacity The minimum capacity to maintain even under high load
|
||||
* @return The calculated job queue capacity
|
||||
*/
|
||||
public int calculateDynamicQueueCapacity(int baseCapacity, int minCapacity) {
|
||||
ResourceMetrics metrics = latestMetrics.get();
|
||||
ResourceStatus status = currentStatus.get();
|
||||
|
||||
// Simple linear reduction based on memory and CPU load
|
||||
double capacityFactor =
|
||||
switch (status) {
|
||||
case OK -> 1.0;
|
||||
case WARNING -> 0.6;
|
||||
case CRITICAL -> 0.3;
|
||||
};
|
||||
|
||||
// Apply additional reduction based on specific memory pressure
|
||||
if (metrics.memoryUsage > 0.8) {
|
||||
capacityFactor *= 0.5; // Further reduce capacity under memory pressure
|
||||
}
|
||||
|
||||
// Calculate capacity with minimum safeguard
|
||||
int capacity = (int) Math.max(minCapacity, Math.ceil(baseCapacity * capacityFactor));
|
||||
|
||||
log.debug(
|
||||
"Dynamic queue capacity: {} (base: {}, factor: {:.2f}, status: {})",
|
||||
capacity,
|
||||
baseCapacity,
|
||||
capacityFactor,
|
||||
status);
|
||||
|
||||
return capacity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a job with the given weight can be executed immediately or should be queued based
|
||||
* on current resource availability.
|
||||
*
|
||||
* @param resourceWeight The resource weight of the job (1-100)
|
||||
* @return true if the job should be queued, false if it can run immediately
|
||||
*/
|
||||
public boolean shouldQueueJob(int resourceWeight) {
|
||||
ResourceStatus status = currentStatus.get();
|
||||
|
||||
// Always run lightweight jobs (weight < 20) unless critical
|
||||
if (resourceWeight < 20 && status != ResourceStatus.CRITICAL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Medium weight jobs run immediately if resources are OK
|
||||
if (resourceWeight < 60 && status == ResourceStatus.OK) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Heavy jobs (weight >= 60) and any job during WARNING/CRITICAL should be queued
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -1,208 +0,0 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
@Slf4j
|
||||
public class SsrfProtectionService {
|
||||
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
private static final Pattern DATA_URL_PATTERN =
|
||||
Pattern.compile("^data:.*", Pattern.CASE_INSENSITIVE);
|
||||
private static final Pattern FRAGMENT_PATTERN = Pattern.compile("^#.*");
|
||||
|
||||
public enum SsrfProtectionLevel {
|
||||
OFF, // No SSRF protection - allows all URLs
|
||||
MEDIUM, // Block internal networks but allow external URLs
|
||||
MAX // Block all external URLs - only data: and fragments
|
||||
}
|
||||
|
||||
public boolean isUrlAllowed(String url) {
|
||||
ApplicationProperties.Html.UrlSecurity config =
|
||||
applicationProperties.getSystem().getHtml().getUrlSecurity();
|
||||
|
||||
if (!config.isEnabled()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (url == null || url.trim().isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
String trimmedUrl = url.trim();
|
||||
|
||||
// Always allow data URLs and fragments
|
||||
if (DATA_URL_PATTERN.matcher(trimmedUrl).matches()
|
||||
|| FRAGMENT_PATTERN.matcher(trimmedUrl).matches()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
SsrfProtectionLevel level = parseProtectionLevel(config.getLevel());
|
||||
|
||||
switch (level) {
|
||||
case OFF:
|
||||
return true;
|
||||
case MAX:
|
||||
return isMaxSecurityAllowed(trimmedUrl, config);
|
||||
case MEDIUM:
|
||||
return isMediumSecurityAllowed(trimmedUrl, config);
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private SsrfProtectionLevel parseProtectionLevel(String level) {
|
||||
try {
|
||||
return SsrfProtectionLevel.valueOf(level.toUpperCase());
|
||||
} catch (IllegalArgumentException e) {
|
||||
log.warn("Invalid SSRF protection level '{}', defaulting to MEDIUM", level);
|
||||
return SsrfProtectionLevel.MEDIUM;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isMaxSecurityAllowed(
|
||||
String url, ApplicationProperties.Html.UrlSecurity config) {
|
||||
// MAX security: only allow explicitly whitelisted domains
|
||||
try {
|
||||
URI uri = new URI(url);
|
||||
String host = uri.getHost();
|
||||
|
||||
if (host == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return config.getAllowedDomains().contains(host.toLowerCase());
|
||||
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to parse URL for MAX security check: {}", url, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isMediumSecurityAllowed(
|
||||
String url, ApplicationProperties.Html.UrlSecurity config) {
|
||||
try {
|
||||
URI uri = new URI(url);
|
||||
String host = uri.getHost();
|
||||
|
||||
if (host == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
String hostLower = host.toLowerCase();
|
||||
|
||||
// Check explicit blocked domains
|
||||
if (config.getBlockedDomains().contains(hostLower)) {
|
||||
log.debug("URL blocked by explicit domain blocklist: {}", url);
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check internal TLD patterns
|
||||
for (String tld : config.getInternalTlds()) {
|
||||
if (hostLower.endsWith(tld.toLowerCase())) {
|
||||
log.debug("URL blocked by internal TLD pattern '{}': {}", tld, url);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// If allowedDomains is specified, only allow those
|
||||
if (!config.getAllowedDomains().isEmpty()) {
|
||||
boolean isAllowed =
|
||||
config.getAllowedDomains().stream()
|
||||
.anyMatch(
|
||||
domain ->
|
||||
hostLower.equals(domain.toLowerCase())
|
||||
|| hostLower.endsWith(
|
||||
"." + domain.toLowerCase()));
|
||||
|
||||
if (!isAllowed) {
|
||||
log.debug("URL not in allowed domains list: {}", url);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve hostname to IP address for network-based checks
|
||||
try {
|
||||
InetAddress address = InetAddress.getByName(host);
|
||||
|
||||
if (config.isBlockPrivateNetworks() && isPrivateAddress(address)) {
|
||||
log.debug("URL blocked - private network address: {}", url);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (config.isBlockLocalhost() && address.isLoopbackAddress()) {
|
||||
log.debug("URL blocked - localhost address: {}", url);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (config.isBlockLinkLocal() && address.isLinkLocalAddress()) {
|
||||
log.debug("URL blocked - link-local address: {}", url);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (config.isBlockCloudMetadata()
|
||||
&& isCloudMetadataAddress(address.getHostAddress())) {
|
||||
log.debug("URL blocked - cloud metadata endpoint: {}", url);
|
||||
return false;
|
||||
}
|
||||
|
||||
} catch (UnknownHostException e) {
|
||||
log.debug("Failed to resolve hostname for SSRF check: {}", host, e);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to parse URL for MEDIUM security check: {}", url, e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isPrivateAddress(InetAddress address) {
|
||||
return address.isSiteLocalAddress()
|
||||
|| address.isAnyLocalAddress()
|
||||
|| isPrivateIPv4Range(address.getHostAddress());
|
||||
}
|
||||
|
||||
private boolean isPrivateIPv4Range(String ip) {
|
||||
return ip.startsWith("10.")
|
||||
|| ip.startsWith("192.168.")
|
||||
|| (ip.startsWith("172.") && isInRange172(ip))
|
||||
|| ip.startsWith("127.")
|
||||
|| "0.0.0.0".equals(ip);
|
||||
}
|
||||
|
||||
private boolean isInRange172(String ip) {
|
||||
String[] parts = ip.split("\\.");
|
||||
if (parts.length >= 2) {
|
||||
try {
|
||||
int secondOctet = Integer.parseInt(parts[1]);
|
||||
return secondOctet >= 16 && secondOctet <= 31;
|
||||
} catch (NumberFormatException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean isCloudMetadataAddress(String ip) {
|
||||
// Cloud metadata endpoints for AWS, GCP, Azure, Oracle Cloud, and IBM Cloud
|
||||
return ip.startsWith("169.254.169.254") // AWS/GCP/Azure
|
||||
|| ip.startsWith("fd00:ec2::254") // AWS IPv6
|
||||
|| ip.startsWith("169.254.169.253") // Oracle Cloud
|
||||
|| ip.startsWith("169.254.169.250"); // IBM Cloud
|
||||
}
|
||||
}
|
||||
@@ -1,466 +0,0 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.time.LocalDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import io.github.pixee.security.ZipSecurity;
|
||||
|
||||
import jakarta.annotation.PreDestroy;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.job.JobResult;
|
||||
import stirling.software.common.model.job.JobStats;
|
||||
import stirling.software.common.model.job.ResultFile;
|
||||
|
||||
/** Manages async tasks and their results */
|
||||
@Service
|
||||
@Slf4j
|
||||
public class TaskManager {
|
||||
private final Map<String, JobResult> jobResults = new ConcurrentHashMap<>();
|
||||
|
||||
@Value("${stirling.jobResultExpiryMinutes:30}")
|
||||
private int jobResultExpiryMinutes = 30;
|
||||
|
||||
private final FileStorage fileStorage;
|
||||
private final ScheduledExecutorService cleanupExecutor =
|
||||
Executors.newSingleThreadScheduledExecutor();
|
||||
|
||||
/** Initialize the task manager and start the cleanup scheduler */
|
||||
public TaskManager(FileStorage fileStorage) {
|
||||
this.fileStorage = fileStorage;
|
||||
|
||||
// Schedule periodic cleanup of old job results
|
||||
cleanupExecutor.scheduleAtFixedRate(
|
||||
this::cleanupOldJobs,
|
||||
10, // Initial delay
|
||||
10, // Interval
|
||||
TimeUnit.MINUTES);
|
||||
|
||||
log.debug(
|
||||
"Task manager initialized with job result expiry of {} minutes",
|
||||
jobResultExpiryMinutes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new task with the given job ID
|
||||
*
|
||||
* @param jobId The job ID
|
||||
*/
|
||||
public void createTask(String jobId) {
|
||||
jobResults.put(jobId, JobResult.createNew(jobId));
|
||||
log.debug("Created task with job ID: {}", jobId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the result of a task as a general object
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @param result The result object
|
||||
*/
|
||||
public void setResult(String jobId, Object result) {
|
||||
JobResult jobResult = getOrCreateJobResult(jobId);
|
||||
jobResult.completeWithResult(result);
|
||||
log.debug("Set result for job ID: {}", jobId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the result of a task as a file
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @param fileId The file ID
|
||||
* @param originalFileName The original file name
|
||||
* @param contentType The content type of the file
|
||||
*/
|
||||
public void setFileResult(
|
||||
String jobId, String fileId, String originalFileName, String contentType) {
|
||||
JobResult jobResult = getOrCreateJobResult(jobId);
|
||||
|
||||
// Check if this is a ZIP file that should be extracted
|
||||
if (isZipFile(contentType, originalFileName)) {
|
||||
try {
|
||||
List<ResultFile> extractedFiles =
|
||||
extractZipToIndividualFiles(fileId, originalFileName);
|
||||
if (!extractedFiles.isEmpty()) {
|
||||
jobResult.completeWithFiles(extractedFiles);
|
||||
log.debug(
|
||||
"Set multiple file results for job ID: {} with {} files extracted from ZIP",
|
||||
jobId,
|
||||
extractedFiles.size());
|
||||
return;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"Failed to extract ZIP file for job {}: {}. Falling back to single file result.",
|
||||
jobId,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// Handle as single file using new ResultFile approach
|
||||
try {
|
||||
long fileSize = fileStorage.getFileSize(fileId);
|
||||
jobResult.completeWithSingleFile(fileId, originalFileName, contentType, fileSize);
|
||||
log.debug("Set single file result for job ID: {} with file ID: {}", jobId, fileId);
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"Failed to get file size for job {}: {}. Using size 0.", jobId, e.getMessage());
|
||||
jobResult.completeWithSingleFile(fileId, originalFileName, contentType, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the result of a task as multiple files
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @param resultFiles The list of result files
|
||||
*/
|
||||
public void setMultipleFileResults(String jobId, List<ResultFile> resultFiles) {
|
||||
JobResult jobResult = getOrCreateJobResult(jobId);
|
||||
jobResult.completeWithFiles(resultFiles);
|
||||
log.debug(
|
||||
"Set multiple file results for job ID: {} with {} files",
|
||||
jobId,
|
||||
resultFiles.size());
|
||||
}
|
||||
|
||||
/**
|
||||
* Set an error for a task
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @param error The error message
|
||||
*/
|
||||
public void setError(String jobId, String error) {
|
||||
JobResult jobResult = getOrCreateJobResult(jobId);
|
||||
jobResult.failWithError(error);
|
||||
log.debug("Set error for job ID: {}: {}", jobId, error);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark a task as complete
|
||||
*
|
||||
* @param jobId The job ID
|
||||
*/
|
||||
public void setComplete(String jobId) {
|
||||
JobResult jobResult = getOrCreateJobResult(jobId);
|
||||
if (jobResult.getResult() == null
|
||||
&& !jobResult.hasFiles()
|
||||
&& jobResult.getError() == null) {
|
||||
// If no result or error has been set, mark it as complete with an empty result
|
||||
jobResult.completeWithResult("Task completed successfully");
|
||||
}
|
||||
log.debug("Marked job ID: {} as complete", jobId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a task is complete
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @return true if the task is complete, false otherwise
|
||||
*/
|
||||
public boolean isComplete(String jobId) {
|
||||
JobResult result = jobResults.get(jobId);
|
||||
return result != null && result.isComplete();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the result of a task
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @return The result object, or null if the task doesn't exist or is not complete
|
||||
*/
|
||||
public JobResult getJobResult(String jobId) {
|
||||
return jobResults.get(jobId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a note to a task. Notes are informational messages that can be attached to a job for
|
||||
* tracking purposes.
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @param note The note to add
|
||||
* @return true if the note was added successfully, false if the job doesn't exist
|
||||
*/
|
||||
public boolean addNote(String jobId, String note) {
|
||||
JobResult jobResult = jobResults.get(jobId);
|
||||
if (jobResult != null) {
|
||||
jobResult.addNote(note);
|
||||
log.debug("Added note to job ID: {}: {}", jobId, note);
|
||||
return true;
|
||||
}
|
||||
log.warn("Attempted to add note to non-existent job ID: {}", jobId);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get statistics about all jobs in the system
|
||||
*
|
||||
* @return Job statistics
|
||||
*/
|
||||
public JobStats getJobStats() {
|
||||
int totalJobs = jobResults.size();
|
||||
int activeJobs = 0;
|
||||
int completedJobs = 0;
|
||||
int failedJobs = 0;
|
||||
int successfulJobs = 0;
|
||||
int fileResultJobs = 0;
|
||||
|
||||
LocalDateTime oldestActiveJobTime = null;
|
||||
LocalDateTime newestActiveJobTime = null;
|
||||
long totalProcessingTimeMs = 0;
|
||||
|
||||
for (JobResult result : jobResults.values()) {
|
||||
if (result.isComplete()) {
|
||||
completedJobs++;
|
||||
|
||||
// Calculate processing time for completed jobs
|
||||
if (result.getCreatedAt() != null && result.getCompletedAt() != null) {
|
||||
long processingTimeMs =
|
||||
java.time.Duration.between(
|
||||
result.getCreatedAt(), result.getCompletedAt())
|
||||
.toMillis();
|
||||
totalProcessingTimeMs += processingTimeMs;
|
||||
}
|
||||
|
||||
if (result.getError() != null) {
|
||||
failedJobs++;
|
||||
} else {
|
||||
successfulJobs++;
|
||||
if (result.hasFiles()) {
|
||||
fileResultJobs++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
activeJobs++;
|
||||
|
||||
// Track oldest and newest active jobs
|
||||
if (result.getCreatedAt() != null) {
|
||||
if (oldestActiveJobTime == null
|
||||
|| result.getCreatedAt().isBefore(oldestActiveJobTime)) {
|
||||
oldestActiveJobTime = result.getCreatedAt();
|
||||
}
|
||||
|
||||
if (newestActiveJobTime == null
|
||||
|| result.getCreatedAt().isAfter(newestActiveJobTime)) {
|
||||
newestActiveJobTime = result.getCreatedAt();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate average processing time
|
||||
long averageProcessingTimeMs =
|
||||
completedJobs > 0 ? totalProcessingTimeMs / completedJobs : 0;
|
||||
|
||||
return JobStats.builder()
|
||||
.totalJobs(totalJobs)
|
||||
.activeJobs(activeJobs)
|
||||
.completedJobs(completedJobs)
|
||||
.failedJobs(failedJobs)
|
||||
.successfulJobs(successfulJobs)
|
||||
.fileResultJobs(fileResultJobs)
|
||||
.oldestActiveJobTime(oldestActiveJobTime)
|
||||
.newestActiveJobTime(newestActiveJobTime)
|
||||
.averageProcessingTimeMs(averageProcessingTimeMs)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create a job result
|
||||
*
|
||||
* @param jobId The job ID
|
||||
* @return The job result
|
||||
*/
|
||||
private JobResult getOrCreateJobResult(String jobId) {
|
||||
return jobResults.computeIfAbsent(jobId, JobResult::createNew);
|
||||
}
|
||||
|
||||
/** Clean up old completed job results */
|
||||
public void cleanupOldJobs() {
|
||||
LocalDateTime expiryThreshold =
|
||||
LocalDateTime.now().minus(jobResultExpiryMinutes, ChronoUnit.MINUTES);
|
||||
int removedCount = 0;
|
||||
|
||||
try {
|
||||
for (Map.Entry<String, JobResult> entry : jobResults.entrySet()) {
|
||||
JobResult result = entry.getValue();
|
||||
|
||||
// Remove completed jobs that are older than the expiry threshold
|
||||
if (result.isComplete()
|
||||
&& result.getCompletedAt() != null
|
||||
&& result.getCompletedAt().isBefore(expiryThreshold)) {
|
||||
|
||||
// Clean up file results
|
||||
cleanupJobFiles(result, entry.getKey());
|
||||
|
||||
// Remove the job result
|
||||
jobResults.remove(entry.getKey());
|
||||
removedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (removedCount > 0) {
|
||||
log.info("Cleaned up {} expired job results", removedCount);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error during job cleanup: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Shutdown the cleanup executor */
|
||||
@PreDestroy
|
||||
public void shutdown() {
|
||||
try {
|
||||
log.info("Shutting down job result cleanup executor");
|
||||
cleanupExecutor.shutdown();
|
||||
if (!cleanupExecutor.awaitTermination(5, TimeUnit.SECONDS)) {
|
||||
cleanupExecutor.shutdownNow();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
cleanupExecutor.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
/** Check if a file is a ZIP file based on content type and filename */
|
||||
private boolean isZipFile(String contentType, String fileName) {
|
||||
if (contentType != null
|
||||
&& (contentType.equals("application/zip")
|
||||
|| contentType.equals("application/x-zip-compressed"))) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (fileName != null && fileName.toLowerCase().endsWith(".zip")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Extract a ZIP file into individual files and store them */
|
||||
private List<ResultFile> extractZipToIndividualFiles(
|
||||
String zipFileId, String originalZipFileName) throws IOException {
|
||||
List<ResultFile> extractedFiles = new ArrayList<>();
|
||||
|
||||
MultipartFile zipFile = fileStorage.retrieveFile(zipFileId);
|
||||
|
||||
try (ZipInputStream zipIn =
|
||||
ZipSecurity.createHardenedInputStream(
|
||||
new ByteArrayInputStream(zipFile.getBytes()))) {
|
||||
ZipEntry entry;
|
||||
while ((entry = zipIn.getNextEntry()) != null) {
|
||||
if (!entry.isDirectory()) {
|
||||
// Use buffered reading for memory safety
|
||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||
byte[] buffer = new byte[4096];
|
||||
int bytesRead;
|
||||
while ((bytesRead = zipIn.read(buffer)) != -1) {
|
||||
out.write(buffer, 0, bytesRead);
|
||||
}
|
||||
byte[] fileContent = out.toByteArray();
|
||||
|
||||
String contentType = determineContentType(entry.getName());
|
||||
String individualFileId = fileStorage.storeBytes(fileContent, entry.getName());
|
||||
|
||||
ResultFile resultFile =
|
||||
ResultFile.builder()
|
||||
.fileId(individualFileId)
|
||||
.fileName(entry.getName())
|
||||
.contentType(contentType)
|
||||
.fileSize(fileContent.length)
|
||||
.build();
|
||||
|
||||
extractedFiles.add(resultFile);
|
||||
log.debug(
|
||||
"Extracted file: {} (size: {} bytes)",
|
||||
entry.getName(),
|
||||
fileContent.length);
|
||||
}
|
||||
zipIn.closeEntry();
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up the original ZIP file after extraction
|
||||
try {
|
||||
fileStorage.deleteFile(zipFileId);
|
||||
log.debug("Cleaned up original ZIP file: {}", zipFileId);
|
||||
} catch (Exception e) {
|
||||
log.warn("Failed to clean up original ZIP file {}: {}", zipFileId, e.getMessage());
|
||||
}
|
||||
|
||||
return extractedFiles;
|
||||
}
|
||||
|
||||
/** Determine content type based on file extension */
|
||||
private String determineContentType(String fileName) {
|
||||
if (fileName == null) {
|
||||
return MediaType.APPLICATION_OCTET_STREAM_VALUE;
|
||||
}
|
||||
|
||||
String lowerName = fileName.toLowerCase();
|
||||
if (lowerName.endsWith(".pdf")) {
|
||||
return MediaType.APPLICATION_PDF_VALUE;
|
||||
} else if (lowerName.endsWith(".txt")) {
|
||||
return MediaType.TEXT_PLAIN_VALUE;
|
||||
} else if (lowerName.endsWith(".json")) {
|
||||
return MediaType.APPLICATION_JSON_VALUE;
|
||||
} else if (lowerName.endsWith(".xml")) {
|
||||
return MediaType.APPLICATION_XML_VALUE;
|
||||
} else if (lowerName.endsWith(".jpg") || lowerName.endsWith(".jpeg")) {
|
||||
return MediaType.IMAGE_JPEG_VALUE;
|
||||
} else if (lowerName.endsWith(".png")) {
|
||||
return MediaType.IMAGE_PNG_VALUE;
|
||||
} else {
|
||||
return MediaType.APPLICATION_OCTET_STREAM_VALUE;
|
||||
}
|
||||
}
|
||||
|
||||
/** Clean up files associated with a job result */
|
||||
private void cleanupJobFiles(JobResult result, String jobId) {
|
||||
// Clean up all result files
|
||||
if (result.hasFiles()) {
|
||||
for (ResultFile resultFile : result.getAllResultFiles()) {
|
||||
try {
|
||||
fileStorage.deleteFile(resultFile.getFileId());
|
||||
} catch (Exception e) {
|
||||
log.warn(
|
||||
"Failed to delete file {} for job {}: {}",
|
||||
resultFile.getFileId(),
|
||||
jobId,
|
||||
e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Find the ResultFile metadata for a given file ID by searching through all job results */
|
||||
public ResultFile findResultFileByFileId(String fileId) {
|
||||
for (JobResult jobResult : jobResults.values()) {
|
||||
if (jobResult.hasFiles()) {
|
||||
for (ResultFile resultFile : jobResult.getAllResultFiles()) {
|
||||
if (fileId.equals(resultFile.getFileId())) {
|
||||
return resultFile;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
-449
@@ -1,449 +0,0 @@
|
||||
package stirling.software.common.service;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Arrays;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.function.Predicate;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import jakarta.annotation.PostConstruct;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.util.GeneralUtils;
|
||||
import stirling.software.common.util.TempFileManager;
|
||||
import stirling.software.common.util.TempFileRegistry;
|
||||
|
||||
/**
|
||||
* Service to periodically clean up temporary files. Runs scheduled tasks to delete old temp files
|
||||
* and directories.
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
@RequiredArgsConstructor
|
||||
public class TempFileCleanupService {
|
||||
|
||||
private final TempFileRegistry registry;
|
||||
private final TempFileManager tempFileManager;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
@Autowired
|
||||
@Qualifier("machineType")
|
||||
private String machineType;
|
||||
|
||||
// Maximum recursion depth for directory traversal
|
||||
private static final int MAX_RECURSION_DEPTH = 5;
|
||||
|
||||
// File patterns that identify our temp files
|
||||
private static final Predicate<String> IS_OUR_TEMP_FILE =
|
||||
fileName ->
|
||||
fileName.startsWith("stirling-pdf-")
|
||||
|| fileName.startsWith("output_")
|
||||
|| fileName.startsWith("compressedPDF")
|
||||
|| fileName.startsWith("pdf-save-")
|
||||
|| fileName.startsWith("pdf-stream-")
|
||||
|| fileName.startsWith("PDFBox")
|
||||
|| fileName.startsWith("input_")
|
||||
|| fileName.startsWith("overlay-");
|
||||
|
||||
// File patterns that identify common system temp files
|
||||
private static final Predicate<String> IS_SYSTEM_TEMP_FILE =
|
||||
fileName ->
|
||||
fileName.matches("lu\\d+[a-z0-9]*\\.tmp")
|
||||
|| fileName.matches("ocr_process\\d+")
|
||||
|| (fileName.startsWith("tmp") && !fileName.contains("jetty"))
|
||||
|| fileName.startsWith("OSL_PIPE_")
|
||||
|| (fileName.endsWith(".tmp") && !fileName.contains("jetty"));
|
||||
|
||||
// File patterns that should be excluded from cleanup
|
||||
private static final Predicate<String> SHOULD_SKIP =
|
||||
fileName ->
|
||||
fileName.contains("jetty")
|
||||
|| fileName.startsWith("jetty-")
|
||||
|| "proc".equals(fileName)
|
||||
|| "sys".equals(fileName)
|
||||
|| "dev".equals(fileName)
|
||||
|| "hsperfdata_stirlingpdfuser".equals(fileName)
|
||||
|| fileName.startsWith("hsperfdata_")
|
||||
|| ".pdfbox.cache".equals(fileName);
|
||||
|
||||
@PostConstruct
|
||||
public void init() {
|
||||
// Create necessary directories
|
||||
ensureDirectoriesExist();
|
||||
|
||||
// Perform startup cleanup if enabled
|
||||
if (applicationProperties.getSystem().getTempFileManagement().isStartupCleanup()) {
|
||||
runStartupCleanup();
|
||||
}
|
||||
}
|
||||
|
||||
/** Ensure that all required temp directories exist */
|
||||
private void ensureDirectoriesExist() {
|
||||
try {
|
||||
ApplicationProperties.TempFileManagement tempFiles =
|
||||
applicationProperties.getSystem().getTempFileManagement();
|
||||
|
||||
// Create the main temp directory
|
||||
String customTempDirectory = tempFiles.getBaseTmpDir();
|
||||
if (customTempDirectory != null && !customTempDirectory.isEmpty()) {
|
||||
Path tempDir = Path.of(customTempDirectory);
|
||||
if (!Files.exists(tempDir)) {
|
||||
Files.createDirectories(tempDir);
|
||||
log.info("Created temp directory: {}", tempDir);
|
||||
}
|
||||
}
|
||||
|
||||
// Create LibreOffice temp directory
|
||||
String libreOfficeTempDir = tempFiles.getLibreofficeDir();
|
||||
if (libreOfficeTempDir != null && !libreOfficeTempDir.isEmpty()) {
|
||||
Path loTempDir = Path.of(libreOfficeTempDir);
|
||||
if (!Files.exists(loTempDir)) {
|
||||
Files.createDirectories(loTempDir);
|
||||
log.info("Created LibreOffice temp directory: {}", loTempDir);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Error creating temp directories", e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Scheduled task to clean up old temporary files. Runs at the configured interval. */
|
||||
@Scheduled(
|
||||
fixedDelayString =
|
||||
"#{applicationProperties.system.tempFileManagement.cleanupIntervalMinutes}",
|
||||
timeUnit = TimeUnit.MINUTES)
|
||||
public void scheduledCleanup() {
|
||||
log.info("Running scheduled temporary file cleanup");
|
||||
long maxAgeMillis = tempFileManager.getMaxAgeMillis();
|
||||
|
||||
// Clean up registered temp files (managed by TempFileRegistry)
|
||||
int registeredDeletedCount = tempFileManager.cleanupOldTempFiles(maxAgeMillis);
|
||||
log.info("Cleaned up {} registered temporary files", registeredDeletedCount);
|
||||
|
||||
// Clean up registered temp directories
|
||||
int directoriesDeletedCount = 0;
|
||||
for (Path directory : registry.getTempDirectories()) {
|
||||
try {
|
||||
if (Files.exists(directory)) {
|
||||
GeneralUtils.deleteDirectory(directory);
|
||||
directoriesDeletedCount++;
|
||||
log.debug("Cleaned up temporary directory: {}", directory);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to clean up temporary directory: {}", directory, e);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up PDFBox cache file
|
||||
cleanupPDFBoxCache();
|
||||
|
||||
// Clean up unregistered temp files based on our cleanup strategy
|
||||
boolean containerMode = isContainerMode();
|
||||
int unregisteredDeletedCount = cleanupUnregisteredFiles(containerMode, true, maxAgeMillis);
|
||||
|
||||
if (registeredDeletedCount > 0
|
||||
|| unregisteredDeletedCount > 0
|
||||
|| directoriesDeletedCount > 0) {
|
||||
log.info(
|
||||
"Scheduled cleanup complete. Deleted {} registered files, {} unregistered files, {} directories",
|
||||
registeredDeletedCount,
|
||||
unregisteredDeletedCount,
|
||||
directoriesDeletedCount);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform startup cleanup of stale temporary files from previous runs. This is especially
|
||||
* important in Docker environments where temp files persist between container restarts.
|
||||
*/
|
||||
private void runStartupCleanup() {
|
||||
boolean containerMode = isContainerMode();
|
||||
|
||||
log.info(
|
||||
"Running in {} mode, using {} cleanup strategy",
|
||||
machineType,
|
||||
containerMode ? "aggressive" : "conservative");
|
||||
|
||||
// For startup cleanup, we use a longer timeout for non-container environments
|
||||
long maxAgeMillis = containerMode ? 0 : 24 * 60 * 60 * 1000; // 0 or 24 hours
|
||||
|
||||
int totalDeletedCount = cleanupUnregisteredFiles(containerMode, false, maxAgeMillis);
|
||||
log.info(
|
||||
"Startup cleanup complete. Deleted {} temporary files/directories",
|
||||
totalDeletedCount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up unregistered temporary files across all configured temp directories.
|
||||
*
|
||||
* @param containerMode Whether we're in container mode (more aggressive cleanup)
|
||||
* @param isScheduled Whether this is a scheduled cleanup or startup cleanup
|
||||
* @param maxAgeMillis Maximum age of files to clean in milliseconds
|
||||
* @return Number of files deleted
|
||||
*/
|
||||
private int cleanupUnregisteredFiles(
|
||||
boolean containerMode, boolean isScheduled, long maxAgeMillis) {
|
||||
AtomicInteger totalDeletedCount = new AtomicInteger(0);
|
||||
|
||||
try {
|
||||
ApplicationProperties.TempFileManagement tempFiles =
|
||||
applicationProperties.getSystem().getTempFileManagement();
|
||||
Path[] dirsToScan;
|
||||
if (tempFiles.isCleanupSystemTemp()
|
||||
&& tempFiles.getSystemTempDir() != null
|
||||
&& !tempFiles.getSystemTempDir().isEmpty()) {
|
||||
Path systemTempPath = getSystemTempPath();
|
||||
dirsToScan =
|
||||
new Path[] {
|
||||
systemTempPath,
|
||||
Path.of(tempFiles.getBaseTmpDir()),
|
||||
Path.of(tempFiles.getLibreofficeDir())
|
||||
};
|
||||
} else {
|
||||
dirsToScan =
|
||||
new Path[] {
|
||||
Path.of(tempFiles.getBaseTmpDir()),
|
||||
Path.of(tempFiles.getLibreofficeDir())
|
||||
};
|
||||
}
|
||||
|
||||
// Process each directory
|
||||
Arrays.stream(dirsToScan)
|
||||
.filter(Files::exists)
|
||||
.forEach(
|
||||
tempDir -> {
|
||||
try {
|
||||
String phase = isScheduled ? "scheduled" : "startup";
|
||||
log.debug(
|
||||
"Scanning directory for {} cleanup: {}",
|
||||
phase,
|
||||
tempDir);
|
||||
|
||||
AtomicInteger dirDeletedCount = new AtomicInteger(0);
|
||||
cleanupDirectoryStreaming(
|
||||
tempDir,
|
||||
containerMode,
|
||||
0,
|
||||
maxAgeMillis,
|
||||
isScheduled,
|
||||
path -> {
|
||||
dirDeletedCount.incrementAndGet();
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug(
|
||||
"Deleted temp file during {} cleanup: {}",
|
||||
phase,
|
||||
path);
|
||||
}
|
||||
});
|
||||
|
||||
int count = dirDeletedCount.get();
|
||||
totalDeletedCount.addAndGet(count);
|
||||
if (count > 0) {
|
||||
log.info(
|
||||
"Cleaned up {} files/directories in {}",
|
||||
count,
|
||||
tempDir);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("Error during cleanup of directory: {}", tempDir, e);
|
||||
}
|
||||
});
|
||||
} catch (Exception e) {
|
||||
log.error("Error during cleanup of unregistered files", e);
|
||||
}
|
||||
|
||||
return totalDeletedCount.get();
|
||||
}
|
||||
|
||||
/** Get the system temp directory path based on configuration or system property. */
|
||||
private Path getSystemTempPath() {
|
||||
String systemTempDir =
|
||||
applicationProperties.getSystem().getTempFileManagement().getSystemTempDir();
|
||||
if (systemTempDir != null && !systemTempDir.isEmpty()) {
|
||||
return Path.of(systemTempDir);
|
||||
} else {
|
||||
return Path.of(System.getProperty("java.io.tmpdir"));
|
||||
}
|
||||
}
|
||||
|
||||
/** Determine if we're running in a container environment. */
|
||||
private boolean isContainerMode() {
|
||||
return "Docker".equals(machineType) || "Kubernetes".equals(machineType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively clean up a directory using a streaming approach to reduce memory usage.
|
||||
*
|
||||
* @param directory The directory to clean
|
||||
* @param containerMode Whether we're in container mode (more aggressive cleanup)
|
||||
* @param depth Current recursion depth
|
||||
* @param maxAgeMillis Maximum age of files to delete
|
||||
* @param isScheduled Whether this is a scheduled cleanup (vs startup)
|
||||
* @param onDeleteCallback Callback function when a file is deleted
|
||||
* @throws IOException If an I/O error occurs
|
||||
*/
|
||||
private void cleanupDirectoryStreaming(
|
||||
Path directory,
|
||||
boolean containerMode,
|
||||
int depth,
|
||||
long maxAgeMillis,
|
||||
boolean isScheduled,
|
||||
Consumer<Path> onDeleteCallback)
|
||||
throws IOException {
|
||||
|
||||
if (depth > MAX_RECURSION_DEPTH) {
|
||||
log.debug("Maximum directory recursion depth reached for: {}", directory);
|
||||
return;
|
||||
}
|
||||
|
||||
java.util.List<Path> subdirectories = new java.util.ArrayList<>();
|
||||
|
||||
try (Stream<Path> pathStream = Files.list(directory)) {
|
||||
pathStream.forEach(
|
||||
path -> {
|
||||
try {
|
||||
String fileName = path.getFileName().toString();
|
||||
|
||||
if (SHOULD_SKIP.test(fileName)) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (Files.isDirectory(path)) {
|
||||
subdirectories.add(path);
|
||||
return;
|
||||
}
|
||||
|
||||
if (registry.contains(path.toFile())) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (shouldDeleteFile(path, fileName, containerMode, maxAgeMillis)) {
|
||||
try {
|
||||
Files.deleteIfExists(path);
|
||||
onDeleteCallback.accept(path);
|
||||
} catch (IOException e) {
|
||||
if (e.getMessage() != null
|
||||
&& e.getMessage()
|
||||
.contains("being used by another process")) {
|
||||
log.debug("File locked, skipping delete: {}", path);
|
||||
} else {
|
||||
log.warn("Failed to delete temp file: {}", path, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("Error processing path: {}", path, e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
for (Path subdirectory : subdirectories) {
|
||||
try {
|
||||
cleanupDirectoryStreaming(
|
||||
subdirectory,
|
||||
containerMode,
|
||||
depth + 1,
|
||||
maxAgeMillis,
|
||||
isScheduled,
|
||||
onDeleteCallback);
|
||||
} catch (IOException e) {
|
||||
log.warn("Error processing subdirectory: {}", subdirectory, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Determine if a file should be deleted based on its name, age, and other criteria. */
|
||||
private boolean shouldDeleteFile(
|
||||
Path path, String fileName, boolean containerMode, long maxAgeMillis) {
|
||||
// First check if it matches our known temp file patterns
|
||||
boolean isOurTempFile = IS_OUR_TEMP_FILE.test(fileName);
|
||||
boolean isSystemTempFile = IS_SYSTEM_TEMP_FILE.test(fileName);
|
||||
|
||||
// Normal operation - check against temp file patterns
|
||||
boolean shouldDelete = isOurTempFile || (containerMode && isSystemTempFile);
|
||||
|
||||
// Get file info for age checks
|
||||
long lastModified = 0;
|
||||
long currentTime = System.currentTimeMillis();
|
||||
boolean isEmptyFile = false;
|
||||
|
||||
try {
|
||||
lastModified = Files.getLastModifiedTime(path).toMillis();
|
||||
// Special case for zero-byte files - these are often corrupted temp files
|
||||
if (Files.size(path) == 0) {
|
||||
isEmptyFile = true;
|
||||
// For empty files, use a shorter timeout (5 minutes)
|
||||
// Delete empty files older than 5 minutes
|
||||
if ((currentTime - lastModified) > 5 * 60 * 1000) {
|
||||
shouldDelete = true;
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.debug("Could not check file info, skipping: {}", path);
|
||||
}
|
||||
|
||||
// Check file age against maxAgeMillis only if it's not an empty file that we've already
|
||||
// decided to delete
|
||||
if (!isEmptyFile && shouldDelete && maxAgeMillis > 0) {
|
||||
// In normal mode, check age against maxAgeMillis
|
||||
shouldDelete = (currentTime - lastModified) > maxAgeMillis;
|
||||
}
|
||||
|
||||
return shouldDelete;
|
||||
}
|
||||
|
||||
/** Clean up LibreOffice temporary files. This method is called after LibreOffice operations. */
|
||||
public void cleanupLibreOfficeTempFiles() {
|
||||
// Cleanup known LibreOffice temp directories
|
||||
try {
|
||||
Set<Path> directories = registry.getTempDirectories();
|
||||
for (Path dir : directories) {
|
||||
if (dir.getFileName().toString().contains("libreoffice") && Files.exists(dir)) {
|
||||
// For directories containing "libreoffice", delete all contents
|
||||
// but keep the directory itself for future use
|
||||
cleanupDirectoryStreaming(
|
||||
dir,
|
||||
isContainerMode(),
|
||||
0,
|
||||
0, // age doesn't matter for LibreOffice cleanup
|
||||
false,
|
||||
path -> log.debug("Cleaned up LibreOffice temp file: {}", path));
|
||||
log.debug("Cleaned up LibreOffice temp directory contents: {}", dir);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to clean up LibreOffice temp files", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up PDFBox cache file from user home directory. This cache file can grow large and
|
||||
* should be periodically cleaned.
|
||||
*/
|
||||
private void cleanupPDFBoxCache() {
|
||||
try {
|
||||
Path userHome = Path.of(System.getProperty("user.home"));
|
||||
Path pdfboxCache = userHome.resolve(".pdfbox.cache");
|
||||
|
||||
if (Files.exists(pdfboxCache)) {
|
||||
Files.deleteIfExists(pdfboxCache);
|
||||
log.debug("Cleaned up PDFBox cache file: {}", pdfboxCache);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.warn("Failed to clean up PDFBox cache file", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
-76
@@ -1,76 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Helper class that provides access to the ApplicationContext. Useful for getting beans in classes
|
||||
* that are not managed by Spring.
|
||||
*/
|
||||
@Component
|
||||
public class ApplicationContextProvider implements ApplicationContextAware {
|
||||
|
||||
private static ApplicationContext applicationContext;
|
||||
|
||||
@Override
|
||||
public void setApplicationContext(ApplicationContext context) throws BeansException {
|
||||
applicationContext = context;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a bean by class type.
|
||||
*
|
||||
* @param <T> The type of the bean
|
||||
* @param beanClass The class of the bean
|
||||
* @return The bean instance, or null if not found
|
||||
*/
|
||||
public static <T> T getBean(Class<T> beanClass) {
|
||||
if (applicationContext == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return applicationContext.getBean(beanClass);
|
||||
} catch (BeansException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a bean by name and class type.
|
||||
*
|
||||
* @param <T> The type of the bean
|
||||
* @param name The name of the bean
|
||||
* @param beanClass The class of the bean
|
||||
* @return The bean instance, or null if not found
|
||||
*/
|
||||
public static <T> T getBean(String name, Class<T> beanClass) {
|
||||
if (applicationContext == null) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
return applicationContext.getBean(name, beanClass);
|
||||
} catch (BeansException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a bean of the specified type exists.
|
||||
*
|
||||
* @param beanClass The class of the bean
|
||||
* @return true if the bean exists, false otherwise
|
||||
*/
|
||||
public static boolean containsBean(Class<?> beanClass) {
|
||||
if (applicationContext == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
applicationContext.getBean(beanClass);
|
||||
return true;
|
||||
} catch (BeansException e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import org.apache.pdfbox.cos.COSDictionary;
|
||||
import org.apache.pdfbox.cos.COSName;
|
||||
import org.apache.pdfbox.pdmodel.PDDocument;
|
||||
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
|
||||
import org.apache.pdfbox.pdmodel.PageMode;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class AttachmentUtils {
|
||||
|
||||
/**
|
||||
* Sets the PDF catalog viewer preferences to display attachments in the viewer.
|
||||
*
|
||||
* @param document The <code>PDDocument</code> to modify.
|
||||
* @param pageMode The <code>PageMode</code> to set for the PDF viewer. <code>PageMode</code>
|
||||
* values: <code>UseNone</code>, <code>UseOutlines</code>, <code>UseThumbs</code>, <code>
|
||||
* FullScreen</code>, <code>UseOC</code>, <code>UseAttachments</code>.
|
||||
*/
|
||||
public static void setCatalogViewerPreferences(PDDocument document, PageMode pageMode) {
|
||||
try {
|
||||
PDDocumentCatalog catalog = document.getDocumentCatalog();
|
||||
if (catalog != null) {
|
||||
COSDictionary catalogDict = catalog.getCOSObject();
|
||||
|
||||
catalog.setPageMode(pageMode);
|
||||
catalogDict.setName(COSName.PAGE_MODE, pageMode.stringValue());
|
||||
|
||||
COSDictionary viewerPrefs =
|
||||
(COSDictionary) catalogDict.getDictionaryObject(COSName.VIEWER_PREFERENCES);
|
||||
if (viewerPrefs == null) {
|
||||
viewerPrefs = new COSDictionary();
|
||||
catalogDict.setItem(COSName.VIEWER_PREFERENCES, viewerPrefs);
|
||||
}
|
||||
|
||||
viewerPrefs.setName(
|
||||
COSName.getPDFName("NonFullScreenPageMode"), pageMode.stringValue());
|
||||
|
||||
viewerPrefs.setBoolean(COSName.getPDFName("DisplayDocTitle"), true);
|
||||
|
||||
log.info(
|
||||
"Set PDF PageMode to UseAttachments to automatically show attachments pane");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to set catalog viewer preferences for attachments", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,71 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import org.owasp.html.AttributePolicy;
|
||||
import org.owasp.html.HtmlPolicyBuilder;
|
||||
import org.owasp.html.PolicyFactory;
|
||||
import org.owasp.html.Sanitizers;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import stirling.software.common.model.ApplicationProperties;
|
||||
import stirling.software.common.service.SsrfProtectionService;
|
||||
|
||||
@Component
|
||||
public class CustomHtmlSanitizer {
|
||||
|
||||
private final SsrfProtectionService ssrfProtectionService;
|
||||
private final ApplicationProperties applicationProperties;
|
||||
|
||||
@Autowired
|
||||
public CustomHtmlSanitizer(
|
||||
SsrfProtectionService ssrfProtectionService,
|
||||
ApplicationProperties applicationProperties) {
|
||||
this.ssrfProtectionService = ssrfProtectionService;
|
||||
this.applicationProperties = applicationProperties;
|
||||
}
|
||||
|
||||
private final AttributePolicy SSRF_SAFE_URL_POLICY =
|
||||
new AttributePolicy() {
|
||||
@Override
|
||||
public String apply(String elementName, String attributeName, String value) {
|
||||
if (value == null || value.trim().isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String trimmedValue = value.trim();
|
||||
|
||||
// Use the SSRF protection service to validate the URL
|
||||
if (ssrfProtectionService != null
|
||||
&& !ssrfProtectionService.isUrlAllowed(trimmedValue)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return trimmedValue;
|
||||
}
|
||||
};
|
||||
|
||||
private final PolicyFactory SSRF_SAFE_IMAGES_POLICY =
|
||||
new HtmlPolicyBuilder()
|
||||
.allowElements("img")
|
||||
.allowAttributes("alt", "width", "height", "title")
|
||||
.onElements("img")
|
||||
.allowAttributes("src")
|
||||
.matching(SSRF_SAFE_URL_POLICY)
|
||||
.onElements("img")
|
||||
.toFactory();
|
||||
|
||||
private final PolicyFactory POLICY =
|
||||
Sanitizers.FORMATTING
|
||||
.and(Sanitizers.BLOCKS)
|
||||
.and(Sanitizers.STYLES)
|
||||
.and(Sanitizers.LINKS)
|
||||
.and(Sanitizers.TABLES)
|
||||
.and(SSRF_SAFE_IMAGES_POLICY)
|
||||
.and(new HtmlPolicyBuilder().disallowElements("noscript").toFactory());
|
||||
|
||||
public String sanitize(String html) {
|
||||
boolean disableSanitize =
|
||||
Boolean.TRUE.equals(applicationProperties.getSystem().getDisableSanitize());
|
||||
return disableSanitize ? html : POLICY.sanitize(html);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,327 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.text.MessageFormat;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Utility class for handling exceptions with internationalized error messages. Provides consistent
|
||||
* error handling and user-friendly messages across the application.
|
||||
*/
|
||||
@Slf4j
|
||||
public class ExceptionUtils {
|
||||
|
||||
/**
|
||||
* Create an IOException with internationalized message for PDF corruption.
|
||||
*
|
||||
* @param cause the original exception
|
||||
* @return IOException with user-friendly message
|
||||
*/
|
||||
public static IOException createPdfCorruptedException(Exception cause) {
|
||||
return createPdfCorruptedException(null, cause);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an IOException with internationalized message for PDF corruption with context.
|
||||
*
|
||||
* @param context additional context (e.g., "during merge", "during image extraction")
|
||||
* @param cause the original exception
|
||||
* @return IOException with user-friendly message
|
||||
*/
|
||||
public static IOException createPdfCorruptedException(String context, Exception cause) {
|
||||
String message;
|
||||
if (context != null && !context.isEmpty()) {
|
||||
message =
|
||||
String.format(
|
||||
"Error %s: PDF file appears to be corrupted or damaged. Please try using the 'Repair PDF' feature first to fix the file before proceeding with this operation.",
|
||||
context);
|
||||
} else {
|
||||
message =
|
||||
"PDF file appears to be corrupted or damaged. Please try using the 'Repair PDF' feature first to fix the file before proceeding with this operation.";
|
||||
}
|
||||
return new IOException(message, cause);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an IOException with internationalized message for multiple corrupted PDFs.
|
||||
*
|
||||
* @param cause the original exception
|
||||
* @return IOException with user-friendly message
|
||||
*/
|
||||
public static IOException createMultiplePdfCorruptedException(Exception cause) {
|
||||
String message =
|
||||
"One or more PDF files appear to be corrupted or damaged. Please try using the 'Repair PDF' feature on each file first before attempting to merge them.";
|
||||
return new IOException(message, cause);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an IOException with internationalized message for PDF encryption issues.
|
||||
*
|
||||
* @param cause the original exception
|
||||
* @return IOException with user-friendly message
|
||||
*/
|
||||
public static IOException createPdfEncryptionException(Exception cause) {
|
||||
String message =
|
||||
"The PDF appears to have corrupted encryption data. This can happen when the PDF was created with incompatible encryption methods. Please try using the 'Repair PDF' feature first, or contact the document creator for a new copy.";
|
||||
return new IOException(message, cause);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an IOException with internationalized message for PDF password issues.
|
||||
*
|
||||
* @param cause the original exception
|
||||
* @return IOException with user-friendly message
|
||||
*/
|
||||
public static IOException createPdfPasswordException(Exception cause) {
|
||||
String message =
|
||||
"The PDF Document is passworded and either the password was not provided or was incorrect";
|
||||
return new IOException(message, cause);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an IOException with internationalized message for file processing errors.
|
||||
*
|
||||
* @param operation the operation being performed (e.g., "merge", "split", "convert")
|
||||
* @param cause the original exception
|
||||
* @return IOException with user-friendly message
|
||||
*/
|
||||
public static IOException createFileProcessingException(String operation, Exception cause) {
|
||||
String message =
|
||||
String.format(
|
||||
"An error occurred while processing the file during %s operation: %s",
|
||||
operation, cause.getMessage());
|
||||
return new IOException(message, cause);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a generic IOException with internationalized message.
|
||||
*
|
||||
* @param messageKey the i18n message key
|
||||
* @param defaultMessage the default message if i18n is not available
|
||||
* @param cause the original exception
|
||||
* @param args optional arguments for the message
|
||||
* @return IOException with user-friendly message
|
||||
*/
|
||||
public static IOException createIOException(
|
||||
String messageKey, String defaultMessage, Exception cause, Object... args) {
|
||||
String message = MessageFormat.format(defaultMessage, args);
|
||||
return new IOException(message, cause);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a generic RuntimeException with internationalized message.
|
||||
*
|
||||
* @param messageKey the i18n message key
|
||||
* @param defaultMessage the default message if i18n is not available
|
||||
* @param cause the original exception
|
||||
* @param args optional arguments for the message
|
||||
* @return RuntimeException with user-friendly message
|
||||
*/
|
||||
public static RuntimeException createRuntimeException(
|
||||
String messageKey, String defaultMessage, Exception cause, Object... args) {
|
||||
String message = MessageFormat.format(defaultMessage, args);
|
||||
return new RuntimeException(message, cause);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an IllegalArgumentException with internationalized message.
|
||||
*
|
||||
* @param messageKey the i18n message key
|
||||
* @param defaultMessage the default message if i18n is not available
|
||||
* @param args optional arguments for the message
|
||||
* @return IllegalArgumentException with user-friendly message
|
||||
*/
|
||||
public static IllegalArgumentException createIllegalArgumentException(
|
||||
String messageKey, String defaultMessage, Object... args) {
|
||||
String message = MessageFormat.format(defaultMessage, args);
|
||||
return new IllegalArgumentException(message);
|
||||
}
|
||||
|
||||
/** Create file validation exceptions. */
|
||||
public static IllegalArgumentException createHtmlFileRequiredException() {
|
||||
return createIllegalArgumentException(
|
||||
"error.fileFormatRequired", "File must be in {0} format", "HTML or ZIP");
|
||||
}
|
||||
|
||||
public static IllegalArgumentException createPdfFileRequiredException() {
|
||||
return createIllegalArgumentException(
|
||||
"error.fileFormatRequired", "File must be in {0} format", "PDF");
|
||||
}
|
||||
|
||||
public static IllegalArgumentException createInvalidPageSizeException(String size) {
|
||||
return createIllegalArgumentException(
|
||||
"error.invalidFormat", "Invalid {0} format: {1}", "page size", size);
|
||||
}
|
||||
|
||||
/** Create OCR-related exceptions. */
|
||||
public static IOException createOcrLanguageRequiredException() {
|
||||
return createIOException(
|
||||
"error.optionsNotSpecified", "{0} options are not specified", null, "OCR language");
|
||||
}
|
||||
|
||||
public static IOException createOcrInvalidLanguagesException() {
|
||||
return createIOException(
|
||||
"error.invalidFormat",
|
||||
"Invalid {0} format: {1}",
|
||||
null,
|
||||
"OCR languages",
|
||||
"none of the selected languages are valid");
|
||||
}
|
||||
|
||||
public static IOException createOcrToolsUnavailableException() {
|
||||
return createIOException(
|
||||
"error.toolNotInstalled", "{0} is not installed", null, "OCR tools");
|
||||
}
|
||||
|
||||
/** Create system requirement exceptions. */
|
||||
public static IOException createPythonRequiredForWebpException() {
|
||||
return createIOException(
|
||||
"error.toolRequired", "{0} is required for {1}", null, "Python", "WebP conversion");
|
||||
}
|
||||
|
||||
/** Create file operation exceptions. */
|
||||
public static IOException createFileNotFoundException(String fileId) {
|
||||
return createIOException("error.fileNotFound", "File not found with ID: {0}", null, fileId);
|
||||
}
|
||||
|
||||
public static RuntimeException createPdfaConversionFailedException() {
|
||||
return createRuntimeException(
|
||||
"error.conversionFailed", "{0} conversion failed", null, "PDF/A");
|
||||
}
|
||||
|
||||
public static IllegalArgumentException createInvalidComparatorException() {
|
||||
return createIllegalArgumentException(
|
||||
"error.invalidFormat",
|
||||
"Invalid {0} format: {1}",
|
||||
"comparator",
|
||||
"only 'greater', 'equal', and 'less' are supported");
|
||||
}
|
||||
|
||||
/** Create compression-related exceptions. */
|
||||
public static RuntimeException createMd5AlgorithmException(Exception cause) {
|
||||
return createRuntimeException(
|
||||
"error.algorithmNotAvailable", "{0} algorithm not available", cause, "MD5");
|
||||
}
|
||||
|
||||
public static IllegalArgumentException createCompressionOptionsException() {
|
||||
return createIllegalArgumentException(
|
||||
"error.optionsNotSpecified",
|
||||
"{0} options are not specified",
|
||||
"compression (expected output size and optimize level)");
|
||||
}
|
||||
|
||||
public static IOException createGhostscriptCompressionException() {
|
||||
return createIOException(
|
||||
"error.commandFailed", "{0} command failed", null, "Ghostscript compression");
|
||||
}
|
||||
|
||||
public static IOException createGhostscriptCompressionException(Exception cause) {
|
||||
return createIOException(
|
||||
"error.commandFailed", "{0} command failed", cause, "Ghostscript compression");
|
||||
}
|
||||
|
||||
public static IOException createQpdfCompressionException(Exception cause) {
|
||||
return createIOException("error.commandFailed", "{0} command failed", cause, "QPDF");
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an exception indicates a corrupted PDF and wrap it with appropriate message.
|
||||
*
|
||||
* @param e the exception to check
|
||||
* @return the original exception if not PDF corruption, or a new IOException with user-friendly
|
||||
* message
|
||||
*/
|
||||
public static IOException handlePdfException(IOException e) {
|
||||
return handlePdfException(e, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an exception indicates a corrupted PDF and wrap it with appropriate message.
|
||||
*
|
||||
* @param e the exception to check
|
||||
* @param context additional context for the error
|
||||
* @return the original exception if not PDF corruption, or a new IOException with user-friendly
|
||||
* message
|
||||
*/
|
||||
public static IOException handlePdfException(IOException e, String context) {
|
||||
if (PdfErrorUtils.isCorruptedPdfError(e)) {
|
||||
return createPdfCorruptedException(context, e);
|
||||
}
|
||||
|
||||
if (isEncryptionError(e)) {
|
||||
return createPdfEncryptionException(e);
|
||||
}
|
||||
|
||||
if (isPasswordError(e)) {
|
||||
return createPdfPasswordException(e);
|
||||
}
|
||||
|
||||
return e; // Return original exception if no specific handling needed
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an exception indicates a PDF encryption/decryption error.
|
||||
*
|
||||
* @param e the exception to check
|
||||
* @return true if it's an encryption error, false otherwise
|
||||
*/
|
||||
public static boolean isEncryptionError(IOException e) {
|
||||
String message = e.getMessage();
|
||||
if (message == null) return false;
|
||||
|
||||
return message.contains("BadPaddingException")
|
||||
|| message.contains("Given final block not properly padded")
|
||||
|| message.contains("AES initialization vector not fully read")
|
||||
|| message.contains("Failed to decrypt");
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if an exception indicates a PDF password error.
|
||||
*
|
||||
* @param e the exception to check
|
||||
* @return true if it's a password error, false otherwise
|
||||
*/
|
||||
public static boolean isPasswordError(IOException e) {
|
||||
String message = e.getMessage();
|
||||
if (message == null) return false;
|
||||
|
||||
return message.contains("password is incorrect")
|
||||
|| message.contains("Password is not provided")
|
||||
|| message.contains("PDF contains an encryption dictionary");
|
||||
}
|
||||
|
||||
/**
|
||||
* Log an exception with appropriate level based on its type.
|
||||
*
|
||||
* @param operation the operation being performed
|
||||
* @param e the exception that occurred
|
||||
*/
|
||||
public static void logException(String operation, Exception e) {
|
||||
if (PdfErrorUtils.isCorruptedPdfError(e)) {
|
||||
log.warn("PDF corruption detected during {}: {}", operation, e.getMessage());
|
||||
} else if (e instanceof IOException
|
||||
&& (isEncryptionError((IOException) e) || isPasswordError((IOException) e))) {
|
||||
log.info("PDF security issue during {}: {}", operation, e.getMessage());
|
||||
} else {
|
||||
log.error("Unexpected error during {}", operation, e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Create common validation exceptions. */
|
||||
public static IllegalArgumentException createInvalidArgumentException(String argumentName) {
|
||||
return createIllegalArgumentException(
|
||||
"error.invalidArgument", "Invalid argument: {0}", argumentName);
|
||||
}
|
||||
|
||||
public static IllegalArgumentException createInvalidArgumentException(
|
||||
String argumentName, String value) {
|
||||
return createIllegalArgumentException(
|
||||
"error.invalidFormat", "Invalid {0} format: {1}", argumentName, value);
|
||||
}
|
||||
|
||||
public static IllegalArgumentException createNullArgumentException(String argumentName) {
|
||||
return createIllegalArgumentException(
|
||||
"error.argumentRequired", "{0} must not be null", argumentName);
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public class ExecutorFactory {
|
||||
|
||||
/**
|
||||
* Creates an ExecutorService using virtual threads if available (Java 21+), or falls back to a
|
||||
* cached thread pool on older Java versions.
|
||||
*/
|
||||
public static ExecutorService newVirtualOrCachedThreadExecutor() {
|
||||
try {
|
||||
ExecutorService executor =
|
||||
(ExecutorService)
|
||||
Executors.class
|
||||
.getMethod("newVirtualThreadPerTaskExecutor")
|
||||
.invoke(null);
|
||||
return executor;
|
||||
} catch (NoSuchMethodException e) {
|
||||
log.debug("Virtual threads not available; falling back to cached thread pool.");
|
||||
} catch (Exception e) {
|
||||
log.debug("Error initializing virtual thread executor: {}", e.getMessage(), e);
|
||||
}
|
||||
|
||||
return Executors.newCachedThreadPool();
|
||||
}
|
||||
}
|
||||
@@ -1,218 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.FileVisitResult;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.nio.file.SimpleFileVisitor;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Stream;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
import java.util.zip.ZipOutputStream;
|
||||
|
||||
import io.github.pixee.security.ZipSecurity;
|
||||
|
||||
import stirling.software.common.model.api.converters.HTMLToPdfRequest;
|
||||
import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult;
|
||||
|
||||
public class FileToPdf {
|
||||
|
||||
public static byte[] convertHtmlToPdf(
|
||||
String weasyprintPath,
|
||||
HTMLToPdfRequest request,
|
||||
byte[] fileBytes,
|
||||
String fileName,
|
||||
TempFileManager tempFileManager,
|
||||
CustomHtmlSanitizer customHtmlSanitizer)
|
||||
throws IOException, InterruptedException {
|
||||
|
||||
try (TempFile tempOutputFile = new TempFile(tempFileManager, ".pdf")) {
|
||||
try (TempFile tempInputFile =
|
||||
new TempFile(
|
||||
tempFileManager,
|
||||
fileName.toLowerCase().endsWith(".html") ? ".html" : ".zip")) {
|
||||
|
||||
if (fileName.toLowerCase().endsWith(".html")) {
|
||||
String sanitizedHtml =
|
||||
sanitizeHtmlContent(
|
||||
new String(fileBytes, StandardCharsets.UTF_8),
|
||||
customHtmlSanitizer);
|
||||
Files.write(
|
||||
tempInputFile.getPath(),
|
||||
sanitizedHtml.getBytes(StandardCharsets.UTF_8));
|
||||
} else if (fileName.toLowerCase().endsWith(".zip")) {
|
||||
Files.write(tempInputFile.getPath(), fileBytes);
|
||||
sanitizeHtmlFilesInZip(
|
||||
tempInputFile.getPath(), tempFileManager, customHtmlSanitizer);
|
||||
} else {
|
||||
throw ExceptionUtils.createHtmlFileRequiredException();
|
||||
}
|
||||
|
||||
List<String> command = new ArrayList<>();
|
||||
command.add(weasyprintPath);
|
||||
command.add("-e");
|
||||
command.add("utf-8");
|
||||
command.add("-v");
|
||||
command.add("--pdf-forms");
|
||||
command.add(tempInputFile.getAbsolutePath());
|
||||
command.add(tempOutputFile.getAbsolutePath());
|
||||
|
||||
ProcessExecutorResult returnCode =
|
||||
ProcessExecutor.getInstance(ProcessExecutor.Processes.WEASYPRINT)
|
||||
.runCommandWithOutputHandling(command);
|
||||
|
||||
byte[] pdfBytes = Files.readAllBytes(tempOutputFile.getPath());
|
||||
try {
|
||||
return pdfBytes;
|
||||
} catch (Exception e) {
|
||||
pdfBytes = Files.readAllBytes(tempOutputFile.getPath());
|
||||
if (pdfBytes.length < 1) {
|
||||
throw e;
|
||||
}
|
||||
return pdfBytes;
|
||||
}
|
||||
} // tempInputFile auto-closed
|
||||
} // tempOutputFile auto-closed
|
||||
}
|
||||
|
||||
private static String sanitizeHtmlContent(
|
||||
String htmlContent, CustomHtmlSanitizer customHtmlSanitizer) {
|
||||
return customHtmlSanitizer.sanitize(htmlContent);
|
||||
}
|
||||
|
||||
private static void sanitizeHtmlFilesInZip(
|
||||
Path zipFilePath,
|
||||
TempFileManager tempFileManager,
|
||||
CustomHtmlSanitizer customHtmlSanitizer)
|
||||
throws IOException {
|
||||
try (TempDirectory tempUnzippedDir = new TempDirectory(tempFileManager)) {
|
||||
try (ZipInputStream zipIn =
|
||||
ZipSecurity.createHardenedInputStream(
|
||||
new ByteArrayInputStream(Files.readAllBytes(zipFilePath)))) {
|
||||
ZipEntry entry = zipIn.getNextEntry();
|
||||
while (entry != null) {
|
||||
Path filePath =
|
||||
tempUnzippedDir.getPath().resolve(sanitizeZipFilename(entry.getName()));
|
||||
if (!entry.isDirectory()) {
|
||||
Files.createDirectories(filePath.getParent());
|
||||
if (entry.getName().toLowerCase().endsWith(".html")
|
||||
|| entry.getName().toLowerCase().endsWith(".htm")) {
|
||||
String content =
|
||||
new String(zipIn.readAllBytes(), StandardCharsets.UTF_8);
|
||||
String sanitizedContent =
|
||||
sanitizeHtmlContent(content, customHtmlSanitizer);
|
||||
Files.write(
|
||||
filePath, sanitizedContent.getBytes(StandardCharsets.UTF_8));
|
||||
} else {
|
||||
Files.copy(zipIn, filePath);
|
||||
}
|
||||
}
|
||||
zipIn.closeEntry();
|
||||
entry = zipIn.getNextEntry();
|
||||
}
|
||||
}
|
||||
|
||||
// Repack the sanitized files
|
||||
zipDirectory(tempUnzippedDir.getPath(), zipFilePath);
|
||||
} // tempUnzippedDir auto-cleaned
|
||||
}
|
||||
|
||||
private static void zipDirectory(Path sourceDir, Path zipFilePath) throws IOException {
|
||||
try (ZipOutputStream zos =
|
||||
new ZipOutputStream(new FileOutputStream(zipFilePath.toFile()))) {
|
||||
Files.walk(sourceDir)
|
||||
.filter(path -> !Files.isDirectory(path))
|
||||
.forEach(
|
||||
path -> {
|
||||
ZipEntry zipEntry =
|
||||
new ZipEntry(sourceDir.relativize(path).toString());
|
||||
try {
|
||||
zos.putNextEntry(zipEntry);
|
||||
Files.copy(path, zos);
|
||||
zos.closeEntry();
|
||||
} catch (IOException e) {
|
||||
throw new UncheckedIOException(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private static void deleteDirectory(Path dir) throws IOException {
|
||||
Files.walkFileTree(
|
||||
dir,
|
||||
new SimpleFileVisitor<Path>() {
|
||||
@Override
|
||||
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
|
||||
throws IOException {
|
||||
Files.delete(file);
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileVisitResult postVisitDirectory(Path dir, IOException exc)
|
||||
throws IOException {
|
||||
Files.delete(dir);
|
||||
return FileVisitResult.CONTINUE;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private static Path unzipAndGetMainHtml(byte[] fileBytes) throws IOException {
|
||||
Path tempDirectory = Files.createTempDirectory("unzipped_");
|
||||
try (ZipInputStream zipIn =
|
||||
ZipSecurity.createHardenedInputStream(new ByteArrayInputStream(fileBytes))) {
|
||||
ZipEntry entry = zipIn.getNextEntry();
|
||||
while (entry != null) {
|
||||
Path filePath = tempDirectory.resolve(sanitizeZipFilename(entry.getName()));
|
||||
if (entry.isDirectory()) {
|
||||
Files.createDirectories(filePath); // Explicitly create the directory structure
|
||||
} else {
|
||||
Files.createDirectories(
|
||||
filePath.getParent()); // Create parent directories if they don't exist
|
||||
Files.copy(zipIn, filePath);
|
||||
}
|
||||
zipIn.closeEntry();
|
||||
entry = zipIn.getNextEntry();
|
||||
}
|
||||
}
|
||||
|
||||
// Search for the main HTML file.
|
||||
try (Stream<Path> walk = Files.walk(tempDirectory)) {
|
||||
List<Path> htmlFiles = walk.filter(file -> file.toString().endsWith(".html")).toList();
|
||||
|
||||
if (htmlFiles.isEmpty()) {
|
||||
throw new IOException("No HTML files found in the unzipped directory.");
|
||||
}
|
||||
|
||||
// Prioritize 'index.html' if it exists, otherwise use the first .html file
|
||||
for (Path htmlFile : htmlFiles) {
|
||||
if ("index.html".equals(htmlFile.getFileName().toString())) {
|
||||
return htmlFile;
|
||||
}
|
||||
}
|
||||
|
||||
return htmlFiles.get(0);
|
||||
}
|
||||
}
|
||||
|
||||
static String sanitizeZipFilename(String entryName) {
|
||||
if (entryName == null || entryName.trim().isEmpty()) {
|
||||
return "";
|
||||
}
|
||||
// Remove any drive letters (e.g., "C:\") and leading forward/backslashes
|
||||
entryName = entryName.replaceAll("^[a-zA-Z]:[\\\\/]+", "");
|
||||
entryName = entryName.replaceAll("^[\\\\/]+", "");
|
||||
|
||||
// Recursively remove path traversal sequences
|
||||
while (entryName.contains("../") || entryName.contains("..\\")) {
|
||||
entryName = entryName.replace("../", "").replace("..\\", "");
|
||||
}
|
||||
// Normalize all backslashes to forward slashes
|
||||
entryName = entryName.replaceAll("\\\\", "/");
|
||||
return entryName;
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
/** Utility class for detecting and handling PDF-related errors. */
|
||||
public class PdfErrorUtils {
|
||||
|
||||
/**
|
||||
* Checks if an IOException indicates a corrupted PDF file.
|
||||
*
|
||||
* @param e the IOException to check
|
||||
* @return true if the error indicates PDF corruption, false otherwise
|
||||
*/
|
||||
public static boolean isCorruptedPdfError(IOException e) {
|
||||
return isCorruptedPdfError(e.getMessage());
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if any Exception indicates a corrupted PDF file.
|
||||
*
|
||||
* @param e the Exception to check
|
||||
* @return true if the error indicates PDF corruption, false otherwise
|
||||
*/
|
||||
public static boolean isCorruptedPdfError(Exception e) {
|
||||
return isCorruptedPdfError(e.getMessage());
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if an error message indicates a corrupted PDF file.
|
||||
*
|
||||
* @param message the error message to check
|
||||
* @return true if the message indicates PDF corruption, false otherwise
|
||||
*/
|
||||
private static boolean isCorruptedPdfError(String message) {
|
||||
if (message == null) return false;
|
||||
|
||||
// Check for common corruption indicators
|
||||
return message.contains("Missing root object specification")
|
||||
|| message.contains("Header doesn't contain versioninfo")
|
||||
|| message.contains("Expected trailer")
|
||||
|| message.contains("Invalid PDF")
|
||||
|| message.contains("Corrupted")
|
||||
|| message.contains("damaged")
|
||||
|| message.contains("Unknown dir object")
|
||||
|| message.contains("Can't dereference COSObject")
|
||||
|| message.contains("parseCOSString string should start with")
|
||||
|| message.contains("ICCBased colorspace array must have a stream")
|
||||
|| message.contains("1-based index not found")
|
||||
|| message.contains("Invalid dictionary, found:")
|
||||
|| message.contains("AES initialization vector not fully read")
|
||||
|| message.contains("BadPaddingException")
|
||||
|| message.contains("Given final block not properly padded")
|
||||
|| message.contains("End-of-File, expected line");
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import static stirling.software.common.util.ValidationUtils.isCollectionEmpty;
|
||||
import static stirling.software.common.util.ValidationUtils.isStringEmpty;
|
||||
|
||||
import stirling.software.common.model.oauth2.Provider;
|
||||
|
||||
public class ProviderUtils {
|
||||
|
||||
public static boolean validateProvider(Provider provider) {
|
||||
if (provider == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isStringEmpty(provider.getClientId())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isStringEmpty(provider.getClientSecret())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isCollectionEmpty(provider.getScopes())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import org.springframework.beans.BeansException;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.context.ApplicationContextAware;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Utility class to access Spring managed beans from non-Spring managed classes. This is especially
|
||||
* useful for classes that are instantiated by frameworks or created dynamically.
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
public class SpringContextHolder implements ApplicationContextAware {
|
||||
|
||||
private static ApplicationContext applicationContext;
|
||||
|
||||
@Override
|
||||
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
|
||||
SpringContextHolder.applicationContext = applicationContext;
|
||||
log.debug("Spring context holder initialized");
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a Spring bean by class type
|
||||
*
|
||||
* @param <T> The bean type
|
||||
* @param beanClass The bean class
|
||||
* @return The bean instance, or null if not found
|
||||
*/
|
||||
public static <T> T getBean(Class<T> beanClass) {
|
||||
if (applicationContext == null) {
|
||||
log.warn(
|
||||
"Application context not initialized when attempting to get bean of type {}",
|
||||
beanClass.getName());
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return applicationContext.getBean(beanClass);
|
||||
} catch (BeansException e) {
|
||||
log.error("Error getting bean of type {}: {}", beanClass.getName(), e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a Spring bean by name
|
||||
*
|
||||
* @param <T> The bean type
|
||||
* @param beanName The bean name
|
||||
* @return The bean instance, or null if not found
|
||||
*/
|
||||
public static <T> T getBean(String beanName) {
|
||||
if (applicationContext == null) {
|
||||
log.warn(
|
||||
"Application context not initialized when attempting to get bean '{}'",
|
||||
beanName);
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
@SuppressWarnings("unchecked")
|
||||
T bean = (T) applicationContext.getBean(beanName);
|
||||
return bean;
|
||||
} catch (BeansException e) {
|
||||
log.error("Error getting bean '{}': {}", beanName, e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the application context is initialized
|
||||
*
|
||||
* @return true if initialized, false otherwise
|
||||
*/
|
||||
public static boolean isInitialized() {
|
||||
return applicationContext != null;
|
||||
}
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* A wrapper class for a temporary directory that implements AutoCloseable. Can be used with
|
||||
* try-with-resources for automatic cleanup.
|
||||
*/
|
||||
@Slf4j
|
||||
public class TempDirectory implements AutoCloseable {
|
||||
|
||||
private final TempFileManager manager;
|
||||
private final Path directory;
|
||||
|
||||
public TempDirectory(TempFileManager manager) throws IOException {
|
||||
this.manager = manager;
|
||||
this.directory = manager.createTempDirectory();
|
||||
}
|
||||
|
||||
public Path getPath() {
|
||||
return directory;
|
||||
}
|
||||
|
||||
public String getAbsolutePath() {
|
||||
return directory.toAbsolutePath().toString();
|
||||
}
|
||||
|
||||
public boolean exists() {
|
||||
return java.nio.file.Files.exists(directory);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
manager.deleteTempDirectory(directory);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TempDirectory{" + directory.toAbsolutePath() + "}";
|
||||
}
|
||||
}
|
||||
@@ -1,49 +0,0 @@
|
||||
package stirling.software.common.util;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* A wrapper class for a temporary file that implements AutoCloseable. Can be used with
|
||||
* try-with-resources for automatic cleanup.
|
||||
*/
|
||||
@Slf4j
|
||||
public class TempFile implements AutoCloseable {
|
||||
|
||||
private final TempFileManager manager;
|
||||
private final File file;
|
||||
|
||||
public TempFile(TempFileManager manager, String suffix) throws IOException {
|
||||
this.manager = manager;
|
||||
this.file = manager.createTempFile(suffix);
|
||||
}
|
||||
|
||||
public File getFile() {
|
||||
return file;
|
||||
}
|
||||
|
||||
public Path getPath() {
|
||||
return file.toPath();
|
||||
}
|
||||
|
||||
public String getAbsolutePath() {
|
||||
return file.getAbsolutePath();
|
||||
}
|
||||
|
||||
public boolean exists() {
|
||||
return file.exists();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
manager.deleteTempFile(file);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "TempFile{" + file.getAbsolutePath() + "}";
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user