diff --git a/.github/config/.files.yaml b/.github/config/.files.yaml index d120123d7b..f866770b55 100644 --- a/.github/config/.files.yaml +++ b/.github/config/.files.yaml @@ -6,22 +6,27 @@ app: &app - app/(common|core|proprietary)/src/main/java/** openapi: &openapi - - build.gradle - - app/(common|core|proprietary)/build.gradle - - app/(common|core|proprietary)/src/main/java/** + - *build + - *app -project: &project - - app/(common|core|proprietary)/src/(main|test)/java/** - - app/(common|core|proprietary)/build.gradle - - 'app/(common|core|proprietary)/src/(main|test)/resources/**/!(messages_*.properties|*.md)*' - - exampleYmlFiles/** - - gradle/** - - libs/** - - 'testing/**/!(requirements*.txt|requirements*.in)*' - - build.gradle +docker: &docker - Dockerfile - Dockerfile.fat - Dockerfile.ultra-lite + - ".github/workflows/build.yml" + - scripts/init.sh + - scripts/init-without-ocr.sh + - exampleYmlFiles/** + +project: &project + - app/(common|core|proprietary)/src/(main|test)/java/** + - *build + - "app/(common|core|proprietary)/src/(main|test)/resources/**/!(messages_*.properties|*.md)*" + - exampleYmlFiles/** + - gradle/** + - libs/** + - "testing/**/!(requirements*.txt|requirements*.in)*" + - *docker - gradle.properties - gradlew - gradlew.bat diff --git a/.github/workflows/PR-Demo-Comment-with-react.yml b/.github/workflows/PR-Demo-Comment-with-react.yml index b22c1b11b7..9c3f7a392d 100644 --- a/.github/workflows/PR-Demo-Comment-with-react.yml +++ b/.github/workflows/PR-Demo-Comment-with-react.yml @@ -44,7 +44,7 @@ jobs: egress-policy: audit - name: Checkout PR - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Setup GitHub App Bot if: github.actor != 'dependabot[bot]' @@ -132,7 +132,7 @@ jobs: egress-policy: audit - name: Checkout PR - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Setup GitHub App Bot if: github.actor != 'dependabot[bot]' @@ -144,7 +144,7 @@ jobs: private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - name: Checkout PR - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: ref: refs/pull/${{ needs.check-comment.outputs.pr_number }}/merge token: ${{ steps.setup-bot.outputs.token }} @@ -366,7 +366,7 @@ jobs: egress-policy: audit - name: Check out the repository - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Setup GitHub App Bot id: setup-bot diff --git a/.github/workflows/PR-Demo-cleanup.yml b/.github/workflows/PR-Demo-cleanup.yml index c070bb6e8e..621322df73 100644 --- a/.github/workflows/PR-Demo-cleanup.yml +++ b/.github/workflows/PR-Demo-cleanup.yml @@ -26,7 +26,7 @@ jobs: egress-policy: audit - name: Checkout PR - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Setup GitHub App Bot if: github.actor != 'dependabot[bot]' diff --git a/.github/workflows/ai_pr_title_review.yml b/.github/workflows/ai_pr_title_review.yml index ce9e75496b..e93a0dc391 100644 --- a/.github/workflows/ai_pr_title_review.yml +++ b/.github/workflows/ai_pr_title_review.yml @@ -23,7 +23,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: fetch-depth: 0 diff --git a/.github/workflows/auto-labelerV2.yml b/.github/workflows/auto-labelerV2.yml index 4b93793fb9..d776aa7ac8 100644 --- a/.github/workflows/auto-labelerV2.yml +++ b/.github/workflows/auto-labelerV2.yml @@ -20,7 +20,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Setup GitHub App Bot id: setup-bot diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index da5d911315..1d7c637779 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,8 +33,9 @@ jobs: app: ${{ steps.changes.outputs.app }} project: ${{ steps.changes.outputs.project }} openapi: ${{ steps.changes.outputs.openapi }} + docker: ${{ steps.changes.outputs.docker }} steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Check for file changes uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 @@ -61,21 +62,17 @@ jobs: egress-policy: audit - name: Checkout repository - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up JDK ${{ matrix.jdk-version }} uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 with: java-version: ${{ matrix.jdk-version }} distribution: "temurin" - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@4d9f0ba0025fe599b4ebab900eb7f3a1d93ef4c2 # v5.0.0 - with: - gradle-version: 8.14 + cache: gradle - name: Build with Gradle and spring security ${{ matrix.spring-security }} - run: ./gradlew clean build + run: ./gradlew clean build -x spotlessApply -x spotlessCheck -x sonarqube env: DISABLE_ADDITIONAL_FEATURES: ${{ matrix.spring-security }} @@ -100,12 +97,14 @@ jobs: if [ ${#missing_reports[@]} -gt 0 ]; then echo "ERROR: The following required test report directories are missing:" printf '%s\n' "${missing_reports[@]}" - exit 1 + echo "reports-present=false" >> "$GITHUB_OUTPUT" + else + echo "All required test report directories are present" + echo "reports-present=true" >> "$GITHUB_OUTPUT" fi - echo "All required test report directories are present" - name: Upload Test Reports - if: always() + if: always() && steps.check-reports.outputs.reports-present == 'true' uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: test-reports-jdk-${{ matrix.jdk-version }}-spring-security-${{ matrix.spring-security }} @@ -127,6 +126,7 @@ jobs: if-no-files-found: warn - name: Add coverage to PR with spring security ${{ matrix.spring-security }} and JDK ${{ matrix.jdk-version }} + if: steps.check-reports.outputs.reports-present == 'true' id: jacoco uses: madrapps/jacoco-report@50d3aff4548aa991e6753342d9ba291084e63848 # v1.7.2 with: @@ -148,22 +148,20 @@ jobs: egress-policy: audit - name: Checkout repository - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up JDK 17 uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 with: java-version: "17" distribution: "temurin" - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@4d9f0ba0025fe599b4ebab900eb7f3a1d93ef4c2 # v5.0.0 + cache: gradle - name: Generate OpenAPI documentation run: ./gradlew :stirling-pdf:generateOpenApiDocs env: DISABLE_ADDITIONAL_FEATURES: true - + - name: Upload OpenAPI Documentation uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: @@ -181,13 +179,14 @@ jobs: egress-policy: audit - name: Checkout repository - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up JDK 17 uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 with: java-version: "17" distribution: "temurin" + cache: gradle - name: Check licenses for compatibility run: ./gradlew clean checkLicense @@ -205,8 +204,14 @@ jobs: retention-days: 3 docker-compose-tests: - if: needs.files-changed.outputs.project == 'true' - needs: files-changed + if: | + needs.files-changed.outputs.project == 'true' && + ( + needs.files-changed.outputs.docker != 'true' || + needs.test-build-docker-images.result == 'success' || + needs.test-build-docker-images.result == 'skipped' + ) + needs: [files-changed, test-build-docker-images] # if: github.event_name == 'push' && github.ref == 'refs/heads/main' || # (github.event_name == 'pull_request' && # contains(github.event.pull_request.labels.*.name, 'licenses') == false && @@ -230,27 +235,28 @@ jobs: egress-policy: audit - name: Checkout Repository - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up Java 17 uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 with: java-version: "17" distribution: "temurin" + cache: gradle - name: Set up Docker Buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - name: Install Docker Compose run: | - sudo curl -SL "https://github.com/docker/compose/releases/download/v2.37.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo curl -SL "https://github.com/docker/compose/releases/download/v2.40.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose sudo chmod +x /usr/local/bin/docker-compose - name: Set up Python uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.12" - cache: 'pip' # caching pip dependencies + cache: "pip" # caching pip dependencies cache-dependency-path: ./testing/cucumber/requirements.txt - name: Pip requirements @@ -265,13 +271,22 @@ jobs: ./testing/test.sh test-build-docker-images: - if: github.event_name == 'pull_request' && needs.files-changed.outputs.project == 'true' + if: github.event_name == 'pull_request' && needs.files-changed.outputs.docker == 'true' needs: [files-changed, build] runs-on: ubuntu-latest + permissions: + contents: read + packages: write strategy: fail-fast: false matrix: - docker-rev: ["Dockerfile", "Dockerfile.ultra-lite", "Dockerfile.fat"] + docker: + - name: "Dockerfile.ultra-lite" + tag: "ultra-lite" + - name: "Dockerfile.fat" + tag: "fat" + - name: "Dockerfile" + tag: "latest" steps: - name: Harden Runner uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 @@ -279,53 +294,227 @@ jobs: egress-policy: audit - name: Checkout Repository - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up JDK 17 uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 with: java-version: "17" distribution: "temurin" - - - name: Set up Gradle - uses: gradle/actions/setup-gradle@4d9f0ba0025fe599b4ebab900eb7f3a1d93ef4c2 # v5.0.0 - with: - gradle-version: 8.14 + cache: gradle - name: Build application - run: ./gradlew clean build + run: ./gradlew clean build -x spotlessApply -x spotlessCheck -x test -x sonarqube env: DISABLE_ADDITIONAL_FEATURES: true STIRLING_PDF_DESKTOP_UI: false + # - name: Free disk space on runner + # run: | + # echo "Disk space before cleanup:" && df -h + # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/boost + # docker system prune -af || true + # echo "Disk space after cleanup:" && df -h + - name: Set up QEMU uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + with: + platforms: linux/amd64,linux/arm64/v8 - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + with: + platforms: linux/amd64,linux/arm64/v8 - - name: Build ${{ matrix.docker-rev }} + - name: Prepare branch tag + id: branch_tag + shell: bash + run: | + BRANCH_SOURCE="${GITHUB_HEAD_REF:-${GITHUB_REF_NAME}}" + BRANCH_LOWER=$(echo "$BRANCH_SOURCE" | tr '[:upper:]' '[:lower:]') + SAFE_BRANCH=$(echo "$BRANCH_LOWER" | sed 's/[^a-z0-9_.-]/-/g' | sed 's/^-\+//' | sed 's/-\+$//' | sed 's/--\+/-/g') + if [ -z "$SAFE_BRANCH" ]; then + SAFE_BRANCH="branch" + fi + SHORT_SHA=$(echo "${GITHUB_SHA:-${{ github.sha }}}" | cut -c1-8) + echo "safe_branch=$SAFE_BRANCH" >> "$GITHUB_OUTPUT" + echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT" + + - name: Convert repository owner to lowercase + id: repoowner + run: echo "lowercase=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT + + - name: Docker meta + id: meta + uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v5.9.0 + with: + images: | + # ${{ secrets.DOCKER_HUB_USERNAME }}/stirling-pdf-test + ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test + flavor: | + latest=false + tags: | + type=raw,value=${{ matrix.docker.tag }},enable=true + # type=raw,value=${{ matrix.docker.tag }}-${{ steps.branch_tag.outputs.safe_branch }},enable=true + # type=raw,value=${{ matrix.docker.tag }}-${{ steps.branch_tag.outputs.safe_branch }}-${{ steps.branch_tag.outputs.short_sha }},enable=true + labels: | + org.opencontainers.image.title=Stirling-PDF Test + org.opencontainers.image.description=CI test image for Stirling-PDF + org.opencontainers.image.url=https://www.stirlingpdf.com + org.opencontainers.image.documentation=https://docs.stirlingpdf.com + org.opencontainers.image.authors=Stirling-Tools + org.opencontainers.image.licenses=MIT + org.opencontainers.image.version=${{ matrix.docker.tag }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.source=${{ github.repository }} + maintainer=Stirling-Tools + + - name: Choose primary tag for tests + id: testtag + shell: bash + run: | + IMAGE="ghcr.io/${{ steps.repoowner.outputs.lowercase }}/stirling-pdf-test" + VARIANT="${{ matrix.docker.tag }}" + BRANCH="${{ steps.branch_tag.outputs.safe_branch }}" + SHA_SHORT="${{ steps.branch_tag.outputs.short_sha }}" + CANDIDATE="$IMAGE:$VARIANT-$BRANCH-$SHA_SHORT" + SECONDARY="$IMAGE:$VARIANT-$BRANCH" + ALL_TAGS="$(echo '${{ steps.meta.outputs.tags }}' | tr ' ' '\n')" + if echo "$ALL_TAGS" | grep -qx "$CANDIDATE"; then + SELECTED="$CANDIDATE" + elif echo "$ALL_TAGS" | grep -qx "$SECONDARY"; then + SELECTED="$SECONDARY" + else + SELECTED="$(echo "$ALL_TAGS" | head -n1)" + fi + echo "tag=$SELECTED" >> $GITHUB_OUTPUT + echo "Using test tag: $SELECTED" + + # - name: Log in to Docker Hub + # uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + # with: + # username: ${{ secrets.DOCKER_HUB_USERNAME }} + # password: ${{ secrets.DOCKER_HUB_API }} + + # - name: Log in to GitHub Container Registry + # uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + # with: + # registry: ghcr.io + # username: ${{ github.actor }} + # password: ${{ github.token }} + + - name: Build and push amd64 image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: builder: ${{ steps.buildx.outputs.name }} context: . - file: ./${{ matrix.docker-rev }} + file: ./${{ matrix.docker.name }} push: false + load: true cache-from: type=gha cache-to: type=gha,mode=max - platforms: linux/amd64,linux/arm64/v8 - provenance: true - sbom: true + tags: ${{ steps.meta.outputs.tags }} # ALLE Tags publishen + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 + provenance: false + sbom: false - - name: Upload Reports + - name: Show amd64 image size + run: | + IMAGE_TAG="${{ steps.testtag.outputs.tag }}" + echo "Inspecting image: ${IMAGE_TAG}" + SIZE=$(docker image inspect "${IMAGE_TAG}" --format='{{.Size}}') + FORMATTED=$(numfmt --to=iec --suffix=B "${SIZE}") + echo "Image size (amd64): ${FORMATTED}" + + - name: Start amd64 image for 2 minutes + run: | + IMAGE_TAG="${{ steps.testtag.outputs.tag }}" + CONTAINER_NAME="stirling-pdf-test-${{ matrix.docker.tag }}-amd64" + echo "Starting container ${CONTAINER_NAME} from ${IMAGE_TAG}" + docker run -d --name "${CONTAINER_NAME}" "${IMAGE_TAG}" + echo "Waiting up to 2 minutes..." + sleep 120 || true + echo "===== Logs for ${CONTAINER_NAME} =====" + docker logs "${CONTAINER_NAME}" || true + echo "Stopping container ${CONTAINER_NAME} after 2 minutes" + docker stop "${CONTAINER_NAME}" || true + docker rm "${CONTAINER_NAME}" || true + + - name: Prune amd64 image and cache if: always() - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + run: | + docker image rm -f ${{ steps.testtag.outputs.tag }} || true + docker builder prune --force || true + + - name: Build and push arm64 image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: - name: reports-docker-${{ matrix.docker-rev }} - path: | - build/reports/tests/ - build/test-results/ - build/reports/problems/ - retention-days: 3 - if-no-files-found: warn + builder: ${{ steps.buildx.outputs.name }} + context: . + file: ./${{ matrix.docker.name }} + push: false + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + tags: ${{ steps.meta.outputs.tags }} # ALLE Tags publishen + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/arm64/v8 + provenance: false + sbom: false + + - name: Show arm64 image size + run: | + IMAGE_TAG="${{ steps.testtag.outputs.tag }}" + echo "Inspecting image: ${IMAGE_TAG}" + SIZE=$(docker image inspect "${IMAGE_TAG}" --format='{{.Size}}') + FORMATTED=$(numfmt --to=iec --suffix=B "${SIZE}") + echo "Image size (arm64): ${FORMATTED}" + + - name: Start arm64 image for 2 minutes + run: | + IMAGE_TAG="${{ steps.testtag.outputs.tag }}" + CONTAINER_NAME="stirling-pdf-test-${{ matrix.docker.tag }}-arm64" + echo "Starting container ${CONTAINER_NAME} from ${IMAGE_TAG}" + docker run -d --name "${CONTAINER_NAME}" "${IMAGE_TAG}" + echo "Waiting up to 2 minutes..." + sleep 120 || true + echo "===== Logs for ${CONTAINER_NAME} =====" + docker logs "${CONTAINER_NAME}" || true + echo "Stopping container ${CONTAINER_NAME} after 2 minutes" + docker stop "${CONTAINER_NAME}" || true + docker rm "${CONTAINER_NAME}" || true + + - name: Cleanup arm64 image and cache + if: always() + run: | + docker image rm -f ${{ steps.testtag.outputs.tag }} || true + docker builder prune --force || true + + # - name: Build and push multi-arch image + # uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + # with: + # builder: ${{ steps.buildx.outputs.name }} + # context: . + # file: ./${{ matrix.docker.name }} + # push: true + # cache-from: type=gha + # cache-to: type=gha,mode=max + # tags: ${{ steps.meta.outputs.tags }} + # labels: ${{ steps.meta.outputs.labels }} + # platforms: linux/amd64,linux/arm64/v8 + # provenance: false + # sbom: false + + # - name: Upload Docker build reports + # if: always() + # uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + # with: + # name: reports-docker-${{ matrix.docker.name }} + # path: | + # build/reports/ + # build/test-results/ + # build/reports/problems/ + # retention-days: 3 + # if-no-files-found: warn diff --git a/.github/workflows/check_properties.yml b/.github/workflows/check_properties.yml index f842a67c6e..a3fab958e1 100644 --- a/.github/workflows/check_properties.yml +++ b/.github/workflows/check_properties.yml @@ -37,7 +37,7 @@ jobs: egress-policy: audit - name: Checkout main branch first - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Setup GitHub App Bot id: setup-bot diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 535f4593a3..a346fae0ed 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -22,6 +22,6 @@ jobs: egress-policy: audit - name: "Checkout Repository" - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: "Dependency Review" uses: actions/dependency-review-action@3c4e3dcb1aa7874d2c16be7d79418e9b7efd6261 # v4.8.2 diff --git a/.github/workflows/licenses-update.yml b/.github/workflows/licenses-update.yml index 41a6137987..7685c013ae 100644 --- a/.github/workflows/licenses-update.yml +++ b/.github/workflows/licenses-update.yml @@ -36,7 +36,7 @@ jobs: egress-policy: audit - name: Check out code - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: fetch-depth: 0 @@ -82,7 +82,7 @@ jobs: - name: Create Pull Request id: cpr if: env.CHANGES_DETECTED == 'true' - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 with: token: ${{ steps.setup-bot.outputs.token }} commit-message: "Update 3rd Party Licenses" diff --git a/.github/workflows/manage-label.yml b/.github/workflows/manage-label.yml index 355a63b573..ee6bb3492e 100644 --- a/.github/workflows/manage-label.yml +++ b/.github/workflows/manage-label.yml @@ -20,7 +20,7 @@ jobs: egress-policy: audit - name: Check out the repository - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Run Labeler uses: crazy-max/ghaction-github-labeler@24d110aa46a59976b8a7f35518cb7f14f434c916 # v5.3.0 diff --git a/.github/workflows/multiOSReleases.yml b/.github/workflows/multiOSReleases.yml index aaa97b53f3..e1ac603b33 100644 --- a/.github/workflows/multiOSReleases.yml +++ b/.github/workflows/multiOSReleases.yml @@ -25,7 +25,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up JDK uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 @@ -64,7 +64,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up JDK 21 uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 @@ -152,7 +152,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up JDK 21 uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 diff --git a/.github/workflows/pre_commit.yml b/.github/workflows/pre_commit.yml index 10d171052f..9466788b19 100644 --- a/.github/workflows/pre_commit.yml +++ b/.github/workflows/pre_commit.yml @@ -26,7 +26,7 @@ jobs: egress-policy: audit - name: Checkout repository - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: fetch-depth: 0 @@ -67,7 +67,7 @@ jobs: - name: Create Pull Request if: env.CHANGES_DETECTED == 'true' - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 with: token: ${{ steps.setup-bot.outputs.token }} commit-message: ":file_folder: pre-commit" diff --git a/.github/workflows/push-docker.yml b/.github/workflows/push-docker.yml index 1c109df3ed..a0c873d037 100644 --- a/.github/workflows/push-docker.yml +++ b/.github/workflows/push-docker.yml @@ -34,7 +34,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up JDK 17 uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 diff --git a/.github/workflows/releaseArtifacts.yml b/.github/workflows/releaseArtifacts.yml index 3276d66c0e..c0882b22df 100644 --- a/.github/workflows/releaseArtifacts.yml +++ b/.github/workflows/releaseArtifacts.yml @@ -27,7 +27,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up JDK 17 uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 22a71bbf62..b3d1910e00 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -39,7 +39,7 @@ jobs: egress-policy: audit - name: "Checkout code" - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: persist-credentials: false @@ -74,6 +74,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@014f16e7ab1402f30e7c3329d33797e7948572db # v3.29.5 + uses: github/codeql-action/upload-sarif@fdbfb4d2750291e159f0156def62b853c2798ca2 # v3.29.5 with: sarif_file: results.sarif diff --git a/.github/workflows/swagger.yml b/.github/workflows/swagger.yml index 3eeab28b0d..369bb4f467 100644 --- a/.github/workflows/swagger.yml +++ b/.github/workflows/swagger.yml @@ -30,7 +30,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up JDK 17 uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 diff --git a/.github/workflows/sync_files.yml b/.github/workflows/sync_files.yml index 00023d8a12..8b7e430946 100644 --- a/.github/workflows/sync_files.yml +++ b/.github/workflows/sync_files.yml @@ -40,7 +40,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Setup GitHub App Bot id: setup-bot @@ -80,7 +80,7 @@ jobs: - name: Create Pull Request if: always() - uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + uses: peter-evans/create-pull-request@84ae59a2cdc2258d6fa0732dd66352dddae2a412 # v7.0.9 with: token: ${{ steps.setup-bot.outputs.token }} commit-message: Update files diff --git a/.github/workflows/testdriver.yml b/.github/workflows/testdriver.yml index 1c2a0f07b2..cf30635d5b 100644 --- a/.github/workflows/testdriver.yml +++ b/.github/workflows/testdriver.yml @@ -29,7 +29,7 @@ jobs: egress-policy: audit - name: Checkout repository - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up JDK uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0 @@ -124,7 +124,7 @@ jobs: outputs: frontend: ${{ steps.changes.outputs.frontend }} steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Check for file changes uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 @@ -143,7 +143,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up Node uses: actions/setup-node@2028fbc5c25fe9cf00d9f06a71cc4710d4507903 # v6.0.0 diff --git a/Dockerfile b/Dockerfile index bcb62ed58d..14ac55099d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,88 @@ -# Main stage -FROM alpine:3.22.2@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412 +# ============================================================================== +# Multi-stage Dockerfile for Stirling-PDF – image with everything included +# Includes: LibreOffice, Calibre, Tesseract, OCRmyPDF, unoserver, WeasyPrint, etc. +# ============================================================================== -# Copy necessary files -COPY scripts /scripts -COPY app/core/src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/ +# ======================================== +# STAGE 1: Runtime image based on Debian stable-slim +# Contains Java runtime + LibreOffice + Calibre + all PDF tools +# ======================================== +FROM debian:stable-slim@sha256:7cb087f19bcc175b96fbe4c2aef42ed00733a659581a80f6ebccfd8fe3185a3d + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV DEBIAN_FRONTEND=noninteractive + +ENV TESS_BASE_PATH=/usr/share/tesseract-ocr/5/tessdata + +# Install core runtime dependencies + tools required by Stirling-PDF features +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates tzdata tini bash fontconfig \ + openjdk-21-jre-headless \ + ffmpeg poppler-utils ocrmypdf \ + libreoffice-nogui libreoffice-java-common \ + python3 python3-venv python3-uno \ + tesseract-ocr tesseract-ocr-eng tesseract-ocr-deu tesseract-ocr-fra \ + tesseract-ocr-por tesseract-ocr-chi-sim \ + libcairo2 libpango-1.0-0 libpangoft2-1.0-0 libgdk-pixbuf-2.0-0 \ + gosu unpaper \ + # AWT headless support (required for some Java graphics operations) + libfreetype6 libfontconfig1 libx11-6 libxt6 libxext6 libxrender1 libxtst6 libxi6 \ + libxinerama1 libxkbcommon0 libxkbfile1 libsm6 libice6 \ + # Qt WebEngine dependencies for Calibre + libegl1 libopengl0 libgl1 libxdamage1 libxfixes3 libxshmfence1 libdrm2 libgbm1 \ + libxkbcommon-x11-0 libxrandr2 libxcomposite1 libnss3 libx11-xcb1 \ + libxcb-cursor0 libdbus-1-3 libglib2.0-0 \ + # Virtual framebuffer (required for headless LibreOffice) + xvfb x11-utils coreutils \ + # Temporary packages only needed for Calibre installer + xz-utils gpgv curl xdg-utils \ + \ + # Install Calibre from official installer script + && curl -fsSL https://download.calibre-ebook.com/linux-installer.sh | sh /dev/stdin \ + \ + # Clean up installer-only packages + && apt-get purge -y xz-utils gpgv xdg-utils \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* + +# Make ebook-convert available in PATH +RUN ln -sf /opt/calibre/ebook-convert /usr/bin/ebook-convert \ + && /opt/calibre/ebook-convert --version + +# ============================================================================== +# Create non-root user (stirlingpdfuser) with configurable UID/GID +# ============================================================================== +ARG PUID=1000 +ARG PGID=1000 + +RUN set -eux; \ + # Create group if it doesn't exist + if ! getent group stirlingpdfgroup >/dev/null 2>&1; then \ + if getent group "${PGID}" >/dev/null 2>&1; then \ + groupadd -o -g "${PGID}" stirlingpdfgroup; \ + else \ + groupadd -g "${PGID}" stirlingpdfgroup; \ + fi; \ + fi; \ + # Create user if it doesn't exist, avoid UID conflicts + if ! id -u stirlingpdfuser >/dev/null 2>&1; then \ + if getent passwd | awk -F: -v id="${PUID}" '$3==id{found=1} END{exit !found}'; then \ + echo "UID ${PUID} already in use – creating stirlingpdfuser with automatic UID"; \ + useradd -m -g stirlingpdfgroup -d /home/stirlingpdfuser -s /bin/bash stirlingpdfuser; \ + else \ + useradd -m -u "${PUID}" -g stirlingpdfgroup -d /home/stirlingpdfuser -s /bin/bash stirlingpdfuser; \ + fi; \ + fi + +# Compatibility alias for older entrypoint scripts expecting su-exec +RUN ln -sf /usr/sbin/gosu /usr/local/bin/su-exec + +# Copy application files from build stage +COPY scripts/ /scripts/ +COPY app/core/src/main/resources/static/fonts/*.ttf /usr/share/fonts/truetype/ COPY app/core/build/libs/*.jar app.jar +# Optional version tag (can be passed at build time) ARG VERSION_TAG LABEL org.opencontainers.image.title="Stirling-PDF" @@ -20,91 +97,68 @@ LABEL org.opencontainers.image.authors="Stirling-Tools" LABEL org.opencontainers.image.version="${VERSION_TAG}" LABEL org.opencontainers.image.keywords="PDF, manipulation, merge, split, convert, OCR, watermark" -# Set Environment Variables +# ============================================================================== +# Runtime environment variables +# ============================================================================== ENV DISABLE_ADDITIONAL_FEATURES=true \ - VERSION_TAG=$VERSION_TAG \ - JAVA_BASE_OPTS="-XX:+UnlockExperimentalVMOptions -XX:MaxRAMPercentage=75 -XX:InitiatingHeapOccupancyPercent=20 -XX:+G1PeriodicGCInvokesConcurrent -XX:G1PeriodicGCInterval=10000 -XX:+UseStringDeduplication -XX:G1PeriodicGCSystemLoadThreshold=70" \ + JAVA_BASE_OPTS="-XX:+UnlockExperimentalVMOptions -XX:MaxRAMPercentage=75 -XX:InitiatingHeapOccupancyPercent=20 \ + -XX:+G1PeriodicGCInvokesConcurrent -XX:G1PeriodicGCInterval=10000 \ + -XX:+UseStringDeduplication -XX:G1PeriodicGCSystemLoadThreshold=70 \ + -Djava.awt.headless=true" \ JAVA_CUSTOM_OPTS="" \ HOME=/home/stirlingpdfuser \ - PUID=1000 \ - PGID=1000 \ + PUID=${PUID} \ + PGID=${PGID} \ UMASK=022 \ - PYTHONPATH=/usr/lib/libreoffice/program:/opt/venv/lib/python3.12/site-packages \ UNO_PATH=/usr/lib/libreoffice/program \ - URE_BOOTSTRAP=file:///usr/lib/libreoffice/program/fundamentalrc \ - PATH=$PATH:/opt/venv/bin \ STIRLING_TEMPFILES_DIRECTORY=/tmp/stirling-pdf \ TMPDIR=/tmp/stirling-pdf \ TEMP=/tmp/stirling-pdf \ TMP=/tmp/stirling-pdf -# JDK for app -RUN apk add --no-cache bash \ - && ln -sf /bin/bash /bin/sh \ - && printf '%s\n' \ - 'https://dl-cdn.alpinelinux.org/alpine/edge/main' \ - 'https://dl-cdn.alpinelinux.org/alpine/edge/community' \ - 'https://dl-cdn.alpinelinux.org/alpine/edge/testing' \ - > /etc/apk/repositories && \ - apk upgrade --no-cache -a && \ - apk add --no-cache \ - ca-certificates \ - tzdata \ - tini \ - bash \ - curl \ - shadow \ - su-exec \ - openssl \ - openssl-dev \ - openjdk21-jre \ - ffmpeg \ - # Doc conversion - gcompat \ - libc6-compat \ - libreoffice \ - # pdftohtml - poppler-utils \ - # OCR MY PDF (unpaper for descew and other advanced features) - tesseract-ocr-data-eng \ - tesseract-ocr-data-chi_sim \ - tesseract-ocr-data-deu \ - tesseract-ocr-data-fra \ - tesseract-ocr-data-por \ - unpaper \ - # CV / Python - py3-opencv \ - python3 \ - ocrmypdf \ - py3-pip \ - py3-pillow \ - py3-pdf2image \ - # Calibre - calibre \ - # URW Base 35 fonts for better PDF rendering - font-urw-base35 && \ - # Calibre fixes - apk fix --no-cache calibre && \ - python3 -m venv /opt/venv && \ - /opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools && \ - /opt/venv/bin/pip install --no-cache-dir --upgrade unoserver weasyprint && \ - ln -s /usr/lib/libreoffice/program/uno.py /opt/venv/lib/python3.12/site-packages/ && \ - ln -s /usr/lib/libreoffice/program/unohelper.py /opt/venv/lib/python3.12/site-packages/ && \ - ln -s /usr/lib/libreoffice/program /opt/venv/lib/python3.12/site-packages/LibreOffice && \ - mv /usr/share/tessdata /usr/share/tessdata-original && \ - mkdir -p $HOME /configs /logs /customFiles /pipeline/watchedFolders /pipeline/finishedFolders /tmp/stirling-pdf && \ - # Configure URW Base 35 fonts - ln -s /usr/share/fontconfig/conf.avail/69-urw-*.conf /etc/fonts/conf.d/ && \ - fc-cache -f -v && \ - chmod +x /scripts/* && \ - # User permissions - addgroup -S stirlingpdfgroup && adduser -S stirlingpdfuser -G stirlingpdfgroup && \ - chown -R stirlingpdfuser:stirlingpdfgroup $HOME /scripts /usr/share/fonts/opentype/noto /configs /customFiles /pipeline /tmp/stirling-pdf && \ - chown stirlingpdfuser:stirlingpdfgroup /app.jar && \ - ln -sf /bin/busybox /bin/sh +# ============================================================================== +# Python virtual environment for additional Python tools (WeasyPrint, OpenCV, etc.) +# ============================================================================== +RUN python3 -m venv /opt/venv --system-site-packages \ + && /opt/venv/bin/pip install --no-cache-dir weasyprint pdf2image opencv-python-headless \ + && /opt/venv/bin/python -c "import cv2; print('OpenCV version:', cv2.__version__)" +# Separate venv for unoserver (keeps it isolated) +RUN python3 -m venv /opt/unoserver-venv --system-site-packages \ + && /opt/unoserver-venv/bin/pip install --no-cache-dir unoserver + +# Make unoserver tools available in main venv PATH +RUN ln -sf /opt/unoserver-venv/bin/unoconvert /opt/venv/bin/unoconvert \ + && ln -sf /opt/unoserver-venv/bin/unoserver /opt/venv/bin/unoserver + +# Extend PATH to include both virtual environments +ENV PATH="/opt/venv/bin:/opt/unoserver-venv/bin:${PATH}" + +# ============================================================================== +# Final permissions, directories and font cache +# ============================================================================== +RUN set -eux; \ + chmod +x /scripts/*; \ + mkdir -p /configs /logs /customFiles /pipeline/watchedFolders /pipeline/finishedFolders /tmp/stirling-pdf; \ + chown -R stirlingpdfuser:stirlingpdfgroup \ + /home/stirlingpdfuser /configs /logs /customFiles /pipeline /tmp/stirling-pdf \ + /app.jar /usr/share/fonts/truetype /scripts; \ + chmod -R 755 /tmp/stirling-pdf + +# Rebuild font cache +RUN fc-cache -f -v + +# Force Qt/WebEngine to run headlessly (required for Calibre in Docker) +ENV QT_QPA_PLATFORM=offscreen \ + QTWEBENGINE_CHROMIUM_FLAGS="--disable-gpu --disable-dev-shm-usage" + +# Expose web UI port EXPOSE 8080/tcp -# Set user and run command +STOPSIGNAL SIGTERM + +# Use tini as init (handles signals and zombies correctly) ENTRYPOINT ["tini", "--", "/scripts/init.sh"] -CMD ["sh", "-c", "java -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/tmp/stirling-pdf -jar /app.jar & /opt/venv/bin/unoserver --port 2003 --interface 127.0.0.1"] + +# CMD is empty – actual start command is defined in init.sh +CMD [] diff --git a/Dockerfile.fat b/Dockerfile.fat index 5609ffd20f..b260e94218 100644 --- a/Dockerfile.fat +++ b/Dockerfile.fat @@ -1,122 +1,209 @@ -# Build the application -FROM gradle:8.14-jdk21 AS build +# ============================================================================== +# Multi-stage Dockerfile for Stirling-PDF – "fat" image with everything included +# Includes: LibreOffice, Calibre, Tesseract, OCRmyPDF, unoserver, WeasyPrint, etc. +# ============================================================================== -COPY build.gradle . -COPY settings.gradle . -COPY gradlew . -COPY gradle gradle/ +# ======================================== +# STAGE 1: Build Stirling-PDF with Gradle (Alpine) +# ======================================== +FROM eclipse-temurin:21-jdk-alpine@sha256:c4799f335a65b1ecca8a31239b05522f2b0a184d6818f6349e83484ee6956198 AS build + +# Install build tools +RUN apk add --no-cache bash unzip curl git + +WORKDIR /workspace + +# Copy Gradle wrapper and configuration files +COPY build.gradle settings.gradle gradlew ./ +COPY gradle ./gradle/ + +# Make gradlew executable +RUN chmod +x gradlew + +# Create module directories and copy module build files (for Gradle layer caching) +RUN mkdir -p core common proprietary COPY app/core/build.gradle core/. COPY app/common/build.gradle common/. COPY app/proprietary/build.gradle proprietary/. -RUN ./gradlew build -x spotlessApply -x spotlessCheck -x test -x sonarqube || return 0 -# Set the working directory +# Warm-up Gradle dependency cache (optional but improves subsequent builds) +RUN ./gradlew --no-daemon printVersion --quiet | tail -1 > /tmp/version_tag || true +RUN ./gradlew --no-daemon build -x spotlessApply -x spotlessCheck -x test -x sonarqube || true + +# Switch to final source directory and copy full source code WORKDIR /app - -# Copy the entire project to the working directory COPY . . -# Build the application with DISABLE_ADDITIONAL_FEATURES=false +# Environment variables (can be overridden at build time) ENV DISABLE_ADDITIONAL_FEATURES=false \ STIRLING_PDF_DESKTOP_UI=false -RUN ./gradlew clean build -x spotlessApply -x spotlessCheck -x test -x sonarqube -# Main stage -FROM alpine:3.22.2@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412 +# Final build – produce the fat JAR +RUN ./gradlew --no-daemon clean build \ + -x spotlessApply -x spotlessCheck -x test -x sonarqube \ + && apk del bash unzip curl git -# Copy necessary files -COPY scripts /scripts -COPY app/core/src/main/resources/static/fonts/*.ttf /usr/share/fonts/opentype/noto/ -# first /app directory is for the build stage, second is for the final image -COPY --from=build /app/app/core/build/libs/*.jar app.jar +# ======================================== +# STAGE 2: Runtime image based on Debian stable-slim +# Contains Java runtime + LibreOffice + Calibre + all PDF tools +# ======================================== +FROM debian:stable-slim@sha256:7cb087f19bcc175b96fbe4c2aef42ed00733a659581a80f6ebccfd8fe3185a3d + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV DEBIAN_FRONTEND=noninteractive + +# Install core runtime dependencies + tools required by Stirling-PDF features +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates tzdata tini bash fontconfig \ + openjdk-21-jre-headless \ + ffmpeg poppler-utils qpdf ghostscript ocrmypdf \ + libreoffice-nogui libreoffice-java-common \ + python3 python3-venv python3-uno \ + tesseract-ocr tesseract-ocr-eng tesseract-ocr-deu tesseract-ocr-fra \ + tesseract-ocr-por tesseract-ocr-chi-sim \ + libcairo2 libpango-1.0-0 libpangoft2-1.0-0 libgdk-pixbuf-2.0-0 \ + gosu unpaper \ + # AWT headless support (required for some Java graphics operations) + libfreetype6 libfontconfig1 libx11-6 libxt6 libxext6 libxrender1 libxtst6 libxi6 \ + libxinerama1 libxkbcommon0 libxkbfile1 libsm6 libice6 \ + # Qt WebEngine dependencies for Calibre + libegl1 libopengl0 libgl1 libxdamage1 libxfixes3 libxshmfence1 libdrm2 libgbm1 \ + libxkbcommon-x11-0 libxrandr2 libxcomposite1 libnss3 libx11-xcb1 \ + libxcb-cursor0 libdbus-1-3 libglib2.0-0 \ + # Virtual framebuffer (required for headless LibreOffice) + xvfb x11-utils coreutils \ + # Temporary packages only needed for Calibre installer + xz-utils gpgv curl xdg-utils \ + \ + # Install Calibre from official installer script + && curl -fsSL https://download.calibre-ebook.com/linux-installer.sh | sh /dev/stdin \ + \ + # Clean up installer-only packages + && apt-get purge -y xz-utils gpgv xdg-utils \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* + +# Make ebook-convert available in PATH +RUN ln -sf /opt/calibre/ebook-convert /usr/bin/ebook-convert \ + && /opt/calibre/ebook-convert --version + +# ============================================================================== +# Create non-root user (stirlingpdfuser) with configurable UID/GID +# ============================================================================== +ARG PUID=1000 +ARG PGID=1000 + +RUN set -eux; \ + # Create group if it doesn't exist + if ! getent group stirlingpdfgroup >/dev/null 2>&1; then \ + if getent group "${PGID}" >/dev/null 2>&1; then \ + groupadd -o -g "${PGID}" stirlingpdfgroup; \ + else \ + groupadd -g "${PGID}" stirlingpdfgroup; \ + fi; \ + fi; \ + # Create user if it doesn't exist, avoid UID conflicts + if ! id -u stirlingpdfuser >/dev/null 2>&1; then \ + if getent passwd | awk -F: -v id="${PUID}" '$3==id{found=1} END{exit !found}'; then \ + echo "UID ${PUID} already in use – creating stirlingpdfuser with automatic UID"; \ + useradd -m -g stirlingpdfgroup -d /home/stirlingpdfuser -s /bin/bash stirlingpdfuser; \ + else \ + useradd -m -u "${PUID}" -g stirlingpdfgroup -d /home/stirlingpdfuser -s /bin/bash stirlingpdfuser; \ + fi; \ + fi + +# Compatibility alias for older entrypoint scripts expecting su-exec +RUN ln -sf /usr/sbin/gosu /usr/local/bin/su-exec + +# Copy application files from build stage +COPY scripts/ /scripts/ +COPY app/core/src/main/resources/static/fonts/*.ttf /usr/share/fonts/truetype/ +COPY --from=build /app/app/core/build/libs/*.jar /app.jar + +# Copy version tag generated during build +COPY --from=build /tmp/version_tag /etc/stirling_version + +# Optional version tag (can be passed at build time) ARG VERSION_TAG -# Set Environment Variables +# Metadata labels +LABEL org.opencontainers.image.title="Stirling-PDF" +LABEL org.opencontainers.image.description="A powerful locally hosted web-based PDF manipulation tool supporting 50+ operations including merging, splitting, conversion, OCR, watermarking, and more." +LABEL org.opencontainers.image.source="https://github.com/Stirling-Tools/Stirling-PDF" +LABEL org.opencontainers.image.licenses="MIT" +LABEL org.opencontainers.image.vendor="Stirling-Tools" +LABEL org.opencontainers.image.url="https://www.stirlingpdf.com" +LABEL org.opencontainers.image.documentation="https://docs.stirlingpdf.com" +LABEL maintainer="Stirling-Tools" +LABEL org.opencontainers.image.authors="Stirling-Tools" +LABEL org.opencontainers.image.version="${VERSION_TAG}" +LABEL org.opencontainers.image.keywords="PDF, manipulation, merge, split, convert, OCR, watermark" + +# ============================================================================== +# Runtime environment variables +# ============================================================================== ENV DISABLE_ADDITIONAL_FEATURES=true \ - VERSION_TAG=$VERSION_TAG \ - JAVA_BASE_OPTS="-XX:+UnlockExperimentalVMOptions -XX:MaxRAMPercentage=75 -XX:InitiatingHeapOccupancyPercent=20 -XX:+G1PeriodicGCInvokesConcurrent -XX:G1PeriodicGCInterval=10000 -XX:+UseStringDeduplication -XX:G1PeriodicGCSystemLoadThreshold=70" \ + JAVA_BASE_OPTS="-XX:+UnlockExperimentalVMOptions -XX:MaxRAMPercentage=75 -XX:InitiatingHeapOccupancyPercent=20 \ + -XX:+G1PeriodicGCInvokesConcurrent -XX:G1PeriodicGCInterval=10000 \ + -XX:+UseStringDeduplication -XX:G1PeriodicGCSystemLoadThreshold=70 \ + -Djava.awt.headless=true" \ JAVA_CUSTOM_OPTS="" \ HOME=/home/stirlingpdfuser \ - PUID=1000 \ - PGID=1000 \ + PUID=${PUID} \ + PGID=${PGID} \ UMASK=022 \ FAT_DOCKER=true \ INSTALL_BOOK_AND_ADVANCED_HTML_OPS=false \ - PYTHONPATH=/usr/lib/libreoffice/program:/opt/venv/lib/python3.12/site-packages \ UNO_PATH=/usr/lib/libreoffice/program \ - URE_BOOTSTRAP=file:///usr/lib/libreoffice/program/fundamentalrc \ - PATH=$PATH:/opt/venv/bin \ STIRLING_TEMPFILES_DIRECTORY=/tmp/stirling-pdf \ TMPDIR=/tmp/stirling-pdf \ TEMP=/tmp/stirling-pdf \ TMP=/tmp/stirling-pdf -# JDK for app -RUN apk add --no-cache bash \ - && ln -sf /bin/bash /bin/sh \ - && printf '%s\n' \ - 'https://dl-cdn.alpinelinux.org/alpine/edge/main' \ - 'https://dl-cdn.alpinelinux.org/alpine/edge/community' \ - 'https://dl-cdn.alpinelinux.org/alpine/edge/testing' \ - > /etc/apk/repositories && \ - apk upgrade --no-cache -a && \ - apk add --no-cache \ - ca-certificates \ - tzdata \ - tini \ - bash \ - curl \ - shadow \ - su-exec \ - openssl \ - openssl-dev \ - openjdk21-jre \ - ffmpeg \ - # Doc conversion - gcompat \ - libc6-compat \ - libreoffice \ - # pdftohtml - poppler-utils \ - # OCR MY PDF (unpaper for descew and other advanced featues) - tesseract-ocr-data-eng \ - tesseract-ocr-data-chi_sim \ - tesseract-ocr-data-deu \ - tesseract-ocr-data-fra \ - tesseract-ocr-data-por \ - unpaper \ - font-terminus font-dejavu font-noto font-noto-cjk font-awesome font-noto-extra font-liberation font-linux-libertine font-urw-base35 \ - # CV / Python - py3-opencv \ - python3 \ - ocrmypdf \ - py3-pip \ - py3-pillow \ - py3-pdf2image \ - # Calibre (musl-native) + QtWebEngine Runtime - calibre && \ - # Calibre fixes - apk fix --no-cache calibre && \ - python3 -m venv /opt/venv && \ - /opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools && \ - /opt/venv/bin/pip install --no-cache-dir --upgrade unoserver weasyprint && \ - ln -s /usr/lib/libreoffice/program/uno.py /opt/venv/lib/python3.12/site-packages/ && \ - ln -s /usr/lib/libreoffice/program/unohelper.py /opt/venv/lib/python3.12/site-packages/ && \ - ln -s /usr/lib/libreoffice/program /opt/venv/lib/python3.12/site-packages/LibreOffice && \ - mv /usr/share/tessdata /usr/share/tessdata-original && \ - mkdir -p $HOME /configs /logs /customFiles /pipeline/watchedFolders /pipeline/finishedFolders /tmp/stirling-pdf && \ - # Configure URW Base 35 fonts - ln -s /usr/share/fontconfig/conf.avail/69-urw-*.conf /etc/fonts/conf.d/ && \ - fc-cache -f -v && \ - chmod +x /scripts/* && \ - # User permissions - addgroup -S stirlingpdfgroup && adduser -S stirlingpdfuser -G stirlingpdfgroup && \ - chown -R stirlingpdfuser:stirlingpdfgroup $HOME /scripts /usr/share/fonts/opentype/noto /configs /customFiles /pipeline /tmp/stirling-pdf && \ - chown stirlingpdfuser:stirlingpdfgroup /app.jar && \ - ln -sf /bin/busybox /bin/sh +# ============================================================================== +# Python virtual environment for additional Python tools (WeasyPrint, OpenCV, etc.) +# ============================================================================== +RUN python3 -m venv /opt/venv --system-site-packages \ + && /opt/venv/bin/pip install --no-cache-dir weasyprint pdf2image opencv-python-headless \ + && /opt/venv/bin/python -c "import cv2; print('OpenCV version:', cv2.__version__)" +# Separate venv for unoserver (keeps it isolated) +RUN python3 -m venv /opt/unoserver-venv --system-site-packages \ + && /opt/unoserver-venv/bin/pip install --no-cache-dir unoserver + +# Make unoserver tools available in main venv PATH +RUN ln -sf /opt/unoserver-venv/bin/unoconvert /opt/venv/bin/unoconvert \ + && ln -sf /opt/unoserver-venv/bin/unoserver /opt/venv/bin/unoserver + +# Extend PATH to include both virtual environments +ENV PATH="/opt/venv/bin:/opt/unoserver-venv/bin:${PATH}" + +# ============================================================================== +# Final permissions, directories and font cache +# ============================================================================== +RUN set -eux; \ + chmod +x /scripts/*; \ + mkdir -p /configs /logs /customFiles /pipeline/watchedFolders /pipeline/finishedFolders /tmp/stirling-pdf; \ + chown -R stirlingpdfuser:stirlingpdfgroup \ + /home/stirlingpdfuser /configs /logs /customFiles /pipeline /tmp/stirling-pdf \ + /app.jar /usr/share/fonts/truetype /scripts; \ + chmod -R 755 /tmp/stirling-pdf + +# Rebuild font cache +RUN fc-cache -f -v + +# Force Qt/WebEngine to run headlessly (required for Calibre in Docker) +ENV QT_QPA_PLATFORM=offscreen \ + QTWEBENGINE_CHROMIUM_FLAGS="--disable-gpu --disable-dev-shm-usage" + +# Expose web UI port EXPOSE 8080/tcp -# Set user and run command + +STOPSIGNAL SIGTERM + +# Use tini as init (handles signals and zombies correctly) ENTRYPOINT ["tini", "--", "/scripts/init.sh"] -CMD ["sh", "-c", "java -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/tmp/stirling-pdf -jar /app.jar & /opt/venv/bin/unoserver --port 2003 --interface 127.0.0.1"] + +# CMD is empty – actual start command is defined in init.sh +CMD [] diff --git a/Dockerfile.ultra-lite b/Dockerfile.ultra-lite index a49362d605..e364ba0a73 100644 --- a/Dockerfile.ultra-lite +++ b/Dockerfile.ultra-lite @@ -56,4 +56,4 @@ EXPOSE 8080/tcp # Run the application ENTRYPOINT ["tini", "--", "/scripts/init-without-ocr.sh"] -CMD ["java", "-Dfile.encoding=UTF-8", "-Djava.io.tmpdir=/tmp/stirling-pdf", "-jar", "/app.jar"] +CMD [] diff --git a/app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java b/app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java index f8bc38a6bb..7e6da0f0f2 100644 --- a/app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java +++ b/app/common/src/main/java/stirling/software/common/configuration/RuntimePathConfig.java @@ -10,8 +10,10 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import stirling.software.common.model.ApplicationProperties; +import stirling.software.common.model.ApplicationProperties.CustomPaths; import stirling.software.common.model.ApplicationProperties.CustomPaths.Operations; import stirling.software.common.model.ApplicationProperties.CustomPaths.Pipeline; +import stirling.software.common.model.ApplicationProperties.System; @Slf4j @Configuration @@ -19,9 +21,16 @@ import stirling.software.common.model.ApplicationProperties.CustomPaths.Pipeline public class RuntimePathConfig { private final ApplicationProperties properties; private final String basePath; + + // Operation paths private final String weasyPrintPath; private final String unoConvertPath; private final String calibrePath; + private final String ocrMyPdfPath; + private final String sOfficePath; + + // Tesseract data path + private final String tessDataPath; // Pipeline paths private final String pipelineWatchedFoldersPath; @@ -38,7 +47,10 @@ public class RuntimePathConfig { String defaultFinishedFolders = Path.of(this.pipelinePath, "finishedFolders").toString(); String defaultWebUIConfigs = Path.of(this.pipelinePath, "defaultWebUIConfigs").toString(); - Pipeline pipeline = properties.getSystem().getCustomPaths().getPipeline(); + System system = properties.getSystem(); + CustomPaths customPaths = system.getCustomPaths(); + + Pipeline pipeline = customPaths.getPipeline(); this.pipelineWatchedFoldersPath = resolvePath( @@ -58,9 +70,11 @@ public class RuntimePathConfig { // Initialize Operation paths String defaultWeasyPrintPath = isDocker ? "/opt/venv/bin/weasyprint" : "weasyprint"; String defaultUnoConvertPath = isDocker ? "/opt/venv/bin/unoconvert" : "unoconvert"; - String defaultCalibrePath = isDocker ? "/usr/bin/ebook-convert" : "ebook-convert"; + String defaultCalibrePath = isDocker ? "/opt/calibre/ebook-convert" : "ebook-convert"; + String defaultOcrMyPdfPath = isDocker ? "/usr/bin/ocrmypdf" : "ocrmypdf"; + String defaultSOfficePath = isDocker ? "/usr/bin/soffice" : "soffice"; - Operations operations = properties.getSystem().getCustomPaths().getOperations(); + Operations operations = customPaths.getOperations(); this.weasyPrintPath = resolvePath( defaultWeasyPrintPath, @@ -72,6 +86,25 @@ public class RuntimePathConfig { this.calibrePath = resolvePath( defaultCalibrePath, operations != null ? operations.getCalibre() : null); + this.ocrMyPdfPath = + resolvePath( + defaultOcrMyPdfPath, operations != null ? operations.getOcrmypdf() : null); + this.sOfficePath = + resolvePath( + defaultSOfficePath, operations != null ? operations.getSoffice() : null); + + // Initialize Tesseract data path + String defaultTessDataPath = + isDocker ? "/usr/share/tesseract-ocr/5/tessdata" : "/usr/share/tessdata"; + + String tessPath = system.getTessdataDir(); + String tessdataDir = java.lang.System.getenv("TESSDATA_PREFIX"); + + this.tessDataPath = + resolvePath( + defaultTessDataPath, + (tessPath != null && !tessPath.isEmpty()) ? tessPath : tessdataDir); + log.info("Using Tesseract data path: {}", this.tessDataPath); } private String resolvePath(String defaultPath, String customPath) { diff --git a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java index e8606b1f9e..fbd36c6d49 100644 --- a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java +++ b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java @@ -372,6 +372,8 @@ public class ApplicationProperties { private String weasyprint; private String unoconvert; private String calibre; + private String ocrmypdf; + private String soffice; } } @@ -454,10 +456,10 @@ public class ApplicationProperties { @Override public String toString() { return """ - Driver { - driverName='%s' - } - """ + Driver { + driverName='%s' + } + """ .formatted(driverName); } } diff --git a/app/common/src/main/java/stirling/software/common/service/CustomPDFDocumentFactory.java b/app/common/src/main/java/stirling/software/common/service/CustomPDFDocumentFactory.java index 8d7419eb71..4e2b4c789f 100644 --- a/app/common/src/main/java/stirling/software/common/service/CustomPDFDocumentFactory.java +++ b/app/common/src/main/java/stirling/software/common/service/CustomPDFDocumentFactory.java @@ -478,11 +478,6 @@ public class CustomPDFDocumentFactory { return file; } - /** Create a uniquely named temporary directory */ - private Path createTempDirectory(String prefix) throws IOException { - return Files.createTempDirectory(prefix + tempCounter.incrementAndGet() + "-"); - } - /** Create new document bytes based on an existing document */ public byte[] createNewBytesBasedOnOldDocument(byte[] oldDocument) throws IOException { try (PDDocument document = load(oldDocument)) { diff --git a/app/common/src/main/java/stirling/software/common/service/PostHogService.java b/app/common/src/main/java/stirling/software/common/service/PostHogService.java index 6c42e093ff..199890f184 100644 --- a/app/common/src/main/java/stirling/software/common/service/PostHogService.java +++ b/app/common/src/main/java/stirling/software/common/service/PostHogService.java @@ -25,6 +25,7 @@ import org.springframework.stereotype.Service; import com.posthog.java.PostHog; +import stirling.software.common.configuration.RuntimePathConfig; import stirling.software.common.model.ApplicationProperties; @Service @@ -33,6 +34,7 @@ public class PostHogService { private final String uniqueId; private final String appVersion; private final ApplicationProperties applicationProperties; + private final RuntimePathConfig runtimePathConfig; private final UserServiceInterface userService; private final Environment env; private boolean configDirMounted; @@ -43,12 +45,14 @@ public class PostHogService { @Qualifier("configDirMounted") boolean configDirMounted, @Qualifier("appVersion") String appVersion, ApplicationProperties applicationProperties, + RuntimePathConfig runtimePathConfig, @Autowired(required = false) UserServiceInterface userService, Environment env) { this.postHog = postHog; this.uniqueId = uuid; this.appVersion = appVersion; this.applicationProperties = applicationProperties; + this.runtimePathConfig = runtimePathConfig; this.userService = userService; this.env = env; this.configDirMounted = configDirMounted; @@ -313,10 +317,7 @@ public class PostHogService { properties, "system_customHTMLFiles", applicationProperties.getSystem().isCustomHTMLFiles()); - addIfNotEmpty( - properties, - "system_tessdataDir", - applicationProperties.getSystem().getTessdataDir()); + addIfNotEmpty(properties, "system_tessdataDir", runtimePathConfig.getTessDataPath()); addIfNotEmpty( properties, "system_enableAlphaFunctionality", diff --git a/app/common/src/main/java/stirling/software/common/util/FileToPdf.java b/app/common/src/main/java/stirling/software/common/util/FileToPdf.java index 30a478294b..6e0fec08ee 100644 --- a/app/common/src/main/java/stirling/software/common/util/FileToPdf.java +++ b/app/common/src/main/java/stirling/software/common/util/FileToPdf.java @@ -5,11 +5,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; -import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.List; import java.util.Locale; @@ -148,64 +145,6 @@ public class FileToPdf { } } - private static void deleteDirectory(Path dir) throws IOException { - Files.walkFileTree( - dir, - new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) - throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - }); - } - - private static Path unzipAndGetMainHtml(byte[] fileBytes) throws IOException { - Path tempDirectory = Files.createTempDirectory("unzipped_"); - try (ZipInputStream zipIn = - ZipSecurity.createHardenedInputStream(new ByteArrayInputStream(fileBytes))) { - ZipEntry entry = zipIn.getNextEntry(); - while (entry != null) { - Path filePath = tempDirectory.resolve(sanitizeZipFilename(entry.getName())); - if (entry.isDirectory()) { - Files.createDirectories(filePath); // Explicitly create the directory structure - } else { - Files.createDirectories( - filePath.getParent()); // Create parent directories if they don't exist - Files.copy(zipIn, filePath); - } - zipIn.closeEntry(); - entry = zipIn.getNextEntry(); - } - } - - // Search for the main HTML file. - try (Stream walk = Files.walk(tempDirectory)) { - List htmlFiles = walk.filter(file -> file.toString().endsWith(".html")).toList(); - - if (htmlFiles.isEmpty()) { - throw new IOException("No HTML files found in the unzipped directory."); - } - - // Prioritize 'index.html' if it exists, otherwise use the first .html file - for (Path htmlFile : htmlFiles) { - if ("index.html".equals(htmlFile.getFileName().toString())) { - return htmlFile; - } - } - - return htmlFiles.get(0); - } - } - static String sanitizeZipFilename(String entryName) { if (entryName == null || entryName.trim().isEmpty()) { return ""; diff --git a/app/common/src/main/java/stirling/software/common/util/GeneralUtils.java b/app/common/src/main/java/stirling/software/common/util/GeneralUtils.java index 874417990c..1379b74590 100644 --- a/app/common/src/main/java/stirling/software/common/util/GeneralUtils.java +++ b/app/common/src/main/java/stirling/software/common/util/GeneralUtils.java @@ -94,32 +94,6 @@ public class GeneralUtils { return tempFile; } - /* - * Gets the configured temporary directory, creating it if necessary. - * - * @return Path to the temporary directory - * @throws IOException if directory creation fails - */ - private Path getTempDirectory() throws IOException { - String customTempDir = System.getenv("STIRLING_TEMPFILES_DIRECTORY"); - if (customTempDir == null || customTempDir.isEmpty()) { - customTempDir = System.getProperty("stirling.tempfiles.directory"); - } - - Path tempDir; - if (customTempDir != null && !customTempDir.isEmpty()) { - tempDir = Path.of(customTempDir); - } else { - tempDir = Path.of(System.getProperty("java.io.tmpdir"), "stirling-pdf"); - } - - if (!Files.exists(tempDir)) { - Files.createDirectories(tempDir); - } - - return tempDir; - } - /* * Remove file extension * diff --git a/app/common/src/main/java/stirling/software/common/util/PDFToFile.java b/app/common/src/main/java/stirling/software/common/util/PDFToFile.java index 32f2cc874b..b00cdae868 100644 --- a/app/common/src/main/java/stirling/software/common/util/PDFToFile.java +++ b/app/common/src/main/java/stirling/software/common/util/PDFToFile.java @@ -27,15 +27,22 @@ import io.github.pixee.security.Filenames; import lombok.extern.slf4j.Slf4j; +import stirling.software.common.configuration.RuntimePathConfig; import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult; @Slf4j public class PDFToFile { private final TempFileManager tempFileManager; + private final RuntimePathConfig runtimePathConfig; public PDFToFile(TempFileManager tempFileManager) { + this(tempFileManager, null); + } + + public PDFToFile(TempFileManager tempFileManager, RuntimePathConfig runtimePathConfig) { this.tempFileManager = tempFileManager; + this.runtimePathConfig = runtimePathConfig; } public ResponseEntity processPdfToMarkdown(MultipartFile inputFile) @@ -241,31 +248,65 @@ public class PDFToFile { byte[] fileBytes; String fileName; + Path libreOfficeProfile = null; try (TempFile inputFileTemp = new TempFile(tempFileManager, ".pdf"); TempDirectory outputDirTemp = new TempDirectory(tempFileManager)) { Path tempInputFile = inputFileTemp.getPath(); Path tempOutputDir = outputDirTemp.getPath(); + Path unoOutputFile = + tempOutputDir.resolve( + pdfBaseName + "." + resolvePrimaryExtension(outputFormat)); // Save the uploaded file to a temporary location inputFile.transferTo(tempInputFile); // Run the LibreOffice command - List command = - new ArrayList<>( - Arrays.asList( - "soffice", - "--headless", - "--nologo", - "--infilter=" + libreOfficeFilter, - "--convert-to", - outputFormat, - "--outdir", - tempOutputDir.toString(), - tempInputFile.toString())); - ProcessExecutorResult returnCode = - ProcessExecutor.getInstance(ProcessExecutor.Processes.LIBRE_OFFICE) - .runCommandWithOutputHandling(command); + ProcessExecutorResult returnCode = null; + IOException unoconvertException = null; + + if (isUnoConvertEnabled()) { + try { + List unoCommand = + buildUnoConvertCommand( + tempInputFile, unoOutputFile, outputFormat, libreOfficeFilter); + returnCode = + ProcessExecutor.getInstance(ProcessExecutor.Processes.LIBRE_OFFICE) + .runCommandWithOutputHandling(unoCommand); + } catch (IOException e) { + unoconvertException = e; + log.warn( + "Unoconvert command failed ({}). Falling back to soffice command.", + e.getMessage()); + } + } + + if (returnCode == null) { + // Run the LibreOffice command as a fallback + libreOfficeProfile = Files.createTempDirectory("libreoffice_profile_"); + List command = new ArrayList<>(); + command.add(runtimePathConfig.getSOfficePath()); + command.add("-env:UserInstallation=" + libreOfficeProfile.toUri().toString()); + command.add("--headless"); + command.add("--nologo"); + command.add("--infilter=" + libreOfficeFilter); + command.add("--convert-to"); + command.add(outputFormat); + command.add("--outdir"); + command.add(tempOutputDir.toString()); + command.add(tempInputFile.toString()); + + try { + returnCode = + ProcessExecutor.getInstance(ProcessExecutor.Processes.LIBRE_OFFICE) + .runCommandWithOutputHandling(command); + } catch (IOException e) { + if (unoconvertException != null) { + e.addSuppressed(unoconvertException); + } + throw e; + } + } // Get output files List outputFiles = Arrays.asList(tempOutputDir.toFile().listFiles()); @@ -300,8 +341,42 @@ public class PDFToFile { fileBytes = byteArrayOutputStream.toByteArray(); } + } finally { + if (libreOfficeProfile != null) { + FileUtils.deleteQuietly(libreOfficeProfile.toFile()); + } } return WebResponseUtils.bytesToWebResponse( fileBytes, fileName, MediaType.APPLICATION_OCTET_STREAM); } + + private boolean isUnoConvertEnabled() { + return runtimePathConfig != null + && runtimePathConfig.getUnoConvertPath() != null + && !runtimePathConfig.getUnoConvertPath().isBlank(); + } + + private List buildUnoConvertCommand( + Path inputFile, Path outputFile, String outputFormat, String libreOfficeFilter) { + List command = new ArrayList<>(); + command.add(runtimePathConfig.getUnoConvertPath()); + command.add("--port"); + command.add("2003"); + command.add("--convert-to"); + command.add(outputFormat); + if (libreOfficeFilter != null && !libreOfficeFilter.isBlank()) { + command.add("--input-filter=" + libreOfficeFilter); + } + command.add(inputFile.toString()); + command.add(outputFile.toString()); + return command; + } + + private String resolvePrimaryExtension(String outputFormat) { + if (outputFormat == null) { + return ""; + } + int colonIndex = outputFormat.indexOf(':'); + return colonIndex > 0 ? outputFormat.substring(0, colonIndex) : outputFormat; + } } diff --git a/app/common/src/test/java/stirling/software/common/util/PDFToFileTest.java b/app/common/src/test/java/stirling/software/common/util/PDFToFileTest.java index 19c3d43220..528125eacf 100644 --- a/app/common/src/test/java/stirling/software/common/util/PDFToFileTest.java +++ b/app/common/src/test/java/stirling/software/common/util/PDFToFileTest.java @@ -32,6 +32,7 @@ import org.springframework.web.multipart.MultipartFile; import io.github.pixee.security.ZipSecurity; +import stirling.software.common.configuration.RuntimePathConfig; import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult; /** @@ -48,6 +49,7 @@ class PDFToFileTest { @Mock private ProcessExecutor mockProcessExecutor; @Mock private ProcessExecutorResult mockExecutorResult; @Mock private TempFileManager mockTempFileManager; + @Mock private RuntimePathConfig mockRuntimePathConfig; @BeforeEach void setUp() throws IOException { @@ -61,7 +63,9 @@ class PDFToFileTest { .when(mockTempFileManager.createTempDirectory()) .thenAnswer(invocation -> Files.createTempDirectory("test")); - pdfToFile = new PDFToFile(mockTempFileManager); + lenient().when(mockRuntimePathConfig.getSOfficePath()).thenReturn("/usr/bin/soffice"); + + pdfToFile = new PDFToFile(mockTempFileManager, mockRuntimePathConfig); } @Test @@ -363,7 +367,8 @@ class PDFToFileTest { when(mockProcessExecutor.runCommandWithOutputHandling( argThat( args -> - args.contains("--convert-to") + args != null + && args.contains("--convert-to") && args.contains("docx")))) .thenAnswer( invocation -> { @@ -424,7 +429,11 @@ class PDFToFileTest { .thenReturn(mockProcessExecutor); when(mockProcessExecutor.runCommandWithOutputHandling( - argThat(args -> args.contains("--convert-to") && args.contains("odp")))) + argThat( + args -> + args != null + && args.contains("--convert-to") + && args.contains("odp")))) .thenAnswer( invocation -> { // When command is executed, find the output directory argument @@ -513,7 +522,8 @@ class PDFToFileTest { when(mockProcessExecutor.runCommandWithOutputHandling( argThat( args -> - args.contains("--convert-to") + args != null + && args.contains("--convert-to") && args.contains("txt:Text")))) .thenAnswer( invocation -> { @@ -611,4 +621,110 @@ class PDFToFileTest { .contains("output.docx")); } } + + @Test + void testProcessPdfToOfficeFormat_UsesUnoconvertWhenConfigured() + throws IOException, InterruptedException { + when(mockRuntimePathConfig.getUnoConvertPath()).thenReturn("/custom/unoconvert"); + PDFToFile pdfToFileWithUno = new PDFToFile(mockTempFileManager, mockRuntimePathConfig); + + try (MockedStatic mockedStaticProcessExecutor = + mockStatic(ProcessExecutor.class)) { + MultipartFile pdfFile = + new MockMultipartFile( + "file", + "document.pdf", + MediaType.APPLICATION_PDF_VALUE, + "Fake PDF content".getBytes()); + + mockedStaticProcessExecutor + .when(() -> ProcessExecutor.getInstance(ProcessExecutor.Processes.LIBRE_OFFICE)) + .thenReturn(mockProcessExecutor); + + when(mockProcessExecutor.runCommandWithOutputHandling( + argThat(args -> args != null && args.contains("/custom/unoconvert")))) + .thenAnswer( + invocation -> { + List args = invocation.getArgument(0); + String outputPath = args.get(args.size() - 1); + Files.write(Path.of(outputPath), "Fake DOCX content".getBytes()); + return mockExecutorResult; + }); + + ResponseEntity response = + pdfToFileWithUno.processPdfToOfficeFormat(pdfFile, "docx", "writer_pdf_import"); + + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertNotNull(response.getBody()); + assertTrue(response.getBody().length > 0); + assertTrue( + response.getHeaders() + .getContentDisposition() + .toString() + .contains("document.docx")); + } + } + + @Test + void testProcessPdfToOfficeFormat_FallsBackWhenUnoconvertFails() + throws IOException, InterruptedException { + when(mockRuntimePathConfig.getUnoConvertPath()).thenReturn("/custom/unoconvert"); + PDFToFile pdfToFileWithUno = new PDFToFile(mockTempFileManager, mockRuntimePathConfig); + + try (MockedStatic mockedStaticProcessExecutor = + mockStatic(ProcessExecutor.class)) { + MultipartFile pdfFile = + new MockMultipartFile( + "file", + "document.pdf", + MediaType.APPLICATION_PDF_VALUE, + "Fake PDF content".getBytes()); + + mockedStaticProcessExecutor + .when(() -> ProcessExecutor.getInstance(ProcessExecutor.Processes.LIBRE_OFFICE)) + .thenReturn(mockProcessExecutor); + + when(mockProcessExecutor.runCommandWithOutputHandling( + argThat(args -> args != null && args.contains("/custom/unoconvert")))) + .thenThrow(new IOException("Conversion failed")); + + when(mockProcessExecutor.runCommandWithOutputHandling( + argThat( + args -> + args != null + && args.stream() + .anyMatch( + arg -> + arg.contains( + "soffice"))))) + .thenAnswer( + invocation -> { + List args = invocation.getArgument(0); + String outDir = null; + for (int i = 0; i < args.size(); i++) { + if ("--outdir".equals(args.get(i)) && i + 1 < args.size()) { + outDir = args.get(i + 1); + break; + } + } + assertNotNull(outDir); + Files.write( + Path.of(outDir, "document.docx"), + "Fallback DOCX content".getBytes()); + return mockExecutorResult; + }); + + ResponseEntity response = + pdfToFileWithUno.processPdfToOfficeFormat(pdfFile, "docx", "writer_pdf_import"); + + assertEquals(HttpStatus.OK, response.getStatusCode()); + assertNotNull(response.getBody()); + assertTrue(response.getBody().length > 0); + assertTrue( + response.getHeaders() + .getContentDisposition() + .toString() + .contains("document.docx")); + } + } } diff --git a/app/core/src/main/java/stirling/software/SPDF/config/ExternalAppDepConfig.java b/app/core/src/main/java/stirling/software/SPDF/config/ExternalAppDepConfig.java index 22ad2adf47..7dd806aa7f 100644 --- a/app/core/src/main/java/stirling/software/SPDF/config/ExternalAppDepConfig.java +++ b/app/core/src/main/java/stirling/software/SPDF/config/ExternalAppDepConfig.java @@ -41,6 +41,8 @@ public class ExternalAppDepConfig { private final String weasyprintPath; private final String unoconvPath; private final String calibrePath; + private final String ocrMyPdfPath; + private final String sOfficePath; /** * Map of command(binary) -> affected groups (e.g. "gs" -> ["Ghostscript"]). Immutable to avoid @@ -58,11 +60,13 @@ public class ExternalAppDepConfig { this.weasyprintPath = runtimePathConfig.getWeasyPrintPath(); this.unoconvPath = runtimePathConfig.getUnoConvertPath(); this.calibrePath = runtimePathConfig.getCalibrePath(); + this.ocrMyPdfPath = runtimePathConfig.getOcrMyPdfPath(); + this.sOfficePath = runtimePathConfig.getSOfficePath(); Map> tmp = new HashMap<>(); tmp.put("gs", List.of("Ghostscript")); - tmp.put("ocrmypdf", List.of("OCRmyPDF")); - tmp.put("soffice", List.of("LibreOffice")); + tmp.put(ocrMyPdfPath, List.of("OCRmyPDF")); + tmp.put(sOfficePath, List.of("LibreOffice")); tmp.put(weasyprintPath, List.of("Weasyprint")); tmp.put("pdftohtml", List.of("Pdftohtml")); tmp.put(unoconvPath, List.of("Unoconvert")); diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java index a4ce593808..167293ef13 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertOfficeController.java @@ -93,6 +93,7 @@ public class ConvertOfficeController { Files.copy(inputFile.getInputStream(), inputPath, StandardCopyOption.REPLACE_EXISTING); } + Path libreOfficeProfile = null; try { ProcessExecutorResult result; // Run Unoconvert command @@ -112,8 +113,10 @@ public class ConvertOfficeController { .runCommandWithOutputHandling(command); } // Run soffice command else { + libreOfficeProfile = Files.createTempDirectory("libreoffice_profile_"); List command = new ArrayList<>(); - command.add("soffice"); + command.add(runtimePathConfig.getSOfficePath()); + command.add("-env:UserInstallation=" + libreOfficeProfile.toUri().toString()); command.add("--headless"); command.add("--nologo"); command.add("--convert-to"); @@ -169,6 +172,9 @@ public class ConvertOfficeController { } catch (IOException e) { log.warn("Failed to delete temp input file: {}", inputPath, e); } + if (libreOfficeProfile != null) { + FileUtils.deleteQuietly(libreOfficeProfile.toFile()); + } } } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToHtml.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToHtml.java index 76414ca576..e647468720 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToHtml.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToHtml.java @@ -13,6 +13,7 @@ import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; +import stirling.software.common.configuration.RuntimePathConfig; import stirling.software.common.model.api.PDFFile; import stirling.software.common.util.PDFToFile; import stirling.software.common.util.TempFileManager; @@ -24,6 +25,7 @@ import stirling.software.common.util.TempFileManager; public class ConvertPDFToHtml { private final TempFileManager tempFileManager; + private final RuntimePathConfig runtimePathConfig; @PostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/pdf/html") @Operation( @@ -32,7 +34,7 @@ public class ConvertPDFToHtml { "This endpoint converts a PDF file to HTML format. Input:PDF Output:HTML Type:SISO") public ResponseEntity processPdfToHTML(@ModelAttribute PDFFile file) throws Exception { MultipartFile inputFile = file.getFileInput(); - PDFToFile pdfToFile = new PDFToFile(tempFileManager); + PDFToFile pdfToFile = new PDFToFile(tempFileManager, runtimePathConfig); return pdfToFile.processPdfToHtml(inputFile); } } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToOffice.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToOffice.java index d9538de585..753b8a0758 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToOffice.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToOffice.java @@ -20,6 +20,7 @@ import lombok.RequiredArgsConstructor; import stirling.software.SPDF.model.api.converters.PdfToPresentationRequest; import stirling.software.SPDF.model.api.converters.PdfToTextOrRTFRequest; import stirling.software.SPDF.model.api.converters.PdfToWordRequest; +import stirling.software.common.configuration.RuntimePathConfig; import stirling.software.common.model.api.PDFFile; import stirling.software.common.service.CustomPDFDocumentFactory; import stirling.software.common.util.GeneralUtils; @@ -35,6 +36,7 @@ public class ConvertPDFToOffice { private final CustomPDFDocumentFactory pdfDocumentFactory; private final TempFileManager tempFileManager; + private final RuntimePathConfig runtimePathConfig; @PostMapping(consumes = MediaType.MULTIPART_FORM_DATA_VALUE, value = "/pdf/presentation") @Operation( @@ -47,7 +49,7 @@ public class ConvertPDFToOffice { throws IOException, InterruptedException { MultipartFile inputFile = request.getFileInput(); String outputFormat = request.getOutputFormat(); - PDFToFile pdfToFile = new PDFToFile(tempFileManager); + PDFToFile pdfToFile = new PDFToFile(tempFileManager, runtimePathConfig); return pdfToFile.processPdfToOfficeFormat(inputFile, outputFormat, "impress_pdf_import"); } @@ -72,7 +74,7 @@ public class ConvertPDFToOffice { MediaType.TEXT_PLAIN); } } else { - PDFToFile pdfToFile = new PDFToFile(tempFileManager); + PDFToFile pdfToFile = new PDFToFile(tempFileManager, runtimePathConfig); return pdfToFile.processPdfToOfficeFormat(inputFile, outputFormat, "writer_pdf_import"); } } @@ -87,7 +89,7 @@ public class ConvertPDFToOffice { throws IOException, InterruptedException { MultipartFile inputFile = request.getFileInput(); String outputFormat = request.getOutputFormat(); - PDFToFile pdfToFile = new PDFToFile(tempFileManager); + PDFToFile pdfToFile = new PDFToFile(tempFileManager, runtimePathConfig); return pdfToFile.processPdfToOfficeFormat(inputFile, outputFormat, "writer_pdf_import"); } @@ -100,7 +102,7 @@ public class ConvertPDFToOffice { public ResponseEntity processPdfToXML(@ModelAttribute PDFFile file) throws Exception { MultipartFile inputFile = file.getFileInput(); - PDFToFile pdfToFile = new PDFToFile(tempFileManager); + PDFToFile pdfToFile = new PDFToFile(tempFileManager, runtimePathConfig); return pdfToFile.processPdfToOfficeFormat(inputFile, "xml", "writer_pdf_import"); } } diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFA.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFA.java index bdc57a9844..5c388b5046 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFA.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/converters/ConvertPDFToPDFA.java @@ -71,9 +71,11 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import stirling.software.SPDF.model.api.converters.PdfToPdfARequest; +import stirling.software.common.configuration.RuntimePathConfig; import stirling.software.common.util.ExceptionUtils; import stirling.software.common.util.ProcessExecutor; import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult; @@ -83,8 +85,11 @@ import stirling.software.common.util.WebResponseUtils; @RequestMapping("/api/v1/convert") @Slf4j @Tag(name = "Convert", description = "Convert APIs") +@RequiredArgsConstructor public class ConvertPDFToPDFA { + private final RuntimePathConfig runtimePathConfig; + private static final String ICC_RESOURCE_PATH = "/icc/sRGB2014.icc"; private static final int PDFA_COMPATIBILITY_POLICY = 1; @@ -1043,26 +1048,33 @@ public class ConvertPDFToPDFA { ? "pdf:writer_pdf_Export:{\"SelectPdfVersion\":{\"type\":\"long\",\"value\":\"2\"}}" : "pdf:writer_pdf_Export:{\"SelectPdfVersion\":{\"type\":\"long\",\"value\":\"1\"}}"; - // Prepare LibreOffice command - List command = - new ArrayList<>( - Arrays.asList( - "soffice", - "--headless", - "--nologo", - "--convert-to", - pdfFilter, - "--outdir", - tempOutputDir.toString(), - tempInputFile.toString())); + Path libreOfficeProfile = Files.createTempDirectory("libreoffice_profile_"); + try { + // Prepare LibreOffice command + List command = + new ArrayList<>( + Arrays.asList( + runtimePathConfig.getSOfficePath(), + "-env:UserInstallation=" + + libreOfficeProfile.toUri().toString(), + "--headless", + "--nologo", + "--convert-to", + pdfFilter, + "--outdir", + tempOutputDir.toString(), + tempInputFile.toString())); - ProcessExecutorResult returnCode = - ProcessExecutor.getInstance(ProcessExecutor.Processes.LIBRE_OFFICE) - .runCommandWithOutputHandling(command); + ProcessExecutorResult returnCode = + ProcessExecutor.getInstance(ProcessExecutor.Processes.LIBRE_OFFICE) + .runCommandWithOutputHandling(command); - if (returnCode.getRc() != 0) { - log.error("PDF/A conversion failed with return code: {}", returnCode.getRc()); - throw ExceptionUtils.createPdfaConversionFailedException(); + if (returnCode.getRc() != 0) { + log.error("PDF/A conversion failed with return code: {}", returnCode.getRc()); + throw ExceptionUtils.createPdfaConversionFailedException(); + } + } finally { + FileUtils.deleteQuietly(libreOfficeProfile.toFile()); } // Get the output file diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java index 2916246297..e9f558e7de 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/api/misc/OCRController.java @@ -37,10 +37,17 @@ import lombok.extern.slf4j.Slf4j; import stirling.software.SPDF.config.EndpointConfiguration; import stirling.software.SPDF.model.api.misc.ProcessPdfWithOcrRequest; +import stirling.software.common.configuration.RuntimePathConfig; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.service.CustomPDFDocumentFactory; -import stirling.software.common.util.*; +import stirling.software.common.util.ExceptionUtils; +import stirling.software.common.util.GeneralUtils; +import stirling.software.common.util.ProcessExecutor; import stirling.software.common.util.ProcessExecutor.ProcessExecutorResult; +import stirling.software.common.util.TempDirectory; +import stirling.software.common.util.TempFile; +import stirling.software.common.util.TempFileManager; +import stirling.software.common.util.WebResponseUtils; @RestController @RequestMapping("/api/v1/misc") @@ -53,6 +60,7 @@ public class OCRController { private final CustomPDFDocumentFactory pdfDocumentFactory; private final TempFileManager tempFileManager; private final EndpointConfiguration endpointConfiguration; + private final RuntimePathConfig runtimePathConfig; private boolean isOcrMyPdfEnabled() { return endpointConfiguration.isGroupEnabled("OCRmyPDF"); @@ -64,7 +72,7 @@ public class OCRController { /** Gets the list of available Tesseract languages from the tessdata directory */ public List getAvailableTesseractLanguages() { - String tessdataDir = applicationProperties.getSystem().getTessdataDir(); + String tessdataDir = runtimePathConfig.getTessDataPath(); File[] files = new File(tessdataDir).listFiles(); if (files == null) { return Collections.emptyList(); @@ -80,9 +88,10 @@ public class OCRController { @Operation( summary = "Process a PDF file with OCR", description = - "This endpoint processes a PDF file using OCR (Optical Character Recognition). " - + "Users can specify languages, sidecar, deskew, clean, cleanFinal, ocrType, ocrRenderType, and removeImagesAfter options. " - + "Uses OCRmyPDF if available, falls back to Tesseract. Input:PDF Output:PDF Type:SI-Conditional") + "This endpoint processes a PDF file using OCR (Optical Character Recognition). Users can" + + " specify languages, sidecar, deskew, clean, cleanFinal, ocrType, ocrRenderType," + + " and removeImagesAfter options. Uses OCRmyPDF if available, falls back to" + + " Tesseract. Input:PDF Output:PDF Type:SI-Conditional") public ResponseEntity processPdfWithOCR( @ModelAttribute ProcessPdfWithOcrRequest request) throws IOException, InterruptedException { @@ -217,7 +226,7 @@ public class OCRController { List command = new ArrayList<>( Arrays.asList( - "ocrmypdf", + runtimePathConfig.getOcrMyPdfPath(), "--verbose", "2", "--output-type", diff --git a/app/core/src/main/java/stirling/software/SPDF/controller/web/OtherWebController.java b/app/core/src/main/java/stirling/software/SPDF/controller/web/OtherWebController.java index 09dd46cec7..9549ac4dba 100644 --- a/app/core/src/main/java/stirling/software/SPDF/controller/web/OtherWebController.java +++ b/app/core/src/main/java/stirling/software/SPDF/controller/web/OtherWebController.java @@ -14,16 +14,20 @@ import io.swagger.v3.oas.annotations.Hidden; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import stirling.software.common.configuration.RuntimePathConfig; import stirling.software.common.model.ApplicationProperties; import stirling.software.common.util.CheckProgramInstall; @Controller @Tag(name = "Misc", description = "Miscellaneous APIs") @RequiredArgsConstructor +@Slf4j public class OtherWebController { private final ApplicationProperties applicationProperties; + private final RuntimePathConfig runtimePathConfig; @GetMapping("/compress-pdf") @Hidden @@ -120,7 +124,7 @@ public class OtherWebController { } public List getAvailableTesseractLanguages() { - String tessdataDir = applicationProperties.getSystem().getTessdataDir(); + String tessdataDir = runtimePathConfig.getTessDataPath(); File[] files = new File(tessdataDir).listFiles(); if (files == null) { return Collections.emptyList(); diff --git a/app/core/src/main/resources/settings.yml.template b/app/core/src/main/resources/settings.yml.template index 734f3f793e..64ddaa857a 100644 --- a/app/core/src/main/resources/settings.yml.template +++ b/app/core/src/main/resources/settings.yml.template @@ -115,7 +115,7 @@ system: showUpdate: false # see when a new update is available showUpdateOnlyAdmin: false # only admins can see when a new update is available, depending on showUpdate it must be set to 'true' customHTMLFiles: false # enable to have files placed in /customFiles/templates override the existing template HTML files - tessdataDir: /usr/share/tessdata # path to the directory containing the Tessdata files. This setting is relevant for Windows systems. For Windows users, this path should be adjusted to point to the appropriate directory where the Tessdata files are stored. + tessdataDir: "" # path to the directory containing the Tessdata files. This setting is relevant for Windows systems. For Windows users, this path should be adjusted to point to the appropriate directory where the Tessdata files are stored. enableAnalytics: null # Master toggle for analytics: set to 'true' to enable all analytics, 'false' to disable all analytics, or leave as 'null' to prompt admin on first launch enablePosthog: null # Enable PostHog analytics (open-source product analytics): set to 'true' to enable, 'false' to disable, or 'null' to enable by default when analytics is enabled enableScarf: null # Enable Scarf pixel: set to 'true' to enable, 'false' to disable, or 'null' to enable by default when analytics is enabled @@ -150,6 +150,8 @@ system: weasyprint: '' # Defaults to /opt/venv/bin/weasyprint unoconvert: '' # Defaults to /opt/venv/bin/unoconvert calibre: '' # Defaults to /usr/bin/ebook-convert + ocrmypdf: '' # Defaults to /usr/bin/ocrmypdf + soffice: '' # Defaults to /usr/bin/soffice fileUploadLimit: '' # Defaults to "". No limit when string is empty. Set a number, between 0 and 999, followed by one of the following strings to set a limit. "KB", "MB", "GB". tempFileManagement: baseTmpDir: '' # Defaults to java.io.tmpdir/stirling-pdf diff --git a/app/core/src/main/resources/static/js/DecryptFiles.js b/app/core/src/main/resources/static/js/DecryptFiles.js index 057ca98374..b19fb6e12e 100644 --- a/app/core/src/main/resources/static/js/DecryptFiles.js +++ b/app/core/src/main/resources/static/js/DecryptFiles.js @@ -1,3 +1,9 @@ +const PDFJS_DEFAULT_OPTIONS = { + cMapUrl: pdfjsPath + 'cmaps/', + cMapPacked: true, + standardFontDataUrl: pdfjsPath + 'standard_fonts/', +}; + function formatProblemDetailsJson(input) { try { const obj = typeof input === 'string' ? JSON.parse(input) : input; @@ -238,7 +244,7 @@ export class DecryptFile { return {isEncrypted: false, requiresPassword: false}; } - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; const arrayBuffer = await file.arrayBuffer(); const arrayBufferForPdfLib = arrayBuffer.slice(0); @@ -246,12 +252,14 @@ export class DecryptFile { if(this.decryptWorker == null){ loadingTask = pdfjsLib.getDocument({ + ...PDFJS_DEFAULT_OPTIONS, data: arrayBuffer, }); this.decryptWorker = loadingTask._worker }else { loadingTask = pdfjsLib.getDocument({ + ...PDFJS_DEFAULT_OPTIONS, data: arrayBuffer, worker: this.decryptWorker }); diff --git a/app/core/src/main/resources/static/js/compare/pdfWorker.js b/app/core/src/main/resources/static/js/compare/pdfWorker.js index 78952bd759..86fb66383d 100644 --- a/app/core/src/main/resources/static/js/compare/pdfWorker.js +++ b/app/core/src/main/resources/static/js/compare/pdfWorker.js @@ -1,18 +1,39 @@ importScripts('./diff.js'); +let complexMessage = 'One or both of the provided documents are large files, accuracy of comparison may be reduced'; +let largeFilesMessage = 'One or Both of the provided documents are too large to process'; + +// Early: Listener for SET messages (before onmessage) +self.addEventListener('message', (event) => { + if (event.data.type === 'SET_COMPLEX_MESSAGE') { + complexMessage = event.data.message; + } else if (event.data.type === 'SET_TOO_LARGE_MESSAGE') { + largeFilesMessage = event.data.message; + } +}); + self.onmessage = async function (e) { - const { text1, text2, color1, color2 } = e.data; - console.log('Received text for comparison:', { text1, text2 }); + const data = e.data; + if (data.type !== 'COMPARE') { + console.log('Worker ignored non-COMPARE message'); + return; + } + + const { text1, text2, color1, color2 } = data; + console.log('Received text for comparison:', { lengths: { text1: text1.length, text2: text2.length } }); // Safe Log const startTime = performance.now(); - if (text1.trim() === "" || text2.trim() === "") { + // Safe Trim + if (!text1 || !text2 || text1.trim() === "" || text2.trim() === "") { self.postMessage({ status: 'error', message: 'One or both of the texts are empty.' }); return; } - const words1 = text1.split(' '); - const words2 = text2.split(' '); + // Robust Word-Split (handles spaces/punctuation better) + const words1 = text1.trim().split(/\s+/).filter(w => w.length > 0); + const words2 = text2.trim().split(/\s+/).filter(w => w.length > 0); + const MAX_WORD_COUNT = 150000; const COMPLEX_WORD_COUNT = 50000; const BATCH_SIZE = 5000; // Define a suitable batch size for processing @@ -21,44 +42,28 @@ self.onmessage = async function (e) { const isComplex = words1.length > COMPLEX_WORD_COUNT || words2.length > COMPLEX_WORD_COUNT; const isTooLarge = words1.length > MAX_WORD_COUNT || words2.length > MAX_WORD_COUNT; - let complexMessage = 'One or both of the provided documents are large files, accuracy of comparison may be reduced'; - let tooLargeMessage = 'One or Both of the provided documents are too large to process'; - - // Listen for messages from the main thread - self.addEventListener('message', (event) => { - if (event.data.type === 'SET_TOO_LARGE_MESSAGE') { - tooLargeMessage = event.data.message; - } - if (event.data.type === 'SET_COMPLEX_MESSAGE') { - complexMessage = event.data.message; - } - }); - if (isTooLarge) { - self.postMessage({ - status: 'warning', - message: tooLargeMessage, - }); + self.postMessage({ status: 'error', message: largeFilesMessage }); return; - } else { - - if (isComplex) { - self.postMessage({ - status: 'warning', - message: complexMessage, - }); - } - // Perform diff operation depending on document size - const differences = isComplex - ? await staggeredBatchDiff(words1, words2, color1, color2, BATCH_SIZE, OVERLAP_SIZE) - : diff(words1, words2, color1, color2); - - console.log(`Diff operation took ${performance.now() - startTime} milliseconds`); - self.postMessage({ status: 'success', differences }); } + + if (isComplex) { + self.postMessage({ status: 'warning', message: complexMessage }); + } + + // Diff based on size + let differences; + if (isComplex) { + differences = await staggeredBatchDiff(words1, words2, color1 || '#ff0000', color2 || '#008000', BATCH_SIZE, OVERLAP_SIZE); + } else { + differences = diff(words1, words2, color1 || '#ff0000', color2 || '#008000'); + } + + console.log(`Diff took ${performance.now() - startTime} ms for ${words1.length + words2.length} words`); + self.postMessage({ status: 'success', differences }); }; -//Splits text into smaller batches to run through diff checking algorithms. overlaps the batches to help ensure +// Splits text into smaller batches to run through diff checking algorithms. overlaps the batches to help ensure async function staggeredBatchDiff(words1, words2, color1, color2, batchSize, overlapSize) { const differences = []; const totalWords1 = words1.length; @@ -67,10 +72,9 @@ async function staggeredBatchDiff(words1, words2, color1, color2, batchSize, ove let previousEnd1 = 0; // Track where the last batch ended in words1 let previousEnd2 = 0; // Track where the last batch ended in words2 - // Function to determine if differences are large, differences that are too large indicate potential error in batching - const isLargeDifference = (differences) => { - return differences.length > 50; - }; + // Track processed indices to dedupe overlaps + const processed1 = new Set(); + const processed2 = new Set(); while (previousEnd1 < totalWords1 || previousEnd2 < totalWords2) { // Define the next chunk boundaries @@ -80,66 +84,130 @@ async function staggeredBatchDiff(words1, words2, color1, color2, batchSize, ove const start2 = previousEnd2; const end2 = Math.min(start2 + batchSize, totalWords2); - //If difference is too high decrease batch size for more granular check - const dynamicBatchSize = isLargeDifference(differences) ? batchSize / 2 : batchSize; + // Adaptive: If many diffs, smaller batch (max 3x downscale) + const recentDiffs = differences.slice(-100).filter(([c]) => c !== 'black').length; + // If difference is too high decrease batch size for more granular check + const dynamicBatchSize = Math.max(batchSize / Math.min(8, 1 + recentDiffs / 50), batchSize / 8); - // Adjust the size of the current chunk using dynamic batch size - const batchWords1 = words1.slice(start1, end1 + dynamicBatchSize); - const batchWords2 = words2.slice(start2, end2 + dynamicBatchSize); + const extendedEnd1 = Math.min(end1 + dynamicBatchSize, totalWords1); + const extendedEnd2 = Math.min(end2 + dynamicBatchSize, totalWords2); + + const batchWords1 = words1.slice(start1, extendedEnd1); + const batchWords2 = words2.slice(start2, extendedEnd2); // Include overlap from the previous chunk - const overlapWords1 = previousEnd1 > 0 ? words1.slice(Math.max(0, previousEnd1 - overlapSize), previousEnd1) : []; - const overlapWords2 = previousEnd2 > 0 ? words2.slice(Math.max(0, previousEnd2 - overlapSize), previousEnd2) : []; + const overlapStart1 = Math.max(0, previousEnd1 - overlapSize); + const overlapStart2 = Math.max(0, previousEnd2 - overlapSize); + const overlapWords1 = previousEnd1 > 0 ? words1.slice(overlapStart1, previousEnd1) : []; + const overlapWords2 = previousEnd2 > 0 ? words2.slice(overlapStart2, previousEnd2) : []; + // Combine overlaps and current batches for comparison - const combinedWords1 = overlapWords1.concat(batchWords1); - const combinedWords2 = overlapWords2.concat(batchWords2); + const combinedWords1 = [...overlapWords1, ...batchWords1]; + const combinedWords2 = [...overlapWords2, ...batchWords2]; // Perform the diff on the combined words const batchDifferences = diff(combinedWords1, combinedWords2, color1, color2); - differences.push(...batchDifferences); - // Update the previous end indices based on the results of this batch + const combinedIndices1 = []; + for (let i = overlapStart1; i < previousEnd1; i++) { + combinedIndices1.push(i); + } + for (let i = start1; i < extendedEnd1; i++) { + combinedIndices1.push(i); + } + + const combinedIndices2 = []; + for (let i = overlapStart2; i < previousEnd2; i++) { + combinedIndices2.push(i); + } + for (let i = start2; i < extendedEnd2; i++) { + combinedIndices2.push(i); + } + + let pointer1 = 0; + let pointer2 = 0; + + const filteredBatch = []; + batchDifferences.forEach(([color, word]) => { + if (color === color1) { + const globalIndex1 = combinedIndices1[pointer1]; + if (globalIndex1 === undefined || !processed1.has(globalIndex1)) { + filteredBatch.push([color, word]); + } + if (globalIndex1 !== undefined) { + processed1.add(globalIndex1); + } + pointer1++; + } else if (color === color2) { + const globalIndex2 = combinedIndices2[pointer2]; + if (globalIndex2 === undefined || !processed2.has(globalIndex2)) { + filteredBatch.push([color, word]); + } + if (globalIndex2 !== undefined) { + processed2.add(globalIndex2); + } + pointer2++; + } else { + const globalIndex1 = combinedIndices1[pointer1]; + const globalIndex2 = combinedIndices2[pointer2]; + const alreadyProcessed = (globalIndex1 !== undefined && processed1.has(globalIndex1)) && (globalIndex2 !== undefined && processed2.has(globalIndex2)); + if (!alreadyProcessed) { + filteredBatch.push([color, word]); + } + if (globalIndex1 !== undefined) { + processed1.add(globalIndex1); + } + if (globalIndex2 !== undefined) { + processed2.add(globalIndex2); + } + pointer1++; + pointer2++; + } + }); + + differences.push(...filteredBatch); + + // Mark as processed + for (let k = start1; k < end1; k++) processed1.add(k); + for (let k = start2; k < end2; k++) processed2.add(k); + previousEnd1 = end1; previousEnd2 = end2; + + // Yield for async (avoids blocking) + await new Promise(resolve => setTimeout(resolve, 0)); } return differences; } - // Standard diff function for small text comparisons function diff(words1, words2, color1, color2) { - console.log(`Starting diff between ${words1.length} words and ${words2.length} words`); - const matrix = Array.from({ length: words1.length + 1 }, () => Array(words2.length + 1).fill(0)); + console.log(`Diff: ${words1.length} vs ${words2.length} words`); + const oldStr = words1.join(' '); // As string for diff.js + const newStr = words2.join(' '); + // Static method: No 'new' needed, avoids constructor error + const changes = Diff.diffWords(oldStr, newStr, { ignoreWhitespace: true }); - for (let i = 1; i <= words1.length; i++) { - for (let j = 1; j <= words2.length; j++) { - matrix[i][j] = words1[i - 1] === words2[j - 1] - ? matrix[i - 1][j - 1] + 1 - : Math.max(matrix[i][j - 1], matrix[i - 1][j]); - } - } - return backtrack(matrix, words1, words2, color1, color2); -} - -// Backtrack function to find differences -function backtrack(matrix, words1, words2, color1, color2) { - let i = words1.length, j = words2.length; + // Map changes to [color, word] format (change.value and added/removed) const differences = []; + changes.forEach(change => { + const value = change.value; + const op = change.added ? 1 : change.removed ? -1 : 0; - while (i > 0 || j > 0) { - if (i > 0 && j > 0 && words1[i - 1] === words2[j - 1]) { - differences.unshift(['black', words1[i - 1]]); - i--; j--; - } else if (j > 0 && (i === 0 || matrix[i][j] === matrix[i][j - 1])) { - differences.unshift([color2, words2[j - 1]]); - j--; - } else { - differences.unshift([color1, words1[i - 1]]); - i--; - } - } + // Split value into words and process + const words = value.split(/\s+/).filter(w => w.length > 0); + words.forEach(word => { + if (op === 0) { // Equal + differences.push(['black', word]); + } else if (op === 1) { // Insert + differences.push([color2, word]); + } else if (op === -1) { // Delete + differences.push([color1, word]); + } + }); + }); return differences; } diff --git a/app/core/src/main/resources/static/js/downloader.js b/app/core/src/main/resources/static/js/downloader.js index 52113b731f..2afdb2f75c 100644 --- a/app/core/src/main/resources/static/js/downloader.js +++ b/app/core/src/main/resources/static/js/downloader.js @@ -2,6 +2,12 @@ if (window.isDownloadScriptInitialized) return; // Prevent re-execution window.isDownloadScriptInitialized = true; + const PDFJS_DEFAULT_OPTIONS = { + cMapUrl: pdfjsPath + 'cmaps/', + cMapPacked: true, + standardFontDataUrl: pdfjsPath + 'standard_fonts/', + }; + // Global PDF processing count tracking for survey system window.incrementPdfProcessingCount = function() { let pdfProcessingCount = parseInt(localStorage.getItem('pdfProcessingCount') || '0'); @@ -234,8 +240,13 @@ async function getPDFPageCount(file) { try { const arrayBuffer = await file.arrayBuffer(); - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; - const pdf = await pdfjsLib.getDocument({data: arrayBuffer}).promise; + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; + const pdf = await pdfjsLib + .getDocument({ + ...PDFJS_DEFAULT_OPTIONS, + data: arrayBuffer, + }) + .promise; return pdf.numPages; } catch (error) { console.error('Error getting PDF page count:', error); @@ -245,7 +256,7 @@ async function checkAndDecryptFiles(url, files) { const decryptedFiles = []; - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; // Extract the base URL const baseUrl = new URL(url); @@ -271,7 +282,10 @@ } try { const arrayBuffer = await file.arrayBuffer(); - const loadingTask = pdfjsLib.getDocument({data: arrayBuffer}); + const loadingTask = pdfjsLib.getDocument({ + ...PDFJS_DEFAULT_OPTIONS, + data: arrayBuffer, + }); console.log(`Attempting to load PDF: ${file.name}`); const pdf = await loadingTask.promise; diff --git a/app/core/src/main/resources/static/js/homecard.js b/app/core/src/main/resources/static/js/homecard.js index 2e321b66dc..22c0caaf58 100644 --- a/app/core/src/main/resources/static/js/homecard.js +++ b/app/core/src/main/resources/static/js/homecard.js @@ -220,7 +220,7 @@ document.addEventListener('DOMContentLoaded', async function () { }); } try { - const response = await fetch('/files/popularity.txt'); + const response = await fetch('./files/popularity.txt'); if (!response.ok) { const errorText = await response.text().catch(() => ''); const errorMsg = errorText || response.statusText || 'Request failed'; diff --git a/app/core/src/main/resources/static/js/merge.js b/app/core/src/main/resources/static/js/merge.js index 82a0a0d88e..af5037c27b 100644 --- a/app/core/src/main/resources/static/js/merge.js +++ b/app/core/src/main/resources/static/js/merge.js @@ -1,3 +1,9 @@ +const PDFJS_DEFAULT_OPTIONS = { + cMapUrl: pdfjsPath + 'cmaps/', + cMapPacked: true, + standardFontDataUrl: pdfjsPath + 'standard_fonts/', +}; + let currentSort = { field: null, descending: false, @@ -73,7 +79,13 @@ async function displayFiles(files) { async function getPDFPageCount(file) { const blobUrl = URL.createObjectURL(file); - const pdf = await pdfjsLib.getDocument(blobUrl).promise; + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; + const pdf = await pdfjsLib + .getDocument({ + ...PDFJS_DEFAULT_OPTIONS, + url: blobUrl, + }) + .promise; URL.revokeObjectURL(blobUrl); return pdf.numPages; } diff --git a/app/core/src/main/resources/static/js/multitool/PdfContainer.js b/app/core/src/main/resources/static/js/multitool/PdfContainer.js index a3fa4c116d..866fd69faf 100644 --- a/app/core/src/main/resources/static/js/multitool/PdfContainer.js +++ b/app/core/src/main/resources/static/js/multitool/PdfContainer.js @@ -8,6 +8,12 @@ import { AddFilesCommand } from './commands/add-page.js'; import { DecryptFile } from '../DecryptFiles.js'; import { CommandSequence } from './commands/commands-sequence.js'; +const PDFJS_DEFAULT_OPTIONS = { + cMapUrl: pdfjsPath + 'cmaps/', + cMapPacked: true, + standardFontDataUrl: pdfjsPath + 'standard_fonts/', +}; + const isSvgFile = (file) => { if (!file) return false; const type = (file.type || '').toLowerCase(); @@ -479,8 +485,11 @@ class PdfContainer { } async toRenderer(objectUrl) { - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; - const pdf = await pdfjsLib.getDocument(objectUrl).promise; + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; + const pdf = await pdfjsLib.getDocument({ + url: objectUrl, + ...PDFJS_DEFAULT_OPTIONS, + }).promise; return { document: pdf, pageCount: pdf.numPages, diff --git a/app/core/src/main/resources/static/js/pages/add-image.js b/app/core/src/main/resources/static/js/pages/add-image.js index 2bafd86ecc..6a40145b7f 100644 --- a/app/core/src/main/resources/static/js/pages/add-image.js +++ b/app/core/src/main/resources/static/js/pages/add-image.js @@ -1,3 +1,9 @@ +const PDFJS_DEFAULT_OPTIONS = { + cMapUrl: pdfjsPath + 'cmaps/', + cMapPacked: true, + standardFontDataUrl: pdfjsPath + 'standard_fonts/', +}; + window.goToFirstOrLastPage = goToFirstOrLastPage; document.getElementById('download-pdf').addEventListener('click', async () => { @@ -31,8 +37,11 @@ document.querySelector('input[name=pdf-upload]').addEventListener('change', asyn const file = allFiles[0]; originalFileName = file.name.replace(/\.[^/.]+$/, ''); const pdfData = await file.arrayBuffer(); - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; - const pdfDoc = await pdfjsLib.getDocument({ data: pdfData }).promise; + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; + const pdfDoc = await pdfjsLib.getDocument({ + ...PDFJS_DEFAULT_OPTIONS, + data: pdfData, + }).promise; await DraggableUtils.renderPage(pdfDoc, 0); document.querySelectorAll('.show-on-file-selected').forEach((el) => { diff --git a/app/core/src/main/resources/static/js/pages/adjust-contrast.js b/app/core/src/main/resources/static/js/pages/adjust-contrast.js index a9692d2bc4..506af1f8e6 100644 --- a/app/core/src/main/resources/static/js/pages/adjust-contrast.js +++ b/app/core/src/main/resources/static/js/pages/adjust-contrast.js @@ -1,3 +1,9 @@ +const PDFJS_DEFAULT_OPTIONS = { + cMapUrl: pdfjsPath + 'cmaps/', + cMapPacked: true, + standardFontDataUrl: pdfjsPath + 'standard_fonts/', +}; + var canvas = document.getElementById('contrast-pdf-canvas'); var context = canvas.getContext('2d'); var originalImageData = null; @@ -9,8 +15,11 @@ async function renderPDFAndSaveOriginalImageData(file) { var fileReader = new FileReader(); fileReader.onload = async function () { var data = new Uint8Array(this.result); - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; - pdf = await pdfjsLib.getDocument({data: data}).promise; + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; + pdf = await pdfjsLib.getDocument({ + ...PDFJS_DEFAULT_OPTIONS, + data: data, + }).promise; // Get the number of pages in the PDF var numPages = pdf.numPages; diff --git a/app/core/src/main/resources/static/js/pages/change-metadata.js b/app/core/src/main/resources/static/js/pages/change-metadata.js index bdc5426b71..25d8565a06 100644 --- a/app/core/src/main/resources/static/js/pages/change-metadata.js +++ b/app/core/src/main/resources/static/js/pages/change-metadata.js @@ -1,3 +1,9 @@ +const PDFJS_DEFAULT_OPTIONS = { + cMapUrl: pdfjsPath + 'cmaps/', + cMapPacked: true, + standardFontDataUrl: pdfjsPath + 'standard_fonts/', +}; + const deleteAllCheckbox = document.querySelector('#deleteAll'); let inputs = document.querySelectorAll('input'); const customMetadataDiv = document.getElementById('customMetadata'); @@ -43,8 +49,13 @@ fileInput.addEventListener('change', async function () { customMetadataFormContainer.removeChild(customMetadataFormContainer.firstChild); } var url = URL.createObjectURL(file); - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; - const pdf = await pdfjsLib.getDocument(url).promise; + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; + const pdf = await pdfjsLib + .getDocument({ + ...PDFJS_DEFAULT_OPTIONS, + url: url, + }) + .promise; const pdfMetadata = await pdf.getMetadata(); lastPDFFile = pdfMetadata?.info; console.log(pdfMetadata); diff --git a/app/core/src/main/resources/static/js/pages/crop.js b/app/core/src/main/resources/static/js/pages/crop.js index c61cbaeccb..f6af3e5e23 100644 --- a/app/core/src/main/resources/static/js/pages/crop.js +++ b/app/core/src/main/resources/static/js/pages/crop.js @@ -1,3 +1,9 @@ +const PDFJS_DEFAULT_OPTIONS = { + cMapUrl: pdfjsPath + 'cmaps/', + cMapPacked: true, + standardFontDataUrl: pdfjsPath + 'standard_fonts/', +}; + let pdfCanvas = document.getElementById('cropPdfCanvas'); let overlayCanvas = document.getElementById('overlayCanvas'); let canvasesContainer = document.getElementById('canvasesContainer'); @@ -42,12 +48,17 @@ function renderPageFromFile(file) { let reader = new FileReader(); reader.onload = function (ev) { let typedArray = new Uint8Array(reader.result); - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; - pdfjsLib.getDocument(typedArray).promise.then(function (pdf) { - pdfDoc = pdf; - totalPages = pdf.numPages; - renderPage(currentPage); - }); + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; + pdfjsLib + .getDocument({ + ...PDFJS_DEFAULT_OPTIONS, + data: typedArray, + }) + .promise.then(function (pdf) { + pdfDoc = pdf; + totalPages = pdf.numPages; + renderPage(currentPage); + }); }; reader.readAsArrayBuffer(file); } diff --git a/app/core/src/main/resources/static/js/pages/pdf-to-csv.js b/app/core/src/main/resources/static/js/pages/pdf-to-csv.js index 9a06aac5b4..1c3cc01248 100644 --- a/app/core/src/main/resources/static/js/pages/pdf-to-csv.js +++ b/app/core/src/main/resources/static/js/pages/pdf-to-csv.js @@ -1,3 +1,9 @@ +const PDFJS_DEFAULT_OPTIONS = { + cMapUrl: pdfjsPath + 'cmaps/', + cMapPacked: true, + standardFontDataUrl: pdfjsPath + 'standard_fonts/', +}; + let pdfCanvas = document.getElementById('cropPdfCanvas'); let overlayCanvas = document.getElementById('overlayCanvas'); let canvasesContainer = document.getElementById('canvasesContainer'); @@ -37,12 +43,17 @@ btn1Object.addEventListener('click', function (e) { let reader = new FileReader(); reader.onload = function (ev) { let typedArray = new Uint8Array(reader.result); - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; - pdfjsLib.getDocument(typedArray).promise.then(function (pdf) { - pdfDoc = pdf; - totalPages = pdf.numPages; - renderPage(currentPage); - }); + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; + pdfjsLib + .getDocument({ + ...PDFJS_DEFAULT_OPTIONS, + data: typedArray, + }) + .promise.then(function (pdf) { + pdfDoc = pdf; + totalPages = pdf.numPages; + renderPage(currentPage); + }); }; reader.readAsArrayBuffer(file); } @@ -58,12 +69,17 @@ btn2Object.addEventListener('click', function (e) { let reader = new FileReader(); reader.onload = function (ev) { let typedArray = new Uint8Array(reader.result); - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; - pdfjsLib.getDocument(typedArray).promise.then(function (pdf) { - pdfDoc = pdf; - totalPages = pdf.numPages; - renderPage(currentPage); - }); + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; + pdfjsLib + .getDocument({ + ...PDFJS_DEFAULT_OPTIONS, + data: typedArray, + }) + .promise.then(function (pdf) { + pdfDoc = pdf; + totalPages = pdf.numPages; + renderPage(currentPage); + }); }; reader.readAsArrayBuffer(file); } @@ -75,12 +91,17 @@ function renderPageFromFile(file) { let reader = new FileReader(); reader.onload = function (ev) { let typedArray = new Uint8Array(reader.result); - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; - pdfjsLib.getDocument(typedArray).promise.then(function (pdf) { - pdfDoc = pdf; - totalPages = pdf.numPages; - renderPage(currentPage); - }); + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; + pdfjsLib + .getDocument({ + ...PDFJS_DEFAULT_OPTIONS, + data: typedArray, + }) + .promise.then(function (pdf) { + pdfDoc = pdf; + totalPages = pdf.numPages; + renderPage(currentPage); + }); pageNumbers.value = currentPage; }; reader.readAsArrayBuffer(file); diff --git a/app/core/src/main/resources/static/js/pages/sign.js b/app/core/src/main/resources/static/js/pages/sign.js index ec02e75b36..48a15c7807 100644 --- a/app/core/src/main/resources/static/js/pages/sign.js +++ b/app/core/src/main/resources/static/js/pages/sign.js @@ -1,3 +1,9 @@ +const PDFJS_DEFAULT_OPTIONS = { + cMapUrl: pdfjsPath + 'cmaps/', + cMapPacked: true, + standardFontDataUrl: pdfjsPath + 'standard_fonts/', +}; + window.toggleSignatureView = toggleSignatureView; window.previewSignature = previewSignature; window.addSignatureFromPreview = addSignatureFromPreview; @@ -70,9 +76,11 @@ document const file = allFiles[0]; originalFileName = file.name.replace(/\.[^/.]+$/, ""); const pdfData = await file.arrayBuffer(); - pdfjsLib.GlobalWorkerOptions.workerSrc = - "./pdfjs-legacy/pdf.worker.mjs"; - const pdfDoc = await pdfjsLib.getDocument({ data: pdfData }).promise; + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; + const pdfDoc = await pdfjsLib.getDocument({ + ...PDFJS_DEFAULT_OPTIONS, + data: pdfData, + }).promise; await DraggableUtils.renderPage(pdfDoc, 0); document.querySelectorAll(".show-on-file-selected").forEach((el) => { diff --git a/app/core/src/main/resources/templates/convert/pdf-to-pdfa.html b/app/core/src/main/resources/templates/convert/pdf-to-pdfa.html index 5b44ff5b62..6035e7561d 100644 --- a/app/core/src/main/resources/templates/convert/pdf-to-pdfa.html +++ b/app/core/src/main/resources/templates/convert/pdf-to-pdfa.html @@ -41,7 +41,7 @@ diff --git a/app/core/src/main/resources/templates/merge-pdfs.html b/app/core/src/main/resources/templates/merge-pdfs.html index 794eac8056..8878c3c257 100644 --- a/app/core/src/main/resources/templates/merge-pdfs.html +++ b/app/core/src/main/resources/templates/merge-pdfs.html @@ -58,7 +58,7 @@ diff --git a/app/core/src/main/resources/templates/misc/compare.html b/app/core/src/main/resources/templates/misc/compare.html index 27a7ed9edc..f7bab9589a 100644 --- a/app/core/src/main/resources/templates/misc/compare.html +++ b/app/core/src/main/resources/templates/misc/compare.html @@ -79,7 +79,7 @@ - +
@@ -105,7 +105,8 @@ result2.addEventListener('scroll', function () { result1.scrollTop = result2.scrollTop; }); - async function comparePDFs() { + + async function comparePDFs(event) { const file1 = document.getElementById("fileInput-input").files[0]; const file2 = document.getElementById("fileInput2-input").files[0]; var color1 = document.getElementById('color-box1').value; @@ -113,137 +114,216 @@ const complexMessage = /*[[#{compare.complex.message}]]*/ 'One or both of the provided documents are large files, accuracy of comparison may be reduced'; const largeFilesMessage = /*[[#{compare.large.file.message}]]*/ 'One or Both of the provided documents are too large to process'; - const noTextMessage = /*[[#{compare.no.text.message}]]*/ 'One or both of the selected PDFs have no text content. Please choose PDFs with text for comparison."'; + const noTextMessage = /*[[#{compare.no.text.message}]]*/ 'One or both of the selected PDFs have no text content. Please choose PDFs with text for comparison.'; + const invalidPdfMessage = /*[[#{compare.invalid.pdf.message}]]*/ 'One or both files are not valid PDFs. Please check and re-upload.'; + const submitText = /*[[#{compare.submit}]]*/ 'Compare'; if (!file1 || !file2) { - console.error("Please select two PDF files to compare"); + alert('Please select two PDF files to compare'); return; } - pdfjsLib.GlobalWorkerOptions.workerSrc = './pdfjs-legacy/pdf.worker.mjs'; - - const [pdf1, pdf2] = await Promise.all([ - pdfjsLib.getDocument(URL.createObjectURL(file1)).promise, - pdfjsLib.getDocument(URL.createObjectURL(file2)).promise - ]); - - const extractText = async (pdf) => { - const pages = []; - for (let i = 1; i <= pdf.numPages; i++) { - const page = await pdf.getPage(i); - const content = await page.getTextContent(); - const strings = content.items.map(item => item.str); - pages.push(strings.join(" ")); - } - return pages.join(" "); - }; - - const [text1, text2] = await Promise.all([ - extractText(pdf1), - extractText(pdf2) - ]); - - if (text1.trim() === "" || text2.trim() === "") { - alert(noTextMessage); + // Basic checks + if (file1.size === 0 || file2.size === 0) { + alert('One or both files are empty.'); + return; + } + if (file1.size > 100 * 1024 * 1024 || file2.size > 100 * 1024 * 1024) { + alert(largeFilesMessage); return; } - const resultDiv1 = document.getElementById("result1"); - const resultDiv2 = document.getElementById("result2"); - const loading = /*[[#{loading}]]*/ 'Loading...'; - - resultDiv1.innerHTML = loading; - resultDiv2.innerHTML = loading; - - // Create a new Worker - const worker = new Worker('./js/compare/pdfWorker.js'); - - - // Post messages to the worker - worker.postMessage({ - type: 'SET_COMPLEX_MESSAGE', - message: complexMessage - }); - - worker.postMessage({ - type: 'SET_TOO_LARGE_MESSAGE', - message: largeFilesMessage - }); - - // Error handling for the worker - worker.onerror = function (error) { - console.error('Worker error:', error); + // PDF.js setup (Legacy-safe: Worker disabled) + const PDFJS_DEFAULT_OPTIONS = { + cMapUrl: pdfjsPath + 'cmaps/', + cMapPacked: true, + standardFontDataUrl: pdfjsPath + 'standard_fonts/', + disableWorker: true // Avoids Legacy CMap errors without changing PDF.js }; - worker.onmessage = function (e) { - const { status, differences, message } = e.data; - if (status === 'error') { + pdfjsLib.GlobalWorkerOptions.workerSrc = pdfjsPath + 'pdf.worker.mjs'; - resultDiv1.innerHTML = ''; - resultDiv2.innerHTML = ''; - alert(message); - return; + const button = event.target; + button.disabled = true; + button.textContent = 'Processing...'; + + try { + // Load ArrayBuffer + const [data1, data2] = await Promise.all([ + readFileAsArrayBuffer(file1), + readFileAsArrayBuffer(file2) + ]); + + // Header validation (prevents InvalidPDFException) + await validatePdfHeader(data1, 'File 1'); + await validatePdfHeader(data2, 'File 2'); + + // Load PDFs + const [pdf1, pdf2] = await Promise.all([ + loadPdfWithErrorHandling({ ...PDFJS_DEFAULT_OPTIONS, data: data1 }, 'File 1'), + loadPdfWithErrorHandling({ ...PDFJS_DEFAULT_OPTIONS, data: data2 }, 'File 2') + ]); + + // Extract text + result1.innerHTML = 'Extracting text from File 1...'; + result2.innerHTML = 'Extracting text from File 2...'; + const [text1, text2] = await Promise.all([ + extractText(pdf1, 'File 1', result1), + extractText(pdf2, 'File 2', result2) + ]); + + if (text1.trim() === "" || text2.trim() === "") { + throw new Error(noTextMessage); } - if (status === 'success' && differences) { - console.log('Differences:', differences); - displayDifferences(differences); - } - if (event.data.status === 'warning') { - console.warn(event.data.message); - alert(event.data.message); - } - }; - worker.postMessage({ text1, text2, color1, color2 }); + // Worker diff + await processWithWorker(text1, text2, color1, color2, complexMessage, largeFilesMessage); - const displayDifferences = (differences) => { - const resultDiv1 = document.getElementById("result1"); - const resultDiv2 = document.getElementById("result2"); - resultDiv1.innerHTML = ""; - resultDiv2.innerHTML = ""; - - differences.forEach(([color, word]) => { - const span1 = document.createElement("span"); - const span2 = document.createElement("span"); - - if (color === color2) { - span1.style.color = "transparent"; - span1.style.userSelect = "none"; - span2.style.color = color; - } - // If it's a deletion, show it in in the first document and transparent in the second - else if (color === color1) { - span1.style.color = color; - span2.style.color = "transparent"; - span2.style.userSelect = "none"; - } - // If it's unchanged, show it in black in both - else { - span1.style.color = color; - span2.style.color = color; - } - - span1.textContent = word; - span2.textContent = word; - resultDiv1.appendChild(span1); - resultDiv2.appendChild(span2); - - // Add space after each word, or a new line if the word ends with a full stop - const spaceOrNewline1 = document.createElement("span"); - const spaceOrNewline2 = document.createElement("span"); - if (word.endsWith(".")) { - spaceOrNewline1.innerHTML = "
"; - spaceOrNewline2.innerHTML = "
"; - } else { - spaceOrNewline1.textContent = " "; - spaceOrNewline2.textContent = " "; - } - resultDiv1.appendChild(spaceOrNewline1); - resultDiv2.appendChild(spaceOrNewline2); - }); - }; + } catch (error) { + console.error('Comparison failed:', error); + alert(error.message || invalidPdfMessage); + result1.innerHTML = ''; + result2.innerHTML = ''; + } finally { + button.disabled = false; + button.textContent = submitText; + } } + // FileReader helper + function readFileAsArrayBuffer(file) { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result); + reader.onerror = reject; + reader.readAsArrayBuffer(file); + }); + } + // Header validation (PDF.js-specific, but client-side) + async function validatePdfHeader(data, fileName) { + const header = new Uint8Array(data.slice(0, 8)); + const headerStr = String.fromCharCode(...header); + console.log(`${fileName} header:`, headerStr); + if (!headerStr.startsWith('%PDF-')) { + throw new Error(`${fileName} is not a valid PDF (header: ${headerStr}).`); + } + if (data.byteLength < 100) { + throw new Error(`${fileName} is too short.`); + } + } + + // PDF loading with catch + function loadPdfWithErrorHandling(options, fileName) { + return pdfjsLib.getDocument(options).promise + .then(pdf => { + console.log(`${fileName} loaded: ${pdf.numPages} pages`); + return pdf; + }) + .catch(err => { + console.error(`${fileName} load failed:`, err); + if (err.name === 'InvalidPDFException') { + throw new Error(`${fileName}: Invalid PDF structure. Re-upload.`); + } + throw err; + }); + } + + // Text extraction + async function extractText(pdf, fileName, statusElement) { + const pages = []; + const totalPages = pdf.numPages; + for (let i = 1; i <= totalPages; i++) { + const page = await pdf.getPage(i); + const content = await page.getTextContent(); + const strings = content.items.map(item => item.str).join(' '); + pages.push(strings); + statusElement.innerHTML = `${fileName}: ${Math.round((i / totalPages) * 100)}%`; + } + return pages.join(' '); + } + + // Worker processing + async function processWithWorker(text1, text2, color1, color2, complexMessage, largeFilesMessage) { + return new Promise((resolve, reject) => { + const worker = new Worker('./js/compare/pdfWorker.js'); + const timeout = setTimeout(() => { + worker.terminate(); + reject(new Error('Timeout: Files too complex.')); + }, 30000); + + worker.postMessage({ type: 'SET_COMPLEX_MESSAGE', message: complexMessage }); + worker.postMessage({ type: 'SET_TOO_LARGE_MESSAGE', message: largeFilesMessage }); + + worker.onerror = (error) => { + clearTimeout(timeout); + worker.terminate(); + reject(new Error('Worker error: ' + error.message)); + }; + + worker.onmessage = (e) => { + clearTimeout(timeout); + const { status, differences, message } = e.data; + if (status === 'error') { + worker.terminate(); + reject(new Error(message)); + return; + } + if (status === 'warning') { + alert(message); + } + if (status === 'success' && differences) { + displayDifferences(differences, color1, color2); + worker.terminate(); + resolve(); + } + }; + + worker.postMessage({ type: 'COMPARE', text1, text2, color1, color2 }); + }); + } + + // Display differences + function displayDifferences(differences, color1, color2) { + const resultDiv1 = document.getElementById("result1"); + const resultDiv2 = document.getElementById("result2"); + resultDiv1.innerHTML = ""; + resultDiv2.innerHTML = ""; + + differences.forEach(([color, word]) => { + const span1 = document.createElement("span"); + const span2 = document.createElement("span"); + + if (color === color2) { + span1.style.color = "transparent"; + span1.style.userSelect = "none"; + span2.style.color = color; + } else if (color === color1) { + span1.style.color = color; + span2.style.color = "transparent"; + span2.style.userSelect = "none"; + } else { + span1.style.color = color || 'black'; + span2.style.color = color || 'black'; + } + + span1.textContent = word; + span2.textContent = word; + resultDiv1.appendChild(span1); + resultDiv2.appendChild(span2); + + const spaceOrNewline1 = document.createElement("span"); + const spaceOrNewline2 = document.createElement("span"); + if (word.endsWith(".")) { + spaceOrNewline1.innerHTML = "
"; + spaceOrNewline2.innerHTML = "
"; + } else { + spaceOrNewline1.textContent = " "; + spaceOrNewline2.textContent = " "; + } + resultDiv1.appendChild(spaceOrNewline1); + resultDiv2.appendChild(spaceOrNewline2); + }); + }
diff --git a/app/core/src/main/resources/templates/rotate-pdf.html b/app/core/src/main/resources/templates/rotate-pdf.html index 3d54ce3177..6baa242683 100644 --- a/app/core/src/main/resources/templates/rotate-pdf.html +++ b/app/core/src/main/resources/templates/rotate-pdf.html @@ -59,12 +59,24 @@ - +