From 8f5c95f0f4aee6d50ec0c82d75133cd5621cd6cf Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sun, 26 Apr 2020 05:33:26 -0700 Subject: [PATCH] Remove last vestiges of command line usage of qpdf - change to check_pdf --- docs/advanced.rst | 3 +- docs/errors.rst | 8 +-- docs/release_notes.rst | 1 + docs/security.rst | 9 ++-- src/ocrmypdf/_sync.py | 5 +- src/ocrmypdf/_validation.py | 7 --- src/ocrmypdf/exec/qpdf.py | 63 ----------------------- src/ocrmypdf/helpers.py | 37 +++++++++++++ tests/{test_qpdf.py => test_check_pdf.py} | 8 +-- tests/test_hocrtransform.py | 4 +- tests/test_main.py | 7 +-- tests/test_stdio.py | 4 +- tests/test_userunit.py | 2 +- 13 files changed, 63 insertions(+), 95 deletions(-) delete mode 100644 src/ocrmypdf/exec/qpdf.py rename tests/{test_qpdf.py => test_check_pdf.py} (82%) diff --git a/docs/advanced.rst b/docs/advanced.rst index 6f8567ba..9c30f1c5 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -148,7 +148,8 @@ In addition to tesseract, OCRmyPDF uses the following external binaries: - ``gs`` (Ghostscript) - ``unpaper`` -- ``qpdf`` +- ``pngquant`` +- ``jbig2`` In each case OCRmyPDF will search the ``PATH`` environment variable to locate the binaries. diff --git a/docs/errors.rst b/docs/errors.rst index 328080ad..bf0b53d0 100644 --- a/docs/errors.rst +++ b/docs/errors.rst @@ -32,10 +32,10 @@ As the error message suggests, your options are: Input file 'filename' is not a valid PDF ======================================== -OCRmyPDF passes files through qpdf, a program that fixes errors in PDFs, -before it tries to work on them. In most cases this happens because the -PDF is corrupt and truncated (incomplete file copying) and not much can -be done. +OCRmyPDF checks files with pikepdf, a library that in turn uses libqpdf to fixes +errors in PDFs, before it tries to work on them. In most cases this happens +because the PDF is corrupt and truncated (incomplete file copying) and not much +can be done. You can try rewriting the file with Ghostscript: diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 3c01b831..3c0bfa90 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -30,6 +30,7 @@ v10.0.0 (not yet released) ``ocrmypdf.helpers.Resolution`` class. - A deprecated parameter in ``ocrmypdf.exec.ghostscript.generate_pdfa`` was removed. +- The deprecated module ``ocrmypdf.exec.qpdf`` was removed. - The ``ocrmypdf.hocrtransform`` module has been updated to follow PEP8 naming conventions. diff --git a/docs/security.rst b/docs/security.rst index bcc69e8e..36246960 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -68,7 +68,7 @@ license, OCRmyPDF's GPL license, and any other licenses. Setting aside these concerns, a side effect of OCRmyPDF is it may incidentally sanitize PDFs that contain certain types of malware. It -runs ``qpdf`` to repair the PDF, which could correct malformed PDF +repairs the PDF with pikepdf/libqpdf, which could correct malformed PDF structures that are part of an attack. When PDF/A output is selected (the default), the input PDF is partially reconstructed by Ghostscript. When ``--force-ocr`` is used, all pages are rasterized and reconverted @@ -144,10 +144,9 @@ set, the document cannot be viewed without the password. Either way, OCRmyPDF does not remove passwords from PDFs and exits with an error on encountering them. -``qpdf``, one of OCRmyPDF's dependencies, can remove passwords. If the -owner and user password are set, a password is required for ``qpdf``. If -only the owner password is set, then the password can be stripped, even -if one does not have the owner password. +``qpdf`` can remove passwords. If the owner and user password are set, a +password is required for ``qpdf``. If only the owner password is set, then the +password can be stripped, even if one does not have the owner password. After OCR is applied, password protection is not permitted on PDF/A documents but the file can be converted to regular PDF. diff --git a/src/ocrmypdf/_sync.py b/src/ocrmypdf/_sync.py index e6e44271..77ecfa9b 100644 --- a/src/ocrmypdf/_sync.py +++ b/src/ocrmypdf/_sync.py @@ -64,8 +64,7 @@ from ._validation import ( report_output_file_size, ) from .exceptions import ExitCode, ExitCodeException -from .exec import qpdf -from .helpers import available_cpu_count +from .helpers import available_cpu_count, check_pdf from .pdfa import file_claims_pdfa log = logging.getLogger(__name__) @@ -357,7 +356,7 @@ def run_pipeline(options, api=False): pdfa_info['conformance'], ) return ExitCode.pdfa_conversion_failed - if not qpdf.check(options.output_file): + if not check_pdf(options.output_file): log.warning('Output file: The generated PDF is INVALID') return ExitCode.invalid_output_pdf report_output_file_size(options, start_input_file, options.output_file) diff --git a/src/ocrmypdf/_validation.py b/src/ocrmypdf/_validation.py index ef440e0f..8be0b16e 100644 --- a/src/ocrmypdf/_validation.py +++ b/src/ocrmypdf/_validation.py @@ -38,7 +38,6 @@ from .exec import ( ghostscript, jbig2enc, pngquant, - qpdf, tesseract, unpaper, ) @@ -473,9 +472,3 @@ def check_dependency_versions(options): "supported. Please upgrade to a newer version, or downgrade to the " "previous version." ) - check_external_program( - program='qpdf', - package='qpdf', - version_checker=qpdf.version, - need_version='8.0.2', - ) diff --git a/src/ocrmypdf/exec/qpdf.py b/src/ocrmypdf/exec/qpdf.py deleted file mode 100644 index 32ef8cb6..00000000 --- a/src/ocrmypdf/exec/qpdf.py +++ /dev/null @@ -1,63 +0,0 @@ -# © 2017 James R. Barlow: github.com/jbarlow83 -# -# This file is part of OCRmyPDF. -# -# OCRmyPDF is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# OCRmyPDF is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with OCRmyPDF. If not, see . - -"""Interface to qpdf executable""" - -import logging -from io import StringIO - -import pikepdf - -log = logging.getLogger(__name__) - - -def version(): - return pikepdf.__libqpdf_version__ - - -def check(input_file): - pdf = None - try: - pdf = pikepdf.open(input_file) - except pikepdf.PdfError as e: - log.error(e) - return False - else: - messages = pdf.check() - for msg in messages: - if 'error' in msg.lower(): - log.error(msg) - else: - log.warning(msg) - - sio = StringIO() - linearize = None - try: - pdf.check_linearization(sio) - except RuntimeError: - pass - else: - linearize = sio.getvalue() - if linearize: - log.warning(linearize) - - if not messages and not linearize: - return True - return False - finally: - if pdf: - pdf.close() diff --git a/src/ocrmypdf/helpers.py b/src/ocrmypdf/helpers.py index c57aca35..69f4d9bf 100644 --- a/src/ocrmypdf/helpers.py +++ b/src/ocrmypdf/helpers.py @@ -24,9 +24,12 @@ from collections import namedtuple from collections.abc import Iterable from contextlib import suppress from functools import wraps +from io import StringIO from math import inf, isclose from pathlib import Path +import pikepdf + log = logging.getLogger(__name__) @@ -165,6 +168,40 @@ def is_file_writable(test_file: os.PathLike): return False +def check_pdf(input_file): + pdf = None + try: + pdf = pikepdf.open(input_file) + except pikepdf.PdfError as e: + log.error(e) + return False + else: + messages = pdf.check() + for msg in messages: + if 'error' in msg.lower(): + log.error(msg) + else: + log.warning(msg) + + sio = StringIO() + linearize = None + try: + pdf.check_linearization(sio) + except RuntimeError: + pass + else: + linearize = sio.getvalue() + if linearize: + log.warning(linearize) + + if not messages and not linearize: + return True + return False + finally: + if pdf: + pdf.close() + + def deprecated(func): """Warn that function is deprecated""" diff --git a/tests/test_qpdf.py b/tests/test_check_pdf.py similarity index 82% rename from tests/test_qpdf.py rename to tests/test_check_pdf.py index 0e925249..b3516e90 100644 --- a/tests/test_qpdf.py +++ b/tests/test_check_pdf.py @@ -17,9 +17,9 @@ import pytest -import ocrmypdf.exec.qpdf as qpdf +from ocrmypdf.helpers import check_pdf -def test_qpdf_error(resources): - assert qpdf.check(resources / 'blank.pdf') - assert not qpdf.check(__file__) +def test_pdf_error(resources): + assert check_pdf(resources / 'blank.pdf') + assert not check_pdf(__file__) diff --git a/tests/test_hocrtransform.py b/tests/test_hocrtransform.py index 13b1f601..e00f8365 100644 --- a/tests/test_hocrtransform.py +++ b/tests/test_hocrtransform.py @@ -21,8 +21,8 @@ import pytest from PIL import Image from ocrmypdf import hocrtransform -from ocrmypdf.exec import qpdf from ocrmypdf.exec.tesseract import HOCR_TEMPLATE +from ocrmypdf.helpers import check_pdf # pylint: disable=redefined-outer-name @@ -43,4 +43,4 @@ def test_mono_image(blank_hocr, outdir): hocr = hocrtransform.HocrTransform(str(blank_hocr), 300) hocr.to_pdf(str(outdir / 'mono.pdf'), image_filename=str(outdir / 'mono.tif')) - qpdf.check(str(outdir / 'mono.pdf')) + check_pdf(str(outdir / 'mono.pdf')) diff --git a/tests/test_main.py b/tests/test_main.py index 9b0cd17e..e69ed192 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -29,7 +29,8 @@ from PIL import Image import ocrmypdf from ocrmypdf.exceptions import ExitCode, MissingDependencyError -from ocrmypdf.exec import ghostscript, qpdf, tesseract +from ocrmypdf.exec import ghostscript, tesseract +from ocrmypdf.helpers import check_pdf from ocrmypdf.pdfa import file_claims_pdfa from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo @@ -529,8 +530,8 @@ def test_skip_big_with_no_images(spoof_tesseract_noop, resources, outpdf): @pytest.mark.skipif( - '8.0.0' <= qpdf.version() <= '8.0.1', - reason="qpdf regression on pages with no contents", + '8.0.0' <= pikepdf.__libqpdf_version__ <= '8.0.1', + reason="libqpdf regression on pages with no contents", ) def test_no_contents(spoof_tesseract_noop, resources, outpdf): check_ocrmypdf( diff --git a/tests/test_stdio.py b/tests/test_stdio.py index 150e40ff..e57c11f0 100644 --- a/tests/test_stdio.py +++ b/tests/test_stdio.py @@ -23,7 +23,7 @@ from subprocess import DEVNULL, PIPE, CalledProcessError, Popen, run import pytest from ocrmypdf.exceptions import ExitCode -from ocrmypdf.exec import qpdf +from ocrmypdf.helpers import check_pdf # pytest.helpers is dynamic # pylint: disable=no-member,redefined-outer-name @@ -74,7 +74,7 @@ def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf): ) assert p.returncode == ExitCode.ok - assert qpdf.check(output_file) + assert check_pdf(output_file) @pytest.mark.skipif( diff --git a/tests/test_userunit.py b/tests/test_userunit.py index 83ad01d4..41d21aa2 100644 --- a/tests/test_userunit.py +++ b/tests/test_userunit.py @@ -39,7 +39,7 @@ def test_userunit_ghostscript_fails(poster, no_outpdf, caplog): assert 'not supported by Ghostscript' in caplog.text -def test_userunit_qpdf_passes(spoof_tesseract_cache, poster, outpdf): +def test_userunit_pdf_passes(spoof_tesseract_cache, poster, outpdf): before = PdfInfo(poster) check_ocrmypdf(poster, outpdf, '--output-type=pdf', env=spoof_tesseract_cache)