haeaeae/ocrmypdf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'windows'

Browse Source
This commit is contained in:
James R. Barlow
2019-12-06 15:09:09 -08:00
parent fbf271a3ec 66bda3420a
commit 6b745d892f
34 changed files with 562 additions and 409 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.pre-commit-config.yaml
+1
View File
@@ -4,6 +4,7 @@ repos:
hooks:
- id: black
language_version: python3.7
exclude: ^src/ocrmypdf/lib/_leptonica.py
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.4.0
hooks:
docs/installation.rst
+30 -11
View File
@@ -435,8 +435,36 @@ See `OCRmyPDF Docker Image <docker>`__ for more information.
Installing on Windows
=====================
Direct installation on Windows is not currently possible, but it works well in
Windows Subsystem for Linux:
.. warning::
Native Windows support is new. Consider it "beta" software. Some
functionality is missing or may be more difficult to enable. If you need a
production-ready solution, use Windows Subsystem for Linux or a Docker
image.
You must install the following for Windows using their installers:
* Python 3.7 (64-bit recommended)
* Tesseract 4.0 or later
* Ghostscript 9.50 or later
* QPDF 9.0.2 or later
You can install these with the Chocolatey package manager:
* ``choco install python3``
* ``choco install tesseract``
* ``choco install ghostscript``
* ``choco install qpdf``
Also consider adding:
* ``choco install pngquant``
Modify your ``PATH`` environment variable so that Tesseract, Ghostscript and QPDF
executables on the ``PATH``.
Installing on Windows Subsystem for Linux
=========================================
#. Install Ubuntu 18.04 for Windows Subsystem for Linux, if not already installed.
#. Follow the procedure to install :ref:`OCRmyPDF on Ubuntu 18.04 <ubuntu-lts-latest>`.
@@ -455,15 +483,6 @@ Then confirm that the expected version from PyPI (|latest|) is installed:
You can then run OCRmyPDF in the Windows command prompt or Powershell, prefixing
``wsl``, and call it from Windows programs or batch files.
Why no native Windows?
^^^^^^^^^^^^^^^^^^^^^^
It would probably not be too difficult to port on Windows. The main
reason this has been avoided is the difficulty of packaging and
installing the various non-Python dependencies: Tesseract, QPDF,
Ghostscript, Leptonica. Pull requests to add or improve Windows support
would be quite welcome.
Docker
^^^^^^
src/ocrmypdf/_graft.py
+1 -1
View File
@@ -21,7 +21,7 @@ from pathlib import Path
import pikepdf
MAX_REPLACE_PAGES = int(os.environ.get('_OCRMYPDF_MAX_REPLACE_PAGES', 100))
MAX_REPLACE_PAGES = 100
def _update_page_resources(*, page, font, font_key, procset):
src/ocrmypdf/_pipeline.py
+3 -3
View File
@@ -37,7 +37,7 @@ from .exceptions import (
UnsupportedImageFormatError,
)
from .exec import ghostscript, tesseract
from .helpers import re_symlink
from .helpers import safe_symlink
from .hocrtransform import HocrTransform
from .optimize import optimize
from .pdfa import generate_pdfa_ps
@@ -132,7 +132,7 @@ def triage(input_file, output_file, options, log):
"input file is a PDF, not an image."
)
# Origin file is a pdf create a symlink with pdf extension
re_symlink(input_file, output_file)
safe_symlink(input_file, output_file)
return output_file
except EnvironmentError as e:
log.error(e)
@@ -701,7 +701,7 @@ def convert_to_pdfa(input_pdf, input_ps_stub, context):
if modified:
pdf_file.save(fix_docinfo_file)
else:
os.symlink(input_pdf, fix_docinfo_file)
safe_symlink(input_pdf, fix_docinfo_file)
ghostscript.generate_pdfa(
pdf_version=input_pdfinfo.min_version,
src/ocrmypdf/_sync.py
+21 -5
View File
@@ -26,6 +26,7 @@ from collections import namedtuple
from tempfile import mkdtemp
from tqdm import tqdm
import PIL
from ._graft import OcrGrafter
from ._jobcontext import PDFContext, cleanup_working_files, make_logger
@@ -176,7 +177,7 @@ def post_process(pdf_file, context):
return optimize_pdf(pdf_out, context)
def worker_init(queue):
def worker_init(queue, max_pixels):
"""Initialize a process pool worker"""
# Ignore SIGINT (our parent process will kill us gracefully)
@@ -188,9 +189,15 @@ def worker_init(queue):
root.handlers = []
root.addHandler(h)
# In Windows, child process will not inherit our change to this value in
# the parent process, so ensure workers get it set
PIL.Image.MAX_IMAGE_PIXELS = max_pixels
def worker_thread_init(_queue):
pass
def worker_thread_init(_queue, max_pixels):
# This is probably not needed since threads should all see the same memory,
# but done for consistency.
PIL.Image.MAX_IMAGE_PIXELS = max_pixels
def log_listener(queue):
@@ -261,7 +268,9 @@ def exec_concurrent(context):
unit_scale=0.5,
disable=not context.options.progress_bar,
) as pbar, Pool(
processes=max_workers, initializer=initializer, initargs=(log_queue,)
processes=max_workers,
initializer=initializer,
initargs=(log_queue, PIL.Image.MAX_IMAGE_PIXELS),
) as pool:
results = pool.imap_unordered(exec_page_sync, context.get_page_contexts())
while True:
@@ -304,6 +313,13 @@ class NeverRaise(Exception):
pass # pylint: disable=unnecessary-pass
def samefile(f1, f2):
if os.name == 'nt':
return f1 == f2
else:
return os.path.samefile(f1, f2)
def run_pipeline(options, api=False):
log = make_logger(options, __name__)
@@ -339,7 +355,7 @@ def run_pipeline(options, api=False):
if options.output_file == '-':
log.info("Output sent to stdout")
elif os.path.samefile(options.output_file, os.devnull):
elif samefile(options.output_file, os.devnull):
pass # Say nothing when sending to dev null
else:
if options.output_type.startswith('pdfa'):
src/ocrmypdf/_validation.py
+3 -3
View File
@@ -42,7 +42,7 @@ from .exec import (
tesseract,
unpaper,
)
from .helpers import is_file_writable, is_iterable_notstr, monotonic, re_symlink
from .helpers import is_file_writable, is_iterable_notstr, monotonic, safe_symlink
# -------------
# External dependencies
@@ -374,7 +374,7 @@ def create_input_file(options, work_folder):
else:
try:
target = os.path.join(work_folder, 'origin')
re_symlink(options.input_file, target)
safe_symlink(options.input_file, target)
return target
except FileNotFoundError:
raise InputFileError(f"File not found - {options.input_file}")
@@ -446,7 +446,7 @@ def report_output_file_size(options, input_file, output_file):
def check_dependency_versions(options):
check_external_program(
program='tesseract',
package={'darwin': 'tesseract', 'linux': 'tesseract-ocr'},
package={'linux': 'tesseract-ocr'},
version_checker=tesseract.version,
need_version='4.0.0', # using backport for Travis CI
)
src/ocrmypdf/api.py
+2 -1
View File
@@ -18,6 +18,7 @@
import logging
import os
import sys
import warnings
from enum import IntEnum
from pathlib import Path
@@ -158,7 +159,7 @@ def create_options(*, input_file, output_file, **kwargs):
# If we are running a Tesseract spoof, ensure it knows what the input file is
if os.environ.get('PYTEST_CURRENT_TEST') and options.tesseract_env:
options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = input_file
options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = os.fspath(input_file)
return options
src/ocrmypdf/exec/__init__.py
+33 -2
View File
@@ -21,14 +21,35 @@ import logging
import os
import re
import sys
import shutil
from collections.abc import Mapping
from subprocess import PIPE, STDOUT, CalledProcessError, run
from subprocess import PIPE, STDOUT, CalledProcessError, run as subprocess_run
from ..exceptions import ExitCode, MissingDependencyError
log = logging.Logger(__name__)
def _get_program(args, env=None):
program = args[0]
test_path = env.get('_OCRMYPDF_TEST_PATH', '')
if test_path:
program = shutil.which(program, path=test_path)
return program
def run(args, *, env=None, **kwargs):
if not env:
env = os.environ
program = _get_program(args, env)
if os.name == 'nt' and program.lower().endswith('.py'):
args = [sys.executable, program] + args[1:]
else:
args = [program] + args[1:]
log.debug(args)
return subprocess_run(args, env=env, **kwargs)
def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)', env=None):
"Get the version of the specified program"
args_prog = [program, version_arg]
@@ -111,23 +132,33 @@ On RPM-based systems (Red Hat, Fedora), search for instructions on
installing the RPM for {program}.
'''
windows_install_advice = '''
If not already installed, install the Chocolatey package manager. Then use
a command prompt to install the missing package:
choco install {package}
'''
def _get_platform():
if sys.platform.startswith('freebsd'):
return 'freebsd'
elif sys.platform.startswith('linux'):
return 'linux'
elif sys.platform.startswith('win'):
return 'windows'
return sys.platform
def _error_trailer(program, package, **kwargs):
if isinstance(package, Mapping):
package = package[_get_platform()]
package = package.get(_get_platform(), program)
if _get_platform() == 'darwin':
log.info(osx_install_advice.format(**locals()))
elif _get_platform() == 'linux':
log.info(linux_install_advice.format(**locals()))
elif _get_platform() == 'windows':
log.info(windows_install_advice.format(**locals()))
def _error_missing_program(program, package, required_for, recommended):
src/ocrmypdf/exec/ghostscript.py
+118 -97
View File
@@ -19,24 +19,36 @@
import logging
import re
import os
import warnings
from contextlib import suppress
from functools import lru_cache
from io import BytesIO
from os import fspath
from shutil import copy
from subprocess import PIPE, STDOUT, run
from tempfile import NamedTemporaryFile
from pathlib import Path
from subprocess import PIPE, CalledProcessError
from shutil import which
from PIL import Image
from ..exceptions import SubprocessOutputError
from . import get_version
from ..exceptions import SubprocessOutputError, MissingDependencyError
from . import get_version, run
gslog = logging.getLogger()
GS = 'gs'
if os.name == 'nt':
GS = which('gswin64c')
if not GS:
GS = which('gswin32c')
if not GS:
raise MissingDependencyError("Ghostscript (gswin64c or gswin32c)")
GS = Path(GS).stem
@lru_cache(maxsize=1)
def version():
return get_version('gs')
return get_version(GS)
def jpeg_passthrough_available():
@@ -83,7 +95,7 @@ def extract_text(input_file, pageno=1):
args_gs = (
[
'gs',
GS,
'-dQUIET',
'-dSAFER',
'-dBATCH',
@@ -92,14 +104,15 @@ def extract_text(input_file, pageno=1):
'-dTextFormat=0',
]
+ pages
+ ['-o', '-', fspath(input_file)]
+ ['-o', '-', fspath(input_file), "-sstdout=%stderr"]
)
p = run(args_gs, stdout=PIPE, stderr=PIPE)
if p.returncode != 0:
try:
p = run(args_gs, stdout=PIPE, stderr=PIPE, check=True)
except CalledProcessError as e:
raise SubprocessOutputError(
'Ghostscript text extraction failed\n%s\n%s\n%s'
% (input_file, p.stdout.decode(), p.stderr.decode())
'Ghostscript text extraction failed\n%s\n%s'
% (input_file, e.stderr.decode(errors='replace'))
)
return p.stdout
@@ -141,54 +154,58 @@ def rasterize_pdf(
if not log:
log = gslog
with NamedTemporaryFile(delete=True) as tmp:
args_gs = (
[
'gs',
'-dQUIET',
'-dSAFER',
'-dBATCH',
'-dNOPAUSE',
f'-sDEVICE={raster_device}',
f'-dFirstPage={pageno}',
f'-dLastPage={pageno}',
f'-r{res[0]:f}x{res[1]:f}',
]
+ (['-dFILTERVECTOR'] if filter_vector else [])
+ [
'-o',
tmp.name,
'-dAutoRotatePages=/None', # Probably has no effect on raster
'-f',
fspath(input_file),
]
)
args_gs = (
[
GS,
'-dQUIET',
'-dSAFER',
'-dBATCH',
'-dNOPAUSE',
f'-sDEVICE={raster_device}',
f'-dFirstPage={pageno}',
f'-dLastPage={pageno}',
f'-r{res[0]:f}x{res[1]:f}',
]
+ (['-dFILTERVECTOR'] if filter_vector else [])
+ [
'-o',
'-',
'-sstdout=%stderr',
'-dAutoRotatePages=/None', # Probably has no effect on raster
'-f',
fspath(input_file),
]
)
log.debug(args_gs)
p = run(args_gs, stdout=PIPE, stderr=STDOUT, universal_newlines=True)
if _gs_error_reported(p.stdout):
log.error(p.stdout)
elif p.stdout:
log.debug(p.stdout)
log.debug(args_gs)
try:
p = run(args_gs, stdout=PIPE, stderr=PIPE, check=True)
except CalledProcessError as e:
with suppress(OSError):
Path(output_file).unlink() # no unfinished files
log.error(e.stderr.decode(errors='replace'))
raise SubprocessOutputError('Ghostscript rasterizing failed')
else:
stderr = p.stderr.decode(errors='replace')
if _gs_error_reported(stderr):
log.error(stderr)
elif stderr:
log.debug(stderr)
if p.returncode != 0:
raise SubprocessOutputError('Ghostscript rasterizing failed')
tmp.seek(0)
with Image.open(tmp) as im:
if rotation is not None:
log.debug("Rotating output by %i", rotation)
# rotation is a clockwise angle and Image.ROTATE_* is
# counterclockwise so this cancels out the rotation
if rotation == 90:
im = im.transpose(Image.ROTATE_90)
elif rotation == 180:
im = im.transpose(Image.ROTATE_180)
elif rotation == 270:
im = im.transpose(Image.ROTATE_270)
if rotation % 180 == 90:
page_dpi = page_dpi[1], page_dpi[0]
im.save(fspath(output_file), dpi=page_dpi)
with Image.open(BytesIO(p.stdout)) as im:
if rotation is not None:
log.debug("Rotating output by %i", rotation)
# rotation is a clockwise angle and Image.ROTATE_* is
# counterclockwise so this cancels out the rotation
if rotation == 90:
im = im.transpose(Image.ROTATE_90)
elif rotation == 180:
im = im.transpose(Image.ROTATE_180)
elif rotation == 270:
im = im.transpose(Image.ROTATE_270)
if rotation % 180 == 90:
page_dpi = page_dpi[1], page_dpi[0]
im.save(fspath(output_file), dpi=page_dpi)
def generate_pdfa(
@@ -256,37 +273,48 @@ def generate_pdfa(
# https://bugs.ghostscript.com/show_bug.cgi?id=699216
compression_args.append('-dPassThroughJPEGImages=false')
with NamedTemporaryFile(delete=True) as gs_pdf:
# nb no need to specify ProcessColorModel when ColorConversionStrategy
# is set; see:
# https://bugs.ghostscript.com/show_bug.cgi?id=699392
args_gs = (
[
"gs",
"-dQUIET",
"-dBATCH",
"-dNOPAUSE",
"-dSAFER",
"-dCompatibilityLevel=" + str(pdf_version),
"-sDEVICE=pdfwrite",
"-dAutoRotatePages=/None",
"-sColorConversionStrategy=" + strategy,
]
+ compression_args
+ [
"-dJPEGQ=95",
"-dPDFA=" + pdfa_part,
"-dPDFACompatibilityPolicy=1",
"-sOutputFile=" + gs_pdf.name,
]
)
args_gs.extend(fspath(s) for s in pdf_pages) # Stringify Path objs
log.debug(args_gs)
p = run(args_gs, stdout=PIPE, stderr=STDOUT, universal_newlines=True)
if _gs_error_reported(p.stdout):
log.error(p.stdout)
elif 'overprint mode not set' in p.stdout:
# nb no need to specify ProcessColorModel when ColorConversionStrategy
# is set; see:
# https://bugs.ghostscript.com/show_bug.cgi?id=699392
args_gs = (
[
GS,
"-dQUIET",
"-dBATCH",
"-dNOPAUSE",
"-dSAFER",
"-dCompatibilityLevel=" + str(pdf_version),
"-sDEVICE=pdfwrite",
"-dAutoRotatePages=/None",
"-sColorConversionStrategy=" + strategy,
]
+ compression_args
+ [
"-dJPEGQ=95",
"-dPDFA=" + pdfa_part,
"-dPDFACompatibilityPolicy=1",
"-o",
"-",
"-sstdout=%stderr",
]
)
args_gs.extend(fspath(s) for s in pdf_pages) # Stringify Path objs
log.debug(args_gs)
try:
with Path(output_file).open('wb') as output:
p = run(args_gs, stdout=output, stderr=PIPE, check=True)
except CalledProcessError as e:
# Ghostscript does not change return code when it fails to create
# PDF/A - check PDF/A status elsewhere
with suppress(OSError):
Path(output_file).unlink()
log.error(e.stderr.decode(errors='replace'))
raise SubprocessOutputError('Ghostscript PDF/A rendering failed')
else:
stderr = p.stderr.decode('utf-8', errors='replace')
if _gs_error_reported(stderr):
log.error(stderr)
elif 'overprint mode not set' in stderr:
# Unless someone is going to print PDF/A documents on a
# magical sRGB printer I can't see the removal of overprinting
# being a problem....
@@ -295,11 +323,4 @@ def generate_pdfa(
"input file to complete PDF/A conversion. "
)
else:
log.debug(p.stdout)
if p.returncode == 0:
# Ghostscript does not change return code when it fails to create
# PDF/A - check PDF/A status elsewhere
copy(gs_pdf.name, fspath(output_file))
else:
raise SubprocessOutputError('Ghostscript PDF/A rendering failed')
log.debug(stderr)
src/ocrmypdf/exec/jbig2enc.py
+2 -2
View File
@@ -18,10 +18,10 @@
"""Interface to jbig2 executable"""
from functools import lru_cache
from subprocess import PIPE, run
from subprocess import PIPE
from ..exceptions import MissingDependencyError
from . import get_version
from . import get_version, run
@lru_cache(maxsize=1)
src/ocrmypdf/exec/qpdf.py
+2 -2
View File
@@ -19,9 +19,9 @@
from functools import lru_cache
from os import fspath
from subprocess import PIPE, STDOUT, CalledProcessError, run
from subprocess import PIPE, STDOUT, CalledProcessError
from . import get_version
from . import get_version, run
@lru_cache(maxsize=1)
src/ocrmypdf/exec/tesseract.py
+8 -12
View File
@@ -23,15 +23,15 @@ from collections import namedtuple
from contextlib import suppress
import logging
from os import fspath
from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired, run
from subprocess import PIPE, STDOUT, CalledProcessError, TimeoutExpired
from ..exceptions import (
MissingDependencyError,
SubprocessOutputError,
TesseractConfigError,
)
from ..helpers import page_number
from . import get_version
from ..helpers import page_number, safe_symlink
from . import get_version, run
OrientationConfidence = namedtuple('OrientationConfidence', ('angle', 'confidence'))
@@ -128,9 +128,10 @@ def languages(tesseract_env=None):
except CalledProcessError as e:
raise MissingDependencyError(lang_error(e.output)) from e
for line in output.splitlines():
if line.startswith('Error'):
raise MissingDependencyError(lang_error(output))
header, *rest = output.splitlines()
if not header.startswith('List of available languages'):
raise MissingDependencyError(lang_error(output))
return set(lang.strip() for lang in rest)
@@ -194,12 +195,7 @@ def tesseract_log_output(mainlog, stdout, input_file):
try:
text = stdout.decode()
except UnicodeDecodeError:
log.error(
"Tesseract's output was not utf-8. "
"This usually means Tesseract's language packs do not match "
"the installed version of Tesseract."
)
text = stdout.decode('utf-8', 'backslashreplace')
text = stdout.decode('utf-8', 'ignore')
lines = text.splitlines()
for line in lines:
@@ -325,7 +321,7 @@ def use_skip_page(text_only, skip_pdf, output_pdf, output_text):
# Substitute a "skipped page"
with suppress(FileNotFoundError):
os.remove(output_pdf) # In case it was partially created
os.symlink(skip_pdf, output_pdf)
safe_symlink(skip_pdf, output_pdf)
return
# Or normally, just write a 0 byte file to the output to indicate a skip
src/ocrmypdf/exec/unpaper.py
+2 -3
View File
@@ -22,7 +22,6 @@
import os
import shlex
import subprocess
from functools import lru_cache
from subprocess import PIPE, STDOUT, CalledProcessError
from tempfile import TemporaryDirectory
@@ -30,7 +29,7 @@ from tempfile import TemporaryDirectory
from PIL import Image
from ..exceptions import MissingDependencyError, SubprocessOutputError
from . import get_version
from . import get_version, run as external_run
@lru_cache(maxsize=1)
@@ -77,7 +76,7 @@ def run(input_file, output_file, dpi, log, mode_args):
# their unpaper arguments (whether intentionally or otherwise)
args_unpaper.extend([input_pnm, output_pnm])
try:
proc = subprocess.run(
proc = external_run(
args_unpaper,
check=True,
close_fds=True,
src/ocrmypdf/helpers.py
+9 -3
View File
@@ -18,6 +18,7 @@
import logging
import multiprocessing
import os
import shutil
import warnings
from collections.abc import Iterable
from contextlib import suppress
@@ -27,14 +28,14 @@ from pathlib import Path
log = logging.getLogger(__name__)
def re_symlink(input_file, soft_link_name, *args, **kwargs):
def safe_symlink(input_file, soft_link_name, *args, **kwargs):
"""
Helper function: relinks soft symbolic link if necessary
"""
if len(args) == 1 and isinstance(args[0], logging.Logger):
log.warning("Deprecated: re_symlink(,log)")
log.warning("Deprecated: safe_symlink(,log)")
if 'log' in kwargs:
log.warning('Deprecated: re_symlink(...log=)')
log.warning('Deprecated: safe_symlink(...log=)')
input_file = os.fspath(input_file)
soft_link_name = os.fspath(soft_link_name)
@@ -60,6 +61,11 @@ def re_symlink(input_file, soft_link_name, *args, **kwargs):
if not os.path.exists(input_file):
raise FileNotFoundError(f"trying to create a broken symlink to {input_file}")
if os.name == 'nt':
# Don't actually use symlinks on Windows due to permission issues
shutil.copyfile(input_file, soft_link_name)
return
log.debug("os.symlink(%s, %s)", input_file, soft_link_name)
# Create symbolic link using absolute path
src/ocrmypdf/leptonica.py
+34 -17
View File
@@ -34,12 +34,22 @@ from os import fspath
from tempfile import TemporaryFile
from .lib._leptonica import ffi
from .exceptions import MissingDependencyError
# pylint: disable=protected-access
logger = logging.getLogger(__name__)
lept = ffi.dlopen(find_library('lept'))
if os.name == 'nt':
libname = 'liblept-5'
else:
libname = 'lept'
_libpath = find_library(libname)
if not _libpath and os.name == 'nt':
raise MissingDependencyError(
"Please ensure that 'tesseract' is on your PATH environment variable. "
)
lept = ffi.dlopen(_libpath)
lept.setMsgSeverity(lept.L_SEVERITY_WARNING)
@@ -292,9 +302,11 @@ class Pix(LeptonicaObject):
Leptonica can load TIFF, PNM (PBM, PGM, PPM), PNG, and JPEG. If
loading fails then the object will wrap a C null pointer.
"""
filename = fspath(path)
with _LeptonicaErrorTrap():
return cls(lept.pixRead(os.fsencode(filename)))
with open(path, 'rb') as py_file:
data = py_file.read()
buffer = ffi.from_buffer(data)
with _LeptonicaErrorTrap():
return cls(lept.pixReadMem(buffer, len(buffer)))
def write_implied_format(self, path, jpeg_quality=0, jpeg_progressive=0):
"""Write pix to the filename, with the extension indicating format.
@@ -302,11 +314,19 @@ class Pix(LeptonicaObject):
jpeg_quality -- quality (iff JPEG; 1 - 100, 0 for default)
jpeg_progressive -- (iff JPEG; 0 for baseline seq., 1 for progressive)
"""
filename = fspath(path)
with _LeptonicaErrorTrap():
lept.pixWriteImpliedFormat(
os.fsencode(filename), self._cdata, jpeg_quality, jpeg_progressive
)
lept_format = lept.getImpliedFileFormat(os.fsencode(path))
with open(path, 'wb') as py_file:
data = ffi.new('l_uint8 **pdata')
size = ffi.new('size_t *psize')
with _LeptonicaErrorTrap():
if lept_format == lept.L_JPEG_ENCODE:
lept.pixWriteMemJpeg(
data, size, self._cdata, jpeg_quality, jpeg_progressive
)
else:
lept.pixWriteMem(data, size, self._cdata, lept_format)
buffer = ffi.buffer(data[0], size[0])
py_file.write(buffer)
@classmethod
def frompil(self, pillow_image):
@@ -502,17 +522,14 @@ class Pix(LeptonicaObject):
display=0,
pdfdir=ffi.NULL,
):
if get_leptonica_version() < 'leptonica-1.76':
# Leptonica 1.76 changed the API for pixFindPageForeground; we don't
# support the old version
raise LeptonicaError("Not available in this version of Leptonica")
with _LeptonicaErrorTrap():
cropbox = Box(
lept.pixFindPageForeground(
self._cdata,
threshold,
mindist,
erasedist,
pagenum,
showmorph,
display,
pdfdir,
self._cdata, threshold, mindist, erasedist, showmorph, ffi.NULL
)
)
src/ocrmypdf/lib/_leptonica.py
+5 -5
View File
File diff suppressed because one or more lines are too long
src/ocrmypdf/lib/compile_leptonica.py
+34 -9
View File
@@ -16,6 +16,8 @@
# You should have received a copy of the GNU General Public License
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
from pathlib import Path
from cffi import FFI
ffibuilder = FFI()
@@ -74,6 +76,17 @@ struct Pixa
};
typedef struct Pixa PIXA;
/*! Array of compressed pix */
struct PixaComp
{
l_int32 n; /*!< number of PixComp in ptr array */
l_int32 nalloc; /*!< number of PixComp ptrs allocated */
l_int32 offset; /*!< indexing offset into ptr array */
struct PixComp **pixc; /*!< the array of ptrs to PixComp */
struct Boxa *boxa; /*!< array of boxes */
};
typedef struct PixaComp PIXAC;
struct Box
{
l_int32 x;
@@ -210,9 +223,15 @@ ffibuilder.cdef(
"""
PIX * pixRead ( const char *filename );
PIX * pixReadMem ( const l_uint8 *data, size_t size );
PIX * pixReadStream ( FILE *fp, l_int32 hint );
PIX * pixScale ( PIX *pixs, l_float32 scalex, l_float32 scaley );
l_int32 pixFindSkew ( PIX *pixs, l_float32 *pangle, l_float32 *pconf );
l_int32 pixWriteImpliedFormat ( const char *filename, PIX *pix, l_int32 quality, l_int32 progressive );
l_int32 getImpliedFileFormat ( const char *filename );
l_ok pixWriteStream ( FILE *fp, PIX *pix, l_int32 format );
l_ok pixWriteStreamJpeg ( FILE *fp, PIX *pixs, l_int32 quality, l_int32 progressive );
l_ok pixWriteMem ( l_uint8 **pdata, size_t *psize, PIX *pix, l_int32 format );
l_ok pixWriteMemJpeg ( l_uint8 **pdata, size_t *psize, PIX *pix, l_int32 quality, l_int32 progressive );
l_int32
pixWriteMemPng(l_uint8 **pdata,
size_t *psize,
@@ -294,14 +313,12 @@ pixCleanBackgroundToWhite(PIX *pixs,
l_int32 whiteval);
BOX *
pixFindPageForeground(PIX *pixs,
l_int32 threshold,
l_int32 mindist,
l_int32 erasedist,
l_int32 pagenum,
l_int32 showmorph,
l_int32 display,
const char *pdfdir);
pixFindPageForeground ( PIX *pixs,
l_int32 threshold,
l_int32 mindist,
l_int32 erasedist,
l_int32 showmorph,
PIXAC *pixac );
PIX *
pixClipRectangle(PIX *pixs,
@@ -414,7 +431,10 @@ pixExtractBarcodes(PIX *pixs,
l_int32 debugflag);
BOXA *
pixLocateBarcodes ( PIX *pixs, l_int32 thresh, PIX **ppixb, PIX **ppixm );
pixLocateBarcodes ( PIX *pixs,
l_int32 thresh,
PIX **ppixb,
PIX **ppixm );
SARRAY *
pixReadBarcodes(PIXA *pixa,
@@ -491,3 +511,8 @@ ffibuilder.set_source("ocrmypdf.lib._leptonica", None)
if __name__ == '__main__':
ffibuilder.compile(verbose=True)
if Path('ocrmypdf/lib/_leptonica.py').exists() and Path('src/ocrmypdf').exists():
output = Path('ocrmypdf/lib/_leptonica.py')
output.rename('src/ocrmypdf/lib/_leptonica.py')
Path('ocrmypdf/lib').rmdir()
Path('ocrmypdf').rmdir()
src/ocrmypdf/optimize.py
+3 -3
View File
@@ -31,7 +31,7 @@ from . import leptonica
from ._jobcontext import PDFContext
from .exceptions import OutputFileAccessError
from .exec import jbig2enc, pngquant
from .helpers import re_symlink
from .helpers import safe_symlink
DEFAULT_JPEG_QUALITY = 75
DEFAULT_PNG_QUALITY = 70
@@ -492,7 +492,7 @@ def optimize(input_file, output_file, context, save_settings):
log = context.log
options = context.options
if options.optimize == 0:
re_symlink(input_file, output_file)
safe_symlink(input_file, output_file)
return
if options.jpeg_quality == 0:
@@ -538,7 +538,7 @@ def optimize(input_file, output_file, context, save_settings):
pike.remove_unreferenced_resources()
pike.save(output_file, **save_settings)
else:
re_symlink(target_file, output_file)
safe_symlink(target_file, output_file)
def main(infile, outfile, level, jobs=1):
src/ocrmypdf/pdfinfo/ghosttext.py
+1
View File
@@ -96,6 +96,7 @@ def extract_text_xml(infile, pdf, pageno=None, log=gslog):
page_count_difference = len(pdf.pages) - len(page_xml)
if page_count_difference != 0:
log.error("The number of pages in the input file is inconsistent.")
log.error(f"Expected {len(pdf.pages)}, txtwrite says {len(page_xml)}")
if page_count_difference > 0:
page_xml.extend([None] * page_count_difference)
return page_xml
tests/conftest.py
+54 -31
View File
@@ -15,10 +15,10 @@
# You should have received a copy of the GNU General Public License
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import ast
import os
import platform
import sys
from contextlib import contextmanager
from pathlib import Path
from subprocess import PIPE, run
from ocrmypdf import api, cli
@@ -81,6 +81,23 @@ PROJECT_ROOT = os.path.dirname(TESTS_ROOT)
OCRMYPDF = [sys.executable, '-m', 'ocrmypdf']
WINDOWS_SHIM_TEMPLATE = """
# This is a shim for Windows that has the same effect as a symlink to the target .py
# file
import os
import subprocess
import sys
args = [sys.executable, {spoofer}, *sys.argv[1:]]
p = subprocess.run(args, check=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
sys.stdout.buffer.write(p.stdout)
sys.stderr.buffer.write(p.stderr)
sys.exit(p.returncode)
"""
assert ast.parse(WINDOWS_SHIM_TEMPLATE.format(spoofer=repr(r"C:\\Temp\\file.py")))
@pytest.helpers.register
def spoof(tmp_path_factory, **kwargs):
"""Modify PATH to override subprocess executables
@@ -98,39 +115,27 @@ def spoof(tmp_path_factory, **kwargs):
for replace_program, with_spoof in kwargs.items():
spoofer = Path(SPOOF_PATH) / with_spoof
spoofer.chmod(0o755)
(tmpdir / replace_program).symlink_to(spoofer)
env['_OCRMYPDF_SAVE_PATH'] = env['PATH']
env['PATH'] = str(tmpdir) + ":" + env['PATH']
if os.name != 'nt':
spoofer.chmod(0o755)
(tmpdir / replace_program).symlink_to(spoofer)
else:
py_file = WINDOWS_SHIM_TEMPLATE.format(
spoofer=repr(os.fspath(spoofer.absolute()))
)
if replace_program == 'gs':
programs = ['gswin64c', 'gswin32c']
else:
programs = [replace_program]
for prog in programs:
(tmpdir / f'{prog}.py').write_text(py_file, encoding='utf-8')
env['_OCRMYPDF_TEST_PATH'] = str(tmpdir) + os.pathsep + env['PATH']
if os.name == 'nt':
if '.py' not in env['PATHEXT'].lower():
raise EnvironmentError("PATHEXT is not configured to support .py")
return env
@pytest.helpers.register
@contextmanager
def os_environ(new_env):
old_env = os.environ.copy()
if new_env is None:
new_env = {}
for k, v in new_env.items():
if k != 'PYTEST_CURRENT_TEST':
os.environ[k] = v
yield
new_keys = set(os.environ.copy()) - set(old_env)
for k in new_keys:
if k != 'PYTEST_CURRENT_TEST':
del os.environ[k]
for k in old_env:
if k != 'PYTEST_CURRENT_TEST':
os.environ[k] = old_env[k]
for k, v in os.environ.copy().items():
if k != 'PYTEST_CURRENT_TEST':
assert v == old_env[k]
@pytest.fixture(scope='session')
def spoof_tesseract_noop(tmp_path_factory):
return spoof(tmp_path_factory, tesseract='tesseract_noop.py')
@@ -183,7 +188,7 @@ def check_ocrmypdf(input_file, output_file, *args, env=None):
api.check_options(options)
if env:
options.tesseract_env = env
options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = input_file
options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = os.fspath(input_file)
result = api.run_pipeline(options, api=True)
assert result == 0
@@ -193,6 +198,24 @@ def check_ocrmypdf(input_file, output_file, *args, env=None):
return output_file
@pytest.helpers.register
def run_ocrmypdf_api(input_file, output_file, *args, env=None):
"Run ocrmypdf and let caller deal with results"
options = cli.parser.parse_args(
[str(input_file), str(output_file)]
+ [str(arg) for arg in args if arg is not None]
)
api.check_options(options)
if env:
options.tesseract_env = env.copy()
options.tesseract_env['_OCRMYPDF_TEST_INFILE'] = os.fspath(input_file)
if options.tesseract_env:
assert all(isinstance(v, (str, bytes)) for v in options.tesseract_env.values())
return api.run_pipeline(options, api=False)
@pytest.helpers.register
def run_ocrmypdf(input_file, output_file, *args, env=None, universal_newlines=True):
"Run ocrmypdf and let caller deal with results"
tests/spoof/gs.py
+40
View File
@@ -0,0 +1,40 @@
# © 2019 James R. Barlow: github.com/jbarlow83
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""Find Ghostscript executable"""
import os
import shutil
def real_ghostscript(argv):
if os.name != 'nt':
gs = shutil.which('gs')
gs_args = [gs] + argv[1:]
os.execv(gs_args[0], gs_args)
else:
gs = shutil.which('gswin64c')
if not gs:
gs = shutil.which('gswin32c')
os.execv(gs, argv[1:])
return # Not reachable
tests/spoof/gs_feature_elision.py
+1 -5
View File
@@ -30,10 +30,7 @@ from subprocess import check_call
PDF/A creation."""
def real_ghostscript(argv):
gs_args = ['gs'] + argv[1:]
os.execvp("gs", gs_args)
return # Not reachable
from gs import real_ghostscript
elision_warning = """GPL Ghostscript 9.20: Setting Overprint Mode to 1
@@ -41,7 +38,6 @@ not permitted in PDF/A-2, overprint mode not set"""
def main():
os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH']
if '--version' in sys.argv:
print('9.20')
print('SPOOFED: ' + os.path.basename(__file__))
tests/spoof/gs_pdfa_failure.py
+1 -6
View File
@@ -27,15 +27,10 @@ import sys
"""Replicate Ghostscript PDF/A conversion failure by suppressing some
arguments"""
def real_ghostscript(argv):
gs_args = ['gs'] + argv[1:]
os.execvp("gs", gs_args)
return # Not reachable
from gs import real_ghostscript
def main():
os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH']
if '--version' in sys.argv:
print('9.20')
print('SPOOFED: ' + os.path.basename(__file__))
tests/spoof/gs_raster_failure.py
+1 -5
View File
@@ -27,14 +27,10 @@ import sys
"""Replicate Ghostscript raster failure while allowing rendering"""
def real_ghostscript(argv):
gs_args = ['gs'] + argv[1:]
os.execvp("gs", gs_args)
return # Not reachable
from gs import real_ghostscript
def main():
os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH']
if '--version' in sys.argv:
print('9.20')
print('SPOOFED: ' + os.path.basename(__file__))
tests/spoof/gs_render_failure.py
+1 -5
View File
@@ -26,14 +26,10 @@ import os
import sys
def real_ghostscript(argv):
gs_args = ['gs'] + argv[1:]
os.execvp("gs", gs_args)
return # Not reachable
from gs import real_ghostscript
def main():
os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH']
if '--version' in sys.argv:
print('9.20')
print('SPOOFED: ' + os.path.basename(__file__))
tests/spoof/tesseract_cache.py
-2
View File
@@ -59,8 +59,6 @@ import subprocess
import sys
from pathlib import Path
if '_OCRMYPDF_SAVE_PATH' in os.environ:
os.environ['PATH'] = os.environ['_OCRMYPDF_SAVE_PATH']
__version__ = subprocess.check_output(
['tesseract', '--version'], stderr=subprocess.STDOUT
tests/test_ghostscript.py
+64
View File
@@ -18,12 +18,47 @@
import logging
from decimal import Decimal
import pikepdf
import pytest
from PIL import Image
from ocrmypdf.exceptions import ExitCode
from ocrmypdf.exec.ghostscript import rasterize_pdf
check_ocrmypdf = pytest.helpers.check_ocrmypdf
run_ocrmypdf = pytest.helpers.run_ocrmypdf
run_ocrmypdf_api = pytest.helpers.run_ocrmypdf_api
spoof = pytest.helpers.spoof
@pytest.fixture(scope='session')
def spoof_no_tess_gs_render_fail(tmp_path_factory):
return spoof(
tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_render_failure.py'
)
@pytest.fixture(scope='session')
def spoof_no_tess_gs_raster_fail(tmp_path_factory):
return spoof(
tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_raster_failure.py'
)
@pytest.fixture(scope='session')
def spoof_no_tess_no_pdfa(tmp_path_factory):
return spoof(
tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_pdfa_failure.py'
)
@pytest.fixture(scope='session')
def spoof_no_tess_pdfa_warning(tmp_path_factory):
return spoof(
tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_feature_elision.py'
)
@pytest.fixture
def linn(resources):
@@ -79,3 +114,32 @@ def test_rasterize_rotated(linn, outdir, caplog):
with Image.open(outdir / 'out.png') as im:
assert im.size == (target_size[1], target_size[0])
assert im.info['dpi'] == (target_dpi[1], target_dpi[0])
def test_gs_render_failure(spoof_no_tess_gs_render_fail, resources, outpdf):
p, out, err = run_ocrmypdf(
resources / 'blank.pdf', outpdf, env=spoof_no_tess_gs_render_fail
)
print(err)
assert p.returncode == ExitCode.child_process_error
def test_gs_raster_failure(spoof_no_tess_gs_raster_fail, resources, outpdf):
p, out, err = run_ocrmypdf(
resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_gs_raster_fail
)
print(err)
assert p.returncode == ExitCode.child_process_error
def test_ghostscript_pdfa_failure(spoof_no_tess_no_pdfa, resources, outpdf):
p, out, err = run_ocrmypdf(
resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_no_pdfa
)
assert (
p.returncode == ExitCode.pdfa_conversion_failed
), "Unexpected return when PDF/A fails"
def test_ghostscript_feature_elision(spoof_no_tess_pdfa_warning, resources, outpdf):
check_ocrmypdf(resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_pdfa_warning)
tests/test_graft.py
+2 -5
View File
@@ -16,14 +16,13 @@
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
import os
from unittest.mock import patch
import pytest
import ocrmypdf
import pikepdf
os_environ = pytest.helpers.os_environ
def test_no_glyphless_graft(resources, outdir):
pdf = pikepdf.open(resources / 'francais.pdf')
@@ -33,9 +32,7 @@ def test_no_glyphless_graft(resources, outdir):
pdf.pages.extend(pdf_cmyk.pages)
pdf.save(outdir / 'test.pdf')
env = os.environ.copy()
env['_OCRMYPDF_MAX_REPLACE_PAGES'] = '2'
with os_environ(env):
with patch('ocrmypdf._graft.MAX_REPLACE_PAGES', 2):
ocrmypdf.ocr(
outdir / 'test.pdf', outdir / 'out.pdf', deskew=True, tesseract_timeout=0
)
tests/test_lept.py
+7 -13
View File
@@ -63,6 +63,10 @@ def test_pix_otsu(crom_pix):
assert im1bpp.mode == '1'
@pytest.mark.skipif(
lept.get_leptonica_version() < 'leptonica-1.76',
reason="needs new leptonica for API change",
)
def test_crop(resources):
pix = lept.Pix.open(resources / 'linn.png')
foreground = pix.crop_to_foreground()
@@ -90,16 +94,6 @@ def test_leptonica_compile(tmp_path):
ffibuilder.compile(tmpdir=fspath(tmp_path), target=fspath(tmp_path / 'lepttest.*'))
def test_with_stderr(capsys):
# pytest redirects stderr too; we must disable this for the test to be valid
with capsys.disabled():
with pytest.raises(FileNotFoundError):
lept.Pix.open("does_not_exist1")
def test_without_stderr(capsys):
# pytest redirects stderr too; we must disable this for the test to be valid
with capsys.disabled():
with patch('sys.stderr', new=None):
with pytest.raises(FileNotFoundError):
lept.Pix.open("does_not_exist2")
def test_file_not_found():
with pytest.raises(FileNotFoundError):
lept.Pix.open("does_not_exist1")
tests/test_main.py
+55 -121
View File
@@ -39,6 +39,7 @@ from ocrmypdf.pdfinfo import Colorspace, Encoding, PdfInfo
check_ocrmypdf = pytest.helpers.check_ocrmypdf
run_ocrmypdf = pytest.helpers.run_ocrmypdf
run_ocrmypdf_api = pytest.helpers.run_ocrmypdf_api
spoof = pytest.helpers.spoof
@@ -55,34 +56,6 @@ def spoof_tesseract_big_image_error(tmp_path_factory):
return spoof(tmp_path_factory, tesseract='tesseract_big_image_error.py')
@pytest.fixture(scope='session')
def spoof_no_tess_no_pdfa(tmp_path_factory):
return spoof(
tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_pdfa_failure.py'
)
@pytest.fixture(scope='session')
def spoof_no_tess_pdfa_warning(tmp_path_factory):
return spoof(
tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_feature_elision.py'
)
@pytest.fixture(scope='session')
def spoof_no_tess_gs_render_fail(tmp_path_factory):
return spoof(
tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_render_failure.py'
)
@pytest.fixture(scope='session')
def spoof_no_tess_gs_raster_fail(tmp_path_factory):
return spoof(
tmp_path_factory, tesseract='tesseract_noop.py', gs='gs_raster_failure.py'
)
def test_quick(spoof_tesseract_cache, resources, outpdf):
check_ocrmypdf(resources / 'ccitt.pdf', outpdf, env=spoof_tesseract_cache)
@@ -197,8 +170,8 @@ def test_oversample(spoof_tesseract_cache, renderer, resources, outpdf):
def test_repeat_ocr(resources, no_outpdf):
p, _, _ = run_ocrmypdf(resources / 'graph_ocred.pdf', no_outpdf)
assert p.returncode != 0
result = run_ocrmypdf_api(resources / 'graph_ocred.pdf', no_outpdf)
assert result == ExitCode.already_done_ocr
def test_force_ocr(spoof_tesseract_cache, resources, outpdf):
@@ -300,34 +273,34 @@ def test_maximum_options(
)
def test_tesseract_missing_tessdata(resources, no_outpdf):
def test_tesseract_missing_tessdata(resources, no_outpdf, tmpdir):
env = os.environ.copy()
env['TESSDATA_PREFIX'] = '/tmp'
env['TESSDATA_PREFIX'] = os.fspath(tmpdir)
p, _, err = run_ocrmypdf(
resources / 'graph_ocred.pdf', no_outpdf, '-v', '1', '--skip-text', env=env
returncode = run_ocrmypdf_api(
resources / 'graph.pdf', no_outpdf, '-v', '1', '--skip-text', env=env
)
assert p.returncode == ExitCode.missing_dependency, err
assert returncode == ExitCode.missing_dependency
def test_invalid_input_pdf(resources, no_outpdf):
p, out, err = run_ocrmypdf(resources / 'invalid.pdf', no_outpdf)
assert p.returncode == ExitCode.input_file, err
result = run_ocrmypdf_api(resources / 'invalid.pdf', no_outpdf)
assert result == ExitCode.input_file
def test_blank_input_pdf(resources, outpdf):
p, out, err = run_ocrmypdf(resources / 'blank.pdf', outpdf)
assert p.returncode == ExitCode.ok
result = run_ocrmypdf_api(resources / 'blank.pdf', outpdf)
assert result == ExitCode.ok
def test_force_ocr_on_pdf_with_no_images(spoof_tesseract_crash, resources, no_outpdf):
# As a correctness test, make sure that --force-ocr on a PDF with no
# content still triggers tesseract. If tesseract crashes, then it was
# called.
p, _, err = run_ocrmypdf(
p, _, _ = run_ocrmypdf(
resources / 'blank.pdf', no_outpdf, '--force-ocr', env=spoof_tesseract_crash
)
assert p.returncode == ExitCode.child_process_error, err
assert p.returncode == ExitCode.child_process_error
assert not os.path.exists(no_outpdf)
@@ -340,27 +313,29 @@ def test_german(spoof_tesseract_cache, resources, outdir):
# properly. It is fine that we are testing -l deu on a French file because
# we are exercising the functionality not going for accuracy.
sidecar = outdir / 'francais.txt'
p, out, err = run_ocrmypdf(
resources / 'francais.pdf',
outdir / 'francais.pdf',
'-l',
'deu', # more commonly installed
'--sidecar',
sidecar,
env=spoof_tesseract_cache,
)
if 'deu' not in tesseract.languages():
pytest.xfail(reason="tesseract-deu language pack not installed")
assert p.returncode == ExitCode.ok, "Requires tesseract deu language pack"
try:
check_ocrmypdf(
resources / 'francais.pdf',
outdir / 'francais.pdf',
'-l',
'deu', # more commonly installed
'--sidecar',
sidecar,
env=spoof_tesseract_cache,
)
except MissingDependencyError:
if 'deu' not in tesseract.languages():
pytest.xfail(reason="tesseract-deu language pack not installed")
raise
def test_klingon(resources, outpdf):
p, out, err = run_ocrmypdf(resources / 'francais.pdf', outpdf, '-l', 'klz')
p, _, _ = run_ocrmypdf(resources / 'francais.pdf', outpdf, '-l', 'klz')
assert p.returncode == ExitCode.missing_dependency
def test_missing_docinfo(spoof_tesseract_noop, resources, outpdf):
p, out, err = run_ocrmypdf(
result = run_ocrmypdf_api(
resources / 'missing_docinfo.pdf',
outpdf,
'-l',
@@ -368,7 +343,7 @@ def test_missing_docinfo(spoof_tesseract_noop, resources, outpdf):
'--skip-text',
env=spoof_tesseract_noop,
)
assert p.returncode == ExitCode.ok, err
assert result == ExitCode.ok
def test_uppercase_extension(spoof_tesseract_noop, resources, outdir):
@@ -379,24 +354,25 @@ def test_uppercase_extension(spoof_tesseract_noop, resources, outdir):
)
def test_input_file_not_found(no_outpdf):
def test_input_file_not_found(caplog, no_outpdf):
input_file = "does not exist.pdf"
p, out, err = run_ocrmypdf(input_file, no_outpdf)
assert p.returncode == ExitCode.input_file
assert input_file in out or input_file in err
result = run_ocrmypdf_api(input_file, no_outpdf)
assert result == ExitCode.input_file
assert input_file in caplog.text
def test_input_file_not_a_pdf(no_outpdf):
def test_input_file_not_a_pdf(caplog, no_outpdf):
input_file = __file__ # Try to OCR this file
p, out, err = run_ocrmypdf(input_file, no_outpdf)
assert p.returncode == ExitCode.input_file
assert input_file in out or input_file in err
result = run_ocrmypdf_api(input_file, no_outpdf)
assert result == ExitCode.input_file
if os.name != 'nt': # name will be mangled with \\'s on nt
assert input_file in caplog.text
def test_encrypted(resources, no_outpdf):
p, out, err = run_ocrmypdf(resources / 'skew-encrypted.pdf', no_outpdf)
assert p.returncode == ExitCode.encrypted_pdf
assert out.find('encrypted')
def test_encrypted(resources, caplog, no_outpdf):
result = run_ocrmypdf_api(resources / 'skew-encrypted.pdf', no_outpdf)
assert result == ExitCode.encrypted_pdf
assert 'encryption must be removed' in caplog.text
@pytest.mark.parametrize('renderer', RENDERERS)
@@ -415,8 +391,8 @@ def test_pagesegmode(renderer, spoof_tesseract_cache, resources, outpdf):
@pytest.mark.parametrize('renderer', RENDERERS)
def test_tesseract_crash(renderer, spoof_tesseract_crash, resources, no_outpdf):
p, out, err = run_ocrmypdf(
def test_tesseract_crash(renderer, spoof_tesseract_crash, resources, no_outpdf, caplog):
p, _, err = run_ocrmypdf(
resources / 'ccitt.pdf',
no_outpdf,
'-v',
@@ -427,7 +403,7 @@ def test_tesseract_crash(renderer, spoof_tesseract_crash, resources, no_outpdf):
)
assert p.returncode == ExitCode.child_process_error
assert not os.path.exists(no_outpdf)
assert "ERROR" in err
assert "SubprocessOutputError" in err
def test_tesseract_crash_autorotate(spoof_tesseract_crash, resources, no_outpdf):
@@ -556,19 +532,6 @@ def test_linearized_pdf_and_indirect_object(spoof_tesseract_noop, resources, out
check_ocrmypdf(resources / 'epson.pdf', outpdf, env=spoof_tesseract_noop)
def test_ghostscript_pdfa_failure(spoof_no_tess_no_pdfa, resources, outpdf):
p, out, err = run_ocrmypdf(
resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_no_pdfa
)
assert (
p.returncode == ExitCode.pdfa_conversion_failed
), "Unexpected return when PDF/A fails"
def test_ghostscript_feature_elision(spoof_no_tess_pdfa_warning, resources, outpdf):
check_ocrmypdf(resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_pdfa_warning)
def test_very_high_dpi(spoof_tesseract_cache, resources, outpdf):
"Checks for a Decimal quantize error with high DPI, etc"
check_ocrmypdf(resources / '2400dpi.pdf', outpdf, env=spoof_tesseract_cache)
@@ -586,7 +549,7 @@ def test_overlay(spoof_tesseract_noop, resources, outpdf):
def test_destination_not_writable(spoof_tesseract_noop, resources, outdir):
if os.getuid() == 0 or os.geteuid() == 0:
if os.name != 'nt' and (os.getuid() == 0 or os.geteuid() == 0):
pytest.xfail(reason="root can write to anything")
protected_file = outdir / 'protected.pdf'
protected_file.touch()
@@ -613,23 +576,6 @@ language_model_penalty_non_freq_dict_word 0
)
@pytest.mark.slow # This test sometimes times out in CI
@pytest.mark.parametrize('renderer', RENDERERS)
def test_tesseract_config_notfound(renderer, resources, outdir):
cfg_file = outdir / 'nofile.cfg'
p, out, err = run_ocrmypdf(
resources / 'ccitt.pdf',
outdir / 'out.pdf',
'--pdf-renderer',
renderer,
'--tesseract-config',
cfg_file,
)
assert "Can't open" in err, "No error message about missing config file"
assert p.returncode == ExitCode.ok, err
@pytest.mark.slow # This test sometimes times out in CI
@pytest.mark.parametrize('renderer', RENDERERS)
def test_tesseract_config_invalid(renderer, resources, outdir):
@@ -649,7 +595,10 @@ THIS FILE IS INVALID
'--tesseract-config',
cfg_file,
)
assert "parameter not found" in err.lower(), "No error message"
assert (
"parameter not found" in err.lower()
or "error occurred while parsing" in err.lower()
), "No error message"
assert p.returncode == ExitCode.invalid_config
@@ -717,22 +666,6 @@ def test_skip_big_with_no_images(spoof_tesseract_noop, resources, outpdf):
)
def test_gs_render_failure(spoof_no_tess_gs_render_fail, resources, outpdf):
p, out, err = run_ocrmypdf(
resources / 'blank.pdf', outpdf, env=spoof_no_tess_gs_render_fail
)
print(err)
assert p.returncode == ExitCode.child_process_error
def test_gs_raster_failure(spoof_no_tess_gs_raster_fail, resources, outpdf):
p, out, err = run_ocrmypdf(
resources / 'ccitt.pdf', outpdf, env=spoof_no_tess_gs_raster_fail
)
print(err)
assert p.returncode == ExitCode.child_process_error
@pytest.mark.skipif(
'8.0.0' <= qpdf.version() <= '8.0.1',
reason="qpdf regression on pages with no contents",
@@ -872,7 +805,7 @@ def test_sidecar_pagecount(spoof_tesseract_cache, resources, outpdf):
pdfinfo = PdfInfo(resources / 'multipage.pdf')
num_pages = len(pdfinfo)
with open(sidecar, 'r') as f:
with open(sidecar, 'r', encoding='utf-8') as f:
ocr_text = f.read()
# There should a formfeed between each pair of pages, so the count of
@@ -888,7 +821,7 @@ def test_sidecar_nonempty(spoof_tesseract_cache, resources, outpdf):
resources / 'ccitt.pdf', outpdf, '--sidecar', sidecar, env=spoof_tesseract_cache
)
with open(sidecar, 'r') as f:
with open(sidecar, 'r', encoding='utf-8') as f:
ocr_text = f.read()
assert 'the' in ocr_text
@@ -946,6 +879,7 @@ def test_output_is_dir(spoof_tesseract_noop, resources, outdir):
assert 'is not a writable file' in err
@pytest.mark.skipif(os.name == 'nt', reason="symlink needs admin permissions")
def test_output_is_symlink(spoof_tesseract_noop, resources, outdir):
sym = Path(outdir / 'this_is_a_symlink')
sym.symlink_to(outdir / 'out.pdf')
tests/test_metadata.py
+3 -4
View File
@@ -21,6 +21,7 @@ from datetime import timezone
import logging
import mmap
from os import fspath
import os
from pathlib import Path
from shutil import copyfile, move
from unittest.mock import MagicMock, patch
@@ -330,10 +331,8 @@ def test_prevent_gs_invalid_xml(resources, outdir):
str(outdir / 'layers.rendered.pdf'), str(outdir / 'pdfa.ps'), context
)
with open(outdir / 'pdfa.pdf', 'rb') as f:
with mmap.mmap(
f.fileno(), 0, flags=mmap.MAP_PRIVATE, prot=mmap.PROT_READ
) as mm:
with open(outdir / 'pdfa.pdf', 'r+b') as f:
with mmap.mmap(f.fileno(), 0) as mm:
# Since the XML may be invalid, we scan instead of actually feeding it
# to a parser.
XMP_MAGIC = b'W5M0MpCehiHzreSzNTczkc9d'
tests/test_pdfinfo.py
+13 -13
View File
@@ -91,21 +91,21 @@ def test_single_page_image(outdir):
def test_single_page_inline_image(outdir):
filename = outdir / 'image-mono-inline.pdf'
pdf = Canvas(str(filename), pagesize=(8 * 72, 6 * 72))
with NamedTemporaryFile() as im_tmp:
im = Image.new('1', (8, 8), 0)
for n in range(8):
im.putpixel((n, n), 1)
im.save(im_tmp.name, format='PNG')
# Draw image in a 72x72 pt or 1"x1" area
pdf.drawInlineImage(im_tmp.name, 0, 0, width=72, height=72)
pdf.showPage()
pdf.save()
pdf = pdfinfo.PdfInfo(filename)
print(pdf)
pdfimage = pdf[0].images[0]
im = Image.new('1', (8, 8), 0)
for n in range(8):
im.putpixel((n, n), 1)
# Draw image in a 72x72 pt or 1"x1" area
pdf.drawInlineImage(im, 0, 0, width=72, height=72)
pdf.showPage()
pdf.save()
info = pdfinfo.PdfInfo(filename)
print(info)
pdfimage = info[0].images[0]
assert isclose(pdfimage.xres, 8)
assert pdfimage.color == Colorspace.rgb # reportlab produces color image
assert pdfimage.color == Colorspace.gray
assert pdfimage.width == 8
tests/test_stdio.py
+3 -17
View File
@@ -18,7 +18,7 @@
import os
import sys
from pathlib import Path
from subprocess import DEVNULL, PIPE, run, Popen
from subprocess import DEVNULL, PIPE, run, Popen, CalledProcessError
import pytest
@@ -29,6 +29,7 @@ from ocrmypdf.exec import qpdf
# pylint: disable=no-member,redefined-outer-name
run_ocrmypdf = pytest.helpers.run_ocrmypdf
run_ocrmypdf_api = pytest.helpers.run_ocrmypdf
spoof = pytest.helpers.spoof
@@ -76,6 +77,7 @@ def test_stdout(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
@pytest.mark.skipif(
sys.version_info[0:3] >= (3, 6, 4), reason="issue fixed in Python 3.6.4"
)
@pytest.mark.skipif(os.name == 'nt', reason="POSIX problem")
def test_closed_streams(spoof_tesseract_noop, ocrmypdf_exec, resources, outpdf):
input_file = str(resources / 'francais.pdf')
output_file = str(outpdf)
@@ -113,22 +115,6 @@ def test_bad_locale():
assert 'configured to use ASCII as encoding' in err, "should whine"
@pytest.mark.parametrize('renderer', ['hocr', 'sandwich'])
def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf):
p, out, err = run_ocrmypdf(
resources / 'ccitt.pdf',
no_outpdf,
'--pdf-renderer',
renderer,
env=spoof_tess_bad_utf8,
)
assert out == '', "stdout not clean"
assert p.returncode != 0
assert 'not utf-8' in err, "should whine about utf-8"
assert '\\x96' in err, 'should repeat backslash encoded output'
def test_dev_null(spoof_tesseract_noop, resources):
p, out, err = run_ocrmypdf(
resources / 'trivial.pdf', os.devnull, '--force-ocr', env=spoof_tesseract_noop
tests/test_userunit.py
+5 -3
View File
@@ -24,6 +24,7 @@ from ocrmypdf.pdfinfo import PdfInfo
check_ocrmypdf = pytest.helpers.check_ocrmypdf
run_ocrmypdf = pytest.helpers.run_ocrmypdf
run_ocrmypdf_api = pytest.helpers.run_ocrmypdf_api
spoof = pytest.helpers.spoof
@@ -32,9 +33,10 @@ def poster(resources):
return resources / 'poster.pdf'
def test_userunit_ghostscript_fails(poster, no_outpdf):
p, out, err = run_ocrmypdf(poster, no_outpdf, '--output-type=pdfa')
assert p.returncode == ExitCode.input_file
def test_userunit_ghostscript_fails(poster, no_outpdf, caplog):
result = run_ocrmypdf_api(poster, no_outpdf, '--output-type=pdfa')
assert result == ExitCode.input_file
assert 'not supported by Ghostscript' in caplog.text
def test_userunit_qpdf_passes(spoof_tesseract_cache, poster, outpdf):