From 2ccb3edc584dd5500361f036b7fd971c30ee7ca1 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 10 May 2018 00:49:36 -0700 Subject: [PATCH 01/15] Ignore masks when deciding what color to rasterize at --- src/ocrmypdf/pipeline.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/ocrmypdf/pipeline.py b/src/ocrmypdf/pipeline.py index 5f185864..b9e990c9 100644 --- a/src/ocrmypdf/pipeline.py +++ b/src/ocrmypdf/pipeline.py @@ -496,17 +496,23 @@ def rasterize_with_ghostscript( options = context.get_options() pageinfo = get_pageinfo(input_file, context) - device = 'png16m' # 24-bit - if pageinfo.images: - if all(image.comp == 1 for image in pageinfo.images): - if all(image.bpc == 1 for image in pageinfo.images): - device = 'pngmono' - elif all(image.bpc > 1 and image.color == Colorspace.index - for image in pageinfo.images): - device = 'png256' - elif all(image.bpc > 1 and image.color == Colorspace.gray - for image in pageinfo.images): - device = 'pnggray' + colorspaces = ['pngmono', 'pnggray', 'png256', 'png16m'] + device_idx = 0 + def at_least(cs): + return max(device_idx, colorspaces.index(cs)) + + for image in pageinfo.images: + if image.type_ != 'image': + continue # ignore masks + if image.bpc > 1: + if image.color == Colorspace.index: + device_idx = at_least('png256') + elif image.color == Colorspace.gray: + device_idx = at_least('pnggray') + else: + device_idx = at_least('png16m') + + device = colorspaces[device_idx] log.debug("Rasterize {0} with {1}".format( os.path.basename(input_file), device)) From 70aa644c108dd79374dff4dc37050dc5ceb01882 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Mon, 9 Jul 2018 13:56:23 -0700 Subject: [PATCH 02/15] Backport Python 3.7 fix for ruffus 2.7.0 from ocrmypdf v7.0.0 --- .gitignore | 1 + docs/release_notes.rst | 6 ++++ requirements.txt | 6 ++-- setup.py | 4 +-- src/ocrmypdf/__main__.py | 67 ++++++++++++++++++++-------------------- 5 files changed, 45 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 3a636a86..f06aefdc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.pyc *.sublime-* venv*/ +.venv/ pyvenv.cfg tasks.py .bash_history diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 082fdec6..c1575260 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -10,6 +10,12 @@ The OCRmyPDF package itself does not contain a public API, although it is fairly replace: `#$1 `_ +v6.2.2 +------ + +- Backport v7.0.0 fix for ruffus 2.7.0 + + v6.2.1 ------ diff --git a/requirements.txt b/requirements.txt index d7dec8b0..73998a86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ # requirements.txt can be used to replicate the developer's build environment # setup.py lists a separate set of requirements that are looser to simplify # installation -ruffus == 2.6.3 -Pillow == 5.1.0 +ruffus == 2.7.0 +Pillow == 5.2.0 reportlab == 3.4.0 PyPDF2 == 1.26.0 img2pdf == 0.2.4 cffi == 1.11.5 -PyMuPDF == 1.12.5 \ No newline at end of file +PyMuPDF == 1.12.5 diff --git a/setup.py b/setup.py index 5420aa6e..badc576e 100644 --- a/setup.py +++ b/setup.py @@ -243,12 +243,12 @@ setup( 'cffi >= 1.9.1', # must be a setup and install requirement 'defusedxml >= 0.5.0', # pure Python, so track HEAD closely 'img2pdf >= 0.2.4', # pure Python, so track HEAD closely - 'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"', + 'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"', # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3 # block 5.1.0, broken wheels 'PyPDF2 >= 1.26', # pure Python, so track HEAD closely 'reportlab >= 3.3.0', # oldest released version with sane image handling - 'ruffus == 2.6.3', # pinned - ocrmypdf implements a 2.6.3 workaround + 'ruffus >= 2.7.0', ], extras_require={ 'fitz': ['PyMuPDF >= 1.12.5'] # for table of contents bug diff --git a/src/ocrmypdf/__main__.py b/src/ocrmypdf/__main__.py index aa4490c4..9290daf2 100755 --- a/src/ocrmypdf/__main__.py +++ b/src/ocrmypdf/__main__.py @@ -70,7 +70,7 @@ def complain(message): if 'IDE_PROJECT_ROOTS' in os.environ: os.environ['PATH'] = '/usr/local/bin:' + os.environ['PATH'] -# -------- +# -------- # Critical environment tests verify_python3_env() @@ -156,7 +156,7 @@ parser.add_argument( '--image-dpi', metavar='DPI', type=int, help="For input image instead of PDF, use this DPI instead of file's.") parser.add_argument( - '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'], + '--output-type', choices=['pdfa', 'pdf', 'pdfa-1', 'pdfa-2', 'pdfa-3'], default='pdfa', help="Choose output type. 'pdfa' creates a PDF/A-2b compliant file for " "long term archiving (default, recommended) but may not suitable " @@ -574,7 +574,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log): if exc_name == 'builtins.SystemExit': match = re.search(r"\.(.+?)\)", exc_value) exit_code_name = match.groups()[0] - exit_code = getattr(ExitCode, exit_code_name, 'other_error') + exit_code = getattr(ExitCode, exit_code_name, 'other_error') elif exc_name == 'ruffus.ruffus_exceptions.MissingInputFileError': log.error(cleanup_ruffus_error_message(exc_value)) exit_code = ExitCode.input_file @@ -598,7 +598,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log): (exc_name == 'ocrmypdf.exceptions.EncryptedPdfError'): log.error(textwrap.dedent("""\ Input PDF is encrypted. The encryption must be removed to - perform OCR. + perform OCR. For information about this PDF's security use qpdf --show-encryption infilename @@ -607,7 +607,7 @@ def do_ruffus_exception(ruffus_five_tuple, options, log): qpdf --decrypt [--password=[password]] infilename """)) - exit_code = ExitCode.encrypted_pdf + exit_code = ExitCode.encrypted_pdf elif exc_name == 'ocrmypdf.exceptions.PdfMergeFailedError': log.error(textwrap.dedent("""\ Failed to merge PDF image layer with OCR layer @@ -638,33 +638,31 @@ def do_ruffus_exception(ruffus_five_tuple, options, log): return ExitCode.other_error -def traverse_ruffus_exception(e_args, options, log): - """Walk through a RethrownJobError and find the first exception. +def traverse_ruffus_exception(exceptions, options, log): + """Traverse a RethrownJobError and output the exceptions - Ruffus flattens exception to 5 element tuples. Because of a bug - in <= 2.6.3 it may present either the single: - (task, job, exc, value, stack) - or something like: - [[(task, job, exc, value, stack)]] - - Generally cross-process exception marshalling doesn't work well - and ruffus doesn't support because BaseException has its own - implementation of __reduce__ that attempts to reconstruct the - exception based on e.__init__(e.args). - - Attempting to log the exception directly marshalls it to the logger - which is probably in another process, so it's better to log only - data from the exception at this point. + Ruffus presents exceptions as 5 element tuples. The RethrownJobException + has a list of exceptions like + e.job_exceptions = [(5-tuple), (5-tuple), ...] + + ruffus < 2.7.0 had a bug with exception marshalling that would give + different output whether the main or child process raised the exception. + We no longer support this. + + Attempting to log the exception itself will re-marshall it to the logger + which is normally running in another process. It's better to avoid re- + marshalling. The exit code will be based on this, even if multiple exceptions occurred at the same time.""" - if isinstance(e_args, Sequence) and isinstance(e_args[0], str) and \ - len(e_args) == 5: - return do_ruffus_exception(e_args, options, log) - elif is_iterable_notstr(e_args): - for exc in e_args: - return traverse_ruffus_exception(exc, options, log) + exit_codes = [] + for exc in exceptions: + exit_code = do_ruffus_exception(exceptions, options, log) + exit_codes.append(exit_code) + + return exit_codes[0] # Multiple codes are rare so take the first one + def check_closed_streams(options): @@ -749,7 +747,7 @@ def check_environ(options, _log): for k in old_envvars: if k in os.environ: _log.warning(textwrap.dedent("""\ - OCRmyPDF no longer uses the environment variable {}. + OCRmyPDF no longer uses the environment variable {}. Change PATH to select alternate programs.""".format(k))) @@ -792,14 +790,14 @@ def report_output_file_size(options, _log, input_file, output_file): ratio = output_size / input_size if ratio < 1.35 or input_size < 25000: return # Seems fine - + reasons = [] if not fitz: reasons.append("The optional dependency PyMuPDF is not installed.") image_preproc = { - 'deskew', - 'clean_final', - 'remove_background', + 'deskew', + 'clean_final', + 'remove_background', 'oversample', 'force_ocr' } @@ -886,7 +884,8 @@ def run_pipeline(): except ruffus_exceptions.RethrownJobError as e: if options.verbose: _log.debug(str(e)) # stringify exception so logger doesn't have to - exitcode = traverse_ruffus_exception(e.args, options, _log) + exceptions = e.job_exceptions + exitcode = traverse_ruffus_exception(exceptions, options, _log) if exitcode is None: _log.error("Unexpected ruffus exception: " + str(e)) _log.error(repr(e)) @@ -919,7 +918,7 @@ def run_pipeline(): _log.warning('Output file: The generated PDF is INVALID') return ExitCode.invalid_output_pdf - report_output_file_size(options, _log, start_input_file, + report_output_file_size(options, _log, start_input_file, options.output_file) pdfinfo = context.get_pdfinfo() From 465007442853c060b1b469f4c1f814dab0dafb63 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 28 Jun 2018 13:57:45 -0700 Subject: [PATCH 03/15] Cherrypick Python 3.7 documentation updates from v7.0.0 From b0eacd6 --- README.rst | 16 +++++++--------- docs/installation.rst | 18 +++++++++--------- setup.py | 1 + src/ocrmypdf/_unicodefun.py | 5 +++++ 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index bf323c21..655a7e93 100644 --- a/README.rst +++ b/README.rst @@ -4,10 +4,10 @@ OCRmyPDF .. image:: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master :target: https://travis-ci.org/jbarlow83/OCRmyPDF -.. image:: https://img.shields.io/pypi/v/ocrmypdf.svg +.. image:: https://img.shields.io/pypi/v/ocrmypdf.svg :target: https://pypi.org/project/ocrmypdf/ -.. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg +.. image:: https://img.shields.io/homebrew/v/ocrmypdf.svg :alt: homebrew :target: http://brewformulas.org/Ocrmypdf @@ -54,7 +54,7 @@ Motivation I searched the web for a free command line tool to OCR PDF files on Linux/UNIX: I found many, but none of them were really satisfying. -- Either they produced PDF files with misplaced text under the image (making copy/paste impossible) +- Either they produced PDF files with misplaced text under the image (making copy/paste impossible) - Or they did not handle accents and multilingual characters - Or they changed the resolution of the embedded images - Or they generated ridiculously large PDF files @@ -63,7 +63,7 @@ Linux/UNIX: I found many, but none of them were really satisfying. - On top of that none of them produced PDF/A files (format dedicated for long time storage) ...so I decided to develop my own tool (using various existing scripts -as an inspiration). +as an inspiration). Installation ------------ @@ -97,7 +97,7 @@ you can often find packages that provide language packs: # Debian/Ubuntu users apt-get install tesseract-ocr-chi-sim # Example: Install Chinese Simplified language back - + You can then pass the ``-l LANG`` argument to OCRmyPDF to give a hint as to what languages it should search for. Multiple languages can be requested. @@ -126,9 +126,7 @@ If you detect an issue, please: Requirements ------------ -Runs on CPython 3.6, and requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings. - -Python 3.5 is also supported. +Runs on CPython 3.5, 3.6 and 3.7. Requires external program installations of Ghostscript, Tesseract OCR, QPDF, and Leptonica. ocrmypdf is pure Python, but uses CFFI to portably generate library bindings. Press & Media ------------- @@ -142,7 +140,7 @@ Press & Media License ------- -The OCRmyPDF software is licensed under the GNU GPLv3. Certain files are covered by other licenses, as noted in their source files. +The OCRmyPDF software is licensed under the GNU GPLv3. Certain files are covered by other licenses, as noted in their source files. The license for each test file varies, and is noted in tests/resources/README.rst. The documentation is licensed under Creative Commons Attribution-ShareAlike 4.0 (CC-BY-SA 4.0). diff --git a/docs/installation.rst b/docs/installation.rst index 8efaac2e..99978aab 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1,7 +1,7 @@ Installation ============ -The easiest way to install OCRmyPDF to follow the steps for your operating system/platform. +The easiest way to install OCRmyPDF to follow the steps for your operating system/platform. If you want to use the latest version of OCRmyPDF, your best bet is to install the most recent version your platform provides, and then upgrade that version by installing the Python binary wheels. @@ -23,7 +23,7 @@ To see what versions are available, check the `Debian Package Tracker `_, and both the input and output file must be inside the writable volume. This example command uses the current working directory as the writable volume: .. code-block:: bash docker run --rm -v "$(pwd):/home/docker" ocrmypdf -In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``: +In this worked example, the current working directory contains an input file called ``test.pdf`` and the output will go to ``output.pdf``: .. code-block:: bash @@ -167,11 +167,11 @@ Install or upgrade the required Homebrew packages, if any are missing: brew install python3 brew install libxml2 libffi leptonica brew install unpaper # optional - -Python 3.5 and 3.6 are supported. + +Python 3.5, 3.6 and 3.7 are supported. Install the required Tesseract OCR engine with the language packs you plan to use: - + .. code-block:: bash brew install tesseract # Option 1: for English, French, German, Spanish @@ -398,7 +398,7 @@ The following dependencies are recommended: - unpaper 6.1 - PyMuPDF 1.12.5 or newer -These are in addition to the Python packaging dependencies, meaning that unfortunately, the ``pip install`` command cannot satisfy all of them. +These are in addition to the Python packaging dependencies, meaning that unfortunately, the ``pip install`` command cannot satisfy all of them. Python 3.6 and Tesseract 4.0.0-beta.1 are recommended for best OCR results and best performance. diff --git a/setup.py b/setup.py index badc576e..1362399a 100644 --- a/setup.py +++ b/setup.py @@ -215,6 +215,7 @@ setup( classifiers=[ "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: End Users/Desktop", diff --git a/src/ocrmypdf/_unicodefun.py b/src/ocrmypdf/_unicodefun.py index a916077f..e80d3d21 100644 --- a/src/ocrmypdf/_unicodefun.py +++ b/src/ocrmypdf/_unicodefun.py @@ -40,6 +40,11 @@ import codecs def verify_python3_env(): """Ensures that the environment is good for unicode on Python 3.""" + + # PEP 538 changes in Python 3.7 should make this wrangling unnecessary + if sys.version_info[0:3] >= (3, 7, 0): + return + try: import locale fs_enc = codecs.lookup(locale.getpreferredencoding()).name From ed9fb110b120e20d1f76e5d96145aaa32a8ae047 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 1 May 2018 21:31:09 -0700 Subject: [PATCH 04/15] Fix a comment about Tesseract behavior in certain versions --- src/ocrmypdf/exec/tesseract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ocrmypdf/exec/tesseract.py b/src/ocrmypdf/exec/tesseract.py index eb745749..b4df6e1c 100644 --- a/src/ocrmypdf/exec/tesseract.py +++ b/src/ocrmypdf/exec/tesseract.py @@ -160,7 +160,7 @@ def get_orientation(input_file, language: list, engine_mode, timeout: float, assert 'Rotate' not in osd angle = -angle % 360 else: - # Tesseract == 3.04.01, hopefully also Tesseract > 3.04.01 + # Tesseract >= 3.04.01 # reports "Orientation in degrees" as a clockwise angle assert 'Rotate' in osd From 7394a4cf49f91ee2d5a84a87fcf33bf8adaf049f Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 11 May 2018 02:31:07 -0700 Subject: [PATCH 05/15] Cherrypick warning about --user-words not having any effect Might be available in full release of Tess4 --- src/ocrmypdf/__main__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ocrmypdf/__main__.py b/src/ocrmypdf/__main__.py index 9290daf2..5510578c 100755 --- a/src/ocrmypdf/__main__.py +++ b/src/ocrmypdf/__main__.py @@ -490,6 +490,10 @@ def check_options_advanced(options, log): "--pdfa-image-compression argument has no effect when " "--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'" ) + + if tesseract.v4() and (options.user_words or options.user_patterns): + log.warning( + 'Tesseract 4.x ignores --user-words, so this has no effect') def check_options_metadata(options, log): From 9f02de55be47e5af791aca32b52afe63bb8fdaae Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 11 May 2018 22:31:09 -0700 Subject: [PATCH 06/15] main: do better parameter validation --- src/ocrmypdf/__main__.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/ocrmypdf/__main__.py b/src/ocrmypdf/__main__.py index 5510578c..cdd0ffc7 100755 --- a/src/ocrmypdf/__main__.py +++ b/src/ocrmypdf/__main__.py @@ -85,6 +85,20 @@ if tesseract.version() < MINIMUM_TESS_VERSION: # ------------- # Parser +def numeric(basetype, min_=None, max_=None): + "Validator for numeric params" + min_ = basetype(min_) if min_ is not None else None + max_ = basetype(max_) if max_ is not None else None + def _numeric(string): + value = basetype(string) + if (min_ is not None and value < min_ + or max_ is not None and value > max_): + msg = "%r not in valid range %r" % (string, (min_, max_)) + raise argparse.ArgumentTypeError(msg) + return value + return _numeric + + parser = argparse.ArgumentParser( prog=PROGRAM_NAME, fromfile_prefix_chars='@', @@ -233,7 +247,7 @@ preprocessing.add_argument( help="Clean page as above, and incorporate the cleaned image in the final " "PDF. Might remove desired content.") preprocessing.add_argument( - '--oversample', metavar='DPI', type=int, default=0, + '--oversample', metavar='DPI', type=numeric(int, 0, 5000), default=0, help="Oversample images to at least the specified DPI, to improve OCR " "results slightly") @@ -255,7 +269,7 @@ ocrsettings.add_argument( # "pages") ocrsettings.add_argument( - '--skip-big', type=float, metavar='MPixels', + '--skip-big', type=numeric(float, 0, 5000), metavar='MPixels', help="Skip OCR on pages larger than the specified amount of megapixels, " "but include skipped pages in final output") @@ -263,7 +277,7 @@ advanced = parser.add_argument_group( "Advanced", "Advanced options to control Tesseract's OCR behavior") advanced.add_argument( - '--max-image-mpixels', action='store', type=float, metavar='MPixels', + '--max-image-mpixels', action='store', type=numeric(float, 0), metavar='MPixels', help="Set maximum number of pixels to unpack before treating an image as a " "decompression bomb", default=128.0) @@ -296,11 +310,11 @@ advanced.add_argument( " of Ghostscript; deprecated" ) advanced.add_argument( - '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS', + '--tesseract-timeout', default=180.0, type=numeric(float, 0), metavar='SECONDS', help='Give up on OCR after the timeout, but copy the preprocessed page ' 'into the final output') advanced.add_argument( - '--rotate-pages-threshold', default=14.0, type=float, metavar='CONFIDENCE', + '--rotate-pages-threshold', default=14.0, type=numeric(float, 1000), metavar='CONFIDENCE', help="Only rotate pages when confidence is above this value (arbitrary " "units reported by tesseract)") advanced.add_argument( From eb343b1e37c385af7228b963f45a675ce858be49 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 1 May 2018 15:52:00 -0700 Subject: [PATCH 07/15] Tests: Add ability to disable use of cache (cherrypicked from v7) --- tests/spoof/tesseract_cache.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/spoof/tesseract_cache.py b/tests/spoof/tesseract_cache.py index 6e0110fa..92366f2b 100755 --- a/tests/spoof/tesseract_cache.py +++ b/tests/spoof/tesseract_cache.py @@ -106,6 +106,8 @@ def main(): source = os.environ['_OCRMYPDF_TEST_INFILE'] # required args = parser.parse_args() + cache_disabled = os.environ.get('_OCRMYPDF_CACHE_DISABLED', False) + if args.imagename == 'stdin': real_tesseract() @@ -128,7 +130,7 @@ def main(): print("Tesseract cache folder {} - ".format(cache_folder), end='', file=sys.stderr) - if (cache_folder / 'stderr.bin').exists(): + if (cache_folder / 'stderr.bin').exists() and not cache_disabled: # Cache hit print("HIT", file=sys.stderr) From ea69883386b87bf5f77a74d3bfa126c7b4769e92 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sat, 23 Jun 2018 01:54:09 -0700 Subject: [PATCH 08/15] Tests: Speed up a slow test (cherry-picked from v7) --- tests/test_main.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_main.py b/tests/test_main.py index c65f3cdd..37a8f333 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -300,7 +300,8 @@ def test_autorotate_threshold( @pytest.mark.parametrize('renderer',RENDERERS) def test_ocr_timeout(renderer, resources, outpdf): out = check_ocrmypdf(resources / 'skew.pdf', outpdf, - '--tesseract-timeout', '1.0') + '--tesseract-timeout', '0.01', + '--pdf-renderer', renderer) pdfinfo = PdfInfo(out) assert not pdfinfo[0].has_text @@ -472,7 +473,7 @@ def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error, resources, outpdf): check_ocrmypdf( resources / 'hugemono.pdf', outpdf, '-r', - '--pdf-renderer', renderer, + '--pdf-renderer', renderer, '--max-image-mpixels', '0', env=spoof_tesseract_big_image_error) @@ -993,7 +994,7 @@ def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf): @pytest.mark.skipif( - not tesseract.has_textonly_pdf(), + not tesseract.has_textonly_pdf(), reason="issue only affects sandwich") def test_rotate_deskew_timeout(resources, outdir): check_ocrmypdf( @@ -1043,10 +1044,10 @@ def test_text_curves(spoof_tesseract_noop, resources, outpdf): check_ocrmypdf( resources / 'vector.pdf', outpdf, '--force-ocr', env=spoof_tesseract_noop) - + info = PdfInfo(outpdf) assert len(info.pages[0].images) != 0, "force did not rasterize" - + def test_dev_null(spoof_tesseract_noop, resources): p, out, err = run_ocrmypdf( From 5252b88f0fb2e492eee844818a6db4468e1dd1bf Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 12 Jul 2018 02:09:31 -0700 Subject: [PATCH 09/15] Travis: modernize with v7.0.0 updates Travis: add 3.7 testing Travis: remove deploy to testpypi since it's broken Travis: cherry-pick change to declarative APT From efb9572 Removed pngquant Travis: Remove linux_before_install.sh Eliminate Homebrew autobrewing --- .travis.yml | 74 +++++++++++++------------- .travis/autobrew.py | 93 --------------------------------- .travis/linux_before_install.sh | 31 ----------- .travis/osx_brew.sh | 23 -------- 4 files changed, 36 insertions(+), 185 deletions(-) delete mode 100644 .travis/autobrew.py delete mode 100644 .travis/linux_before_install.sh delete mode 100644 .travis/osx_brew.sh diff --git a/.travis.yml b/.travis.yml index 2ba7ba3d..2f8c94a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,32 +1,53 @@ -dist: trusty language: python cache: - ccache: true pip: true directories: - $HOME/Library/Caches/Homebrew -env: - global: - - secure: "hsf6MT+n2x3OiDM2fQyJZdV0/PWYmv81LdVqC6cfnHBE/8N3DloJRqQ7WfO14TxhiK9PEC7MpyCj0lSabUHEO7gSH6Vks6I1asoSkt8S9/bSMlhT4hei+pwVpeGEiU5xHVATNjY+D919VC3IFvc3XmjT74h/2SLhaZ+jhEmDggM=" # HOMEBREW_OCRMYPDF_TOKEN +addons: + apt: + update: true + sources: + - sourceline: 'ppa:alex-p/tesseract-ocr' + - sourceline: 'ppa:heyarje/libav-11' + - sourceline: 'ppa:vshn/ghostscript' + packages: + - ghostscript + - libavcodec56 + - libavformat56 + - libavutil54 + - libffi-dev + - poppler-utils + - tesseract-ocr + - tesseract-ocr-deu + - tesseract-ocr-eng + - tesseract-ocr-fra matrix: include: - os: linux + dist: trusty sudo: required language: python - python: 3.5 + python: "3.5" env: EXTRAS= - os: linux + dist: trusty sudo: required language: python - python: 3.6 + python: "3.6" env: EXTRAS= - os: linux + dist: trusty sudo: required language: python - python: 3.6 + python: "3.6" env: EXTRAS=[fitz] + - os: linux + dist: xenial + sudo: required + language: python + python: "3.7" - os: osx osx_image: xcode8 language: generic @@ -41,7 +62,10 @@ before_cache: before_install: | if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - bash .travis/linux_before_install.sh + pip install --upgrade pip + mkdir -p packages + wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb + sudo dpkg -i packages/unpaper_6.1-1.deb elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update && brew bundle --file=.travis/Brewfile pip3 install --upgrade pip @@ -49,7 +73,9 @@ before_install: | fi install: -- pip3 install ".$EXTRAS" +- pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251 +- pip3 install -r requirements.txt +- pip3 install --no-deps ".$EXTRAS" - pip3 install -r test_requirements.txt script: @@ -73,31 +99,3 @@ deploy: tags: true condition: $TRAVIS_PYTHON_VERSION == "3.6" && $TRAVIS_OS_NAME == "linux" && $EXTRAS == "" skip_upload_docs: true - - # test pypi -- provider: pypi - server: https://testpypi.pypi.org/legacy/ - user: ocrmypdf-travis - password: - secure: "DTFOmmNL6olA0+yXvp4u9jXZlZeqrJsJ0526jzqf4a3gZ6jnGTq5UI6WzRsslSyoMMfXKtHQebqHM6ogSgCZinyZ3ufHJo8fn9brxbEc2gsiWkbj5o3bGwdWMT1vNNE7XW0VCpw87rZ1EEwjl4FJHFudMlPR1yfU5+uq0k0PACo=" - distributions: "sdist" - on: - branch: develop - tags: false - condition: $TRAVIS_OS_NAME == "osx" - skip_upload_docs: true - - # null deploy for osx - # we really just want to run after_deploy *after* pypi upload is done, but - # after_deploy on runs if a given box deployed -- provider: script - script: /usr/bin/true - on: - branch: master - tags: true - condition: $TRAVIS_OS_NAME == "osx" - -after_deploy: | - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - bash .travis/osx_brew.sh - fi diff --git a/.travis/autobrew.py b/.travis/autobrew.py deleted file mode 100644 index 35b7c3fa..00000000 --- a/.travis/autobrew.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -# © 2017-18 James R. Barlow: github.com/jbarlow83 - -from string import Template -from subprocess import run, PIPE -import re - -recipe_template = Template(""" -class Ocrmypdf < Formula - include Language::Python::Virtualenv - - desc "Adds an OCR text layer to scanned PDF files" - homepage "https://github.com/jbarlow83/OCRmyPDF" - ${ocrmypdf_url} - ${ocrmypdf_sha256} - - depends_on "pkg-config" => :build - depends_on "mupdf-tools" => :build # statically links libmupdf.a - depends_on "freetype" - depends_on "ghostscript" - depends_on "jpeg" - depends_on "libpng" - depends_on "python" - depends_on "qpdf" - depends_on "tesseract" - depends_on "unpaper" - -${resources} - def install - venv = virtualenv_create(libexec, "python3") - - resource("Pillow").stage do - inreplace "setup.py" do |s| - sdkprefix = MacOS::CLT.installed? ? "" : MacOS.sdk_path - s.gsub! "openjpeg.h", "probably_not_a_header_called_this_eh.h" - s.gsub! "ZLIB_ROOT = None", "ZLIB_ROOT = ('#{sdkprefix}/usr/lib', '#{sdkprefix}/usr/include')" - s.gsub! "JPEG_ROOT = None", "JPEG_ROOT = ('#{Formula["jpeg"].opt_prefix}/lib', '#{Formula["jpeg"].opt_prefix}/include')" - s.gsub! "FREETYPE_ROOT = None", "FREETYPE_ROOT = ('#{Formula["freetype"].opt_prefix}/lib', '#{Formula["freetype"].opt_prefix}/include')" - end - - # avoid triggering "helpful" distutils code that doesn't recognize Xcode 7 .tbd stubs - ENV.append "CFLAGS", "-I#{MacOS.sdk_path}/System/Library/Frameworks/Tk.framework/Versions/8.5/Headers" unless MacOS::CLT.installed? - venv.pip_install Pathname.pwd - end - - res = resources.map(&:name).to_set - ["Pillow"] - - res.each do |r| - venv.pip_install resource(r) - end - - venv.pip_install_and_link buildpath - end - - test do - # Since we use Python 3, we require a UTF-8 locale - ENV["LC_ALL"] = "en_US.UTF-8" - - system "#{bin}/ocrmypdf", "-f", "-q", "--deskew", - test_fixtures("test.pdf"), "ocr.pdf" - assert_predicate testpath/"ocr.pdf", :exist? - end -end -""") - -def main(): - p = run(['poet', '--single', 'ocrmypdf'], - encoding='utf-8', stdout=PIPE, check=True) - - ocrmypdf_lines = p.stdout.splitlines() - ocrmypdf_url = ocrmypdf_lines[1].strip() - ocrmypdf_sha256 = ocrmypdf_lines[2].strip() - - ocrmypdf_version = re.search( - r'ocrmypdf-(.+)\.tar.*', ocrmypdf_url).group(1) - print(f"Autobrewing {ocrmypdf_version}") - - p = run(['poet', '--resources', 'ocrmypdf'], - encoding='utf-8', stdout=PIPE, check=True) - - poet_resources = p.stdout - - # Remove the duplicate "ocrmypdf" resource block - all_resources = poet_resources.split('resource') - kept_resources = [block for block in all_resources if 'ocrmypdf' not in block] - resources = 'resource'.join(kept_resources) - - with open('ocrmypdf.rb', 'w') as out: - out.write(recipe_template.substitute(**locals())) - - -if __name__ == '__main__': - main() diff --git a/.travis/linux_before_install.sh b/.travis/linux_before_install.sh deleted file mode 100644 index 438a161f..00000000 --- a/.travis/linux_before_install.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# © 2017 James R. Barlow: github.com/jbarlow83 -set -euo pipefail -set -x - -sudo add-apt-repository ppa:vshn/ghostscript -y -sudo add-apt-repository ppa:heyarje/libav-11 -y -sudo apt-get update -qq -sudo apt-get install -y \ - ghostscript \ - poppler-utils \ - libavformat56 \ - libavcodec56 \ - libavutil54 \ - libffi-dev \ - qpdf - -sudo add-apt-repository ppa:alex-p/tesseract-ocr -y - -sudo apt-get update -sudo apt-get autoremove -y -sudo apt-get install -y --no-install-recommends \ - tesseract-ocr \ - tesseract-ocr-eng \ - tesseract-ocr-fra \ - tesseract-ocr-deu - -pip install --upgrade pip -mkdir -p packages -wget -q 'https://www.dropbox.com/s/vaq0kbwi6e6au80/unpaper_6.1-1.deb?raw=1' -O packages/unpaper_6.1-1.deb -sudo dpkg -i packages/unpaper_6.1-1.deb diff --git a/.travis/osx_brew.sh b/.travis/osx_brew.sh deleted file mode 100644 index 540af33e..00000000 --- a/.travis/osx_brew.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# © 2017 James R. Barlow: github.com/jbarlow83 -set -uo pipefail -set -x - -pip3 install homebrew-pypi-poet -python3 .travis/autobrew.py -cat ocrmypdf.rb - -# brew audit crashes Travis -#brew audit ocrmypdf.rb - -# Important: disable debug output so token is hidden -set +x -git clone https://$HOMEBREW_OCRMYPDF_TOKEN@github.com/jbarlow83/homebrew-ocrmypdf.git -set -x - -pushd homebrew-ocrmypdf -cp ../ocrmypdf.rb Formula/ocrmypdf.rb -git add Formula/ocrmypdf.rb -git commit -m "homebrew-ocrmypdf: automatic release $TRAVIS_BUILD_NUMBER $TRAVIS_TAG" -git push origin master -popd From 8f8e6dcdd465e309a102ae17e9a52f4cbb34ad15 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 12 Jul 2018 02:59:38 -0700 Subject: [PATCH 10/15] Fix problem iterating ruffus exceptions and rotate-pages-threshold parameter validation --- src/ocrmypdf/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ocrmypdf/__main__.py b/src/ocrmypdf/__main__.py index cdd0ffc7..1f2bda70 100755 --- a/src/ocrmypdf/__main__.py +++ b/src/ocrmypdf/__main__.py @@ -314,7 +314,7 @@ advanced.add_argument( help='Give up on OCR after the timeout, but copy the preprocessed page ' 'into the final output') advanced.add_argument( - '--rotate-pages-threshold', default=14.0, type=numeric(float, 1000), metavar='CONFIDENCE', + '--rotate-pages-threshold', default=14.0, type=numeric(float, max_=1000), metavar='CONFIDENCE', help="Only rotate pages when confidence is above this value (arbitrary " "units reported by tesseract)") advanced.add_argument( @@ -504,7 +504,7 @@ def check_options_advanced(options, log): "--pdfa-image-compression argument has no effect when " "--output-type is not 'pdfa', 'pdfa-1', or 'pdfa-2'" ) - + if tesseract.v4() and (options.user_words or options.user_patterns): log.warning( 'Tesseract 4.x ignores --user-words, so this has no effect') @@ -676,7 +676,7 @@ def traverse_ruffus_exception(exceptions, options, log): exit_codes = [] for exc in exceptions: - exit_code = do_ruffus_exception(exceptions, options, log) + exit_code = do_ruffus_exception(exc, options, log) exit_codes.append(exit_code) return exit_codes[0] # Multiple codes are rare so take the first one From e774b4650bd20e28a3bbc0f06f83857a47701ba0 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 3 Jul 2018 00:05:49 -0700 Subject: [PATCH 11/15] ocrmypdf.exec: trap FileNotFoundError too --- src/ocrmypdf/exec/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ocrmypdf/exec/__init__.py b/src/ocrmypdf/exec/__init__.py index f670c726..3af2b8fb 100644 --- a/src/ocrmypdf/exec/__init__.py +++ b/src/ocrmypdf/exec/__init__.py @@ -25,7 +25,7 @@ from ..exceptions import MissingDependencyError -def get_version(program, *, +def get_version(program, *, version_arg='--version', regex=r'(\d+(\.\d+)*)'): "Get the version of the specified program" args_prog = [ @@ -37,6 +37,10 @@ def get_version(program, *, args_prog, close_fds=True, universal_newlines=True, stdout=PIPE, stderr=STDOUT, check=True) output = proc.stdout + except FileNotFoundError as e: + raise MissingDependencyError( + "Could not find program '{}' on the PATH".format( + program)) from e except CalledProcessError as e: if e.returncode < 0: raise MissingDependencyError( From e7d21dd826cc7505d8d7eb9ed3bf1bf711585600 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 12 Jul 2018 03:03:34 -0700 Subject: [PATCH 12/15] Skip locale check on Python 3.7 --- tests/test_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_main.py b/tests/test_main.py index 37a8f333..b932084e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -967,6 +967,7 @@ def test_pdfa_n(spoof_tesseract_cache, pdfa_level, resources, outpdf): assert pdfa_info['conformance'] == 'PDF/A-{}B'.format(pdfa_level) +@pytest.mark.skipif(sys.version_info >= (3, 7, 0), reason='fixed') def test_bad_locale(): env = os.environ.copy() env['LC_ALL'] = 'C' From d240fc1ea61f2b6d8cdf54d2e4c1d9a51efa8e13 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 12 Jul 2018 03:07:47 -0700 Subject: [PATCH 13/15] Update release notes for v6.2.2 --- docs/release_notes.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/release_notes.rst b/docs/release_notes.rst index c1575260..e417f644 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -13,8 +13,9 @@ The OCRmyPDF package itself does not contain a public API, although it is fairly v6.2.2 ------ -- Backport v7.0.0 fix for ruffus 2.7.0 - +- Backport compatibility fixes for Python 3.7 and ruffus 2.7.0 from v7.0.0 +- Backport fix to ignore masks when deciding what colors are on a page +- Backport some minor improvements from v7.0.0: better argument validation and warnings about the Tesseract 4.0.0 ``--user-words`` regression v6.2.1 ------ From 1d9cc239ee25bc6539e0b6dcf4f3022ba80f7169 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 12 Jul 2018 03:22:03 -0700 Subject: [PATCH 14/15] Travis: Fix v6 build failures --- .travis.yml | 4 ++-- requirements.txt | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2f8c94a4..804fcce2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ addons: - libavutil54 - libffi-dev - poppler-utils + - qpdf - tesseract-ocr - tesseract-ocr-deu - tesseract-ocr-eng @@ -74,8 +75,7 @@ before_install: | install: - pip3 install pycparser # py3.7 workaround for https://github.com/eliben/pycparser/issues/251 -- pip3 install -r requirements.txt -- pip3 install --no-deps ".$EXTRAS" +- pip3 install ".$EXTRAS" - pip3 install -r test_requirements.txt script: diff --git a/requirements.txt b/requirements.txt index 73998a86..5ba424bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ PyPDF2 == 1.26.0 img2pdf == 0.2.4 cffi == 1.11.5 PyMuPDF == 1.12.5 +defusedxml == 0.5.0 From 6a302fdb88f311e1253490a83c8184fd464ba686 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 12 Jul 2018 03:27:40 -0700 Subject: [PATCH 15/15] Travis: nevermind xenial, then Gave a weird build error --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 804fcce2..725e2eaf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +dist: trusty language: python cache: pip: true @@ -27,28 +28,24 @@ addons: matrix: include: - os: linux - dist: trusty sudo: required language: python python: "3.5" env: EXTRAS= - os: linux - dist: trusty sudo: required language: python python: "3.6" env: EXTRAS= - os: linux - dist: trusty sudo: required language: python python: "3.6" env: EXTRAS=[fitz] - os: linux - dist: xenial sudo: required language: python - python: "3.7" + python: "3.7-dev" - os: osx osx_image: xcode8 language: generic