From ec3f506500175b37cb76154e4a6c4dc33edd58bd Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 5 Jun 2020 16:36:11 -0700 Subject: [PATCH] Convert tesseract_badutf8 to plugin --- tests/plugins/tesseract_badutf8.py | 63 +++++++++++++++++++++++ tests/spoof/tesseract_badutf8.py | 80 ------------------------------ tests/test_stdio.py | 5 -- 3 files changed, 63 insertions(+), 85 deletions(-) create mode 100644 tests/plugins/tesseract_badutf8.py delete mode 100755 tests/spoof/tesseract_badutf8.py diff --git a/tests/plugins/tesseract_badutf8.py b/tests/plugins/tesseract_badutf8.py new file mode 100644 index 00000000..b87a3dac --- /dev/null +++ b/tests/plugins/tesseract_badutf8.py @@ -0,0 +1,63 @@ +# © 2020 James R. Barlow: github.com/jbarlow83 +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""Tesseract bad utf8 + +In some cases, some versions of Tesseract can output binary gibberish or data +that is not UTF-8 compatible, so we are forced to check that we can convert it +and present it to the user. +""" + +from subprocess import CalledProcessError +from unittest.mock import patch + +from ocrmypdf import hookimpl +from ocrmypdf.builtin_plugins.tesseract_ocr import TesseractOcrEngine + + +def bad_utf8(*args, **kwargs): + raise CalledProcessError( + 1, + 'tesseract', + output=b'\x96\xb3\x8c\xf8\x82\xc8UTF-8\x0a', # "Invalid UTF-8" in Shift JIS + stderr=b"", + ) + + +class BadUtf8OcrEngine(TesseractOcrEngine): + @staticmethod + def generate_hocr(input_file, output_hocr, output_text, options): + with patch('ocrmypdf.exec.tesseract.run', new=bad_utf8): + TesseractOcrEngine.generate_hocr( + input_file, output_hocr, output_text, options + ) + + @staticmethod + def generate_pdf(input_file, output_pdf, output_text, options): + with patch('ocrmypdf.exec.tesseract.run', new=bad_utf8): + TesseractOcrEngine.generate_pdf( + input_file, output_pdf, output_text, options + ) + + +@hookimpl +def get_ocr_engine(): + return BadUtf8OcrEngine() diff --git a/tests/spoof/tesseract_badutf8.py b/tests/spoof/tesseract_badutf8.py deleted file mode 100755 index 3cbb6625..00000000 --- a/tests/spoof/tesseract_badutf8.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -# © 2017 James R. Barlow: github.com/jbarlow83 -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to -# permit persons to whom the Software is furnished to do so, subject to -# the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -import sys - -"""Tesseract bad utf8 spoof - -In 'hocr' mode or 'pdf' mode, return error code 1 and some non-Unicode -text because tesseract seems to do that in some cases related to -language pack version mismatches - -""" - - -VERSION_STRING = '''tesseract 4.0.0 - leptonica-1.77.0 - libjpeg 9c : libpng 1.6.35 : libtiff 4.0.10 : zlib 1.2.11 : libopenjp2 2.3.0 - Found AVX2 - Found AVX - Found SSE -SPOOFED -''' - -# Japanese "Invalid UTF-8" encoded in Shift JIS -BAD_UTF8 = b'\x96\xb3\x8c\xf8\x82\xc8UTF-8\x0a' - - -def main(): - if sys.argv[1] == '--version': - print(VERSION_STRING, file=sys.stderr) - sys.exit(0) - elif sys.argv[1] == '--list-langs': - print('List of available languages (1):\neng', file=sys.stderr) - sys.exit(0) - elif sys.argv[-2] == '--print-parameters': - print("Some parameters", file=sys.stderr) - print("textonly_pdf\t1\tSome help text") - sys.exit(0) - elif sys.argv[-2] in ('hocr', 'pdf'): - sys.stdout.buffer.write(BAD_UTF8) - sys.exit(1) - elif sys.argv[-1] == 'stdout': - # input file is at sys.argv[-2] but we don't look at it - print( - """Orientation: 0 -Orientation in degrees: 0 -Orientation confidence: 100.00 -Script: 1 -Script confidence: 100.00""", - file=sys.stderr, - ) - else: - print("Spoof doesn't understand arguments", file=sys.stderr) - print(sys.argv, file=sys.stderr) - sys.exit(1) - - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/tests/test_stdio.py b/tests/test_stdio.py index d78b1644..2f58306a 100644 --- a/tests/test_stdio.py +++ b/tests/test_stdio.py @@ -33,11 +33,6 @@ run_ocrmypdf_api = pytest.helpers.run_ocrmypdf spoof = pytest.helpers.spoof -@pytest.fixture -def spoof_tess_bad_utf8(tmp_path_factory): - return spoof(tmp_path_factory, tesseract='tesseract_badutf8.py') - - def test_stdin(ocrmypdf_exec, resources, outpdf): input_file = str(resources / 'francais.pdf') output_file = str(outpdf)