From 9db01c7ff5cdea61f5b5d807fd9d5493754ad8b9 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Wed, 4 Dec 2019 15:00:12 -0800 Subject: [PATCH] Remove test_bad_utf8 Due to difficulties of getting this to work on Python 3.8, Windows, and high probability that this behavior is now gone from Tesseract 4.0+. Originally added in 2017. --- src/ocrmypdf/exec/tesseract.py | 7 +------ tests/test_stdio.py | 18 +----------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/src/ocrmypdf/exec/tesseract.py b/src/ocrmypdf/exec/tesseract.py index 34bd8983..a4a42b7d 100644 --- a/src/ocrmypdf/exec/tesseract.py +++ b/src/ocrmypdf/exec/tesseract.py @@ -194,12 +194,7 @@ def tesseract_log_output(mainlog, stdout, input_file): try: text = stdout.decode() except UnicodeDecodeError: - log.error( - "command line output was not utf-8. " - + "This usually means Tesseract's language packs do not match " - "the installed version of Tesseract." - ) - text = stdout.decode('utf-8', 'backslashreplace') + text = stdout.decode('utf-8', 'ignore') lines = text.splitlines() for line in lines: diff --git a/tests/test_stdio.py b/tests/test_stdio.py index 53c0398f..0f2609af 100644 --- a/tests/test_stdio.py +++ b/tests/test_stdio.py @@ -18,7 +18,7 @@ import os import sys from pathlib import Path -from subprocess import DEVNULL, PIPE, run, Popen +from subprocess import DEVNULL, PIPE, run, Popen, CalledProcessError import pytest @@ -115,22 +115,6 @@ def test_bad_locale(): assert 'configured to use ASCII as encoding' in err, "should whine" -@pytest.mark.parametrize('renderer', ['hocr', 'sandwich']) -def test_bad_utf8(spoof_tess_bad_utf8, renderer, resources, no_outpdf): - p, out, err = run_ocrmypdf( - resources / 'ccitt.pdf', - no_outpdf, - '--pdf-renderer', - renderer, - env=spoof_tess_bad_utf8, - ) - - assert out == '', "stdout not clean" - assert p.returncode != 0 - assert 'not utf-8' in err, "should whine about utf-8" - assert '\\x96' in err, 'should repeat backslash encoded output' - - def test_dev_null(spoof_tesseract_noop, resources): p, out, err = run_ocrmypdf( resources / 'trivial.pdf', os.devnull, '--force-ocr', env=spoof_tesseract_noop