From 414d80fc161927fc411eae26b468d885097d3bbb Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Thu, 11 Sep 2025 17:13:04 -0700 Subject: [PATCH] Deprecate semfree and don't auto activate it Instead the standard executor will fall back to threads. semfree caused test failures with Py3.14: https://github.com/ocrmypdf/OCRmyPDF/issues/1558 In retrospect and with emerging Python tech like freethreading, semfree is becoming less necessary. We can use threads for the time being. A consequence is that performance may be lower on Lambda and Termux when we are using threads and not shelling out work. --- src/ocrmypdf/_plugin_manager.py | 13 ++------ src/ocrmypdf/builtin_plugins/concurrency.py | 34 ++++++++++++++++----- src/ocrmypdf/extra_plugins/__init__.py | 6 +--- src/ocrmypdf/extra_plugins/semfree.py | 9 ++++++ tests/test_semfree.py | 25 +++++++-------- 5 files changed, 51 insertions(+), 36 deletions(-) diff --git a/src/ocrmypdf/_plugin_manager.py b/src/ocrmypdf/_plugin_manager.py index e380e7bb..090aaf27 100644 --- a/src/ocrmypdf/_plugin_manager.py +++ b/src/ocrmypdf/_plugin_manager.py @@ -73,19 +73,10 @@ class OcrmypdfPluginManager(pluggy.PluginManager): module = importlib.import_module(name) self.register(module) - # 2. Install semfree if needed - try: - # pylint: disable=import-outside-toplevel - from multiprocessing.synchronize import SemLock - - del SemLock - except ImportError: - self.register(importlib.import_module('ocrmypdf.extra_plugins.semfree')) - - # 3. Register setuptools plugins + # 2. Register setuptools plugins self.load_setuptools_entrypoints('ocrmypdf') - # 4. Register plugins specified on command line + # 3. Register plugins specified on command line for name in self.__plugins: if isinstance(name, Path) or name.endswith('.py'): # Import by filename diff --git a/src/ocrmypdf/builtin_plugins/concurrency.py b/src/ocrmypdf/builtin_plugins/concurrency.py index 48442bda..86603a9d 100644 --- a/src/ocrmypdf/builtin_plugins/concurrency.py +++ b/src/ocrmypdf/builtin_plugins/concurrency.py @@ -96,6 +96,31 @@ def thread_init(q: Queue, user_init: UserInit, loglevel) -> None: return +def setup_executor(use_threads: bool) -> tuple[Queue, Executor, WorkerInit]: + if not use_threads: + # Some execution environments like AWS Lambda and Termux do not support + # semaphores. Check if semaphore support is available, and if not, fall back + # to using threads. + try: + # pylint: disable=import-outside-toplevel + from multiprocessing.synchronize import SemLock + + del SemLock + except ImportError: + use_threads = True + + if use_threads: + loq_queue = queue.Queue(-1) + executor_class = ThreadPoolExecutor + initializer = thread_init + else: + loq_queue = multiprocessing.Queue(-1) + executor_class = ProcessPoolExecutor + initializer = process_init + + return loq_queue, executor_class, initializer + + class StandardExecutor(Executor): """Standard OCRmyPDF concurrent task executor.""" @@ -110,14 +135,7 @@ class StandardExecutor(Executor): task_arguments: Iterable, task_finished: Callable, ): - if use_threads: - log_queue: Queue = queue.Queue(-1) - executor_class: FuturesExecutorClass = ThreadPoolExecutor - initializer: WorkerInit = thread_init - else: - log_queue = multiprocessing.Queue(-1) - executor_class = ProcessPoolExecutor - initializer = process_init + log_queue, executor_class, initializer = setup_executor(use_threads) # Regardless of whether we use_threads for worker processes, the log_listener # must be a thread. Make sure we create the listener after the worker pool, diff --git a/src/ocrmypdf/extra_plugins/__init__.py b/src/ocrmypdf/extra_plugins/__init__.py index abfdfbbb..a39eeea4 100644 --- a/src/ocrmypdf/extra_plugins/__init__.py +++ b/src/ocrmypdf/extra_plugins/__init__.py @@ -2,8 +2,4 @@ # # SPDX-License-Identifier: MPL-2.0 -"""Extra plugins. These are not automatically inserted when ocrmypdf is run. - -You can use these plugins by specifying them on the command line, e.g.: -ocrmypdf --plugin ocrmypdf.extra_plugins.semfree ... -""" +"""Extra plugins. These are not automatically inserted when ocrmypdf is run.""" diff --git a/src/ocrmypdf/extra_plugins/semfree.py b/src/ocrmypdf/extra_plugins/semfree.py index 92a75adc..49e8143c 100644 --- a/src/ocrmypdf/extra_plugins/semfree.py +++ b/src/ocrmypdf/extra_plugins/semfree.py @@ -13,6 +13,9 @@ worker communicates only with the main process. This is not without drawbacks. If the tasks are not "even" in size, which cannot be guaranteed, some workers may end up with too much work while others are idle. It is less efficient than the standard implementation, so not the default. + +This module is deprecated and will be removed in a future release. The standard +executor will fall back to threads in these environments. """ from __future__ import annotations @@ -20,6 +23,7 @@ from __future__ import annotations import logging import logging.handlers import signal +import warnings from collections.abc import Callable, Iterable, Iterator from contextlib import suppress from enum import Enum, auto @@ -32,6 +36,11 @@ from ocrmypdf._concurrent import NullProgressBar from ocrmypdf.exceptions import InputFileError from ocrmypdf.helpers import remove_all_log_handlers +warnings.warn( + "semfree.py is deprecated and will be removed in a future release.", + DeprecationWarning, +) + class MessageType(Enum): """Implement basic IPC messaging.""" diff --git a/tests/test_semfree.py b/tests/test_semfree.py index 8537afd7..1c897b52 100644 --- a/tests/test_semfree.py +++ b/tests/test_semfree.py @@ -12,15 +12,16 @@ from .conftest import is_linux, run_ocrmypdf_api @pytest.mark.skipif(not is_linux(), reason='semfree plugin only works on Linux') def test_semfree(resources, outpdf): - exitcode = run_ocrmypdf_api( - resources / 'multipage.pdf', - outpdf, - '--skip-text', - '--skip-big', - '2', - '--plugin', - 'ocrmypdf.extra_plugins.semfree', - '--plugin', - 'tests/plugins/tesseract_noop.py', - ) - assert exitcode in (ExitCode.ok, ExitCode.pdfa_conversion_failed) + with pytest.warns(DeprecationWarning, match="semfree.py is deprecated"): + exitcode = run_ocrmypdf_api( + resources / 'multipage.pdf', + outpdf, + '--skip-text', + '--skip-big', + '2', + '--plugin', + 'ocrmypdf.extra_plugins.semfree', + '--plugin', + 'tests/plugins/tesseract_noop.py', + ) + assert exitcode in (ExitCode.ok, ExitCode.pdfa_conversion_failed)