From dea8fcfb5b9842aa9105de6fa0c46926fcfe4703 Mon Sep 17 00:00:00 2001
From: "James R. Barlow" <jim@purplerock.ca>
Date: Thu, 22 Mar 2018 23:18:07 -0700
Subject: [PATCH] Optimize page splitting by multiprocessing

Previously page splitting occurred in a single process because it was
not believed to affect performance much. It turned out to be an expensive
operation.

It now scales better with large page sizes although this has a negative
effect on small files.

Overall time changes as follows:

7 page file, 9.02s -> 9.56s
731 page file, 213s -> 97s

WITH --tesseract-timeout 0 --output-type pdf --skip-text

i.e. you don't get a 2.2x speed gain when OCR is available.

Squashed a commit to ix test suite failure on --rotate-pages
Squashed a commit to remove debug code
---
 ocrmypdf/pipeline.py | 64 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 52 insertions(+), 12 deletions(-)

diff --git a/ocrmypdf/pipeline.py b/ocrmypdf/pipeline.py
index 6aa68aa9..9db5d16e 100644
--- a/ocrmypdf/pipeline.py
+++ b/ocrmypdf/pipeline.py
@@ -305,7 +305,7 @@ def is_ocr_required(pageinfo, log, options):
     return ocr_required
 
 
-def split_pages(
+def pre_split_pages(
         input_files,
         output_files,
         log,
@@ -331,20 +331,45 @@ def split_pages(
 
     pdfinfo = context.get_pdfinfo()
     npages = len(pdfinfo)
-    qpdf.split_pages(input_file, work_folder, npages)
 
-    from glob import glob
-    for filename in glob(os.path.join(work_folder, '*.page.pdf')):
-        pageinfo = get_pageinfo(filename, context)
+    # Ruffus needs to see a file for any task it generates, so create
+    # empty placeholders for every page.
+    for n in range(npages):
+        page = Path(work_folder) / '{0:06d}.presplit.pdf'.format(n + 1)
+        page.touch()
 
+
+def split_page(
+        placeholder_file,
+        output_file,
+        log,
+        context):
+    pageno = page_number(placeholder_file) - 1
+    input_pdf = context.get_pdfinfo().filename
+    qpdf.extract_page(input_pdf, output_file, pageno)
+
+
+def ocr_or_skip(
+        input_files,
+        output_files,
+        log,
+        context):
+    options = context.get_options()
+    work_folder = context.get_work_folder()
+    pdfinfo =  context.get_pdfinfo()
+
+    for input_file in input_files:
+        pageno = page_number(input_file) - 1
+        pageinfo = pdfinfo[pageno]
         alt_suffix = \
             '.ocr.page.pdf' if is_ocr_required(pageinfo, log, options) \
             else '.skip.page.pdf'
+
         re_symlink(
-            filename,
+            input_file,
             os.path.join(
                 work_folder,
-                os.path.basename(filename)[0:6] + alt_suffix),
+                os.path.basename(input_file)[0:6] + alt_suffix),
             log)
 
 
@@ -1035,16 +1060,31 @@ def build_pipeline(options, work_folder, log, context):
         extras=[log, context])
 
     # Split (kwargs for split seems to be broken, so pass plain args)
-    task_split_pages = main_pipeline.split(
-        split_pages,
+    task_pre_split_pages = main_pipeline.split(
+        pre_split_pages,
         task_repair_pdf,
-        os.path.join(work_folder, '*.page.pdf'),
+        os.path.join(work_folder, '*.presplit.pdf'),
+        extras=[log, context])
+
+    task_split_pages = main_pipeline.transform(
+        task_func=split_page,
+        input=task_pre_split_pages,
+        filter=suffix('.presplit.pdf'),
+        output='.page.pdf',
+        output_dir=work_folder,
+        extras=[log, context])
+
+    task_ocr_or_skip = main_pipeline.split(
+        ocr_or_skip,
+        task_split_pages,
+        [os.path.join(work_folder, '*.ocr.page.pdf'),
+         os.path.join(work_folder, '*.skip.page.pdf')],
         extras=[log, context])
 
     # Rasterize preview
     task_rasterize_preview = main_pipeline.transform(
         task_func=rasterize_preview,
-        input=task_split_pages,
+        input=task_ocr_or_skip,
         filter=suffix('.page.pdf'),
         output='.preview.jpg',
         output_dir=work_folder,
@@ -1054,7 +1094,7 @@ def build_pipeline(options, work_folder, log, context):
     # Orient
     task_orient_page = main_pipeline.collate(
         task_func=orient_page,
-        input=[task_split_pages, task_rasterize_preview],
+        input=[task_ocr_or_skip, task_rasterize_preview],
         filter=regex(r".*/(\d{6})(\.ocr|\.skip)(?:\.page\.pdf|\.preview\.jpg)"),
         output=os.path.join(work_folder, r'\1\2.oriented.pdf'),
         extras=[log, context])