From 537aaf56d763463fe1f2cd763dd3c99c46db7888 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 23 Mar 2018 13:38:05 -0700 Subject: [PATCH 1/4] Remove duplication between page merge functions --- ocrmypdf/pipeline.py | 69 ++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/ocrmypdf/pipeline.py b/ocrmypdf/pipeline.py index 9db5d16e..a5682feb 100644 --- a/ocrmypdf/pipeline.py +++ b/ocrmypdf/pipeline.py @@ -900,29 +900,42 @@ def skip_page( re_symlink(input_file, output_file, log) +def _merge_pages_common( + input_files_groups, + output_file, + log, + context): + """Determine ordered list of PDF pages to merge. Returns PDF from which + metadata should be drawn if present (for qpdf).""" + + input_files = list(f for f in flatten_groups(input_files_groups) + if not f.endswith('.txt')) + metadata_file = next( + (ii for ii in input_files if ii.endswith('.repaired.pdf')), None) + input_files.remove(metadata_file) + + def input_file_order(s): + '''Sort order: All rendered pages followed + by their debug page.''' + key = page_number(s) * 10 + if 'debug' in os.path.basename(s): + key += 1 + return key + + pdf_pages = sorted(input_files, key=input_file_order) + log.debug("Final pages: " + "\n".join(pdf_pages)) + + return pdf_pages, metadata_file + + def merge_pages_ghostscript( input_files_groups, output_file, log, context): options = context.get_options() - def input_file_order(s): - '''Sort order: All rendered pages followed - by their debug page, if any, followed by Postscript stub. - Ghostscript documentation has the Postscript stub at the - beginning, but it works at the end and also gets document info - right that way.''' - if s.endswith('.ps'): - return 99999999 - key = page_number(s) * 10 - if 'debug' in os.path.basename(s): - key += 1 - return key - - input_files = (f for f in flatten_groups(input_files_groups) - if not f.endswith('.txt')) - pdf_pages = sorted(input_files, key=input_file_order) - log.debug("Final pages: " + "\n".join(pdf_pages)) + pdf_pages, _ = _merge_pages_common( + input_files_groups, output_file, log, context) input_pdfinfo = context.get_pdfinfo() ghostscript.generate_pdfa( pdf_version=input_pdfinfo.min_version, @@ -940,23 +953,8 @@ def merge_pages_qpdf( log, context): options = context.get_options() - - input_files = list(f for f in flatten_groups(input_files_groups) - if not f.endswith('.txt')) - metadata_file = next( - (ii for ii in input_files if ii.endswith('.repaired.pdf'))) - input_files.remove(metadata_file) - - def input_file_order(s): - '''Sort order: All rendered pages followed - by their debug page.''' - key = page_number(s) * 10 - if 'debug' in os.path.basename(s): - key += 1 - return key - - pdf_pages = sorted(input_files, key=input_file_order) - log.debug("Final pages: " + "\n".join(pdf_pages)) + pdf_pages, metadata_file = _merge_pages_common( + input_files_groups, output_file, log, context) reader_metadata = pypdf.PdfFileReader(metadata_file) pdfmark = get_pdfmark(reader_metadata, options) @@ -965,7 +963,8 @@ def merge_pages_qpdf( first_page = pypdf.PdfFileReader(pdf_pages[0]) writer = pypdf.PdfFileWriter() - writer._header = b'%PDF-' + _pdf_guess_version(pdf_pages[0]) # copy version from source + # copy version from source + writer._header = b'%PDF-' + _pdf_guess_version(pdf_pages[0]) writer.appendPagesFromReader(first_page) writer.addMetadata(pdfmark) writer_file = pdf_pages[0].replace('.pdf', '.metadata.pdf') From 3795d6720f88692101ba13c537776b79283b4d0a Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 23 Mar 2018 13:39:16 -0700 Subject: [PATCH 2/4] Try out pymupdf merging With garbage collection it reduces waste on the worst case file. That's nice. 1 MB -> 105 MB -> 1.5 MB. Indicates really problem is using PyPDF2 to watermark. Currently hacked into --output-type pdf. --- ocrmypdf/pipeline.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/ocrmypdf/pipeline.py b/ocrmypdf/pipeline.py index a5682feb..c2cc39f7 100644 --- a/ocrmypdf/pipeline.py +++ b/ocrmypdf/pipeline.py @@ -978,6 +978,29 @@ def merge_pages_qpdf( log=log) +def merge_pages_mupdf( + input_files_groups, + output_file, + log, + context): + options = context.get_options() + + pdf_pages, metadata_file = _merge_pages_common( + input_files_groups, output_file, log, context) + + import fitz + doc = fitz.Document() + + for pdf_page in pdf_pages: + page = fitz.open(pdf_page) + doc.insertPDF(page) + + metadata = fitz.open(metadata_file) + doc.setToC(metadata.getToC()) + doc.setMetadata(metadata.metadata) + doc.save(output_file, garbage=4) + + def merge_sidecars( input_files_groups, output_file, @@ -1254,7 +1277,7 @@ def build_pipeline(options, work_folder, log, context): task_merge_pages_ghostscript.active_if(options.output_type.startswith('pdfa')) task_merge_pages_qpdf = main_pipeline.merge( - task_func=merge_pages_qpdf, + task_func=merge_pages_mupdf, input=[task_combine_layers, task_render_hocr_debug_page, task_skip_page, From 376dfdba1c07d1a64554a9e697b7892d6bf085a7 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 23 Mar 2018 13:48:37 -0700 Subject: [PATCH 3/4] Fix text/image files not closed in combine_layers --- ocrmypdf/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ocrmypdf/pipeline.py b/ocrmypdf/pipeline.py index c2cc39f7..b3cb7f1c 100644 --- a/ocrmypdf/pipeline.py +++ b/ocrmypdf/pipeline.py @@ -705,8 +705,8 @@ def combine_layers( image = next(ii for ii in flatten_groups(infiles) if ii.endswith('.image-layer.pdf')) - pdf_text = pypdf.PdfFileReader(open(text, "rb")) - pdf_image = pypdf.PdfFileReader(open(image, "rb")) + pdf_text = pypdf.PdfFileReader(text) + pdf_image = pypdf.PdfFileReader(image) page_text = pdf_text.getPage(0) From 74bdfc07fb9c9e9d6e63f55fc44c7e06f960f127 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sat, 24 Mar 2018 01:18:22 -0700 Subject: [PATCH 4/4] mumerge: fix regressions --- ocrmypdf/pipeline.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/ocrmypdf/pipeline.py b/ocrmypdf/pipeline.py index b3cb7f1c..fddd93c4 100644 --- a/ocrmypdf/pipeline.py +++ b/ocrmypdf/pipeline.py @@ -912,11 +912,17 @@ def _merge_pages_common( if not f.endswith('.txt')) metadata_file = next( (ii for ii in input_files if ii.endswith('.repaired.pdf')), None) - input_files.remove(metadata_file) + if metadata_file in input_files: + input_files.remove(metadata_file) def input_file_order(s): '''Sort order: All rendered pages followed - by their debug page.''' + by their debug page, if any, followed by Postscript stub. + Ghostscript documentation has the Postscript stub at the + beginning, but it works at the end and also gets document info + right that way.''' + if s.endswith('.ps'): + return 99999999 key = page_number(s) * 10 if 'debug' in os.path.basename(s): key += 1 @@ -990,14 +996,19 @@ def merge_pages_mupdf( import fitz doc = fitz.Document() - + + reader_metadata = pypdf.PdfFileReader(metadata_file) + pdfmark = get_pdfmark(reader_metadata, options) + pdfmark['/Producer'] = 'qpdf ' + qpdf.version() + pymupdf_metadata = {k[1:].lower() : v for k, v in pdfmark.items()} + for pdf_page in pdf_pages: page = fitz.open(pdf_page) doc.insertPDF(page) metadata = fitz.open(metadata_file) doc.setToC(metadata.getToC()) - doc.setMetadata(metadata.metadata) + doc.setMetadata(pymupdf_metadata) doc.save(output_file, garbage=4)