diff --git a/setup.py b/setup.py index 560434f8..81003662 100644 --- a/setup.py +++ b/setup.py @@ -99,7 +99,7 @@ setup( 'cffi >= 1.9.1', # must be a setup and install requirement 'img2pdf >= 0.3.0, < 0.4', # pure Python, so track HEAD closely 'pdfminer.six == 20181108 ; sys_platform != "darwin"', - 'pikepdf >= 1.1.0, < 2', + 'pikepdf >= 1.2.0, < 2', 'Pillow >= 4.0.0, != 5.1.0 ; sys_platform == "darwin"', # Pillow < 4 has BytesIO/TIFF bug w/img2pdf 0.2.3 # block 5.1.0, broken wheels diff --git a/src/ocrmypdf/_weave.py b/src/ocrmypdf/_weave.py index 17e37144..a66244a2 100644 --- a/src/ocrmypdf/_weave.py +++ b/src/ocrmypdf/_weave.py @@ -25,7 +25,7 @@ from .exec import tesseract from .helpers import flatten_groups, page_number -MAX_OPEN_PAGE_PDFS = int(os.environ.get('_OCRMYPDF_MAX_OPEN_PAGE_PDFS', 100)) +MAX_REPLACE_PAGES = int(os.environ.get('_OCRMYPDF_MAX_REPLACE_PAGES', 100)) def _update_page_resources(*, page, font, font_key, procset): @@ -321,7 +321,6 @@ def weave_layers(infiles, output_file, log, context): base = list(basegroup)[0] path_base = Path(base).resolve() pdf_base = pikepdf.open(path_base) - keep_open = [] font, font_key, procset = None, None, None pdfinfo = context.get_pdfinfo() pagerefs = {} @@ -330,6 +329,8 @@ def weave_layers(infiles, output_file, log, context): pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]') ) + replacements = 0 + # Iterate rest for page_num, layers in groups: layers = list(layers) @@ -353,7 +354,7 @@ def weave_layers(infiles, output_file, log, context): old_objgen = pdf_base.pages[page_num - 1].objgen with pikepdf.open(image) as pdf_image: - keep_open.append(pdf_image) + replacements += 1 image_page = pdf_image.pages[0] pdf_base.pages[page_num - 1] = image_page @@ -394,20 +395,19 @@ def weave_layers(infiles, output_file, log, context): content_rotation - autorotate_correction ) % 360 - if len(keep_open) > MAX_OPEN_PAGE_PDFS: - # qpdf limitations require us to keep files open when we intend - # to copy content from them before saving. However, we want to keep - # a lid on file handles and memory usage, so for big files we're - # going to stop and save periodically. Attach the font to page 1 - # even if page 1 doesn't use it, so we have a way to get it back. + if replacements % MAX_REPLACE_PAGES == 0: + # Periodically save and reload the Pdf object. This will keep a + # lid on our memory usage for very large files. Attach the font to + # page 1 even if page 1 doesn't use it, so we have a way to get it + # back. + # TODO refactor this to outside the loop page0 = pdf_base.pages[0] _update_page_resources( page=page0, font=font, font_key=font_key, procset=procset ) interim = output_file + f'_working{page_num}.pdf' pdf_base.save(interim) - del pdf_base - keep_open = [] + pdf_base.close() pdf_base = pikepdf.open(interim) procset = pdf_base.pages[0].Resources.ProcSet @@ -415,3 +415,4 @@ def weave_layers(infiles, output_file, log, context): _fix_toc(pdf_base, pagerefs, log) pdf_base.save(output_file) + pdf_base.close() diff --git a/tests/test_weave.py b/tests/test_weave.py index 34452215..372eddca 100644 --- a/tests/test_weave.py +++ b/tests/test_weave.py @@ -53,7 +53,7 @@ def test_no_glyphless_weave(resources, outdir): pdf.save(outdir / 'test.pdf') env = os.environ.copy() - env['_OCRMYPDF_MAX_OPEN_PAGE_PDFS'] = '2' + env['_OCRMYPDF_MAX_REPLACE_PAGES'] = '2' check_ocrmypdf( outdir / 'test.pdf', outdir / 'out.pdf',