From 795019b0c1de8964d6a7cd279d08cdf3fb37b2ab Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 11 Sep 2018 14:44:16 -0700 Subject: [PATCH] Work around invalid TOC entries Kodak Capture Desktop and probably other software creates a /Outlines entry with /First being set to an invalid indirect reference to an object that hasn't been created. This is legal in the PDF spec but problematic for qpdf. The objgen will be (max valid object ID + 1, 0). Because we create new objects in _weave, some TOC entries will end up assigned to new objects we create. Typically /ProcSet. We solve the issue by refactoring page traversal and then doing it twice, once to resolve all references (eliminating the null reference problem) and a second pass to make our changes. --- src/ocrmypdf/_weave.py | 91 ++++++++++++++++++++++++++----------- tests/resources/README.rst | 5 ++ tests/resources/kcs.pdf | Bin 0 -> 4372 bytes tests/test_metadata.py | 12 +++++ 4 files changed, 81 insertions(+), 27 deletions(-) create mode 100755 tests/resources/kcs.pdf diff --git a/src/ocrmypdf/_weave.py b/src/ocrmypdf/_weave.py index 450e6417..c4c20b79 100644 --- a/src/ocrmypdf/_weave.py +++ b/src/ocrmypdf/_weave.py @@ -137,14 +137,9 @@ def _find_font(text, pdf_base): return font, font_key -def _fix_toc(pdf_base, pageref_remap, log): - """Repair the table of contents - - Whenever we replace a page wholesale, it gets assigned a new objgen number - and other references to it within the PDF become invalid, most notably in - the table of contents (/Outlines in PDF-speak). In weave_layers we collect - pageref_remap, a mapping that describes the new objgen number given an old - one. (objgen is a tuple, and the gen is almost always zero.) +def _traverse_toc(pdf_base, visitor_fn, log): + """ + Walk the table of contents, calling visitor_fn() at each node The /Outlines data structure is a messy data structure, but rather than navigating hierarchically we just track unique nodes. Enqueue nodes when @@ -152,8 +147,6 @@ def _fix_toc(pdf_base, pageref_remap, log): the two types of object in the table of contents that can be page bookmarks and update the page entry. - It may ultimately be better to find a way to rebuild a page in place. - """ visited = set() @@ -162,10 +155,54 @@ def _fix_toc(pdf_base, pageref_remap, log): if not '/Outlines' in pdf_base.root: return + + queue.add(pdf_base.root.Outlines.objgen) + while queue: + objgen = queue.pop() + visited.add(objgen) + node = pdf_base.get_object(objgen) + log.debug('fix toc: exploring outline entries at %r', objgen) + + # Enumerate other nodes we could visit from here + for key in link_keys: + if key not in node: + continue + item = node[key] + if not item.is_indirect: + # or not isinstance(item, pikepdf.Dictionary): + # # If there is garbage data, replace the key with an indirect + # # ref to None. Kodak Capture Desktop produces keys like these. + # log.error('Removing invalid reference from TOC: %s', repr(item)) + # node[key] = pdf_base.make_indirect(None) + continue + objgen = item.objgen + if objgen not in visited: + queue.add(objgen) + + visitor_fn(pdf_base, node, log) + + +def _fix_toc(pdf_base, pageref_remap, log): + """Repair the table of contents + + Whenever we replace a page wholesale, it gets assigned a new objgen number + and other references to it within the PDF become invalid, most notably in + the table of contents (/Outlines in PDF-speak). In weave_layers we collect + pageref_remap, a mapping that describes the new objgen number given an old + one. (objgen is a tuple, and the gen is almost always zero.) + + It may ultimately be better to find a way to rebuild a page in place. + + """ + if not pageref_remap: return def remap_dest(dest_node): + """ + Inner helper function: change the objgen for any page from the old we + invalidated to its new one. + """ if not isinstance(dest_node, pikepdf.Array): return pageref = dest_node[0] @@ -174,30 +211,24 @@ def _fix_toc(pdf_base, pageref_remap, log): new_objgen = pageref_remap[pageref.objgen] dest_node[0] = pdf_base.get_object(new_objgen) - queue.add(pdf_base.root.Outlines.objgen) - while queue: - objgen = queue.pop() - visited.add(objgen) - node = pdf_base.get_object(objgen) - log.debug('fix toc: visiting %r', objgen) - - # Enumerate other nodes we could visit from here - for key in link_keys: - if key not in node: - continue - item = node[key] - if not item.is_indirect: - continue - objgen = item.objgen - if objgen not in visited: - queue.add(objgen) + def visit_remap_dest(pdf_base, node, log): + """ + Visitor function to fix ToC entries + Test for the two types of references to pages that can occur in ToCs. + Both types have the same final format (an indirect reference to the + target page). + """ if '/Dest' in node: + # /Dest reference to another page (old method) remap_dest(node['/Dest']) elif '/A' in node: + # /A (action) command set to "GoTo" (newer method) if '/S' in node['/A'] and node['/A']['/S'] == '/GoTo': remap_dest(node['/A']['/D']) + _traverse_toc(pdf_base, visit_remap_dest, log) + def weave_layers( infiles, @@ -247,6 +278,12 @@ def weave_layers( pdfinfo = context.get_pdfinfo() pagerefs = {} + # Walk the table of contents first, to trigger pikepdf/qpdf to resolve all + # page references in the table of contents. Some PDF generators put invalid + # references in the ToC, so we want to resolve them to null before we + # create any references, or the ToC will be corrupted + _traverse_toc(pdf_base, lambda *args: None, log) + procset = pdf_base.make_indirect( pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]')) diff --git a/tests/resources/README.rst b/tests/resources/README.rst index cc2caa8a..63fdd705 100644 --- a/tests/resources/README.rst +++ b/tests/resources/README.rst @@ -104,6 +104,11 @@ licensed under the specified license. - @jbarlow83 - @jbarlow83 - CC-BY-SA 4.0 + * - kcs.pdf + - PDF file generated by Kodak Capture Desktop Software 1.2; has invalid table of contents + - @jbarlow83 + - @jbarlow83 + - CC-BY-SA 4.0 * - masks.pdf - file containing explicit masks and a stencil mask drawn without a proper transformation matrix; printout of a German Wikipedia article (CC-BY-SA) - @supergrobi diff --git a/tests/resources/kcs.pdf b/tests/resources/kcs.pdf new file mode 100755 index 0000000000000000000000000000000000000000..b0d46990e5839d3a82f6f744fbcf79f893ecd302 GIT binary patch literal 4372 zcmY!laBSW1m|7|r85t|Ya_Kwg=ar=9l@u$Og3RHvv*Sw5O98nD z#0SwfHu^rPdFdq?3T7Y`c6MCFB}J);xm<-@AY(wrfs6vt$+=u$C%NQv1#*GRfvSX> z2+|KS(by0q2V#NT4x-_Hhx(06Ke#lh1RNS*UvTOBCFZ7rykx+o@0OWUl3Jvo@9gXu z65^Iv;gXu1pOVU@AD)>~lA&N|X=uWw?~$6Bo>8J;X>7oy@03|m9FSV%oS$2ep9cz4 zLy)g?@{58C5|dLE^j%WRGLuu?ixMlj^dV*iBo^fsgTmKaLDx`0-#I_0G&ippY&0k^ zp}}E+8XW&2;Aa}+uYL&g|Nohc|EI@yzzM z!9Vru+GUX~i|T_2WjSr&V{G73GDG2XVdxPJ|0gBFoL^4-YL%UMXU%Lz9o{ zp1!8A6z>+X#T?y?tPY3%s!TTG=9>0U@X~@U*B=SAzJAyuHpBZN%L;?Nhd8fRx2$xu z6J^=;k+bSEC+owi1-m&#kH`x&eR0^txtIMyXp8WrTGodG5zZy=IP{uWzSgoe`xw5^foA#_2t9b zdhbKrfAl8(2wKJZ>S4X3eQZxFW8b6?F14z!{t3kG4WJHYxEcFop`Bs zX~dzgTyK=DIBQEccx+esvnu<(BY(@Q)ecQ-P4*;AZ_2Du2yc1)VA|_HsxJ@y&`qr8 z=$rhtW$Mw=_``nxHrssss(YHz)?|^2veLAfd0M>Rxt{Lq6$nbvxpHbtOiD@PuaFiy zj^>Cr0v+jTj!R!=U+|q-AZ~2(N7X&6vE_%uoRbG*6h3b}MYyp2$me3)^)!P+UX>>ikSl}DsaHvIYnlOtjlRzcgkq2CEwhnDa=Xnn4Dm8gh8MZ7hTU=yMn4R!wdg#x2nQMn?Q;Xt>tP?pVUv6d1 z6Uxy$$i!PP+s|=bLGz`p?X8@zw>U1adTG{?CZ8wl*J#%4V)Q!m!PFLKtyPH=7N0wL zU{cvU$8}Pi*BxyXzPX7xM)-#NE|OfX*cIS-P3uBT+g5%agOKJnQGv71SGOFLYFx^~ z*I{Dm{Lm^!pgH-luZH^VjK-1&jhB=fE;Vx4s5e!q95^QIt0GW;h>gRX!)Q@c)s~%X zjq1y0cgJU-*O@J z{jy7!suTac)n{w+tj>D%^)}no-SrE(nqv3vy0oLJqGvrT_B$wP~EclEmI zIYw5>zpCp?yk2o{`!{dijc)z$znLsfRjmL2mYp!wx&CF)X^Ub+Y z`~I~Fi=>pB!}s6zP-;Jv+mSKddSoY{jUW5zsZQ0{QtMU z?*7_`ZHIGuSmR$-#O{BeeVfO^phH5&{(8y2f4g4YR$TW^vwgjHKxp2#|9jh>H=C-- z|37rQ{^L3cgTp4w9FLbdn@7ZlC+x1zyt+H5Ig3d=H|XmP?d|oqZkS2(ZaCC(xL}v} zCT`wWZ?EqDV{ntzhHpmM{v9bIv!{o>-}*kskhkS0*C)~=LTsJ$`c6(;KV`ey8 zh2o4m-nrM_?=d)>Yo4;kSUgJD;o|QbS+BRrwn!(Om|^;Bk9^|2TV=D`cut6jImfhx zPL0}Y_d9oW*2_2M3C%~sTW`O;6>zAEpT}5XfjOISbk^SYxe?jriET31cn^D4_~&ou z)#iL*+s5bJ&bF}X!|xo^#HO-2W`{df*jHaJS_aZOf+iiVktUh#Tly7)< zr#Uc;=Yw(9;ls_&TpOk|7EE*Y-M8vC+hifB!`m*UNOj%FvC(IpaK&}P^}O0`j$4&7 z-_7JTtWI2^eo&+2gwcZ2E6Wwu8C|^*$#z_W@8OCVr~D+>m@OxcZn()*?Q6YM$m>S2 zvOyrr``txp=Z;jCZaW%WCX_d4eWjK1Cmu^yhYhQ4<A^37zGl#*-I|i}L9B(y5t(!R~ zT-RVZ#B1?JA(5H;w!EQvS#KrOcV;q4>Y^WqcH zTW)?GLMBryj4o-r6x==`x}~ALEy{^8n`cW0^J|{XJ`-DF_=Qtvd9gRwPCaum)uUHX zR)+cDqMNZP0nIGRHyjs6hiwyHW2@8L_LKQA^MvKxE(vBu>^+`2t9kVVKICS%{bW`D zQ0E+DpPyo2d0c75hrF9MPKA1hpE)lS{9U1tfBQmxjqtnM>z94~T{eIFpLOE@c4;?p z-u}+=`~LFV|KILg`Tzg^|M_44|Ihos{r|VuzxMvW|LgzXW%vL7US0dXZr@&?DT4L8 ze*b>^pU1B6mGJsQx2->1C@c84r<_Id4%_+P&(juM3Ol&?^Iq4#EvHsw{nVEfk}E5S ze<#fO)b~ZlGN(5pZAXtiui>(KFssaw<%(KakmL-7`?pfoh^|nd>&&vkOtz6Zd9p+H z`=)iA&l2tkTL*CMsxJswx3l03=iDo7zv?ghb4&3EA1ePaBSvJ6!a{-Eo2w6%crSX` zWHmAV&^nQB$qm{Z5_%#_nI#m(*B)B8ke5yPvSMYysm%vh%KNQ#@?5pTiI=}h;j-03 zg_SLL4Q9kIXkNEcO`L0{HuLq1JX^jvYi(^=X)sqwviZ=&0^7e#RXjQ>lTu>%S+5Eh zdibb*nJaK(VuqHFAgl3hhvw|KOBK!wCIxgDz4{_h>iwea;93iZCHyTN60(bUSAA); zDq?SY)3&dvoxZb4rtqZ-OT>gT+7i?zEpc2W(zIFOY!Em5QV;E>T0Lj0nzqXd{tG5oab7ABuHruU zy2>#{sw~^l);w-H?@C?^?p+R3OdUN0+?Hs3;l31+I3<9iern>h!&_$Z$41@CU)!dC zt&x@AbLJ_wOPPBwU%S7{B~2)Ii9mn%zv(=ZBK03y4XUDIe=X%*zH15B-KCf7z8;qU zmaN=%N3OBDl{!f6*|%o6z@_*ZcSN1&Ql^ zver)L{rmlj`Im*VOL#Tv|8q!JeQ@03u)BTx|2Fy330H6Io{-eIclR%j_Xk&2`80+YwROJ#(5?oygnHX(GGevg4aGp@$Vtt@!ss;j-h* zP|uywtxas6ZAWf0H66(R(3D}KaFC<@5F4j8+u`3$9B&?+yI|m((H8K>i09A}DYt5Y z#t9lBol_mRe2mU%TllZ<2(v~av*I${8f%dq4hKyn!lQS;c*85s*1VtBXy**`SIzlB z+J}S|Td$E~R(MsmVp5~>`X=E`4;ED2UDYEPGsAGhnx#z(%GF-kDD7Cg)#0*lg>!Ct znN;GM9Nm_SHyNk!T@z&!c28n#o6mJ{S>G8R6A2x4tG48Bj-0l#_DK(R$}&4#6I0?* zR=RNZnwm?)v5gyIS8tei=<)ym_kZpC|Nr*aZ~wo%|F>`dwS}v>&RA%GIw4O43KACo z|Nno=3H`^@7S6ig|NrFHc(8b#+L>_9ysP&dxl*uWGd3t}6>yS3n+X-a8w zYLSA5YhrOpZepH-cYaD@wgRY2mY7$mprD|j$))dHl$uzQU!BuOD$Hg1ohOA^n*IpAUDFh zouDpvF{oEqnpdJ=$ffU{nNq9}tpMtp2XVzB=>(Ys(WwBn8fFQoUs_aLqF@N>2nTWL z`y>{ZKrPmH&Ig%mfMgC=MNw)Rmw|$z0ha+BD43a=8k;JlDL}=HEetFaz_JQ?a4|~* z157b96Lc{H0|QG8F+(#$3^5}UV@xqjb;jlv7;Z5!Ffu~uEh$RO%t74QY55AEWDx`kh~kpOq7rZ@m>5`^a;d7i`nv%DSe0i+ literal 0 HcmV?d00001 diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 0bd0736e..ac110081 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -265,3 +265,15 @@ def test_srgb_in_unicode_path(tmpdir): with patch('ocrmypdf.pdfa.SRGB_ICC_PROFILE', new=str(dst)): generate_pdfa_ps(dstdir / 'out.ps', {}) + + +def test_kodak_toc(resources, outpdf, spoof_tesseract_noop): + output = check_ocrmypdf( + resources / 'kcs.pdf', outpdf, + '--output-type', 'pdf', + env=spoof_tesseract_noop) + + p = pikepdf.open(outpdf) + + if pikepdf.Name.First in p.root.Outlines: + assert isinstance(p.root.Outlines.First, pikepdf.Dictionary)