From 6f3ac46b1c176d48782347cfa14d9ef6ce773f37 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sat, 20 Feb 2016 04:53:02 -0800 Subject: [PATCH] Gracefully recover from tesseract's failure to process very large images And test cases to check this --- ocrmypdf/tesseract.py | 22 +++++++--- tests/resources/hugemono.pdf | Bin 0 -> 11779 bytes tests/spoof/tesseract_big_image_error.py | 50 +++++++++++++++++++++++ tests/spoof/tesseract_cache.py | 7 ++++ tests/test_main.py | 15 +++++++ 5 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 tests/resources/hugemono.pdf create mode 100755 tests/spoof/tesseract_big_image_error.py diff --git a/ocrmypdf/tesseract.py b/ocrmypdf/tesseract.py index f25d0d3e..020540cc 100644 --- a/ocrmypdf/tesseract.py +++ b/ocrmypdf/tesseract.py @@ -98,6 +98,8 @@ def get_orientation(input_file, language: list, timeout: float, log): return OrientationConfidence(angle=0, confidence=0.0) except CalledProcessError as e: tesseract_log_output(log, e.output, input_file) + if 'Image too large' in e.output: + return OrientationConfidence(0, 0) raise e from e else: osd = {} @@ -148,6 +150,13 @@ def page_timedout(log, input_file): log.warning(prefix + " took too long to OCR - skipping") +def _generate_null_hocr(output_hocr, pageinfo): + with open(output_hocr, 'w', encoding="utf-8") as f: + f.write(HOCR_TEMPLATE.format( + pageinfo['width_pixels'], + pageinfo['height_pixels'])) + + def generate_hocr(input_file, output_hocr, language: list, tessconfig: list, timeout: float, pageinfo_getter, pagesegmode: int, log): @@ -175,13 +184,13 @@ def generate_hocr(input_file, output_hocr, language: list, tessconfig: list, # Temporary workaround to hocrTransform not being able to function if # it does not have a valid hOCR file. page_timedout(log, input_file) - with open(output_hocr, 'w', encoding="utf-8") as f: - pageinfo = pageinfo_getter() - f.write(HOCR_TEMPLATE.format( - pageinfo['width_pixels'], - pageinfo['height_pixels'])) + _generate_null_hocr(output_hocr, pageinfo_getter()) except CalledProcessError as e: tesseract_log_output(log, e.output, input_file) + if 'Image too large' in e.output: + _generate_null_hocr(output_hocr, pageinfo_getter()) + return + raise e from e else: tesseract_log_output(log, stdout, input_file) @@ -242,6 +251,9 @@ def generate_pdf(input_image, skip_pdf, output_pdf, language: list, shutil.copy(skip_pdf, output_pdf) except CalledProcessError as e: tesseract_log_output(log, e.output, input_image) + if 'Image too large' in e.output: + shutil.copy(skip_pdf, output_pdf) + return raise e from e else: tesseract_log_output(log, stdout, input_image) diff --git a/tests/resources/hugemono.pdf b/tests/resources/hugemono.pdf new file mode 100644 index 0000000000000000000000000000000000000000..30700b3e401a76d444ab73f138cc07dbe5b7efa2 GIT binary patch literal 11779 zcmY!laB5EaQ;QTdax#-jGSku&^c6sqQ9(+Yg06y*fuX5^rJ<1~mp<5<{2~Pn z$CUh}RE2ZBj8)*$cg`=(D^V~6`4+ApBn#0G3n7qpaPSnR=9R$0N8dL!B{R_}zd|9} zK+nLyz(4^?S(q50vmjRGm89mC6f2m5ZRgStN-fSWElN%;Rsg#+!apf1HMvB=#)eDZ zGuKeT9IVXFj!QqFC_gzkwL~FWLEkesF+J5uAr=%nhzJ6i2yw9jm%dMGUV2G}f*Dvl zmx7%gS8+*EYGN)|p@IU!X;2qJDO}>oxgaYI6@_n1$$?lP zM}ugvkDxBGfVc#j+F;(~(hn|8Dgh@zu%{Hb^!*ZZQ^EFe>4#^glw>Fvn;IAxaOr!b zW~OJ9z<5rXCB*@$Mb7!T1^IcP_%H-TV@`fia6w{ns)D{tYFTD-s(Vplr2?0}TV_s4 zYLSAzv$JPNh+ATXOKNg{3Rn?{4oED@1&6-Aw}P&rg1&QpPHAplF~lYX{h<7EDBI2s zIUxU!g3%BlJp>f&m><|gNIqddP{ZEZxXz)5{Xm6>+0v$l6TF*zjLvlMY`VAMv~FSJ zwA2Nu(;g*Gi#jKnp6NI((%V@_`2YVO@&Es~La6os|Gc{v$r%>y+Ua*`qWRVO|A!tH zEL2!n|NmjZj!-V~|Nq3fTLTu>|9`!1)$0HMxmypd2-yGs!;Tm|5&i#vORuWN|NjxA zC+gJt|9|VDhannZt~f~Fu2-wz3jgSdb4>`?|Nrat*l@6khg$#t|Nr{`|Ns9SH?9Bw z|Nl>3-&ytl|Nm$GG@F&@{{R19_W%F?{}3!(Kq|ie7d+zr@&Et-@&EtUgAh#e|Ns9W zc_c0Y=7|jk;ikN-yBaQpi!wdAnP5CqoqwW3%sL6Z=%q~0Q<$E`u%21LJyU|`pKD{& zjtk;vra3m{c(9&n0!xNTp4{FnIVmFj^QH~w|Ma#cPFj&E_MBH~M?;RuTjI?XX|ijWBlEUN z?wqJ#`t!yD(|R@LfD>EHy32VN?PBzp&C8sjP^l)B=B>CYdJ5YDC8=sIg{51Sq&9ux zT{H7!!xil_BAXQguBshKSdx+@)zFdoMx!~q)JSUgwUi~i`^rps_a9`oUXVJskIT&= zaax&$rsy3pp9zx76Pa~pHY?suSfG&da8{Xs(FTWeUwAe+oZ6sQmLi!F!`;FoV7#dz zl7(kA)AOok#i@=n${tQDXGmDVvq^wooP}py7Pp!U`yFBnoVH+7 zxq+U5fy22|j#J(ToVH$&I%$I*hh!Aml<&_8$rd**GK=J~;fXXW4s z3JHz(1Z))EZD>Bk|Dl<~z~K--hy4E*35EI=*$WC4jx7?7{{$)=e>mLLf5_ezm z!SRLtA@&yehin1~%^WrYxohhL5}N-g{1E`DcKFX>qfp0Ty&kMOk?kdmTyyCmeh&LX z83xT9HVVJv5Ai7c=P(awK9peqGFst}!tMAMHVPk_Ijj%y`@MF2Xn!HmpqcX_`~MdP z&41ZT1gdz{n*Xvd_@i)F|Di4Wf~VkrQ@o`Oo># z-gnFX1s|FZ@hIF?Xui7jLWTdALy2s{hj`!rWfQ1ha66#+GG~kY3mb($3h(wWU%qVa zL-zGiRs3Hr`?9yluYbAxhyRidA*95(7dd<|y!`@jA$ z!~e^D&W9!!>X$G0Yj)}1ym_c{6+z{`+&-bC}1!eED+OOZ6qMmoNX7xoqD2|I6mT z+_ik!tor54Uv8WJ<;$0AU%u|TeEF|w$d`Znee3;RmRz;JeEAF5>GPLgfBEv?tjy)h zZmoa0{8!z)S(*Rl&0qHVudS`gU6rq;moNVf_O-S9f7$=b-QcSR^Iw7kXMOGR3jgS} zw)U4V`*Z%STlsIh=gXJ>cGU@Q*}r_<>Pxlu_vXKvwS3vF|I79-`}GR!kiB&v6Z50J z^!rd8_zorwyFaF!pw6FgnYi@LAox=w66WJ=Cq)uwIU63xB$h$Ou z`4chjFOFN}FS2cT(HzMuaEm#kIm1eE%i8^a8WVB_ZZX#{l1kiuk#~XlBHqM{Y@4JM zw+a61$ZY%KoTL23IWzk8Csu}EEW9rS%sR5W|1v2ju3ftn#I+ZaT>43}^+nr`y-e&5 z9XGP0)_EP)b66lNa9w<6MCwO{3;dZ%QjdO2IIBP5jJ}IB!~G-84wfN2TO2p=PHi}C zWRbeSCC7Aj5Qk(PhxtMF2hE4r-a9nce`u~z_#wUr@Nc^oK$%i}VG@Edu49EOMIH1o9o(AF?~LU2p`!-~TvdS>zwG zmpZby$g)T~vi%dNWs&|OP`&<8-TdW_Y=2p#Cj{HQ5~yX7Z#kk4(Y5XC55FefKMLPH z6z-@g+9>?zu;;K}(44#eA={VcD#soEr5xr8>*m_~e)*#&kSBck5bx`F4s*_b0=Yl@ zIa|PX#9_uTKf z+xx?O&oA?4e-Wr~e7AD1`YYkfnN}Ru_dm>={VVU)>=k~K|C%rP*Ti?XZuZN4vpsLG z%4_z&Tx;)ZQdayzq5`0Ljt|0Z9*T+9A)x#tTjzyBZRs{hKrRF(Pi-|Uxv z)pz|hUtP7`ciU8R|I07!Loa9AnwL$#Typ>Ne&4H^=3jpK{xy63>sQ_Uzv`>KX1{*9 z{MV)0`t|?jUHX;z^56XBU*>MvzyJS|?;&^X{g?a=UGmb_{QtYjfAg1o4VzpaH@V*T zZr%L*_wNSPN98Ynx%^kj+_(D6|1bLr(hRb^^lw%8(vttaSFhSl{(pJNm%IP3`~NTX z{}uj!X}#~Z>979&z4UHURsH_Dm6yJIesSFK|J9{`bHD6be*fj)zv>{Hf3J;S>VN(6 z|JdkX|Neh{S!)NjHSGWE|9|&~FZ)%i{_ESN()wM$zWu-b?bplyzv{zt_20k!otb;p z{_pRvFRL%#`u}eFmw(gawtx9`|MIQ)_5c6Jy}Gpg(!cqae!u*`{{LJ3mmpnlU;qFA z^~>b<*WbS{eK&dk(*NEH+{Rd{(AoURsVMFe*f$L>+7$!%>MOd z_OET1{{7v=<85}rP&xY7*V4H4|JJSBH~rtM|JzH}|NnRY-~RaZPgFWS3B~PO`ThIr z*RR*j{qFsG{p-42zv|a7|5bMV?{|~7yv*?1aewvx{@YMr_y7Nf`u`sCucQB8`@ikd z{~teQYB1StU-s+E{!bh%nbr5}e@#qOR+Fv(cGkpK=|B+L}zrK_VE&1j9x61eb zw&~yh%dg$L|I7a6vif=d{~rIJFTec%|Nr^k{|`TWzx4Y1_y6BbUwi%k@Bjb*{r?qR z`v3jD>%YDB|NsAeZT-v8$RU;e-JZ|(Q5zyJQf{{P?q>wojV{{35D|84sJ#5cBE zm;Keh^#9AQynp-u|JDEc|LecmRsTQysP}&R<=_AKUGKK-|Np!G`v3pG-qio&Z@&Kv z)QAE#(Ecy~|MLH>|8L|De%t@D{{R2K@8kdfn+}qV|Ht3Vef{m!y|NsC0&)XbP`}Q{~M~=!b6zq`}Y6;Z_UM2-}i6-sc$U* z|Aj9A#n+$Q`k%}{MV|0q*V^@O`q%$+r!rlwez*UbTf)}=OTsrq{r~^}@BiQI8&V4l zY~JnNzyJTaBMqPbNB=+5>G1h~zL7Wc`G5ca{m%wlBEjBdUw`@4|L0i;KDbuDUjJDk zz()B0nW+vR{ICB1TE7bvx>H}UxAoS0|Nn38$KKZe{`#L23;22Ef4MmPV!c$h?L5kp8x;RxIMw-lrwg3PBe|7!8gKs1!9gySI z|NsC0^E&os|6h0OXG$<1T>k$|E5nCOP@8-IZ~nxYnv4&ZF9iqA0XeDquj_wKYOpc+ zf7|0k4nME`|NoFUZ@T~g|No|c_y0fG|Bw6s|Np=Lf8$R$wEu7Y|A+P8*MHdmmlbT@ zkHm*<+?mzc|Nl4rUt0e||92Kxas1!r8wQ}p`NR7B>wok?eUATs|AQ2Z$6o@q@Bh}f z{=WoL`nUeWJ!$UJxBvhDi2qs-PD|iuY?G770^9I48x#oOwmy%I!6AhT3Hhb(rvLvZ z$b8_y|J%R*b2L6ksQDoW|K{;${nD8&DFWCA7T{r`9$+)}7WclW*>|NpOmjq&2&rKSHm&i2VQm|MmaP6=ofB z31L5ZcCvu&Nr<0tI!5z*{ENnCtpC3_di@ke5U%>a^#9$u`Twn@nQGqTa9-ryvi|S)|NlC2m)8HS2lHOE-3tHy|NH;{KX0rM zN>5tg$+~^}*Z=ack98_ zx2pnIZ-WBR9~3-IAm$hG|Nnn+{|8b3|Nj3_BPV`+Z~gxd`~Q3Y6aV)WlBj-|H)mG; zUH@(W%d*|8|G%vNwi1-{m;XQbp(48e-?aaKbpNmZ|FHf)_rL4$Z|nE}ljocC5)_ax z_3!?Qf2sd_$s|xVU;m%?;jaJh_y7O(zx4mF|L@m-U;p>@7ss{t|MR!6-TQC<|6hN9 z)&JZ7Z}tEG|G#_v|NU^Z&PRdtd*$|N3A3_~q-r?Ef3|zdru{zuEua z?_0n9{@?$th90)mDA|zdlxA)2hAo*Z;2tg~KB8y!HPl79^k8Vfg#I!oj>( z|Nj5q_e5gim)HOQe~$;{w;~?Z`2Q0Z7(U71$=dGMAtAPZ-Tz;!-*re#{r&%H{r%9t zyRLtGeOJI>#q9qN);H~~`cSw3*8l6-2k-y?`u{b^Hy<+N|NsB{>U!0d{r~@Oz3vTm zTl~N4Z_EGx|F?Z_`M+ELU)|bkYyW@!-(Ua!?ce|2)BOMKZ~sfTe*ORd|Jyone%Lbq z-=*Inpkn0u`>^TR|G(|8kN^Mg>;J$1|NpiB|NrZ^OYi^w{~MpHt6Tr||Ni*fvGuvN zTc+;+Uthm%`@5;@v;TkFzVHA4|GVNr3Arl#|NZ;d*VbRUTmQB6`n&yK|NpJudi{6& zwb}oF#ZB9HxBUM7|7+h~UAO<&|NHCT-~avZuJ-@mzyEGoTfW*mxBlT48 zaqRj3%f7Gm|4Z*?XPWIU&A#>jb=Ixd|6l*RoBg|X{o8-Hme&8hUYc8X>G76r*Z*$K z%~jJ}|Lyuev+KXT-04s#vm-d}g? z{_pJHna4fX{Xfjjqn%lL|1fv`9)tZG4s8>E+a|v4 zlIQOK9}a;mZWDTSD4LD?-;FRHF$vFq^4rAZzubs_*s`$oN5P>Oi$s;#?f-8i>b41O z*v%&NBT;ukDF6S&NY;Qi5?=cxum9%edHw%J{Eb942_?zJ|0J&`w(LlcZRQN;dHw%K zqMn3j8`otX@hl#_XPsOzZ1gO<}~fi zY-ai9I6d@}r1OMq-umBcoNt=-R)723%yRpG!a@6+thI*m+fHb|yqzLh8oK@ejkR49 z!dA3h_%>thI_(sGrpN!Eh^|uHVmrZxyFY35f3V#DZ%aU8_CJ>Gf3&wN?EnAYqHpW2 z{r@ex@AB4vN9%u!{<~Vg^{vYN|F>>sXRiIWZU47z+xGum|M%Aa%=O>)uG;^9>(;pI z(b4gLb92M@U;BTxe#`&e(OcI2|GRdXaCQ8(L;r91ZrHZ#f1>&S4YU6k%=*uxpQ-l$ z-?hupzPY)H>;JAX-SYna*86Ww|6Q%mExQz6|99;&@vXn3<9~B!XM@th8~tyy{@%K~ zs{H@kT+_F8*RKD&cI|37IQ9DIf7`n>|Nk4^mP6Z`xwEsg|0hOAB--!au=amuYyE~< ze{V#$359`NGC%v9m_+<Paod0>$ z_7@zwV=#3)NR2>ZoJ4%vhQ#n6i4g`X4EERaXxD!@wEr-N0a(!_up(jBhpoT4dBp!6 zOl&&Lxt{-xnEc_y9)*Jy2K#x$$^+8=KWsX8L2`$|_QQ1&I_Cc*^gkpt zbAPyD5RoI{ow)xWj~I`Df$s%_=m&?+9c-5WVcv^wuItuSy>PjgsdqSz#O#Ko~m@Z1rm1s&~zY#h!Vz0531O63t| z%`8faXWE!E+O1_oS)6+s7G1~^`^@XCY>=(?PAW21A@%753Df6Uhdnkiwq@q6;7u)P zoIUk8+ot>oQ`uQ;nR$AW%iI`OZ1<2{p24ehzuC+o#hj`DIt=l3kww9I%McxZWI)h2?=p= z7Zel}{D0}o-thR?+WMDO@&9~JOxYqaeW{`u%M!H%lj0nEn@<>SsN2N)eVxO;uj>|E zlZS`ueORIM6TBDj7 z<*0R-n@8(XuG%FpF^PyorxW~}eG@!4OiomqF^4n7;lrVTLqQ5P!EBrc2~$EJ985SE z@6)t5b3*vq@T3{*eNCfQt-Bn%Y*x0fX;kK_x27*!4y|Qt4$WLP>p~)%XeNj!kQmnN zxats(K*GW(HUWcKU{)eC3y+9KqL|ZzRgqU3LkyO%-(1ehI)THK+v-M^$e{p@3G!*L zI3l^46Vn3BM9v&kQUWz-K@F2pJQ@O{Aut*OqaiRF0;3@?8UmvsFd71*Aut*OqaiRF z0)sXL4zMsdfR>4(E+GXi4h1b3H8e0ag|8y5C`wJ^GElHo-~z3yg(#LS%3A}-KE z-r&rtR0T^e{h<7O&@yB2!fJibytI4;(4y)f1ulJ0muMT1H%ts2%?%A)%#0n~49yIj r%pILwER7wVObwik9PJ3J0IvuxE=epZ0SBI;p^1qpm#V6(zZ(|-lw@w* literal 0 HcmV?d00001 diff --git a/tests/spoof/tesseract_big_image_error.py b/tests/spoof/tesseract_big_image_error.py new file mode 100755 index 00000000..827d6585 --- /dev/null +++ b/tests/spoof/tesseract_big_image_error.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +import sys + + +VERSION_STRING = '''tesseract 3.04.00 + leptonica-1.72 + libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5 +SPOOFED: return error claiming image too big +''' + +"""Simulates a Tesseract crash + +It isn't strictly necessary to crash the process and that has unwanted +side effects like triggering core dumps or error reporting, logging and such. +It's enough to dump some text to stderr and return an error code. + +Follows the POSIX? convention of returning 128 + signal number. + +""" + + +def main(): + if sys.argv[1] == '--version': + print(VERSION_STRING, file=sys.stderr) + sys.exit(0) + elif sys.argv[1] == '--list-langs': + print('List of available languages (1):\neng', file=sys.stderr) + sys.exit(0) + elif sys.argv[-1] == 'hocr': + print("Image too large: (33830, 14959)\n" + "Error during processing.", file=sys.stderr) + sys.exit(1) + elif sys.argv[-1] == 'pdf': + print("Image too large: (33830, 14959)\n" + "Error during processing.", file=sys.stderr) + sys.exit(1) + elif sys.argv[-1] == 'stdout': + print("Image too large: (33830, 14959)\n" + "Error during processing.", file=sys.stderr) + sys.exit(1) + else: + print("Spoof doesn't understand arguments", file=sys.stderr) + print(sys.argv, file=sys.stderr) + sys.exit(1) + + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/tests/spoof/tesseract_cache.py b/tests/spoof/tesseract_cache.py index fd3655e0..96485e14 100755 --- a/tests/spoof/tesseract_cache.py +++ b/tests/spoof/tesseract_cache.py @@ -90,6 +90,13 @@ def main(): universal_newlines=True) stdout, stderr = p.communicate() + if p.returncode != 0: + # Do not cache errors or crashes + print("Tesseract error", file=sys.stderr) + print(stdout, end='') + print(stderr, end='', file=sys.stderr) + return p.returncode + with open(cache_name + '.stdout', 'w') as f: f.write(stdout) with open(cache_name + '.stderr', 'w') as f: diff --git a/tests/test_main.py b/tests/test_main.py index 52669dd8..bf3c3d1e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -113,6 +113,11 @@ def spoof_tesseract_crash(): return spoof('tesseract', 'tesseract_crash.py') +@pytest.fixture +def spoof_tesseract_big_image_error(): + return spoof('tesseract', 'tesseract_big_image_error.py') + + def test_quick(spoof_tesseract_noop): check_ocrmypdf('c02-22.pdf', 'test_quick.pdf', env=spoof_tesseract_noop) @@ -477,3 +482,13 @@ def test_tesseract_crash_autorotate(spoof_tesseract_crash): assert sh.returncode == ExitCode.child_process_error assert not os.path.exists(_outfile('wontwork.pdf')) assert "ERROR" in err + + +@pytest.mark.parametrize('renderer', [ + 'hocr', + 'tesseract', + ]) +def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error): + check_ocrmypdf( + 'hugemono.pdf', 'hugemono_%s.pdf' % renderer, '-r', + '--pdf-renderer', renderer, env=spoof_tesseract_big_image_error)