diff --git a/.gitattributes b/.gitattributes index 9e64aba4..be2a4386 100644 --- a/.gitattributes +++ b/.gitattributes @@ -13,5 +13,6 @@ *.jpg binary *.bin binary *.afdesign binary +*.ttf binary .git_archival.txt export-subst diff --git a/REUSE.toml b/REUSE.toml index 5f21ab3f..dbee971d 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -167,15 +167,9 @@ SPDX-FileCopyrightText = [ SPDX-License-Identifier = "Zlib" [[annotations]] -path = "src/ocrmypdf/data/pdf.ttf" +path = "src/ocrmypdf/data/Occulta.ttf" precedence = "aggregate" -SPDX-FileCopyrightText = [ - "(C) 2014 Ray Smith", - "(C) 2015 Ken Sharp", - "(C) 2016 James R. Barlow", - "(C) 2016 Jeff Breidenbach", - "(C) 2017 Zdenko Podobný", -] +SPDX-FileCopyrightText = ["(C) 2026 James R. Barlow"] SPDX-License-Identifier = "Apache-2.0" [[annotations]] diff --git a/scripts/generate_glyphless_font.py b/scripts/generate_glyphless_font.py new file mode 100644 index 00000000..988863f5 --- /dev/null +++ b/scripts/generate_glyphless_font.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2024 James R. Barlow +# SPDX-License-Identifier: MPL-2.0 + +"""Generate the Occulta glyphless font for OCRmyPDF. + +Occulta (Latin for "hidden") is a glyphless font designed for invisible text layers +in searchable PDFs. It has proper Unicode cmap coverage using format 13 (many-to-one) +for efficient mapping of all BMP codepoints to a small set of width-specific glyphs. + +Features: +- Full BMP coverage (U+0000 to U+FFFF) +- Width-aware glyphs for proper text selection: + - Zero-width for combining marks and invisible characters + - Regular width (500 units) for Latin, Greek, Cyrillic, Arabic, Hebrew, etc. + - Double width (1000 units) for CJK and fullwidth characters +- Uses cmap format 13 (many-to-one) for ~12KB size vs ~780KB with format 12 +- Compatible with fpdf2 and other modern PDF libraries + +Usage: + python scripts/generate_glyphless_font.py + +Output: + src/ocrmypdf/data/Occulta.ttf +""" + +from __future__ import annotations + +import unicodedata +from pathlib import Path + +from fontTools.fontBuilder import FontBuilder +from fontTools.ttLib import TTFont +from fontTools.ttLib.tables._c_m_a_p import CmapSubtable +from fontTools.ttLib.tables._g_l_y_f import Glyph + +# Output path relative to this script +OUTPUT_PATH = Path(__file__).parent.parent / "src" / "ocrmypdf" / "data" / "Occulta.ttf" + +# Font metrics (units per em = 1000) +UNITS_PER_EM = 1000 +ASCENT = 800 +DESCENT = -200 + +# Glyph definitions: (name, advance_width, left_side_bearing) +GLYPHS = [ + (".notdef", 500, 0), # Required, used for unmapped characters + ("space", 500, 0), # U+0020 SPACE + ("nbspace", 500, 0), # U+00A0 NO-BREAK SPACE + ("blank0", 0, 0), # Zero-width (combining marks, ZWNJ, ZWJ, BOM) + ("blank1", 500, 0), # Regular width (most scripts) + ("blank2", 1000, 0), # Double width (CJK, fullwidth) +] + +# Explicit zero-width character codepoints +ZERO_WIDTH_CHARS = frozenset( + [ + 0x200B, # ZERO WIDTH SPACE + 0x200C, # ZERO WIDTH NON-JOINER + 0x200D, # ZERO WIDTH JOINER + 0xFEFF, # ZERO WIDTH NO-BREAK SPACE (BOM) + 0x200E, # LEFT-TO-RIGHT MARK + 0x200F, # RIGHT-TO-LEFT MARK + 0x202A, # LEFT-TO-RIGHT EMBEDDING + 0x202B, # RIGHT-TO-LEFT EMBEDDING + 0x202C, # POP DIRECTIONAL FORMATTING + 0x202D, # LEFT-TO-RIGHT OVERRIDE + 0x202E, # RIGHT-TO-LEFT OVERRIDE + 0x2060, # WORD JOINER + 0x2061, # FUNCTION APPLICATION + 0x2062, # INVISIBLE TIMES + 0x2063, # INVISIBLE SEPARATOR + 0x2064, # INVISIBLE PLUS + ] +) + + +def classify_codepoint(codepoint: int) -> str: + """Classify a Unicode codepoint into one of our glyph categories. + + Args: + codepoint: Unicode codepoint (0x0000 to 0xFFFF) + + Returns: + Glyph name to map this codepoint to + """ + # Special cases first + if codepoint == 0x0020: + return "space" + if codepoint == 0x00A0: + return "nbspace" + if codepoint in ZERO_WIDTH_CHARS: + return "blank0" + + # Use Unicode properties for the rest + char = chr(codepoint) + try: + category = unicodedata.category(char) + east_asian_width = unicodedata.east_asian_width(char) + + # Combining marks are zero-width + if category.startswith("M"): + return "blank0" + + # Wide and Fullwidth characters are double-width + if east_asian_width in ("W", "F"): + return "blank2" + + # Everything else is regular width + return "blank1" + + except (ValueError, TypeError): + # Fallback for any edge cases + return "blank1" + + +def build_cmap() -> dict[int, str]: + """Build the Unicode to glyph name mapping for the entire BMP. + + Returns: + Dictionary mapping codepoints to glyph names + """ + return {cp: classify_codepoint(cp) for cp in range(0x10000)} + + +def create_font() -> TTFont: + """Create the Occulta glyphless font. + + Returns: + TTFont object ready to be saved + """ + glyph_names = [g[0] for g in GLYPHS] + + # Start building the font + fb = FontBuilder(UNITS_PER_EM, isTTF=True) + fb.setupGlyphOrder(glyph_names) + + # Create empty (invisible) glyphs + glyphs = {} + for name, _, _ in GLYPHS: + glyph = Glyph() + glyph.numberOfContours = 0 + glyphs[name] = glyph + fb.setupGlyf(glyphs) + + # Set up horizontal metrics + metrics = {name: (width, lsb) for name, width, lsb in GLYPHS} + fb.setupHorizontalMetrics(metrics) + + # Minimal cmap to satisfy FontBuilder (we'll replace it later) + fb.setupCharacterMap({0x0020: "space", 0x00A0: "nbspace"}) + + # Set up other required tables + fb.setupHorizontalHeader(ascent=ASCENT, descent=DESCENT) + fb.setupOS2( + sTypoAscender=ASCENT, + sTypoDescender=DESCENT, + sTypoLineGap=0, + usWinAscent=UNITS_PER_EM, + usWinDescent=abs(DESCENT), + sxHeight=500, + sCapHeight=700, + ) + import time + + # Use current time for font timestamps + now = int(time.time()) + fb.setupHead(unitsPerEm=UNITS_PER_EM, created=now, modified=now) + fb.setupPost() + fb.setupNameTable( + { + "familyName": "Occulta", + "styleName": "Regular", + "uniqueFontIdentifier": "OCRmyPDF;Occulta-Regular;2026", + "fullName": "Occulta Regular", + "version": "Version 2.0", + "psName": "Occulta-Regular", + } + ) + + # Build the font + font = fb.font + + # Now replace the cmap with format 13 for efficient many-to-one mapping + char_to_glyph = build_cmap() + + cmap13 = CmapSubtable.newSubtable(13) + cmap13.platformID = 3 # Windows + cmap13.platEncID = 10 # Unicode full repertoire + cmap13.language = 0 + cmap13.cmap = char_to_glyph + + font["cmap"].tables = [cmap13] + + return font + + +def main() -> None: + """Generate the Occulta font and save it.""" + print("Generating Occulta glyphless font...") + + font = create_font() + + # Create output directory if needed + OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + + # Save the font + font.save(str(OUTPUT_PATH)) + font.close() + + # Report statistics + size = OUTPUT_PATH.stat().st_size + print(f"Saved to: {OUTPUT_PATH}") + print(f"Size: {size:,} bytes") + + # Verify cmap + font = TTFont(str(OUTPUT_PATH)) + for table in font["cmap"].tables: + print( + f"cmap: Platform {table.platformID}, " + f"Encoding {table.platEncID}, " + f"Format {table.format}, " + f"{len(table.cmap)} mappings" + ) + font.close() + + print("Done!") + + +if __name__ == "__main__": + main() diff --git a/src/ocrmypdf/data/NotoSans-Regular.ttf b/src/ocrmypdf/data/NotoSans-Regular.ttf new file mode 100644 index 00000000..7552fbe8 Binary files /dev/null and b/src/ocrmypdf/data/NotoSans-Regular.ttf differ diff --git a/src/ocrmypdf/data/Occulta.ttf b/src/ocrmypdf/data/Occulta.ttf new file mode 100644 index 00000000..e80c8850 Binary files /dev/null and b/src/ocrmypdf/data/Occulta.ttf differ diff --git a/src/ocrmypdf/data/pdf.ttf b/src/ocrmypdf/data/pdf.ttf deleted file mode 100644 index d1472b20..00000000 Binary files a/src/ocrmypdf/data/pdf.ttf and /dev/null differ diff --git a/src/ocrmypdf/font/__init__.py b/src/ocrmypdf/font/__init__.py new file mode 100644 index 00000000..306808d7 --- /dev/null +++ b/src/ocrmypdf/font/__init__.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2025 James R. Barlow +# SPDX-License-Identifier: MPL-2.0 + +"""Font management for OCRmyPDF PDF rendering. + +This module provides font infrastructure for the fpdf2 PDF renderer. It includes: + +- FontManager: Base class for font loading and glyph checking +- FontProvider: Protocol and implementations for font discovery +- MultiFontManager: Automatic font selection for multilingual documents +- SystemFontProvider: System font discovery +""" + +from ocrmypdf.font.font_manager import FontManager +from ocrmypdf.font.font_provider import ( + BuiltinFontProvider, + ChainedFontProvider, + FontProvider, +) +from ocrmypdf.font.multi_font_manager import MultiFontManager +from ocrmypdf.font.system_font_provider import SystemFontProvider + +__all__ = [ + "FontManager", + "FontProvider", + "BuiltinFontProvider", + "ChainedFontProvider", + "MultiFontManager", + "SystemFontProvider", +] diff --git a/src/ocrmypdf/font/font_manager.py b/src/ocrmypdf/font/font_manager.py new file mode 100644 index 00000000..97fb6c34 --- /dev/null +++ b/src/ocrmypdf/font/font_manager.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: 2025 James R. Barlow +# SPDX-License-Identifier: MPL-2.0 + +"""Base font management for PDF rendering. + +This module provides the base FontManager class that handles font loading +and glyph checking using uharfbuzz. +""" + +from __future__ import annotations + +from pathlib import Path + +import uharfbuzz as hb + + +class FontManager: + """Manages font loading and glyph checking for PDF rendering. + + This base class handles loading fonts with uharfbuzz for glyph checking + and text shaping. Renderer-specific subclasses should extend this to + add their own font objects. + + Attributes: + font_path: Path to the font file + font_data: Raw font file bytes + font_index: Index within TTC collection (0 for single-font files) + hb_face: uharfbuzz Face object + hb_font: uharfbuzz Font object + """ + + def __init__(self, font_path: Path, font_index: int = 0): + """Initialize font manager. + + Args: + font_path: Path to TrueType/OpenType font file + font_index: Index of font within a TTC collection (default 0). + For single-font files (.ttf, .otf), use 0. + """ + self.font_path = font_path + self.font_index = font_index + + # Load font data + self.font_data = font_path.read_bytes() + + # Load font with uharfbuzz for glyph checking and text measurement + # Note: uharfbuzz Face also supports font_index for TTC files + self.hb_face = hb.Face(self.font_data, font_index) + self.hb_font = hb.Font(self.hb_face) + + def get_hb_font(self) -> hb.Font: + """Get uharfbuzz Font object for text measurement. + + Returns: + UHarfBuzz Font instance + """ + return self.hb_font + + def has_glyph(self, codepoint: int) -> bool: + """Check if font has a glyph for given codepoint. + + Args: + codepoint: Unicode codepoint + + Returns: + True if font has a real glyph (not .notdef) + """ + glyph_id = self.hb_font.get_nominal_glyph(codepoint) + return glyph_id is not None and glyph_id != 0 + + def get_font_metrics(self) -> tuple[float, float, float]: + """Get normalized font metrics (ascent, descent, units_per_em). + + Returns: + Tuple of (ascent, descent, units_per_em) where ascent and descent + are in font units. Ascent is positive (above baseline), descent + is typically negative (below baseline). + """ + extents = self.hb_font.get_font_extents('ltr') + units_per_em = self.hb_face.upem + return (extents.ascender, extents.descender, units_per_em) + + def get_left_side_bearing(self, char: str, font_size: float) -> float: + """Get the left side bearing of a character at a given font size. + + The left side bearing (lsb) is the horizontal distance from the glyph + origin (x=0) to the leftmost pixel of the glyph. A positive lsb means + there's whitespace before the glyph starts. + + Args: + char: Single character to get lsb for + font_size: Font size in points + + Returns: + Left side bearing in points. Returns 0 if character not found. + """ + if not char: + return 0.0 + + codepoint = ord(char) + glyph_id = self.hb_font.get_nominal_glyph(codepoint) + if glyph_id is None or glyph_id == 0: + return 0.0 + + # Get glyph extents which include left/right bearing info + extents = self.hb_font.get_glyph_extents(glyph_id) + if extents is None: + return 0.0 + + # x_bearing is the left side bearing in font units + units_per_em = self.hb_face.upem + lsb_units = extents.x_bearing + lsb_pt = lsb_units * font_size / units_per_em + + return lsb_pt diff --git a/src/ocrmypdf/font/font_provider.py b/src/ocrmypdf/font/font_provider.py new file mode 100644 index 00000000..90c5035e --- /dev/null +++ b/src/ocrmypdf/font/font_provider.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: 2025 James R. Barlow +# SPDX-License-Identifier: MPL-2.0 + +"""Font provider protocol and implementations for PDF rendering.""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Protocol + +from ocrmypdf.font.font_manager import FontManager + +log = logging.getLogger(__name__) + + +class FontProvider(Protocol): + """Protocol for providing fonts to MultiFontManager. + + Implementations are responsible for knowing where fonts are located + and loading them. MultiFontManager asks for fonts by name and uses + them for glyph coverage checking. + """ + + def get_font(self, font_name: str) -> FontManager | None: + """Get a FontManager for the named font. + + Args: + font_name: Logical font name (e.g., 'NotoSans-Regular') + + Returns: + FontManager if font is available, None otherwise + """ + ... + + def get_available_fonts(self) -> list[str]: + """Get list of available font names. + + Returns: + List of font names that can be retrieved with get_font() + """ + ... + + def get_fallback_font(self) -> FontManager: + """Get the glyphless fallback font. + + This font must always be available and handles any codepoint. + + Returns: + FontManager for the glyphless fallback font (Occulta.ttf) + """ + ... + + +class BuiltinFontProvider: + """Font provider using builtin fonts from ocrmypdf/data directory.""" + + # Mapping of logical font names to filenames + # Only Latin (NotoSans) and the glyphless fallback (Occulta.ttf) are bundled. + # All other scripts (Arabic, Devanagari, CJK, etc.) are discovered from + # system fonts by SystemFontProvider to reduce package size. + FONT_FILES = { + 'NotoSans-Regular': 'NotoSans-Regular.ttf', + 'Occulta': 'Occulta.ttf', + } + + def __init__(self, font_dir: Path | None = None): + """Initialize builtin font provider. + + Args: + font_dir: Directory containing font files. If None, uses + the default ocrmypdf/data directory. + """ + if font_dir is None: + font_dir = Path(__file__).parent.parent / "data" + self.font_dir = font_dir + self._fonts: dict[str, FontManager] = {} + self._load_fonts() + + def _load_fonts(self) -> None: + """Load available fonts, logging warnings for missing ones.""" + for font_name, font_file in self.FONT_FILES.items(): + font_path = self.font_dir / font_file + if not font_path.exists(): + if font_name == 'Occulta': + raise FileNotFoundError( + f"Required fallback font not found: {font_path}" + ) + log.warning( + "Font %s not found at %s - OCR output quality for some " + "scripts may be affected", + font_name, + font_path, + ) + continue + + try: + self._fonts[font_name] = FontManager(font_path) + except Exception as e: + if font_name == 'Occulta': + raise ValueError( + f"Failed to load required fallback font {font_file}: {e}" + ) from e + log.warning( + "Failed to load font %s: %s - OCR output quality may be affected", + font_name, + e, + ) + + def get_font(self, font_name: str) -> FontManager | None: + """Get a FontManager for the named font.""" + return self._fonts.get(font_name) + + def get_available_fonts(self) -> list[str]: + """Get list of available font names.""" + return list(self._fonts.keys()) + + def get_fallback_font(self) -> FontManager: + """Get the glyphless fallback font.""" + return self._fonts['Occulta'] + + +class ChainedFontProvider: + """Font provider that tries multiple providers in order. + + This allows combining builtin fonts with system fonts, trying + the builtin provider first and falling back to system fonts + for fonts not bundled with the package. + """ + + def __init__(self, providers: list[FontProvider]): + """Initialize chained font provider. + + Args: + providers: List of font providers to try in order. + The first provider that returns a font wins. + """ + if not providers: + raise ValueError("At least one provider is required") + self.providers = providers + + def get_font(self, font_name: str) -> FontManager | None: + """Get a FontManager for the named font. + + Tries each provider in order until one returns a font. + + Args: + font_name: Logical font name (e.g., 'NotoSans-Regular') + + Returns: + FontManager if any provider has the font, None otherwise + """ + for provider in self.providers: + if font := provider.get_font(font_name): + return font + return None + + def get_available_fonts(self) -> list[str]: + """Get list of available font names from all providers. + + Returns: + Combined list of font names (deduplicated, order preserved) + """ + seen: set[str] = set() + result: list[str] = [] + for provider in self.providers: + for name in provider.get_available_fonts(): + if name not in seen: + seen.add(name) + result.append(name) + return result + + def get_fallback_font(self) -> FontManager: + """Get the glyphless fallback font. + + Tries each provider until one provides a fallback font. + + Returns: + FontManager for the fallback font + + Raises: + RuntimeError: If no provider can provide a fallback font + """ + for provider in self.providers: + try: + return provider.get_fallback_font() + except (NotImplementedError, AttributeError, KeyError): + continue + raise RuntimeError("No fallback font available from any provider") diff --git a/src/ocrmypdf/font/multi_font_manager.py b/src/ocrmypdf/font/multi_font_manager.py new file mode 100644 index 00000000..4cd672ca --- /dev/null +++ b/src/ocrmypdf/font/multi_font_manager.py @@ -0,0 +1,323 @@ +# SPDX-FileCopyrightText: 2025 James R. Barlow +# SPDX-License-Identifier: MPL-2.0 + +"""Multi-font management for PDF rendering. + +Provides automatic font selection for multilingual documents based on +language hints and glyph coverage analysis. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +from ocrmypdf.font.font_manager import FontManager +from ocrmypdf.font.font_provider import ( + BuiltinFontProvider, + ChainedFontProvider, + FontProvider, +) +from ocrmypdf.font.system_font_provider import SystemFontProvider + +log = logging.getLogger(__name__) + + +class MultiFontManager: + """Manages multiple fonts with automatic selection and fallback. + + This class orchestrates multiple FontManager instances to provide + word-level font selection for multilingual documents. It uses a hybrid + approach combining language hints from hOCR with glyph coverage analysis. + + Font selection strategy: + 1. Try language-preferred font (if language hint available) + 2. Try fallback fonts in order by glyph coverage + 3. Fall back to Occulta.ttf (glyphless fallback) + """ + + # Language to font mapping + # Keys are ISO 639-2/3 codes or Tesseract language codes + LANGUAGE_FONT_MAP = { + # Arabic script + 'ara': 'NotoSansArabic-Regular', # Arabic + 'per': 'NotoSansArabic-Regular', # Persian (uses Arabic script) + 'fas': 'NotoSansArabic-Regular', # Farsi (alternative code for Persian) + 'urd': 'NotoSansArabic-Regular', # Urdu (uses Arabic script) + 'pus': 'NotoSansArabic-Regular', # Pashto + 'kur': 'NotoSansArabic-Regular', # Kurdish (Arabic script variant) + # Devanagari script + 'hin': 'NotoSansDevanagari-Regular', # Hindi + 'san': 'NotoSansDevanagari-Regular', # Sanskrit + 'mar': 'NotoSansDevanagari-Regular', # Marathi + 'nep': 'NotoSansDevanagari-Regular', # Nepali + 'kok': 'NotoSansDevanagari-Regular', # Konkani + 'bho': 'NotoSansDevanagari-Regular', # Bhojpuri + 'mai': 'NotoSansDevanagari-Regular', # Maithili + # CJK + 'chi': 'NotoSansCJK-Regular', # Chinese (generic) + 'zho': 'NotoSansCJK-Regular', # Chinese (ISO 639-3) + 'chi_sim': 'NotoSansCJK-Regular', # Chinese Simplified (Tesseract) + 'chi_tra': 'NotoSansCJK-Regular', # Chinese Traditional (Tesseract) + 'jpn': 'NotoSansCJK-Regular', # Japanese + 'kor': 'NotoSansCJK-Regular', # Korean + # Thai + 'tha': 'NotoSansThai-Regular', # Thai + # Hebrew + 'heb': 'NotoSansHebrew-Regular', # Hebrew + 'yid': 'NotoSansHebrew-Regular', # Yiddish (uses Hebrew script) + # Bengali script + 'ben': 'NotoSansBengali-Regular', # Bengali + 'asm': 'NotoSansBengali-Regular', # Assamese (uses Bengali script) + # Tamil + 'tam': 'NotoSansTamil-Regular', # Tamil + # Gujarati + 'guj': 'NotoSansGujarati-Regular', # Gujarati + # Telugu + 'tel': 'NotoSansTelugu-Regular', # Telugu + # Kannada + 'kan': 'NotoSansKannada-Regular', # Kannada + # Malayalam + 'mal': 'NotoSansMalayalam-Regular', # Malayalam + # Myanmar (Burmese) + 'mya': 'NotoSansMyanmar-Regular', # Myanmar + # Khmer (Cambodian) + 'khm': 'NotoSansKhmer-Regular', # Khmer + # Lao + 'lao': 'NotoSansLao-Regular', # Lao + # Georgian + 'kat': 'NotoSansGeorgian-Regular', # Georgian + 'geo': 'NotoSansGeorgian-Regular', # Georgian (alternative) + # Armenian + 'hye': 'NotoSansArmenian-Regular', # Armenian + 'arm': 'NotoSansArmenian-Regular', # Armenian (alternative) + # Ethiopic + 'amh': 'NotoSansEthiopic-Regular', # Amharic + 'tir': 'NotoSansEthiopic-Regular', # Tigrinya + # Sinhala + 'sin': 'NotoSansSinhala-Regular', # Sinhala + # Gurmukhi (Punjabi) + 'pan': 'NotoSansGurmukhi-Regular', # Punjabi + 'pnb': 'NotoSansGurmukhi-Regular', # Western Punjabi + # Oriya + 'ori': 'NotoSansOriya-Regular', # Oriya + 'ory': 'NotoSansOriya-Regular', # Oriya (alternative) + # Tibetan + 'bod': 'NotoSansTibetan-Regular', # Tibetan + 'tib': 'NotoSansTibetan-Regular', # Tibetan (alternative) + } + + # Ordered fallback chain for fonts (after language-preferred font) + # Order matters: most common scripts first for faster matching + FALLBACK_FONTS = [ + 'NotoSans-Regular', # Latin, Greek, Cyrillic + 'NotoSansArabic-Regular', + 'NotoSansDevanagari-Regular', + 'NotoSansCJK-Regular', + 'NotoSansThai-Regular', + 'NotoSansHebrew-Regular', + 'NotoSansBengali-Regular', + 'NotoSansTamil-Regular', + 'NotoSansGujarati-Regular', + 'NotoSansTelugu-Regular', + 'NotoSansKannada-Regular', + 'NotoSansMalayalam-Regular', + 'NotoSansMyanmar-Regular', + 'NotoSansKhmer-Regular', + 'NotoSansLao-Regular', + 'NotoSansGeorgian-Regular', + 'NotoSansArmenian-Regular', + 'NotoSansEthiopic-Regular', + 'NotoSansSinhala-Regular', + 'NotoSansGurmukhi-Regular', + 'NotoSansOriya-Regular', + 'NotoSansTibetan-Regular', + ] + + def __init__( + self, + font_dir: Path | None = None, + *, + font_provider: FontProvider | None = None, + ): + """Initialize multi-font manager. + + Args: + font_dir: Directory containing font files. If font_provider is + not specified, this is passed to BuiltinFontProvider. + font_provider: Provider for loading fonts. If None, uses a + ChainedFontProvider that tries builtin fonts first, + then searches system fonts. + """ + if font_provider is not None: + self.font_provider = font_provider + else: + # Use chained provider: try builtin fonts first, then system fonts + self.font_provider = ChainedFontProvider([ + BuiltinFontProvider(font_dir), + SystemFontProvider(), + ]) + + # Font selection cache: (word_text, language) -> font_name + self._selection_cache: dict[tuple[str, str | None], str] = {} + # Track whether we've warned about missing fonts (warn once per script) + self._warned_scripts: set[str] = set() + + @property + def fonts(self) -> dict[str, FontManager]: + """Get all loaded fonts (backward compatibility).""" + return self.get_all_fonts() + + def _try_font( + self, font_name: str, word_text: str, cache_key: tuple[str, str | None] + ) -> FontManager | None: + """Try to use a font for the given word. + + Args: + font_name: Name of font to try + word_text: Text content to check + cache_key: Cache key for storing successful result + + Returns: + FontManager if font exists and has all glyphs, None otherwise + """ + font = self.font_provider.get_font(font_name) + if font is None: + return None + if self._has_all_glyphs(font, word_text): + self._selection_cache[cache_key] = font_name + return font + return None + + def select_font_for_word( + self, word_text: str, line_language: str | None + ) -> FontManager: + """Select appropriate font for a word. + + Uses a hybrid approach: + 1. Language-based selection (if language hint available) + 2. Ordered fallback through available fonts by glyph coverage + 3. Final fallback to Occulta.ttf (glyphless) + + Args: + word_text: The text content of the word + line_language: Language code from hOCR (e.g., 'ara', 'eng') + + Returns: + FontManager instance to use for rendering this word + """ + cache_key = (word_text, line_language) + if cache_key in self._selection_cache: + cached_name = self._selection_cache[cache_key] + font = self.font_provider.get_font(cached_name) + if font: + return font + + tried_fonts: set[str] = set() + + # Phase 1: Try language-preferred font + if line_language and line_language in self.LANGUAGE_FONT_MAP: + preferred = self.LANGUAGE_FONT_MAP[line_language] + tried_fonts.add(preferred) + if result := self._try_font(preferred, word_text, cache_key): + return result + + # Phase 2: Try fallback fonts in order + for font_name in self.FALLBACK_FONTS: + if font_name in tried_fonts: + continue + if result := self._try_font(font_name, word_text, cache_key): + return result + + # Phase 3: Glyphless fallback (always succeeds) + # Warn if we're falling back for non-ASCII text (likely missing font) + self._warn_missing_font(word_text, line_language) + self._selection_cache[cache_key] = 'Occulta' + return self.font_provider.get_fallback_font() + + def _warn_missing_font( + self, word_text: str, line_language: str | None + ) -> None: + """Warn user about missing font for non-Latin text. + + Only warns once per language/script to avoid log spam. + """ + # Determine a key for deduplication (language or 'non-ascii') + warn_key = line_language if line_language else 'unknown' + + # Only warn for non-ASCII text and only once per key + if warn_key in self._warned_scripts: + return + + # Check if text contains non-ASCII characters + if not any(ord(c) > 127 for c in word_text): + return + + self._warned_scripts.add(warn_key) + + if line_language and line_language in self.LANGUAGE_FONT_MAP: + font_name = self.LANGUAGE_FONT_MAP[line_language] + log.warning( + "No font found with glyphs for '%s' text. " + "Install %s for better rendering. " + "See https://fonts.google.com/noto", + line_language, + font_name, + ) + else: + log.warning( + "No font found with glyphs for some text. " + "Install Noto fonts for better rendering. " + "See https://fonts.google.com/noto" + ) + + def _has_all_glyphs(self, font: FontManager, text: str) -> bool: + """Check if a font has glyphs for all characters in text. + + Args: + font: FontManager instance to check + text: Text to verify coverage for + + Returns: + True if font has real glyphs for all characters (not .notdef) + """ + if not text: + return True + + hb_font = font.get_hb_font() + + for char in text: + codepoint = ord(char) + glyph_id = hb_font.get_nominal_glyph(codepoint) + if glyph_id is None or glyph_id == 0: # 0 = .notdef glyph + return False + + return True + + def has_all_glyphs(self, font_name: str, text: str) -> bool: + """Check if a named font has glyphs for all characters in text. + + Args: + font_name: Name of font to check + text: Text to verify coverage for + + Returns: + True if font has real glyphs for all characters (not .notdef) + """ + font = self.font_provider.get_font(font_name) + if font is None: + return False + return self._has_all_glyphs(font, text) + + def get_all_fonts(self) -> dict[str, FontManager]: + """Get all loaded font managers. + + Returns: + Dictionary mapping font names to FontManager instances + """ + result = {} + for name in self.font_provider.get_available_fonts(): + font = self.font_provider.get_font(name) + if font is not None: + result[name] = font + return result diff --git a/src/ocrmypdf/font/system_font_provider.py b/src/ocrmypdf/font/system_font_provider.py new file mode 100644 index 00000000..d0d17bc3 --- /dev/null +++ b/src/ocrmypdf/font/system_font_provider.py @@ -0,0 +1,297 @@ +# SPDX-FileCopyrightText: 2025 James R. Barlow +# SPDX-License-Identifier: MPL-2.0 + +"""System font discovery for PDF rendering. + +Provides lazy discovery of Noto fonts installed on the system across +Linux, macOS, and Windows platforms. +""" + +from __future__ import annotations + +import logging +import os +import sys +from pathlib import Path + +from ocrmypdf.font.font_manager import FontManager + +log = logging.getLogger(__name__) + + +class SystemFontProvider: + """Discovers and provides system-installed Noto fonts with lazy scanning. + + This provider searches standard system font directories for Noto fonts. + Scanning is performed lazily - only when a font is actually requested + and not found in the builtin fonts. Results are cached for the lifetime + of the provider instance. + """ + + # System font directories by platform + SYSTEM_FONT_DIRS: dict[str, list[Path]] = { + 'linux': [ + Path('/usr/share/fonts'), + Path('/usr/local/share/fonts'), + Path.home() / '.fonts', + Path.home() / '.local/share/fonts', + ], + 'freebsd': [ + Path('/usr/local/share/fonts'), + Path.home() / '.fonts', + ], + 'darwin': [ + Path('/Library/Fonts'), + Path('/System/Library/Fonts'), + Path.home() / 'Library/Fonts', + ], + # Windows is handled dynamically in _get_font_dirs() + } + + # Noto font logical names → possible filenames (priority order) + # The first match found will be used + NOTO_FONT_PATTERNS: dict[str, list[str]] = { + 'NotoSans-Regular': [ + 'NotoSans-Regular.ttf', + 'NotoSans-Regular.otf', + ], + 'NotoSansArabic-Regular': [ + 'NotoSansArabic-Regular.ttf', + 'NotoSansArabic-Regular.otf', + ], + 'NotoSansDevanagari-Regular': [ + 'NotoSansDevanagari-Regular.ttf', + 'NotoSansDevanagari-Regular.otf', + ], + 'NotoSansCJK-Regular': [ + # Language-specific variants (any will work for CJK) + 'NotoSansCJKsc-Regular.otf', # Simplified Chinese + 'NotoSansCJKtc-Regular.otf', # Traditional Chinese + 'NotoSansCJKjp-Regular.otf', # Japanese + 'NotoSansCJKkr-Regular.otf', # Korean + # TTC collections (common on Linux distros) + 'NotoSansCJK-Regular.ttc', + 'NotoSansCJKsc-Regular.ttc', + # Variable fonts + 'NotoSansCJKsc-VF.otf', + ], + 'NotoSansThai-Regular': [ + 'NotoSansThai-Regular.ttf', + 'NotoSansThai-Regular.otf', + ], + 'NotoSansHebrew-Regular': [ + 'NotoSansHebrew-Regular.ttf', + 'NotoSansHebrew-Regular.otf', + ], + 'NotoSansBengali-Regular': [ + 'NotoSansBengali-Regular.ttf', + 'NotoSansBengali-Regular.otf', + ], + 'NotoSansTamil-Regular': [ + 'NotoSansTamil-Regular.ttf', + 'NotoSansTamil-Regular.otf', + ], + 'NotoSansGujarati-Regular': [ + 'NotoSansGujarati-Regular.ttf', + 'NotoSansGujarati-Regular.otf', + ], + 'NotoSansTelugu-Regular': [ + 'NotoSansTelugu-Regular.ttf', + 'NotoSansTelugu-Regular.otf', + ], + 'NotoSansKannada-Regular': [ + 'NotoSansKannada-Regular.ttf', + 'NotoSansKannada-Regular.otf', + ], + 'NotoSansMalayalam-Regular': [ + 'NotoSansMalayalam-Regular.ttf', + 'NotoSansMalayalam-Regular.otf', + ], + 'NotoSansMyanmar-Regular': [ + 'NotoSansMyanmar-Regular.ttf', + 'NotoSansMyanmar-Regular.otf', + ], + 'NotoSansKhmer-Regular': [ + 'NotoSansKhmer-Regular.ttf', + 'NotoSansKhmer-Regular.otf', + ], + 'NotoSansLao-Regular': [ + 'NotoSansLao-Regular.ttf', + 'NotoSansLao-Regular.otf', + ], + 'NotoSansGeorgian-Regular': [ + 'NotoSansGeorgian-Regular.ttf', + 'NotoSansGeorgian-Regular.otf', + ], + 'NotoSansArmenian-Regular': [ + 'NotoSansArmenian-Regular.ttf', + 'NotoSansArmenian-Regular.otf', + ], + 'NotoSansEthiopic-Regular': [ + 'NotoSansEthiopic-Regular.ttf', + 'NotoSansEthiopic-Regular.otf', + ], + 'NotoSansSinhala-Regular': [ + 'NotoSansSinhala-Regular.ttf', + 'NotoSansSinhala-Regular.otf', + ], + 'NotoSansGurmukhi-Regular': [ + 'NotoSansGurmukhi-Regular.ttf', + 'NotoSansGurmukhi-Regular.otf', + ], + 'NotoSansOriya-Regular': [ + 'NotoSansOriya-Regular.ttf', + 'NotoSansOriya-Regular.otf', + ], + 'NotoSansTibetan-Regular': [ + 'NotoSansTibetan-Regular.ttf', + 'NotoSansTibetan-Regular.otf', + ], + } + + def __init__(self) -> None: + """Initialize system font provider with empty caches.""" + # Cache: font_name -> FontManager (successfully loaded fonts) + self._font_cache: dict[str, FontManager] = {} + # Negative cache: font names we've searched for but not found + self._not_found: set[str] = set() + # Cached font directories (computed lazily) + self._font_dirs: list[Path] | None = None + + def _get_platform(self) -> str: + """Get the current platform identifier. + + Returns: + Platform string: 'linux', 'darwin', 'windows', or 'freebsd' + """ + if sys.platform == 'win32': + return 'windows' + elif sys.platform == 'darwin': + return 'darwin' + elif 'freebsd' in sys.platform: + return 'freebsd' + else: + return 'linux' + + def _get_font_dirs(self) -> list[Path]: + """Get font directories for the current platform. + + Returns: + List of paths to search for fonts (may include non-existent paths) + """ + if self._font_dirs is not None: + return self._font_dirs + + platform = self._get_platform() + + if platform == 'windows': + # Get Windows font directories from environment + windir = os.environ.get('WINDIR', r'C:\Windows') + self._font_dirs = [Path(windir) / 'Fonts'] + # User-installed fonts (Windows 10+) + localappdata = os.environ.get('LOCALAPPDATA') + if localappdata: + self._font_dirs.append( + Path(localappdata) / 'Microsoft' / 'Windows' / 'Fonts' + ) + else: + self._font_dirs = list(self.SYSTEM_FONT_DIRS.get(platform, [])) + + return self._font_dirs + + def _find_font_file(self, font_name: str) -> Path | None: + """Search system directories for a font file. + + Args: + font_name: Logical font name (e.g., 'NotoSansCJK-Regular') + + Returns: + Path to font file if found, None otherwise + """ + if font_name not in self.NOTO_FONT_PATTERNS: + return None + + patterns = self.NOTO_FONT_PATTERNS[font_name] + + for font_dir in self._get_font_dirs(): + if not font_dir.exists(): + continue + + for pattern in patterns: + # Search recursively for the font file + try: + matches = list(font_dir.rglob(pattern)) + if matches: + log.debug( + "Found system font %s at %s", font_name, matches[0] + ) + return matches[0] + except PermissionError: + # Skip directories we can't read + continue + + return None + + def get_font(self, font_name: str) -> FontManager | None: + """Get a FontManager for the named font (lazy loading). + + This method implements lazy scanning: fonts are only searched for + when first requested. Results (both positive and negative) are + cached for subsequent calls. + + Args: + font_name: Logical font name (e.g., 'NotoSansCJK-Regular') + + Returns: + FontManager if font is found and loadable, None otherwise + """ + # Check positive cache first + if font_name in self._font_cache: + return self._font_cache[font_name] + + # Check negative cache (already searched, not found) + if font_name in self._not_found: + return None + + # Lazy scan for this specific font + font_path = self._find_font_file(font_name) + if font_path is not None: + try: + fm = FontManager(font_path) + self._font_cache[font_name] = fm + return fm + except Exception as e: + log.warning( + "Found font %s at %s but failed to load: %s", + font_name, + font_path, + e, + ) + + # Cache negative result + self._not_found.add(font_name) + return None + + def get_available_fonts(self) -> list[str]: + """Get list of font names this provider can potentially find. + + Note: This returns all font names we know patterns for, not + necessarily fonts that are actually installed. Use get_font() + to check if a specific font is available. + + Returns: + List of logical font names + """ + return list(self.NOTO_FONT_PATTERNS.keys()) + + def get_fallback_font(self) -> FontManager: + """Get the glyphless fallback font. + + Raises: + NotImplementedError: System provider doesn't provide fallback. + Use BuiltinFontProvider for the fallback font. + """ + raise NotImplementedError( + "SystemFontProvider does not provide a fallback font. " + "Use BuiltinFontProvider for Occulta.ttf fallback." + )