"""PDF text-layer extraction and quality assessment for hybrid OCR.""" from __future__ import annotations import re import unicodedata from dataclasses import asdict, dataclass from typing import Any @dataclass class TextLayerPolicy: min_chars: int = 50 min_printable_ratio: float = 0.85 min_content_ratio: float = 0.60 max_replacement_ratio: float = 0.02 min_chars_per_megapixel: float = 25.0 @dataclass class TextLayerAssessment: usable: bool reason: str raw_chars: int non_whitespace_chars: int printable_ratio: float content_ratio: float replacement_ratio: float chars_per_megapixel: float def to_dict(self) -> dict[str, Any]: return asdict(self) def normalize_text(text: str) -> str: text = text.replace("\x00", "") text = text.replace("\r\n", "\n").replace("\r", "\n") lines = [re.sub(r"[ \t]+", " ", line).strip() for line in text.splitlines()] compact_lines: list[str] = [] previous_blank = False for line in lines: if line: compact_lines.append(line) previous_blank = False elif compact_lines and not previous_blank: compact_lines.append("") previous_blank = True return "\n".join(compact_lines).strip() def _is_content_character(character: str) -> bool: if character.isalnum(): return True code = ord(character) return ( 0x3400 <= code <= 0x4DBF or 0x4E00 <= code <= 0x9FFF or 0xF900 <= code <= 0xFAFF or 0x3040 <= code <= 0x30FF or 0xAC00 <= code <= 0xD7AF ) def assess_text_layer( text: str, *, width_points: float, height_points: float, policy: TextLayerPolicy, ) -> TextLayerAssessment: normalized = normalize_text(text) compact = [character for character in normalized if not character.isspace()] count = len(compact) if count == 0: return TextLayerAssessment(False, "empty_text_layer", len(text), 0, 0.0, 0.0, 0.0, 0.0) printable = sum(character.isprintable() and unicodedata.category(character) != "Cc" for character in compact) content = sum(_is_content_character(character) for character in compact) replacements = sum(character in {"\ufffd", "�"} for character in compact) page_megapixels = max((width_points * height_points) / 1_000_000.0, 0.01) printable_ratio = printable / count content_ratio = content / count replacement_ratio = replacements / count density = count / page_megapixels checks = ( (count >= policy.min_chars, "too_few_characters"), (printable_ratio >= policy.min_printable_ratio, "low_printable_ratio"), (content_ratio >= policy.min_content_ratio, "low_content_ratio"), (replacement_ratio <= policy.max_replacement_ratio, "high_replacement_ratio"), (density >= policy.min_chars_per_megapixel, "low_text_density"), ) reason = "usable_text_layer" usable = True for passed, failure_reason in checks: if not passed: usable = False reason = failure_reason break return TextLayerAssessment( usable=usable, reason=reason, raw_chars=len(text), non_whitespace_chars=count, printable_ratio=round(printable_ratio, 4), content_ratio=round(content_ratio, 4), replacement_ratio=round(replacement_ratio, 4), chars_per_megapixel=round(density, 3), ) def extract_page_text(page: Any, policy: TextLayerPolicy) -> tuple[str, TextLayerAssessment]: text_page = page.get_textpage() try: raw_text = text_page.get_text_bounded() finally: text_page.close() width, height = page.get_size() normalized = normalize_text(raw_text) assessment = assess_text_layer( normalized, width_points=width, height_points=height, policy=policy, ) return normalized, assessment