128 lines
3.9 KiB
Python
128 lines
3.9 KiB
Python
"""PDF text-layer extraction and quality assessment for hybrid OCR."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import re
|
||
import unicodedata
|
||
from dataclasses import asdict, dataclass
|
||
from typing import Any
|
||
|
||
|
||
@dataclass
|
||
class TextLayerPolicy:
|
||
min_chars: int = 50
|
||
min_printable_ratio: float = 0.85
|
||
min_content_ratio: float = 0.60
|
||
max_replacement_ratio: float = 0.02
|
||
min_chars_per_megapixel: float = 25.0
|
||
|
||
|
||
@dataclass
|
||
class TextLayerAssessment:
|
||
usable: bool
|
||
reason: str
|
||
raw_chars: int
|
||
non_whitespace_chars: int
|
||
printable_ratio: float
|
||
content_ratio: float
|
||
replacement_ratio: float
|
||
chars_per_megapixel: float
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
return asdict(self)
|
||
|
||
|
||
def normalize_text(text: str) -> str:
|
||
text = text.replace("\x00", "")
|
||
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
||
lines = [re.sub(r"[ \t]+", " ", line).strip() for line in text.splitlines()]
|
||
compact_lines: list[str] = []
|
||
previous_blank = False
|
||
for line in lines:
|
||
if line:
|
||
compact_lines.append(line)
|
||
previous_blank = False
|
||
elif compact_lines and not previous_blank:
|
||
compact_lines.append("")
|
||
previous_blank = True
|
||
return "\n".join(compact_lines).strip()
|
||
|
||
|
||
def _is_content_character(character: str) -> bool:
|
||
if character.isalnum():
|
||
return True
|
||
code = ord(character)
|
||
return (
|
||
0x3400 <= code <= 0x4DBF
|
||
or 0x4E00 <= code <= 0x9FFF
|
||
or 0xF900 <= code <= 0xFAFF
|
||
or 0x3040 <= code <= 0x30FF
|
||
or 0xAC00 <= code <= 0xD7AF
|
||
)
|
||
|
||
|
||
def assess_text_layer(
|
||
text: str,
|
||
*,
|
||
width_points: float,
|
||
height_points: float,
|
||
policy: TextLayerPolicy,
|
||
) -> TextLayerAssessment:
|
||
normalized = normalize_text(text)
|
||
compact = [character for character in normalized if not character.isspace()]
|
||
count = len(compact)
|
||
if count == 0:
|
||
return TextLayerAssessment(False, "empty_text_layer", len(text), 0, 0.0, 0.0, 0.0, 0.0)
|
||
|
||
printable = sum(character.isprintable() and unicodedata.category(character) != "Cc" for character in compact)
|
||
content = sum(_is_content_character(character) for character in compact)
|
||
replacements = sum(character in {"\ufffd", "<EFBFBD>"} for character in compact)
|
||
page_megapixels = max((width_points * height_points) / 1_000_000.0, 0.01)
|
||
printable_ratio = printable / count
|
||
content_ratio = content / count
|
||
replacement_ratio = replacements / count
|
||
density = count / page_megapixels
|
||
|
||
checks = (
|
||
(count >= policy.min_chars, "too_few_characters"),
|
||
(printable_ratio >= policy.min_printable_ratio, "low_printable_ratio"),
|
||
(content_ratio >= policy.min_content_ratio, "low_content_ratio"),
|
||
(replacement_ratio <= policy.max_replacement_ratio, "high_replacement_ratio"),
|
||
(density >= policy.min_chars_per_megapixel, "low_text_density"),
|
||
)
|
||
reason = "usable_text_layer"
|
||
usable = True
|
||
for passed, failure_reason in checks:
|
||
if not passed:
|
||
usable = False
|
||
reason = failure_reason
|
||
break
|
||
|
||
return TextLayerAssessment(
|
||
usable=usable,
|
||
reason=reason,
|
||
raw_chars=len(text),
|
||
non_whitespace_chars=count,
|
||
printable_ratio=round(printable_ratio, 4),
|
||
content_ratio=round(content_ratio, 4),
|
||
replacement_ratio=round(replacement_ratio, 4),
|
||
chars_per_megapixel=round(density, 3),
|
||
)
|
||
|
||
|
||
def extract_page_text(page: Any, policy: TextLayerPolicy) -> tuple[str, TextLayerAssessment]:
|
||
text_page = page.get_textpage()
|
||
try:
|
||
raw_text = text_page.get_text_bounded()
|
||
finally:
|
||
text_page.close()
|
||
width, height = page.get_size()
|
||
normalized = normalize_text(raw_text)
|
||
assessment = assess_text_layer(
|
||
normalized,
|
||
width_points=width,
|
||
height_points=height,
|
||
policy=policy,
|
||
)
|
||
return normalized, assessment
|