128 lines
3.5 KiB
Python
128 lines
3.5 KiB
Python
import json
|
|
from pathlib import Path
|
|
|
|
from benchmark import (
|
|
BenchmarkCase,
|
|
CaseResult,
|
|
load_config,
|
|
profile_cases,
|
|
render_report,
|
|
run_case,
|
|
)
|
|
|
|
|
|
def test_smoke_profile_has_image_and_pdf_cases():
|
|
cases = profile_cases("smoke", "cpu")
|
|
assert {case.kind for case in cases} == {"image", "pdf"}
|
|
assert len({case.id for case in cases}) == len(cases)
|
|
|
|
|
|
def test_load_custom_config(tmp_path):
|
|
config = tmp_path / "matrix.json"
|
|
config.write_text(
|
|
json.dumps(
|
|
{
|
|
"cases": [
|
|
{
|
|
"id": "custom-image",
|
|
"title": "Custom",
|
|
"group": "custom",
|
|
"kind": "image",
|
|
"params": {"threads": 4},
|
|
}
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
cases = load_config(config)
|
|
assert cases == [
|
|
BenchmarkCase(
|
|
id="custom-image",
|
|
title="Custom",
|
|
group="custom",
|
|
kind="image",
|
|
params={"threads": 4},
|
|
)
|
|
]
|
|
|
|
|
|
def test_render_report_contains_speed_and_hash_comparison(tmp_path):
|
|
cases = [
|
|
BenchmarkCase("base", "Base", "image", "image", {"threads": 1}),
|
|
BenchmarkCase("fast", "Fast", "image", "image", {"threads": 8}),
|
|
]
|
|
results = [
|
|
CaseResult(
|
|
case=cases[0],
|
|
status="completed",
|
|
exit_code=0,
|
|
command=["python", "ocr.py"],
|
|
started_at="2026-01-01T00:00:00+08:00",
|
|
wall_seconds=20.0,
|
|
output_dir="out/base",
|
|
log_file="base.log",
|
|
completed_files=2,
|
|
discovered_files=2,
|
|
average_file_seconds=8.0,
|
|
throughput_files_per_minute=6.0,
|
|
result_hash="same",
|
|
recognized_chars=100,
|
|
),
|
|
CaseResult(
|
|
case=cases[1],
|
|
status="completed",
|
|
exit_code=0,
|
|
command=["python", "ocr.py"],
|
|
started_at="2026-01-01T00:00:00+08:00",
|
|
wall_seconds=10.0,
|
|
output_dir="out/fast",
|
|
log_file="fast.log",
|
|
completed_files=2,
|
|
discovered_files=2,
|
|
average_file_seconds=4.0,
|
|
throughput_files_per_minute=12.0,
|
|
result_hash="same",
|
|
recognized_chars=100,
|
|
),
|
|
]
|
|
report = tmp_path / "report.md"
|
|
render_report(
|
|
report_path=report,
|
|
run_id="test",
|
|
profile="test",
|
|
device="cpu",
|
|
data_dir=Path("data"),
|
|
cases=cases,
|
|
results=results,
|
|
run_root=Path("runs"),
|
|
started_at="2026-01-01T00:00:00+08:00",
|
|
)
|
|
text = report.read_text(encoding="utf-8")
|
|
assert "2.00x" in text
|
|
assert "结果一致" in text
|
|
assert "| 是 |" in text
|
|
|
|
|
|
def test_run_case_timeout_returns_124(tmp_path, monkeypatch):
|
|
script = tmp_path / "slow.py"
|
|
script.write_text("import time; time.sleep(10)", encoding="utf-8")
|
|
case = BenchmarkCase("slow", "Slow", "timeout", "image", {})
|
|
|
|
def fake_build_command(*args, **kwargs):
|
|
import sys
|
|
|
|
return [sys.executable, str(script)]
|
|
|
|
monkeypatch.setattr("benchmark.build_command", fake_build_command)
|
|
result = run_case(
|
|
case,
|
|
data_dir=tmp_path,
|
|
device="cpu",
|
|
run_root=tmp_path / "runs",
|
|
timeout=0.1,
|
|
)
|
|
assert result.exit_code == 124
|
|
assert result.status == "failed"
|
|
assert "timeout" in (result.error or "")
|