release: public alpha 0.6.0 baseline
This commit is contained in:
JWeBot Contributors
Executable
+764
@@ -0,0 +1,764 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure local JWeBot latency and summarize runtime telemetry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gc
|
||||
import importlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[1]
|
||||
CONFIG_SRC = PROJECT_ROOT / "packages" / "jwebot-config" / "src"
|
||||
SDK_SRC = PROJECT_ROOT / "packages" / "jwebot-sdk" / "src"
|
||||
KERNEL_SRC = PROJECT_ROOT / "packages" / "jwebot-kernel" / "src"
|
||||
HOST_SRC = PROJECT_ROOT / "packages" / "jwebot-extension-host" / "src"
|
||||
OFFICIAL_SRC = (
|
||||
PROJECT_ROOT
|
||||
/ "packages"
|
||||
/ "jwebot-official-extensions"
|
||||
/ "src"
|
||||
)
|
||||
sys.path[:0] = [
|
||||
str(PROJECT_ROOT),
|
||||
str(CONFIG_SRC),
|
||||
str(SDK_SRC),
|
||||
str(OFFICIAL_SRC),
|
||||
str(KERNEL_SRC),
|
||||
str(HOST_SRC),
|
||||
]
|
||||
os.environ["JWEBOT_CONFIG_ROOT"] = str(
|
||||
PROJECT_ROOT / ".benchmark-config-does-not-exist"
|
||||
)
|
||||
|
||||
import wechat_local_ocr as ocr # noqa: E402
|
||||
from jwebot_kernel import ( # noqa: E402
|
||||
ExtensionHostClient,
|
||||
build_release,
|
||||
install_release,
|
||||
sign_release_record,
|
||||
)
|
||||
from jwebot_kernel import edge_runtime # noqa: E402
|
||||
|
||||
|
||||
REPORT_SCHEMA_VERSION = 1
|
||||
PUBLISHER = "performance-baseline"
|
||||
|
||||
|
||||
def percentile(values: Sequence[float], quantile: float) -> float:
|
||||
"""Return an interpolated percentile for a non-empty sample."""
|
||||
|
||||
if not values:
|
||||
raise ValueError("percentile requires at least one value")
|
||||
if not 0 <= quantile <= 100:
|
||||
raise ValueError("percentile must be between 0 and 100")
|
||||
ordered = sorted(float(value) for value in values)
|
||||
position = (len(ordered) - 1) * quantile / 100
|
||||
lower = math.floor(position)
|
||||
upper = math.ceil(position)
|
||||
if lower == upper:
|
||||
return ordered[lower]
|
||||
fraction = position - lower
|
||||
return ordered[lower] + (ordered[upper] - ordered[lower]) * fraction
|
||||
|
||||
|
||||
def summarize_ms(values: Sequence[float]) -> dict[str, float | int]:
|
||||
if not values:
|
||||
raise ValueError("latency summary requires at least one value")
|
||||
return {
|
||||
"samples": len(values),
|
||||
"mean_ms": round(sum(values) / len(values), 3),
|
||||
"min_ms": round(min(values), 3),
|
||||
"p50_ms": round(percentile(values, 50), 3),
|
||||
"p95_ms": round(percentile(values, 95), 3),
|
||||
"max_ms": round(max(values), 3),
|
||||
}
|
||||
|
||||
|
||||
def measure_case(
|
||||
name: str,
|
||||
operation: Callable[[], object],
|
||||
*,
|
||||
iterations: int,
|
||||
warmups: int,
|
||||
category: str,
|
||||
) -> dict[str, Any]:
|
||||
for _ in range(warmups):
|
||||
operation()
|
||||
samples: list[float] = []
|
||||
gc_enabled = gc.isenabled()
|
||||
gc.disable()
|
||||
try:
|
||||
for _ in range(iterations):
|
||||
started = time.perf_counter_ns()
|
||||
operation()
|
||||
samples.append((time.perf_counter_ns() - started) / 1_000_000)
|
||||
finally:
|
||||
if gc_enabled:
|
||||
gc.enable()
|
||||
return {
|
||||
"name": name,
|
||||
"category": category,
|
||||
"status": "measured",
|
||||
**summarize_ms(samples),
|
||||
}
|
||||
|
||||
|
||||
def skipped_case(name: str, category: str, reason: str) -> dict[str, Any]:
|
||||
return {
|
||||
"name": name,
|
||||
"category": category,
|
||||
"status": "skipped",
|
||||
"reason": reason[:160],
|
||||
}
|
||||
|
||||
|
||||
def _synthetic_bubble(width: int = 500, height: int = 180) -> Image.Image:
|
||||
image = Image.new("RGB", (width, height), ocr.INCOMING_COLOR)
|
||||
draw = ImageDraw.Draw(image)
|
||||
for top in (22, 58, 94, 130):
|
||||
draw.rectangle((24, top, width - 28, top + 12), fill=(38, 38, 38))
|
||||
return image
|
||||
|
||||
|
||||
def _synthetic_message_view(width: int, height: int) -> Image.Image:
|
||||
image = Image.new("RGB", (width, height), (245, 245, 245))
|
||||
draw = ImageDraw.Draw(image)
|
||||
for index, top in enumerate((60, 180, 310, 455)):
|
||||
bubble_height = 72 + index * 8
|
||||
draw.rounded_rectangle(
|
||||
(64, top, 430, top + bubble_height),
|
||||
radius=6,
|
||||
fill=ocr.INCOMING_COLOR,
|
||||
)
|
||||
draw.rectangle(
|
||||
(86, top + 20, 380, top + 31),
|
||||
fill=(35, 35, 35),
|
||||
)
|
||||
return image
|
||||
|
||||
|
||||
def benchmark_ocr(iterations: int, warmups: int) -> list[dict[str, Any]]:
|
||||
bubble = _synthetic_bubble()
|
||||
messages = [
|
||||
ocr.IncomingMessage(
|
||||
sender=f"sender-{index}",
|
||||
text=f"message-{index}",
|
||||
confidence=0.99,
|
||||
sender_confidence=0.99,
|
||||
body_confidence=0.99,
|
||||
y=index * 30,
|
||||
)
|
||||
for index in range(18)
|
||||
]
|
||||
previous = [message.signature for message in messages[:-1]]
|
||||
|
||||
frame_image = Image.new("RGB", (2, 2), "white")
|
||||
sequence = 0
|
||||
|
||||
def buffer_round_trip() -> None:
|
||||
nonlocal sequence
|
||||
buffer = ocr.CaptureFrameBuffer(12)
|
||||
for _ in range(12):
|
||||
sequence += 1
|
||||
buffer.offer(
|
||||
ocr.CapturedFrame(
|
||||
image=frame_image,
|
||||
layout_hash=f"layout-{sequence}",
|
||||
visual_hash=f"visual-{sequence}",
|
||||
observed_at="2026-10-14T00:00:00+00:00",
|
||||
captured_monotonic=float(sequence),
|
||||
sequence=sequence,
|
||||
latest_synced=True,
|
||||
)
|
||||
)
|
||||
buffer.get_latest()
|
||||
|
||||
results = [
|
||||
measure_case(
|
||||
"ocr.text_line_regions",
|
||||
lambda: ocr.text_line_regions(bubble),
|
||||
iterations=iterations,
|
||||
warmups=warmups,
|
||||
category="ocr-preprocess",
|
||||
),
|
||||
measure_case(
|
||||
"ocr.sequence_suffix",
|
||||
lambda: ocr.new_sequence_suffix(previous, messages[1:]),
|
||||
iterations=iterations,
|
||||
warmups=warmups,
|
||||
category="ocr-sequence",
|
||||
),
|
||||
measure_case(
|
||||
"ocr.capture_buffer_latest",
|
||||
buffer_round_trip,
|
||||
iterations=iterations,
|
||||
warmups=warmups,
|
||||
category="capture-queue",
|
||||
),
|
||||
]
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="jwebot-performance-ocr-queue-"
|
||||
) as value:
|
||||
original_queue = ocr.QUEUE_DIR
|
||||
ocr.QUEUE_DIR = Path(value)
|
||||
counter = 0
|
||||
|
||||
def durable_spool() -> None:
|
||||
nonlocal counter
|
||||
counter += 1
|
||||
ocr.queue_event(
|
||||
{
|
||||
"event_id": f"queue-{counter}",
|
||||
"observed_at": "2026-10-14T00:00:00+00:00",
|
||||
"sender": "PerformanceUser",
|
||||
"text": "@JWeBot /翻硬币",
|
||||
"mention_bot": True,
|
||||
"group_fingerprint": "performance-group",
|
||||
}
|
||||
)
|
||||
|
||||
try:
|
||||
results.append(
|
||||
measure_case(
|
||||
"ocr.local_event.durable_spool",
|
||||
durable_spool,
|
||||
iterations=iterations,
|
||||
warmups=warmups,
|
||||
category="ocr-persistence",
|
||||
)
|
||||
)
|
||||
finally:
|
||||
ocr.QUEUE_DIR = original_queue
|
||||
ocr.QUEUE_WAKE_EVENT.clear()
|
||||
try:
|
||||
importlib.import_module("cv2")
|
||||
except ModuleNotFoundError:
|
||||
results.extend(
|
||||
[
|
||||
skipped_case(
|
||||
"ocr.frame_fingerprint.full_view",
|
||||
"ocr-preprocess",
|
||||
"opencv-python-headless is not installed",
|
||||
),
|
||||
skipped_case(
|
||||
"ocr.frame_fingerprint.roi",
|
||||
"ocr-preprocess",
|
||||
"opencv-python-headless is not installed",
|
||||
),
|
||||
]
|
||||
)
|
||||
return results
|
||||
|
||||
message_width = ocr.MESSAGE_BOX[2] - ocr.MESSAGE_BOX[0]
|
||||
message_height = ocr.MESSAGE_BOX[3] - ocr.MESSAGE_BOX[1]
|
||||
full_view = _synthetic_message_view(message_width, message_height)
|
||||
region = _synthetic_message_view(540, 220)
|
||||
results.extend(
|
||||
[
|
||||
measure_case(
|
||||
"ocr.frame_fingerprint.full_view",
|
||||
lambda: ocr.incoming_frame_hashes(full_view),
|
||||
iterations=iterations,
|
||||
warmups=warmups,
|
||||
category="ocr-preprocess",
|
||||
),
|
||||
measure_case(
|
||||
"ocr.frame_fingerprint.roi",
|
||||
lambda: ocr.incoming_frame_hashes(region),
|
||||
iterations=iterations,
|
||||
warmups=warmups,
|
||||
category="ocr-preprocess",
|
||||
),
|
||||
]
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def _edge_config(root: Path) -> edge_runtime.EdgeConfig:
|
||||
return edge_runtime.EdgeConfig(
|
||||
socket_path=root / "edge.sock",
|
||||
database_path=root / "edge.sqlite3",
|
||||
cloud_host="",
|
||||
cloud_agent="wechat-responder",
|
||||
deepseek_key="",
|
||||
deepseek_base_url="https://api.deepseek.com",
|
||||
deepseek_model=edge_runtime.EDGE_FLASH_MODEL,
|
||||
memory_remote_path="/home/node/.openclaw/workspace/MEMORY.md",
|
||||
delivery_enabled=False,
|
||||
steam_api_key="",
|
||||
steam_id64="",
|
||||
steam_proxy_url=edge_runtime.DEFAULT_STEAM_MIHOMO_PROXY_URL,
|
||||
extension_mode="in-process",
|
||||
extension_state_root=root / "extensions",
|
||||
extension_allowed_signers=root / "allowed_signers",
|
||||
extension_socket_root=root / "sockets",
|
||||
telemetry_enabled=False,
|
||||
)
|
||||
|
||||
|
||||
def _event(identifier: str, text: str) -> dict[str, object]:
|
||||
return {
|
||||
"event_id": identifier,
|
||||
"group_fingerprint": "performance-group",
|
||||
"sender": "PerformanceUser",
|
||||
"text": text,
|
||||
"observed_at": "2026-10-14T00:00:00+00:00",
|
||||
"mention_bot": True,
|
||||
"confidence": 0.99,
|
||||
"sender_confidence": 0.99,
|
||||
"body_confidence": 0.99,
|
||||
"source": "performance-baseline",
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _edge_application() -> Iterator[edge_runtime.EdgeApplication]:
|
||||
with tempfile.TemporaryDirectory(prefix="jwebot-performance-edge-") as value:
|
||||
application = edge_runtime.EdgeApplication(_edge_config(Path(value)))
|
||||
try:
|
||||
yield application
|
||||
finally:
|
||||
application.close()
|
||||
|
||||
|
||||
def benchmark_edge(iterations: int, warmups: int) -> list[dict[str, Any]]:
|
||||
total = iterations + warmups
|
||||
results: list[dict[str, Any]] = []
|
||||
with _edge_application() as application:
|
||||
counter = 0
|
||||
|
||||
def local_command() -> None:
|
||||
nonlocal counter
|
||||
counter += 1
|
||||
response = application.accept(
|
||||
_event(f"local-{counter}", "@JWeBot /翻硬币")
|
||||
)
|
||||
if response.get("status") != "reply":
|
||||
raise RuntimeError("local benchmark did not return a reply")
|
||||
|
||||
results.append(
|
||||
measure_case(
|
||||
"edge.local_command.in_process",
|
||||
local_command,
|
||||
iterations=iterations,
|
||||
warmups=warmups,
|
||||
category="edge-ledger",
|
||||
)
|
||||
)
|
||||
|
||||
with _edge_application() as application:
|
||||
counter = 0
|
||||
|
||||
def prepare_ack() -> None:
|
||||
nonlocal counter
|
||||
counter += 1
|
||||
response = application.accept(
|
||||
_event(f"ack-prepare-{counter}", "@JWeBot benchmark chat")
|
||||
)
|
||||
if response.get("fast_ack") is not True:
|
||||
raise RuntimeError("chat benchmark did not prepare an ACK")
|
||||
|
||||
results.append(
|
||||
measure_case(
|
||||
"edge.chat_ack.prepare",
|
||||
prepare_ack,
|
||||
iterations=iterations,
|
||||
warmups=warmups,
|
||||
category="edge-ledger",
|
||||
)
|
||||
)
|
||||
|
||||
with _edge_application() as application:
|
||||
identifiers = [f"ack-confirm-{index}" for index in range(total)]
|
||||
for identifier in identifiers:
|
||||
application.accept(
|
||||
_event(identifier, "@JWeBot benchmark confirmation")
|
||||
)
|
||||
iterator = iter(identifiers)
|
||||
|
||||
def confirm_ack() -> None:
|
||||
response = application.confirm_fast_acknowledgement(
|
||||
next(iterator),
|
||||
delivered=True,
|
||||
)
|
||||
if response.get("acknowledged") is not True:
|
||||
raise RuntimeError("chat benchmark did not confirm an ACK")
|
||||
|
||||
results.append(
|
||||
measure_case(
|
||||
"edge.chat_ack.confirm",
|
||||
confirm_ack,
|
||||
iterations=iterations,
|
||||
warmups=warmups,
|
||||
category="edge-ledger",
|
||||
)
|
||||
)
|
||||
|
||||
with _edge_application() as application:
|
||||
identifiers = [f"queue-claim-{index}" for index in range(total)]
|
||||
for identifier in identifiers:
|
||||
application.accept(_event(identifier, "@JWeBot queued benchmark"))
|
||||
application.confirm_fast_acknowledgement(
|
||||
identifier,
|
||||
delivered=True,
|
||||
)
|
||||
|
||||
def claim_queue() -> None:
|
||||
row = application.ledger.claim_direct_chat()
|
||||
if row is None:
|
||||
raise RuntimeError("chat benchmark queue is empty")
|
||||
|
||||
results.append(
|
||||
measure_case(
|
||||
"edge.direct_queue.claim",
|
||||
claim_queue,
|
||||
iterations=iterations,
|
||||
warmups=warmups,
|
||||
category="edge-ledger",
|
||||
)
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def _publisher_key(root: Path) -> tuple[Path, Path]:
|
||||
executable = shutil.which("ssh-keygen")
|
||||
if executable is None:
|
||||
raise RuntimeError("ssh-keygen is unavailable")
|
||||
private_key = root / "publisher"
|
||||
completed = subprocess.run(
|
||||
[
|
||||
executable,
|
||||
"-q",
|
||||
"-t",
|
||||
"ed25519",
|
||||
"-N",
|
||||
"",
|
||||
"-f",
|
||||
str(private_key),
|
||||
],
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
raise RuntimeError("cannot create the benchmark publisher key")
|
||||
allowed_signers = root / "allowed_signers"
|
||||
allowed_signers.write_text(
|
||||
f"{PUBLISHER} "
|
||||
+ private_key.with_suffix(".pub").read_text(encoding="utf-8"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return private_key, allowed_signers
|
||||
|
||||
|
||||
def _host_wrapper(root: Path) -> Path:
|
||||
paths = [str(SDK_SRC), str(KERNEL_SRC), str(HOST_SRC)]
|
||||
try:
|
||||
import tomli
|
||||
except ModuleNotFoundError:
|
||||
pass
|
||||
else:
|
||||
paths.append(str(Path(tomli.__file__).resolve().parent.parent))
|
||||
wrapper = root / "host-wrapper.py"
|
||||
wrapper.write_text(
|
||||
"\n".join(
|
||||
(
|
||||
"import runpy, sys",
|
||||
f"sys.path[:0] = {paths!r}",
|
||||
"runpy.run_module('jwebot_extension_host', run_name='__main__')",
|
||||
)
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return wrapper
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _signed_coin_host() -> Iterator[
|
||||
tuple[ExtensionHostClient, Path, Mapping[str, object]]
|
||||
]:
|
||||
with tempfile.TemporaryDirectory(prefix="jwebot-performance-host-") as value:
|
||||
root = Path(value)
|
||||
private_key, allowed_signers = _publisher_key(root)
|
||||
artifact, record_path, _ = build_release(
|
||||
PROJECT_ROOT / "extensions" / "alice" / "coin",
|
||||
root / "dist",
|
||||
)
|
||||
signature_path, _ = sign_release_record(
|
||||
record_path,
|
||||
private_key,
|
||||
PUBLISHER,
|
||||
)
|
||||
release_dir = install_release(
|
||||
artifact,
|
||||
record_path,
|
||||
root / "state" / "coin",
|
||||
signature=signature_path,
|
||||
allowed_signers=allowed_signers,
|
||||
expected_publisher=PUBLISHER,
|
||||
health_check=lambda release: None,
|
||||
)
|
||||
client = ExtensionHostClient(
|
||||
command=(sys.executable, str(_host_wrapper(root))),
|
||||
timeout_seconds=8,
|
||||
allowed_signers=allowed_signers,
|
||||
expected_publisher=PUBLISHER,
|
||||
plugin_id="coin",
|
||||
)
|
||||
request = {
|
||||
"op": "invoke_command",
|
||||
"command": "/翻硬币",
|
||||
"arguments": "",
|
||||
"event": _event("host-benchmark", "@JWeBot /翻硬币"),
|
||||
}
|
||||
yield client, release_dir, request
|
||||
|
||||
|
||||
def benchmark_host(iterations: int, warmups: int) -> list[dict[str, Any]]:
|
||||
host_iterations = min(iterations, 25)
|
||||
host_warmups = min(warmups, 3)
|
||||
try:
|
||||
with _signed_coin_host() as (client, release_dir, request):
|
||||
health = measure_case(
|
||||
"host.signed_process.health",
|
||||
lambda: client.invoke(release_dir, {"op": "health"}),
|
||||
iterations=host_iterations,
|
||||
warmups=host_warmups,
|
||||
category="extension-host",
|
||||
)
|
||||
|
||||
def invoke_command() -> None:
|
||||
response = client.invoke(
|
||||
release_dir,
|
||||
request,
|
||||
service_handler=(
|
||||
lambda profile, operation, payload: {
|
||||
"text": "benchmark reply"
|
||||
}
|
||||
),
|
||||
)
|
||||
if response.get("status") != "ok":
|
||||
raise RuntimeError("signed Host benchmark failed")
|
||||
|
||||
command = measure_case(
|
||||
"host.signed_process.command",
|
||||
invoke_command,
|
||||
iterations=host_iterations,
|
||||
warmups=host_warmups,
|
||||
category="extension-host",
|
||||
)
|
||||
return [health, command]
|
||||
except Exception as error:
|
||||
reason = f"{type(error).__name__}: {error}"
|
||||
return [
|
||||
skipped_case(
|
||||
"host.signed_process.health",
|
||||
"extension-host",
|
||||
reason,
|
||||
),
|
||||
skipped_case(
|
||||
"host.signed_process.command",
|
||||
"extension-host",
|
||||
reason,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def parse_telemetry_line(line: str) -> dict[str, Any] | None:
|
||||
text = line.strip()
|
||||
marker = "telemetry "
|
||||
if marker in text:
|
||||
text = text.split(marker, 1)[1].strip()
|
||||
elif not text.startswith("{"):
|
||||
return None
|
||||
try:
|
||||
value = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if not isinstance(value, dict) or not isinstance(value.get("name"), str):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def read_telemetry(paths: Iterable[str]) -> list[dict[str, Any]]:
|
||||
records: list[dict[str, Any]] = []
|
||||
for value in paths:
|
||||
if value == "-":
|
||||
lines = sys.stdin
|
||||
records.extend(
|
||||
record
|
||||
for line in lines
|
||||
if (record := parse_telemetry_line(line)) is not None
|
||||
)
|
||||
continue
|
||||
with Path(value).open("r", encoding="utf-8", errors="replace") as source:
|
||||
records.extend(
|
||||
record
|
||||
for line in source
|
||||
if (record := parse_telemetry_line(line)) is not None
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
def summarize_telemetry(records: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
latencies: dict[str, list[float]] = {}
|
||||
statuses: dict[str, int] = {}
|
||||
event_hashes: set[str] = set()
|
||||
for record in records:
|
||||
name = str(record.get("name", "unknown"))[:80]
|
||||
status = str(record.get("status", "unknown"))[:24]
|
||||
statuses[f"{name}:{status}"] = statuses.get(f"{name}:{status}", 0) + 1
|
||||
event_hash = record.get("event_hash")
|
||||
if isinstance(event_hash, str) and event_hash:
|
||||
event_hashes.add(event_hash)
|
||||
elapsed = record.get("elapsed_ms")
|
||||
if (
|
||||
isinstance(elapsed, (int, float))
|
||||
and not isinstance(elapsed, bool)
|
||||
and math.isfinite(float(elapsed))
|
||||
and float(elapsed) >= 0
|
||||
):
|
||||
latencies.setdefault(name, []).append(float(elapsed))
|
||||
stages = [
|
||||
{"name": name, **summarize_ms(values)}
|
||||
for name, values in latencies.items()
|
||||
]
|
||||
stages.sort(
|
||||
key=lambda value: (
|
||||
-float(value["p95_ms"]),
|
||||
str(value["name"]),
|
||||
)
|
||||
)
|
||||
return {
|
||||
"records": len(records),
|
||||
"events": len(event_hashes),
|
||||
"stages": stages,
|
||||
"statuses": dict(sorted(statuses.items())),
|
||||
}
|
||||
|
||||
|
||||
def build_report(
|
||||
*,
|
||||
iterations: int,
|
||||
warmups: int,
|
||||
include_local: bool,
|
||||
include_host: bool,
|
||||
telemetry_paths: Sequence[str],
|
||||
) -> dict[str, Any]:
|
||||
benchmarks: list[dict[str, Any]] = []
|
||||
if include_local:
|
||||
benchmarks.extend(benchmark_ocr(iterations, warmups))
|
||||
benchmarks.extend(benchmark_edge(iterations, warmups))
|
||||
if include_host:
|
||||
benchmarks.extend(benchmark_host(iterations, warmups))
|
||||
report: dict[str, Any] = {
|
||||
"schema_version": REPORT_SCHEMA_VERSION,
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(
|
||||
timespec="seconds"
|
||||
),
|
||||
"version": (PROJECT_ROOT / "VERSION").read_text(
|
||||
encoding="utf-8"
|
||||
).strip(),
|
||||
"environment": {
|
||||
"platform": platform.platform(),
|
||||
"python": platform.python_version(),
|
||||
"machine": platform.machine(),
|
||||
},
|
||||
"parameters": {
|
||||
"iterations": iterations,
|
||||
"warmups": warmups,
|
||||
"host_enabled": include_host,
|
||||
},
|
||||
"configured_latency_windows_ms": {
|
||||
"damage_debounce": round(
|
||||
ocr.DAMAGE_DEBOUNCE_SECONDS * 1000
|
||||
),
|
||||
"damage_max_burst": round(
|
||||
ocr.DAMAGE_MAX_BURST_SECONDS * 1000
|
||||
),
|
||||
"edge_idle_worker_poll": 1000,
|
||||
"follow_latest_settle": round(
|
||||
ocr.FOLLOW_LATEST_SETTLE_SECONDS * 1000
|
||||
),
|
||||
"ocr_responder_fallback_poll": 1000,
|
||||
"window_layout_settle": round(
|
||||
ocr.WINDOW_LAYOUT_SETTLE_SECONDS * 1000
|
||||
),
|
||||
},
|
||||
"benchmarks": benchmarks,
|
||||
}
|
||||
if telemetry_paths:
|
||||
report["telemetry"] = summarize_telemetry(
|
||||
read_telemetry(telemetry_paths)
|
||||
)
|
||||
return report
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--iterations", type=int, default=50)
|
||||
parser.add_argument("--warmups", type=int, default=5)
|
||||
parser.add_argument(
|
||||
"--skip-host",
|
||||
action="store_true",
|
||||
help="Skip signed Extension Host process measurements.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--telemetry",
|
||||
action="append",
|
||||
default=[],
|
||||
metavar="PATH",
|
||||
help="Read a telemetry log; use '-' for standard input.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--telemetry-only",
|
||||
action="store_true",
|
||||
help="Analyze telemetry without running local microbenchmarks.",
|
||||
)
|
||||
parser.add_argument("--output", type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
if args.iterations < 1:
|
||||
parser.error("--iterations must be positive")
|
||||
if args.warmups < 0:
|
||||
parser.error("--warmups cannot be negative")
|
||||
if args.telemetry_only and not args.telemetry:
|
||||
parser.error("--telemetry-only requires at least one --telemetry path")
|
||||
|
||||
report = build_report(
|
||||
iterations=args.iterations,
|
||||
warmups=args.warmups,
|
||||
include_local=not args.telemetry_only,
|
||||
include_host=not args.skip_host,
|
||||
telemetry_paths=args.telemetry,
|
||||
)
|
||||
payload = json.dumps(
|
||||
report,
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
sort_keys=True,
|
||||
)
|
||||
if args.output is not None:
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(payload + "\n", encoding="utf-8")
|
||||
print(payload)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user