release: public alpha 0.6.0 baseline
This commit is contained in:
JWeBot Contributors
@@ -0,0 +1,18 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=68"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "jwebot-kernel"
|
||||
version = "0.6.0a0+1015"
|
||||
description = "Stable routing and extension authority for JWeBot"
|
||||
requires-python = ">=3.9"
|
||||
dependencies = [
|
||||
"jwebot-config==0.6.0a0+1015",
|
||||
"jwebot-sdk==0.6.0a0+1015",
|
||||
"jwebot-official-extensions==0.6.0a0+1015",
|
||||
"tomli>=2; python_version < '3.11'",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["src"]
|
||||
@@ -0,0 +1,110 @@
|
||||
"""Stable kernel contracts for JWeBot."""
|
||||
|
||||
from .activation import (
|
||||
ActivationError,
|
||||
install_release,
|
||||
rollback_release,
|
||||
verify_installed_release,
|
||||
verify_installed_release_signature,
|
||||
)
|
||||
from .catalog import CatalogError, CommandMatch, ExtensionCatalog
|
||||
from .extension_dispatch import (
|
||||
ExpectedExtension,
|
||||
HostedExtensionError,
|
||||
HostedExtensionRuntime,
|
||||
HostedRelease,
|
||||
SignedExtensionInventory,
|
||||
)
|
||||
from .host_client import (
|
||||
ExtensionHostClient,
|
||||
HostClientError,
|
||||
HostTimeout,
|
||||
UnixExtensionHostClient,
|
||||
)
|
||||
from .manifest import (
|
||||
CapabilityRequest,
|
||||
ManifestError,
|
||||
PluginManifest,
|
||||
PluginType,
|
||||
load_manifest,
|
||||
parse_manifest,
|
||||
)
|
||||
from .routing import (
|
||||
ExecutionPath,
|
||||
RouteDecision,
|
||||
RouteKind,
|
||||
classify_question,
|
||||
parse_local_command,
|
||||
)
|
||||
from .release import (
|
||||
ReleaseError,
|
||||
ReleaseRecord,
|
||||
build_release,
|
||||
load_release_record,
|
||||
release_record_bytes,
|
||||
verify_release,
|
||||
)
|
||||
from .signing import (
|
||||
ReleaseSignature,
|
||||
SignatureError,
|
||||
load_release_signature,
|
||||
release_signature_bytes,
|
||||
sign_release_record,
|
||||
verify_release_signature,
|
||||
)
|
||||
from .source_policy import (
|
||||
PolicyFinding,
|
||||
SourcePolicyError,
|
||||
validate_source_tree,
|
||||
)
|
||||
from .telemetry import RuntimeTelemetry, hash_identifier
|
||||
|
||||
__all__ = [
|
||||
"ActivationError",
|
||||
"CapabilityRequest",
|
||||
"CatalogError",
|
||||
"CommandMatch",
|
||||
"ExtensionCatalog",
|
||||
"ExpectedExtension",
|
||||
"ExecutionPath",
|
||||
"ExtensionHostClient",
|
||||
"HostClientError",
|
||||
"HostTimeout",
|
||||
"HostedExtensionError",
|
||||
"HostedExtensionRuntime",
|
||||
"HostedRelease",
|
||||
"UnixExtensionHostClient",
|
||||
"ManifestError",
|
||||
"PluginManifest",
|
||||
"PluginType",
|
||||
"PolicyFinding",
|
||||
"RouteDecision",
|
||||
"RouteKind",
|
||||
"RuntimeTelemetry",
|
||||
"ReleaseError",
|
||||
"ReleaseRecord",
|
||||
"ReleaseSignature",
|
||||
"SignatureError",
|
||||
"SignedExtensionInventory",
|
||||
"SourcePolicyError",
|
||||
"classify_question",
|
||||
"build_release",
|
||||
"hash_identifier",
|
||||
"load_manifest",
|
||||
"load_release_record",
|
||||
"load_release_signature",
|
||||
"parse_local_command",
|
||||
"parse_manifest",
|
||||
"release_record_bytes",
|
||||
"release_signature_bytes",
|
||||
"install_release",
|
||||
"rollback_release",
|
||||
"sign_release_record",
|
||||
"validate_source_tree",
|
||||
"verify_release",
|
||||
"verify_release_signature",
|
||||
"verify_installed_release",
|
||||
"verify_installed_release_signature",
|
||||
]
|
||||
|
||||
__version__ = "0.6.0a0+1015"
|
||||
@@ -0,0 +1,319 @@
|
||||
"""Verified release installation with atomic activation and rollback pointers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Callable, Iterator
|
||||
|
||||
from .release import (
|
||||
ReleaseError,
|
||||
ReleaseRecord,
|
||||
load_release_record,
|
||||
source_digest_for_files,
|
||||
verify_release,
|
||||
)
|
||||
from .signing import (
|
||||
ReleaseSignature,
|
||||
load_release_signature,
|
||||
release_signature_bytes,
|
||||
verify_release_signature,
|
||||
)
|
||||
|
||||
|
||||
class ActivationError(ValueError):
|
||||
"""Raised when a verified release cannot be installed or activated safely."""
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _activation_lock(state_root: Path) -> Iterator[None]:
|
||||
state_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
if state_root.is_symlink():
|
||||
raise ActivationError("activation state root cannot be a symlink")
|
||||
lock_path = state_root / ".activation.lock"
|
||||
descriptor = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o600)
|
||||
try:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_EX)
|
||||
yield
|
||||
finally:
|
||||
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
||||
os.close(descriptor)
|
||||
|
||||
|
||||
def _release_target(digest: str) -> Path:
|
||||
if len(digest) != 64 or any(value not in "0123456789abcdef" for value in digest):
|
||||
raise ActivationError("invalid release digest")
|
||||
return Path("releases") / digest
|
||||
|
||||
|
||||
def _read_pointer(state_root: Path, name: str) -> Path | None:
|
||||
pointer = state_root / name
|
||||
if not pointer.exists() and not pointer.is_symlink():
|
||||
return None
|
||||
if not pointer.is_symlink():
|
||||
raise ActivationError(f"{name} pointer is not a symlink")
|
||||
target = Path(os.readlink(pointer))
|
||||
if target.is_absolute() or len(target.parts) != 2 or target.parts[0] != "releases":
|
||||
raise ActivationError(f"{name} pointer has an unsafe target")
|
||||
if not (state_root / target).is_dir():
|
||||
raise ActivationError(f"{name} pointer target is unavailable")
|
||||
return target
|
||||
|
||||
|
||||
def _replace_pointer(state_root: Path, name: str, target: Path) -> None:
|
||||
temporary = state_root / f".{name}.{os.getpid()}.tmp"
|
||||
temporary.unlink(missing_ok=True)
|
||||
temporary.symlink_to(target)
|
||||
try:
|
||||
os.replace(temporary, state_root / name)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _installed_files(
|
||||
release_dir: Path,
|
||||
record: ReleaseRecord,
|
||||
) -> tuple[tuple[str, bytes], ...]:
|
||||
values: list[tuple[str, bytes]] = []
|
||||
for path in sorted(release_dir.rglob("*")):
|
||||
relative = path.relative_to(release_dir).as_posix()
|
||||
if relative in {".release.json", ".release.sig.json"}:
|
||||
continue
|
||||
if path.is_symlink():
|
||||
raise ActivationError(f"installed release contains a symlink: {relative}")
|
||||
if path.is_file():
|
||||
values.append((relative, path.read_bytes()))
|
||||
return tuple(values)
|
||||
|
||||
|
||||
def _verify_installed(
|
||||
release_dir: Path,
|
||||
record: ReleaseRecord,
|
||||
signature: ReleaseSignature | None = None,
|
||||
) -> None:
|
||||
installed = _installed_files(release_dir, record)
|
||||
if tuple(path for path, _ in installed) != record.files:
|
||||
raise ActivationError("installed release file list mismatch")
|
||||
if source_digest_for_files(installed) != record.source_digest:
|
||||
raise ActivationError("installed release source digest mismatch")
|
||||
stored_record = release_dir / ".release.json"
|
||||
try:
|
||||
stored = ReleaseRecord.from_mapping(
|
||||
json.loads(stored_record.read_text(encoding="utf-8"))
|
||||
)
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError, ReleaseError) as error:
|
||||
raise ActivationError("installed release record is invalid") from error
|
||||
if stored != record:
|
||||
raise ActivationError("installed release record mismatch")
|
||||
if signature is not None:
|
||||
stored_signature_path = release_dir / ".release.sig.json"
|
||||
try:
|
||||
stored_signature = load_release_signature(stored_signature_path)
|
||||
except ReleaseError as error:
|
||||
raise ActivationError(
|
||||
"installed release signature is invalid"
|
||||
) from error
|
||||
if stored_signature != signature:
|
||||
raise ActivationError("installed release signature mismatch")
|
||||
|
||||
|
||||
def _extract_verified(
|
||||
artifact: Path,
|
||||
record: ReleaseRecord,
|
||||
destination: Path,
|
||||
signature: ReleaseSignature | None = None,
|
||||
) -> None:
|
||||
with tarfile.open(artifact, mode="r:gz") as archive:
|
||||
members = archive.getmembers()
|
||||
for member in members:
|
||||
path = PurePosixPath(member.name)
|
||||
relative = PurePosixPath(*path.parts[1:])
|
||||
target = destination.joinpath(*relative.parts)
|
||||
target.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
stream = archive.extractfile(member)
|
||||
if stream is None:
|
||||
raise ActivationError("verified artifact member cannot be read")
|
||||
descriptor = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as output:
|
||||
shutil.copyfileobj(stream, output)
|
||||
except Exception:
|
||||
try:
|
||||
os.close(descriptor)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
(destination / ".release.json").write_text(
|
||||
json.dumps(
|
||||
record.to_mapping(),
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.chmod(destination / ".release.json", 0o600)
|
||||
if signature is not None:
|
||||
(destination / ".release.sig.json").write_bytes(
|
||||
release_signature_bytes(signature)
|
||||
)
|
||||
os.chmod(destination / ".release.sig.json", 0o600)
|
||||
|
||||
|
||||
def install_release(
|
||||
artifact: Path,
|
||||
release_record: Path,
|
||||
state_root: Path,
|
||||
*,
|
||||
signature: Path | None = None,
|
||||
allowed_signers: Path | None = None,
|
||||
expected_publisher: str | None = None,
|
||||
allow_unsigned: bool = False,
|
||||
health_check: Callable[[Path], None] | None = None,
|
||||
allow_unhealthy_local: bool = False,
|
||||
) -> Path:
|
||||
record = load_release_record(release_record)
|
||||
if allow_unsigned:
|
||||
if (
|
||||
signature is not None
|
||||
or allowed_signers is not None
|
||||
or expected_publisher is not None
|
||||
):
|
||||
raise ActivationError(
|
||||
"unsigned local activation cannot include signature inputs"
|
||||
)
|
||||
signature_record = None
|
||||
else:
|
||||
if signature is None or allowed_signers is None:
|
||||
raise ActivationError("signed release activation is required")
|
||||
signature_record = verify_release_signature(
|
||||
release_record,
|
||||
signature,
|
||||
allowed_signers,
|
||||
expected_publisher=expected_publisher,
|
||||
)
|
||||
if health_check is None and not allow_unhealthy_local:
|
||||
raise ActivationError("signed release health check is required")
|
||||
verify_release(artifact, record)
|
||||
relative_target = _release_target(record.source_digest)
|
||||
releases = state_root / "releases"
|
||||
release_dir = state_root / relative_target
|
||||
|
||||
with _activation_lock(state_root):
|
||||
created = False
|
||||
releases.mkdir(mode=0o700, exist_ok=True)
|
||||
if releases.is_symlink():
|
||||
raise ActivationError("releases directory cannot be a symlink")
|
||||
if release_dir.is_symlink():
|
||||
raise ActivationError("release destination is unsafe")
|
||||
if release_dir.exists():
|
||||
if not release_dir.is_dir():
|
||||
raise ActivationError("release destination is unsafe")
|
||||
_verify_installed(release_dir, record, signature_record)
|
||||
else:
|
||||
staging = Path(
|
||||
tempfile.mkdtemp(
|
||||
prefix=f".{record.source_digest}.",
|
||||
dir=releases,
|
||||
)
|
||||
)
|
||||
os.chmod(staging, 0o700)
|
||||
try:
|
||||
_extract_verified(
|
||||
artifact,
|
||||
record,
|
||||
staging,
|
||||
signature_record,
|
||||
)
|
||||
_verify_installed(staging, record, signature_record)
|
||||
staging.replace(release_dir)
|
||||
created = True
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
if health_check is not None:
|
||||
try:
|
||||
health_check(release_dir)
|
||||
except Exception as error:
|
||||
if created:
|
||||
shutil.rmtree(release_dir, ignore_errors=True)
|
||||
raise ActivationError("release health check failed") from error
|
||||
|
||||
current = _read_pointer(state_root, "current")
|
||||
if current == relative_target:
|
||||
return release_dir
|
||||
if current is not None:
|
||||
_replace_pointer(state_root, "previous", current)
|
||||
_replace_pointer(state_root, "current", relative_target)
|
||||
return release_dir
|
||||
|
||||
|
||||
def verify_installed_release(release_dir: Path) -> ReleaseRecord:
|
||||
if not release_dir.is_dir() or release_dir.is_symlink():
|
||||
raise ActivationError("installed release directory is unsafe")
|
||||
record_path = release_dir / ".release.json"
|
||||
try:
|
||||
record = ReleaseRecord.from_mapping(
|
||||
json.loads(record_path.read_text(encoding="utf-8"))
|
||||
)
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError, ReleaseError) as error:
|
||||
raise ActivationError("installed release record is invalid") from error
|
||||
if release_dir.name != record.source_digest:
|
||||
raise ActivationError("installed release directory digest mismatch")
|
||||
_verify_installed(release_dir, record)
|
||||
return record
|
||||
|
||||
|
||||
def verify_installed_release_signature(
|
||||
release_dir: Path,
|
||||
allowed_signers: Path,
|
||||
*,
|
||||
expected_publisher: str | None = None,
|
||||
) -> tuple[ReleaseRecord, ReleaseSignature]:
|
||||
record = verify_installed_release(release_dir)
|
||||
signature = verify_release_signature(
|
||||
release_dir / ".release.json",
|
||||
release_dir / ".release.sig.json",
|
||||
allowed_signers,
|
||||
expected_publisher=expected_publisher,
|
||||
)
|
||||
_verify_installed(release_dir, record, signature)
|
||||
return record, signature
|
||||
|
||||
|
||||
def rollback_release(
|
||||
state_root: Path,
|
||||
*,
|
||||
allowed_signers: Path | None = None,
|
||||
expected_publisher: str | None = None,
|
||||
allow_unsigned: bool = False,
|
||||
) -> Path:
|
||||
with _activation_lock(state_root):
|
||||
current = _read_pointer(state_root, "current")
|
||||
previous = _read_pointer(state_root, "previous")
|
||||
if current is None or previous is None:
|
||||
raise ActivationError("no verified previous release is available")
|
||||
previous_dir = state_root / previous
|
||||
if allow_unsigned:
|
||||
if allowed_signers is not None or expected_publisher is not None:
|
||||
raise ActivationError(
|
||||
"unsigned local rollback cannot include trusted publishers"
|
||||
)
|
||||
verify_installed_release(previous_dir)
|
||||
else:
|
||||
if allowed_signers is None:
|
||||
raise ActivationError("signed release rollback is required")
|
||||
verify_installed_release_signature(
|
||||
previous_dir,
|
||||
allowed_signers,
|
||||
expected_publisher=expected_publisher,
|
||||
)
|
||||
_replace_pointer(state_root, "current", previous)
|
||||
_replace_pointer(state_root, "previous", current)
|
||||
return previous_dir
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Deterministic startup registration and runtime-frozen extension catalog."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Mapping
|
||||
|
||||
from .manifest import PluginManifest
|
||||
|
||||
|
||||
class CatalogError(ValueError):
|
||||
"""Raised when approved manifests cannot form one deterministic catalog."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CommandMatch:
|
||||
plugin_id: str
|
||||
command: str
|
||||
arguments: str
|
||||
|
||||
|
||||
class ExtensionCatalog:
|
||||
"""Mutable during startup, immutable after the kernel freezes registration."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._manifests: dict[str, PluginManifest] = {}
|
||||
self._commands: dict[str, str] = {}
|
||||
self._frozen = False
|
||||
|
||||
@property
|
||||
def frozen(self) -> bool:
|
||||
return self._frozen
|
||||
|
||||
@property
|
||||
def manifests(self) -> Mapping[str, PluginManifest]:
|
||||
return MappingProxyType(self._manifests)
|
||||
|
||||
@property
|
||||
def commands(self) -> Mapping[str, str]:
|
||||
return MappingProxyType(self._commands)
|
||||
|
||||
def register(self, manifest: PluginManifest) -> None:
|
||||
if self._frozen:
|
||||
raise CatalogError("extension catalog is frozen")
|
||||
if manifest.plugin_id in self._manifests:
|
||||
raise CatalogError(f"duplicate plugin id: {manifest.plugin_id}")
|
||||
for command in manifest.capabilities.commands:
|
||||
owner = self._commands.get(command.casefold())
|
||||
if owner is not None:
|
||||
raise CatalogError(
|
||||
f"command {command} already belongs to {owner}"
|
||||
)
|
||||
self._manifests[manifest.plugin_id] = manifest
|
||||
for command in manifest.capabilities.commands:
|
||||
self._commands[command.casefold()] = manifest.plugin_id
|
||||
|
||||
def freeze(self) -> None:
|
||||
self._frozen = True
|
||||
|
||||
def resolve_command(self, question: str) -> CommandMatch | None:
|
||||
if not self._frozen:
|
||||
raise CatalogError("extension catalog must be frozen before routing")
|
||||
normalized = question.lstrip()
|
||||
folded = normalized.casefold()
|
||||
for command_key, plugin_id in sorted(
|
||||
self._commands.items(),
|
||||
key=lambda item: len(item[0]),
|
||||
reverse=True,
|
||||
):
|
||||
if folded == command_key:
|
||||
return CommandMatch(
|
||||
plugin_id=plugin_id,
|
||||
command=normalized[: len(command_key)],
|
||||
arguments="",
|
||||
)
|
||||
prefix = command_key + " "
|
||||
if folded.startswith(prefix):
|
||||
return CommandMatch(
|
||||
plugin_id=plugin_id,
|
||||
command=normalized[: len(command_key)],
|
||||
arguments=normalized[len(prefix) :].strip(),
|
||||
)
|
||||
return None
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,257 @@
|
||||
"""Signed startup catalog and isolated extension-host dispatch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Protocol
|
||||
|
||||
from jwebot_sdk import API_VERSION, ContractViolation, ExtensionResult
|
||||
|
||||
from .activation import verify_installed_release_signature
|
||||
from .catalog import ExtensionCatalog
|
||||
from .host_client import ServiceHandler
|
||||
from .manifest import PluginManifest, load_manifest
|
||||
|
||||
|
||||
class HostedExtensionError(ValueError):
|
||||
"""Controlled signed-catalog, health, or invocation failure."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExpectedExtension:
|
||||
plugin_id: str
|
||||
author: str
|
||||
commands: tuple[str, ...]
|
||||
service_profiles: tuple[str, ...]
|
||||
description: str = ""
|
||||
help_entries: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HostedRelease:
|
||||
manifest: PluginManifest
|
||||
release_dir: Path
|
||||
|
||||
|
||||
class HostInvoker(Protocol):
|
||||
def __call__(
|
||||
self,
|
||||
request: Mapping[str, Any],
|
||||
service_handler: ServiceHandler | None,
|
||||
) -> Mapping[str, Any]: ...
|
||||
|
||||
|
||||
def _active_release_dir(plugin_root: Path) -> Path | None:
|
||||
current = plugin_root / "current"
|
||||
if not current.exists() and not current.is_symlink():
|
||||
return None
|
||||
if not current.is_symlink():
|
||||
raise HostedExtensionError("extension current pointer is not a symlink")
|
||||
target = Path(os.readlink(current))
|
||||
if (
|
||||
target.is_absolute()
|
||||
or len(target.parts) != 2
|
||||
or target.parts[0] != "releases"
|
||||
):
|
||||
raise HostedExtensionError("extension current pointer is unsafe")
|
||||
release_dir = plugin_root / target
|
||||
if not release_dir.is_dir() or release_dir.is_symlink():
|
||||
raise HostedExtensionError("extension current release is unavailable")
|
||||
return release_dir
|
||||
|
||||
|
||||
class SignedExtensionInventory:
|
||||
"""Load reviewed signed manifests during startup, then freeze routing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
catalog: ExtensionCatalog,
|
||||
releases: Mapping[str, HostedRelease],
|
||||
) -> None:
|
||||
if not catalog.frozen:
|
||||
raise HostedExtensionError("signed extension catalog must be frozen")
|
||||
self.catalog = catalog
|
||||
self._releases = MappingProxyType(dict(releases))
|
||||
|
||||
@property
|
||||
def releases(self) -> Mapping[str, HostedRelease]:
|
||||
return self._releases
|
||||
|
||||
@classmethod
|
||||
def load(
|
||||
cls,
|
||||
state_root: Path,
|
||||
allowed_signers: Path,
|
||||
expected: Sequence[ExpectedExtension],
|
||||
*,
|
||||
expected_publisher: str | None = None,
|
||||
require_all: bool = False,
|
||||
) -> "SignedExtensionInventory":
|
||||
catalog = ExtensionCatalog()
|
||||
releases: dict[str, HostedRelease] = {}
|
||||
missing: list[str] = []
|
||||
for descriptor in sorted(expected, key=lambda item: item.plugin_id):
|
||||
plugin_root = state_root / descriptor.plugin_id
|
||||
release_dir = _active_release_dir(plugin_root)
|
||||
if release_dir is None:
|
||||
missing.append(descriptor.plugin_id)
|
||||
continue
|
||||
try:
|
||||
record, _ = verify_installed_release_signature(
|
||||
release_dir,
|
||||
allowed_signers,
|
||||
expected_publisher=expected_publisher,
|
||||
)
|
||||
manifest = load_manifest(release_dir / "plugin.toml")
|
||||
except Exception as error:
|
||||
raise HostedExtensionError(
|
||||
f"signed extension verification failed: {descriptor.plugin_id}"
|
||||
) from error
|
||||
if (
|
||||
record.plugin_id != descriptor.plugin_id
|
||||
or record.author != descriptor.author
|
||||
or manifest.plugin_id != descriptor.plugin_id
|
||||
or manifest.author != descriptor.author
|
||||
or (
|
||||
descriptor.description
|
||||
and manifest.description != descriptor.description
|
||||
)
|
||||
or (
|
||||
descriptor.help_entries
|
||||
and manifest.help_entries != descriptor.help_entries
|
||||
)
|
||||
or manifest.capabilities.commands != descriptor.commands
|
||||
or manifest.capabilities.service_profiles
|
||||
!= descriptor.service_profiles
|
||||
):
|
||||
raise HostedExtensionError(
|
||||
f"signed extension inventory mismatch: {descriptor.plugin_id}"
|
||||
)
|
||||
catalog.register(manifest)
|
||||
releases[descriptor.plugin_id] = HostedRelease(
|
||||
manifest=manifest,
|
||||
release_dir=release_dir,
|
||||
)
|
||||
if require_all and missing:
|
||||
raise HostedExtensionError(
|
||||
"required signed extensions are missing: "
|
||||
+ ", ".join(sorted(missing))
|
||||
)
|
||||
catalog.freeze()
|
||||
return cls(catalog, releases)
|
||||
|
||||
|
||||
class HostedExtensionRuntime:
|
||||
"""Health-gated dispatch over signed, frozen Extension Host releases."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
inventory: SignedExtensionInventory,
|
||||
invokers: Mapping[str, HostInvoker],
|
||||
service_handler: ServiceHandler,
|
||||
*,
|
||||
require_all_healthy: bool = False,
|
||||
) -> None:
|
||||
self.inventory = inventory
|
||||
self.service_handler = service_handler
|
||||
active: dict[str, HostInvoker] = {}
|
||||
failures: dict[str, str] = {}
|
||||
for plugin_id, release in inventory.releases.items():
|
||||
invoker = invokers.get(plugin_id)
|
||||
if invoker is None:
|
||||
failures[plugin_id] = "HostInvokerMissing"
|
||||
continue
|
||||
try:
|
||||
response = invoker({"op": "health"}, None)
|
||||
self._validate_health(plugin_id, release.manifest, response)
|
||||
except Exception as error:
|
||||
failures[plugin_id] = type(error).__name__
|
||||
continue
|
||||
active[plugin_id] = invoker
|
||||
if require_all_healthy and len(active) != len(inventory.releases):
|
||||
unavailable = sorted(set(inventory.releases) - set(active))
|
||||
raise HostedExtensionError(
|
||||
"required extension hosts are unhealthy: "
|
||||
+ ", ".join(unavailable)
|
||||
)
|
||||
self._active = MappingProxyType(active)
|
||||
self._failures = MappingProxyType(failures)
|
||||
|
||||
@staticmethod
|
||||
def _validate_health(
|
||||
plugin_id: str,
|
||||
manifest: PluginManifest,
|
||||
response: Mapping[str, Any],
|
||||
) -> None:
|
||||
if (
|
||||
response.get("status") != "ok"
|
||||
or response.get("api_version") != API_VERSION
|
||||
or response.get("plugin_id") != plugin_id
|
||||
or response.get("author") != manifest.author
|
||||
or response.get("description") != manifest.description
|
||||
or tuple(response.get("help", ())) != manifest.help_entries
|
||||
or tuple(response.get("commands", ()))
|
||||
!= manifest.capabilities.commands
|
||||
or tuple(response.get("service_profiles", ()))
|
||||
!= manifest.capabilities.service_profiles
|
||||
):
|
||||
raise HostedExtensionError("extension host health response mismatch")
|
||||
|
||||
@property
|
||||
def active_plugin_ids(self) -> frozenset[str]:
|
||||
return frozenset(self._active)
|
||||
|
||||
@property
|
||||
def failures(self) -> Mapping[str, str]:
|
||||
return self._failures
|
||||
|
||||
def available(self, plugin_id: str) -> bool:
|
||||
return plugin_id in self._active
|
||||
|
||||
def invoke(
|
||||
self,
|
||||
plugin_id: str,
|
||||
command: str,
|
||||
arguments: str,
|
||||
event: Mapping[str, Any],
|
||||
) -> ExtensionResult:
|
||||
invoker = self._active.get(plugin_id)
|
||||
if invoker is None:
|
||||
raise HostedExtensionError("extension host is not active")
|
||||
release = self.inventory.releases.get(plugin_id)
|
||||
if release is None:
|
||||
raise HostedExtensionError("extension release is not registered")
|
||||
if command.casefold() not in {
|
||||
value.casefold()
|
||||
for value in release.manifest.capabilities.commands
|
||||
}:
|
||||
raise HostedExtensionError("extension command is not declared")
|
||||
try:
|
||||
response = invoker(
|
||||
{
|
||||
"op": "invoke_command",
|
||||
"command": command,
|
||||
"arguments": arguments,
|
||||
"event": dict(event),
|
||||
},
|
||||
self.service_handler,
|
||||
)
|
||||
except Exception as error:
|
||||
raise HostedExtensionError("extension host invocation failed") from error
|
||||
if (
|
||||
response.get("status") != "ok"
|
||||
or response.get("api_version") != API_VERSION
|
||||
or response.get("plugin_id") != plugin_id
|
||||
or not isinstance(response.get("result"), Mapping)
|
||||
):
|
||||
raise HostedExtensionError("extension host response mismatch")
|
||||
try:
|
||||
return ExtensionResult.from_mapping(response["result"])
|
||||
except ContractViolation as error:
|
||||
raise HostedExtensionError(
|
||||
"extension host result violates the SDK contract"
|
||||
) from error
|
||||
@@ -0,0 +1,579 @@
|
||||
"""Parent-side clients for one isolated extension-host invocation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import select
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .telemetry import RuntimeTelemetry
|
||||
|
||||
|
||||
MAX_REQUEST_BYTES = 256 * 1024
|
||||
MAX_RESPONSE_BYTES = 1024 * 1024
|
||||
MAX_STDERR_BYTES = 64 * 1024
|
||||
MAX_SERVICE_CALLS = 8
|
||||
ServiceHandler = Callable[
|
||||
[str, str, Mapping[str, object]],
|
||||
Mapping[str, object],
|
||||
]
|
||||
|
||||
|
||||
class HostClientError(ValueError):
|
||||
"""Controlled parent-side host transport failure."""
|
||||
|
||||
|
||||
class HostTimeout(HostClientError):
|
||||
"""Extension host exceeded its parent-enforced wall-time deadline."""
|
||||
|
||||
|
||||
def _message_payload(value: Mapping[str, Any]) -> bytes:
|
||||
try:
|
||||
payload = json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
except (TypeError, ValueError) as error:
|
||||
raise HostClientError("extension message is not JSON serializable") from error
|
||||
if not payload or len(payload) > MAX_REQUEST_BYTES:
|
||||
raise HostClientError("extension request exceeds the size limit")
|
||||
return payload + b"\n"
|
||||
|
||||
|
||||
def _decode_message(payload: bytes) -> dict[str, Any]:
|
||||
if not payload or len(payload) > MAX_RESPONSE_BYTES:
|
||||
raise HostClientError("extension host output exceeds the size limit")
|
||||
try:
|
||||
response = json.loads(payload)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as error:
|
||||
raise HostClientError("extension host returned invalid JSON") from error
|
||||
if not isinstance(response, dict):
|
||||
raise HostClientError("extension host returned an invalid response")
|
||||
return response
|
||||
|
||||
|
||||
def _final_response(response: Mapping[str, Any]) -> dict[str, Any]:
|
||||
if response.get("status") == "error":
|
||||
raise HostClientError("extension host returned an error")
|
||||
if response.get("status") != "ok":
|
||||
raise HostClientError("extension host returned an invalid response")
|
||||
return dict(response)
|
||||
|
||||
|
||||
def _service_request(
|
||||
response: Mapping[str, Any],
|
||||
) -> tuple[str, str, str, Mapping[str, object]]:
|
||||
call_id = response.get("call_id")
|
||||
profile = response.get("profile")
|
||||
operation = response.get("operation")
|
||||
payload = response.get("payload")
|
||||
if (
|
||||
not isinstance(call_id, str)
|
||||
or not call_id
|
||||
or len(call_id) > 40
|
||||
or not isinstance(profile, str)
|
||||
or not profile
|
||||
or len(profile) > 80
|
||||
or not isinstance(operation, str)
|
||||
or not operation
|
||||
or len(operation) > 80
|
||||
or not isinstance(payload, Mapping)
|
||||
):
|
||||
raise HostClientError("extension host service request is invalid")
|
||||
return call_id, profile, operation, payload
|
||||
|
||||
|
||||
class _MessageReader:
|
||||
def __init__(
|
||||
self,
|
||||
read_chunk: Callable[[float], bytes],
|
||||
*,
|
||||
deadline: float,
|
||||
) -> None:
|
||||
self.read_chunk = read_chunk
|
||||
self.deadline = deadline
|
||||
self.buffer = bytearray()
|
||||
self.total = 0
|
||||
|
||||
def read(self) -> dict[str, Any]:
|
||||
while True:
|
||||
newline = self.buffer.find(b"\n")
|
||||
if newline >= 0:
|
||||
payload = bytes(self.buffer[:newline])
|
||||
del self.buffer[: newline + 1]
|
||||
return _decode_message(payload)
|
||||
remaining = self.deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise HostTimeout("extension host timed out")
|
||||
chunk = self.read_chunk(remaining)
|
||||
if not chunk:
|
||||
if self.buffer:
|
||||
payload = bytes(self.buffer)
|
||||
self.buffer.clear()
|
||||
return _decode_message(payload)
|
||||
raise HostClientError("extension host closed without a response")
|
||||
self.total += len(chunk)
|
||||
if self.total > MAX_RESPONSE_BYTES:
|
||||
raise HostClientError(
|
||||
"extension host output exceeds the size limit"
|
||||
)
|
||||
self.buffer.extend(chunk)
|
||||
|
||||
|
||||
def _protocol_exchange(
|
||||
request: Mapping[str, Any],
|
||||
*,
|
||||
send: Callable[[bytes], None],
|
||||
read_chunk: Callable[[float], bytes],
|
||||
timeout_seconds: float,
|
||||
service_handler: ServiceHandler | None,
|
||||
telemetry: RuntimeTelemetry | None = None,
|
||||
plugin_id: str = "",
|
||||
transport: str = "",
|
||||
) -> dict[str, Any]:
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
reader = _MessageReader(read_chunk, deadline=deadline)
|
||||
send(_message_payload(request))
|
||||
service_calls = 0
|
||||
while True:
|
||||
response = reader.read()
|
||||
if response.get("status") != "service_request":
|
||||
return _final_response(response)
|
||||
service_calls += 1
|
||||
if service_calls > MAX_SERVICE_CALLS:
|
||||
raise HostClientError("extension host exceeded the service call limit")
|
||||
call_id, profile, operation, payload = _service_request(response)
|
||||
service_started = time.monotonic()
|
||||
if telemetry is not None:
|
||||
telemetry.emit(
|
||||
"host.service.request",
|
||||
plugin_id=plugin_id,
|
||||
transport=transport,
|
||||
operation=operation,
|
||||
)
|
||||
if service_handler is None:
|
||||
send(
|
||||
_message_payload(
|
||||
{
|
||||
"op": "service_error",
|
||||
"call_id": call_id,
|
||||
"error": "ServiceUnavailable",
|
||||
}
|
||||
)
|
||||
)
|
||||
if telemetry is not None:
|
||||
telemetry.elapsed(
|
||||
"host.service.complete",
|
||||
service_started,
|
||||
status="error",
|
||||
plugin_id=plugin_id,
|
||||
transport=transport,
|
||||
operation=operation,
|
||||
error_type="ServiceUnavailable",
|
||||
)
|
||||
continue
|
||||
try:
|
||||
result = service_handler(profile, operation, payload)
|
||||
if not isinstance(result, Mapping):
|
||||
raise HostClientError(
|
||||
"kernel service handler returned an invalid result"
|
||||
)
|
||||
reply = {
|
||||
"op": "service_result",
|
||||
"call_id": call_id,
|
||||
"result": dict(result),
|
||||
}
|
||||
except Exception as error:
|
||||
reply = {
|
||||
"op": "service_error",
|
||||
"call_id": call_id,
|
||||
"error": type(error).__name__[:80],
|
||||
}
|
||||
if time.monotonic() > deadline:
|
||||
raise HostTimeout("kernel service exceeded the host deadline")
|
||||
if telemetry is not None:
|
||||
telemetry.elapsed(
|
||||
"host.service.complete",
|
||||
service_started,
|
||||
status=(
|
||||
"ok"
|
||||
if reply["op"] == "service_result"
|
||||
else "error"
|
||||
),
|
||||
plugin_id=plugin_id,
|
||||
transport=transport,
|
||||
operation=operation,
|
||||
error_type=(
|
||||
""
|
||||
if reply["op"] == "service_result"
|
||||
else str(reply["error"])
|
||||
),
|
||||
)
|
||||
send(_message_payload(reply))
|
||||
|
||||
|
||||
class _StderrCollector(threading.Thread):
|
||||
def __init__(self, stream: Any) -> None:
|
||||
super().__init__(name="jwebot-extension-stderr", daemon=True)
|
||||
self.stream = stream
|
||||
self.content = bytearray()
|
||||
self.overflow = False
|
||||
|
||||
def run(self) -> None:
|
||||
while True:
|
||||
chunk = self.stream.read(16 * 1024)
|
||||
if not chunk:
|
||||
return
|
||||
remaining = MAX_STDERR_BYTES + 1 - len(self.content)
|
||||
if remaining > 0:
|
||||
self.content.extend(chunk[:remaining])
|
||||
if len(self.content) > MAX_STDERR_BYTES:
|
||||
self.overflow = True
|
||||
|
||||
|
||||
def _kill_process_group(process: subprocess.Popen[bytes]) -> None:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
process.kill()
|
||||
|
||||
|
||||
class ExtensionHostClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
command: Sequence[str] | None = None,
|
||||
timeout_seconds: float = 8.0,
|
||||
allowed_signers: Path | None = None,
|
||||
expected_publisher: str | None = None,
|
||||
allow_unsigned_local: bool = False,
|
||||
plugin_id: str = "",
|
||||
telemetry: RuntimeTelemetry | None = None,
|
||||
) -> None:
|
||||
self.command = tuple(
|
||||
command
|
||||
or (
|
||||
sys.executable,
|
||||
"-I",
|
||||
"-m",
|
||||
"jwebot_extension_host",
|
||||
)
|
||||
)
|
||||
if (
|
||||
not self.command
|
||||
or timeout_seconds <= 0
|
||||
or (
|
||||
allow_unsigned_local
|
||||
and (
|
||||
allowed_signers is not None
|
||||
or expected_publisher is not None
|
||||
)
|
||||
)
|
||||
):
|
||||
raise HostClientError("invalid extension host client configuration")
|
||||
if command is None and allowed_signers is None and not allow_unsigned_local:
|
||||
raise HostClientError("default extension host requires publisher trust")
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self.allowed_signers = allowed_signers
|
||||
self.expected_publisher = expected_publisher
|
||||
self.allow_unsigned_local = allow_unsigned_local
|
||||
self.plugin_id = plugin_id[:64]
|
||||
self.telemetry = telemetry
|
||||
|
||||
def _command(self, release_dir: Path) -> list[str]:
|
||||
command = [
|
||||
*self.command,
|
||||
"--release-dir",
|
||||
str(release_dir),
|
||||
]
|
||||
if self.allow_unsigned_local:
|
||||
command.append("--allow-unsigned-local")
|
||||
elif self.allowed_signers is not None:
|
||||
command.extend(
|
||||
[
|
||||
"--allowed-signers",
|
||||
str(self.allowed_signers),
|
||||
]
|
||||
)
|
||||
if self.expected_publisher is not None:
|
||||
command.extend(["--publisher", self.expected_publisher])
|
||||
return command
|
||||
|
||||
def invoke(
|
||||
self,
|
||||
release_dir: Path,
|
||||
request: Mapping[str, Any],
|
||||
*,
|
||||
service_handler: ServiceHandler | None = None,
|
||||
) -> dict[str, Any]:
|
||||
invocation_started = time.monotonic()
|
||||
try:
|
||||
resolved_release = release_dir.resolve(strict=True)
|
||||
except OSError as error:
|
||||
if self.telemetry is not None:
|
||||
self.telemetry.elapsed(
|
||||
"host.invoke",
|
||||
invocation_started,
|
||||
status="error",
|
||||
plugin_id=self.plugin_id,
|
||||
transport="process",
|
||||
operation=str(request.get("op", "")),
|
||||
error_type=type(error).__name__,
|
||||
)
|
||||
raise HostClientError(
|
||||
"extension release directory is unavailable"
|
||||
) from error
|
||||
if not resolved_release.is_dir():
|
||||
if self.telemetry is not None:
|
||||
self.telemetry.elapsed(
|
||||
"host.invoke",
|
||||
invocation_started,
|
||||
status="error",
|
||||
plugin_id=self.plugin_id,
|
||||
transport="process",
|
||||
operation=str(request.get("op", "")),
|
||||
error_type="ReleaseUnavailable",
|
||||
)
|
||||
raise HostClientError("extension release path is not a directory")
|
||||
|
||||
environment = {
|
||||
key: value
|
||||
for key, value in os.environ.items()
|
||||
if key in {"LANG", "LC_ALL", "TZ"}
|
||||
}
|
||||
environment["PYTHONDONTWRITEBYTECODE"] = "1"
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
self._command(resolved_release),
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
cwd="/",
|
||||
env=environment,
|
||||
close_fds=True,
|
||||
start_new_session=True,
|
||||
bufsize=0,
|
||||
)
|
||||
except OSError as error:
|
||||
if self.telemetry is not None:
|
||||
self.telemetry.elapsed(
|
||||
"host.invoke",
|
||||
invocation_started,
|
||||
status="error",
|
||||
plugin_id=self.plugin_id,
|
||||
transport="process",
|
||||
error_type=type(error).__name__,
|
||||
)
|
||||
raise HostClientError("cannot start extension host") from error
|
||||
if self.telemetry is not None:
|
||||
self.telemetry.elapsed(
|
||||
"host.process.spawn",
|
||||
invocation_started,
|
||||
plugin_id=self.plugin_id,
|
||||
transport="process",
|
||||
process="extension-host",
|
||||
)
|
||||
assert process.stdin is not None
|
||||
assert process.stdout is not None
|
||||
assert process.stderr is not None
|
||||
stderr = _StderrCollector(process.stderr)
|
||||
stderr.start()
|
||||
deadline = time.monotonic() + self.timeout_seconds
|
||||
|
||||
def send(payload: bytes) -> None:
|
||||
try:
|
||||
process.stdin.write(payload)
|
||||
except (BrokenPipeError, OSError) as error:
|
||||
raise HostClientError(
|
||||
"extension host closed its input"
|
||||
) from error
|
||||
|
||||
def read_chunk(timeout: float) -> bytes:
|
||||
ready, _, _ = select.select(
|
||||
[process.stdout.fileno()],
|
||||
[],
|
||||
[],
|
||||
timeout,
|
||||
)
|
||||
if not ready:
|
||||
raise HostTimeout("extension host timed out")
|
||||
return os.read(process.stdout.fileno(), 64 * 1024)
|
||||
|
||||
try:
|
||||
response = _protocol_exchange(
|
||||
request,
|
||||
send=send,
|
||||
read_chunk=read_chunk,
|
||||
timeout_seconds=self.timeout_seconds,
|
||||
service_handler=service_handler,
|
||||
telemetry=self.telemetry,
|
||||
plugin_id=self.plugin_id,
|
||||
transport="process",
|
||||
)
|
||||
process.stdin.close()
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
raise HostTimeout("extension host timed out")
|
||||
try:
|
||||
return_code = process.wait(timeout=remaining)
|
||||
except subprocess.TimeoutExpired as error:
|
||||
raise HostTimeout("extension host timed out") from error
|
||||
stderr.join(timeout=1)
|
||||
if stderr.overflow:
|
||||
raise HostClientError(
|
||||
"extension host output exceeds the size limit"
|
||||
)
|
||||
if return_code != 0:
|
||||
raise HostClientError("extension host returned an error")
|
||||
if self.telemetry is not None:
|
||||
self.telemetry.elapsed(
|
||||
"host.invoke",
|
||||
invocation_started,
|
||||
plugin_id=self.plugin_id,
|
||||
transport="process",
|
||||
operation=str(request.get("op", "")),
|
||||
)
|
||||
return response
|
||||
except Exception as error:
|
||||
if process.poll() is None:
|
||||
_kill_process_group(process)
|
||||
process.wait()
|
||||
stderr.join(timeout=1)
|
||||
if self.telemetry is not None:
|
||||
self.telemetry.elapsed(
|
||||
"host.invoke",
|
||||
invocation_started,
|
||||
status="error",
|
||||
plugin_id=self.plugin_id,
|
||||
transport="process",
|
||||
operation=str(request.get("op", "")),
|
||||
error_type=type(error).__name__,
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
for stream in (process.stdin, process.stdout, process.stderr):
|
||||
if not stream.closed:
|
||||
stream.close()
|
||||
|
||||
|
||||
class UnixExtensionHostClient:
|
||||
"""Client for a systemd-activated AF_UNIX extension-host socket."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
socket_path: Path,
|
||||
*,
|
||||
timeout_seconds: float = 8.0,
|
||||
plugin_id: str = "",
|
||||
telemetry: RuntimeTelemetry | None = None,
|
||||
) -> None:
|
||||
if not socket_path.is_absolute() or timeout_seconds <= 0:
|
||||
raise HostClientError("invalid Unix host client configuration")
|
||||
self.socket_path = socket_path
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self.plugin_id = plugin_id[:64]
|
||||
self.telemetry = telemetry
|
||||
|
||||
def invoke(
|
||||
self,
|
||||
request: Mapping[str, Any],
|
||||
*,
|
||||
service_handler: ServiceHandler | None = None,
|
||||
) -> dict[str, Any]:
|
||||
invocation_started = time.monotonic()
|
||||
try:
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection:
|
||||
connection.settimeout(self.timeout_seconds)
|
||||
connection.connect(str(self.socket_path))
|
||||
|
||||
def send(payload: bytes) -> None:
|
||||
connection.sendall(payload)
|
||||
|
||||
def read_chunk(timeout: float) -> bytes:
|
||||
connection.settimeout(timeout)
|
||||
return connection.recv(64 * 1024)
|
||||
|
||||
response = _protocol_exchange(
|
||||
request,
|
||||
send=send,
|
||||
read_chunk=read_chunk,
|
||||
timeout_seconds=self.timeout_seconds,
|
||||
service_handler=service_handler,
|
||||
telemetry=self.telemetry,
|
||||
plugin_id=self.plugin_id,
|
||||
transport="unix",
|
||||
)
|
||||
try:
|
||||
connection.shutdown(socket.SHUT_WR)
|
||||
except OSError:
|
||||
pass
|
||||
if self.telemetry is not None:
|
||||
self.telemetry.elapsed(
|
||||
"host.invoke",
|
||||
invocation_started,
|
||||
plugin_id=self.plugin_id,
|
||||
transport="unix",
|
||||
operation=str(request.get("op", "")),
|
||||
)
|
||||
return response
|
||||
except HostTimeout as error:
|
||||
if self.telemetry is not None:
|
||||
self.telemetry.elapsed(
|
||||
"host.invoke",
|
||||
invocation_started,
|
||||
status="error",
|
||||
plugin_id=self.plugin_id,
|
||||
transport="unix",
|
||||
operation=str(request.get("op", "")),
|
||||
error_type=type(error).__name__,
|
||||
)
|
||||
raise
|
||||
except HostClientError as error:
|
||||
if self.telemetry is not None:
|
||||
self.telemetry.elapsed(
|
||||
"host.invoke",
|
||||
invocation_started,
|
||||
status="error",
|
||||
plugin_id=self.plugin_id,
|
||||
transport="unix",
|
||||
operation=str(request.get("op", "")),
|
||||
error_type=type(error).__name__,
|
||||
)
|
||||
raise
|
||||
except socket.timeout as error:
|
||||
if self.telemetry is not None:
|
||||
self.telemetry.elapsed(
|
||||
"host.invoke",
|
||||
invocation_started,
|
||||
status="error",
|
||||
plugin_id=self.plugin_id,
|
||||
transport="unix",
|
||||
operation=str(request.get("op", "")),
|
||||
error_type="HostTimeout",
|
||||
)
|
||||
raise HostTimeout("extension host socket timed out") from error
|
||||
except OSError as error:
|
||||
if self.telemetry is not None:
|
||||
self.telemetry.elapsed(
|
||||
"host.invoke",
|
||||
invocation_started,
|
||||
status="error",
|
||||
plugin_id=self.plugin_id,
|
||||
transport="unix",
|
||||
operation=str(request.get("op", "")),
|
||||
error_type=type(error).__name__,
|
||||
)
|
||||
raise HostClientError(
|
||||
"extension host socket is unavailable"
|
||||
) from error
|
||||
@@ -0,0 +1,244 @@
|
||||
"""Strict parser for reviewed extension manifests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from jwebot_sdk import API_VERSION
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError:
|
||||
try:
|
||||
import tomli as tomllib
|
||||
except ModuleNotFoundError:
|
||||
tomllib = None # type: ignore[assignment]
|
||||
|
||||
|
||||
PLUGIN_ID = re.compile(r"^[a-z][a-z0-9-]{2,63}$")
|
||||
SEMVER = re.compile(r"^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)$")
|
||||
AUTHOR = re.compile(r"^[A-Za-z][A-Za-z0-9._ -]{0,63}$")
|
||||
ENTRYPOINT = re.compile(
|
||||
r"^[A-Za-z_][A-Za-z0-9_.]*:[A-Za-z_][A-Za-z0-9_]*$"
|
||||
)
|
||||
COMMAND = re.compile(r"^/[\w.-]{1,40}$")
|
||||
NETWORK_PROFILE = re.compile(r"^[a-z][a-z0-9-]{1,39}$")
|
||||
ALLOWED_TOP_LEVEL = {
|
||||
"id",
|
||||
"author",
|
||||
"description",
|
||||
"help",
|
||||
"version",
|
||||
"api_version",
|
||||
"entrypoint",
|
||||
"type",
|
||||
"capabilities",
|
||||
}
|
||||
ALLOWED_CAPABILITIES = {
|
||||
"commands",
|
||||
"storage",
|
||||
"network_profiles",
|
||||
"service_profiles",
|
||||
}
|
||||
|
||||
|
||||
class ManifestError(ValueError):
|
||||
"""Raised when an extension manifest violates the kernel policy."""
|
||||
|
||||
|
||||
class PluginType(str, Enum):
|
||||
COMMAND = "command"
|
||||
ENRICHMENT = "enrichment"
|
||||
SCHEDULED = "scheduled"
|
||||
|
||||
|
||||
def _required_string(
|
||||
value: Mapping[str, Any],
|
||||
key: str,
|
||||
*,
|
||||
pattern: re.Pattern[str] | None = None,
|
||||
) -> str:
|
||||
raw = value.get(key)
|
||||
if not isinstance(raw, str) or not raw:
|
||||
raise ManifestError(f"{key} must be a non-empty string")
|
||||
if pattern is not None and not pattern.fullmatch(raw):
|
||||
raise ManifestError(f"{key} has an invalid format")
|
||||
return raw
|
||||
|
||||
|
||||
def _string_list(
|
||||
value: object,
|
||||
*,
|
||||
key: str,
|
||||
pattern: re.Pattern[str],
|
||||
limit: int,
|
||||
) -> tuple[str, ...]:
|
||||
if value is None:
|
||||
return ()
|
||||
if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
|
||||
raise ManifestError(f"{key} must be a list of strings")
|
||||
if len(value) > limit:
|
||||
raise ManifestError(f"{key} exceeds {limit} entries")
|
||||
if any(not pattern.fullmatch(item) for item in value):
|
||||
raise ManifestError(f"{key} contains an invalid value")
|
||||
if len(set(value)) != len(value):
|
||||
raise ManifestError(f"{key} contains duplicates")
|
||||
return tuple(value)
|
||||
|
||||
|
||||
def _optional_description(value: Mapping[str, Any]) -> str:
|
||||
raw = value.get("description", "")
|
||||
if not isinstance(raw, str):
|
||||
raise ManifestError("description must be a string")
|
||||
description = raw.strip()
|
||||
if len(description) > 120 or "\n" in description or "\r" in description:
|
||||
raise ManifestError("description exceeds the display boundary")
|
||||
return description
|
||||
|
||||
|
||||
def _help_entries(value: Mapping[str, Any]) -> tuple[str, ...]:
|
||||
raw = value.get("help", [])
|
||||
if not isinstance(raw, list) or not all(
|
||||
isinstance(item, str)
|
||||
for item in raw
|
||||
):
|
||||
raise ManifestError("help must be a list of strings")
|
||||
if len(raw) > 8:
|
||||
raise ManifestError("help exceeds 8 entries")
|
||||
entries = tuple(item.strip() for item in raw)
|
||||
if any(
|
||||
not item
|
||||
or not item.startswith("/")
|
||||
or len(item) > 160
|
||||
or "\n" in item
|
||||
or "\r" in item
|
||||
for item in entries
|
||||
):
|
||||
raise ManifestError("help contains an invalid display entry")
|
||||
if len(set(entries)) != len(entries):
|
||||
raise ManifestError("help contains duplicates")
|
||||
return entries
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CapabilityRequest:
|
||||
commands: tuple[str, ...] = ()
|
||||
storage: bool = False
|
||||
network_profiles: tuple[str, ...] = ()
|
||||
service_profiles: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PluginManifest:
|
||||
plugin_id: str
|
||||
author: str
|
||||
description: str
|
||||
help_entries: tuple[str, ...]
|
||||
version: str
|
||||
api_version: str
|
||||
entrypoint: str
|
||||
plugin_type: PluginType
|
||||
capabilities: CapabilityRequest
|
||||
|
||||
|
||||
def parse_manifest(value: Mapping[str, Any]) -> PluginManifest:
|
||||
if not isinstance(value, Mapping):
|
||||
raise ManifestError("manifest must be an object")
|
||||
unknown = set(value) - ALLOWED_TOP_LEVEL
|
||||
if unknown:
|
||||
raise ManifestError(f"unknown manifest keys: {', '.join(sorted(unknown))}")
|
||||
|
||||
plugin_id = _required_string(value, "id", pattern=PLUGIN_ID)
|
||||
author = _required_string(value, "author", pattern=AUTHOR)
|
||||
description = _optional_description(value)
|
||||
help_entries = _help_entries(value)
|
||||
if bool(description) != bool(help_entries):
|
||||
raise ManifestError(
|
||||
"description and help must be declared together"
|
||||
)
|
||||
version = _required_string(value, "version", pattern=SEMVER)
|
||||
api_version = _required_string(value, "api_version")
|
||||
if api_version != API_VERSION:
|
||||
raise ManifestError(f"unsupported api_version: {api_version}")
|
||||
entrypoint = _required_string(value, "entrypoint", pattern=ENTRYPOINT)
|
||||
entry_module = entrypoint.partition(":")[0].partition(".")[0]
|
||||
if entry_module != plugin_id.replace("-", "_"):
|
||||
raise ManifestError("entrypoint must use the plugin module namespace")
|
||||
try:
|
||||
plugin_type = PluginType(_required_string(value, "type"))
|
||||
except ValueError as error:
|
||||
raise ManifestError("type must be command, enrichment, or scheduled") from error
|
||||
|
||||
raw_capabilities = value.get("capabilities", {})
|
||||
if not isinstance(raw_capabilities, Mapping):
|
||||
raise ManifestError("capabilities must be an object")
|
||||
unknown_capabilities = set(raw_capabilities) - ALLOWED_CAPABILITIES
|
||||
if unknown_capabilities:
|
||||
raise ManifestError(
|
||||
"unknown capabilities: " + ", ".join(sorted(unknown_capabilities))
|
||||
)
|
||||
storage = raw_capabilities.get("storage", False)
|
||||
if not isinstance(storage, bool):
|
||||
raise ManifestError("capabilities.storage must be boolean")
|
||||
commands = _string_list(
|
||||
raw_capabilities.get("commands"),
|
||||
key="capabilities.commands",
|
||||
pattern=COMMAND,
|
||||
limit=16,
|
||||
)
|
||||
if len({command.casefold() for command in commands}) != len(commands):
|
||||
raise ManifestError("capabilities.commands contains case-insensitive duplicates")
|
||||
capabilities = CapabilityRequest(
|
||||
commands=commands,
|
||||
storage=storage,
|
||||
network_profiles=_string_list(
|
||||
raw_capabilities.get("network_profiles"),
|
||||
key="capabilities.network_profiles",
|
||||
pattern=NETWORK_PROFILE,
|
||||
limit=8,
|
||||
),
|
||||
service_profiles=_string_list(
|
||||
raw_capabilities.get("service_profiles"),
|
||||
key="capabilities.service_profiles",
|
||||
pattern=NETWORK_PROFILE,
|
||||
limit=8,
|
||||
),
|
||||
)
|
||||
|
||||
if plugin_type is PluginType.COMMAND and not capabilities.commands:
|
||||
raise ManifestError("command extension must declare at least one command")
|
||||
if plugin_type is PluginType.COMMAND and not description:
|
||||
raise ManifestError(
|
||||
"command extension must declare description and help"
|
||||
)
|
||||
if plugin_type is not PluginType.COMMAND and capabilities.commands:
|
||||
raise ManifestError("only command extensions may declare commands")
|
||||
if plugin_type is not PluginType.COMMAND and help_entries:
|
||||
raise ManifestError("only command extensions may declare help")
|
||||
|
||||
return PluginManifest(
|
||||
plugin_id=plugin_id,
|
||||
author=author,
|
||||
description=description,
|
||||
help_entries=help_entries,
|
||||
version=version,
|
||||
api_version=api_version,
|
||||
entrypoint=entrypoint,
|
||||
plugin_type=plugin_type,
|
||||
capabilities=capabilities,
|
||||
)
|
||||
|
||||
|
||||
def load_manifest(path: Path) -> PluginManifest:
|
||||
if tomllib is None:
|
||||
raise ManifestError("TOML parser unavailable; install tomli on Python < 3.11")
|
||||
try:
|
||||
with path.open("rb") as stream:
|
||||
value = tomllib.load(stream)
|
||||
except (OSError, tomllib.TOMLDecodeError) as error:
|
||||
raise ManifestError("cannot read plugin manifest") from error
|
||||
return parse_manifest(value)
|
||||
@@ -0,0 +1,315 @@
|
||||
"""Deterministic extension release artifacts and digest verification."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tarfile
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .manifest import AUTHOR, PluginManifest, load_manifest
|
||||
from .source_policy import validate_source_tree
|
||||
|
||||
|
||||
MAX_RELEASE_FILES = 66
|
||||
MAX_RELEASE_BYTES = 1024 * 1024
|
||||
MAX_ARTIFACT_BYTES = 2 * 1024 * 1024
|
||||
|
||||
|
||||
class ReleaseError(ValueError):
|
||||
"""Raised when a release tree or artifact is not immutable and bounded."""
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def release_record_bytes(record: "ReleaseRecord") -> bytes:
|
||||
return _canonical_json(record.to_mapping()) + b"\n"
|
||||
|
||||
|
||||
def source_digest_for_files(files: tuple[tuple[str, bytes], ...]) -> str:
|
||||
digest = hashlib.sha256()
|
||||
for relative, content in files:
|
||||
path_bytes = relative.encode("utf-8")
|
||||
digest.update(len(path_bytes).to_bytes(4, "big"))
|
||||
digest.update(path_bytes)
|
||||
digest.update(len(content).to_bytes(8, "big"))
|
||||
digest.update(content)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for chunk in iter(lambda: stream.read(128 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def collect_release_files(
|
||||
root: Path,
|
||||
manifest: PluginManifest,
|
||||
) -> tuple[tuple[str, bytes], ...]:
|
||||
if not root.is_dir() or root.is_symlink():
|
||||
raise ReleaseError("extension root must be a real directory")
|
||||
validate_source_tree(root, manifest)
|
||||
|
||||
allowed: list[Path] = [root / "plugin.toml"]
|
||||
readme = root / "README.md"
|
||||
if readme.exists():
|
||||
allowed.append(readme)
|
||||
|
||||
source_root = root / "src"
|
||||
for path in sorted(source_root.rglob("*")):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if path.is_symlink():
|
||||
raise ReleaseError(f"release symlink is forbidden: {relative}")
|
||||
if path.is_dir():
|
||||
continue
|
||||
if (
|
||||
"__pycache__" in path.relative_to(source_root).parts
|
||||
and path.suffix == ".pyc"
|
||||
):
|
||||
continue
|
||||
if not path.is_file() or path.suffix != ".py":
|
||||
raise ReleaseError(f"unsupported release file: {relative}")
|
||||
allowed.append(path)
|
||||
|
||||
values: list[tuple[str, bytes]] = []
|
||||
total_bytes = 0
|
||||
for path in sorted(set(allowed), key=lambda item: item.relative_to(root).as_posix()):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if path.is_symlink() or not path.is_file():
|
||||
raise ReleaseError(f"release file is unavailable: {relative}")
|
||||
if PurePosixPath(relative).is_absolute() or ".." in PurePosixPath(relative).parts:
|
||||
raise ReleaseError(f"unsafe release path: {relative}")
|
||||
content = path.read_bytes()
|
||||
total_bytes += len(content)
|
||||
values.append((relative, content))
|
||||
|
||||
if len(values) > MAX_RELEASE_FILES:
|
||||
raise ReleaseError(f"release exceeds {MAX_RELEASE_FILES} files")
|
||||
if total_bytes > MAX_RELEASE_BYTES:
|
||||
raise ReleaseError(f"release exceeds {MAX_RELEASE_BYTES} bytes")
|
||||
return tuple(values)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReleaseRecord:
|
||||
plugin_id: str
|
||||
author: str
|
||||
plugin_version: str
|
||||
api_version: str
|
||||
source_digest: str
|
||||
artifact_name: str
|
||||
artifact_sha256: str
|
||||
artifact_size: int
|
||||
files: tuple[str, ...]
|
||||
|
||||
def to_mapping(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": 2,
|
||||
"plugin_id": self.plugin_id,
|
||||
"author": self.author,
|
||||
"plugin_version": self.plugin_version,
|
||||
"api_version": self.api_version,
|
||||
"source_digest": self.source_digest,
|
||||
"artifact_name": self.artifact_name,
|
||||
"artifact_sha256": self.artifact_sha256,
|
||||
"artifact_size": self.artifact_size,
|
||||
"files": list(self.files),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: Mapping[str, Any]) -> "ReleaseRecord":
|
||||
try:
|
||||
if value.get("schema_version") != 2:
|
||||
raise ReleaseError("unsupported release schema")
|
||||
files = value["files"]
|
||||
if not isinstance(files, list) or not all(
|
||||
isinstance(item, str) for item in files
|
||||
):
|
||||
raise ReleaseError("release files must be a string list")
|
||||
author = value["author"]
|
||||
if not isinstance(author, str) or not AUTHOR.fullmatch(author):
|
||||
raise ReleaseError("release author has an invalid format")
|
||||
record = cls(
|
||||
plugin_id=str(value["plugin_id"]),
|
||||
author=author,
|
||||
plugin_version=str(value["plugin_version"]),
|
||||
api_version=str(value["api_version"]),
|
||||
source_digest=str(value["source_digest"]),
|
||||
artifact_name=str(value["artifact_name"]),
|
||||
artifact_sha256=str(value["artifact_sha256"]),
|
||||
artifact_size=int(value["artifact_size"]),
|
||||
files=tuple(files),
|
||||
)
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise ReleaseError("invalid release record") from error
|
||||
for digest in (record.source_digest, record.artifact_sha256):
|
||||
if len(digest) != 64 or any(character not in "0123456789abcdef" for character in digest):
|
||||
raise ReleaseError("invalid release digest")
|
||||
if record.artifact_size < 1 or record.artifact_size > MAX_ARTIFACT_BYTES:
|
||||
raise ReleaseError("invalid artifact size")
|
||||
if len(record.files) > MAX_RELEASE_FILES or len(set(record.files)) != len(record.files):
|
||||
raise ReleaseError("invalid release file list")
|
||||
return record
|
||||
|
||||
|
||||
def _write_artifact(
|
||||
path: Path,
|
||||
files: tuple[tuple[str, bytes], ...],
|
||||
) -> None:
|
||||
with path.open("wb") as raw:
|
||||
with gzip.GzipFile(
|
||||
filename="",
|
||||
mode="wb",
|
||||
compresslevel=9,
|
||||
fileobj=raw,
|
||||
mtime=0,
|
||||
) as compressed:
|
||||
with tarfile.open(
|
||||
fileobj=compressed,
|
||||
mode="w",
|
||||
format=tarfile.USTAR_FORMAT,
|
||||
) as archive:
|
||||
for relative, content in files:
|
||||
info = tarfile.TarInfo(name=f"plugin/{relative}")
|
||||
info.size = len(content)
|
||||
info.mode = 0o644
|
||||
info.uid = 0
|
||||
info.gid = 0
|
||||
info.uname = ""
|
||||
info.gname = ""
|
||||
info.mtime = 0
|
||||
archive.addfile(info, io.BytesIO(content))
|
||||
|
||||
|
||||
def _publish_immutable(temporary: Path, destination: Path) -> None:
|
||||
if destination.exists():
|
||||
if destination.read_bytes() != temporary.read_bytes():
|
||||
raise ReleaseError(f"immutable release already exists: {destination.name}")
|
||||
temporary.unlink()
|
||||
return
|
||||
temporary.replace(destination)
|
||||
|
||||
|
||||
def build_release(
|
||||
root: Path,
|
||||
output: Path,
|
||||
*,
|
||||
manifest: PluginManifest | None = None,
|
||||
) -> tuple[Path, Path, ReleaseRecord]:
|
||||
manifest = manifest or load_manifest(root / "plugin.toml")
|
||||
files = collect_release_files(root, manifest)
|
||||
source_digest = source_digest_for_files(files)
|
||||
stem = f"{manifest.plugin_id}-{manifest.version}-{source_digest[:16]}"
|
||||
artifact = output / f"{stem}.tar.gz"
|
||||
record_path = output / f"{stem}.release.json"
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{stem}.",
|
||||
suffix=".tmp",
|
||||
dir=output,
|
||||
)
|
||||
os.close(descriptor)
|
||||
temporary = Path(temporary_name)
|
||||
try:
|
||||
_write_artifact(temporary, files)
|
||||
artifact_size = temporary.stat().st_size
|
||||
if artifact_size > MAX_ARTIFACT_BYTES:
|
||||
raise ReleaseError(f"artifact exceeds {MAX_ARTIFACT_BYTES} bytes")
|
||||
record = ReleaseRecord(
|
||||
plugin_id=manifest.plugin_id,
|
||||
author=manifest.author,
|
||||
plugin_version=manifest.version,
|
||||
api_version=manifest.api_version,
|
||||
source_digest=source_digest,
|
||||
artifact_name=artifact.name,
|
||||
artifact_sha256=_sha256_file(temporary),
|
||||
artifact_size=artifact_size,
|
||||
files=tuple(relative for relative, _ in files),
|
||||
)
|
||||
_publish_immutable(temporary, artifact)
|
||||
record_bytes = release_record_bytes(record)
|
||||
record_temporary = record_path.with_name(f".{record_path.name}.{os.getpid()}.tmp")
|
||||
record_temporary.write_bytes(record_bytes)
|
||||
try:
|
||||
_publish_immutable(record_temporary, record_path)
|
||||
finally:
|
||||
record_temporary.unlink(missing_ok=True)
|
||||
return artifact, record_path, record
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def load_release_record(path: Path) -> ReleaseRecord:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error:
|
||||
raise ReleaseError("cannot read release record") from error
|
||||
if not isinstance(value, Mapping):
|
||||
raise ReleaseError("release record must be an object")
|
||||
return ReleaseRecord.from_mapping(value)
|
||||
|
||||
|
||||
def verify_release(artifact: Path, record: ReleaseRecord) -> None:
|
||||
if artifact.name != record.artifact_name:
|
||||
raise ReleaseError("artifact name does not match release record")
|
||||
try:
|
||||
size = artifact.stat().st_size
|
||||
except OSError as error:
|
||||
raise ReleaseError("artifact is unavailable") from error
|
||||
if size != record.artifact_size:
|
||||
raise ReleaseError("artifact size mismatch")
|
||||
if _sha256_file(artifact) != record.artifact_sha256:
|
||||
raise ReleaseError("artifact digest mismatch")
|
||||
|
||||
files: list[tuple[str, bytes]] = []
|
||||
try:
|
||||
with tarfile.open(artifact, mode="r:gz") as archive:
|
||||
members = archive.getmembers()
|
||||
if len(members) > MAX_RELEASE_FILES:
|
||||
raise ReleaseError("artifact contains too many files")
|
||||
total_bytes = 0
|
||||
for member in members:
|
||||
path = PurePosixPath(member.name)
|
||||
if (
|
||||
not member.isfile()
|
||||
or path.is_absolute()
|
||||
or ".." in path.parts
|
||||
or not path.parts
|
||||
or path.parts[0] != "plugin"
|
||||
):
|
||||
raise ReleaseError("artifact contains an unsafe member")
|
||||
relative = PurePosixPath(*path.parts[1:]).as_posix()
|
||||
if not relative or relative.startswith("/"):
|
||||
raise ReleaseError("artifact contains an unsafe path")
|
||||
total_bytes += member.size
|
||||
if total_bytes > MAX_RELEASE_BYTES:
|
||||
raise ReleaseError("artifact expands beyond the size limit")
|
||||
stream = archive.extractfile(member)
|
||||
if stream is None:
|
||||
raise ReleaseError("artifact member cannot be read")
|
||||
files.append((relative, stream.read()))
|
||||
except (OSError, tarfile.TarError, EOFError) as error:
|
||||
raise ReleaseError("cannot read release artifact") from error
|
||||
|
||||
normalized = tuple(sorted(files))
|
||||
if tuple(path for path, _ in normalized) != record.files:
|
||||
raise ReleaseError("artifact file list mismatch")
|
||||
if source_digest_for_files(normalized) != record.source_digest:
|
||||
raise ReleaseError("artifact source digest mismatch")
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Pure routing primitives with no model, storage, or GUI dependencies."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Mapping
|
||||
|
||||
|
||||
class RouteKind(str, Enum):
|
||||
LOCAL = "local"
|
||||
CHAT = "chat"
|
||||
CLOUD = "cloud"
|
||||
|
||||
|
||||
class ExecutionPath(str, Enum):
|
||||
LOCAL_COMMAND = "local-command"
|
||||
DEEPSEEK = "deepseek"
|
||||
OPENCLAW = "openclaw"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RouteDecision:
|
||||
route: RouteKind
|
||||
reason: str
|
||||
command_kind: str = ""
|
||||
arguments: str = ""
|
||||
|
||||
@property
|
||||
def execution_path(self) -> ExecutionPath:
|
||||
return {
|
||||
RouteKind.LOCAL: ExecutionPath.LOCAL_COMMAND,
|
||||
RouteKind.CHAT: ExecutionPath.DEEPSEEK,
|
||||
RouteKind.CLOUD: ExecutionPath.OPENCLAW,
|
||||
}[self.route]
|
||||
|
||||
|
||||
def parse_local_command(
|
||||
question: str,
|
||||
aliases: Mapping[str, str],
|
||||
) -> tuple[str, str]:
|
||||
"""Resolve the longest registered slash-command prefix."""
|
||||
|
||||
normalized = question.lstrip()
|
||||
if not normalized.startswith("/"):
|
||||
return "", ""
|
||||
raw = normalized[1:].lstrip()
|
||||
folded = raw.casefold()
|
||||
for command, kind in sorted(
|
||||
aliases.items(),
|
||||
key=lambda item: len(item[0]),
|
||||
reverse=True,
|
||||
):
|
||||
command_key = command.casefold()
|
||||
if folded == command_key:
|
||||
return kind, ""
|
||||
if folded.startswith(command_key):
|
||||
return kind, raw[len(command):].strip()
|
||||
command, _, arguments = raw.partition(" ")
|
||||
return aliases.get(command.casefold(), ""), arguments.strip()
|
||||
|
||||
|
||||
def classify_question(
|
||||
question: str,
|
||||
aliases: Mapping[str, str],
|
||||
) -> RouteDecision:
|
||||
normalized = question.lstrip()
|
||||
command_kind, arguments = parse_local_command(normalized, aliases)
|
||||
if command_kind:
|
||||
return RouteDecision(
|
||||
route=RouteKind.LOCAL,
|
||||
reason=f"local_{command_kind}",
|
||||
command_kind=command_kind,
|
||||
arguments=arguments,
|
||||
)
|
||||
if normalized.startswith("/"):
|
||||
return RouteDecision(route=RouteKind.LOCAL, reason="local_unknown")
|
||||
if normalized.startswith(("#", "＃")):
|
||||
return RouteDecision(route=RouteKind.CLOUD, reason="hash_command")
|
||||
return RouteDecision(route=RouteKind.CHAT, reason="direct_chat")
|
||||
@@ -0,0 +1,396 @@
|
||||
"""OpenSSH publisher signatures for canonical extension release records."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .release import (
|
||||
ReleaseError,
|
||||
ReleaseRecord,
|
||||
load_release_record,
|
||||
release_record_bytes,
|
||||
verify_release,
|
||||
)
|
||||
|
||||
|
||||
SIGNATURE_SCHEMA_VERSION = 1
|
||||
SIGNATURE_FORMAT = "sshsig"
|
||||
SIGNATURE_NAMESPACE = "jwebot-release-v1"
|
||||
MAX_RELEASE_RECORD_BYTES = 128 * 1024
|
||||
MAX_SIGNATURE_BYTES = 16 * 1024
|
||||
MAX_ALLOWED_SIGNERS_BYTES = 64 * 1024
|
||||
PUBLISHER_ID = re.compile(r"^[a-z][a-z0-9._-]{1,63}$")
|
||||
RELEASE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{1,159}\.release\.json$")
|
||||
SSH_SIGNATURE_HEADER = b"-----BEGIN SSH SIGNATURE-----"
|
||||
SSH_SIGNATURE_FOOTER = b"-----END SSH SIGNATURE-----"
|
||||
SIGNATURE_PAYLOAD_PREFIX = b"JWEBOT-RELEASE-SIGNATURE-V1\0"
|
||||
|
||||
|
||||
class SignatureError(ReleaseError):
|
||||
"""Raised when a release signature or publisher trust check fails."""
|
||||
|
||||
|
||||
def _canonical_json(value: object) -> bytes:
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256(value: bytes) -> str:
|
||||
return hashlib.sha256(value).hexdigest()
|
||||
|
||||
|
||||
def _signature_payload(
|
||||
publisher: str,
|
||||
release_name: str,
|
||||
release_bytes: bytes,
|
||||
) -> bytes:
|
||||
publisher_bytes = publisher.encode("ascii")
|
||||
name_bytes = release_name.encode("ascii")
|
||||
return b"".join(
|
||||
(
|
||||
SIGNATURE_PAYLOAD_PREFIX,
|
||||
len(publisher_bytes).to_bytes(2, "big"),
|
||||
publisher_bytes,
|
||||
len(name_bytes).to_bytes(2, "big"),
|
||||
name_bytes,
|
||||
len(release_bytes).to_bytes(8, "big"),
|
||||
release_bytes,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _release_name(record: ReleaseRecord) -> str:
|
||||
suffix = ".tar.gz"
|
||||
if not record.artifact_name.endswith(suffix):
|
||||
raise SignatureError("release artifact name is invalid")
|
||||
name = f"{record.artifact_name[: -len(suffix)]}.release.json"
|
||||
if not RELEASE_NAME.fullmatch(name):
|
||||
raise SignatureError("release record name is invalid")
|
||||
return name
|
||||
|
||||
|
||||
def _read_canonical_release(path: Path) -> tuple[ReleaseRecord, bytes]:
|
||||
record = load_release_record(path)
|
||||
try:
|
||||
content = path.read_bytes()
|
||||
except OSError as error:
|
||||
raise SignatureError("release record is unavailable") from error
|
||||
if len(content) > MAX_RELEASE_RECORD_BYTES:
|
||||
raise SignatureError("release record exceeds the size limit")
|
||||
if content != release_record_bytes(record):
|
||||
raise SignatureError("release record is not canonical")
|
||||
return record, content
|
||||
|
||||
|
||||
def _publish_immutable(content: bytes, destination: Path) -> None:
|
||||
if destination.exists():
|
||||
try:
|
||||
current = destination.read_bytes()
|
||||
except OSError as error:
|
||||
raise SignatureError("signature destination is unavailable") from error
|
||||
if current != content:
|
||||
raise SignatureError(
|
||||
f"immutable signature already exists: {destination.name}"
|
||||
)
|
||||
return
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=f".{destination.name}.",
|
||||
suffix=".tmp",
|
||||
dir=destination.parent,
|
||||
)
|
||||
temporary = Path(temporary_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
stream.write(content)
|
||||
temporary.replace(destination)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _signature_path(release_path: Path) -> Path:
|
||||
suffix = ".release.json"
|
||||
if not release_path.name.endswith(suffix):
|
||||
raise SignatureError("release record name is invalid")
|
||||
stem = release_path.name[: -len(suffix)]
|
||||
return release_path.with_name(f"{stem}.release.sig.json")
|
||||
|
||||
|
||||
def _run_ssh_keygen(
|
||||
arguments: list[str],
|
||||
*,
|
||||
input_bytes: bytes,
|
||||
ssh_keygen: Path,
|
||||
) -> subprocess.CompletedProcess[bytes]:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[str(ssh_keygen), *arguments],
|
||||
input=input_bytes,
|
||||
capture_output=True,
|
||||
timeout=10,
|
||||
check=False,
|
||||
env={
|
||||
"LANG": "C",
|
||||
"LC_ALL": "C",
|
||||
"PATH": "/usr/bin:/bin",
|
||||
},
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired) as error:
|
||||
raise SignatureError("ssh-keygen execution failed") from error
|
||||
if len(result.stdout) > MAX_SIGNATURE_BYTES or len(result.stderr) > 16 * 1024:
|
||||
raise SignatureError("ssh-keygen output exceeds the size limit")
|
||||
return result
|
||||
|
||||
|
||||
def _validate_allowed_signers(path: Path) -> None:
|
||||
try:
|
||||
content = path.read_bytes()
|
||||
except OSError as error:
|
||||
raise SignatureError("trusted publisher file is unavailable") from error
|
||||
if not 1 <= len(content) <= MAX_ALLOWED_SIGNERS_BYTES:
|
||||
raise SignatureError("trusted publisher file is invalid")
|
||||
try:
|
||||
text = content.decode("ascii")
|
||||
except UnicodeDecodeError as error:
|
||||
raise SignatureError("trusted publisher file must be ASCII") from error
|
||||
key_count = 0
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
fields = stripped.split()
|
||||
if len(fields) < 3 or fields[1] != "ssh-ed25519":
|
||||
raise SignatureError(
|
||||
"trusted publishers must use simple Ed25519 entries"
|
||||
)
|
||||
principals = fields[0].split(",")
|
||||
if not principals or any(
|
||||
not PUBLISHER_ID.fullmatch(principal)
|
||||
for principal in principals
|
||||
):
|
||||
raise SignatureError("trusted publisher principal is invalid")
|
||||
try:
|
||||
key_blob = base64.b64decode(fields[2], validate=True)
|
||||
except (ValueError, binascii.Error) as error:
|
||||
raise SignatureError("trusted publisher key is invalid") from error
|
||||
if not 32 <= len(key_blob) <= 256:
|
||||
raise SignatureError("trusted publisher key is invalid")
|
||||
key_count += 1
|
||||
if key_count > 64:
|
||||
raise SignatureError("trusted publisher file has too many keys")
|
||||
if key_count == 0:
|
||||
raise SignatureError("trusted publisher file contains no keys")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ReleaseSignature:
|
||||
publisher: str
|
||||
release_name: str
|
||||
release_sha256: str
|
||||
signature: bytes
|
||||
|
||||
def to_mapping(self) -> dict[str, object]:
|
||||
return {
|
||||
"schema_version": SIGNATURE_SCHEMA_VERSION,
|
||||
"format": SIGNATURE_FORMAT,
|
||||
"namespace": SIGNATURE_NAMESPACE,
|
||||
"publisher": self.publisher,
|
||||
"release_name": self.release_name,
|
||||
"release_sha256": self.release_sha256,
|
||||
"signature_base64": base64.b64encode(self.signature).decode("ascii"),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: Mapping[str, Any]) -> "ReleaseSignature":
|
||||
expected = {
|
||||
"schema_version",
|
||||
"format",
|
||||
"namespace",
|
||||
"publisher",
|
||||
"release_name",
|
||||
"release_sha256",
|
||||
"signature_base64",
|
||||
}
|
||||
if set(value) != expected:
|
||||
raise SignatureError("invalid release signature fields")
|
||||
if value.get("schema_version") != SIGNATURE_SCHEMA_VERSION:
|
||||
raise SignatureError("unsupported release signature schema")
|
||||
if value.get("format") != SIGNATURE_FORMAT:
|
||||
raise SignatureError("unsupported release signature format")
|
||||
if value.get("namespace") != SIGNATURE_NAMESPACE:
|
||||
raise SignatureError("invalid release signature namespace")
|
||||
publisher = value.get("publisher")
|
||||
release_name = value.get("release_name")
|
||||
release_sha256 = value.get("release_sha256")
|
||||
encoded = value.get("signature_base64")
|
||||
if not isinstance(publisher, str) or not PUBLISHER_ID.fullmatch(publisher):
|
||||
raise SignatureError("publisher has an invalid format")
|
||||
if not isinstance(release_name, str) or not RELEASE_NAME.fullmatch(
|
||||
release_name
|
||||
):
|
||||
raise SignatureError("signed release name has an invalid format")
|
||||
if (
|
||||
not isinstance(release_sha256, str)
|
||||
or len(release_sha256) != 64
|
||||
or any(value not in "0123456789abcdef" for value in release_sha256)
|
||||
):
|
||||
raise SignatureError("signed release digest is invalid")
|
||||
if not isinstance(encoded, str) or len(encoded) > MAX_SIGNATURE_BYTES * 2:
|
||||
raise SignatureError("encoded release signature is invalid")
|
||||
try:
|
||||
signature = base64.b64decode(encoded, validate=True)
|
||||
except (ValueError, binascii.Error) as error:
|
||||
raise SignatureError("encoded release signature is invalid") from error
|
||||
if (
|
||||
not signature.startswith(SSH_SIGNATURE_HEADER)
|
||||
or SSH_SIGNATURE_FOOTER not in signature
|
||||
or len(signature) > MAX_SIGNATURE_BYTES
|
||||
):
|
||||
raise SignatureError("OpenSSH release signature is invalid")
|
||||
return cls(
|
||||
publisher=publisher,
|
||||
release_name=release_name,
|
||||
release_sha256=release_sha256,
|
||||
signature=signature,
|
||||
)
|
||||
|
||||
|
||||
def release_signature_bytes(signature: ReleaseSignature) -> bytes:
|
||||
return _canonical_json(signature.to_mapping()) + b"\n"
|
||||
|
||||
|
||||
def load_release_signature(path: Path) -> ReleaseSignature:
|
||||
try:
|
||||
content = path.read_bytes()
|
||||
value = json.loads(content.decode("utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error:
|
||||
raise SignatureError("cannot read release signature") from error
|
||||
if not isinstance(value, Mapping):
|
||||
raise SignatureError("release signature must be an object")
|
||||
signature = ReleaseSignature.from_mapping(value)
|
||||
if content != release_signature_bytes(signature):
|
||||
raise SignatureError("release signature record is not canonical")
|
||||
return signature
|
||||
|
||||
|
||||
def sign_release_record(
|
||||
release_path: Path,
|
||||
private_key: Path,
|
||||
publisher: str,
|
||||
*,
|
||||
artifact: Path | None = None,
|
||||
output: Path | None = None,
|
||||
ssh_keygen: Path = Path("/usr/bin/ssh-keygen"),
|
||||
) -> tuple[Path, ReleaseSignature]:
|
||||
if not PUBLISHER_ID.fullmatch(publisher):
|
||||
raise SignatureError("publisher has an invalid format")
|
||||
if not private_key.is_file():
|
||||
raise SignatureError("publisher private key is unavailable")
|
||||
record, release_bytes = _read_canonical_release(release_path)
|
||||
expected_release_name = _release_name(record)
|
||||
if release_path.name != expected_release_name:
|
||||
raise SignatureError("release record name does not match its artifact")
|
||||
verify_release(
|
||||
artifact or release_path.with_name(record.artifact_name),
|
||||
record,
|
||||
)
|
||||
result = _run_ssh_keygen(
|
||||
[
|
||||
"-Y",
|
||||
"sign",
|
||||
"-q",
|
||||
"-f",
|
||||
str(private_key),
|
||||
"-n",
|
||||
SIGNATURE_NAMESPACE,
|
||||
"-",
|
||||
],
|
||||
input_bytes=_signature_payload(
|
||||
publisher,
|
||||
expected_release_name,
|
||||
release_bytes,
|
||||
),
|
||||
ssh_keygen=ssh_keygen,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SignatureError("release signing failed")
|
||||
signature = ReleaseSignature(
|
||||
publisher=publisher,
|
||||
release_name=expected_release_name,
|
||||
release_sha256=_sha256(release_bytes),
|
||||
signature=result.stdout,
|
||||
)
|
||||
signature = ReleaseSignature.from_mapping(signature.to_mapping())
|
||||
output = output or _signature_path(release_path)
|
||||
_publish_immutable(release_signature_bytes(signature), output)
|
||||
return output, signature
|
||||
|
||||
|
||||
def verify_release_signature(
|
||||
release_path: Path,
|
||||
signature_path: Path,
|
||||
allowed_signers: Path,
|
||||
*,
|
||||
expected_publisher: str | None = None,
|
||||
ssh_keygen: Path = Path("/usr/bin/ssh-keygen"),
|
||||
) -> ReleaseSignature:
|
||||
record, release_bytes = _read_canonical_release(release_path)
|
||||
signature = load_release_signature(signature_path)
|
||||
if expected_publisher is not None and signature.publisher != expected_publisher:
|
||||
raise SignatureError("release publisher does not match policy")
|
||||
if signature.release_name != _release_name(record):
|
||||
raise SignatureError("signature references a different release record")
|
||||
if signature.release_sha256 != _sha256(release_bytes):
|
||||
raise SignatureError("signed release digest mismatch")
|
||||
if not allowed_signers.is_file():
|
||||
raise SignatureError("trusted publisher file is invalid")
|
||||
_validate_allowed_signers(allowed_signers)
|
||||
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix=".jwebot-release-signature.",
|
||||
suffix=".sshsig",
|
||||
)
|
||||
temporary = Path(temporary_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as stream:
|
||||
stream.write(signature.signature)
|
||||
result = _run_ssh_keygen(
|
||||
[
|
||||
"-Y",
|
||||
"verify",
|
||||
"-q",
|
||||
"-f",
|
||||
str(allowed_signers),
|
||||
"-I",
|
||||
signature.publisher,
|
||||
"-n",
|
||||
SIGNATURE_NAMESPACE,
|
||||
"-s",
|
||||
str(temporary),
|
||||
],
|
||||
input_bytes=_signature_payload(
|
||||
signature.publisher,
|
||||
signature.release_name,
|
||||
release_bytes,
|
||||
),
|
||||
ssh_keygen=ssh_keygen,
|
||||
)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
if result.returncode != 0:
|
||||
raise SignatureError("release signature verification failed")
|
||||
return signature
|
||||
@@ -0,0 +1,223 @@
|
||||
"""Static preflight policy for reviewed extension source trees."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from .manifest import PluginManifest
|
||||
|
||||
|
||||
MAX_SOURCE_FILES = 64
|
||||
MAX_SOURCE_BYTES = 512 * 1024
|
||||
ALLOWED_STDLIB_IMPORTS = frozenset(
|
||||
{
|
||||
"collections",
|
||||
"dataclasses",
|
||||
"datetime",
|
||||
"decimal",
|
||||
"enum",
|
||||
"functools",
|
||||
"itertools",
|
||||
"math",
|
||||
"operator",
|
||||
"re",
|
||||
"statistics",
|
||||
"string",
|
||||
"typing",
|
||||
}
|
||||
)
|
||||
FORBIDDEN_CALLS = frozenset(
|
||||
{
|
||||
"__import__",
|
||||
"breakpoint",
|
||||
"compile",
|
||||
"delattr",
|
||||
"eval",
|
||||
"exec",
|
||||
"getattr",
|
||||
"globals",
|
||||
"input",
|
||||
"locals",
|
||||
"open",
|
||||
"setattr",
|
||||
"vars",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PolicyFinding:
|
||||
path: str
|
||||
line: int
|
||||
code: str
|
||||
message: str
|
||||
|
||||
|
||||
class SourcePolicyError(ValueError):
|
||||
def __init__(self, findings: tuple[PolicyFinding, ...]) -> None:
|
||||
self.findings = findings
|
||||
summary = "; ".join(
|
||||
f"{finding.path}:{finding.line} {finding.code} {finding.message}"
|
||||
for finding in findings[:5]
|
||||
)
|
||||
super().__init__(summary or "extension source policy failed")
|
||||
|
||||
|
||||
class _PolicyVisitor(ast.NodeVisitor):
|
||||
def __init__(self, path: str, local_module: str) -> None:
|
||||
self.path = path
|
||||
self.local_module = local_module
|
||||
self.findings: list[PolicyFinding] = []
|
||||
|
||||
def add(self, node: ast.AST, code: str, message: str) -> None:
|
||||
self.findings.append(
|
||||
PolicyFinding(
|
||||
path=self.path,
|
||||
line=getattr(node, "lineno", 1),
|
||||
code=code,
|
||||
message=message,
|
||||
)
|
||||
)
|
||||
|
||||
def import_allowed(self, module: str) -> bool:
|
||||
top_level = module.partition(".")[0]
|
||||
return (
|
||||
top_level == "jwebot_sdk"
|
||||
or top_level == self.local_module
|
||||
or top_level in ALLOWED_STDLIB_IMPORTS
|
||||
)
|
||||
|
||||
def visit_Import(self, node: ast.Import) -> None:
|
||||
for alias in node.names:
|
||||
if not self.import_allowed(alias.name):
|
||||
self.add(
|
||||
node,
|
||||
"JWP001",
|
||||
f"import is not allowed: {alias.name}",
|
||||
)
|
||||
self.generic_visit(node)
|
||||
|
||||
def visit_ImportFrom(self, node: ast.ImportFrom) -> None:
|
||||
if node.level == 0 and not self.import_allowed(node.module or ""):
|
||||
self.add(
|
||||
node,
|
||||
"JWP001",
|
||||
f"import is not allowed: {node.module or '<unknown>'}",
|
||||
)
|
||||
self.generic_visit(node)
|
||||
|
||||
def visit_Call(self, node: ast.Call) -> None:
|
||||
if isinstance(node.func, ast.Name) and node.func.id in FORBIDDEN_CALLS:
|
||||
self.add(
|
||||
node,
|
||||
"JWP002",
|
||||
f"dynamic or privileged builtin is not allowed: {node.func.id}",
|
||||
)
|
||||
self.generic_visit(node)
|
||||
|
||||
def visit_Attribute(self, node: ast.Attribute) -> None:
|
||||
if node.attr.startswith("__") and node.attr.endswith("__"):
|
||||
self.add(
|
||||
node,
|
||||
"JWP003",
|
||||
f"dunder attribute access is not allowed: {node.attr}",
|
||||
)
|
||||
self.generic_visit(node)
|
||||
|
||||
|
||||
def validate_source_tree(
|
||||
root: Path,
|
||||
manifest: PluginManifest,
|
||||
) -> tuple[Path, ...]:
|
||||
source_root = root / "src"
|
||||
if not source_root.is_dir() or source_root.is_symlink():
|
||||
raise SourcePolicyError(
|
||||
(
|
||||
PolicyFinding(
|
||||
path="src",
|
||||
line=1,
|
||||
code="JWP004",
|
||||
message="src must be a real directory",
|
||||
),
|
||||
)
|
||||
)
|
||||
paths = sorted(source_root.rglob("*.py"))
|
||||
findings: list[PolicyFinding] = []
|
||||
if not paths:
|
||||
findings.append(
|
||||
PolicyFinding(
|
||||
path="src",
|
||||
line=1,
|
||||
code="JWP004",
|
||||
message="extension contains no Python source",
|
||||
)
|
||||
)
|
||||
if len(paths) > MAX_SOURCE_FILES:
|
||||
findings.append(
|
||||
PolicyFinding(
|
||||
path="src",
|
||||
line=1,
|
||||
code="JWP005",
|
||||
message=f"extension exceeds {MAX_SOURCE_FILES} source files",
|
||||
)
|
||||
)
|
||||
|
||||
local_module = manifest.entrypoint.partition(":")[0].partition(".")[0]
|
||||
total_bytes = 0
|
||||
for path in paths:
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if path.is_symlink() or not path.is_file():
|
||||
findings.append(
|
||||
PolicyFinding(
|
||||
path=relative,
|
||||
line=1,
|
||||
code="JWP006",
|
||||
message="source symlinks and non-regular files are forbidden",
|
||||
)
|
||||
)
|
||||
continue
|
||||
raw = path.read_bytes()
|
||||
total_bytes += len(raw)
|
||||
try:
|
||||
source = raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
findings.append(
|
||||
PolicyFinding(
|
||||
path=relative,
|
||||
line=1,
|
||||
code="JWP007",
|
||||
message="source must be UTF-8",
|
||||
)
|
||||
)
|
||||
continue
|
||||
try:
|
||||
tree = ast.parse(source, filename=relative)
|
||||
except SyntaxError as error:
|
||||
findings.append(
|
||||
PolicyFinding(
|
||||
path=relative,
|
||||
line=error.lineno or 1,
|
||||
code="JWP008",
|
||||
message="source contains invalid Python syntax",
|
||||
)
|
||||
)
|
||||
continue
|
||||
visitor = _PolicyVisitor(relative, local_module)
|
||||
visitor.visit(tree)
|
||||
findings.extend(visitor.findings)
|
||||
|
||||
if total_bytes > MAX_SOURCE_BYTES:
|
||||
findings.append(
|
||||
PolicyFinding(
|
||||
path="src",
|
||||
line=1,
|
||||
code="JWP009",
|
||||
message=f"source exceeds {MAX_SOURCE_BYTES} bytes",
|
||||
)
|
||||
)
|
||||
if findings:
|
||||
raise SourcePolicyError(tuple(findings))
|
||||
return tuple(paths)
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
"""Privacy-safe structured runtime telemetry for latency and failures."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Union
|
||||
|
||||
|
||||
TELEMETRY_SCHEMA_VERSION = 1
|
||||
EVENT_NAME = re.compile(r"^[a-z][a-z0-9_.-]{2,79}$")
|
||||
FIELD_NAME = re.compile(r"^[a-z][a-z0-9_]{1,39}$")
|
||||
ALLOWED_FIELDS = frozenset(
|
||||
{
|
||||
"ack_kind",
|
||||
"attempts",
|
||||
"bytes",
|
||||
"chars",
|
||||
"count",
|
||||
"confirmed_by",
|
||||
"delivered",
|
||||
"depth",
|
||||
"duplicate",
|
||||
"error_type",
|
||||
"execution_path",
|
||||
"fallback",
|
||||
"flag",
|
||||
"flag_wait_ms",
|
||||
"height",
|
||||
"mode",
|
||||
"operation",
|
||||
"outcome",
|
||||
"phase",
|
||||
"plugin_id",
|
||||
"pixels",
|
||||
"polls",
|
||||
"process",
|
||||
"queue",
|
||||
"queue_state",
|
||||
"reason",
|
||||
"roi",
|
||||
"route",
|
||||
"source",
|
||||
"threshold",
|
||||
"timeout_ms",
|
||||
"transport",
|
||||
"worker",
|
||||
"width",
|
||||
"x",
|
||||
"y",
|
||||
}
|
||||
)
|
||||
Scalar = Union[str, int, float, bool, None]
|
||||
|
||||
|
||||
def hash_identifier(value: object) -> str:
|
||||
text = str(value or "")
|
||||
if not text:
|
||||
return ""
|
||||
return hashlib.sha256(
|
||||
text.encode("utf-8", "replace")
|
||||
).hexdigest()[:12]
|
||||
|
||||
|
||||
class RuntimeTelemetry:
|
||||
"""Emit bounded JSON logs and retain low-cardinality process aggregates."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
component: str,
|
||||
enabled: bool = True,
|
||||
observer: Callable[[Mapping[str, Scalar]], None] | None = None,
|
||||
logger: logging.Logger | None = None,
|
||||
monotonic: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
self.component = component[:40]
|
||||
self.enabled = enabled
|
||||
self.observer = observer
|
||||
self.logger = logger or logging.getLogger("jwebot.telemetry")
|
||||
self.monotonic = monotonic
|
||||
self._lock = threading.Lock()
|
||||
self._counts: dict[str, int] = {}
|
||||
self._latencies: dict[str, dict[str, float]] = {}
|
||||
|
||||
def emit(
|
||||
self,
|
||||
name: str,
|
||||
*,
|
||||
status: str = "ok",
|
||||
elapsed_ms: float | None = None,
|
||||
event_id: object = "",
|
||||
group_id: object = "",
|
||||
**fields: Scalar,
|
||||
) -> Mapping[str, Scalar]:
|
||||
if not self.enabled:
|
||||
return MappingProxyType({})
|
||||
safe_name = name if EVENT_NAME.fullmatch(name) else "telemetry.invalid"
|
||||
safe_status = (
|
||||
status[:24]
|
||||
if isinstance(status, str) and status
|
||||
else "unknown"
|
||||
)
|
||||
value: dict[str, Scalar] = {
|
||||
"schema_version": TELEMETRY_SCHEMA_VERSION,
|
||||
"at": datetime.now(timezone.utc).isoformat(timespec="milliseconds"),
|
||||
"name": safe_name,
|
||||
"status": safe_status,
|
||||
"component": self.component,
|
||||
"pid": os.getpid(),
|
||||
}
|
||||
event_hash = hash_identifier(event_id)
|
||||
group_hash = hash_identifier(group_id)
|
||||
if event_hash:
|
||||
value["event_hash"] = event_hash
|
||||
if group_hash:
|
||||
value["group_hash"] = group_hash
|
||||
if elapsed_ms is not None:
|
||||
value["elapsed_ms"] = round(max(0.0, float(elapsed_ms)), 3)
|
||||
for key, item in fields.items():
|
||||
if (
|
||||
key in ALLOWED_FIELDS
|
||||
and FIELD_NAME.fullmatch(key)
|
||||
and isinstance(item, (str, int, float, bool, type(None)))
|
||||
):
|
||||
value[key] = item[:120] if isinstance(item, str) else item
|
||||
metric_key = f"{safe_name}:{safe_status}"
|
||||
with self._lock:
|
||||
self._counts[metric_key] = self._counts.get(metric_key, 0) + 1
|
||||
if elapsed_ms is not None:
|
||||
latency = self._latencies.setdefault(
|
||||
safe_name,
|
||||
{"count": 0.0, "total_ms": 0.0, "max_ms": 0.0},
|
||||
)
|
||||
latency["count"] += 1
|
||||
latency["total_ms"] += max(0.0, float(elapsed_ms))
|
||||
latency["max_ms"] = max(
|
||||
latency["max_ms"],
|
||||
max(0.0, float(elapsed_ms)),
|
||||
)
|
||||
try:
|
||||
self.logger.info(
|
||||
"telemetry %s",
|
||||
json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
if self.observer is not None:
|
||||
try:
|
||||
self.observer(MappingProxyType(dict(value)))
|
||||
except Exception:
|
||||
pass
|
||||
return MappingProxyType(value)
|
||||
|
||||
def elapsed(
|
||||
self,
|
||||
name: str,
|
||||
started: float,
|
||||
**kwargs: Any,
|
||||
) -> Mapping[str, Scalar]:
|
||||
return self.emit(
|
||||
name,
|
||||
elapsed_ms=(self.monotonic() - started) * 1000,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def span(
|
||||
self,
|
||||
name: str,
|
||||
**fields: Scalar,
|
||||
) -> Iterator[None]:
|
||||
started = self.monotonic()
|
||||
try:
|
||||
yield
|
||||
except BaseException as error:
|
||||
self.elapsed(
|
||||
name,
|
||||
started,
|
||||
status="error",
|
||||
error_type=type(error).__name__,
|
||||
**fields,
|
||||
)
|
||||
raise
|
||||
else:
|
||||
self.elapsed(name, started, **fields)
|
||||
|
||||
def snapshot(self) -> dict[str, object]:
|
||||
with self._lock:
|
||||
counts = dict(self._counts)
|
||||
latencies = {
|
||||
name: {
|
||||
"count": int(value["count"]),
|
||||
"average_ms": round(
|
||||
value["total_ms"] / value["count"],
|
||||
3,
|
||||
),
|
||||
"max_ms": round(value["max_ms"], 3),
|
||||
}
|
||||
for name, value in self._latencies.items()
|
||||
if value["count"]
|
||||
}
|
||||
return {
|
||||
"schema_version": TELEMETRY_SCHEMA_VERSION,
|
||||
"counts": counts,
|
||||
"latencies": latencies,
|
||||
}
|
||||
Reference in New Issue
Block a user