From 6a22e66de681cac05118ba6f548f7ae0cec9533a Mon Sep 17 00:00:00 2001 From: The Beast Date: Mon, 20 Jul 2026 00:03:58 +0000 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20Phase=201=20=E2=80=94=20trust-tiere?= =?UTF-8?q?d=20adapter=20registry,=20isolation,=20permissions,=20checkpoin?= =?UTF-8?q?ts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sapiens.registry: SYNTHETIC/CORE/UNTRUSTED trust tiers derived from manifest provenance facts, replacing the Phase-0 synthetic-only gate - sapiens.permissions: owner-permission/licence manifest (permissions.json, empty by default — zero third-party permissions, clean-room invariant enforced by code); UNTRUSTED adapters require an active matching entry - sapiens.isolation: subprocess execution with POSIX rlimits (CPU, address space, open files) + wall-clock timeout for UNTRUSTED adapters; every failure mode contained fail-closed - sapiens.checkpoints: HMAC-SHA256 signed ledger checkpoints (env-held key, never stored) + external anchor export/verify; ledger gains checkpoint event kind with continuity verification - kernel: UNTRUSTED-tier adapters validate only via isolation; L4 stays human-gated; no real-data adapter ships in this phase - models: AdapterManifest gains code_origin / data_sources / third_party_source with coherence checks - tests: 45 new (permissions, registry, isolation incl. CPU/memory-hog and timeout containment, checkpoints, kernel isolation); fixed stale module-level time-budgeted ExecutionContext in photometry tests - docs: README/ROADMAP/ARCHITECTURE/PROVENANCE truthful for Phase 1; version 0.2.0 Closes #12, closes #13, closes #14, closes #15, closes #16. Part of #7. --- ARCHITECTURE.md | 16 ++- PROVENANCE.md | 13 ++ README.md | 54 +++++--- ROADMAP.md | 30 +++- permissions.json | 5 + pyproject.toml | 2 +- src/sapiens/__init__.py | 8 +- src/sapiens/adapter.py | 17 ++- src/sapiens/checkpoints.py | 163 ++++++++++++++++++++++ src/sapiens/isolation.py | 229 +++++++++++++++++++++++++++++++ src/sapiens/kernel.py | 20 ++- src/sapiens/ledger.py | 19 ++- src/sapiens/models.py | 24 +++- src/sapiens/permissions.py | 144 +++++++++++++++++++ src/sapiens/registry.py | 98 +++++++++++++ tests/isolation_doubles.py | 87 ++++++++++++ tests/test_boundaries.py | 72 ++++++++-- tests/test_checkpoints.py | 128 +++++++++++++++++ tests/test_isolation.py | 104 ++++++++++++++ tests/test_kernel_isolation.py | 66 +++++++++ tests/test_permissions.py | 115 ++++++++++++++++ tests/test_photometry_adapter.py | 14 +- tests/test_registry.py | 115 ++++++++++++++++ 23 files changed, 1479 insertions(+), 64 deletions(-) create mode 100644 permissions.json create mode 100644 src/sapiens/checkpoints.py create mode 100644 src/sapiens/isolation.py create mode 100644 src/sapiens/permissions.py create mode 100644 src/sapiens/registry.py create mode 100644 tests/isolation_doubles.py create mode 100644 tests/test_checkpoints.py create mode 100644 tests/test_isolation.py create mode 100644 tests/test_kernel_isolation.py create mode 100644 tests/test_permissions.py create mode 100644 tests/test_registry.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ca5d6e2..4592aaf 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,4 +1,4 @@ -# SAPIENS Phase-0 Architecture +# SAPIENS Architecture (Phases 0–1) ## Design goals @@ -13,7 +13,11 @@ ```text src/sapiens/ models.py immutable candidate/evidence/manifest models - adapter.py DomainAdapter protocol and Phase-0 adapter validation + adapter.py DomainAdapter protocol; validation routes to the registry + registry.py trust-tiered adapter registry (Phase 1) + permissions.py owner-permission/licence manifest for third-party code (Phase 1) + isolation.py subprocess + rlimit execution for UNTRUSTED adapters (Phase 1) + checkpoints.py signed ledger checkpoints + external anchor export (Phase 1) ledger.py JSONL hash-chain ledger and L0→L4 transition guard kernel.py domain-neutral candidate registration and next-gate validation bridge.py cross-domain structure transfer with mandatory L0 reset @@ -32,7 +36,9 @@ Adapters may propose candidates and produce evidence, but they cannot set eviden Required methods: -- `manifest`: domain name/version/vocabulary and `synthetic_only=True` for Phase 0. +- `manifest`: domain name/version/vocabulary plus provenance facts + (`synthetic_only`, `code_origin`, `data_sources`, `third_party_source`). + The Phase-1 registry derives a trust tier from these facts. - `propose(seed, limit)`: deterministic candidates. - `validate(candidate, stage, seed, context)`: bounded evidence for `internal`, `replication`, or `review`. - `import_structure(structure, candidate_id)`: target-domain candidate from cross-domain structure. @@ -41,7 +47,7 @@ Required methods: The ledger is newline-delimited canonical JSON. Each event stores the previous event hash and its own hash. Replay validates sequence, hashes, candidate creation, evidence scope, one-step promotion, required evidence kinds, demotion reasons, and the L4 human gate. -Hash chaining detects tampering but does **not** prove authorship, signatures, scientific truth, or external timestamping. +Hash chaining detects tampering but does **not** prove authorship, scientific truth, or external timestamping. Phase-1 `checkpoint` events summarise the chain (event count + head hash) and may carry an HMAC-SHA256 signature (environment-held key, never stored); `sapiens.checkpoints` also exports/verifies external anchor files. HMAC is symmetric: it proves key possession, not third-party authorship. ## Cross-domain bridge @@ -51,4 +57,4 @@ Hash chaining detects tampering but does **not** prove authorship, signatures, s `WorkQueue` gives bounded jobs, serialized payload-size limits, idempotency keys, leases, stale-lease rejection, and retry/dead states. `DiscoveryDaemon.run_bounded` executes only explicitly registered handlers under time/step budgets; it does not dynamically import or shell out from queue payloads. -Phase 0 uses cooperative in-process preemption. Subprocess/cgroup isolation is a roadmap item before untrusted adapters. +Synthetic and CORE (first-party, real-data) adapters run in-process with cooperative preemption. UNTRUSTED (third-party) adapters run only via `sapiens.isolation`: a child process applies POSIX rlimits (CPU, address space, open files) to itself, the parent enforces a wall-clock timeout, and every failure mode is contained fail-closed (no evidence on failure). Third-party adapters additionally require a recorded owner permission (`permissions.json`, empty by default) before the registry will validate them at all. rlimits bound resources; they are not a full security sandbox. diff --git a/PROVENANCE.md b/PROVENANCE.md index 1ecdd46..60aac67 100644 --- a/PROVENANCE.md +++ b/PROVENANCE.md @@ -33,3 +33,16 @@ No third-party source code is included. Therefore no third-party notices are emb ## Known licence gate ASTRA-dev/ASTRA/GEODISC/BIODISC lack explicit licences at inspected refs. Any future kernel extraction from those repositories requires a signed/committed compatible licence or separate written permission. Until then: architecture-only references or clean-room reimplementation only. + +## Phase 1 — permission manifest mechanism + +Phase 1 operationalises the gate: [`permissions.json`](permissions.json) is the +machine-readable owner-permission manifest consumed by `sapiens.permissions` +and enforced by `sapiens.registry`. Any adapter declaring +`code_origin="third-party"` is UNTRUSTED-tier: it cannot validate without a +matching active permission entry (`adapter:` scope for its declared +`third_party_source`), and it executes only inside the resource-limited +subprocess (`sapiens.isolation`). **The shipped manifest is empty** — zero +ASTRA-family permissions — so the clean-room boundary above is enforced by +code, not by convention. Entries may be added only with explicit owner +sign-off (recorded grantor, licence, evidence reference, validity window). diff --git a/README.md b/README.md index ae08089..cb73b00 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,8 @@ SAPIENS is an **experimental platform for traceable cross-domain scientific-discovery workflows** — a `DomainAdapter` boundary, a hash-chained -L0→L4 evidence ledger, and kernel-gated promotions. **Phase 0: synthetic -adapters only, no discoveries claimed.** +L0→L4 evidence ledger, and kernel-gated promotions. **Phases 0–1 shipped: +synthetic adapters only in practice, no discoveries claimed.** It provides the plumbing a discovery system needs before it can be trusted: a domain-neutral adapter boundary, an append-only hash-chained evidence @@ -20,11 +20,13 @@ budgets. > **Read this first — despite the acronym:** SAPIENS is **not** AGI, ASI, or > superintelligence, and does not claim to be. It is an experimental research -> platform. **No scientific discoveries are claimed.** Phase 0 ships with -> deterministic **synthetic adapters only**; the included examples discover +> platform. **No scientific discoveries are claimed.** The shipped adapters +> remain deterministic and **synthetic only**; the included examples discover > nothing about nature. The CLI reports `"scientific_discoveries_claimed": 0` > by construction, and the test suite enforces the honesty and boundary -> invariants described below. +> invariants described below. Phase 1 added the *machinery* for real-domain +> work (trust tiers, isolation, permissions) — but no real-data adapter +> ships yet. ## Why @@ -39,7 +41,11 @@ inherits traceability and bounded confidence instead of retrofitting them. ```text src/sapiens/ models.py immutable candidate / evidence / manifest models - adapter.py DomainAdapter protocol + Phase-0 adapter validation + adapter.py DomainAdapter protocol; validation routes to the registry + registry.py trust-tiered adapter registry (SYNTHETIC / CORE / UNTRUSTED) + permissions.py owner-permission/licence manifest for third-party code + isolation.py subprocess + rlimit execution for UNTRUSTED adapters + checkpoints.py HMAC-signed ledger checkpoints + external anchor export ledger.py JSONL hash-chained evidence ledger, L0→L4 transition guard kernel.py domain-neutral DiscoveryKernel; owns all promotions bridge.py cross-domain structure transfer — ALWAYS resets target to L0 @@ -68,7 +74,7 @@ Key design rules (enforced by tests in [`tests/`](tests/)): - **L2 Replication** — passed held-out / reproducibility checks. - **L3 Review** — passed bounded structured review / adversarial checks. - **L4 External-ready** — requires an explicit **human gate**; autonomous - promotion to L4 is disabled in Phase 0. + promotion to L4 is disabled. - **Kernel-owned promotions** — adapters propose, only the `DiscoveryKernel` promotes, and only through the ledger's transition guard. - **Cross-domain bridge resets to L0** — transfer moves *structure and @@ -156,33 +162,43 @@ astrophysical result**. ## Status & roadmap -**Phase 0 — shipped** (current package version `0.1.0`): clean-room foundation, +**Phase 0 — shipped** (package version `0.1.0`): clean-room foundation, three deterministic synthetic adapters, synthetic-only orchestration, hash-chained ledger, kernel gates, bridge, bounded queue/daemon, and CI on -Python 3.10/3.11/3.12. The test suite currently includes positive and negative -period-detection fixtures, L0 reset/provenance checks, ledger tamper checks, -promotion guards, and bounded queue/daemon behavior. +Python 3.10/3.11/3.12. + +**Phase 1 — shipped** (current package version `0.2.0`): the synthetic-only +gate is replaced by a **trust-tiered adapter registry** (SYNTHETIC / CORE / +UNTRUSTED), an **owner-permission/licence manifest** +([`permissions.json`](permissions.json) — empty by default: no third-party +code may power an adapter without a recorded entry), **subprocess isolation +with OS-level resource limits** for UNTRUSTED adapters (rlimit CPU / +address-space / open-files plus wall-clock timeout, fail-closed), and +**HMAC-signed ledger checkpoints** with external anchor export +(key from the environment only, never stored). No real-data adapter ships in +Phase 1; tiers are exercised by synthetic adapters and test doubles. Next, in order (see [`ROADMAP.md`](ROADMAP.md)): -1. **Phase 1** — legal/licence gate and adapter hardening (trust-tiered - adapter registry, sandboxing, signed ledger checkpoints). +1. ~~**Phase 1** — legal/licence gate and adapter hardening~~ **shipped**. 2. **Phase 2** — validation framework v1 (statistical gates, holdout protocols, leakage controls, seeded-bias fixtures, calibration). 3. **Phase 3** — structured L3 review panels (role-specialized reviewers, multi-round objection tracking, catch-rate scoring). -4. **Phase 4** — **real domain adapters** (ASTRA / GEODISC / BIODISC / SLATE) - — only after licence and owner review. +4. **Phase 4** — **real domain adapters** — first a clean-room Kepler + photometry adapter on public NASA/MAST data; ASTRA / GEODISC / BIODISC / + SLATE adapters only after licence and owner review. 5. **Phase 5** — external-review workflows with human L4 gates and reproduction bundles. ## Provenance & legal boundary -SAPIENS Phase 0 is a **clean-room implementation**: no source code from +SAPIENS is a **clean-room implementation**: no source code from ASTRA-dev, ASTRA, GEODISC, BIODISC, or SLATE was copied into this repository -(those codebases carry unresolved licensing; reuse is explicitly gated to -Phase 1+ with owner permission). See [`PROVENANCE.md`](PROVENANCE.md) for the -documented boundary. +(those codebases carry unresolved licensing; reuse is gated on recorded owner +permission — the Phase-1 permission manifest ships empty, so every +third-party adapter is refused today). See [`PROVENANCE.md`](PROVENANCE.md) +for the documented boundary. ## Credits diff --git a/ROADMAP.md b/ROADMAP.md index 47c5d8a..387e546 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -7,12 +7,28 @@ - Synthetic integrated orchestration. - Tests and Python matrix CI. -## Phase 1 — legal/licence gate and adapter hardening - -- Obtain explicit licences/permissions for any ASTRA-family code reuse before extraction. -- Replace synthetic-only adapter gate with a trust-tiered adapter registry. -- Add subprocess isolation and OS-level resource limits for untrusted adapters. -- Add signed ledger checkpoints or external anchoring. +## Phase 1 — shipped (package version 0.2.0) + +Legal/licence gate and adapter hardening: + +- ~~Obtain explicit licences/permissions for any ASTRA-family code reuse + before extraction.~~ Mechanism shipped: `permissions.json` + + `sapiens.permissions` record owner grants; the manifest is **empty** — no + ASTRA-family (or any third-party) permissions exist, so every third-party + adapter is refused until an owner records one. +- ~~Replace synthetic-only adapter gate with a trust-tiered adapter + registry.~~ Shipped: `sapiens.registry` (SYNTHETIC / CORE / UNTRUSTED). +- ~~Add subprocess isolation and OS-level resource limits for untrusted + adapters.~~ Shipped: `sapiens.isolation` — rlimit CPU / address-space / + open-files plus wall-clock timeout, fail-closed; the kernel runs + UNTRUSTED-tier adapters only through it. +- ~~Add signed ledger checkpoints or external anchoring.~~ Shipped: + `sapiens.checkpoints` — HMAC-SHA256 checkpoint events (key from + environment only) plus external anchor export/verify. + +Honest limits: rlimits bound resource use but are not a security sandbox; +HMAC proves local key possession, not third-party authorship. No real-data +adapter ships in Phase 1. ## Phase 2 — validation framework v1 @@ -28,6 +44,8 @@ ## Phase 4 — real domain adapters +- First: one clean-room Kepler photometry adapter on public NASA/MAST data + (reuses this repository's own Apache-2.0 demo pipeline path). - ASTRA/GEODISC/BIODISC/SLATE adapters only after licence and owner review. - Domain-specific validators remain sandboxed behind adapters. - Cross-domain method transfer enters target domain at L0 every time. diff --git a/permissions.json b/permissions.json new file mode 100644 index 0000000..823128d --- /dev/null +++ b/permissions.json @@ -0,0 +1,5 @@ +{ + "version": 1, + "comment": "Owner permissions for third-party code reuse (Phase 1). EMPTY BY DEFAULT: no ASTRA-family (or any other third-party) permissions are recorded, so every third-party adapter is refused. Add entries only with explicit owner sign-off; see PROVENANCE.md.", + "entries": [] +} diff --git a/pyproject.toml b/pyproject.toml index 56feaa4..eec17b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sapiens-discovery" -version = "0.1.0" +version = "0.2.0" description = "Experimental foundation for traceable cross-domain scientific-discovery workflows" readme = "README.md" requires-python = ">=3.10" diff --git a/src/sapiens/__init__.py b/src/sapiens/__init__.py index 0d6a2fc..eff7afe 100644 --- a/src/sapiens/__init__.py +++ b/src/sapiens/__init__.py @@ -5,16 +5,22 @@ from .kernel import DiscoveryKernel from .ledger import EvidenceLedger from .models import AdapterManifest, Candidate, Evidence, EvidenceLevel +from .permissions import PermissionEntry, PermissionManifest +from .registry import AdapterRegistry, TrustTier __all__ = [ "AdapterManifest", + "AdapterRegistry", "Candidate", "DiscoveryKernel", "DomainAdapter", "Evidence", "EvidenceLedger", "EvidenceLevel", + "PermissionEntry", + "PermissionManifest", "TransferEnvelope", + "TrustTier", "transfer", ] -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/src/sapiens/adapter.py b/src/sapiens/adapter.py index 9829e4b..6627cb6 100644 --- a/src/sapiens/adapter.py +++ b/src/sapiens/adapter.py @@ -25,8 +25,15 @@ def import_structure(self, structure: dict[str, object], *, candidate_id: str) - def validate_adapter(adapter: DomainAdapter) -> None: - if not isinstance(adapter, DomainAdapter): - raise TypeError("adapter does not implement DomainAdapter") - manifest = adapter.manifest - if not manifest.synthetic_only: - raise ValueError("Phase 0 refuses non-synthetic adapters") + """Phase-1 gate: auto-tiered registry validation. + + Kept for backward compatibility with Phase-0 callers. Synthetic adapters + pass exactly as before; first-party clean-room real-data adapters pass at + CORE tier; third-party adapters require a recorded permission and raise + ``MissingPermissionError`` without one. Callers that need the tier (e.g. + the kernel, to decide on isolation) should use ``AdapterRegistry`` + directly. + """ + from .registry import AdapterRegistry + + AdapterRegistry().validate_adapter(adapter) diff --git a/src/sapiens/checkpoints.py b/src/sapiens/checkpoints.py new file mode 100644 index 0000000..d944b38 --- /dev/null +++ b/src/sapiens/checkpoints.py @@ -0,0 +1,163 @@ +"""Signed ledger checkpoints and external anchor export (Phase 1). + +A checkpoint is a ledger event that summarises the chain so far: the event +count and the current head hash. With an HMAC key (environment-only, never +stored) the checkpoint is *signed*; without one it is still a structural +marker the verifier checks for continuity. + +Honest scope, matching the ledger's own warnings: + +- HMAC is symmetric. A signed checkpoint proves the writer held the local + key; it is **not** a public signature and proves nothing to a third party + who does not hold the key. +- ``export_anchor`` writes the head hash to a separate file so it can be + published or archived elsewhere. Comparing an anchor against a ledger + detects whole-file rewrites by anyone who could not also update the + anchor. Anchoring to a truly external system (timestamping service, + public chain) is a Phase-5 workflow; this module provides the hook. + +The key is read from the environment (``SAPIENS_CHECKPOINT_KEY``) at call +time and is never written to the ledger, the anchor file, logs, or any +artifact. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import os +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .ledger import GENESIS, EvidenceLedger, LedgerEvent + +CHECKPOINT_KEY_ENV = "SAPIENS_CHECKPOINT_KEY" +SIGNATURE_SCHEME = "hmac-sha256" + + +def _canonical(value: Any) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False, ensure_ascii=False + ).encode("utf-8") + + +def key_from_env(env: Mapping[str, str] | None = None) -> bytes | None: + """Read the HMAC key from the environment. Never log or persist the result.""" + source = env if env is not None else os.environ + raw = source.get(CHECKPOINT_KEY_ENV) + if not raw: + return None + return raw.encode("utf-8") + + +def signature_payload(*, event_count: int, head_hash: str) -> bytes: + return _canonical({"event_count": event_count, "head_hash": head_hash}) + + +def sign(*, event_count: int, head_hash: str, key: bytes) -> str: + if not key: + raise ValueError("signing key must be non-empty") + return hmac.new( + key, signature_payload(event_count=event_count, head_hash=head_hash), hashlib.sha256 + ).hexdigest() + + +@dataclass(frozen=True) +class CheckpointVerification: + """Outcome of checkpoint signature verification. Data, not vibes.""" + + checkpoints: int + signed: int + signatures_verified: int + signatures_unverifiable: tuple[int, ...] # seq numbers: signed, but no key available + signature_failures: tuple[int, ...] # seq numbers: signature does not match + + +def verify_checkpoints( + events: tuple[LedgerEvent, ...], *, key: bytes | None +) -> CheckpointVerification: + """Check every checkpoint event's signature (when verifiable). + + Structural continuity (event count + head hash) is already enforced by + ``EvidenceLedger.verify``; this layer handles only signatures. + """ + checkpoints = signed = verified = 0 + unverifiable: list[int] = [] + failures: list[int] = [] + for event in events: + if event.kind != "checkpoint": + continue + checkpoints += 1 + signature = event.payload.get("signature") + if not signature: + continue + signed += 1 + if key is None: + unverifiable.append(event.seq) + continue + expected = sign( + event_count=int(event.payload["event_count"]), + head_hash=str(event.payload["head_hash"]), + key=key, + ) + if hmac.compare_digest(expected, str(signature)): + verified += 1 + else: + failures.append(event.seq) + return CheckpointVerification( + checkpoints=checkpoints, + signed=signed, + signatures_verified=verified, + signatures_unverifiable=tuple(unverifiable), + signature_failures=tuple(failures), + ) + + +def record_checkpoint(ledger: EvidenceLedger, *, key: bytes | None = None) -> LedgerEvent: + """Append a checkpoint over the current chain. Key from env when omitted.""" + if key is None: + key = key_from_env() + events = ledger.events() + head = events[-1].event_hash if events else GENESIS + payload: dict[str, Any] = { + "event_count": len(events), + "head_hash": head, + "signed": key is not None, + "signature": None, + "scheme": SIGNATURE_SCHEME if key is not None else None, + } + if key is not None: + payload["signature"] = sign(event_count=len(events), head_hash=head, key=key) + return ledger.append("checkpoint", "__ledger__", payload) + + +def export_anchor(ledger: EvidenceLedger, path: str | Path) -> dict[str, Any]: + """Write the current head hash to a separate anchor file (JSON).""" + events = ledger.events() + head = events[-1].event_hash if events else GENESIS + anchor = { + "kind": "sapiens-ledger-anchor", + "version": 1, + "event_count": len(events), + "head_hash": head, + } + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(anchor, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return anchor + + +def verify_anchor(ledger: EvidenceLedger, path: str | Path) -> bool: + """True iff the anchor file matches the ledger's current head.""" + try: + anchor = json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"unreadable anchor file: {exc}") from exc + if anchor.get("kind") != "sapiens-ledger-anchor" or anchor.get("version") != 1: + raise ValueError("not a sapiens ledger anchor file") + events = ledger.events() + head = events[-1].event_hash if events else GENESIS + return anchor.get("head_hash") == head and anchor.get("event_count") == len(events) diff --git a/src/sapiens/isolation.py b/src/sapiens/isolation.py new file mode 100644 index 0000000..520cf6f --- /dev/null +++ b/src/sapiens/isolation.py @@ -0,0 +1,229 @@ +"""Subprocess isolation with OS-level resource limits for untrusted adapters (Phase 1). + +UNTRUSTED-tier adapters never execute in the kernel's process. Instead the +kernel serialises the work order (adapter location, candidate, stage, seed, +budget) to a child Python process that: + +1. applies POSIX rlimits to *itself* before importing any adapter code + (CPU seconds, address space, open files — thread-safe, unlike preexec_fn), +2. loads the adapter class from its source file, constructs it with no + arguments, and runs ``validate`` under a fresh ``ExecutionContext``, +3. prints exactly one JSON result line on stdout (adapter stdout is + redirected to stderr so a noisy adapter cannot corrupt the protocol). + +The parent enforces a wall-clock timeout and maps every failure — crash, +limit kill, timeout, malformed output — to :class:`IsolationError`. Isolation +is fail-closed: a contained failure produces no evidence. + +Honest limits: rlimits bound resource use; they are not a security sandbox. +A malicious adapter could still attempt network or filesystem mischief within +those bounds — which is why third-party code additionally requires a recorded +owner permission before it may run at all. +""" + +from __future__ import annotations + +import importlib.util +import inspect +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .budget import ExecutionContext +from .models import Candidate, Evidence + +PROTOCOL_VERSION = 1 + + +class IsolationError(RuntimeError): + """An isolated run failed or its result could not be trusted.""" + + +@dataclass(frozen=True) +class ResourceLimits: + cpu_seconds: int = 30 + address_space_bytes: int = 1 << 30 # 1 GiB + max_open_files: int = 64 + + def __post_init__(self) -> None: + if self.cpu_seconds <= 0 or self.address_space_bytes <= 0 or self.max_open_files <= 0: + raise ValueError("resource limits must be positive") + + +DEFAULT_LIMITS = ResourceLimits() + + +def _apply_limits(limits: ResourceLimits) -> None: + import resource + + resource.setrlimit(resource.RLIMIT_CPU, (limits.cpu_seconds, limits.cpu_seconds)) + resource.setrlimit( + resource.RLIMIT_AS, (limits.address_space_bytes, limits.address_space_bytes) + ) + resource.setrlimit(resource.RLIMIT_NOFILE, (limits.max_open_files, limits.max_open_files)) + + +def _load_adapter(path: str, qualname: str) -> Any: + if "" in qualname: + raise IsolationError("isolated adapters must be module-level classes") + spec = importlib.util.spec_from_file_location("sapiens_isolated_adapter", path) + if spec is None or spec.loader is None: + raise IsolationError(f"cannot load adapter module from {path!r}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + obj: Any = module + for part in qualname.split("."): + obj = getattr(obj, part, None) + if obj is None: + raise IsolationError(f"adapter class {qualname!r} not found in {path!r}") + return obj() + + +def _serialize_evidence(item: Evidence) -> dict[str, Any]: + return { + "evidence_id": item.evidence_id, + "candidate_id": item.candidate_id, + "kind": item.kind, + "passed": item.passed, + "protocol": item.protocol, + "dataset": item.dataset, + "seed": item.seed, + "score": item.score, + "details": dict(item.details), + } + + +def _child_main() -> None: + """Entry point: ``python -m sapiens.isolation`` (reads the work order on stdin).""" + try: + spec = json.loads(sys.stdin.read()) + if spec.get("protocol") != PROTOCOL_VERSION: + raise IsolationError("isolation protocol version mismatch") + limits = ResourceLimits(**spec["limits"]) + _apply_limits(limits) + adapter = _load_adapter(spec["adapter_path"], spec["adapter_qualname"]) + raw = spec["candidate"] + candidate = Candidate( + raw["candidate_id"], + raw["domain"], + raw["claim"], + raw.get("parameters") or {}, + raw.get("parent_id"), + raw.get("source_adapter", ""), + ) + budget = spec["budget"] + context = ExecutionContext( + max_steps=int(budget["max_steps"]), max_seconds=float(budget["max_seconds"]) + ) + import contextlib + + with contextlib.redirect_stdout(sys.stderr): + evidence = adapter.validate( + candidate, stage=spec["stage"], seed=int(spec["seed"]), context=context + ) + payload = [_serialize_evidence(item) for item in evidence] + # Fail loudly inside the child if anything is not canonical-JSON safe. + result = json.dumps( + {"ok": True, "evidence": payload}, allow_nan=False, ensure_ascii=False + ) + except BaseException as exc: # contained: report, never traceback-spam the parent + result = json.dumps({"ok": False, "error": f"{type(exc).__name__}: {exc}"}) + sys.stdout.write(result + "\n") + sys.stdout.flush() + + +def adapter_location(adapter: object) -> tuple[str, str]: + """(source file, qualname) for an adapter instance; raises if not locatable.""" + cls = type(adapter) + path = inspect.getsourcefile(cls) + if path is None or not Path(path).is_file(): + raise IsolationError(f"cannot locate source file for adapter class {cls.__name__!r}") + qualname = cls.__qualname__ + if "" in qualname: + raise IsolationError("isolated adapters must be defined at module level") + return path, qualname + + +def run_validate_isolated( + adapter: object, + candidate: Candidate, + *, + stage: str, + seed: int, + context: ExecutionContext, + limits: ResourceLimits = DEFAULT_LIMITS, + timeout_seconds: float = 60.0, +) -> tuple[Evidence, ...]: + """Run ``adapter.validate`` in a resource-limited subprocess. Fail-closed.""" + if sys.platform == "win32": + raise IsolationError("subprocess isolation requires POSIX rlimits") + if timeout_seconds <= 0: + raise ValueError("timeout must be positive") + path, qualname = adapter_location(adapter) + spec = { + "protocol": PROTOCOL_VERSION, + "adapter_path": path, + "adapter_qualname": qualname, + "candidate": { + "candidate_id": candidate.candidate_id, + "domain": candidate.domain, + "claim": candidate.claim, + "parameters": dict(candidate.parameters), + "parent_id": candidate.parent_id, + "source_adapter": candidate.source_adapter, + }, + "stage": stage, + "seed": seed, + "budget": {"max_steps": context.max_steps, "max_seconds": context.max_seconds}, + "limits": { + "cpu_seconds": limits.cpu_seconds, + "address_space_bytes": limits.address_space_bytes, + "max_open_files": limits.max_open_files, + }, + } + try: + order = json.dumps(spec, allow_nan=False) + except (TypeError, ValueError) as exc: + raise IsolationError(f"work order is not JSON-serialisable: {exc}") from exc + try: + proc = subprocess.run( + [sys.executable, "-m", "sapiens.isolation"], + input=order, + capture_output=True, + text=True, + timeout=timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise IsolationError( + f"isolated adapter exceeded the {timeout_seconds}s wall-clock timeout" + ) from exc + if proc.returncode != 0: + tail = (proc.stderr or "").strip().splitlines() + detail = tail[-1] if tail else "no stderr" + raise IsolationError( + f"isolated adapter died (exit {proc.returncode}; killed by rlimit or crash): {detail}" + ) + lines = [line for line in proc.stdout.splitlines() if line.strip()] + if len(lines) != 1: + raise IsolationError("isolated adapter produced malformed stdout") + try: + result = json.loads(lines[0]) + except json.JSONDecodeError as exc: + raise IsolationError("isolated adapter produced invalid JSON") from exc + if not result.get("ok"): + raise IsolationError(f"isolated adapter failed: {result.get('error', 'unknown')}") + evidence: list[Evidence] = [] + for raw in result["evidence"]: + try: + evidence.append(Evidence(**raw)) + except (TypeError, ValueError) as exc: + raise IsolationError(f"isolated adapter returned invalid evidence: {exc}") from exc + return tuple(evidence) + + +if __name__ == "__main__": + _child_main() diff --git a/src/sapiens/kernel.py b/src/sapiens/kernel.py index 370489a..fad11cf 100644 --- a/src/sapiens/kernel.py +++ b/src/sapiens/kernel.py @@ -2,10 +2,12 @@ from __future__ import annotations -from .adapter import DomainAdapter, validate_adapter +from .adapter import DomainAdapter from .budget import ExecutionContext +from .isolation import run_validate_isolated from .ledger import EvidenceLedger from .models import Candidate, EvidenceLevel +from .registry import AdapterRegistry, TrustTier _STAGE_BY_LEVEL = { EvidenceLevel.L1: "internal", @@ -15,8 +17,9 @@ class DiscoveryKernel: - def __init__(self, ledger: EvidenceLedger) -> None: + def __init__(self, ledger: EvidenceLedger, registry: AdapterRegistry | None = None) -> None: self.ledger = ledger + self.registry = registry if registry is not None else AdapterRegistry() def register(self, candidate: Candidate, *, transferred_from: str | None = None) -> None: self.ledger.record_candidate(candidate.candidate_id, transferred_from=transferred_from) @@ -29,15 +32,22 @@ def validate_next( seed: int, context: ExecutionContext, ) -> EvidenceLevel: - validate_adapter(adapter) + tier = self.registry.validate_adapter(adapter) if candidate.domain != adapter.manifest.domain: raise ValueError("candidate domain does not match adapter") current = self.ledger.state(candidate.candidate_id).level if current >= EvidenceLevel.L3: - raise ValueError("Phase 0 automated kernel cannot promote beyond L3") + raise ValueError("automated kernel cannot promote beyond L3; L4 stays human-gated") target = EvidenceLevel(current + 1) stage = _STAGE_BY_LEVEL[target] - evidence = adapter.validate(candidate, stage=stage, seed=seed, context=context) + if tier == TrustTier.UNTRUSTED: + # Third-party code never runs in this process. A contained + # isolation failure yields no evidence and no promotion. + evidence = run_validate_isolated( + adapter, candidate, stage=stage, seed=seed, context=context + ) + else: + evidence = adapter.validate(candidate, stage=stage, seed=seed, context=context) refs: list[str] = [] for item in evidence: if item.candidate_id != candidate.candidate_id or item.kind != stage: diff --git a/src/sapiens/ledger.py b/src/sapiens/ledger.py index 58c6ec2..a1ccca6 100644 --- a/src/sapiens/ledger.py +++ b/src/sapiens/ledger.py @@ -2,7 +2,9 @@ Hash chaining detects accidental or after-the-fact modification; it does not prove identity, authorship, scientific validity, or resistance to an actor who can rewrite -an entire file. Signatures and external anchoring are roadmap items. +an entire file. Phase 1 adds ``checkpoint`` events (see ``sapiens.checkpoints``): +HMAC-signed or unsigned markers that summarise the chain, plus external anchor +export. Signatures are symmetric and prove key possession, not authorship. """ from __future__ import annotations @@ -19,7 +21,8 @@ from .models import Evidence, EvidenceLevel GENESIS = "0" * 64 -_ALLOWED_KINDS = {"candidate", "evidence", "promotion", "demotion", "transfer"} +_ALLOWED_KINDS = {"candidate", "evidence", "promotion", "demotion", "transfer", "checkpoint"} +CHECKPOINT_ACTOR = "__ledger__" def _canonical(value: Any) -> bytes: @@ -105,6 +108,15 @@ def _verify(events: Iterable[LedgerEvent]) -> None: bool(event.payload.get("passed")), str(event.payload.get("kind", "")), ) + elif event.kind == "checkpoint": + if event.candidate_id != CHECKPOINT_ACTOR: + raise LedgerIntegrityError("checkpoint must be recorded by the ledger actor") + if int(event.payload.get("event_count", -1)) != event.seq - 1: + raise LedgerIntegrityError("checkpoint event count does not match the chain") + if event.payload.get("head_hash") != previous: + raise LedgerIntegrityError("checkpoint head hash does not match the chain") + if bool(event.payload.get("signed")) != bool(event.payload.get("signature")): + raise LedgerIntegrityError("checkpoint signed flag and signature disagree") else: if current is None: raise LedgerIntegrityError("transition references unknown candidate") @@ -155,8 +167,9 @@ def state(self, candidate_id: str) -> CandidateState: level = EvidenceLevel.L0 elif event.kind == "evidence": evidence_ids.add(str(event.payload["evidence_id"])) - else: + elif event.kind in {"promotion", "demotion"}: level = EvidenceLevel(int(event.payload["to_level"])) + # checkpoint events carry no per-candidate state if level is None: raise KeyError(candidate_id) return CandidateState(level, frozenset(evidence_ids)) diff --git a/src/sapiens/models.py b/src/sapiens/models.py index 333ccc5..8d953e2 100644 --- a/src/sapiens/models.py +++ b/src/sapiens/models.py @@ -58,19 +58,39 @@ def __post_init__(self) -> None: object.__setattr__(self, "details", frozen_mapping(self.details)) +CODE_ORIGINS = ("first-party-clean-room", "third-party") + + @dataclass(frozen=True) class AdapterManifest: + """What an adapter is made of. Trust *decisions* live in the registry (Phase 1). + + ``synthetic_only`` describes the data: True means the adapter touches only + deterministic synthetic data. ``code_origin`` describes the code: + first-party-clean-room or third-party. Third-party code additionally + requires ``third_party_source`` so the permission manifest can be checked. + """ + name: str version: str domain: str vocabulary: tuple[str, ...] synthetic_only: bool = True + code_origin: str = "first-party-clean-room" + data_sources: tuple[str, ...] = () + third_party_source: str | None = None def __post_init__(self) -> None: if not self.name or not self.version or not self.domain: raise ValueError("adapter manifest fields must be non-empty") - if not self.synthetic_only: - raise ValueError("Phase 0 accepts synthetic-only adapters") + if self.code_origin not in CODE_ORIGINS: + raise ValueError(f"code_origin must be one of {CODE_ORIGINS}") + if self.synthetic_only and self.data_sources: + raise ValueError("synthetic-only adapters must not declare real data sources") + if self.code_origin == "third-party" and not self.third_party_source: + raise ValueError("third-party adapters must declare third_party_source") + if self.code_origin == "first-party-clean-room" and self.third_party_source: + raise ValueError("clean-room adapters must not declare a third_party_source") @dataclass(frozen=True) diff --git a/src/sapiens/permissions.py b/src/sapiens/permissions.py new file mode 100644 index 0000000..50746eb --- /dev/null +++ b/src/sapiens/permissions.py @@ -0,0 +1,144 @@ +"""Owner-permission/licence manifest for third-party code reuse (Phase 1). + +No third-party code may power an adapter without a recorded permission entry. +The repository ships with an **empty** manifest: zero ASTRA-family permissions +exist, and the clean-room invariant holds until an owner records otherwise. + +The manifest is data, not code: a JSON document listing each grant with its +source, scope, licence, grantor, evidence reference, and validity window. +Entries are immutable once loaded; expiry is checked against an explicit date +(injected, never a hidden clock) so verification is deterministic. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import date +from pathlib import Path +from typing import Any + +SCHEMA_VERSION = 1 + + +class MissingPermissionError(PermissionError): + """Raised when third-party code is used without a recorded permission.""" + + +class ManifestFormatError(ValueError): + """Raised when a permission manifest is malformed. Fail loudly, never guess.""" + + +def _parse_date(value: Any, *, field: str) -> date | None: + if value is None: + return None + if not isinstance(value, str): + raise ManifestFormatError(f"{field} must be an ISO date string or null") + try: + return date.fromisoformat(value) + except ValueError as exc: + raise ManifestFormatError(f"{field} is not a valid ISO date: {value!r}") from exc + + +@dataclass(frozen=True) +class PermissionEntry: + """One recorded owner grant for third-party material.""" + + source: str # upstream project, e.g. "ASTRA-dev" + scope: str # what the grant covers, e.g. "adapter:astra-photometry" or "module:swarm/*" + licence: str # licence or permission basis, e.g. "MIT" or "written-permission" + granted_by: str # identity of the owner who granted it + reference: str # evidence: URL, document id, or message reference + granted_on: date + expires_on: date | None = None + + def __post_init__(self) -> None: + for name in ("source", "scope", "licence", "granted_by", "reference"): + if not getattr(self, name): + raise ManifestFormatError(f"permission entry field {name} must be non-empty") + if self.expires_on is not None and self.expires_on < self.granted_on: + raise ManifestFormatError("permission expires before it was granted") + + @classmethod + def from_dict(cls, raw: Any) -> PermissionEntry: + if not isinstance(raw, dict): + raise ManifestFormatError("permission entries must be objects") + known = { + "source", "scope", "licence", "granted_by", "reference", "granted_on", "expires_on", + } + unknown = set(raw) - known + if unknown: + raise ManifestFormatError(f"unknown permission entry fields: {sorted(unknown)}") + granted_on = _parse_date(raw.get("granted_on"), field="granted_on") + if granted_on is None: + raise ManifestFormatError("granted_on is required") + return cls( + source=str(raw.get("source", "")), + scope=str(raw.get("scope", "")), + licence=str(raw.get("licence", "")), + granted_by=str(raw.get("granted_by", "")), + reference=str(raw.get("reference", "")), + granted_on=granted_on, + expires_on=_parse_date(raw.get("expires_on"), field="expires_on"), + ) + + def active(self, *, on: date) -> bool: + return self.granted_on <= on and (self.expires_on is None or on <= self.expires_on) + + def covers(self, *, source: str, scope: str) -> bool: + if self.source != source: + return False + if self.scope == scope: + return True + # Prefix wildcard: "module:swarm/*" covers "module:swarm/pheromone_dynamics". + return self.scope.endswith("/*") and scope.startswith(self.scope[:-1]) + + +@dataclass(frozen=True) +class PermissionManifest: + """Immutable set of recorded owner permissions.""" + + entries: tuple[PermissionEntry, ...] = () + + @classmethod + def empty(cls) -> PermissionManifest: + return cls(()) + + @classmethod + def load(cls, path: str | Path) -> PermissionManifest: + try: + raw = json.loads(Path(path).read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ManifestFormatError(f"permission manifest is not valid JSON: {exc}") from exc + if not isinstance(raw, dict): + raise ManifestFormatError("permission manifest must be a JSON object") + version = raw.get("version") + if version != SCHEMA_VERSION: + raise ManifestFormatError(f"unsupported manifest version: {version!r}") + entries_raw = raw.get("entries") + if not isinstance(entries_raw, list): + raise ManifestFormatError("manifest entries must be a list") + entries = tuple(PermissionEntry.from_dict(item) for item in entries_raw) + sources = [(e.source, e.scope) for e in entries] + if len(set(sources)) != len(sources): + raise ManifestFormatError("duplicate (source, scope) permission entries") + return cls(entries) + + def permits(self, *, source: str, scope: str, on: date) -> bool: + return any( + entry.covers(source=source, scope=scope) and entry.active(on=on) + for entry in self.entries + ) + + def require(self, *, source: str, scope: str, on: date) -> PermissionEntry: + for entry in self.entries: + if entry.covers(source=source, scope=scope): + if entry.active(on=on): + return entry + raise MissingPermissionError( + f"permission for {source!r} scope {scope!r} is not active on {on}" + ) + raise MissingPermissionError( + f"no recorded owner permission for {source!r} scope {scope!r}; " + "third-party code requires an explicit permission entry" + ) diff --git a/src/sapiens/registry.py b/src/sapiens/registry.py new file mode 100644 index 0000000..840a4fe --- /dev/null +++ b/src/sapiens/registry.py @@ -0,0 +1,98 @@ +"""Trust-tiered adapter registry (Phase 1). + +Replaces the Phase-0 synthetic-only gate. Three tiers: + +- ``SYNTHETIC``: deterministic synthetic data only. Runs in-process. +- ``CORE``: first-party clean-room code; real data allowed. Runs in-process. +- ``UNTRUSTED``: third-party code. Requires a recorded owner permission + (``PermissionManifest``) *and* subprocess isolation for every execution. + +An adapter's tier derives from its manifest, never from adapter-supplied +claims at call time: synthetic data + any first-party code ⇒ SYNTHETIC; +real data + first-party clean-room code ⇒ CORE; third-party code ⇒ UNTRUSTED. +Explicit registration pins the tier so a manifest edit cannot silently +downgrade a check that already happened. +""" + +from __future__ import annotations + +from datetime import date +from enum import IntEnum + +from .adapter import DomainAdapter +from .permissions import PermissionManifest + + +class TrustTier(IntEnum): + SYNTHETIC = 0 + CORE = 1 + UNTRUSTED = 2 + + +def derive_tier(adapter: DomainAdapter) -> TrustTier: + """Tier from manifest facts alone.""" + manifest = adapter.manifest + if manifest.code_origin == "third-party": + return TrustTier.UNTRUSTED + return TrustTier.SYNTHETIC if manifest.synthetic_only else TrustTier.CORE + + +class AdapterRegistry: + """Validates adapters against trust tiers and recorded permissions.""" + + def __init__( + self, + permissions: PermissionManifest | None = None, + *, + today: date | None = None, + ) -> None: + self._permissions = permissions if permissions is not None else PermissionManifest.empty() + self._today = today if today is not None else date.today() + self._tiers: dict[str, TrustTier] = {} + + @property + def permissions(self) -> PermissionManifest: + return self._permissions + + def _check(self, adapter: DomainAdapter, tier: TrustTier) -> None: + manifest = adapter.manifest + if tier == TrustTier.SYNTHETIC and not manifest.synthetic_only: + raise ValueError( + f"adapter {manifest.name!r} registered SYNTHETIC but declares real data sources" + ) + if tier == TrustTier.CORE and manifest.code_origin != "first-party-clean-room": + raise ValueError( + f"adapter {manifest.name!r} registered CORE but is not first-party clean-room code" + ) + if tier == TrustTier.UNTRUSTED: + self._permissions.require( + source=manifest.third_party_source or "", + scope=f"adapter:{manifest.name}", + on=self._today, + ) + + def register(self, adapter: DomainAdapter, tier: TrustTier) -> None: + """Pin an adapter to an explicit tier after validating manifest coherence.""" + if not isinstance(adapter, DomainAdapter): + raise TypeError("adapter does not implement DomainAdapter") + self._check(adapter, tier) + name = adapter.manifest.name + existing = self._tiers.get(name) + if existing is not None and existing != tier: + raise ValueError(f"adapter {name!r} already registered at tier {existing.name}") + self._tiers[name] = tier + + def tier_of(self, adapter: DomainAdapter) -> TrustTier: + registered = self._tiers.get(adapter.manifest.name) + return registered if registered is not None else derive_tier(adapter) + + def validate_adapter(self, adapter: DomainAdapter) -> TrustTier: + """Phase-1 gate: structural check + tier rules + permission enforcement.""" + if not isinstance(adapter, DomainAdapter): + raise TypeError("adapter does not implement DomainAdapter") + tier = self.tier_of(adapter) + self._check(adapter, tier) + return tier + + def requires_isolation(self, adapter: DomainAdapter) -> bool: + return self.validate_adapter(adapter) == TrustTier.UNTRUSTED diff --git a/tests/isolation_doubles.py b/tests/isolation_doubles.py new file mode 100644 index 0000000..4ca6632 --- /dev/null +++ b/tests/isolation_doubles.py @@ -0,0 +1,87 @@ +"""Module-level adapter doubles for isolation tests. + +Isolation loads adapters by source-file path in a child process, so these +must live at module level in an importable file (not inside test functions). +Not a pytest module (no test_ functions); ruff still applies. +""" + +from __future__ import annotations + +import time + +from sapiens.models import AdapterManifest, Candidate, Evidence + + +class WellBehavedAdapter: + manifest = AdapterManifest( + "well-behaved", + "1", + "iso-domain", + ("x",), + synthetic_only=False, + code_origin="third-party", + third_party_source="doubles", + data_sources=("https://example.org/public",), + ) + + def propose(self, *, seed: int, limit: int): + return () + + def validate(self, candidate, *, stage: str, seed: int, context): + context.checkpoint() + return ( + Evidence( + f"ev-{candidate.candidate_id}-{stage}", + candidate.candidate_id, + stage, + True, + "double-protocol", + f"double-{stage}", + seed, + 0.75, + {"isolated": True}, + ), + ) + + def import_structure(self, structure, *, candidate_id: str): + return Candidate(candidate_id, "iso-domain", "claim") + + +class NoisyAdapter(WellBehavedAdapter): + def validate(self, candidate, *, stage: str, seed: int, context): + print("NOISE THAT MUST NOT CORRUPT THE PROTOCOL") # noqa: T201 + return super().validate(candidate, stage=stage, seed=seed, context=context) + + +class CpuHogAdapter(WellBehavedAdapter): + def validate(self, candidate, *, stage: str, seed: int, context): + while True: + pass + + +class MemoryHogAdapter(WellBehavedAdapter): + def validate(self, candidate, *, stage: str, seed: int, context): + blob = bytearray(1 << 31) # 2 GiB, exceeds any test rlimit + return blob # unreachable under limits; keeps linters quiet about unused work + + +class SleeperAdapter(WellBehavedAdapter): + def validate(self, candidate, *, stage: str, seed: int, context): + time.sleep(30) + return () + + +class BadEvidenceAdapter(WellBehavedAdapter): + def validate(self, candidate, *, stage: str, seed: int, context): + return ( + Evidence( + "ev-bad", + candidate.candidate_id, + stage, + True, + "double-protocol", + "double-data", + seed, + 1.5, # out of range: model must reject on the parent side + ), + ) diff --git a/tests/test_boundaries.py b/tests/test_boundaries.py index 269318a..298f513 100644 --- a/tests/test_boundaries.py +++ b/tests/test_boundaries.py @@ -5,6 +5,8 @@ from sapiens.adapter import validate_adapter from sapiens.models import AdapterManifest, Candidate, Evidence +from sapiens.permissions import MissingPermissionError +from sapiens.registry import TrustTier ROOT = Path(__file__).resolve().parents[1] @@ -21,23 +23,69 @@ def test_core_does_not_import_adapters(): ) -def test_phase0_rejects_non_synthetic_adapter(): - class RealAdapter: - @property - def manifest(self): # type: ignore[override] - return AdapterManifest("real", "0", "unsafe", ("x",), synthetic_only=False) +class _RealDataAdapter: + @property + def manifest(self): # type: ignore[override] + return AdapterManifest( + "real", + "0", + "unsafe", + ("x",), + synthetic_only=False, + data_sources=("https://example.org/public-data",), + ) - def propose(self, *, seed: int, limit: int): - return () + def propose(self, *, seed: int, limit: int): + return () - def validate(self, candidate, *, stage: str, seed: int, context): - return () + def validate(self, candidate, *, stage: str, seed: int, context): + return () - def import_structure(self, structure, *, candidate_id: str): - return Candidate(candidate_id, "unsafe", "claim") + def import_structure(self, structure, *, candidate_id: str): + return Candidate(candidate_id, "unsafe", "claim") + +class _ThirdPartyAdapter(_RealDataAdapter): + @property + def manifest(self): # type: ignore[override] + return AdapterManifest( + "third-party-real", + "0", + "unsafe", + ("x",), + synthetic_only=False, + code_origin="third-party", + third_party_source="ASTRA-dev", + data_sources=("https://example.org/public-data",), + ) + + +def test_phase1_accepts_first_party_real_data_adapter_at_core_tier(): + # Phase 1 replaced the synthetic-only gate with trust tiers: first-party + # clean-room code on real data is CORE and runs in-process. + validate_adapter(_RealDataAdapter()) + + +def test_phase1_rejects_third_party_adapter_without_permission(): + with pytest.raises(MissingPermissionError): + validate_adapter(_ThirdPartyAdapter()) + + +def test_phase1_registry_reports_untrusted_tier_for_third_party(): + from sapiens.registry import AdapterRegistry + + registry = AdapterRegistry() + assert registry.tier_of(_ThirdPartyAdapter()) == TrustTier.UNTRUSTED + assert registry.tier_of(_RealDataAdapter()) == TrustTier.CORE + + +def test_manifest_rejects_incoherent_provenance(): + with pytest.raises(ValueError): + AdapterManifest("m", "1", "d", ("x",), code_origin="third-party") # no source + with pytest.raises(ValueError): + AdapterManifest("m", "1", "d", ("x",), code_origin="not-a-real-origin") with pytest.raises(ValueError): - validate_adapter(RealAdapter()) + AdapterManifest("m", "1", "d", ("x",), data_sources=("real-data",)) # synthetic lie def test_evidence_rejects_invalid_confidence_score(): diff --git a/tests/test_checkpoints.py b/tests/test_checkpoints.py new file mode 100644 index 0000000..b28211e --- /dev/null +++ b/tests/test_checkpoints.py @@ -0,0 +1,128 @@ +import json + +import pytest + +from sapiens.checkpoints import ( + export_anchor, + key_from_env, + record_checkpoint, + sign, + verify_anchor, + verify_checkpoints, +) +from sapiens.ledger import EvidenceLedger, LedgerIntegrityError +from sapiens.models import Evidence + +KEY = b"test-key-not-a-secret" +OTHER_KEY = b"different-key" + + +def filled_ledger(tmp_path): + ledger = EvidenceLedger(tmp_path / "evidence.jsonl") + ledger.record_candidate("cand-1") + ledger.record_evidence( + Evidence("ev-1", "cand-1", "internal", True, "proto", "data", 7, 0.9) + ) + ledger.promote("cand-1", 1, ("ev-1",)) + return ledger + + +def test_unsigned_checkpoint_roundtrip(tmp_path): + ledger = filled_ledger(tmp_path) + event = record_checkpoint(ledger, key=None) + assert event.kind == "checkpoint" + assert event.payload["event_count"] == 3 + assert event.payload["signed"] is False + assert event.payload["signature"] is None + assert ledger.verify() + report = verify_checkpoints(ledger.events(), key=None) + assert report.checkpoints == 1 and report.signed == 0 + # More events may follow a checkpoint; the chain stays valid. + ledger.record_candidate("cand-2") + assert ledger.verify() + + +def test_signed_checkpoint_verifies_with_key(tmp_path): + ledger = filled_ledger(tmp_path) + record_checkpoint(ledger, key=KEY) + assert ledger.verify() + events = ledger.events() + report = verify_checkpoints(events, key=KEY) + assert report.signed == 1 and report.signatures_verified == 1 + assert not report.signature_failures + + +def test_signed_checkpoint_reports_unverifiable_without_key(tmp_path): + ledger = filled_ledger(tmp_path) + record_checkpoint(ledger, key=KEY) + report = verify_checkpoints(ledger.events(), key=None) + assert report.signed == 1 + assert report.signatures_verified == 0 + assert report.signatures_unverifiable == (4,) + + +def test_wrong_key_detected(tmp_path): + ledger = filled_ledger(tmp_path) + record_checkpoint(ledger, key=KEY) + report = verify_checkpoints(ledger.events(), key=OTHER_KEY) + assert report.signature_failures == (4,) + + +def test_tampered_history_breaks_checkpoint_continuity(tmp_path): + ledger = filled_ledger(tmp_path) + record_checkpoint(ledger, key=KEY) + path = tmp_path / "evidence.jsonl" + lines = path.read_text().splitlines() + # Rewrite a historical payload without fixing the chain. + first = json.loads(lines[0]) + first["payload"]["level"] = 99 + lines[0] = json.dumps(first) + path.write_text("\n".join(lines) + "\n") + with pytest.raises(LedgerIntegrityError): + ledger.verify() + + +def test_forged_checkpoint_event_rejected(tmp_path): + ledger = filled_ledger(tmp_path) + with pytest.raises(LedgerIntegrityError): + # Wrong head hash: verifier catches it at append time. + ledger.append( + "checkpoint", + "__ledger__", + {"event_count": 3, "head_hash": "0" * 64, "signed": False, "signature": None}, + ) + + +def test_anchor_export_and_verify(tmp_path): + ledger = filled_ledger(tmp_path) + anchor_path = tmp_path / "anchor.json" + anchor = export_anchor(ledger, anchor_path) + assert anchor["event_count"] == 3 + assert verify_anchor(ledger, anchor_path) + # Ledger moves on: the anchor no longer matches the head. + ledger.record_candidate("cand-2") + assert not verify_anchor(ledger, anchor_path) + + +def test_anchor_rejects_foreign_file(tmp_path): + ledger = filled_ledger(tmp_path) + bogus = tmp_path / "bogus.json" + bogus.write_text(json.dumps({"kind": "something-else"})) + with pytest.raises(ValueError): + verify_anchor(ledger, bogus) + + +def test_key_from_env_never_returns_empty(monkeypatch): + monkeypatch.delenv("SAPIENS_CHECKPOINT_KEY", raising=False) + assert key_from_env() is None + monkeypatch.setenv("SAPIENS_CHECKPOINT_KEY", "env-key") + assert key_from_env() == b"env-key" + monkeypatch.setenv("SAPIENS_CHECKPOINT_KEY", "") + assert key_from_env() is None + + +def test_sign_is_deterministic_and_key_dependent(): + a = sign(event_count=3, head_hash="ab" * 32, key=KEY) + assert a == sign(event_count=3, head_hash="ab" * 32, key=KEY) + assert a != sign(event_count=3, head_hash="ab" * 32, key=OTHER_KEY) + assert a != sign(event_count=4, head_hash="ab" * 32, key=KEY) diff --git a/tests/test_isolation.py b/tests/test_isolation.py new file mode 100644 index 0000000..fa50e26 --- /dev/null +++ b/tests/test_isolation.py @@ -0,0 +1,104 @@ +import pytest +from isolation_doubles import ( + BadEvidenceAdapter, + CpuHogAdapter, + MemoryHogAdapter, + NoisyAdapter, + SleeperAdapter, + WellBehavedAdapter, +) + +from sapiens.budget import ExecutionContext +from sapiens.isolation import ( + IsolationError, + ResourceLimits, + run_validate_isolated, +) +from sapiens.models import Candidate + +CANDIDATE = Candidate("cand-1", "iso-domain", "isolated validation works", {"k": 1}) +CONTEXT = ExecutionContext(max_steps=10, max_seconds=5.0) +FAST_LIMITS = ResourceLimits(cpu_seconds=2, address_space_bytes=256 << 20, max_open_files=32) + + +def test_well_behaved_adapter_round_trips_evidence(): + evidence = run_validate_isolated( + WellBehavedAdapter(), CANDIDATE, stage="internal", seed=7, context=CONTEXT + ) + assert len(evidence) == 1 + item = evidence[0] + assert item.candidate_id == "cand-1" + assert item.kind == "internal" + assert item.passed is True + assert item.score == 0.75 + assert item.details["isolated"] is True + + +def test_noisy_adapter_stdout_does_not_corrupt_protocol(): + evidence = run_validate_isolated( + NoisyAdapter(), CANDIDATE, stage="internal", seed=7, context=CONTEXT + ) + assert len(evidence) == 1 and evidence[0].passed + + +def test_cpu_hog_killed_by_rlimit(): + with pytest.raises(IsolationError, match="died"): + run_validate_isolated( + CpuHogAdapter(), + CANDIDATE, + stage="internal", + seed=7, + context=CONTEXT, + limits=FAST_LIMITS, + timeout_seconds=20.0, + ) + + +def test_memory_hog_contained_by_rlimit(): + with pytest.raises(IsolationError): + run_validate_isolated( + MemoryHogAdapter(), + CANDIDATE, + stage="internal", + seed=7, + context=CONTEXT, + limits=FAST_LIMITS, + timeout_seconds=20.0, + ) + + +def test_sleeper_killed_by_wall_clock_timeout(): + with pytest.raises(IsolationError, match="wall-clock"): + run_validate_isolated( + SleeperAdapter(), + CANDIDATE, + stage="internal", + seed=7, + context=CONTEXT, + timeout_seconds=2.0, + ) + + +def test_invalid_adapter_evidence_is_contained(): + with pytest.raises(IsolationError): + run_validate_isolated( + BadEvidenceAdapter(), CANDIDATE, stage="internal", seed=7, context=CONTEXT + ) + + +def test_non_serialisable_candidate_parameters_rejected_before_spawn(): + candidate = Candidate("cand-2", "iso-domain", "bad params", {"fn": object()}) + with pytest.raises(IsolationError, match="JSON"): + run_validate_isolated( + WellBehavedAdapter(), candidate, stage="internal", seed=7, context=CONTEXT + ) + + +def test_locally_defined_adapter_cannot_be_isolated(): + class Local(WellBehavedAdapter): + pass + + with pytest.raises(IsolationError, match="module level"): + run_validate_isolated( + Local(), CANDIDATE, stage="internal", seed=7, context=CONTEXT + ) diff --git a/tests/test_kernel_isolation.py b/tests/test_kernel_isolation.py new file mode 100644 index 0000000..1d08247 --- /dev/null +++ b/tests/test_kernel_isolation.py @@ -0,0 +1,66 @@ +import json +from datetime import date + +import pytest +from isolation_doubles import WellBehavedAdapter + +from sapiens.budget import ExecutionContext +from sapiens.kernel import DiscoveryKernel +from sapiens.ledger import EvidenceLedger +from sapiens.models import Candidate, EvidenceLevel +from sapiens.permissions import MissingPermissionError, PermissionManifest +from sapiens.registry import AdapterRegistry + +PERMISSION = { + "source": "doubles", + "scope": "adapter:well-behaved", + "licence": "MIT", + "granted_by": "owner@example.org", + "reference": "https://example.org/permission/1", + "granted_on": "2026-07-01", + "expires_on": None, +} + + +def registry(tmp_path, entries): + path = tmp_path / "permissions.json" + path.write_text(json.dumps({"version": 1, "entries": entries})) + return AdapterRegistry(PermissionManifest.load(path), today=date(2026, 7, 20)) + + +def candidate(): + return Candidate("iso-cand", "iso-domain", "claim under isolation") + + +def test_kernel_runs_untrusted_adapter_isolated(tmp_path): + kernel = DiscoveryKernel( + EvidenceLedger(tmp_path / "evidence.jsonl"), registry(tmp_path, [PERMISSION]) + ) + cand = candidate() + kernel.register(cand) + reached = kernel.validate_next( + WellBehavedAdapter(), + cand, + seed=7, + context=ExecutionContext(max_steps=10, max_seconds=5.0), + ) + assert reached == EvidenceLevel.L1 + state = kernel.ledger.state("iso-cand") + assert state.level == EvidenceLevel.L1 + + +def test_kernel_refuses_third_party_without_permission(tmp_path): + kernel = DiscoveryKernel( + EvidenceLedger(tmp_path / "evidence.jsonl"), registry(tmp_path, []) + ) + cand = candidate() + kernel.register(cand) + with pytest.raises(MissingPermissionError): + kernel.validate_next( + WellBehavedAdapter(), + cand, + seed=7, + context=ExecutionContext(max_steps=10, max_seconds=5.0), + ) + # No promotion, no evidence: the ledger shows the candidate still at L0. + assert kernel.ledger.state("iso-cand").level == EvidenceLevel.L0 diff --git a/tests/test_permissions.py b/tests/test_permissions.py new file mode 100644 index 0000000..8863570 --- /dev/null +++ b/tests/test_permissions.py @@ -0,0 +1,115 @@ +import json +from datetime import date + +import pytest + +from sapiens.permissions import ( + ManifestFormatError, + MissingPermissionError, + PermissionEntry, + PermissionManifest, +) + +TODAY = date(2026, 7, 20) + +ENTRY = { + "source": "ASTRA-dev", + "scope": "adapter:astra-photometry", + "licence": "MIT", + "granted_by": "owner@example.org", + "reference": "https://example.org/permission/1", + "granted_on": "2026-07-01", + "expires_on": "2026-12-31", +} + + +def make_manifest(tmp_path, entries): + path = tmp_path / "permissions.json" + path.write_text(json.dumps({"version": 1, "entries": entries})) + return PermissionManifest.load(path) + + +def test_repo_manifest_is_empty_and_valid(): + from pathlib import Path + + root = Path(__file__).resolve().parents[1] + manifest = PermissionManifest.load(root / "permissions.json") + assert manifest.entries == () + # The clean-room invariant: zero ASTRA-family permissions recorded. + assert not any("ASTRA" in e.source or "astra" in e.source for e in manifest.entries) + + +def test_empty_manifest_refuses_everything(): + manifest = PermissionManifest.empty() + assert not manifest.permits(source="ASTRA-dev", scope="adapter:x", on=TODAY) + with pytest.raises(MissingPermissionError): + manifest.require(source="ASTRA-dev", scope="adapter:x", on=TODAY) + + +def test_load_roundtrip_and_permit(tmp_path): + manifest = make_manifest(tmp_path, [ENTRY]) + assert manifest.permits( + source="ASTRA-dev", scope="adapter:astra-photometry", on=date(2026, 8, 1) + ) + entry = manifest.require( + source="ASTRA-dev", scope="adapter:astra-photometry", on=date(2026, 8, 1) + ) + assert entry.licence == "MIT" + + +def test_expired_permission_refused(tmp_path): + manifest = make_manifest(tmp_path, [ENTRY]) + assert not manifest.permits( + source="ASTRA-dev", scope="adapter:astra-photometry", on=date(2027, 1, 1) + ) + with pytest.raises(MissingPermissionError, match="not active"): + manifest.require( + source="ASTRA-dev", scope="adapter:astra-photometry", on=date(2027, 1, 1) + ) + + +def test_not_yet_granted_permission_refused(tmp_path): + manifest = make_manifest(tmp_path, [ENTRY]) + assert not manifest.permits( + source="ASTRA-dev", scope="adapter:astra-photometry", on=date(2026, 6, 1) + ) + + +def test_scope_wildcard(tmp_path): + wildcard = {**ENTRY, "scope": "module:swarm/*"} + manifest = make_manifest(tmp_path, [wildcard]) + assert manifest.permits( + source="ASTRA-dev", scope="module:swarm/pheromone_dynamics", on=date(2026, 8, 1) + ) + assert not manifest.permits(source="ASTRA-dev", scope="module:other/x", on=date(2026, 8, 1)) + + +def test_wrong_source_refused(tmp_path): + manifest = make_manifest(tmp_path, [ENTRY]) + assert not manifest.permits( + source="SLATE", scope="adapter:astra-photometry", on=date(2026, 8, 1) + ) + + +@pytest.mark.parametrize( + "raw", + [ + {"version": 2, "entries": []}, # unsupported version + {"version": 1}, # missing entries list + {"version": 1, "entries": [{"source": "x"}]}, # incomplete entry + {"version": 1, "entries": [{**ENTRY, "granted_on": "not-a-date"}]}, + {"version": 1, "entries": [{**ENTRY, "expires_on": "2026-01-01"}]}, # before grant + {"version": 1, "entries": [{**ENTRY, "surprise_field": 1}]}, # unknown field + {"version": 1, "entries": [ENTRY, ENTRY]}, # duplicates + ], +) +def test_malformed_manifests_fail_loudly(tmp_path, raw): + path = tmp_path / "bad.json" + path.write_text(json.dumps(raw)) + with pytest.raises(ManifestFormatError): + PermissionManifest.load(path) + + +def test_entry_requires_nonempty_fields(): + with pytest.raises(ManifestFormatError): + PermissionEntry.from_dict({**ENTRY, "licence": ""}) diff --git a/tests/test_photometry_adapter.py b/tests/test_photometry_adapter.py index a57316f..7a669b7 100644 --- a/tests/test_photometry_adapter.py +++ b/tests/test_photometry_adapter.py @@ -4,7 +4,11 @@ from sapiens.adapters import SyntheticPhotometryAdapter, SyntheticThresholdAdapter from sapiens.budget import ExecutionContext -CTX = ExecutionContext(10, 2) + +def ctx() -> ExecutionContext: + # Fresh context per call: a module-level time-budgeted context starts its + # wall clock at import/collection time and goes stale as the suite grows. + return ExecutionContext(10, 10) def test_photometry_true_period_promotes_to_l3(tmp_path: Path): @@ -13,9 +17,9 @@ def test_photometry_true_period_promotes_to_l3(tmp_path: Path): adapter = SyntheticPhotometryAdapter() candidate = adapter.propose(seed=5, limit=1)[0] kernel.register(candidate) - assert kernel.validate_next(adapter, candidate, seed=40, context=CTX) == EvidenceLevel.L1 - assert kernel.validate_next(adapter, candidate, seed=41, context=CTX) == EvidenceLevel.L2 - assert kernel.validate_next(adapter, candidate, seed=42, context=CTX) == EvidenceLevel.L3 + assert kernel.validate_next(adapter, candidate, seed=40, context=ctx()) == EvidenceLevel.L1 + assert kernel.validate_next(adapter, candidate, seed=41, context=ctx()) == EvidenceLevel.L2 + assert kernel.validate_next(adapter, candidate, seed=42, context=ctx()) == EvidenceLevel.L3 assert ledger.verify() is True @@ -25,7 +29,7 @@ def test_photometry_wrong_period_does_not_promote(tmp_path: Path): adapter = SyntheticPhotometryAdapter() candidate = adapter.propose(seed=5, limit=2)[1] # the wrong-period candidate kernel.register(candidate) - assert kernel.validate_next(adapter, candidate, seed=40, context=CTX) == EvidenceLevel.L0 + assert kernel.validate_next(adapter, candidate, seed=40, context=ctx()) == EvidenceLevel.L0 def test_photometry_evidence_is_well_formed(): diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 0000000..fe18151 --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,115 @@ +import json +from datetime import date + +import pytest + +from sapiens.models import AdapterManifest, Candidate +from sapiens.permissions import MissingPermissionError, PermissionManifest +from sapiens.registry import AdapterRegistry, TrustTier, derive_tier + + +class SyntheticDouble: + manifest = AdapterManifest("synth", "1", "synth-domain", ("x",)) + + def propose(self, *, seed: int, limit: int): + return () + + def validate(self, candidate, *, stage: str, seed: int, context): + return () + + def import_structure(self, structure, *, candidate_id: str): + return Candidate(candidate_id, "synth-domain", "claim") + + +class CoreDouble(SyntheticDouble): + manifest = AdapterManifest( + "core-real", + "1", + "core-domain", + ("x",), + synthetic_only=False, + data_sources=("https://example.org/public",), + ) + + +class ThirdPartyDouble(SyntheticDouble): + manifest = AdapterManifest( + "tp", + "1", + "tp-domain", + ("x",), + synthetic_only=False, + code_origin="third-party", + third_party_source="ASTRA-dev", + data_sources=("https://example.org/public",), + ) + + +PERMISSION = { + "source": "ASTRA-dev", + "scope": "adapter:tp", + "licence": "MIT", + "granted_by": "owner@example.org", + "reference": "https://example.org/permission/1", + "granted_on": "2026-07-01", + "expires_on": None, +} + + +def manifest_with(entries, tmp_path): + path = tmp_path / "permissions.json" + path.write_text(json.dumps({"version": 1, "entries": entries})) + return PermissionManifest.load(path) + + +def test_derive_tier_from_manifest_facts(): + assert derive_tier(SyntheticDouble()) == TrustTier.SYNTHETIC + assert derive_tier(CoreDouble()) == TrustTier.CORE + assert derive_tier(ThirdPartyDouble()) == TrustTier.UNTRUSTED + + +def test_unregistered_adapters_auto_tier_on_validate(): + registry = AdapterRegistry(today=date(2026, 7, 20)) + assert registry.validate_adapter(SyntheticDouble()) == TrustTier.SYNTHETIC + assert registry.validate_adapter(CoreDouble()) == TrustTier.CORE + + +def test_third_party_requires_permission_entry(tmp_path): + registry = AdapterRegistry(today=date(2026, 7, 20)) + with pytest.raises(MissingPermissionError): + registry.validate_adapter(ThirdPartyDouble()) + permitted = AdapterRegistry( + manifest_with([PERMISSION], tmp_path), today=date(2026, 7, 20) + ) + assert permitted.validate_adapter(ThirdPartyDouble()) == TrustTier.UNTRUSTED + assert permitted.requires_isolation(ThirdPartyDouble()) + + +def test_expired_permission_blocks_validation(tmp_path): + expired = {**PERMISSION, "expires_on": "2026-07-19"} + registry = AdapterRegistry(manifest_with([expired], tmp_path), today=date(2026, 7, 20)) + with pytest.raises(MissingPermissionError): + registry.validate_adapter(ThirdPartyDouble()) + + +def test_registration_pins_tier_and_rejects_incoherent(tmp_path): + registry = AdapterRegistry(today=date(2026, 7, 20)) + registry.register(SyntheticDouble(), TrustTier.SYNTHETIC) + with pytest.raises(ValueError, match="already registered"): + registry.register(SyntheticDouble(), TrustTier.CORE) + with pytest.raises(ValueError, match="real data sources"): + registry.register(CoreDouble(), TrustTier.SYNTHETIC) + with pytest.raises(ValueError, match="clean-room"): + registry.register(ThirdPartyDouble(), TrustTier.CORE) + + +def test_register_third_party_untrusted_with_permission(tmp_path): + registry = AdapterRegistry(manifest_with([PERMISSION], tmp_path), today=date(2026, 7, 20)) + registry.register(ThirdPartyDouble(), TrustTier.UNTRUSTED) + assert registry.tier_of(ThirdPartyDouble()) == TrustTier.UNTRUSTED + + +def test_non_adapter_rejected(): + registry = AdapterRegistry() + with pytest.raises(TypeError): + registry.validate_adapter(object()) From b6b0dabb2701a35fd7bdbe79122c5b44ac06c777 Mon Sep 17 00:00:00 2001 From: The Beast Date: Mon, 20 Jul 2026 00:17:23 +0000 Subject: [PATCH 2/3] =?UTF-8?q?feat:=20Phase=202=20=E2=80=94=20validation?= =?UTF-8?q?=20framework=20v1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sapiens.validation: L1 internal-consistency gate (score presence/range, determinism across identical reruns, degenerate constant-score rejection) and L2 holdout-replication gate (declared HoldoutProtocol per domain; leakage controls on dataset collision and (dataset, seed) reuse; pass-fraction threshold); ValidationGates wiring config - sapiens.fixtures: labelled seeded-bias suite (known-good / overfit / leakage / degenerate) with expected gate outcomes - sapiens.calibration: CalibrationReport over the fixture suite — catch rate, false-reject rate, sample counts, content-hash report id - sapiens.confidence: aggregate_confidence raises UncalibratedError without sufficient calibration; emits documented heuristic (raw pass fraction x demonstrated catch rate) with provenance — no invented precision - kernel: opt-in validation wiring (fail-closed when a configured domain lacks a declared protocol); gate verdicts logged, never fabricated as ledger evidence - 33 new tests; docs (README/ROADMAP/ARCHITECTURE/VALIDATION) truthful; version 0.3.0 Closes #17, closes #18, closes #19, closes #20, closes #21, closes #22. Part of #8. --- ARCHITECTURE.md | 10 +- README.md | 18 ++- ROADMAP.md | 25 +++- VALIDATION.md | 17 ++- pyproject.toml | 2 +- src/sapiens/__init__.py | 17 ++- src/sapiens/calibration.py | 127 +++++++++++++++++++ src/sapiens/confidence.py | 73 +++++++++++ src/sapiens/fixtures.py | 141 +++++++++++++++++++++ src/sapiens/kernel.py | 49 +++++++- src/sapiens/validation.py | 176 +++++++++++++++++++++++++++ tests/test_calibration_confidence.py | 96 +++++++++++++++ tests/test_kernel_gates.py | 133 ++++++++++++++++++++ tests/test_validation.py | 170 ++++++++++++++++++++++++++ 14 files changed, 1040 insertions(+), 14 deletions(-) create mode 100644 src/sapiens/calibration.py create mode 100644 src/sapiens/confidence.py create mode 100644 src/sapiens/fixtures.py create mode 100644 src/sapiens/validation.py create mode 100644 tests/test_calibration_confidence.py create mode 100644 tests/test_kernel_gates.py create mode 100644 tests/test_validation.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 4592aaf..b447c51 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,4 +1,4 @@ -# SAPIENS Architecture (Phases 0–1) +# SAPIENS Architecture (Phases 0–2) ## Design goals @@ -18,6 +18,10 @@ src/sapiens/ permissions.py owner-permission/licence manifest for third-party code (Phase 1) isolation.py subprocess + rlimit execution for UNTRUSTED adapters (Phase 1) checkpoints.py signed ledger checkpoints + external anchor export (Phase 1) + validation.py L1/L2 automated gates + holdout protocols + leakage controls (Phase 2) + fixtures.py seeded-bias fixture suite with labelled outcomes (Phase 2) + calibration.py gate-performance calibration reports (Phase 2) + confidence.py calibration-gated confidence aggregation (Phase 2) ledger.py JSONL hash-chain ledger and L0→L4 transition guard kernel.py domain-neutral candidate registration and next-gate validation bridge.py cross-domain structure transfer with mandatory L0 reset @@ -49,6 +53,10 @@ The ledger is newline-delimited canonical JSON. Each event stores the previous e Hash chaining detects tampering but does **not** prove authorship, scientific truth, or external timestamping. Phase-1 `checkpoint` events summarise the chain (event count + head hash) and may carry an HMAC-SHA256 signature (environment-held key, never stored); `sapiens.checkpoints` also exports/verifies external anchor files. HMAC is symmetric: it proves key possession, not third-party authorship. +## Validation gates (Phase 2) + +`DiscoveryKernel(validation=ValidationGates(...))` opts into automated L1/L2 gates. L1 runs statistical sanity checks over a candidate's internal evidence (determinism across identical reruns, degenerate constant scores, score presence). L2 requires a declared `HoldoutProtocol` for the domain and enforces holdout discipline: replication evidence must come from declared holdout datasets, dataset collisions and (dataset, seed) reuse across the boundary are leakage and reject the gate, and a minimum pass fraction applies. Gate verdicts are appended to `kernel.gate_log` (inspectable, recomputable) — the kernel never fabricates gate outcomes as ledger evidence. Gates are pure functions in `sapiens.validation`; `sapiens.fixtures` ships a labelled seeded-bias suite; `sapiens.calibration` scores gates against it; `sapiens.confidence` refuses to aggregate confidence without the resulting report. + ## Cross-domain bridge `transfer(source, source_level, target_adapter, candidate_id)` extracts only a small structural envelope and returns a target-domain candidate plus `EvidenceLevel.L0`. The discarded source level is retained only as provenance. The target candidate must climb target-domain gates from scratch. diff --git a/README.md b/README.md index cb73b00..b60863b 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,10 @@ src/sapiens/ permissions.py owner-permission/licence manifest for third-party code isolation.py subprocess + rlimit execution for UNTRUSTED adapters checkpoints.py HMAC-signed ledger checkpoints + external anchor export + validation.py L1/L2 automated gates: sanity checks, holdout + leakage + fixtures.py seeded-bias fixture suite (labelled ground truth) + calibration.py gate-performance calibration reports + confidence.py calibration-gated confidence aggregation (refuses blindly) ledger.py JSONL hash-chained evidence ledger, L0→L4 transition guard kernel.py domain-neutral DiscoveryKernel; owns all promotions bridge.py cross-domain structure transfer — ALWAYS resets target to L0 @@ -167,7 +171,7 @@ three deterministic synthetic adapters, synthetic-only orchestration, hash-chained ledger, kernel gates, bridge, bounded queue/daemon, and CI on Python 3.10/3.11/3.12. -**Phase 1 — shipped** (current package version `0.2.0`): the synthetic-only +**Phase 1 — shipped** (package version `0.2.0`): the synthetic-only gate is replaced by a **trust-tiered adapter registry** (SYNTHETIC / CORE / UNTRUSTED), an **owner-permission/licence manifest** ([`permissions.json`](permissions.json) — empty by default: no third-party @@ -178,11 +182,19 @@ address-space / open-files plus wall-clock timeout, fail-closed), and (key from the environment only, never stored). No real-data adapter ships in Phase 1; tiers are exercised by synthetic adapters and test doubles. +**Phase 2 — shipped** (current package version `0.3.0`): automated L0→L2 +**validation gates** — L1 statistical sanity checks (determinism, +degenerate-score rejection) and L2 declared holdout protocols with explicit +leakage controls (dataset collision, (dataset, seed) reuse) — plus a +labelled **seeded-bias fixture suite** (known-good / overfit / leakage / +degenerate), **calibration reports** (catch rate and false-reject rate with +sample counts), and **calibration-gated confidence aggregation** that +refuses to emit a number without sufficient calibration data. + Next, in order (see [`ROADMAP.md`](ROADMAP.md)): 1. ~~**Phase 1** — legal/licence gate and adapter hardening~~ **shipped**. -2. **Phase 2** — validation framework v1 (statistical gates, holdout - protocols, leakage controls, seeded-bias fixtures, calibration). +2. ~~**Phase 2** — validation framework v1~~ **shipped**. 3. **Phase 3** — structured L3 review panels (role-specialized reviewers, multi-round objection tracking, catch-rate scoring). 4. **Phase 4** — **real domain adapters** — first a clean-room Kepler diff --git a/ROADMAP.md b/ROADMAP.md index 387e546..9c6cac8 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -30,11 +30,28 @@ Honest limits: rlimits bound resource use but are not a security sandbox; HMAC proves local key possession, not third-party authorship. No real-data adapter ships in Phase 1. -## Phase 2 — validation framework v1 +## Phase 2 — shipped (package version 0.3.0) -- Expand L0→L2 automated gates with statistical sanity checks, holdout protocols, and explicit leakage controls. -- Add seeded-bias fixtures and calibration reports. -- Add confidence aggregation only after calibration data exists; do not invent precision. +Validation framework v1: + +- ~~Expand L0→L2 automated gates~~ Shipped: `sapiens.validation` — L1 + internal-consistency gate (score presence/range, determinism across reruns, + degenerate constant-score rejection) and L2 holdout-replication gate + (declared `HoldoutProtocol` per domain; explicit leakage controls on + dataset collision and (dataset, seed) reuse; pass-fraction threshold). + Opt-in kernel wiring via `DiscoveryKernel(validation=...)`; gate verdicts + are logged, never fabricated as evidence; a configured domain without a + declared protocol fails closed. +- ~~Add seeded-bias fixtures and calibration reports.~~ Shipped: + `sapiens.fixtures` (known-good / overfit / leakage / degenerate, labelled + with expected outcomes) and `sapiens.calibration` (`CalibrationReport`: + catch rate, false-reject rate, sample counts; report ids are content + hashes). +- ~~Add confidence aggregation only after calibration data exists~~ Shipped: + `sapiens.confidence.aggregate_confidence` raises `UncalibratedError` + without a sufficiently sampled calibration report; with one, it emits a + documented heuristic (raw pass fraction × demonstrated catch rate) with + full provenance. No invented precision. ## Phase 3 — structured L3 review panels diff --git a/VALIDATION.md b/VALIDATION.md index 14477d4..2c8c1e7 100644 --- a/VALIDATION.md +++ b/VALIDATION.md @@ -23,7 +23,7 @@ python -m sapiens.cli - Queue capacity, idempotency, stale-lease rejection, oversized-payload rejection. - Daemon executes only registered handlers under bounded context. - Boundary test: core modules do not import synthetic adapters. -- Phase-0 rejects non-synthetic adapters. +- Phase-0 rejects non-synthetic adapters (superseded in Phase 1 by trust-tier tests). ## CI @@ -44,10 +44,23 @@ GitHub Actions runs Python 3.10, 3.11, and 3.12 with: 6. Ledger uses canonical JSON and rejects hash/sequence/transition violations. 7. Cross-domain bridge discards source confidence and starts target at L0. +## Phase 1–2 additions + +- Trust-tier registry: third-party adapters refused without permission; tier derivation from manifest facts. +- Isolation: CPU-hog / memory-hog / sleeper children killed and contained; protocol corruption impossible from child stdout. +- Checkpoints: continuity verified; HMAC sign/verify; wrong-key detection; anchor export/verify. +- L1 gate: determinism, degenerate-score, missing-score rejection. +- L2 gate: holdout discipline, dataset-collision and (dataset, seed) leakage rejection, pass-fraction threshold. +- Calibration: fixture labels verified against gate behaviour; report rates match ground truth. +- Confidence: refuses without calibration / with thin calibration / without evidence. + ## Known limits - Hash-chain integrity is not cryptographic authorship or external timestamping. -- Phase-0 daemon is cooperative and in-process; hostile adapters need subprocess/cgroup isolation later. +- rlimits bound resource use but are not a full security sandbox; third-party code additionally requires a recorded owner permission. +- HMAC checkpoint signatures are symmetric: key possession, not third-party authorship. +- Confidence values are documented heuristics (raw pass fraction × demonstrated catch rate), not probability estimates; fixture suites are small and rates are exact only for the fixtures included. +- The daemon is cooperative and in-process for SYNTHETIC/CORE tiers; UNTRUSTED adapters run via subprocess isolation. - Synthetic adapters are toy harnesses, not scientific models. - L3 is represented as a bounded `review` evidence gate; full multi-agent review panels are roadmap. - L4 is only a human-gated transition rule, not automated. diff --git a/pyproject.toml b/pyproject.toml index eec17b9..7ec1513 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sapiens-discovery" -version = "0.2.0" +version = "0.3.0" description = "Experimental foundation for traceable cross-domain scientific-discovery workflows" readme = "README.md" requires-python = ">=3.10" diff --git a/src/sapiens/__init__.py b/src/sapiens/__init__.py index eff7afe..1af7229 100644 --- a/src/sapiens/__init__.py +++ b/src/sapiens/__init__.py @@ -2,25 +2,40 @@ from .adapter import DomainAdapter from .bridge import TransferEnvelope, transfer +from .calibration import CalibrationReport, run_calibration +from .confidence import CalibratedConfidence, UncalibratedError, aggregate_confidence +from .fixtures import FixtureKind, SeededFixture, fixture_suite from .kernel import DiscoveryKernel from .ledger import EvidenceLedger from .models import AdapterManifest, Candidate, Evidence, EvidenceLevel from .permissions import PermissionEntry, PermissionManifest from .registry import AdapterRegistry, TrustTier +from .validation import GateVerdict, HoldoutProtocol, ValidationGates __all__ = [ "AdapterManifest", "AdapterRegistry", + "CalibratedConfidence", + "CalibrationReport", "Candidate", "DiscoveryKernel", "DomainAdapter", "Evidence", "EvidenceLedger", "EvidenceLevel", + "FixtureKind", + "GateVerdict", + "HoldoutProtocol", "PermissionEntry", "PermissionManifest", + "SeededFixture", "TransferEnvelope", "TrustTier", + "UncalibratedError", + "ValidationGates", + "aggregate_confidence", + "fixture_suite", + "run_calibration", "transfer", ] -__version__ = "0.2.0" +__version__ = "0.3.0" diff --git a/src/sapiens/calibration.py b/src/sapiens/calibration.py new file mode 100644 index 0000000..7ea8377 --- /dev/null +++ b/src/sapiens/calibration.py @@ -0,0 +1,127 @@ +"""Calibration reports (Phase 2): gate performance against ground truth. + +A calibration report runs the validation gates over the seeded-bias fixture +suite and records, per fixture, whether each gate behaved as the fixture's +label demands. The resulting rates — known-bad catch rate, known-good false +-reject rate — are the *only* legitimate basis for confidence aggregation +(see ``sapiens.confidence``). A report also carries its sample counts so a +thin report cannot masquerade as a strong one. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass + +from .fixtures import FixtureKind, SeededFixture +from .validation import check_internal_consistency, check_replication + +KNOWN_BAD_KINDS = (FixtureKind.OVERFIT, FixtureKind.LEAKAGE, FixtureKind.DEGENERATE) + + +@dataclass(frozen=True) +class FixtureOutcome: + fixture_id: str + kind: str + l1_passed: bool + l2_passed: bool + l1_reasons: tuple[str, ...] + l2_reasons: tuple[str, ...] + + +@dataclass(frozen=True) +class CalibrationReport: + """Immutable gate-performance record over a labelled fixture suite.""" + + report_id: str + outcomes: tuple[FixtureOutcome, ...] + known_good_total: int + known_good_accepted: int + known_bad_total: int + known_bad_caught: int + + @property + def catch_rate(self) -> float: + """Fraction of known-bad fixtures at least one gate correctly rejected.""" + return self.known_bad_caught / self.known_bad_total if self.known_bad_total else 0.0 + + @property + def false_reject_rate(self) -> float: + """Fraction of known-good fixtures some gate wrongly rejected.""" + if not self.known_good_total: + return 1.0 # no evidence of reliability: assume the worst, honestly + return 1.0 - self.known_good_accepted / self.known_good_total + + def meets_minimum(self, *, min_known_bad: int, min_known_good: int) -> bool: + return self.known_bad_total >= min_known_bad and self.known_good_total >= min_known_good + + def to_dict(self) -> dict[str, object]: + return { + "report_id": self.report_id, + "known_good_total": self.known_good_total, + "known_good_accepted": self.known_good_accepted, + "known_bad_total": self.known_bad_total, + "known_bad_caught": self.known_bad_caught, + "catch_rate": self.catch_rate, + "false_reject_rate": self.false_reject_rate, + "outcomes": [ + { + "fixture_id": o.fixture_id, + "kind": o.kind, + "l1_passed": o.l1_passed, + "l2_passed": o.l2_passed, + } + for o in self.outcomes + ], + } + + +def _report_id(outcomes: tuple[FixtureOutcome, ...]) -> str: + canonical = json.dumps( + [(o.fixture_id, o.l1_passed, o.l2_passed) for o in outcomes], + separators=(",", ":"), + ) + return hashlib.sha256(canonical.encode()).hexdigest()[:16] + + +def run_calibration(fixtures: tuple[SeededFixture, ...]) -> CalibrationReport: + """Evaluate both gates on every fixture and tally against labels.""" + outcomes: list[FixtureOutcome] = [] + good_total = good_accepted = bad_total = bad_caught = 0 + for fixture in fixtures: + l1 = check_internal_consistency(fixture.internal) + l2 = check_replication(fixture.internal, fixture.replication, fixture.protocol) + if l1.passed != fixture.expect_l1_pass or l2.passed != fixture.expect_l2_pass: + raise AssertionError( + f"fixture {fixture.fixture_id!r} label disagrees with gates: " + f"expected L1={fixture.expect_l1_pass} L2={fixture.expect_l2_pass}, " + f"got L1={l1.passed} L2={l2.passed} — fix the fixture or the gate" + ) + outcomes.append( + FixtureOutcome( + fixture.fixture_id, + fixture.kind.value, + l1.passed, + l2.passed, + l1.reasons, + l2.reasons, + ) + ) + if fixture.kind == FixtureKind.KNOWN_GOOD: + good_total += 1 + if l1.passed and l2.passed: + good_accepted += 1 + elif fixture.kind in KNOWN_BAD_KINDS: + bad_total += 1 + if not (l1.passed and l2.passed): + bad_caught += 1 + result = tuple(outcomes) + return CalibrationReport( + _report_id(result), + result, + good_total, + good_accepted, + bad_total, + bad_caught, + ) diff --git a/src/sapiens/confidence.py b/src/sapiens/confidence.py new file mode 100644 index 0000000..853eedb --- /dev/null +++ b/src/sapiens/confidence.py @@ -0,0 +1,73 @@ +"""Confidence aggregation (Phase 2) — only on top of calibration data. + +There is no confidence without calibration. ``aggregate_confidence`` raises +:class:`UncalibratedError` unless handed a :class:`CalibrationReport` built +from enough labelled fixtures; the gates' *demonstrated* catch rate is the +only discount applied, and the formula is a documented heuristic, not a +probability estimate. We do not invent precision. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .calibration import CalibrationReport +from .models import Evidence + +FORMULA_VERSION = "catch-rate-discount-v1" + + +class UncalibratedError(RuntimeError): + """No trustworthy calibration data exists; refuse to emit a number.""" + + +@dataclass(frozen=True) +class CalibratedConfidence: + """A bounded heuristic score with its full provenance attached.""" + + value: float # raw_pass_fraction * catch_rate, in [0, 1] + raw_pass_fraction: float + catch_rate: float + false_reject_rate: float + calibration_report_id: str + formula: str + caveat: str = ( + "Heuristic score, not a probability: raw pass fraction discounted by " + "the gates' demonstrated catch rate on seeded fixtures." + ) + + +def aggregate_confidence( + evidence: tuple[Evidence, ...], + calibration: CalibrationReport | None, + *, + min_known_bad: int = 2, + min_known_good: int = 1, +) -> CalibratedConfidence: + """Aggregate evidence into a calibrated score, or refuse. + + Refusal is the point: without a calibration report meeting minimum + fixture counts, any number would be invented precision. + """ + if calibration is None: + raise UncalibratedError("no calibration report supplied; confidence refused") + if not calibration.meets_minimum( + min_known_bad=min_known_bad, min_known_good=min_known_good + ): + raise UncalibratedError( + f"calibration report {calibration.report_id} is too thin " + f"(known_bad={calibration.known_bad_total}, " + f"known_good={calibration.known_good_total}); confidence refused" + ) + if not evidence: + raise UncalibratedError("no evidence supplied; confidence refused") + raw = sum(1 for item in evidence if item.passed) / len(evidence) + value = raw * calibration.catch_rate + return CalibratedConfidence( + value=value, + raw_pass_fraction=raw, + catch_rate=calibration.catch_rate, + false_reject_rate=calibration.false_reject_rate, + calibration_report_id=calibration.report_id, + formula=FORMULA_VERSION, + ) diff --git a/src/sapiens/fixtures.py b/src/sapiens/fixtures.py new file mode 100644 index 0000000..72b527d --- /dev/null +++ b/src/sapiens/fixtures.py @@ -0,0 +1,141 @@ +"""Seeded-bias fixture suite (Phase 2). + +Deterministic, labelled candidates with planted failure modes, used to +calibrate the validation gates (Phase 2) and to score review-panel catch +rates (Phase 3). Every fixture declares its expected gate outcome; the +calibration report is only meaningful because this ground truth exists. + +Fixtures are pure evidence data — no adapter behaviour is simulated — so the +suite is stable across Python versions and never touches the ledger. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +from .models import Candidate, Evidence +from .validation import HoldoutProtocol, synthetic_holdout_protocol + +PROTOCOL = synthetic_holdout_protocol() +DOMAIN = "synthetic-fixtures" + + +class FixtureKind(Enum): + KNOWN_GOOD = "known-good" # honest evidence; gates must accept + OVERFIT = "overfit" # passes train, fails holdout; L2 must reject + LEAKAGE = "leakage" # train/holdout boundary violated; L2 must reject + DEGENERATE = "degenerate" # constant scores across seeds; L1 must reject + + +@dataclass(frozen=True) +class SeededFixture: + fixture_id: str + kind: FixtureKind + candidate: Candidate + internal: tuple[Evidence, ...] + replication: tuple[Evidence, ...] + protocol: HoldoutProtocol + expect_l1_pass: bool + expect_l2_pass: bool + rationale: str + + +def _evidence( + fixture: str, stage: str, runs: tuple[tuple[int, str, float, bool], ...] +) -> tuple[Evidence, ...]: + return tuple( + Evidence( + f"{fixture}-{stage}-{index}", + f"cand-{fixture}", + stage, + passed, + f"fixture-{stage}-v1", + dataset, + seed, + score, + {"fixture": fixture}, + ) + for index, (seed, dataset, score, passed) in enumerate(runs) + ) + + +def _candidate(fixture: str, claim: str) -> Candidate: + return Candidate(f"cand-{fixture}", DOMAIN, claim, source_adapter="fixture-suite") + + +def fixture_suite() -> tuple[SeededFixture, ...]: + """The canonical seeded-bias suite. Deterministic; do not mutate.""" + good = SeededFixture( + "good-1", + FixtureKind.KNOWN_GOOD, + _candidate("good-1", "honest signal replicates on holdout"), + _evidence( + "good-1", + "internal", + ((101, "synthetic-train", 0.91, True), (102, "synthetic-train", 0.89, True)), + ), + _evidence( + "good-1", + "replication", + ((201, "synthetic-holdout", 0.88, True), (202, "synthetic-holdout", 0.9, True)), + ), + PROTOCOL, + expect_l1_pass=True, + expect_l2_pass=True, + rationale="varying scores, clean split, independent seeds, all runs pass", + ) + overfit = SeededFixture( + "overfit-1", + FixtureKind.OVERFIT, + _candidate("overfit-1", "signal that only exists in the training split"), + _evidence( + "overfit-1", + "internal", + ((101, "synthetic-train", 0.95, True), (102, "synthetic-train", 0.93, True)), + ), + _evidence( + "overfit-1", + "replication", + ((201, "synthetic-holdout", 0.31, False), (202, "synthetic-holdout", 0.28, False)), + ), + PROTOCOL, + expect_l1_pass=True, + expect_l2_pass=False, + rationale="strong on train, collapses on holdout — L2 must catch it", + ) + leakage = SeededFixture( + "leakage-1", + FixtureKind.LEAKAGE, + _candidate("leakage-1", "'replication' run on the training data itself"), + _evidence("leakage-1", "internal", ((101, "synthetic-train", 0.92, True),)), + _evidence("leakage-1", "replication", ((101, "synthetic-train", 0.92, True),)), + PROTOCOL, + expect_l1_pass=True, + expect_l2_pass=False, + rationale="same dataset and seed on both sides — leakage controls must fire", + ) + degenerate = SeededFixture( + "degenerate-1", + FixtureKind.DEGENERATE, + _candidate("degenerate-1", "constant score regardless of seed"), + _evidence( + "degenerate-1", + "internal", + ( + (101, "synthetic-train", 0.5, True), + (102, "synthetic-train", 0.5, True), + (103, "synthetic-train", 0.5, True), + ), + ), + _evidence( + "degenerate-1", + "replication", + ((201, "synthetic-holdout", 0.5, True),), + ), + PROTOCOL, + expect_l1_pass=False, + expect_l2_pass=True, + rationale="identical score across three independent seeds — no information", + ) + return (good, overfit, leakage, degenerate) diff --git a/src/sapiens/kernel.py b/src/sapiens/kernel.py index fad11cf..8d6d023 100644 --- a/src/sapiens/kernel.py +++ b/src/sapiens/kernel.py @@ -6,8 +6,9 @@ from .budget import ExecutionContext from .isolation import run_validate_isolated from .ledger import EvidenceLedger -from .models import Candidate, EvidenceLevel +from .models import Candidate, Evidence, EvidenceLevel from .registry import AdapterRegistry, TrustTier +from .validation import GateVerdict, ValidationGates _STAGE_BY_LEVEL = { EvidenceLevel.L1: "internal", @@ -17,9 +18,16 @@ class DiscoveryKernel: - def __init__(self, ledger: EvidenceLedger, registry: AdapterRegistry | None = None) -> None: + def __init__( + self, + ledger: EvidenceLedger, + registry: AdapterRegistry | None = None, + validation: ValidationGates | None = None, + ) -> None: self.ledger = ledger self.registry = registry if registry is not None else AdapterRegistry() + self.validation = validation + self.gate_log: list[GateVerdict] = [] def register(self, candidate: Candidate, *, transferred_from: str | None = None) -> None: self.ledger.record_candidate(candidate.candidate_id, transferred_from=transferred_from) @@ -57,5 +65,42 @@ def validate_next( refs.append(item.evidence_id) if not refs: return current + if not self._passes_validation_gates(candidate, target): + return current self.ledger.promote(candidate.candidate_id, target, tuple(refs)) return target + + def _evidence_for(self, candidate_id: str) -> tuple[Evidence, ...]: + items: list[Evidence] = [] + for event in self.ledger.events(): + if event.kind == "evidence" and event.candidate_id == candidate_id: + items.append(Evidence(**event.payload)) + return tuple(items) + + def _passes_validation_gates(self, candidate: Candidate, target: EvidenceLevel) -> bool: + """Phase-2 opt-in gates. Verdicts are logged, never fabricated as evidence.""" + if self.validation is None or target not in (EvidenceLevel.L1, EvidenceLevel.L2): + return True + from .validation import check_internal_consistency, check_replication + + recorded = self._evidence_for(candidate.candidate_id) + if target == EvidenceLevel.L1: + verdict = check_internal_consistency(recorded) + self.gate_log.append(verdict) + return verdict.passed + protocol = self.validation.protocol_for(candidate.domain) + if protocol is None: + raise ValueError( + f"validation gates configured but no holdout protocol declared for " + f"domain {candidate.domain!r}; refusing L2 promotion" + ) + internal = tuple(item for item in recorded if item.kind == "internal") + replication = tuple(item for item in recorded if item.kind == "replication") + verdict = check_replication( + internal, + replication, + protocol, + min_pass_fraction=self.validation.min_replication_pass_fraction, + ) + self.gate_log.append(verdict) + return verdict.passed diff --git a/src/sapiens/validation.py b/src/sapiens/validation.py new file mode 100644 index 0000000..45449ae --- /dev/null +++ b/src/sapiens/validation.py @@ -0,0 +1,176 @@ +"""Validation framework v1 (Phase 2): automated L0→L2 gates. + +Two gate families, both pure functions over recorded evidence: + +- **Internal consistency (L1)**: statistical sanity checks — scores present + and in range, determinism (identical protocol/dataset/seed reruns must + agree), degenerate-distribution rejection (constant scores across + independent seeds carry no information). +- **Replication (L2)**: a declared :class:`HoldoutProtocol` per domain plus + explicit leakage controls — replication evidence must come from declared + holdout datasets, and any dataset collision or (dataset, seed) pair reuse + across the train/holdout boundary rejects the gate. + +Gates return :class:`GateVerdict` with explicit reasons; they never fabricate +or mutate evidence. Wiring them into promotion is the kernel's job +(``DiscoveryKernel(validation=...)``); here they stay inspectable and testable +in isolation. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType + +from .models import Evidence + +INTERNAL_KIND = "internal" +REPLICATION_KIND = "replication" + + +@dataclass(frozen=True) +class GateVerdict: + gate: str + passed: bool + reasons: tuple[str, ...] = () + + def __post_init__(self) -> None: + if self.passed and self.reasons: + raise ValueError("a passing verdict carries no failure reasons") + if not self.passed and not self.reasons: + raise ValueError("a failing verdict must say why") + + +@dataclass(frozen=True) +class HoldoutProtocol: + """Declared train/holdout split for a domain. Leakage controls key off this.""" + + name: str + train_datasets: tuple[str, ...] + holdout_datasets: tuple[str, ...] + + def __post_init__(self) -> None: + if not self.name or not self.train_datasets or not self.holdout_datasets: + raise ValueError("holdout protocol requires a name and both dataset splits") + collision = set(self.train_datasets) & set(self.holdout_datasets) + if collision: + raise ValueError(f"holdout protocol is self-leaking: {sorted(collision)}") + + +def _scores(items: list[Evidence]) -> list[float]: + return [item.score for item in items if item.score is not None] # type: ignore[misc] + + +def check_internal_consistency(evidence: tuple[Evidence, ...]) -> GateVerdict: + """L1 gate: statistical sanity over internal evidence for one candidate.""" + gate = "L1-internal-consistency" + reasons: list[str] = [] + items = [item for item in evidence if item.kind == INTERNAL_KIND] + if not items: + return GateVerdict(gate, False, ("no internal evidence to validate",)) + for item in items: + if item.score is None: + reasons.append(f"evidence {item.evidence_id} carries no score") + elif not 0.0 <= item.score <= 1.0: + reasons.append(f"evidence {item.evidence_id} score out of range") + # Determinism: reruns of the identical protocol/dataset/seed must agree. + by_run: dict[tuple[str, str, int], set[tuple[float | None, bool]]] = {} + for item in items: + key = (item.protocol, item.dataset, item.seed) + by_run.setdefault(key, set()).add((item.score, item.passed)) + for (protocol, dataset, seed), outcomes in by_run.items(): + if len(outcomes) > 1: + reasons.append( + f"non-deterministic rerun: {protocol}/{dataset}/seed={seed} " + f"yielded {len(outcomes)} distinct outcomes" + ) + # Degenerate distribution: identical scores across >=3 independent seeds + # carry no information about the candidate. + seeds = {item.seed for item in items} + scores = _scores(items) + if len(seeds) >= 3 and len(scores) >= 3 and len(set(scores)) == 1: + reasons.append( + f"degenerate evidence: constant score {scores[0]} across {len(seeds)} seeds" + ) + if reasons: + return GateVerdict(gate, False, tuple(reasons)) + return GateVerdict(gate, True) + + +def check_replication( + internal: tuple[Evidence, ...], + replication: tuple[Evidence, ...], + protocol: HoldoutProtocol, + *, + min_pass_fraction: float = 1.0, +) -> GateVerdict: + """L2 gate: holdout discipline + leakage controls over replication evidence.""" + gate = "L2-holdout-replication" + if not 0.0 < min_pass_fraction <= 1.0: + raise ValueError("min_pass_fraction must be in (0, 1]") + reasons: list[str] = [] + train = [item for item in internal if item.kind == INTERNAL_KIND] + holdout = [item for item in replication if item.kind == REPLICATION_KIND] + if not holdout: + return GateVerdict(gate, False, ("no replication evidence to validate",)) + for item in train: + if item.dataset not in protocol.train_datasets: + reasons.append( + f"internal evidence {item.evidence_id} uses undeclared dataset " + f"{item.dataset!r} (protocol {protocol.name!r})" + ) + for item in holdout: + if item.dataset not in protocol.holdout_datasets: + reasons.append( + f"replication evidence {item.evidence_id} uses non-holdout dataset " + f"{item.dataset!r} (protocol {protocol.name!r})" + ) + # Leakage control 1: dataset identifiers must not cross the boundary. + train_ids = {item.dataset for item in train} + holdout_ids = {item.dataset for item in holdout} + collision = train_ids & holdout_ids + if collision: + reasons.append(f"dataset leakage across train/holdout boundary: {sorted(collision)}") + # Leakage control 2: an identical (dataset, seed) pair on both sides means + # the "independent" run literally reproduced the training draw. + train_draws = {(item.dataset, item.seed) for item in train} + holdout_draws = {(item.dataset, item.seed) for item in holdout} + reused = train_draws & holdout_draws + if reused: + reasons.append(f"(dataset, seed) reuse across boundary: {sorted(reused)}") + if holdout: + pass_fraction = sum(1 for item in holdout if item.passed) / len(holdout) + if pass_fraction < min_pass_fraction: + reasons.append( + f"replication pass fraction {pass_fraction:.3f} below " + f"required {min_pass_fraction:.3f}" + ) + if reasons: + return GateVerdict(gate, False, tuple(reasons)) + return GateVerdict(gate, True) + + +@dataclass(frozen=True) +class ValidationGates: + """Opt-in kernel wiring: per-domain holdout protocols + replication threshold.""" + + protocols: Mapping[str, HoldoutProtocol] = field( + default_factory=lambda: MappingProxyType({}) + ) + min_replication_pass_fraction: float = 1.0 + + def __post_init__(self) -> None: + object.__setattr__(self, "protocols", MappingProxyType(dict(self.protocols))) + + def protocol_for(self, domain: str) -> HoldoutProtocol | None: + return self.protocols.get(domain) + + +def synthetic_holdout_protocol() -> HoldoutProtocol: + """The split every shipped synthetic adapter already honours.""" + return HoldoutProtocol( + "synthetic-v1", + train_datasets=("synthetic-train",), + holdout_datasets=("synthetic-holdout",), + ) diff --git a/tests/test_calibration_confidence.py b/tests/test_calibration_confidence.py new file mode 100644 index 0000000..8ea754b --- /dev/null +++ b/tests/test_calibration_confidence.py @@ -0,0 +1,96 @@ +import json + +import pytest + +from sapiens.calibration import CalibrationReport, run_calibration +from sapiens.confidence import UncalibratedError, aggregate_confidence +from sapiens.fixtures import FixtureKind, fixture_suite +from sapiens.models import Evidence + + +def test_fixture_suite_labels_match_gate_behaviour(): + # run_calibration raises if any fixture's label disagrees with the gates. + report = run_calibration(fixture_suite()) + assert len(report.outcomes) == 4 + + +def test_calibration_report_rates_match_ground_truth(): + report = run_calibration(fixture_suite()) + assert report.known_good_total == 1 + assert report.known_good_accepted == 1 + assert report.known_bad_total == 3 + assert report.known_bad_caught == 3 + assert report.catch_rate == 1.0 + assert report.false_reject_rate == 0.0 + + +def test_fixture_kinds_cover_documented_failure_modes(): + kinds = {fixture.kind for fixture in fixture_suite()} + assert FixtureKind.KNOWN_GOOD in kinds + assert FixtureKind.OVERFIT in kinds + assert FixtureKind.LEAKAGE in kinds + assert FixtureKind.DEGENERATE in kinds + for fixture in fixture_suite(): + assert fixture.rationale # every fixture explains itself + + +def test_report_is_json_serialisable_and_identified(): + report = run_calibration(fixture_suite()) + blob = json.dumps(report.to_dict()) + assert report.report_id in blob + # Same fixtures ⇒ same report id (deterministic). + assert run_calibration(fixture_suite()).report_id == report.report_id + + +def test_empty_suite_report_is_honest_about_thinness(): + report = run_calibration(()) + assert report.known_bad_total == 0 + assert report.catch_rate == 0.0 + assert report.false_reject_rate == 1.0 # no evidence: assume the worst + assert not report.meets_minimum(min_known_bad=2, min_known_good=1) + + +def ev(eid, passed): + return Evidence(eid, "cand-x", "internal", passed, "p", "synthetic-train", 1, 0.9) + + +def test_confidence_refused_without_calibration(): + with pytest.raises(UncalibratedError, match="no calibration"): + aggregate_confidence((ev("a", True),), None) + + +def test_confidence_refused_with_thin_calibration(): + good_only = tuple(f for f in fixture_suite() if f.kind == FixtureKind.KNOWN_GOOD) + thin = run_calibration(good_only) + with pytest.raises(UncalibratedError, match="too thin"): + aggregate_confidence((ev("a", True),), thin) + + +def test_confidence_refused_without_evidence(): + report = run_calibration(fixture_suite()) + with pytest.raises(UncalibratedError, match="no evidence"): + aggregate_confidence((), report) + + +def test_calibrated_confidence_value_and_provenance(): + report = run_calibration(fixture_suite()) + result = aggregate_confidence((ev("a", True), ev("b", False)), report) + assert result.raw_pass_fraction == 0.5 + assert result.catch_rate == 1.0 + assert result.value == 0.5 # raw * catch_rate + assert result.calibration_report_id == report.report_id + assert "not a probability" in result.caveat + + +def test_confidence_discounted_by_weak_calibration(): + weak = CalibrationReport( + "weak-report", + (), + known_good_total=2, + known_good_accepted=2, + known_bad_total=4, + known_bad_caught=2, + ) + result = aggregate_confidence((ev("a", True),), weak) + assert result.catch_rate == 0.5 + assert result.value == 0.5 # perfect evidence, halved by demonstrated weakness diff --git a/tests/test_kernel_gates.py b/tests/test_kernel_gates.py new file mode 100644 index 0000000..5da55d8 --- /dev/null +++ b/tests/test_kernel_gates.py @@ -0,0 +1,133 @@ +"""Kernel + Phase-2 validation gates integration.""" + +import pytest + +from sapiens.adapters import SyntheticLinearAdapter +from sapiens.budget import ExecutionContext +from sapiens.kernel import DiscoveryKernel +from sapiens.ledger import EvidenceLedger +from sapiens.models import AdapterManifest, Candidate, Evidence, EvidenceLevel +from sapiens.validation import ValidationGates, synthetic_holdout_protocol + + +def ctx(): + return ExecutionContext(max_steps=10, max_seconds=10.0) + + +def gated_kernel(tmp_path, adapter_domain, protocol=None): + gates = ValidationGates({adapter_domain: protocol or synthetic_holdout_protocol()}) + return DiscoveryKernel(EvidenceLedger(tmp_path / "events.jsonl"), validation=gates) + + +def test_synthetic_adapter_promotes_through_gates(tmp_path): + adapter = SyntheticLinearAdapter() + kernel = gated_kernel(tmp_path, adapter.manifest.domain) + candidate = adapter.propose(seed=5, limit=1)[0] + kernel.register(candidate) + assert kernel.validate_next(adapter, candidate, seed=40, context=ctx()) == EvidenceLevel.L1 + assert kernel.validate_next(adapter, candidate, seed=41, context=ctx()) == EvidenceLevel.L2 + assert all(verdict.passed for verdict in kernel.gate_log) + + +class LeakyReplicationAdapter: + """Replication 'evidence' produced on the training dataset itself.""" + + manifest = AdapterManifest("leaky", "1", "leaky-domain", ("x",)) + + def propose(self, *, seed: int, limit: int): + return (Candidate("cand-leaky", "leaky-domain", "leaks holdout"),) + + def validate(self, candidate, *, stage: str, seed: int, context): + context.checkpoint() + dataset = "synthetic-train" # same dataset for every stage: leakage + return ( + Evidence( + f"ev-{stage}", candidate.candidate_id, stage, True, "leaky-v1", dataset, seed, 0.9 + ), + ) + + def import_structure(self, structure, *, candidate_id: str): + return Candidate(candidate_id, "leaky-domain", "claim") + + +def test_kernel_blocks_l2_on_dataset_leakage(tmp_path): + adapter = LeakyReplicationAdapter() + kernel = gated_kernel(tmp_path, "leaky-domain") + candidate = adapter.propose(seed=1, limit=1)[0] + kernel.register(candidate) + assert kernel.validate_next(adapter, candidate, seed=40, context=ctx()) == EvidenceLevel.L1 + assert kernel.validate_next(adapter, candidate, seed=41, context=ctx()) == EvidenceLevel.L1 + assert kernel.ledger.state("cand-leaky").level == EvidenceLevel.L1 + failing = [v for v in kernel.gate_log if not v.passed] + assert failing and any("leakage" in r for r in failing[-1].reasons) + + +class NonDeterministicAdapter: + """Same seed, different outcomes across calls: fails the determinism check.""" + + manifest = AdapterManifest("flaky", "1", "flaky-domain", ("x",)) + + def __init__(self): + self.calls = 0 + + def propose(self, *, seed: int, limit: int): + return (Candidate("cand-flaky", "flaky-domain", "unstable signal"),) + + def validate(self, candidate, *, stage: str, seed: int, context): + context.checkpoint() + self.calls += 1 + score = 0.1 if self.calls == 1 else 0.9 # first attempt fails, then "recovers" + return ( + Evidence( + f"ev-{self.calls}", + candidate.candidate_id, + stage, + score > 0.5, + "flaky-v1", + "synthetic-train", + seed, + score, + ), + ) + + def import_structure(self, structure, *, candidate_id: str): + return Candidate(candidate_id, "flaky-domain", "claim") + + +def test_kernel_blocks_l1_on_nondeterminism(tmp_path): + adapter = NonDeterministicAdapter() + kernel = gated_kernel(tmp_path, "flaky-domain") + candidate = adapter.propose(seed=1, limit=1)[0] + kernel.register(candidate) + # Attempt 1 records failing evidence → stays L0. Attempt 2 reruns the same + # (protocol, dataset, seed) but reports a contradictory passing outcome, so + # the determinism check must block promotion even though evidence passed. + assert kernel.validate_next(adapter, candidate, seed=42, context=ctx()) == EvidenceLevel.L0 + assert kernel.validate_next(adapter, candidate, seed=42, context=ctx()) == EvidenceLevel.L0 + assert kernel.ledger.state("cand-flaky").level == EvidenceLevel.L0 + failing = [v for v in kernel.gate_log if not v.passed] + assert failing + assert any("non-deterministic" in reason for reason in failing[-1].reasons) + + +def test_kernel_refuses_l2_without_declared_protocol(tmp_path): + adapter = SyntheticLinearAdapter() + gates = ValidationGates({}) # configured but declares nothing + kernel = DiscoveryKernel( + EvidenceLedger(tmp_path / "events.jsonl"), validation=gates + ) + candidate = adapter.propose(seed=5, limit=1)[0] + kernel.register(candidate) + assert kernel.validate_next(adapter, candidate, seed=40, context=ctx()) == EvidenceLevel.L1 + with pytest.raises(ValueError, match="no holdout protocol"): + kernel.validate_next(adapter, candidate, seed=41, context=ctx()) + + +def test_kernel_without_gates_keeps_phase1_behaviour(tmp_path): + adapter = LeakyReplicationAdapter() # would fail gates; no gates configured + kernel = DiscoveryKernel(EvidenceLedger(tmp_path / "events.jsonl")) + candidate = adapter.propose(seed=1, limit=1)[0] + kernel.register(candidate) + assert kernel.validate_next(adapter, candidate, seed=40, context=ctx()) == EvidenceLevel.L1 + assert kernel.validate_next(adapter, candidate, seed=41, context=ctx()) == EvidenceLevel.L2 + assert kernel.gate_log == [] diff --git a/tests/test_validation.py b/tests/test_validation.py new file mode 100644 index 0000000..83aca79 --- /dev/null +++ b/tests/test_validation.py @@ -0,0 +1,170 @@ +import pytest + +from sapiens.models import Evidence +from sapiens.validation import ( + GateVerdict, + HoldoutProtocol, + ValidationGates, + check_internal_consistency, + check_replication, + synthetic_holdout_protocol, +) + +PROTOCOL = synthetic_holdout_protocol() + + +def ev(eid, kind, dataset, seed, score, passed=True, protocol="proto-v1"): + return Evidence(eid, "cand-x", kind, passed, protocol, dataset, seed, score) + + +class TestGateVerdictCoherence: + def test_passing_verdict_has_no_reasons(self): + with pytest.raises(ValueError): + GateVerdict("g", True, ("reason",)) + + def test_failing_verdict_must_explain(self): + with pytest.raises(ValueError): + GateVerdict("g", False, ()) + + +class TestHoldoutProtocolCoherence: + def test_self_leaking_protocol_rejected(self): + with pytest.raises(ValueError, match="self-leaking"): + HoldoutProtocol("bad", ("train",), ("train",)) + + def test_empty_splits_rejected(self): + with pytest.raises(ValueError): + HoldoutProtocol("bad", (), ("holdout",)) + + +class TestInternalConsistency: + def test_honest_evidence_passes(self): + evidence = ( + ev("a", "internal", "synthetic-train", 1, 0.9), + ev("b", "internal", "synthetic-train", 2, 0.85), + ) + assert check_internal_consistency(evidence).passed + + def test_no_evidence_fails(self): + verdict = check_internal_consistency(()) + assert not verdict.passed + assert "no internal evidence" in verdict.reasons[0] + + def test_missing_score_fails(self): + verdict = check_internal_consistency( + (ev("a", "internal", "synthetic-train", 1, None),) + ) + assert not verdict.passed + assert "no score" in verdict.reasons[0] + + def test_nondeterministic_rerun_fails(self): + evidence = ( + ev("a", "internal", "synthetic-train", 7, 0.9, passed=True), + ev("b", "internal", "synthetic-train", 7, 0.2, passed=False), + ) + verdict = check_internal_consistency(evidence) + assert not verdict.passed + assert any("non-deterministic" in reason for reason in verdict.reasons) + + def test_deterministic_rerun_agrees_passes(self): + evidence = ( + ev("a", "internal", "synthetic-train", 7, 0.9), + ev("b", "internal", "synthetic-train", 7, 0.9), + ) + assert check_internal_consistency(evidence).passed + + def test_degenerate_constant_scores_fail(self): + evidence = tuple( + ev(f"e{seed}", "internal", "synthetic-train", seed, 0.5) for seed in (1, 2, 3) + ) + verdict = check_internal_consistency(evidence) + assert not verdict.passed + assert any("degenerate" in reason for reason in verdict.reasons) + + def test_two_seeds_not_enough_for_degenerate_check(self): + evidence = tuple( + ev(f"e{seed}", "internal", "synthetic-train", seed, 0.5) for seed in (1, 2) + ) + assert check_internal_consistency(evidence).passed + + def test_other_kinds_ignored(self): + evidence = (ev("a", "replication", "synthetic-holdout", 1, 0.9),) + verdict = check_internal_consistency(evidence) + assert not verdict.passed # no internal items at all + + +class TestReplicationGate: + def honest(self): + internal = (ev("i1", "internal", "synthetic-train", 1, 0.9),) + replication = ( + ev("r1", "replication", "synthetic-holdout", 2, 0.88), + ev("r2", "replication", "synthetic-holdout", 3, 0.91), + ) + return internal, replication + + def test_honest_holdout_passes(self): + internal, replication = self.honest() + assert check_replication(internal, replication, PROTOCOL).passed + + def test_no_replication_evidence_fails(self): + internal, _ = self.honest() + verdict = check_replication(internal, (), PROTOCOL) + assert not verdict.passed + + def test_undeclared_train_dataset_fails(self): + internal = (ev("i1", "internal", "secret-train", 1, 0.9),) + _, replication = self.honest() + verdict = check_replication(internal, replication, PROTOCOL) + assert not verdict.passed + assert any("undeclared dataset" in reason for reason in verdict.reasons) + + def test_non_holdout_replication_dataset_fails(self): + internal = (ev("i1", "internal", "synthetic-train", 1, 0.9),) + replication = (ev("r1", "replication", "synthetic-train", 2, 0.9),) + verdict = check_replication(internal, replication, PROTOCOL) + assert not verdict.passed + assert any("non-holdout dataset" in reason for reason in verdict.reasons) + + def test_dataset_collision_is_leakage(self): + internal = (ev("i1", "internal", "synthetic-train", 1, 0.9),) + replication = (ev("r1", "replication", "synthetic-train", 2, 0.9),) + verdict = check_replication(internal, replication, PROTOCOL) + assert any("leakage" in reason for reason in verdict.reasons) + + def test_dataset_seed_pair_reuse_is_leakage(self): + internal = (ev("i1", "internal", "synthetic-train", 7, 0.9),) + replication = (ev("r1", "replication", "synthetic-train", 7, 0.9),) + verdict = check_replication(internal, replication, PROTOCOL) + assert any("reuse" in reason for reason in verdict.reasons) + + def test_same_seed_different_dataset_is_not_leakage(self): + # Numeric seed overlap across disjoint datasets is how the shipped + # synthetic adapters derive independent holdout noise; it must pass. + internal = (ev("i1", "internal", "synthetic-train", 7, 0.9),) + replication = (ev("r1", "replication", "synthetic-holdout", 7, 0.88),) + assert check_replication(internal, replication, PROTOCOL).passed + + def test_pass_fraction_enforced(self): + internal = (ev("i1", "internal", "synthetic-train", 1, 0.9),) + replication = ( + ev("r1", "replication", "synthetic-holdout", 2, 0.9, passed=True), + ev("r2", "replication", "synthetic-holdout", 3, 0.2, passed=False), + ) + verdict = check_replication(internal, replication, PROTOCOL) + assert not verdict.passed + assert any("pass fraction" in reason for reason in verdict.reasons) + relaxed = check_replication( + internal, replication, PROTOCOL, min_pass_fraction=0.5 + ) + assert relaxed.passed + + def test_invalid_threshold_rejected(self): + internal, replication = self.honest() + with pytest.raises(ValueError): + check_replication(internal, replication, PROTOCOL, min_pass_fraction=0.0) + + +def test_validation_gates_protocol_lookup(): + gates = ValidationGates({"synthetic-x": PROTOCOL}) + assert gates.protocol_for("synthetic-x") is PROTOCOL + assert gates.protocol_for("unknown") is None From ec3c7ae5dbf285e5fa42060203fa71d05801d80d Mon Sep 17 00:00:00 2001 From: The Beast Date: Mon, 20 Jul 2026 00:36:31 +0000 Subject: [PATCH 3/3] =?UTF-8?q?feat:=20Phase=203=20=E2=80=94=20structured?= =?UTF-8?q?=20L3=20review=20panels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sapiens.review: role-specialized schemas (statistician, domain theorist, methodologist, devil's advocate), typed verdicts with severity-graded objections; bounded deterministic multi-round protocol with objection lifecycle (raised/sustained/withdrawn); disagreement gate — sustained MAJOR/BLOCKING rejects, MINOR caveats recorded but non-fatal - sapiens.reviewers: four deterministic reference reviewers re-running Phase-2 gates independently + hunting seeded-bias signatures (degenerate scores, train/holdout leakage, implausible perfection, thin stages); MAJOR escalates to BLOCKING on re-affirmation - sapiens.catchrate: per-role and panel catch-rate scoring over the seeded fixture suite with explicit small-sample caveat - kernel: opt-in L3 panel gate — panel verdict recorded in the ledger as review evidence (no side channel); approval required for L3 promotion; behaviour without a panel unchanged - 44 new tests; docs truthful; version 0.4.0 Closes #23, closes #24, closes #25, closes #26, closes #27, closes #28. Part of #9. --- ARCHITECTURE.md | 9 +- README.md | 18 +- ROADMAP.md | 23 ++- VALIDATION.md | 5 + pyproject.toml | 2 +- src/sapiens/__init__.py | 12 +- src/sapiens/catchrate.py | 114 ++++++++++++ src/sapiens/kernel.py | 39 +++++ src/sapiens/review.py | 270 +++++++++++++++++++++++++++++ src/sapiens/reviewers.py | 347 +++++++++++++++++++++++++++++++++++++ tests/test_catchrate.py | 53 ++++++ tests/test_kernel_panel.py | 132 ++++++++++++++ tests/test_review.py | 209 ++++++++++++++++++++++ tests/test_reviewers.py | 161 +++++++++++++++++ 14 files changed, 1384 insertions(+), 10 deletions(-) create mode 100644 src/sapiens/catchrate.py create mode 100644 src/sapiens/review.py create mode 100644 src/sapiens/reviewers.py create mode 100644 tests/test_catchrate.py create mode 100644 tests/test_kernel_panel.py create mode 100644 tests/test_review.py create mode 100644 tests/test_reviewers.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b447c51..00a1257 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,4 +1,4 @@ -# SAPIENS Architecture (Phases 0–2) +# SAPIENS Architecture (Phases 0–3) ## Design goals @@ -22,6 +22,9 @@ src/sapiens/ fixtures.py seeded-bias fixture suite with labelled outcomes (Phase 2) calibration.py gate-performance calibration reports (Phase 2) confidence.py calibration-gated confidence aggregation (Phase 2) + review.py L3 panel protocol: roles, objections, rounds, gates (Phase 3) + reviewers.py deterministic reference reviewers, four roles (Phase 3) + catchrate.py panel catch-rate scoring over seeded fixtures (Phase 3) ledger.py JSONL hash-chain ledger and L0→L4 transition guard kernel.py domain-neutral candidate registration and next-gate validation bridge.py cross-domain structure transfer with mandatory L0 reset @@ -57,6 +60,10 @@ Hash chaining detects tampering but does **not** prove authorship, scientific tr `DiscoveryKernel(validation=ValidationGates(...))` opts into automated L1/L2 gates. L1 runs statistical sanity checks over a candidate's internal evidence (determinism across identical reruns, degenerate constant scores, score presence). L2 requires a declared `HoldoutProtocol` for the domain and enforces holdout discipline: replication evidence must come from declared holdout datasets, dataset collisions and (dataset, seed) reuse across the boundary are leakage and reject the gate, and a minimum pass fraction applies. Gate verdicts are appended to `kernel.gate_log` (inspectable, recomputable) — the kernel never fabricates gate outcomes as ledger evidence. Gates are pure functions in `sapiens.validation`; `sapiens.fixtures` ships a labelled seeded-bias suite; `sapiens.calibration` scores gates against it; `sapiens.confidence` refuses to aggregate confidence without the resulting report. +## L3 review panels (Phase 3) + +`DiscoveryKernel(panel=ReviewPanel(...))` gates L3 promotion on a structured panel. Reviewers are pure deterministic functions in four roles (statistician, domain theorist, methodologist, devil's advocate). The panel convenes bounded rounds; objections carry severity (MINOR/MAJOR/BLOCKING) and a tracked lifecycle (raised/sustained/withdrawn); reference reviewers escalate re-affirmed MAJOR findings to BLOCKING. Approval requires no sustained MAJOR/BLOCKING objection; MINOR caveats are recorded but non-fatal. The panel's verdict is recorded in the ledger as review evidence (`panel-transcript` dataset) — approval adds it to the promotion refs, rejection leaves the candidate at L2 with the rejection on record. `sapiens.catchrate` scores panels against the seeded fixture suite. + ## Cross-domain bridge `transfer(source, source_level, target_adapter, candidate_id)` extracts only a small structural envelope and returns a target-domain candidate plus `EvidenceLevel.L0`. The discarded source level is retained only as provenance. The target candidate must climb target-domain gates from scratch. diff --git a/README.md b/README.md index b60863b..124bd19 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,9 @@ src/sapiens/ fixtures.py seeded-bias fixture suite (labelled ground truth) calibration.py gate-performance calibration reports confidence.py calibration-gated confidence aggregation (refuses blindly) + review.py L3 panel protocol: roles, objections, multi-round gate + reviewers.py deterministic reference reviewers (four roles) + catchrate.py panel catch-rate scoring over the seeded fixtures ledger.py JSONL hash-chained evidence ledger, L0→L4 transition guard kernel.py domain-neutral DiscoveryKernel; owns all promotions bridge.py cross-domain structure transfer — ALWAYS resets target to L0 @@ -182,7 +185,7 @@ address-space / open-files plus wall-clock timeout, fail-closed), and (key from the environment only, never stored). No real-data adapter ships in Phase 1; tiers are exercised by synthetic adapters and test doubles. -**Phase 2 — shipped** (current package version `0.3.0`): automated L0→L2 +**Phase 2 — shipped** (package version `0.3.0`): automated L0→L2 **validation gates** — L1 statistical sanity checks (determinism, degenerate-score rejection) and L2 declared holdout protocols with explicit leakage controls (dataset collision, (dataset, seed) reuse) — plus a @@ -191,12 +194,21 @@ degenerate), **calibration reports** (catch rate and false-reject rate with sample counts), and **calibration-gated confidence aggregation** that refuses to emit a number without sufficient calibration data. +**Phase 3 — shipped** (current package version `0.4.0`): structured **L3 +review panels** — four role-specialized deterministic reviewers +(statistician, domain theorist, methodologist, devil's advocate), a bounded +multi-round protocol with objection lifecycle tracking (raised / sustained +/ withdrawn) and disagreement gates (sustained MAJOR/BLOCKING objections +reject; MINOR caveats are recorded but non-fatal), panel verdicts recorded +in the ledger as review evidence, and **catch-rate scoring** over the +seeded fixture suite (panel catches 3/3 known-bad, 0 false rejects — exact +for this suite, not an estimate). + Next, in order (see [`ROADMAP.md`](ROADMAP.md)): 1. ~~**Phase 1** — legal/licence gate and adapter hardening~~ **shipped**. 2. ~~**Phase 2** — validation framework v1~~ **shipped**. -3. **Phase 3** — structured L3 review panels (role-specialized reviewers, - multi-round objection tracking, catch-rate scoring). +3. ~~**Phase 3** — structured L3 review panels~~ **shipped**. 4. **Phase 4** — **real domain adapters** — first a clean-room Kepler photometry adapter on public NASA/MAST data; ASTRA / GEODISC / BIODISC / SLATE adapters only after licence and owner review. diff --git a/ROADMAP.md b/ROADMAP.md index 9c6cac8..d5d80f1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -53,11 +53,26 @@ Validation framework v1: documented heuristic (raw pass fraction × demonstrated catch rate) with full provenance. No invented precision. -## Phase 3 — structured L3 review panels +## Phase 3 — shipped (package version 0.4.0) -- Role-specialized reviewer schemas: statistician, domain theorist, methodologist, devil's advocate. -- Multi-round reports, objection tracking, disagreement gates. -- Catch-rate scoring on seeded known-bad and known-good candidates. +Structured L3 review panels: + +- ~~Role-specialized reviewer schemas~~ Shipped: `sapiens.review` — + statistician, domain theorist, methodologist, devil's advocate; typed + approve/object/abstain verdicts with severity-graded objections. +- ~~Multi-round reports, objection tracking, disagreement gates~~ Shipped: + bounded deterministic multi-round protocol; objection lifecycle + (raised/sustained/withdrawn) tracked in every `PanelReport`; sustained + MAJOR/BLOCKING objections reject, MINOR caveats are recorded but + non-fatal (documented gate semantics). Deterministic reference reviewers + in `sapiens.reviewers` re-run the Phase-2 gates independently and hunt + seeded-bias signatures. +- ~~Catch-rate scoring~~ Shipped: `sapiens.catchrate.score_panel` over the + seeded fixture suite — per-role and panel-level catch rates plus + false-reject rate, with an explicit small-sample caveat. +- Kernel integration: with a panel configured, L3 promotion requires panel + approval, and the verdict is recorded in the ledger as review evidence + (no side channel). Without a panel, Phase-2 behaviour is unchanged. ## Phase 4 — real domain adapters diff --git a/VALIDATION.md b/VALIDATION.md index 2c8c1e7..faa51fe 100644 --- a/VALIDATION.md +++ b/VALIDATION.md @@ -53,6 +53,10 @@ GitHub Actions runs Python 3.10, 3.11, and 3.12 with: - L2 gate: holdout discipline, dataset-collision and (dataset, seed) leakage rejection, pass-fraction threshold. - Calibration: fixture labels verified against gate behaviour; report rates match ground truth. - Confidence: refuses without calibration / with thin calibration / without evidence. +- Panels: schema coherence (objections only on OBJECT), multi-round lifecycle (raise/sustain/withdraw/escalate), disagreement gates, budget bounds. +- Reference reviewers: each catches its designed fixture class; clean per-role attribution. +- Catch-rate harness: panel 3/3 known-bad, 0 false rejects on the seeded suite (regression thresholds). +- Kernel panel gate: approval path recorded end-to-end in the ledger; rejection blocks L3 with verdict on record; repeated attempts do not collide. ## Known limits @@ -60,6 +64,7 @@ GitHub Actions runs Python 3.10, 3.11, and 3.12 with: - rlimits bound resource use but are not a full security sandbox; third-party code additionally requires a recorded owner permission. - HMAC checkpoint signatures are symmetric: key possession, not third-party authorship. - Confidence values are documented heuristics (raw pass fraction × demonstrated catch rate), not probability estimates; fixture suites are small and rates are exact only for the fixtures included. +- Reference reviewers are deterministic heuristics, not domain experts; panel approval is procedural evidence, not peer review by scientists. Catch rates are exact for the seeded suite only. - The daemon is cooperative and in-process for SYNTHETIC/CORE tiers; UNTRUSTED adapters run via subprocess isolation. - Synthetic adapters are toy harnesses, not scientific models. - L3 is represented as a bounded `review` evidence gate; full multi-agent review panels are roadmap. diff --git a/pyproject.toml b/pyproject.toml index 7ec1513..3a89c73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "sapiens-discovery" -version = "0.3.0" +version = "0.4.0" description = "Experimental foundation for traceable cross-domain scientific-discovery workflows" readme = "README.md" requires-python = ">=3.10" diff --git a/src/sapiens/__init__.py b/src/sapiens/__init__.py index 1af7229..918233c 100644 --- a/src/sapiens/__init__.py +++ b/src/sapiens/__init__.py @@ -3,6 +3,7 @@ from .adapter import DomainAdapter from .bridge import TransferEnvelope, transfer from .calibration import CalibrationReport, run_calibration +from .catchrate import CatchRateReport, score_panel from .confidence import CalibratedConfidence, UncalibratedError, aggregate_confidence from .fixtures import FixtureKind, SeededFixture, fixture_suite from .kernel import DiscoveryKernel @@ -10,6 +11,8 @@ from .models import AdapterManifest, Candidate, Evidence, EvidenceLevel from .permissions import PermissionEntry, PermissionManifest from .registry import AdapterRegistry, TrustTier +from .review import PanelOutcome, PanelReport, ReviewerRole, ReviewPanel +from .reviewers import reference_panel from .validation import GateVerdict, HoldoutProtocol, ValidationGates __all__ = [ @@ -18,6 +21,7 @@ "CalibratedConfidence", "CalibrationReport", "Candidate", + "CatchRateReport", "DiscoveryKernel", "DomainAdapter", "Evidence", @@ -26,8 +30,12 @@ "FixtureKind", "GateVerdict", "HoldoutProtocol", + "PanelOutcome", + "PanelReport", "PermissionEntry", "PermissionManifest", + "ReviewerRole", + "ReviewPanel", "SeededFixture", "TransferEnvelope", "TrustTier", @@ -35,7 +43,9 @@ "ValidationGates", "aggregate_confidence", "fixture_suite", + "reference_panel", "run_calibration", + "score_panel", "transfer", ] -__version__ = "0.3.0" +__version__ = "0.4.0" diff --git a/src/sapiens/catchrate.py b/src/sapiens/catchrate.py new file mode 100644 index 0000000..20af3e0 --- /dev/null +++ b/src/sapiens/catchrate.py @@ -0,0 +1,114 @@ +"""Catch-rate scoring for review panels (Phase 3). + +Runs a panel over the labelled seeded-bias fixture suite and reports, per +role and for the panel as a whole, how often known-bad candidates draw +substantive (MAJOR/BLOCKING) objections and known-good candidates are +approved. Rates are exact for the fixtures included — they are *not* +estimates of performance on unseeded candidates, and the report says so. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from .calibration import KNOWN_BAD_KINDS +from .fixtures import FixtureKind, SeededFixture +from .review import PanelOutcome, ReviewPanel, Severity + +CAVEAT = ( + "Catch rates are exact for the seeded fixture suite only; they do not " + "estimate performance on unseeded candidates." +) + + +@dataclass(frozen=True) +class RoleCatchStats: + role: str + bad_fixtures_objected: int + bad_fixtures_total: int + + @property + def rate(self) -> float: + return ( + self.bad_fixtures_objected / self.bad_fixtures_total + if self.bad_fixtures_total + else 0.0 + ) + + +@dataclass(frozen=True) +class CatchRateReport: + per_role: tuple[RoleCatchStats, ...] + panel_catches: int + known_bad_total: int + known_good_total: int + known_good_approved: int + caveat: str = CAVEAT + + @property + def panel_catch_rate(self) -> float: + return self.panel_catches / self.known_bad_total if self.known_bad_total else 0.0 + + @property + def false_reject_rate(self) -> float: + if not self.known_good_total: + return 1.0 + return 1.0 - self.known_good_approved / self.known_good_total + + def to_dict(self) -> dict[str, object]: + return { + "per_role": { + stats.role: { + "bad_fixtures_objected": stats.bad_fixtures_objected, + "bad_fixtures_total": stats.bad_fixtures_total, + "rate": stats.rate, + } + for stats in self.per_role + }, + "panel_catches": self.panel_catches, + "known_bad_total": self.known_bad_total, + "panel_catch_rate": self.panel_catch_rate, + "known_good_total": self.known_good_total, + "known_good_approved": self.known_good_approved, + "false_reject_rate": self.false_reject_rate, + "caveat": self.caveat, + } + + +def score_panel( + panel: ReviewPanel, fixtures: tuple[SeededFixture, ...], *, seed: int = 0 +) -> CatchRateReport: + """Convene the panel over every fixture and tally against labels.""" + roles = [reviewer.role.value for reviewer in panel.reviewers] + role_catches = {role: 0 for role in roles} + bad_total = catches = good_total = good_approved = 0 + for fixture in fixtures: + evidence = fixture.internal + fixture.replication + report = panel.convene(fixture.candidate, evidence, seed=seed) + if fixture.kind == FixtureKind.KNOWN_GOOD: + good_total += 1 + if report.outcome == PanelOutcome.APPROVED: + good_approved += 1 + elif fixture.kind in KNOWN_BAD_KINDS: + bad_total += 1 + if report.outcome == PanelOutcome.REJECTED: + catches += 1 + substantive_roles = { + objection.role.value + for objection in ( + obj for rnd in report.rounds for obj in rnd.objections() + ) + if objection.severity in (Severity.MAJOR, Severity.BLOCKING) + } + for role in substantive_roles: + if role in role_catches: + role_catches[role] += 1 + return CatchRateReport( + tuple( + RoleCatchStats(role, role_catches[role], bad_total) for role in sorted(role_catches) + ), + catches, + bad_total, + good_total, + good_approved, + ) diff --git a/src/sapiens/kernel.py b/src/sapiens/kernel.py index 8d6d023..40070e7 100644 --- a/src/sapiens/kernel.py +++ b/src/sapiens/kernel.py @@ -8,6 +8,7 @@ from .ledger import EvidenceLedger from .models import Candidate, Evidence, EvidenceLevel from .registry import AdapterRegistry, TrustTier +from .review import PanelReport, ReviewPanel from .validation import GateVerdict, ValidationGates _STAGE_BY_LEVEL = { @@ -23,11 +24,14 @@ def __init__( ledger: EvidenceLedger, registry: AdapterRegistry | None = None, validation: ValidationGates | None = None, + panel: ReviewPanel | None = None, ) -> None: self.ledger = ledger self.registry = registry if registry is not None else AdapterRegistry() self.validation = validation + self.panel = panel self.gate_log: list[GateVerdict] = [] + self.panel_log: list[PanelReport] = [] def register(self, candidate: Candidate, *, transferred_from: str | None = None) -> None: self.ledger.record_candidate(candidate.candidate_id, transferred_from=transferred_from) @@ -67,9 +71,44 @@ def validate_next( return current if not self._passes_validation_gates(candidate, target): return current + if target == EvidenceLevel.L3 and self.panel is not None: + panel_evidence_id = self._convene_panel(candidate, seed=seed) + if panel_evidence_id is None: + return current # panel rejected; verdict already on the ledger + refs.append(panel_evidence_id) self.ledger.promote(candidate.candidate_id, target, tuple(refs)) return target + def _convene_panel(self, candidate: Candidate, *, seed: int) -> str | None: + """Convene the L3 panel and record its verdict as review evidence. + + Returns the panel evidence id on approval, None on rejection. The + verdict is ledger-recorded either way — no side channels. + """ + from .review import PanelOutcome + + recorded = self._evidence_for(candidate.candidate_id) + report = self.panel.convene(candidate, recorded, seed=seed) # type: ignore[union-attr] + self.panel_log.append(report) + approved = report.outcome == PanelOutcome.APPROVED + # Unique per attempt: an identical re-convened report must not collide + # with the previously recorded panel verdict (ledger forbids id reuse). + evidence_id = f"panel-{report.report_id}-e{len(self.ledger.events())}" + self.ledger.record_evidence( + Evidence( + evidence_id, + candidate.candidate_id, + "review", + approved, + "review-panel-v1", + "panel-transcript", + seed, + None, + {"report": report.to_dict()}, + ) + ) + return evidence_id if approved else None + def _evidence_for(self, candidate_id: str) -> tuple[Evidence, ...]: items: list[Evidence] = [] for event in self.ledger.events(): diff --git a/src/sapiens/review.py b/src/sapiens/review.py new file mode 100644 index 0000000..7ccf048 --- /dev/null +++ b/src/sapiens/review.py @@ -0,0 +1,270 @@ +"""Structured L3 review panels (Phase 3). + +A panel is a bounded, deterministic, multi-round protocol over typed +reviewer verdicts. Four roles are defined — statistician, domain theorist, +methodologist, devil's advocate — each emitting approve / object / abstain +verdicts with severity-graded objections and rationales. + +Protocol (deliberately strict, documented here and nowhere else): + +1. **Round 1** — every reviewer verdicts independently over the candidate's + recorded evidence. +2. **Objection lifecycle** — an objection is RAISED in its round; in each + later round its raiser must re-affirm it (SUSTAINED) or drop it + (WITHDRAWN). Objections cannot silently vanish: the transcript records + every transition. +3. **Rebuttal rounds** — while any objection stands and the round budget + lasts, the panel convenes another round. Reference reviewers escalate a + re-affirmed MAJOR to BLOCKING; MINOR findings stay MINOR. +4. **Disagreement gate** — the panel approves only if no MAJOR or BLOCKING + objection is sustained in the final round. MINOR objections are recorded + as standing caveats in the transcript but do not block: the gate is + strict on substance, tolerant of caveats. L3 is the last automated rung + before the human L4 gate, so any substantive disagreement rejects. + +Panels produce :class:`PanelReport` data — the kernel may record it as +review evidence, but the report itself is the verdict, never a promotion. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from enum import Enum +from typing import Protocol, runtime_checkable + +from .models import Candidate, Evidence + + +class ReviewerRole(Enum): + STATISTICIAN = "statistician" + DOMAIN_THEORIST = "domain-theorist" + METHODOLOGIST = "methodologist" + DEVILS_ADVOCATE = "devils-advocate" + + +class VerdictKind(Enum): + APPROVE = "approve" + OBJECT = "object" + ABSTAIN = "abstain" + + +class Severity(Enum): + MINOR = "minor" + MAJOR = "major" + BLOCKING = "blocking" + + +class ObjectionStatus(Enum): + RAISED = "raised" + SUSTAINED = "sustained" + WITHDRAWN = "withdrawn" + + +class PanelOutcome(Enum): + APPROVED = "approved" + REJECTED = "rejected" + + +@dataclass(frozen=True) +class Objection: + objection_id: str + role: ReviewerRole + severity: Severity + text: str + raised_round: int + + def __post_init__(self) -> None: + if not self.objection_id or not self.text: + raise ValueError("objections require an id and text") + if self.raised_round < 1: + raise ValueError("rounds are 1-indexed") + + @property + def key(self) -> str: + """Identity across rounds: same reviewer, same concern.""" + return hashlib.sha256(f"{self.role.value}|{self.text}".encode()).hexdigest()[:16] + + +@dataclass(frozen=True) +class ReviewerVerdict: + role: ReviewerRole + verdict: VerdictKind + rationale: str + objections: tuple[Objection, ...] = () + + def __post_init__(self) -> None: + if self.verdict != VerdictKind.OBJECT and self.objections: + raise ValueError("only an objecting verdict carries objections") + if self.verdict == VerdictKind.OBJECT and not self.objections: + raise ValueError("an objecting verdict must raise at least one objection") + if not self.rationale: + raise ValueError("every verdict requires a rationale") + for objection in self.objections: + if objection.role != self.role: + raise ValueError("objections must belong to their verdict's role") + + +@dataclass(frozen=True) +class ReviewRound: + round_number: int + verdicts: tuple[ReviewerVerdict, ...] + + def objections(self) -> tuple[Objection, ...]: + return tuple(obj for verdict in self.verdicts for obj in verdict.objections) + + +@runtime_checkable +class Reviewer(Protocol): + """A role-specialized, deterministic reviewer. Pure function of its inputs.""" + + @property + def role(self) -> ReviewerRole: ... + + def review( + self, + candidate: Candidate, + evidence: tuple[Evidence, ...], + *, + round_number: int, + prior_objections: tuple[Objection, ...], + seed: int, + ) -> ReviewerVerdict: ... + + +@dataclass(frozen=True) +class PanelReport: + candidate_id: str + outcome: PanelOutcome + rounds: tuple[ReviewRound, ...] + sustained_blocking: tuple[Objection, ...] + withdrawn: tuple[Objection, ...] + lifecycle: tuple[tuple[str, ObjectionStatus], ...] = () + + def to_dict(self) -> dict[str, object]: + return { + "candidate_id": self.candidate_id, + "outcome": self.outcome.value, + "rounds": [ + { + "round_number": r.round_number, + "verdicts": [ + { + "role": v.role.value, + "verdict": v.verdict.value, + "rationale": v.rationale, + "objections": [ + { + "objection_id": o.objection_id, + "severity": o.severity.value, + "text": o.text, + "raised_round": o.raised_round, + } + for o in v.objections + ], + } + for v in r.verdicts + ], + } + for r in self.rounds + ], + "sustained_blocking": [o.text for o in self.sustained_blocking], + "withdrawn": [o.text for o in self.withdrawn], + "lifecycle": {key: status.value for key, status in self.lifecycle}, + } + + @property + def report_id(self) -> str: + canonical = json.dumps(self.to_dict(), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode()).hexdigest()[:16] + + +class ReviewPanel: + """Convenes reviewers over bounded multi-round protocols. Deterministic.""" + + def __init__(self, reviewers: tuple[Reviewer, ...], *, max_rounds: int = 3) -> None: + if not reviewers: + raise ValueError("a panel requires at least one reviewer") + roles = [reviewer.role for reviewer in reviewers] + if len(set(roles)) != len(roles): + raise ValueError("reviewer roles must be unique on a panel") + if max_rounds < 1: + raise ValueError("max_rounds must be positive") + self._reviewers = reviewers + self._max_rounds = max_rounds + + @property + def reviewers(self) -> tuple[Reviewer, ...]: + return self._reviewers + + def convene( + self, candidate: Candidate, evidence: tuple[Evidence, ...], *, seed: int + ) -> PanelReport: + rounds: list[ReviewRound] = [] + active: dict[str, Objection] = {} # key -> latest objection form + withdrawn: list[Objection] = [] + for round_number in range(1, self._max_rounds + 1): + prior = tuple(active.values()) + verdicts = tuple( + reviewer.review( + candidate, + evidence, + round_number=round_number, + prior_objections=prior, + seed=seed, + ) + for reviewer in self._reviewers + ) + rounds.append(ReviewRound(round_number, verdicts)) + current: dict[str, Objection] = {} + for verdict in verdicts: + for objection in verdict.objections: + key = objection.key + if key in active: + current[key] = Objection( + objection.objection_id, + objection.role, + objection.severity, + objection.text, + active[key].raised_round, + ) + else: + current[key] = objection + for key, previous in active.items(): + if key not in current: + withdrawn.append(previous) + active = current + # Continue convening while any objection stands: rebuttal rounds + # give raisers the chance to sustain, escalate, or withdraw. + # Termination is bounded by the round budget. + if not active: + break # clean consensus (or everything withdrawn) + final = rounds[-1] + sustained_blocking = tuple( + obj for obj in active.values() if obj.severity == Severity.BLOCKING + ) + sustained_substantive = tuple( + obj + for obj in active.values() + if obj.severity in (Severity.MAJOR, Severity.BLOCKING) + ) + approved = not sustained_substantive + lifecycle: list[tuple[str, ObjectionStatus]] = [] + for key, obj in active.items(): + status = ( + ObjectionStatus.RAISED + if obj.raised_round == final.round_number + else ObjectionStatus.SUSTAINED + ) + lifecycle.append((key, status)) + for obj in withdrawn: + lifecycle.append((obj.key, ObjectionStatus.WITHDRAWN)) + return PanelReport( + candidate.candidate_id, + PanelOutcome.APPROVED if approved else PanelOutcome.REJECTED, + tuple(rounds), + sustained_blocking, + tuple(withdrawn), + tuple(lifecycle), + ) diff --git a/src/sapiens/reviewers.py b/src/sapiens/reviewers.py new file mode 100644 index 0000000..49c88b5 --- /dev/null +++ b/src/sapiens/reviewers.py @@ -0,0 +1,347 @@ +"""Deterministic reference reviewers for the L3 panel (Phase 3). + +Four role-specialized reviewers, each a pure function of (candidate, +evidence, round, prior objections, seed). They overlap the Phase-2 gates +deliberately: the statistician re-runs them as an independent check, and +the devil's advocate hunts planted-bias signatures directly. Redundancy is +the point of a panel. + +Escalation policy (uniform): findings are recomputed every round; a MAJOR +finding re-affirmed in round 2 or later escalates to BLOCKING. MINOR stays +MINOR. The seed is accepted for interface uniformity; every check here is +deterministic without it. +""" + +from __future__ import annotations + +from .calibration import CalibrationReport +from .models import Candidate, Evidence +from .review import ( + Objection, + ReviewerRole, + ReviewerVerdict, + ReviewPanel, + Severity, + VerdictKind, +) +from .validation import ( + HoldoutProtocol, + check_internal_consistency, + check_replication, +) + + +def _objection( + role: ReviewerRole, severity: Severity, text: str, round_number: int, index: int +) -> Objection: + return Objection( + f"{role.value}-r{round_number}-{index}", + role, + severity, + text, + round_number, + ) + + +def _escalate(severity: Severity, round_number: int) -> Severity: + if severity == Severity.MAJOR and round_number >= 2: + return Severity.BLOCKING + return severity + + +class StatisticianReviewer: + """Re-runs the Phase-2 gates as an independent statistical check.""" + + role = ReviewerRole.STATISTICIAN + + def __init__( + self, + protocol: HoldoutProtocol | None = None, + calibration: CalibrationReport | None = None, + ) -> None: + self._protocol = protocol + self._calibration = calibration + + def review( + self, + candidate: Candidate, + evidence: tuple[Evidence, ...], + *, + round_number: int, + prior_objections: tuple[Objection, ...], + seed: int, + ) -> ReviewerVerdict: + objections: list[Objection] = [] + internal = tuple(item for item in evidence if item.kind == "internal") + replication = tuple(item for item in evidence if item.kind == "replication") + l1 = check_internal_consistency(evidence) + if not l1.passed: + for reason in l1.reasons: + objections.append( + _objection( + self.role, Severity.BLOCKING, f"L1 gate: {reason}", round_number, 0 + ) + ) + if self._protocol is not None: + l2 = check_replication(internal, replication, self._protocol) + if not l2.passed: + for reason in l2.reasons: + objections.append( + _objection( + self.role, Severity.BLOCKING, f"L2 gate: {reason}", round_number, 1 + ) + ) + if ( + self._calibration is not None + and self._calibration.known_bad_total > 0 + and self._calibration.catch_rate < 1.0 + ): + objections.append( + _objection( + self.role, + Severity.MINOR, + f"calibration report {self._calibration.report_id} shows gates catch " + f"only {self._calibration.catch_rate:.2f} of known-bad fixtures", + round_number, + 2, + ) + ) + if objections: + return ReviewerVerdict( + self.role, + VerdictKind.OBJECT, + f"statistical checks found {len(objections)} issue(s)", + tuple(objections), + ) + return ReviewerVerdict( + self.role, VerdictKind.APPROVE, "statistical gates satisfied" + ) + + +class MethodologistReviewer: + """Protocol and dataset hygiene: consistency within stages, no cross-stage reuse.""" + + role = ReviewerRole.METHODOLOGIST + + def review( + self, + candidate: Candidate, + evidence: tuple[Evidence, ...], + *, + round_number: int, + prior_objections: tuple[Objection, ...], + seed: int, + ) -> ReviewerVerdict: + objections: list[Objection] = [] + if not evidence: + objections.append( + _objection( + self.role, Severity.BLOCKING, "no evidence to review", round_number, 0 + ) + ) + by_stage: dict[str, list[Evidence]] = {} + for item in evidence: + by_stage.setdefault(item.kind, []).append(item) + index = 1 + for stage, items in sorted(by_stage.items()): + protocols = {item.protocol for item in items} + if len(protocols) > 1: + objections.append( + _objection( + self.role, + Severity.BLOCKING, + f"stage {stage!r} mixes protocols {sorted(protocols)}", + round_number, + index, + ) + ) + index += 1 + # Leakage signal: an internal-stage (train) dataset reappearing in a + # later stage. Holdout reuse across replication/review is legitimate. + internal_datasets = {item.dataset for item in by_stage.get("internal", [])} + later_datasets = { + item.dataset + for stage, items in by_stage.items() + if stage != "internal" + for item in items + } + reused = internal_datasets & later_datasets + if reused: + objections.append( + _objection( + self.role, + Severity.BLOCKING, + f"train dataset(s) {sorted(reused)} reappear beyond the internal " + "stage — leakage signal", + round_number, + index, + ) + ) + if objections: + return ReviewerVerdict( + self.role, + VerdictKind.OBJECT, + f"methodology checks found {len(objections)} issue(s)", + tuple(objections), + ) + return ReviewerVerdict( + self.role, VerdictKind.APPROVE, "protocol and dataset hygiene satisfied" + ) + + +class DomainTheoristReviewer: + """Claim coherence: the claim must be expressible in the domain vocabulary.""" + + role = ReviewerRole.DOMAIN_THEORIST + + def __init__(self, vocabulary: tuple[str, ...]) -> None: + if not vocabulary: + raise ValueError("domain theorist requires a vocabulary") + self._vocabulary = vocabulary + + def review( + self, + candidate: Candidate, + evidence: tuple[Evidence, ...], + *, + round_number: int, + prior_objections: tuple[Objection, ...], + seed: int, + ) -> ReviewerVerdict: + haystack = " ".join( + [candidate.claim, *(str(key) for key in candidate.parameters)] + ).lower() + hits = [term for term in self._vocabulary if term.lower() in haystack] + if not hits: + severity = _escalate(Severity.MAJOR, round_number) + return ReviewerVerdict( + self.role, + VerdictKind.OBJECT, + "claim is not expressible in the declared domain vocabulary", + ( + _objection( + self.role, + severity, + f"claim references none of {self._vocabulary}", + round_number, + 0, + ), + ), + ) + return ReviewerVerdict( + self.role, + VerdictKind.APPROVE, + f"claim coheres with domain vocabulary ({', '.join(hits)})", + ) + + +class DevilsAdvocateReviewer: + """Adversarial hunt for seeded-bias signatures. Never satisfied by default.""" + + role = ReviewerRole.DEVILS_ADVOCATE + + def review( + self, + candidate: Candidate, + evidence: tuple[Evidence, ...], + *, + round_number: int, + prior_objections: tuple[Objection, ...], + seed: int, + ) -> ReviewerVerdict: + objections: list[Objection] = [] + # Signature 1: constant scores across independent seeds (degenerate). + by_stage: dict[str, list[Evidence]] = {} + for item in evidence: + by_stage.setdefault(item.kind, []).append(item) + index = 0 + for stage, items in sorted(by_stage.items()): + seeds = {item.seed for item in items} + scores = [item.score for item in items if item.score is not None] + if len(seeds) >= 3 and len(scores) >= 3 and len(set(scores)) == 1: + objections.append( + _objection( + self.role, + Severity.BLOCKING, + f"constant score {scores[0]} across {len(seeds)} seeds in stage " + f"{stage!r} — degenerate signature", + round_number, + index, + ) + ) + index += 1 + # Signature 2: a train (internal-stage) dataset reappearing in a later + # stage (leakage). Holdout reuse across replication/review is fine. + internal_datasets = {item.dataset for item in by_stage.get("internal", [])} + for stage, items in sorted(by_stage.items()): + if stage == "internal": + continue + overlap = internal_datasets & {item.dataset for item in items} + if overlap: + objections.append( + _objection( + self.role, + Severity.BLOCKING, + f"train dataset(s) {sorted(overlap)} reappear in stage " + f"{stage!r} — leakage signature", + round_number, + index, + ) + ) + index += 1 + # Signature 3: everything perfect (all scores 1.0) — too clean to trust. + all_scores = [item.score for item in evidence if item.score is not None] + if all_scores and all(score == 1.0 for score in all_scores): + objections.append( + _objection( + self.role, + _escalate(Severity.MAJOR, round_number), + "every score is exactly 1.0 — implausibly perfect", + round_number, + index, + ) + ) + index += 1 + # Signature 4: thin stages (a single run decides a stage). Caveat only. + thin = [stage for stage, items in sorted(by_stage.items()) if len(items) == 1] + if thin: + objections.append( + _objection( + self.role, + Severity.MINOR, + f"stage(s) {thin} rest on a single run", + round_number, + index, + ) + ) + if objections: + return ReviewerVerdict( + self.role, + VerdictKind.OBJECT, + f"adversarial hunt found {len(objections)} signature(s)", + tuple(objections), + ) + return ReviewerVerdict( + self.role, + VerdictKind.APPROVE, + "adversarial hunt found no seeded-bias signature", + ) + + +def reference_panel( + vocabulary: tuple[str, ...], + protocol: HoldoutProtocol | None = None, + calibration: CalibrationReport | None = None, + *, + max_rounds: int = 3, +) -> ReviewPanel: + """A panel of the four deterministic reference reviewers.""" + return ReviewPanel( + ( + StatisticianReviewer(protocol, calibration), + DomainTheoristReviewer(vocabulary), + MethodologistReviewer(), + DevilsAdvocateReviewer(), + ), + max_rounds=max_rounds, + ) diff --git a/tests/test_catchrate.py b/tests/test_catchrate.py new file mode 100644 index 0000000..a3a594b --- /dev/null +++ b/tests/test_catchrate.py @@ -0,0 +1,53 @@ +"""Catch-rate scoring harness (Phase 3) with regression thresholds.""" + +import json + +from sapiens.catchrate import score_panel +from sapiens.fixtures import fixture_suite +from sapiens.reviewers import reference_panel +from sapiens.validation import synthetic_holdout_protocol + +VOCAB = ("signal", "score", "data", "training", "replication", "holdout") + + +def make_report(): + panel = reference_panel(VOCAB, synthetic_holdout_protocol()) + return score_panel(panel, fixture_suite(), seed=0) + + +def test_panel_catches_every_seeded_bad_fixture(): + # Regression threshold on this suite: exact, not estimated (see caveat). + report = make_report() + assert report.known_bad_total == 3 + assert report.panel_catches == 3 + assert report.panel_catch_rate == 1.0 + + +def test_panel_does_not_false_reject_known_good(): + report = make_report() + assert report.known_good_total == 1 + assert report.known_good_approved == 1 + assert report.false_reject_rate == 0.0 + + +def test_per_role_attribution_is_honest(): + report = make_report() + stats = {entry.role: entry for entry in report.per_role} + assert set(stats) == { + "statistician", + "domain-theorist", + "methodologist", + "devils-advocate", + } + # The theorist's job is coherence, not bias: zero catches is the honest + # result, and the report must show it rather than inflate roles. + assert stats["domain-theorist"].bad_fixtures_objected == 0 + assert stats["statistician"].bad_fixtures_objected == 3 + assert stats["devils-advocate"].bad_fixtures_objected >= 2 + + +def test_report_serialisable_with_caveat(): + report = make_report() + blob = json.dumps(report.to_dict()) + assert "do not estimate" in report.caveat + assert "statistician" in blob diff --git a/tests/test_kernel_panel.py b/tests/test_kernel_panel.py new file mode 100644 index 0000000..6446ae9 --- /dev/null +++ b/tests/test_kernel_panel.py @@ -0,0 +1,132 @@ +"""Kernel + L3 review panel integration (Phase 3).""" + +import json + +from sapiens.adapters import SyntheticPhotometryAdapter +from sapiens.budget import ExecutionContext +from sapiens.kernel import DiscoveryKernel +from sapiens.ledger import EvidenceLedger +from sapiens.models import AdapterManifest, Candidate, Evidence, EvidenceLevel +from sapiens.review import PanelOutcome +from sapiens.reviewers import reference_panel +from sapiens.validation import synthetic_holdout_protocol + + +def ctx(): + return ExecutionContext(max_steps=20, max_seconds=20.0) + + +def climb(kernel, adapter, candidate, seeds=(40, 41, 42)): + reached = EvidenceLevel.L0 + for seed in seeds: + reached = kernel.validate_next(adapter, candidate, seed=seed, context=ctx()) + return reached + + +def test_panel_approval_path_recorded_end_to_end(tmp_path): + adapter = SyntheticPhotometryAdapter() + panel = reference_panel(adapter.manifest.vocabulary, synthetic_holdout_protocol()) + kernel = DiscoveryKernel(EvidenceLedger(tmp_path / "events.jsonl"), panel=panel) + candidate = adapter.propose(seed=5, limit=1)[0] + kernel.register(candidate) + assert climb(kernel, adapter, candidate) == EvidenceLevel.L3 + assert kernel.panel_log[-1].outcome == PanelOutcome.APPROVED + # The verdict lives in the ledger as review evidence — no side channel. + panel_records = [ + event + for event in kernel.ledger.events() + if event.kind == "evidence" and event.payload["dataset"] == "panel-transcript" + ] + assert len(panel_records) == 1 + assert panel_records[0].payload["passed"] is True + assert panel_records[0].payload["details"]["report"]["outcome"] == "approved" + assert kernel.ledger.verify() + + +class PerfectScoreAdapter: + """Everything exactly 1.0: passes naive gates, trips the devil's advocate.""" + + manifest = AdapterManifest("perfect", "1", "perfect-domain", ("signal",)) + + def propose(self, *, seed: int, limit: int): + return (Candidate("cand-perfect", "perfect-domain", "a signal claim"),) + + def validate(self, candidate, *, stage: str, seed: int, context): + context.checkpoint() + dataset = "synthetic-train" if stage == "internal" else "synthetic-holdout" + return ( + Evidence( + f"ev-{stage}-{seed}", + candidate.candidate_id, + stage, + True, + f"perfect-{stage}-v1", + dataset, + seed, + 1.0, + ), + ) + + def import_structure(self, structure, *, candidate_id: str): + return Candidate(candidate_id, "perfect-domain", "claim") + + +def test_panel_rejection_blocks_l3_and_records_verdict(tmp_path): + adapter = PerfectScoreAdapter() + panel = reference_panel(adapter.manifest.vocabulary, synthetic_holdout_protocol()) + kernel = DiscoveryKernel(EvidenceLedger(tmp_path / "events.jsonl"), panel=panel) + candidate = adapter.propose(seed=1, limit=1)[0] + kernel.register(candidate) + assert climb(kernel, adapter, candidate) == EvidenceLevel.L2 + assert kernel.ledger.state("cand-perfect").level == EvidenceLevel.L2 + report = kernel.panel_log[-1] + assert report.outcome == PanelOutcome.REJECTED + assert len(report.rounds) == 3 # escalated objection held through the budget + assert any("implausibly perfect" in o.text for o in report.sustained_blocking) + panel_records = [ + event + for event in kernel.ledger.events() + if event.kind == "evidence" and event.payload["dataset"] == "panel-transcript" + ] + assert len(panel_records) == 1 + assert panel_records[0].payload["passed"] is False + assert kernel.ledger.verify() + + +def test_repeated_rejection_attempts_do_not_collide(tmp_path): + adapter = PerfectScoreAdapter() + panel = reference_panel(adapter.manifest.vocabulary, synthetic_holdout_protocol()) + kernel = DiscoveryKernel(EvidenceLedger(tmp_path / "events.jsonl"), panel=panel) + candidate = adapter.propose(seed=1, limit=1)[0] + kernel.register(candidate) + assert kernel.validate_next(adapter, candidate, seed=40, context=ctx()) == EvidenceLevel.L1 + assert kernel.validate_next(adapter, candidate, seed=41, context=ctx()) == EvidenceLevel.L2 + assert kernel.validate_next(adapter, candidate, seed=42, context=ctx()) == EvidenceLevel.L2 + # A second L3 attempt with a fresh seed must not hit duplicate evidence ids. + assert kernel.validate_next(adapter, candidate, seed=43, context=ctx()) == EvidenceLevel.L2 + assert kernel.ledger.verify() + reports = [ + event + for event in kernel.ledger.events() + if event.kind == "evidence" and event.payload["dataset"] == "panel-transcript" + ] + assert len(reports) == 2 + + +def test_kernel_without_panel_unchanged(tmp_path): + adapter = SyntheticPhotometryAdapter() + kernel = DiscoveryKernel(EvidenceLedger(tmp_path / "events.jsonl")) + candidate = adapter.propose(seed=5, limit=1)[0] + kernel.register(candidate) + assert climb(kernel, adapter, candidate) == EvidenceLevel.L3 + assert kernel.panel_log == [] + + +def test_panel_report_blob_is_json_safe(tmp_path): + adapter = PerfectScoreAdapter() + panel = reference_panel(adapter.manifest.vocabulary, synthetic_holdout_protocol()) + kernel = DiscoveryKernel(EvidenceLedger(tmp_path / "events.jsonl"), panel=panel) + candidate = adapter.propose(seed=1, limit=1)[0] + kernel.register(candidate) + climb(kernel, adapter, candidate) + json.dumps(kernel.panel_log[-1].to_dict()) diff --git a/tests/test_review.py b/tests/test_review.py new file mode 100644 index 0000000..369e87f --- /dev/null +++ b/tests/test_review.py @@ -0,0 +1,209 @@ +"""Panel protocol + schema tests (scripted reviewer doubles).""" + +import json + +import pytest + +from sapiens.models import Candidate, Evidence +from sapiens.review import ( + Objection, + ObjectionStatus, + PanelOutcome, + ReviewerRole, + ReviewerVerdict, + ReviewPanel, + Severity, + VerdictKind, +) + +CANDIDATE = Candidate("cand-p", "dom", "a claim about signal") +EVIDENCE = (Evidence("e1", "cand-p", "internal", True, "p", "d", 1, 0.9),) + + +class Scripted: + """Reviewer double following a per-round script of (verdict, [(severity, text)]).""" + + def __init__(self, role, script): + self.role = role + self._script = script + + def review(self, candidate, evidence, *, round_number, prior_objections, seed): + kind, objections = self._script.get(round_number, (VerdictKind.APPROVE, [])) + return ReviewerVerdict( + self.role, + kind, + f"scripted round {round_number}", + tuple( + Objection( + f"{self.role.value}-r{round_number}-{i}", + self.role, + severity, + text, + round_number, + ) + for i, (severity, text) in enumerate(objections) + ), + ) + + +def approve_all(role): + return Scripted(role, {}) + + +class TestSchema: + def test_approve_carries_no_objections(self): + with pytest.raises(ValueError): + ReviewerVerdict( + ReviewerRole.STATISTICIAN, + VerdictKind.APPROVE, + "rationale", + (Objection("o1", ReviewerRole.STATISTICIAN, Severity.MINOR, "x", 1),), + ) + + def test_object_requires_objections(self): + with pytest.raises(ValueError): + ReviewerVerdict(ReviewerRole.STATISTICIAN, VerdictKind.OBJECT, "rationale") + + def test_abstain_carries_no_objections(self): + with pytest.raises(ValueError): + ReviewerVerdict( + ReviewerRole.STATISTICIAN, + VerdictKind.ABSTAIN, + "rationale", + (Objection("o1", ReviewerRole.STATISTICIAN, Severity.MINOR, "x", 1),), + ) + + def test_rationale_required(self): + with pytest.raises(ValueError): + ReviewerVerdict(ReviewerRole.STATISTICIAN, VerdictKind.APPROVE, "") + + def test_objection_role_must_match(self): + with pytest.raises(ValueError): + ReviewerVerdict( + ReviewerRole.STATISTICIAN, + VerdictKind.OBJECT, + "rationale", + (Objection("o1", ReviewerRole.METHODOLOGIST, Severity.MINOR, "x", 1),), + ) + + def test_objection_needs_text_and_valid_round(self): + with pytest.raises(ValueError): + Objection("o1", ReviewerRole.STATISTICIAN, Severity.MINOR, "", 1) + with pytest.raises(ValueError): + Objection("o1", ReviewerRole.STATISTICIAN, Severity.MINOR, "x", 0) + + def test_panel_requires_unique_roles_and_budget(self): + with pytest.raises(ValueError): + ReviewPanel(()) + with pytest.raises(ValueError): + ReviewPanel((approve_all(ReviewerRole.STATISTICIAN),) * 2) + with pytest.raises(ValueError): + ReviewPanel((approve_all(ReviewerRole.STATISTICIAN),), max_rounds=0) + + +class TestProtocol: + def panel(self, *reviewers, max_rounds=3): + return ReviewPanel(reviewers, max_rounds=max_rounds) + + def test_clean_consensus_approves_in_one_round(self): + panel = self.panel( + approve_all(ReviewerRole.STATISTICIAN), approve_all(ReviewerRole.METHODOLOGIST) + ) + report = panel.convene(CANDIDATE, EVIDENCE, seed=1) + assert report.outcome == PanelOutcome.APPROVED + assert len(report.rounds) == 1 + + def test_minor_raised_then_withdrawn_approves(self): + wavering = Scripted( + ReviewerRole.STATISTICIAN, + {1: (VerdictKind.OBJECT, [(Severity.MINOR, "caveat")])}, + ) + panel = self.panel(wavering, approve_all(ReviewerRole.METHODOLOGIST)) + report = panel.convene(CANDIDATE, EVIDENCE, seed=1) + assert report.outcome == PanelOutcome.APPROVED + assert len(report.rounds) == 2 + assert [status for _, status in report.lifecycle] == [ObjectionStatus.WITHDRAWN] + assert report.withdrawn[0].text == "caveat" + + def test_major_escalates_and_rejects(self): + # The raiser holds the objection through the whole budget: MAJOR in + # round 1, escalated to BLOCKING in the rebuttals, never withdrawn. + escalating = Scripted( + ReviewerRole.STATISTICIAN, + { + 1: (VerdictKind.OBJECT, [(Severity.MAJOR, "suspicious")]), + 2: (VerdictKind.OBJECT, [(Severity.BLOCKING, "suspicious")]), + 3: (VerdictKind.OBJECT, [(Severity.BLOCKING, "suspicious")]), + }, + ) + panel = self.panel(escalating, approve_all(ReviewerRole.METHODOLOGIST)) + report = panel.convene(CANDIDATE, EVIDENCE, seed=1) + assert report.outcome == PanelOutcome.REJECTED + assert len(report.rounds) == 3 + assert report.sustained_blocking[0].text == "suspicious" + statuses = dict(report.lifecycle) + assert list(statuses.values()) == [ObjectionStatus.SUSTAINED] + + def test_escalated_objection_late_withdrawal_approves(self): + # Escalated to BLOCKING in round 2, then withdrawn in round 3: the + # lifecycle must record the withdrawal and the panel approves. + relenting = Scripted( + ReviewerRole.STATISTICIAN, + { + 1: (VerdictKind.OBJECT, [(Severity.MAJOR, "suspicious")]), + 2: (VerdictKind.OBJECT, [(Severity.BLOCKING, "suspicious")]), + }, + ) + panel = self.panel(relenting, approve_all(ReviewerRole.METHODOLOGIST)) + report = panel.convene(CANDIDATE, EVIDENCE, seed=1) + assert report.outcome == PanelOutcome.APPROVED + assert len(report.rounds) == 3 + assert report.withdrawn[0].text == "suspicious" + assert report.withdrawn[0].severity == Severity.BLOCKING + + def test_blocking_withdrawn_in_rebuttal_approves(self): + relenting = Scripted( + ReviewerRole.DEVILS_ADVOCATE, + {1: (VerdictKind.OBJECT, [(Severity.BLOCKING, "fatal flaw")])}, + ) + panel = self.panel(relenting, approve_all(ReviewerRole.METHODOLOGIST)) + report = panel.convene(CANDIDATE, EVIDENCE, seed=1) + assert report.outcome == PanelOutcome.APPROVED + assert len(report.rounds) == 2 + assert report.withdrawn[0].text == "fatal flaw" + + def test_residual_minor_is_caveat_not_fatal(self): + stubborn_minor = Scripted( + ReviewerRole.METHODOLOGIST, + { + 1: (VerdictKind.OBJECT, [(Severity.MINOR, "nit")]), + 2: (VerdictKind.OBJECT, [(Severity.MINOR, "nit")]), + 3: (VerdictKind.OBJECT, [(Severity.MINOR, "nit")]), + }, + ) + panel = self.panel(stubborn_minor, approve_all(ReviewerRole.STATISTICIAN)) + report = panel.convene(CANDIDATE, EVIDENCE, seed=1) + assert report.outcome == PanelOutcome.APPROVED + assert len(report.rounds) == 3 # budget exhausted on the caveat + + def test_sustained_major_at_budget_rejects(self): + stubborn_major = Scripted( + ReviewerRole.METHODOLOGIST, + { + 1: (VerdictKind.OBJECT, [(Severity.MAJOR, "concern")]), + 2: (VerdictKind.OBJECT, [(Severity.MAJOR, "concern")]), + 3: (VerdictKind.OBJECT, [(Severity.MAJOR, "concern")]), + }, + ) + panel = self.panel(stubborn_major, approve_all(ReviewerRole.STATISTICIAN)) + report = panel.convene(CANDIDATE, EVIDENCE, seed=1) + assert report.outcome == PanelOutcome.REJECTED + assert len(report.rounds) == 3 + + def test_report_serialisable_and_deterministic_id(self): + panel = self.panel(approve_all(ReviewerRole.STATISTICIAN)) + first = panel.convene(CANDIDATE, EVIDENCE, seed=1) + second = panel.convene(CANDIDATE, EVIDENCE, seed=1) + assert first.report_id == second.report_id + blob = json.dumps(first.to_dict()) + assert "cand-p" in blob and "approved" in blob diff --git a/tests/test_reviewers.py b/tests/test_reviewers.py new file mode 100644 index 0000000..5e5985d --- /dev/null +++ b/tests/test_reviewers.py @@ -0,0 +1,161 @@ +"""Reference reviewers against the seeded fixture suite.""" + +from sapiens.fixtures import FixtureKind, fixture_suite +from sapiens.models import Candidate, Evidence +from sapiens.review import Severity, VerdictKind +from sapiens.reviewers import ( + DevilsAdvocateReviewer, + DomainTheoristReviewer, + MethodologistReviewer, + StatisticianReviewer, + reference_panel, +) +from sapiens.validation import synthetic_holdout_protocol + +FIXTURES = {f.kind: f for f in fixture_suite()} +PROTOCOL = synthetic_holdout_protocol() +VOCAB = ("signal", "score", "data", "training", "replication", "holdout") + + +def review(reviewer, fixture, round_number=1): + return reviewer.review( + fixture.candidate, + fixture.internal + fixture.replication, + round_number=round_number, + prior_objections=(), + seed=0, + ) + + +class TestStatistician: + reviewer = StatisticianReviewer(PROTOCOL) + + def test_approves_known_good(self): + assert review(self.reviewer, FIXTURES[FixtureKind.KNOWN_GOOD]).verdict == ( + VerdictKind.APPROVE + ) + + def test_catches_overfit_at_l2(self): + verdict = review(self.reviewer, FIXTURES[FixtureKind.OVERFIT]) + assert verdict.verdict == VerdictKind.OBJECT + assert any( + o.severity == Severity.BLOCKING and "L2 gate" in o.text for o in verdict.objections + ) + + def test_catches_leakage_at_l2(self): + verdict = review(self.reviewer, FIXTURES[FixtureKind.LEAKAGE]) + assert verdict.verdict == VerdictKind.OBJECT + assert any("leakage" in o.text for o in verdict.objections) + + def test_catches_degenerate_at_l1(self): + verdict = review(self.reviewer, FIXTURES[FixtureKind.DEGENERATE]) + assert verdict.verdict == VerdictKind.OBJECT + assert any("L1 gate" in o.text for o in verdict.objections) + + +class TestMethodologist: + reviewer = MethodologistReviewer() + + def test_approves_known_good(self): + assert review(self.reviewer, FIXTURES[FixtureKind.KNOWN_GOOD]).verdict == ( + VerdictKind.APPROVE + ) + + def test_catches_cross_stage_dataset_reuse(self): + verdict = review(self.reviewer, FIXTURES[FixtureKind.LEAKAGE]) + assert verdict.verdict == VerdictKind.OBJECT + assert any( + o.severity == Severity.BLOCKING and "leakage signal" in o.text + for o in verdict.objections + ) + + def test_blocks_on_empty_evidence(self): + verdict = self.reviewer.review( + Candidate("c", "d", "claim about signal"), + (), + round_number=1, + prior_objections=(), + seed=0, + ) + assert verdict.verdict == VerdictKind.OBJECT + assert verdict.objections[0].severity == Severity.BLOCKING + + +class TestDomainTheorist: + reviewer = DomainTheoristReviewer(VOCAB) + + def test_approves_all_fixture_claims(self): + # Fixture claims are deliberately vocabulary-coherent so per-role + # attribution stays clean: the theorist's job is coherence, not bias. + for fixture in FIXTURES.values(): + assert review(self.reviewer, fixture).verdict == VerdictKind.APPROVE + + def test_incoherent_claim_escalates_across_rounds(self): + candidate = Candidate("c", "d", "zzz qqq nothing coherent") + evidence = (Evidence("e", "c", "internal", True, "p", "d", 1, 0.9),) + first = self.reviewer.review( + candidate, evidence, round_number=1, prior_objections=(), seed=0 + ) + second = self.reviewer.review( + candidate, evidence, round_number=2, prior_objections=(), seed=0 + ) + assert first.objections[0].severity == Severity.MAJOR + assert second.objections[0].severity == Severity.BLOCKING + + +class TestDevilsAdvocate: + reviewer = DevilsAdvocateReviewer() + + def test_approves_known_good(self): + assert review(self.reviewer, FIXTURES[FixtureKind.KNOWN_GOOD]).verdict == ( + VerdictKind.APPROVE + ) + + def test_catches_degenerate_signature(self): + verdict = review(self.reviewer, FIXTURES[FixtureKind.DEGENERATE]) + assert any( + o.severity == Severity.BLOCKING and "degenerate" in o.text + for o in verdict.objections + ) + + def test_catches_leakage_signature(self): + verdict = review(self.reviewer, FIXTURES[FixtureKind.LEAKAGE]) + assert any( + o.severity == Severity.BLOCKING and "leakage" in o.text + for o in verdict.objections + ) + + def test_overfit_alone_not_advocate_territory(self): + # Overfit passes the advocate's signature checks; the statistician + # owns that catch. Attribution honesty matters for per-role rates. + verdict = review(self.reviewer, FIXTURES[FixtureKind.OVERFIT]) + assert all( + o.severity != Severity.BLOCKING or "single run" in o.text + for o in verdict.objections + ) + + def test_perfect_scores_escalate(self): + candidate = Candidate("c", "d", "signal claim") + evidence = ( + Evidence("e1", "c", "internal", True, "p", "train", 1, 1.0), + Evidence("e2", "c", "replication", True, "p", "holdout", 2, 1.0), + ) + first = self.reviewer.review( + candidate, evidence, round_number=1, prior_objections=(), seed=0 + ) + second = self.reviewer.review( + candidate, evidence, round_number=2, prior_objections=(), seed=0 + ) + assert any(o.severity == Severity.MAJOR for o in first.objections) + assert any(o.severity == Severity.BLOCKING for o in second.objections) + + def test_thin_stage_is_minor_caveat(self): + verdict = review(self.reviewer, FIXTURES[FixtureKind.LEAKAGE]) + # leakage fixture has one run per stage: minor caveat rides alongside + assert any(o.severity == Severity.MINOR for o in verdict.objections) + + +def test_reference_panel_has_four_unique_roles(): + panel = reference_panel(VOCAB, PROTOCOL) + assert len(panel.reviewers) == 4 + assert len({reviewer.role for reviewer in panel.reviewers}) == 4