From 300c995093f011e4174e92324e24b4a252e8647e Mon Sep 17 00:00:00 2001 From: Baixin Guo <141702260+GBX-Max1220@users.noreply.github.com> Date: Sat, 24 Oct 2026 03:51:31 +0800 Subject: [PATCH 1/2] test(sleep): capture paired-session scope evidence contracts --- tests/test_preference_scope_contracts.py | 60 ++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/test_preference_scope_contracts.py diff --git a/tests/test_preference_scope_contracts.py b/tests/test_preference_scope_contracts.py new file mode 100644 index 00000000..6bb4f86e --- /dev/null +++ b/tests/test_preference_scope_contracts.py @@ -0,0 +1,60 @@ +"""Synthetic evidence contracts for #154; no real preference inference is tested.""" +import json +import pytest + +from skillopt_sleep.types import SessionDigest +from skillopt_sleep.mine import heuristic_mine, mine +from skillopt_sleep.llm_miner import _digest_to_prompt + +class ScopeEvidenceLoss(AssertionError): + """Only this known retention gap is allowed to xfail.""" + + +CASES = [ + ("durable", "For all future status reports, use bullet points.", + "Keep that preference for future reports.", "eligible_for_review"), + ("one_off", "For this report only, use bullet points.", + "Do not apply that formatting to other reports.", "withhold_generalization"), + ("reversal", "For all future status reports, use bullet points.", + "I withdraw that preference; use prose in future reports.", "withhold_stale_rule"), + ("ambiguous", "Maybe bullet points would be better.", + "I am not sure whether I want that in future reports.", "withhold_pending_clarification"), +] + + +def sessions(case): + name, first, later, _ = case + return [SessionDigest( + session_id=f"{name}-{i}", project="synthetic-project", + started_at=f"2026-10-0{i+1}T00:00:00Z", + user_prompts=["Prepare a status report for this project.", text], + assistant_finals=["Synthetic status report."], + feedback_signals=["Thanks"], n_user_turns=2, + ) for i, text in enumerate((first, later))] + + +@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES]) +def test_short_scope_evidence_reaches_real_miner_prompt(case): + for digest in sessions(case): + assert digest.user_prompts[-1] in _digest_to_prompt(digest) + + +@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES]) +def test_scope_evidence_survives_individual_heuristic_mining(case): + for digest in sessions(case): + record = heuristic_mine([digest])[0] + assert digest.user_prompts[-1] in record.context_excerpt + assert record.source_sessions == [digest.session_id] + + +@pytest.mark.parametrize("case", CASES, ids=[c[0] for c in CASES]) +@pytest.mark.xfail(strict=True, raises=ScopeEvidenceLoss, reason="Proposed #154 contract: dedup currently retains first context only") +def test_paired_scope_evidence_survives_public_mine_and_dedup(case): + digests = sessions(case) + records = mine(digests, val_fraction=0, test_fraction=0) + assert len(records) == 1 + assert records[0].source_sessions == [d.session_id for d in digests] + payload = json.dumps(records[0].to_dict()) + missing = [d.session_id for d in digests if d.user_prompts[-1] not in payload] + if missing: + raise ScopeEvidenceLoss(f"Missing scope evidence from sessions: {missing}") From 5e5a44077e090a371dbc8dc36619f304baa25b57 Mon Sep 17 00:00:00 2001 From: Baixin Guo <141702260+GBX-Max1220@users.noreply.github.com> Date: Sat, 24 Oct 2026 03:51:33 +0800 Subject: [PATCH 2/2] docs(test): describe bounded preference-scope matrix --- tests/PREFERENCE_SCOPE_MATRIX_154.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 tests/PREFERENCE_SCOPE_MATRIX_154.md diff --git a/tests/PREFERENCE_SCOPE_MATRIX_154.md b/tests/PREFERENCE_SCOPE_MATRIX_154.md new file mode 100644 index 00000000..46fcb3d3 --- /dev/null +++ b/tests/PREFERENCE_SCOPE_MATRIX_154.md @@ -0,0 +1,16 @@ +# Issue 154: bounded paired-session evidence matrix + +Base: 84b36cd5564e8893569e50e37f02a81d10b55e27. +Synthetic fixtures, not user data or human preference labels. + +| Category | Evidence required | Proposed persistence policy (not implemented) | +|---|---|---| +| Durable | explicit future scope and confirmation | eligible for review, not automatic update | +| One-off | explicit local scope and exclusion of future applicability | withhold generalized rule | +| Reversal | earlier scope and later withdrawal, both with provenance/order | withhold stale rule; replacement needs review | +| Ambiguous | uncertainty and absence of durable consent | withhold pending clarification | + +The tests call the real prompt constructor, heuristic miner, and public mine/dedup path. They make no model calls and do not mock a successful semantic inference. +Four passing prompt checks and four individual-record checks are expected. Four strict xfails express the proposed retention contract: same-intent dedup currently merges source IDs, but not the second record's scope evidence. + +A task surviving mining is not proof of a skill update. The current matrix detects evidence loss only. Persistence-policy labels above are expectations for discussion, not code outputs. Clean-context replay and model-level preference inference are outside this patch. No production code is changed.