Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type {AssessmentCitation, AssessmentVerdict, TrajectoryAssessment, Trajec
import type {TrajectoryNativeOutcomes} from "../../../../../../loopx/control_plane/capabilities/trajectory_native_outcomes.js";
import {ContextHelp} from "./context-help";
import {ResearchSynthesis} from "./goal-research-synthesis";
import {ToolContextInventory} from "./goal-tool-context";
import "./goal-research-progress.css";

// Read models from benchmark-toolkit. This view formats existing counts and
Expand Down Expand Up @@ -306,7 +307,7 @@ export function ResearchProgress({pack: p, packs, selectedId, zh, onSelect, onCi
</ContextHelp></div>
<p>{zh ? "其中输入" : "Of which input"} <strong>{incomplete.usage ? number(incomplete.usage.input_tokens) : "—"}</strong> · {zh ? "输出" : "output"} <strong>{incomplete.usage ? number(incomplete.usage.output_tokens) : "—"}</strong> tokens</p>
</div> : null}
<div className="research-usage-secondary"><LoopXInputShare usage={usage} zh={zh}/>
<div className="research-usage-secondary"><LoopXInputShare usage={usage} zh={zh}/><ToolContextInventory value={usage?.tool_context} zh={zh}/>
{usage ? <div className="research-request-coverage">
<div className="research-usage-heading"><h4>{zh ? "请求覆盖" : "Request coverage"}</h4>
{coverage?.status !== "verified" ? <span className="research-usage-scope">{coverage ? (zh ? "核对失败" : "Reconciliation failed") : (zh ? "未核对" : "Not reconciled")}</span> : null}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import type {TrajectoryToolContext} from "../../../../../../loopx/control_plane/capabilities/trajectory_assessment_types.js";
import {ContextHelp} from "./context-help";

/** Formats the toolkit observation; never classifies commands or infers savings. */
export function ToolContextInventory({value, zh}: {value: TrajectoryToolContext | undefined; zh: boolean}) {
if (!value) return null;
const number = (n: number) => n.toLocaleString(zh ? "zh-CN" : "en");
const fraction = value.status === "observed" ? value.loopx_tool_output_fraction : null;
const percent = fraction === null ? "—" : fraction > 0 && fraction < 0.001 ? "<0.1%"
: fraction < 1 && fraction > 0.999 ? ">99.9%"
: new Intl.NumberFormat(zh ? "zh-CN" : "en", {style: "percent", maximumFractionDigits: 1}).format(fraction);
return <section className="research-request-coverage" aria-label={zh ? "LoopX 工具上下文" : "LoopX tool context"}>
<div className="research-usage-heading"><h4>{zh ? "LoopX 工具返回 · 字节" : "LoopX tool returns · bytes"}</h4>
<ContextHelp label={zh ? "工具上下文统计口径" : "Tool context measurement basis"}>
<p>{zh ? "分母是捕获中追加的用户、系统、开发者文本及工具返回;分子仅识别简单 LoopX CLI 调用的返回。复合脚本、间接调用、文件读取和其他内容保持未归因,图片等非文本内容不计字节。" : "The denominator is appended user, system, developer text and tool returns. Only returns from simple LoopX CLI calls are assigned. Compound scripts, indirect calls, file reads and other content remain unassigned; non-text content such as images has no byte count."}</p>
<p>{zh ? "这是已记录内容的组成,不是逐请求重放的上下文或计费 token 占比。重复内容不等于无益重复,LoopX 返回也可能携带有用研究证据;不能据此推断净成本或提分。" : "This describes recorded content, not per-request replay or billable token share. Repeated content need not be waste; LoopX returns can carry useful research evidence. It does not establish net cost or score benefit."}</p>
</ContextHelp></div>
{value.status === "unavailable" ? <p role="status">{zh ? "捕获未通过核对 · 占比未知" : "Capture could not be reconciled · share unknown"}</p> : <>
<p className="research-input-share-value"><strong data-loopx-tool-context-share>{percent}</strong><span>{zh ? "占已记录输入文本" : "of recorded input text"}</span></p>
<details className="research-usage-detail"><summary>{zh ? "返回明细" : "Return details"}</summary><dl className="research-usage-basis">
<dt>{zh ? "LoopX 工具返回" : "LoopX tool returns"}</dt><dd>{number(value.loopx_tool_output_bytes)} B</dd>
<dt>{zh ? "已记录输入文本" : "Recorded input text"}</dt><dd>{number(value.observed_input_text_bytes)} B</dd>
<dt>{zh ? "未归因文本" : "Unassigned text"}</dt><dd>{number(value.unassigned_input_text_bytes)} B</dd>
<dt>{zh ? "未计量内容块" : "Unmeasured content parts"}</dt><dd>{number(value.unmeasured_parts)}</dd>
<dt>{zh ? "重复相同返回" : "Repeated identical returns"}</dt><dd>{number(value.repeated_loopx_output_bytes)} B</dd>
<dt>{zh ? "已去重返回记录" : "Duplicate return records excluded"}</dt><dd>{number(value.duplicate_output_records)}</dd>
{value.operations.map(row => <div key={row.operation}><dt>{row.operation}</dt><dd>{number(row.calls)} {zh ? "次" : "calls"} · {number(row.bytes)} B</dd></div>)}
</dl></details>
</>}
</section>;
}
36 changes: 36 additions & 0 deletions docs/reference/trajectory-assessment.md
Original file line number Diff line number Diff line change
Expand Up @@ -1064,6 +1064,42 @@ values with the existing warning and never borrow another pack's measurement.
不能反推 packet token;未采集、部分覆盖与真实零分开。比例是结构开销,不能单独
当作净效率损失、可避免成本或因果贡献。普通阅读不调用模型。

### Captured tool context inventory

Add `--include-tool-context` to `trajectory-pack-register` with the existing
`--usage-jsonl` and `--usage-session-id` to measure appended input text. The
Research workbench's existing Sources and usage disclosure displays the saved
inventory beside request-token attribution; browsing never launches a model.
Omitting the option preserves historical registration and projections.

The numerator is the UTF-8 text returned by recognized simple LoopX CLI calls;
the denominator includes captured user/system/developer text and tool returns.
Native `response_item` records are used, not their event-message mirrors.
Duplicate output records for one call count once. Identical output from two
different calls is still two pieces of appended input; it is not automatically
redundant research. Operation counts distinguish context reads/projects from
other LoopX returns. No output or command argument text is sent to the browser.
Task lease acquisition, inspection, renewal, release and transfer have separate
rows, as do common Todo operations such as claim and completion. These count
observed replies, not successful mutations or proved unnecessary recovery.

Only literal simple shell argv, Python `-m loopx`, `-m loopx.cli` or
`-m loopx.entrypoint`, and an exact
single literal `const/await tools.exec_command/text` wrapper are recognized.
Compound shell, arbitrary JavaScript, indirect commands, continuation tools and
file reads remain unassigned. Non-text content is counted as unmeasured parts.
The byte share covers the recorded component only: injected LoopX instructions
can remain unassigned, and useful domain evidence may be present in LoopX
returns. It is neither whole control-plane overhead nor per-request replay,
billable tokens, avoidable cost, or causal benefit. Keep the separate Turn-packet
estimate and provider counters; do not add their different units or denominators.

中文:可选统计在原轨迹分析工作台中展示 LoopX 工具返回占已记录输入文本的字节
比例,并拆出 context-read/project 等操作。未知和非文本内容分别保留;同内容的
再次返回不等于无益重复。这不是全量控制面、逐请求重放或计费 token 的占比。
租约获取、检查、续租、释放、转移及常见 Todo 操作分别列示;计数表示可识别的返回,
不证明状态写入成功或恢复操作可以省略。

### Recorded Turn packet estimate

The existing native `codex-cli` Turn host can record its actual injected packet
Expand Down
2 changes: 2 additions & 0 deletions examples/personal-workspace-browser-smoke.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {reportDownloadScenario} from "./personal-workspace-browser/report-downlo
import { selectScenarioShard } from "./personal-workspace-browser/shard.mjs";
import {completedHistoryIsolationScenario} from "./personal-workspace-browser/completed-history-isolation.mjs";
import {researchWorkerReadbackScenario} from "./personal-workspace-browser/research-worker-readback.mjs";
import {researchToolContextScenario} from "./personal-workspace-browser/research-tool-context.mjs";
import { zcodeGoalScenario } from "./personal-workspace-browser/zcode-goal.mjs";
import {goalWorkspaceCreateScenario} from "./personal-workspace-browser/goal-workspace-create.mjs";
import {agentModelSelectionScenario} from "./personal-workspace-browser/agent-model-selection.mjs";
Expand Down Expand Up @@ -132,6 +133,7 @@ scenarioCatalog.push(attentionUserActionScenario);
scenarioCatalog.push(researchResultsScenario);
scenarioCatalog.push(researchHistoryScenario);
scenarioCatalog.push(researchMeasurementsScenario);
scenarioCatalog.push(researchToolContextScenario);
scenarioCatalog.push(researchNativeReadingScenario);
scenarioCatalog.push(researchFindingReaderScenario);
scenarioCatalog.push(researchRecordSearchScenario);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,9 @@ def main():
scope = dict(runtime_root=runtime, owner_goal_id=capture["goal_id"])
registered = register_pack(**scope, input_path=input_path, source_root=source_root, title=analysis["title"],
config=analysis.get("config", fixture_config),
usage_jsonl=analysis.get("usage_jsonl"),
usage_session_id=analysis.get("usage_session_id"),
include_tool_context=analysis.get("include_tool_context", False),
native_outcomes_json=analysis.get("native_outcomes_json"))
for language, output in analysis["outputs"].items():
provider = write_python_host_fixture(root / f"analysis-provider-{index}-{language}",
Expand Down
110 changes: 110 additions & 0 deletions examples/personal-workspace-browser/research-tool-context.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
import assert from "node:assert/strict";
import {execFileSync} from "node:child_process";
import {mkdtemp, rm} from "node:fs/promises";
import {tmpdir} from "node:os";
import {resolve} from "node:path";
import {resolveTestPython} from "../../scripts/test-python.mjs";
import {outputDir, repoRoot} from "./fixture.mjs";
import {startHistoryOwner} from "./research-history.mjs";

// Real source verification, TS reduction, pack registry, HTTP and packaged UI.
// The fixture has no model outputs: ordinary metric reading needs no analysis.
export const researchToolContextScenario = {
id: "research-tool-context",
async run({browser}) {
const temporary = await mkdtemp(resolve(tmpdir(), "loopx-tool-context-"));
let owner, page;
try {
const fixture = resolve(temporary, "capture.json");
execFileSync(resolveTestPython(), ["-c", `
import hashlib,json,sys
from pathlib import Path
from loopx.capabilities.explore.result_log import build_explore_node_event
goal='synthetic-context-reading'
text='Synthetic research observation; utility remains unknown.'
scope=dict(goal_id=goal,case_id='synthetic',run_id='fixed',revision='fixture',window='fixed capture')
pack=dict(schema_version='trajectory_assessment_input_v1',scope=scope,objective='Compare research outcomes and complete costs',
selection=dict(rule='one synthetic episode',population_episodes=1,exhaustive=True),
sources=[dict(id='facts',artifact='facts.txt',artifact_sha256=hashlib.sha256(text.encode()).hexdigest(),first_line=1,text=text)],
episodes=[dict(id='episode',question='Did new evidence change the next experiment?',source_ids=['facts'],eligible_metrics=['evidence_adoption'])])
usage=dict(input_tokens=100,cached_input_tokens=60,output_tokens=5)
rows=[dict(type='session_meta',payload=dict(id='session')),
dict(type='event_msg',payload=dict(type='task_started',turn_id='turn')),
dict(type='response_item',payload=dict(type='message',role='user',content=[dict(type='input_text',text='x'*100)])),
dict(type='response_item',payload=dict(type='function_call',name='exec_command',call_id='control',arguments=json.dumps(dict(cmd='loopx quota context-read')))),
dict(type='response_item',payload=dict(type='function_call_output',call_id='control',output='x'*60)),
dict(type='response_item',payload=dict(type='function_call_output',call_id='task',output='x'*40)),
dict(type='event_msg',payload=dict(type='token_count',info=dict(total_token_usage=usage,last_token_usage=usage))),
dict(type='event_msg',payload=dict(type='task_complete',turn_id='turn'))]
for i,row in enumerate(rows):row['timestamp']=f'2026-01-01T00:00:0{i}Z'
analyses=[]
for enabled in [False,True]:
analyses.append(dict(title='Captured tool context' if enabled else 'Without tool inventory',input=pack,
artifacts={'facts.txt':text,'usage.jsonl':'\\n'.join(map(json.dumps,rows))},outputs={},
usage_jsonl='usage.jsonl',usage_session_id='session',include_tool_context=enabled))
broken=json.loads(json.dumps(analyses[-1]))
broken['title']='Conflicting tool capture'
conflicting=rows[:5]+[dict(type='response_item',payload=dict(type='function_call_output',call_id='control',output='different'))]+rows[5:]
broken['artifacts']['usage.jsonl']='\\n'.join(map(json.dumps,conflicting))
analyses.append(broken)
Path(sys.argv[1]).write_text(json.dumps(dict(goal_id=goal,display_name='Synthetic context cost study',
events=[build_explore_node_event(goal_id=goal,node_id='question',title='Synthetic research question',recorded_at='2026-01-01T00:00:00Z')],analyses=analyses)))
`, fixture], {cwd: repoRoot});
owner = await startHistoryOwner(fixture);
page = await browser.newPage({viewport: {width: 1440, height: 900}});
const errors = [];
page.on("pageerror", error => errors.push(error.message));
await page.goto(`${owner.base}/chat/?goalId=synthetic-context-reading&view=research&research_goal=synthetic-context-reading&statusUrl=%2Fstatus.json`, {waitUntil: "networkidle"});
const progress = page.locator('.research-progress');
const selector = progress.locator('select');
await selector.selectOption({label: 'Without tool inventory'});
await progress.locator('.research-usage').waitFor({state: 'attached'});
await page.screenshot({path: resolve(outputDir, 'research-tool-context-before.png'), animations: 'disabled'});
assert.equal(await progress.locator('[data-loopx-tool-context-share]').count(), 0);
await selector.selectOption({label: 'Captured tool context'});
const basis = progress.locator('.research-progress-basis');
await basis.locator(':scope > summary').focus(); await page.keyboard.press('Enter');
const share = progress.locator('[data-loopx-tool-context-share]');
await share.waitFor({state: 'visible'});
assert.equal(await share.textContent(), '30%');
const inventory = progress.getByRole('region', {name: 'LoopX 工具上下文', exact: true});
const details = inventory.locator('details');
await details.locator('summary').focus(); await page.keyboard.press('Enter');
assert.match(await details.innerText(), /60 B/);
assert.match(await details.innerText(), /200 B/);
assert.match(await details.innerText(), /quota context-read/);
for (const width of [1440, 390]) {
await page.setViewportSize({width, height: 900});
await inventory.scrollIntoViewIfNeeded();
assert(await inventory.evaluate(node => node.scrollWidth <= node.clientWidth + 1));
await page.screenshot({path: resolve(outputDir, `research-tool-context-${width}.png`), animations: 'disabled'});
}
await page.evaluate(() => localStorage.setItem('loopx-pw-locale', 'en'));
await page.reload({waitUntil: 'networkidle'});
const english = progress.getByRole('region', {name: 'LoopX tool context', exact: true});
await share.waitFor({state: 'attached'});
if (!await progress.locator('.research-progress-basis').evaluate(node => node.open)) {
await progress.locator('.research-progress-basis > summary').focus(); await page.keyboard.press('Enter');
}
await english.waitFor({state: 'visible'});
assert.equal(await share.textContent(), '30%');
await selector.selectOption({label: 'Conflicting tool capture'});
const unavailable = progress.getByRole('region', {name: 'LoopX tool context', exact: true, includeHidden: true});
await unavailable.locator('[role="status"]').waitFor({state: 'attached'});
if (!await progress.locator('.research-progress-basis').evaluate(node => node.open)) {
await progress.locator('.research-progress-basis > summary').focus(); await page.keyboard.press('Enter');
}
assert.match(await unavailable.innerText(), /share unknown/);
assert.equal(await progress.locator('[data-loopx-tool-context-share]').count(), 0);
const readback = await (await page.request.get(`${owner.base}/fixture/readback`)).json();
assert(readback.source_unchanged && readback.mutation_attempts === 0);
assert.deepEqual(errors, []);
return {coverageEntries: [], note: 'real-owner byte inventory, opt-out parity, unknown analysis, keyboard, desktop/phone and English; zero model calls'};
} catch (error) {
if (page && !page.isClosed()) await page.screenshot({path: resolve(outputDir, 'research-tool-context-failed.png')});
throw error;
} finally {
await page?.close(); await owner?.close(); await rm(temporary, {recursive: true, force: true});
}
},
};
Loading
Loading