Feat/advisor eval

docs/agents/telemetry-change.md

@@ -27,19 +27,19 @@
 
 ### 1. 三套鉴权与埋点标识
 
-| 命令声明          | 凭证 / 请求域                                       | 主要请求出口                                                                          | 后端埋点标识                                                                      | 前端埋点标识（AEM）                              |
-| ----------------- | --------------------------------------------------- | ------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------------------ |
-| `auth: "apiKey"`  | API Key；DashScope / OpenAI-compatible 模型域       | `Client.request/requestJson`、`McpClient`、Managed Agent instrumented fetch、上传策略 | 有：`User-Agent`、`x-dashscope-source-config`、`X-Dashscope-Service: bailian-cli` | 有：`pid=bailian-cli-node`、`authMethod=apiKey`  |
-| `auth: "console"` | Console access token；Bailian Console Gateway       | `callConsoleGateway()` → `/cli/api.json`                                              | 有：Console Gateway 后端直接标识                                                  | 有：`pid=bailian-cli-node`、`authMethod=console` |
-| `auth: "openapi"` | AccessKey ID/Secret，可选 STS token；阿里云 OpenAPI | `Client.openApiJson()`                                                                | 有：`x-dashscope-source-config`、`X-Dashscope-Service: bailian-cli`               | 有：`pid=bailian-cli-node`、`authMethod=openapi` |
-| `auth: "none"`    | 无凭证域                                            | 本地逻辑或命令自行管理的登录/配置流程                                                 | 无                                                                                | 有：`pid=bailian-cli-node`、`authMethod=none`    |
+| 命令声明          | 凭证 / 请求域                                       | 主要请求出口                                                                                    | 后端埋点标识                                                                      | 前端埋点标识（AEM）                              |
+| ----------------- | --------------------------------------------------- | ----------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | ------------------------------------------------ |
+| `auth: "apiKey"`  | API Key；DashScope / OpenAI-compatible 模型域       | `Client.request/requestJson`、`McpClient`、Managed Agent / Advisor instrumented fetch、上传策略 | 有：`User-Agent`、`x-dashscope-source-config`、`X-Dashscope-Service: bailian-cli` | 有：`pid=bailian-cli-node`、`authMethod=apiKey`  |
+| `auth: "console"` | Console access token；Bailian Console Gateway       | `callConsoleGateway()` → `/cli/api.json`                                                        | 有：Console Gateway 后端直接标识                                                  | 有：`pid=bailian-cli-node`、`authMethod=console` |
+| `auth: "openapi"` | AccessKey ID/Secret，可选 STS token；阿里云 OpenAPI | `Client.openApiJson()`                                                                          | 有：`x-dashscope-source-config`、`X-Dashscope-Service: bailian-cli`               | 有：`pid=bailian-cli-node`、`authMethod=openapi` |
+| `auth: "none"`    | 无凭证域                                            | 本地逻辑或命令自行管理的登录/配置流程                                                           | 无                                                                                | 有：`pid=bailian-cli-node`、`authMethod=none`    |
 
 `authMethod` 记录的是命令声明的鉴权域，不是凭证来源。它不会区分 API Key 来自 flag、env 还是 config。
 鉴权域是命令的准入门槛和主请求域，不保证命令内部只有一种网络出口；例如部分 `apiKey` 命令也可能读取匿名 Console 公共目录，Managed Agent 还可能访问其他 provider。
 
 表中的后端埋点按该鉴权域的主要业务请求填写：
 
-- Managed Agent 的 `User-Agent` 对所有 SDK 请求注入；`x-dashscope-source-config` 与 `X-Dashscope-Service` 仅对阿里云 host 注入
+- Managed Agent 与 Advisor 通过宿主注入的 `createInstrumentedFetch` 统一携带 `User-Agent`；渠道头仅对阿里云 host 注入。Advisor 的引擎保持独立，普通聊天、工具调用、SSE 与 JSON 重试共用该传输，`--verbose` 在 stderr 输出 endpoint、HTTP 状态与响应头中的 request_id，凭证脱敏且不输出请求体
 - DashScope 上传策略 `getPolicy` 走 `trackingHeaders()`（含 source-config 与 `X-Dashscope-Service`），没有显式 CLI `User-Agent`
 - Console Gateway 用户数据由后端直接区分，不依赖 `x-dashscope-source-config`；AEM 的 `authMethod=console` 是独立的命令侧统计
 - OpenAPI 的 ACS 签名头，以及 Console Gateway 的 `product`、`action`、`api` 是鉴权或路由字段，不计为埋点标识

packages/commands/src/commands/advisor/recommend.ts

@@ -1,6 +1,7 @@
 import {
   agentLoop,
   BailianError,
+  createInstrumentedFetch,
   createLLMClient,
   createReadlineInteraction,
   DEFAULT_LLM_MODEL,
@@ -136,8 +137,8 @@ const flags = {
   noThinking: {
     type: "switch",
     description: {
-      "en-US": "Disable thinking mode for intent analysis (on by default)",
-      "zh-CN": "关闭意图分析的思考模式（默认开启）",
+      "en-US": "Deprecated no-op: thinking is always disabled for advisor LLM calls",
+      "zh-CN": "已废弃且无效果：advisor 的 LLM 调用现已始终关闭思考模式",
     },
   },
 } as const;
@@ -225,11 +226,17 @@ export default defineCommand({
       llm: createLLMClient({
         baseUrl: ctx.client.baseUrl,
         apiKey: credential.token,
+        fetch: createInstrumentedFetch({
+          identity: ctx.identity,
+          settings: ctx.settings,
+          debug: (diagnostic) => ctx.output.debug(diagnostic),
+        }),
         defaultModel: runtimeFlags.model ?? DEFAULT_LLM_MODEL,
-        // Intent analysis is the only LLM call that decides the slate, so it
-        // gets thinking mode by default (--no-thinking to disable). Other
-        // stages stay in plain mode: thinking tokens are billed per call.
-        thinkingStages: runtimeFlags.noThinking ? [] : ["intent"],
+        // Disable thinking for every advisor LLM call. Intent analysis was the
+        // only thinking stage, but it dominated latency without improving
+        // clear recommendation requests. `--no-thinking` remains accepted as
+        // a deprecated no-op for existing scripts.
+        thinkingStages: [],
       }),
       catalog: () => loadCatalog(),
       families: () => loadFamilies(),

packages/commands/src/commands/finetune/capability.ts

@@ -9,6 +9,7 @@ import {
   TRAINING_TYPES_CLI,
   anonymousConsoleCall,
   UsageError,
+  type HttpDeps,
   type Settings,
   type ModelCapability,
   type FlagsDef,
@@ -20,8 +21,11 @@ import {
  * records so capability fields (`supports` / `trainingTypes`) are preserved
  * for filtering.
  */
-async function fetchAllFoundationModels(settings: Settings): Promise<ModelCapability[]> {
-  const all = await fetchModelListAll(anonymousConsoleCall(settings));
+async function fetchAllFoundationModels(
+  settings: Settings,
+  diagnostics: Pick<HttpDeps, "debug">,
+): Promise<ModelCapability[]> {
+  const all = await fetchModelListAll(anonymousConsoleCall(settings, diagnostics));
   return all as ModelCapability[];
 }
 
@@ -85,6 +89,9 @@ export default defineCommand({
   },
   async run(ctx) {
     const { settings, flags } = ctx;
+    const diagnostics: Pick<HttpDeps, "debug"> = {
+      debug: (diagnostic) => ctx.output.debug(diagnostic),
+    };
     const model = flags.baseModel || undefined;
     const trainingType = flags.trainingType || undefined;
 
@@ -103,7 +110,7 @@ export default defineCommand({
 
     // Direction 1: by model → which training types it supports.
     if (model) {
-      const capability = await fetchModelCapability(settings, model);
+      const capability = await fetchModelCapability(settings, model, diagnostics);
       if (!capability) {
         const resultData2 = { model, error: `No foundation model found matching "${model}".` };
         ctx.output.result(
@@ -136,7 +143,7 @@ export default defineCommand({
       );
     }
     const { method, variant } = trainingTypeMethodVariant(trainingType);
-    const all = await fetchAllFoundationModels(settings);
+    const all = await fetchAllFoundationModels(settings, diagnostics);
     const matched = all
       .filter((record) => modelSupportsTrainingType(record, trainingType))
       .map((record) => ({

packages/commands/src/commands/finetune/create.ts

@@ -14,6 +14,7 @@ import {
   formatIssue,
   BailianError,
   ExitCode,
+  type HttpDeps,
   type CommandContext,
   type CreateFineTuneRequest,
   type FineTuneHyperParameters,
@@ -562,6 +563,9 @@ async function runCreate<F extends FlagsDef>(
   ctx: CommandContext<F>,
 ): Promise<void> {
   const { settings } = ctx;
+  const diagnostics: Pick<HttpDeps, "debug"> = {
+    debug: (diagnostic) => ctx.output.debug(diagnostic),
+  };
   const flags = ctx.flags as Record<string, unknown>;
   const model = flags.baseModel as string;
   const datasetsRaw = flags.datasets as string;
@@ -818,7 +822,7 @@ async function runCreate<F extends FlagsDef>(
   if (!settings.dryRun && !profile.shouldSkipCapabilityCheck(modality)) {
     let capability: Awaited<ReturnType<typeof fetchModelCapability>> | undefined;
     try {
-      capability = await fetchModelCapability(settings, model);
+      capability = await fetchModelCapability(settings, model, diagnostics);
     } catch (error) {
       ctx.output.warning({
         code: "TRAINING_CAPABILITY_LOOKUP_FAILED",

packages/commands/src/commands/finetune/fee.ts

@@ -13,6 +13,7 @@ import {
   effectiveConsoleGatewayConfig,
   unwrapResponse,
   MODEL_LIST_API,
+  type HttpDeps,
   type Settings,
   type ModelPriceInfo,
 } from "bailian-cli-core";
@@ -31,6 +32,7 @@ export interface ActualFee {
 async function fetchTrainingPrice(
   settings: Settings,
   model: string,
+  diagnostics: Pick<HttpDeps, "debug">,
 ): Promise<ModelPriceInfo | null> {
   const eff = effectiveConsoleGatewayConfig(settings);
   const result = await callConsoleGateway(
@@ -55,6 +57,7 @@ async function fetchTrainingPrice(
         },
       },
     },
+    diagnostics,
   );
   const responseData = unwrapResponse(result as Record<string, unknown>);
   const list = (responseData.list as Record<string, unknown>[]) ?? [];
@@ -90,9 +93,10 @@ export async function computeActualFee(
   settings: Settings,
   model: string,
   usageTokens: number,
+  diagnostics: Pick<HttpDeps, "debug"> = {},
 ): Promise<ActualFee | null> {
   try {
-    const ftEntry = await fetchTrainingPrice(settings, model);
+    const ftEntry = await fetchTrainingPrice(settings, model, diagnostics);
     const unitPrice = Number(ftEntry?.price);
     if (!Number.isFinite(unitPrice) || unitPrice <= 0) return null;
     const priceUnit = ftEntry?.priceUnit ?? "每百万tokens";

packages/commands/src/commands/finetune/get.ts

@@ -1,4 +1,4 @@
-import { defineCommand, getFineTune, type FlagsDef } from "bailian-cli-core";
+import { defineCommand, getFineTune, type HttpDeps, type FlagsDef } from "bailian-cli-core";
 
 import { computeActualFee } from "./fee.ts";
 
@@ -23,6 +23,9 @@ export default defineCommand({
   exampleArgs: ["--job-id ft-xxx", "--job-id ft-xxx --output json"],
   async run(ctx) {
     const { settings, flags } = ctx;
+    const diagnostics: Pick<HttpDeps, "debug"> = {
+      debug: (diagnostic) => ctx.output.debug(diagnostic),
+    };
     const jobId = flags.jobId;
 
     if (settings.dryRun) {
@@ -77,7 +80,7 @@ export default defineCommand({
     // Actual fee: only when the platform reports a concrete token count
     // (SUCCEEDED / CANCELED). Best-effort — silently omitted on lookup failure.
     if (usageTokens !== undefined && usageTokens > 0 && job.model) {
-      const fee = await computeActualFee(settings, job.model, usageTokens);
+      const fee = await computeActualFee(settings, job.model, usageTokens, diagnostics);
       if (fee) {
         item.training_cost = fee.cost;
         item.cost_basis = `${fee.unitPrice} 元/${fee.priceUnit}`;

packages/commands/src/commands/finetune/watch.ts

@@ -3,6 +3,7 @@ import {
   getFineTune,
   BailianError,
   ExitCode,
+  type HttpDeps,
   type FlagsDef,
 } from "bailian-cli-core";
 
@@ -134,6 +135,9 @@ export default defineCommand({
   ],
   async run(ctx) {
     const { settings, flags } = ctx;
+    const diagnostics: Pick<HttpDeps, "debug"> = {
+      debug: (diagnostic) => ctx.output.debug(diagnostic),
+    };
     const jobId = flags.jobId;
     const follow = flags.follow;
     const intervalSec = Math.max(MIN_INTERVAL_SEC, flags.interval ?? DEFAULT_INTERVAL_SEC);
@@ -175,7 +179,12 @@ export default defineCommand({
         const usageTokens = typeof job?.usage === "number" ? job.usage : undefined;
         if (terminal && usageTokens && usageTokens > 0 && job?.model) {
           output.usage_tokens = usageTokens;
-          const fee = await computeActualFee(settings, job.model as string, usageTokens);
+          const fee = await computeActualFee(
+            settings,
+            job.model as string,
+            usageTokens,
+            diagnostics,
+          );
           if (fee) {
             output.training_cost = fee.cost;
             output.cost_basis = `${fee.unitPrice} 元/${fee.priceUnit}`;
@@ -226,7 +235,12 @@ export default defineCommand({
             const usageTokens = typeof job?.usage === "number" ? job.usage : undefined;
             const enriched: Record<string, unknown> = { ...response };
             if (usageTokens && usageTokens > 0 && job?.model) {
-              const fee = await computeActualFee(settings, job.model as string, usageTokens);
+              const fee = await computeActualFee(
+                settings,
+                job.model as string,
+                usageTokens,
+                diagnostics,
+              );
               if (fee) {
                 enriched.training_cost = fee.cost;
                 enriched.usage_tokens = usageTokens;

packages/commands/tests/e2e/advisor-recommend.e2e.test.ts

@@ -1,5 +1,13 @@
-import { describe, expect, test } from "vite-plus/test";
-import { isDashScopeE2EReady, parseStdoutJson, runCommandHelp, runCommandE2e } from "./helpers.ts";
+import { afterEach, beforeEach, describe, expect, test, vi } from "vite-plus/test";
+import * as core from "bailian-cli-core";
+import {
+  isDashScopeE2EReady,
+  parseStdoutJson,
+  runCommandHelp,
+  runCommandE2e,
+  runCommandInProcess,
+} from "./helpers.ts";
+import { E2E_HARNESS_IDENTITY } from "./harness/identity.ts";
 import { ADVISOR_ROUTES } from "./topic-routes.ts";
 
 describe("e2e: advisor recommend", () => {
@@ -14,6 +22,112 @@ describe("e2e: advisor recommend", () => {
   });
 });
 
+describe("e2e: advisor recommend 传输（离线）", () => {
+  beforeEach(() => {
+    // 隔离目录、同步与原生网络，回归不读取真实目录或调用付费模型。
+    vi.spyOn(core, "maybeSyncWikiData").mockResolvedValue(false);
+    vi.spyOn(core, "loadCatalog").mockResolvedValue([]);
+    vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("禁止原生网络请求")));
+  });
+  afterEach(() => {
+    vi.restoreAllMocks();
+    vi.unstubAllGlobals();
+  });
+
+  test.each([true, false])(
+    "真实命令注入统一传输且默认关闭 thinking，verbose=%s",
+    async (verbose) => {
+      const endpoint = "https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions";
+      const requests: RequestInit[] = [];
+      vi.spyOn(core.httpTransport, "fetch").mockImplementation(async (input, init) => {
+        expect(input).toBe(endpoint);
+        requests.push(init ?? {});
+        const body = JSON.parse(init?.body as string);
+        const content = JSON.stringify(
+          requests.length === 1
+            ? {
+                taskSummary: "文本生成",
+                inputModality: ["Text"],
+                outputModality: ["Text"],
+                requiredCapabilities: ["TG"],
+                matchedScenes: [],
+                sceneConfidence: 1,
+              }
+            : { scores: [] },
+        );
+        const headers = { "x-request-id": "advisor-success-request" };
+        return body.stream
+          ? new Response(
+              `data: ${JSON.stringify({ choices: [{ delta: { content } }] })}\n\ndata: [DONE]\n\n`,
+              { headers },
+            )
+          : Response.json({ choices: [{ message: { content } }] }, { headers });
+      });
+      const { stdout, stderr, exitCode } = await runCommandInProcess(
+        ADVISOR_ROUTES,
+        [
+          "advisor",
+          "recommend",
+          "--message",
+          "文本生成",
+          "--no-agent",
+          ...(verbose ? ["--verbose"] : ["--no-thinking"]),
+        ],
+        {
+          DASHSCOPE_API_KEY: "test-advisor-secret-key",
+          DASHSCOPE_BASE_URL: "https://dashscope.aliyuncs.com",
+          DO_NOT_TRACK: "1",
+          NODE_OPTIONS: "",
+          HTTP_PROXY: "",
+          HTTPS_PROXY: "",
+          http_proxy: "",
+          https_proxy: "",
+        },
+      );
+      expect(exitCode, stderr).toBe(0);
+      expect(requests).toHaveLength(2);
+      for (const request of requests) {
+        const headers = new Headers(request.headers);
+        expect(headers.get("authorization")).toBe("Bearer test-advisor-secret-key");
+        expect(headers.get("user-agent")).toBe(
+          `${E2E_HARNESS_IDENTITY.clientName}/${E2E_HARNESS_IDENTITY.version}`,
+        );
+        expect(headers.get("x-dashscope-source-config")).toBe(
+          core.sourceConfig(E2E_HARNESS_IDENTITY),
+        );
+        expect(headers.get("x-dashscope-openapisource")).toBe("BailianCLI");
+        expect(headers.get("x-dashscope-service")).toBe("bailian-cli");
+        expect(JSON.parse(request.body as string).model).toBe(core.DEFAULT_LLM_MODEL);
+      }
+      expect(JSON.parse(requests[0].body as string).enable_thinking).toBeUndefined();
+      expect(JSON.parse(requests[0].body as string).stream).toBeUndefined();
+      expect(JSON.parse(requests[1].body as string).stream).toBeUndefined();
+      expect(parseStdoutJson<{ intent: { taskSummary: string } }>(stdout).intent.taskSummary).toBe(
+        "文本生成",
+      );
+      expect(stdout).not.toContain("request_id:");
+      expect(stderr).not.toContain("test-advisor-secret-key");
+      if (verbose) {
+        const debugLines = stderr
+          .trim()
+          .split("\n")
+          .filter((line) => line.startsWith('{"debug":'));
+        expect(debugLines.length).toBeGreaterThanOrEqual(4);
+        const requestDebug = JSON.parse(debugLines[0]!).debug;
+        const responseDebug = JSON.parse(debugLines[1]!).debug;
+        expect(requestDebug.details).toMatchObject({ method: "POST", url: endpoint });
+        expect(responseDebug.details).toMatchObject({
+          status: 200,
+          request_id: "advisor-success-request",
+        });
+      } else {
+        expect(stderr).not.toContain('{"debug":');
+        expect(stderr).not.toContain("request_id:");
+      }
+    },
+  );
+});
+
 describe.skipIf(!isDashScopeE2EReady())("e2e: advisor recommend (DashScope)", () => {
   test("advisor recommend without --message errors as usage error (2)", async () => {
     const { stdout, stderr, exitCode } = await runCommandE2e(ADVISOR_ROUTES, [