diff --git a/agentx/evaluations/client.py b/agentx/evaluations/client.py index e4c8456..5f83a09 100644 --- a/agentx/evaluations/client.py +++ b/agentx/evaluations/client.py @@ -75,6 +75,22 @@ def _resolve_scorer_id(scorer_id: Optional[str], evaluation_settings_id: Optiona return scorer_id or evaluation_settings_id +# "Argument not given" for update_tool_schema_meta, where None is itself a meaningful value +# (clear the price). +_UNSET: object = object() + + +def _validated_tool_price(value: float) -> float: + """Local mirror of the engine's check: USD per call, finite and non-negative.""" + try: + price = float(value) + except (TypeError, ValueError): + raise ValueError(f"price_per_call_usd must be a number of USD per call, got {value!r}") from None + if price != price or price in (float("inf"), float("-inf")) or price < 0: + raise ValueError(f"price_per_call_usd must be a finite, non-negative number, got {value!r}") + return price + + class EvaluationsClient: def __init__( self, @@ -723,13 +739,39 @@ def list_tool_schemas(self) -> List[dict]: data = self._request("GET", "/evaluate/tool-schemas", base=self._api_root) return data.get("toolSchemas", []) if isinstance(data, dict) else data - def create_tool_schema(self, *, name: str, definition: str, description: Optional[str] = None) -> dict: + def create_tool_schema( + self, *, name: str, definition: str, description: Optional[str] = None, + price_per_call_usd: Optional[float] = None, + ) -> dict: payload: dict = {"name": name, "definition": definition} if description is not None: payload["description"] = description + if price_per_call_usd is not None: + payload["pricePerCallUsd"] = _validated_tool_price(price_per_call_usd) # Server-side write - no transport retry (see init_run's comment). return self._request("POST", "/evaluate/tool-schemas", base=self._api_root, json=payload, retry=False) + def update_tool_schema_meta( + self, tool_schema_id: str, *, description: Optional[str] = None, + price_per_call_usd: Optional[float] = _UNSET, test_endpoint_url: Optional[str] = _UNSET, # type: ignore[assignment] + ) -> dict: + """Metadata-only edit (PATCH): description, per-call price, test endpoint. Never touches + the version log. ``price_per_call_usd=None`` clears the price (the tool becomes free).""" + payload: dict = {} + if description is not None: + payload["description"] = description + if price_per_call_usd is not _UNSET: + payload["pricePerCallUsd"] = None if price_per_call_usd is None else _validated_tool_price(price_per_call_usd) + if test_endpoint_url is not _UNSET: + payload["testEndpointUrl"] = test_endpoint_url + return self._request( + "PATCH", f"/evaluate/tool-schemas/{tool_schema_id}", base=self._api_root, json=payload, retry=False, + ) + + def delete_tool_schema(self, tool_schema_id: str) -> None: + """Deletes the tool schema and its version history.""" + self._request("DELETE", f"/evaluate/tool-schemas/{tool_schema_id}", base=self._api_root, retry=False) + def get_tool_schema_examples(self, tool_schema_id: str, window: Optional[str] = None) -> dict: params = {"window": window} if window else None return self._request( diff --git a/agentx/evaluations/tool_schemas.py b/agentx/evaluations/tool_schemas.py index 4c58aeb..4d75256 100644 --- a/agentx/evaluations/tool_schemas.py +++ b/agentx/evaluations/tool_schemas.py @@ -19,15 +19,39 @@ def __init__(self, client: "EvaluationsClient"): def list(self) -> List[dict]: return self._client.list_tool_schemas() - def create(self, *, name: str, definition: str, description: Optional[str] = None) -> dict: - return self._client.create_tool_schema(name=name, definition=definition, description=description) + def create( + self, *, name: str, definition: str, description: Optional[str] = None, + price_per_call_usd: Optional[float] = None, + ) -> dict: + """``price_per_call_usd`` is what one call of this tool costs you (a paid external API + billed per request). Every recorded call is charged, failed ones included; the spend + shows as the "Tool calls" segment of the Overview cost chart and counts toward the + ``estimatedCostUsd`` alert metric. Leave it unset for a free tool.""" + return self._client.create_tool_schema( + name=name, definition=definition, description=description, price_per_call_usd=price_per_call_usd, + ) - def get_or_create(self, *, name: str, definition: str, description: Optional[str] = None) -> dict: - """Idempotent register: returns the existing schema of this name if present.""" + def get_or_create( + self, *, name: str, definition: str, description: Optional[str] = None, + price_per_call_usd: Optional[float] = None, + ) -> dict: + """Idempotent register: returns the existing schema of this name if present (its stored + price wins - use :meth:`set_price` to change it).""" existing = next((t for t in self.list() if t.get("name") == name), None) if existing: return existing - return self.create(name=name, definition=definition, description=description) + return self.create( + name=name, definition=definition, description=description, price_per_call_usd=price_per_call_usd, + ) + + def set_price(self, tool_schema_id: str, price_per_call_usd: Optional[float]) -> dict: + """Set (or clear with ``None``) a registered tool's USD price per call. Metadata only - + the definition's version log is untouched.""" + return self._client.update_tool_schema_meta(tool_schema_id, price_per_call_usd=price_per_call_usd) + + def delete(self, tool_schema_id: str) -> None: + """Deletes the tool schema and its version history.""" + self._client.delete_tool_schema(tool_schema_id) def examples(self, tool_schema_id: str, window: Optional[str] = None) -> dict: """Failure evidence recorded against this tool (agent-tool-failure signals, low-rated diff --git a/agentx/monitor/client.py b/agentx/monitor/client.py index 8af4152..803dc04 100644 --- a/agentx/monitor/client.py +++ b/agentx/monitor/client.py @@ -360,6 +360,16 @@ def metrics( params[key] = value return self._request("GET", "/metrics", params=params) + def cost_trend(self, window: str = "7d") -> dict: + """Overview's "Total cost" chart over a window ("24h", "7d", "30d"): priced spend per + bucket and in total, stacked by model, with two reserved segments - ``"eval runs"`` + (offline-evaluation spend) and ``"tool calls"`` (registered tools' per-call prices, split + per tool in ``totalsByTool``). Unpriced models and tools contribute $0.""" + return self._request( + "GET", "/agent-monitoring/cost-trend", + base=self._api_root(), params={"window": window}, + ) + def topics(self, window: str = "7d") -> dict: """The Topics view's data over a window ("24h", "7d", "30d"): LLM-classified themes of sampled production traffic with per-topic counts and sentiment. Empty until Topics is diff --git a/tests/test_tool_schemas.py b/tests/test_tool_schemas.py new file mode 100644 index 0000000..6e86286 --- /dev/null +++ b/tests/test_tool_schemas.py @@ -0,0 +1,81 @@ +"""Unit tests for the tool schema registry client - wire-level, no engine required: the +per-call price rides the camelCase wire on create, PATCH edits are metadata-only and can clear +the price with None, and a bad price is refused locally before any request.""" + +from typing import Any, Dict, List + +import pytest + +from agentx.evaluations.client import EvaluationsClient +from agentx.evaluations.tool_schemas import ToolSchemaClient + + +class RecordingClient(EvaluationsClient): + def __init__(self, responses: List[Any]): + super().__init__(api_key="agtx_test", base_url="http://engine:4700/api/v1") + self.calls: List[Dict[str, Any]] = [] + self._responses = responses + + def _request(self, method: str, path: str, timeout: int = 30, base=None, retry: bool = True, **kwargs: Any) -> Any: # type: ignore[override] + self.calls.append({"method": method, "path": path, "retry": retry, **kwargs}) + return self._responses.pop(0) if self._responses else {} + + +def test_create_sends_the_price_on_the_camelcase_wire_without_retry(): + client = RecordingClient([{"_id": "t1", "name": "geocode", "pricePerCallUsd": 0.25}]) + created = ToolSchemaClient(client).create(name="geocode", definition="{}", price_per_call_usd=0.25) + call = client.calls[0] + assert call["method"] == "POST" + assert call["path"] == "/evaluate/tool-schemas" + assert call["retry"] is False + assert call["json"] == {"name": "geocode", "definition": "{}", "pricePerCallUsd": 0.25} + assert created["pricePerCallUsd"] == 0.25 + + +def test_create_omits_the_price_when_not_given(): + client = RecordingClient([{}]) + ToolSchemaClient(client).create(name="clock", definition="{}") + assert "pricePerCallUsd" not in client.calls[0]["json"] + + +def test_set_price_patches_metadata_and_none_clears_it(): + client = RecordingClient([{"ok": True}, {"ok": True}]) + tools = ToolSchemaClient(client) + tools.set_price("t1", 0.01) + tools.set_price("t1", None) + assert [c["method"] for c in client.calls] == ["PATCH", "PATCH"] + assert client.calls[0]["path"] == "/evaluate/tool-schemas/t1" + assert client.calls[0]["json"] == {"pricePerCallUsd": 0.01} + assert client.calls[1]["json"] == {"pricePerCallUsd": None} + assert all(c["retry"] is False for c in client.calls) + + +def test_update_meta_sends_only_the_fields_given(): + client = RecordingClient([{"ok": True}]) + client.update_tool_schema_meta("t1", description="Order lookup") + assert client.calls[0]["json"] == {"description": "Order lookup"} + + +def test_a_bad_price_is_refused_before_any_request(): + client = RecordingClient([]) + tools = ToolSchemaClient(client) + with pytest.raises(ValueError, match="non-negative"): + tools.create(name="x", definition="{}", price_per_call_usd=-1) + with pytest.raises(ValueError, match="number"): + tools.set_price("t1", "cheap") # type: ignore[arg-type] + assert client.calls == [] + + +def test_delete_never_retries(): + client = RecordingClient([{}]) + ToolSchemaClient(client).delete("t1") + assert client.calls[0]["method"] == "DELETE" + assert client.calls[0]["path"] == "/evaluate/tool-schemas/t1" + assert client.calls[0]["retry"] is False + + +def test_get_or_create_keeps_the_existing_row_and_its_price(): + client = RecordingClient([{"toolSchemas": [{"_id": "t1", "name": "geocode", "pricePerCallUsd": 0.25}]}]) + existing = ToolSchemaClient(client).get_or_create(name="geocode", definition="{}", price_per_call_usd=9) + assert existing["pricePerCallUsd"] == 0.25 + assert [c["method"] for c in client.calls] == ["GET"]