Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 43 additions & 1 deletion agentx/evaluations/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,22 @@ def _resolve_scorer_id(scorer_id: Optional[str], evaluation_settings_id: Optiona
return scorer_id or evaluation_settings_id


# "Argument not given" for update_tool_schema_meta, where None is itself a meaningful value
# (clear the price).
_UNSET: object = object()


def _validated_tool_price(value: float) -> float:
"""Local mirror of the engine's check: USD per call, finite and non-negative."""
try:
price = float(value)
except (TypeError, ValueError):
raise ValueError(f"price_per_call_usd must be a number of USD per call, got {value!r}") from None
if price != price or price in (float("inf"), float("-inf")) or price < 0:
raise ValueError(f"price_per_call_usd must be a finite, non-negative number, got {value!r}")
return price


class EvaluationsClient:
def __init__(
self,
Expand Down Expand Up @@ -723,13 +739,39 @@ def list_tool_schemas(self) -> List[dict]:
data = self._request("GET", "/evaluate/tool-schemas", base=self._api_root)
return data.get("toolSchemas", []) if isinstance(data, dict) else data

def create_tool_schema(self, *, name: str, definition: str, description: Optional[str] = None) -> dict:
def create_tool_schema(
self, *, name: str, definition: str, description: Optional[str] = None,
price_per_call_usd: Optional[float] = None,
) -> dict:
payload: dict = {"name": name, "definition": definition}
if description is not None:
payload["description"] = description
if price_per_call_usd is not None:
payload["pricePerCallUsd"] = _validated_tool_price(price_per_call_usd)
# Server-side write - no transport retry (see init_run's comment).
return self._request("POST", "/evaluate/tool-schemas", base=self._api_root, json=payload, retry=False)

def update_tool_schema_meta(
self, tool_schema_id: str, *, description: Optional[str] = None,
price_per_call_usd: Optional[float] = _UNSET, test_endpoint_url: Optional[str] = _UNSET, # type: ignore[assignment]
) -> dict:
"""Metadata-only edit (PATCH): description, per-call price, test endpoint. Never touches
the version log. ``price_per_call_usd=None`` clears the price (the tool becomes free)."""
payload: dict = {}
if description is not None:
payload["description"] = description
if price_per_call_usd is not _UNSET:
payload["pricePerCallUsd"] = None if price_per_call_usd is None else _validated_tool_price(price_per_call_usd)
if test_endpoint_url is not _UNSET:
payload["testEndpointUrl"] = test_endpoint_url
return self._request(
"PATCH", f"/evaluate/tool-schemas/{tool_schema_id}", base=self._api_root, json=payload, retry=False,
)

def delete_tool_schema(self, tool_schema_id: str) -> None:
"""Deletes the tool schema and its version history."""
self._request("DELETE", f"/evaluate/tool-schemas/{tool_schema_id}", base=self._api_root, retry=False)

def get_tool_schema_examples(self, tool_schema_id: str, window: Optional[str] = None) -> dict:
params = {"window": window} if window else None
return self._request(
Expand Down
34 changes: 29 additions & 5 deletions agentx/evaluations/tool_schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,39 @@ def __init__(self, client: "EvaluationsClient"):
def list(self) -> List[dict]:
return self._client.list_tool_schemas()

def create(self, *, name: str, definition: str, description: Optional[str] = None) -> dict:
return self._client.create_tool_schema(name=name, definition=definition, description=description)
def create(
self, *, name: str, definition: str, description: Optional[str] = None,
price_per_call_usd: Optional[float] = None,
) -> dict:
"""``price_per_call_usd`` is what one call of this tool costs you (a paid external API
billed per request). Every recorded call is charged, failed ones included; the spend
shows as the "Tool calls" segment of the Overview cost chart and counts toward the
``estimatedCostUsd`` alert metric. Leave it unset for a free tool."""
return self._client.create_tool_schema(
name=name, definition=definition, description=description, price_per_call_usd=price_per_call_usd,
)

def get_or_create(self, *, name: str, definition: str, description: Optional[str] = None) -> dict:
"""Idempotent register: returns the existing schema of this name if present."""
def get_or_create(
self, *, name: str, definition: str, description: Optional[str] = None,
price_per_call_usd: Optional[float] = None,
) -> dict:
"""Idempotent register: returns the existing schema of this name if present (its stored
price wins - use :meth:`set_price` to change it)."""
existing = next((t for t in self.list() if t.get("name") == name), None)
if existing:
return existing
return self.create(name=name, definition=definition, description=description)
return self.create(
name=name, definition=definition, description=description, price_per_call_usd=price_per_call_usd,
)

def set_price(self, tool_schema_id: str, price_per_call_usd: Optional[float]) -> dict:
"""Set (or clear with ``None``) a registered tool's USD price per call. Metadata only -
the definition's version log is untouched."""
return self._client.update_tool_schema_meta(tool_schema_id, price_per_call_usd=price_per_call_usd)

def delete(self, tool_schema_id: str) -> None:
"""Deletes the tool schema and its version history."""
self._client.delete_tool_schema(tool_schema_id)

def examples(self, tool_schema_id: str, window: Optional[str] = None) -> dict:
"""Failure evidence recorded against this tool (agent-tool-failure signals, low-rated
Expand Down
10 changes: 10 additions & 0 deletions agentx/monitor/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,16 @@ def metrics(
params[key] = value
return self._request("GET", "/metrics", params=params)

def cost_trend(self, window: str = "7d") -> dict:
"""Overview's "Total cost" chart over a window ("24h", "7d", "30d"): priced spend per
bucket and in total, stacked by model, with two reserved segments - ``"eval runs"``
(offline-evaluation spend) and ``"tool calls"`` (registered tools' per-call prices, split
per tool in ``totalsByTool``). Unpriced models and tools contribute $0."""
return self._request(
"GET", "/agent-monitoring/cost-trend",
base=self._api_root(), params={"window": window},
)

def topics(self, window: str = "7d") -> dict:
"""The Topics view's data over a window ("24h", "7d", "30d"): LLM-classified themes of
sampled production traffic with per-topic counts and sentiment. Empty until Topics is
Expand Down
81 changes: 81 additions & 0 deletions tests/test_tool_schemas.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
"""Unit tests for the tool schema registry client - wire-level, no engine required: the
per-call price rides the camelCase wire on create, PATCH edits are metadata-only and can clear
the price with None, and a bad price is refused locally before any request."""

from typing import Any, Dict, List

import pytest

from agentx.evaluations.client import EvaluationsClient
from agentx.evaluations.tool_schemas import ToolSchemaClient


class RecordingClient(EvaluationsClient):
def __init__(self, responses: List[Any]):
super().__init__(api_key="agtx_test", base_url="http://engine:4700/api/v1")
self.calls: List[Dict[str, Any]] = []
self._responses = responses

def _request(self, method: str, path: str, timeout: int = 30, base=None, retry: bool = True, **kwargs: Any) -> Any: # type: ignore[override]
self.calls.append({"method": method, "path": path, "retry": retry, **kwargs})
return self._responses.pop(0) if self._responses else {}


def test_create_sends_the_price_on_the_camelcase_wire_without_retry():
client = RecordingClient([{"_id": "t1", "name": "geocode", "pricePerCallUsd": 0.25}])
created = ToolSchemaClient(client).create(name="geocode", definition="{}", price_per_call_usd=0.25)
call = client.calls[0]
assert call["method"] == "POST"
assert call["path"] == "/evaluate/tool-schemas"
assert call["retry"] is False
assert call["json"] == {"name": "geocode", "definition": "{}", "pricePerCallUsd": 0.25}
assert created["pricePerCallUsd"] == 0.25


def test_create_omits_the_price_when_not_given():
client = RecordingClient([{}])
ToolSchemaClient(client).create(name="clock", definition="{}")
assert "pricePerCallUsd" not in client.calls[0]["json"]


def test_set_price_patches_metadata_and_none_clears_it():
client = RecordingClient([{"ok": True}, {"ok": True}])
tools = ToolSchemaClient(client)
tools.set_price("t1", 0.01)
tools.set_price("t1", None)
assert [c["method"] for c in client.calls] == ["PATCH", "PATCH"]
assert client.calls[0]["path"] == "/evaluate/tool-schemas/t1"
assert client.calls[0]["json"] == {"pricePerCallUsd": 0.01}
assert client.calls[1]["json"] == {"pricePerCallUsd": None}
assert all(c["retry"] is False for c in client.calls)


def test_update_meta_sends_only_the_fields_given():
client = RecordingClient([{"ok": True}])
client.update_tool_schema_meta("t1", description="Order lookup")
assert client.calls[0]["json"] == {"description": "Order lookup"}


def test_a_bad_price_is_refused_before_any_request():
client = RecordingClient([])
tools = ToolSchemaClient(client)
with pytest.raises(ValueError, match="non-negative"):
tools.create(name="x", definition="{}", price_per_call_usd=-1)
with pytest.raises(ValueError, match="number"):
tools.set_price("t1", "cheap") # type: ignore[arg-type]
assert client.calls == []


def test_delete_never_retries():
client = RecordingClient([{}])
ToolSchemaClient(client).delete("t1")
assert client.calls[0]["method"] == "DELETE"
assert client.calls[0]["path"] == "/evaluate/tool-schemas/t1"
assert client.calls[0]["retry"] is False


def test_get_or_create_keeps_the_existing_row_and_its_price():
client = RecordingClient([{"toolSchemas": [{"_id": "t1", "name": "geocode", "pricePerCallUsd": 0.25}]}])
existing = ToolSchemaClient(client).get_or_create(name="geocode", definition="{}", price_per_call_usd=9)
assert existing["pricePerCallUsd"] == 0.25
assert [c["method"] for c in client.calls] == ["GET"]
Loading