Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions docs/how-to/just-run-blueapi-and-services-locally.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,23 @@ To log in through the BlueAPI CLI:
4. Run `blueapi controller plans` to check that the log in has succeeded

By default the BlueAPI instance will be available via the OAuth2 proxy at `localhost:4180`, and Tiled through its OAuth2 proxy at `localhost:4181`.

## Metrics and traces

BlueAPI reports metrics and traces with OpenTelemetry, pushed over OTLP when `OTLP_EXPORT_ENABLED=true` (to wherever the standard `OTEL_EXPORTER_OTLP_*` variables point). To look at them locally, the compose file has an optional `monitoring` profile with Prometheus, Grafana, Jaeger and Tempo, behind an OpenTelemetry Collector that sends traces to both Jaeger and Tempo:

1. `just compose --profile monitoring up -d` starts the usual services plus the monitoring stack
2. `just serve-monitored` runs the server with its telemetry pointed at that stack (see `tests/system_tests/services/monitoring/otel.env`)
3. Run some plans, e.g. `just run sleep '{"time": 2}'`
4. Open:
- Grafana at <http://localhost:3000>, which opens on the BlueAPI dashboard
- Prometheus at <http://localhost:9090>
- Jaeger at <http://localhost:16686>
- Tempo through Grafana: Explore, then the Tempo datasource

| Metric | Labels | Meaning |
| --- | --- | --- |
| `blueapi.task.duration` (`blueapi_task_duration_seconds_*` in Prometheus) | `task.name`, `outcome` (`success`/`failure`) | Histogram of how long each task took; its count is the number of runs |
| `http.server.*` | method, route, status code | Request metrics for the REST API, from the FastAPI instrumentation |

Every metric also carries `service.name` (`BlueAPI` for the API process, `BlueAPIWorker` for the worker subprocess) and `service.instrument`, taken from the `INSTRUMENT` or `BEAMLINE` environment variable.
6 changes: 6 additions & 0 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ serve *OPTS:
source tests/system_tests/.env
uv run blueapi -c {{ TEST_CONFIG }} {{ OPTS }} serve

[doc('Run the server exporting metrics and traces to the monitoring stack (`just compose --profile monitoring up -d`)')]
serve-monitored *OPTS:
#!/usr/bin/env bash
source tests/system_tests/services/monitoring/otel.env
just serve {{ OPTS }}

[doc('Run a plan using the blueapi CLI')]
run PLAN PARAMS:
uv run blueapi -c {{ TEST_CONFIG }} controller run -i {{ SESSION }} {{ PLAN }} '{{ PARAMS }}'
Expand Down
2 changes: 2 additions & 0 deletions src/blueapi/cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
)
from blueapi.core import OTLP_EXPORT_ENABLED, DataEvent
from blueapi.log import set_up_logging
from blueapi.metrics import setup_metrics
from blueapi.service.model import DeviceResponse, PlanResponse, SourceInfo, TaskRequest
from blueapi.worker import ProgressEvent, WorkerEvent
from blueapi.worker.event import TaskError, TaskResult
Expand Down Expand Up @@ -249,6 +250,7 @@ def start_application(obj: dict):
observability context.
"""
setup_tracing("BlueAPI", OTLP_EXPORT_ENABLED)
setup_metrics(OTLP_EXPORT_ENABLED)
start(config)


Expand Down
35 changes: 35 additions & 0 deletions src/blueapi/metrics.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
from opentelemetry.metrics import get_meter, set_meter_provider
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.trace import get_tracer_provider

METER = get_meter("blueapi")

TASK_DURATION = METER.create_histogram(
"blueapi.task.duration",
unit="s",
description="Time taken to run a task, by task name and outcome",
# Plans run for anything between a fraction of a second and hours
explicit_bucket_boundaries_advisory=[
0.1, 0.5, 1, 5, 10, 30, 60, 300, 600, 1800, 3600, 7200, 14400
],
) # fmt: skip


def setup_metrics(with_otlp_export: bool = True) -> None:
"""Push metrics over OTLP, to the same place and with the same resource
(service name, instrument) as the traces. Call after setup_tracing.

Pushing rather than serving a /metrics endpoint means the API process and
the worker subprocess each report for themselves, with nothing shared.
"""
if with_otlp_export:
# The API only allows the global provider to be set once, later calls
# are ignored with a warning
set_meter_provider(
MeterProvider(
metric_readers=[PeriodicExportingMetricReader(OTLPMetricExporter())],
resource=getattr(get_tracer_provider(), "resource", None),
)
)
11 changes: 11 additions & 0 deletions src/blueapi/worker/task_worker.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import logging
import time
import uuid
from collections.abc import Iterable, Mapping
from concurrent.futures import Future, ThreadPoolExecutor
Expand Down Expand Up @@ -32,6 +33,7 @@
)
from blueapi.core.bluesky_event_loop import configure_bluesky_event_loop
from blueapi.log import plan_tag_filter_context
from blueapi.metrics import TASK_DURATION, setup_metrics
from blueapi.utils.base_model import BlueapiBaseModel
from blueapi.utils.thread_exception import handle_all_exceptions

Expand Down Expand Up @@ -152,6 +154,7 @@ def __init__(
self._broadcast_statuses = broadcast_statuses
self._current_task_otel_context = None
setup_tracing("BlueAPIWorker", OTLP_EXPORT_ENABLED)
setup_metrics(OTLP_EXPORT_ENABLED)

@start_as_current_span(TRACER, "task_id")
def clear_task(self, task_id: str) -> str:
Expand Down Expand Up @@ -432,16 +435,24 @@ def process_task():
self._current = next_task
self._current.is_pending = False
meta = {"task_id": self._current.task_id}
start = time.monotonic()
outcome = "failure"
try:
result = self._current.task.do_task(self._ctx)
LOGGER.info(
"Task ran successfully - returned: %s", result, extra=meta
)
self._current.set_result(result)
outcome = "success"
except Exception as e:
LOGGER.error("Task failed", extra=meta)
self._current.set_exception(e)
self._report_error(e)
finally:
TASK_DURATION.record(
time.monotonic() - start,
{"task.name": next_task.task.name, "outcome": outcome},
)

with plan_tag_filter_context(next_task.task.name, LOGGER):
if self._current_task_otel_context is not None:
Expand Down
59 changes: 59 additions & 0 deletions tests/system_tests/compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -154,3 +154,62 @@ services:
condition: service_healthy
security_opt:
- label=disable

# Local dev only: `just compose --profile monitoring up -d` then `just serve-monitored`
otel-collector:
profiles: [monitoring]
image: otel/opentelemetry-collector:0.139.0
command: ["--config=/etc/otel-collector.yaml"]
ports:
- "4318:4318" # OTLP http, metrics and traces from blueapi
volumes:
- ./services/monitoring/otel-collector.yaml:/etc/otel-collector.yaml:ro
security_opt:
- label=disable

jaeger:
profiles: [monitoring]
image: jaegertracing/jaeger:2.11.0
ports:
- "16686:16686" # UI
security_opt:
- label=disable

# Queried through Grafana, it has no UI of its own
tempo:
profiles: [monitoring]
image: grafana/tempo:2.9.0
command: ["-config.file=/etc/tempo.yaml"]
volumes:
- ./services/monitoring/tempo.yaml:/etc/tempo.yaml:ro
security_opt:
- label=disable

prometheus:
profiles: [monitoring]
image: prom/prometheus:v3.7.3
command:
- --config.file=/etc/prometheus/prometheus.yaml
- --web.enable-otlp-receiver # metrics from the collector
ports:
- "9090:9090"
volumes:
- ./services/monitoring/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro
security_opt:
- label=disable

grafana:
profiles: [monitoring]
image: grafana/grafana:12.3.0
environment:
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
- GF_AUTH_DISABLE_LOGIN_FORM=true
- GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/var/lib/grafana/dashboards/blueapi.json
ports:
- "3000:3000"
volumes:
- ./services/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
- ./services/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
security_opt:
- label=disable
Loading
Loading