diff --git a/asap-tools/experiments/constants.py b/asap-tools/experiments/constants.py index 487e7e2c..fc9cdd40 100644 --- a/asap-tools/experiments/constants.py +++ b/asap-tools/experiments/constants.py @@ -41,6 +41,12 @@ PROMETHEUS_CONFIG_DIR = "prometheus_config" PROMETHEUS_CONFIG_FILE = "prometheus.yml" +# Long-window baseline queries can run for minutes; Prometheus would cancel +# them after its 2-minute default. +PROMETHEUS_QUERY_TIMEOUT = "24h" +# The 50M default rejects 24h windows over a few hundred 1s-scraped series. +# Peak query memory is roughly 16 B per sample (~8 GB at this cap). +PROMETHEUS_QUERY_MAX_SAMPLES = 500_000_000 PROMETHEUS_REMOTE_WRITE_ENDPOINT = "/api/v1/write" HTTP_READINESS_CURL_TIMEOUT_SECONDS = 5 QUERY_ENGINE_RUNTIME_INFO_ENDPOINT = "/api/v1/status/runtimeinfo" diff --git a/asap-tools/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 b/asap-tools/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 index 176a6e7a..adf14db5 100644 --- a/asap-tools/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 +++ b/asap-tools/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 @@ -11,7 +11,14 @@ services: command: - "-storageDataPath=/victoria-metrics-data" - "-httpListenAddr=:8428" - - "-retentionPeriod=1d" + # Keep a 24 h window plus the run, and let long-window queries finish + # (the defaults are 30 s per query and 10 s waiting in the queue). + - "-retentionPeriod=3d" + - "-search.maxQueryDuration=24h" + - "-search.maxQueueDuration=24h" + # Raise the per-query caps so 24 h windows over many series aren't rejected. + - "-search.maxSamplesPerQuery=10000000000" + - "-search.maxPointsPerTimeseries=1000000" {% if cpu_limit or memory_limit -%} deploy: resources: diff --git a/asap-tools/experiments/experiment_utils/services/docker_prometheus.py b/asap-tools/experiments/experiment_utils/services/docker_prometheus.py index 88c0e519..90ac419b 100644 --- a/asap-tools/experiments/experiment_utils/services/docker_prometheus.py +++ b/asap-tools/experiments/experiment_utils/services/docker_prometheus.py @@ -6,7 +6,12 @@ from .base import DockerServiceBase from experiment_utils.providers.base import InfrastructureProvider -from constants import PROMETHEUS_CONFIG_DIR, PROMETHEUS_CONFIG_FILE +from constants import ( + PROMETHEUS_CONFIG_DIR, + PROMETHEUS_CONFIG_FILE, + PROMETHEUS_QUERY_MAX_SAMPLES, + PROMETHEUS_QUERY_TIMEOUT, +) class DockerPrometheusService(DockerServiceBase): @@ -98,6 +103,8 @@ def start( "--web.console.libraries=/etc/prometheus/console_libraries", "--web.console.templates=/etc/prometheus/consoles", "--web.enable-lifecycle", + f"--query.timeout={PROMETHEUS_QUERY_TIMEOUT}", + f"--query.max-samples={PROMETHEUS_QUERY_MAX_SAMPLES}", ] ) diff --git a/asap-tools/experiments/experiment_utils/services/prometheus.py b/asap-tools/experiments/experiment_utils/services/prometheus.py index 07d54137..60d210ea 100644 --- a/asap-tools/experiments/experiment_utils/services/prometheus.py +++ b/asap-tools/experiments/experiment_utils/services/prometheus.py @@ -8,7 +8,12 @@ from .base import BaseService from experiment_utils.providers.base import InfrastructureProvider -from constants import PROMETHEUS_CONFIG_DIR, PROMETHEUS_CONFIG_FILE +from constants import ( + PROMETHEUS_CONFIG_DIR, + PROMETHEUS_CONFIG_FILE, + PROMETHEUS_QUERY_MAX_SAMPLES, + PROMETHEUS_QUERY_TIMEOUT, +) class PrometheusService(BaseService): @@ -89,7 +94,12 @@ def _start_prometheus(self, experiment_output_dir: str) -> None: time.sleep(3) # Start prometheus - cmd = f"./prometheus --config.file={PROMETHEUS_CONFIG_FILE} > /dev/null 2>&1 < /dev/null &" + cmd = ( + f"./prometheus --config.file={PROMETHEUS_CONFIG_FILE}" + f" --query.timeout={PROMETHEUS_QUERY_TIMEOUT}" + f" --query.max-samples={PROMETHEUS_QUERY_MAX_SAMPLES}" + " > /dev/null 2>&1 < /dev/null &" + ) self.provider.execute_command( node_idx=self.node_offset, cmd=cmd, diff --git a/asap-tools/queriers/prometheus-client/classes/clickhouse_query_client.py b/asap-tools/queriers/prometheus-client/classes/clickhouse_query_client.py index 1ab97859..d42a9bc5 100644 --- a/asap-tools/queriers/prometheus-client/classes/clickhouse_query_client.py +++ b/asap-tools/queriers/prometheus-client/classes/clickhouse_query_client.py @@ -1,5 +1,5 @@ import requests -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple from requests.adapters import HTTPAdapter from .query_client import QueryClient, QueryResponse @@ -15,7 +15,9 @@ def __init__( database: str = "default", user: str = "default", password: str = "", - timeout: int = 30, + # (connect, read): long-window baseline queries can take minutes, but a + # dead server must still fail instead of hanging the querier. + timeout: Tuple[float, float] = (10, 24 * 60 * 60), **kwargs: Any, ): super().__init__(server_url, server_name) @@ -96,7 +98,7 @@ def execute_query( return QueryResponse( success=False, data=None, - error_message=f"Request timed out after {self.timeout}s", + error_message=f"Request timed out (connect, read timeout: {self.timeout}s)", ) except Exception as e: return QueryResponse(