diff --git a/.design_docs/accelerated-random-workload-eval.md b/.design_docs/accelerated-random-workload-eval.md new file mode 100644 index 00000000..e8983734 --- /dev/null +++ b/.design_docs/accelerated-random-workload-eval.md @@ -0,0 +1,262 @@ +# Accelerated random-workload E2E evaluation + +## Purpose + +Run a practical, end-to-end version of the paper's §6.2 experiment before the +full PR 862 workload. The experiment should retain the important workload +shape—many input series and few returned series—and show the benefit of +planning a *workload* of related RQEs rather than isolated queries. + +The experiment has ten deterministic random workloads. Each workload contains +exactly ten atomic user queries; binary arithmetic queries are out of scope for +this first version. + +The systems are ASAPQuery, Prometheus, VictoriaMetrics, raw ClickHouse, and +ClickHouse with exact materialized views (MVs). All arms receive identical +data and use the same explicit logical evaluation timestamps. + +## Data model + +One Prometheus-style gauge, `data`, and a companion cumulative counter, +`data_total`, have 10,000 series. Their labels form a hierarchy: + +```text +region: 4 + service: 5 per region + job: 10 per service + label_0: 50 per job +``` + +Thus `4 * 5 * 10 * 50 = 10,000` leaf series. Samples arrive every 100 ms +(10 samples/s per series). Values should have the existing heavy-tailed +distribution used by the synthetic feeder; event/counter increments must be +Zipf-skewed so count-weighted top-k is non-degenerate. + +The generator only exposes hierarchy-closed grouping sets: + +| Grouping `G` | Number of output groups | +| --- | ---: | +| `region` | 4 | +| `region, service` | 20 | +| `region, service, job` | 200 | + +`label_0` is retained for internal, full-cardinality work but is not a +standalone visible grouping: it would produce 10,000 output series. + +## Time grid + +Each RQE chooses one of these lookback/repetition pairs. The alternatives +give the seeded generator some variation without making the query schedule +unbounded. + +| Lookback `W` | Allowed repeat intervals `T` | +| ---: | ---: | +| 100 ms | 1 s | +| 1 min | 1 s, 2 s | +| 5 min | 5 s, 10 s | +| 10 min | 10 s, 20 s | +| 20 min | 30 s, 60 s | + +The maximum accelerated backfill is therefore only 20 minutes of event-time +history. The first 100 ms class is for spatial queries; temporal queries use +the 1--20 minute classes. The spatial class targets a 100 ms repetition +interval. Current exact-MV support is whole-second only, so this class is +gated on [#876](https://github.com/ProjectASAP/ASAPQuery/issues/876), which +adds millisecond interval metadata, `DateTime64` bucket flooring and edge +handling, a sub-second query schedule, and raw-versus-MV equivalence tests. +This is a limitation of the current baseline generator, not of ClickHouse. + +## Query-template catalogue + +Every template instance emits both the optimizer-facing RQE(s) and a concrete +query rendering. The generator records the query-to-RQE mapping in its +manifest. + +### Sum + +```promql +sum by (G) (sum_over_time(data[W])) +``` + +This is a low-output exact control query: it returns 4, 20, or 200 values. + +### Top-k by sum + +```promql +topk by (G) (k, sum_over_time(data[W])) +``` + +This is a value-weighted top-k RQE. Use `k` in `{4, 8}`. It returns at most +`k` values per group; `G = region, service` with `k = 8`, for example, returns +at most 160 values. + +### Top-k by count + +Use a skewed event/counter metric and rank the per-series increase: + +```promql +topk by (G) (k, increase(data_total[W])) +``` + +This is a count-weighted top-k RQE. `count_over_time(data[W])` is not an +acceptable substitute: at a fixed scrape cadence its values are nearly equal +for every healthy series. The engine's per-series counter-delta path must be +smoke-tested before this template is admitted to an E2E workload. + +### Rate + +The rate RQE is computed at the full series grouping and must have a final +low-cardinality reducer in the rendered query: + +```promql +topk by (G) (k, rate(data_total[W])) +``` + +A bare `rate(data_total[W])` returns all 10,000 leaf series and is excluded. +Rate-plus-top-k shares the counter-delta prerequisite with count-weighted +top-k, so it is a gated template rather than an unconditional member of the +first pilot workload. + +### Quantile + +For the 100 ms spatial class, PromQL has exact grouped semantics: + +```promql +quantile by (G) (phi, data) +``` + +For a temporal grouped quantile, PromQL cannot directly express a pooled +quantile over every raw sample in each group. The PromQL-compatible rendering +is instead the average of each leaf series' temporal quantile: + +```promql +avg by (G) (quantile_over_time(phi, data[W])) +``` + +This is deliberately called **average of per-series quantiles**, not a group +quantile. It has low output cardinality and is the semantics that ClickHouse +must emulate when comparing the same query across all five systems. + +SQL can also express the distinct, stronger pooled-quantile semantics: + +```sql +SELECT G, QUANTILE(phi, value) +FROM data +WHERE ts > :eval_time - INTERVAL W AND ts <= :eval_time +GROUP BY G +``` + +That SQL query is a separate logical query and belongs only to the SQL-capable +comparison; it must not be reported as equivalent to the PromQL average of +per-series quantiles. + +### Cardinality + +True cardinality uses a record/value field, such as `user_id`, rather than +counting Prometheus series: + +```sql +SELECT G, COUNT(DISTINCT user_id) +FROM events +WHERE ts > :eval_time - INTERVAL W AND ts <= :eval_time +GROUP BY G +``` + +It is an HLL/cardinality RQE and has 4, 20, or 200 outputs. Prometheus and +VictoriaMetrics do not have an equivalent distinct-value query. Cardinality +queries are therefore executed and reported in the ASAPQuery/ClickHouse/raw-MV +SQL arm, not as a five-system like-for-like comparison. + +## Workload generator + +The generator takes a stable workload seed and writes one immutable manifest +per workload. It produces ten workloads with ten user queries each, for 100 +queries total. A query specification contains: + +```text +id, capability, metric/value field, grouping G, lookback W, interval T, +quantile phi or top-k k, accuracy SLA, rendered PromQL or SQL, RQE IDs +``` + +The manifest also contains the label hierarchy, cardinalities of every legal +grouping, value/key distribution facts, feeder seed, and support class +(`common-promql`, `counter-gated-promql`, or `sql-only`). + +Sampling is constrained rather than purely independent: + +- exactly ten atomic queries per workload and no binary queries; +- at least three quantile queries, at least two top-k queries, and at least + one sum query; +- queries from at least three time classes; +- at least one roll-up chain: same metric, capability, and `(W, T)` at nested + groupings, for example `(region, service, job)` and `(region, service)`; +- no duplicate `(capability, metric, G, W, T, parameters)` RQEs; +- visible output cardinality is normally at most 160; the 200-group quantile + and sum cases are allowed as a limited high-output control; +- `phi` is sampled from `{0.50, 0.75, 0.90, 0.95, 0.99}` and `k` from `{4, 8}`. + +The generator must validate all rendering constraints before writing a +workload. In particular, `W` must be compatible with the MV bucket/interval +policy, each RQE needs facts and accuracy coverage, and every requested +grouping must be legal in the hierarchy. + +## Planning and rendering model + +RQEs are the source of truth for optimization, but query text is not produced +by an arbitrary RQE-to-PromQL converter. A template emits both in parallel: + +```text +template instance + -> RQEs: capability, grouping, W, T, SLA, distribution facts + -> user query: PromQL or SQL, annotated with the RQE IDs it consumes +``` + +The offline optimizer chooses deployments for the RQEs. Its frozen output is +then converted into ASAPQuery's engine configuration. The generated manifest +and the converted plan must agree: every query leaf has exactly one planned, +writable deployment. + +Fine-to-coarse roll-up is a desired optimization opportunity, not an assumed +property. Before accepting a random workload, the offline-plan/conversion +preflight must verify that its selected roll-up mappings are supported by the +current planner and engine. If they are not, retain the workload only with +separate deployments or reject/regenerate it; do not silently claim a roll-up +benefit. + +## Staged execution + +1. Implement the manifest-driven random workload generator and its validation. +2. Generate the ten seeded workloads; inspect them and freeze the manifests. +3. Choose one representative workload with only `common-promql` templates for + the five-system pilot. Do not block this pilot on cardinality or the + counter-delta-gated templates. +4. Run the optimizer offline at the actual 10 samples/s/series rate, with the + chosen accuracy SLA and no latency SLA. Archive inputs, curves/costs, + selected plan, and converted ASAPQuery configuration. +5. Render baseline inputs from the same manifest: PromQL for Prometheus and + VictoriaMetrics; semantically equivalent SQL for raw ClickHouse and MV + ClickHouse. For a five-system quantile comparison, ClickHouse emulates the + PromQL average-of-per-series-quantiles semantics. +6. Run the one workload end to end on ASAPQuery, Prometheus, VictoriaMetrics, + ClickHouse, and ClickHouse+MV. Accelerated-backfill 20 minutes of logical + history, drain ingest backlogs, then run a paced measurement tail at the + frozen explicit evaluation times. +7. Check baseline agreement, ASAPQuery accuracy, query completion/late + firings, CPU, memory, disk, and per-query latency. Only after this pilot + passes should the other nine workloads run. +8. Run SQL-only cardinality and pooled-quantile queries as a separately + labelled ASAPQuery/ClickHouse/ClickHouse+MV arm; never fold their results + into a five-system apples-to-apples aggregate. + +## Pilot acceptance gates + +- The generator reproduces a byte-identical manifest for a fixed seed. +- Every selected RQE has curve and workload-fact coverage. +- The frozen plan converts to a writable ASAPQuery configuration. +- Every common query completes on all five systems at the same logical + evaluation time. +- Raw ClickHouse and MV ClickHouse agree within floating-point tolerance for + their shared exact semantics. +- ASAPQuery's observed error satisfies the workload's accuracy SLA. +- Counter-based top-k/rate enters only after its dedicated correctness and + planner-conversion smoke test passes. diff --git a/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py b/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py new file mode 100644 index 00000000..bc7db547 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Render frozen random-workload manifests into offline planner inputs.""" + +import argparse +import json +from collections import defaultdict +from pathlib import Path + +import yaml + + +ACCURACY_SLA = 0.95 +PROMQL_METRICS = ("data", "data_total") +SQL_EVENTS_TABLE = { + "name": "events", + "time_column": "ts", + "value_columns": ["user_id"], + "metadata_columns": ["region", "service", "job", "label_0"], +} + + +def query_groups(queries, query_key): + """Group rendered query text by its fixed repetition interval.""" + grouped = defaultdict(list) + for query in queries: + rendered = query[query_key] + if rendered: + grouped[query["interval_ms"]].append(rendered) + return [ + { + "id": index, + "repetition_delay_ms": interval_ms, + "queries": rendered, + "controller_options": {"accuracy_sla": ACCURACY_SLA}, + } + for index, (interval_ms, rendered) in enumerate(sorted(grouped.items()), start=1) + ] + + +def render(workload): + """Return planner inputs and query schedule derived from one manifest.""" + labels = [label["name"] for label in workload["labels"]] + queries = workload["queries"] + return { + "queries": queries, + "promql_planner": { + "metrics": [{"metric": metric, "labels": labels} for metric in PROMQL_METRICS], + "query_groups": query_groups(queries, "promql"), + "aggregate_cleanup": {"policy": "read_based"}, + }, + "sql_planner": { + "tables": [SQL_EVENTS_TABLE], + "query_groups": query_groups(queries, "sql"), + "aggregate_cleanup": {"policy": "read_based"}, + }, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + workload = json.loads(args.manifest.read_text()) + rendered = render(workload) + args.out.mkdir(parents=True, exist_ok=True) + for name in ("promql_planner", "sql_planner"): + path = args.out / f"{name}.yaml" + path.write_text(yaml.safe_dump(rendered[name], sort_keys=False)) + print(path) + schedule = args.out / "queries.json" + schedule.write_text(json.dumps(rendered["queries"], indent=2, sort_keys=True) + "\n") + print(schedule) + + +if __name__ == "__main__": + main() diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads.py b/asap-tools/experiments/datasets/fake_metric/random_workloads.py new file mode 100644 index 00000000..1029fc80 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Generate deterministic manifests for accelerated random E2E workloads. + +The manifest is the source of truth for an optimizer RQE and the query text +that the E2E runners execute. It deliberately does not invoke a planner or +write runner-specific configuration. +""" + +import argparse +import json +import math +import random +from pathlib import Path + + +WORKLOAD_COUNT = 10 +QUERIES_PER_WORKLOAD = 10 +SAMPLES_PER_SEC_PER_SERIES = 10 +ACCURACY_SLA = 0.95 +SECOND_MS = 1_000 +MINUTE_MS = 60 * SECOND_MS +SPATIAL_LOOKBACK_MS = 100 +SPATIAL_INTERVAL_MS = 100 +LABEL_HIERARCHY = ( + ("region", 4, None), + ("service", 5, "region"), + ("job", 10, "service"), + ("label_0", 50, "job"), +) +LABEL_CARDINALITIES = {name: cardinality for name, cardinality, _ in LABEL_HIERARCHY} +METRIC_LABELS = [ + {"name": name, "cardinality": cardinality, **({"child_of": parent} if parent else {})} + for name, cardinality, parent in LABEL_HIERARCHY +] +GROUPINGS = (("region",), ("region", "service"), ("region", "service", "job")) +GROUP_COUNTS = { + grouping: math.prod(LABEL_CARDINALITIES[label] for label in grouping) + for grouping in GROUPINGS +} +TEMPORAL_TIMES = ( + (MINUTE_MS, (SECOND_MS, 2 * SECOND_MS)), + (5 * MINUTE_MS, (5 * SECOND_MS, 10 * SECOND_MS)), + (10 * MINUTE_MS, (10 * SECOND_MS, 20 * SECOND_MS)), + (20 * MINUTE_MS, (30 * SECOND_MS, MINUTE_MS)), +) +QUANTILES = (0.50, 0.75, 0.90, 0.95, 0.99) +TOPK_VALUES = (4, 8) + + +def duration(milliseconds): + """Render a whole-minute duration for the supported PromQL templates.""" + return f"{milliseconds // MINUTE_MS}m" + + +def grouping_text(grouping): + return ", ".join(grouping) + + +def choose_temporal_time(rng): + lookback_ms, intervals = rng.choice(TEMPORAL_TIMES) + return lookback_ms, rng.choice(intervals) + + +def query( + query_id, + capability, + grouping, + lookback_ms, + interval_ms, + rendered, + support, + **params, +): + """Create one manifest query and its single optimizer-facing RQE.""" + rqe = { + "id": f"{query_id}/rqe", + "capability": capability, + "metric": "data_total" if capability in ("rate", "topk_by_count") else "data", + "grouping_labels": list(grouping), + "lookback_ms": lookback_ms, + "interval_ms": interval_ms, + "accuracy_sla": ACCURACY_SLA, + **params, + } + return { + "id": query_id, + "support": support, + "grouping": list(grouping), + "output_groups": GROUP_COUNTS[grouping], + "output_series_upper_bound": GROUP_COUNTS[grouping] * params.get("k", 1), + "lookback_ms": lookback_ms, + "interval_ms": interval_ms, + "promql": rendered if support != "sql-only" else None, + "sql": rendered if support == "sql-only" else None, + "rqes": [rqe], + } + + +def sum_query(query_id, grouping, lookback_ms, interval_ms): + rendered = f"sum by ({grouping_text(grouping)}) (sum_over_time(data[{duration(lookback_ms)}]))" + return query( + query_id, "sum", grouping, lookback_ms, interval_ms, rendered, "common-promql" + ) + + +def quantile_query(query_id, grouping, phi): + rendered = f"quantile by ({grouping_text(grouping)}) ({phi:.2f}, data)" + return query( + query_id, + "quantile", + grouping, + SPATIAL_LOOKBACK_MS, + SPATIAL_INTERVAL_MS, + rendered, + "common-promql", + phi=phi, + ) + + +def topk_sum_query(query_id, grouping, lookback_ms, interval_ms, k): + rendered = f"topk by ({grouping_text(grouping)}) ({k}, sum_over_time(data[{duration(lookback_ms)}]))" + return query( + query_id, + "topk_by_value", + grouping, + lookback_ms, + interval_ms, + rendered, + "common-promql", + k=k, + ) + + +def topk_count_query(query_id, grouping, lookback_ms, interval_ms, k): + rendered = f"topk by ({grouping_text(grouping)}) ({k}, increase(data_total[{duration(lookback_ms)}]))" + return query( + query_id, + "topk_by_count", + grouping, + lookback_ms, + interval_ms, + rendered, + "counter-gated-promql", + k=k, + ) + + +def rate_query(query_id, grouping, lookback_ms, interval_ms, k): + rendered = f"topk by ({grouping_text(grouping)}) ({k}, rate(data_total[{duration(lookback_ms)}]))" + return query( + query_id, + "rate", + grouping, + lookback_ms, + interval_ms, + rendered, + "counter-gated-promql", + k=k, + ) + + +def cardinality_query(query_id, grouping, lookback_ms, interval_ms): + group = grouping_text(grouping) + lookback_secs = lookback_ms // SECOND_MS + rendered = ( + f"SELECT {group}, COUNT(DISTINCT user_id) FROM events " + f"WHERE ts BETWEEN DATEADD(s, -{lookback_secs}, NOW()) AND NOW() " + f"GROUP BY {group}" + ) + return query( + query_id, + "cardinality", + grouping, + lookback_ms, + interval_ms, + rendered, + "sql-only", + value="user_id", + ) + + +def generate_workload(seed, index): + """Return one validated ten-query workload for `seed` and one-based `index`.""" + rng = random.Random(f"{seed}/{index}") + fine, coarse = GROUPINGS[2], GROUPINGS[1] + rollup_w, rollup_t = choose_temporal_time(rng) + short_w, short_t = MINUTE_MS, rng.choice((SECOND_MS, 2 * SECOND_MS)) + long_w, long_t = 20 * MINUTE_MS, rng.choice((30 * SECOND_MS, MINUTE_MS)) + quantile_specs = rng.sample( + [(grouping, phi) for grouping in GROUPINGS for phi in QUANTILES], 3 + ) + queries = [ + sum_query("q01-sum-fine", fine, rollup_w, rollup_t), + sum_query("q02-sum-coarse", coarse, rollup_w, rollup_t), + quantile_query("q03-quantile", *quantile_specs[0]), + quantile_query("q04-quantile", *quantile_specs[1]), + quantile_query("q05-quantile", *quantile_specs[2]), + topk_sum_query( + "q06-topk-sum", + rng.choice(GROUPINGS[:2]), + short_w, + short_t, + rng.choice(TOPK_VALUES), + ), + topk_sum_query( + "q07-topk-sum", + rng.choice(GROUPINGS[:2]), + long_w, + long_t, + rng.choice(TOPK_VALUES), + ), + topk_count_query( + "q08-topk-count", + rng.choice(GROUPINGS[:2]), + rollup_w, + rollup_t, + rng.choice(TOPK_VALUES), + ), + rate_query( + "q09-rate", + rng.choice(GROUPINGS[:2]), + long_w, + long_t, + rng.choice(TOPK_VALUES), + ), + cardinality_query("q10-cardinality", rng.choice(GROUPINGS), rollup_w, rollup_t), + ] + workload = { + "version": 1, + "id": f"workload-{index:02d}", + "seed": seed, + "labels": METRIC_LABELS, + "samples_per_sec_per_series": SAMPLES_PER_SEC_PER_SERIES, + "queries": queries, + } + validate_workload(workload) + return workload + + +def validate_workload(workload): + """Reject manifests that violate the experiment's stable public contract.""" + queries = workload["queries"] + if len(queries) != QUERIES_PER_WORKLOAD: + raise ValueError(f"a workload must contain exactly {QUERIES_PER_WORKLOAD} queries") + if len({query["id"] for query in queries}) != len(queries): + raise ValueError("query IDs must be unique") + rqe_keys = { + ( + rqe["capability"], + tuple(rqe["grouping_labels"]), + rqe["lookback_ms"], + rqe["interval_ms"], + rqe.get("phi"), + rqe.get("k"), + ) + for item in queries + for rqe in item["rqes"] + } + if len(rqe_keys) != len(queries): + raise ValueError("a workload must not repeat an atomic RQE") + if sum(query["rqes"][0]["capability"] == "quantile" for query in queries) < 3: + raise ValueError("a workload must contain at least three quantile queries") + if sum(query["rqes"][0]["capability"].startswith("topk_") for query in queries) < 2: + raise ValueError("a workload must contain at least two top-k queries") + if not any(query["rqes"][0]["capability"] == "sum" for query in queries): + raise ValueError("a workload must contain a sum query") + for item in queries: + grouping = tuple(item["grouping"]) + if grouping not in GROUP_COUNTS: + raise ValueError(f"unsupported grouping: {grouping}") + if len(item["rqes"]) != 1: + raise ValueError(f"{item['id']} must map to one RQE") + if not item["rqes"][0]["id"]: + raise ValueError(f"{item['id']} has no RQE ID") + if not any( + q["rqes"][0]["capability"] == "sum" and tuple(q["grouping"]) == GROUPINGS[2] + for q in queries + ): + raise ValueError("a workload must contain the fine side of a roll-up chain") + + +def generate_workloads(seed, workload_count=WORKLOAD_COUNT): + """Generate the deterministic workload suite for one experiment seed.""" + return [generate_workload(seed, index) for index in range(1, workload_count + 1)] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seed", type=int, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + for workload in generate_workloads(args.seed): + path = args.out / f"{workload['id']}.json" + path.write_text(json.dumps(workload, indent=2, sort_keys=True) + "\n") + print(path) + + +if __name__ == "__main__": + main() diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01.json new file mode 100644 index 00000000..0e3f9250 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01.json @@ -0,0 +1,319 @@ +{ + "id": "workload-01", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "quantile by (region, service, job) (0.99, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q04-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "quantile by (region) (0.95, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q04-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q05-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "quantile by (region, service, job) (0.75, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q05-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 20000, + "k": 8, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q10-cardinality", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data", + "value": "user_id" + } + ], + "sql": "SELECT region, service, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region, service", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02.json new file mode 100644 index 00000000..171e2d0a --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02.json @@ -0,0 +1,317 @@ +{ + "id": "workload-02", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q03-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "quantile by (region) (0.75, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q03-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "quantile by (region, service) (0.95, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q05-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "quantile by (region, service, job) (0.99, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q05-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 5000, + "k": 4, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03.json new file mode 100644 index 00000000..5d93cf32 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03.json @@ -0,0 +1,317 @@ +{ + "id": "workload-03", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 1000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 1000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "quantile by (region, service) (0.95, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "quantile by (region, service) (0.90, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q05-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "quantile by (region, service, job) (0.90, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q05-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 1000, + "k": 8, + "lookback_ms": 60000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 1000, + "lookback_ms": 60000, + "metric": "data", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -60, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04.json new file mode 100644 index 00000000..2bf9ab31 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04.json @@ -0,0 +1,313 @@ +{ + "id": "workload-04", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 10000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 10000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q03-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "quantile by (region) (0.75, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q03-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "quantile by (region, service) (0.75, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "quantile by (region) (0.95, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 10000, + "k": 4, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 10000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 10000, + "lookback_ms": 600000, + "metric": "data", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05.json new file mode 100644 index 00000000..f12afc78 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05.json @@ -0,0 +1,317 @@ +{ + "id": "workload-05", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "quantile by (region, service) (0.99, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q04-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "quantile by (region, service, job) (0.50, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q04-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "quantile by (region) (0.90, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 8, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 10000, + "k": 8, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06.json new file mode 100644 index 00000000..ae2ad018 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06.json @@ -0,0 +1,321 @@ +{ + "id": "workload-06", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "quantile by (region, service, job) (0.50, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "quantile by (region, service) (0.50, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "quantile by (region) (0.90, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 5000, + "k": 8, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality", + "interval_ms": 5000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 5000, + "lookback_ms": 300000, + "metric": "data", + "value": "user_id" + } + ], + "sql": "SELECT region, service, job, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, job", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07.json new file mode 100644 index 00000000..5d7053cf --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07.json @@ -0,0 +1,315 @@ +{ + "id": "workload-07", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "quantile by (region, service, job) (0.99, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q04-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "quantile by (region, service, job) (0.95, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q04-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "quantile by (region) (0.50, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.5 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 20000, + "k": 4, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q10-cardinality", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data", + "value": "user_id" + } + ], + "sql": "SELECT region, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08.json new file mode 100644 index 00000000..dd6f1df6 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08.json @@ -0,0 +1,307 @@ +{ + "id": "workload-08", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "quantile by (region, service) (0.75, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q04-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "quantile by (region, service) (0.95, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q04-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "quantile by (region) (0.95, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 4, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, increase(data_total[5m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 10000, + "k": 8, + "lookback_ms": 300000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q10-cardinality", + "interval_ms": 10000, + "lookback_ms": 300000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 10000, + "lookback_ms": 300000, + "metric": "data", + "value": "user_id" + } + ], + "sql": "SELECT region, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09.json new file mode 100644 index 00000000..f1c095f4 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09.json @@ -0,0 +1,321 @@ +{ + "id": "workload-09", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 2000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 2000, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q03-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "quantile by (region, service, job) (0.75, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q03-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q04-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "quantile by (region, service, job) (0.95, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q04-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.95 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q05-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "quantile by (region, service) (0.75, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q05-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q06-topk-sum", + "interval_ms": 1000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 1000, + "k": 8, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 30000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q08-topk-count", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 16, + "promql": "topk by (region) (4, increase(data_total[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 2000, + "k": 4, + "lookback_ms": 60000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 30000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 30000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q10-cardinality", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region", + "service" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 2000, + "lookback_ms": 60000, + "metric": "data", + "value": "user_id" + } + ], + "sql": "SELECT region, service, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -60, NOW()) AND NOW() GROUP BY region, service", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "version": 1 +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10.json new file mode 100644 index 00000000..d93c5ac3 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10.json @@ -0,0 +1,311 @@ +{ + "id": "workload-10", + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "queries": [ + { + "grouping": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 200, + "output_series_upper_bound": 200, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service", + "job" + ], + "id": "q01-sum-fine/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q02-sum-coarse", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "sum", + "grouping_labels": [ + "region", + "service" + ], + "id": "q02-sum-coarse/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q03-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 20, + "output_series_upper_bound": 20, + "promql": "quantile by (region, service) (0.99, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region", + "service" + ], + "id": "q03-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.99 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q04-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "quantile by (region) (0.75, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q04-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.75 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q05-quantile", + "interval_ms": 100, + "lookback_ms": 100, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": "quantile by (region) (0.90, data)", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "quantile", + "grouping_labels": [ + "region" + ], + "id": "q05-quantile/rqe", + "interval_ms": 100, + "lookback_ms": 100, + "metric": "data", + "phi": 0.9 + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q06-topk-sum", + "interval_ms": 2000, + "lookback_ms": 60000, + "output_groups": 4, + "output_series_upper_bound": 32, + "promql": "topk by (region) (8, sum_over_time(data[1m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region" + ], + "id": "q06-topk-sum/rqe", + "interval_ms": 2000, + "k": 8, + "lookback_ms": 60000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q07-topk-sum", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_value", + "grouping_labels": [ + "region", + "service" + ], + "id": "q07-topk-sum/rqe", + "interval_ms": 60000, + "k": 8, + "lookback_ms": 1200000, + "metric": "data" + } + ], + "sql": null, + "support": "common-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q08-topk-count", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 20, + "output_series_upper_bound": 160, + "promql": "topk by (region, service) (8, increase(data_total[10m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "topk_by_count", + "grouping_labels": [ + "region", + "service" + ], + "id": "q08-topk-count/rqe", + "interval_ms": 20000, + "k": 8, + "lookback_ms": 600000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region", + "service" + ], + "id": "q09-rate", + "interval_ms": 60000, + "lookback_ms": 1200000, + "output_groups": 20, + "output_series_upper_bound": 80, + "promql": "topk by (region, service) (4, rate(data_total[20m]))", + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "rate", + "grouping_labels": [ + "region", + "service" + ], + "id": "q09-rate/rqe", + "interval_ms": 60000, + "k": 4, + "lookback_ms": 1200000, + "metric": "data_total" + } + ], + "sql": null, + "support": "counter-gated-promql" + }, + { + "grouping": [ + "region" + ], + "id": "q10-cardinality", + "interval_ms": 20000, + "lookback_ms": 600000, + "output_groups": 4, + "output_series_upper_bound": 4, + "promql": null, + "rqes": [ + { + "accuracy_sla": 0.95, + "capability": "cardinality", + "grouping_labels": [ + "region" + ], + "id": "q10-cardinality/rqe", + "interval_ms": 20000, + "lookback_ms": 600000, + "metric": "data", + "value": "user_id" + } + ], + "sql": "SELECT region, COUNT(DISTINCT user_id) FROM events WHERE ts BETWEEN DATEADD(s, -600, NOW()) AND NOW() GROUP BY region", + "support": "sql-only" + } + ], + "samples_per_sec_per_series": 10, + "seed": 862, + "version": 1 +} diff --git a/asap-tools/experiments/tests/test_random_workload_artifacts.py b/asap-tools/experiments/tests/test_random_workload_artifacts.py new file mode 100644 index 00000000..a3f21f93 --- /dev/null +++ b/asap-tools/experiments/tests/test_random_workload_artifacts.py @@ -0,0 +1,41 @@ +"""Tests for manifest-to-planner workload rendering.""" + +import os +import sys +import unittest + + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "datasets", "fake_metric")) + +import random_workload_artifacts as artifacts # noqa: E402 +import random_workloads as workloads # noqa: E402 + + +class RenderArtifactsTest(unittest.TestCase): + def test_promql_input_groups_non_sql_queries_by_repeat_interval(self): + rendered = artifacts.render(workloads.generate_workloads(seed=862)[0]) + + planner = rendered["promql_planner"] + expected = [q for q in rendered["queries"] if q["promql"]] + self.assertEqual( + sum(len(group["queries"]) for group in planner["query_groups"]), len(expected) + ) + self.assertEqual( + {tuple(metric["labels"]) for metric in planner["metrics"]}, + {("region", "service", "job", "label_0")}, + ) + for group in planner["query_groups"]: + self.assertTrue(all(query["interval_ms"] == group["repetition_delay_ms"] + for query in expected if query["promql"] in group["queries"])) + + def test_sql_input_contains_only_sql_queries_and_their_table_schema(self): + rendered = artifacts.render(workloads.generate_workloads(seed=862)[0]) + + self.assertTrue(rendered["sql_planner"]["query_groups"]) + self.assertEqual(rendered["sql_planner"]["tables"][0]["name"], "events") + sql = [query for group in rendered["sql_planner"]["query_groups"] for query in group["queries"]] + self.assertTrue(all("COUNT(DISTINCT user_id)" in query for query in sql)) + + +if __name__ == "__main__": + unittest.main() diff --git a/asap-tools/experiments/tests/test_random_workloads.py b/asap-tools/experiments/tests/test_random_workloads.py new file mode 100644 index 00000000..1cbad9f7 --- /dev/null +++ b/asap-tools/experiments/tests/test_random_workloads.py @@ -0,0 +1,82 @@ +"""Behavior tests for the accelerated random-workload manifest generator.""" + +import os +import sys +import unittest +from copy import deepcopy + + +sys.path.insert( + 0, os.path.join(os.path.dirname(__file__), "..", "datasets", "fake_metric") +) + +import random_workloads as workloads # noqa: E402 + + +class GenerateWorkloadsTest(unittest.TestCase): + def test_seeded_generation_produces_ten_distinct_ten_query_workloads(self): + generated = workloads.generate_workloads(seed=17) + + self.assertEqual(len(generated), 10) + self.assertEqual(generated, workloads.generate_workloads(seed=17)) + self.assertEqual( + {workload["id"] for workload in generated}, + {f"workload-{index:02d}" for index in range(1, 11)}, + ) + for workload in generated: + queries = workload["queries"] + self.assertEqual(len(queries), 10) + self.assertEqual(len({query["id"] for query in queries}), 10) + self.assertTrue(all(query["rqes"] for query in queries)) + + def test_manifest_declares_each_querys_rendering_and_output_cardinality(self): + queries = workloads.generate_workloads(seed=1)[0]["queries"] + + for query in queries: + self.assertIn( + query["support"], {"common-promql", "counter-gated-promql", "sql-only"} + ) + self.assertIn(query["output_groups"], {4, 20, 200}) + if query["support"] == "sql-only": + self.assertIsNone(query["promql"]) + self.assertIn("COUNT(DISTINCT user_id)", query["sql"]) + self.assertIn("DATEADD(s, -", query["sql"]) + else: + self.assertIsNone(query["sql"]) + self.assertIn(" by (", query["promql"]) + + def test_visible_query_outputs_stay_bounded(self): + for workload in workloads.generate_workloads(seed=4): + for query in workload["queries"]: + rqe = query["rqes"][0] + expected = query["output_groups"] * rqe.get("k", 1) + self.assertEqual(query["output_series_upper_bound"], expected) + self.assertLessEqual(query["output_series_upper_bound"], 200) + + def test_workloads_do_not_repeat_an_atomic_rqe(self): + for workload in workloads.generate_workloads(seed=1): + keys = [] + for query in workload["queries"]: + rqe = query["rqes"][0] + keys.append( + ( + rqe["capability"], + tuple(rqe["grouping_labels"]), + rqe["lookback_ms"], + rqe["interval_ms"], + rqe.get("phi"), + rqe.get("k"), + ) + ) + self.assertEqual(len(keys), len(set(keys))) + + def test_validation_rejects_duplicate_atomic_rqes(self): + workload = deepcopy(workloads.generate_workloads(seed=1)[0]) + workload["queries"][4]["rqes"][0] = deepcopy(workload["queries"][3]["rqes"][0]) + + with self.assertRaisesRegex(ValueError, "must not repeat an atomic RQE"): + workloads.validate_workload(workload) + + +if __name__ == "__main__": + unittest.main()